forked from zhoulin/zl_bench
bench: log every A/B cell as it finishes, and cap it with a timeout
The three markers the README requires be checked — AUTOTUNE_REPLAY_FALLBACK, BENCHMARK_DIRECT_NO_CUDAGRAPH, BENCHMARK_CAPTURE_POISONED — were only reachable by grepping run.log afterwards, which happens when someone remembers to. The NV run that reproduced PR763 wrote a per-cell status row instead; a 6-round campaign here had no equivalent, so a missing cell, a stalled cell, or a marker count going from 0 to nonzero was invisible until the batch ended. run_cell now writes round/op/side/switch/rc/elapsed_s/success/failed plus the three marker counts to <root>/status.tsv as each cell completes, and wraps the cell in CELL_TIMEOUT (default 2400s, matching the NV run) so one hung cell cannot hold the batch. rc 124/137 is called out by name. grep -c prints 0 and exits 1 when it finds nothing, so each count swallows its status; without that the bookkeeping itself would mark a clean cell failed.
This commit is contained in:
Aharrypotter committed
1 parent
92f3d150c7
commit
d45806eacd
2 files changed
+42
-4
No files matched your search
@@ -231,6 +231,12 @@ SWITCH=FLAGTREE_MUSA_RLC_ENHANCE OPS="topk masked_select" ROUNDS=6 ZL_BACKEND=
|
||||
`REPLAY_FROM` 那一份，config 因此跨轮跨侧全部锁死；逐轮交替先后顺序抵消单向漂移。
|
||||
有轮次失败的算子不出报告——缺轮的报告会让人误以为是完整的 N 轮。
|
||||
|
||||
每跑完一格往 `<root>/status.tsv` 追一行：`round op side switch rc elapsed_s success
|
||||
failed replay_fallback no_cudagraph capture_poisoned`。README 要求必查的三个标记就在
|
||||
这张表里，跑的时候就能看见它们从 0 变成非 0，不用等结束再 grep——事后 grep 只在有人
|
||||
想起来的时候才发生。每格另有 `CELL_TIMEOUT`（默认 2400 秒）兜底，超时记 `rc=124` 并打
|
||||
`!!! CELL_TIMEOUT`，不会让一格卡死拖住整批。
|
||||
|
||||
第 1 轮 A/B 跑完后，脚本比两侧 `ttgir/` 的指纹（相对路径 + 内容）。**逐字节相同就
|
||||
跳过这个算子**，并打 `!!! SWITCH_NO_IR_DIFF op=... switch=...`：这个开关没改变该算子的
|
||||
生成代码，剩下 N-1 轮测的全是噪声，最后还会报成一个"无差异"的结论。先查开关名拼写
|
||||
|
||||
+36
-4
@@ -47,6 +47,7 @@ ir_fingerprint() {
|
||||
OPS="${OPS:-}"
|
||||
: "${OPS:?OPS is required, e.g. \"topk masked_select\"}"
|
||||
ROUNDS="${ROUNDS:-6}"
|
||||
CELL_TIMEOUT="${CELL_TIMEOUT:-2400}"
|
||||
|
||||
if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then
|
||||
export FLAGGEMS_PERF_COLOR=always
|
||||
@@ -55,6 +56,38 @@ fi
|
||||
ROOT="runs/ab_repeat_${SWITCH}_$(date +%Y%m%d-%H%M%S)"
|
||||
mkdir -p "$ROOT"
|
||||
LOG="$ROOT/summary.log"
|
||||
STATUS="$ROOT/status.tsv"
|
||||
printf 'round\top\tside\tswitch\trc\telapsed_s\tsuccess\tfailed\treplay_fallback\tno_cudagraph\tcapture_poisoned\n' > "$STATUS"
|
||||
|
||||
# 跑一格并逐格登记。事后 grep 也能拿到这些数，但只在有人想起来时才发生；
|
||||
# 缺格、超时、标记从 0 变成非 0，都要在跑的时候就看得见。
|
||||
run_cell() { # round side op op_file out_dir [replay_from]
|
||||
local round="$1" side="$2" op="$3" op_file="$4" dir="$5" replay="${6:-}"
|
||||
local val=0; [[ "$side" == B ]] && val=1
|
||||
local start rc elapsed log ok fail fb ng po
|
||||
start=$(date +%s)
|
||||
if [[ -n "$replay" ]]; then
|
||||
OP="$op" OP_FILE="$op_file" OUT_DIR="$dir" REPLAY_FROM="$replay" \
|
||||
timeout -k 30 "$CELL_TIMEOUT" env "$SWITCH=$val" bash run_pytest.sh
|
||||
else
|
||||
OP="$op" OP_FILE="$op_file" OUT_DIR="$dir" \
|
||||
timeout -k 30 "$CELL_TIMEOUT" env "$SWITCH=$val" bash run_pytest.sh
|
||||
fi
|
||||
rc=$?
|
||||
elapsed=$(( $(date +%s) - start ))
|
||||
(( rc == 124 || rc == 137 )) && echo "!!! CELL_TIMEOUT ${CELL_TIMEOUT}s op=$op round=$round side=$side"
|
||||
log="$dir/run.log"
|
||||
# grep -c 没命中时打印 0 但以 1 退出；不吞掉的话会被当成本格失败
|
||||
ok=$(grep -c '^SUCCESS' "$log" 2>/dev/null || true)
|
||||
fail=$(grep -c '^FAILED' "$log" 2>/dev/null || true)
|
||||
fb=$(grep -c 'AUTOTUNE_REPLAY_FALLBACK' "$log" 2>/dev/null || true)
|
||||
ng=$(grep -c 'BENCHMARK_DIRECT_NO_CUDAGRAPH' "$log" 2>/dev/null || true)
|
||||
po=$(grep -c 'BENCHMARK_CAPTURE_POISONED' "$log" 2>/dev/null || true)
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$round" "$op" "$side" "$val" "$rc" "$elapsed" \
|
||||
"${ok:-0}" "${fail:-0}" "${fb:-0}" "${ng:-0}" "${po:-0}" >> "$STATUS"
|
||||
return $rc
|
||||
}
|
||||
|
||||
{
|
||||
echo ">>> switch=$SWITCH rounds=$ROUNDS ops=$OPS"
|
||||
@@ -67,8 +100,7 @@ LOG="$ROOT/summary.log"
|
||||
|
||||
# 第 1 轮 A 侧 record，其余全部回放它，config 跨轮跨侧一致
|
||||
echo "########## $op | round 1 | A ($SWITCH=0, record) ##########"
|
||||
if ! OP="$op" OP_FILE="$op_file" OUT_DIR="$out/r1_A" \
|
||||
env "$SWITCH=0" bash run_pytest.sh; then
|
||||
if ! run_cell 1 A "$op" "$op_file" "$out/r1_A"; then
|
||||
echo "!!! FAILED $op round 1 A; skipping this operator"
|
||||
continue
|
||||
fi
|
||||
@@ -82,8 +114,7 @@ LOG="$ROOT/summary.log"
|
||||
[[ $k -eq 1 && $side == A ]] && continue # 已经跑过
|
||||
val=$([[ $side == A ]] && echo 0 || echo 1)
|
||||
echo "########## $op | round $k | $side ($SWITCH=$val, replay) ##########"
|
||||
if ! OP="$op" OP_FILE="$op_file" OUT_DIR="$out/r${k}_${side}" \
|
||||
REPLAY_FROM="$REC" env "$SWITCH=$val" bash run_pytest.sh; then
|
||||
if ! run_cell "$k" "$side" "$op" "$op_file" "$out/r${k}_${side}" "$REC"; then
|
||||
echo "!!! FAILED $op round $k $side"
|
||||
ok=0
|
||||
fi
|
||||
@@ -125,5 +156,6 @@ LOG="$ROOT/summary.log"
|
||||
sed -i -E $'s/\x1b\\[[0-9;]*[A-Za-z]//g' "$LOG"
|
||||
|
||||
echo ">>> 总 log: $(pwd)/$LOG"
|
||||
echo ">>> 逐格状态表: $(pwd)/$STATUS"
|
||||
echo ">>> 逐算子重出报告: python batch_summary.py --ab-rounds $ROOT/<op> --label <op>"
|
||||
echo ">>> 解读要点: 逐轮区间宽说明判定不稳；单侧噪声大说明机器不稳；两者含义不同"
|
||||
Reference in new issue
Block a user