diff --git a/README.md b/README.md index 6f70d20..826003f 100644 --- a/README.md +++ b/README.md @@ -231,6 +231,12 @@ SWITCH=FLAGTREE_MUSA_RLC_ENHANCE OPS="topk masked_select" ROUNDS=6 ZL_BACKEND= `REPLAY_FROM` 那一份,config 因此跨轮跨侧全部锁死;逐轮交替先后顺序抵消单向漂移。 有轮次失败的算子不出报告——缺轮的报告会让人误以为是完整的 N 轮。 +每跑完一格往 `/status.tsv` 追一行:`round op side switch rc elapsed_s success +failed replay_fallback no_cudagraph capture_poisoned`。README 要求必查的三个标记就在 +这张表里,跑的时候就能看见它们从 0 变成非 0,不用等结束再 grep——事后 grep 只在有人 +想起来的时候才发生。每格另有 `CELL_TIMEOUT`(默认 2400 秒)兜底,超时记 `rc=124` 并打 +`!!! CELL_TIMEOUT`,不会让一格卡死拖住整批。 + 第 1 轮 A/B 跑完后,脚本比两侧 `ttgir/` 的指纹(相对路径 + 内容)。**逐字节相同就 跳过这个算子**,并打 `!!! SWITCH_NO_IR_DIFF op=... switch=...`:这个开关没改变该算子的 生成代码,剩下 N-1 轮测的全是噪声,最后还会报成一个"无差异"的结论。先查开关名拼写 diff --git a/ab_repeat.sh b/ab_repeat.sh index f52aefc..6e70c1c 100644 --- a/ab_repeat.sh +++ b/ab_repeat.sh @@ -47,6 +47,7 @@ ir_fingerprint() { OPS="${OPS:-}" : "${OPS:?OPS is required, e.g. \"topk masked_select\"}" ROUNDS="${ROUNDS:-6}" +CELL_TIMEOUT="${CELL_TIMEOUT:-2400}" if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then export FLAGGEMS_PERF_COLOR=always @@ -55,6 +56,38 @@ fi ROOT="runs/ab_repeat_${SWITCH}_$(date +%Y%m%d-%H%M%S)" mkdir -p "$ROOT" LOG="$ROOT/summary.log" +STATUS="$ROOT/status.tsv" +printf 'round\top\tside\tswitch\trc\telapsed_s\tsuccess\tfailed\treplay_fallback\tno_cudagraph\tcapture_poisoned\n' > "$STATUS" + +# 跑一格并逐格登记。事后 grep 也能拿到这些数,但只在有人想起来时才发生; +# 缺格、超时、标记从 0 变成非 0,都要在跑的时候就看得见。 +run_cell() { # round side op op_file out_dir [replay_from] + local round="$1" side="$2" op="$3" op_file="$4" dir="$5" replay="${6:-}" + local val=0; [[ "$side" == B ]] && val=1 + local start rc elapsed log ok fail fb ng po + start=$(date +%s) + if [[ -n "$replay" ]]; then + OP="$op" OP_FILE="$op_file" OUT_DIR="$dir" REPLAY_FROM="$replay" \ + timeout -k 30 "$CELL_TIMEOUT" env "$SWITCH=$val" bash run_pytest.sh + else + OP="$op" OP_FILE="$op_file" OUT_DIR="$dir" \ + timeout -k 30 "$CELL_TIMEOUT" env "$SWITCH=$val" bash run_pytest.sh + fi + rc=$? + elapsed=$(( $(date +%s) - start )) + (( rc == 124 || rc == 137 )) && echo "!!! CELL_TIMEOUT ${CELL_TIMEOUT}s op=$op round=$round side=$side" + log="$dir/run.log" + # grep -c 没命中时打印 0 但以 1 退出;不吞掉的话会被当成本格失败 + ok=$(grep -c '^SUCCESS' "$log" 2>/dev/null || true) + fail=$(grep -c '^FAILED' "$log" 2>/dev/null || true) + fb=$(grep -c 'AUTOTUNE_REPLAY_FALLBACK' "$log" 2>/dev/null || true) + ng=$(grep -c 'BENCHMARK_DIRECT_NO_CUDAGRAPH' "$log" 2>/dev/null || true) + po=$(grep -c 'BENCHMARK_CAPTURE_POISONED' "$log" 2>/dev/null || true) + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$round" "$op" "$side" "$val" "$rc" "$elapsed" \ + "${ok:-0}" "${fail:-0}" "${fb:-0}" "${ng:-0}" "${po:-0}" >> "$STATUS" + return $rc +} { echo ">>> switch=$SWITCH rounds=$ROUNDS ops=$OPS" @@ -67,8 +100,7 @@ LOG="$ROOT/summary.log" # 第 1 轮 A 侧 record,其余全部回放它,config 跨轮跨侧一致 echo "########## $op | round 1 | A ($SWITCH=0, record) ##########" - if ! OP="$op" OP_FILE="$op_file" OUT_DIR="$out/r1_A" \ - env "$SWITCH=0" bash run_pytest.sh; then + if ! run_cell 1 A "$op" "$op_file" "$out/r1_A"; then echo "!!! FAILED $op round 1 A; skipping this operator" continue fi @@ -82,8 +114,7 @@ LOG="$ROOT/summary.log" [[ $k -eq 1 && $side == A ]] && continue # 已经跑过 val=$([[ $side == A ]] && echo 0 || echo 1) echo "########## $op | round $k | $side ($SWITCH=$val, replay) ##########" - if ! OP="$op" OP_FILE="$op_file" OUT_DIR="$out/r${k}_${side}" \ - REPLAY_FROM="$REC" env "$SWITCH=$val" bash run_pytest.sh; then + if ! run_cell "$k" "$side" "$op" "$op_file" "$out/r${k}_${side}" "$REC"; then echo "!!! FAILED $op round $k $side" ok=0 fi @@ -125,5 +156,6 @@ LOG="$ROOT/summary.log" sed -i -E $'s/\x1b\\[[0-9;]*[A-Za-z]//g' "$LOG" echo ">>> 总 log: $(pwd)/$LOG" +echo ">>> 逐格状态表: $(pwd)/$STATUS" echo ">>> 逐算子重出报告: python batch_summary.py --ab-rounds $ROOT/ --label " echo ">>> 解读要点: 逐轮区间宽说明判定不稳;单侧噪声大说明机器不稳;两者含义不同"