bench: log every A/B cell as it finishes, and cap it with a timeout

The three markers the README requires be checked — AUTOTUNE_REPLAY_FALLBACK,
BENCHMARK_DIRECT_NO_CUDAGRAPH, BENCHMARK_CAPTURE_POISONED — were only
reachable by grepping run.log afterwards, which happens when someone
remembers to. The NV run that reproduced PR763 wrote a per-cell status row
instead; a 6-round campaign here had no equivalent, so a missing cell, a
stalled cell, or a marker count going from 0 to nonzero was invisible until
the batch ended.

run_cell now writes round/op/side/switch/rc/elapsed_s/success/failed plus the
three marker counts to <root>/status.tsv as each cell completes, and wraps the
cell in CELL_TIMEOUT (default 2400s, matching the NV run) so one hung cell
cannot hold the batch. rc 124/137 is called out by name.

grep -c prints 0 and exits 1 when it finds nothing, so each count swallows its
status; without that the bookkeeping itself would mark a clean cell failed.

ab_repeat.sh

@@ -47,6 +47,7 @@ ir_fingerprint() {
 OPS="${OPS:-}"
 : "${OPS:?OPS is required, e.g. \"topk masked_select\"}"
 ROUNDS="${ROUNDS:-6}"
+CELL_TIMEOUT="${CELL_TIMEOUT:-2400}"
 
 if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then
   export FLAGGEMS_PERF_COLOR=always
@@ -55,6 +56,38 @@ fi
 ROOT="runs/ab_repeat_${SWITCH}_$(date +%Y%m%d-%H%M%S)"
 mkdir -p "$ROOT"
 LOG="$ROOT/summary.log"
+STATUS="$ROOT/status.tsv"
+printf 'round\top\tside\tswitch\trc\telapsed_s\tsuccess\tfailed\treplay_fallback\tno_cudagraph\tcapture_poisoned\n' > "$STATUS"
+
+# 跑一格并逐格登记。事后 grep 也能拿到这些数，但只在有人想起来时才发生；
+# 缺格、超时、标记从 0 变成非 0，都要在跑的时候就看得见。
+run_cell() {   # round side op op_file out_dir [replay_from]
+  local round="$1" side="$2" op="$3" op_file="$4" dir="$5" replay="${6:-}"
+  local val=0; [[ "$side" == B ]] && val=1
+  local start rc elapsed log ok fail fb ng po
+  start=$(date +%s)
+  if [[ -n "$replay" ]]; then
+    OP="$op" OP_FILE="$op_file" OUT_DIR="$dir" REPLAY_FROM="$replay" \
+      timeout -k 30 "$CELL_TIMEOUT" env "$SWITCH=$val" bash run_pytest.sh
+  else
+    OP="$op" OP_FILE="$op_file" OUT_DIR="$dir" \
+      timeout -k 30 "$CELL_TIMEOUT" env "$SWITCH=$val" bash run_pytest.sh
+  fi
+  rc=$?
+  elapsed=$(( $(date +%s) - start ))
+  (( rc == 124 || rc == 137 )) && echo "!!! CELL_TIMEOUT ${CELL_TIMEOUT}s op=$op round=$round side=$side"
+  log="$dir/run.log"
+  # grep -c 没命中时打印 0 但以 1 退出；不吞掉的话会被当成本格失败
+  ok=$(grep -c '^SUCCESS' "$log" 2>/dev/null || true)
+  fail=$(grep -c '^FAILED' "$log" 2>/dev/null || true)
+  fb=$(grep -c 'AUTOTUNE_REPLAY_FALLBACK' "$log" 2>/dev/null || true)
+  ng=$(grep -c 'BENCHMARK_DIRECT_NO_CUDAGRAPH' "$log" 2>/dev/null || true)
+  po=$(grep -c 'BENCHMARK_CAPTURE_POISONED' "$log" 2>/dev/null || true)
+  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+    "$round" "$op" "$side" "$val" "$rc" "$elapsed" \
+    "${ok:-0}" "${fail:-0}" "${fb:-0}" "${ng:-0}" "${po:-0}" >> "$STATUS"
+  return $rc
+}
 
 {
   echo ">>> switch=$SWITCH  rounds=$ROUNDS  ops=$OPS"
@@ -67,8 +100,7 @@ LOG="$ROOT/summary.log"
 
     # 第 1 轮 A 侧 record，其余全部回放它，config 跨轮跨侧一致
     echo "########## $op | round 1 | A ($SWITCH=0, record) ##########"
-    if ! OP="$op" OP_FILE="$op_file" OUT_DIR="$out/r1_A" \
-         env "$SWITCH=0" bash run_pytest.sh; then
+    if ! run_cell 1 A "$op" "$op_file" "$out/r1_A"; then
       echo "!!! FAILED $op round 1 A; skipping this operator"
       continue
     fi
@@ -82,8 +114,7 @@ LOG="$ROOT/summary.log"
         [[ $k -eq 1 && $side == A ]] && continue   # 已经跑过
         val=$([[ $side == A ]] && echo 0 || echo 1)
         echo "########## $op | round $k | $side ($SWITCH=$val, replay) ##########"
-        if ! OP="$op" OP_FILE="$op_file" OUT_DIR="$out/r${k}_${side}" \
-             REPLAY_FROM="$REC" env "$SWITCH=$val" bash run_pytest.sh; then
+        if ! run_cell "$k" "$side" "$op" "$op_file" "$out/r${k}_${side}" "$REC"; then
           echo "!!! FAILED $op round $k $side"
           ok=0
         fi
@@ -125,5 +156,6 @@ LOG="$ROOT/summary.log"
 sed -i -E $'s/\x1b\\[[0-9;]*[A-Za-z]//g' "$LOG"
 
 echo ">>> 总 log: $(pwd)/$LOG"
+echo ">>> 逐格状态表: $(pwd)/$STATUS"
 echo ">>> 逐算子重出报告: python batch_summary.py --ab-rounds $ROOT/<op> --label <op>"
 echo ">>> 解读要点: 逐轮区间宽说明判定不稳；单侧噪声大说明机器不稳；两者含义不同"