feat: repeat an A/B over rounds and report whether the verdict holds

A single pair cannot say whether its own verdict is stable, and on this device
that is not a theoretical gap. Measured on an S5000, topk eliminated 32
convert_layout ops yet 17 of its 42 shapes changed the SIGN of their delta
between two equally valid runs. The cause is not the timer: graph capture and
direct timing agree to 1.00x on that operator, and the verdict moved a median of
1.1pp between them, inside the 1.7pp noise floor a same-timer control
established. The deltas are simply near zero, and one measurement cannot orient
them.

ab_repeat.sh runs an A/B over N rounds in independent processes. Round 1's A
side records the autotune configs and every later run -- including A's own later
rounds -- replays that one set, so configs are pinned across rounds and sides and
only the switch under test and measurement noise remain. Round order alternates
to cancel one-directional drift. An operator with a failed round gets no report,
since a report missing rounds reads exactly like a complete one.

batch_summary --ab-rounds aggregates <root>/r<k>_<A|B>/run.log and separates the
two dispersions, which answer different questions and were previously conflated:
the spread within one side's samples says how noisy the machine was, while the
spread of the per-round deltas says whether the verdict holds. Shapes whose
delta changes sign across rounds are counted and called out, because their
direction is not a conclusion at any timer.

The path is already validated on NVIDIA: six rounds narrowed router_gemm to
-16.6% ~ -16.2% with 0.4% side noise, which is what turned a single pair into a
statement.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

batch_summary.py

@@ -16,8 +16,10 @@ from __future__ import annotations
 
 import argparse
 import csv
+import itertools
 import os
 import re
+import statistics
 import sys
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple
@@ -244,6 +246,118 @@ def report_ab(before: Path, after: Path, label: str = "") -> List[Dict[str, str]
     return rows
 
 
+def _lat_index(text: str) -> Dict[Tuple[str, str, int], float]:
+    """(dtype, size_detail, occurrence) -> gems latency, SUCCESS rows only.
+
+    Occurrence disambiguates a shape a benchmark measures twice in one table,
+    which BLAS does at --level comprehensive.
+    """
+    out: Dict[Tuple[str, str, int], float] = {}
+    seen: Dict[Tuple[str, str], int] = {}
+    for row in parse_run_log(text):
+        if row.get("row_status") != "SUCCESS":
+            continue
+        try:
+            lat = float(row.get("gems_lat", ""))
+        except ValueError:
+            continue
+        if lat <= 0:
+            continue
+        base = (row.get("dtype", ""), row.get("size_detail", ""))
+        i = seen.get(base, 0)
+        seen[base] = i + 1
+        out[base + (i,)] = lat
+    return out
+
+
+def ab_round_rows(root: Path) -> Tuple[List[Dict[str, str]], List[float], int]:
+    """Aggregate an N-round A/B laid out as <root>/r<k>_<A|B>/run.log.
+
+    Returns the per-shape rows, the per-round deltas, and the round count.
+
+    A single pair cannot say whether its own verdict is stable, which is the
+    question that matters once a delta lands near zero: measured on an S5000,
+    topk eliminated 32 converts yet 17 of its 42 shapes changed the SIGN of
+    their delta between two equally valid runs. Two dispersions answer different
+    halves of that and both are reported:
+
+      side spread  max/min-1 within one side's samples -> how noisy the machine
+                   was while measuring
+      round spread min~max of the per-round deltas -> whether the VERDICT holds,
+                   which is what a single pair cannot show
+    """
+    rounds: List[Tuple[Dict, Dict]] = []
+    for k in itertools.count(1):
+        a, b = root / f"r{k}_A" / "run.log", root / f"r{k}_B" / "run.log"
+        if not (a.is_file() and b.is_file()):
+            break
+        rounds.append((
+            _lat_index(a.read_text(encoding="utf-8", errors="replace")),
+            _lat_index(b.read_text(encoding="utf-8", errors="replace")),
+        ))
+    if not rounds:
+        return [], [], 0
+
+    keys = set(rounds[0][0]) & set(rounds[0][1])
+    for a, b in rounds[1:]:
+        keys &= set(a) & set(b)
+
+    per_round = [
+        statistics.median((a[k] / b[k] - 1.0) * 100.0 for k in keys)
+        for a, b in rounds
+    ] if keys else []
+
+    rows: List[Dict[str, str]] = []
+    for key in sorted(keys):
+        av = [a[key] for a, _ in rounds]
+        bv = [b[key] for _, b in rounds]
+        am, bm = statistics.median(av), statistics.median(bv)
+        deltas = [(a / b - 1.0) * 100.0 for a, b in zip(av, bv)]
+        rows.append({
+            "dtype": key[0], "size_detail": key[1],
+            "before_lat": f"{am:.6f}", "after_lat": f"{bm:.6f}",
+            "speedup_percent": f"{(am / bm - 1.0) * 100.0:.2f}",
+            "delta_min": f"{min(deltas):.2f}", "delta_max": f"{max(deltas):.2f}",
+            "sign_stable": "yes" if all(d > 0 for d in deltas)
+                           or all(d <= 0 for d in deltas) else "NO",
+            "before_spread": f"{(max(av) / min(av) - 1.0) * 100.0:.2f}",
+            "after_spread": f"{(max(bv) / min(bv) - 1.0) * 100.0:.2f}",
+        })
+    return rows, per_round, len(rounds)
+
+
+def report_ab_rounds(root: Path, label: str = "") -> List[Dict[str, str]]:
+    """Print the multi-round A/B: per-shape medians plus both dispersions."""
+    rows, per_round, n = ab_round_rows(root)
+    name = label or root.name
+    if n < 2:
+        print(f"[ab-rounds] {name}: 需要至少 2 轮 <root>/r<k>_<A|B>/run.log"
+              f"（找到 {n} 轮）")
+        return rows
+    if not rows:
+        print(f"[ab-rounds] {name}: {n} 轮，但没有全轮次都成功的配对行")
+        return rows
+    for r in rows:
+        flag = "" if r["sign_stable"] == "yes" else "  符号不稳"
+        print(f"  {r['dtype']:<16} {float(r['speedup_percent']):+7.2f}%  "
+              f"[{float(r['delta_min']):+.2f}%~{float(r['delta_max']):+.2f}%]  "
+              f"{r['before_lat']} -> {r['after_lat']}  "
+              f"{r['size_detail']}{flag}")
+    med = [float(r["speedup_percent"]) for r in rows]
+    spread = [float(r["before_spread"]) for r in rows] + \
+             [float(r["after_spread"]) for r in rows]
+    unstable = sum(1 for r in rows if r["sign_stable"] != "yes")
+    print(f"[ab-rounds] {name}: {n} 轮 × {len(rows)} shape，"
+          f"speedup 中位 {statistics.median(med):+.2f}%，"
+          f"逐轮 {min(per_round):+.2f}% ~ {max(per_round):+.2f}%，"
+          f"单侧噪声中位 {statistics.median(spread):.2f}%")
+    if unstable:
+        # The verdict, not the machine, is what these rows fail to pin down.
+        print(f"[ab-rounds] {name}: {unstable}/{len(rows)} 个 shape 的 delta "
+              "在轮次间改变符号——这些行的方向不可结论，与计时口径无关")
+    return rows
+
+
 def main() -> None:
     p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
     p.add_argument("batch_dir", type=Path, nargs="?",
@@ -252,8 +366,16 @@ def main() -> None:
                    help="输出 CSV（默认 <batch_dir>/summary.csv）")
     p.add_argument("--ab", nargs=2, type=Path, metavar=("BEFORE", "AFTER"),
                    help="两份 run.log，按 shape 配对输出该算子的 speedup 区间")
-    p.add_argument("--label", default="", help="--ab 报告里的算子名")
+    p.add_argument("--ab-rounds", type=Path, default=None, metavar="ROOT",
+                   help="多轮 A/B 目录，含 r<k>_A/run.log 与 r<k>_B/run.log；"
+                        "逐 shape 取轮次中位，并报逐轮区间与符号稳定性")
+    p.add_argument("--label", default="", help="--ab / --ab-rounds 报告里的算子名")
     args = p.parse_args()
+    if args.ab_rounds:
+        if not args.ab_rounds.is_dir():
+            raise SystemExit(f"目录不存在: {args.ab_rounds}")
+        report_ab_rounds(args.ab_rounds, args.label)
+        return
     if args.ab:
         for path in args.ab:
             if not path.is_file():