bench: refuse to report a comparison whose two sides have different calibers

Two runs on different calibers produce a report that looks exactly like a
valid one — no step errors, the numbers just mean something else. This repo
has published three figures that were an artifact of such a difference:
router_gemm -8.2% (record vs record), mm "945 variants -> 33" (shared vs
inherited libtuner cache), replication_pad3d +43% vs +124.5% (timer).

batch_summary --ab / --ab-rounds now reads every participating run's
run_identity.json and compares backend/device/timer/bench_level/shape_mode/
tuner_cache/direct_repeat before parsing a single latency. Mismatch, or a
missing identity, prints !!! [caliber] and exits 4. autotune_mode is judged
separately: record pinning a replay is the normal shape, both sides record
is the exact trap above and is refused by name. --allow-caliber-mismatch
releases it, still printing what it found.

Recording on a *shared* tuner cache warns rather than refuses: the pinned
config depends on what else ran on the machine, which leaves relative
conclusions intact and absolute latencies unquotable. Refusing it would
reject the default every A/B here currently uses.

ab_repeat.sh closes the one axis that had no check: after round 1 it
fingerprints both sides' ttgir (relative path + content, so a changed
variant *set* counts too). Byte-identical means the switch changed no
generated code for this op, and the remaining N-1 rounds would measure
noise and report it as "no difference" — so the op is skipped with
SWITCH_NO_IR_DIFF unless SWITCH_ALLOW_NO_IR_DIFF=1.

README leads with a caliber table: each axis, its measured magnitude, what
pins it, and which code checks it. The fourth column is code because
documentation did not hold — the README already said "the B side must use
REPLAY_FROM" when the record-vs-record number was published.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

batch_summary.py

@@ -222,8 +222,16 @@ def ab_rows(before_text: str, after_text: str) -> List[Dict[str, str]]:
     return paired
 
 
-def report_ab(before: Path, after: Path, label: str = "") -> List[Dict[str, str]]:
-    """打印一个算子的前后对比：逐行 speedup 与整体区间。"""
+def report_ab(before: Path, after: Path, label: str = "",
+              strict: bool = True) -> List[Dict[str, str]]:
+    """打印一个算子的前后对比：逐行 speedup 与整体区间。
+
+    先核对两侧口径；不一致时默认拒绝出报告，因为不可比的两次运行会产出
+    一份和有效报告长得一模一样的表。
+    """
+    if not _report_caliber([("before", before.parent), ("after", after.parent)],
+                           label or "ab", strict):
+        raise CaliberMismatch()
     before_text = before.read_text(encoding="utf-8", errors="replace")
     after_text = after.read_text(encoding="utf-8", errors="replace")
     rows = ab_rows(before_text, after_text)
@@ -246,6 +254,109 @@ def report_ab(before: Path, after: Path, label: str = "") -> List[Dict[str, str]
     return rows
 
 
+# Fields whose disagreement makes two runs incomparable. Measured magnitudes,
+# so the list is not precautionary: an unpinned autotune config moved
+# router_gemm by 8pp; the timer moved replication_pad3d from +43% to +124.5%;
+# a shared tuner cache made mm look like 945 compiled variants had become 33;
+# --level core sees 5 mm shapes where comprehensive sees 29.
+CALIBER_FIELDS = (
+    "backend", "device", "timer", "bench_level", "shape_mode",
+    "tuner_cache", "direct_repeat",
+)
+
+
+def caliber_of(run_dir: Path) -> Optional[Dict[str, str]]:
+    """Caliber fields from a run's run_identity.json, or None if unreadable."""
+    if not (run_dir / "run_identity.json").is_file():
+        return None
+    try:
+        from _run_identity import load_identity
+        ident = load_identity(str(run_dir))
+    except Exception:
+        return None
+    return {k: str(ident.get(k, "")) for k in CALIBER_FIELDS} | {
+        "autotune_mode": str(ident.get("autotune_mode", ""))
+    }
+
+
+def caliber_complaints(runs: List[Tuple[str, Path]]) -> List[str]:
+    """Everything about this set of runs that makes their comparison unsound.
+
+    Reported rather than silently tolerated because the failure is invisible in
+    the numbers: two runs on different calibers produce a report that looks
+    exactly like a valid one. This repository has already published three
+    figures that were an artifact of one such difference.
+    """
+    out: List[str] = []
+    cals = [(name, caliber_of(d)) for name, d in runs]
+    missing = [n for n, c in cals if c is None]
+    if missing:
+        out.append(f"口径不可核实：{', '.join(missing)} 缺 run_identity.json"
+                   "（早于该字段的产物）")
+    known = [(n, c) for n, c in cals if c is not None]
+    if len(known) < 2:
+        return out
+
+    for field in CALIBER_FIELDS:
+        seen = {c[field] for _, c in known}
+        if len(seen) > 1:
+            vals = ", ".join(f"{n}={c[field] or '<空>'}" for n, c in known)
+            out.append(f"{field} 不一致：{vals}")
+
+    # A/B needs one side to pin the configs the other replays. Two record runs
+    # each sweep independently, and the winner can differ; the README puts that
+    # at the 10% level and one-directional per shape, which is enough to swamp
+    # what is being measured.
+    modes = [(n, c["autotune_mode"]) for n, c in known]
+    if all(m.startswith("record") for _, m in modes):
+        out.append("两侧都是 record 模式：config 未锁定，各自 sweep 可能选出不同 "
+                   "winner。按 README，只有 record 数据支撑的结论按未验证处理")
+    return out
+
+
+def caliber_hazards(runs: List[Tuple[str, Path]]) -> List[str]:
+    """Not mismatches — shared state that makes a matched caliber less solid.
+
+    Warned about rather than refused, because refusing would reject the
+    default configuration every A/B in this repo currently uses.
+    """
+    cals = [(n, c) for n, c in ((n, caliber_of(d)) for n, d in runs) if c]
+    recording = [n for n, c in cals if c["autotune_mode"].startswith("record")]
+    shared = [n for n, c in cals if c["tuner_cache"].startswith("shared")]
+    if recording and shared:
+        # 一次 record 跑在共享 libtuner cache 上，sweep 的起点是别的运行留下的
+        # 结果。曾把 mm 的 945 个变体读成 33 个，误报为编译期收益。A/B 两侧都
+        # replay 同一份 config 时结论仍成立，绝对值不成立。
+        return [f"{', '.join(recording)} 在共享 tuner cache 上 record："
+                "被 pin 的 config 取决于此前跑过什么。相对结论不受影响，"
+                "绝对延迟/变体数不可跨机器引用（ZL_ISOLATE_TUNER_CACHE=1 可消除）"]
+    return []
+
+
+class CaliberMismatch(SystemExit):
+    """Strict-mode refusal. Exit 4 so a driver script can tell it from a crash."""
+
+    def __init__(self) -> None:
+        super().__init__(4)
+
+
+def _report_caliber(runs: List[Tuple[str, Path]], label: str, strict: bool) -> bool:
+    """Print caliber problems. Returns False when strict mode should stop."""
+    for h in caliber_hazards(runs):
+        print(f"警告 [caliber] {label}: {h}")
+    problems = caliber_complaints(runs)
+    if not problems:
+        return True
+    head = "!!!" if strict else "警告"
+    for p in problems:
+        print(f"{head} [caliber] {label}: {p}")
+    if strict:
+        print(f"!!! [caliber] {label}: 拒绝出报告。确认口径确实可比后加 "
+              "--allow-caliber-mismatch")
+        return False
+    return True
+
+
 def _lat_index(text: str) -> Dict[Tuple[str, str, int], float]:
     """(dtype, size_detail, occurrence) -> gems latency, SUCCESS rows only.
 
@@ -326,10 +437,14 @@ def ab_round_rows(root: Path) -> Tuple[List[Dict[str, str]], List[float], int]:
     return rows, per_round, len(rounds)
 
 
-def report_ab_rounds(root: Path, label: str = "") -> List[Dict[str, str]]:
+def report_ab_rounds(root: Path, label: str = "",
+                     strict: bool = True) -> List[Dict[str, str]]:
     """Print the multi-round A/B: per-shape medians plus both dispersions."""
-    rows, per_round, n = ab_round_rows(root)
     name = label or root.name
+    cells = sorted(d for d in root.glob("r*_[AB]") if (d / "run.log").is_file())
+    if cells and not _report_caliber([(d.name, d) for d in cells], name, strict):
+        raise CaliberMismatch()
+    rows, per_round, n = ab_round_rows(root)
     if n < 2:
         print(f"[ab-rounds] {name}: 需要至少 2 轮 <root>/r<k>_<A|B>/run.log"
               f"（找到 {n} 轮）")
@@ -370,17 +485,22 @@ def main() -> None:
                    help="多轮 A/B 目录，含 r<k>_A/run.log 与 r<k>_B/run.log；"
                         "逐 shape 取轮次中位，并报逐轮区间与符号稳定性")
     p.add_argument("--label", default="", help="--ab / --ab-rounds 报告里的算子名")
+    p.add_argument("--allow-caliber-mismatch", action="store_true",
+                   help="口径不一致时仍出报告（默认拒绝）。只在确认两次运行"
+                        "确实可比时使用")
     args = p.parse_args()
     if args.ab_rounds:
         if not args.ab_rounds.is_dir():
             raise SystemExit(f"目录不存在: {args.ab_rounds}")
-        report_ab_rounds(args.ab_rounds, args.label)
+        report_ab_rounds(args.ab_rounds, args.label,
+                         strict=not args.allow_caliber_mismatch)
         return
     if args.ab:
         for path in args.ab:
             if not path.is_file():
                 raise SystemExit(f"run.log 不存在: {path}")
-        report_ab(args.ab[0], args.ab[1], args.label)
+        report_ab(args.ab[0], args.ab[1], args.label,
+                  strict=not args.allow_caliber_mismatch)
         return
     if args.batch_dir is None:
         raise SystemExit("需要 batch_dir，或用 --ab BEFORE AFTER")