feat: report a per-operator speedup range across two runs

The speedup column in a result table compares gems against torch, not one build
against another, so an A/B needs the two runs paired by shape. batch_summary
gains an --ab mode that indexes both run.logs by operator, dtype and size
detail, computes before/after per pair, and prints the per-shape rows plus the
range. Only rows that succeeded on both sides with a positive latency count, so
an incomplete pairing cannot masquerade as a complete range. When either side
carries a BENCHMARK_DIRECT_NO_CUDAGRAPH marker the range is annotated, because
a fallback row was timed under a different regime than a captured one.

ab_rlc_test.sh drives the pairing for the FlagTree RemoveLayoutConversions
switch the way ab_fold_test.sh does for the FlagGems one: per operator, run with
the enhancement off while recording autotune configs, then replay those configs
with it on, so the two sides differ by that switch alone. Each operator
therefore leaves exactly two run.log files, and the summary of each pair is
printed as it completes.

Checked on an S5000 against gelu_backward over five shapes: both sides captured
with no fallback, five rows paired, range -0.43% to +0.09%.
This commit is contained in:
Aharrypotter committed 2026-09-02 12:19:26 +08:00
1 parent 7522143747
commit abf9475ff3
2 files changed
+139 -1

No files matched your search

+71 -1
View File
@@ -165,12 +165,82 @@ def summarize(batch_dir: Path, out_path: Optional[Path] = None) -> Path:
return out_path
def ab_rows(before_text: str, after_text: str) -> List[Dict[str, str]]:
"""把两次 run.log 的测量行按 (op, dtype, size_detail) 配对。
speedup 列是 gems 对 torch,不是前后对比,所以这里用 gems_lat 自己算
before/after - 1。只有两侧都 SUCCESS 且延迟为正的行才计入;对不上的行
单独返回,避免用不完整的配对报一个看起来完整的区间。
"""
def index(text: str) -> Dict[Tuple[str, str, str], float]:
out: Dict[Tuple[str, str, str], float] = {}
for row in parse_run_log(text):
if row.get("row_status") != "SUCCESS":
continue
try:
lat = float(row.get("gems_lat", ""))
except ValueError:
continue
if lat > 0:
out[(row.get("fg_op_name", ""), row.get("dtype", ""),
row.get("size_detail", ""))] = lat
return out
before, after = index(before_text), index(after_text)
paired: List[Dict[str, str]] = []
for key in sorted(before.keys() & after.keys()):
op, dtype, size = key
b, a = before[key], after[key]
paired.append({
"fg_op_name": op, "dtype": dtype, "size_detail": size,
"before_lat": f"{b:.6f}", "after_lat": f"{a:.6f}",
"speedup_percent": f"{(b / a - 1.0) * 100.0:.2f}",
})
return paired
def report_ab(before: Path, after: Path, label: str = "") -> List[Dict[str, str]]:
"""打印一个算子的前后对比:逐行 speedup 与整体区间。"""
before_text = before.read_text(encoding="utf-8", errors="replace")
after_text = after.read_text(encoding="utf-8", errors="replace")
rows = ab_rows(before_text, after_text)
name = label or (rows[0]["fg_op_name"] if rows else "?")
if not rows:
print(f"[ab] {name}: 没有可配对的测量行"
f"(before={before} after={after})")
return rows
values = [float(r["speedup_percent"]) for r in rows]
for r in rows:
print(f" {r['dtype']:<16} {float(r['speedup_percent']):+7.2f}% "
f"{r['before_lat']} -> {r['after_lat']} {r['size_detail']}")
fallback = ""
if DIRECT_NO_CUDAGRAPH_RE.search(before_text) or \
DIRECT_NO_CUDAGRAPH_RE.search(after_text):
# 回退行用的是 do_bench 口径,与 cudagraph 行不同表可比性
fallback = " [含 BENCHMARK_DIRECT_NO_CUDAGRAPH 回退行]"
print(f"[ab] {name}: {len(rows)} 行配对,speedup "
f"{min(values):+.2f}% ~ {max(values):+.2f}%{fallback}")
return rows
def main() -> None:
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
p.add_argument("batch_dir", type=Path, help="run_batch.py 的产物目录")
p.add_argument("batch_dir", type=Path, nargs="?",
help="run_batch.py 的产物目录")
p.add_argument("-o", "--output", type=Path, default=None,
help="输出 CSV(默认 <batch_dir>/summary.csv)")
p.add_argument("--ab", nargs=2, type=Path, metavar=("BEFORE", "AFTER"),
help="两份 run.log,按 shape 配对输出该算子的 speedup 区间")
p.add_argument("--label", default="", help="--ab 报告里的算子名")
args = p.parse_args()
if args.ab:
for path in args.ab:
if not path.is_file():
raise SystemExit(f"run.log 不存在: {path}")
report_ab(args.ab[0], args.ab[1], args.label)
return
if args.batch_dir is None:
raise SystemExit("需要 batch_dir,或用 --ab BEFORE AFTER")
if not args.batch_dir.is_dir():
raise SystemExit(f"目录不存在: {args.batch_dir}")
out = summarize(args.batch_dir, args.output)