forked from zhoulin/zl_bench
feat: absorb the MThreads compat plugin and the A/B range report
Three pieces from the fork's feature/musa-benchmark-support branch that never reached main, rebased onto the backend abstraction rather than copied verbatim. _musa_benchmark_compat_plugin drops FlagGems benchmark skip guards written as `torch.cuda.is_available()` where the code under test dispatches through flag_gems.device. On MThreads those turn a working operator into "not measured": test_fft.py skipped entirely, and unskipped on an S5000 it produces 25 rows. The plugin pins each override to one file and one exact skip reason and only acts when flag_gems has dispatched to the mthreads vendor; two guards were probed on device and deliberately left in place, which is why they are named in its docstring rather than silently absent. It is wired through a new DeviceBackend.compat_plugins rather than a name test in the planner, so the policy stays where every other backend difference lives. Compat plugins are planned first because they rewrite collection-time markers and nothing may read the item list before them; a test pins that position and another pins that they do not leak to other backends. batch_summary gains --ab: the speedup column compares gems against torch, not one build against another, so an A/B needs the two run.logs paired by operator, dtype and size detail. Only rows that succeeded on both sides with a positive latency count, so an incomplete pairing cannot look like a complete range, and the range is annotated when either side carries a BENCHMARK_DIRECT_NO_CUDAGRAPH marker, since a fallback row was timed under a different regime. ab_rlc_test.sh is the same caliber as ab_fold_test.sh with the comparison axis moved to FLAGTREE_<VENDOR>_RLC_ENHANCE. Both are caliber examples, not regression scripts: if the switch under test no longer exists the script still runs every round and prints a full table showing no difference, so confirm the axis is live before reusing either. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c9f9b7885f
commit
29af59be9b
8 files changed
+258
-3
No files matched your search
+71
-1
@@ -186,12 +186,82 @@ def summarize(batch_dir: Path, out_path: Optional[Path] = None) -> Path:
|
||||
return out_path
|
||||
|
||||
|
||||
def ab_rows(before_text: str, after_text: str) -> List[Dict[str, str]]:
|
||||
"""把两次 run.log 的测量行按 (op, dtype, size_detail) 配对。
|
||||
|
||||
speedup 列是 gems 对 torch,不是前后对比,所以这里用 gems_lat 自己算
|
||||
before/after - 1。只有两侧都 SUCCESS 且延迟为正的行才计入;对不上的行
|
||||
单独返回,避免用不完整的配对报一个看起来完整的区间。
|
||||
"""
|
||||
def index(text: str) -> Dict[Tuple[str, str, str], float]:
|
||||
out: Dict[Tuple[str, str, str], float] = {}
|
||||
for row in parse_run_log(text):
|
||||
if row.get("row_status") != "SUCCESS":
|
||||
continue
|
||||
try:
|
||||
lat = float(row.get("gems_lat", ""))
|
||||
except ValueError:
|
||||
continue
|
||||
if lat > 0:
|
||||
out[(row.get("fg_op_name", ""), row.get("dtype", ""),
|
||||
row.get("size_detail", ""))] = lat
|
||||
return out
|
||||
|
||||
before, after = index(before_text), index(after_text)
|
||||
paired: List[Dict[str, str]] = []
|
||||
for key in sorted(before.keys() & after.keys()):
|
||||
op, dtype, size = key
|
||||
b, a = before[key], after[key]
|
||||
paired.append({
|
||||
"fg_op_name": op, "dtype": dtype, "size_detail": size,
|
||||
"before_lat": f"{b:.6f}", "after_lat": f"{a:.6f}",
|
||||
"speedup_percent": f"{(b / a - 1.0) * 100.0:.2f}",
|
||||
})
|
||||
return paired
|
||||
|
||||
|
||||
def report_ab(before: Path, after: Path, label: str = "") -> List[Dict[str, str]]:
|
||||
"""打印一个算子的前后对比:逐行 speedup 与整体区间。"""
|
||||
before_text = before.read_text(encoding="utf-8", errors="replace")
|
||||
after_text = after.read_text(encoding="utf-8", errors="replace")
|
||||
rows = ab_rows(before_text, after_text)
|
||||
name = label or (rows[0]["fg_op_name"] if rows else "?")
|
||||
if not rows:
|
||||
print(f"[ab] {name}: 没有可配对的测量行"
|
||||
f"(before={before} after={after})")
|
||||
return rows
|
||||
values = [float(r["speedup_percent"]) for r in rows]
|
||||
for r in rows:
|
||||
print(f" {r['dtype']:<16} {float(r['speedup_percent']):+7.2f}% "
|
||||
f"{r['before_lat']} -> {r['after_lat']} {r['size_detail']}")
|
||||
fallback = ""
|
||||
if DIRECT_NO_CUDAGRAPH_RE.search(before_text) or \
|
||||
DIRECT_NO_CUDAGRAPH_RE.search(after_text):
|
||||
# 回退行用的是 do_bench 口径,与 cudagraph 行不同表可比性
|
||||
fallback = " [含 BENCHMARK_DIRECT_NO_CUDAGRAPH 回退行]"
|
||||
print(f"[ab] {name}: {len(rows)} 行配对,speedup "
|
||||
f"{min(values):+.2f}% ~ {max(values):+.2f}%{fallback}")
|
||||
return rows
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("batch_dir", type=Path, help="run_batch.py 的产物目录")
|
||||
p.add_argument("batch_dir", type=Path, nargs="?",
|
||||
help="run_batch.py 的产物目录")
|
||||
p.add_argument("-o", "--output", type=Path, default=None,
|
||||
help="输出 CSV(默认 <batch_dir>/summary.csv)")
|
||||
p.add_argument("--ab", nargs=2, type=Path, metavar=("BEFORE", "AFTER"),
|
||||
help="两份 run.log,按 shape 配对输出该算子的 speedup 区间")
|
||||
p.add_argument("--label", default="", help="--ab 报告里的算子名")
|
||||
args = p.parse_args()
|
||||
if args.ab:
|
||||
for path in args.ab:
|
||||
if not path.is_file():
|
||||
raise SystemExit(f"run.log 不存在: {path}")
|
||||
report_ab(args.ab[0], args.ab[1], args.label)
|
||||
return
|
||||
if args.batch_dir is None:
|
||||
raise SystemExit("需要 batch_dir,或用 --ab BEFORE AFTER")
|
||||
if not args.batch_dir.is_dir():
|
||||
raise SystemExit(f"目录不存在: {args.batch_dir}")
|
||||
out = summarize(args.batch_dir, args.output)
|
||||
|
||||
Reference in new issue
Block a user