feat: absorb the MThreads compat plugin and the A/B range report

Three pieces from the fork's feature/musa-benchmark-support branch that never
reached main, rebased onto the backend abstraction rather than copied verbatim.

_musa_benchmark_compat_plugin drops FlagGems benchmark skip guards written as
`torch.cuda.is_available()` where the code under test dispatches through
flag_gems.device. On MThreads those turn a working operator into "not measured":
test_fft.py skipped entirely, and unskipped on an S5000 it produces 25 rows. The
plugin pins each override to one file and one exact skip reason and only acts
when flag_gems has dispatched to the mthreads vendor; two guards were probed on
device and deliberately left in place, which is why they are named in its
docstring rather than silently absent.

It is wired through a new DeviceBackend.compat_plugins rather than a name test
in the planner, so the policy stays where every other backend difference lives.
Compat plugins are planned first because they rewrite collection-time markers
and nothing may read the item list before them; a test pins that position and
another pins that they do not leak to other backends.

batch_summary gains --ab: the speedup column compares gems against torch, not
one build against another, so an A/B needs the two run.logs paired by operator,
dtype and size detail. Only rows that succeeded on both sides with a positive
latency count, so an incomplete pairing cannot look like a complete range, and
the range is annotated when either side carries a BENCHMARK_DIRECT_NO_CUDAGRAPH
marker, since a fallback row was timed under a different regime.

ab_rlc_test.sh is the same caliber as ab_fold_test.sh with the comparison axis
moved to FLAGTREE_<VENDOR>_RLC_ENHANCE. Both are caliber examples, not
regression scripts: if the switch under test no longer exists the script still
runs every round and prints a full table showing no difference, so confirm the
axis is live before reusing either.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
AharrypotterandClaude Opus 5 committed 2026-09-10 10:31:59 +08:00
1 parent c9f9b7885f
commit 29af59be9b
8 files changed
+258 -3

No files matched your search

+71 -1
View File
@@ -186,12 +186,82 @@ def summarize(batch_dir: Path, out_path: Optional[Path] = None) -> Path:
return out_path
def ab_rows(before_text: str, after_text: str) -> List[Dict[str, str]]:
"""把两次 run.log 的测量行按 (op, dtype, size_detail) 配对。
speedup 列是 gems 对 torch,不是前后对比,所以这里用 gems_lat 自己算
before/after - 1。只有两侧都 SUCCESS 且延迟为正的行才计入;对不上的行
单独返回,避免用不完整的配对报一个看起来完整的区间。
"""
def index(text: str) -> Dict[Tuple[str, str, str], float]:
out: Dict[Tuple[str, str, str], float] = {}
for row in parse_run_log(text):
if row.get("row_status") != "SUCCESS":
continue
try:
lat = float(row.get("gems_lat", ""))
except ValueError:
continue
if lat > 0:
out[(row.get("fg_op_name", ""), row.get("dtype", ""),
row.get("size_detail", ""))] = lat
return out
before, after = index(before_text), index(after_text)
paired: List[Dict[str, str]] = []
for key in sorted(before.keys() & after.keys()):
op, dtype, size = key
b, a = before[key], after[key]
paired.append({
"fg_op_name": op, "dtype": dtype, "size_detail": size,
"before_lat": f"{b:.6f}", "after_lat": f"{a:.6f}",
"speedup_percent": f"{(b / a - 1.0) * 100.0:.2f}",
})
return paired
def report_ab(before: Path, after: Path, label: str = "") -> List[Dict[str, str]]:
"""打印一个算子的前后对比:逐行 speedup 与整体区间。"""
before_text = before.read_text(encoding="utf-8", errors="replace")
after_text = after.read_text(encoding="utf-8", errors="replace")
rows = ab_rows(before_text, after_text)
name = label or (rows[0]["fg_op_name"] if rows else "?")
if not rows:
print(f"[ab] {name}: 没有可配对的测量行"
f"(before={before} after={after})")
return rows
values = [float(r["speedup_percent"]) for r in rows]
for r in rows:
print(f" {r['dtype']:<16} {float(r['speedup_percent']):+7.2f}% "
f"{r['before_lat']} -> {r['after_lat']} {r['size_detail']}")
fallback = ""
if DIRECT_NO_CUDAGRAPH_RE.search(before_text) or \
DIRECT_NO_CUDAGRAPH_RE.search(after_text):
# 回退行用的是 do_bench 口径,与 cudagraph 行不同表可比性
fallback = " [含 BENCHMARK_DIRECT_NO_CUDAGRAPH 回退行]"
print(f"[ab] {name}: {len(rows)} 行配对,speedup "
f"{min(values):+.2f}% ~ {max(values):+.2f}%{fallback}")
return rows
def main() -> None:
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
p.add_argument("batch_dir", type=Path, help="run_batch.py 的产物目录")
p.add_argument("batch_dir", type=Path, nargs="?",
help="run_batch.py 的产物目录")
p.add_argument("-o", "--output", type=Path, default=None,
help="输出 CSV(默认 <batch_dir>/summary.csv)")
p.add_argument("--ab", nargs=2, type=Path, metavar=("BEFORE", "AFTER"),
help="两份 run.log,按 shape 配对输出该算子的 speedup 区间")
p.add_argument("--label", default="", help="--ab 报告里的算子名")
args = p.parse_args()
if args.ab:
for path in args.ab:
if not path.is_file():
raise SystemExit(f"run.log 不存在: {path}")
report_ab(args.ab[0], args.ab[1], args.label)
return
if args.batch_dir is None:
raise SystemExit("需要 batch_dir,或用 --ab BEFORE AFTER")
if not args.batch_dir.is_dir():
raise SystemExit(f"目录不存在: {args.batch_dir}")
out = summarize(args.batch_dir, args.output)