feat: make the harness backend-agnostic

Everything about which accelerator a run targets was previously spelled CUDA:
device discovery went through nvidia-smi, pinning through CUDA_VISIBLE_DEVICES,
the vendor guard only recognised NVIDIA, and the -p plugin list was hardcoded in
run_pytest.sh with no way to drop the CUDA-only members. Running the same
operator corpus on another vendor's device therefore meant editing the drivers.

The selection now hangs off one ZL_BACKEND variable (cuda/musa/rocm/ascend, with
the usual aliases):

- backends/ owns per-backend device enumeration, visibility env keys and
  capability flags, so nothing above it names a vendor;
- plugin_plan.py derives the -p list from those capabilities instead of a
  hardcoded array, which is what lets a non-CUDA run omit the graph-capture and
  cluster-fix plugins rather than watch them fail;
- _timing_plugin selects the timer from the same capabilities: CUDA keeps
  graph capture with its per-shape fallback, other backends stay on upstream
  do_bench and record timer=direct, because capture-then-fallback on a device
  without graph support only produces noise;
- _device_setup_plugin replaces the NVIDIA-only vendor guard;
- providers/flaggems.py resolves the pytest node, so a caller may pass TEST_NODE
  directly instead of the OP/OP_FILE pair, and SHAPE_MODE=upstream can defer to
  FlagGems' own core_shapes;
- _run_identity.py writes run_identity.json next to each run, recording backend,
  device, timer, autotune mode and the resolved test node. Two runs whose
  calibers differ are otherwise indistinguishable after the fact.

run_batch gains --backend and --devices (--gpus kept as an alias) and carries
backend/device/timer into ops_status.csv. test_backends.py covers the registry,
alias resolution, visibility env and plugin planning.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
AharrypotterandClaude Opus 5 committed 2026-09-09 16:39:14 +08:00
1 parent 35a928a516
commit c8ea586d0f
24 files changed
+1448 -159

No files matched your search

+24 -3
View File
@@ -39,12 +39,31 @@ DIRECT_NO_CUDAGRAPH_RE = re.compile(r"BENCHMARK_DIRECT_NO_CUDAGRAPH\b")
REPLAY_FALLBACK_RE = re.compile(r"AUTOTUNE_REPLAY_FALLBACK\b")
SUMMARY_FIELDS = [
"repo", "op", "fg_op_name", "dtype", "case_id", "row_status",
"repo", "op", "backend", "device", "timer",
"fg_op_name", "dtype", "case_id", "row_status",
"torch_lat", "gems_lat", "speedup", "extra_metrics", "size_detail",
"op_status", "notes",
]
def _identity_from_status(status_row: Dict[str, str], op_dir: Path) -> Dict[str, str]:
"""Backend/device/timer from ops_status.csv, falling back to run_identity.json."""
backend = (status_row.get("backend") or "").strip()
device = (status_row.get("device") or status_row.get("gpu") or "").strip()
timer = (status_row.get("timer") or "").strip()
ident_path = op_dir / "run_identity.json"
if ident_path.is_file() and not (backend and device and timer):
try:
from _run_identity import load_identity
ident = load_identity(str(op_dir))
backend = backend or str(ident.get("backend") or "")
device = device or str(ident.get("device") or ident.get("gpu") or "")
timer = timer or str(ident.get("timer") or "")
except Exception:
pass
return {"backend": backend, "device": device, "timer": timer}
def parse_run_log(text: str) -> List[Dict[str, str]]:
"""把 run.log 里的全部结果表解析为行列表(跨多个 dtype 表)。"""
rows: List[Dict[str, str]] = []
@@ -133,10 +152,11 @@ def summarize(batch_dir: Path, out_path: Optional[Path] = None) -> Path:
text = log.read_text(errors="replace")
status_row = status_by_dir.get(f"{repo}/{op_dir.name}", {})
op = status_row.get("op") or op_dir.name
ident = _identity_from_status(status_row, op_dir)
table_rows = parse_run_log(text)
if not table_rows:
all_rows.append({
"repo": repo, "op": op, "fg_op_name": "", "dtype": "",
"repo": repo, "op": op, **ident, "fg_op_name": "", "dtype": "",
"case_id": "", "row_status": "", "torch_lat": "",
"gems_lat": "", "speedup": "", "extra_metrics": "",
"size_detail": "",
@@ -150,7 +170,8 @@ def summarize(batch_dir: Path, out_path: Optional[Path] = None) -> Path:
idx = case_counter.get(key, 0)
case_counter[key] = idx + 1
all_rows.append({
"repo": repo, "op": op, "case_id": f"case_{idx:03d}",
"repo": repo, "op": op, **ident,
"case_id": f"case_{idx:03d}",
"op_status": status_row.get("status", ""),
"notes": status_row.get("notes", ""),
**row,