forked from zhoulin/zl_bench
feat: re-account a sealed MUSA RLC run without re-running it
A sealed run already holds every TTGIR dump and its launch index, so a change in how convert topology is counted can be applied to it directly instead of spending accelerator time. The tool re-reads each cell's dump, recomputes the scoped count, re-runs the campaign classifier against the sealed correctness verdicts, and writes a derived receipt and matrix outside the run root. It refuses to write inside a sealed run and rejects a directory with no campaign receipt. A derived verdict can only demote a topology claim. Promoting one to G or C needs reversed-order timing, which a sealed run holds only for tasks that already showed a reduction, so a reduction appearing for the first time under scoping is reported as X for missing round-b evidence rather than guessed. Run against the sealed 32-workload S5000 campaign, a floor of 1 changes exactly one task: gelu_backward moves from C to B, and the only excluded files in the whole campaign are its five once-launched randn_kernel dumps. A floor of 0 reproduces the sealed counts with no task changed, which is the equivalence that makes the scoped number auditable rather than merely different.
This commit is contained in:
1 parent
8bbea472bb
commit
8a685638bb
2 files changed
+380
No files matched your search
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
CAMPAIGN = HERE.parent
|
||||
SPEC = importlib.util.spec_from_file_location(
|
||||
"reaccount_musa_rlc_topology", CAMPAIGN / "reaccount_musa_rlc_topology.py"
|
||||
)
|
||||
assert SPEC is not None and SPEC.loader is not None
|
||||
TOOL = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(TOOL)
|
||||
|
||||
|
||||
def _write_cell(cell_dir: Path, rows):
|
||||
ttgir = cell_dir / "ttgir"
|
||||
ttgir.mkdir(parents=True)
|
||||
lines = [
|
||||
"\t".join(
|
||||
[
|
||||
"shape", "kernel", "file", "launches", "sweep_launches",
|
||||
"num_warps", "num_stages", "constexprs", "cache_hash",
|
||||
]
|
||||
)
|
||||
]
|
||||
for shape, kernel, name, launches, converts in rows:
|
||||
path = ttgir / shape / kernel / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("ttg.convert_layout\n" * converts, encoding="utf-8")
|
||||
lines.append(
|
||||
"\t".join([shape, kernel, name, str(launches), "0", "4", "3", "{}", "H"])
|
||||
)
|
||||
(ttgir / "index.tsv").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _seal_run(root: Path):
|
||||
"""A minimal sealed run shaped like the gelu_backward task."""
|
||||
(root / "campaign-receipt.json").write_text(
|
||||
json.dumps({"campaign_id": "sealed-v2"}), encoding="utf-8"
|
||||
)
|
||||
task_root = root / "tasks" / "gelu_backward"
|
||||
cells = {}
|
||||
for key, round_name, cell_name, setup, measured in (
|
||||
("a_off", "round-a", "off", 4, 0),
|
||||
("a_mask0", "round-a", "mask0", 4, 0),
|
||||
("a_candidate", "round-a", "candidate", 0, 0),
|
||||
("b_off", "round-b", "off", 4, 0),
|
||||
("b_candidate", "round-b", "candidate", 0, 0),
|
||||
):
|
||||
cell_dir = task_root / round_name / cell_name
|
||||
_write_cell(
|
||||
cell_dir,
|
||||
[
|
||||
("s", "randn_kernel", "w16s3.ttgir", 1, setup),
|
||||
("s", "gelu_backward_kernel", "w4s3.ttgir", 3795, measured),
|
||||
],
|
||||
)
|
||||
cells[key] = {
|
||||
"round": round_name,
|
||||
"cell": cell_name,
|
||||
"status": "PASS",
|
||||
"convert_count": setup + measured,
|
||||
"median_gems_latency_ms": 0.1168 if cell_name == "off" else 0.1166,
|
||||
"replay_fallback_count": 0,
|
||||
}
|
||||
task_root.mkdir(parents=True, exist_ok=True)
|
||||
(task_root / "task-receipt.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"task_id": "gelu_backward",
|
||||
"op": "gelu_backward",
|
||||
"family": "elementwise-backward",
|
||||
"classification": "C",
|
||||
"classification_detail": {
|
||||
"reason": "topology reduction without stable >=5% speedup"
|
||||
},
|
||||
"correctness": {"status": "PASS"},
|
||||
"correctness_isolation": {"candidate": {"status": "PASS"}},
|
||||
"cells": cells,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
class ReaccountTests(unittest.TestCase):
|
||||
def test_input_construction_reduction_is_demoted_to_b(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
base = Path(temporary)
|
||||
run_root = base / "sealed"
|
||||
run_root.mkdir()
|
||||
_seal_run(run_root)
|
||||
result = TOOL.reaccount(run_root, base / "derived", 1)
|
||||
|
||||
self.assertEqual(result["changed_task_ids"], ["gelu_backward"])
|
||||
task = result["tasks"][0]
|
||||
self.assertEqual(task["sealed_classification"], "C")
|
||||
self.assertEqual(task["reaccounted_classification"], "B")
|
||||
self.assertEqual(task["sealed_convert"]["a_off"], 4)
|
||||
self.assertEqual(task["reaccounted_convert"]["a_off"], 0)
|
||||
self.assertEqual(
|
||||
task["excluded_unmeasured_files"], ["s/randn_kernel/w16s3.ttgir"]
|
||||
)
|
||||
|
||||
def test_a_zero_floor_reproduces_the_sealed_verdict(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
base = Path(temporary)
|
||||
run_root = base / "sealed"
|
||||
run_root.mkdir()
|
||||
_seal_run(run_root)
|
||||
result = TOOL.reaccount(run_root, base / "derived", 0)
|
||||
|
||||
# Floor 0 admits the once-launched builder again, so the derived verdict
|
||||
# must match the sealed one. That equivalence is what makes the scoped
|
||||
# count auditable rather than an unexplained new number.
|
||||
self.assertEqual(result["changed_task_ids"], [])
|
||||
self.assertEqual(result["tasks"][0]["reaccounted_classification"], "C")
|
||||
|
||||
def test_derived_output_is_written_and_never_inside_the_sealed_run(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
base = Path(temporary)
|
||||
run_root = base / "sealed"
|
||||
run_root.mkdir()
|
||||
_seal_run(run_root)
|
||||
|
||||
with self.assertRaises(TOOL.ReaccountError):
|
||||
TOOL.reaccount(run_root, run_root / "derived", 1)
|
||||
with self.assertRaises(TOOL.ReaccountError):
|
||||
TOOL.reaccount(run_root, run_root, 1)
|
||||
|
||||
output = base / "derived"
|
||||
TOOL.reaccount(run_root, output, 1)
|
||||
self.assertTrue((output / "reaccount-receipt.json").is_file())
|
||||
self.assertTrue((output / "reaccount-matrix.csv").is_file())
|
||||
# The sealed receipt must survive untouched.
|
||||
sealed = json.loads(
|
||||
(run_root / "tasks" / "gelu_backward" / "task-receipt.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
self.assertEqual(sealed["classification"], "C")
|
||||
self.assertEqual(sealed["cells"]["a_off"]["convert_count"], 4)
|
||||
|
||||
def test_a_directory_without_a_campaign_receipt_is_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
base = Path(temporary)
|
||||
with self.assertRaises(TOOL.ReaccountError):
|
||||
TOOL.reaccount(base, base.parent / "derived", 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user