forked from zhoulin/zl_bench
Aharrypotter
8bbea472bb
A frozen 32-workload campaign that measures how FlagTree's RemoveLayoutConversions pass behaves on a Moore Threads S5000, through FlagGems public dispatch and this repository's existing plugins. The manifest is the contract: source commits, the single physical device, the three screen cells off/mask0/candidate, the native-event timer, and per-workload dtypes, timeouts and canary membership. prepare validates it against ops/inventory_main.csv and freezes an immutable task matrix; the runner executes the matrix serially, and the worker drives one operator cell through pytest with _ir_meta_plugin, _autotune_record_plugin and the rest. Three properties are worth calling out because they decide whether a verdict means anything. Convert topology is scoped to the measured operator. A benchmark call builds its inputs once and then runs the operator many times under the timer, so counting ttg.convert_layout over every executed kernel credits the candidate with converts removed from input construction that no timer can observe. Each dumped TTGIR file is joined to its launch count from index.tsv and only variants launched more than measured_launch_floor contribute; the unscoped total and the excluded files stay in the receipt, and a cell that cannot be joined to its index is classified X rather than trusted. Correctness is scoped to the variant a workload measures. Running a whole test file lets an unrelated variant decide the verdict, so a workload may pin node ids or a -k selector, never both, and any narrowing must state its reason. Targets are resolved before the campaign starts. The inventory can lag the pinned FlagGems commit, and such a task previously failed only inside the worker's preflight, which exited silently and left an empty log. The runner now resolves every benchmark and correctness target against the real tree, a workload may name a target the inventory does not know about if it says why, and each preflight check names itself on failure.
552 lines
17 KiB
JSON
552 lines
17 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"campaign_id": "s5000-musa-rlc-32-product-v1-20260902-v3",
|
|
"experiment_kind": "musa-rlc-product-validation",
|
|
"permitted_claim": "S5000 MUSA product-dispatch correctness, layout-topology response, and same-contract native-event latency for this frozen corpus only",
|
|
"production_default_policy": "unchanged-off",
|
|
"source": {
|
|
"flagtree_commit": "3059917e2df16f8ea35c5efe1ed1af2d87117396",
|
|
"flaggems_commit": "46ef8080329190d8da8738340240de2d5a475950",
|
|
"zl_bench_commit": "da2288564584039366e7be07fe4e94539a560c81"
|
|
},
|
|
"device": {
|
|
"ssh_alias": "S5000",
|
|
"container": "zixu_flagos_S5000",
|
|
"backend": "mthreads",
|
|
"model": "Moore Threads S5000",
|
|
"physical_device": "7",
|
|
"logical_device": "0",
|
|
"visible_device_count": 1
|
|
},
|
|
"protocol": {
|
|
"screen_cells": [
|
|
"off",
|
|
"mask0",
|
|
"candidate"
|
|
],
|
|
"timing_cells": [
|
|
"off",
|
|
"candidate"
|
|
],
|
|
"canary_order": [
|
|
"dropout",
|
|
"softmax",
|
|
"instance_norm",
|
|
"fft",
|
|
"sort",
|
|
"batch_norm",
|
|
"topk",
|
|
"conv1d"
|
|
],
|
|
"candidate_phase_mask": 13,
|
|
"candidate_profitability_policy": true,
|
|
"candidate_product_launch_count": 1,
|
|
"default_benchmark_dtypes": [
|
|
"float16"
|
|
],
|
|
"timer": "triton.testing.do_bench via MUSA device-interface Event",
|
|
"autotune_contract": "off-record then mask0/candidate replay from off",
|
|
"fresh_process_per_operator_cell": true,
|
|
"same_physical_device_for_all_cells": true,
|
|
"serial_measurement": true,
|
|
"full_campaign_requires_canary_terminal": true,
|
|
"correctness": "FlagGems public dispatch quick tests under candidate, narrowed to the variant each workload measures and run with the MThreads compatibility plugin, plus benchmark execution for every cell",
|
|
"classification": {
|
|
"G": "correct, convert count decreases, and stable speedup >= 5%",
|
|
"C": "correct and convert count decreases, but latency intervals overlap or speedup < 5%",
|
|
"B": "correct but candidate does not reduce the executed convert topology",
|
|
"R": "correct but stable regression <= -3%",
|
|
"K": "candidate correctness failure",
|
|
"X": "unsupported, timeout, harness failure, or same-contract replay violation"
|
|
},
|
|
"timeout_seconds_by_cost": {
|
|
"light": 300,
|
|
"medium": 900,
|
|
"heavy": 1800
|
|
},
|
|
"measured_launch_floor": 1,
|
|
"supported_benchmark_dtypes": [
|
|
"float16",
|
|
"float32",
|
|
"bfloat16",
|
|
"int16",
|
|
"int32",
|
|
"bool",
|
|
"complex64"
|
|
]
|
|
},
|
|
"workloads": [
|
|
{
|
|
"id": "batch_norm",
|
|
"repo": "main",
|
|
"op": "batch_norm",
|
|
"op_file": "batch_norm",
|
|
"shape_key": "batch_norm",
|
|
"correctness_file": "tests/test_batch_norm.py",
|
|
"family": "normalization",
|
|
"nv_evidence": "strong-positive",
|
|
"nv_gain_percent": 43.1,
|
|
"prior_s5000": "B 0->0",
|
|
"cost": "medium",
|
|
"canary": true
|
|
},
|
|
{
|
|
"id": "replication_pad3d",
|
|
"repo": "main",
|
|
"op": "replication_pad3d",
|
|
"op_file": "replication_pad3d",
|
|
"shape_key": "replication_pad3d",
|
|
"correctness_file": "tests/test_replication_pad3d.py",
|
|
"family": "padding",
|
|
"nv_evidence": "strong-positive",
|
|
"nv_gain_percent": 38.6,
|
|
"prior_s5000": "B 0->0 at author shape",
|
|
"cost": "medium",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "fft",
|
|
"repo": "main",
|
|
"op": "perf_fft",
|
|
"op_file": "fft",
|
|
"shape_key": "fft",
|
|
"correctness_file": "tests/test_fft.py",
|
|
"family": "fft",
|
|
"nv_evidence": "strong-positive",
|
|
"nv_gain_percent": 36.5,
|
|
"prior_s5000": "G 38->2",
|
|
"cost": "heavy",
|
|
"canary": true,
|
|
"benchmark_dtypes": [
|
|
"complex64"
|
|
]
|
|
},
|
|
{
|
|
"id": "topk",
|
|
"repo": "main",
|
|
"op": "topk",
|
|
"op_file": "topk",
|
|
"shape_key": "topk",
|
|
"correctness_file": "tests/test_topk.py",
|
|
"family": "selection",
|
|
"nv_evidence": "strong-positive-shared-network-risk",
|
|
"nv_gain_percent": 20.8,
|
|
"prior_s5000": "B 0->0 at large author shape",
|
|
"cost": "medium",
|
|
"canary": true
|
|
},
|
|
{
|
|
"id": "instance_norm",
|
|
"repo": "main",
|
|
"op": "instance_norm",
|
|
"op_file": "instance_norm",
|
|
"shape_key": "instance_norm",
|
|
"correctness_file": "tests/test_instance_norm.py",
|
|
"family": "normalization",
|
|
"nv_evidence": "strong-positive",
|
|
"nv_gain_percent": 17.8,
|
|
"prior_s5000": "G 1->0",
|
|
"cost": "medium",
|
|
"canary": true
|
|
},
|
|
{
|
|
"id": "dropout",
|
|
"repo": "main",
|
|
"op": "dropout",
|
|
"op_file": "dropout",
|
|
"shape_key": "dropout",
|
|
"correctness_file": "tests/test_dropout.py",
|
|
"family": "rng",
|
|
"nv_evidence": "strong-positive-shared-network-risk",
|
|
"nv_gain_percent": 15.2,
|
|
"prior_s5000": "product G 8->0 +9.15%",
|
|
"cost": "light",
|
|
"canary": true
|
|
},
|
|
{
|
|
"id": "router_gemm",
|
|
"repo": "main",
|
|
"op": "perf_router_gemm",
|
|
"op_file": "router_gemm",
|
|
"shape_key": "router_gemm",
|
|
"correctness_file": "tests/test_router_gemm.py",
|
|
"family": "mma",
|
|
"nv_evidence": "strong-positive-mma",
|
|
"nv_gain_percent": 14.5,
|
|
"prior_s5000": "public screen available; product v1 pending",
|
|
"cost": "heavy",
|
|
"canary": false,
|
|
"benchmark_dtypes": [
|
|
"bfloat16"
|
|
]
|
|
},
|
|
{
|
|
"id": "fused_recurrent_gated_delta_rule",
|
|
"repo": "main",
|
|
"op": "perf_fused_recurrent_gated_delta_rule",
|
|
"op_file": "FLA/test_fused_recurrent_gated_delta_rule_perf",
|
|
"shape_key": null,
|
|
"correctness_file": "tests/test_FLA/test_fused_recurrent_gated_delta_rule.py",
|
|
"family": "recurrent",
|
|
"nv_evidence": "strong-positive-recurrent",
|
|
"nv_gain_percent": 14.2,
|
|
"prior_s5000": "not product-qualified",
|
|
"cost": "heavy",
|
|
"canary": false,
|
|
"benchmark_dtypes": [
|
|
"bfloat16"
|
|
]
|
|
},
|
|
{
|
|
"id": "top_k_per_row_decode",
|
|
"repo": "main",
|
|
"op": "top_k_per_row_decode",
|
|
"op_file": "top_k_per_row_decode",
|
|
"shape_key": "top_k_per_row_decode",
|
|
"correctness_file": "tests/test_top_k_per_row_decode.py",
|
|
"family": "selection",
|
|
"nv_evidence": "strong-positive-shared-network-risk",
|
|
"nv_gain_percent": 13.0,
|
|
"prior_s5000": "not product-qualified",
|
|
"cost": "medium",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "sort",
|
|
"repo": "main",
|
|
"op": "perf_sort",
|
|
"op_file": "sort",
|
|
"shape_key": "sort",
|
|
"correctness_file": "tests/test_sort.py",
|
|
"family": "sort",
|
|
"nv_evidence": "strong-positive-shared-network-risk",
|
|
"nv_gain_percent": 12.3,
|
|
"prior_s5000": "G 2->1",
|
|
"cost": "medium",
|
|
"canary": true
|
|
},
|
|
{
|
|
"id": "sort_stable",
|
|
"repo": "main",
|
|
"op": "perf_sort_stable",
|
|
"op_file": "sort",
|
|
"shape_key": "sort_stable",
|
|
"correctness_file": "tests/test_sort.py",
|
|
"family": "sort",
|
|
"nv_evidence": "strong-positive-shared-network-risk",
|
|
"nv_gain_percent": 12.3,
|
|
"prior_s5000": "G 2->1",
|
|
"cost": "medium",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "uniform",
|
|
"repo": "main",
|
|
"op": "uniform_",
|
|
"op_file": "uniform_",
|
|
"shape_key": "uniform_",
|
|
"correctness_file": "tests/test_uniform_.py",
|
|
"family": "rng",
|
|
"nv_evidence": "strong-positive-rng",
|
|
"nv_gain_percent": 10.3,
|
|
"prior_s5000": "public boundary risk; product v1 pending",
|
|
"cost": "light",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "rand",
|
|
"repo": "main",
|
|
"op": "rand",
|
|
"op_file": "rand",
|
|
"shape_key": "rand",
|
|
"correctness_file": "tests/test_rand.py",
|
|
"family": "rng",
|
|
"nv_evidence": "strong-positive-rng",
|
|
"nv_gain_percent": 8.7,
|
|
"prior_s5000": "product v1 pending",
|
|
"cost": "light",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "rand_like",
|
|
"repo": "main",
|
|
"op": "rand_like",
|
|
"op_file": "rand_like",
|
|
"shape_key": "rand_like",
|
|
"correctness_file": "tests/test_rand_like.py",
|
|
"family": "rng",
|
|
"nv_evidence": "strong-positive-rng",
|
|
"nv_gain_percent": 8.6,
|
|
"prior_s5000": "product v1 pending",
|
|
"cost": "light",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "max_pool2d_with_indices",
|
|
"repo": "main",
|
|
"op": "max_pool2d_with_indices",
|
|
"op_file": "max_pool2d",
|
|
"shape_key": "max_pool2d_with_indices",
|
|
"correctness_file": "tests/test_max_pool2d.py",
|
|
"family": "reduction-index",
|
|
"nv_evidence": "positive-reduction-index",
|
|
"nv_gain_percent": 6.1,
|
|
"prior_s5000": "B 0->0",
|
|
"cost": "medium",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "randperm",
|
|
"repo": "main",
|
|
"op": "randperm",
|
|
"op_file": "randperm",
|
|
"shape_key": "randperm",
|
|
"correctness_file": "tests/test_randperm.py",
|
|
"family": "rng",
|
|
"nv_evidence": "small-positive-rng",
|
|
"nv_gain_percent": 4.0,
|
|
"prior_s5000": "C 7->1 latency overlap",
|
|
"cost": "medium",
|
|
"canary": false,
|
|
"benchmark_dtypes": [
|
|
"int32"
|
|
]
|
|
},
|
|
{
|
|
"id": "randn",
|
|
"repo": "main",
|
|
"op": "randn",
|
|
"op_file": "randn",
|
|
"shape_key": "randn",
|
|
"correctness_file": "tests/test_randn.py",
|
|
"family": "rng",
|
|
"nv_evidence": "diluted-positive-rng",
|
|
"nv_gain_percent": 4.0,
|
|
"prior_s5000": "product v1 pending",
|
|
"cost": "light",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "randn_like",
|
|
"repo": "main",
|
|
"op": "randn_like",
|
|
"op_file": "randn_like",
|
|
"shape_key": "randn_like",
|
|
"correctness_file": "tests/test_randn_like.py",
|
|
"family": "rng",
|
|
"nv_evidence": "diluted-positive-rng",
|
|
"nv_gain_percent": 4.0,
|
|
"prior_s5000": "product v1 pending",
|
|
"cost": "light",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "upsample_bicubic2d_aa",
|
|
"repo": "main",
|
|
"op": "upsample_bicubic2d_aa",
|
|
"op_file": "upsample_bicubic2d_aa",
|
|
"shape_key": "upsample_bicubic2d_aa",
|
|
"correctness_file": "tests/test_upsample_bicubic2d_aa.py",
|
|
"family": "interpolation",
|
|
"nv_evidence": "diluted-positive-cold-path",
|
|
"nv_gain_percent": 1.7,
|
|
"prior_s5000": "C 10->0 latency flat",
|
|
"cost": "medium",
|
|
"canary": false,
|
|
"correctness_selector": "not backward",
|
|
"correctness_scope_reason": "the benchmarked op is the forward path (benchmark/test_upsample_bicubic2d_aa.py::test_upsample_bicubic2d_aa); the backward baseline defect fails identically under off/mask0/candidate and is tracked as a separate MUSA numerics issue"
|
|
},
|
|
{
|
|
"id": "grid_sample",
|
|
"repo": "main",
|
|
"op": "grid_sample",
|
|
"op_file": "grid_sample",
|
|
"shape_key": "grid_sample",
|
|
"correctness_file": "tests/test_grid_sample.py",
|
|
"family": "sampling",
|
|
"nv_evidence": "structure-change-no-win",
|
|
"nv_gain_percent": 0.0,
|
|
"prior_s5000": "B 0->0",
|
|
"cost": "medium",
|
|
"canary": false,
|
|
"benchmark_selector": "dtype0",
|
|
"benchmark_scope_reason": "benchmark/test_grid_sample.py parametrises over consts.FLOAT_DTYPES and declares dtypes=[dtype] per case, so a campaign-wide --dtypes float16 filter makes the float32 and bfloat16 cases raise; dtype0 is the float16 case this workload measures"
|
|
},
|
|
{
|
|
"id": "max_pool2d_backward",
|
|
"repo": "main",
|
|
"op": "max_pool2d_backward",
|
|
"op_file": "max_pool2d",
|
|
"shape_key": "max_pool2d_backward",
|
|
"correctness_file": "tests/test_max_pool2d.py",
|
|
"family": "reduction-index",
|
|
"nv_evidence": "structure-change-no-win",
|
|
"nv_gain_percent": 0.0,
|
|
"prior_s5000": "B 0->0",
|
|
"cost": "medium",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "conv1d",
|
|
"repo": "main",
|
|
"op": "conv1d",
|
|
"op_file": "conv1d",
|
|
"shape_key": "conv1d",
|
|
"correctness_file": "tests/test_conv1d.py",
|
|
"family": "convolution",
|
|
"nv_evidence": "light-regression-review",
|
|
"nv_gain_percent": -4.2,
|
|
"prior_s5000": "C 1->0 via phase3, slight slowdown",
|
|
"cost": "heavy",
|
|
"canary": true
|
|
},
|
|
{
|
|
"id": "layer_norm_backward",
|
|
"repo": "main",
|
|
"op": "layer_norm_backward",
|
|
"op_file": "layer_norm",
|
|
"shape_key": "layer_norm_backward",
|
|
"correctness_file": "tests/test_layer_norm.py",
|
|
"family": "normalization",
|
|
"nv_evidence": "light-regression-review",
|
|
"nv_gain_percent": -1.9,
|
|
"prior_s5000": "C 4->2 latency flat",
|
|
"cost": "medium",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "cutlass_scaled_mm",
|
|
"repo": "main",
|
|
"op": "cutlass_scaled_mm_benchmark",
|
|
"op_file": "cutlass_scaled_mm",
|
|
"shape_key": "cutlass_scaled_mm",
|
|
"correctness_file": "tests/test_cutlass_scaled_mm.py",
|
|
"family": "mma",
|
|
"nv_evidence": "light-regression-mma",
|
|
"nv_gain_percent": -1.2,
|
|
"prior_s5000": "likely unsupported CUDA-specific control",
|
|
"cost": "heavy",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "gelu_backward",
|
|
"repo": "main",
|
|
"op": "gelu_backward",
|
|
"op_file": "gelu",
|
|
"shape_key": "gelu_backward",
|
|
"correctness_file": "tests/test_gelu.py",
|
|
"family": "elementwise-backward",
|
|
"nv_evidence": "direct-no-cudagraph-noise",
|
|
"nv_gain_percent": null,
|
|
"prior_s5000": "B 0->0 with LLVM optimizations",
|
|
"cost": "medium",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "fused_marlin_moe_mxfp4",
|
|
"repo": "main",
|
|
"op": "fused_marlin_moe_w4a16_mxfp4",
|
|
"op_file": "fused_marlin_moe_w4a16_mxfp4",
|
|
"shape_key": null,
|
|
"correctness_file": "tests/test_fused_marlin_moe.py",
|
|
"family": "quantized-moe",
|
|
"nv_evidence": "autotune-config-change",
|
|
"nv_gain_percent": null,
|
|
"prior_s5000": "likely unsupported CUDA-specific control",
|
|
"cost": "heavy",
|
|
"canary": false,
|
|
"correctness_selector": "mxfp4",
|
|
"correctness_scope_reason": "tests/test_fused_marlin_moe.py also holds w4a16_int4 and w8a16_int8 variants; only the mxfp4 variant corresponds to this workload, and the int8 failures are a separate baseline defect",
|
|
"inventory_override_reason": "zl_bench da2288564584039366e7be07fe4e94539a560c81 lists benchmark/test_fused_marlin_moe.py, but FlagGems 46ef8080329190d8da8738340240de2d5a475950 split it into per-quantisation files; the mxfp4 target is benchmark/test_fused_marlin_moe_w4a16_mxfp4.py::test_fused_marlin_moe_w4a16_mxfp4"
|
|
},
|
|
{
|
|
"id": "bincount",
|
|
"repo": "main",
|
|
"op": "bincount",
|
|
"op_file": "bincount",
|
|
"shape_key": "bincount",
|
|
"correctness_file": "tests/test_bincount.py",
|
|
"family": "histogram",
|
|
"nv_evidence": "autotune-config-change",
|
|
"nv_gain_percent": null,
|
|
"prior_s5000": "product v1 pending",
|
|
"cost": "medium",
|
|
"canary": false,
|
|
"benchmark_dtypes": [
|
|
"float32"
|
|
]
|
|
},
|
|
{
|
|
"id": "masked_select",
|
|
"repo": "main",
|
|
"op": "masked_select",
|
|
"op_file": "masked_select",
|
|
"shape_key": "masked_select",
|
|
"correctness_file": "tests/test_masked_select.py",
|
|
"family": "selection",
|
|
"nv_evidence": "holdout-shared-network",
|
|
"nv_gain_percent": null,
|
|
"prior_s5000": "public qualify available",
|
|
"cost": "medium",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "mm",
|
|
"repo": "main",
|
|
"op": "mm",
|
|
"op_file": "mm",
|
|
"shape_key": "mm",
|
|
"correctness_file": "tests/test_mm.py",
|
|
"family": "mma-control",
|
|
"nv_evidence": "holdout-mma-control",
|
|
"nv_gain_percent": null,
|
|
"prior_s5000": "control only",
|
|
"cost": "heavy",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "scaled_dot_product_attention",
|
|
"repo": "main",
|
|
"op": "scaled_dot_product_attention",
|
|
"op_file": "scaled_dot_product_attention",
|
|
"shape_key": "scaled_dot_product_attention",
|
|
"correctness_file": "tests/test_scaled_dot_product_attention.py",
|
|
"family": "attention-control",
|
|
"nv_evidence": "holdout-attention-control",
|
|
"nv_gain_percent": null,
|
|
"prior_s5000": "control only",
|
|
"cost": "heavy",
|
|
"canary": false
|
|
},
|
|
{
|
|
"id": "softmax",
|
|
"repo": "main",
|
|
"op": "softmax",
|
|
"op_file": "softmax",
|
|
"shape_key": "softmax",
|
|
"correctness_file": "tests/test_softmax.py",
|
|
"family": "reduction-control",
|
|
"nv_evidence": "holdout-reduction-control",
|
|
"nv_gain_percent": null,
|
|
"prior_s5000": "product B 0->0",
|
|
"cost": "light",
|
|
"canary": true
|
|
},
|
|
{
|
|
"id": "layer_norm",
|
|
"repo": "main",
|
|
"op": "layer_norm",
|
|
"op_file": "layer_norm",
|
|
"shape_key": "layer_norm",
|
|
"correctness_file": "tests/test_layer_norm.py",
|
|
"family": "normalization-control",
|
|
"nv_evidence": "holdout-normalization-control",
|
|
"nv_gain_percent": null,
|
|
"prior_s5000": "control only",
|
|
"cost": "medium",
|
|
"canary": false
|
|
}
|
|
],
|
|
"predecessor_campaign_id": "s5000-musa-rlc-32-product-v1-20260901-v2"
|
|
}
|