Stabilize first-run cudagraph timing and colorize terminal output
- _cudagraph_plugin: warm up autotune/JIT explicitly before graph capture (internal 5-iter warmup is too short), tag fallback markers with the failing phase; first-run latency no longer jitters run-to-run. - _device_guard_plugin (new): set GEMS_VENDOR via torch probe before importing flag_gems, avoiding its timeout-less nvidia-smi subprocess probe that can hang import in fork-broken environments. - _pretty_report_plugin (new) + _term_style (new): fold inputs identical across all result rows into a legend line, color status/plugin tags/markers on the live terminal; run.log is ANSI-stripped and keeps upstream SUCCESS/column wording for grep compatibility. - run_pytest.sh: make USE_FLAGTUNE overridable, group all knobs into a config section with Chinese comments, add start/end banners. - README: document the warmup semantics, new plugins/env vars, and the A/B rule that both sides must use the same USE_FLAGTUNE.
This commit is contained in:
@@ -92,22 +92,26 @@ def _warmup_load_backend():
|
||||
torch.mm(a, b)
|
||||
torch.cuda.synchronize()
|
||||
except (ImportError, AttributeError) as e:
|
||||
print(f"[mm-cluster-fix-plugin] warmup skipped (benign): {e!r}",
|
||||
from _term_style import tag
|
||||
print(f"{tag('[mm-cluster-fix-plugin]')} warmup skipped (benign): {e!r}",
|
||||
file=sys.stderr, flush=True)
|
||||
except Exception as e: # noqa: BLE001
|
||||
# e.g. a CUDA error — do NOT hide it; the harness needs to see it.
|
||||
print(f"[mm-cluster-fix-plugin] WARNING: warmup mm failed unexpectedly: {e!r}",
|
||||
from _term_style import tag
|
||||
print(f"{tag('[mm-cluster-fix-plugin]')} WARNING: warmup mm failed "
|
||||
f"unexpectedly: {e!r}",
|
||||
file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
from _term_style import tag
|
||||
_patch_loaded()
|
||||
_warmup_load_backend()
|
||||
_patch_loaded()
|
||||
guarded = _guarded_modules()
|
||||
if guarded:
|
||||
print(
|
||||
"[mm-cluster-fix-plugin] guarded cluster_remote_mm_scenario "
|
||||
f"{tag('[mm-cluster-fix-plugin]')} guarded cluster_remote_mm_scenario "
|
||||
f"(odd N-tile unmasked OOB) in: {guarded}",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
@@ -125,7 +129,7 @@ def pytest_configure(config):
|
||||
else:
|
||||
# Non-Hopper: the cluster path never runs, so an unpatched state is fine.
|
||||
print(
|
||||
"[mm-cluster-fix-plugin] no cluster_remote_mm_scenario found; "
|
||||
f"{tag('[mm-cluster-fix-plugin]')} no cluster_remote_mm_scenario found; "
|
||||
"non-Hopper device, guard not needed (no-op).",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
|
||||
Reference in New Issue
Block a user