From ee5f70fa896af14159af86f2f153dddd8ee6c15b Mon Sep 17 00:00:00 2001 From: Aharrypotter Date: Thu, 10 Sep 2026 16:00:01 +0800 Subject: [PATCH] bench: record device clocks, temperature and power for the whole campaign A 6-round A/B here reports side noise at the 0.01% level. Clock throttling and thermal drift move latency by far more than that and leave no trace in run.log, so the report comes out looking exactly as clean as a run on a stable machine. The NV run that reproduced PR763 sampled nvidia-smi every 5s alongside the campaign; the MUSA campaigns had no equivalent, and the parallel plan makes it worse by heating the whole box. telemetry.py samples through the backend, so cuda and musa emit the same columns (index, sm_clock_mhz, mem_clock_mhz, temp_c, power_w, util_pct) and two machines' campaigns can be read side by side. mthreads-gmi offers no CSV query mode, so the MUSA path parses --json and strips the units glued onto each value ("36C", "1888MHz", "477.68W"); its "Power Draw " key carries a trailing space, matched by stripping rather than by literal. The device comes from the backend's own visibility variable when not pinned. A list of devices resolves to no -i at all, sampling every one, rather than silently picking the first. Dropping the placeholder drops its flag too: a bare trailing -i fails the sampler instead of widening it. Telemetry is evidence, not a gate. A missing or unpermitted sampler exits quietly with its reason in telemetry.err and the campaign runs unchanged. --- README.md | 13 +++++++ ab_repeat.sh | 15 ++++++++ backends/base.py | 28 ++++++++++++++ backends/cuda.py | 21 ++++++++++ backends/musa.py | 49 ++++++++++++++++++++++++ telemetry.py | 99 ++++++++++++++++++++++++++++++++++++++++++++++++ test_backends.py | 73 +++++++++++++++++++++++++++++++++++ 7 files changed, 298 insertions(+) create mode 100644 telemetry.py diff --git a/README.md b/README.md index 826003f..3391c47 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,19 @@ failed replay_fallback no_cudagraph capture_poisoned`。README 要求必查的 想起来的时候才发生。每格另有 `CELL_TIMEOUT`（默认 2400 秒）兜底，超时记 `rc=124` 并打 `!!! CELL_TIMEOUT`，不会让一格卡死拖住整批。 +全程还采一份 `/telemetry.tsv`： + +``` +timestamp index sm_clock_mhz mem_clock_mhz temp_c power_w util_pct +2026-09-10T07:41:12 7 1888 2500 36 477.68 87 +``` + +列在 cuda 与 musa 上完全一致（`nvidia-smi --query-gpu` 与 `mthreads-gmi -q --json` +各自解析后归一），所以两台机器的 campaign 可以并排读。理由很直接：六轮 A/B 声称的是 +0.01% 量级的单侧噪声，**降频和热漂移能移动比这大得多的延迟，而且在 run.log 里不留 +任何痕迹**——报告照样干净。`TELEMETRY_INTERVAL`（默认 5 秒）设 0 关闭；采样器缺失或 +没权限时 `telemetry.py` 安静退出并把原因写进 `telemetry.err`，不影响测试本身。 + 第 1 轮 A/B 跑完后，脚本比两侧 `ttgir/` 的指纹（相对路径 + 内容）。**逐字节相同就 跳过这个算子**，并打 `!!! SWITCH_NO_IR_DIFF op=... switch=...`：这个开关没改变该算子的 生成代码，剩下 N-1 轮测的全是噪声，最后还会报成一个"无差异"的结论。先查开关名拼写 diff --git a/ab_repeat.sh b/ab_repeat.sh index 6e70c1c..44ca537 100644 --- a/ab_repeat.sh +++ b/ab_repeat.sh @@ -57,6 +57,19 @@ ROOT="runs/ab_repeat_${SWITCH}_$(date +%Y%m%d-%H%M%S)" mkdir -p "$ROOT" LOG="$ROOT/summary.log" STATUS="$ROOT/status.tsv" + +# 全程采时钟/温度/功耗。六轮 A/B 声称的是 0.01% 量级的单侧噪声，降频和热漂移 +# 能移动比这大得多的延迟，而且在 run.log 里不留痕迹——报告照样干净。 +# TELEMETRY_INTERVAL=0 关闭。采不到时 telemetry.py 自己安静退出，不影响测试。 +TELEMETRY_INTERVAL="${TELEMETRY_INTERVAL:-5}" +TELE_PID="" +if [[ "$TELEMETRY_INTERVAL" != "0" ]]; then + python telemetry.py --backend "${ZL_BACKEND:-cuda}" --device "${ZL_DEVICE:-}" \ + --interval "$TELEMETRY_INTERVAL" --run \ + > "$ROOT/telemetry.tsv" 2> "$ROOT/telemetry.err" & + TELE_PID=$! + trap '[[ -n "$TELE_PID" ]] && kill "$TELE_PID" 2>/dev/null' EXIT +fi printf 'round\top\tside\tswitch\trc\telapsed_s\tsuccess\tfailed\treplay_fallback\tno_cudagraph\tcapture_poisoned\n' > "$STATUS" # 跑一格并逐格登记。事后 grep 也能拿到这些数，但只在有人想起来时才发生； @@ -157,5 +170,7 @@ sed -i -E $'s/\x1b\\[[0-9;]*[A-Za-z]//g' "$LOG" echo ">>> 总 log: $(pwd)/$LOG" echo ">>> 逐格状态表: $(pwd)/$STATUS" +[[ -s "$ROOT/telemetry.tsv" ]] && echo ">>> 时钟/温度: $(pwd)/$ROOT/telemetry.tsv" +[[ -s "$ROOT/telemetry.err" ]] && echo ">>> 遥测提示: $(cat "$ROOT/telemetry.err")" echo ">>> 逐算子重出报告: python batch_summary.py --ab-rounds $ROOT/ --label " echo ">>> 解读要点: 逐轮区间宽说明判定不稳；单侧噪声大说明机器不稳；两者含义不同" diff --git a/backends/base.py b/backends/base.py index c1afae7..21618a0 100644 --- a/backends/base.py +++ b/backends/base.py @@ -72,6 +72,34 @@ class DeviceBackend: # backend. Skipped up front so one of them cannot cost a whole run. capture_deny_ops: Tuple[str, ...] = () + # One-shot device telemetry sampler. Empty means this backend has none, and + # the campaign runs without a clock/thermal record rather than pretending. + # "{dev}" is dropped from the argv when no device is pinned, so the sampler + # covers every visible device instead of guessing an index. + telemetry_argv: Tuple[str, ...] = () + + def telemetry_command(self, device: str = "") -> List[str]: + if not self.telemetry_argv: + return [] + out: List[str] = [] + for a in self.telemetry_argv: + if "{dev}" in a: + if not device: + # Drop the flag the placeholder belonged to as well; a bare + # trailing "-i" makes the sampler fail instead of widening. + if out and out[-1].startswith("-"): + out.pop() + continue + a = a.replace("{dev}", device) + out.append(a) + return out + + def parse_telemetry(self, raw: str) -> List[Dict[str, str]]: + """One dict per device: index, sm_clock_mhz, mem_clock_mhz, temp_c, + power_w, util_pct. Normalised so a CUDA and a MUSA campaign produce the + same columns and can be read side by side.""" + return [] + def classify_capture_error(self, exc: BaseException) -> str: """-> "fatal" | "poisoned" | "recoverable". diff --git a/backends/cuda.py b/backends/cuda.py index 0c3ae54..8429382 100644 --- a/backends/cuda.py +++ b/backends/cuda.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Dict, List + from .base import DeviceBackend @@ -26,3 +28,22 @@ class CudaBackend(DeviceBackend): # No CUDA wording is known to poison the context without also matching one # of the fatal markers above, so nothing is listed here rather than guessed. poisoning_error_markers = () + + telemetry_argv = ( + "nvidia-smi", + "--query-gpu=index,clocks.sm,clocks.mem,temperature.gpu,power.draw,utilization.gpu", + "--format=csv,noheader,nounits", + "-i", "{dev}", + ) + + def parse_telemetry(self, raw: str) -> List[Dict[str, str]]: + out: List[Dict[str, str]] = [] + for line in raw.splitlines(): + parts = [p.strip() for p in line.split(",")] + if len(parts) != 6 or not parts[0].isdigit(): + continue + out.append(dict(zip( + ("index", "sm_clock_mhz", "mem_clock_mhz", + "temp_c", "power_w", "util_pct"), parts))) + return out + diff --git a/backends/musa.py b/backends/musa.py index 066af8d..a0dbfbf 100644 --- a/backends/musa.py +++ b/backends/musa.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Dict, List + from .base import DeviceBackend @@ -50,3 +52,50 @@ class MusaBackend(DeviceBackend): # Device count comes from torch.musa or MUSA_VISIBLE_DEVICES. Do not guess # a vendor CLI here; confirm the real listing tool before adding it. list_command = None + + # mthreads-gmi has no CSV query mode; --json is the only machine-readable + # form it offers. Values arrive with their units glued on ("36C", "1888MHz", + # "477.68W", "87%"), so the parser strips trailing non-numerics rather than + # trusting a fixed suffix. + telemetry_argv = ( + "mthreads-gmi", "-q", + "-d", "TEMPERATURE,CLOCK,POWER,UTILIZATION", + "--json", + "-i", "{dev}", + ) + + def parse_telemetry(self, raw: str) -> List[Dict[str, str]]: + import json + import re + + def num(v): + m = re.match(r"^\s*(-?[\d.]+)", str(v or "")) + return m.group(1) if m else "" + + start = raw.find("{") + if start < 0: + return [] + try: + data = json.loads(raw[start:]) + except Exception: + return [] + gpus = data.get("GPU") or [] + if isinstance(gpus, dict): + gpus = [gpus] + out: List[Dict[str, str]] = [] + for g in gpus: + clocks = g.get("Clocks") or {} + power = g.get("Power Readings") or {} + # 键名带尾随空格（"Power Draw "），别按字面写死 + draw = next((v for k, v in power.items() + if k.strip() == "Power Draw"), "") + out.append({ + "index": str(g.get("Index", "")).strip(), + "sm_clock_mhz": num(clocks.get("Graphics")), + "mem_clock_mhz": num(clocks.get("Memory")), + "temp_c": num((g.get("Temperature") or {}).get("GPU Current Temp")), + "power_w": num(draw), + "util_pct": num((g.get("Utilization") or {}).get("Gpu")), + }) + return out + diff --git a/telemetry.py b/telemetry.py new file mode 100644 index 0000000..0ba1fdd --- /dev/null +++ b/telemetry.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""采样设备时钟/温度/功耗，跨后端归一化成同一组列。 + +为什么要有它：六轮 A/B 会声称单侧噪声在 0.01% 量级。降频和热漂移足以移动比这大 +得多的延迟，而且在 run.log 里不留任何痕迹——报告照样干净。NV 上复现 PR763 的那次 +全程 `nvidia-smi -l 5` 写 clocks.csv，S5000 这边一直没有对应的东西。 + +输出每设备每次采样一行 TSV： + + timestamp index sm_clock_mhz mem_clock_mhz temp_c power_w util_pct + +列在 cuda 与 musa 上完全一致，所以两边的 campaign 可以并排读。采样器不存在时安静 +退出：拿不到遥测是缺一份证据，不是让整批测试失败的理由。 + +用法： + python telemetry.py --backend musa --device 7 --interval 5 --run > telemetry.tsv + python telemetry.py --backend cuda # 只打印将要执行的采样命令 +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +import time +from typing import List, Optional + +FIELDS = ("index", "sm_clock_mhz", "mem_clock_mhz", "temp_c", "power_w", "util_pct") + + +def resolve_device(backend, explicit: Optional[str]) -> str: + """显式设备优先，否则从后端自己的可见性变量取。 + + 单值才用得上：`MUSA_VISIBLE_DEVICES=7` 指一张卡，`0,1,2` 指一组，后者不传 + `-i`、采全部，比挑第一张更诚实。 + """ + if explicit: + return explicit.strip() + for key in getattr(backend, "visibility_env_keys", ()): + raw = os.environ.get(key, "").strip() + if raw and "," not in raw: + return raw + return "" + + +def sample_once(backend, argv: List[str]) -> List[dict]: + try: + out = subprocess.run(argv, capture_output=True, text=True, timeout=30) + except (OSError, subprocess.SubprocessError): + return [] + return backend.parse_telemetry(out.stdout or "") + + +def run(backend, device: str, interval: float, out=sys.stdout) -> int: + argv = backend.telemetry_command(device) + if not argv: + print(f"[telemetry] backend={backend.name} 没有采样器；本批不记时钟/温度", + file=sys.stderr, flush=True) + return 0 + if not sample_once(backend, argv): + # 第一次就取不到，说明工具缺失或没权限。不要留一个空文件让人以为采过了 + print(f"[telemetry] {argv[0]} 取不到读数；本批不记时钟/温度", + file=sys.stderr, flush=True) + return 0 + print("timestamp\t" + "\t".join(FIELDS), file=out, flush=True) + while True: + ts = time.strftime("%Y-%m-%dT%H:%M:%S") + for row in sample_once(backend, argv): + print(ts + "\t" + "\t".join(row.get(f, "") for f in FIELDS), + file=out, flush=True) + time.sleep(interval) + + +def main(argv: Optional[List[str]] = None) -> int: + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--backend", default=os.environ.get("ZL_BACKEND", "cuda")) + p.add_argument("--device", default=os.environ.get("ZL_DEVICE", "")) + p.add_argument("--interval", type=float, default=5.0) + p.add_argument("--run", action="store_true", + help="持续采样直到被杀；不给则只打印采样命令") + args = p.parse_args(argv) + + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from backends import get_backend + backend = get_backend(args.backend) + device = resolve_device(backend, args.device) + if not args.run: + print(" ".join(backend.telemetry_command(device))) + return 0 + try: + return run(backend, device, args.interval) + except KeyboardInterrupt: + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test_backends.py b/test_backends.py index 03e7315..0f6baed 100644 --- a/test_backends.py +++ b/test_backends.py @@ -541,5 +541,78 @@ class CaliberGuardTests(unittest.TestCase): len(report_ab(a / "run.log", b / "run.log", "op", strict=False)), 1) +class TelemetryTests(unittest.TestCase): + """采样器命令与解析。两边归一到同一组列，campaign 才能并排读。""" + + # 取自 S5000 实机：值带单位，"Power Draw " 的键名有尾随空格 + MUSA_JSON = """ +{ + "Attached GPUs": "8", + "GPU": [ + { + "Index": "7", + "Temperature": {"GPU Current Temp": "36C"}, + "Clocks": {"Graphics": "1888MHz", "Memory": "2500MHz", "Video": "0MHz"}, + "Power Readings": {"Power Draw ": "477.68W", "Current Power Limit": "950.00W"}, + "Utilization": {"Gpu": "87%", "Memory": "14%"} + } + ] +} +""" + CUDA_CSV = "0, 1980, 2619, 41, 122.35, 63\n" + + def test_musa_parses_units_off_values(self): + rows = get_backend("musa").parse_telemetry(self.MUSA_JSON) + self.assertEqual(rows, [{ + "index": "7", "sm_clock_mhz": "1888", "mem_clock_mhz": "2500", + "temp_c": "36", "power_w": "477.68", "util_pct": "87"}]) + + def test_cuda_parses_csv(self): + rows = get_backend("cuda").parse_telemetry(self.CUDA_CSV) + self.assertEqual(rows, [{ + "index": "0", "sm_clock_mhz": "1980", "mem_clock_mhz": "2619", + "temp_c": "41", "power_w": "122.35", "util_pct": "63"}]) + + def test_both_backends_yield_the_same_columns(self): + import telemetry + a = get_backend("musa").parse_telemetry(self.MUSA_JSON)[0] + b = get_backend("cuda").parse_telemetry(self.CUDA_CSV)[0] + self.assertEqual(set(a), set(b)) + self.assertEqual(set(a), set(telemetry.FIELDS)) + + def test_garbage_parses_to_nothing_rather_than_raising(self): + for backend in ("musa", "cuda"): + for raw in ("", "no such tool", "{broken json"): + self.assertEqual(get_backend(backend).parse_telemetry(raw), []) + + def test_no_device_drops_the_flag_not_just_the_value(self): + # 留下裸 "-i" 会让采样器直接失败，比采全部更糟 + argv = get_backend("musa").telemetry_command("") + self.assertNotIn("-i", argv) + self.assertIn("--json", argv) + self.assertEqual(get_backend("musa").telemetry_command("7")[-2:], ["-i", "7"]) + + def test_backend_without_sampler_returns_empty(self): + from backends.base import DeviceBackend + self.assertEqual(DeviceBackend().telemetry_command("0"), []) + + def test_device_comes_from_the_visibility_var_when_unpinned(self): + import telemetry + backend = get_backend("musa") + old = os.environ.get("MUSA_VISIBLE_DEVICES") + try: + os.environ["MUSA_VISIBLE_DEVICES"] = "7" + self.assertEqual(telemetry.resolve_device(backend, ""), "7") + # 一组卡不挑第一张，交给采样器采全部 + os.environ["MUSA_VISIBLE_DEVICES"] = "0,1,2" + self.assertEqual(telemetry.resolve_device(backend, ""), "") + self.assertEqual(telemetry.resolve_device(backend, "3"), "3") + finally: + if old is None: + os.environ.pop("MUSA_VISIBLE_DEVICES", None) + else: + os.environ["MUSA_VISIBLE_DEVICES"] = old + + if __name__ == "__main__": unittest.main()