diff --git a/README.md b/README.md index 826003f..3391c47 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,19 @@ failed replay_fallback no_cudagraph capture_poisoned`。README 要求必查的 想起来的时候才发生。每格另有 `CELL_TIMEOUT`(默认 2400 秒)兜底,超时记 `rc=124` 并打 `!!! CELL_TIMEOUT`,不会让一格卡死拖住整批。 +全程还采一份 `/telemetry.tsv`: + +``` +timestamp index sm_clock_mhz mem_clock_mhz temp_c power_w util_pct +2026-09-10T07:41:12 7 1888 2500 36 477.68 87 +``` + +列在 cuda 与 musa 上完全一致(`nvidia-smi --query-gpu` 与 `mthreads-gmi -q --json` +各自解析后归一),所以两台机器的 campaign 可以并排读。理由很直接:六轮 A/B 声称的是 +0.01% 量级的单侧噪声,**降频和热漂移能移动比这大得多的延迟,而且在 run.log 里不留 +任何痕迹**——报告照样干净。`TELEMETRY_INTERVAL`(默认 5 秒)设 0 关闭;采样器缺失或 +没权限时 `telemetry.py` 安静退出并把原因写进 `telemetry.err`,不影响测试本身。 + 第 1 轮 A/B 跑完后,脚本比两侧 `ttgir/` 的指纹(相对路径 + 内容)。**逐字节相同就 跳过这个算子**,并打 `!!! SWITCH_NO_IR_DIFF op=... switch=...`:这个开关没改变该算子的 生成代码,剩下 N-1 轮测的全是噪声,最后还会报成一个"无差异"的结论。先查开关名拼写 diff --git a/ab_repeat.sh b/ab_repeat.sh index 6e70c1c..44ca537 100644 --- a/ab_repeat.sh +++ b/ab_repeat.sh @@ -57,6 +57,19 @@ ROOT="runs/ab_repeat_${SWITCH}_$(date +%Y%m%d-%H%M%S)" mkdir -p "$ROOT" LOG="$ROOT/summary.log" STATUS="$ROOT/status.tsv" + +# 全程采时钟/温度/功耗。六轮 A/B 声称的是 0.01% 量级的单侧噪声,降频和热漂移 +# 能移动比这大得多的延迟,而且在 run.log 里不留痕迹——报告照样干净。 +# TELEMETRY_INTERVAL=0 关闭。采不到时 telemetry.py 自己安静退出,不影响测试。 +TELEMETRY_INTERVAL="${TELEMETRY_INTERVAL:-5}" +TELE_PID="" +if [[ "$TELEMETRY_INTERVAL" != "0" ]]; then + python telemetry.py --backend "${ZL_BACKEND:-cuda}" --device "${ZL_DEVICE:-}" \ + --interval "$TELEMETRY_INTERVAL" --run \ + > "$ROOT/telemetry.tsv" 2> "$ROOT/telemetry.err" & + TELE_PID=$! + trap '[[ -n "$TELE_PID" ]] && kill "$TELE_PID" 2>/dev/null' EXIT +fi printf 'round\top\tside\tswitch\trc\telapsed_s\tsuccess\tfailed\treplay_fallback\tno_cudagraph\tcapture_poisoned\n' > "$STATUS" # 跑一格并逐格登记。事后 grep 也能拿到这些数,但只在有人想起来时才发生; @@ -157,5 +170,7 @@ sed -i -E $'s/\x1b\\[[0-9;]*[A-Za-z]//g' "$LOG" echo ">>> 总 log: $(pwd)/$LOG" echo ">>> 逐格状态表: $(pwd)/$STATUS" +[[ -s "$ROOT/telemetry.tsv" ]] && echo ">>> 时钟/温度: $(pwd)/$ROOT/telemetry.tsv" +[[ -s "$ROOT/telemetry.err" ]] && echo ">>> 遥测提示: $(cat "$ROOT/telemetry.err")" echo ">>> 逐算子重出报告: python batch_summary.py --ab-rounds $ROOT/ --label " echo ">>> 解读要点: 逐轮区间宽说明判定不稳;单侧噪声大说明机器不稳;两者含义不同" diff --git a/backends/base.py b/backends/base.py index c1afae7..21618a0 100644 --- a/backends/base.py +++ b/backends/base.py @@ -72,6 +72,34 @@ class DeviceBackend: # backend. Skipped up front so one of them cannot cost a whole run. capture_deny_ops: Tuple[str, ...] = () + # One-shot device telemetry sampler. Empty means this backend has none, and + # the campaign runs without a clock/thermal record rather than pretending. + # "{dev}" is dropped from the argv when no device is pinned, so the sampler + # covers every visible device instead of guessing an index. + telemetry_argv: Tuple[str, ...] = () + + def telemetry_command(self, device: str = "") -> List[str]: + if not self.telemetry_argv: + return [] + out: List[str] = [] + for a in self.telemetry_argv: + if "{dev}" in a: + if not device: + # Drop the flag the placeholder belonged to as well; a bare + # trailing "-i" makes the sampler fail instead of widening. + if out and out[-1].startswith("-"): + out.pop() + continue + a = a.replace("{dev}", device) + out.append(a) + return out + + def parse_telemetry(self, raw: str) -> List[Dict[str, str]]: + """One dict per device: index, sm_clock_mhz, mem_clock_mhz, temp_c, + power_w, util_pct. Normalised so a CUDA and a MUSA campaign produce the + same columns and can be read side by side.""" + return [] + def classify_capture_error(self, exc: BaseException) -> str: """-> "fatal" | "poisoned" | "recoverable". diff --git a/backends/cuda.py b/backends/cuda.py index 0c3ae54..8429382 100644 --- a/backends/cuda.py +++ b/backends/cuda.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Dict, List + from .base import DeviceBackend @@ -26,3 +28,22 @@ class CudaBackend(DeviceBackend): # No CUDA wording is known to poison the context without also matching one # of the fatal markers above, so nothing is listed here rather than guessed. poisoning_error_markers = () + + telemetry_argv = ( + "nvidia-smi", + "--query-gpu=index,clocks.sm,clocks.mem,temperature.gpu,power.draw,utilization.gpu", + "--format=csv,noheader,nounits", + "-i", "{dev}", + ) + + def parse_telemetry(self, raw: str) -> List[Dict[str, str]]: + out: List[Dict[str, str]] = [] + for line in raw.splitlines(): + parts = [p.strip() for p in line.split(",")] + if len(parts) != 6 or not parts[0].isdigit(): + continue + out.append(dict(zip( + ("index", "sm_clock_mhz", "mem_clock_mhz", + "temp_c", "power_w", "util_pct"), parts))) + return out + diff --git a/backends/musa.py b/backends/musa.py index 066af8d..a0dbfbf 100644 --- a/backends/musa.py +++ b/backends/musa.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Dict, List + from .base import DeviceBackend @@ -50,3 +52,50 @@ class MusaBackend(DeviceBackend): # Device count comes from torch.musa or MUSA_VISIBLE_DEVICES. Do not guess # a vendor CLI here; confirm the real listing tool before adding it. list_command = None + + # mthreads-gmi has no CSV query mode; --json is the only machine-readable + # form it offers. Values arrive with their units glued on ("36C", "1888MHz", + # "477.68W", "87%"), so the parser strips trailing non-numerics rather than + # trusting a fixed suffix. + telemetry_argv = ( + "mthreads-gmi", "-q", + "-d", "TEMPERATURE,CLOCK,POWER,UTILIZATION", + "--json", + "-i", "{dev}", + ) + + def parse_telemetry(self, raw: str) -> List[Dict[str, str]]: + import json + import re + + def num(v): + m = re.match(r"^\s*(-?[\d.]+)", str(v or "")) + return m.group(1) if m else "" + + start = raw.find("{") + if start < 0: + return [] + try: + data = json.loads(raw[start:]) + except Exception: + return [] + gpus = data.get("GPU") or [] + if isinstance(gpus, dict): + gpus = [gpus] + out: List[Dict[str, str]] = [] + for g in gpus: + clocks = g.get("Clocks") or {} + power = g.get("Power Readings") or {} + # 键名带尾随空格("Power Draw "),别按字面写死 + draw = next((v for k, v in power.items() + if k.strip() == "Power Draw"), "") + out.append({ + "index": str(g.get("Index", "")).strip(), + "sm_clock_mhz": num(clocks.get("Graphics")), + "mem_clock_mhz": num(clocks.get("Memory")), + "temp_c": num((g.get("Temperature") or {}).get("GPU Current Temp")), + "power_w": num(draw), + "util_pct": num((g.get("Utilization") or {}).get("Gpu")), + }) + return out + diff --git a/telemetry.py b/telemetry.py new file mode 100644 index 0000000..0ba1fdd --- /dev/null +++ b/telemetry.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""采样设备时钟/温度/功耗,跨后端归一化成同一组列。 + +为什么要有它:六轮 A/B 会声称单侧噪声在 0.01% 量级。降频和热漂移足以移动比这大 +得多的延迟,而且在 run.log 里不留任何痕迹——报告照样干净。NV 上复现 PR763 的那次 +全程 `nvidia-smi -l 5` 写 clocks.csv,S5000 这边一直没有对应的东西。 + +输出每设备每次采样一行 TSV: + + timestamp index sm_clock_mhz mem_clock_mhz temp_c power_w util_pct + +列在 cuda 与 musa 上完全一致,所以两边的 campaign 可以并排读。采样器不存在时安静 +退出:拿不到遥测是缺一份证据,不是让整批测试失败的理由。 + +用法: + python telemetry.py --backend musa --device 7 --interval 5 --run > telemetry.tsv + python telemetry.py --backend cuda # 只打印将要执行的采样命令 +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +import time +from typing import List, Optional + +FIELDS = ("index", "sm_clock_mhz", "mem_clock_mhz", "temp_c", "power_w", "util_pct") + + +def resolve_device(backend, explicit: Optional[str]) -> str: + """显式设备优先,否则从后端自己的可见性变量取。 + + 单值才用得上:`MUSA_VISIBLE_DEVICES=7` 指一张卡,`0,1,2` 指一组,后者不传 + `-i`、采全部,比挑第一张更诚实。 + """ + if explicit: + return explicit.strip() + for key in getattr(backend, "visibility_env_keys", ()): + raw = os.environ.get(key, "").strip() + if raw and "," not in raw: + return raw + return "" + + +def sample_once(backend, argv: List[str]) -> List[dict]: + try: + out = subprocess.run(argv, capture_output=True, text=True, timeout=30) + except (OSError, subprocess.SubprocessError): + return [] + return backend.parse_telemetry(out.stdout or "") + + +def run(backend, device: str, interval: float, out=sys.stdout) -> int: + argv = backend.telemetry_command(device) + if not argv: + print(f"[telemetry] backend={backend.name} 没有采样器;本批不记时钟/温度", + file=sys.stderr, flush=True) + return 0 + if not sample_once(backend, argv): + # 第一次就取不到,说明工具缺失或没权限。不要留一个空文件让人以为采过了 + print(f"[telemetry] {argv[0]} 取不到读数;本批不记时钟/温度", + file=sys.stderr, flush=True) + return 0 + print("timestamp\t" + "\t".join(FIELDS), file=out, flush=True) + while True: + ts = time.strftime("%Y-%m-%dT%H:%M:%S") + for row in sample_once(backend, argv): + print(ts + "\t" + "\t".join(row.get(f, "") for f in FIELDS), + file=out, flush=True) + time.sleep(interval) + + +def main(argv: Optional[List[str]] = None) -> int: + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--backend", default=os.environ.get("ZL_BACKEND", "cuda")) + p.add_argument("--device", default=os.environ.get("ZL_DEVICE", "")) + p.add_argument("--interval", type=float, default=5.0) + p.add_argument("--run", action="store_true", + help="持续采样直到被杀;不给则只打印采样命令") + args = p.parse_args(argv) + + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from backends import get_backend + backend = get_backend(args.backend) + device = resolve_device(backend, args.device) + if not args.run: + print(" ".join(backend.telemetry_command(device))) + return 0 + try: + return run(backend, device, args.interval) + except KeyboardInterrupt: + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test_backends.py b/test_backends.py index 03e7315..0f6baed 100644 --- a/test_backends.py +++ b/test_backends.py @@ -541,5 +541,78 @@ class CaliberGuardTests(unittest.TestCase): len(report_ab(a / "run.log", b / "run.log", "op", strict=False)), 1) +class TelemetryTests(unittest.TestCase): + """采样器命令与解析。两边归一到同一组列,campaign 才能并排读。""" + + # 取自 S5000 实机:值带单位,"Power Draw " 的键名有尾随空格 + MUSA_JSON = """ +{ + "Attached GPUs": "8", + "GPU": [ + { + "Index": "7", + "Temperature": {"GPU Current Temp": "36C"}, + "Clocks": {"Graphics": "1888MHz", "Memory": "2500MHz", "Video": "0MHz"}, + "Power Readings": {"Power Draw ": "477.68W", "Current Power Limit": "950.00W"}, + "Utilization": {"Gpu": "87%", "Memory": "14%"} + } + ] +} +""" + CUDA_CSV = "0, 1980, 2619, 41, 122.35, 63\n" + + def test_musa_parses_units_off_values(self): + rows = get_backend("musa").parse_telemetry(self.MUSA_JSON) + self.assertEqual(rows, [{ + "index": "7", "sm_clock_mhz": "1888", "mem_clock_mhz": "2500", + "temp_c": "36", "power_w": "477.68", "util_pct": "87"}]) + + def test_cuda_parses_csv(self): + rows = get_backend("cuda").parse_telemetry(self.CUDA_CSV) + self.assertEqual(rows, [{ + "index": "0", "sm_clock_mhz": "1980", "mem_clock_mhz": "2619", + "temp_c": "41", "power_w": "122.35", "util_pct": "63"}]) + + def test_both_backends_yield_the_same_columns(self): + import telemetry + a = get_backend("musa").parse_telemetry(self.MUSA_JSON)[0] + b = get_backend("cuda").parse_telemetry(self.CUDA_CSV)[0] + self.assertEqual(set(a), set(b)) + self.assertEqual(set(a), set(telemetry.FIELDS)) + + def test_garbage_parses_to_nothing_rather_than_raising(self): + for backend in ("musa", "cuda"): + for raw in ("", "no such tool", "{broken json"): + self.assertEqual(get_backend(backend).parse_telemetry(raw), []) + + def test_no_device_drops_the_flag_not_just_the_value(self): + # 留下裸 "-i" 会让采样器直接失败,比采全部更糟 + argv = get_backend("musa").telemetry_command("") + self.assertNotIn("-i", argv) + self.assertIn("--json", argv) + self.assertEqual(get_backend("musa").telemetry_command("7")[-2:], ["-i", "7"]) + + def test_backend_without_sampler_returns_empty(self): + from backends.base import DeviceBackend + self.assertEqual(DeviceBackend().telemetry_command("0"), []) + + def test_device_comes_from_the_visibility_var_when_unpinned(self): + import telemetry + backend = get_backend("musa") + old = os.environ.get("MUSA_VISIBLE_DEVICES") + try: + os.environ["MUSA_VISIBLE_DEVICES"] = "7" + self.assertEqual(telemetry.resolve_device(backend, ""), "7") + # 一组卡不挑第一张,交给采样器采全部 + os.environ["MUSA_VISIBLE_DEVICES"] = "0,1,2" + self.assertEqual(telemetry.resolve_device(backend, ""), "") + self.assertEqual(telemetry.resolve_device(backend, "3"), "3") + finally: + if old is None: + os.environ.pop("MUSA_VISIBLE_DEVICES", None) + else: + os.environ["MUSA_VISIBLE_DEVICES"] = old + + if __name__ == "__main__": unittest.main()