bench: record device clocks, temperature and power for the whole campaign

A 6-round A/B here reports side noise at the 0.01% level. Clock throttling and
thermal drift move latency by far more than that and leave no trace in
run.log, so the report comes out looking exactly as clean as a run on a stable
machine. The NV run that reproduced PR763 sampled nvidia-smi every 5s
alongside the campaign; the MUSA campaigns had no equivalent, and the parallel
plan makes it worse by heating the whole box.

telemetry.py samples through the backend, so cuda and musa emit the same
columns (index, sm_clock_mhz, mem_clock_mhz, temp_c, power_w, util_pct) and
two machines' campaigns can be read side by side. mthreads-gmi offers no CSV
query mode, so the MUSA path parses --json and strips the units glued onto
each value ("36C", "1888MHz", "477.68W"); its "Power Draw " key carries a
trailing space, matched by stripping rather than by literal.

The device comes from the backend's own visibility variable when not pinned. A
list of devices resolves to no -i at all, sampling every one, rather than
silently picking the first. Dropping the placeholder drops its flag too: a
bare trailing -i fails the sampler instead of widening it.

Telemetry is evidence, not a gate. A missing or unpermitted sampler exits
quietly with its reason in telemetry.err and the campaign runs unchanged.

test_backends.py

@@ -541,5 +541,78 @@ class CaliberGuardTests(unittest.TestCase):
                 len(report_ab(a / "run.log", b / "run.log", "op", strict=False)), 1)
 
 
+class TelemetryTests(unittest.TestCase):
+    """采样器命令与解析。两边归一到同一组列，campaign 才能并排读。"""
+
+    # 取自 S5000 实机：值带单位，"Power Draw " 的键名有尾随空格
+    MUSA_JSON = """
+{
+    "Attached GPUs": "8",
+    "GPU": [
+        {
+            "Index": "7",
+            "Temperature": {"GPU Current Temp": "36C"},
+            "Clocks": {"Graphics": "1888MHz", "Memory": "2500MHz", "Video": "0MHz"},
+            "Power Readings": {"Power Draw ": "477.68W", "Current Power Limit": "950.00W"},
+            "Utilization": {"Gpu": "87%", "Memory": "14%"}
+        }
+    ]
+}
+"""
+    CUDA_CSV = "0, 1980, 2619, 41, 122.35, 63\n"
+
+    def test_musa_parses_units_off_values(self):
+        rows = get_backend("musa").parse_telemetry(self.MUSA_JSON)
+        self.assertEqual(rows, [{
+            "index": "7", "sm_clock_mhz": "1888", "mem_clock_mhz": "2500",
+            "temp_c": "36", "power_w": "477.68", "util_pct": "87"}])
+
+    def test_cuda_parses_csv(self):
+        rows = get_backend("cuda").parse_telemetry(self.CUDA_CSV)
+        self.assertEqual(rows, [{
+            "index": "0", "sm_clock_mhz": "1980", "mem_clock_mhz": "2619",
+            "temp_c": "41", "power_w": "122.35", "util_pct": "63"}])
+
+    def test_both_backends_yield_the_same_columns(self):
+        import telemetry
+        a = get_backend("musa").parse_telemetry(self.MUSA_JSON)[0]
+        b = get_backend("cuda").parse_telemetry(self.CUDA_CSV)[0]
+        self.assertEqual(set(a), set(b))
+        self.assertEqual(set(a), set(telemetry.FIELDS))
+
+    def test_garbage_parses_to_nothing_rather_than_raising(self):
+        for backend in ("musa", "cuda"):
+            for raw in ("", "no such tool", "{broken json"):
+                self.assertEqual(get_backend(backend).parse_telemetry(raw), [])
+
+    def test_no_device_drops_the_flag_not_just_the_value(self):
+        # 留下裸 "-i" 会让采样器直接失败，比采全部更糟
+        argv = get_backend("musa").telemetry_command("")
+        self.assertNotIn("-i", argv)
+        self.assertIn("--json", argv)
+        self.assertEqual(get_backend("musa").telemetry_command("7")[-2:], ["-i", "7"])
+
+    def test_backend_without_sampler_returns_empty(self):
+        from backends.base import DeviceBackend
+        self.assertEqual(DeviceBackend().telemetry_command("0"), [])
+
+    def test_device_comes_from_the_visibility_var_when_unpinned(self):
+        import telemetry
+        backend = get_backend("musa")
+        old = os.environ.get("MUSA_VISIBLE_DEVICES")
+        try:
+            os.environ["MUSA_VISIBLE_DEVICES"] = "7"
+            self.assertEqual(telemetry.resolve_device(backend, ""), "7")
+            # 一组卡不挑第一张，交给采样器采全部
+            os.environ["MUSA_VISIBLE_DEVICES"] = "0,1,2"
+            self.assertEqual(telemetry.resolve_device(backend, ""), "")
+            self.assertEqual(telemetry.resolve_device(backend, "3"), "3")
+        finally:
+            if old is None:
+                os.environ.pop("MUSA_VISIBLE_DEVICES", None)
+            else:
+                os.environ["MUSA_VISIBLE_DEVICES"] = old
+
+
 if __name__ == "__main__":
     unittest.main()