forked from zhoulin/zl_bench
Aharrypotter
ee5f70fa89
A 6-round A/B here reports side noise at the 0.01% level. Clock throttling and
thermal drift move latency by far more than that and leave no trace in
run.log, so the report comes out looking exactly as clean as a run on a stable
machine. The NV run that reproduced PR763 sampled nvidia-smi every 5s
alongside the campaign; the MUSA campaigns had no equivalent, and the parallel
plan makes it worse by heating the whole box.
telemetry.py samples through the backend, so cuda and musa emit the same
columns (index, sm_clock_mhz, mem_clock_mhz, temp_c, power_w, util_pct) and
two machines' campaigns can be read side by side. mthreads-gmi offers no CSV
query mode, so the MUSA path parses --json and strips the units glued onto
each value ("36C", "1888MHz", "477.68W"); its "Power Draw " key carries a
trailing space, matched by stripping rather than by literal.
The device comes from the backend's own visibility variable when not pinned. A
list of devices resolves to no -i at all, sampling every one, rather than
silently picking the first. Dropping the placeholder drops its flag too: a
bare trailing -i fails the sampler instead of widening it.
Telemetry is evidence, not a gate. A missing or unpermitted sampler exits
quietly with its reason in telemetry.err and the campaign runs unchanged.
177 lines
7.9 KiB
Bash
177 lines
7.9 KiB
Bash
#!/bin/bash
|
||
# 多轮 A/B：同一对比在 N 轮独立进程里各跑一次，逐 shape 取轮次中位。
|
||
#
|
||
# 为什么需要它：单对观测无法说明自己稳不稳。S5000 实测 topk 消除了 32 个
|
||
# convert_layout，42 个 shape 里却有 17 个在两次同等有效的运行之间改变了 delta
|
||
# 的符号——那不是计时口径的问题（图捕获与 direct 的延迟比是 1.00x），是收益本身
|
||
# 落在零附近、单次测量给不出方向。多轮之后才谈得上"这个判定成立"。
|
||
#
|
||
# 口径：第 1 轮 A 侧 record autotune config，此后每一次运行（含 A 侧自己的后续
|
||
# 轮次）都 REPLAY_FROM 那一份。config 因此跨轮跨侧全部锁死，剩下的变量只有被测
|
||
# 开关与测量噪声。逐轮交替先后顺序，抵消单向漂移。
|
||
#
|
||
# 用法：
|
||
# SWITCH=FLAGTREE_MUSA_RLC_ENHANCE OPS="topk masked_select" ROUNDS=6 \
|
||
# ZL_BACKEND=musa FLAGGEMS_DIR=/path/to/FlagGems bash ab_repeat.sh
|
||
#
|
||
# OPS 元素为 OP 或 OP:OP_FILE（省略时 OP_FILE=OP）。
|
||
# SWITCH 是被测环境变量，A 侧设 0、B 侧设 1。
|
||
#
|
||
# 对比轴有效性由脚本自己查：第 1 轮 A/B 跑完后比两侧的 ttgir 指纹，完全一致说明
|
||
# 这个开关在这个算子上没改变任何生成代码——继续跑满 N 轮只会得到一个"无差异"的假
|
||
# 结论（同 ab_fold_test.sh 的告诫）。这类算子默认跳过；开关确实只影响运行时而不影响
|
||
# IR 时，用 SWITCH_ALLOW_NO_IR_DIFF=1 放行。
|
||
#
|
||
# 产物：runs/ab_repeat_<开关>_<时间戳>/<op>/r<k>_<A|B>/，跑完对每个算子调用
|
||
# batch_summary.py --ab-rounds 出报告。
|
||
set -uo pipefail
|
||
|
||
cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
SWITCH_ALLOW_NO_IR_DIFF="${SWITCH_ALLOW_NO_IR_DIFF:-0}"
|
||
|
||
# 一个 run 生成代码的指纹：ttgir 相对路径 + 内容。路径进指纹是因为"变体集合变了"
|
||
# 与"某个变体内容变了"同样算改变，而只 cat 内容会把前者洗掉。
|
||
ir_fingerprint() {
|
||
local dir="$1/ttgir"
|
||
[[ -d "$dir" ]] || return 1
|
||
local out
|
||
out=$( { find "$dir" -type f | LC_ALL=C sort | while IFS= read -r f; do
|
||
printf '%s
|
||
' "${f#"$dir"/}"; cat "$f"
|
||
done; } | sha256sum | cut -d' ' -f1 )
|
||
[[ -n "$out" ]] && printf '%s' "$out"
|
||
}
|
||
|
||
: "${SWITCH:?SWITCH is required, e.g. FLAGTREE_MUSA_RLC_ENHANCE}"
|
||
OPS="${OPS:-}"
|
||
: "${OPS:?OPS is required, e.g. \"topk masked_select\"}"
|
||
ROUNDS="${ROUNDS:-6}"
|
||
CELL_TIMEOUT="${CELL_TIMEOUT:-2400}"
|
||
|
||
if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then
|
||
export FLAGGEMS_PERF_COLOR=always
|
||
fi
|
||
|
||
ROOT="runs/ab_repeat_${SWITCH}_$(date +%Y%m%d-%H%M%S)"
|
||
mkdir -p "$ROOT"
|
||
LOG="$ROOT/summary.log"
|
||
STATUS="$ROOT/status.tsv"
|
||
|
||
# 全程采时钟/温度/功耗。六轮 A/B 声称的是 0.01% 量级的单侧噪声，降频和热漂移
|
||
# 能移动比这大得多的延迟，而且在 run.log 里不留痕迹——报告照样干净。
|
||
# TELEMETRY_INTERVAL=0 关闭。采不到时 telemetry.py 自己安静退出，不影响测试。
|
||
TELEMETRY_INTERVAL="${TELEMETRY_INTERVAL:-5}"
|
||
TELE_PID=""
|
||
if [[ "$TELEMETRY_INTERVAL" != "0" ]]; then
|
||
python telemetry.py --backend "${ZL_BACKEND:-cuda}" --device "${ZL_DEVICE:-}" \
|
||
--interval "$TELEMETRY_INTERVAL" --run \
|
||
> "$ROOT/telemetry.tsv" 2> "$ROOT/telemetry.err" &
|
||
TELE_PID=$!
|
||
trap '[[ -n "$TELE_PID" ]] && kill "$TELE_PID" 2>/dev/null' EXIT
|
||
fi
|
||
printf 'round\top\tside\tswitch\trc\telapsed_s\tsuccess\tfailed\treplay_fallback\tno_cudagraph\tcapture_poisoned\n' > "$STATUS"
|
||
|
||
# 跑一格并逐格登记。事后 grep 也能拿到这些数，但只在有人想起来时才发生；
|
||
# 缺格、超时、标记从 0 变成非 0，都要在跑的时候就看得见。
|
||
run_cell() { # round side op op_file out_dir [replay_from]
|
||
local round="$1" side="$2" op="$3" op_file="$4" dir="$5" replay="${6:-}"
|
||
local val=0; [[ "$side" == B ]] && val=1
|
||
local start rc elapsed log ok fail fb ng po
|
||
start=$(date +%s)
|
||
if [[ -n "$replay" ]]; then
|
||
OP="$op" OP_FILE="$op_file" OUT_DIR="$dir" REPLAY_FROM="$replay" \
|
||
timeout -k 30 "$CELL_TIMEOUT" env "$SWITCH=$val" bash run_pytest.sh
|
||
else
|
||
OP="$op" OP_FILE="$op_file" OUT_DIR="$dir" \
|
||
timeout -k 30 "$CELL_TIMEOUT" env "$SWITCH=$val" bash run_pytest.sh
|
||
fi
|
||
rc=$?
|
||
elapsed=$(( $(date +%s) - start ))
|
||
(( rc == 124 || rc == 137 )) && echo "!!! CELL_TIMEOUT ${CELL_TIMEOUT}s op=$op round=$round side=$side"
|
||
log="$dir/run.log"
|
||
# grep -c 没命中时打印 0 但以 1 退出；不吞掉的话会被当成本格失败
|
||
ok=$(grep -c '^SUCCESS' "$log" 2>/dev/null || true)
|
||
fail=$(grep -c '^FAILED' "$log" 2>/dev/null || true)
|
||
fb=$(grep -c 'AUTOTUNE_REPLAY_FALLBACK' "$log" 2>/dev/null || true)
|
||
ng=$(grep -c 'BENCHMARK_DIRECT_NO_CUDAGRAPH' "$log" 2>/dev/null || true)
|
||
po=$(grep -c 'BENCHMARK_CAPTURE_POISONED' "$log" 2>/dev/null || true)
|
||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||
"$round" "$op" "$side" "$val" "$rc" "$elapsed" \
|
||
"${ok:-0}" "${fail:-0}" "${fb:-0}" "${ng:-0}" "${po:-0}" >> "$STATUS"
|
||
return $rc
|
||
}
|
||
|
||
{
|
||
echo ">>> switch=$SWITCH rounds=$ROUNDS ops=$OPS"
|
||
for entry in $OPS; do
|
||
op="${entry%%:*}"
|
||
op_file="${entry#*:}"
|
||
[[ "$op_file" == "$entry" ]] && op_file="$op"
|
||
out="$ROOT/$op"
|
||
mkdir -p "$out"
|
||
|
||
# 第 1 轮 A 侧 record，其余全部回放它，config 跨轮跨侧一致
|
||
echo "########## $op | round 1 | A ($SWITCH=0, record) ##########"
|
||
if ! run_cell 1 A "$op" "$op_file" "$out/r1_A"; then
|
||
echo "!!! FAILED $op round 1 A; skipping this operator"
|
||
continue
|
||
fi
|
||
REC="$out/r1_A"
|
||
|
||
ok=1; reason="有失败轮次"
|
||
for ((k = 1; k <= ROUNDS; k++)); do
|
||
# 奇数轮先 A 后 B，偶数轮反过来
|
||
if (( k % 2 == 1 )); then order="A B"; else order="B A"; fi
|
||
for side in $order; do
|
||
[[ $k -eq 1 && $side == A ]] && continue # 已经跑过
|
||
val=$([[ $side == A ]] && echo 0 || echo 1)
|
||
echo "########## $op | round $k | $side ($SWITCH=$val, replay) ##########"
|
||
if ! run_cell "$k" "$side" "$op" "$op_file" "$out/r${k}_${side}" "$REC"; then
|
||
echo "!!! FAILED $op round $k $side"
|
||
ok=0
|
||
fi
|
||
done
|
||
|
||
if (( k == 1 && ok )); then
|
||
fa=$(ir_fingerprint "$out/r1_A") || fa=""
|
||
fb=$(ir_fingerprint "$out/r1_B") || fb=""
|
||
if [[ -z "$fa" || -z "$fb" ]]; then
|
||
echo "??? $op: 没有 ttgir 产物，跳过对比轴有效性检查"
|
||
elif [[ "$fa" == "$fb" ]]; then
|
||
echo "!!! SWITCH_NO_IR_DIFF op=$op switch=$SWITCH"
|
||
echo "!!! 两侧 ttgir 逐字节相同：这个开关没有改变 $op 的生成代码。"
|
||
echo "!!! 继续跑 $ROUNDS 轮只会测出噪声，并报成一个'无差异'结论。"
|
||
echo "!!! 先确认开关名拼写与它在被测源码里的读取点；确属只影响运行时的"
|
||
echo "!!! 开关，用 SWITCH_ALLOW_NO_IR_DIFF=1 重跑。"
|
||
if [[ "$SWITCH_ALLOW_NO_IR_DIFF" != "1" ]]; then
|
||
ok=0; reason="开关未改变该算子的生成代码"
|
||
break
|
||
fi
|
||
else
|
||
echo ">>> 对比轴有效：$op 两侧 ttgir 不同（A=${fa:0:12} B=${fb:0:12}）"
|
||
fi
|
||
fi
|
||
done
|
||
|
||
echo "########## $op | report ##########"
|
||
if (( ok )); then
|
||
python batch_summary.py --ab-rounds "$out" --label "$op" \
|
||
|| echo "!!! report failed: $op"
|
||
else
|
||
# 缺轮的算子照样出报告会让人误以为是完整的 N 轮
|
||
echo "!!! $op $reason，不出报告；产物在 $out"
|
||
fi
|
||
done
|
||
echo "########## ALL DONE ##########"
|
||
} 2>&1 | tee "$LOG"
|
||
|
||
sed -i -E $'s/\x1b\\[[0-9;]*[A-Za-z]//g' "$LOG"
|
||
|
||
echo ">>> 总 log: $(pwd)/$LOG"
|
||
echo ">>> 逐格状态表: $(pwd)/$STATUS"
|
||
[[ -s "$ROOT/telemetry.tsv" ]] && echo ">>> 时钟/温度: $(pwd)/$ROOT/telemetry.tsv"
|
||
[[ -s "$ROOT/telemetry.err" ]] && echo ">>> 遥测提示: $(cat "$ROOT/telemetry.err")"
|
||
echo ">>> 逐算子重出报告: python batch_summary.py --ab-rounds $ROOT/<op> --label <op>"
|
||
echo ">>> 解读要点: 逐轮区间宽说明判定不稳；单侧噪声大说明机器不稳；两者含义不同"
|