forked from zhoulin/zl_bench
Aharrypotter
2786eb9886
campaign.sh fans a screened operator list across cards, one ab_repeat.sh per card, serial within a card. It does no measuring: ab_repeat.sh is the path that produced the four operator verdicts already on record, and it stays byte for byte as it was. Each card gets its own AB_ROOT. Eight ab_repeat.sh started in the same second would otherwise compute the same timestamped directory name and two cards would write into one. ZL_ISOLATE_TUNER_CACHE=1 is forced, since eight processes on one sqlite is the thing to avoid. The cost is that pinned configs differ from a shared-cache batch, so absolute latencies cannot be quoted across the two; the caliber guard refuses such comparisons on its own, which is why this is a note and not a warning someone has to remember. partition.py's shell output gained the per-card weight, because EXTRA_OPS has to land on the lightest card and counting operators gets that wrong — one heavy operator outweighs three light ones. It writes through stdout.buffer: Python turns "\n" into CRLF on Windows, and the trailing \r then rides into the last field and breaks the numeric comparison that picks the card.
169 lines
7.7 KiB
Bash
169 lines
7.7 KiB
Bash
#!/bin/bash
|
||
# 多轮 A/B：同一对比在 N 轮独立进程里各跑一次，逐 shape 取轮次中位。
|
||
#
|
||
# 为什么需要它：单对观测无法说明自己稳不稳。S5000 实测 topk 消除了 32 个
|
||
# convert_layout，42 个 shape 里却有 17 个在两次同等有效的运行之间改变了 delta
|
||
# 的符号——那不是计时口径的问题（图捕获与 direct 的延迟比是 1.00x），是收益本身
|
||
# 落在零附近、单次测量给不出方向。多轮之后才谈得上"这个判定成立"。
|
||
#
|
||
# 口径：第 1 轮 A 侧 record autotune config，此后每一次运行（含 A 侧自己的后续
|
||
# 轮次）都 REPLAY_FROM 那一份。config 因此跨轮跨侧全部锁死，剩下的变量只有被测
|
||
# 开关与测量噪声。逐轮交替先后顺序，抵消单向漂移。
|
||
#
|
||
# 用法：
|
||
# SWITCH=FLAGTREE_MUSA_RLC_ENHANCE OPS="topk masked_select" ROUNDS=6 \
|
||
# ZL_BACKEND=musa FLAGGEMS_DIR=/path/to/FlagGems bash ab_repeat.sh
|
||
#
|
||
# OPS 元素为 OP 或 OP:OP_FILE（省略时 OP_FILE=OP）。
|
||
# SWITCH 是被测环境变量，A 侧设 0、B 侧设 1。
|
||
#
|
||
# 对比轴有效性由脚本自己查：第 1 轮 A/B 跑完后比两侧的 ttgir 指纹，完全一致说明
|
||
# 这个开关在这个算子上没改变任何生成代码——继续跑满 N 轮只会得到一个"无差异"的假
|
||
# 结论（同 ab_fold_test.sh 的告诫）。这类算子默认跳过；开关确实只影响运行时而不影响
|
||
# IR 时，用 SWITCH_ALLOW_NO_IR_DIFF=1 放行。
|
||
#
|
||
# 产物：runs/ab_repeat_<开关>_<时间戳>/<op>/r<k>_<A|B>/，跑完对每个算子调用
|
||
# batch_summary.py --ab-rounds 出报告。
|
||
set -uo pipefail
|
||
|
||
cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
SWITCH_ALLOW_NO_IR_DIFF="${SWITCH_ALLOW_NO_IR_DIFF:-0}"
|
||
|
||
source "$(dirname "${BASH_SOURCE[0]}")/_ir_fingerprint.sh"
|
||
|
||
: "${SWITCH:?SWITCH is required, e.g. FLAGTREE_MUSA_RLC_ENHANCE}"
|
||
OPS="${OPS:-}"
|
||
: "${OPS:?OPS is required, e.g. \"topk masked_select\"}"
|
||
ROUNDS="${ROUNDS:-6}"
|
||
CELL_TIMEOUT="${CELL_TIMEOUT:-2400}"
|
||
|
||
if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then
|
||
export FLAGGEMS_PERF_COLOR=always
|
||
fi
|
||
|
||
# AB_ROOT 供 campaign.sh 指定每卡独立的产物根：八个 ab_repeat.sh 同一秒启动时，
|
||
# 默认的时间戳目录名会撞在一起，两张卡往同一个目录里写。
|
||
ROOT="${AB_ROOT:-runs/ab_repeat_${SWITCH}_$(date +%Y%m%d-%H%M%S)}"
|
||
mkdir -p "$ROOT"
|
||
LOG="$ROOT/summary.log"
|
||
STATUS="$ROOT/status.tsv"
|
||
|
||
# 全程采时钟/温度/功耗。六轮 A/B 声称的是 0.01% 量级的单侧噪声，降频和热漂移
|
||
# 能移动比这大得多的延迟，而且在 run.log 里不留痕迹——报告照样干净。
|
||
# TELEMETRY_INTERVAL=0 关闭。采不到时 telemetry.py 自己安静退出，不影响测试。
|
||
TELEMETRY_INTERVAL="${TELEMETRY_INTERVAL:-5}"
|
||
TELE_PID=""
|
||
if [[ "$TELEMETRY_INTERVAL" != "0" ]]; then
|
||
python telemetry.py --backend "${ZL_BACKEND:-cuda}" --device "${ZL_DEVICE:-}" \
|
||
--interval "$TELEMETRY_INTERVAL" --run \
|
||
> "$ROOT/telemetry.tsv" 2> "$ROOT/telemetry.err" &
|
||
TELE_PID=$!
|
||
trap '[[ -n "$TELE_PID" ]] && kill "$TELE_PID" 2>/dev/null' EXIT
|
||
fi
|
||
printf 'round\top\tside\tswitch\trc\telapsed_s\tsuccess\tfailed\treplay_fallback\tno_cudagraph\tcapture_poisoned\n' > "$STATUS"
|
||
|
||
# 跑一格并逐格登记。事后 grep 也能拿到这些数，但只在有人想起来时才发生；
|
||
# 缺格、超时、标记从 0 变成非 0，都要在跑的时候就看得见。
|
||
run_cell() { # round side op op_file out_dir [replay_from]
|
||
local round="$1" side="$2" op="$3" op_file="$4" dir="$5" replay="${6:-}"
|
||
local val=0; [[ "$side" == B ]] && val=1
|
||
local start rc elapsed log ok fail fb ng po
|
||
start=$(date +%s)
|
||
if [[ -n "$replay" ]]; then
|
||
OP="$op" OP_FILE="$op_file" OUT_DIR="$dir" REPLAY_FROM="$replay" \
|
||
timeout -k 30 "$CELL_TIMEOUT" env "$SWITCH=$val" bash run_pytest.sh
|
||
else
|
||
OP="$op" OP_FILE="$op_file" OUT_DIR="$dir" \
|
||
timeout -k 30 "$CELL_TIMEOUT" env "$SWITCH=$val" bash run_pytest.sh
|
||
fi
|
||
rc=$?
|
||
elapsed=$(( $(date +%s) - start ))
|
||
(( rc == 124 || rc == 137 )) && echo "!!! CELL_TIMEOUT ${CELL_TIMEOUT}s op=$op round=$round side=$side"
|
||
log="$dir/run.log"
|
||
# grep -c 没命中时打印 0 但以 1 退出；不吞掉的话会被当成本格失败
|
||
ok=$(grep -c '^SUCCESS' "$log" 2>/dev/null || true)
|
||
fail=$(grep -c '^FAILED' "$log" 2>/dev/null || true)
|
||
fb=$(grep -c 'AUTOTUNE_REPLAY_FALLBACK' "$log" 2>/dev/null || true)
|
||
ng=$(grep -c 'BENCHMARK_DIRECT_NO_CUDAGRAPH' "$log" 2>/dev/null || true)
|
||
po=$(grep -c 'BENCHMARK_CAPTURE_POISONED' "$log" 2>/dev/null || true)
|
||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||
"$round" "$op" "$side" "$val" "$rc" "$elapsed" \
|
||
"${ok:-0}" "${fail:-0}" "${fb:-0}" "${ng:-0}" "${po:-0}" >> "$STATUS"
|
||
return $rc
|
||
}
|
||
|
||
{
|
||
echo ">>> switch=$SWITCH rounds=$ROUNDS ops=$OPS"
|
||
for entry in $OPS; do
|
||
op="${entry%%:*}"
|
||
op_file="${entry#*:}"
|
||
[[ "$op_file" == "$entry" ]] && op_file="$op"
|
||
out="$ROOT/$op"
|
||
mkdir -p "$out"
|
||
|
||
# 第 1 轮 A 侧 record，其余全部回放它，config 跨轮跨侧一致
|
||
echo "########## $op | round 1 | A ($SWITCH=0, record) ##########"
|
||
if ! run_cell 1 A "$op" "$op_file" "$out/r1_A"; then
|
||
echo "!!! FAILED $op round 1 A; skipping this operator"
|
||
continue
|
||
fi
|
||
REC="$out/r1_A"
|
||
|
||
ok=1; reason="有失败轮次"
|
||
for ((k = 1; k <= ROUNDS; k++)); do
|
||
# 奇数轮先 A 后 B，偶数轮反过来
|
||
if (( k % 2 == 1 )); then order="A B"; else order="B A"; fi
|
||
for side in $order; do
|
||
[[ $k -eq 1 && $side == A ]] && continue # 已经跑过
|
||
val=$([[ $side == A ]] && echo 0 || echo 1)
|
||
echo "########## $op | round $k | $side ($SWITCH=$val, replay) ##########"
|
||
if ! run_cell "$k" "$side" "$op" "$op_file" "$out/r${k}_${side}" "$REC"; then
|
||
echo "!!! FAILED $op round $k $side"
|
||
ok=0
|
||
fi
|
||
done
|
||
|
||
if (( k == 1 && ok )); then
|
||
fa=$(ir_fingerprint "$out/r1_A") || fa=""
|
||
fb=$(ir_fingerprint "$out/r1_B") || fb=""
|
||
if [[ -z "$fa" || -z "$fb" ]]; then
|
||
echo "??? $op: 没有编出任何 .ttgir（可能该算子没走 Triton，或本轮失败），"
|
||
echo "??? 跳过对比轴有效性检查——这不等于开关无作用"
|
||
elif [[ "$fa" == "$fb" ]]; then
|
||
echo "!!! SWITCH_NO_IR_DIFF op=$op switch=$SWITCH"
|
||
echo "!!! 两侧 ttgir 逐字节相同：这个开关没有改变 $op 的生成代码。"
|
||
echo "!!! 继续跑 $ROUNDS 轮只会测出噪声，并报成一个'无差异'结论。"
|
||
echo "!!! 先确认开关名拼写与它在被测源码里的读取点；确属只影响运行时的"
|
||
echo "!!! 开关，用 SWITCH_ALLOW_NO_IR_DIFF=1 重跑。"
|
||
if [[ "$SWITCH_ALLOW_NO_IR_DIFF" != "1" ]]; then
|
||
ok=0; reason="开关未改变该算子的生成代码"
|
||
break
|
||
fi
|
||
else
|
||
echo ">>> 对比轴有效：$op 两侧 ttgir 不同（A=${fa:0:12} B=${fb:0:12}）"
|
||
fi
|
||
fi
|
||
done
|
||
|
||
echo "########## $op | report ##########"
|
||
if (( ok )); then
|
||
python batch_summary.py --ab-rounds "$out" --label "$op" \
|
||
|| echo "!!! report failed: $op"
|
||
else
|
||
# 缺轮的算子照样出报告会让人误以为是完整的 N 轮
|
||
echo "!!! $op $reason，不出报告；产物在 $out"
|
||
fi
|
||
done
|
||
echo "########## ALL DONE ##########"
|
||
} 2>&1 | tee "$LOG"
|
||
|
||
sed -i -E $'s/\x1b\\[[0-9;]*[A-Za-z]//g' "$LOG"
|
||
|
||
echo ">>> 总 log: $(pwd)/$LOG"
|
||
echo ">>> 逐格状态表: $(pwd)/$STATUS"
|
||
[[ -s "$ROOT/telemetry.tsv" ]] && echo ">>> 时钟/温度: $(pwd)/$ROOT/telemetry.tsv"
|
||
[[ -s "$ROOT/telemetry.err" ]] && echo ">>> 遥测提示: $(cat "$ROOT/telemetry.err")"
|
||
echo ">>> 逐算子重出报告: python batch_summary.py --ab-rounds $ROOT/<op> --label <op>"
|
||
echo ">>> 解读要点: 逐轮区间宽说明判定不稳；单侧噪声大说明机器不稳；两者含义不同"
|