forked from zhoulin/zl_bench
Aharrypotter
5236d0745f
ir_fingerprint hashed every file under the dump directory, which includes index.tsv and naming.md. An operator that compiles no Triton kernel at all leaves only those two, identical on both sides, so the switch-effectiveness check reported "byte-identical ttgir" and skipped the operator with SWITCH_NO_IR_DIFF — "the switch changed no generated code" — when the truth was that nothing was generated. router_gemm on MUSA is exactly this case: its kernel cannot be instantiated there (missing BLOCK_M/N/K/SPLIT_K from the autotune path), zero .ttgir files, and the check called it a no-op switch. Now only *.ttgir files are hashed, and a run with none returns failure so the caller treats it as "no comparable IR" rather than "identical". The message on that path says so, and says explicitly that it does not mean the switch is inert.
184 lines
8.5 KiB
Bash
184 lines
8.5 KiB
Bash
#!/bin/bash
|
||
# 多轮 A/B：同一对比在 N 轮独立进程里各跑一次，逐 shape 取轮次中位。
|
||
#
|
||
# 为什么需要它：单对观测无法说明自己稳不稳。S5000 实测 topk 消除了 32 个
|
||
# convert_layout，42 个 shape 里却有 17 个在两次同等有效的运行之间改变了 delta
|
||
# 的符号——那不是计时口径的问题（图捕获与 direct 的延迟比是 1.00x），是收益本身
|
||
# 落在零附近、单次测量给不出方向。多轮之后才谈得上"这个判定成立"。
|
||
#
|
||
# 口径：第 1 轮 A 侧 record autotune config，此后每一次运行（含 A 侧自己的后续
|
||
# 轮次）都 REPLAY_FROM 那一份。config 因此跨轮跨侧全部锁死，剩下的变量只有被测
|
||
# 开关与测量噪声。逐轮交替先后顺序，抵消单向漂移。
|
||
#
|
||
# 用法：
|
||
# SWITCH=FLAGTREE_MUSA_RLC_ENHANCE OPS="topk masked_select" ROUNDS=6 \
|
||
# ZL_BACKEND=musa FLAGGEMS_DIR=/path/to/FlagGems bash ab_repeat.sh
|
||
#
|
||
# OPS 元素为 OP 或 OP:OP_FILE（省略时 OP_FILE=OP）。
|
||
# SWITCH 是被测环境变量，A 侧设 0、B 侧设 1。
|
||
#
|
||
# 对比轴有效性由脚本自己查：第 1 轮 A/B 跑完后比两侧的 ttgir 指纹，完全一致说明
|
||
# 这个开关在这个算子上没改变任何生成代码——继续跑满 N 轮只会得到一个"无差异"的假
|
||
# 结论（同 ab_fold_test.sh 的告诫）。这类算子默认跳过；开关确实只影响运行时而不影响
|
||
# IR 时，用 SWITCH_ALLOW_NO_IR_DIFF=1 放行。
|
||
#
|
||
# 产物：runs/ab_repeat_<开关>_<时间戳>/<op>/r<k>_<A|B>/，跑完对每个算子调用
|
||
# batch_summary.py --ab-rounds 出报告。
|
||
set -uo pipefail
|
||
|
||
cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
SWITCH_ALLOW_NO_IR_DIFF="${SWITCH_ALLOW_NO_IR_DIFF:-0}"
|
||
|
||
# 一个 run 生成代码的指纹：只取 .ttgir 的相对路径 + 内容。
|
||
# 路径进指纹是因为"变体集合变了"与"某个变体内容变了"同样算改变，而只 cat 内容
|
||
# 会把前者洗掉。只认 .ttgir 则是因为 dump 目录里还有 index.tsv 和 naming.md：
|
||
# 一个 kernel 都没编出来的 run 只剩这两个文件、两侧完全相同，会把"什么都没生成"
|
||
# 报成"开关没改变生成代码"——router_gemm 在 MUSA 上就是这样。
|
||
# 零 kernel 返回非零，交给调用方当作"没有可比的 IR"，而不是"一致"。
|
||
ir_fingerprint() {
|
||
local dir="$1/ttgir"
|
||
[[ -d "$dir" ]] || return 1
|
||
local n
|
||
n=$(find "$dir" -name '*.ttgir' -type f 2>/dev/null | wc -l)
|
||
(( n > 0 )) || return 1
|
||
local out
|
||
out=$( { find "$dir" -name '*.ttgir' -type f | LC_ALL=C sort | while IFS= read -r f; do
|
||
printf '%s\n' "${f#"$dir"/}"; cat "$f"
|
||
done; } | sha256sum | cut -d' ' -f1 )
|
||
[[ -n "$out" ]] && printf '%s' "$out"
|
||
}
|
||
|
||
: "${SWITCH:?SWITCH is required, e.g. FLAGTREE_MUSA_RLC_ENHANCE}"
|
||
OPS="${OPS:-}"
|
||
: "${OPS:?OPS is required, e.g. \"topk masked_select\"}"
|
||
ROUNDS="${ROUNDS:-6}"
|
||
CELL_TIMEOUT="${CELL_TIMEOUT:-2400}"
|
||
|
||
if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then
|
||
export FLAGGEMS_PERF_COLOR=always
|
||
fi
|
||
|
||
ROOT="runs/ab_repeat_${SWITCH}_$(date +%Y%m%d-%H%M%S)"
|
||
mkdir -p "$ROOT"
|
||
LOG="$ROOT/summary.log"
|
||
STATUS="$ROOT/status.tsv"
|
||
|
||
# 全程采时钟/温度/功耗。六轮 A/B 声称的是 0.01% 量级的单侧噪声，降频和热漂移
|
||
# 能移动比这大得多的延迟，而且在 run.log 里不留痕迹——报告照样干净。
|
||
# TELEMETRY_INTERVAL=0 关闭。采不到时 telemetry.py 自己安静退出，不影响测试。
|
||
TELEMETRY_INTERVAL="${TELEMETRY_INTERVAL:-5}"
|
||
TELE_PID=""
|
||
if [[ "$TELEMETRY_INTERVAL" != "0" ]]; then
|
||
python telemetry.py --backend "${ZL_BACKEND:-cuda}" --device "${ZL_DEVICE:-}" \
|
||
--interval "$TELEMETRY_INTERVAL" --run \
|
||
> "$ROOT/telemetry.tsv" 2> "$ROOT/telemetry.err" &
|
||
TELE_PID=$!
|
||
trap '[[ -n "$TELE_PID" ]] && kill "$TELE_PID" 2>/dev/null' EXIT
|
||
fi
|
||
printf 'round\top\tside\tswitch\trc\telapsed_s\tsuccess\tfailed\treplay_fallback\tno_cudagraph\tcapture_poisoned\n' > "$STATUS"
|
||
|
||
# 跑一格并逐格登记。事后 grep 也能拿到这些数，但只在有人想起来时才发生；
|
||
# 缺格、超时、标记从 0 变成非 0，都要在跑的时候就看得见。
|
||
run_cell() { # round side op op_file out_dir [replay_from]
|
||
local round="$1" side="$2" op="$3" op_file="$4" dir="$5" replay="${6:-}"
|
||
local val=0; [[ "$side" == B ]] && val=1
|
||
local start rc elapsed log ok fail fb ng po
|
||
start=$(date +%s)
|
||
if [[ -n "$replay" ]]; then
|
||
OP="$op" OP_FILE="$op_file" OUT_DIR="$dir" REPLAY_FROM="$replay" \
|
||
timeout -k 30 "$CELL_TIMEOUT" env "$SWITCH=$val" bash run_pytest.sh
|
||
else
|
||
OP="$op" OP_FILE="$op_file" OUT_DIR="$dir" \
|
||
timeout -k 30 "$CELL_TIMEOUT" env "$SWITCH=$val" bash run_pytest.sh
|
||
fi
|
||
rc=$?
|
||
elapsed=$(( $(date +%s) - start ))
|
||
(( rc == 124 || rc == 137 )) && echo "!!! CELL_TIMEOUT ${CELL_TIMEOUT}s op=$op round=$round side=$side"
|
||
log="$dir/run.log"
|
||
# grep -c 没命中时打印 0 但以 1 退出；不吞掉的话会被当成本格失败
|
||
ok=$(grep -c '^SUCCESS' "$log" 2>/dev/null || true)
|
||
fail=$(grep -c '^FAILED' "$log" 2>/dev/null || true)
|
||
fb=$(grep -c 'AUTOTUNE_REPLAY_FALLBACK' "$log" 2>/dev/null || true)
|
||
ng=$(grep -c 'BENCHMARK_DIRECT_NO_CUDAGRAPH' "$log" 2>/dev/null || true)
|
||
po=$(grep -c 'BENCHMARK_CAPTURE_POISONED' "$log" 2>/dev/null || true)
|
||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||
"$round" "$op" "$side" "$val" "$rc" "$elapsed" \
|
||
"${ok:-0}" "${fail:-0}" "${fb:-0}" "${ng:-0}" "${po:-0}" >> "$STATUS"
|
||
return $rc
|
||
}
|
||
|
||
{
|
||
echo ">>> switch=$SWITCH rounds=$ROUNDS ops=$OPS"
|
||
for entry in $OPS; do
|
||
op="${entry%%:*}"
|
||
op_file="${entry#*:}"
|
||
[[ "$op_file" == "$entry" ]] && op_file="$op"
|
||
out="$ROOT/$op"
|
||
mkdir -p "$out"
|
||
|
||
# 第 1 轮 A 侧 record，其余全部回放它，config 跨轮跨侧一致
|
||
echo "########## $op | round 1 | A ($SWITCH=0, record) ##########"
|
||
if ! run_cell 1 A "$op" "$op_file" "$out/r1_A"; then
|
||
echo "!!! FAILED $op round 1 A; skipping this operator"
|
||
continue
|
||
fi
|
||
REC="$out/r1_A"
|
||
|
||
ok=1; reason="有失败轮次"
|
||
for ((k = 1; k <= ROUNDS; k++)); do
|
||
# 奇数轮先 A 后 B，偶数轮反过来
|
||
if (( k % 2 == 1 )); then order="A B"; else order="B A"; fi
|
||
for side in $order; do
|
||
[[ $k -eq 1 && $side == A ]] && continue # 已经跑过
|
||
val=$([[ $side == A ]] && echo 0 || echo 1)
|
||
echo "########## $op | round $k | $side ($SWITCH=$val, replay) ##########"
|
||
if ! run_cell "$k" "$side" "$op" "$op_file" "$out/r${k}_${side}" "$REC"; then
|
||
echo "!!! FAILED $op round $k $side"
|
||
ok=0
|
||
fi
|
||
done
|
||
|
||
if (( k == 1 && ok )); then
|
||
fa=$(ir_fingerprint "$out/r1_A") || fa=""
|
||
fb=$(ir_fingerprint "$out/r1_B") || fb=""
|
||
if [[ -z "$fa" || -z "$fb" ]]; then
|
||
echo "??? $op: 没有编出任何 .ttgir（可能该算子没走 Triton，或本轮失败），"
|
||
echo "??? 跳过对比轴有效性检查——这不等于开关无作用"
|
||
elif [[ "$fa" == "$fb" ]]; then
|
||
echo "!!! SWITCH_NO_IR_DIFF op=$op switch=$SWITCH"
|
||
echo "!!! 两侧 ttgir 逐字节相同：这个开关没有改变 $op 的生成代码。"
|
||
echo "!!! 继续跑 $ROUNDS 轮只会测出噪声，并报成一个'无差异'结论。"
|
||
echo "!!! 先确认开关名拼写与它在被测源码里的读取点；确属只影响运行时的"
|
||
echo "!!! 开关，用 SWITCH_ALLOW_NO_IR_DIFF=1 重跑。"
|
||
if [[ "$SWITCH_ALLOW_NO_IR_DIFF" != "1" ]]; then
|
||
ok=0; reason="开关未改变该算子的生成代码"
|
||
break
|
||
fi
|
||
else
|
||
echo ">>> 对比轴有效：$op 两侧 ttgir 不同（A=${fa:0:12} B=${fb:0:12}）"
|
||
fi
|
||
fi
|
||
done
|
||
|
||
echo "########## $op | report ##########"
|
||
if (( ok )); then
|
||
python batch_summary.py --ab-rounds "$out" --label "$op" \
|
||
|| echo "!!! report failed: $op"
|
||
else
|
||
# 缺轮的算子照样出报告会让人误以为是完整的 N 轮
|
||
echo "!!! $op $reason，不出报告；产物在 $out"
|
||
fi
|
||
done
|
||
echo "########## ALL DONE ##########"
|
||
} 2>&1 | tee "$LOG"
|
||
|
||
sed -i -E $'s/\x1b\\[[0-9;]*[A-Za-z]//g' "$LOG"
|
||
|
||
echo ">>> 总 log: $(pwd)/$LOG"
|
||
echo ">>> 逐格状态表: $(pwd)/$STATUS"
|
||
[[ -s "$ROOT/telemetry.tsv" ]] && echo ">>> 时钟/温度: $(pwd)/$ROOT/telemetry.tsv"
|
||
[[ -s "$ROOT/telemetry.err" ]] && echo ">>> 遥测提示: $(cat "$ROOT/telemetry.err")"
|
||
echo ">>> 逐算子重出报告: python batch_summary.py --ab-rounds $ROOT/<op> --label <op>"
|
||
echo ">>> 解读要点: 逐轮区间宽说明判定不稳；单侧噪声大说明机器不稳；两者含义不同"
|