From 2786eb98864d6c80b54bf0eced8014a4747585aa Mon Sep 17 00:00:00 2001 From: Aharrypotter Date: Thu, 10 Sep 2026 19:43:37 +0800 Subject: [PATCH] bench: orchestrate a multi-card campaign without touching the measurement path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit campaign.sh fans a screened operator list across cards, one ab_repeat.sh per card, serial within a card. It does no measuring: ab_repeat.sh is the path that produced the four operator verdicts already on record, and it stays byte for byte as it was. Each card gets its own AB_ROOT. Eight ab_repeat.sh started in the same second would otherwise compute the same timestamped directory name and two cards would write into one. ZL_ISOLATE_TUNER_CACHE=1 is forced, since eight processes on one sqlite is the thing to avoid. The cost is that pinned configs differ from a shared-cache batch, so absolute latencies cannot be quoted across the two; the caliber guard refuses such comparisons on its own, which is why this is a note and not a warning someone has to remember. partition.py's shell output gained the per-card weight, because EXTRA_OPS has to land on the lightest card and counting operators gets that wrong — one heavy operator outweighs three light ones. It writes through stdout.buffer: Python turns "\n" into CRLF on Windows, and the trailing \r then rides into the last field and breaks the numeric comparison that picks the card. --- README.md | 23 +++++++++ ab_repeat.sh | 4 +- ab_rlc_test.sh | 2 +- campaign.sh | 128 +++++++++++++++++++++++++++++++++++++++++++++++++ partition.py | 8 +++- 5 files changed, 162 insertions(+), 3 deletions(-) create mode 100644 campaign.sh diff --git a/README.md b/README.md index 8567777..434c0ac 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,29 @@ ttgir 关键设计： - **命名 = 组内有区分度的参数**：同一 (shape, kernel) 组内取值相同的 constexpr 不进文件名；多词参数缩写为首字母（`BLOCK_SIZE_M` → `BSM`，图例见 naming.md）；同名冲突依次用 dtype（`__fp16`）、参数对齐特化（`__EMdiv16`）、hash 前缀消歧。 - **崩溃也能拿到 IR**：dump 挂在进程 atexit 上，CUDA crash 后仍会落盘已编译部分。 +## 多卡跑一批：campaign.sh + +`ab_repeat.sh` 是单卡串行的。一批算子要并行时用 `campaign.sh` 编排——它只分活， +测量仍然全部由 `ab_repeat.sh` 做，那个脚本一行不动。 + +```bash +SWITCH=FLAGTREE_MUSA_RLC_ENHANCE DEVICES=0,1,2,3,4,5,6,7 SCREEN=runs/ab_rlc_XXX/screen.tsv ZL_BACKEND=musa FLAGGEMS_DIR=/path/to/FlagGems DRY_RUN=1 bash campaign.sh +``` + +`DRY_RUN=1` 先看方案，确认了再去掉它起跑。 + +- **只取筛选表里 `IR_CHANGED` 的算子**——第一步的判定在这里生效 +- **按实测耗时贪心分片**（`partition.py`），不是 hash 取模。S5000 上 16 个算子实测： + hash 分片关键路径 120 分钟、有一张卡空转；贪心 70 分钟。**下界由最重的单个算子 + 决定**（`topk` 一个就 70.6 分钟），加卡不解决，分片只保证别的卡不空等 +- **每卡独立 `AB_ROOT`**——八个 `ab_repeat.sh` 同一秒启动，默认的时间戳目录名会撞 +- **强制 `ZL_ISOLATE_TUNER_CACHE=1`**。8 个进程写同一个 sqlite 是要避开的事。代价是 + pin 的 config 与共享缓存批次不同，**两批的绝对延迟不可互引**——口径守卫会自己拒绝 +- `EXTRA_OPS` 追加筛选表里没有的算子（如 benchmark 刚修好的），放到**负载最小**的卡上 +- 跑完合并各卡 `status.tsv`（首列是卡号），并统计非零退出的格与三个标记的总数 + +设备可见性变量名从 backends 层取，不写死 `MUSA_VISIBLE_DEVICES`。 + ## 两步：先筛选，再实测 跑一个算子的六轮 A/B 要 13~44 分钟（S5000 实测：`rand` 15 shape 13.0 分钟， diff --git a/ab_repeat.sh b/ab_repeat.sh index 79cfcf6..b65b7fb 100644 --- a/ab_repeat.sh +++ b/ab_repeat.sh @@ -42,7 +42,9 @@ if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then export FLAGGEMS_PERF_COLOR=always fi -ROOT="runs/ab_repeat_${SWITCH}_$(date +%Y%m%d-%H%M%S)" +# AB_ROOT 供 campaign.sh 指定每卡独立的产物根：八个 ab_repeat.sh 同一秒启动时， +# 默认的时间戳目录名会撞在一起，两张卡往同一个目录里写。 +ROOT="${AB_ROOT:-runs/ab_repeat_${SWITCH}_$(date +%Y%m%d-%H%M%S)}" mkdir -p "$ROOT" LOG="$ROOT/summary.log" STATUS="$ROOT/status.tsv" diff --git a/ab_rlc_test.sh b/ab_rlc_test.sh index 2448ace..59ae113 100644 --- a/ab_rlc_test.sh +++ b/ab_rlc_test.sh @@ -80,7 +80,7 @@ if [[ -n "$RESCORE" ]]; then done < <(tail -n +2 "$TSV.before-rescore") echo ">>> 重打分完成，$changed 个算子的判定变了；原表存为 $TSV.before-rescore" - awk -F'\t' 'NR>1{c[$NF]++} END{for (k in c) printf(" %-12s %d\n", k, c[k])}' "$TSV" + awk -v i="$c_v" -F'\t' 'NR>1{c[$i]++} END{for (k in c) printf(" %-12s %d\n", k, c[k])}' "$TSV" exit 0 fi diff --git a/campaign.sh b/campaign.sh new file mode 100644 index 0000000..69e5e4b --- /dev/null +++ b/campaign.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# 多卡跑一批算子的六轮 A/B：每卡一个 ab_repeat.sh，卡内串行。 +# +# 它只做编排，不做测量。ab_repeat.sh 一行不动——那是已经跑出结论的、验证过的串行 +# 路径，不该为了并行去改它。分片交给 partition.py（按实测耗时贪心，比 hash 分片在 +# 这批活上快 50 分钟）。 +# +# 并行必须隔离 tuner cache：8 个进程写同一个 sqlite 是要避开的事。代价是 pin 下来 +# 的 config 与共享缓存下跑的批次不同，两批的绝对延迟不可互引——batch_summary.py +# 的口径守卫会自己拒绝这种对比，不用记着。 +# +# 用法： +# SWITCH=FLAGTREE_MUSA_RLC_ENHANCE DEVICES=0,1,2,3,4,5,6,7 \ +# SCREEN=runs/ab_rlc_XXX/screen.tsv \ +# ZL_BACKEND=musa FLAGGEMS_DIR=/path/to/FlagGems bash campaign.sh +# +# DRY_RUN=1 只打印分片方案，不起跑 +# EXTRA_OPS=".." 追加筛选表里没有的算子（如 benchmark 刚修好的），塞给最空的卡 +# ROUNDS=6 CELL_TIMEOUT=2400 VERDICT=IR_CHANGED +set -uo pipefail + +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +: "${SWITCH:?SWITCH is required, e.g. FLAGTREE_MUSA_RLC_ENHANCE}" +: "${SCREEN:?SCREEN is required, 指向 ab_rlc_test.sh 的 screen.tsv}" +DEVICES="${DEVICES:-0}" +ROUNDS="${ROUNDS:-6}" +CELL_TIMEOUT="${CELL_TIMEOUT:-2400}" +VERDICT="${VERDICT:-IR_CHANGED}" +EXTRA_OPS="${EXTRA_OPS:-}" +DRY_RUN="${DRY_RUN:-0}" +TAB=$'\t' + +[[ -f "$SCREEN" ]] || { echo "!!! 筛选表不存在: $SCREEN" >&2; exit 2; } + +# 设备可见性变量名从 backends 层拿，不写死 MUSA_VISIBLE_DEVICES +VIS_KEY=$(python -c "import sys; sys.path.insert(0, '.') +from backends import get_backend +print(get_backend().visibility_env_keys[0])" 2>/dev/null) || VIS_KEY="" +[[ -n "$VIS_KEY" ]] || { echo "!!! 取不到后端的可见性变量名，检查 ZL_BACKEND" >&2; exit 2; } + +echo ">>> switch=$SWITCH rounds=$ROUNDS devices=$DEVICES 可见性变量=$VIS_KEY" +python partition.py --screen "$SCREEN" --devices "$DEVICES" \ + --verdict "$VERDICT" --rounds "$ROUNDS" || exit 2 + +# 每卡一行 "<权重秒>" +mapfile -t PLAN < <(python partition.py --screen "$SCREEN" --devices "$DEVICES" \ + --verdict "$VERDICT" --format shell) || exit 2 + +if [[ -n "$EXTRA_OPS" ]]; then + # 追加到当前负载最小的卡。按算子个数选是错的——一个重算子能顶三个轻的， + # partition.py 的第三列给的就是权重秒数 + lightest=0; least="" + for i in "${!PLAN[@]}"; do + IFS="$TAB" read -r _d _o _w <<< "${PLAN[$i]}" + if [[ -z "$least" ]] || (( ${_w:-0} < least )); then least=${_w:-0}; lightest=$i; fi + done + IFS="$TAB" read -r d o w <<< "${PLAN[$lightest]}" + PLAN[$lightest]="${d}${TAB}${o:+$o }${EXTRA_OPS}${TAB}${w}" + echo ">>> EXTRA_OPS 追加到卡${d}（当前最空，$(( ${w:-0} * ROUNDS / 60 )) 分钟）: $EXTRA_OPS" +fi + +if [[ "$DRY_RUN" == "1" ]]; then + echo ">>> DRY_RUN=1，不起跑。逐卡将执行：" + for row in "${PLAN[@]}"; do + IFS="$TAB" read -r d o w <<< "$row" + printf ' 卡%s %4s 分钟 OPS="%s"\n' "$d" "$(( ${w:-0} * ROUNDS / 60 ))" "$o" + done + exit 0 +fi + +ROOT="runs/campaign_${SWITCH}_$(date +%Y%m%d-%H%M%S)" +mkdir -p "$ROOT" +printf '%s\n' "${PLAN[@]}" > "$ROOT/plan.tsv" +echo ">>> 产物: $(pwd)/$ROOT" + +declare -a PIDS=() CARDS=() +for row in "${PLAN[@]}"; do + IFS="$TAB" read -r dev ops _w <<< "$row" + [[ -n "$ops" ]] || { echo ">>> 卡$dev 没有分到活，跳过"; continue; } + out="$ROOT/card${dev}" + mkdir -p "$out" + # AB_ROOT 让每卡有独立产物根：八个 ab_repeat.sh 同一秒启动，默认的时间戳目录名会撞 + env "$VIS_KEY=$dev" ZL_DEVICE="$dev" ZL_ISOLATE_TUNER_CACHE=1 \ + AB_ROOT="$out" SWITCH="$SWITCH" ROUNDS="$ROUNDS" \ + CELL_TIMEOUT="$CELL_TIMEOUT" OPS="$ops" NO_COLOR=1 \ + bash ab_repeat.sh > "$ROOT/card${dev}.log" 2>&1 & + PIDS+=($!); CARDS+=("$dev") + echo ">>> 卡$dev pid=$! ops=$ops" +done + +(( ${#PIDS[@]} )) || { echo "!!! 没有一张卡分到活" >&2; exit 2; } +echo ">>> ${#PIDS[@]} 张卡已启动，等待全部结束……" + +fail=0 +for i in "${!PIDS[@]}"; do + if wait "${PIDS[$i]}"; then + echo ">>> 卡${CARDS[$i]} 结束 rc=0" + else + rc=$?; fail=$((fail+1)) + echo "!!! 卡${CARDS[$i]} 结束 rc=$rc —— 看 $ROOT/card${CARDS[$i]}.log" + fi +done + +# 合并逐格记录：卡号加在最前面，便于事后按卡追 +{ + first=1 + for d in "${CARDS[@]}"; do + f="$ROOT/card${d}/status.tsv" + [[ -f "$f" ]] || continue + if (( first )); then printf 'device%s%s\n' "$TAB" "$(head -1 "$f")"; first=0; fi + tail -n +2 "$f" | sed "s/^/${d}${TAB}/" + done +} > "$ROOT/status.tsv" + +echo +echo "########## 汇总 ##########" +echo ">>> 失败的卡: $fail / ${#PIDS[@]}" +if [[ -s "$ROOT/status.tsv" ]]; then + echo ">>> 逐格记录: $(pwd)/$ROOT/status.tsv ($(( $(wc -l < "$ROOT/status.tsv") - 1 )) 格)" + awk -F"$TAB" 'NR>1 && ($6+0)!=0 {n++} END{printf ">>> 非零退出的格: %d\n", n+0}' "$ROOT/status.tsv" + awk -F"$TAB" 'NR>1 {fb+=$10; ng+=$11; po+=$12} + END{printf ">>> 三个标记合计: replay_fallback=%d no_cudagraph=%d capture_poisoned=%d\n", + fb+0, ng+0, po+0}' "$ROOT/status.tsv" +fi +echo ">>> 逐算子报告在各卡的 summary.log 里；单独重出：" +echo " python batch_summary.py --ab-rounds $ROOT/card/ --label " +echo ">>> 注意：本批 ZL_ISOLATE_TUNER_CACHE=1，绝对延迟不可与共享缓存的批次互引" diff --git a/partition.py b/partition.py index 8eda301..44e1ca1 100644 --- a/partition.py +++ b/partition.py @@ -113,9 +113,15 @@ def main(argv: Optional[List[str]] = None) -> int: assignment = plan(items, devices) if args.format == "shell": + # 第三列是该卡的权重秒数，供驱动脚本决定往哪张卡追加算子——按负载选，不是 + # 按算子个数：一个重算子能顶三个轻的。 + # 显式写 "\n" 而不用 print：Windows 上 print 会发 CRLF，读的一侧会把 \r + # 带进最后一个字段，数值比较就废了。 for d in devices: ops = " ".join(f"{op}:{files[op]}" for op, _ in assignment[d]) - print(f"{d}\t{ops}") + load = sum(w for _, w in assignment[d]) + sys.stdout.buffer.write(f"{d}\t{ops}\t{load:.0f}\n".encode()) + sys.stdout.buffer.flush() return 0 total = sum(w for _, w in items) * args.rounds