forked from zhoulin/zl_bench
bench: bound each screening pair and record what it cost
Screening is first contact with an operator, which makes it the place most likely to meet one that hangs — and it had no upper bound at all, so a single operator could hold the whole pass indefinitely. The entire argument for having a screening step is that it is fast. CELL_TIMEOUT (default 2400s, as in ab_repeat.sh) now caps each side, and a truncated pair says so instead of being folded into the dtype explanation: rc 124/137 is a cost problem, not a verdict about the switch. elapsed_before/elapsed_after join screen.tsv. The two-step design rests on the ratio between a pair and six rounds — 0.8 minutes against 13-44 on S5000 — so the pass should record its own half of that ratio rather than leave it to be reconstructed from directory timestamps.
This commit is contained in:
Aharrypotter committed
1 parent
1fd3abd56d
commit
a11863399a
1 file changed
+23
-11
+23
-11
@@ -45,6 +45,10 @@ ENHANCE_VAR="FLAGTREE_${VENDOR}_RLC_ENHANCE"
|
||||
# 空格分隔的算子表。元素为 OP 或 OP:OP_FILE（省略时 OP_FILE=OP）。
|
||||
OPS="${OPS:-softmax}"
|
||||
|
||||
# 筛选是第一次接触一个算子的地方，最容易撞上挂死的那个。没有上限时一个算子
|
||||
# 能把整批停在那里，而筛选的全部意义就是快。
|
||||
CELL_TIMEOUT="${CELL_TIMEOUT:-2400}"
|
||||
|
||||
if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then
|
||||
export FLAGGEMS_PERF_COLOR=always
|
||||
fi
|
||||
@@ -55,8 +59,7 @@ ROOT="runs/ab_rlc_$(date +%Y%m%d-%H%M%S)"
|
||||
mkdir -p "$ROOT"
|
||||
LOG="$ROOT/summary.log"
|
||||
TSV="$ROOT/screen.tsv"
|
||||
printf 'op op_file dtypes rows_before rows_after kernels_before kernels_after cvt_before cvt_after verdict
|
||||
' > "$TSV"
|
||||
printf 'op op_file dtypes rows_before rows_after kernels_before kernels_after cvt_before cvt_after elapsed_before elapsed_after verdict\n' > "$TSV"
|
||||
|
||||
count_cvt() { find "$1/ttgir" -name '*.ttgir' -type f -exec cat {} + 2>/dev/null | grep -c convert_layout || true; }
|
||||
count_kernels() { find "$1/ttgir" -name '*.ttgir' -type f 2>/dev/null | wc -l; }
|
||||
@@ -74,14 +77,19 @@ count_rows() { grep -c '^SUCCESS' "$1/run.log" 2>/dev/null || true; }
|
||||
BEFORE="$ROOT/${op}_before"; AFTER="$ROOT/${op}_after"
|
||||
|
||||
echo "########## $op | $ENHANCE_VAR=0 before (record) ##########"
|
||||
t0=$(date +%s)
|
||||
OP="$op" OP_FILE="$op_file" OUT_DIR="$BEFORE" DTYPES="$dtypes" \
|
||||
env "$ENHANCE_VAR=0" bash run_pytest.sh
|
||||
rc_before=$?
|
||||
timeout -k 30 "$CELL_TIMEOUT" env "$ENHANCE_VAR=0" bash run_pytest.sh
|
||||
rc_before=$?; el_before=$(( $(date +%s) - t0 ))
|
||||
|
||||
echo "########## $op | $ENHANCE_VAR=1 after (replay $BEFORE) ##########"
|
||||
t0=$(date +%s)
|
||||
OP="$op" OP_FILE="$op_file" OUT_DIR="$AFTER" DTYPES="$dtypes" \
|
||||
REPLAY_FROM="$BEFORE" env "$ENHANCE_VAR=1" bash run_pytest.sh
|
||||
rc_after=$?
|
||||
REPLAY_FROM="$BEFORE" timeout -k 30 "$CELL_TIMEOUT" env "$ENHANCE_VAR=1" bash run_pytest.sh
|
||||
rc_after=$?; el_after=$(( $(date +%s) - t0 ))
|
||||
for rc in "$rc_before" "$rc_after"; do
|
||||
(( rc == 124 || rc == 137 )) && echo "!!! CELL_TIMEOUT ${CELL_TIMEOUT}s op=$op"
|
||||
done
|
||||
|
||||
rb=$(count_rows "$BEFORE"); ra=$(count_rows "$AFTER")
|
||||
kb=$(count_kernels "$BEFORE"); ka=$(count_kernels "$AFTER")
|
||||
@@ -91,7 +99,11 @@ count_rows() { grep -c '^SUCCESS' "$1/run.log" 2>/dev/null || true; }
|
||||
|
||||
if (( rc_before != 0 || rc_after != 0 )) || [[ "${rb:-0}" == 0 || "${ra:-0}" == 0 ]]; then
|
||||
verdict=NO_ROWS
|
||||
note="跑不出测量行（rc=$rc_before/$rc_after）。看 run.log——多半是 dtype 不被该算子接受，或它在本后端拿不到 config"
|
||||
if (( rc_before == 124 || rc_after == 124 || rc_before == 137 || rc_after == 137 )); then
|
||||
note="超过 CELL_TIMEOUT=${CELL_TIMEOUT}s 被截断（rc=$rc_before/$rc_after）。这是耗时问题，不是对开关的判定——先看它为什么这么慢"
|
||||
else
|
||||
note="跑不出测量行（rc=$rc_before/$rc_after）。看 run.log——多半是 dtype 不被该算子接受，或它在本后端拿不到 config"
|
||||
fi
|
||||
elif [[ -z "$fb" || -z "$fa" ]]; then
|
||||
verdict=NO_TRITON
|
||||
note="跑出了行但没编出 .ttgir：本次运行没有任何东西经过 Triton。算子本身可能完全正常（多半跑的是 aten 或厂商库），只是被测的编译器 pass 没有作用对象。不是'开关无作用'，也不是'没有 kernel'"
|
||||
@@ -102,9 +114,9 @@ count_rows() { grep -c '^SUCCESS' "$1/run.log" 2>/dev/null || true; }
|
||||
verdict=IR_CHANGED
|
||||
note="生成代码有变（cvt ${cb}→${ca}，指纹 ${fb:0:8}/${fa:0:8}）：值得排六轮 A/B"
|
||||
fi
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$op" "$op_file" "${dtypes:-<all>}" "${rb:-0}" "${ra:-0}" \
|
||||
"$kb" "$ka" "${cb:-0}" "${ca:-0}" "$verdict" >> "$TSV"
|
||||
"$kb" "$ka" "${cb:-0}" "${ca:-0}" "$el_before" "$el_after" "$verdict" >> "$TSV"
|
||||
echo ">>> $op: $verdict — $note"
|
||||
|
||||
if [[ "$verdict" == NO_ROWS ]]; then
|
||||
@@ -117,8 +129,8 @@ count_rows() { grep -c '^SUCCESS' "$1/run.log" 2>/dev/null || true; }
|
||||
done
|
||||
|
||||
echo "########## 筛选汇总 ##########"
|
||||
awk -F'\t' 'NR>1{printf " %-12s %-22s cvt %s->%s kernels %s/%s rows %s/%s\n", \
|
||||
$10,$1,$8,$9,$6,$7,$4,$5}' "$TSV" | sort
|
||||
awk -F'\t' 'NR>1{printf " %-12s %-22s cvt %s->%s kernels %s/%s rows %s/%s %ss\n", \
|
||||
$12,$1,$8,$9,$6,$7,$4,$5,$10+$11}' "$TSV" | sort
|
||||
echo ">>> 只把 IR_CHANGED 的算子送进 ab_repeat.sh。"
|
||||
echo "########## ALL DONE ##########"
|
||||
} 2>&1 | tee "$LOG"
|
||||
|
||||
Reference in new issue
Block a user