diff --git a/ab_rlc_test.sh b/ab_rlc_test.sh index 6e8c2ab..e22751a 100644 --- a/ab_rlc_test.sh +++ b/ab_rlc_test.sh @@ -45,6 +45,10 @@ ENHANCE_VAR="FLAGTREE_${VENDOR}_RLC_ENHANCE" # 空格分隔的算子表。元素为 OP 或 OP:OP_FILE(省略时 OP_FILE=OP)。 OPS="${OPS:-softmax}" +# 筛选是第一次接触一个算子的地方,最容易撞上挂死的那个。没有上限时一个算子 +# 能把整批停在那里,而筛选的全部意义就是快。 +CELL_TIMEOUT="${CELL_TIMEOUT:-2400}" + if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then export FLAGGEMS_PERF_COLOR=always fi @@ -55,8 +59,7 @@ ROOT="runs/ab_rlc_$(date +%Y%m%d-%H%M%S)" mkdir -p "$ROOT" LOG="$ROOT/summary.log" TSV="$ROOT/screen.tsv" -printf 'op op_file dtypes rows_before rows_after kernels_before kernels_after cvt_before cvt_after verdict -' > "$TSV" +printf 'op op_file dtypes rows_before rows_after kernels_before kernels_after cvt_before cvt_after elapsed_before elapsed_after verdict\n' > "$TSV" count_cvt() { find "$1/ttgir" -name '*.ttgir' -type f -exec cat {} + 2>/dev/null | grep -c convert_layout || true; } count_kernels() { find "$1/ttgir" -name '*.ttgir' -type f 2>/dev/null | wc -l; } @@ -74,14 +77,19 @@ count_rows() { grep -c '^SUCCESS' "$1/run.log" 2>/dev/null || true; } BEFORE="$ROOT/${op}_before"; AFTER="$ROOT/${op}_after" echo "########## $op | $ENHANCE_VAR=0 before (record) ##########" + t0=$(date +%s) OP="$op" OP_FILE="$op_file" OUT_DIR="$BEFORE" DTYPES="$dtypes" \ - env "$ENHANCE_VAR=0" bash run_pytest.sh - rc_before=$? + timeout -k 30 "$CELL_TIMEOUT" env "$ENHANCE_VAR=0" bash run_pytest.sh + rc_before=$?; el_before=$(( $(date +%s) - t0 )) echo "########## $op | $ENHANCE_VAR=1 after (replay $BEFORE) ##########" + t0=$(date +%s) OP="$op" OP_FILE="$op_file" OUT_DIR="$AFTER" DTYPES="$dtypes" \ - REPLAY_FROM="$BEFORE" env "$ENHANCE_VAR=1" bash run_pytest.sh - rc_after=$? + REPLAY_FROM="$BEFORE" timeout -k 30 "$CELL_TIMEOUT" env "$ENHANCE_VAR=1" bash run_pytest.sh + rc_after=$?; el_after=$(( $(date +%s) - t0 )) + for rc in "$rc_before" "$rc_after"; do + (( rc == 124 || rc == 137 )) && echo "!!! CELL_TIMEOUT ${CELL_TIMEOUT}s op=$op" + done rb=$(count_rows "$BEFORE"); ra=$(count_rows "$AFTER") kb=$(count_kernels "$BEFORE"); ka=$(count_kernels "$AFTER") @@ -91,7 +99,11 @@ count_rows() { grep -c '^SUCCESS' "$1/run.log" 2>/dev/null || true; } if (( rc_before != 0 || rc_after != 0 )) || [[ "${rb:-0}" == 0 || "${ra:-0}" == 0 ]]; then verdict=NO_ROWS - note="跑不出测量行(rc=$rc_before/$rc_after)。看 run.log——多半是 dtype 不被该算子接受,或它在本后端拿不到 config" + if (( rc_before == 124 || rc_after == 124 || rc_before == 137 || rc_after == 137 )); then + note="超过 CELL_TIMEOUT=${CELL_TIMEOUT}s 被截断(rc=$rc_before/$rc_after)。这是耗时问题,不是对开关的判定——先看它为什么这么慢" + else + note="跑不出测量行(rc=$rc_before/$rc_after)。看 run.log——多半是 dtype 不被该算子接受,或它在本后端拿不到 config" + fi elif [[ -z "$fb" || -z "$fa" ]]; then verdict=NO_TRITON note="跑出了行但没编出 .ttgir:本次运行没有任何东西经过 Triton。算子本身可能完全正常(多半跑的是 aten 或厂商库),只是被测的编译器 pass 没有作用对象。不是'开关无作用',也不是'没有 kernel'" @@ -102,9 +114,9 @@ count_rows() { grep -c '^SUCCESS' "$1/run.log" 2>/dev/null || true; } verdict=IR_CHANGED note="生成代码有变(cvt ${cb}→${ca},指纹 ${fb:0:8}/${fa:0:8}):值得排六轮 A/B" fi - printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ "$op" "$op_file" "${dtypes:-}" "${rb:-0}" "${ra:-0}" \ - "$kb" "$ka" "${cb:-0}" "${ca:-0}" "$verdict" >> "$TSV" + "$kb" "$ka" "${cb:-0}" "${ca:-0}" "$el_before" "$el_after" "$verdict" >> "$TSV" echo ">>> $op: $verdict — $note" if [[ "$verdict" == NO_ROWS ]]; then @@ -117,8 +129,8 @@ count_rows() { grep -c '^SUCCESS' "$1/run.log" 2>/dev/null || true; } done echo "########## 筛选汇总 ##########" - awk -F'\t' 'NR>1{printf " %-12s %-22s cvt %s->%s kernels %s/%s rows %s/%s\n", \ - $10,$1,$8,$9,$6,$7,$4,$5}' "$TSV" | sort + awk -F'\t' 'NR>1{printf " %-12s %-22s cvt %s->%s kernels %s/%s rows %s/%s %ss\n", \ + $12,$1,$8,$9,$6,$7,$4,$5,$10+$11}' "$TSV" | sort echo ">>> 只把 IR_CHANGED 的算子送进 ab_repeat.sh。" echo "########## ALL DONE ##########" } 2>&1 | tee "$LOG"