chore: harden generate_ar30.sh (drop no-op mock, add zero-gain early stop)

2026-07-14 21:08:00 -04:00
parent b1c1bf7aac
commit d84f9bf777
1 changed files with 108 additions and 0 deletions
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# AR 专项出题 — 循环直到生成够 30 题
+#
+# 用法:
+#   bash scripts/generate_ar30.sh                    # 正常运行（需真实 LLM/VLM 凭据）
+#   EARLY_STOP_ROUNDS=3 bash scripts/generate_ar30.sh  # 放宽早停阈值
+#
+# 行为:
+#   - 每轮调用 generate-v2 --task-types "Action Recognition"
+#   - AR 走 grounded selector（候选池 + VLM 视觉打分），产出真实 near-miss 干扰项
+#   - 检查 output JSON 中的题目数
+#   - 不够 30 题则换 seed 重跑（断点续跑 + 追加写入）
+#   - 够了就停；连续 EARLY_STOP_ROUNDS 轮 0 产出也停（避免空跑满 MAX_ROUNDS）
+#
+# 注意:
+#   - 利用管线的断点续跑 + on_accept 逐题追加机制
+#   - 每轮换 seed 使采样多样化，但 output_dir 不变（追加到同一文件）
+#   - selector 让每题多次 VLM 调用：更慢更贵、AR 产量低于普通题型，属正常
+#   - 无 mock 模式：本管线不读 LLM_MOCK/VLM_MOCK，需 .env 配好真实 VL_LLM_* / JUDGE_LLM_*
+
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+TARGET=30
+CONFIG="${CONFIG:-config/question_gen_ar30.yaml}"
+STORE_DIR="${STORE_DIR:-store}"
+DB_PATH="${DB_PATH:-logs/question_gen_ar30.db}"
+OUTPUT="store/questions/generated-ar30/accepted_questions.json"
+MAX_ROUNDS=10
+BASE_SEED=2024
+EARLY_STOP_ROUNDS="${EARLY_STOP_ROUNDS:-2}"   # 连续 N 轮 0 产出则提前停止
+
+export PYTHONUNBUFFERED=1
+export HF_HUB_OFFLINE=1
+export TRANSFORMERS_OFFLINE=1
+
+# shellcheck source=../.env
+source .env
+
+PYTHON="$(conda run -n Video-Tree-TRM which python)"
+
+count_accepted() {
+    if [ -f "$OUTPUT" ]; then
+        "$PYTHON" -c "
+import json, sys
+try:
+    data = json.loads(open('$OUTPUT').read())
+    print(len(data))
+except Exception:
+    print(0)
+"
+    else
+        echo 0
+    fi
+}
+
+echo "=== AR 出题循环 — 目标 ${TARGET} 题 ==="
+echo "配置: ${CONFIG}"
+echo "输出: ${OUTPUT}"
+echo "早停: 连续 ${EARLY_STOP_ROUNDS} 轮 0 产出则停止"
+echo ""
+
+zero_streak=0
+
+for round in $(seq 1 "$MAX_ROUNDS"); do
+    current=$(count_accepted)
+    echo "[Round ${round}/${MAX_ROUNDS}] 当前已有 ${current}/${TARGET} 题"
+
+    if [ "$current" -ge "$TARGET" ]; then
+        echo "✓ 已达目标 ${TARGET} 题，停止"
+        break
+    fi
+
+    remaining=$((TARGET - current))
+    seed=$((BASE_SEED + round - 1))
+    echo "  需要 ${remaining} 题，seed=${seed}"
+
+    "$PYTHON" tools/generate_questions.py generate-v2 \
+        --store-dir "$STORE_DIR" \
+        --config "$CONFIG" \
+        --db-path "$DB_PATH" \
+        --seed "$seed" \
+        --task-types "Action Recognition" \
+        2>&1 | tee -a "logs/generate_ar30_round${round}.log"
+
+    new_count=$(count_accepted)
+    gained=$((new_count - current))
+    echo "  本轮生成 ${gained} 题 (累计 ${new_count}/${TARGET})"
+
+    if [ "$gained" -eq 0 ]; then
+        zero_streak=$((zero_streak + 1))
+        echo "  ⚠ 本轮 0 产出（连续 ${zero_streak}/${EARLY_STOP_ROUNDS}）"
+        if [ "$zero_streak" -ge "$EARLY_STOP_ROUNDS" ]; then
+            echo "⚠ 连续 ${zero_streak} 轮 0 产出（selector/gate 可能持续拒题），提前停止"
+            break
+        fi
+    else
+        zero_streak=0
+    fi
+    echo ""
+done
+
+final=$(count_accepted)
+echo "=== 完成：共 ${final} 道 AR 题 ==="
+if [ "$final" -lt "$TARGET" ]; then
+    echo "⚠ 未达目标（${final}/${TARGET}），可能需要调整 retry_limit 或检查 gate 通过率"
+fi