diff --git a/scripts/generate_ar30.sh b/scripts/generate_ar30.sh new file mode 100755 index 0000000..72e7c11 --- /dev/null +++ b/scripts/generate_ar30.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# AR 专项出题 — 循环直到生成够 30 题 +# +# 用法: +# bash scripts/generate_ar30.sh # 正常运行(需真实 LLM/VLM 凭据) +# EARLY_STOP_ROUNDS=3 bash scripts/generate_ar30.sh # 放宽早停阈值 +# +# 行为: +# - 每轮调用 generate-v2 --task-types "Action Recognition" +# - AR 走 grounded selector(候选池 + VLM 视觉打分),产出真实 near-miss 干扰项 +# - 检查 output JSON 中的题目数 +# - 不够 30 题则换 seed 重跑(断点续跑 + 追加写入) +# - 够了就停;连续 EARLY_STOP_ROUNDS 轮 0 产出也停(避免空跑满 MAX_ROUNDS) +# +# 注意: +# - 利用管线的断点续跑 + on_accept 逐题追加机制 +# - 每轮换 seed 使采样多样化,但 output_dir 不变(追加到同一文件) +# - selector 让每题多次 VLM 调用:更慢更贵、AR 产量低于普通题型,属正常 +# - 无 mock 模式:本管线不读 LLM_MOCK/VLM_MOCK,需 .env 配好真实 VL_LLM_* / JUDGE_LLM_* + +set -euo pipefail + +cd "$(dirname "$0")/.." + +TARGET=30 +CONFIG="${CONFIG:-config/question_gen_ar30.yaml}" +STORE_DIR="${STORE_DIR:-store}" +DB_PATH="${DB_PATH:-logs/question_gen_ar30.db}" +OUTPUT="store/questions/generated-ar30/accepted_questions.json" +MAX_ROUNDS=10 +BASE_SEED=2024 +EARLY_STOP_ROUNDS="${EARLY_STOP_ROUNDS:-2}" # 连续 N 轮 0 产出则提前停止 + +export PYTHONUNBUFFERED=1 +export HF_HUB_OFFLINE=1 +export TRANSFORMERS_OFFLINE=1 + +# shellcheck source=../.env +source .env + +PYTHON="$(conda run -n Video-Tree-TRM which python)" + +count_accepted() { + if [ -f "$OUTPUT" ]; then + "$PYTHON" -c " +import json, sys +try: + data = json.loads(open('$OUTPUT').read()) + print(len(data)) +except Exception: + print(0) +" + else + echo 0 + fi +} + +echo "=== AR 出题循环 — 目标 ${TARGET} 题 ===" +echo "配置: ${CONFIG}" +echo "输出: ${OUTPUT}" +echo "早停: 连续 ${EARLY_STOP_ROUNDS} 轮 0 产出则停止" +echo "" + +zero_streak=0 + +for round in $(seq 1 "$MAX_ROUNDS"); do + current=$(count_accepted) + echo "[Round ${round}/${MAX_ROUNDS}] 当前已有 ${current}/${TARGET} 题" + + if [ "$current" -ge "$TARGET" ]; then + echo "✓ 已达目标 ${TARGET} 题,停止" + break + fi + + remaining=$((TARGET - current)) + seed=$((BASE_SEED + round - 1)) + echo " 需要 ${remaining} 题,seed=${seed}" + + "$PYTHON" tools/generate_questions.py generate-v2 \ + --store-dir "$STORE_DIR" \ + --config "$CONFIG" \ + --db-path "$DB_PATH" \ + --seed "$seed" \ + --task-types "Action Recognition" \ + 2>&1 | tee -a "logs/generate_ar30_round${round}.log" + + new_count=$(count_accepted) + gained=$((new_count - current)) + echo " 本轮生成 ${gained} 题 (累计 ${new_count}/${TARGET})" + + if [ "$gained" -eq 0 ]; then + zero_streak=$((zero_streak + 1)) + echo " ⚠ 本轮 0 产出(连续 ${zero_streak}/${EARLY_STOP_ROUNDS})" + if [ "$zero_streak" -ge "$EARLY_STOP_ROUNDS" ]; then + echo "⚠ 连续 ${zero_streak} 轮 0 产出(selector/gate 可能持续拒题),提前停止" + break + fi + else + zero_streak=0 + fi + echo "" +done + +final=$(count_accepted) +echo "=== 完成:共 ${final} 道 AR 题 ===" +if [ "$final" -lt "$TARGET" ]; then + echo "⚠ 未达目标(${final}/${TARGET}),可能需要调整 retry_limit 或检查 gate 通过率" +fi