114 lines
3.6 KiB
Bash
Executable File
114 lines
3.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# AR 专项出题 — 循环直到生成够 30 题
|
||
#
|
||
# 用法:
|
||
# bash scripts/generate_ar30.sh # 正常运行（需真实 LLM/VLM 凭据）
|
||
# EARLY_STOP_ROUNDS=3 bash scripts/generate_ar30.sh # 放宽早停阈值
|
||
#
|
||
# 行为:
|
||
# - 每轮调用 generate-v2 --task-types "Action Recognition"
|
||
# - AR 走 grounded selector（候选池 + VLM 视觉打分），产出真实 near-miss 干扰项
|
||
# - 检查 output JSON 中的题目数
|
||
# - 不够 30 题则换 seed 重跑（断点续跑 + 追加写入）
|
||
# - 够了就停；连续 EARLY_STOP_ROUNDS 轮 0 产出也停（避免空跑满 MAX_ROUNDS）
|
||
#
|
||
# 注意:
|
||
# - 利用管线的断点续跑 + on_accept 逐题追加机制
|
||
# - 每轮换 seed 使采样多样化，但 output_dir 不变（追加到同一文件）
|
||
# - selector 让每题多次 VLM 调用：更慢更贵、AR 产量低于普通题型，属正常
|
||
# - 无 mock 模式：本管线不读 LLM_MOCK/VLM_MOCK，需 .env 配好真实 VL_LLM_* / JUDGE_LLM_*
|
||
|
||
set -euo pipefail
|
||
|
||
cd "$(dirname "$0")/.."
|
||
|
||
TARGET=30
|
||
CONFIG="${CONFIG:-config/question_gen_ar30.yaml}"
|
||
STORE_DIR="${STORE_DIR:-store}"
|
||
DB_PATH="${DB_PATH:-logs/question_gen_ar30.db}"
|
||
MAX_ROUNDS=10
|
||
BASE_SEED=2024
|
||
EARLY_STOP_ROUNDS="${EARLY_STOP_ROUNDS:-2}" # 连续 N 轮 0 产出则提前停止
|
||
|
||
export PYTHONUNBUFFERED=1
|
||
export HF_HUB_OFFLINE=1
|
||
export TRANSFORMERS_OFFLINE=1
|
||
|
||
# shellcheck source=../.env
|
||
source .env
|
||
|
||
PYTHON="$(conda run -n Video-Tree-TRM which python)"
|
||
|
||
# OUTPUT 从 CONFIG 的 output_dir 推导，使 CONFIG 覆盖时计数路径自动跟随；可用 OUTPUT env 覆盖
|
||
if [ -z "${OUTPUT:-}" ]; then
|
||
OUTPUT_DIR="$("$PYTHON" -c "import yaml; print(yaml.safe_load(open('$CONFIG'))['question_gen_v2']['output_dir'])")"
|
||
OUTPUT="${OUTPUT_DIR}/accepted_questions.json"
|
||
fi
|
||
|
||
count_accepted() {
|
||
if [ -f "$OUTPUT" ]; then
|
||
"$PYTHON" -c "
|
||
import json, sys
|
||
try:
|
||
data = json.loads(open('$OUTPUT').read())
|
||
print(len(data))
|
||
except Exception:
|
||
print(0)
|
||
"
|
||
else
|
||
echo 0
|
||
fi
|
||
}
|
||
|
||
echo "=== AR 出题循环 — 目标 ${TARGET} 题 ==="
|
||
echo "配置: ${CONFIG}"
|
||
echo "输出: ${OUTPUT}"
|
||
echo "早停: 连续 ${EARLY_STOP_ROUNDS} 轮 0 产出则停止"
|
||
echo ""
|
||
|
||
zero_streak=0
|
||
|
||
for round in $(seq 1 "$MAX_ROUNDS"); do
|
||
current=$(count_accepted)
|
||
echo "[Round ${round}/${MAX_ROUNDS}] 当前已有 ${current}/${TARGET} 题"
|
||
|
||
if [ "$current" -ge "$TARGET" ]; then
|
||
echo "✓ 已达目标 ${TARGET} 题，停止"
|
||
break
|
||
fi
|
||
|
||
remaining=$((TARGET - current))
|
||
seed=$((BASE_SEED + round - 1))
|
||
echo " 需要 ${remaining} 题，seed=${seed}"
|
||
|
||
"$PYTHON" tools/generate_questions.py generate-v2 \
|
||
--store-dir "$STORE_DIR" \
|
||
--config "$CONFIG" \
|
||
--db-path "$DB_PATH" \
|
||
--seed "$seed" \
|
||
--task-types "Action Recognition" \
|
||
2>&1 | tee -a "logs/generate_ar30_round${round}.log"
|
||
|
||
new_count=$(count_accepted)
|
||
gained=$((new_count - current))
|
||
echo " 本轮生成 ${gained} 题 (累计 ${new_count}/${TARGET})"
|
||
|
||
if [ "$gained" -eq 0 ]; then
|
||
zero_streak=$((zero_streak + 1))
|
||
echo " ⚠ 本轮 0 产出（连续 ${zero_streak}/${EARLY_STOP_ROUNDS}）"
|
||
if [ "$zero_streak" -ge "$EARLY_STOP_ROUNDS" ]; then
|
||
echo "⚠ 连续 ${zero_streak} 轮 0 产出（selector/gate 可能持续拒题），提前停止"
|
||
break
|
||
fi
|
||
else
|
||
zero_streak=0
|
||
fi
|
||
echo ""
|
||
done
|
||
|
||
final=$(count_accepted)
|
||
echo "=== 完成：共 ${final} 道 AR 题 ==="
|
||
if [ "$final" -lt "$TARGET" ]; then
|
||
echo "⚠ 未达目标（${final}/${TARGET}），可能需要调整 retry_limit 或检查 gate 通过率"
|
||
fi
|