55 lines
1.8 KiB
Bash
Executable File
55 lines
1.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# 生成题推理 + 与 benchmark 基线对比
|
|
# 职责：对 240 道生成题跑推理，然后自动与 benchmark 基线做 Fisher 精确检验对比。
|
|
# 前提：benchmark 基线已跑完（workspaces/default/harness.db 中有 infer_adhoc）。
|
|
# 用法:
|
|
# bash scripts/infer_generated.sh # 全量 240 题 + 对比
|
|
# N_SAMPLES=10 bash scripts/infer_generated.sh # smoke test
|
|
# SKILLS_VERSION=v2 PROMPTS_VERSION=v2 bash scripts/infer_generated.sh # 指定版本
|
|
# BASELINE_RUN_ID=infer_v2 bash scripts/infer_generated.sh # 指定基线 run_id
|
|
set -euo pipefail
|
|
|
|
cd "$(dirname "$0")/.."
|
|
|
|
CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}"
|
|
export CUDA_VISIBLE_DEVICES
|
|
|
|
export HF_HUB_OFFLINE=1
|
|
export TRANSFORMERS_OFFLINE=1
|
|
export PYTHONUNBUFFERED=1
|
|
|
|
set -a
|
|
source .env
|
|
set +a
|
|
|
|
PYTHON="$(conda run -n Video-Tree-TRM which python)"
|
|
|
|
RUN_ID="${RUN_ID:-gen240}"
|
|
BASELINE_DB="${BASELINE_DB:-workspaces/default/harness.db}"
|
|
BASELINE_RUN_ID="${BASELINE_RUN_ID:-infer_adhoc}"
|
|
|
|
# ── Phase 1: 推理 ──
|
|
echo "========== Phase 1: 推理 240 道生成题 =========="
|
|
|
|
"${PYTHON}" main.py \
|
|
--workspace-dir workspaces/default \
|
|
--store-dir store \
|
|
--mode infer \
|
|
--concurrency 24 \
|
|
--max-steps 40 \
|
|
--skill-mode auto \
|
|
--n-samples "${N_SAMPLES:-0}" \
|
|
--questions generated \
|
|
--run-id "${RUN_ID}" \
|
|
--skills-version "${SKILLS_VERSION:-v1}" \
|
|
--prompts-version "${PROMPTS_VERSION:-v1}"
|
|
|
|
# ── Phase 2: 对比 ──
|
|
echo "========== Phase 2: 与 benchmark 基线对比 =========="
|
|
|
|
"${PYTHON}" tools/generate_questions.py calibrate \
|
|
--baseline-db "${BASELINE_DB}" \
|
|
--baseline-run-id "${BASELINE_RUN_ID}" \
|
|
--target-db workspaces/default/harness.db \
|
|
--target-run-id "infer_${RUN_ID}"
|