feat(harness): add Action Recognition training experiment

- PerCategoryPoolStrategy: filter test pool by task_types - RunConfig: add run_holdout_eval toggle (default true) - load_config: fix YAML task_types list-to-tuple conversion - Runner: conditionally skip _holdout_four_way when disabled - CLI: add --no-run-holdout-eval flag - New config/train_action_recognition.yaml (3 epochs, per_category) - New scripts/train_action_recognition.sh (baseline + seed + train)
2026-07-14 00:58:54 -04:00
parent 37d4519905
commit dec7346da3
12 changed files with 1423 additions and 52 deletions
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# Action Recognition 单题型训练实验
+# 设计文档: research-wiki/designs/2026-07-14-action-recognition-training-design.md
+#
+# 三阶段:
+#   Phase 0: baseline infer (v2-360 Action Recognition 30 题)
+#   Phase 1: create seed (v2ar-baseline)
+#   Phase 2: train (3 epochs, per_category)
+#
+# 用法:
+#   CUDA_VISIBLE_DEVICES=0 bash scripts/train_action_recognition.sh
+#   MODE=mock bash scripts/train_action_recognition.sh   # 跳过 Phase 0/1
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}"
+export CUDA_VISIBLE_DEVICES
+
+export HF_HUB_OFFLINE=1
+export TRANSFORMERS_OFFLINE=1
+export PYTHONUNBUFFERED=1
+
+set -a
+source .env
+set +a
+
+PYTHON="$(conda run -n Video-Tree-TRM which python)"
+
+# ── Phase 0: Baseline infer ──
+if [[ "${MODE:-}" != "mock" ]]; then
+    echo "=== Phase 0: Baseline infer (v2-360 Action Recognition 30 题) ==="
+    "${PYTHON}" main.py \
+        --config config/default.yaml \
+        --workspace-dir workspaces/default \
+        --store-dir store \
+        --mode infer \
+        --concurrency 24 \
+        --max-steps 40 \
+        --skill-mode auto \
+        --n-samples 0 \
+        --questions "generated-v2-360" \
+        --skills-version v1 \
+        --prompts-version v1 \
+        --run-id v2ar_baseline \
+        --task-types "Action Recognition"
+fi
+
+# ── Phase 1: Create seed ──
+if [[ "${MODE:-}" != "mock" && ! -d "store/seeds/v2ar-baseline" ]]; then
+    echo "=== Phase 1: Create seed v2ar-baseline ==="
+    "${PYTHON}" -c "
+from pathlib import Path
+from app.harness.store import extract_run_db, init_seed
+import tempfile
+
+tmp = Path(tempfile.mkdtemp()) / 'baseline.db'
+extract_run_db(
+    Path('workspaces/default/harness.db'),
+    tmp,
+    'infer_v2ar_baseline',
+)
+init_seed(
+    store_dir=Path('store'),
+    name='v2ar-baseline',
+    skills_dir=Path('store/skills/v1'),
+    prompts_dir=Path('store/prompts/v1'),
+    baseline_db=tmp,
+    baseline_run_id='infer_v2ar_baseline',
+    parent=None,
+    description='v2-360 Action Recognition 30 题 baseline (skills/v1)',
+)
+tmp.unlink()
+print('Seed created: store/seeds/v2ar-baseline/')
+"
+elif [[ -d "store/seeds/v2ar-baseline" ]]; then
+    echo "=== Phase 1: Seed v2ar-baseline 已存在，跳过 ==="
+fi
+
+# ── Phase 2: Train ──
+echo "=== Phase 2: Train (3 epochs, Action Recognition) ==="
+"${PYTHON}" main.py \
+    --config config/train_action_recognition.yaml \
+    --fresh \
+    --seed v2ar-baseline
+
+echo "=== 训练完成 ==="
+echo "结果查看:"
+echo "  cat workspaces/train-action-recognition/analyses/final_test_eval.json"
+echo "  sqlite3 workspaces/train-action-recognition/harness.db 'SELECT * FROM dual_metric'"