feat(harness): add Action Recognition training experiment

- PerCategoryPoolStrategy: filter test pool by task_types - RunConfig: add run_holdout_eval toggle (default true) - load_config: fix YAML task_types list-to-tuple conversion - Runner: conditionally skip _holdout_four_way when disabled - CLI: add --no-run-holdout-eval flag - New config/train_action_recognition.yaml (3 epochs, per_category) - New scripts/train_action_recognition.sh (baseline + seed + train)
2026-07-14 00:58:54 -04:00
parent 37d4519905
commit dec7346da3
12 changed files with 1423 additions and 52 deletions
@@ -90,6 +90,7 @@ class RunConfig:
        pool_split_mode: 池划分策略，"global"（全局统一划分）/ "per_category"（按类别独立划分）。
        train_ratio: 训练集占比，范围 (0, 1)。
        test_questions: 测试题目集路径（相对路径）。
+        run_holdout_eval: 是否在 epoch 末执行 held-out 四向评估，默认 True。
    """

    # ── 必填字段（无默认值，来自 YAML 或 CLI） ──
@@ -145,6 +146,7 @@ class RunConfig:
    pool_split_mode: str = "global"
    train_ratio: float = 0.667
    test_questions: str = "benchmarks/Video-MME"
+    run_holdout_eval: bool = True


 def _validate(config: RunConfig) -> None:
@@ -439,6 +441,10 @@ def load_config(
        if field_name in yaml_data:
            yaml_data[field_name] = Path(yaml_data[field_name])

+    # Phase 4b: 类型转换 — task_types list → tuple
+    if "task_types" in yaml_data and yaml_data["task_types"] is not None:
+        yaml_data["task_types"] = tuple(yaml_data["task_types"])
+
    # Phase 5: 构造并校验
    config = RunConfig(**{k: v for k, v in yaml_data.items() if k in valid_fields})
    _validate(config)
@@ -205,6 +205,7 @@ def _q_to_dict(q: GeneratedQuestion) -> dict:
        "answer": q.answer,
        "source_nodes": list(q.source_nodes),
        "difficulty": q.difficulty,
+        "family": q.family,
        "skill_target": q.skill_target,
        "difficulty_steps": q.difficulty_steps,
    }
@@ -228,6 +229,7 @@ def _dict_to_q(d: dict) -> GeneratedQuestion:
        answer=d["answer"],
        source_nodes=tuple(d.get("source_nodes", ())),
        difficulty=d.get("difficulty", "medium"),
+        family=d.get("family"),
        skill_target=d.get("skill_target"),
        difficulty_steps=d.get("difficulty_steps"),
    )
@@ -595,12 +597,15 @@ class PerCategoryPoolStrategy:
            all_train.extend(train)
            all_val.extend(val)

-        # Phase 4: test 池（从外部目录加载，无则空）
+        # Phase 4: test 池（从外部目录加载，无则空；按 task_types 过滤）
        test: list[GeneratedQuestion] = []
        if config.test_questions_dir is not None:
            from app.question_gen import load_benchmark

            test = load_benchmark(config.test_questions_dir)
+            if config.task_types is not None:
+                allowed = set(config.task_types)
+                test = [q for q in test if q.task_type in allowed]

        # Phase 5: 计算 baseline_val_accuracy
        val_correct = sum(1 for q in all_val if correctness.get(q.question_id, False))
@@ -1368,7 +1368,10 @@ class Runner:
            momentum_updated_task_types=momentum_task_types,
            best_val_acc=state.best_val_acc,
        )
-        await self._holdout_four_way(epoch, pools, state, eval_skills_version, eval_prompts_version)
+        if self._config.run_holdout_eval:
+            await self._holdout_four_way(
+                epoch, pools, state, eval_skills_version, eval_prompts_version
+            )

        # Phase 10: gate 阶梯刷新
        self._refresh_gate_ladder(
@@ -0,0 +1,63 @@
+# config/train_action_recognition.yaml
+# Action Recognition 单题型首次训练实验
+# 设计文档: research-wiki/designs/2026-07-14-action-recognition-training-design.md
+
+harness:
+  workspace_dir: "workspaces/train-action-recognition"
+  store_dir: store
+  mode: train
+  run_id: train_ar_v1
+  concurrency: 24
+  max_steps: 40
+  skill_mode: auto
+  n_samples: 0
+  questions: "generated-v2-360"
+  skills_version: v1
+  prompts_version: v1
+  epochs: 3
+  # CE-Gate 参数（沿用 default.yaml）
+  gate_e_confirm: 20.0
+  gate_e_provisional: 3.0
+  gate_w_net_min: 2
+  gate_delta_min: 0.02
+  gate_lambda_dir: -0.642
+  gate_e_rollback: 10.0
+  gate_block: 8
+  gate_n_max: 40
+  gate_p_low: 0.05
+  gate_p_high: 0.95
+  gate_probe_quota: 0.2
+  gate_gamma_decay: 0.9
+  gate_cooldown_steps: 2
+  gate_guard_err: 0.10
+  # 进化参数
+  edit_budget_start: 5
+  edit_budget_end: 2
+  skill_update_mode: patch
+  appendix_consolidate_threshold: 6
+  # 池配置 — per_category 单题型
+  pool_split_mode: per_category
+  task_types:
+    - "Action Recognition"
+  train_ratio: 0.667
+  test_questions: "benchmarks/Video-MME"
+  run_holdout_eval: false
+  # mini-batch
+  batch_size: 10
+  min_class_per_batch: 2
+  batch_correct_ratio: 0.5
+  momentum_samples: 20
+  eval_min_per_class: 2
+  early_stop_patience: 4
+  test_size: 63
+  diag_size: 20
+  diag_correct_ratio: 0.5
+  val_size: 10
+  val_correct_ratio: 0.5
+  use_slow_momentum: true
+
+embed:
+  backend: "local"
+  model_name: "BAAI/bge-base-zh-v1.5"
+  embed_dim: 768
+  device: "cuda"
@@ -203,6 +203,11 @@ def _build_parser() -> argparse.ArgumentParser:
    )
    parser.add_argument("--train-ratio", type=float, dest="train_ratio")
    parser.add_argument("--test-questions", type=str, dest="test_questions")
+    parser.add_argument(
+        "--no-run-holdout-eval",
+        action="store_true",
+        dest="no_run_holdout_eval",
+    )
    return parser


@@ -267,6 +272,9 @@ def main() -> None:
    if cli_args.get("task_types") is not None:
        cli_args["task_types"] = tuple(cli_args["task_types"])
    cli_overrides = {k: v for k, v in cli_args.items() if k != "config"}
+    if cli_overrides.get("no_run_holdout_eval"):
+        cli_overrides["run_holdout_eval"] = False
+    cli_overrides.pop("no_run_holdout_eval", None)
    config = load_config(args.config, cli_overrides)
    logger.info("配置加载完成: mode={}, workspace={}", config.mode, config.workspace_dir)

@@ -0,0 +1,198 @@
+---
+id: action-recognition-training
+title: Action Recognition 单题型首次训练实验设计
+type: design
+created: 2026-07-14
+status: approved
+---
+
+# Action Recognition 单题型首次训练实验设计
+
+## 1. 目标
+
+用 Action Recognition 单题型验证训练管线端到端可用性。选择此题型因其：
+- Video-MME 准确率 65.1%（倒数第三），提升空间大
+- 属于 RETRIEVAL family（权重最高 0.30）
+- v2-360 有 30 道生成题可用于 train/val
+
+## 2. 数据划分
+
+| 池 | 来源 | 数量 | 用途 |
+|----|------|------|------|
+| train (diagnosis) | generated-v2-360 | 20 题 | 错误归因 + 进化 |
+| val (validation) | generated-v2-360 | 10 题 | CE-Gate 验证 + best argmax |
+| test | Video-MME benchmark | 63 题 | 训练结束后终评（epoch 内跳过四向 held-out） |
+
+切分方式：`PerCategoryPoolStrategy`，`train_ratio=0.667`，按 correctness 分层。
+
+## 3. 前置步骤
+
+```
+步骤 1: baseline infer
+  main.py --mode infer \
+    --questions generated-v2-360 \
+    --task-types "Action Recognition" \
+    --run-id v2ar_baseline
+  → workspaces/default/harness.db 新增 30 题 predictions
+
+步骤 2: create seed
+  2a. extract_run_db(
+        src_db="workspaces/default/harness.db",
+        dst_db="/tmp/v2ar_baseline.db",
+        run_id="infer_v2ar_baseline")
+  2b. init_seed(
+        store_dir=Path("store"),
+        name="v2ar-baseline",
+        skills_dir=Path("store/skills/v1"),
+        prompts_dir=Path("store/prompts/v1"),
+        baseline_db=Path("/tmp/v2ar_baseline.db"),
+        baseline_run_id="infer_v2ar_baseline",
+        parent=None,
+        description="v2-360 Action Recognition 30 题 baseline (skills/v1)")
+  → store/seeds/v2ar-baseline/
+
+步骤 3: fresh workspace
+  main.py --mode train --fresh \
+    --seed v2ar-baseline \
+    --run-id train_ar_v1 \
+    --config config/train_action_recognition.yaml
+  → workspaces/train-action-recognition/
+
+前提: workspaces/default/ 已存在（含 harness.db），步骤 1 复用该 workspace。
+```
+
+## 4. 训练配置
+
+| 参数 | 值 | 理由 |
+|------|-----|------|
+| workspace_dir | workspaces/train-action-recognition | 独立 workspace |
+| mode | train | 训练模式 |
+| run_id | train_ar_v1 | 显式命名，observation 可追溯 |
+| questions | generated-v2-360 | v2-360 题目集 |
+| task_types | ["Action Recognition"] | 单题型 |
+| pool_split_mode | per_category | 按类切分 |
+| train_ratio | 0.667 | 20 train / 10 val |
+| test_questions | benchmarks/Video-MME | test 池来源（过滤后 63 题） |
+| run_holdout_eval | false | epoch 内跳过四向评估 + shadow_gate |
+| epochs | 3 | 6 step + 3 轮慢更新 |
+| batch_size | 10 | 20 题 / 2 batch per epoch |
+| min_class_per_batch | 2 | 最小值 |
+| concurrency | 24 | 推理并发 |
+| max_steps | 40 | 沿用默认 |
+| early_stop_patience | 4 | 3 epoch 内有意义 |
+| use_slow_momentum | true | 验证 momentum |
+| skill_mode | auto | 自动加载 skill |
+
+其余 gate 参数沿用 `config/default.yaml` 默认值。
+
+## 5. 训练循环预期
+
+```
+Epoch 1-3 每轮:
+  ├── build_batches: 20 题 → 2 batch
+  ├── Step 0: rollout → diagnose → evolve → CE-Gate
+  ├── Step 1: rollout → diagnose → evolve → CE-Gate
+  └── Slow Update:
+       ├── 全 val 重跑 (10 题)
+       ├── probation 结算
+       ├── best argmax
+       ├── momentum
+       ├── system/tool 慢更新
+       └── 四向 held-out: 跳过 (run_holdout_eval=false)
+           → 同时跳过 _pick_mixed_best 的 shadow val 评估
+收尾:
+  ├── _deliver_best: manifest 指向最佳版本
+  └── _final_test_eval: test 池 63 题终评 (保留，仅跑 1 次)
+```
+
+预计 LLM 调用 ~500-600 次（去掉每 epoch 四向 held-out 后大幅减少）。
+
+## 6. 代码修改
+
+### 6.1 PerCategoryPoolStrategy test 池按 task_types 过滤
+
+**问题**: `PerCategoryPoolStrategy.build()` Phase 4 加载 test 池时调用
+`load_benchmark(config.test_questions_dir)` 会加载全部题目（如 Video-MME 900 题），
+不受 `config.task_types` 过滤。而 train/val 在 Phase 1 已按 task_types 过滤，
+test 池应保持一致。
+
+**修改位置**: `app/harness/pools.py` `PerCategoryPoolStrategy.build()` Phase 4
+
+**修改内容**:
+
+```python
+# Phase 4: test 池（从外部目录加载，无则空）
+test: list[GeneratedQuestion] = []
+if config.test_questions_dir is not None:
+    from app.question_gen import load_benchmark
+    test = load_benchmark(config.test_questions_dir)
+    # 与 train/val 的 Phase 1 过滤保持一致
+    if config.task_types is not None:
+        allowed = set(config.task_types)
+        test = [q for q in test if q.task_type in allowed]
+```
+
+**边界情况**:
+- `config.task_types=None`（全题型）: 不过滤，行为不变
+- 过滤后 test 为空: 不报错（test 池可为空，`_final_test_eval` 会拿到空列表并跳过）
+- 过滤后 test 仅含目标题型: 本次为 63 道 Action Recognition
+
+**对 pools.json 冻结的影响**: test 池过滤后的结果写入 pools.json，
+resume 时直接加载无需重新过滤。fresh workspace 每次从头构建。
+
+**测试**: 新增单测验证 test_questions_dir 混合题型时过滤正确。
+
+### 6.2 RunConfig 新增 run_holdout_eval
+
+**修改位置**: `app/harness/config.py` RunConfig 字段区
+
+```python
+run_holdout_eval: bool = True
+```
+
+**YAML 暴露**: 实验 YAML 中配置（科研实验参数归 YAML）。
+
+**CLI 覆盖**: `main.py` 新增 `--no-run-holdout-eval` 标志，符合 CLAUDE.md §4.5
+"CLI 仅用于单次临时覆盖"。
+
+**Runner 修改**: `app/harness/runner.py` `_slow_update_cycle` Phase 9
+
+```python
+# Phase 9: epoch_report（保留）+ 四向 held-out（可选）
+write_epoch_report(...)
+if self._config.run_holdout_eval:
+    await self._holdout_four_way(...)
+```
+
+**副作用说明**: `run_holdout_eval=false` 同时跳过 `_pick_mixed_best` 的
+shadow val 评估（`best_mixed` 版本选择）。首次试跑只看 `best_hard`，
+shadow/mixed 指标不需要。
+
+**不影响**: `_final_test_eval` 不受此开关控制，训练结束后仍跑一次终评。
+
+**测试**: 新增 runner 单测验证 `run_holdout_eval=false` 时跳过 `_holdout_four_way`。
+
+## 7. 观测产出
+
+| 产出 | 位置 | 内容 |
+|------|------|------|
+| step 报告 | analyses/step_reports/ | 每步 gate 决策（accept/reject/skip） |
+| epoch 报告 | analyses/epoch_reports/ | 慢更新结果、best 变化 |
+| dual_metric | harness.db dual_metric 表 | val 准确率变化 |
+| skill 版本链 | skills/v1→v2→... | 可 diff 查看改动 |
+| checkpoint | checkpoint.json | 断点续训支持 |
+| 终评 | analyses/final_test_eval.json | test 63 题最终准确率 |
+
+## 8. 非功能性需求
+
+| 维度 | 必答问题 | 设计 |
+|------|---------|------|
+| 持久化 | 何时落盘？崩溃丢多少？覆盖还是追加？ | predictions 逐题 INSERT 即时落库（追加）；checkpoint 每 step 结束后原子写入（覆盖）；崩溃最多丢当前 step 的推理结果，resume 从上一 step 重跑 |
+| 幂等性 | 重复执行安全吗？ | 同 seed + 同配置 + fresh → 确定性结果（RNG seed 固定）；predictions INSERT OR IGNORE 防重复 |
+| 断点续跑 | 中断后如何恢复？ | `--resume` 读 checkpoint.json 的 epoch/step_completed/phase，从断点下一 step 继续；epoch_done 阶段恢复到下一 epoch 开头 |
+| 原子性 | 部分写入会损坏数据吗？ | checkpoint.json 整文件写入（非追加）；predictions 逐条 commit 无事务风险；skills 版本目录先 copytree 再改 manifest 指针 |
+
+## 9. 新增文件
+
+- `config/train_action_recognition.yaml` — 实验配置
+- `scripts/train_action_recognition.sh` — 自包含实验脚本（写死参数，零参数复现）
@@ -145,6 +145,21 @@
      "id": "plan:2026-07-11-question-gen-v2",
      "label": "出题管线 v2 实现计划",
      "type": "plan"
+    },
+    {
+      "id": "design:per-category-pool-strategy",
+      "label": "Per-Category Pool Strategy 设计",
+      "type": "design"
+    },
+    {
+      "id": "plan:per-category-pool-strategy",
+      "label": "Per-Category Pool Strategy 实现计划",
+      "type": "plan"
+    },
+    {
+      "id": "plan:action-recognition-training",
+      "label": "Action Recognition 单题型首次训练实验计划",
+      "type": "plan"
    }
  ],
  "links": [
@@ -259,6 +274,20 @@
      "relation": "implements",
      "evidence": "Spec-3 设计的实现计划",
      "added": "2026-07-12T02:57:39.569968+00:00"
+    },
+    {
+      "source": "plan:per-category-pool-strategy",
+      "target": "design:per-category-pool-strategy",
+      "relation": "implements",
+      "evidence": "实现 PoolStrategy Protocol + PerCategoryPoolStrategy 设计",
+      "added": "2026-07-13T02:26:58.551190+00:00"
+    },
+    {
+      "source": "plan:action-recognition-training",
+      "target": "design:action-recognition-training",
+      "relation": "implements",
+      "evidence": "计划实现设计文档中的 2 处代码修改 + 实验配置 + 训练脚本",
+      "added": "2026-07-14T04:50:15.986586+00:00"
    }
  ]
 }
@@ -1,8 +1,8 @@
 # Research Wiki 索引

-> 自动生成，更新时间：2026-07-12 02:57 UTC
+> 自动生成，更新时间：2026-07-14 04:50 UTC

-## design (21)
+## design (24)
 - [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design`
 - [2026-07-07-app-harness-design](designs/2026-07-07-app-harness-design.md) `design:2026-07-07-app-harness-design`
 - [2026-07-07-core-evolution-design](designs/2026-07-07-core-evolution-design.md) `design:2026-07-07-core-evolution-design`
@@ -11,8 +11,11 @@
 - [2026-07-11-agent-runtime-fixes-design](designs/2026-07-11-agent-runtime-fixes-design.md) `design:2026-07-11-agent-runtime-fixes-design`
 - [2026-07-11-batch-tree-build-design](designs/2026-07-11-batch-tree-build-design.md) `design:2026-07-11-batch-tree-build-design`
 - [2026-07-11-question-gen-v2-design](designs/2026-07-11-question-gen-v2-design.md) `design:2026-07-11-question-gen-v2-design`
+- [2026-07-12-per-category-pool-strategy-design](designs/2026-07-12-per-category-pool-strategy-design.md) `design:2026-07-12-per-category-pool-strategy-design`
+- [Action Recognition 单题型首次训练实验设计](designs/2026-07-14-action-recognition-training-design.md) `design:2026-07-14-action-recognition-training-design`
 - [main.py 推理入口 + 初始 Prompt 集设计](designs/2026-07-09-main-inference-entry-design.md) `design:2026-07-09-main-inference-entry-design`
 - [main.py 推理入口 + 初始 Prompt 集设计](designs/main-inference-entry.md) `design:main-inference-entry`
+- [Per-Category Pool Strategy 设计](designs/per-category-pool-strategy.md) `design:per-category-pool-strategy`
 - [Spec-1 Agent 执行环境修复（解析容错+步级重试+摘要附实体）](designs/agent-runtime-fixes.md) `design:agent-runtime-fixes`
 - [Spec-2 建树批量并行入口](designs/batch-tree-build.md) `design:batch-tree-build`
 - [Spec-3 出题管线 v2（失败机理靶向+逐题质量门）](designs/question-gen-v2.md) `design:question-gen-v2`
@@ -31,7 +34,7 @@
 - [Harness 评估: Spec-1 修复验证 (infer_spec1check)](findings/eval-spec1check.md) `finding:eval-spec1check`
 - [Harness 评估: Spec-2 批量并行建树](findings/eval-spec2-batch-tree-build.md) `finding:eval-spec2-batch-tree-build`

-## plan (22)
+## plan (26)
 - [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm`
 - [2026-07-07-app-harness](plans/2026-07-07-app-harness.md) `plan:2026-07-07-app-harness`
 - [2026-07-07-core-evolution](plans/2026-07-07-core-evolution.md) `plan:2026-07-07-core-evolution`
@@ -42,10 +45,14 @@
 - [2026-07-09-tree-repair-resilience](plans/2026-07-09-tree-repair-resilience.md) `plan:2026-07-09-tree-repair-resilience`
 - [2026-07-11-agent-runtime-fixes](plans/2026-07-11-agent-runtime-fixes.md) `plan:2026-07-11-agent-runtime-fixes`
 - [2026-07-11-batch-tree-build](plans/2026-07-11-batch-tree-build.md) `plan:2026-07-11-batch-tree-build`
+- [2026-07-12-per-category-pool-strategy](plans/2026-07-12-per-category-pool-strategy.md) `plan:2026-07-12-per-category-pool-strategy`
+- [2026-07-14-action-recognition-training](plans/2026-07-14-action-recognition-training.md) `plan:2026-07-14-action-recognition-training`
+- [Action Recognition 单题型首次训练实验计划](plans/action-recognition-training.md) `plan:action-recognition-training`
 - [app/harness/ 训练循环编排层实现计划](plans/app-harness.md) `plan:app-harness`
 - [app/search/ 搜索 Agent 装配层实现计划](plans/2026-07-07-search-module.md) `plan:2026-07-07-search-module`
 - [core/agent/ + adapters/llm 基础设施实现计划](plans/core-agent-adapters-llm.md) `plan:core-agent-adapters-llm`
 - [main.py 推理入口 + 初始 Prompt 集实现计划](plans/main-inference-entry.md) `plan:main-inference-entry`
+- [Per-Category Pool Strategy 实现计划](plans/per-category-pool-strategy.md) `plan:per-category-pool-strategy`
 - [question_gen 模块实现计划](plans/question-gen.md) `plan:question-gen`
 - [Spec-1 Agent 执行环境修复实现计划](plans/agent-runtime-fixes-plan.md) `plan:agent-runtime-fixes-plan`
 - [Spec-2 建树批量并行实现计划](plans/batch-tree-build-plan.md) `plan:batch-tree-build-plan`
@@ -65,3 +65,11 @@
 - [2026-07-12 02:38 UTC] 新增 plan: 出题管线 v2 实现计划 (plan:2026-07-11-question-gen-v2)
 - [2026-07-12 02:57 UTC] 新增边: plan:2026-07-11-question-gen-v2 --implements--> design:2026-07-11-question-gen-v2
 - [2026-07-12 02:57 UTC] 重建索引: 53 篇页面
+- [2026-07-13 02:05 UTC] 新增 design: Per-Category Pool Strategy 设计 (design:per-category-pool-strategy)
+- [2026-07-13 02:06 UTC] 重建索引: 55 篇页面
+- [2026-07-13 02:26 UTC] 新增 plan: Per-Category Pool Strategy 实现计划 (plan:per-category-pool-strategy)
+- [2026-07-13 02:26 UTC] 新增边: plan:per-category-pool-strategy --implements--> design:per-category-pool-strategy
+- [2026-07-13 02:27 UTC] 重建索引: 57 篇页面
+- [2026-07-14 04:50 UTC] 新增 plan: Action Recognition 单题型首次训练实验计划 (plan:action-recognition-training)
+- [2026-07-14 04:50 UTC] 新增边: plan:action-recognition-training --implements--> design:action-recognition-training
+- [2026-07-14 04:50 UTC] 重建索引: 60 篇页面
@@ -0,0 +1,661 @@
+# Action Recognition 单题型首次训练实验 Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** 用 Action Recognition 单题型端到端验证训练管线，包含 3 处代码修改、1 套实验配置、1 个前置脚本。
+
+**Architecture:** 修改 `PerCategoryPoolStrategy.build()` 使 test 池受 `task_types` 过滤；RunConfig 新增 `run_holdout_eval` 开关控制 epoch 内四向 held-out；`load_config` 修复 YAML list→tuple 转换；新建实验 YAML 和 sh 脚本驱动训练。
+
+**Tech Stack:** Python 3.11, pytest, YAML, bash
+
+---
+
+### Task 0: 清理 v2-360 目录中的 backup 文件
+
+**Files:**
+- Modify: `store/questions/generated-v2-360/`（重命名文件）
+
+`load_benchmark()` 会加载目录下所有 `*.json`，`accepted_questions_backup_220.json` 含 18 道重复 AR 题会干扰训练。
+
+- [ ] **Step 1: 重命名 backup 文件使其不被 load_benchmark 加载**
+
+```bash
+mv store/questions/generated-v2-360/accepted_questions_backup_220.json \
+   store/questions/generated-v2-360/accepted_questions_backup_220.json.bak
+```
+
+- [ ] **Step 2: 验证 load_benchmark 只加载 180 题**
+
+```bash
+conda activate Video-Tree-TRM & python -c "
+from pathlib import Path
+from app.question_gen import load_benchmark
+qs = load_benchmark(Path('store/questions/generated-v2-360'))
+print(f'Total: {len(qs)}')
+ar = [q for q in qs if q.task_type == 'Action Recognition']
+print(f'Action Recognition: {len(ar)}')
+assert len(qs) == 180, f'Expected 180, got {len(qs)}'
+assert len(ar) == 30, f'Expected 30 AR, got {len(ar)}'
+print('OK')
+"
+```
+
+预期：Total: 180, Action Recognition: 30, OK
+
+- [ ] **Step 3: 提交**
+
+```bash
+git add -A store/questions/generated-v2-360/
+git commit -m "chore: rename v2-360 backup JSON to .bak to exclude from load_benchmark"
+```
+
+---
+
+### Task 1: PerCategoryPoolStrategy test 池 task_types 过滤
+
+**Files:**
+- Modify: `app/harness/pools.py:600-606`
+- Test: `tests/unit/test_harness_pools.py`
+
+- [ ] **Step 1: 写失败测试 — test 池按 task_types 过滤**
+
+在 `tests/unit/test_harness_pools.py` 的 `TestPerCategoryPoolStrategy` 类末尾新增。
+注意：`load_benchmark` 要求每个 JSON 文件内容为**题目数组**（`[{...}]`），不是单个 dict。
+
+```python
+def test_per_category_test_pool_filtered_by_task_types(self, tmp_path: Path):
+    """test_questions_dir 含多题型时，test 池只保留 task_types 指定的题型。"""
+    test_dir = tmp_path / "test_questions"
+    test_dir.mkdir()
+    for tt in ("Action Recognition", "Object Reasoning", "Counting Problem"):
+        items = []
+        for i in range(10):
+            qid = f"{tt.replace(' ', '_')}_{i:03d}"
+            items.append({
+                "question_id": qid,
+                "video_id": "v1",
+                "task_type": tt,
+                "question": f"Q {qid}?",
+                "options": ["A. a", "B. b", "C. c", "D. d"],
+                "answer": "A",
+            })
+        slug = tt.lower().replace(" ", "_")
+        (test_dir / f"{slug}.json").write_text(
+            json.dumps(items, ensure_ascii=False), encoding="utf-8"
+        )
+
+    questions = [_make_question(f"ar_{i:03d}", "Action Recognition") for i in range(30)]
+    correctness = {q.question_id: (i < 20) for i, q in enumerate(questions)}
+
+    config = PoolConfig(
+        task_types=("Action Recognition",),
+        seed=42,
+        baseline_run_id="bl",
+        diag_size=0,
+        diag_correct_ratio=0.0,
+        val_size=0,
+        val_correct_ratio=0.0,
+        test_size=0,
+        eval_min_per_class=0,
+        train_ratio=20 / 30,
+        test_questions_dir=test_dir,
+    )
+    strategy = PerCategoryPoolStrategy()
+    pools = strategy.build(questions, correctness, config)
+
+    assert len(pools.test) == 10
+    assert all(q.task_type == "Action Recognition" for q in pools.test)
+
+def test_per_category_test_pool_no_filter_when_task_types_none(self, tmp_path: Path):
+    """task_types=None 时 test 池不过滤，保留全部题型。"""
+    test_dir = tmp_path / "test_questions"
+    test_dir.mkdir()
+    for tt in ("Action Recognition", "Object Reasoning"):
+        items = []
+        for i in range(5):
+            qid = f"{tt.replace(' ', '_')}_{i:03d}"
+            items.append({
+                "question_id": qid,
+                "video_id": "v1",
+                "task_type": tt,
+                "question": f"Q {qid}?",
+                "options": ["A. a", "B. b", "C. c", "D. d"],
+                "answer": "A",
+            })
+        slug = tt.lower().replace(" ", "_")
+        (test_dir / f"{slug}.json").write_text(
+            json.dumps(items, ensure_ascii=False), encoding="utf-8"
+        )
+
+    questions = [_make_question(f"q_{i:03d}", "Action Recognition") for i in range(10)]
+    correctness = {q.question_id: True for q in questions}
+
+    config = PoolConfig(
+        task_types=None,
+        seed=42,
+        baseline_run_id="bl",
+        diag_size=0,
+        diag_correct_ratio=0.0,
+        val_size=0,
+        val_correct_ratio=0.0,
+        test_size=0,
+        eval_min_per_class=0,
+        train_ratio=0.667,
+        test_questions_dir=test_dir,
+    )
+    strategy = PerCategoryPoolStrategy()
+    pools = strategy.build(questions, correctness, config)
+
+    assert len(pools.test) == 10
+```
+
+- [ ] **Step 2: 运行测试验证失败**
+
+```bash
+conda activate Video-Tree-TRM & pytest tests/unit/test_harness_pools.py::TestPerCategoryPoolStrategy::test_per_category_test_pool_filtered_by_task_types -v
+```
+
+预期：FAIL — `assert len(pools.test) == 10` 失败（实际 30 题，未过滤）。
+
+- [ ] **Step 3: 实现 test 池过滤**
+
+修改 `app/harness/pools.py` `PerCategoryPoolStrategy.build()` 的 Phase 4：
+
+```python
+        # Phase 4: test 池（从外部目录加载，无则空）
+        test: list[GeneratedQuestion] = []
+        if config.test_questions_dir is not None:
+            from app.question_gen import load_benchmark
+
+            test = load_benchmark(config.test_questions_dir)
+            if config.task_types is not None:
+                allowed = set(config.task_types)
+                test = [q for q in test if q.task_type in allowed]
+```
+
+- [ ] **Step 4: 运行测试验证通过**
+
+```bash
+conda activate Video-Tree-TRM & pytest tests/unit/test_harness_pools.py::TestPerCategoryPoolStrategy::test_per_category_test_pool_filtered_by_task_types tests/unit/test_harness_pools.py::TestPerCategoryPoolStrategy::test_per_category_test_pool_no_filter_when_task_types_none -v
+```
+
+预期：PASS
+
+- [ ] **Step 5: 运行全部 pool 测试确保无回归**
+
+```bash
+conda activate Video-Tree-TRM & pytest tests/unit/test_harness_pools.py tests/integration/test_pool_strategy.py -v
+```
+
+预期：全部 PASS
+
+- [ ] **Step 6: 提交**
+
+```bash
+git add app/harness/pools.py tests/unit/test_harness_pools.py
+git commit -m "feat(pools): filter test pool by task_types in PerCategoryPoolStrategy"
+```
+
+---
+
+### Task 2: RunConfig 新增 run_holdout_eval + load_config list→tuple 修复 + Runner 条件跳过
+
+**Files:**
+- Modify: `app/harness/config.py:147` (字段) + `app/harness/config.py:437` (list→tuple)
+- Modify: `app/harness/runner.py:1371`
+- Modify: `main.py:205`
+- Test: `tests/unit/test_harness_pools.py`
+
+- [ ] **Step 1: 写失败测试 — RunConfig 新字段**
+
+在 `tests/unit/test_harness_pools.py` 文件末尾新增：
+
+```python
+class TestRunHoldoutEvalConfig:
+    """run_holdout_eval 字段校验。"""
+
+    def test_default_true(self):
+        """run_holdout_eval 默认值为 True。"""
+        from app.harness.config import RunConfig
+
+        config = RunConfig(
+            workspace_dir=Path("/tmp/ws"),
+            store_dir=Path("/tmp/store"),
+            mode="train",
+            concurrency=4,
+            max_steps=10,
+            skill_mode="auto",
+            n_samples=0,
+            questions="benchmarks/Video-MME",
+            skills_version="v1",
+            prompts_version="v1",
+            epochs=1,
+            diag_size=100,
+            diag_correct_ratio=0.5,
+            val_size=30,
+            val_correct_ratio=0.5,
+            edit_budget_start=5,
+            edit_budget_end=2,
+            batch_size=15,
+            min_class_per_batch=2,
+            eval_min_per_class=2,
+            early_stop_patience=4,
+            test_size=30,
+            use_slow_momentum=True,
+            gate_e_confirm=20.0,
+            gate_e_provisional=3.0,
+            gate_w_net_min=2,
+            gate_delta_min=0.02,
+            gate_lambda_dir=-0.642,
+            gate_e_rollback=10.0,
+            gate_block=8,
+            gate_n_max=40,
+            gate_p_low=0.05,
+            gate_p_high=0.95,
+            gate_probe_quota=0.2,
+            gate_gamma_decay=0.9,
+            gate_cooldown_steps=2,
+            gate_guard_err=0.10,
+            skill_update_mode="patch",
+            appendix_consolidate_threshold=6,
+            run_id="test_run",
+        )
+        assert config.run_holdout_eval is True
+
+    def test_explicit_false(self):
+        """run_holdout_eval 可设为 False。"""
+        from app.harness.config import RunConfig
+
+        config = RunConfig(
+            workspace_dir=Path("/tmp/ws"),
+            store_dir=Path("/tmp/store"),
+            mode="train",
+            concurrency=4,
+            max_steps=10,
+            skill_mode="auto",
+            n_samples=0,
+            questions="benchmarks/Video-MME",
+            skills_version="v1",
+            prompts_version="v1",
+            epochs=1,
+            diag_size=100,
+            diag_correct_ratio=0.5,
+            val_size=30,
+            val_correct_ratio=0.5,
+            edit_budget_start=5,
+            edit_budget_end=2,
+            batch_size=15,
+            min_class_per_batch=2,
+            eval_min_per_class=2,
+            early_stop_patience=4,
+            test_size=30,
+            use_slow_momentum=True,
+            gate_e_confirm=20.0,
+            gate_e_provisional=3.0,
+            gate_w_net_min=2,
+            gate_delta_min=0.02,
+            gate_lambda_dir=-0.642,
+            gate_e_rollback=10.0,
+            gate_block=8,
+            gate_n_max=40,
+            gate_p_low=0.05,
+            gate_p_high=0.95,
+            gate_probe_quota=0.2,
+            gate_gamma_decay=0.9,
+            gate_cooldown_steps=2,
+            gate_guard_err=0.10,
+            skill_update_mode="patch",
+            appendix_consolidate_threshold=6,
+            run_id="test_run",
+            run_holdout_eval=False,
+        )
+        assert config.run_holdout_eval is False
+```
+
+- [ ] **Step 2: 运行测试验证失败**
+
+```bash
+conda activate Video-Tree-TRM & pytest tests/unit/test_harness_pools.py::TestRunHoldoutEvalConfig -v
+```
+
+预期：FAIL — `TypeError: __init__() got an unexpected keyword argument 'run_holdout_eval'`
+
+- [ ] **Step 3: RunConfig 新增 run_holdout_eval 字段**
+
+在 `app/harness/config.py` 的有默认值字段区（`test_questions` 后面）新增：
+
+```python
+    test_questions: str = "benchmarks/Video-MME"
+    run_holdout_eval: bool = True
+```
+
+- [ ] **Step 4: 运行测试验证通过**
+
+```bash
+conda activate Video-Tree-TRM & pytest tests/unit/test_harness_pools.py::TestRunHoldoutEvalConfig -v
+```
+
+预期：PASS
+
+- [ ] **Step 5: load_config 修复 YAML list→tuple 转换**
+
+`RunConfig.task_types` 类型为 `tuple[str, ...] | None`，但 YAML list 加载后不转换。
+在 `app/harness/config.py` `load_config()` Phase 4（类型转换区）后面新增：
+
+```python
+    # Phase 4: 类型转换 — 路径字段转 Path
+    for field_name in _PATH_FIELDS:
+        if field_name in yaml_data:
+            yaml_data[field_name] = Path(yaml_data[field_name])
+
+    # Phase 4b: 类型转换 — task_types list → tuple
+    if "task_types" in yaml_data and yaml_data["task_types"] is not None:
+        yaml_data["task_types"] = tuple(yaml_data["task_types"])
+```
+
+- [ ] **Step 6: main.py 新增 CLI 开关**
+
+在 `main.py` `_build_parser()` 的 `--test-questions` 后面新增：
+
+```python
+    parser.add_argument("--test-questions", type=str, dest="test_questions")
+    parser.add_argument(
+        "--no-run-holdout-eval",
+        action="store_true",
+        dest="no_run_holdout_eval",
+    )
+    return parser
+```
+
+在 `main()` 中 `cli_overrides` 构建处（约第 269 行 `cli_overrides = ...` 之后）处理取反映射：
+
+```python
+    cli_overrides = {k: v for k, v in cli_args.items() if k != "config"}
+    if cli_overrides.get("no_run_holdout_eval"):
+        cli_overrides["run_holdout_eval"] = False
+    cli_overrides.pop("no_run_holdout_eval", None)
+```
+
+- [ ] **Step 7: Runner `_slow_update_cycle` 条件跳过 holdout**
+
+修改 `app/harness/runner.py` `_slow_update_cycle` 的 Phase 9（约第 1371 行）：
+
+将：
+```python
+        await self._holdout_four_way(epoch, pools, state, eval_skills_version, eval_prompts_version)
+```
+
+改为：
+```python
+        if self._config.run_holdout_eval:
+            await self._holdout_four_way(epoch, pools, state, eval_skills_version, eval_prompts_version)
+```
+
+- [ ] **Step 8: 运行全部测试确认无回归**
+
+```bash
+conda activate Video-Tree-TRM & pytest tests/unit/test_harness_pools.py tests/unit/test_harness_store.py tests/integration/test_pool_strategy.py -v
+```
+
+预期：全部 PASS
+
+- [ ] **Step 9: 提交**
+
+```bash
+git add app/harness/config.py app/harness/runner.py main.py tests/unit/test_harness_pools.py
+git commit -m "feat(config): add run_holdout_eval toggle and fix YAML task_types list-to-tuple"
+```
+
+---
+
+### Task 3: 实验配置文件
+
+**Files:**
+- Create: `config/train_action_recognition.yaml`
+
+- [ ] **Step 1: 创建实验 YAML**
+
+```yaml
+# config/train_action_recognition.yaml
+# Action Recognition 单题型首次训练实验
+# 设计文档: research-wiki/designs/2026-07-14-action-recognition-training-design.md
+
+harness:
+  workspace_dir: "workspaces/train-action-recognition"
+  store_dir: store
+  mode: train
+  run_id: train_ar_v1
+  concurrency: 24
+  max_steps: 40
+  skill_mode: auto
+  n_samples: 0
+  questions: "generated-v2-360"
+  skills_version: v1
+  prompts_version: v1
+  epochs: 3
+  # CE-Gate 参数（沿用 default.yaml）
+  gate_e_confirm: 20.0
+  gate_e_provisional: 3.0
+  gate_w_net_min: 2
+  gate_delta_min: 0.02
+  gate_lambda_dir: -0.642
+  gate_e_rollback: 10.0
+  gate_block: 8
+  gate_n_max: 40
+  gate_p_low: 0.05
+  gate_p_high: 0.95
+  gate_probe_quota: 0.2
+  gate_gamma_decay: 0.9
+  gate_cooldown_steps: 2
+  gate_guard_err: 0.10
+  # 进化参数
+  edit_budget_start: 5
+  edit_budget_end: 2
+  skill_update_mode: patch
+  appendix_consolidate_threshold: 6
+  # 池配置 — per_category 单题型
+  pool_split_mode: per_category
+  task_types:
+    - "Action Recognition"
+  train_ratio: 0.667
+  test_questions: "benchmarks/Video-MME"
+  run_holdout_eval: false
+  # mini-batch
+  batch_size: 10
+  min_class_per_batch: 2
+  batch_correct_ratio: 0.5
+  momentum_samples: 20
+  eval_min_per_class: 2
+  early_stop_patience: 4
+  test_size: 63
+  diag_size: 20
+  diag_correct_ratio: 0.5
+  val_size: 10
+  val_correct_ratio: 0.5
+  use_slow_momentum: true
+
+embed:
+  backend: "local"
+  model_name: "BAAI/bge-base-zh-v1.5"
+  embed_dim: 768
+  device: "cuda"
+```
+
+- [ ] **Step 2: 验证 YAML 可正确加载为 RunConfig**
+
+```bash
+conda activate Video-Tree-TRM & python -c "
+from app.harness.config import load_config
+from pathlib import Path
+config = load_config(Path('config/train_action_recognition.yaml'))
+assert config.task_types == ('Action Recognition',), f'task_types={config.task_types}'
+assert config.run_holdout_eval is False
+assert config.pool_split_mode == 'per_category'
+print('Config loaded OK')
+"
+```
+
+预期：Config loaded OK
+
+- [ ] **Step 3: 提交**
+
+```bash
+git add config/train_action_recognition.yaml
+git commit -m "config: add train_action_recognition experiment YAML"
+```
+
+---
+
+### Task 4: 前置脚本 — baseline infer + seed 创建 + 训练
+
+**Files:**
+- Create: `scripts/train_action_recognition.sh`
+
+- [ ] **Step 1: 创建脚本**
+
+```bash
+#!/usr/bin/env bash
+# Action Recognition 单题型训练实验
+# 设计文档: research-wiki/designs/2026-07-14-action-recognition-training-design.md
+#
+# 三阶段:
+#   Phase 0: baseline infer (v2-360 Action Recognition 30 题)
+#   Phase 1: create seed (v2ar-baseline)
+#   Phase 2: train (3 epochs, per_category)
+#
+# 用法:
+#   CUDA_VISIBLE_DEVICES=0 bash scripts/train_action_recognition.sh
+#   MODE=mock bash scripts/train_action_recognition.sh   # 跳过 Phase 0/1
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}"
+export CUDA_VISIBLE_DEVICES
+
+export HF_HUB_OFFLINE=1
+export TRANSFORMERS_OFFLINE=1
+export PYTHONUNBUFFERED=1
+
+set -a
+source .env
+set +a
+
+PYTHON="$(conda run -n Video-Tree-TRM which python)"
+
+# ── Phase 0: Baseline infer ──
+if [[ "${MODE:-}" != "mock" ]]; then
+    echo "=== Phase 0: Baseline infer (v2-360 Action Recognition 30 题) ==="
+    "${PYTHON}" main.py \
+        --config config/default.yaml \
+        --workspace-dir workspaces/default \
+        --store-dir store \
+        --mode infer \
+        --concurrency 24 \
+        --max-steps 40 \
+        --skill-mode auto \
+        --n-samples 0 \
+        --questions "generated-v2-360" \
+        --skills-version v1 \
+        --prompts-version v1 \
+        --run-id v2ar_baseline \
+        --task-types "Action Recognition"
+fi
+
+# ── Phase 1: Create seed ──
+if [[ "${MODE:-}" != "mock" && ! -d "store/seeds/v2ar-baseline" ]]; then
+    echo "=== Phase 1: Create seed v2ar-baseline ==="
+    "${PYTHON}" -c "
+from pathlib import Path
+from app.harness.store import extract_run_db, init_seed
+import tempfile
+
+tmp = Path(tempfile.mkdtemp()) / 'baseline.db'
+extract_run_db(
+    Path('workspaces/default/harness.db'),
+    tmp,
+    'infer_v2ar_baseline',
+)
+init_seed(
+    store_dir=Path('store'),
+    name='v2ar-baseline',
+    skills_dir=Path('store/skills/v1'),
+    prompts_dir=Path('store/prompts/v1'),
+    baseline_db=tmp,
+    baseline_run_id='infer_v2ar_baseline',
+    parent=None,
+    description='v2-360 Action Recognition 30 题 baseline (skills/v1)',
+)
+tmp.unlink()
+print('Seed created: store/seeds/v2ar-baseline/')
+"
+elif [[ -d "store/seeds/v2ar-baseline" ]]; then
+    echo "=== Phase 1: Seed v2ar-baseline 已存在，跳过 ==="
+fi
+
+# ── Phase 2: Train ──
+echo "=== Phase 2: Train (3 epochs, Action Recognition) ==="
+"${PYTHON}" main.py \
+    --config config/train_action_recognition.yaml \
+    --fresh \
+    --seed v2ar-baseline
+
+echo "=== 训练完成 ==="
+echo "结果查看:"
+echo "  cat workspaces/train-action-recognition/analyses/final_test_eval.json"
+echo "  sqlite3 workspaces/train-action-recognition/harness.db 'SELECT * FROM dual_metric'"
+```
+
+- [ ] **Step 2: 设置可执行权限并验证语法**
+
+```bash
+chmod +x scripts/train_action_recognition.sh
+bash -n scripts/train_action_recognition.sh
+```
+
+预期：无语法错误
+
+- [ ] **Step 3: 提交**
+
+```bash
+git add scripts/train_action_recognition.sh
+git commit -m "scripts: add train_action_recognition experiment script"
+```
+
+---
+
+### Task 5: lint 检查 + 全量回归测试
+
+- [ ] **Step 1: Ruff 格式化与检查**
+
+```bash
+conda activate Video-Tree-TRM & ruff format app/ core/ && ruff check app/ core/ --fix
+```
+
+预期：无错误
+
+- [ ] **Step 2: 全量测试**
+
+```bash
+conda activate Video-Tree-TRM & pytest tests/unit/ tests/integration/ -v --tb=short
+```
+
+预期：全部 PASS
+
+- [ ] **Step 3: 最终提交（如有 lint 修复）**
+
+```bash
+git add -A && git commit -m "chore: lint and format training experiment changes"
+```
+
+---
+
+## 核心算法保真校验
+
+本计划不涉及核心算法迁移。修改仅限于：
+- `PerCategoryPoolStrategy.build()` 新增 3 行 test 池过滤（不改 train/val 切分逻辑）
+- `RunConfig` 新增 1 个 bool 字段
+- `load_config` 新增 2 行 list→tuple 转换
+- `_slow_update_cycle` 新增 1 行 `if` 条件（不改 holdout 内部逻辑）
+
+保真校验不适用。
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# Action Recognition 单题型训练实验
+# 设计文档: research-wiki/designs/2026-07-14-action-recognition-training-design.md
+#
+# 三阶段:
+#   Phase 0: baseline infer (v2-360 Action Recognition 30 题)
+#   Phase 1: create seed (v2ar-baseline)
+#   Phase 2: train (3 epochs, per_category)
+#
+# 用法:
+#   CUDA_VISIBLE_DEVICES=0 bash scripts/train_action_recognition.sh
+#   MODE=mock bash scripts/train_action_recognition.sh   # 跳过 Phase 0/1
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}"
+export CUDA_VISIBLE_DEVICES
+
+export HF_HUB_OFFLINE=1
+export TRANSFORMERS_OFFLINE=1
+export PYTHONUNBUFFERED=1
+
+set -a
+source .env
+set +a
+
+PYTHON="$(conda run -n Video-Tree-TRM which python)"
+
+# ── Phase 0: Baseline infer ──
+if [[ "${MODE:-}" != "mock" ]]; then
+    echo "=== Phase 0: Baseline infer (v2-360 Action Recognition 30 题) ==="
+    "${PYTHON}" main.py \
+        --config config/default.yaml \
+        --workspace-dir workspaces/default \
+        --store-dir store \
+        --mode infer \
+        --concurrency 24 \
+        --max-steps 40 \
+        --skill-mode auto \
+        --n-samples 0 \
+        --questions "generated-v2-360" \
+        --skills-version v1 \
+        --prompts-version v1 \
+        --run-id v2ar_baseline \
+        --task-types "Action Recognition"
+fi
+
+# ── Phase 1: Create seed ──
+if [[ "${MODE:-}" != "mock" && ! -d "store/seeds/v2ar-baseline" ]]; then
+    echo "=== Phase 1: Create seed v2ar-baseline ==="
+    "${PYTHON}" -c "
+from pathlib import Path
+from app.harness.store import extract_run_db, init_seed
+import tempfile
+
+tmp = Path(tempfile.mkdtemp()) / 'baseline.db'
+extract_run_db(
+    Path('workspaces/default/harness.db'),
+    tmp,
+    'infer_v2ar_baseline',
+)
+init_seed(
+    store_dir=Path('store'),
+    name='v2ar-baseline',
+    skills_dir=Path('store/skills/v1'),
+    prompts_dir=Path('store/prompts/v1'),
+    baseline_db=tmp,
+    baseline_run_id='infer_v2ar_baseline',
+    parent=None,
+    description='v2-360 Action Recognition 30 题 baseline (skills/v1)',
+)
+tmp.unlink()
+print('Seed created: store/seeds/v2ar-baseline/')
+"
+elif [[ -d "store/seeds/v2ar-baseline" ]]; then
+    echo "=== Phase 1: Seed v2ar-baseline 已存在，跳过 ==="
+fi
+
+# ── Phase 2: Train ──
+echo "=== Phase 2: Train (3 epochs, Action Recognition) ==="
+"${PYTHON}" main.py \
+    --config config/train_action_recognition.yaml \
+    --fresh \
+    --seed v2ar-baseline
+
+echo "=== 训练完成 ==="
+echo "结果查看:"
+echo "  cat workspaces/train-action-recognition/analyses/final_test_eval.json"
+echo "  sqlite3 workspaces/train-action-recognition/harness.db 'SELECT * FROM dual_metric'"
@@ -63,7 +63,7 @@ def _make_question_set(
    返回:
        题目列表。
    """
-    types = task_types or ["Action Reasoning", "Scene Understanding"]
+    types = task_types or ["Action Reasoning", "Information Synopsis"]
    return [_make_question(f"q_{i:04d}", types[i % len(types)]) for i in range(n)]


@@ -347,10 +347,18 @@ class TestGlobalPoolStrategy:
 def _make_per_category_questions():
    """构造 12 类各 30 题，共 360 题。"""
    task_types = [
-        "Action Prediction", "Action Reasoning", "Action Recognition",
-        "Action Sequence", "Causal Reasoning", "Event Reasoning",
-        "Object Interaction", "Object Reasoning", "Object Recognition",
-        "Scene Understanding", "Spatial Reasoning", "Temporal Reasoning",
+        "Action Recognition",
+        "Action Reasoning",
+        "Attribute Perception",
+        "Counting Problem",
+        "Information Synopsis",
+        "Object Recognition",
+        "Object Reasoning",
+        "OCR Problems",
+        "Spatial Perception",
+        "Spatial Reasoning",
+        "Temporal Perception",
+        "Temporal Reasoning",
    ]
    questions = []
    for tt in task_types:
@@ -371,10 +379,17 @@ class TestPerCategoryPoolStrategy:
            correctness[q.question_id] = idx < 18

        config = PoolConfig(
-            task_types=None, seed=42, baseline_run_id="baseline_v2",
-            diag_size=0, diag_correct_ratio=0.0, val_size=0,
-            val_correct_ratio=0.0, test_size=0, eval_min_per_class=0,
-            train_ratio=20 / 30, test_questions_dir=None,
+            task_types=None,
+            seed=42,
+            baseline_run_id="baseline_v2",
+            diag_size=0,
+            diag_correct_ratio=0.0,
+            val_size=0,
+            val_correct_ratio=0.0,
+            test_size=0,
+            eval_min_per_class=0,
+            train_ratio=20 / 30,
+            test_questions_dir=None,
        )
        strategy = PerCategoryPoolStrategy()
        pools = strategy.build(questions, correctness, config)
@@ -382,6 +397,7 @@ class TestPerCategoryPoolStrategy:
        assert len(pools.validation) == 120

        from collections import Counter
+
        diag_counts = Counter(q.task_type for q in pools.diagnosis)
        val_counts = Counter(q.task_type for q in pools.validation)
        for tt in diag_counts:
@@ -401,10 +417,17 @@ class TestPerCategoryPoolStrategy:
            correctness[q.question_id] = idx < 18

        config = PoolConfig(
-            task_types=("Action Reasoning",), seed=42, baseline_run_id="baseline_v2",
-            diag_size=0, diag_correct_ratio=0.0, val_size=0,
-            val_correct_ratio=0.0, test_size=0, eval_min_per_class=0,
-            train_ratio=20 / 30, test_questions_dir=None,
+            task_types=("Action Reasoning",),
+            seed=42,
+            baseline_run_id="baseline_v2",
+            diag_size=0,
+            diag_correct_ratio=0.0,
+            val_size=0,
+            val_correct_ratio=0.0,
+            test_size=0,
+            eval_min_per_class=0,
+            train_ratio=20 / 30,
+            test_questions_dir=None,
        )
        strategy = PerCategoryPoolStrategy()
        pools = strategy.build(questions, correctness, config)
@@ -420,10 +443,17 @@ class TestPerCategoryPoolStrategy:
        questions = [_make_question(f"q_{i:03d}", "Object Recognition") for i in range(30)]
        correctness = {q.question_id: True for q in questions}
        config = PoolConfig(
-            task_types=None, seed=42, baseline_run_id="r",
-            diag_size=0, diag_correct_ratio=0.0, val_size=0,
-            val_correct_ratio=0.0, test_size=0, eval_min_per_class=0,
-            train_ratio=20 / 30, test_questions_dir=None,
+            task_types=None,
+            seed=42,
+            baseline_run_id="r",
+            diag_size=0,
+            diag_correct_ratio=0.0,
+            val_size=0,
+            val_correct_ratio=0.0,
+            test_size=0,
+            eval_min_per_class=0,
+            train_ratio=20 / 30,
+            test_questions_dir=None,
        )
        strategy = PerCategoryPoolStrategy()
        pools = strategy.build(questions, correctness, config)
@@ -435,10 +465,17 @@ class TestPerCategoryPoolStrategy:
        questions = [_make_question(f"q_{i:03d}", "Object Recognition") for i in range(30)]
        correctness = {q.question_id: True for q in questions[:25]}
        config = PoolConfig(
-            task_types=None, seed=42, baseline_run_id="r",
-            diag_size=0, diag_correct_ratio=0.0, val_size=0,
-            val_correct_ratio=0.0, test_size=0, eval_min_per_class=0,
-            train_ratio=20 / 30, test_questions_dir=None,
+            task_types=None,
+            seed=42,
+            baseline_run_id="r",
+            diag_size=0,
+            diag_correct_ratio=0.0,
+            val_size=0,
+            val_correct_ratio=0.0,
+            test_size=0,
+            eval_min_per_class=0,
+            train_ratio=20 / 30,
+            test_questions_dir=None,
        )
        strategy = PerCategoryPoolStrategy()
        with pytest.raises(ValueError, match="correctness 缺失"):
@@ -449,17 +486,131 @@ class TestPerCategoryPoolStrategy:
        questions = _make_per_category_questions()
        correctness = {q.question_id: True for q in questions}
        config = PoolConfig(
-            task_types=("Action Reasoning", "Scene Understanding"), seed=42,
-            baseline_run_id="r", diag_size=0, diag_correct_ratio=0.0,
-            val_size=0, val_correct_ratio=0.0, test_size=0,
-            eval_min_per_class=0, train_ratio=20 / 30, test_questions_dir=None,
+            task_types=("Action Reasoning", "Information Synopsis"),
+            seed=42,
+            baseline_run_id="r",
+            diag_size=0,
+            diag_correct_ratio=0.0,
+            val_size=0,
+            val_correct_ratio=0.0,
+            test_size=0,
+            eval_min_per_class=0,
+            train_ratio=20 / 30,
+            test_questions_dir=None,
        )
        strategy = PerCategoryPoolStrategy()
        pools = strategy.build(questions, correctness, config)
        assert len(pools.diagnosis) == 40
        assert len(pools.validation) == 20
        types_in_diag = {q.task_type for q in pools.diagnosis}
-        assert types_in_diag == {"Action Reasoning", "Scene Understanding"}
+        assert types_in_diag == {"Action Reasoning", "Information Synopsis"}
+
+    def test_per_category_test_pool_filtered_by_task_types(self, tmp_path: Path) -> None:
+        """test_questions_dir 加载的 test 池应按 task_types 过滤。"""
+
+        test_dir = tmp_path / "test_questions"
+        test_dir.mkdir()
+
+        task_types_all = [
+            "Action Recognition",
+            "Action Reasoning",
+            "Temporal Perception",
+        ]
+        for tt in task_types_all:
+            items = []
+            for i in range(10):
+                items.append(
+                    {
+                        "question_id": f"{tt}_{i:03d}",
+                        "video_id": "v1",
+                        "task_type": tt,
+                        "question": f"Q {tt} {i}?",
+                        "options": ["A. a", "B. b", "C. c", "D. d"],
+                        "answer": "A",
+                    }
+                )
+            slug = tt.lower().replace(" ", "_")
+            (test_dir / f"{slug}.json").write_text(json.dumps(items, ensure_ascii=False))
+
+        # train/val 用的题目（与 test 独立）
+        questions = _make_per_category_questions()
+        correctness = {q.question_id: True for q in questions}
+
+        config = PoolConfig(
+            task_types=("Action Recognition",),
+            seed=42,
+            baseline_run_id="baseline_v2",
+            diag_size=0,
+            diag_correct_ratio=0.0,
+            val_size=0,
+            val_correct_ratio=0.0,
+            test_size=0,
+            eval_min_per_class=0,
+            train_ratio=20 / 30,
+            test_questions_dir=test_dir,
+        )
+        strategy = PerCategoryPoolStrategy()
+        pools = strategy.build(questions, correctness, config)
+
+        assert len(pools.test) == 10, (
+            f"test 池应仅含 Action Recognition 的 10 题，实际 {len(pools.test)}"
+        )
+        test_types = {q.task_type for q in pools.test}
+        assert test_types == {"Action Recognition"}, (
+            f"test 池应仅含 Action Recognition，实际含 {test_types}"
+        )
+
+    def test_per_category_test_pool_no_filter_when_task_types_none(self, tmp_path: Path) -> None:
+        """task_types=None 时，test 池不过滤，加载全部题目。"""
+
+        test_dir = tmp_path / "test_questions"
+        test_dir.mkdir()
+
+        task_types_all = [
+            "Action Recognition",
+            "Action Reasoning",
+            "Temporal Perception",
+        ]
+        total_expected = 0
+        for tt in task_types_all:
+            items = []
+            for i in range(10):
+                items.append(
+                    {
+                        "question_id": f"{tt}_{i:03d}",
+                        "video_id": "v1",
+                        "task_type": tt,
+                        "question": f"Q {tt} {i}?",
+                        "options": ["A. a", "B. b", "C. c", "D. d"],
+                        "answer": "A",
+                    }
+                )
+            slug = tt.lower().replace(" ", "_")
+            (test_dir / f"{slug}.json").write_text(json.dumps(items, ensure_ascii=False))
+            total_expected += len(items)
+
+        questions = _make_per_category_questions()
+        correctness = {q.question_id: True for q in questions}
+
+        config = PoolConfig(
+            task_types=None,
+            seed=42,
+            baseline_run_id="baseline_v2",
+            diag_size=0,
+            diag_correct_ratio=0.0,
+            val_size=0,
+            val_correct_ratio=0.0,
+            test_size=0,
+            eval_min_per_class=0,
+            train_ratio=20 / 30,
+            test_questions_dir=test_dir,
+        )
+        strategy = PerCategoryPoolStrategy()
+        pools = strategy.build(questions, correctness, config)
+
+        assert len(pools.test) == total_expected, (
+            f"task_types=None 时应加载全部 {total_expected} 题，实际 {len(pools.test)}"
+        )


 class TestPerCategorySaveLoad:
@@ -470,9 +621,16 @@ class TestPerCategorySaveLoad:
        questions = _make_per_category_questions()
        correctness = {q.question_id: True for q in questions}
        config = PoolConfig(
-            task_types=("Action Reasoning",), seed=42, baseline_run_id="baseline_v2",
-            diag_size=0, diag_correct_ratio=0.0, val_size=0, val_correct_ratio=0.0,
-            test_size=0, eval_min_per_class=0, train_ratio=20 / 30,
+            task_types=("Action Reasoning",),
+            seed=42,
+            baseline_run_id="baseline_v2",
+            diag_size=0,
+            diag_correct_ratio=0.0,
+            val_size=0,
+            val_correct_ratio=0.0,
+            test_size=0,
+            eval_min_per_class=0,
+            train_ratio=20 / 30,
            test_questions_dir=None,
        )
        strategy = PerCategoryPoolStrategy()
@@ -503,8 +661,11 @@ class TestPerCategorySaveLoad:
        from app.harness.pools import Pools

        pools = Pools(
-            diagnosis=[], validation=[], test=[],
-            baseline_run_id="r", baseline_val_accuracy=0.0,
+            diagnosis=[],
+            validation=[],
+            test=[],
+            baseline_run_id="r",
+            baseline_val_accuracy=0.0,
        )
        with pytest.raises(ValueError, match="per_category 模式下.*必须提供 config"):
            save_pools(pools, tmp_path / "pools.json", split_mode="per_category")
@@ -514,11 +675,22 @@ class TestPerCategorySaveLoad:
        questions = _make_question_set(60)
        correctness = _make_correctness(questions, 0.5)
        original = build_pools(
-            questions, correctness,
-            diag_cfg={"size": 10, "correct_ratio": 0.5, "task_types": None,
-                      "seed": 42, "min_per_class": None},
-            val_cfg={"size": 10, "correct_ratio": 0.5, "task_types": None,
-                     "seed": 42, "min_per_class": None},
+            questions,
+            correctness,
+            diag_cfg={
+                "size": 10,
+                "correct_ratio": 0.5,
+                "task_types": None,
+                "seed": 42,
+                "min_per_class": None,
+            },
+            val_cfg={
+                "size": 10,
+                "correct_ratio": 0.5,
+                "task_types": None,
+                "seed": 42,
+                "min_per_class": None,
+            },
            test_cfg={"size": 10},
            baseline_run_id="run_001",
        )
@@ -539,12 +711,20 @@ class TestPerCategorySaveLoad:
            "baseline_run_id": "run_legacy",
            "baseline_val_accuracy": 0.75,
            "correctness": {"q1": True},
-            "diagnosis": [{
-                "question_id": "q1", "video_id": "v1", "task_type": "AR",
-                "question": "Q?", "options": ["A", "B", "C", "D"],
-                "answer": "A", "source_nodes": [], "difficulty": "medium",
-                "skill_target": None, "difficulty_steps": None,
-            }],
+            "diagnosis": [
+                {
+                    "question_id": "q1",
+                    "video_id": "v1",
+                    "task_type": "AR",
+                    "question": "Q?",
+                    "options": ["A", "B", "C", "D"],
+                    "answer": "A",
+                    "source_nodes": [],
+                    "difficulty": "medium",
+                    "skill_target": None,
+                    "difficulty_steps": None,
+                }
+            ],
            "validation": [],
            "test": [],
        }
@@ -559,10 +739,17 @@ class TestPerCategorySaveLoad:
        questions = _make_per_category_questions()
        correctness = {q.question_id: True for q in questions}
        config = PoolConfig(
-            task_types=("Action Reasoning", "Scene Understanding"), seed=0,
-            baseline_run_id="b", diag_size=0, diag_correct_ratio=0.0,
-            val_size=0, val_correct_ratio=0.0, test_size=0,
-            eval_min_per_class=0, train_ratio=20 / 30, test_questions_dir=None,
+            task_types=("Action Reasoning", "Information Synopsis"),
+            seed=0,
+            baseline_run_id="b",
+            diag_size=0,
+            diag_correct_ratio=0.0,
+            val_size=0,
+            val_correct_ratio=0.0,
+            test_size=0,
+            eval_min_per_class=0,
+            train_ratio=20 / 30,
+            test_questions_dir=None,
        )
        strategy = PerCategoryPoolStrategy()
        pools = strategy.build(questions, correctness, config)
@@ -571,7 +758,8 @@ class TestPerCategorySaveLoad:

        data = json.loads(pools_path.read_text())
        assert set(data["categories"].keys()) == {
-            "Action Reasoning", "Scene Understanding",
+            "Action Reasoning",
+            "Information Synopsis",
        }
        for tt in data["categories"]:
            cat = data["categories"][tt]
@@ -579,3 +767,108 @@ class TestPerCategorySaveLoad:
            assert len(cat["val"]) == 10
            # train + val 的 qid 互斥
            assert set(cat["train"]) & set(cat["val"]) == set()
+
+
+class TestRunHoldoutEvalConfig:
+    """run_holdout_eval 字段校验。"""
+
+    def test_default_true(self):
+        """run_holdout_eval 默认值为 True。"""
+        from pathlib import Path
+
+        from app.harness.config import RunConfig
+
+        config = RunConfig(
+            workspace_dir=Path("/tmp/ws"),
+            store_dir=Path("/tmp/store"),
+            mode="train",
+            concurrency=4,
+            max_steps=10,
+            skill_mode="auto",
+            n_samples=0,
+            questions="benchmarks/Video-MME",
+            skills_version="v1",
+            prompts_version="v1",
+            epochs=1,
+            diag_size=100,
+            diag_correct_ratio=0.5,
+            val_size=30,
+            val_correct_ratio=0.5,
+            edit_budget_start=5,
+            edit_budget_end=2,
+            batch_size=15,
+            min_class_per_batch=2,
+            eval_min_per_class=2,
+            early_stop_patience=4,
+            test_size=30,
+            use_slow_momentum=True,
+            gate_e_confirm=20.0,
+            gate_e_provisional=3.0,
+            gate_w_net_min=2,
+            gate_delta_min=0.02,
+            gate_lambda_dir=-0.642,
+            gate_e_rollback=10.0,
+            gate_block=8,
+            gate_n_max=40,
+            gate_p_low=0.05,
+            gate_p_high=0.95,
+            gate_probe_quota=0.2,
+            gate_gamma_decay=0.9,
+            gate_cooldown_steps=2,
+            gate_guard_err=0.10,
+            skill_update_mode="patch",
+            appendix_consolidate_threshold=6,
+            run_id="test_run",
+        )
+        assert config.run_holdout_eval is True
+
+    def test_explicit_false(self):
+        """run_holdout_eval 可设为 False。"""
+        from pathlib import Path
+
+        from app.harness.config import RunConfig
+
+        config = RunConfig(
+            workspace_dir=Path("/tmp/ws"),
+            store_dir=Path("/tmp/store"),
+            mode="train",
+            concurrency=4,
+            max_steps=10,
+            skill_mode="auto",
+            n_samples=0,
+            questions="benchmarks/Video-MME",
+            skills_version="v1",
+            prompts_version="v1",
+            epochs=1,
+            diag_size=100,
+            diag_correct_ratio=0.5,
+            val_size=30,
+            val_correct_ratio=0.5,
+            edit_budget_start=5,
+            edit_budget_end=2,
+            batch_size=15,
+            min_class_per_batch=2,
+            eval_min_per_class=2,
+            early_stop_patience=4,
+            test_size=30,
+            use_slow_momentum=True,
+            gate_e_confirm=20.0,
+            gate_e_provisional=3.0,
+            gate_w_net_min=2,
+            gate_delta_min=0.02,
+            gate_lambda_dir=-0.642,
+            gate_e_rollback=10.0,
+            gate_block=8,
+            gate_n_max=40,
+            gate_p_low=0.05,
+            gate_p_high=0.95,
+            gate_probe_quota=0.2,
+            gate_gamma_decay=0.9,
+            gate_cooldown_steps=2,
+            gate_guard_err=0.10,
+            skill_update_mode="patch",
+            appendix_consolidate_threshold=6,
+            run_id="test_run",
+            run_holdout_eval=False,
+        )
+        assert config.run_holdout_eval is False