feat(harness): add Action Recognition training experiment

- PerCategoryPoolStrategy: filter test pool by task_types
- RunConfig: add run_holdout_eval toggle (default true)
- load_config: fix YAML task_types list-to-tuple conversion
- Runner: conditionally skip _holdout_four_way when disabled
- CLI: add --no-run-holdout-eval flag
- New config/train_action_recognition.yaml (3 epochs, per_category)
- New scripts/train_action_recognition.sh (baseline + seed + train)
This commit is contained in:
2026-07-14 00:58:54 -04:00
parent 37d4519905
commit dec7346da3
12 changed files with 1423 additions and 52 deletions
+6 -1
View File
@@ -205,6 +205,7 @@ def _q_to_dict(q: GeneratedQuestion) -> dict:
"answer": q.answer,
"source_nodes": list(q.source_nodes),
"difficulty": q.difficulty,
"family": q.family,
"skill_target": q.skill_target,
"difficulty_steps": q.difficulty_steps,
}
@@ -228,6 +229,7 @@ def _dict_to_q(d: dict) -> GeneratedQuestion:
answer=d["answer"],
source_nodes=tuple(d.get("source_nodes", ())),
difficulty=d.get("difficulty", "medium"),
family=d.get("family"),
skill_target=d.get("skill_target"),
difficulty_steps=d.get("difficulty_steps"),
)
@@ -595,12 +597,15 @@ class PerCategoryPoolStrategy:
all_train.extend(train)
all_val.extend(val)
# Phase 4: test 池(从外部目录加载,无则空)
# Phase 4: test 池(从外部目录加载,无则空;按 task_types 过滤
test: list[GeneratedQuestion] = []
if config.test_questions_dir is not None:
from app.question_gen import load_benchmark
test = load_benchmark(config.test_questions_dir)
if config.task_types is not None:
allowed = set(config.task_types)
test = [q for q in test if q.task_type in allowed]
# Phase 5: 计算 baseline_val_accuracy
val_correct = sum(1 for q in all_val if correctness.get(q.question_id, False))