feat(harness): add Action Recognition training experiment
- PerCategoryPoolStrategy: filter test pool by task_types - RunConfig: add run_holdout_eval toggle (default true) - load_config: fix YAML task_types list-to-tuple conversion - Runner: conditionally skip _holdout_four_way when disabled - CLI: add --no-run-holdout-eval flag - New config/train_action_recognition.yaml (3 epochs, per_category) - New scripts/train_action_recognition.sh (baseline + seed + train)
This commit is contained in:
@@ -205,6 +205,7 @@ def _q_to_dict(q: GeneratedQuestion) -> dict:
|
||||
"answer": q.answer,
|
||||
"source_nodes": list(q.source_nodes),
|
||||
"difficulty": q.difficulty,
|
||||
"family": q.family,
|
||||
"skill_target": q.skill_target,
|
||||
"difficulty_steps": q.difficulty_steps,
|
||||
}
|
||||
@@ -228,6 +229,7 @@ def _dict_to_q(d: dict) -> GeneratedQuestion:
|
||||
answer=d["answer"],
|
||||
source_nodes=tuple(d.get("source_nodes", ())),
|
||||
difficulty=d.get("difficulty", "medium"),
|
||||
family=d.get("family"),
|
||||
skill_target=d.get("skill_target"),
|
||||
difficulty_steps=d.get("difficulty_steps"),
|
||||
)
|
||||
@@ -595,12 +597,15 @@ class PerCategoryPoolStrategy:
|
||||
all_train.extend(train)
|
||||
all_val.extend(val)
|
||||
|
||||
# Phase 4: test 池(从外部目录加载,无则空)
|
||||
# Phase 4: test 池(从外部目录加载,无则空;按 task_types 过滤)
|
||||
test: list[GeneratedQuestion] = []
|
||||
if config.test_questions_dir is not None:
|
||||
from app.question_gen import load_benchmark
|
||||
|
||||
test = load_benchmark(config.test_questions_dir)
|
||||
if config.task_types is not None:
|
||||
allowed = set(config.task_types)
|
||||
test = [q for q in test if q.task_type in allowed]
|
||||
|
||||
# Phase 5: 计算 baseline_val_accuracy
|
||||
val_correct = sum(1 for q in all_val if correctness.get(q.question_id, False))
|
||||
|
||||
Reference in New Issue
Block a user