feat(harness): refactor build_or_load_pools to accept PoolStrategy + per_category freeze format

- save_pools: extended with split_mode and config params; per_category mode writes categories metadata (seed, train_ratio, test_source) for incremental append and consistency validation - load_pools: compatible with both old format (no split_mode) and new format; extra metadata fields ignored during load - build_or_load_pools: signature changed to (config, strategy, db_path); baseline_run_id read from seed.json (not config.run_id); per_category mode does consistency check on reload and supports incremental category append via strategy.build_incremental - Added _to_pool_config, _read_baseline_run_id, _validate_per_category_consistency helpers - Tests: TestPerCategorySaveLoad with 5 test cases covering roundtrip, missing config error, global split_mode field, legacy format compat, multi-type categories Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-07-12 22:53:19 -04:00
parent e5b07ac974
commit c66a00c924
2 changed files with 359 additions and 42 deletions
@@ -460,3 +460,122 @@ class TestPerCategoryPoolStrategy:
        assert len(pools.validation) == 20
        types_in_diag = {q.task_type for q in pools.diagnosis}
        assert types_in_diag == {"Action Reasoning", "Scene Understanding"}
+
+
+class TestPerCategorySaveLoad:
+    """per_category 格式的 pools.json 冻结/加载。"""
+
+    def test_save_load_per_category_roundtrip(self, tmp_path: Path) -> None:
+        """per_category 模式 save -> load 往返一致。"""
+        questions = _make_per_category_questions()
+        correctness = {q.question_id: True for q in questions}
+        config = PoolConfig(
+            task_types=("Action Reasoning",), seed=42, baseline_run_id="baseline_v2",
+            diag_size=0, diag_correct_ratio=0.0, val_size=0, val_correct_ratio=0.0,
+            test_size=0, eval_min_per_class=0, train_ratio=20 / 30,
+            test_questions_dir=None,
+        )
+        strategy = PerCategoryPoolStrategy()
+        pools = strategy.build(questions, correctness, config)
+        pools_path = tmp_path / "pools.json"
+        save_pools(pools, pools_path, split_mode="per_category", config=config)
+        loaded = load_pools(pools_path)
+        assert loaded.baseline_run_id == pools.baseline_run_id
+        assert len(loaded.diagnosis) == len(pools.diagnosis)
+        assert len(loaded.validation) == len(pools.validation)
+
+        # 验证 per_category 格式内容
+        data = json.loads(pools_path.read_text())
+        assert data["split_mode"] == "per_category"
+        assert "categories" in data
+        assert data["seed"] == 42
+        assert data["train_ratio"] == pytest.approx(20 / 30)
+        assert data["test_source"] is None
+
+        # categories 内容校验
+        cats = data["categories"]
+        assert "Action Reasoning" in cats
+        assert len(cats["Action Reasoning"]["train"]) == 20
+        assert len(cats["Action Reasoning"]["val"]) == 10
+
+    def test_save_per_category_without_config_raises(self, tmp_path: Path) -> None:
+        """per_category 模式未提供 config 时报 ValueError。"""
+        from app.harness.pools import Pools
+
+        pools = Pools(
+            diagnosis=[], validation=[], test=[],
+            baseline_run_id="r", baseline_val_accuracy=0.0,
+        )
+        with pytest.raises(ValueError, match="per_category 模式下.*必须提供 config"):
+            save_pools(pools, tmp_path / "pools.json", split_mode="per_category")
+
+    def test_save_global_mode_has_split_mode_field(self, tmp_path: Path) -> None:
+        """global 模式 save 也写入 split_mode 字段。"""
+        questions = _make_question_set(60)
+        correctness = _make_correctness(questions, 0.5)
+        original = build_pools(
+            questions, correctness,
+            diag_cfg={"size": 10, "correct_ratio": 0.5, "task_types": None,
+                      "seed": 42, "min_per_class": None},
+            val_cfg={"size": 10, "correct_ratio": 0.5, "task_types": None,
+                     "seed": 42, "min_per_class": None},
+            test_cfg={"size": 10},
+            baseline_run_id="run_001",
+        )
+        pools_path = tmp_path / "pools.json"
+        save_pools(original, pools_path, split_mode="global")
+        data = json.loads(pools_path.read_text())
+        assert data["split_mode"] == "global"
+        assert "categories" not in data
+
+        # 仍能正常 load
+        loaded = load_pools(pools_path)
+        assert loaded.baseline_run_id == "run_001"
+        assert len(loaded.diagnosis) == 10
+
+    def test_load_legacy_format_without_split_mode(self, tmp_path: Path) -> None:
+        """旧格式（无 split_mode 字段）仍可加载。"""
+        legacy = {
+            "baseline_run_id": "run_legacy",
+            "baseline_val_accuracy": 0.75,
+            "correctness": {"q1": True},
+            "diagnosis": [{
+                "question_id": "q1", "video_id": "v1", "task_type": "AR",
+                "question": "Q?", "options": ["A", "B", "C", "D"],
+                "answer": "A", "source_nodes": [], "difficulty": "medium",
+                "skill_target": None, "difficulty_steps": None,
+            }],
+            "validation": [],
+            "test": [],
+        }
+        pools_path = tmp_path / "pools.json"
+        pools_path.write_text(json.dumps(legacy), encoding="utf-8")
+        loaded = load_pools(pools_path)
+        assert loaded.baseline_run_id == "run_legacy"
+        assert len(loaded.diagnosis) == 1
+
+    def test_per_category_categories_multi_type(self, tmp_path: Path) -> None:
+        """多类别 per_category save 后 categories 包含所有类别。"""
+        questions = _make_per_category_questions()
+        correctness = {q.question_id: True for q in questions}
+        config = PoolConfig(
+            task_types=("Action Reasoning", "Scene Understanding"), seed=0,
+            baseline_run_id="b", diag_size=0, diag_correct_ratio=0.0,
+            val_size=0, val_correct_ratio=0.0, test_size=0,
+            eval_min_per_class=0, train_ratio=20 / 30, test_questions_dir=None,
+        )
+        strategy = PerCategoryPoolStrategy()
+        pools = strategy.build(questions, correctness, config)
+        pools_path = tmp_path / "pools.json"
+        save_pools(pools, pools_path, split_mode="per_category", config=config)
+
+        data = json.loads(pools_path.read_text())
+        assert set(data["categories"].keys()) == {
+            "Action Reasoning", "Scene Understanding",
+        }
+        for tt in data["categories"]:
+            cat = data["categories"][tt]
+            assert len(cat["train"]) == 20
+            assert len(cat["val"]) == 10
+            # train + val 的 qid 互斥
+            assert set(cat["train"]) & set(cat["val"]) == set()