feat(pools): auto-supplement maintenance correct questions in PerCategoryPoolStrategy

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

app/harness/pools.py

@@ -116,6 +116,8 @@ class GlobalPoolStrategy:
         questions: list[GeneratedQuestion],
         correctness: dict[str, bool],
         config: PoolConfig,
+        *,
+        db_path: Path | None = None,
     ) -> Pools:
         """委托给现有 build_pools 函数。
 
@@ -534,7 +536,7 @@ def build_or_load_pools(
         )
     correctness = {r["question_id"]: r["prediction"] == r["answer"] for r in rows}
 
-    pools = strategy.build(questions, correctness, pool_config)
+    pools = strategy.build(questions, correctness, pool_config, db_path=db_path)
     save_pools(
         pools,
         pools_path,
@@ -558,6 +560,8 @@ class PerCategoryPoolStrategy:
         questions: list[GeneratedQuestion],
         correctness: dict[str, bool],
         config: PoolConfig,
+        *,
+        db_path: Path | None = None,
     ) -> Pools:
         """按题型分组后，每组做 correctness 分层的 train/val 划分。
 
@@ -565,7 +569,9 @@ class PerCategoryPoolStrategy:
             questions: 题目全集。
             correctness: question_id -> 基线是否答对。
             config: 池构建配置（使用 train_ratio, task_types, seed,
-                baseline_run_id, test_questions_dir）。
+                baseline_run_id, test_questions_dir, batch_correct_ratio）。
+            db_path: harness.db 路径，用于查询 benchmark 历史推理记录
+                （maintenance 补入时需要）。
 
         返回:
             冻结的 Pools（diagnosis=train, validation=val,
@@ -583,6 +589,15 @@ class PerCategoryPoolStrategy:
         for q in filtered:
             groups[q.task_type].append(q)
 
+        # Phase 2.5: 正确率检查 + maintenance 补入
+        if config.batch_correct_ratio is not None:
+            self._check_and_supplement_maintenance(
+                groups,
+                correctness,
+                config,
+                db_path,
+            )
+
         # Phase 3: 每组分层划分
         all_train: list[GeneratedQuestion] = []
         all_val: list[GeneratedQuestion] = []
@@ -624,6 +639,124 @@ class PerCategoryPoolStrategy:
             },
         )
 
+    def _check_and_supplement_maintenance(
+        self,
+        groups: dict[str, list[GeneratedQuestion]],
+        correctness: dict[str, bool],
+        config: PoolConfig,
+        db_path: Path | None,
+    ) -> None:
+        """按 task_type 检查正确率，过高警告，过低则从 benchmark 补入正确题。
+
+        修改 groups 和 correctness（原地更新）。
+
+        参数:
+            groups: task_type -> 题目列表映射（原地追加补入题）。
+            correctness: question_id -> 是否正确映射（原地追加补入题标记）。
+            config: 含 batch_correct_ratio 和 test_questions_dir。
+            db_path: harness.db 路径，用于查询 benchmark 历史推理记录。
+        """
+        import sqlite3
+
+        r = config.batch_correct_ratio
+        assert r is not None  # 调用方已保证
+
+        # Phase 2.5a: 正确率检查（不依赖 test_questions_dir）
+        for task_type, group in groups.items():
+            c = sum(1 for q in group if correctness.get(q.question_id, False))
+            n = len(group)
+            ratio = c / n if n > 0 else 0.0
+            if ratio > 1 - r:
+                logger.warning(
+                    "类别 {} 正确率 {:.1%} 过高（阈值 {:.1%}），出题可能太简单",
+                    task_type,
+                    ratio,
+                    1 - r,
+                )
+
+        # Phase 2.5b: maintenance 补入（需要 test_questions_dir）
+        if config.test_questions_dir is None:
+            return
+
+        from app.question_gen import load_benchmark
+
+        bench_questions = load_benchmark(config.test_questions_dir)
+
+        # 查询 DB 中 benchmark 题的历史正确性
+        bench_correctness: dict[str, bool] = {}
+        if db_path is not None and db_path.exists():
+            conn = sqlite3.connect(str(db_path))
+            bench_qids = [q.question_id for q in bench_questions]
+            if bench_qids:
+                placeholders = ",".join("?" for _ in bench_qids)
+                rows = conn.execute(
+                    f"SELECT question_id, prediction, answer FROM predictions "  # noqa: S608
+                    f"WHERE question_id IN ({placeholders}) "
+                    f"ORDER BY timestamp DESC",
+                    bench_qids,
+                ).fetchall()
+                for qid, pred, ans in rows:
+                    if qid not in bench_correctness:
+                        bench_correctness[qid] = pred == ans
+                conn.close()
+
+        # 按 task_type 索引 benchmark 题
+        bench_by_type: dict[str, list[GeneratedQuestion]] = defaultdict(list)
+        for q in bench_questions:
+            bench_by_type[q.task_type].append(q)
+
+        for task_type, group in groups.items():
+            c = sum(1 for q in group if correctness.get(q.question_id, False))
+            w = len(group) - c
+            n = len(group)
+            ratio = c / n if n > 0 else 0.0
+
+            if ratio >= r:
+                continue
+
+            k = math.ceil((r * w - (1 - r) * c) / (1 - r))
+
+            existing_ids = {q.question_id for q in group}
+            candidates = [
+                q
+                for q in bench_by_type.get(task_type, [])
+                if bench_correctness.get(q.question_id, False) and q.question_id not in existing_ids
+            ]
+
+            if not candidates:
+                logger.warning(
+                    "类别 {} 需补入 {} 道正确题，但 benchmark 中无可用候选",
+                    task_type,
+                    k,
+                )
+                continue
+
+            actual = min(k, len(candidates))
+            for q in candidates[:actual]:
+                supplemented = GeneratedQuestion(
+                    question_id=q.question_id,
+                    video_id=q.video_id,
+                    task_type=q.task_type,
+                    question=q.question,
+                    options=q.options,
+                    answer=q.answer,
+                    source_nodes=q.source_nodes,
+                    difficulty=q.difficulty,
+                    family="VME_MAINTENANCE",
+                    skill_target=q.skill_target,
+                    difficulty_steps=q.difficulty_steps,
+                )
+                group.append(supplemented)
+                correctness[supplemented.question_id] = True
+
+            logger.info(
+                "类别 {} 正确率 {:.1%} < {:.1%}，从 benchmark 补入 {} 道 maintenance 正确题",
+                task_type,
+                ratio,
+                r,
+                actual,
+            )
+
     def _split_one_category(
         self,
         questions: list[GeneratedQuestion],