perf: dedup holdout four-way eval (baseline derive, best_hard memo)

app/harness/runner.py

@@ -120,6 +120,9 @@ class _TrainState:
     gate_epoch_observed: bool = False
     probations: dict[str, Probation] = field(default_factory=dict)
     gate_cooldown: dict[str, int] = field(default_factory=dict)
+    # 进程内 holdout 去重备忘录 (skills_v, prompts_v) -> test 评估结果；不进 checkpoint，
+    # resume 后清空（重评一次是可接受代价，换取零 schema 变更）。
+    holdout_memo: dict[tuple[str, str], InferenceResult] = field(default_factory=dict)
 
 
 # ---------------------------------------------------------------------------
@@ -2006,11 +2009,18 @@ class Runner:
     ) -> None:
         """四向 held-out：baseline/best_hard/final/best_mixed 各在 test 池评估。
 
+        去重（进程内备忘录 state.holdout_memo，不改 schema）：
+        - baseline：不跑推理，从基线 predictions 推导 test 结果（0 推理），存 memo 跨 epoch 复用。
+        - final：真评 test，存 memo[(final_sv,final_pv)]。
+        - best_hard：其版本已在 memo（== final 或往轮已评）则引用，否则真评并存 memo。
+        - best_mixed：赢家必是 best_hard 或 final 之一，其结果已在 memo，直接引用（0 推理）。
+
         test 池仅观测落库，绝不进 gate/best/early-stop/调参。
         """
         best_mixed = await self._pick_mixed_best(
             epoch, pools, state, eval_skills_version, eval_prompts_version
         )
+        memo = state.holdout_memo
         versions: dict[str, tuple[str, str] | None] = {
             "baseline": (state.baseline_skills_version, state.baseline_prompts_version),
             "best_hard": (state.best_skills_version, state.best_prompts_version),
@@ -2022,9 +2032,14 @@ class Runner:
                 continue
             sv, pv = version
             run_id = f"{self._config.run_id}_holdout_{version_kind}_e{epoch}"
-            res = await self._eval_version_on_pool(
-                sv, pv, pools.test, run_id, context=f"held-out {version_kind}"
-            )
+            if version not in memo:
+                if version_kind == "baseline":
+                    memo[version] = self._derive_baseline_test_result(pools, run_id)
+                else:
+                    memo[version] = await self._eval_version_on_pool(
+                        sv, pv, pools.test, run_id, context=f"held-out {version_kind}"
+                    )
+            res = memo[version]
             soft = await self._try_soft_score(run_id, pools.test)
             mixed = None if soft is None else 0.5 * res.accuracy + 0.5 * soft
             write_holdout_eval(
@@ -2038,6 +2053,33 @@ class Runner:
                 per_task_type_json=json.dumps(res.per_task_type, ensure_ascii=False),
             )
 
+    def _derive_baseline_test_result(self, pools: Pools, run_id: str) -> InferenceResult:
+        """从基线 run 的 predictions 推导 test 池评估结果（0 推理）。
+
+        基线 run（pools.baseline_run_id）已对全题库推理并落库，test 题在其中；此处
+        按 test 题回读基线预测、经 _aggregate_results 折叠为 unit 级 InferenceResult，
+        避免重复推理基线版本（同版本不重采样）。
+
+        参数:
+            pools: 冻结三池（提供 test 与 baseline_run_id）。
+            run_id: 本次 holdout baseline 向的 run_id（仅用作结果标识）。
+        返回:
+            unit 级 InferenceResult（accuracy / per_task_type 与真评同口径）。
+        """
+        from app.harness.inference import _aggregate_results
+        from app.harness.log import HarnessLog
+
+        qids = [q.question_id for q in pools.test]
+        with HarnessLog(
+            str(self._paths.db_path), pools.baseline_run_id, register_run=False
+        ) as log:
+            placeholders = ", ".join(["?"] * len(qids))
+            rows = log.query(
+                f"SELECT * FROM predictions WHERE run_id=? AND question_id IN ({placeholders})",
+                (pools.baseline_run_id, *qids),
+            )
+        return _aggregate_results(rows, pools.test, run_id)
+
     async def _pick_mixed_best(
         self,
         epoch: int,