feat(harness): correctness 三口径 + gate 块按 unit 跑
进化引擎与 gate e-process 从 question_id 口径迁至 unit_id 口径，AR pair 双向 AND 折叠为单元、不被 P/Q 单题计分污染；逐题 predictions 仅作溯源。 - question_units: 新增 unit_correctness_view(units, per_q)->dict[unit_id,bool] 作为逐题→单元折叠的唯一入口（复用 unit_correctness）。 - core/evolution/validate: pair_block/compute_accuracy 参数改 unit_ids、 分母按单元数（键即 unit_id）。 - app/harness/validate（gate 实际执行路径）：阶梯题序聚合为单元并保持信息 阶梯序（_ladder_units），gate 块按单元切分（AR pair 整锁不跨块拆）； baseline_cache 键含 unit_id、存单元级对错；候选逐题读回后折叠成单元视图； n_used/W/L/四象限/准确率均按单元计；证据行按 unit 口径，candidate_correctness 独立保留逐题对错供 runner 二轨合并。 - runner: probation 结算按 unit 折叠计 W/L（_probation_unit_flips）；quadrant 四象限 id 承载 unit_id。 核心算法保真 #5（信息阶梯 e-process）：本次仅迁移 correctness 口径，不改冷启动 2:1 / gamma-EMA / 反泄漏算法本身（gate_ladder 迁移见 Task 8）。
This commit is contained in:
+63
-21
@@ -42,6 +42,7 @@ from app.harness.observation import (
|
||||
write_shadow_gate,
|
||||
write_step_report,
|
||||
)
|
||||
from app.harness.question_units import build_units, unit_correctness_view
|
||||
from app.harness.store import advance_version
|
||||
from app.harness.validate import Probation, ValidationOutcome
|
||||
from app.harness.workspace import (
|
||||
@@ -61,6 +62,7 @@ from core.evolution import (
|
||||
RejectedEdit,
|
||||
edit_budget_at,
|
||||
momentum_inner,
|
||||
pair_block,
|
||||
probation_verdict,
|
||||
replace_momentum,
|
||||
resolve_skill_file,
|
||||
@@ -72,11 +74,12 @@ if TYPE_CHECKING:
|
||||
from app.harness.pools import Pools
|
||||
from core.evolution.types import (
|
||||
EvolutionRecord,
|
||||
PairResult,
|
||||
SystemCasePack,
|
||||
ToolCasePack,
|
||||
)
|
||||
from core.protocols import LLMProvider, TelemetryRecorder, VLMProvider
|
||||
from core.types import GeneratedQuestion
|
||||
from core.types import GeneratedQuestion, QuestionUnit
|
||||
|
||||
|
||||
class _InterruptError(RuntimeError):
|
||||
@@ -267,15 +270,57 @@ def _compute_total_steps(pools: Pools, correctness: dict[str, bool], config: Run
|
||||
return config.epochs * steps_per_epoch
|
||||
|
||||
|
||||
def _probation_unit_flips(
|
||||
val_units: list[QuestionUnit],
|
||||
snapshot: dict[str, bool],
|
||||
rows: dict[str, dict[str, Any]],
|
||||
task_type: str,
|
||||
eval_run_id: str,
|
||||
) -> PairResult:
|
||||
"""按 unit 折叠锚快照与当前重跑对错，返回单元级翻转统计（W/L）。
|
||||
|
||||
锚快照（开账时逐题对错）与当前全 val 重跑逐题对错各经 unit_correctness_view
|
||||
折叠成单元视图，再走 pair_block 计单元翻转（AR pair 双向 AND，不被 P/Q 单题
|
||||
计分污染，核心算法保真 #5）。
|
||||
|
||||
参数:
|
||||
val_units: 该题型的 val 单元列表。
|
||||
snapshot: 开账时逐题对错快照（question_id -> bool）。
|
||||
rows: 当前全 val 重跑逐题预测行（question_id -> 规范化行）。
|
||||
task_type: 题型（错误信息用）。
|
||||
eval_run_id: 全 val 重跑 run_id（错误信息用）。
|
||||
|
||||
返回:
|
||||
PairResult（单元级 W/L 与 observed）。
|
||||
|
||||
异常:
|
||||
RuntimeError: 重跑缺某 val 题的预测行。
|
||||
"""
|
||||
cur_per_q: dict[str, bool] = {}
|
||||
for q in (q for u in val_units for q in u.questions):
|
||||
row = rows.get(q.question_id)
|
||||
if row is None:
|
||||
raise RuntimeError(
|
||||
f"probation 结算缺预测行: {task_type}/{q.question_id}（run={eval_run_id}）"
|
||||
)
|
||||
cur_per_q[q.question_id] = row["_correct"]
|
||||
snap_units = unit_correctness_view(val_units, snapshot)
|
||||
cur_units = unit_correctness_view(val_units, cur_per_q)
|
||||
return pair_block(snap_units, cur_units, [u.unit_id for u in val_units])
|
||||
|
||||
|
||||
def _outcome_to_quadrant_pairs(task_type: str, outcome: ValidationOutcome) -> list[dict]:
|
||||
"""把 ValidationOutcome 的四象限拍平为逐题 pair（供 quadrant_pair 表落库观测）。
|
||||
"""把 ValidationOutcome 的四象限拍平为单元 pair（供 quadrant_pair 表落库观测）。
|
||||
|
||||
四象限 id 为 **unit_id 口径**（single 即 question_id、AR pair 为 pair_id），
|
||||
与 gate e-process 同粒度；question_id 字段承载 unit_id。
|
||||
|
||||
参数:
|
||||
task_type: 该批 gate 的任务类型。
|
||||
outcome: 局部验证决策结果。
|
||||
|
||||
返回:
|
||||
每条含 question_id/task_type/prev_correct/curr_correct/category 的 dict 列表。
|
||||
每条含 question_id(=unit_id)/task_type/prev_correct/curr_correct/category 的 dict。
|
||||
"""
|
||||
from app.harness.momentum import (
|
||||
IMPROVED,
|
||||
@@ -1294,7 +1339,7 @@ class Runner:
|
||||
self._writeback_val_correctness(eval_r.run_id, pools, state)
|
||||
|
||||
# Phase 4: probation 结算
|
||||
self._settle_probations(eval_r.run_id, state)
|
||||
self._settle_probations(eval_r.run_id, pools, state)
|
||||
|
||||
# Phase 5: best argmax
|
||||
self._maybe_promote_best(
|
||||
@@ -1382,15 +1427,20 @@ class Runner:
|
||||
# 慢更新内部方法
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
def _settle_probations(self, eval_run_id: str, state: _TrainState) -> None:
|
||||
"""epoch 末试用期一次性结算：全 val 重跑逐题结果与锚快照配对。
|
||||
def _settle_probations(self, eval_run_id: str, pools: Pools, state: _TrainState) -> None:
|
||||
"""epoch 末试用期一次性结算：全 val 重跑结果按 unit 与锚快照配对。
|
||||
|
||||
W/L 按 **unit 口径** 统计（AR pair 双向 AND 折叠，不被 P/Q 单题计分污染，
|
||||
核心算法保真 #5）。锚快照与当前重跑均先经 unit_correctness_view 折叠成单元
|
||||
视图再走 pair_block 计翻转。逐题 predictions 仍逐题落库溯源。
|
||||
|
||||
参数:
|
||||
eval_run_id: 本 epoch 全 val 重跑（R）的 run_id。
|
||||
pools: 冻结三池（按 task_type 取 val 子集重建单元）。
|
||||
state: 训练状态（probations 结算后清空）。
|
||||
|
||||
异常:
|
||||
RuntimeError: 重跑缺某快照题的预测行。
|
||||
RuntimeError: 重跑缺某 val 题的预测行。
|
||||
"""
|
||||
if not state.probations:
|
||||
return
|
||||
@@ -1410,20 +1460,12 @@ class Runner:
|
||||
)
|
||||
for task_type in sorted(state.probations):
|
||||
probation = state.probations[task_type]
|
||||
w = l = 0 # noqa: E741
|
||||
for qid, snap_correct in probation.correctness_snapshot.items():
|
||||
row = rows.get(qid)
|
||||
if row is None:
|
||||
raise RuntimeError(
|
||||
f"probation 结算缺预测行: {task_type}/{qid}（run={eval_run_id}）"
|
||||
)
|
||||
cur = row["_correct"]
|
||||
if not snap_correct and cur:
|
||||
w += 1
|
||||
elif snap_correct and not cur:
|
||||
l += 1 # noqa: E741
|
||||
verdict = probation_verdict(w, l, params=params)
|
||||
logger.info("probation 结算[{}]: W={} L={} → {}", task_type, w, l, verdict)
|
||||
val_units = build_units([q for q in pools.validation if q.task_type == task_type])
|
||||
flips = _probation_unit_flips(
|
||||
val_units, probation.correctness_snapshot, rows, task_type, eval_run_id
|
||||
)
|
||||
verdict = probation_verdict(flips.w, flips.l, params=params)
|
||||
logger.info("probation 结算[{}]: W={} L={} → {}", task_type, flips.w, flips.l, verdict)
|
||||
if verdict == "rollback":
|
||||
self._rollback_probation(probation, state)
|
||||
state.probations.clear()
|
||||
|
||||
Reference in New Issue
Block a user