2704 lines
108 KiB
Python
2704 lines
108 KiB
Python
"""实验运行器（瘦编排器），对标 PyTorch Trainer。
|
||
|
||
三级嵌套（epoch → step → per-skill）训练循环 + 慢更新十步序 + 断点续训。
|
||
算法保真 #13：训练循环编排从 TRM4 runner.py（2273 行）迁移，逻辑不可简化。
|
||
|
||
关键重构（TRM4 → TRM5）：
|
||
- sync → async（await run_inference / run_diagnosis / evolve_* / validate_*）
|
||
- LLMClient.from_env → 注入 LLMProvider（self._llm / self._evolve_llm）
|
||
- 直接 DB/文件操作 → 通过模块函数（workspace / store / log / observation）
|
||
- 瘦身 2273 → ~500 行（推理/诊断/进化/验证全委托模块函数）
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import math
|
||
import random
|
||
import shutil
|
||
import sqlite3
|
||
import tempfile
|
||
from collections import Counter
|
||
from dataclasses import dataclass, field, replace
|
||
from pathlib import Path
|
||
from typing import TYPE_CHECKING, Any
|
||
|
||
from loguru import logger
|
||
|
||
from app.harness.batching import build_batches
|
||
from app.harness.checkpoint import (
|
||
check_fingerprint,
|
||
deserialize_state_fields,
|
||
load_checkpoint,
|
||
write_checkpoint,
|
||
)
|
||
from app.harness.config import RunConfig # noqa: TC001 — 运行时 _compute_total_steps 使用
|
||
from app.harness.gate_ladder import BaselineCache, GatePools, build_or_load_gate_pools
|
||
from app.harness.log import HarnessLog
|
||
from app.harness.observation import (
|
||
write_dual_metric,
|
||
write_epoch_report,
|
||
write_gate_evidence,
|
||
write_holdout_eval,
|
||
write_quadrant_pairs,
|
||
write_shadow_gate,
|
||
write_step_report,
|
||
)
|
||
from app.harness.question_units import build_units, unit_correctness_view
|
||
from app.harness.store import advance_version
|
||
from app.harness.validate import (
|
||
GateSpec,
|
||
Probation,
|
||
ValidationOutcome,
|
||
_ladder_units,
|
||
validate_skills_concurrent,
|
||
)
|
||
from app.harness.workspace import (
|
||
ResolvedPaths,
|
||
archive_workspace,
|
||
init_workspace,
|
||
init_workspace_from_seed,
|
||
load_manifest,
|
||
read_best,
|
||
resolve_paths,
|
||
update_best,
|
||
update_manifest,
|
||
)
|
||
from core.evolution import (
|
||
DiagnosisResult,
|
||
GateParams,
|
||
RejectedEdit,
|
||
edit_budget_at,
|
||
momentum_inner,
|
||
pair_block,
|
||
probation_verdict,
|
||
replace_momentum,
|
||
resolve_skill_file,
|
||
)
|
||
from core.evolution.diagnose import merge_system_packs, merge_tool_packs
|
||
|
||
if TYPE_CHECKING:
|
||
from app.harness.inference import InferenceResult
|
||
from app.harness.pools import Pools
|
||
from core.evolution.types import (
|
||
EvolutionRecord,
|
||
PairResult,
|
||
SystemCasePack,
|
||
ToolCasePack,
|
||
)
|
||
from core.protocols import LLMProvider, TelemetryRecorder, VLMProvider
|
||
from core.types import GeneratedQuestion, QuestionUnit
|
||
|
||
|
||
class _InterruptError(RuntimeError):
|
||
"""测试用中断注入信号：_run_step 末尾可选抛出以模拟进程中断。"""
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _TrainState: 19 个可变字段
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass
|
||
class _TrainState:
|
||
"""一次 train() 的跨 step 可变状态（训练循环的"权重/缓冲"）。
|
||
|
||
字段说明见 TRM4 同名 dataclass（完整保留 19 字段语义）。
|
||
TRM5 移除 evolve_client（改走构造注入），其余 18 字段 + gate_epoch_observed 不变。
|
||
"""
|
||
|
||
correctness: dict[str, bool]
|
||
gate_pools: GatePools
|
||
baseline_cache: BaselineCache
|
||
eval_prev_acc: float
|
||
eval_prev_run_id: str
|
||
best_val_acc: float
|
||
best_skills_version: str
|
||
best_prompts_version: str
|
||
baseline_skills_version: str = ""
|
||
baseline_prompts_version: str = ""
|
||
rejected_buffer: dict[str, list[RejectedEdit]] = field(default_factory=dict)
|
||
system_packs: list[SystemCasePack] = field(default_factory=list)
|
||
tool_packs: list[ToolCasePack] = field(default_factory=list)
|
||
global_step: int = 0
|
||
changed_task_types_this_epoch: set[str] = field(default_factory=set)
|
||
epoch_start_skills: dict[str, str] = field(default_factory=dict)
|
||
epochs_since_best_improved: int = 0
|
||
gate_epoch_observed: bool = False
|
||
probations: dict[str, Probation] = field(default_factory=dict)
|
||
gate_cooldown: dict[str, int] = field(default_factory=dict)
|
||
# 进程内 holdout 去重备忘录 (skills_v, prompts_v) -> test 评估结果；不进 checkpoint，
|
||
# resume 后清空（重评一次是可接受代价，换取零 schema 变更）。
|
||
holdout_memo: dict[tuple[str, str], InferenceResult] = field(default_factory=dict)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 纯函数辅助（不依赖 self）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def resume_plan(epoch: int, phase: str, step_completed: int) -> dict:
|
||
"""据 checkpoint 进度算续跑计划（纯函数，便于单测）。
|
||
|
||
参数:
|
||
epoch: checkpoint 落库时的 epoch 序号。
|
||
phase: "in_epoch" 或 "epoch_done"。
|
||
step_completed: 该 epoch 内最后完整完成的 step 序号。
|
||
|
||
返回:
|
||
{"first_epoch": int, "resume_epoch": int | None, "resume_step_from": int}。
|
||
"""
|
||
if phase == "epoch_done":
|
||
return {"first_epoch": epoch + 1, "resume_epoch": None, "resume_step_from": 0}
|
||
return {
|
||
"first_epoch": epoch,
|
||
"resume_epoch": epoch,
|
||
"resume_step_from": step_completed + 1,
|
||
}
|
||
|
||
|
||
def _guard_infra_failures(result: InferenceResult, context: str) -> None:
|
||
"""基础设施失败护栏：stop_reason="error" 占比 > 10% 即硬终止。
|
||
|
||
参数:
|
||
result: 推理聚合结果。
|
||
context: 出错时报错的推理路径名（仅诊断用）。
|
||
|
||
异常:
|
||
RuntimeError: error 占比 > 10%。
|
||
"""
|
||
error_rate = result.stop_reason_counts.get("error", 0) / max(result.total, 1)
|
||
if error_rate > 0.1:
|
||
raise RuntimeError(
|
||
f"{context} 推理基础设施失败率过高 {error_rate:.0%}（stop_reason=error），中止本轮"
|
||
)
|
||
|
||
|
||
def _apply_batch_correctness(
|
||
correctness: dict[str, bool],
|
||
log: Any,
|
||
run_id: str,
|
||
batch: list[GeneratedQuestion],
|
||
) -> None:
|
||
"""从该 run 的 predictions 读 batch 各题新对错，就地增量更新 correctness。
|
||
|
||
参数:
|
||
correctness: question_id -> 是否答对，就地更新。
|
||
log: HarnessLog 实例。
|
||
run_id: rollout 的 run_id。
|
||
batch: 本 step 的题目列表。
|
||
|
||
异常:
|
||
RuntimeError: rollout 不完整（缺预测行）。
|
||
"""
|
||
from app.harness.validate import _load_run_rows
|
||
|
||
rows = _load_run_rows(log, run_id)
|
||
missing = [q.question_id for q in batch if q.question_id not in rows]
|
||
if missing:
|
||
raise RuntimeError(
|
||
f"rollout 不完整：run_id={run_id} 缺 {len(missing)} 道题预测行 {missing}，中止本步"
|
||
)
|
||
for q in batch:
|
||
correctness[q.question_id] = rows[q.question_id]["_correct"]
|
||
|
||
|
||
def _accumulate_slow_packs(diagnosis: DiagnosisResult, state: _TrainState) -> None:
|
||
"""把本 step 诊断的 system/tool 案例包只累加不更新，留给 epoch 末慢更新消费。"""
|
||
if diagnosis.system_case_pack is not None:
|
||
state.system_packs.append(diagnosis.system_case_pack)
|
||
state.tool_packs.extend(diagnosis.tool_case_packs.values())
|
||
|
||
|
||
def _batch_unit_ids(batch: list[GeneratedQuestion]) -> list[str]:
|
||
"""把一个 batch 的扁平题目折叠为 unit_id 序列（孪生对成员去重为单个 unit_id）。
|
||
|
||
checkpoint 存 unit_id 序列而非逐题 question_id：断点续跑恢复时按完整 unit 展开，
|
||
保证孪生对整体重建、绝不被劈开（核心算法保真 #3 断点续跑）。
|
||
|
||
参数:
|
||
batch: 一个 mini-batch 的扁平题目列表（pair 两成员相邻）。
|
||
|
||
返回:
|
||
unit_id 列表，按题目在 batch 中的首次出现顺序去重；single 的 unit_id 即
|
||
question_id，故纯非 AR 输入下与旧逐题 question_id 序列逐字节一致。
|
||
|
||
关键实现:
|
||
用 dict 保序去重（pair 两成员共享 unit_id，仅记一次），无需额外集合。
|
||
"""
|
||
ordered: dict[str, None] = {}
|
||
for q in batch:
|
||
ordered[q.unit_id] = None
|
||
return list(ordered)
|
||
|
||
|
||
def _batch_from_ids(pools: Pools, unit_ids: list[str]) -> list[GeneratedQuestion]:
|
||
"""按 unit_id 序列从诊断池重建一个 batch，按完整 unit 展开成题目列表。
|
||
|
||
与 _batch_unit_ids 对称：恢复时以完整 unit 为单位展开（pair 两成员同进同出），
|
||
断点续跑后孪生对绝不被拆开（核心算法保真 #3）。
|
||
|
||
参数:
|
||
pools: 三池容器。
|
||
unit_ids: 一个 batch 的 unit_id 序列（checkpoint 存的粒度）。
|
||
|
||
返回:
|
||
按 unit_ids 顺序展开的 GeneratedQuestion 列表；每个 unit_id 展开为其全部
|
||
成员题（single 1 题、pair 2 题），顺序与原 batch 一致。
|
||
|
||
关键实现:
|
||
直接以 units_by_id[uid] 取值，unit_id 缺失触发 KeyError（P5 防静默兜底），
|
||
强制 checkpoint 与当前诊断池一致；纯非 AR 下 unit_id==question_id、单元即
|
||
单题，与旧逐题重建逐字节一致。
|
||
"""
|
||
units_by_id = {u.unit_id: u for u in build_units(pools.diagnosis)}
|
||
return [q for uid in unit_ids for q in units_by_id[uid].questions]
|
||
|
||
|
||
def _sample_momentum_candidates(
|
||
pool: list[GeneratedQuestion],
|
||
allowed_task_types: set[str],
|
||
momentum_samples: int,
|
||
epoch: int,
|
||
) -> list[GeneratedQuestion]:
|
||
"""为 momentum 从诊断池按题型过滤后做确定性抽样（逐题粒度）。
|
||
|
||
设计偏差（Phase 1 显式记录）：momentum 采样在逐题粒度进行、不折叠 QuestionUnit，
|
||
故仅保证「纯非 AR 题库」的抽样序列与引入 QuestionUnit 前逐字节一致；混格题库下
|
||
孪生对可能被半采样、且候选集长度/顺序随 AR 成员增减而漂移，**Phase 1 不保证混格
|
||
momentum 的 byte-identical**（属可接受偏差，纯非 AR 必须不漂）。
|
||
|
||
参数:
|
||
pool: 诊断池扁平题目列表。
|
||
allowed_task_types: 允许参与的题型集合。
|
||
momentum_samples: 目标采样数上限。
|
||
epoch: 采样种子（同 epoch 可复现）。
|
||
|
||
返回:
|
||
采样到的题目列表；候选不足则全取，候选为空返回空列表。
|
||
"""
|
||
candidates = [q for q in pool if q.task_type in allowed_task_types]
|
||
n = min(momentum_samples, len(candidates))
|
||
if n <= 0:
|
||
return []
|
||
return random.Random(epoch).sample(candidates, n)
|
||
|
||
|
||
def _snapshot_current_skills(skills_dir: Path) -> dict[str, str]:
|
||
"""快照当前 skills 版本目录下各 skill 文件的正文（文件名 -> 全文）。
|
||
|
||
参数:
|
||
skills_dir: 当前 skills 版本目录。
|
||
|
||
返回:
|
||
{文件名: 全文}，作 momentum 的上一版基准。
|
||
"""
|
||
snapshot: dict[str, str] = {}
|
||
for path in sorted(skills_dir.glob("*.md")):
|
||
snapshot[path.name] = path.read_text(encoding="utf-8")
|
||
return snapshot
|
||
|
||
|
||
def _filter_untrainable_types(
|
||
pools: Pools,
|
||
task_types: list[str] | None,
|
||
eval_min_per_class: int,
|
||
trainable_min_units: int,
|
||
) -> tuple[Pools, list[str] | None]:
|
||
"""剔除不可训练题型（val 单元<eval_min_per_class 或 diag+val 单元<trainable_min_units）。
|
||
|
||
计数以**单元（unit）**为原子：AR pair 孪生对折叠计 1 个单元（等于 gate 阶梯该类
|
||
候选数），非按题目计数——否则 pair 题型会以 2 倍题目数误通过阈值。test 池不过滤
|
||
（继续报告全题型准确率）。在 gate 建立前调用，避免样本不足的微型题型进入信息量
|
||
阶梯导致门控崩溃。
|
||
|
||
参数:
|
||
pools: 冻结三池。
|
||
task_types: 显式题型子集（None 表示全部），过滤后按 keep 收窄。
|
||
eval_min_per_class: 验证池每类保底单元数下限。
|
||
trainable_min_units: 每类可训练所需最小 diag+val 单元数。
|
||
|
||
返回:
|
||
过滤后的 (pools, task_types)：pools.diagnosis/validation 仅保留 keep 题型，
|
||
test 原样；task_types 收窄为 keep（原 None 时返回 sorted(keep)）。
|
||
|
||
异常:
|
||
RuntimeError: 过滤后无任何可训练题型（切分/阈值需调整，fail-fast 不空转训练）。
|
||
"""
|
||
diag_by_type = Counter(u.task_type for u in build_units(pools.diagnosis))
|
||
val_by_type = Counter(u.task_type for u in build_units(pools.validation))
|
||
# 先按调用方显式 task_types 收窄候选集：未请求的题型（哪怕可训练）不得进入
|
||
# keep，否则冻结全局 pools 后 batch/diagnosis 会训练非请求题型，而 gate 只覆盖
|
||
# 请求题型 → 静默语义偏差（I-4）。task_types=None 表示全部题型皆为候选。
|
||
candidates = set(diag_by_type) | set(val_by_type)
|
||
if task_types is not None:
|
||
candidates &= set(task_types)
|
||
keep: set[str] = set()
|
||
dropped: list[tuple[str, str]] = []
|
||
for tt in candidates:
|
||
n_val = val_by_type.get(tt, 0)
|
||
n_units = diag_by_type.get(tt, 0) + n_val
|
||
if n_val < eval_min_per_class:
|
||
dropped.append((tt, f"val_units={n_val}<{eval_min_per_class}"))
|
||
elif n_units < trainable_min_units:
|
||
dropped.append((tt, f"units={n_units}<{trainable_min_units}"))
|
||
else:
|
||
keep.add(tt)
|
||
for tt, why in sorted(dropped):
|
||
logger.warning("可训练性预检剔除题型 {}（{}）", tt, why)
|
||
if not keep:
|
||
detail = "；".join(f"{tt}（{why}）" for tt, why in sorted(dropped))
|
||
raise RuntimeError(
|
||
"可训练性预检剔除了全部题型，无题型满足 "
|
||
f"val_units>={eval_min_per_class} 且 units>={trainable_min_units}："
|
||
f"{detail}。请调整池切分或降低阈值。"
|
||
)
|
||
new_pools = replace(
|
||
pools,
|
||
diagnosis=[q for q in pools.diagnosis if q.task_type in keep],
|
||
validation=[q for q in pools.validation if q.task_type in keep],
|
||
)
|
||
new_types = [t for t in task_types if t in keep] if task_types is not None else sorted(keep)
|
||
return new_pools, new_types
|
||
|
||
|
||
def _should_early_stop(
|
||
workspace_dir: Path,
|
||
epoch: int,
|
||
state: _TrainState,
|
||
patience: int,
|
||
) -> bool:
|
||
"""epoch 粒度 early stop：本 epoch best 未刷新则计数 +1。
|
||
|
||
参数:
|
||
workspace_dir: workspace 目录（读 manifest best）。
|
||
epoch: 当前 epoch。
|
||
state: 训练状态（epochs_since_best_improved 就地更新）。
|
||
patience: early_stop_patience（连续无刷新的 epoch 数上限）。
|
||
|
||
返回:
|
||
是否触发 early stop。
|
||
"""
|
||
best = read_best(workspace_dir)
|
||
improved_this_epoch = best is not None and best.get("epoch") == epoch
|
||
if improved_this_epoch:
|
||
state.epochs_since_best_improved = 0
|
||
return False
|
||
state.epochs_since_best_improved += 1
|
||
return state.epochs_since_best_improved >= patience
|
||
|
||
|
||
def _compute_total_steps(pools: Pools, correctness: dict[str, bool], config: RunConfig) -> int:
|
||
"""退火地平线：用 build_batches 试切一轮拿 selected_count，再乘 epochs。"""
|
||
_, selected_count = build_batches(
|
||
pools.diagnosis,
|
||
correctness,
|
||
config.batch_size,
|
||
config.min_class_per_batch,
|
||
seed=1,
|
||
correct_ratio=config.batch_correct_ratio,
|
||
)
|
||
steps_per_epoch = max(1, math.ceil(selected_count / config.batch_size))
|
||
return config.epochs * steps_per_epoch
|
||
|
||
|
||
def _probation_unit_flips(
|
||
val_units: list[QuestionUnit],
|
||
snapshot: dict[str, bool],
|
||
rows: dict[str, dict[str, Any]],
|
||
task_type: str,
|
||
eval_run_id: str,
|
||
) -> PairResult:
|
||
"""按 unit 折叠锚快照与当前重跑对错，返回单元级翻转统计（W/L）。
|
||
|
||
锚快照（开账时逐题对错）与当前全 val 重跑逐题对错各经 unit_correctness_view
|
||
折叠成单元视图，再走 pair_block 计单元翻转（AR pair 双向 AND，不被 P/Q 单题
|
||
计分污染，核心算法保真 #5）。
|
||
|
||
参数:
|
||
val_units: 该题型的 val 单元列表。
|
||
snapshot: 开账时逐题对错快照（question_id -> bool）。
|
||
rows: 当前全 val 重跑逐题预测行（question_id -> 规范化行）。
|
||
task_type: 题型（错误信息用）。
|
||
eval_run_id: 全 val 重跑 run_id（错误信息用）。
|
||
|
||
返回:
|
||
PairResult（单元级 W/L 与 observed）。
|
||
|
||
异常:
|
||
RuntimeError: 重跑缺某 val 题的预测行。
|
||
"""
|
||
cur_per_q: dict[str, bool] = {}
|
||
for q in (q for u in val_units for q in u.questions):
|
||
row = rows.get(q.question_id)
|
||
if row is None:
|
||
raise RuntimeError(
|
||
f"probation 结算缺预测行: {task_type}/{q.question_id}（run={eval_run_id}）"
|
||
)
|
||
cur_per_q[q.question_id] = row["_correct"]
|
||
snap_units = unit_correctness_view(val_units, snapshot)
|
||
cur_units = unit_correctness_view(val_units, cur_per_q)
|
||
return pair_block(snap_units, cur_units, [u.unit_id for u in val_units])
|
||
|
||
|
||
def _outcome_to_quadrant_pairs(task_type: str, outcome: ValidationOutcome) -> list[dict]:
|
||
"""把 ValidationOutcome 的四象限拍平为单元 pair（供 quadrant_pair 表落库观测）。
|
||
|
||
四象限 id 为 **unit_id 口径**（single 即 question_id、AR pair 为 pair_id），
|
||
与 gate e-process 同粒度；question_id 字段承载 unit_id。
|
||
|
||
参数:
|
||
task_type: 该批 gate 的任务类型。
|
||
outcome: 局部验证决策结果。
|
||
|
||
返回:
|
||
每条含 question_id(=unit_id)/task_type/prev_correct/curr_correct/category 的 dict。
|
||
"""
|
||
from app.harness.momentum import (
|
||
IMPROVED,
|
||
PERSISTENT_FAIL,
|
||
REGRESSED,
|
||
STABLE_SUCCESS,
|
||
)
|
||
|
||
spec = [
|
||
(outcome.improvements, IMPROVED, False, True),
|
||
(outcome.regressions, REGRESSED, True, False),
|
||
(outcome.persistent_fails, PERSISTENT_FAIL, False, False),
|
||
(outcome.stable_successes, STABLE_SUCCESS, True, True),
|
||
]
|
||
pairs: list[dict] = []
|
||
for qids, category, prev_ok, curr_ok in spec:
|
||
for qid in qids:
|
||
pairs.append(
|
||
{
|
||
"question_id": qid,
|
||
"task_type": task_type,
|
||
"prev_correct": prev_ok,
|
||
"curr_correct": curr_ok,
|
||
"category": category,
|
||
}
|
||
)
|
||
return pairs
|
||
|
||
|
||
def _build_comparison_pairs(
|
||
sampled: list[GeneratedQuestion],
|
||
prev_rows: dict[str, dict],
|
||
curr_rows: dict[str, dict],
|
||
) -> list[dict]:
|
||
"""为采样好的诊断池题目构造 momentum 纵向对比对。"""
|
||
pairs: list[dict] = []
|
||
for q in sampled:
|
||
prev = prev_rows.get(q.question_id, {})
|
||
curr = curr_rows.get(q.question_id, {})
|
||
pairs.append(
|
||
{
|
||
"question": q.question,
|
||
"prev_prediction": prev.get("prediction", ""),
|
||
"curr_prediction": curr.get("prediction", ""),
|
||
"correct_prev": prev.get("_correct", False),
|
||
"correct_curr": curr.get("_correct", False),
|
||
}
|
||
)
|
||
return pairs
|
||
|
||
|
||
def _filter_applied_edits(edits: list[dict], reports: list[dict]) -> list[dict] | str:
|
||
"""按 apply_report 过滤出真正 applied 的 edit。
|
||
|
||
参数:
|
||
edits: EvolutionRecord.edits 列表。
|
||
reports: EvolutionRecord.apply_report 列表（与 edits 同序对齐）。
|
||
|
||
返回:
|
||
过滤后的 edit 列表；0 applied 时返回信息性消息字符串。
|
||
reports 为空时返回原 edits 不过滤。
|
||
"""
|
||
if not reports:
|
||
return edits
|
||
applied = [
|
||
edit
|
||
for edit, report in zip(edits, reports, strict=True)
|
||
if str(report.get("status", "")).startswith("applied")
|
||
]
|
||
if not applied:
|
||
return "上轮改法全部未成功应用(0 applied)，本条无已验证信息"
|
||
return applied
|
||
|
||
|
||
def _format_applied_edits(record: Any) -> str | None:
|
||
"""从 EvolutionRecord 中提取真正 applied 的 edit 并格式化为摘要。
|
||
|
||
参数:
|
||
record: EvolutionRecord（duck-typed，需含 edits / apply_report）。
|
||
|
||
返回:
|
||
已 applied edit 的格式化摘要；无 edit 或无 applied 时返回 None，
|
||
0 applied 时返回信息性消息（非 None）。
|
||
"""
|
||
rec_edits = getattr(record, "edits", []) or []
|
||
if not rec_edits:
|
||
return None
|
||
filtered = _filter_applied_edits(rec_edits, getattr(record, "apply_report", []) or [])
|
||
if isinstance(filtered, str):
|
||
return filtered
|
||
summary = "; ".join(
|
||
f"[{edit.get('op')}]{(edit.get('target') or edit.get('content', ''))[:40]}"
|
||
for edit in filtered
|
||
if isinstance(edit, dict)
|
||
)
|
||
return summary or None
|
||
|
||
|
||
def _fallback_summary(record: Any, outcome: Any) -> str:
|
||
"""从 suggestions 或跌幅信息构造兜底黑名单摘要。
|
||
|
||
参数:
|
||
record: EvolutionRecord（duck-typed，需含 suggestions）。
|
||
outcome: ValidationOutcome（duck-typed，需含 delta_hat）。
|
||
|
||
返回:
|
||
兜底摘要字符串。
|
||
"""
|
||
return "; ".join(s.get("change", "") for s in record.suggestions) or (
|
||
f"上轮对本文件改写被拒(delta {outcome.delta_hat:+.2f})，换方向"
|
||
)
|
||
|
||
|
||
def _write_skip_report(
|
||
workspace_dir: Path,
|
||
epoch: int,
|
||
step: int,
|
||
global_step: int,
|
||
task_type: str,
|
||
action: str,
|
||
baseline_acc: float,
|
||
budget: int,
|
||
rank_clip_triggered: bool = False,
|
||
) -> None:
|
||
"""为 cooldown / skipped 路径写 step_report（无 gate 证据）。
|
||
|
||
参数:
|
||
workspace_dir: 工作区目录。
|
||
epoch: 轮次。
|
||
step: epoch 内 step。
|
||
global_step: 全局步计数。
|
||
task_type: 任务类型。
|
||
action: "cooldown" 或 "skipped"。
|
||
baseline_acc: 当前类基线准确率。
|
||
budget: 编辑预算。
|
||
rank_clip_triggered: 是否触发 rank 裁剪（skipped 路径需要）。
|
||
"""
|
||
write_step_report(
|
||
workspace_dir,
|
||
epoch=epoch,
|
||
step=step,
|
||
global_step=global_step,
|
||
task_type=task_type,
|
||
gate_action=action,
|
||
candidate_acc=baseline_acc,
|
||
class_baseline_acc=baseline_acc,
|
||
edit_budget=budget,
|
||
rank_clip_triggered=rank_clip_triggered,
|
||
gate_w=None,
|
||
gate_l=None,
|
||
gate_e_value=None,
|
||
gate_n_used=None,
|
||
gate_stop_reason=None,
|
||
)
|
||
|
||
|
||
def _assert_disjoint_target_files(targets_by_type: dict[str, str]) -> None:
|
||
"""断言本 step 各题型进化目标文件互不相同(设计 v3 §1 fail-fast)。
|
||
|
||
题型并行进化 + 并行 gate 的前提是 skill 文件不相交;两题型 fallback 到
|
||
同一 default-strategy.md 时并行会互相覆盖候选与 accept,必须显式中止
|
||
而非静默串行(当前 12 题型均有专属文件,此断言防未来配置漂移)。
|
||
|
||
参数:
|
||
targets_by_type: {题型: 解析后 skill 文件名}。
|
||
|
||
返回:
|
||
无。
|
||
|
||
异常:
|
||
RuntimeError: 存在两个题型映射同一文件。
|
||
"""
|
||
seen: dict[str, str] = {}
|
||
for task_type, target in targets_by_type.items():
|
||
if target in seen:
|
||
raise RuntimeError(
|
||
f"题型 {seen[target]!r} 与 {task_type!r} 映射同一 skill 文件 {target!r},"
|
||
"并行进化/gate 不支持共享目标文件(设计 v3 §1)"
|
||
)
|
||
seen[target] = task_type
|
||
|
||
|
||
def _escape_sql_like(text: str) -> str:
|
||
"""转义 SQL LIKE 模式中的全部特殊字符(`\\`、`%`、`_`)为字面匹配。
|
||
|
||
参数:
|
||
text: 待作为 LIKE 前缀字面使用的原始字符串。
|
||
|
||
返回:
|
||
可安全拼入 `LIKE ? ESCAPE '\\'` 模式的转义串。
|
||
|
||
关键实现细节:
|
||
反斜杠必须最先转义,否则会二次转义后续替换产生的转义符。
|
||
"""
|
||
return text.replace("\\", "\\\\").replace("%", r"\%").replace("_", r"\_")
|
||
|
||
|
||
def _clear_step_rows(db_path: str, *, baseline_run_id: str, epoch: int, step: int) -> None:
|
||
"""清空一个 step 的全部旧行（rollout + gate 派生），保证崩溃重跑幂等。
|
||
|
||
修复前序潜伏 bug：旧实现只清 rollout run_id，gate 派生 run_id
|
||
（`{step_run_id}_gate_%`）从不清理，重跑会累积重复 predictions（HarnessLog
|
||
无主键去重），_load_run_rows 的 dict 覆盖使结果依赖 SELECT 顺序。
|
||
gate_evidence / quadrant_pair 以 (run_id, epoch, step) 过滤删除；
|
||
表不存在（首个 step）时跳过。step_report 为按文件名覆盖写的 JSON，天然幂等。
|
||
|
||
参数:
|
||
db_path: harness.db 路径。
|
||
baseline_run_id: 基线 run（gate_evidence/quadrant_pair 的 run_id 维度）。
|
||
epoch: 轮次（1-based）。
|
||
step: epoch 内 step 序号（0-based）。
|
||
|
||
返回:
|
||
无。
|
||
|
||
关键实现细节:
|
||
predictions/traces 的 gate 行按 LIKE 前缀删除，`\\`/`%`/`_` 三个 LIKE
|
||
特殊字符全部显式转义（ESCAPE）钉死字面匹配，避免 `..._s1` 误匹配
|
||
`..._s10` 类前缀陷阱，也防 run_id 含 `%`/`\\` 时通配误删他 run 行。
|
||
"""
|
||
from app.harness.inference import PREDICTIONS_SCHEMA, TRACES_SCHEMA
|
||
from app.harness.log import HarnessLog
|
||
|
||
step_run_id = f"{baseline_run_id}_e{epoch}_s{step}"
|
||
escaped = _escape_sql_like(step_run_id)
|
||
with HarnessLog(db_path, step_run_id, register_run=False) as log:
|
||
log.create_table("predictions", PREDICTIONS_SCHEMA)
|
||
log.create_table("traces", TRACES_SCHEMA)
|
||
for table in ("predictions", "traces"):
|
||
log.execute(f"DELETE FROM {table} WHERE run_id=?", (step_run_id,))
|
||
log.execute(
|
||
f"DELETE FROM {table} WHERE run_id LIKE ? ESCAPE '\\'",
|
||
(escaped + r"\_gate\_%",),
|
||
)
|
||
for table in ("gate_evidence", "quadrant_pair"):
|
||
exists = log.query(
|
||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table,)
|
||
)
|
||
if exists:
|
||
log.execute(
|
||
f"DELETE FROM {table} WHERE run_id=? AND epoch=? AND step=?",
|
||
(baseline_run_id, epoch, step),
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Runner 主类
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class Runner:
|
||
"""实验运行器（瘦编排器），通过 RunConfig 驱动训练/推理/诊断/评估等模式。
|
||
|
||
DI 纪律：self 只持注入依赖 + _paths。_TrainState 是 train() 内局部变量，
|
||
显式传参给模块函数。
|
||
|
||
参数:
|
||
config: 运行配置。
|
||
llm: 推理用 LLMProvider。
|
||
evolve_llm: 进化用 LLMProvider（thinking=True）。
|
||
vlm: VLMProvider。
|
||
telemetry: 遥测记录端口。
|
||
tool_dispatch_factory: 工具调度工厂（infer/eval/train 模式必传）。
|
||
prompt_builder_factory: prompt 构建工厂（infer/eval/train 模式必传）。
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
config: RunConfig,
|
||
*,
|
||
llm: LLMProvider,
|
||
evolve_llm: LLMProvider,
|
||
vlm: VLMProvider,
|
||
telemetry: TelemetryRecorder,
|
||
tool_dispatch_factory: Any | None = None,
|
||
prompt_builder_factory: Any | None = None,
|
||
) -> None:
|
||
self._config = config
|
||
self._llm = llm
|
||
self._evolve_llm = evolve_llm
|
||
self._vlm = vlm
|
||
self._telemetry = telemetry
|
||
self._tool_dispatch_factory = tool_dispatch_factory
|
||
self._prompt_builder_factory = prompt_builder_factory
|
||
|
||
# fail-fast: 需要推理的模式必须注入工厂
|
||
if config.mode in {"infer", "eval", "train"} and (
|
||
tool_dispatch_factory is None or prompt_builder_factory is None
|
||
):
|
||
raise ValueError(
|
||
f"mode={config.mode!r} 需要 tool_dispatch_factory 和 "
|
||
f"prompt_builder_factory，但收到 "
|
||
f"tool_dispatch_factory={tool_dispatch_factory!r}, "
|
||
f"prompt_builder_factory={prompt_builder_factory!r}"
|
||
)
|
||
|
||
self._ensure_workspace()
|
||
self._paths: ResolvedPaths = resolve_paths(config.workspace_dir)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# workspace 三态逻辑
|
||
# -----------------------------------------------------------------------
|
||
|
||
def _ensure_workspace(self) -> None:
|
||
"""train 模式按 --resume/--fresh 分三态；其余模式仅要求 ws 已存在并复用。
|
||
|
||
三态逻辑:
|
||
resume+fresh → ValueError
|
||
fresh+已有 → archive + init_from_seed
|
||
resume+无进度 → RuntimeError
|
||
无flag+已有 → SystemExit
|
||
"""
|
||
manifest = self._config.workspace_dir / "manifest.json"
|
||
has_progress = manifest.exists()
|
||
if self._config.mode != "train":
|
||
if not has_progress:
|
||
raise RuntimeError(
|
||
f"{self._config.mode} 模式要求 workspace 已存在: {self._config.workspace_dir}"
|
||
)
|
||
return
|
||
if self._config.resume and self._config.fresh:
|
||
raise ValueError("--resume 与 --fresh 互斥")
|
||
if self._config.fresh:
|
||
if has_progress:
|
||
logger.info(
|
||
"旧 workspace 已归档: {}",
|
||
archive_workspace(self._config.workspace_dir),
|
||
)
|
||
init_workspace_from_seed(
|
||
self._config.workspace_dir,
|
||
self._config.store_dir,
|
||
self._config.seed,
|
||
self._config.questions,
|
||
)
|
||
return
|
||
if self._config.resume:
|
||
if not has_progress:
|
||
raise RuntimeError("--resume 但 workspace 无已有进度")
|
||
return
|
||
if has_progress:
|
||
raise SystemExit("workspace 已有进度；用 --resume 续训 或 --fresh 归档重开")
|
||
init_workspace(
|
||
self._config.workspace_dir,
|
||
self._config.store_dir,
|
||
self._config.questions,
|
||
self._config.skills_version,
|
||
self._config.prompts_version,
|
||
)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# 公共入口：infer / eval / diagnose / promote
|
||
# -----------------------------------------------------------------------
|
||
|
||
async def infer(self, task_types: list[str] | None = None) -> InferenceResult:
|
||
"""执行单次推理（forward-only）。
|
||
|
||
参数:
|
||
task_types: 若非 None，只保留指定题型。
|
||
|
||
返回:
|
||
InferenceResult 冻结实例。
|
||
"""
|
||
from app.harness.inference import run_inference
|
||
from app.harness.log import HarnessLog
|
||
from app.question_gen import load_benchmark
|
||
|
||
# CLI --questions 优先于 manifest 中的 questions（infer 不改 manifest）
|
||
questions_dir = Path(self._config.store_dir) / "questions" / self._config.questions
|
||
questions = load_benchmark(questions_dir)
|
||
if task_types:
|
||
allowed = set(task_types)
|
||
questions = [q for q in questions if q.task_type in allowed]
|
||
if self._config.n_samples > 0:
|
||
questions = questions[: self._config.n_samples]
|
||
|
||
run_id = f"infer_{self._config.run_id}" if self._config.run_id else "infer_adhoc"
|
||
record_run_dir = self._record_run(run_id) # noqa: F841
|
||
|
||
logger.info(
|
||
"启动推理: {} 道题, concurrency={}, max_steps={}, skill_mode={}",
|
||
len(questions),
|
||
self._config.concurrency,
|
||
self._config.max_steps,
|
||
self._config.skill_mode,
|
||
)
|
||
|
||
with HarnessLog(str(self._paths.db_path), run_id) as log:
|
||
return await run_inference(
|
||
questions=questions,
|
||
llm=self._llm,
|
||
tool_dispatch_fn=self._make_tool_dispatch_fn(),
|
||
prompt_builder=self._make_prompt_builder(),
|
||
log=log,
|
||
run_id=run_id,
|
||
concurrency=self._config.concurrency,
|
||
max_steps=self._config.max_steps,
|
||
skill_mode=self._config.skill_mode,
|
||
)
|
||
|
||
async def eval(self, version: str) -> InferenceResult:
|
||
"""用指定 skills 版本跑完整题库，全量记录落 db + 版本回填。
|
||
|
||
参数:
|
||
version: skills 版本号。
|
||
|
||
返回:
|
||
InferenceResult。
|
||
"""
|
||
from datetime import UTC, datetime
|
||
|
||
from app.harness.inference import run_inference
|
||
from app.harness.log import HarnessLog
|
||
from app.question_gen import load_benchmark
|
||
|
||
cur = load_manifest(self._config.workspace_dir)["current"]
|
||
prompts_v = cur["prompts"].split("/")[-1]
|
||
skills_dir = self._paths.workspace_dir / "skills" / version
|
||
prompts_dir = self._paths.workspace_dir / "prompts" / prompts_v
|
||
if not skills_dir.is_dir():
|
||
raise FileNotFoundError(f"skills 版本目录不存在: {skills_dir}")
|
||
if not prompts_dir.is_dir():
|
||
raise FileNotFoundError(f"prompts 版本目录不存在: {prompts_dir}")
|
||
|
||
questions = load_benchmark(self._paths.questions_dir)
|
||
if self._config.n_samples > 0:
|
||
questions = questions[: self._config.n_samples]
|
||
|
||
ts = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
|
||
run_id = f"eval_{version}-{prompts_v}_{ts}"
|
||
self._record_run(run_id)
|
||
logger.info(
|
||
"eval: 版本 skills/{}+prompts/{} 跑 {} 题 (run_id={})",
|
||
version,
|
||
prompts_v,
|
||
len(questions),
|
||
run_id,
|
||
)
|
||
|
||
with HarnessLog(str(self._paths.db_path), run_id) as log:
|
||
result = await run_inference(
|
||
questions=questions,
|
||
llm=self._llm,
|
||
tool_dispatch_fn=self._make_tool_dispatch_fn(skills_dir=skills_dir),
|
||
prompt_builder=self._make_prompt_builder(
|
||
skills_dir=skills_dir, prompts_dir=prompts_dir
|
||
),
|
||
log=log,
|
||
run_id=run_id,
|
||
concurrency=self._config.concurrency,
|
||
max_steps=self._config.max_steps,
|
||
skill_mode=self._config.skill_mode,
|
||
)
|
||
|
||
# C4 回填
|
||
self._backfill_run_versions(run_id, version, prompts_v)
|
||
self._write_eval_report(run_id, version, prompts_v, result)
|
||
return result
|
||
|
||
async def diagnose(self, run_id: str) -> DiagnosisResult:
|
||
"""执行指定 run 的两阶段诊断。
|
||
|
||
参数:
|
||
run_id: 待诊断的 run_id。
|
||
|
||
返回:
|
||
DiagnosisResult。
|
||
"""
|
||
|
||
return await self._run_diagnosis(run_id)
|
||
|
||
def promote(self, version: str, eval_run_id: str, name: str) -> None:
|
||
"""把当前 ws 的指定版本提升为 Store 新种子。
|
||
|
||
参数:
|
||
version: 要提升的 skills 版本号。
|
||
eval_run_id: canonical eval run。
|
||
name: 新种子名。
|
||
"""
|
||
from app.harness.store import promote_to_seed
|
||
|
||
seed_dir = promote_to_seed(
|
||
self._config.workspace_dir,
|
||
self._config.store_dir,
|
||
version,
|
||
eval_run_id,
|
||
name,
|
||
description=f"promote from {self._config.workspace_dir.name} {version}",
|
||
)
|
||
logger.info("已提升为种子: {}", seed_dir)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# train() 三级嵌套（核心）
|
||
# -----------------------------------------------------------------------
|
||
|
||
async def train(self, pools: Pools) -> None:
|
||
"""mini-batch 快慢双速闭环：epoch 内多 step、每 step 按类 per-skill gate。
|
||
|
||
三级嵌套：epoch → batch(step) → per-skill。
|
||
epoch 末 _slow_update_cycle 十步序。
|
||
训练收尾 _deliver_best + _final_test_eval。
|
||
|
||
入口第一步做可训练性预检：剔除样本不足的微型题型（防 gate 阶梯崩溃），
|
||
过滤后的 pools 贯穿 batch/step/slow-update/final-eval 全部消费；filtered_task_types
|
||
透传到 gate 建立（不改 frozen RunConfig）。
|
||
"""
|
||
pools, filtered_task_types = _filter_untrainable_types(
|
||
pools,
|
||
list(self._config.task_types) if self._config.task_types is not None else None,
|
||
self._config.eval_min_per_class,
|
||
self._config.trainable_min_units,
|
||
)
|
||
state, total_steps, plan, saved_batches = await self._setup_train_run(
|
||
pools, filtered_task_types
|
||
)
|
||
for epoch in range(plan["first_epoch"], self._config.epochs + 1):
|
||
if epoch == plan["resume_epoch"]:
|
||
batches = [_batch_from_ids(pools, ids) for ids in saved_batches]
|
||
step_from = plan["resume_step_from"]
|
||
else:
|
||
logger.info("=== Epoch {} ===", epoch)
|
||
state.system_packs = []
|
||
state.tool_packs = []
|
||
state.changed_task_types_this_epoch = set()
|
||
state.epoch_start_skills = _snapshot_current_skills(self._paths.skills_dir)
|
||
batches, _ = build_batches(
|
||
pools.diagnosis,
|
||
state.correctness,
|
||
self._config.batch_size,
|
||
self._config.min_class_per_batch,
|
||
seed=epoch,
|
||
correct_ratio=self._config.batch_correct_ratio,
|
||
)
|
||
step_from = 0
|
||
# checkpoint 存 unit_id 序列：断点续跑按完整 unit 展开，孪生对不拆
|
||
batch_unit_ids = [_batch_unit_ids(b) for b in batches]
|
||
for step in range(step_from, len(batches)):
|
||
await self._run_step(epoch, step, total_steps, batches[step], pools, state)
|
||
state.global_step += 1
|
||
write_checkpoint(
|
||
self._config.workspace_dir,
|
||
state=state,
|
||
epoch=epoch,
|
||
step_completed=step,
|
||
phase="in_epoch",
|
||
global_step=state.global_step,
|
||
total_steps=total_steps,
|
||
version_snapshot=self._current_version_snapshot(),
|
||
epoch_batches=batch_unit_ids,
|
||
config=self._config,
|
||
)
|
||
# checkpoint 落盘移入 _slow_update_cycle 末尾（gate_pools.save 之后立即写），
|
||
# 消除 gate_epoch_observed 在 gate_pools.json 与 checkpoint 间的双计窗口。
|
||
await self._slow_update_cycle(
|
||
epoch,
|
||
pools,
|
||
state,
|
||
total_steps=total_steps,
|
||
step_completed=len(batches) - 1,
|
||
epoch_batches=batch_unit_ids,
|
||
)
|
||
if _should_early_stop(
|
||
self._config.workspace_dir,
|
||
epoch,
|
||
state,
|
||
self._config.early_stop_patience,
|
||
):
|
||
logger.info("Epoch {} 触发 early stop（best 连续无新高）", epoch)
|
||
break
|
||
self._deliver_best(state.best_skills_version, state.best_prompts_version)
|
||
await self._final_test_eval(pools)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# 训练初始化
|
||
# -----------------------------------------------------------------------
|
||
|
||
async def _setup_train_run(
|
||
self, pools: Pools, filtered_task_types: list[str] | None
|
||
) -> tuple[_TrainState, int, dict, list | None]:
|
||
"""据是否 --resume 准备训练起点。
|
||
|
||
参数:
|
||
pools: 已过可训练性预检的三池。
|
||
filtered_task_types: 预检后保留的题型（None 表示不限，由 gate 从 diag 推导）。
|
||
|
||
返回:
|
||
(state, total_steps, plan, saved_batches)。
|
||
"""
|
||
ckpt = load_checkpoint(self._config.workspace_dir) if self._config.resume else None
|
||
if self._config.resume and ckpt is None:
|
||
raise RuntimeError("--resume 但 checkpoint.json 不存在，拒绝静默从头重训")
|
||
gate_pools, baseline_cache = self._init_gate_pools(pools, filtered_task_types)
|
||
if not ckpt:
|
||
state = self._init_train_state(pools, gate_pools, baseline_cache)
|
||
total_steps = _compute_total_steps(pools, state.correctness, self._config)
|
||
plan = {"first_epoch": 1, "resume_epoch": None, "resume_step_from": 0}
|
||
return state, total_steps, plan, None
|
||
struct, decision = check_fingerprint(ckpt["config_fingerprint"], self._config)
|
||
if struct:
|
||
raise RuntimeError(f"结构性配置变化，拒绝 resume: {struct}")
|
||
if decision:
|
||
logger.warning("决策性配置变化，继续 resume: {}", decision)
|
||
state = self._restore_train_state(ckpt, pools, gate_pools, baseline_cache)
|
||
state.global_step = ckpt["progress"]["global_step"]
|
||
update_manifest(
|
||
self._config.workspace_dir,
|
||
skills=ckpt["version_snapshot"]["skills"],
|
||
prompts=ckpt["version_snapshot"]["prompts"],
|
||
)
|
||
self._paths = resolve_paths(self._config.workspace_dir)
|
||
plan = resume_plan(
|
||
ckpt["progress"]["epoch"],
|
||
ckpt["progress"]["phase"],
|
||
ckpt["progress"]["step_completed"],
|
||
)
|
||
return state, ckpt["progress"]["total_steps"], plan, ckpt["epoch_batches"]
|
||
|
||
def _init_gate_pools(
|
||
self, pools: Pools, filtered_task_types: list[str] | None
|
||
) -> tuple[GatePools, BaselineCache]:
|
||
"""构建/加载 CE-Gate 信息量阶梯与基线缓存。
|
||
|
||
副作用：设置 self._gate_questions_by_id（不进 checkpoint）。
|
||
|
||
参数:
|
||
pools: 冻结三池（已过可训练性预检）。
|
||
filtered_task_types: 预检保留的题型；None 时从 pools.diagnosis 推导。
|
||
|
||
返回:
|
||
(GatePools, BaselineCache)。
|
||
"""
|
||
from app.harness.log import HarnessLog
|
||
from app.question_gen import load_benchmark
|
||
|
||
questions = load_benchmark(self._paths.questions_dir)
|
||
self._gate_questions_by_id: dict[str, GeneratedQuestion] = {
|
||
q.question_id: q for q in questions
|
||
}
|
||
# unit 索引：ladder_for 返回 unit_id、update_probs 折叠逐题观测均需按 unit_id
|
||
# 反查成员题（核心算法保真 #5：gate 阶梯 unit 化）。不进 checkpoint、每次启动重建。
|
||
self._gate_units_by_id: dict[str, QuestionUnit] = {
|
||
u.unit_id: u for u in build_units(questions)
|
||
}
|
||
with HarnessLog(
|
||
str(self._paths.db_path), pools.baseline_run_id, register_run=False
|
||
) as log:
|
||
rows = log.query(
|
||
"SELECT question_id, prediction, answer FROM predictions WHERE run_id=?",
|
||
(pools.baseline_run_id,),
|
||
)
|
||
if not rows:
|
||
raise RuntimeError(
|
||
f"基线 run {pools.baseline_run_id} 在 predictions 表无任何行，"
|
||
"无法构建 gate 阶梯（检查种子基线是否完整落库）"
|
||
)
|
||
baseline_correctness = {r["question_id"]: r["prediction"] == r["answer"] for r in rows}
|
||
logger.info("gate 阶梯基线对错覆盖 {} 题", len(baseline_correctness))
|
||
# 预检保留的题型优先；None 时从（已过滤的）诊断池推导，二者一致
|
||
gate_task_types = (
|
||
sorted(filtered_task_types)
|
||
if filtered_task_types is not None
|
||
else sorted({q.task_type for q in pools.diagnosis})
|
||
)
|
||
gate_pools = build_or_load_gate_pools(
|
||
workspace_dir=self._config.workspace_dir,
|
||
questions=questions,
|
||
test_qids={q.question_id for q in pools.test},
|
||
baseline_correctness=baseline_correctness,
|
||
task_types=gate_task_types,
|
||
probe_quota=self._config.gate_probe_quota,
|
||
seed=1,
|
||
baseline_run_id=pools.baseline_run_id,
|
||
)
|
||
baseline_cache = BaselineCache(self._config.workspace_dir / "baseline_cache.json")
|
||
return gate_pools, baseline_cache
|
||
|
||
def _init_train_state(
|
||
self, pools: Pools, gate_pools: GatePools, baseline_cache: BaselineCache
|
||
) -> _TrainState:
|
||
"""初始化跨 step 训练状态。"""
|
||
skills_v = self._current_version("skills")
|
||
prompts_v = self._current_version("prompts")
|
||
update_best(
|
||
self._config.workspace_dir,
|
||
skills=f"skills/{skills_v}",
|
||
prompts=f"prompts/{prompts_v}",
|
||
val_acc=pools.baseline_val_accuracy,
|
||
run_id=pools.baseline_run_id,
|
||
epoch=0,
|
||
)
|
||
if read_best(self._config.workspace_dir) is None:
|
||
raise RuntimeError("best 指针初始化失败")
|
||
return _TrainState(
|
||
correctness=dict(pools.correctness),
|
||
gate_pools=gate_pools,
|
||
baseline_cache=baseline_cache,
|
||
eval_prev_acc=pools.baseline_val_accuracy,
|
||
eval_prev_run_id=pools.baseline_run_id,
|
||
best_val_acc=pools.baseline_val_accuracy,
|
||
best_skills_version=skills_v,
|
||
best_prompts_version=prompts_v,
|
||
baseline_skills_version=skills_v,
|
||
baseline_prompts_version=prompts_v,
|
||
)
|
||
|
||
def _restore_train_state(
|
||
self,
|
||
ckpt: dict,
|
||
pools: Pools,
|
||
gate_pools: GatePools,
|
||
baseline_cache: BaselineCache,
|
||
) -> _TrainState:
|
||
"""从 checkpoint 重建 _TrainState。"""
|
||
fields = deserialize_state_fields(ckpt["state"])
|
||
best = read_best(self._config.workspace_dir) or {}
|
||
return _TrainState(
|
||
gate_pools=gate_pools,
|
||
baseline_cache=baseline_cache,
|
||
best_val_acc=best.get("val_acc", pools.baseline_val_accuracy),
|
||
best_skills_version=best.get("skills", "skills/v1").split("/")[-1],
|
||
best_prompts_version=best.get("prompts", "prompts/v1").split("/")[-1],
|
||
**fields,
|
||
)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# _run_step：rollout → correctness → diagnose → accumulate → gate → cooldown
|
||
# -----------------------------------------------------------------------
|
||
|
||
async def _run_step(
|
||
self,
|
||
epoch: int,
|
||
step: int,
|
||
total_steps: int,
|
||
batch: list[GeneratedQuestion],
|
||
pools: Pools,
|
||
state: _TrainState,
|
||
) -> None:
|
||
"""单 step：rollout → correctness 增量 → 诊断 → 累加 system/tool → 按类 gate。"""
|
||
run_id = f"{pools.baseline_run_id}_e{epoch}_s{step}"
|
||
|
||
from app.harness.log import HarnessLog
|
||
|
||
# 幂等：重跑同一 step 前清 rollout + 全部 gate 派生旧行（修复潜伏 bug：
|
||
# 旧实现只清 rollout，gate 行崩溃重跑会累积重复）。
|
||
_clear_step_rows(
|
||
str(self._paths.db_path),
|
||
baseline_run_id=pools.baseline_run_id,
|
||
epoch=epoch,
|
||
step=step,
|
||
)
|
||
|
||
await self._rollout_batch(batch, run_id)
|
||
|
||
with HarnessLog(str(self._paths.db_path), run_id) as log:
|
||
_apply_batch_correctness(state.correctness, log, run_id, batch)
|
||
|
||
diagnosis = await self._run_diagnosis(run_id, question_ids=[q.question_id for q in batch])
|
||
# 降级占比过高疑似 judge 基础设施故障：不以降级信号驱动进化，直接中止
|
||
n_wrong = sum(1 for q in batch if not state.correctness.get(q.question_id, True))
|
||
if n_wrong > 0 and diagnosis.degraded_count / n_wrong > 0.5:
|
||
raise RuntimeError(
|
||
f"本 step 诊断降级占比 {diagnosis.degraded_count}/{n_wrong} > 50%，"
|
||
"疑似 judge 基础设施故障，中止训练（不以降级信号驱动进化）。"
|
||
)
|
||
_accumulate_slow_packs(diagnosis, state)
|
||
await self._gate_batch_skills(epoch, step, diagnosis, total_steps, pools, state)
|
||
# 冷却计数每 step 递减、归零剔除
|
||
state.gate_cooldown = {t: n - 1 for t, n in state.gate_cooldown.items() if n - 1 > 0}
|
||
# 测试中断注入点
|
||
if getattr(self, "_interrupt_after_step", None) == step:
|
||
raise _InterruptError(f"模拟中断于 epoch{epoch} step{step}")
|
||
|
||
async def _rollout_batch(self, batch: list[GeneratedQuestion], run_id: str) -> None:
|
||
"""用当前 skill 版本重推该 batch。"""
|
||
result = await self._run_inference_on_pool(
|
||
batch, run_id, self._paths.skills_dir, self._paths.prompts_dir
|
||
)
|
||
_guard_infra_failures(result, context="rollout")
|
||
|
||
# -----------------------------------------------------------------------
|
||
# _gate_batch_skills：并行进化 + 连续并发 gate（四阶段）
|
||
# -----------------------------------------------------------------------
|
||
|
||
async def _gate_batch_skills(
|
||
self,
|
||
epoch: int,
|
||
step: int,
|
||
diagnosis: DiagnosisResult,
|
||
total_steps: int,
|
||
pools: Pools,
|
||
state: _TrainState,
|
||
) -> None:
|
||
"""按 task_type 并行 evolve → 连续并发 gate → 字母序统一落账。
|
||
|
||
四阶段(设计 v3 §2.1):Phase A 并行进化(cooldown/无改动照旧跳过);
|
||
Phase B 装配 GateSpec(阶梯出题 + 案例单元排除 + n_max 截断);
|
||
Phase C validate_skills_concurrent(共享题槽,统计按阶梯序前缀推进,
|
||
只读 state);Phase D 唯一写 state 阶段——按字母序 accept/reject 落账,
|
||
与原串行语义等价(题型 skill 文件不相交,合并顺序仅为确定性)。
|
||
|
||
参数:
|
||
epoch / step / total_steps: 训练坐标。
|
||
diagnosis: 本 step 诊断结果(skill_case_packs 按题型分组)。
|
||
pools: 冻结三池。
|
||
state: 训练状态(Phase D 唯一写入点)。
|
||
|
||
返回:
|
||
无。
|
||
"""
|
||
budget = edit_budget_at(
|
||
global_step=state.global_step,
|
||
total_steps=total_steps,
|
||
start=self._config.edit_budget_start,
|
||
end=self._config.edit_budget_end,
|
||
)
|
||
|
||
# ---- Phase A: 并行进化(冷却/无真实改动照旧写 skip 后出清) ----
|
||
records = await self._evolve_types_parallel(epoch, step, diagnosis, budget, pools, state)
|
||
if not records:
|
||
return
|
||
_assert_disjoint_target_files({t: r.target_file for t, r in records.items()})
|
||
|
||
# ---- Phase B: 装配 GateSpec(阶梯出题,收编原 _run_gate_validation 前半) ----
|
||
specs = self._assemble_gate_specs(epoch, step, diagnosis, records, pools, state)
|
||
|
||
# ---- Phase C: 连续并发 gate(只读 state) ----
|
||
with HarnessLog(str(self._paths.db_path), f"gate_e{epoch}_s{step}") as gate_log:
|
||
outcomes = await validate_skills_concurrent(
|
||
workspace_dir=self._config.workspace_dir,
|
||
base_skills_version=self._current_version("skills"),
|
||
specs=specs,
|
||
gate_params=GateParams(
|
||
e_confirm=self._config.gate_e_confirm,
|
||
e_provisional=self._config.gate_e_provisional,
|
||
w_net_min=self._config.gate_w_net_min,
|
||
delta_min=self._config.gate_delta_min,
|
||
lambda_dir=self._config.gate_lambda_dir,
|
||
e_rollback=self._config.gate_e_rollback,
|
||
),
|
||
gate_guard_err=self._config.gate_guard_err,
|
||
baseline_cache=state.baseline_cache,
|
||
prompts_version=self._current_version("prompts"),
|
||
run_inference=self._make_validate_run_inference_fn(gate_log),
|
||
log=gate_log,
|
||
concurrency=self._config.concurrency,
|
||
)
|
||
|
||
# ---- Phase D: 唯一写 state 阶段(字母序确定性落账) ----
|
||
self._settle_gate_outcomes(epoch, step, records, outcomes, budget, pools, state)
|
||
|
||
async def _evolve_types_parallel(
|
||
self,
|
||
epoch: int,
|
||
step: int,
|
||
diagnosis: DiagnosisResult,
|
||
budget: int,
|
||
pools: Pools,
|
||
state: _TrainState,
|
||
) -> dict[str, EvolutionRecord]:
|
||
"""Phase A:各题型进化 asyncio.gather 并行,冷却/无改动路径写 skip 出队。
|
||
|
||
cooldown 与"进化未产出真实改动"(rejected/skipped/内容未变)两类路径
|
||
与原串行实现语义一致:写 skip_report 后不进 gate。进化互相独立
|
||
(各题型 skill 文件不相交,VersionedSkillStore 只读基线版本),
|
||
gather 并行不改变单题型结果。
|
||
|
||
参数:
|
||
epoch / step: 训练坐标。
|
||
diagnosis: 本 step 诊断结果。
|
||
budget: 当步编辑预算。
|
||
pools: 冻结三池。
|
||
state: 训练状态(只读)。
|
||
|
||
返回:
|
||
{题型: EvolutionRecord},仅含产出真实改动、待 gate 的题型。
|
||
"""
|
||
from app.harness.workspace import VersionedSkillStore
|
||
from core.evolution import evolve_single_skill
|
||
|
||
active_types: list[str] = []
|
||
for task_type in sorted(diagnosis.skill_case_packs):
|
||
if state.gate_cooldown.get(task_type, 0) > 0:
|
||
_write_skip_report(
|
||
self._config.workspace_dir,
|
||
epoch,
|
||
step,
|
||
state.global_step,
|
||
task_type,
|
||
action="cooldown",
|
||
baseline_acc=self._class_baseline_acc(
|
||
task_type, pools.validation, state.correctness
|
||
),
|
||
budget=budget,
|
||
)
|
||
continue
|
||
active_types.append(task_type)
|
||
if not active_types:
|
||
return {}
|
||
|
||
evolve_prompts = self._load_evolve_prompts()
|
||
skills_version = self._current_version("skills")
|
||
|
||
async def _evolve_one(task_type: str) -> EvolutionRecord:
|
||
pack = diagnosis.skill_case_packs[task_type]
|
||
skill_store = VersionedSkillStore(self._paths.skills_dir)
|
||
return await evolve_single_skill(
|
||
self._evolve_llm,
|
||
pack,
|
||
skill_store,
|
||
evolve_prompts,
|
||
skills_version,
|
||
budget,
|
||
self._config.appendix_consolidate_threshold,
|
||
skill_update_mode=self._config.skill_update_mode,
|
||
rejected=state.rejected_buffer.get(task_type, []),
|
||
)
|
||
|
||
# 首异常先取消其余进化任务并排水再向上传播(与 validate_skills_concurrent
|
||
# 同款语义):避免失败后残留 in-flight LLM 任务与 pending task 警告。
|
||
tasks = [asyncio.ensure_future(_evolve_one(t)) for t in active_types]
|
||
try:
|
||
evolved = await asyncio.gather(*tasks)
|
||
except BaseException:
|
||
for task in tasks:
|
||
task.cancel()
|
||
await asyncio.gather(*tasks, return_exceptions=True)
|
||
raise
|
||
records = dict(zip(active_types, evolved, strict=True))
|
||
|
||
# 无真实改动的题型照旧写 skipped 后出队
|
||
gated: dict[str, EvolutionRecord] = {}
|
||
for task_type in active_types:
|
||
record = records[task_type]
|
||
if record.status in ("rejected", "skipped") or (
|
||
record.evolved_content == record.original_content
|
||
):
|
||
_write_skip_report(
|
||
self._config.workspace_dir,
|
||
epoch,
|
||
step,
|
||
state.global_step,
|
||
task_type,
|
||
action="skipped",
|
||
baseline_acc=self._class_baseline_acc(
|
||
task_type, pools.validation, state.correctness
|
||
),
|
||
budget=budget,
|
||
rank_clip_triggered=bool(record.clip_info.get("triggered", False)),
|
||
)
|
||
continue
|
||
gated[task_type] = record
|
||
return gated
|
||
|
||
def _assemble_gate_specs(
|
||
self,
|
||
epoch: int,
|
||
step: int,
|
||
diagnosis: DiagnosisResult,
|
||
records: dict[str, EvolutionRecord],
|
||
pools: Pools,
|
||
state: _TrainState,
|
||
) -> list[GateSpec]:
|
||
"""Phase B:为每个待 gate 题型装配 GateSpec(阶梯出题 + 截断)。
|
||
|
||
案例包按 unit 排除:把每个 case 的 question_id 映射到其所属 unit_id,
|
||
命中单元整体排除,防止只排 AR pair 半个成员而给 gate 池灌半个 pair
|
||
(下游 _ladder_units 会 fail-fast)。base_skill_content 读 step 起点
|
||
版本(self._paths 在 Phase D accept 前不变),保证所有题型对同一
|
||
基线版本验证。核心算法保真 #5。
|
||
|
||
参数:
|
||
epoch / step: 训练坐标(拼 gate_run_prefix)。
|
||
diagnosis: 本 step 诊断结果(案例排除来源)。
|
||
records: Phase A 产出的待 gate 进化记录。
|
||
pools: 冻结三池(baseline_run_id)。
|
||
state: 训练状态(只读 gate_pools / gate_epoch_observed)。
|
||
|
||
返回:
|
||
与 records 键序一致的 GateSpec 列表。
|
||
|
||
异常:
|
||
RuntimeError: 阶梯引用了题库中不存在的 unit_id。
|
||
"""
|
||
specs: list[GateSpec] = []
|
||
for task_type, record in records.items():
|
||
pack = diagnosis.skill_case_packs[task_type]
|
||
exclude_units = {
|
||
self._gate_questions_by_id[c.question_id].unit_id
|
||
for c in pack.failure_cases + pack.success_cases
|
||
if c.question_id in self._gate_questions_by_id
|
||
}
|
||
ladder_unit_ids = state.gate_pools.ladder_for(
|
||
task_type,
|
||
exclude_units,
|
||
p_low=self._config.gate_p_low,
|
||
p_high=self._config.gate_p_high,
|
||
cold=not state.gate_epoch_observed,
|
||
)
|
||
missing = [uid for uid in ladder_unit_ids if uid not in self._gate_units_by_id]
|
||
if missing:
|
||
raise RuntimeError(
|
||
f"gate 阶梯引用未知 unit: {missing[:5]}(gate_pools.json 与题库失配)"
|
||
)
|
||
ladder_items = [
|
||
q for uid in ladder_unit_ids for q in self._gate_units_by_id[uid].questions
|
||
]
|
||
slug = task_type.lower().replace(" ", "-")
|
||
specs.append(
|
||
GateSpec(
|
||
task_type=task_type,
|
||
target_file=record.target_file,
|
||
candidate_content=record.evolved_content,
|
||
base_skill_content=(self._paths.skills_dir / record.target_file).read_text(
|
||
encoding="utf-8"
|
||
),
|
||
units=tuple(_ladder_units(ladder_items)[: self._config.gate_n_max]),
|
||
gate_run_prefix=f"{pools.baseline_run_id}_e{epoch}_s{step}_gate_{slug}",
|
||
)
|
||
)
|
||
return specs
|
||
|
||
def _settle_gate_outcomes(
|
||
self,
|
||
epoch: int,
|
||
step: int,
|
||
records: dict[str, EvolutionRecord],
|
||
outcomes: dict[str, ValidationOutcome],
|
||
budget: int,
|
||
pools: Pools,
|
||
state: _TrainState,
|
||
) -> None:
|
||
"""Phase D:按字母序统一落账(观测落库 + accept/reject 写 state)。
|
||
|
||
本阶段是 _gate_batch_skills 唯一写 state 的阶段。字母序仅为确定性
|
||
(题型 skill 文件不相交,accept 串行叠加时 _accept_skill 基于最新
|
||
manifest 版本追加各自 target_file,互不覆盖),与原串行语义等价。
|
||
|
||
参数:
|
||
epoch / step: 训练坐标。
|
||
records: Phase A 产出的进化记录。
|
||
outcomes: Phase C 产出的 gate 判定。
|
||
budget: 当步编辑预算(step_report 落账)。
|
||
pools: 冻结三池。
|
||
state: 训练状态(唯一写入点)。
|
||
|
||
返回:
|
||
无。
|
||
"""
|
||
for task_type in sorted(outcomes):
|
||
record = records[task_type]
|
||
outcome = outcomes[task_type]
|
||
write_gate_evidence(
|
||
str(self._paths.db_path),
|
||
run_id=pools.baseline_run_id,
|
||
epoch=epoch,
|
||
step=step,
|
||
rows=outcome.evidence_rows,
|
||
)
|
||
write_step_report(
|
||
self._config.workspace_dir,
|
||
epoch=epoch,
|
||
step=step,
|
||
global_step=state.global_step,
|
||
task_type=task_type,
|
||
gate_action=outcome.action,
|
||
candidate_acc=outcome.candidate_acc,
|
||
class_baseline_acc=outcome.baseline_acc,
|
||
edit_budget=budget,
|
||
rank_clip_triggered=bool(record.clip_info.get("triggered", False)),
|
||
gate_w=outcome.w,
|
||
gate_l=outcome.l,
|
||
gate_e_value=outcome.e_value,
|
||
gate_n_used=outcome.n_used,
|
||
gate_stop_reason=outcome.stop_reason,
|
||
)
|
||
write_quadrant_pairs(
|
||
str(self._paths.db_path),
|
||
run_id=pools.baseline_run_id,
|
||
epoch=epoch,
|
||
step=step,
|
||
pairs=_outcome_to_quadrant_pairs(task_type, outcome),
|
||
)
|
||
if outcome.accepted:
|
||
self._accept_skill(task_type, record, outcome, state, pools)
|
||
else:
|
||
self._record_rejected_skill(
|
||
state.rejected_buffer, task_type, record, outcome, state.global_step
|
||
)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# accept / reject / probation
|
||
# -----------------------------------------------------------------------
|
||
|
||
def _accept_skill(
|
||
self,
|
||
task_type: str,
|
||
record: EvolutionRecord,
|
||
outcome: ValidationOutcome,
|
||
state: _TrainState,
|
||
pools: Pools,
|
||
) -> None:
|
||
"""accept：写候选为新 skills 版本 → manifest → 路径 → 前移 correctness。
|
||
|
||
probation 分岔：provisional + 无现有试用 + 非 default-strategy.md → 开账。
|
||
试用中追加 pending_edits。
|
||
"""
|
||
pre_accept_version = self._current_version("skills")
|
||
# 开账快照在合并前拍取
|
||
pre_merge_snapshot = {
|
||
q.question_id: state.correctness.get(q.question_id, False)
|
||
for q in pools.validation
|
||
if q.task_type == task_type
|
||
}
|
||
new_version = self._promote_skill_version(record.evolved_content, record.target_file)
|
||
update_manifest(self._config.workspace_dir, skills=f"skills/{new_version}")
|
||
self._paths = resolve_paths(self._config.workspace_dir)
|
||
# correctness 二轨合并（只合并已观测题）
|
||
state.correctness.update(outcome.candidate_correctness)
|
||
# 清该类黑名单
|
||
state.rejected_buffer.pop(task_type, None)
|
||
state.changed_task_types_this_epoch.add(task_type)
|
||
# probation 分岔
|
||
if (
|
||
outcome.action == "accept_provisional"
|
||
and task_type not in state.probations
|
||
and record.target_file != "default-strategy.md"
|
||
):
|
||
state.probations[task_type] = Probation(
|
||
task_type=task_type,
|
||
anchor_skills_version=pre_accept_version,
|
||
target_file=record.target_file,
|
||
correctness_snapshot=pre_merge_snapshot,
|
||
opened_step=state.global_step,
|
||
)
|
||
elif outcome.action == "accept_provisional" and record.target_file == "default-strategy.md":
|
||
logger.warning(
|
||
"按类 gate[{}] provisional 落在共享 default-strategy.md，跳过试用直接转正",
|
||
task_type,
|
||
)
|
||
if task_type in state.probations:
|
||
state.probations[task_type].pending_edits.append(
|
||
RejectedEdit(
|
||
target_file=record.target_file,
|
||
target_type=record.target_type,
|
||
change_summary=self._rejected_summary(record, outcome),
|
||
delta=outcome.delta_hat,
|
||
source_version=record.source_version,
|
||
epoch=state.global_step,
|
||
gate_w=outcome.w,
|
||
gate_l=outcome.l,
|
||
gate_e_value=outcome.e_value,
|
||
gate_delta_shrunk=outcome.delta_shrunk,
|
||
)
|
||
)
|
||
logger.info(
|
||
"按类 gate[{}] accept: 候选{:.1%} (观测基线{:.1%}) → skills/{}",
|
||
task_type,
|
||
outcome.candidate_acc,
|
||
outcome.baseline_acc,
|
||
new_version,
|
||
)
|
||
|
||
def _rollback_probation(self, probation: Probation, state: _TrainState) -> None:
|
||
"""试用期回滚：文件级 revert 到锚版本 + 恢复快照 + 冷却 + 证据入黑名单。"""
|
||
anchor_content = (
|
||
self._config.workspace_dir
|
||
/ "skills"
|
||
/ probation.anchor_skills_version
|
||
/ probation.target_file
|
||
).read_text(encoding="utf-8")
|
||
new_version = self._promote_skill_version(anchor_content, probation.target_file)
|
||
update_manifest(self._config.workspace_dir, skills=f"skills/{new_version}")
|
||
self._paths = resolve_paths(self._config.workspace_dir)
|
||
state.correctness.update(probation.correctness_snapshot)
|
||
state.gate_cooldown[probation.task_type] = self._config.gate_cooldown_steps
|
||
state.rejected_buffer.setdefault(probation.task_type, []).extend(probation.pending_edits)
|
||
logger.info(
|
||
"probation 回滚[{}]: skills 文件 {} 恢复至锚版本 {} → 新版本 {}，冷却 {} step",
|
||
probation.task_type,
|
||
probation.target_file,
|
||
probation.anchor_skills_version,
|
||
new_version,
|
||
self._config.gate_cooldown_steps,
|
||
)
|
||
|
||
@staticmethod
|
||
def _record_rejected_skill(
|
||
rejected_buffer: dict[str, list],
|
||
task_type: str,
|
||
record: EvolutionRecord,
|
||
outcome: ValidationOutcome,
|
||
global_step: int,
|
||
) -> None:
|
||
"""reject：按 task_type 累加 A5 黑名单。"""
|
||
rejected_buffer.setdefault(task_type, []).append(
|
||
RejectedEdit(
|
||
target_file=record.target_file,
|
||
target_type=record.target_type,
|
||
change_summary=Runner._rejected_summary_static(record, outcome),
|
||
delta=outcome.delta_hat,
|
||
source_version=record.source_version,
|
||
epoch=global_step,
|
||
gate_w=outcome.w,
|
||
gate_l=outcome.l,
|
||
gate_e_value=outcome.e_value,
|
||
gate_delta_shrunk=outcome.delta_shrunk,
|
||
)
|
||
)
|
||
logger.info(
|
||
"按类 gate[{}] reject: 候选{:.1%} (观测基线{:.1%}) 回退该 skill",
|
||
task_type,
|
||
outcome.candidate_acc,
|
||
outcome.baseline_acc,
|
||
)
|
||
|
||
def _rejected_summary(self, record: EvolutionRecord, outcome: ValidationOutcome) -> str:
|
||
"""为被拒进化记录生成黑名单摘要：只拼真正 applied 的 edit。"""
|
||
return Runner._rejected_summary_static(record, outcome)
|
||
|
||
@staticmethod
|
||
def _rejected_summary_static(record: EvolutionRecord, outcome: ValidationOutcome) -> str:
|
||
"""黑名单摘要实现（静态方法，供 accept / reject 两侧复用）。
|
||
|
||
为何只记 applied：未 applied 的 edit 从未写进候选正文、从未被 gate
|
||
验证过，进黑名单会污染「已验证无效」语义。
|
||
"""
|
||
applied_summary = _format_applied_edits(record)
|
||
if applied_summary is not None:
|
||
return applied_summary
|
||
return _fallback_summary(record, outcome)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# _slow_update_cycle 十步序
|
||
# -----------------------------------------------------------------------
|
||
|
||
async def _slow_update_cycle(
|
||
self,
|
||
epoch: int,
|
||
pools: Pools,
|
||
state: _TrainState,
|
||
*,
|
||
total_steps: int,
|
||
step_completed: int,
|
||
epoch_batches: list[list[str]],
|
||
) -> None:
|
||
"""epoch 末慢更新十步序。
|
||
|
||
1. 捕获版本快照 → 全 val 重跑 R
|
||
2. soft score + dual_metric 落库
|
||
3. R 逐题对错无条件回写
|
||
4. probation 结算（回滚者覆盖 step 3）
|
||
5. best argmax（严格大于）
|
||
6. momentum（不可变新版本，按 skill 文件分组）
|
||
7. system/tool 慢更新（edit_budget_end）
|
||
8. R2 闭环
|
||
9. 三态标签 + epoch_report + 四向 held-out
|
||
10. gate 阶梯刷新 → 重置 epoch 累加器 → 立即落 epoch_done checkpoint
|
||
|
||
参数:
|
||
total_steps: 全局总 step 数（checkpoint 用）。
|
||
step_completed: 本 epoch 已完成 step 数（checkpoint 用）。
|
||
epoch_batches: 本 epoch batch 的 unit_id 划分（checkpoint 用）。
|
||
"""
|
||
# Phase 1
|
||
eval_skills_version = self._current_version("skills")
|
||
eval_prompts_version = self._current_version("prompts")
|
||
eval_r = await self._eval_full_val(epoch, pools)
|
||
|
||
# Phase 2: soft + dual_metric
|
||
eval_soft = await self._try_soft_score(eval_r.run_id, pools.validation)
|
||
write_dual_metric(
|
||
str(self._paths.db_path),
|
||
run_id=self._config.run_id,
|
||
epoch=epoch,
|
||
version_kind="final",
|
||
skills_version=eval_skills_version,
|
||
prompts_version=eval_prompts_version,
|
||
pool="val",
|
||
hard_acc=eval_r.accuracy,
|
||
soft_score=eval_soft,
|
||
mixed_score=(None if eval_soft is None else 0.5 * eval_r.accuracy + 0.5 * eval_soft),
|
||
)
|
||
|
||
# Phase 3: 无条件回写
|
||
self._writeback_val_correctness(eval_r.run_id, pools, state)
|
||
|
||
# Phase 4: probation 结算
|
||
self._settle_probations(eval_r.run_id, pools, state)
|
||
|
||
# Phase 5: best argmax
|
||
self._maybe_promote_best(
|
||
eval_skills_version,
|
||
eval_prompts_version,
|
||
eval_r.accuracy,
|
||
eval_r.run_id,
|
||
epoch,
|
||
state,
|
||
)
|
||
|
||
# Phase 6: momentum
|
||
momentum_task_types = await self._write_momentum_for_changed_skills(
|
||
state, pools, epoch, eval_skills_version
|
||
)
|
||
|
||
# Phase 7: system/tool 慢更新
|
||
pre_prompts_version = self._current_version("prompts")
|
||
system_tool_updated = await self._update_system_tool(epoch, state)
|
||
system_tool_reverted = False
|
||
|
||
# Phase 8: R2 闭环
|
||
r2_kept_run_ids: list[str] | None = None
|
||
if system_tool_updated:
|
||
r2_skills_version = self._current_version("skills")
|
||
new_prompts_version = self._current_version("prompts")
|
||
eval_r2 = await self._eval_full_val(epoch, pools, run_suffix="_p2")
|
||
# R2 是可能被 revert 的慢更新候选，用 slow_candidate 口径，不占 final
|
||
# （epoch 终值唯一由 Phase 2 的 final 承载）
|
||
write_dual_metric(
|
||
str(self._paths.db_path),
|
||
run_id=self._config.run_id,
|
||
epoch=epoch,
|
||
version_kind="slow_candidate",
|
||
skills_version=r2_skills_version,
|
||
prompts_version=new_prompts_version,
|
||
pool="val",
|
||
hard_acc=eval_r2.accuracy,
|
||
soft_score=None,
|
||
mixed_score=None,
|
||
)
|
||
system_tool_reverted = eval_r2.accuracy < eval_r.accuracy
|
||
if system_tool_reverted:
|
||
self._revert_system_tool(pre_prompts_version)
|
||
else:
|
||
self._writeback_val_correctness(eval_r2.run_id, pools, state)
|
||
self._maybe_promote_best(
|
||
r2_skills_version,
|
||
new_prompts_version,
|
||
eval_r2.accuracy,
|
||
eval_r2.run_id,
|
||
epoch,
|
||
state,
|
||
)
|
||
state.eval_prev_acc = eval_r2.accuracy
|
||
state.eval_prev_run_id = eval_r2.run_id
|
||
r2_kept_run_ids = [eval_r2.run_id]
|
||
if (not system_tool_updated) or system_tool_reverted:
|
||
state.eval_prev_acc = eval_r.accuracy
|
||
state.eval_prev_run_id = eval_r.run_id
|
||
|
||
# Phase 9: 三态标签 + epoch_report + held-out
|
||
if system_tool_reverted:
|
||
system_tool_action = "reverted"
|
||
elif system_tool_updated:
|
||
system_tool_action = "updated"
|
||
else:
|
||
system_tool_action = "none"
|
||
write_epoch_report(
|
||
self._config.workspace_dir,
|
||
epoch=epoch,
|
||
system_tool_action=system_tool_action,
|
||
momentum_updated_task_types=momentum_task_types,
|
||
best_val_acc=state.best_val_acc,
|
||
)
|
||
if self._config.run_holdout_eval:
|
||
await self._holdout_four_way(
|
||
epoch, pools, state, eval_skills_version, eval_prompts_version
|
||
)
|
||
|
||
# Phase 10: gate 阶梯刷新
|
||
self._refresh_gate_ladder(
|
||
epoch, pools.baseline_run_id, state, extra_run_ids=r2_kept_run_ids
|
||
)
|
||
|
||
# 重置 epoch 累加器 + 立即落 epoch_done checkpoint：与 _refresh_gate_ladder 内的
|
||
# gate_pools.save + gate_epoch_observed=True 同刻一致，消除断点续跑的双计窗口。
|
||
state.system_packs = []
|
||
state.tool_packs = []
|
||
state.changed_task_types_this_epoch = set()
|
||
write_checkpoint(
|
||
self._config.workspace_dir,
|
||
state=state,
|
||
epoch=epoch,
|
||
step_completed=step_completed,
|
||
phase="epoch_done",
|
||
global_step=state.global_step,
|
||
total_steps=total_steps,
|
||
version_snapshot=self._current_version_snapshot(),
|
||
epoch_batches=epoch_batches,
|
||
config=self._config,
|
||
)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# 慢更新内部方法
|
||
# -----------------------------------------------------------------------
|
||
|
||
def _settle_probations(self, eval_run_id: str, pools: Pools, state: _TrainState) -> None:
|
||
"""epoch 末试用期一次性结算：全 val 重跑结果按 unit 与锚快照配对。
|
||
|
||
W/L 按 **unit 口径** 统计（AR pair 双向 AND 折叠，不被 P/Q 单题计分污染，
|
||
核心算法保真 #5）。锚快照与当前重跑均先经 unit_correctness_view 折叠成单元
|
||
视图再走 pair_block 计翻转。逐题 predictions 仍逐题落库溯源。
|
||
|
||
参数:
|
||
eval_run_id: 本 epoch 全 val 重跑（R）的 run_id。
|
||
pools: 冻结三池（按 task_type 取 val 子集重建单元）。
|
||
state: 训练状态（probations 结算后清空）。
|
||
|
||
异常:
|
||
RuntimeError: 重跑缺某 val 题的预测行。
|
||
"""
|
||
if not state.probations:
|
||
return
|
||
from app.harness.log import HarnessLog
|
||
from app.harness.validate import _load_run_rows
|
||
|
||
with HarnessLog(str(self._paths.db_path), eval_run_id) as log:
|
||
rows = _load_run_rows(log, eval_run_id)
|
||
|
||
params = GateParams(
|
||
e_confirm=self._config.gate_e_confirm,
|
||
e_provisional=self._config.gate_e_provisional,
|
||
w_net_min=self._config.gate_w_net_min,
|
||
delta_min=self._config.gate_delta_min,
|
||
lambda_dir=self._config.gate_lambda_dir,
|
||
e_rollback=self._config.gate_e_rollback,
|
||
)
|
||
for task_type in sorted(state.probations):
|
||
probation = state.probations[task_type]
|
||
val_units = build_units([q for q in pools.validation if q.task_type == task_type])
|
||
flips = _probation_unit_flips(
|
||
val_units, probation.correctness_snapshot, rows, task_type, eval_run_id
|
||
)
|
||
verdict = probation_verdict(flips.w, flips.l, params=params)
|
||
logger.info("probation 结算[{}]: W={} L={} → {}", task_type, flips.w, flips.l, verdict)
|
||
if verdict == "rollback":
|
||
self._rollback_probation(probation, state)
|
||
state.probations.clear()
|
||
|
||
async def _eval_full_val(
|
||
self, epoch: int, pools: Pools, run_suffix: str = ""
|
||
) -> InferenceResult:
|
||
"""全验证池重跑一次并护栏。"""
|
||
run_id = f"{self._config.run_id}_slow_e{epoch}{run_suffix}"
|
||
result = await self._run_inference_on_pool(
|
||
pools.validation, run_id, self._paths.skills_dir, self._paths.prompts_dir
|
||
)
|
||
_guard_infra_failures(result, context="全 val 重跑")
|
||
return result
|
||
|
||
def _writeback_val_correctness(
|
||
self, eval_run_id: str, pools: Pools, state: _TrainState
|
||
) -> None:
|
||
"""把全 val 重跑逐题对错回写进 state.correctness。"""
|
||
from app.harness.log import HarnessLog
|
||
from app.harness.validate import _load_run_rows
|
||
|
||
with HarnessLog(str(self._paths.db_path), eval_run_id) as log:
|
||
rows = _load_run_rows(log, eval_run_id)
|
||
for q in pools.validation:
|
||
row = rows.get(q.question_id)
|
||
if row is not None:
|
||
state.correctness[q.question_id] = row["_correct"]
|
||
|
||
def _maybe_promote_best(
|
||
self,
|
||
skills_v: str,
|
||
prompts_v: str,
|
||
eval_acc: float,
|
||
run_id: str,
|
||
epoch: int,
|
||
state: _TrainState,
|
||
) -> None:
|
||
"""全局 best argmax（严格大于才推进）。"""
|
||
if eval_acc <= state.best_val_acc:
|
||
return
|
||
state.best_val_acc = eval_acc
|
||
state.best_skills_version = skills_v
|
||
state.best_prompts_version = prompts_v
|
||
state.epochs_since_best_improved = 0
|
||
update_best(
|
||
self._config.workspace_dir,
|
||
skills=f"skills/{skills_v}",
|
||
prompts=f"prompts/{prompts_v}",
|
||
val_acc=eval_acc,
|
||
run_id=run_id,
|
||
epoch=epoch,
|
||
)
|
||
logger.info(
|
||
"全局 best argmax 刷新: {:.1%} → skills/{} prompts/{}",
|
||
eval_acc,
|
||
skills_v,
|
||
prompts_v,
|
||
)
|
||
|
||
async def _write_momentum_for_changed_skills(
|
||
self,
|
||
state: _TrainState,
|
||
pools: Pools,
|
||
epoch: int,
|
||
eval_skills_version: str,
|
||
) -> list[str]:
|
||
"""为本 epoch 改过的题型写 momentum：推进不可变新版本。
|
||
|
||
返回:
|
||
实际写过 momentum 的题型列表。
|
||
"""
|
||
|
||
if not self._config.use_slow_momentum:
|
||
return []
|
||
if not state.changed_task_types_this_epoch:
|
||
return []
|
||
|
||
file_to_task_types = self._group_changed_task_types_by_file(
|
||
state.changed_task_types_this_epoch
|
||
)
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
staged_skills = Path(tmp) / "skills"
|
||
shutil.copytree(self._paths.skills_dir, staged_skills, dirs_exist_ok=True)
|
||
for target_file in sorted(file_to_task_types):
|
||
await self._stage_momentum_for_file(
|
||
target_file,
|
||
file_to_task_types[target_file],
|
||
state,
|
||
pools,
|
||
staged_skills,
|
||
epoch,
|
||
)
|
||
new_version = advance_version(
|
||
self._paths.workspace_dir,
|
||
"skills",
|
||
staged_skills,
|
||
{
|
||
"source": "slow_momentum",
|
||
"parent": eval_skills_version,
|
||
"description": "epoch 末 momentum（不可变新版本）",
|
||
},
|
||
)
|
||
update_manifest(self._config.workspace_dir, skills=f"skills/{new_version}")
|
||
self._paths = resolve_paths(self._config.workspace_dir)
|
||
logger.info(
|
||
"Epoch 末 momentum → skills/{}（不改 eval 版本 {}）",
|
||
new_version,
|
||
eval_skills_version,
|
||
)
|
||
return sorted(state.changed_task_types_this_epoch)
|
||
|
||
async def _stage_momentum_for_file(
|
||
self,
|
||
target_file: str,
|
||
task_types: list[str],
|
||
state: _TrainState,
|
||
pools: Pools,
|
||
staged_skills: Path,
|
||
epoch: int,
|
||
) -> None:
|
||
"""单个 skill 文件的 momentum 生成：诊断池采样 → 两版 rollout → 纵向对比。"""
|
||
from app.harness.log import HarnessLog
|
||
from app.harness.momentum import run_slow_momentum
|
||
from app.harness.validate import _load_run_rows
|
||
|
||
skill_path = staged_skills / target_file
|
||
skill_content = skill_path.read_text(encoding="utf-8")
|
||
prev_skill = state.epoch_start_skills.get(target_file, skill_content)
|
||
prev_guidance = momentum_inner(skill_content)
|
||
|
||
# 采样（逐题粒度，不折叠 unit；混格偏差见 _sample_momentum_candidates docstring）
|
||
sampled = _sample_momentum_candidates(
|
||
pools.diagnosis, set(task_types), self._config.momentum_samples, epoch
|
||
)
|
||
|
||
if not sampled:
|
||
skill_path.write_text(
|
||
replace_momentum(skill_content, prev_guidance or ""),
|
||
encoding="utf-8",
|
||
)
|
||
logger.debug(
|
||
"Epoch {} momentum 跳过 {}：诊断池无匹配题型 {} 的样本",
|
||
epoch,
|
||
target_file,
|
||
sorted(task_types),
|
||
)
|
||
return
|
||
|
||
# 两版 rollout
|
||
prev_run_id = f"momentum_prev_e{epoch}_{target_file.replace('.md', '')}"
|
||
curr_run_id = f"momentum_curr_e{epoch}_{target_file.replace('.md', '')}"
|
||
|
||
with tempfile.TemporaryDirectory() as prev_tmp:
|
||
prev_skills_dir = Path(prev_tmp) / "skills"
|
||
shutil.copytree(self._paths.skills_dir, prev_skills_dir, dirs_exist_ok=True)
|
||
(prev_skills_dir / target_file).write_text(prev_skill, encoding="utf-8")
|
||
await self._run_inference_on_pool(
|
||
sampled, prev_run_id, prev_skills_dir, self._paths.prompts_dir
|
||
)
|
||
|
||
await self._run_inference_on_pool(
|
||
sampled, curr_run_id, self._paths.skills_dir, self._paths.prompts_dir
|
||
)
|
||
|
||
with HarnessLog(str(self._paths.db_path), prev_run_id) as log:
|
||
prev_rows = _load_run_rows(log, prev_run_id)
|
||
with HarnessLog(str(self._paths.db_path), curr_run_id) as log:
|
||
curr_rows = _load_run_rows(log, curr_run_id)
|
||
|
||
comparison_pairs = _build_comparison_pairs(sampled, prev_rows, curr_rows)
|
||
guidance = await run_slow_momentum(
|
||
llm=self._evolve_llm,
|
||
diagnose_prompts_dir=Path("prompts"),
|
||
skill_content=skill_content,
|
||
prev_skill=prev_skill,
|
||
prev_guidance=prev_guidance,
|
||
comparison_pairs=comparison_pairs,
|
||
)
|
||
new_content = replace_momentum(skill_content, guidance)
|
||
skill_path.write_text(new_content, encoding="utf-8")
|
||
logger.info(
|
||
"Epoch {} momentum 写入 skill 文件 {}（题型 {}，采样 {} 题）",
|
||
epoch,
|
||
target_file,
|
||
sorted(task_types),
|
||
len(sampled),
|
||
)
|
||
|
||
async def _update_system_tool(self, epoch: int, state: _TrainState) -> bool:
|
||
"""merge 本 epoch 累加的 system/tool 案例包 → 进化 → accept 写新 prompts 版本。
|
||
|
||
返回:
|
||
是否实际写了新 prompts 版本。
|
||
"""
|
||
from app.harness.workspace import VersionedPromptStore
|
||
from core.evolution import evolve_single_tool, evolve_system_prompt
|
||
|
||
merged_system = merge_system_packs(state.system_packs)
|
||
merged_tools = merge_tool_packs(state.tool_packs)
|
||
source_version = self._current_version("prompts")
|
||
max_edits = self._config.edit_budget_end
|
||
evolve_prompts = self._load_evolve_prompts()
|
||
prompt_store = VersionedPromptStore(self._paths.prompts_dir)
|
||
|
||
records: list[EvolutionRecord] = []
|
||
if merged_system is not None:
|
||
records.append(
|
||
await evolve_system_prompt(
|
||
self._evolve_llm,
|
||
merged_system,
|
||
prompt_store,
|
||
evolve_prompts,
|
||
source_version,
|
||
max_edits,
|
||
)
|
||
)
|
||
for tool_name in sorted(merged_tools):
|
||
records.append(
|
||
await evolve_single_tool(
|
||
self._evolve_llm,
|
||
merged_tools[tool_name],
|
||
prompt_store,
|
||
evolve_prompts,
|
||
source_version,
|
||
max_edits,
|
||
)
|
||
)
|
||
accepted = [r for r in records if r.status == "accepted"]
|
||
if not accepted:
|
||
logger.debug("Epoch {} 慢更新：无 system/tool 改动被接受", epoch)
|
||
return False
|
||
|
||
new_version = self._write_accepted_prompts_version(accepted, source_version)
|
||
if new_version is None:
|
||
return False
|
||
update_manifest(self._config.workspace_dir, prompts=f"prompts/{new_version}")
|
||
self._paths = resolve_paths(self._config.workspace_dir)
|
||
logger.info("Epoch {} 慢更新：system/tool → prompts/{}", epoch, new_version)
|
||
return True
|
||
|
||
def _write_accepted_prompts_version(
|
||
self, accepted: list[EvolutionRecord], source_version: str
|
||
) -> str | None:
|
||
"""将 accepted system/tool records 写成新 prompts 版本。
|
||
|
||
参数:
|
||
accepted: 状态为 accepted 的 EvolutionRecord 列表。
|
||
source_version: 改写前 prompts 版本。
|
||
|
||
返回:
|
||
新版本号，或 None（无实际变化时）。
|
||
"""
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
staged = Path(tmp) / "prompts"
|
||
shutil.copytree(self._paths.prompts_dir, staged, dirs_exist_ok=True)
|
||
any_changed = False
|
||
for rec in accepted:
|
||
if rec.target_type == "tool":
|
||
# tool: evolved_content = json.dumps({"extract": ..., "verify": ...})
|
||
combined = json.loads(rec.evolved_content)
|
||
for key in ("extract", "verify"):
|
||
fname = rec.target_file.replace("_extract.md", f"_{key}.md")
|
||
(staged / fname).write_text(combined[key], encoding="utf-8")
|
||
any_changed = True
|
||
else:
|
||
(staged / rec.target_file).write_text(rec.evolved_content, encoding="utf-8")
|
||
any_changed = True
|
||
if not any_changed:
|
||
return None
|
||
return advance_version(
|
||
self._paths.workspace_dir,
|
||
"prompts",
|
||
staged,
|
||
{
|
||
"source": "evolution",
|
||
"parent": source_version,
|
||
"description": "epoch 末 system/tool 慢更新",
|
||
},
|
||
)
|
||
|
||
def _revert_system_tool(self, pre_prompts_version: str) -> None:
|
||
"""prompts-only delta 退步时回退到更新前版本。"""
|
||
update_manifest(self._config.workspace_dir, prompts=f"prompts/{pre_prompts_version}")
|
||
self._paths = resolve_paths(self._config.workspace_dir)
|
||
logger.info(
|
||
"慢更新 prompts-only delta 退步：system/tool 回退到 prompts/{}",
|
||
pre_prompts_version,
|
||
)
|
||
|
||
def _refresh_gate_ladder(
|
||
self,
|
||
epoch: int,
|
||
base_run_id: str,
|
||
state: _TrainState,
|
||
extra_run_ids: list[str] | None = None,
|
||
) -> None:
|
||
"""用本 epoch 非 gate run 的逐题观测 γ-EMA 更新阶梯 p-hat 并落盘。
|
||
|
||
精确三源：step rollout GLOB 排除 _gate_ + slow R + kept R2。
|
||
"""
|
||
db_path = resolve_paths(self._config.workspace_dir).db_path
|
||
conn = sqlite3.connect(str(db_path))
|
||
conn.row_factory = sqlite3.Row
|
||
try:
|
||
rows = conn.execute(
|
||
"SELECT question_id, prediction, answer FROM predictions "
|
||
"WHERE run_id GLOB ? AND run_id NOT GLOB '*_gate_*' "
|
||
"ORDER BY rowid",
|
||
(f"{base_run_id}_e{epoch}_s*",),
|
||
).fetchall()
|
||
slow_rows = conn.execute(
|
||
"SELECT question_id, prediction, answer FROM predictions "
|
||
"WHERE run_id=? ORDER BY rowid",
|
||
(f"{self._config.run_id}_slow_e{epoch}",),
|
||
).fetchall()
|
||
extra_rows_lists = [
|
||
conn.execute(
|
||
"SELECT question_id, prediction, answer FROM predictions "
|
||
"WHERE run_id=? ORDER BY rowid",
|
||
(rid,),
|
||
).fetchall()
|
||
for rid in (extra_run_ids or [])
|
||
]
|
||
finally:
|
||
conn.close()
|
||
|
||
obs = {r["question_id"]: r["prediction"] == r["answer"] for r in rows}
|
||
obs.update({r["question_id"]: r["prediction"] == r["answer"] for r in slow_rows})
|
||
for extra_rows in extra_rows_lists:
|
||
obs.update({r["question_id"]: r["prediction"] == r["answer"] for r in extra_rows})
|
||
# 逐题观测折叠成单元观测后按 unit_id 匹配更新（防按 qid 匹配 pair 失效致 EMA 停摆）。
|
||
state.gate_pools.update_probs(
|
||
obs, self._gate_units_by_id, gamma=self._config.gate_gamma_decay
|
||
)
|
||
state.gate_pools.save(self._config.workspace_dir / "gate_pools.json")
|
||
state.gate_epoch_observed = True
|
||
|
||
async def _holdout_four_way(
|
||
self,
|
||
epoch: int,
|
||
pools: Pools,
|
||
state: _TrainState,
|
||
eval_skills_version: str,
|
||
eval_prompts_version: str,
|
||
) -> None:
|
||
"""四向 held-out：baseline/best_hard/final/best_mixed 各在 test 池评估。
|
||
|
||
去重（进程内备忘录 state.holdout_memo，不改 schema）：
|
||
- baseline：不跑推理，从基线 predictions 推导 test 结果（0 推理），存 memo 跨 epoch 复用。
|
||
- final：真评 test，存 memo[(final_sv,final_pv)]。
|
||
- best_hard：其版本已在 memo（== final 或往轮已评）则引用，否则真评并存 memo。
|
||
- best_mixed：赢家必是 best_hard 或 final 之一，其结果已在 memo，直接引用（0 推理）。
|
||
|
||
test 池仅观测落库，绝不进 gate/best/early-stop/调参。
|
||
"""
|
||
best_mixed = await self._pick_mixed_best(
|
||
epoch, pools, state, eval_skills_version, eval_prompts_version
|
||
)
|
||
memo = state.holdout_memo
|
||
versions: dict[str, tuple[str, str] | None] = {
|
||
"baseline": (state.baseline_skills_version, state.baseline_prompts_version),
|
||
"best_hard": (state.best_skills_version, state.best_prompts_version),
|
||
"final": (eval_skills_version, eval_prompts_version),
|
||
"best_mixed": best_mixed,
|
||
}
|
||
for version_kind, version in versions.items():
|
||
if version is None:
|
||
continue
|
||
sv, pv = version
|
||
run_id = f"{self._config.run_id}_holdout_{version_kind}_e{epoch}"
|
||
if version not in memo:
|
||
if version_kind == "baseline":
|
||
memo[version] = self._derive_baseline_test_result(pools, run_id)
|
||
else:
|
||
memo[version] = await self._eval_version_on_pool(
|
||
sv, pv, pools.test, run_id, context=f"held-out {version_kind}"
|
||
)
|
||
res = memo[version]
|
||
soft = await self._try_soft_score(run_id, pools.test)
|
||
mixed = None if soft is None else 0.5 * res.accuracy + 0.5 * soft
|
||
write_holdout_eval(
|
||
str(self._paths.db_path),
|
||
run_id=self._config.run_id,
|
||
epoch=epoch,
|
||
version_kind=version_kind,
|
||
hard_acc=res.accuracy,
|
||
soft_score=soft,
|
||
mixed_score=mixed,
|
||
per_task_type_json=json.dumps(res.per_task_type, ensure_ascii=False),
|
||
)
|
||
|
||
def _derive_baseline_test_result(self, pools: Pools, run_id: str) -> InferenceResult:
|
||
"""从基线 run 的 predictions 推导 test 池评估结果（0 推理）。
|
||
|
||
基线 run（pools.baseline_run_id）已对全题库推理并落库，test 题在其中；此处
|
||
按 test 题回读基线预测、经 _aggregate_results 折叠为 unit 级 InferenceResult，
|
||
避免重复推理基线版本（同版本不重采样）。
|
||
|
||
参数:
|
||
pools: 冻结三池（提供 test 与 baseline_run_id）。
|
||
run_id: 本次 holdout baseline 向的 run_id（仅用作结果标识）。
|
||
返回:
|
||
unit 级 InferenceResult（accuracy / per_task_type 与真评同口径）。
|
||
"""
|
||
from app.harness.inference import _aggregate_results
|
||
from app.harness.log import HarnessLog
|
||
|
||
qids = [q.question_id for q in pools.test]
|
||
with HarnessLog(
|
||
str(self._paths.db_path), pools.baseline_run_id, register_run=False
|
||
) as log:
|
||
placeholders = ", ".join(["?"] * len(qids))
|
||
rows = log.query(
|
||
f"SELECT * FROM predictions WHERE run_id=? AND question_id IN ({placeholders})",
|
||
(pools.baseline_run_id, *qids),
|
||
)
|
||
return _aggregate_results(rows, pools.test, run_id)
|
||
|
||
async def _pick_mixed_best(
|
||
self,
|
||
epoch: int,
|
||
pools: Pools,
|
||
state: _TrainState,
|
||
eval_skills_version: str,
|
||
eval_prompts_version: str,
|
||
) -> tuple[str, str] | None:
|
||
"""在 val 池对候选集算 mixed，落 shadow_gate，返回 argmax mixed 版本。
|
||
|
||
只观测落库，绝不改 manifest/best/early-stop。
|
||
"""
|
||
candidates = {
|
||
"best_hard": (state.best_skills_version, state.best_prompts_version),
|
||
"final": (eval_skills_version, eval_prompts_version),
|
||
}
|
||
best_kind: str | None = None
|
||
best_mixed: float | None = None
|
||
for kind, (sv, pv) in candidates.items():
|
||
run_id = f"{self._config.run_id}_shadow_{kind}_e{epoch}"
|
||
res = await self._eval_version_on_pool(
|
||
sv, pv, pools.validation, run_id, context=f"mixed 影子 {kind}"
|
||
)
|
||
soft = await self._try_soft_score(run_id, pools.validation)
|
||
mixed = None if soft is None else 0.5 * res.accuracy + 0.5 * soft
|
||
write_shadow_gate(
|
||
str(self._paths.db_path),
|
||
run_id=self._config.run_id,
|
||
epoch=epoch,
|
||
candidate_version=f"skills/{sv}+prompts/{pv}",
|
||
hard_acc=res.accuracy,
|
||
soft_score=soft,
|
||
mixed_score=mixed,
|
||
is_mixed_best=False,
|
||
)
|
||
if mixed is not None and (best_mixed is None or mixed > best_mixed):
|
||
best_mixed, best_kind = mixed, kind
|
||
if best_kind is None:
|
||
return None
|
||
self._mark_shadow_best(epoch, candidates[best_kind])
|
||
return candidates[best_kind]
|
||
|
||
def _mark_shadow_best(self, epoch: int, best_version: tuple[str, str]) -> None:
|
||
"""回标 shadow_gate 中 argmax mixed 选中的版本 is_mixed_best=1。"""
|
||
sv, pv = best_version
|
||
candidate_version = f"skills/{sv}+prompts/{pv}"
|
||
conn = sqlite3.connect(str(self._paths.db_path))
|
||
try:
|
||
conn.execute(
|
||
"UPDATE shadow_gate SET is_mixed_best=1 WHERE rowid = ("
|
||
" SELECT rowid FROM shadow_gate "
|
||
" WHERE run_id=? AND epoch=? AND candidate_version=? LIMIT 1"
|
||
")",
|
||
(self._config.run_id, epoch, candidate_version),
|
||
)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
# -----------------------------------------------------------------------
|
||
# 收尾
|
||
# -----------------------------------------------------------------------
|
||
|
||
def _deliver_best(self, best_skills_version: str, best_prompts_version: str) -> None:
|
||
"""若当前 current 不是历史最优，回滚 manifest 到 best 并刷新路径。"""
|
||
cur = load_manifest(self._config.workspace_dir)["current"]
|
||
if (
|
||
cur["skills"] != f"skills/{best_skills_version}"
|
||
or cur["prompts"] != f"prompts/{best_prompts_version}"
|
||
):
|
||
update_manifest(
|
||
self._config.workspace_dir,
|
||
skills=f"skills/{best_skills_version}",
|
||
prompts=f"prompts/{best_prompts_version}",
|
||
)
|
||
self._paths = resolve_paths(self._config.workspace_dir)
|
||
logger.info(
|
||
"收尾交付 best：current → skills/{} prompts/{}",
|
||
best_skills_version,
|
||
best_prompts_version,
|
||
)
|
||
|
||
async def _final_test_eval(self, pools: Pools) -> None:
|
||
"""收尾在 held-out test 池跑一次评估。"""
|
||
run_id = f"{self._config.run_id}_final_test"
|
||
result = await self._run_inference_on_pool(
|
||
pools.test, run_id, self._paths.skills_dir, self._paths.prompts_dir
|
||
)
|
||
_guard_infra_failures(result, context="held-out test 评估")
|
||
report = {
|
||
"run_id": result.run_id,
|
||
"accuracy": result.accuracy,
|
||
"total": result.total,
|
||
"correct": result.correct,
|
||
"per_task_type": result.per_task_type,
|
||
}
|
||
path = self._config.workspace_dir / "analyses" / "final_test_eval.json"
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
logger.info("held-out test 评估写入: {} (acc={:.1%})", path, result.accuracy)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# 私有辅助
|
||
# -----------------------------------------------------------------------
|
||
|
||
def _current_version(self, kind: str) -> str:
|
||
"""读取 manifest current 指针中某类资源的当前版本名。"""
|
||
return load_manifest(self._config.workspace_dir)["current"][kind].split("/")[-1]
|
||
|
||
def _current_version_snapshot(self) -> dict[str, str]:
|
||
"""读 manifest.current 的 skills/prompts 指针。"""
|
||
cur = load_manifest(self._config.workspace_dir)["current"]
|
||
return {"skills": cur["skills"], "prompts": cur["prompts"]}
|
||
|
||
def _class_baseline_acc(
|
||
self,
|
||
task_type: str,
|
||
validation: list[GeneratedQuestion],
|
||
correctness: dict[str, bool],
|
||
) -> float:
|
||
"""该 task_type 验证子集在当前 correctness 下的准确率。"""
|
||
class_items = [q for q in validation if q.task_type == task_type]
|
||
assert class_items, f"task_type={task_type} 在验证池中无对应题目"
|
||
correct = sum(1 for q in class_items if correctness.get(q.question_id, False))
|
||
return correct / len(class_items)
|
||
|
||
def _promote_skill_version(self, content: str, target_file: str) -> str:
|
||
"""把候选 skill 内容写成新正式 skills 版本。"""
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
src = Path(tmp) / "skills"
|
||
shutil.copytree(self._paths.skills_dir, src, dirs_exist_ok=True)
|
||
(src / target_file).write_text(content, encoding="utf-8")
|
||
return advance_version(
|
||
self._paths.workspace_dir,
|
||
"skills",
|
||
src,
|
||
{
|
||
"source": "evolution",
|
||
"parent": self._current_version("skills"),
|
||
"description": f"按类 gate accept {target_file}",
|
||
},
|
||
)
|
||
|
||
def _group_changed_task_types_by_file(
|
||
self, changed_task_types: set[str]
|
||
) -> dict[str, list[str]]:
|
||
"""把改过的 task_type 集合经 fallback 解析映射到 skill 文件，按文件分组。"""
|
||
from app.harness.workspace import VersionedSkillStore
|
||
|
||
skill_store = VersionedSkillStore(self._paths.skills_dir)
|
||
grouped: dict[str, list[str]] = {}
|
||
for task_type in changed_task_types:
|
||
skill_file = resolve_skill_file(skill_store, task_type)
|
||
grouped.setdefault(skill_file, []).append(task_type)
|
||
return grouped
|
||
|
||
def _record_run(self, run_id: str) -> Path:
|
||
"""将 current 版本快照追加到 manifest history，创建 run 目录。"""
|
||
from app.harness.workspace import record_run
|
||
|
||
return record_run(self._config.workspace_dir, run_id)
|
||
|
||
def _backfill_run_versions(
|
||
self, run_id: str, skills_version: str, prompts_version: str
|
||
) -> None:
|
||
"""eval run 的 skills/prompts 版本对 + questions_ref 回填进 _runs。"""
|
||
from app.harness.log import HarnessLog
|
||
|
||
with HarnessLog(str(self._paths.db_path), run_id) as log:
|
||
log.execute(
|
||
"UPDATE _runs SET skills_version = ?, prompts_version = ?, "
|
||
"questions_ref = ? WHERE run_id = ?",
|
||
(skills_version, prompts_version, self._config.questions, run_id),
|
||
)
|
||
|
||
def _write_eval_report(
|
||
self,
|
||
run_id: str,
|
||
skills_version: str,
|
||
prompts_version: str,
|
||
result: InferenceResult,
|
||
) -> None:
|
||
"""写 eval 评测报告 analyses/eval_{run_id}.json。"""
|
||
report = {
|
||
"run_id": run_id,
|
||
"skills_version": skills_version,
|
||
"prompts_version": prompts_version,
|
||
"accuracy": result.accuracy,
|
||
"total": result.total,
|
||
"correct": result.correct,
|
||
"stop_reason_counts": result.stop_reason_counts,
|
||
}
|
||
analyses_dir = self._config.workspace_dir / "analyses"
|
||
analyses_dir.mkdir(parents=True, exist_ok=True)
|
||
path = analyses_dir / f"eval_{run_id}.json"
|
||
path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
logger.info("eval 报告写入: {} (acc={:.1%})", path, result.accuracy)
|
||
|
||
async def _run_inference_on_pool(
|
||
self,
|
||
questions: list[GeneratedQuestion],
|
||
run_id: str,
|
||
skills_dir: Path,
|
||
prompts_dir: Path,
|
||
) -> InferenceResult:
|
||
"""用指定版本在给定题池跑一次 run_inference。"""
|
||
from app.harness.inference import run_inference
|
||
from app.harness.log import HarnessLog
|
||
|
||
self._record_run(run_id)
|
||
with HarnessLog(str(self._paths.db_path), run_id) as log:
|
||
return await run_inference(
|
||
questions=questions,
|
||
llm=self._llm,
|
||
tool_dispatch_fn=self._make_tool_dispatch_fn(skills_dir=skills_dir),
|
||
prompt_builder=self._make_prompt_builder(
|
||
skills_dir=skills_dir, prompts_dir=prompts_dir
|
||
),
|
||
log=log,
|
||
run_id=run_id,
|
||
concurrency=self._config.concurrency,
|
||
max_steps=self._config.max_steps,
|
||
skill_mode=self._config.skill_mode,
|
||
)
|
||
|
||
async def _eval_version_on_pool(
|
||
self,
|
||
skills_version: str,
|
||
prompts_version: str,
|
||
questions: list[GeneratedQuestion],
|
||
run_id: str,
|
||
context: str,
|
||
) -> InferenceResult:
|
||
"""用指定版本在给定池跑一次推理并护栏。"""
|
||
skills_dir = self._paths.workspace_dir / "skills" / skills_version
|
||
prompts_dir = self._paths.workspace_dir / "prompts" / prompts_version
|
||
result = await self._run_inference_on_pool(questions, run_id, skills_dir, prompts_dir)
|
||
_guard_infra_failures(result, context=context)
|
||
return result
|
||
|
||
async def _run_diagnosis(
|
||
self, run_id: str, *, question_ids: list[str] | None = None
|
||
) -> DiagnosisResult:
|
||
"""执行两阶段诊断。"""
|
||
from app.harness.baseline_run_log import StepsJsonRunLog
|
||
from app.harness.log import RunLogImpl
|
||
from app.harness.workspace import VersionedSkillStore
|
||
from app.question_gen import load_benchmark
|
||
from core.evolution.diagnose import run_diagnosis
|
||
|
||
questions = load_benchmark(self._paths.questions_dir)
|
||
# traces 表空时（如训练 rollout 只落 steps_json）从 steps_json 重建轨迹，恢复算法 #7
|
||
run_log = StepsJsonRunLog(RunLogImpl(str(self._paths.db_path)))
|
||
skill_store = VersionedSkillStore(self._paths.skills_dir)
|
||
diagnose_prompts = self._load_diagnose_prompts()
|
||
|
||
from app.harness.tree_nodes import load_tree_data_for_videos
|
||
|
||
if question_ids is not None:
|
||
qid_set = set(question_ids)
|
||
video_ids = [q.video_id for q in questions if q.question_id in qid_set]
|
||
else:
|
||
video_ids = [q.video_id for q in questions]
|
||
tree_data = load_tree_data_for_videos(Path(self._config.store_dir), video_ids)
|
||
|
||
return await run_diagnosis(
|
||
run_id=run_id,
|
||
questions=questions,
|
||
tree_data=tree_data,
|
||
llm=self._llm,
|
||
run_log=run_log,
|
||
skill_store=skill_store,
|
||
prompts=diagnose_prompts,
|
||
concurrency=self._config.concurrency,
|
||
question_ids=question_ids,
|
||
)
|
||
|
||
async def _try_soft_score(
|
||
self, run_id: str, questions: list[GeneratedQuestion]
|
||
) -> float | None:
|
||
"""尝试计算 soft score，失败降级为 None。"""
|
||
try:
|
||
# soft score 暂不实现（需诊断 span_evaluations 表），降级为 None
|
||
return None
|
||
except Exception:
|
||
logger.warning("soft score 计算失败（run={}），降级为 None", run_id)
|
||
return None
|
||
|
||
# -----------------------------------------------------------------------
|
||
# 注入工厂（暂用占位，由 main.py 绑定实际实现）
|
||
# -----------------------------------------------------------------------
|
||
|
||
def _make_tool_dispatch_fn(self, *, skills_dir: Path | None = None):
|
||
"""构造工具调度函数（优先使用注入的工厂，否则 noop 降级）。"""
|
||
if self._tool_dispatch_factory is not None:
|
||
return self._tool_dispatch_factory(skills_dir=skills_dir)
|
||
|
||
# noop fallback：diagnose/promote 等不需要推理的模式
|
||
async def _noop_dispatch(tool_name: str, args: dict, *, context: dict) -> str:
|
||
raise NotImplementedError(f"工具 {tool_name} 调度未配置")
|
||
|
||
return _noop_dispatch
|
||
|
||
def _make_prompt_builder(
|
||
self, *, skills_dir: Path | None = None, prompts_dir: Path | None = None
|
||
):
|
||
"""构造 prompt 构建函数（优先使用注入的工厂，否则 noop 降级）。"""
|
||
if self._prompt_builder_factory is not None:
|
||
return self._prompt_builder_factory(skills_dir=skills_dir, prompts_dir=prompts_dir)
|
||
|
||
# noop fallback：diagnose/promote 等不需要推理的模式
|
||
def _noop_builder(qa: GeneratedQuestion) -> tuple[str, str]:
|
||
raise NotImplementedError("prompt_builder 未配置")
|
||
|
||
return _noop_builder
|
||
|
||
def _make_validate_run_inference_fn(self, gate_log: HarnessLog):
|
||
"""构造 validate 用的 RunInferenceFn(绑定共享依赖与共享 HarnessLog)。
|
||
|
||
连续并发 gate 下本函数被逐单元高频并发调用:每次调用新建 HarnessLog
|
||
连接会重现多连接争 SQLite 写锁(遥测同款教训),故复用调用方传入的
|
||
单一 gate_log(单连接 + threading.Lock 串行化)。_record_run 按 run_id
|
||
去重,避免逐单元重复 upsert。
|
||
|
||
参数:
|
||
gate_log: 本 step gate 阶段共享的 HarnessLog 实例。
|
||
|
||
返回:
|
||
符合 RunInferenceFn 协议的异步推理函数。
|
||
"""
|
||
from app.harness.inference import run_inference
|
||
|
||
recorded: set[str] = set()
|
||
|
||
async def _run(
|
||
questions: list[GeneratedQuestion],
|
||
*,
|
||
run_id: str,
|
||
skills_dir: Path,
|
||
) -> InferenceResult:
|
||
if run_id not in recorded:
|
||
recorded.add(run_id)
|
||
self._record_run(run_id)
|
||
return await run_inference(
|
||
questions=questions,
|
||
llm=self._llm,
|
||
tool_dispatch_fn=self._make_tool_dispatch_fn(skills_dir=skills_dir),
|
||
prompt_builder=self._make_prompt_builder(
|
||
skills_dir=skills_dir, prompts_dir=self._paths.prompts_dir
|
||
),
|
||
log=gate_log,
|
||
run_id=run_id,
|
||
concurrency=self._config.concurrency,
|
||
max_steps=self._config.max_steps,
|
||
skill_mode=self._config.skill_mode,
|
||
)
|
||
|
||
return _run
|
||
|
||
def _load_evolve_prompts(self):
|
||
"""加载进化模板束（从项目根 prompts/ 读取诊断标尺模板）。"""
|
||
from core.evolution.types import EvolvePrompts
|
||
|
||
def _read(name: str) -> str:
|
||
p = Path("prompts") / name
|
||
if not p.exists():
|
||
raise FileNotFoundError(f"缺进化/诊断模板: {p}（请从 TRM4 迁移或检查 prompts/）")
|
||
return p.read_text(encoding="utf-8")
|
||
|
||
return EvolvePrompts(
|
||
evolve_skill=_read("evolve_skill.md"),
|
||
evolve_system=_read("evolve_system.md"),
|
||
evolve_tool=_read("evolve_tool.md"),
|
||
evolve_rank=_read("evolve_rank.md"),
|
||
)
|
||
|
||
def _load_diagnose_prompts(self):
|
||
"""加载诊断模板束（从项目根 prompts/ 读取）。"""
|
||
from core.evolution.types import DiagnosePrompts
|
||
|
||
def _read(name: str) -> str:
|
||
p = Path("prompts") / name
|
||
if not p.exists():
|
||
raise FileNotFoundError(f"缺进化/诊断模板: {p}（请从 TRM4 迁移或检查 prompts/）")
|
||
return p.read_text(encoding="utf-8")
|
||
|
||
return DiagnosePrompts(
|
||
defect_vs_lapse=_read("defect_vs_lapse.md"),
|
||
reasoning_sub=_read("reasoning_sub.md"),
|
||
span_eval_system=_read("span_eval_system.md"),
|
||
missed_nodes=_read("missed_nodes.md"),
|
||
skill_adherence=_read("skill_adherence.md"),
|
||
confirmation_bias=_read("confirmation_bias.md"),
|
||
evidence_sufficiency=_read("evidence_sufficiency.md"),
|
||
)
|