iomgaa
2296134f73
Three-level nesting (epoch -> step -> per-skill), slow update 10-step sequence, checkpoint/resume, early stop, probation accept/reject/rollback. Key TRM4->TRM5 changes: - sync -> async (all inference/diagnosis/evolve/validate awaited) - LLMClient.from_env -> injected LLMProvider (DI via constructor) - Direct DB/file access -> module functions (workspace/store/log) - _TrainState as train() local, explicit param passing to helpers Module-level pure functions extracted for testability: resume_plan, _guard_infra_failures, _apply_batch_correctness, _compute_total_steps, _should_early_stop, _format_applied_edits, _fallback_summary, _write_skip_report, _outcome_to_quadrant_pairs, _build_comparison_pairs, _batch_from_ids, _snapshot_current_skills. Tests: 34 unit tests covering 13a-13e sub-tasks. Radon: all functions Grade B or better.
2147 lines
83 KiB
Python
2147 lines
83 KiB
Python
"""实验运行器（瘦编排器），对标 PyTorch Trainer。
|
||
|
||
三级嵌套（epoch → step → per-skill）训练循环 + 慢更新十步序 + 断点续训。
|
||
算法保真 #13：训练循环编排从 TRM4 runner.py（2273 行）迁移，逻辑不可简化。
|
||
|
||
关键重构（TRM4 → TRM5）：
|
||
- sync → async（await run_inference / run_diagnosis / evolve_* / validate_*）
|
||
- LLMClient.from_env → 注入 LLMProvider（self._llm / self._evolve_llm）
|
||
- 直接 DB/文件操作 → 通过模块函数（workspace / store / log / observation）
|
||
- 瘦身 2273 → ~500 行（推理/诊断/进化/验证全委托模块函数）
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import math
|
||
import random
|
||
import shutil
|
||
import sqlite3
|
||
import tempfile
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import TYPE_CHECKING, Any
|
||
|
||
from loguru import logger
|
||
|
||
from app.harness.batching import build_batches
|
||
from app.harness.checkpoint import (
|
||
check_fingerprint,
|
||
deserialize_state_fields,
|
||
load_checkpoint,
|
||
write_checkpoint,
|
||
)
|
||
from app.harness.config import RunConfig # noqa: TC001 — 运行时 _compute_total_steps 使用
|
||
from app.harness.gate_ladder import BaselineCache, GatePools, build_or_load_gate_pools
|
||
from app.harness.observation import (
|
||
write_dual_metric,
|
||
write_epoch_report,
|
||
write_gate_evidence,
|
||
write_holdout_eval,
|
||
write_quadrant_pairs,
|
||
write_shadow_gate,
|
||
write_step_report,
|
||
)
|
||
from app.harness.store import advance_version
|
||
from app.harness.validate import Probation, ValidationOutcome
|
||
from app.harness.workspace import (
|
||
ResolvedPaths,
|
||
archive_workspace,
|
||
init_workspace,
|
||
init_workspace_from_seed,
|
||
load_manifest,
|
||
read_best,
|
||
resolve_paths,
|
||
update_best,
|
||
update_manifest,
|
||
)
|
||
from core.evolution import (
|
||
DiagnosisResult,
|
||
GateParams,
|
||
RejectedEdit,
|
||
edit_budget_at,
|
||
momentum_inner,
|
||
probation_verdict,
|
||
replace_momentum,
|
||
resolve_skill_file,
|
||
)
|
||
from core.evolution.diagnose import merge_system_packs, merge_tool_packs
|
||
|
||
if TYPE_CHECKING:
|
||
from app.harness.inference import InferenceResult
|
||
from app.harness.pools import Pools
|
||
from core.evolution.types import (
|
||
EvolutionRecord,
|
||
SystemCasePack,
|
||
ToolCasePack,
|
||
)
|
||
from core.protocols import LLMProvider, TelemetryRecorder, VLMProvider
|
||
from core.types import GeneratedQuestion
|
||
|
||
|
||
class _InterruptError(RuntimeError):
|
||
"""测试用中断注入信号：_run_step 末尾可选抛出以模拟进程中断。"""
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _TrainState: 19 个可变字段
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass
|
||
class _TrainState:
|
||
"""一次 train() 的跨 step 可变状态（训练循环的"权重/缓冲"）。
|
||
|
||
字段说明见 TRM4 同名 dataclass（完整保留 19 字段语义）。
|
||
TRM5 移除 evolve_client（改走构造注入），其余 18 字段 + gate_epoch_observed 不变。
|
||
"""
|
||
|
||
correctness: dict[str, bool]
|
||
gate_pools: GatePools
|
||
baseline_cache: BaselineCache
|
||
eval_prev_acc: float
|
||
eval_prev_run_id: str
|
||
best_val_acc: float
|
||
best_skills_version: str
|
||
best_prompts_version: str
|
||
baseline_skills_version: str = ""
|
||
baseline_prompts_version: str = ""
|
||
rejected_buffer: dict[str, list[RejectedEdit]] = field(default_factory=dict)
|
||
system_packs: list[SystemCasePack] = field(default_factory=list)
|
||
tool_packs: list[ToolCasePack] = field(default_factory=list)
|
||
global_step: int = 0
|
||
changed_task_types_this_epoch: set[str] = field(default_factory=set)
|
||
epoch_start_skills: dict[str, str] = field(default_factory=dict)
|
||
steps_since_best_improved: int = 0
|
||
gate_epoch_observed: bool = False
|
||
probations: dict[str, Probation] = field(default_factory=dict)
|
||
gate_cooldown: dict[str, int] = field(default_factory=dict)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 纯函数辅助（不依赖 self）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def resume_plan(epoch: int, phase: str, step_completed: int) -> dict:
|
||
"""据 checkpoint 进度算续跑计划（纯函数，便于单测）。
|
||
|
||
参数:
|
||
epoch: checkpoint 落库时的 epoch 序号。
|
||
phase: "in_epoch" 或 "epoch_done"。
|
||
step_completed: 该 epoch 内最后完整完成的 step 序号。
|
||
|
||
返回:
|
||
{"first_epoch": int, "resume_epoch": int | None, "resume_step_from": int}。
|
||
"""
|
||
if phase == "epoch_done":
|
||
return {"first_epoch": epoch + 1, "resume_epoch": None, "resume_step_from": 0}
|
||
return {
|
||
"first_epoch": epoch,
|
||
"resume_epoch": epoch,
|
||
"resume_step_from": step_completed + 1,
|
||
}
|
||
|
||
|
||
def _guard_infra_failures(result: InferenceResult, context: str) -> None:
|
||
"""基础设施失败护栏：stop_reason="error" 占比 > 10% 即硬终止。
|
||
|
||
参数:
|
||
result: 推理聚合结果。
|
||
context: 出错时报错的推理路径名（仅诊断用）。
|
||
|
||
异常:
|
||
RuntimeError: error 占比 > 10%。
|
||
"""
|
||
error_rate = result.stop_reason_counts.get("error", 0) / max(result.total, 1)
|
||
if error_rate > 0.1:
|
||
raise RuntimeError(
|
||
f"{context} 推理基础设施失败率过高 {error_rate:.0%}（stop_reason=error），中止本轮"
|
||
)
|
||
|
||
|
||
def _apply_batch_correctness(
|
||
correctness: dict[str, bool],
|
||
log: Any,
|
||
run_id: str,
|
||
batch: list[GeneratedQuestion],
|
||
) -> None:
|
||
"""从该 run 的 predictions 读 batch 各题新对错，就地增量更新 correctness。
|
||
|
||
参数:
|
||
correctness: question_id -> 是否答对，就地更新。
|
||
log: HarnessLog 实例。
|
||
run_id: rollout 的 run_id。
|
||
batch: 本 step 的题目列表。
|
||
|
||
异常:
|
||
RuntimeError: rollout 不完整（缺预测行）。
|
||
"""
|
||
from app.harness.validate import _load_run_rows
|
||
|
||
rows = _load_run_rows(log, run_id)
|
||
missing = [q.question_id for q in batch if q.question_id not in rows]
|
||
if missing:
|
||
raise RuntimeError(
|
||
f"rollout 不完整：run_id={run_id} 缺 {len(missing)} 道题预测行 {missing}，中止本步"
|
||
)
|
||
for q in batch:
|
||
correctness[q.question_id] = rows[q.question_id]["_correct"]
|
||
|
||
|
||
def _accumulate_slow_packs(diagnosis: DiagnosisResult, state: _TrainState) -> None:
|
||
"""把本 step 诊断的 system/tool 案例包只累加不更新，留给 epoch 末慢更新消费。"""
|
||
if diagnosis.system_case_pack is not None:
|
||
state.system_packs.append(diagnosis.system_case_pack)
|
||
state.tool_packs.extend(diagnosis.tool_case_packs.values())
|
||
|
||
|
||
def _batch_from_ids(pools: Pools, ids: list[str]) -> list[GeneratedQuestion]:
|
||
"""按 question_id 从诊断池重建一个 batch（保持原 epoch 划分）。
|
||
|
||
参数:
|
||
pools: 三池容器。
|
||
ids: 一个 batch 的 question_id 列表。
|
||
|
||
返回:
|
||
按 ids 顺序取出的 GeneratedQuestion 列表。
|
||
"""
|
||
by_id = {q.question_id: q for q in pools.diagnosis}
|
||
return [by_id[i] for i in ids]
|
||
|
||
|
||
def _snapshot_current_skills(skills_dir: Path) -> dict[str, str]:
|
||
"""快照当前 skills 版本目录下各 skill 文件的正文（文件名 -> 全文）。
|
||
|
||
参数:
|
||
skills_dir: 当前 skills 版本目录。
|
||
|
||
返回:
|
||
{文件名: 全文}，作 momentum 的上一版基准。
|
||
"""
|
||
snapshot: dict[str, str] = {}
|
||
for path in sorted(skills_dir.glob("*.md")):
|
||
snapshot[path.name] = path.read_text(encoding="utf-8")
|
||
return snapshot
|
||
|
||
|
||
def _should_early_stop(
|
||
workspace_dir: Path,
|
||
epoch: int,
|
||
steps_this_epoch: int,
|
||
state: _TrainState,
|
||
patience: int,
|
||
) -> bool:
|
||
"""步粒度 early stop：本 epoch best 未刷新则累加本 epoch 步数。
|
||
|
||
参数:
|
||
workspace_dir: workspace 目录（读 manifest best）。
|
||
epoch: 当前 epoch。
|
||
steps_this_epoch: 本 epoch 的 step 总数。
|
||
state: 训练状态（steps_since_best_improved 就地更新）。
|
||
patience: early_stop_patience。
|
||
|
||
返回:
|
||
是否触发 early stop。
|
||
"""
|
||
best = read_best(workspace_dir)
|
||
improved_this_epoch = best is not None and best.get("epoch") == epoch
|
||
if improved_this_epoch:
|
||
state.steps_since_best_improved = 0
|
||
return False
|
||
state.steps_since_best_improved += steps_this_epoch
|
||
return state.steps_since_best_improved >= patience
|
||
|
||
|
||
def _compute_total_steps(pools: Pools, correctness: dict[str, bool], config: RunConfig) -> int:
|
||
"""退火地平线：用 build_batches 试切一轮拿 selected_count，再乘 epochs。"""
|
||
_, selected_count = build_batches(
|
||
pools.diagnosis,
|
||
correctness,
|
||
config.batch_size,
|
||
config.min_class_per_batch,
|
||
seed=1,
|
||
correct_ratio=config.batch_correct_ratio,
|
||
)
|
||
steps_per_epoch = max(1, math.ceil(selected_count / config.batch_size))
|
||
return config.epochs * steps_per_epoch
|
||
|
||
|
||
def _outcome_to_quadrant_pairs(task_type: str, outcome: ValidationOutcome) -> list[dict]:
|
||
"""把 ValidationOutcome 的四象限拍平为逐题 pair（供 quadrant_pair 表落库观测）。
|
||
|
||
参数:
|
||
task_type: 该批 gate 的任务类型。
|
||
outcome: 局部验证决策结果。
|
||
|
||
返回:
|
||
每条含 question_id/task_type/prev_correct/curr_correct/category 的 dict 列表。
|
||
"""
|
||
from app.harness.momentum import (
|
||
IMPROVED,
|
||
PERSISTENT_FAIL,
|
||
REGRESSED,
|
||
STABLE_SUCCESS,
|
||
)
|
||
|
||
spec = [
|
||
(outcome.improvements, IMPROVED, False, True),
|
||
(outcome.regressions, REGRESSED, True, False),
|
||
(outcome.persistent_fails, PERSISTENT_FAIL, False, False),
|
||
(outcome.stable_successes, STABLE_SUCCESS, True, True),
|
||
]
|
||
pairs: list[dict] = []
|
||
for qids, category, prev_ok, curr_ok in spec:
|
||
for qid in qids:
|
||
pairs.append(
|
||
{
|
||
"question_id": qid,
|
||
"task_type": task_type,
|
||
"prev_correct": prev_ok,
|
||
"curr_correct": curr_ok,
|
||
"category": category,
|
||
}
|
||
)
|
||
return pairs
|
||
|
||
|
||
def _build_comparison_pairs(
|
||
sampled: list[GeneratedQuestion],
|
||
prev_rows: dict[str, dict],
|
||
curr_rows: dict[str, dict],
|
||
) -> list[dict]:
|
||
"""为采样好的诊断池题目构造 momentum 纵向对比对。"""
|
||
pairs: list[dict] = []
|
||
for q in sampled:
|
||
prev = prev_rows.get(q.question_id, {})
|
||
curr = curr_rows.get(q.question_id, {})
|
||
pairs.append(
|
||
{
|
||
"question": q.question,
|
||
"prev_prediction": prev.get("prediction", ""),
|
||
"curr_prediction": curr.get("prediction", ""),
|
||
"correct_prev": prev.get("_correct", False),
|
||
"correct_curr": curr.get("_correct", False),
|
||
}
|
||
)
|
||
return pairs
|
||
|
||
|
||
def _filter_applied_edits(edits: list[dict], reports: list[dict]) -> list[dict] | str:
|
||
"""按 apply_report 过滤出真正 applied 的 edit。
|
||
|
||
参数:
|
||
edits: EvolutionRecord.edits 列表。
|
||
reports: EvolutionRecord.apply_report 列表（与 edits 同序对齐）。
|
||
|
||
返回:
|
||
过滤后的 edit 列表；0 applied 时返回信息性消息字符串。
|
||
reports 为空时返回原 edits 不过滤。
|
||
"""
|
||
if not reports:
|
||
return edits
|
||
applied = [
|
||
edit
|
||
for edit, report in zip(edits, reports, strict=True)
|
||
if str(report.get("status", "")).startswith("applied")
|
||
]
|
||
if not applied:
|
||
return "上轮改法全部未成功应用(0 applied)，本条无已验证信息"
|
||
return applied
|
||
|
||
|
||
def _format_applied_edits(record: Any) -> str | None:
|
||
"""从 EvolutionRecord 中提取真正 applied 的 edit 并格式化为摘要。
|
||
|
||
参数:
|
||
record: EvolutionRecord（duck-typed，需含 edits / apply_report）。
|
||
|
||
返回:
|
||
已 applied edit 的格式化摘要；无 edit 或无 applied 时返回 None，
|
||
0 applied 时返回信息性消息（非 None）。
|
||
"""
|
||
rec_edits = getattr(record, "edits", []) or []
|
||
if not rec_edits:
|
||
return None
|
||
filtered = _filter_applied_edits(rec_edits, getattr(record, "apply_report", []) or [])
|
||
if isinstance(filtered, str):
|
||
return filtered
|
||
summary = "; ".join(
|
||
f"[{edit.get('op')}]{(edit.get('target') or edit.get('content', ''))[:40]}"
|
||
for edit in filtered
|
||
if isinstance(edit, dict)
|
||
)
|
||
return summary or None
|
||
|
||
|
||
def _fallback_summary(record: Any, outcome: Any) -> str:
|
||
"""从 suggestions 或跌幅信息构造兜底黑名单摘要。
|
||
|
||
参数:
|
||
record: EvolutionRecord（duck-typed，需含 suggestions）。
|
||
outcome: ValidationOutcome（duck-typed，需含 delta_hat）。
|
||
|
||
返回:
|
||
兜底摘要字符串。
|
||
"""
|
||
return "; ".join(s.get("change", "") for s in record.suggestions) or (
|
||
f"上轮对本文件改写被拒(delta {outcome.delta_hat:+.2f})，换方向"
|
||
)
|
||
|
||
|
||
def _write_skip_report(
|
||
workspace_dir: Path,
|
||
epoch: int,
|
||
step: int,
|
||
global_step: int,
|
||
task_type: str,
|
||
action: str,
|
||
baseline_acc: float,
|
||
budget: int,
|
||
rank_clip_triggered: bool = False,
|
||
) -> None:
|
||
"""为 cooldown / skipped 路径写 step_report（无 gate 证据）。
|
||
|
||
参数:
|
||
workspace_dir: 工作区目录。
|
||
epoch: 轮次。
|
||
step: epoch 内 step。
|
||
global_step: 全局步计数。
|
||
task_type: 任务类型。
|
||
action: "cooldown" 或 "skipped"。
|
||
baseline_acc: 当前类基线准确率。
|
||
budget: 编辑预算。
|
||
rank_clip_triggered: 是否触发 rank 裁剪（skipped 路径需要）。
|
||
"""
|
||
write_step_report(
|
||
workspace_dir,
|
||
epoch=epoch,
|
||
step=step,
|
||
global_step=global_step,
|
||
task_type=task_type,
|
||
gate_action=action,
|
||
candidate_acc=baseline_acc,
|
||
class_baseline_acc=baseline_acc,
|
||
edit_budget=budget,
|
||
rank_clip_triggered=rank_clip_triggered,
|
||
gate_w=None,
|
||
gate_l=None,
|
||
gate_e_value=None,
|
||
gate_n_used=None,
|
||
gate_stop_reason=None,
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Runner 主类
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class Runner:
|
||
"""实验运行器（瘦编排器），通过 RunConfig 驱动训练/推理/诊断/评估等模式。
|
||
|
||
DI 纪律：self 只持注入依赖 + _paths。_TrainState 是 train() 内局部变量，
|
||
显式传参给模块函数。
|
||
|
||
参数:
|
||
config: 运行配置。
|
||
llm: 推理用 LLMProvider。
|
||
evolve_llm: 进化用 LLMProvider（thinking=True）。
|
||
vlm: VLMProvider。
|
||
telemetry: 遥测记录端口。
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
config: RunConfig,
|
||
*,
|
||
llm: LLMProvider,
|
||
evolve_llm: LLMProvider,
|
||
vlm: VLMProvider,
|
||
telemetry: TelemetryRecorder,
|
||
) -> None:
|
||
self._config = config
|
||
self._llm = llm
|
||
self._evolve_llm = evolve_llm
|
||
self._vlm = vlm
|
||
self._telemetry = telemetry
|
||
self._ensure_workspace()
|
||
self._paths: ResolvedPaths = resolve_paths(config.workspace_dir)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# workspace 三态逻辑
|
||
# -----------------------------------------------------------------------
|
||
|
||
def _ensure_workspace(self) -> None:
|
||
"""train 模式按 --resume/--fresh 分三态；其余模式仅要求 ws 已存在并复用。
|
||
|
||
三态逻辑:
|
||
resume+fresh → ValueError
|
||
fresh+已有 → archive + init_from_seed
|
||
resume+无进度 → RuntimeError
|
||
无flag+已有 → SystemExit
|
||
"""
|
||
manifest = self._config.workspace_dir / "manifest.json"
|
||
has_progress = manifest.exists()
|
||
if self._config.mode != "train":
|
||
if not has_progress:
|
||
raise RuntimeError(
|
||
f"{self._config.mode} 模式要求 workspace 已存在: {self._config.workspace_dir}"
|
||
)
|
||
return
|
||
if self._config.resume and self._config.fresh:
|
||
raise ValueError("--resume 与 --fresh 互斥")
|
||
if self._config.fresh:
|
||
if has_progress:
|
||
logger.info(
|
||
"旧 workspace 已归档: {}",
|
||
archive_workspace(self._config.workspace_dir),
|
||
)
|
||
init_workspace_from_seed(
|
||
self._config.workspace_dir,
|
||
self._config.store_dir,
|
||
self._config.seed,
|
||
self._config.questions,
|
||
)
|
||
return
|
||
if self._config.resume:
|
||
if not has_progress:
|
||
raise RuntimeError("--resume 但 workspace 无已有进度")
|
||
return
|
||
if has_progress:
|
||
raise SystemExit("workspace 已有进度；用 --resume 续训 或 --fresh 归档重开")
|
||
init_workspace(
|
||
self._config.workspace_dir,
|
||
self._config.store_dir,
|
||
self._config.questions,
|
||
self._config.skills_version,
|
||
self._config.prompts_version,
|
||
)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# 公共入口：infer / eval / diagnose / promote
|
||
# -----------------------------------------------------------------------
|
||
|
||
async def infer(self, task_types: list[str] | None = None) -> InferenceResult:
|
||
"""执行单次推理（forward-only）。
|
||
|
||
参数:
|
||
task_types: 若非 None，只保留指定题型。
|
||
|
||
返回:
|
||
InferenceResult 冻结实例。
|
||
"""
|
||
from app.harness.inference import run_inference
|
||
from app.harness.log import HarnessLog
|
||
from app.question_gen import load_benchmark
|
||
|
||
questions = load_benchmark(self._paths.questions_dir)
|
||
if task_types:
|
||
allowed = set(task_types)
|
||
questions = [q for q in questions if q.task_type in allowed]
|
||
if self._config.n_samples > 0:
|
||
questions = questions[: self._config.n_samples]
|
||
|
||
run_id = f"infer_{self._config.run_id}" if self._config.run_id else "infer_adhoc"
|
||
record_run_dir = self._record_run(run_id) # noqa: F841
|
||
|
||
logger.info(
|
||
"启动推理: {} 道题, concurrency={}, max_steps={}, skill_mode={}",
|
||
len(questions),
|
||
self._config.concurrency,
|
||
self._config.max_steps,
|
||
self._config.skill_mode,
|
||
)
|
||
|
||
with HarnessLog(str(self._paths.db_path), run_id) as log:
|
||
return await run_inference(
|
||
questions=questions,
|
||
llm=self._llm,
|
||
tool_dispatch_fn=self._make_tool_dispatch_fn(),
|
||
prompt_builder=self._make_prompt_builder(),
|
||
log=log,
|
||
run_id=run_id,
|
||
concurrency=self._config.concurrency,
|
||
max_steps=self._config.max_steps,
|
||
skill_mode=self._config.skill_mode,
|
||
)
|
||
|
||
async def eval(self, version: str) -> InferenceResult:
|
||
"""用指定 skills 版本跑完整题库，全量记录落 db + 版本回填。
|
||
|
||
参数:
|
||
version: skills 版本号。
|
||
|
||
返回:
|
||
InferenceResult。
|
||
"""
|
||
from datetime import UTC, datetime
|
||
|
||
from app.harness.inference import run_inference
|
||
from app.harness.log import HarnessLog
|
||
from app.question_gen import load_benchmark
|
||
|
||
cur = load_manifest(self._config.workspace_dir)["current"]
|
||
prompts_v = cur["prompts"].split("/")[-1]
|
||
skills_dir = self._paths.workspace_dir / "skills" / version
|
||
prompts_dir = self._paths.workspace_dir / "prompts" / prompts_v
|
||
if not skills_dir.is_dir():
|
||
raise FileNotFoundError(f"skills 版本目录不存在: {skills_dir}")
|
||
if not prompts_dir.is_dir():
|
||
raise FileNotFoundError(f"prompts 版本目录不存在: {prompts_dir}")
|
||
|
||
questions = load_benchmark(self._paths.questions_dir)
|
||
if self._config.n_samples > 0:
|
||
questions = questions[: self._config.n_samples]
|
||
|
||
ts = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
|
||
run_id = f"eval_{version}-{prompts_v}_{ts}"
|
||
self._record_run(run_id)
|
||
logger.info(
|
||
"eval: 版本 skills/{}+prompts/{} 跑 {} 题 (run_id={})",
|
||
version,
|
||
prompts_v,
|
||
len(questions),
|
||
run_id,
|
||
)
|
||
|
||
with HarnessLog(str(self._paths.db_path), run_id) as log:
|
||
result = await run_inference(
|
||
questions=questions,
|
||
llm=self._llm,
|
||
tool_dispatch_fn=self._make_tool_dispatch_fn(skills_dir=skills_dir),
|
||
prompt_builder=self._make_prompt_builder(
|
||
skills_dir=skills_dir, prompts_dir=prompts_dir
|
||
),
|
||
log=log,
|
||
run_id=run_id,
|
||
concurrency=self._config.concurrency,
|
||
max_steps=self._config.max_steps,
|
||
skill_mode=self._config.skill_mode,
|
||
)
|
||
|
||
# C4 回填
|
||
self._backfill_run_versions(run_id, version, prompts_v)
|
||
self._write_eval_report(run_id, version, prompts_v, result)
|
||
return result
|
||
|
||
async def diagnose(self, run_id: str) -> DiagnosisResult:
|
||
"""执行指定 run 的两阶段诊断。
|
||
|
||
参数:
|
||
run_id: 待诊断的 run_id。
|
||
|
||
返回:
|
||
DiagnosisResult。
|
||
"""
|
||
|
||
return await self._run_diagnosis(run_id)
|
||
|
||
def promote(self, version: str, eval_run_id: str, name: str) -> None:
|
||
"""把当前 ws 的指定版本提升为 Store 新种子。
|
||
|
||
参数:
|
||
version: 要提升的 skills 版本号。
|
||
eval_run_id: canonical eval run。
|
||
name: 新种子名。
|
||
"""
|
||
from app.harness.store import promote_to_seed
|
||
|
||
seed_dir = promote_to_seed(
|
||
self._config.workspace_dir,
|
||
self._config.store_dir,
|
||
version,
|
||
eval_run_id,
|
||
name,
|
||
description=f"promote from {self._config.workspace_dir.name} {version}",
|
||
)
|
||
logger.info("已提升为种子: {}", seed_dir)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# train() 三级嵌套（核心）
|
||
# -----------------------------------------------------------------------
|
||
|
||
async def train(self, pools: Pools) -> None:
|
||
"""mini-batch 快慢双速闭环：epoch 内多 step、每 step 按类 per-skill gate。
|
||
|
||
三级嵌套：epoch → batch(step) → per-skill。
|
||
epoch 末 _slow_update_cycle 十步序。
|
||
训练收尾 _deliver_best + _final_test_eval。
|
||
"""
|
||
state, total_steps, plan, saved_batches = await self._setup_train_run(pools)
|
||
for epoch in range(plan["first_epoch"], self._config.epochs + 1):
|
||
if epoch == plan["resume_epoch"]:
|
||
batches = [_batch_from_ids(pools, ids) for ids in saved_batches]
|
||
step_from = plan["resume_step_from"]
|
||
else:
|
||
logger.info("=== Epoch {} ===", epoch)
|
||
state.system_packs = []
|
||
state.tool_packs = []
|
||
state.changed_task_types_this_epoch = set()
|
||
state.epoch_start_skills = _snapshot_current_skills(self._paths.skills_dir)
|
||
batches, _ = build_batches(
|
||
pools.diagnosis,
|
||
state.correctness,
|
||
self._config.batch_size,
|
||
self._config.min_class_per_batch,
|
||
seed=epoch,
|
||
correct_ratio=self._config.batch_correct_ratio,
|
||
)
|
||
step_from = 0
|
||
batch_ids = [[q.question_id for q in b] for b in batches]
|
||
for step in range(step_from, len(batches)):
|
||
await self._run_step(epoch, step, total_steps, batches[step], pools, state)
|
||
state.global_step += 1
|
||
write_checkpoint(
|
||
self._config.workspace_dir,
|
||
state=state,
|
||
epoch=epoch,
|
||
step_completed=step,
|
||
phase="in_epoch",
|
||
global_step=state.global_step,
|
||
total_steps=total_steps,
|
||
version_snapshot=self._current_version_snapshot(),
|
||
epoch_batches=batch_ids,
|
||
config=self._config,
|
||
)
|
||
await self._slow_update_cycle(epoch, pools, state)
|
||
state.system_packs = []
|
||
state.tool_packs = []
|
||
state.changed_task_types_this_epoch = set()
|
||
write_checkpoint(
|
||
self._config.workspace_dir,
|
||
state=state,
|
||
epoch=epoch,
|
||
step_completed=len(batches) - 1,
|
||
phase="epoch_done",
|
||
global_step=state.global_step,
|
||
total_steps=total_steps,
|
||
version_snapshot=self._current_version_snapshot(),
|
||
epoch_batches=batch_ids,
|
||
config=self._config,
|
||
)
|
||
if _should_early_stop(
|
||
self._config.workspace_dir,
|
||
epoch,
|
||
len(batches),
|
||
state,
|
||
self._config.early_stop_patience,
|
||
):
|
||
logger.info("Epoch {} 触发 early stop（best 连续无新高）", epoch)
|
||
break
|
||
self._deliver_best(state.best_skills_version, state.best_prompts_version)
|
||
await self._final_test_eval(pools)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# 训练初始化
|
||
# -----------------------------------------------------------------------
|
||
|
||
async def _setup_train_run(self, pools: Pools) -> tuple[_TrainState, int, dict, list | None]:
|
||
"""据是否 --resume 准备训练起点。
|
||
|
||
返回:
|
||
(state, total_steps, plan, saved_batches)。
|
||
"""
|
||
ckpt = load_checkpoint(self._config.workspace_dir) if self._config.resume else None
|
||
if self._config.resume and ckpt is None:
|
||
raise RuntimeError("--resume 但 checkpoint.json 不存在，拒绝静默从头重训")
|
||
gate_pools, baseline_cache = self._init_gate_pools(pools)
|
||
if not ckpt:
|
||
state = self._init_train_state(pools, gate_pools, baseline_cache)
|
||
total_steps = _compute_total_steps(pools, state.correctness, self._config)
|
||
plan = {"first_epoch": 1, "resume_epoch": None, "resume_step_from": 0}
|
||
return state, total_steps, plan, None
|
||
struct, decision = check_fingerprint(ckpt["config_fingerprint"], self._config)
|
||
if struct:
|
||
raise RuntimeError(f"结构性配置变化，拒绝 resume: {struct}")
|
||
if decision:
|
||
logger.warning("决策性配置变化，继续 resume: {}", decision)
|
||
state = self._restore_train_state(ckpt, pools, gate_pools, baseline_cache)
|
||
state.global_step = ckpt["progress"]["global_step"]
|
||
update_manifest(
|
||
self._config.workspace_dir,
|
||
skills=ckpt["version_snapshot"]["skills"],
|
||
prompts=ckpt["version_snapshot"]["prompts"],
|
||
)
|
||
self._paths = resolve_paths(self._config.workspace_dir)
|
||
plan = resume_plan(
|
||
ckpt["progress"]["epoch"],
|
||
ckpt["progress"]["phase"],
|
||
ckpt["progress"]["step_completed"],
|
||
)
|
||
return state, ckpt["progress"]["total_steps"], plan, ckpt["epoch_batches"]
|
||
|
||
def _init_gate_pools(self, pools: Pools) -> tuple[GatePools, BaselineCache]:
|
||
"""构建/加载 CE-Gate 信息量阶梯与基线缓存。
|
||
|
||
副作用：设置 self._gate_questions_by_id（不进 checkpoint）。
|
||
|
||
参数:
|
||
pools: 冻结三池。
|
||
|
||
返回:
|
||
(GatePools, BaselineCache)。
|
||
"""
|
||
from app.harness.log import HarnessLog
|
||
from app.question_gen import load_benchmark
|
||
|
||
questions = load_benchmark(self._paths.questions_dir)
|
||
self._gate_questions_by_id: dict[str, GeneratedQuestion] = {
|
||
q.question_id: q for q in questions
|
||
}
|
||
with HarnessLog(str(self._paths.db_path), pools.baseline_run_id) as log:
|
||
rows = log.query(
|
||
"SELECT question_id, prediction, answer FROM predictions WHERE run_id=?",
|
||
(pools.baseline_run_id,),
|
||
)
|
||
if not rows:
|
||
raise RuntimeError(
|
||
f"基线 run {pools.baseline_run_id} 在 predictions 表无任何行，"
|
||
"无法构建 gate 阶梯（检查种子基线是否完整落库）"
|
||
)
|
||
baseline_correctness = {r["question_id"]: r["prediction"] == r["answer"] for r in rows}
|
||
logger.info("gate 阶梯基线对错覆盖 {} 题", len(baseline_correctness))
|
||
gate_task_types = sorted({q.task_type for q in pools.diagnosis})
|
||
gate_pools = build_or_load_gate_pools(
|
||
workspace_dir=self._config.workspace_dir,
|
||
questions=questions,
|
||
test_qids={q.question_id for q in pools.test},
|
||
baseline_correctness=baseline_correctness,
|
||
task_types=gate_task_types,
|
||
probe_quota=self._config.gate_probe_quota,
|
||
seed=1,
|
||
baseline_run_id=pools.baseline_run_id,
|
||
)
|
||
baseline_cache = BaselineCache(self._config.workspace_dir / "baseline_cache.json")
|
||
return gate_pools, baseline_cache
|
||
|
||
def _init_train_state(
|
||
self, pools: Pools, gate_pools: GatePools, baseline_cache: BaselineCache
|
||
) -> _TrainState:
|
||
"""初始化跨 step 训练状态。"""
|
||
skills_v = self._current_version("skills")
|
||
prompts_v = self._current_version("prompts")
|
||
update_best(
|
||
self._config.workspace_dir,
|
||
skills=f"skills/{skills_v}",
|
||
prompts=f"prompts/{prompts_v}",
|
||
val_acc=pools.baseline_val_accuracy,
|
||
run_id=pools.baseline_run_id,
|
||
epoch=0,
|
||
)
|
||
if read_best(self._config.workspace_dir) is None:
|
||
raise RuntimeError("best 指针初始化失败")
|
||
return _TrainState(
|
||
correctness=dict(pools.correctness),
|
||
gate_pools=gate_pools,
|
||
baseline_cache=baseline_cache,
|
||
eval_prev_acc=pools.baseline_val_accuracy,
|
||
eval_prev_run_id=pools.baseline_run_id,
|
||
best_val_acc=pools.baseline_val_accuracy,
|
||
best_skills_version=skills_v,
|
||
best_prompts_version=prompts_v,
|
||
baseline_skills_version=skills_v,
|
||
baseline_prompts_version=prompts_v,
|
||
)
|
||
|
||
def _restore_train_state(
|
||
self,
|
||
ckpt: dict,
|
||
pools: Pools,
|
||
gate_pools: GatePools,
|
||
baseline_cache: BaselineCache,
|
||
) -> _TrainState:
|
||
"""从 checkpoint 重建 _TrainState。"""
|
||
fields = deserialize_state_fields(ckpt["state"])
|
||
best = read_best(self._config.workspace_dir) or {}
|
||
return _TrainState(
|
||
gate_pools=gate_pools,
|
||
baseline_cache=baseline_cache,
|
||
best_val_acc=best.get("val_acc", pools.baseline_val_accuracy),
|
||
best_skills_version=best.get("skills", "skills/v1").split("/")[-1],
|
||
best_prompts_version=best.get("prompts", "prompts/v1").split("/")[-1],
|
||
**fields,
|
||
)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# _run_step：rollout → correctness → diagnose → accumulate → gate → cooldown
|
||
# -----------------------------------------------------------------------
|
||
|
||
async def _run_step(
|
||
self,
|
||
epoch: int,
|
||
step: int,
|
||
total_steps: int,
|
||
batch: list[GeneratedQuestion],
|
||
pools: Pools,
|
||
state: _TrainState,
|
||
) -> None:
|
||
"""单 step：rollout → correctness 增量 → 诊断 → 累加 system/tool → 按类 gate。"""
|
||
run_id = f"{pools.baseline_run_id}_e{epoch}_s{step}"
|
||
await self._rollout_batch(batch, run_id)
|
||
|
||
from app.harness.log import HarnessLog
|
||
|
||
with HarnessLog(str(self._paths.db_path), run_id) as log:
|
||
_apply_batch_correctness(state.correctness, log, run_id, batch)
|
||
|
||
diagnosis = await self._run_diagnosis(run_id, question_ids=[q.question_id for q in batch])
|
||
_accumulate_slow_packs(diagnosis, state)
|
||
await self._gate_batch_skills(epoch, step, diagnosis, total_steps, pools, state)
|
||
# 冷却计数每 step 递减、归零剔除
|
||
state.gate_cooldown = {t: n - 1 for t, n in state.gate_cooldown.items() if n - 1 > 0}
|
||
# 测试中断注入点
|
||
if getattr(self, "_interrupt_after_step", None) == step:
|
||
raise _InterruptError(f"模拟中断于 epoch{epoch} step{step}")
|
||
|
||
async def _rollout_batch(self, batch: list[GeneratedQuestion], run_id: str) -> None:
|
||
"""用当前 skill 版本重推该 batch。"""
|
||
result = await self._run_inference_on_pool(
|
||
batch, run_id, self._paths.skills_dir, self._paths.prompts_dir
|
||
)
|
||
_guard_infra_failures(result, context="rollout")
|
||
|
||
# -----------------------------------------------------------------------
|
||
# _gate_batch_skills：per task_type gate
|
||
# -----------------------------------------------------------------------
|
||
|
||
async def _gate_batch_skills(
|
||
self,
|
||
epoch: int,
|
||
step: int,
|
||
diagnosis: DiagnosisResult,
|
||
total_steps: int,
|
||
pools: Pools,
|
||
state: _TrainState,
|
||
) -> None:
|
||
"""按 task_type 独立 evolve → 局部验证 → accept/reject。"""
|
||
from app.harness.workspace import VersionedSkillStore
|
||
from core.evolution import evolve_single_skill
|
||
|
||
budget = edit_budget_at(
|
||
global_step=state.global_step,
|
||
total_steps=total_steps,
|
||
start=self._config.edit_budget_start,
|
||
end=self._config.edit_budget_end,
|
||
)
|
||
for task_type in sorted(diagnosis.skill_case_packs):
|
||
# 冷却 admission control
|
||
if state.gate_cooldown.get(task_type, 0) > 0:
|
||
_write_skip_report(
|
||
self._config.workspace_dir,
|
||
epoch,
|
||
step,
|
||
state.global_step,
|
||
task_type,
|
||
action="cooldown",
|
||
baseline_acc=self._class_baseline_acc(
|
||
task_type, pools.validation, state.correctness
|
||
),
|
||
budget=budget,
|
||
)
|
||
continue
|
||
|
||
pack = diagnosis.skill_case_packs[task_type]
|
||
skill_store = VersionedSkillStore(self._paths.skills_dir)
|
||
evolve_prompts = self._load_evolve_prompts()
|
||
record = await evolve_single_skill(
|
||
self._evolve_llm,
|
||
pack,
|
||
skill_store,
|
||
evolve_prompts,
|
||
self._current_version("skills"),
|
||
budget,
|
||
self._config.appendix_consolidate_threshold,
|
||
skill_update_mode=self._config.skill_update_mode,
|
||
rejected=state.rejected_buffer.get(task_type, []),
|
||
)
|
||
# 进化未产出真实改动
|
||
if record.status in ("rejected", "skipped") or (
|
||
record.evolved_content == record.original_content
|
||
):
|
||
_write_skip_report(
|
||
self._config.workspace_dir,
|
||
epoch,
|
||
step,
|
||
state.global_step,
|
||
task_type,
|
||
action="skipped",
|
||
baseline_acc=self._class_baseline_acc(
|
||
task_type, pools.validation, state.correctness
|
||
),
|
||
budget=budget,
|
||
rank_clip_triggered=bool(record.clip_info.get("triggered", False)),
|
||
)
|
||
continue
|
||
|
||
outcome = await self._run_gate_validation(
|
||
epoch, step, task_type, pack, record, pools, state
|
||
)
|
||
# 观测落库
|
||
write_gate_evidence(
|
||
str(self._paths.db_path),
|
||
run_id=pools.baseline_run_id,
|
||
epoch=epoch,
|
||
step=step,
|
||
rows=outcome.evidence_rows,
|
||
)
|
||
write_step_report(
|
||
self._config.workspace_dir,
|
||
epoch=epoch,
|
||
step=step,
|
||
global_step=state.global_step,
|
||
task_type=task_type,
|
||
gate_action=outcome.action,
|
||
candidate_acc=outcome.candidate_acc,
|
||
class_baseline_acc=outcome.baseline_acc,
|
||
edit_budget=budget,
|
||
rank_clip_triggered=bool(record.clip_info.get("triggered", False)),
|
||
gate_w=outcome.w,
|
||
gate_l=outcome.l,
|
||
gate_e_value=outcome.e_value,
|
||
gate_n_used=outcome.n_used,
|
||
gate_stop_reason=outcome.stop_reason,
|
||
)
|
||
write_quadrant_pairs(
|
||
str(self._paths.db_path),
|
||
run_id=pools.baseline_run_id,
|
||
epoch=epoch,
|
||
step=step,
|
||
pairs=_outcome_to_quadrant_pairs(task_type, outcome),
|
||
)
|
||
if outcome.accepted:
|
||
self._accept_skill(task_type, record, outcome, state, pools)
|
||
else:
|
||
self._record_rejected_skill(
|
||
state.rejected_buffer, task_type, record, outcome, state.global_step
|
||
)
|
||
|
||
async def _run_gate_validation(
|
||
self,
|
||
epoch: int,
|
||
step: int,
|
||
task_type: str,
|
||
pack: Any,
|
||
record: EvolutionRecord,
|
||
pools: Pools,
|
||
state: _TrainState,
|
||
) -> ValidationOutcome:
|
||
"""CE-Gate 块序贯配对验证：阶梯出题 → 基线/候选逐块配对 → e-process 四出口。
|
||
|
||
参数:
|
||
epoch: 轮次。
|
||
step: epoch 内 step。
|
||
task_type: 待验证题型。
|
||
pack: SkillCasePack。
|
||
record: 进化产物。
|
||
pools: 冻结三池。
|
||
state: 训练状态。
|
||
|
||
返回:
|
||
ValidationOutcome。
|
||
"""
|
||
from app.harness.log import HarnessLog
|
||
from app.harness.validate import validate_skill_local
|
||
|
||
exclude_qids = {c.question_id for c in pack.failure_cases + pack.success_cases}
|
||
ladder_qids = state.gate_pools.ladder_for(
|
||
task_type,
|
||
exclude_qids,
|
||
p_low=self._config.gate_p_low,
|
||
p_high=self._config.gate_p_high,
|
||
cold=not state.gate_epoch_observed,
|
||
)
|
||
missing = [qid for qid in ladder_qids if qid not in self._gate_questions_by_id]
|
||
if missing:
|
||
raise ValueError(
|
||
f"gate 阶梯[{task_type}] 含 benchmark 中不存在的题: "
|
||
f"{missing[:5]}（gate_pools.json 与题库失配）"
|
||
)
|
||
ladder_items = [self._gate_questions_by_id[qid] for qid in ladder_qids]
|
||
base_skill_content = (self._paths.skills_dir / record.target_file).read_text(
|
||
encoding="utf-8"
|
||
)
|
||
slug = task_type.lower().replace(" ", "-")
|
||
run_inference_fn = self._make_validate_run_inference_fn()
|
||
with HarnessLog(str(self._paths.db_path), f"gate_{slug}") as gate_log:
|
||
return await validate_skill_local(
|
||
workspace_dir=self._config.workspace_dir,
|
||
base_skills_version=self._current_version("skills"),
|
||
task_type=task_type,
|
||
target_file=record.target_file,
|
||
candidate_content=record.evolved_content,
|
||
base_skill_content=base_skill_content,
|
||
ladder_items=ladder_items,
|
||
gate_params=GateParams(
|
||
e_confirm=self._config.gate_e_confirm,
|
||
e_provisional=self._config.gate_e_provisional,
|
||
w_net_min=self._config.gate_w_net_min,
|
||
delta_min=self._config.gate_delta_min,
|
||
lambda_dir=self._config.gate_lambda_dir,
|
||
e_rollback=self._config.gate_e_rollback,
|
||
),
|
||
gate_block=self._config.gate_block,
|
||
gate_n_max=self._config.gate_n_max,
|
||
gate_guard_err=self._config.gate_guard_err,
|
||
baseline_cache=state.baseline_cache,
|
||
prompts_version=self._current_version("prompts"),
|
||
run_inference=run_inference_fn,
|
||
log=gate_log,
|
||
gate_run_prefix=(f"{pools.baseline_run_id}_e{epoch}_s{step}_gate_{slug}"),
|
||
)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# accept / reject / probation
|
||
# -----------------------------------------------------------------------
|
||
|
||
def _accept_skill(
|
||
self,
|
||
task_type: str,
|
||
record: EvolutionRecord,
|
||
outcome: ValidationOutcome,
|
||
state: _TrainState,
|
||
pools: Pools,
|
||
) -> None:
|
||
"""accept：写候选为新 skills 版本 → manifest → 路径 → 前移 correctness。
|
||
|
||
probation 分岔：provisional + 无现有试用 + 非 default-strategy.md → 开账。
|
||
试用中追加 pending_edits。
|
||
"""
|
||
pre_accept_version = self._current_version("skills")
|
||
# 开账快照在合并前拍取
|
||
pre_merge_snapshot = {
|
||
q.question_id: state.correctness.get(q.question_id, False)
|
||
for q in pools.validation
|
||
if q.task_type == task_type
|
||
}
|
||
new_version = self._promote_skill_version(record.evolved_content, record.target_file)
|
||
update_manifest(self._config.workspace_dir, skills=f"skills/{new_version}")
|
||
self._paths = resolve_paths(self._config.workspace_dir)
|
||
# correctness 二轨合并（只合并已观测题）
|
||
state.correctness.update(outcome.candidate_correctness)
|
||
# 清该类黑名单
|
||
state.rejected_buffer.pop(task_type, None)
|
||
state.changed_task_types_this_epoch.add(task_type)
|
||
# probation 分岔
|
||
if (
|
||
outcome.action == "accept_provisional"
|
||
and task_type not in state.probations
|
||
and record.target_file != "default-strategy.md"
|
||
):
|
||
state.probations[task_type] = Probation(
|
||
task_type=task_type,
|
||
anchor_skills_version=pre_accept_version,
|
||
target_file=record.target_file,
|
||
correctness_snapshot=pre_merge_snapshot,
|
||
opened_step=state.global_step,
|
||
)
|
||
elif outcome.action == "accept_provisional" and record.target_file == "default-strategy.md":
|
||
logger.warning(
|
||
"按类 gate[{}] provisional 落在共享 default-strategy.md，跳过试用直接转正",
|
||
task_type,
|
||
)
|
||
if task_type in state.probations:
|
||
state.probations[task_type].pending_edits.append(
|
||
RejectedEdit(
|
||
target_file=record.target_file,
|
||
target_type=record.target_type,
|
||
change_summary=self._rejected_summary(record, outcome),
|
||
delta=outcome.delta_hat,
|
||
source_version=record.source_version,
|
||
epoch=state.global_step,
|
||
gate_w=outcome.w,
|
||
gate_l=outcome.l,
|
||
gate_e_value=outcome.e_value,
|
||
gate_delta_shrunk=outcome.delta_shrunk,
|
||
)
|
||
)
|
||
logger.info(
|
||
"按类 gate[{}] accept: 候选{:.1%} (观测基线{:.1%}) → skills/{}",
|
||
task_type,
|
||
outcome.candidate_acc,
|
||
outcome.baseline_acc,
|
||
new_version,
|
||
)
|
||
|
||
def _rollback_probation(self, probation: Probation, state: _TrainState) -> None:
|
||
"""试用期回滚：文件级 revert 到锚版本 + 恢复快照 + 冷却 + 证据入黑名单。"""
|
||
anchor_content = (
|
||
self._config.workspace_dir
|
||
/ "skills"
|
||
/ probation.anchor_skills_version
|
||
/ probation.target_file
|
||
).read_text(encoding="utf-8")
|
||
new_version = self._promote_skill_version(anchor_content, probation.target_file)
|
||
update_manifest(self._config.workspace_dir, skills=f"skills/{new_version}")
|
||
self._paths = resolve_paths(self._config.workspace_dir)
|
||
state.correctness.update(probation.correctness_snapshot)
|
||
state.gate_cooldown[probation.task_type] = self._config.gate_cooldown_steps
|
||
state.rejected_buffer.setdefault(probation.task_type, []).extend(probation.pending_edits)
|
||
logger.info(
|
||
"probation 回滚[{}]: skills 文件 {} 恢复至锚版本 {} → 新版本 {}，冷却 {} step",
|
||
probation.task_type,
|
||
probation.target_file,
|
||
probation.anchor_skills_version,
|
||
new_version,
|
||
self._config.gate_cooldown_steps,
|
||
)
|
||
|
||
@staticmethod
|
||
def _record_rejected_skill(
|
||
rejected_buffer: dict[str, list],
|
||
task_type: str,
|
||
record: EvolutionRecord,
|
||
outcome: ValidationOutcome,
|
||
global_step: int,
|
||
) -> None:
|
||
"""reject：按 task_type 累加 A5 黑名单。"""
|
||
rejected_buffer.setdefault(task_type, []).append(
|
||
RejectedEdit(
|
||
target_file=record.target_file,
|
||
target_type=record.target_type,
|
||
change_summary=Runner._rejected_summary_static(record, outcome),
|
||
delta=outcome.delta_hat,
|
||
source_version=record.source_version,
|
||
epoch=global_step,
|
||
gate_w=outcome.w,
|
||
gate_l=outcome.l,
|
||
gate_e_value=outcome.e_value,
|
||
gate_delta_shrunk=outcome.delta_shrunk,
|
||
)
|
||
)
|
||
logger.info(
|
||
"按类 gate[{}] reject: 候选{:.1%} (观测基线{:.1%}) 回退该 skill",
|
||
task_type,
|
||
outcome.candidate_acc,
|
||
outcome.baseline_acc,
|
||
)
|
||
|
||
def _rejected_summary(self, record: EvolutionRecord, outcome: ValidationOutcome) -> str:
|
||
"""为被拒进化记录生成黑名单摘要：只拼真正 applied 的 edit。"""
|
||
return Runner._rejected_summary_static(record, outcome)
|
||
|
||
@staticmethod
|
||
def _rejected_summary_static(record: EvolutionRecord, outcome: ValidationOutcome) -> str:
|
||
"""黑名单摘要实现（静态方法，供 accept / reject 两侧复用）。
|
||
|
||
为何只记 applied：未 applied 的 edit 从未写进候选正文、从未被 gate
|
||
验证过，进黑名单会污染「已验证无效」语义。
|
||
"""
|
||
applied_summary = _format_applied_edits(record)
|
||
if applied_summary is not None:
|
||
return applied_summary
|
||
return _fallback_summary(record, outcome)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# _slow_update_cycle 十步序
|
||
# -----------------------------------------------------------------------
|
||
|
||
async def _slow_update_cycle(self, epoch: int, pools: Pools, state: _TrainState) -> None:
|
||
"""epoch 末慢更新十步序。
|
||
|
||
1. 捕获版本快照 → 全 val 重跑 R
|
||
2. soft score + dual_metric 落库
|
||
3. R 逐题对错无条件回写
|
||
4. probation 结算（回滚者覆盖 step 3）
|
||
5. best argmax（严格大于）
|
||
6. momentum（不可变新版本，按 skill 文件分组）
|
||
7. system/tool 慢更新（edit_budget_end）
|
||
8. R2 闭环
|
||
9. 三态标签 + epoch_report + 四向 held-out
|
||
10. gate 阶梯刷新
|
||
"""
|
||
# Phase 1
|
||
eval_skills_version = self._current_version("skills")
|
||
eval_prompts_version = self._current_version("prompts")
|
||
eval_r = await self._eval_full_val(epoch, pools)
|
||
|
||
# Phase 2: soft + dual_metric
|
||
eval_soft = await self._try_soft_score(eval_r.run_id, pools.validation)
|
||
write_dual_metric(
|
||
str(self._paths.db_path),
|
||
run_id=self._config.run_id,
|
||
epoch=epoch,
|
||
version_kind="final",
|
||
skills_version=eval_skills_version,
|
||
prompts_version=eval_prompts_version,
|
||
pool="val",
|
||
hard_acc=eval_r.accuracy,
|
||
soft_score=eval_soft,
|
||
mixed_score=(None if eval_soft is None else 0.5 * eval_r.accuracy + 0.5 * eval_soft),
|
||
)
|
||
|
||
# Phase 3: 无条件回写
|
||
self._writeback_val_correctness(eval_r.run_id, pools, state)
|
||
|
||
# Phase 4: probation 结算
|
||
self._settle_probations(eval_r.run_id, state)
|
||
|
||
# Phase 5: best argmax
|
||
self._maybe_promote_best(
|
||
eval_skills_version,
|
||
eval_prompts_version,
|
||
eval_r.accuracy,
|
||
eval_r.run_id,
|
||
epoch,
|
||
state,
|
||
)
|
||
|
||
# Phase 6: momentum
|
||
momentum_task_types = await self._write_momentum_for_changed_skills(
|
||
state, pools, epoch, eval_skills_version
|
||
)
|
||
|
||
# Phase 7: system/tool 慢更新
|
||
pre_prompts_version = self._current_version("prompts")
|
||
system_tool_updated = await self._update_system_tool(epoch, state)
|
||
system_tool_reverted = False
|
||
|
||
# Phase 8: R2 闭环
|
||
r2_kept_run_ids: list[str] | None = None
|
||
if system_tool_updated:
|
||
r2_skills_version = self._current_version("skills")
|
||
new_prompts_version = self._current_version("prompts")
|
||
eval_r2 = await self._eval_full_val(epoch, pools, run_suffix="_p2")
|
||
write_dual_metric(
|
||
str(self._paths.db_path),
|
||
run_id=self._config.run_id,
|
||
epoch=epoch,
|
||
version_kind="final",
|
||
skills_version=r2_skills_version,
|
||
prompts_version=new_prompts_version,
|
||
pool="val",
|
||
hard_acc=eval_r2.accuracy,
|
||
soft_score=None,
|
||
mixed_score=None,
|
||
)
|
||
system_tool_reverted = eval_r2.accuracy < eval_r.accuracy
|
||
if system_tool_reverted:
|
||
self._revert_system_tool(pre_prompts_version)
|
||
else:
|
||
self._writeback_val_correctness(eval_r2.run_id, pools, state)
|
||
self._maybe_promote_best(
|
||
r2_skills_version,
|
||
new_prompts_version,
|
||
eval_r2.accuracy,
|
||
eval_r2.run_id,
|
||
epoch,
|
||
state,
|
||
)
|
||
state.eval_prev_acc = eval_r2.accuracy
|
||
state.eval_prev_run_id = eval_r2.run_id
|
||
r2_kept_run_ids = [eval_r2.run_id]
|
||
if (not system_tool_updated) or system_tool_reverted:
|
||
state.eval_prev_acc = eval_r.accuracy
|
||
state.eval_prev_run_id = eval_r.run_id
|
||
|
||
# Phase 9: 三态标签 + epoch_report + held-out
|
||
if system_tool_reverted:
|
||
system_tool_action = "reverted"
|
||
elif system_tool_updated:
|
||
system_tool_action = "updated"
|
||
else:
|
||
system_tool_action = "none"
|
||
write_epoch_report(
|
||
self._config.workspace_dir,
|
||
epoch=epoch,
|
||
system_tool_action=system_tool_action,
|
||
momentum_updated_task_types=momentum_task_types,
|
||
best_val_acc=state.best_val_acc,
|
||
)
|
||
await self._holdout_four_way(epoch, pools, state, eval_skills_version, eval_prompts_version)
|
||
|
||
# Phase 10: gate 阶梯刷新
|
||
self._refresh_gate_ladder(
|
||
epoch, pools.baseline_run_id, state, extra_run_ids=r2_kept_run_ids
|
||
)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# 慢更新内部方法
|
||
# -----------------------------------------------------------------------
|
||
|
||
def _settle_probations(self, eval_run_id: str, state: _TrainState) -> None:
|
||
"""epoch 末试用期一次性结算：全 val 重跑逐题结果与锚快照配对。
|
||
|
||
参数:
|
||
eval_run_id: 本 epoch 全 val 重跑（R）的 run_id。
|
||
state: 训练状态（probations 结算后清空）。
|
||
|
||
异常:
|
||
RuntimeError: 重跑缺某快照题的预测行。
|
||
"""
|
||
if not state.probations:
|
||
return
|
||
from app.harness.log import HarnessLog
|
||
from app.harness.validate import _load_run_rows
|
||
|
||
with HarnessLog(str(self._paths.db_path), eval_run_id) as log:
|
||
rows = _load_run_rows(log, eval_run_id)
|
||
|
||
params = GateParams(
|
||
e_confirm=self._config.gate_e_confirm,
|
||
e_provisional=self._config.gate_e_provisional,
|
||
w_net_min=self._config.gate_w_net_min,
|
||
delta_min=self._config.gate_delta_min,
|
||
lambda_dir=self._config.gate_lambda_dir,
|
||
e_rollback=self._config.gate_e_rollback,
|
||
)
|
||
for task_type in sorted(state.probations):
|
||
probation = state.probations[task_type]
|
||
w = l = 0 # noqa: E741
|
||
for qid, snap_correct in probation.correctness_snapshot.items():
|
||
row = rows.get(qid)
|
||
if row is None:
|
||
raise RuntimeError(
|
||
f"probation 结算缺预测行: {task_type}/{qid}（run={eval_run_id}）"
|
||
)
|
||
cur = row["_correct"]
|
||
if not snap_correct and cur:
|
||
w += 1
|
||
elif snap_correct and not cur:
|
||
l += 1 # noqa: E741
|
||
verdict = probation_verdict(w, l, params=params)
|
||
logger.info("probation 结算[{}]: W={} L={} → {}", task_type, w, l, verdict)
|
||
if verdict == "rollback":
|
||
self._rollback_probation(probation, state)
|
||
state.probations.clear()
|
||
|
||
async def _eval_full_val(
|
||
self, epoch: int, pools: Pools, run_suffix: str = ""
|
||
) -> InferenceResult:
|
||
"""全验证池重跑一次并护栏。"""
|
||
run_id = f"{self._config.run_id}_slow_e{epoch}{run_suffix}"
|
||
result = await self._run_inference_on_pool(
|
||
pools.validation, run_id, self._paths.skills_dir, self._paths.prompts_dir
|
||
)
|
||
_guard_infra_failures(result, context="全 val 重跑")
|
||
return result
|
||
|
||
def _writeback_val_correctness(
|
||
self, eval_run_id: str, pools: Pools, state: _TrainState
|
||
) -> None:
|
||
"""把全 val 重跑逐题对错回写进 state.correctness。"""
|
||
from app.harness.log import HarnessLog
|
||
from app.harness.validate import _load_run_rows
|
||
|
||
with HarnessLog(str(self._paths.db_path), eval_run_id) as log:
|
||
rows = _load_run_rows(log, eval_run_id)
|
||
for q in pools.validation:
|
||
row = rows.get(q.question_id)
|
||
if row is not None:
|
||
state.correctness[q.question_id] = row["_correct"]
|
||
|
||
def _maybe_promote_best(
|
||
self,
|
||
skills_v: str,
|
||
prompts_v: str,
|
||
eval_acc: float,
|
||
run_id: str,
|
||
epoch: int,
|
||
state: _TrainState,
|
||
) -> None:
|
||
"""全局 best argmax（严格大于才推进）。"""
|
||
if eval_acc <= state.best_val_acc:
|
||
return
|
||
state.best_val_acc = eval_acc
|
||
state.best_skills_version = skills_v
|
||
state.best_prompts_version = prompts_v
|
||
state.steps_since_best_improved = 0
|
||
update_best(
|
||
self._config.workspace_dir,
|
||
skills=f"skills/{skills_v}",
|
||
prompts=f"prompts/{prompts_v}",
|
||
val_acc=eval_acc,
|
||
run_id=run_id,
|
||
epoch=epoch,
|
||
)
|
||
logger.info(
|
||
"全局 best argmax 刷新: {:.1%} → skills/{} prompts/{}",
|
||
eval_acc,
|
||
skills_v,
|
||
prompts_v,
|
||
)
|
||
|
||
async def _write_momentum_for_changed_skills(
|
||
self,
|
||
state: _TrainState,
|
||
pools: Pools,
|
||
epoch: int,
|
||
eval_skills_version: str,
|
||
) -> list[str]:
|
||
"""为本 epoch 改过的题型写 momentum：推进不可变新版本。
|
||
|
||
返回:
|
||
实际写过 momentum 的题型列表。
|
||
"""
|
||
|
||
if not self._config.use_slow_momentum:
|
||
return []
|
||
if not state.changed_task_types_this_epoch:
|
||
return []
|
||
|
||
file_to_task_types = self._group_changed_task_types_by_file(
|
||
state.changed_task_types_this_epoch
|
||
)
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
staged_skills = Path(tmp) / "skills"
|
||
shutil.copytree(self._paths.skills_dir, staged_skills, dirs_exist_ok=True)
|
||
for target_file in sorted(file_to_task_types):
|
||
await self._stage_momentum_for_file(
|
||
target_file,
|
||
file_to_task_types[target_file],
|
||
state,
|
||
pools,
|
||
staged_skills,
|
||
epoch,
|
||
)
|
||
new_version = advance_version(
|
||
self._paths.workspace_dir,
|
||
"skills",
|
||
staged_skills,
|
||
{
|
||
"source": "slow_momentum",
|
||
"parent": eval_skills_version,
|
||
"description": "epoch 末 momentum（不可变新版本）",
|
||
},
|
||
)
|
||
update_manifest(self._config.workspace_dir, skills=f"skills/{new_version}")
|
||
self._paths = resolve_paths(self._config.workspace_dir)
|
||
logger.info(
|
||
"Epoch 末 momentum → skills/{}（不改 eval 版本 {}）",
|
||
new_version,
|
||
eval_skills_version,
|
||
)
|
||
return sorted(state.changed_task_types_this_epoch)
|
||
|
||
async def _stage_momentum_for_file(
|
||
self,
|
||
target_file: str,
|
||
task_types: list[str],
|
||
state: _TrainState,
|
||
pools: Pools,
|
||
staged_skills: Path,
|
||
epoch: int,
|
||
) -> None:
|
||
"""单个 skill 文件的 momentum 生成：诊断池采样 → 两版 rollout → 纵向对比。"""
|
||
from app.harness.log import HarnessLog
|
||
from app.harness.momentum import run_slow_momentum
|
||
from app.harness.validate import _load_run_rows
|
||
|
||
skill_path = staged_skills / target_file
|
||
skill_content = skill_path.read_text(encoding="utf-8")
|
||
prev_skill = state.epoch_start_skills.get(target_file, skill_content)
|
||
prev_guidance = momentum_inner(skill_content)
|
||
|
||
# 采样
|
||
allowed = set(task_types)
|
||
candidates = [q for q in pools.diagnosis if q.task_type in allowed]
|
||
rng = random.Random(epoch)
|
||
n = min(self._config.momentum_samples, len(candidates))
|
||
sampled = rng.sample(candidates, n) if n > 0 else []
|
||
|
||
if not sampled:
|
||
skill_path.write_text(
|
||
replace_momentum(skill_content, prev_guidance or ""),
|
||
encoding="utf-8",
|
||
)
|
||
logger.debug(
|
||
"Epoch {} momentum 跳过 {}：诊断池无匹配题型 {} 的样本",
|
||
epoch,
|
||
target_file,
|
||
sorted(task_types),
|
||
)
|
||
return
|
||
|
||
# 两版 rollout
|
||
prev_run_id = f"momentum_prev_e{epoch}_{target_file.replace('.md', '')}"
|
||
curr_run_id = f"momentum_curr_e{epoch}_{target_file.replace('.md', '')}"
|
||
|
||
with tempfile.TemporaryDirectory() as prev_tmp:
|
||
prev_skills_dir = Path(prev_tmp) / "skills"
|
||
shutil.copytree(self._paths.skills_dir, prev_skills_dir, dirs_exist_ok=True)
|
||
(prev_skills_dir / target_file).write_text(prev_skill, encoding="utf-8")
|
||
await self._run_inference_on_pool(
|
||
sampled, prev_run_id, prev_skills_dir, self._paths.prompts_dir
|
||
)
|
||
|
||
await self._run_inference_on_pool(
|
||
sampled, curr_run_id, self._paths.skills_dir, self._paths.prompts_dir
|
||
)
|
||
|
||
with HarnessLog(str(self._paths.db_path), prev_run_id) as log:
|
||
prev_rows = _load_run_rows(log, prev_run_id)
|
||
with HarnessLog(str(self._paths.db_path), curr_run_id) as log:
|
||
curr_rows = _load_run_rows(log, curr_run_id)
|
||
|
||
comparison_pairs = _build_comparison_pairs(sampled, prev_rows, curr_rows)
|
||
guidance = await run_slow_momentum(
|
||
llm=self._evolve_llm,
|
||
diagnose_prompts_dir=Path("prompts"),
|
||
skill_content=skill_content,
|
||
prev_skill=prev_skill,
|
||
prev_guidance=prev_guidance,
|
||
comparison_pairs=comparison_pairs,
|
||
)
|
||
new_content = replace_momentum(skill_content, guidance)
|
||
skill_path.write_text(new_content, encoding="utf-8")
|
||
logger.info(
|
||
"Epoch {} momentum 写入 skill 文件 {}（题型 {}，采样 {} 题）",
|
||
epoch,
|
||
target_file,
|
||
sorted(task_types),
|
||
len(sampled),
|
||
)
|
||
|
||
async def _update_system_tool(self, epoch: int, state: _TrainState) -> bool:
|
||
"""merge 本 epoch 累加的 system/tool 案例包 → 进化 → accept 写新 prompts 版本。
|
||
|
||
返回:
|
||
是否实际写了新 prompts 版本。
|
||
"""
|
||
from app.harness.workspace import VersionedPromptStore
|
||
from core.evolution import evolve_single_tool, evolve_system_prompt
|
||
|
||
merged_system = merge_system_packs(state.system_packs)
|
||
merged_tools = merge_tool_packs(state.tool_packs)
|
||
source_version = self._current_version("prompts")
|
||
max_edits = self._config.edit_budget_end
|
||
evolve_prompts = self._load_evolve_prompts()
|
||
prompt_store = VersionedPromptStore(self._paths.prompts_dir)
|
||
|
||
records: list[EvolutionRecord] = []
|
||
if merged_system is not None:
|
||
records.append(
|
||
await evolve_system_prompt(
|
||
self._evolve_llm,
|
||
merged_system,
|
||
prompt_store,
|
||
evolve_prompts,
|
||
source_version,
|
||
max_edits,
|
||
)
|
||
)
|
||
for tool_name in sorted(merged_tools):
|
||
records.append(
|
||
await evolve_single_tool(
|
||
self._evolve_llm,
|
||
merged_tools[tool_name],
|
||
prompt_store,
|
||
evolve_prompts,
|
||
source_version,
|
||
max_edits,
|
||
)
|
||
)
|
||
accepted = [r for r in records if r.status == "accepted"]
|
||
if not accepted:
|
||
logger.debug("Epoch {} 慢更新：无 system/tool 改动被接受", epoch)
|
||
return False
|
||
|
||
new_version = self._write_accepted_prompts_version(accepted, source_version)
|
||
if new_version is None:
|
||
return False
|
||
update_manifest(self._config.workspace_dir, prompts=f"prompts/{new_version}")
|
||
self._paths = resolve_paths(self._config.workspace_dir)
|
||
logger.info("Epoch {} 慢更新：system/tool → prompts/{}", epoch, new_version)
|
||
return True
|
||
|
||
def _write_accepted_prompts_version(
|
||
self, accepted: list[EvolutionRecord], source_version: str
|
||
) -> str | None:
|
||
"""将 accepted system/tool records 写成新 prompts 版本。
|
||
|
||
参数:
|
||
accepted: 状态为 accepted 的 EvolutionRecord 列表。
|
||
source_version: 改写前 prompts 版本。
|
||
|
||
返回:
|
||
新版本号，或 None（无实际变化时）。
|
||
"""
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
staged = Path(tmp) / "prompts"
|
||
shutil.copytree(self._paths.prompts_dir, staged, dirs_exist_ok=True)
|
||
any_changed = False
|
||
for rec in accepted:
|
||
if rec.target_type == "tool":
|
||
# tool: evolved_content = json.dumps({"extract": ..., "verify": ...})
|
||
combined = json.loads(rec.evolved_content)
|
||
for key in ("extract", "verify"):
|
||
fname = rec.target_file.replace("_extract.md", f"_{key}.md")
|
||
(staged / fname).write_text(combined[key], encoding="utf-8")
|
||
any_changed = True
|
||
else:
|
||
(staged / rec.target_file).write_text(rec.evolved_content, encoding="utf-8")
|
||
any_changed = True
|
||
if not any_changed:
|
||
return None
|
||
return advance_version(
|
||
self._paths.workspace_dir,
|
||
"prompts",
|
||
staged,
|
||
{
|
||
"source": "evolution",
|
||
"parent": source_version,
|
||
"description": "epoch 末 system/tool 慢更新",
|
||
},
|
||
)
|
||
|
||
def _revert_system_tool(self, pre_prompts_version: str) -> None:
|
||
"""prompts-only delta 退步时回退到更新前版本。"""
|
||
update_manifest(self._config.workspace_dir, prompts=f"prompts/{pre_prompts_version}")
|
||
self._paths = resolve_paths(self._config.workspace_dir)
|
||
logger.info(
|
||
"慢更新 prompts-only delta 退步：system/tool 回退到 prompts/{}",
|
||
pre_prompts_version,
|
||
)
|
||
|
||
def _refresh_gate_ladder(
|
||
self,
|
||
epoch: int,
|
||
base_run_id: str,
|
||
state: _TrainState,
|
||
extra_run_ids: list[str] | None = None,
|
||
) -> None:
|
||
"""用本 epoch 非 gate run 的逐题观测 γ-EMA 更新阶梯 p-hat 并落盘。
|
||
|
||
精确三源：step rollout GLOB 排除 _gate_ + slow R + kept R2。
|
||
"""
|
||
db_path = resolve_paths(self._config.workspace_dir).db_path
|
||
conn = sqlite3.connect(str(db_path))
|
||
conn.row_factory = sqlite3.Row
|
||
try:
|
||
rows = conn.execute(
|
||
"SELECT question_id, prediction, answer FROM predictions "
|
||
"WHERE run_id GLOB ? AND run_id NOT GLOB '*_gate_*' "
|
||
"ORDER BY rowid",
|
||
(f"{base_run_id}_e{epoch}_s*",),
|
||
).fetchall()
|
||
slow_rows = conn.execute(
|
||
"SELECT question_id, prediction, answer FROM predictions "
|
||
"WHERE run_id=? ORDER BY rowid",
|
||
(f"{self._config.run_id}_slow_e{epoch}",),
|
||
).fetchall()
|
||
extra_rows_lists = [
|
||
conn.execute(
|
||
"SELECT question_id, prediction, answer FROM predictions "
|
||
"WHERE run_id=? ORDER BY rowid",
|
||
(rid,),
|
||
).fetchall()
|
||
for rid in (extra_run_ids or [])
|
||
]
|
||
finally:
|
||
conn.close()
|
||
|
||
obs = {r["question_id"]: r["prediction"] == r["answer"] for r in rows}
|
||
obs.update({r["question_id"]: r["prediction"] == r["answer"] for r in slow_rows})
|
||
for extra_rows in extra_rows_lists:
|
||
obs.update({r["question_id"]: r["prediction"] == r["answer"] for r in extra_rows})
|
||
state.gate_pools.update_probs(obs, gamma=self._config.gate_gamma_decay)
|
||
state.gate_pools.save(self._config.workspace_dir / "gate_pools.json")
|
||
state.gate_epoch_observed = True
|
||
|
||
async def _holdout_four_way(
|
||
self,
|
||
epoch: int,
|
||
pools: Pools,
|
||
state: _TrainState,
|
||
eval_skills_version: str,
|
||
eval_prompts_version: str,
|
||
) -> None:
|
||
"""四向 held-out：baseline/best_hard/final/best_mixed 各在 test 池评估。
|
||
|
||
test 池仅观测落库，绝不进 gate/best/early-stop/调参。
|
||
"""
|
||
best_mixed = await self._pick_mixed_best(
|
||
epoch, pools, state, eval_skills_version, eval_prompts_version
|
||
)
|
||
versions: dict[str, tuple[str, str] | None] = {
|
||
"baseline": (state.baseline_skills_version, state.baseline_prompts_version),
|
||
"best_hard": (state.best_skills_version, state.best_prompts_version),
|
||
"final": (eval_skills_version, eval_prompts_version),
|
||
"best_mixed": best_mixed,
|
||
}
|
||
for version_kind, version in versions.items():
|
||
if version is None:
|
||
continue
|
||
sv, pv = version
|
||
run_id = f"{self._config.run_id}_holdout_{version_kind}_e{epoch}"
|
||
res = await self._eval_version_on_pool(
|
||
sv, pv, pools.test, run_id, context=f"held-out {version_kind}"
|
||
)
|
||
soft = await self._try_soft_score(run_id, pools.test)
|
||
mixed = None if soft is None else 0.5 * res.accuracy + 0.5 * soft
|
||
write_holdout_eval(
|
||
str(self._paths.db_path),
|
||
run_id=self._config.run_id,
|
||
epoch=epoch,
|
||
version_kind=version_kind,
|
||
hard_acc=res.accuracy,
|
||
soft_score=soft,
|
||
mixed_score=mixed,
|
||
per_task_type_json=json.dumps(res.per_task_type, ensure_ascii=False),
|
||
)
|
||
|
||
async def _pick_mixed_best(
|
||
self,
|
||
epoch: int,
|
||
pools: Pools,
|
||
state: _TrainState,
|
||
eval_skills_version: str,
|
||
eval_prompts_version: str,
|
||
) -> tuple[str, str] | None:
|
||
"""在 val 池对候选集算 mixed，落 shadow_gate，返回 argmax mixed 版本。
|
||
|
||
只观测落库，绝不改 manifest/best/early-stop。
|
||
"""
|
||
candidates = {
|
||
"best_hard": (state.best_skills_version, state.best_prompts_version),
|
||
"final": (eval_skills_version, eval_prompts_version),
|
||
}
|
||
best_kind: str | None = None
|
||
best_mixed: float | None = None
|
||
for kind, (sv, pv) in candidates.items():
|
||
run_id = f"{self._config.run_id}_shadow_{kind}_e{epoch}"
|
||
res = await self._eval_version_on_pool(
|
||
sv, pv, pools.validation, run_id, context=f"mixed 影子 {kind}"
|
||
)
|
||
soft = await self._try_soft_score(run_id, pools.validation)
|
||
mixed = None if soft is None else 0.5 * res.accuracy + 0.5 * soft
|
||
write_shadow_gate(
|
||
str(self._paths.db_path),
|
||
run_id=self._config.run_id,
|
||
epoch=epoch,
|
||
candidate_version=f"skills/{sv}+prompts/{pv}",
|
||
hard_acc=res.accuracy,
|
||
soft_score=soft,
|
||
mixed_score=mixed,
|
||
is_mixed_best=False,
|
||
)
|
||
if mixed is not None and (best_mixed is None or mixed > best_mixed):
|
||
best_mixed, best_kind = mixed, kind
|
||
if best_kind is None:
|
||
return None
|
||
self._mark_shadow_best(epoch, candidates[best_kind])
|
||
return candidates[best_kind]
|
||
|
||
def _mark_shadow_best(self, epoch: int, best_version: tuple[str, str]) -> None:
|
||
"""回标 shadow_gate 中 argmax mixed 选中的版本 is_mixed_best=1。"""
|
||
sv, pv = best_version
|
||
candidate_version = f"skills/{sv}+prompts/{pv}"
|
||
conn = sqlite3.connect(str(self._paths.db_path))
|
||
try:
|
||
conn.execute(
|
||
"UPDATE shadow_gate SET is_mixed_best=1 WHERE rowid = ("
|
||
" SELECT rowid FROM shadow_gate "
|
||
" WHERE run_id=? AND epoch=? AND candidate_version=? LIMIT 1"
|
||
")",
|
||
(self._config.run_id, epoch, candidate_version),
|
||
)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
# -----------------------------------------------------------------------
|
||
# 收尾
|
||
# -----------------------------------------------------------------------
|
||
|
||
def _deliver_best(self, best_skills_version: str, best_prompts_version: str) -> None:
|
||
"""若当前 current 不是历史最优，回滚 manifest 到 best 并刷新路径。"""
|
||
cur = load_manifest(self._config.workspace_dir)["current"]
|
||
if (
|
||
cur["skills"] != f"skills/{best_skills_version}"
|
||
or cur["prompts"] != f"prompts/{best_prompts_version}"
|
||
):
|
||
update_manifest(
|
||
self._config.workspace_dir,
|
||
skills=f"skills/{best_skills_version}",
|
||
prompts=f"prompts/{best_prompts_version}",
|
||
)
|
||
self._paths = resolve_paths(self._config.workspace_dir)
|
||
logger.info(
|
||
"收尾交付 best：current → skills/{} prompts/{}",
|
||
best_skills_version,
|
||
best_prompts_version,
|
||
)
|
||
|
||
async def _final_test_eval(self, pools: Pools) -> None:
|
||
"""收尾在 held-out test 池跑一次评估。"""
|
||
run_id = f"{self._config.run_id}_final_test"
|
||
result = await self._run_inference_on_pool(
|
||
pools.test, run_id, self._paths.skills_dir, self._paths.prompts_dir
|
||
)
|
||
_guard_infra_failures(result, context="held-out test 评估")
|
||
report = {
|
||
"run_id": result.run_id,
|
||
"accuracy": result.accuracy,
|
||
"total": result.total,
|
||
"correct": result.correct,
|
||
"per_task_type": result.per_task_type,
|
||
}
|
||
path = self._config.workspace_dir / "analyses" / "final_test_eval.json"
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
logger.info("held-out test 评估写入: {} (acc={:.1%})", path, result.accuracy)
|
||
|
||
# -----------------------------------------------------------------------
|
||
# 私有辅助
|
||
# -----------------------------------------------------------------------
|
||
|
||
def _current_version(self, kind: str) -> str:
|
||
"""读取 manifest current 指针中某类资源的当前版本名。"""
|
||
return load_manifest(self._config.workspace_dir)["current"][kind].split("/")[-1]
|
||
|
||
def _current_version_snapshot(self) -> dict[str, str]:
|
||
"""读 manifest.current 的 skills/prompts 指针。"""
|
||
cur = load_manifest(self._config.workspace_dir)["current"]
|
||
return {"skills": cur["skills"], "prompts": cur["prompts"]}
|
||
|
||
def _class_baseline_acc(
|
||
self,
|
||
task_type: str,
|
||
validation: list[GeneratedQuestion],
|
||
correctness: dict[str, bool],
|
||
) -> float:
|
||
"""该 task_type 验证子集在当前 correctness 下的准确率。"""
|
||
class_items = [q for q in validation if q.task_type == task_type]
|
||
assert class_items, f"task_type={task_type} 在验证池中无对应题目"
|
||
correct = sum(1 for q in class_items if correctness.get(q.question_id, False))
|
||
return correct / len(class_items)
|
||
|
||
def _promote_skill_version(self, content: str, target_file: str) -> str:
|
||
"""把候选 skill 内容写成新正式 skills 版本。"""
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
src = Path(tmp) / "skills"
|
||
shutil.copytree(self._paths.skills_dir, src, dirs_exist_ok=True)
|
||
(src / target_file).write_text(content, encoding="utf-8")
|
||
return advance_version(
|
||
self._paths.workspace_dir,
|
||
"skills",
|
||
src,
|
||
{
|
||
"source": "evolution",
|
||
"parent": self._current_version("skills"),
|
||
"description": f"按类 gate accept {target_file}",
|
||
},
|
||
)
|
||
|
||
def _group_changed_task_types_by_file(
|
||
self, changed_task_types: set[str]
|
||
) -> dict[str, list[str]]:
|
||
"""把改过的 task_type 集合经 fallback 解析映射到 skill 文件，按文件分组。"""
|
||
from app.harness.workspace import VersionedSkillStore
|
||
|
||
skill_store = VersionedSkillStore(self._paths.skills_dir)
|
||
grouped: dict[str, list[str]] = {}
|
||
for task_type in changed_task_types:
|
||
skill_file = resolve_skill_file(skill_store, task_type)
|
||
grouped.setdefault(skill_file, []).append(task_type)
|
||
return grouped
|
||
|
||
def _record_run(self, run_id: str) -> Path:
|
||
"""将 current 版本快照追加到 manifest history，创建 run 目录。"""
|
||
from app.harness.workspace import record_run
|
||
|
||
return record_run(self._config.workspace_dir, run_id)
|
||
|
||
def _backfill_run_versions(
|
||
self, run_id: str, skills_version: str, prompts_version: str
|
||
) -> None:
|
||
"""eval run 的 skills/prompts 版本对 + questions_ref 回填进 _runs。"""
|
||
from app.harness.log import HarnessLog
|
||
|
||
with HarnessLog(str(self._paths.db_path), run_id) as log:
|
||
log.execute(
|
||
"UPDATE _runs SET skills_version = ?, prompts_version = ?, "
|
||
"questions_ref = ? WHERE run_id = ?",
|
||
(skills_version, prompts_version, self._config.questions, run_id),
|
||
)
|
||
|
||
def _write_eval_report(
|
||
self,
|
||
run_id: str,
|
||
skills_version: str,
|
||
prompts_version: str,
|
||
result: InferenceResult,
|
||
) -> None:
|
||
"""写 eval 评测报告 analyses/eval_{run_id}.json。"""
|
||
report = {
|
||
"run_id": run_id,
|
||
"skills_version": skills_version,
|
||
"prompts_version": prompts_version,
|
||
"accuracy": result.accuracy,
|
||
"total": result.total,
|
||
"correct": result.correct,
|
||
"stop_reason_counts": result.stop_reason_counts,
|
||
}
|
||
analyses_dir = self._config.workspace_dir / "analyses"
|
||
analyses_dir.mkdir(parents=True, exist_ok=True)
|
||
path = analyses_dir / f"eval_{run_id}.json"
|
||
path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
logger.info("eval 报告写入: {} (acc={:.1%})", path, result.accuracy)
|
||
|
||
async def _run_inference_on_pool(
|
||
self,
|
||
questions: list[GeneratedQuestion],
|
||
run_id: str,
|
||
skills_dir: Path,
|
||
prompts_dir: Path,
|
||
) -> InferenceResult:
|
||
"""用指定版本在给定题池跑一次 run_inference。"""
|
||
from app.harness.inference import run_inference
|
||
from app.harness.log import HarnessLog
|
||
|
||
self._record_run(run_id)
|
||
with HarnessLog(str(self._paths.db_path), run_id) as log:
|
||
return await run_inference(
|
||
questions=questions,
|
||
llm=self._llm,
|
||
tool_dispatch_fn=self._make_tool_dispatch_fn(skills_dir=skills_dir),
|
||
prompt_builder=self._make_prompt_builder(
|
||
skills_dir=skills_dir, prompts_dir=prompts_dir
|
||
),
|
||
log=log,
|
||
run_id=run_id,
|
||
concurrency=self._config.concurrency,
|
||
max_steps=self._config.max_steps,
|
||
skill_mode=self._config.skill_mode,
|
||
)
|
||
|
||
async def _eval_version_on_pool(
|
||
self,
|
||
skills_version: str,
|
||
prompts_version: str,
|
||
questions: list[GeneratedQuestion],
|
||
run_id: str,
|
||
context: str,
|
||
) -> InferenceResult:
|
||
"""用指定版本在给定池跑一次推理并护栏。"""
|
||
skills_dir = self._paths.workspace_dir / "skills" / skills_version
|
||
prompts_dir = self._paths.workspace_dir / "prompts" / prompts_version
|
||
result = await self._run_inference_on_pool(questions, run_id, skills_dir, prompts_dir)
|
||
_guard_infra_failures(result, context=context)
|
||
return result
|
||
|
||
async def _run_diagnosis(
|
||
self, run_id: str, *, question_ids: list[str] | None = None
|
||
) -> DiagnosisResult:
|
||
"""执行两阶段诊断。"""
|
||
from app.harness.log import RunLogImpl
|
||
from app.harness.workspace import VersionedSkillStore
|
||
from app.question_gen import load_benchmark
|
||
from core.evolution.diagnose import run_diagnosis
|
||
|
||
questions = load_benchmark(self._paths.questions_dir)
|
||
run_log = RunLogImpl(str(self._paths.db_path))
|
||
skill_store = VersionedSkillStore(self._paths.skills_dir)
|
||
diagnose_prompts = self._load_diagnose_prompts()
|
||
|
||
return await run_diagnosis(
|
||
run_id=run_id,
|
||
questions=questions,
|
||
tree_data={}, # tree_data 由诊断管线内部按需加载
|
||
llm=self._llm,
|
||
run_log=run_log,
|
||
skill_store=skill_store,
|
||
prompts=diagnose_prompts,
|
||
concurrency=self._config.concurrency,
|
||
question_ids=question_ids,
|
||
)
|
||
|
||
async def _try_soft_score(
|
||
self, run_id: str, questions: list[GeneratedQuestion]
|
||
) -> float | None:
|
||
"""尝试计算 soft score，失败降级为 None。"""
|
||
try:
|
||
# soft score 暂不实现（需诊断 span_evaluations 表），降级为 None
|
||
return None
|
||
except Exception:
|
||
logger.warning("soft score 计算失败（run={}），降级为 None", run_id)
|
||
return None
|
||
|
||
# -----------------------------------------------------------------------
|
||
# 注入工厂（暂用占位，由 main.py 绑定实际实现）
|
||
# -----------------------------------------------------------------------
|
||
|
||
def _make_tool_dispatch_fn(self, *, skills_dir: Path | None = None):
|
||
"""构造工具调度函数（由子类或 main.py 覆盖）。"""
|
||
|
||
async def _noop_dispatch(tool_name: str, args: dict, *, context: dict) -> str:
|
||
raise NotImplementedError(
|
||
f"工具 {tool_name} 调度未配置（需由 main.py 注入 tool_dispatch_fn）"
|
||
)
|
||
|
||
return _noop_dispatch
|
||
|
||
def _make_prompt_builder(
|
||
self, *, skills_dir: Path | None = None, prompts_dir: Path | None = None
|
||
):
|
||
"""构造 prompt 构建函数（由子类或 main.py 覆盖）。"""
|
||
|
||
def _noop_builder(qa: GeneratedQuestion) -> tuple[str, str]:
|
||
raise NotImplementedError("prompt_builder 未配置（需由 main.py 注入）")
|
||
|
||
return _noop_builder
|
||
|
||
def _make_validate_run_inference_fn(self):
|
||
"""构造 validate 用的 RunInferenceFn（绑定共享依赖）。"""
|
||
from app.harness.inference import run_inference
|
||
from app.harness.log import HarnessLog
|
||
|
||
async def _run(
|
||
questions: list[GeneratedQuestion],
|
||
*,
|
||
run_id: str,
|
||
skills_dir: Path,
|
||
) -> InferenceResult:
|
||
self._record_run(run_id)
|
||
with HarnessLog(str(self._paths.db_path), run_id) as log:
|
||
return await run_inference(
|
||
questions=questions,
|
||
llm=self._llm,
|
||
tool_dispatch_fn=self._make_tool_dispatch_fn(skills_dir=skills_dir),
|
||
prompt_builder=self._make_prompt_builder(
|
||
skills_dir=skills_dir, prompts_dir=self._paths.prompts_dir
|
||
),
|
||
log=log,
|
||
run_id=run_id,
|
||
concurrency=self._config.concurrency,
|
||
max_steps=self._config.max_steps,
|
||
skill_mode=self._config.skill_mode,
|
||
)
|
||
|
||
return _run
|
||
|
||
def _load_evolve_prompts(self):
|
||
"""加载进化模板束（从项目根 prompts/ 读取诊断标尺模板）。"""
|
||
from core.evolution.types import EvolvePrompts
|
||
|
||
def _read(name: str) -> str:
|
||
p = Path("prompts") / name
|
||
return p.read_text(encoding="utf-8") if p.exists() else ""
|
||
|
||
return EvolvePrompts(
|
||
evolve_skill=_read("evolve_skill.md"),
|
||
evolve_system=_read("evolve_system.md"),
|
||
evolve_tool=_read("evolve_tool.md"),
|
||
evolve_rank=_read("evolve_rank.md"),
|
||
consolidate_system=_read("consolidate_system.md"),
|
||
)
|
||
|
||
def _load_diagnose_prompts(self):
|
||
"""加载诊断模板束（从项目根 prompts/ 读取）。"""
|
||
from core.evolution.types import DiagnosePrompts
|
||
|
||
def _read(name: str) -> str:
|
||
p = Path("prompts") / name
|
||
return p.read_text(encoding="utf-8") if p.exists() else ""
|
||
|
||
return DiagnosePrompts(
|
||
defect_vs_lapse=_read("defect_vs_lapse.md"),
|
||
reasoning_sub=_read("reasoning_sub.md"),
|
||
span_eval_system=_read("span_eval_system.md"),
|
||
span_eval_user=_read("span_eval_user.md"),
|
||
missed_nodes=_read("missed_nodes.md"),
|
||
skill_adherence=_read("skill_adherence.md"),
|
||
confirmation_bias=_read("confirmation_bias.md"),
|
||
evidence_sufficiency=_read("evidence_sufficiency.md"),
|
||
)
|