feat(harness): checkpoint.py — TrainState 序列化 + 原子写 + 指纹校验
This commit is contained in:
@@ -0,0 +1,301 @@
|
|||||||
|
"""step 级续训 checkpoint:_TrainState 可持久化字段的序列化 / 反序列化。
|
||||||
|
|
||||||
|
_TrainState 的累加包均为扁平纯数据 dataclass,经 dataclasses.asdict 序列化为
|
||||||
|
纯 JSON dict;反序列化时用 Cls(**d) 还原,其中 SystemCasePack 含嵌套 CaseSample
|
||||||
|
列表、Probation 含嵌套 RejectedEdit 列表,需逐个重建。
|
||||||
|
|
||||||
|
不持久化的字段:gate_pools / baseline_cache(各自文件级自持久化,resume 时按
|
||||||
|
指纹重载)、best_*(从 manifest best 指针读)、global_step(存 progress 块,
|
||||||
|
由 train 单独赋值)。gate_epoch_observed 持久化:warm p-hat 在 gate_pools.json
|
||||||
|
幸存,观测开关须随行,否则 resume 后阶梯排序回退冷启动序。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from dataclasses import asdict, dataclass, field
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
from core.evolution.types import (
|
||||||
|
CaseSample,
|
||||||
|
RejectedEdit,
|
||||||
|
SystemCasePack,
|
||||||
|
ToolCasePack,
|
||||||
|
)
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
CHECKPOINT_SCHEMA_VERSION = 1
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Probation 类型(Task 10 validate.py 尚未就绪,暂定义在此供 checkpoint 使用)
|
||||||
|
# Task 10 完成后迁移至 app/harness/validate.py 并改为 re-export。
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Probation:
|
||||||
|
"""一个题型的在途试用账本(每题型至多一个)。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
task_type: 题型。
|
||||||
|
anchor_skills_version: 锚版本名(最近一个 CONFIRMED 的 skills 版本)。
|
||||||
|
target_file: 该题型解析后的 skill 文件名。
|
||||||
|
correctness_snapshot: 开账时该题型 val 题的对错快照(回滚时恢复)。
|
||||||
|
opened_step: 开账时的 global_step(观测用)。
|
||||||
|
pending_edits: 试用链上全部候选 edit 的黑名单素材(回滚时整链入黑名单)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
task_type: str
|
||||||
|
anchor_skills_version: str
|
||||||
|
target_file: str
|
||||||
|
correctness_snapshot: dict[str, bool]
|
||||||
|
opened_step: int
|
||||||
|
pending_edits: list[RejectedEdit] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 结构性 / 决策性指纹键
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_STRUCTURAL_KEYS = (
|
||||||
|
"batch_size",
|
||||||
|
"min_class_per_batch",
|
||||||
|
"epochs",
|
||||||
|
"diag_size",
|
||||||
|
"val_size",
|
||||||
|
"batch_correct_ratio",
|
||||||
|
)
|
||||||
|
|
||||||
|
_DECISION_KEYS = (
|
||||||
|
"edit_budget_start",
|
||||||
|
"edit_budget_end",
|
||||||
|
"early_stop_patience",
|
||||||
|
"use_slow_momentum",
|
||||||
|
"skill_update_mode",
|
||||||
|
"appendix_consolidate_threshold",
|
||||||
|
"momentum_samples",
|
||||||
|
"gate_e_confirm",
|
||||||
|
"gate_e_provisional",
|
||||||
|
"gate_w_net_min",
|
||||||
|
"gate_delta_min",
|
||||||
|
"gate_lambda_dir",
|
||||||
|
"gate_e_rollback",
|
||||||
|
"gate_block",
|
||||||
|
"gate_n_max",
|
||||||
|
"gate_p_low",
|
||||||
|
"gate_p_high",
|
||||||
|
"gate_probe_quota",
|
||||||
|
"gate_gamma_decay",
|
||||||
|
"gate_cooldown_steps",
|
||||||
|
"gate_guard_err",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 序列化 / 反序列化
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def serialize_state(state: Any) -> dict[str, Any]:
|
||||||
|
"""把 _TrainState 的可持久化字段转为纯 JSON dict。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
state: _TrainState 实例(duck-typed,仅需含可持久化字段)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
纯 JSON 可序列化的 dict,不含 gate_pools / baseline_cache /
|
||||||
|
best_* / global_step。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
- changed_task_types_this_epoch 是 set,JSON 无 set,故 sorted 成有序列表。
|
||||||
|
- dataclass 均经 asdict 递归转 dict(含 SystemCasePack 嵌套 CaseSample、
|
||||||
|
Probation 嵌套 RejectedEdit)。
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"correctness": state.correctness,
|
||||||
|
"eval_prev_acc": state.eval_prev_acc,
|
||||||
|
"eval_prev_run_id": state.eval_prev_run_id,
|
||||||
|
"baseline_skills_version": state.baseline_skills_version,
|
||||||
|
"baseline_prompts_version": state.baseline_prompts_version,
|
||||||
|
"steps_since_best_improved": state.steps_since_best_improved,
|
||||||
|
"epoch_start_skills": state.epoch_start_skills,
|
||||||
|
"changed_task_types_this_epoch": sorted(state.changed_task_types_this_epoch),
|
||||||
|
"rejected_buffer": {k: [asdict(x) for x in v] for k, v in state.rejected_buffer.items()},
|
||||||
|
"system_packs": [asdict(x) for x in state.system_packs],
|
||||||
|
"tool_packs": [asdict(x) for x in state.tool_packs],
|
||||||
|
"probations": {t: asdict(p) for t, p in state.probations.items()},
|
||||||
|
"gate_cooldown": state.gate_cooldown,
|
||||||
|
"gate_epoch_observed": state.gate_epoch_observed,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _restore_system_pack(d: dict[str, Any]) -> SystemCasePack:
|
||||||
|
"""还原 SystemCasePack,含嵌套 CaseSample 列表。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
d: asdict(SystemCasePack) 产出的 dict。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
复活的 SystemCasePack;failure_cases / success_cases 重建为 CaseSample 实例。
|
||||||
|
"""
|
||||||
|
return SystemCasePack(
|
||||||
|
stats=d["stats"],
|
||||||
|
failure_cases=[CaseSample(**c) for c in d["failure_cases"]],
|
||||||
|
success_cases=[CaseSample(**c) for c in d["success_cases"]],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def deserialize_state_fields(d: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
"""把序列化 dict 还原为可填入 _TrainState 的字段字典(dataclass 复活)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
d: serialize_state 产出并经 JSON 往返的 dict。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
字段名 -> 值的 dict,可直接铺到 _TrainState;其中各 dataclass 已复活、
|
||||||
|
changed_task_types_this_epoch 还原为 set。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
- RejectedEdit / ToolCasePack 字段均为标量/dict/list[dict],Cls(**d) 直接构造。
|
||||||
|
- SystemCasePack 含嵌套 CaseSample,交由 _restore_system_pack 重建。
|
||||||
|
- Probation 含嵌套 RejectedEdit 列表(pending_edits),先重建内层再构造外层。
|
||||||
|
- 直接取 d[...] 不用 .get 兜底:serialize 后的 checkpoint 必带全部键,
|
||||||
|
缺键即 checkpoint 损坏,应硬失败(P5 不掩盖)。
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"correctness": d["correctness"],
|
||||||
|
"eval_prev_acc": d["eval_prev_acc"],
|
||||||
|
"eval_prev_run_id": d["eval_prev_run_id"],
|
||||||
|
"baseline_skills_version": d["baseline_skills_version"],
|
||||||
|
"baseline_prompts_version": d["baseline_prompts_version"],
|
||||||
|
"steps_since_best_improved": d["steps_since_best_improved"],
|
||||||
|
"epoch_start_skills": d["epoch_start_skills"],
|
||||||
|
"changed_task_types_this_epoch": set(d["changed_task_types_this_epoch"]),
|
||||||
|
"rejected_buffer": {
|
||||||
|
k: [RejectedEdit(**x) for x in v] for k, v in d["rejected_buffer"].items()
|
||||||
|
},
|
||||||
|
"system_packs": [_restore_system_pack(x) for x in d["system_packs"]],
|
||||||
|
"tool_packs": [ToolCasePack(**x) for x in d["tool_packs"]],
|
||||||
|
"probations": {
|
||||||
|
t: Probation(
|
||||||
|
**{
|
||||||
|
**d_p,
|
||||||
|
"pending_edits": [RejectedEdit(**x) for x in d_p["pending_edits"]],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
for t, d_p in d["probations"].items()
|
||||||
|
},
|
||||||
|
"gate_cooldown": d["gate_cooldown"],
|
||||||
|
"gate_epoch_observed": d["gate_epoch_observed"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 配置指纹
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def compute_fingerprint(config: Any) -> dict[str, Any]:
|
||||||
|
"""采集影响训练轨迹的配置项(结构性 + 决策性)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
config: 训练配置对象(duck-typed,需含 _STRUCTURAL_KEYS + _DECISION_KEYS 属性)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
指纹 dict,键为配置项名,值为对应配置值。
|
||||||
|
"""
|
||||||
|
return {k: getattr(config, k) for k in _STRUCTURAL_KEYS + _DECISION_KEYS}
|
||||||
|
|
||||||
|
|
||||||
|
def check_fingerprint(saved: dict[str, Any], config: Any) -> tuple[list[str], list[str]]:
|
||||||
|
"""比对保存的指纹与当前配置。返回 (结构性不一致项, 决策性不一致项)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
saved: checkpoint 中保存的 config_fingerprint。
|
||||||
|
config: 当前训练配置对象。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
(structural, decision) 两个不一致项名列表。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
结构性不一致(batch_size/min_class_per_batch/epochs/diag_size/val_size/
|
||||||
|
batch_correct_ratio)→ 调用方应拒绝 resume;决策性不一致 → 仅告警放行。
|
||||||
|
"""
|
||||||
|
cur = compute_fingerprint(config)
|
||||||
|
structural = [k for k in _STRUCTURAL_KEYS if saved.get(k) != cur[k]]
|
||||||
|
decision = [k for k in _DECISION_KEYS if saved.get(k) != cur[k]]
|
||||||
|
return structural, decision
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 读写 checkpoint
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def write_checkpoint(
|
||||||
|
workspace_dir: Path,
|
||||||
|
*,
|
||||||
|
state: Any,
|
||||||
|
epoch: int,
|
||||||
|
step_completed: int,
|
||||||
|
phase: str,
|
||||||
|
global_step: int,
|
||||||
|
total_steps: int,
|
||||||
|
version_snapshot: dict[str, str],
|
||||||
|
epoch_batches: list[list[str]],
|
||||||
|
config: Any,
|
||||||
|
) -> None:
|
||||||
|
"""原子写 checkpoint.json(.tmp 再 os.replace)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
workspace_dir: workspace 目录,checkpoint.json 写入其下。
|
||||||
|
state: _TrainState 实例,交由 serialize_state 序列化。
|
||||||
|
epoch: 当前 epoch 序号。
|
||||||
|
step_completed: 本 epoch 内已完成的 step 数。
|
||||||
|
phase: 续训阶段标识(如 "in_epoch")。
|
||||||
|
global_step: 全局 step 序号。
|
||||||
|
total_steps: 全局总 step 数。
|
||||||
|
version_snapshot: skills/prompts 版本快照。
|
||||||
|
epoch_batches: 本 epoch 的 batch 划分(question_id 列表的列表)。
|
||||||
|
config: 训练配置对象,用于计算 config_fingerprint。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
先写 checkpoint.json.tmp 再 os.replace,保证 checkpoint 不被写一半的中断破坏。
|
||||||
|
"""
|
||||||
|
payload = {
|
||||||
|
"schema_version": CHECKPOINT_SCHEMA_VERSION,
|
||||||
|
"progress": {
|
||||||
|
"epoch": epoch,
|
||||||
|
"step_completed": step_completed,
|
||||||
|
"phase": phase,
|
||||||
|
"global_step": global_step,
|
||||||
|
"total_steps": total_steps,
|
||||||
|
},
|
||||||
|
"version_snapshot": version_snapshot,
|
||||||
|
"epoch_batches": epoch_batches,
|
||||||
|
"config_fingerprint": compute_fingerprint(config),
|
||||||
|
"state": serialize_state(state),
|
||||||
|
}
|
||||||
|
path = workspace_dir / "checkpoint.json"
|
||||||
|
tmp = path.with_name("checkpoint.json.tmp")
|
||||||
|
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||||
|
os.replace(tmp, path)
|
||||||
|
|
||||||
|
|
||||||
|
def load_checkpoint(workspace_dir: Path) -> dict[str, Any] | None:
|
||||||
|
"""读 checkpoint.json,不存在返回 None。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
workspace_dir: workspace 目录。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
checkpoint payload dict;checkpoint.json 不存在时返回 None。
|
||||||
|
"""
|
||||||
|
path = workspace_dir / "checkpoint.json"
|
||||||
|
if not path.exists():
|
||||||
|
return None
|
||||||
|
return json.loads(path.read_text())
|
||||||
@@ -0,0 +1,413 @@
|
|||||||
|
"""app/harness/checkpoint.py 单元测试。
|
||||||
|
|
||||||
|
覆盖序列化/反序列化往返、嵌套 dataclass 复活、缺键硬失败、
|
||||||
|
配置指纹结构性 vs 决策性判定、原子写与 load 缺失场景。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.harness.checkpoint import (
|
||||||
|
CHECKPOINT_SCHEMA_VERSION,
|
||||||
|
Probation,
|
||||||
|
check_fingerprint,
|
||||||
|
compute_fingerprint,
|
||||||
|
deserialize_state_fields,
|
||||||
|
load_checkpoint,
|
||||||
|
serialize_state,
|
||||||
|
write_checkpoint,
|
||||||
|
)
|
||||||
|
from core.evolution.types import (
|
||||||
|
CaseSample,
|
||||||
|
RejectedEdit,
|
||||||
|
SystemCasePack,
|
||||||
|
ToolCasePack,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# fixtures: 模拟 _TrainState 与 RunConfig
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _make_case_sample(**overrides: Any) -> CaseSample:
|
||||||
|
"""构造一个最小可用 CaseSample。"""
|
||||||
|
defaults: dict[str, Any] = {
|
||||||
|
"question_id": "q001",
|
||||||
|
"video_id": "v001",
|
||||||
|
"task_type": "temporal",
|
||||||
|
"question": "What happened?",
|
||||||
|
"options": ["A", "B", "C"],
|
||||||
|
"answer": "A",
|
||||||
|
"prediction": "B",
|
||||||
|
"correct": False,
|
||||||
|
"error_type": "reasoning",
|
||||||
|
"selection_reason": "worst",
|
||||||
|
"metrics": {"acc": 0.5},
|
||||||
|
"trace": [{"step": 1, "action": "search"}],
|
||||||
|
}
|
||||||
|
defaults.update(overrides)
|
||||||
|
return CaseSample(**defaults)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_rejected_edit(**overrides: Any) -> RejectedEdit:
|
||||||
|
"""构造一个最小可用 RejectedEdit。"""
|
||||||
|
defaults: dict[str, Any] = {
|
||||||
|
"target_file": "temporal-reasoning.md",
|
||||||
|
"target_type": "skill",
|
||||||
|
"change_summary": "added step",
|
||||||
|
"delta": -0.05,
|
||||||
|
"source_version": "v2",
|
||||||
|
"epoch": 1,
|
||||||
|
"gate_w": 3,
|
||||||
|
"gate_l": 5,
|
||||||
|
"gate_e_value": 0.8,
|
||||||
|
"gate_delta_shrunk": -0.02,
|
||||||
|
}
|
||||||
|
defaults.update(overrides)
|
||||||
|
return RejectedEdit(**defaults)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_system_pack() -> SystemCasePack:
|
||||||
|
"""构造包含嵌套 CaseSample 的 SystemCasePack。"""
|
||||||
|
return SystemCasePack(
|
||||||
|
stats={"pattern": "repeat_visit", "count": 3},
|
||||||
|
failure_cases=[_make_case_sample(question_id="q010")],
|
||||||
|
success_cases=[_make_case_sample(question_id="q011", correct=True, error_type=None)],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_tool_pack() -> ToolCasePack:
|
||||||
|
"""构造 ToolCasePack。"""
|
||||||
|
return ToolCasePack(
|
||||||
|
tool_name="search_subtree",
|
||||||
|
target_files=["search_subtree_extract.md"],
|
||||||
|
stats={"completeness": 0.8},
|
||||||
|
failure_spans=[{"step": 2, "issue": "missing"}],
|
||||||
|
success_spans=[{"step": 3, "quality": "good"}],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_probation() -> Probation:
|
||||||
|
"""构造包含嵌套 RejectedEdit 的 Probation。"""
|
||||||
|
return Probation(
|
||||||
|
task_type="temporal",
|
||||||
|
anchor_skills_version="v1",
|
||||||
|
target_file="temporal-reasoning.md",
|
||||||
|
correctness_snapshot={"q001": True, "q002": False},
|
||||||
|
opened_step=5,
|
||||||
|
pending_edits=[_make_rejected_edit()],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _FakeState:
|
||||||
|
"""模拟 _TrainState 全部可持久化字段。"""
|
||||||
|
|
||||||
|
correctness: dict[str, bool]
|
||||||
|
eval_prev_acc: float
|
||||||
|
eval_prev_run_id: str
|
||||||
|
baseline_skills_version: str
|
||||||
|
baseline_prompts_version: str
|
||||||
|
steps_since_best_improved: int
|
||||||
|
epoch_start_skills: str
|
||||||
|
changed_task_types_this_epoch: set[str]
|
||||||
|
rejected_buffer: dict[str, list[RejectedEdit]]
|
||||||
|
system_packs: list[SystemCasePack]
|
||||||
|
tool_packs: list[ToolCasePack]
|
||||||
|
probations: dict[str, Probation]
|
||||||
|
gate_cooldown: dict[str, int]
|
||||||
|
gate_epoch_observed: dict[str, bool]
|
||||||
|
|
||||||
|
|
||||||
|
def _make_state() -> _FakeState:
|
||||||
|
"""构造一个填满全部字段的 _FakeState。"""
|
||||||
|
return _FakeState(
|
||||||
|
correctness={"q001": True, "q002": False},
|
||||||
|
eval_prev_acc=0.65,
|
||||||
|
eval_prev_run_id="run-abc",
|
||||||
|
baseline_skills_version="v1",
|
||||||
|
baseline_prompts_version="v1",
|
||||||
|
steps_since_best_improved=2,
|
||||||
|
epoch_start_skills="v1",
|
||||||
|
changed_task_types_this_epoch={"temporal", "causal"},
|
||||||
|
rejected_buffer={"temporal": [_make_rejected_edit()]},
|
||||||
|
system_packs=[_make_system_pack()],
|
||||||
|
tool_packs=[_make_tool_pack()],
|
||||||
|
probations={"temporal": _make_probation()},
|
||||||
|
gate_cooldown={"temporal": 3},
|
||||||
|
gate_epoch_observed={"temporal": True},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class _FakeConfig:
|
||||||
|
"""模拟 RunConfig 的指纹相关字段。"""
|
||||||
|
|
||||||
|
batch_size: int = 8
|
||||||
|
min_class_per_batch: int = 2
|
||||||
|
epochs: int = 5
|
||||||
|
diag_size: int = 30
|
||||||
|
val_size: int = 50
|
||||||
|
batch_correct_ratio: float = 0.5
|
||||||
|
edit_budget_start: int = 6
|
||||||
|
edit_budget_end: int = 3
|
||||||
|
early_stop_patience: int = 3
|
||||||
|
use_slow_momentum: bool = True
|
||||||
|
skill_update_mode: str = "patch"
|
||||||
|
appendix_consolidate_threshold: int = 10
|
||||||
|
momentum_samples: int = 20
|
||||||
|
gate_e_confirm: float = 20.0
|
||||||
|
gate_e_provisional: float = 6.0
|
||||||
|
gate_w_net_min: int = 2
|
||||||
|
gate_delta_min: float = 0.02
|
||||||
|
gate_lambda_dir: float = -3.0
|
||||||
|
gate_e_rollback: float = 10.0
|
||||||
|
gate_block: int = 4
|
||||||
|
gate_n_max: int = 40
|
||||||
|
gate_p_low: float = 0.1
|
||||||
|
gate_p_high: float = 0.9
|
||||||
|
gate_probe_quota: float = 0.2
|
||||||
|
gate_gamma_decay: float = 0.9
|
||||||
|
gate_cooldown_steps: int = 2
|
||||||
|
gate_guard_err: float = 0.3
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# 测试用例
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestSerializeDeserializeRoundtrip:
|
||||||
|
"""序列化 → JSON 往返 → 反序列化应完全复原。"""
|
||||||
|
|
||||||
|
def test_serialize_deserialize_roundtrip(self) -> None:
|
||||||
|
state = _make_state()
|
||||||
|
serialized = serialize_state(state)
|
||||||
|
# JSON 往返(模拟实际落盘-读回)
|
||||||
|
json_str = json.dumps(serialized, ensure_ascii=False)
|
||||||
|
loaded = json.loads(json_str)
|
||||||
|
restored = deserialize_state_fields(loaded)
|
||||||
|
|
||||||
|
assert restored["correctness"] == state.correctness
|
||||||
|
assert restored["eval_prev_acc"] == state.eval_prev_acc
|
||||||
|
assert restored["eval_prev_run_id"] == state.eval_prev_run_id
|
||||||
|
assert restored["baseline_skills_version"] == state.baseline_skills_version
|
||||||
|
assert restored["baseline_prompts_version"] == state.baseline_prompts_version
|
||||||
|
assert restored["steps_since_best_improved"] == state.steps_since_best_improved
|
||||||
|
assert restored["epoch_start_skills"] == state.epoch_start_skills
|
||||||
|
assert restored["changed_task_types_this_epoch"] == state.changed_task_types_this_epoch
|
||||||
|
assert restored["gate_cooldown"] == state.gate_cooldown
|
||||||
|
assert restored["gate_epoch_observed"] == state.gate_epoch_observed
|
||||||
|
|
||||||
|
|
||||||
|
class TestSerializeSetToSortedList:
|
||||||
|
"""set 字段序列化为排序列表。"""
|
||||||
|
|
||||||
|
def test_serialize_set_to_sorted_list(self) -> None:
|
||||||
|
state = _make_state()
|
||||||
|
state.changed_task_types_this_epoch = {"z_type", "a_type", "m_type"}
|
||||||
|
serialized = serialize_state(state)
|
||||||
|
assert serialized["changed_task_types_this_epoch"] == ["a_type", "m_type", "z_type"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeserializeNestedSystemPack:
|
||||||
|
"""SystemCasePack 内嵌套的 CaseSample 正确复活。"""
|
||||||
|
|
||||||
|
def test_deserialize_nested_system_pack(self) -> None:
|
||||||
|
state = _make_state()
|
||||||
|
serialized = serialize_state(state)
|
||||||
|
json_str = json.dumps(serialized, ensure_ascii=False)
|
||||||
|
loaded = json.loads(json_str)
|
||||||
|
restored = deserialize_state_fields(loaded)
|
||||||
|
|
||||||
|
packs = restored["system_packs"]
|
||||||
|
assert len(packs) == 1
|
||||||
|
pack = packs[0]
|
||||||
|
assert isinstance(pack, SystemCasePack)
|
||||||
|
assert len(pack.failure_cases) == 1
|
||||||
|
assert isinstance(pack.failure_cases[0], CaseSample)
|
||||||
|
assert pack.failure_cases[0].question_id == "q010"
|
||||||
|
assert len(pack.success_cases) == 1
|
||||||
|
assert isinstance(pack.success_cases[0], CaseSample)
|
||||||
|
assert pack.success_cases[0].question_id == "q011"
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeserializeNestedProbation:
|
||||||
|
"""Probation 内嵌套的 RejectedEdit 正确复活。"""
|
||||||
|
|
||||||
|
def test_deserialize_nested_probation(self) -> None:
|
||||||
|
state = _make_state()
|
||||||
|
serialized = serialize_state(state)
|
||||||
|
json_str = json.dumps(serialized, ensure_ascii=False)
|
||||||
|
loaded = json.loads(json_str)
|
||||||
|
restored = deserialize_state_fields(loaded)
|
||||||
|
|
||||||
|
probations = restored["probations"]
|
||||||
|
assert "temporal" in probations
|
||||||
|
prob = probations["temporal"]
|
||||||
|
assert isinstance(prob, Probation)
|
||||||
|
assert prob.task_type == "temporal"
|
||||||
|
assert prob.anchor_skills_version == "v1"
|
||||||
|
assert prob.correctness_snapshot == {"q001": True, "q002": False}
|
||||||
|
assert len(prob.pending_edits) == 1
|
||||||
|
edit = prob.pending_edits[0]
|
||||||
|
assert isinstance(edit, RejectedEdit)
|
||||||
|
assert edit.target_file == "temporal-reasoning.md"
|
||||||
|
assert edit.delta == -0.05
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeserializeMissingKeyRaises:
|
||||||
|
"""缺键即 checkpoint 损坏,应硬失败。"""
|
||||||
|
|
||||||
|
def test_deserialize_missing_key_raises(self) -> None:
|
||||||
|
state = _make_state()
|
||||||
|
serialized = serialize_state(state)
|
||||||
|
del serialized["gate_epoch_observed"]
|
||||||
|
with pytest.raises(KeyError):
|
||||||
|
deserialize_state_fields(serialized)
|
||||||
|
|
||||||
|
|
||||||
|
class TestFingerprintStructuralVsDecision:
|
||||||
|
"""compute_fingerprint 包含全部结构性 + 决策性键。"""
|
||||||
|
|
||||||
|
def test_fingerprint_structural_vs_decision(self) -> None:
|
||||||
|
config = _FakeConfig()
|
||||||
|
fp = compute_fingerprint(config)
|
||||||
|
|
||||||
|
structural = {
|
||||||
|
"batch_size",
|
||||||
|
"min_class_per_batch",
|
||||||
|
"epochs",
|
||||||
|
"diag_size",
|
||||||
|
"val_size",
|
||||||
|
"batch_correct_ratio",
|
||||||
|
}
|
||||||
|
decision = {
|
||||||
|
"edit_budget_start",
|
||||||
|
"edit_budget_end",
|
||||||
|
"early_stop_patience",
|
||||||
|
"use_slow_momentum",
|
||||||
|
"skill_update_mode",
|
||||||
|
"appendix_consolidate_threshold",
|
||||||
|
"momentum_samples",
|
||||||
|
"gate_e_confirm",
|
||||||
|
"gate_e_provisional",
|
||||||
|
"gate_w_net_min",
|
||||||
|
"gate_delta_min",
|
||||||
|
"gate_lambda_dir",
|
||||||
|
"gate_e_rollback",
|
||||||
|
"gate_block",
|
||||||
|
"gate_n_max",
|
||||||
|
"gate_p_low",
|
||||||
|
"gate_p_high",
|
||||||
|
"gate_probe_quota",
|
||||||
|
"gate_gamma_decay",
|
||||||
|
"gate_cooldown_steps",
|
||||||
|
"gate_guard_err",
|
||||||
|
}
|
||||||
|
assert structural | decision == set(fp.keys())
|
||||||
|
assert fp["batch_size"] == 8
|
||||||
|
assert fp["gate_e_confirm"] == 20.0
|
||||||
|
|
||||||
|
|
||||||
|
class TestCheckFingerprintStructuralReject:
|
||||||
|
"""结构性键变化应出现在 structural 列表中。"""
|
||||||
|
|
||||||
|
def test_check_fingerprint_structural_reject(self) -> None:
|
||||||
|
config_old = _FakeConfig()
|
||||||
|
saved = compute_fingerprint(config_old)
|
||||||
|
# 修改结构性参数
|
||||||
|
config_new = _FakeConfig(batch_size=16, epochs=10)
|
||||||
|
structural, decision = check_fingerprint(saved, config_new)
|
||||||
|
assert "batch_size" in structural
|
||||||
|
assert "epochs" in structural
|
||||||
|
assert len(decision) == 0
|
||||||
|
|
||||||
|
|
||||||
|
class TestCheckFingerprintDecisionWarn:
|
||||||
|
"""决策性键变化应出现在 decision 列表中,structural 为空。"""
|
||||||
|
|
||||||
|
def test_check_fingerprint_decision_warn(self) -> None:
|
||||||
|
config_old = _FakeConfig()
|
||||||
|
saved = compute_fingerprint(config_old)
|
||||||
|
config_new = _FakeConfig(early_stop_patience=10, gate_e_confirm=50.0)
|
||||||
|
structural, decision = check_fingerprint(saved, config_new)
|
||||||
|
assert len(structural) == 0
|
||||||
|
assert "early_stop_patience" in decision
|
||||||
|
assert "gate_e_confirm" in decision
|
||||||
|
|
||||||
|
|
||||||
|
class TestWriteCheckpointAtomic:
|
||||||
|
"""原子写:先 .tmp 再 os.replace。"""
|
||||||
|
|
||||||
|
def test_write_checkpoint_atomic(self, tmp_path: Path) -> None:
|
||||||
|
state = _make_state()
|
||||||
|
config = _FakeConfig()
|
||||||
|
write_checkpoint(
|
||||||
|
tmp_path,
|
||||||
|
state=state,
|
||||||
|
epoch=2,
|
||||||
|
step_completed=5,
|
||||||
|
phase="in_epoch",
|
||||||
|
global_step=15,
|
||||||
|
total_steps=40,
|
||||||
|
version_snapshot={"skills": "v3", "prompts": "v2"},
|
||||||
|
epoch_batches=[["q001", "q002"], ["q003"]],
|
||||||
|
config=config,
|
||||||
|
)
|
||||||
|
ckpt_path = tmp_path / "checkpoint.json"
|
||||||
|
assert ckpt_path.exists()
|
||||||
|
# .tmp 应已被 os.replace 移除
|
||||||
|
assert not (tmp_path / "checkpoint.json.tmp").exists()
|
||||||
|
|
||||||
|
payload = json.loads(ckpt_path.read_text())
|
||||||
|
assert payload["schema_version"] == CHECKPOINT_SCHEMA_VERSION
|
||||||
|
assert payload["progress"]["epoch"] == 2
|
||||||
|
assert payload["progress"]["step_completed"] == 5
|
||||||
|
assert payload["progress"]["phase"] == "in_epoch"
|
||||||
|
assert payload["progress"]["global_step"] == 15
|
||||||
|
assert payload["progress"]["total_steps"] == 40
|
||||||
|
assert payload["version_snapshot"] == {"skills": "v3", "prompts": "v2"}
|
||||||
|
assert payload["epoch_batches"] == [["q001", "q002"], ["q003"]]
|
||||||
|
assert "config_fingerprint" in payload
|
||||||
|
assert "state" in payload
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoadCheckpointMissing:
|
||||||
|
"""checkpoint.json 不存在时返回 None。"""
|
||||||
|
|
||||||
|
def test_load_checkpoint_missing(self, tmp_path: Path) -> None:
|
||||||
|
result = load_checkpoint(tmp_path)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_load_checkpoint_exists(self, tmp_path: Path) -> None:
|
||||||
|
"""checkpoint.json 存在时正确读回。"""
|
||||||
|
state = _make_state()
|
||||||
|
config = _FakeConfig()
|
||||||
|
write_checkpoint(
|
||||||
|
tmp_path,
|
||||||
|
state=state,
|
||||||
|
epoch=1,
|
||||||
|
step_completed=3,
|
||||||
|
phase="post_evolve",
|
||||||
|
global_step=8,
|
||||||
|
total_steps=20,
|
||||||
|
version_snapshot={"skills": "v2", "prompts": "v1"},
|
||||||
|
epoch_batches=[["q001"]],
|
||||||
|
config=config,
|
||||||
|
)
|
||||||
|
loaded = load_checkpoint(tmp_path)
|
||||||
|
assert loaded is not None
|
||||||
|
assert loaded["schema_version"] == CHECKPOINT_SCHEMA_VERSION
|
||||||
|
assert loaded["progress"]["epoch"] == 1
|
||||||
|
# 完整往返测试:state 可 deserialize
|
||||||
|
restored = deserialize_state_fields(loaded["state"])
|
||||||
|
assert restored["eval_prev_acc"] == 0.65
|
||||||
Reference in New Issue
Block a user