feat(harness): checkpoint.py — TrainState 序列化 + 原子写 + 指纹校验

2026-07-07 13:13:37 -04:00
parent d7f1bdeea6
commit a6b816db94
2 changed files with 714 additions and 0 deletions
@@ -0,0 +1,413 @@
+"""app/harness/checkpoint.py 单元测试。
+
+覆盖序列化/反序列化往返、嵌套 dataclass 复活、缺键硬失败、
+配置指纹结构性 vs 决策性判定、原子写与 load 缺失场景。
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+from app.harness.checkpoint import (
+    CHECKPOINT_SCHEMA_VERSION,
+    Probation,
+    check_fingerprint,
+    compute_fingerprint,
+    deserialize_state_fields,
+    load_checkpoint,
+    serialize_state,
+    write_checkpoint,
+)
+from core.evolution.types import (
+    CaseSample,
+    RejectedEdit,
+    SystemCasePack,
+    ToolCasePack,
+)
+
+# ---------------------------------------------------------------------------
+# fixtures: 模拟 _TrainState 与 RunConfig
+# ---------------------------------------------------------------------------
+
+
+def _make_case_sample(**overrides: Any) -> CaseSample:
+    """构造一个最小可用 CaseSample。"""
+    defaults: dict[str, Any] = {
+        "question_id": "q001",
+        "video_id": "v001",
+        "task_type": "temporal",
+        "question": "What happened?",
+        "options": ["A", "B", "C"],
+        "answer": "A",
+        "prediction": "B",
+        "correct": False,
+        "error_type": "reasoning",
+        "selection_reason": "worst",
+        "metrics": {"acc": 0.5},
+        "trace": [{"step": 1, "action": "search"}],
+    }
+    defaults.update(overrides)
+    return CaseSample(**defaults)
+
+
+def _make_rejected_edit(**overrides: Any) -> RejectedEdit:
+    """构造一个最小可用 RejectedEdit。"""
+    defaults: dict[str, Any] = {
+        "target_file": "temporal-reasoning.md",
+        "target_type": "skill",
+        "change_summary": "added step",
+        "delta": -0.05,
+        "source_version": "v2",
+        "epoch": 1,
+        "gate_w": 3,
+        "gate_l": 5,
+        "gate_e_value": 0.8,
+        "gate_delta_shrunk": -0.02,
+    }
+    defaults.update(overrides)
+    return RejectedEdit(**defaults)
+
+
+def _make_system_pack() -> SystemCasePack:
+    """构造包含嵌套 CaseSample 的 SystemCasePack。"""
+    return SystemCasePack(
+        stats={"pattern": "repeat_visit", "count": 3},
+        failure_cases=[_make_case_sample(question_id="q010")],
+        success_cases=[_make_case_sample(question_id="q011", correct=True, error_type=None)],
+    )
+
+
+def _make_tool_pack() -> ToolCasePack:
+    """构造 ToolCasePack。"""
+    return ToolCasePack(
+        tool_name="search_subtree",
+        target_files=["search_subtree_extract.md"],
+        stats={"completeness": 0.8},
+        failure_spans=[{"step": 2, "issue": "missing"}],
+        success_spans=[{"step": 3, "quality": "good"}],
+    )
+
+
+def _make_probation() -> Probation:
+    """构造包含嵌套 RejectedEdit 的 Probation。"""
+    return Probation(
+        task_type="temporal",
+        anchor_skills_version="v1",
+        target_file="temporal-reasoning.md",
+        correctness_snapshot={"q001": True, "q002": False},
+        opened_step=5,
+        pending_edits=[_make_rejected_edit()],
+    )
+
+
+@dataclass
+class _FakeState:
+    """模拟 _TrainState 全部可持久化字段。"""
+
+    correctness: dict[str, bool]
+    eval_prev_acc: float
+    eval_prev_run_id: str
+    baseline_skills_version: str
+    baseline_prompts_version: str
+    steps_since_best_improved: int
+    epoch_start_skills: str
+    changed_task_types_this_epoch: set[str]
+    rejected_buffer: dict[str, list[RejectedEdit]]
+    system_packs: list[SystemCasePack]
+    tool_packs: list[ToolCasePack]
+    probations: dict[str, Probation]
+    gate_cooldown: dict[str, int]
+    gate_epoch_observed: dict[str, bool]
+
+
+def _make_state() -> _FakeState:
+    """构造一个填满全部字段的 _FakeState。"""
+    return _FakeState(
+        correctness={"q001": True, "q002": False},
+        eval_prev_acc=0.65,
+        eval_prev_run_id="run-abc",
+        baseline_skills_version="v1",
+        baseline_prompts_version="v1",
+        steps_since_best_improved=2,
+        epoch_start_skills="v1",
+        changed_task_types_this_epoch={"temporal", "causal"},
+        rejected_buffer={"temporal": [_make_rejected_edit()]},
+        system_packs=[_make_system_pack()],
+        tool_packs=[_make_tool_pack()],
+        probations={"temporal": _make_probation()},
+        gate_cooldown={"temporal": 3},
+        gate_epoch_observed={"temporal": True},
+    )
+
+
+@dataclass(frozen=True)
+class _FakeConfig:
+    """模拟 RunConfig 的指纹相关字段。"""
+
+    batch_size: int = 8
+    min_class_per_batch: int = 2
+    epochs: int = 5
+    diag_size: int = 30
+    val_size: int = 50
+    batch_correct_ratio: float = 0.5
+    edit_budget_start: int = 6
+    edit_budget_end: int = 3
+    early_stop_patience: int = 3
+    use_slow_momentum: bool = True
+    skill_update_mode: str = "patch"
+    appendix_consolidate_threshold: int = 10
+    momentum_samples: int = 20
+    gate_e_confirm: float = 20.0
+    gate_e_provisional: float = 6.0
+    gate_w_net_min: int = 2
+    gate_delta_min: float = 0.02
+    gate_lambda_dir: float = -3.0
+    gate_e_rollback: float = 10.0
+    gate_block: int = 4
+    gate_n_max: int = 40
+    gate_p_low: float = 0.1
+    gate_p_high: float = 0.9
+    gate_probe_quota: float = 0.2
+    gate_gamma_decay: float = 0.9
+    gate_cooldown_steps: int = 2
+    gate_guard_err: float = 0.3
+
+
+# =========================================================================
+# 测试用例
+# =========================================================================
+
+
+class TestSerializeDeserializeRoundtrip:
+    """序列化 → JSON 往返 → 反序列化应完全复原。"""
+
+    def test_serialize_deserialize_roundtrip(self) -> None:
+        state = _make_state()
+        serialized = serialize_state(state)
+        # JSON 往返（模拟实际落盘-读回）
+        json_str = json.dumps(serialized, ensure_ascii=False)
+        loaded = json.loads(json_str)
+        restored = deserialize_state_fields(loaded)
+
+        assert restored["correctness"] == state.correctness
+        assert restored["eval_prev_acc"] == state.eval_prev_acc
+        assert restored["eval_prev_run_id"] == state.eval_prev_run_id
+        assert restored["baseline_skills_version"] == state.baseline_skills_version
+        assert restored["baseline_prompts_version"] == state.baseline_prompts_version
+        assert restored["steps_since_best_improved"] == state.steps_since_best_improved
+        assert restored["epoch_start_skills"] == state.epoch_start_skills
+        assert restored["changed_task_types_this_epoch"] == state.changed_task_types_this_epoch
+        assert restored["gate_cooldown"] == state.gate_cooldown
+        assert restored["gate_epoch_observed"] == state.gate_epoch_observed
+
+
+class TestSerializeSetToSortedList:
+    """set 字段序列化为排序列表。"""
+
+    def test_serialize_set_to_sorted_list(self) -> None:
+        state = _make_state()
+        state.changed_task_types_this_epoch = {"z_type", "a_type", "m_type"}
+        serialized = serialize_state(state)
+        assert serialized["changed_task_types_this_epoch"] == ["a_type", "m_type", "z_type"]
+
+
+class TestDeserializeNestedSystemPack:
+    """SystemCasePack 内嵌套的 CaseSample 正确复活。"""
+
+    def test_deserialize_nested_system_pack(self) -> None:
+        state = _make_state()
+        serialized = serialize_state(state)
+        json_str = json.dumps(serialized, ensure_ascii=False)
+        loaded = json.loads(json_str)
+        restored = deserialize_state_fields(loaded)
+
+        packs = restored["system_packs"]
+        assert len(packs) == 1
+        pack = packs[0]
+        assert isinstance(pack, SystemCasePack)
+        assert len(pack.failure_cases) == 1
+        assert isinstance(pack.failure_cases[0], CaseSample)
+        assert pack.failure_cases[0].question_id == "q010"
+        assert len(pack.success_cases) == 1
+        assert isinstance(pack.success_cases[0], CaseSample)
+        assert pack.success_cases[0].question_id == "q011"
+
+
+class TestDeserializeNestedProbation:
+    """Probation 内嵌套的 RejectedEdit 正确复活。"""
+
+    def test_deserialize_nested_probation(self) -> None:
+        state = _make_state()
+        serialized = serialize_state(state)
+        json_str = json.dumps(serialized, ensure_ascii=False)
+        loaded = json.loads(json_str)
+        restored = deserialize_state_fields(loaded)
+
+        probations = restored["probations"]
+        assert "temporal" in probations
+        prob = probations["temporal"]
+        assert isinstance(prob, Probation)
+        assert prob.task_type == "temporal"
+        assert prob.anchor_skills_version == "v1"
+        assert prob.correctness_snapshot == {"q001": True, "q002": False}
+        assert len(prob.pending_edits) == 1
+        edit = prob.pending_edits[0]
+        assert isinstance(edit, RejectedEdit)
+        assert edit.target_file == "temporal-reasoning.md"
+        assert edit.delta == -0.05
+
+
+class TestDeserializeMissingKeyRaises:
+    """缺键即 checkpoint 损坏，应硬失败。"""
+
+    def test_deserialize_missing_key_raises(self) -> None:
+        state = _make_state()
+        serialized = serialize_state(state)
+        del serialized["gate_epoch_observed"]
+        with pytest.raises(KeyError):
+            deserialize_state_fields(serialized)
+
+
+class TestFingerprintStructuralVsDecision:
+    """compute_fingerprint 包含全部结构性 + 决策性键。"""
+
+    def test_fingerprint_structural_vs_decision(self) -> None:
+        config = _FakeConfig()
+        fp = compute_fingerprint(config)
+
+        structural = {
+            "batch_size",
+            "min_class_per_batch",
+            "epochs",
+            "diag_size",
+            "val_size",
+            "batch_correct_ratio",
+        }
+        decision = {
+            "edit_budget_start",
+            "edit_budget_end",
+            "early_stop_patience",
+            "use_slow_momentum",
+            "skill_update_mode",
+            "appendix_consolidate_threshold",
+            "momentum_samples",
+            "gate_e_confirm",
+            "gate_e_provisional",
+            "gate_w_net_min",
+            "gate_delta_min",
+            "gate_lambda_dir",
+            "gate_e_rollback",
+            "gate_block",
+            "gate_n_max",
+            "gate_p_low",
+            "gate_p_high",
+            "gate_probe_quota",
+            "gate_gamma_decay",
+            "gate_cooldown_steps",
+            "gate_guard_err",
+        }
+        assert structural | decision == set(fp.keys())
+        assert fp["batch_size"] == 8
+        assert fp["gate_e_confirm"] == 20.0
+
+
+class TestCheckFingerprintStructuralReject:
+    """结构性键变化应出现在 structural 列表中。"""
+
+    def test_check_fingerprint_structural_reject(self) -> None:
+        config_old = _FakeConfig()
+        saved = compute_fingerprint(config_old)
+        # 修改结构性参数
+        config_new = _FakeConfig(batch_size=16, epochs=10)
+        structural, decision = check_fingerprint(saved, config_new)
+        assert "batch_size" in structural
+        assert "epochs" in structural
+        assert len(decision) == 0
+
+
+class TestCheckFingerprintDecisionWarn:
+    """决策性键变化应出现在 decision 列表中，structural 为空。"""
+
+    def test_check_fingerprint_decision_warn(self) -> None:
+        config_old = _FakeConfig()
+        saved = compute_fingerprint(config_old)
+        config_new = _FakeConfig(early_stop_patience=10, gate_e_confirm=50.0)
+        structural, decision = check_fingerprint(saved, config_new)
+        assert len(structural) == 0
+        assert "early_stop_patience" in decision
+        assert "gate_e_confirm" in decision
+
+
+class TestWriteCheckpointAtomic:
+    """原子写：先 .tmp 再 os.replace。"""
+
+    def test_write_checkpoint_atomic(self, tmp_path: Path) -> None:
+        state = _make_state()
+        config = _FakeConfig()
+        write_checkpoint(
+            tmp_path,
+            state=state,
+            epoch=2,
+            step_completed=5,
+            phase="in_epoch",
+            global_step=15,
+            total_steps=40,
+            version_snapshot={"skills": "v3", "prompts": "v2"},
+            epoch_batches=[["q001", "q002"], ["q003"]],
+            config=config,
+        )
+        ckpt_path = tmp_path / "checkpoint.json"
+        assert ckpt_path.exists()
+        # .tmp 应已被 os.replace 移除
+        assert not (tmp_path / "checkpoint.json.tmp").exists()
+
+        payload = json.loads(ckpt_path.read_text())
+        assert payload["schema_version"] == CHECKPOINT_SCHEMA_VERSION
+        assert payload["progress"]["epoch"] == 2
+        assert payload["progress"]["step_completed"] == 5
+        assert payload["progress"]["phase"] == "in_epoch"
+        assert payload["progress"]["global_step"] == 15
+        assert payload["progress"]["total_steps"] == 40
+        assert payload["version_snapshot"] == {"skills": "v3", "prompts": "v2"}
+        assert payload["epoch_batches"] == [["q001", "q002"], ["q003"]]
+        assert "config_fingerprint" in payload
+        assert "state" in payload
+
+
+class TestLoadCheckpointMissing:
+    """checkpoint.json 不存在时返回 None。"""
+
+    def test_load_checkpoint_missing(self, tmp_path: Path) -> None:
+        result = load_checkpoint(tmp_path)
+        assert result is None
+
+    def test_load_checkpoint_exists(self, tmp_path: Path) -> None:
+        """checkpoint.json 存在时正确读回。"""
+        state = _make_state()
+        config = _FakeConfig()
+        write_checkpoint(
+            tmp_path,
+            state=state,
+            epoch=1,
+            step_completed=3,
+            phase="post_evolve",
+            global_step=8,
+            total_steps=20,
+            version_snapshot={"skills": "v2", "prompts": "v1"},
+            epoch_batches=[["q001"]],
+            config=config,
+        )
+        loaded = load_checkpoint(tmp_path)
+        assert loaded is not None
+        assert loaded["schema_version"] == CHECKPOINT_SCHEMA_VERSION
+        assert loaded["progress"]["epoch"] == 1
+        # 完整往返测试：state 可 deserialize
+        restored = deserialize_state_fields(loaded["state"])
+        assert restored["eval_prev_acc"] == 0.65