diff --git a/app/harness/checkpoint.py b/app/harness/checkpoint.py index f686d3b..5c57544 100644 --- a/app/harness/checkpoint.py +++ b/app/harness/checkpoint.py @@ -94,7 +94,7 @@ def serialize_state(state: Any) -> dict[str, Any]: "eval_prev_run_id": state.eval_prev_run_id, "baseline_skills_version": state.baseline_skills_version, "baseline_prompts_version": state.baseline_prompts_version, - "steps_since_best_improved": state.steps_since_best_improved, + "epochs_since_best_improved": state.epochs_since_best_improved, "epoch_start_skills": state.epoch_start_skills, "changed_task_types_this_epoch": sorted(state.changed_task_types_this_epoch), "rejected_buffer": {k: [asdict(x) for x in v] for k, v in state.rejected_buffer.items()}, @@ -145,7 +145,7 @@ def deserialize_state_fields(d: dict[str, Any]) -> dict[str, Any]: "eval_prev_run_id": d["eval_prev_run_id"], "baseline_skills_version": d["baseline_skills_version"], "baseline_prompts_version": d["baseline_prompts_version"], - "steps_since_best_improved": d["steps_since_best_improved"], + "epochs_since_best_improved": d["epochs_since_best_improved"], "epoch_start_skills": d["epoch_start_skills"], "changed_task_types_this_epoch": set(d["changed_task_types_this_epoch"]), "rejected_buffer": { diff --git a/app/harness/runner.py b/app/harness/runner.py index f7bc167..97ddda9 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -115,7 +115,7 @@ class _TrainState: global_step: int = 0 changed_task_types_this_epoch: set[str] = field(default_factory=set) epoch_start_skills: dict[str, str] = field(default_factory=dict) - steps_since_best_improved: int = 0 + epochs_since_best_improved: int = 0 gate_epoch_observed: bool = False probations: dict[str, Probation] = field(default_factory=dict) gate_cooldown: dict[str, int] = field(default_factory=dict) @@ -291,18 +291,16 @@ def _snapshot_current_skills(skills_dir: Path) -> dict[str, str]: def _should_early_stop( workspace_dir: Path, epoch: int, - steps_this_epoch: int, state: _TrainState, patience: int, ) -> bool: - """步粒度 early stop:本 epoch best 未刷新则累加本 epoch 步数。 + """epoch 粒度 early stop:本 epoch best 未刷新则计数 +1。 参数: workspace_dir: workspace 目录(读 manifest best)。 epoch: 当前 epoch。 - steps_this_epoch: 本 epoch 的 step 总数。 - state: 训练状态(steps_since_best_improved 就地更新)。 - patience: early_stop_patience。 + state: 训练状态(epochs_since_best_improved 就地更新)。 + patience: early_stop_patience(连续无刷新的 epoch 数上限)。 返回: 是否触发 early stop。 @@ -310,10 +308,10 @@ def _should_early_stop( best = read_best(workspace_dir) improved_this_epoch = best is not None and best.get("epoch") == epoch if improved_this_epoch: - state.steps_since_best_improved = 0 + state.epochs_since_best_improved = 0 return False - state.steps_since_best_improved += steps_this_epoch - return state.steps_since_best_improved >= patience + state.epochs_since_best_improved += 1 + return state.epochs_since_best_improved >= patience def _compute_total_steps(pools: Pools, correctness: dict[str, bool], config: RunConfig) -> int: @@ -849,7 +847,6 @@ class Runner: if _should_early_stop( self._config.workspace_dir, epoch, - len(batches), state, self._config.early_stop_patience, ): @@ -1588,7 +1585,7 @@ class Runner: state.best_val_acc = eval_acc state.best_skills_version = skills_v state.best_prompts_version = prompts_v - state.steps_since_best_improved = 0 + state.epochs_since_best_improved = 0 update_best( self._config.workspace_dir, skills=f"skills/{skills_v}", diff --git a/tests/integration/test_checkpoint_pair.py b/tests/integration/test_checkpoint_pair.py index 0b5591a..c6b28dc 100644 --- a/tests/integration/test_checkpoint_pair.py +++ b/tests/integration/test_checkpoint_pair.py @@ -90,7 +90,7 @@ class _FakeState: eval_prev_run_id: str = "run-0" baseline_skills_version: str = "v1" baseline_prompts_version: str = "v1" - steps_since_best_improved: int = 0 + epochs_since_best_improved: int = 0 epoch_start_skills: str = "v1" changed_task_types_this_epoch: set[str] = field(default_factory=set) rejected_buffer: dict = field(default_factory=dict) diff --git a/tests/unit/test_harness_checkpoint.py b/tests/unit/test_harness_checkpoint.py index e5f0b95..1d5d40e 100644 --- a/tests/unit/test_harness_checkpoint.py +++ b/tests/unit/test_harness_checkpoint.py @@ -116,7 +116,7 @@ class _FakeState: eval_prev_run_id: str baseline_skills_version: str baseline_prompts_version: str - steps_since_best_improved: int + epochs_since_best_improved: int epoch_start_skills: str changed_task_types_this_epoch: set[str] rejected_buffer: dict[str, list[RejectedEdit]] @@ -135,7 +135,7 @@ def _make_state() -> _FakeState: eval_prev_run_id="run-abc", baseline_skills_version="v1", baseline_prompts_version="v1", - steps_since_best_improved=2, + epochs_since_best_improved=2, epoch_start_skills="v1", changed_task_types_this_epoch={"temporal", "causal"}, rejected_buffer={"temporal": [_make_rejected_edit()]}, @@ -201,7 +201,7 @@ class TestSerializeDeserializeRoundtrip: assert restored["eval_prev_run_id"] == state.eval_prev_run_id assert restored["baseline_skills_version"] == state.baseline_skills_version assert restored["baseline_prompts_version"] == state.baseline_prompts_version - assert restored["steps_since_best_improved"] == state.steps_since_best_improved + assert restored["epochs_since_best_improved"] == state.epochs_since_best_improved assert restored["epoch_start_skills"] == state.epoch_start_skills assert restored["changed_task_types_this_epoch"] == state.changed_task_types_this_epoch assert restored["gate_cooldown"] == state.gate_cooldown diff --git a/tests/unit/test_harness_runner.py b/tests/unit/test_harness_runner.py index c7b5f29..619a689 100644 --- a/tests/unit/test_harness_runner.py +++ b/tests/unit/test_harness_runner.py @@ -349,92 +349,56 @@ class TestBuildComparisonPairs: # ========================================================================= +def _write_manifest_with_best(tmp_path: Path, best_epoch: int) -> None: + """写含 best.epoch 的 manifest,供 _should_early_stop 读 read_best。""" + manifest = { + "name": "test", + "store": ".", + "current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"}, + "best": {"epoch": best_epoch, "val_acc": 0.5}, + "history": [], + } + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + + class TestShouldEarlyStop: - """_should_early_stop 步粒度 early stop。""" + """_should_early_stop epoch 粒度 early stop(patience 以 epoch 计)。""" def test_improved_this_epoch_resets(self, tmp_path: Path) -> None: """本 epoch best 刷新时重置计数器。""" - # 写 manifest + best - manifest = { - "name": "test", - "store": ".", - "current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"}, - "best": {"epoch": 2, "val_acc": 0.9}, - "history": [], - } - (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + _write_manifest_with_best(tmp_path, best_epoch=2) state = MagicMock() - state.steps_since_best_improved = 10 + state.epochs_since_best_improved = 3 - result = _should_early_stop(tmp_path, epoch=2, steps_this_epoch=5, state=state, patience=20) + result = _should_early_stop(tmp_path, epoch=2, state=state, patience=2) assert result is False - assert state.steps_since_best_improved == 0 - - def test_no_improvement_accumulates(self, tmp_path: Path) -> None: - """未刷新时累加步数。""" - manifest = { - "name": "test", - "store": ".", - "current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"}, - "best": {"epoch": 1, "val_acc": 0.5}, - "history": [], - } - (tmp_path / "manifest.json").write_text(json.dumps(manifest)) - - state = MagicMock() - state.steps_since_best_improved = 15 - - result = _should_early_stop(tmp_path, epoch=3, steps_this_epoch=5, state=state, patience=20) - assert result is True # 15 + 5 = 20 >= 20 - assert state.steps_since_best_improved == 20 + assert state.epochs_since_best_improved == 0 def test_below_patience_continues(self, tmp_path: Path) -> None: - """累计步数未达阈值时继续。""" - manifest = { - "name": "test", - "store": ".", - "current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"}, - "best": {"epoch": 1, "val_acc": 0.5}, - "history": [], - } - (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + """未达 patience 个 epoch 无刷新时继续。""" + _write_manifest_with_best(tmp_path, best_epoch=1) state = MagicMock() - state.steps_since_best_improved = 10 + state.epochs_since_best_improved = 0 - result = _should_early_stop(tmp_path, epoch=3, steps_this_epoch=5, state=state, patience=20) + result = _should_early_stop(tmp_path, epoch=2, state=state, patience=3) assert result is False - assert state.steps_since_best_improved == 15 + assert state.epochs_since_best_improved == 1 - def test_step_granularity(self, tmp_path: Path) -> None: - """步粒度而非 epoch 粒度。""" - manifest = { - "name": "test", - "store": ".", - "current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"}, - "best": {"epoch": 1, "val_acc": 0.5}, - "history": [], - } - (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + def test_early_stop_counts_epochs_not_steps(self, tmp_path: Path) -> None: + """patience=2 表示连续 2 个 epoch 无 best 刷新才停(不是步数)。""" + _write_manifest_with_best(tmp_path, best_epoch=1) state = MagicMock() - # 连续 3 个 epoch,每个 3 步 - state.steps_since_best_improved = 0 - for ep in range(2, 5): - stopped = _should_early_stop( - tmp_path, epoch=ep, steps_this_epoch=3, state=state, patience=10 - ) - if ep < 4: - assert stopped is False - else: - # 3+3+3=9 < 10 但第三轮后 9+3=12>=10 在 ep=5 触发 - # 实际:ep=2 → 3, ep=3 → 6, ep=4 → 9 - assert stopped is False - stopped = _should_early_stop( - tmp_path, epoch=5, steps_this_epoch=3, state=state, patience=10 - ) - assert stopped is True # 9+3=12>=10 + state.epochs_since_best_improved = 0 + + # epoch 2 无刷新 → 1 → 不停 + assert _should_early_stop(tmp_path, epoch=2, state=state, patience=2) is False + assert state.epochs_since_best_improved == 1 + # epoch 3 无刷新 → 2 → 停 + assert _should_early_stop(tmp_path, epoch=3, state=state, patience=2) is True + assert state.epochs_since_best_improved == 2 # ========================================================================= @@ -667,7 +631,7 @@ class TestTrainState: assert state.system_packs == [] assert state.tool_packs == [] assert state.changed_task_types_this_epoch == set() - assert state.steps_since_best_improved == 0 + assert state.epochs_since_best_improved == 0 # =========================================================================