fix: early_stop patience counts epochs not steps

This commit is contained in:
2026-07-16 06:09:11 -04:00
parent 25918a73ff
commit c44f6010eb
5 changed files with 48 additions and 87 deletions
+2 -2
View File
@@ -94,7 +94,7 @@ def serialize_state(state: Any) -> dict[str, Any]:
"eval_prev_run_id": state.eval_prev_run_id,
"baseline_skills_version": state.baseline_skills_version,
"baseline_prompts_version": state.baseline_prompts_version,
"steps_since_best_improved": state.steps_since_best_improved,
"epochs_since_best_improved": state.epochs_since_best_improved,
"epoch_start_skills": state.epoch_start_skills,
"changed_task_types_this_epoch": sorted(state.changed_task_types_this_epoch),
"rejected_buffer": {k: [asdict(x) for x in v] for k, v in state.rejected_buffer.items()},
@@ -145,7 +145,7 @@ def deserialize_state_fields(d: dict[str, Any]) -> dict[str, Any]:
"eval_prev_run_id": d["eval_prev_run_id"],
"baseline_skills_version": d["baseline_skills_version"],
"baseline_prompts_version": d["baseline_prompts_version"],
"steps_since_best_improved": d["steps_since_best_improved"],
"epochs_since_best_improved": d["epochs_since_best_improved"],
"epoch_start_skills": d["epoch_start_skills"],
"changed_task_types_this_epoch": set(d["changed_task_types_this_epoch"]),
"rejected_buffer": {
+8 -11
View File
@@ -115,7 +115,7 @@ class _TrainState:
global_step: int = 0
changed_task_types_this_epoch: set[str] = field(default_factory=set)
epoch_start_skills: dict[str, str] = field(default_factory=dict)
steps_since_best_improved: int = 0
epochs_since_best_improved: int = 0
gate_epoch_observed: bool = False
probations: dict[str, Probation] = field(default_factory=dict)
gate_cooldown: dict[str, int] = field(default_factory=dict)
@@ -291,18 +291,16 @@ def _snapshot_current_skills(skills_dir: Path) -> dict[str, str]:
def _should_early_stop(
workspace_dir: Path,
epoch: int,
steps_this_epoch: int,
state: _TrainState,
patience: int,
) -> bool:
"""粒度 early stop:本 epoch best 未刷新则累加本 epoch 步数
"""epoch 粒度 early stop:本 epoch best 未刷新则计数 +1
参数:
workspace_dir: workspace 目录(读 manifest best)。
epoch: 当前 epoch。
steps_this_epoch: 本 epoch 的 step 总数
state: 训练状态(steps_since_best_improved 就地更新)。
patience: early_stop_patience。
state: 训练状态(epochs_since_best_improved 就地更新)
patience: early_stop_patience(连续无刷新的 epoch 数上限)。
返回:
是否触发 early stop。
@@ -310,10 +308,10 @@ def _should_early_stop(
best = read_best(workspace_dir)
improved_this_epoch = best is not None and best.get("epoch") == epoch
if improved_this_epoch:
state.steps_since_best_improved = 0
state.epochs_since_best_improved = 0
return False
state.steps_since_best_improved += steps_this_epoch
return state.steps_since_best_improved >= patience
state.epochs_since_best_improved += 1
return state.epochs_since_best_improved >= patience
def _compute_total_steps(pools: Pools, correctness: dict[str, bool], config: RunConfig) -> int:
@@ -849,7 +847,6 @@ class Runner:
if _should_early_stop(
self._config.workspace_dir,
epoch,
len(batches),
state,
self._config.early_stop_patience,
):
@@ -1588,7 +1585,7 @@ class Runner:
state.best_val_acc = eval_acc
state.best_skills_version = skills_v
state.best_prompts_version = prompts_v
state.steps_since_best_improved = 0
state.epochs_since_best_improved = 0
update_best(
self._config.workspace_dir,
skills=f"skills/{skills_v}",