fix: idempotent _run_step (DELETE stale) + checkpoint after gate save
This commit is contained in:
+53
-16
@@ -887,21 +887,15 @@ class Runner:
|
||||
epoch_batches=batch_unit_ids,
|
||||
config=self._config,
|
||||
)
|
||||
await self._slow_update_cycle(epoch, pools, state)
|
||||
state.system_packs = []
|
||||
state.tool_packs = []
|
||||
state.changed_task_types_this_epoch = set()
|
||||
write_checkpoint(
|
||||
self._config.workspace_dir,
|
||||
state=state,
|
||||
epoch=epoch,
|
||||
step_completed=len(batches) - 1,
|
||||
phase="epoch_done",
|
||||
global_step=state.global_step,
|
||||
# checkpoint 落盘移入 _slow_update_cycle 末尾(gate_pools.save 之后立即写),
|
||||
# 消除 gate_epoch_observed 在 gate_pools.json 与 checkpoint 间的双计窗口。
|
||||
await self._slow_update_cycle(
|
||||
epoch,
|
||||
pools,
|
||||
state,
|
||||
total_steps=total_steps,
|
||||
version_snapshot=self._current_version_snapshot(),
|
||||
step_completed=len(batches) - 1,
|
||||
epoch_batches=batch_unit_ids,
|
||||
config=self._config,
|
||||
)
|
||||
if _should_early_stop(
|
||||
self._config.workspace_dir,
|
||||
@@ -1081,10 +1075,21 @@ class Runner:
|
||||
) -> None:
|
||||
"""单 step:rollout → correctness 增量 → 诊断 → 累加 system/tool → 按类 gate。"""
|
||||
run_id = f"{pools.baseline_run_id}_e{epoch}_s{step}"
|
||||
await self._rollout_batch(batch, run_id)
|
||||
|
||||
from app.harness.inference import PREDICTIONS_SCHEMA, TRACES_SCHEMA
|
||||
from app.harness.log import HarnessLog
|
||||
|
||||
# 幂等:重跑同一 step 前先清旧行,避免断点续跑重复累计双计。
|
||||
# 先 CREATE TABLE IF NOT EXISTS(fresh workspace 首跑时表尚未由 run_inference 建),
|
||||
# register_run=False 避免只读清理污染 _runs 运行状态。
|
||||
with HarnessLog(str(self._paths.db_path), run_id, register_run=False) as log:
|
||||
log.create_table("predictions", PREDICTIONS_SCHEMA)
|
||||
log.create_table("traces", TRACES_SCHEMA)
|
||||
log.execute("DELETE FROM predictions WHERE run_id=?", (run_id,))
|
||||
log.execute("DELETE FROM traces WHERE run_id=?", (run_id,))
|
||||
|
||||
await self._rollout_batch(batch, run_id)
|
||||
|
||||
with HarnessLog(str(self._paths.db_path), run_id) as log:
|
||||
_apply_batch_correctness(state.correctness, log, run_id, batch)
|
||||
|
||||
@@ -1454,7 +1459,16 @@ class Runner:
|
||||
# _slow_update_cycle 十步序
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
async def _slow_update_cycle(self, epoch: int, pools: Pools, state: _TrainState) -> None:
|
||||
async def _slow_update_cycle(
|
||||
self,
|
||||
epoch: int,
|
||||
pools: Pools,
|
||||
state: _TrainState,
|
||||
*,
|
||||
total_steps: int,
|
||||
step_completed: int,
|
||||
epoch_batches: list[list[str]],
|
||||
) -> None:
|
||||
"""epoch 末慢更新十步序。
|
||||
|
||||
1. 捕获版本快照 → 全 val 重跑 R
|
||||
@@ -1466,7 +1480,12 @@ class Runner:
|
||||
7. system/tool 慢更新(edit_budget_end)
|
||||
8. R2 闭环
|
||||
9. 三态标签 + epoch_report + 四向 held-out
|
||||
10. gate 阶梯刷新
|
||||
10. gate 阶梯刷新 → 重置 epoch 累加器 → 立即落 epoch_done checkpoint
|
||||
|
||||
参数:
|
||||
total_steps: 全局总 step 数(checkpoint 用)。
|
||||
step_completed: 本 epoch 已完成 step 数(checkpoint 用)。
|
||||
epoch_batches: 本 epoch batch 的 unit_id 划分(checkpoint 用)。
|
||||
"""
|
||||
# Phase 1
|
||||
eval_skills_version = self._current_version("skills")
|
||||
@@ -1578,6 +1597,24 @@ class Runner:
|
||||
epoch, pools.baseline_run_id, state, extra_run_ids=r2_kept_run_ids
|
||||
)
|
||||
|
||||
# 重置 epoch 累加器 + 立即落 epoch_done checkpoint:与 _refresh_gate_ladder 内的
|
||||
# gate_pools.save + gate_epoch_observed=True 同刻一致,消除断点续跑的双计窗口。
|
||||
state.system_packs = []
|
||||
state.tool_packs = []
|
||||
state.changed_task_types_this_epoch = set()
|
||||
write_checkpoint(
|
||||
self._config.workspace_dir,
|
||||
state=state,
|
||||
epoch=epoch,
|
||||
step_completed=step_completed,
|
||||
phase="epoch_done",
|
||||
global_step=state.global_step,
|
||||
total_steps=total_steps,
|
||||
version_snapshot=self._current_version_snapshot(),
|
||||
epoch_batches=epoch_batches,
|
||||
config=self._config,
|
||||
)
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# 慢更新内部方法
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user