From 21c360bc8711cea1067aa2ee81369aff0d6ccb82 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 23:44:32 -0400 Subject: [PATCH 01/17] feat: gate prefix-ordered consumption core (algo #6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CE-Gate 语义修订获批:块序贯 → 阶梯序前缀逐对序贯。新增 GateSpec/_UnitSlot/ _GateRun 数据结构与 _advance_prefix 纯逻辑(乱序到达下统计严格按预声明阶梯序 消费,INFRA 剔除后重判防 continue 悬置,过线即冻结)。旧块路径共存,Task 6 删。 --- app/harness/validate.py | 135 +++++++++++++++++++++++++++++++++ tests/unit/test_gate_prefix.py | 134 ++++++++++++++++++++++++++++++++ 2 files changed, 269 insertions(+) create mode 100644 tests/unit/test_gate_prefix.py diff --git a/app/harness/validate.py b/app/harness/validate.py index 3e37756..0e2adf7 100644 --- a/app/harness/validate.py +++ b/app/harness/validate.py @@ -793,3 +793,138 @@ async def validate_skill_local( shutil.rmtree(cand_dir) except OSError as e: logger.warning("候选临时目录清理失败 {}: {}", cand_dir, e) + + +# --------------------------------------------------------------------------- +# 连续并发 gate:数据结构 + 前缀消费(algo #6 语义修订:块序贯 → 阶梯序前缀逐对序贯) +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class GateSpec: + """单题型 gate 验证规格(runner 装配阶段产物,调度器输入)。 + + 字段: + task_type: 题型。 + target_file: 解析后生效 skill 文件名(候选物化写此文件)。 + candidate_content: 候选 skill 全文。 + base_skill_content: 基线侧生效 skill 全文(skill_hash 作缓存键)。 + units: 阶梯序单元列表(已排除案例单元、截断 gate_n_max)。 + gate_run_prefix: run_id 前缀,必须含 "_gate_"(防泄露过滤依赖)。 + """ + + task_type: str + target_file: str + candidate_content: str + base_skill_content: str + units: list[QuestionUnit] + gate_run_prefix: str + + +@dataclass +class _UnitSlot: + """单个阶梯单元的双臂到达状态。 + + base 为单元级对错(AR pair 已折叠);cand_per_q 为逐题对错(折叠交给消费时, + 以复用 unit_correctness_view 并保留逐题溯源)。INFRA 标志与结果互斥。 + """ + + unit: QuestionUnit + base: bool | None = None + cand_per_q: dict[str, bool] | None = None + base_infra: bool = False + cand_infra: bool = False + + def resolved(self) -> bool: + """双臂均已出结果(含 INFRA 判定)。""" + base_done = self.base is not None or self.base_infra + cand_done = self.cand_per_q is not None or self.cand_infra + return base_done and cand_done + + def excluded(self) -> bool: + """任一臂 INFRA 即整单元剔除(不入配对)。""" + return self.base_infra or self.cand_infra + + +@dataclass +class _GateRun: + """单题型 gate 的运行时状态(计数器 + 前缀指针 + 证据)。""" + + spec: GateSpec + slots: list[_UnitSlot] + s_hash: str + prefix_ptr: int = 0 + w: int = 0 + l: int = 0 # noqa: E741 + n_used: int = 0 + n_excluded: int = 0 + errors: int = 0 + infra_denom: int = 0 + frozen: bool = False + verdict: GateVerdict | None = None + base_obs: dict[str, bool] = field(default_factory=dict) + cand_obs: dict[str, bool] = field(default_factory=dict) + candidate_per_q: dict[str, bool] = field(default_factory=dict) + evidence_rows: list[dict] = field(default_factory=list) + + @classmethod + def from_spec(cls, spec: GateSpec) -> _GateRun: + """由规格构造初始状态(slots 与阶梯序一一对应)。""" + return cls( + spec=spec, + slots=[_UnitSlot(unit=u) for u in spec.units], + s_hash=skill_hash(spec.base_skill_content), + ) + + +def _advance_prefix(run: _GateRun, params: GateParams) -> None: + """沿阶梯序消费"已配齐前缀",逐单元更新 (W,L) 并判定,过线即冻结。 + + 统计合法性关键(设计 v3 §1 / Codex C1):严禁按到达序消费——base 臂缓存命中 + 瞬间返回、cand 臂必新鲜跑,两臂延迟不对称,若 cand 延迟与对错相关,早到翻转 + 对系统性偏向 W 型 → e-值虚高假接受。前缀消费把判定顺序钉回预声明阶梯序, + anytime-valid 无条件成立;INFRA 单元视为"已解决(剔除)"不阻塞前缀。 + """ + while not run.frozen and run.prefix_ptr < len(run.slots): + slot = run.slots[run.prefix_ptr] + if not slot.resolved(): + return + rank = run.prefix_ptr + run.prefix_ptr += 1 + if slot.excluded(): + run.n_excluded += 1 + # 剔除使 n_remaining 缩小,必须重判(Codex plan 审 C1):否则尾部全 INFRA + # 时 verdict 停留在 "continue",绕过题尽第四出口且 _finalize_outcome + # 查 stop_reason 映射 KeyError。n_used==0(纯前导 INFRA)时无证据可判,跳过。 + if run.n_used > 0: + n_remaining = (len(run.slots) - run.n_excluded) - run.n_used + run.verdict = gate_decision(run.w, run.l, run.n_used, n_remaining, params=params) + if run.verdict.decision != "continue": + run.frozen = True + continue + uid = slot.unit.unit_id + assert slot.base is not None and slot.cand_per_q is not None + c_units = unit_correctness_view([slot.unit], slot.cand_per_q) + pair_result = pair_block({uid: slot.base}, c_units, [uid]) + run.candidate_per_q.update(slot.cand_per_q) + for u, (b, c) in pair_result.observed.items(): + run.base_obs[u] = b + run.cand_obs[u] = c + run.w += pair_result.w + run.l += pair_result.l + run.n_used += 1 + n_remaining = (len(run.slots) - run.n_excluded) - run.n_used + run.verdict = gate_decision(run.w, run.l, run.n_used, n_remaining, params=params) + run.evidence_rows.append( + { + "question_id": uid, + "task_type": run.spec.task_type, + "ladder_rank": rank, + "baseline_correct": slot.base, + "candidate_correct": c_units[uid], + "e_value": run.verdict.e_value, + "stop_reason": "", + } + ) + if run.verdict.decision != "continue": + run.frozen = True diff --git a/tests/unit/test_gate_prefix.py b/tests/unit/test_gate_prefix.py new file mode 100644 index 0000000..bf68ad6 --- /dev/null +++ b/tests/unit/test_gate_prefix.py @@ -0,0 +1,134 @@ +"""连续并发 gate 的前缀消费纯逻辑测试。""" + +from __future__ import annotations + +from app.harness.validate import GateSpec, _advance_prefix, _GateRun +from core.evolution import GateParams +from core.types import GeneratedQuestion, QuestionUnit + + +def _mk_question(qid: str, task_type: str = "Action Reasoning") -> GeneratedQuestion: + """构造最小可用的 single 题（字段以 core.types 真实定义为准，缺省值从简）。""" + return GeneratedQuestion( + question_id=qid, + video_id="v1", + task_type=task_type, + question=f"q-{qid}", + options=("A. x", "B. y"), + answer="A", + source_nodes=(), + difficulty="easy", + ) + + +def _mk_unit(qid: str, task_type: str = "Action Reasoning") -> QuestionUnit: + """由单条题目构造 single 单元（unit_id 回填为 question_id）。""" + return QuestionUnit.from_single(_mk_question(qid, task_type)) + + +def _mk_run(n_units: int) -> _GateRun: + """构造含 n_units 个 single 单元的初始 gate 运行时状态。""" + spec = GateSpec( + task_type="Action Reasoning", + target_file="action-reasoning.md", + candidate_content="cand", + base_skill_content="base", + units=[_mk_unit(f"q{i}") for i in range(n_units)], + gate_run_prefix="r_e1_s0_gate_action-reasoning", + ) + return _GateRun.from_spec(spec) + + +_PARAMS = GateParams( + e_confirm=20.0, + e_provisional=3.0, + w_net_min=2, + delta_min=0.02, + lambda_dir=-0.642, + e_rollback=10.0, +) + + +def test_prefix_blocks_on_unresolved_head() -> None: + """阶梯头部单元未配齐时,即使尾部全部配齐也一个都不消费。""" + run = _mk_run(4) + for i in (1, 2, 3): # 尾部三个先到 + run.slots[i].base = False + run.slots[i].cand_per_q = {f"q{i}": True} + _advance_prefix(run, _PARAMS) + assert run.n_used == 0 and run.w == 0 and run.verdict is None + + +def test_prefix_consumes_in_ladder_order_after_head_arrives() -> None: + """头部补齐后一次性顺序消费到最长已配齐前缀。""" + run = _mk_run(4) + for i in (0, 1, 2): + run.slots[i].base = False + run.slots[i].cand_per_q = {f"q{i}": True} + _advance_prefix(run, _PARAMS) + assert run.n_used == 3 and run.w == 3 and run.l == 0 + assert [r["ladder_rank"] for r in run.evidence_rows] == [0, 1, 2] + + +def test_freeze_on_terminal_verdict_stops_consumption() -> None: + """过线即冻结,后续已配齐单元不再消费。 + + 数值:W 连胜 L=0 时 E=(2^(W+1)-1)/(W+1),W=6→18.14<20,W=7→31.875≥20, + 故 7 连胜恰好 confirmed 过线(Codex 复核)。 + """ + run = _mk_run(12) + for i in range(12): + run.slots[i].base = False + run.slots[i].cand_per_q = {f"q{i}": True} + _advance_prefix(run, _PARAMS) + assert run.frozen and run.verdict is not None + assert run.verdict.decision == "accept_confirmed" + assert run.n_used == 7 # 第 7 个净胜恰好过线,早停不吃满 + + +def test_tail_infra_reaches_terminal_not_continue() -> None: + """尾部全 INFRA:剔除后须重判(n_remaining 归 0 → 题尽第四出口), + verdict 不得停留在 continue(Codex plan 审 C1 回归锁)。""" + run = _mk_run(4) + run.slots[0].base = False + run.slots[0].cand_per_q = {"q0": True} + run.slots[1].base = True + run.slots[1].cand_per_q = {"q1": True} + for i in (2, 3): + run.slots[i].base_infra = True + run.slots[i].cand_per_q = {f"q{i}": True} + _advance_prefix(run, _PARAMS) + assert run.verdict is not None and run.verdict.decision != "continue" + assert run.frozen + + +def test_infra_unit_skipped_not_counted() -> None: + """INFRA 单元(任一臂)剔除:不入 (W,L)、计入 n_excluded、前缀继续推进。 + + 注意 futility 出口在小 n_remaining 下很敏感,用 6 单元(首个 INFRA、其余 5 个 + W 翻转)保证消费全程不提前触发 futility:题尽走 accept_provisional 终态。 + """ + run = _mk_run(6) + run.slots[0].base_infra = True + run.slots[0].cand_per_q = {"q0": True} + for i in range(1, 6): + run.slots[i].base = False + run.slots[i].cand_per_q = {f"q{i}": True} + _advance_prefix(run, _PARAMS) + assert run.n_excluded == 1 and run.n_used == 5 and run.w == 5 and run.l == 0 + assert run.verdict is not None and run.verdict.decision == "accept_provisional" + + +def test_ties_hit_futility_early_and_freeze() -> None: + """全打平(无翻转对)时 futility 出口尽早触发并冻结——早停语义(数值:W=L=0 + 时乐观 E = E(n_remaining, 0),n 小易 Date: Fri, 17 Jul 2026 00:00:56 -0400 Subject: [PATCH 02/17] fix: lock all-INFRA contract, idempotency tests, tuple units (algo #6) --- app/harness/validate.py | 13 +++++++--- tests/unit/test_gate_prefix.py | 44 +++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/app/harness/validate.py b/app/harness/validate.py index 0e2adf7..4f35a43 100644 --- a/app/harness/validate.py +++ b/app/harness/validate.py @@ -809,7 +809,8 @@ class GateSpec: target_file: 解析后生效 skill 文件名(候选物化写此文件)。 candidate_content: 候选 skill 全文。 base_skill_content: 基线侧生效 skill 全文(skill_hash 作缓存键)。 - units: 阶梯序单元列表(已排除案例单元、截断 gate_n_max)。 + units: 阶梯序单元元组(已排除案例单元、截断 gate_n_max);元组,装配后 + 不可变,防 spec.units 与 run.slots 漂移。 gate_run_prefix: run_id 前缀,必须含 "_gate_"(防泄露过滤依赖)。 """ @@ -817,7 +818,7 @@ class GateSpec: target_file: str candidate_content: str base_skill_content: str - units: list[QuestionUnit] + units: tuple[QuestionUnit, ...] gate_run_prefix: str @@ -884,6 +885,10 @@ def _advance_prefix(run: _GateRun, params: GateParams) -> None: 瞬间返回、cand 臂必新鲜跑,两臂延迟不对称,若 cand 延迟与对错相关,早到翻转 对系统性偏向 W 型 → e-值虚高假接受。前缀消费把判定顺序钉回预声明阶梯序, anytime-valid 无条件成立;INFRA 单元视为"已解决(剔除)"不阻塞前缀。 + + 契约:全部单元被剔除时 verdict 保持 None、frozen 保持 False,由调度编排层 + (Task 3 的 validate_skills_concurrent)检测 verdict None 并 raise + RuntimeError;本函数不负责该终态。 """ while not run.frozen and run.prefix_ptr < len(run.slots): slot = run.slots[run.prefix_ptr] @@ -903,7 +908,9 @@ def _advance_prefix(run: _GateRun, params: GateParams) -> None: run.frozen = True continue uid = slot.unit.unit_id - assert slot.base is not None and slot.cand_per_q is not None + assert slot.base is not None and slot.cand_per_q is not None, ( + f"slot 未配齐即被消费: unit={slot.unit.unit_id}" + ) c_units = unit_correctness_view([slot.unit], slot.cand_per_q) pair_result = pair_block({uid: slot.base}, c_units, [uid]) run.candidate_per_q.update(slot.cand_per_q) diff --git a/tests/unit/test_gate_prefix.py b/tests/unit/test_gate_prefix.py index bf68ad6..8b6749b 100644 --- a/tests/unit/test_gate_prefix.py +++ b/tests/unit/test_gate_prefix.py @@ -33,7 +33,7 @@ def _mk_run(n_units: int) -> _GateRun: target_file="action-reasoning.md", candidate_content="cand", base_skill_content="base", - units=[_mk_unit(f"q{i}") for i in range(n_units)], + units=tuple(_mk_unit(f"q{i}") for i in range(n_units)), gate_run_prefix="r_e1_s0_gate_action-reasoning", ) return _GateRun.from_spec(spec) @@ -119,6 +119,48 @@ def test_infra_unit_skipped_not_counted() -> None: assert run.verdict is not None and run.verdict.decision == "accept_provisional" +def test_all_infra_leaves_verdict_none() -> None: + """全部单元被剔除:verdict 保持 None、frozen 保持 False——终态由调度编排层 + (Task 3 的 validate_skills_concurrent)检测 verdict None 并 raise,本函数不管。""" + run = _mk_run(3) + for i in range(3): + run.slots[i].base_infra = True + run.slots[i].cand_per_q = {f"q{i}": True} + _advance_prefix(run, _PARAMS) + assert run.verdict is None and not run.frozen and run.n_excluded == 3 and run.prefix_ptr == 3 + + +def test_repeated_calls_are_idempotent() -> None: + """部分前缀消费后重复调用不改变状态;补齐剩余单元后再调用正常推进。""" + run = _mk_run(4) + for i in (0, 1): + run.slots[i].base = False + run.slots[i].cand_per_q = {f"q{i}": True} + _advance_prefix(run, _PARAMS) + snapshot = (run.n_used, run.w, run.l, run.prefix_ptr, len(run.evidence_rows)) + _advance_prefix(run, _PARAMS) + _advance_prefix(run, _PARAMS) + assert (run.n_used, run.w, run.l, run.prefix_ptr, len(run.evidence_rows)) == snapshot + for i in (2, 3): + run.slots[i].base = False + run.slots[i].cand_per_q = {f"q{i}": True} + _advance_prefix(run, _PARAMS) + assert run.n_used > snapshot[0] and run.prefix_ptr > snapshot[3] + + +def test_frozen_run_call_is_noop() -> None: + """frozen 后再调用是 no-op:不再消费已配齐单元、证据不再追加。""" + run = _mk_run(12) + for i in range(12): + run.slots[i].base = False + run.slots[i].cand_per_q = {f"q{i}": True} + _advance_prefix(run, _PARAMS) + assert run.frozen and run.n_used == 7 + n_evidence = len(run.evidence_rows) + _advance_prefix(run, _PARAMS) + assert run.n_used == 7 and len(run.evidence_rows) == n_evidence + + def test_ties_hit_futility_early_and_freeze() -> None: """全打平(无翻转对)时 futility 出口尽早触发并冻结——早停语义(数值:W=L=0 时乐观 E = E(n_remaining, 0),n 小易 Date: Fri, 17 Jul 2026 00:03:17 -0400 Subject: [PATCH 03/17] test: strengthen frozen-noop full-state assertion (algo #6) --- tests/unit/test_gate_prefix.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_gate_prefix.py b/tests/unit/test_gate_prefix.py index 8b6749b..ac72a07 100644 --- a/tests/unit/test_gate_prefix.py +++ b/tests/unit/test_gate_prefix.py @@ -156,9 +156,26 @@ def test_frozen_run_call_is_noop() -> None: run.slots[i].cand_per_q = {f"q{i}": True} _advance_prefix(run, _PARAMS) assert run.frozen and run.n_used == 7 - n_evidence = len(run.evidence_rows) + before = ( + run.w, + run.l, + run.n_used, + run.prefix_ptr, + len(run.evidence_rows), + run.verdict, + run.frozen, + ) _advance_prefix(run, _PARAMS) - assert run.n_used == 7 and len(run.evidence_rows) == n_evidence + after = ( + run.w, + run.l, + run.n_used, + run.prefix_ptr, + len(run.evidence_rows), + run.verdict, + run.frozen, + ) + assert after == before def test_ties_hit_futility_early_and_freeze() -> None: From 0a8e1ad18b950e3e5c2c0d3156f87b34a8e9587f Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 17 Jul 2026 00:06:35 -0400 Subject: [PATCH 04/17] docs: complete four-part docstrings for gate prefix section (algo #6) --- app/harness/validate.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/app/harness/validate.py b/app/harness/validate.py index 4f35a43..d3d016d 100644 --- a/app/harness/validate.py +++ b/app/harness/validate.py @@ -837,13 +837,21 @@ class _UnitSlot: cand_infra: bool = False def resolved(self) -> bool: - """双臂均已出结果(含 INFRA 判定)。""" + """双臂均已出结果(含 INFRA 判定)。 + + 返回: + base 臂(结果或 INFRA)与 cand 臂(结果或 INFRA)都已到达时为 True。 + """ base_done = self.base is not None or self.base_infra cand_done = self.cand_per_q is not None or self.cand_infra return base_done and cand_done def excluded(self) -> bool: - """任一臂 INFRA 即整单元剔除(不入配对)。""" + """任一臂 INFRA 即整单元剔除(不入配对)。 + + 返回: + base_infra 或 cand_infra 任一为 True 时为 True。 + """ return self.base_infra or self.cand_infra @@ -870,7 +878,14 @@ class _GateRun: @classmethod def from_spec(cls, spec: GateSpec) -> _GateRun: - """由规格构造初始状态(slots 与阶梯序一一对应)。""" + """由规格构造初始状态(slots 与阶梯序一一对应)。 + + 参数: + spec: 单题型 gate 规格(units 已阶梯序)。 + + 返回: + 计数器归零、slots 逐单元初始化、s_hash 已计算的 _GateRun。 + """ return cls( spec=spec, slots=[_UnitSlot(unit=u) for u in spec.units], @@ -889,6 +904,13 @@ def _advance_prefix(run: _GateRun, params: GateParams) -> None: 契约:全部单元被剔除时 verdict 保持 None、frozen 保持 False,由调度编排层 (Task 3 的 validate_skills_concurrent)检测 verdict None 并 raise RuntimeError;本函数不负责该终态。 + + 参数: + run: 单题型 gate 运行时状态(原地更新计数器/指针/证据)。 + params: e-process 判据阈值组。 + + 返回: + 无(所有效果原地写入 run;可重复调用,已消费前缀不重复消费)。 """ while not run.frozen and run.prefix_ptr < len(run.slots): slot = run.slots[run.prefix_ptr] From 232afd525b7860dcd5bbb7d711e35f61f251a39c Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 17 Jul 2026 00:13:51 -0400 Subject: [PATCH 05/17] feat: gate unit-arm tasks with question-slot gate (algo #6) --- app/harness/validate.py | 187 +++++++++++++++++++++++++++++ tests/unit/test_gate_unit_arm.py | 198 +++++++++++++++++++++++++++++++ 2 files changed, 385 insertions(+) create mode 100644 tests/unit/test_gate_unit_arm.py diff --git a/app/harness/validate.py b/app/harness/validate.py index d3d016d..0235b13 100644 --- a/app/harness/validate.py +++ b/app/harness/validate.py @@ -15,6 +15,7 @@ from __future__ import annotations +import asyncio import json import shutil import tempfile @@ -957,3 +958,189 @@ def _advance_prefix(run: _GateRun, params: GateParams) -> None: ) if run.verdict.decision != "continue": run.frozen = True + + +class _QuestionSlots: + """按题数计数的共享并发闸:峰值在飞请求恒 ≤ width(设计 v3 §2.4)。 + + 多槽获取(AR pair 一单元两题)经内部锁串行化,防多任务半持有交错死锁。 + asyncio.Semaphore 等待队列 FIFO,任务按创建序(题型 round-robin)获得槽, + 即公平调度的实现载体(Codex I2)。 + """ + + def __init__(self, width: int) -> None: + """初始化题槽闸。 + + 参数: + width: 并发宽度(全 gate 同时在飞的题数上限),必须为正。 + """ + assert width > 0, f"并发宽度必须为正: {width}" + self._width = width + self._sem = asyncio.Semaphore(width) + self._acquire_lock = asyncio.Lock() + + async def acquire(self, n: int) -> None: + """原子获取 n 个题槽。 + + fail-fast:n > 宽度时任务持锁等待永不满足的槽位 → 自死锁 + (AR pair 单元 2 题 + width=1 的病态配置,Codex plan 审 C2),直接报错。 + + 参数: + n: 申请的题槽数(单元内题目数,single=1 / AR pair=2)。 + + 返回: + 无(成功返回即持有 n 个槽,须与 release(n) 配对)。 + + 异常: + ValueError: n 超过并发宽度(否则自死锁)。 + """ + if n > self._width: + raise ValueError(f"单次申请题槽 {n} 超过并发宽度 {self._width},将自死锁") + async with self._acquire_lock: + for _ in range(n): + await self._sem.acquire() + + def release(self, n: int) -> None: + """归还 n 个题槽。 + + 参数: + n: 与 acquire 对应的题槽数。 + + 返回: + 无。 + """ + for _ in range(n): + self._sem.release() + + +async def _run_unit_arm( + run: _GateRun, + slot_idx: int, + arm: str, + slots: _QuestionSlots, + run_inference: RunInferenceFn, + log: HarnessLog, + baseline_cache: BaselineCache, + prompts_version: str, + base_skills_dir: Path, + cand_dir: Path, + gate_params: GateParams, + gate_guard_err: float, +) -> None: + """执行一个 (单元, 臂) 任务:缓存/推理 → 到达登记 → 前缀消费推进。 + + 冻结检查两次:启动时(排队任务撤销点)与获得题槽后(获槽期间被冻结)。 + base 臂缓存命中不占题槽(零推理);INFRA 单元不写缓存(不永久污染基线快照)。 + 护栏在每次臂完成时检查(等价迁移自跨块累计,设计 v3 §2.3),超阈值 raise + 中止整轮(与现行行为一致)。 + + 参数: + run: 该题型的 gate 运行时状态。 + slot_idx: 单元在阶梯中的下标。 + arm: "base" 或 "cand"。 + slots: 全 gate 共享题槽闸。 + run_inference: 注入推理函数。 + log: HarnessLog 共享实例(推理后读预测)。 + baseline_cache / prompts_version: 基线缓存及键成分。 + base_skills_dir / cand_dir: 两臂各自的 skills 目录。 + gate_params: e-process 判据(前缀消费用)。 + gate_guard_err: INFRA 错误率护栏阈值。 + + 返回: + 无(结果写入 run.slots[slot_idx] 并触发 _advance_prefix)。 + + 异常: + RuntimeError: 累计 INFRA 错误率超护栏阈值(经 _check_infra_guard)。 + """ + assert arm in ("base", "cand") + if run.frozen: + return + slot = run.slots[slot_idx] + spec = run.spec + + if arm == "base": + cached = baseline_cache.get(spec.task_type, run.s_hash, prompts_version, slot.unit.unit_id) + if cached is not None: + slot.base = cached + _advance_prefix(run, gate_params) + return + + questions = list(slot.unit.questions) + await slots.acquire(len(questions)) + try: + if run.frozen: + return + run_id = f"{spec.gate_run_prefix}_{arm}" + skills_dir = base_skills_dir if arm == "base" else cand_dir + r = await run_inference(questions, run_id=run_id, skills_dir=skills_dir) + _register_arm_arrival( + run=run, + slot=slot, + arm=arm, + questions=questions, + inference_run_id=r.run_id, + inference_total=r.total, + log=log, + baseline_cache=baseline_cache, + prompts_version=prompts_version, + ) + _check_infra_guard(run.errors, run.infra_denom, gate_guard_err) + finally: + slots.release(len(questions)) + _advance_prefix(run, gate_params) + + +def _register_arm_arrival( + run: _GateRun, + slot: _UnitSlot, + arm: str, + questions: list[GeneratedQuestion], + inference_run_id: str, + inference_total: int, + log: HarnessLog, + baseline_cache: BaselineCache, + prompts_version: str, +) -> None: + """把一次臂推理结果登记进 slot 与 run 计数器(INFRA 判定 + 对错折叠 + 回写缓存)。 + + INFRA 臂只标记不写缓存(不永久污染基线快照);正常 base 臂折叠为单元级对错并 + 回写 BaselineCache,正常 cand 臂保留逐题对错(折叠交给前缀消费,保留逐题溯源)。 + + 参数: + run: 该题型的 gate 运行时状态(errors / infra_denom 原地累加)。 + slot: 本单元的双臂到达状态(结果或 INFRA 标志原地写入)。 + arm: "base" 或 "cand"。 + questions: 本单元展开后的题目列表。 + inference_run_id: 本次推理的 run_id(DB 回读键)。 + inference_total: 本次推理的题次数(护栏分母增量)。 + log: HarnessLog 共享实例(推理后读预测)。 + baseline_cache / prompts_version: 基线缓存及键成分。 + + 返回: + 无(所有效果原地写入 run 与 slot)。 + + 关键实现细节: + errors 按单元级去重(Codex plan 审 I3):同一单元双臂都 INFRA 只计 1 个 + error,与设计 §2.3"分子=INFRA 单元数(任一臂)"及旧块实现口径一致 + (旧实现 cand 不跑 base-INFRA 单元,天然无双计)。 + """ + spec = run.spec + infra_qids = _infra_question_ids_from_db(log, inference_run_id, questions) + run.infra_denom += inference_total + if infra_qids: + if not slot.excluded(): + run.errors += 1 + if arm == "base": + slot.base_infra = True + else: + slot.cand_infra = True + return + per_q = _candidate_correctness_from_db(log, inference_run_id, questions) + if arm == "base": + folded = unit_correctness_view([slot.unit], per_q) + slot.base = folded[slot.unit.unit_id] + baseline_cache.put( + spec.task_type, run.s_hash, prompts_version, slot.unit.unit_id, slot.base + ) + else: + slot.cand_per_q = per_q diff --git a/tests/unit/test_gate_unit_arm.py b/tests/unit/test_gate_unit_arm.py new file mode 100644 index 0000000..ddb9b8b --- /dev/null +++ b/tests/unit/test_gate_unit_arm.py @@ -0,0 +1,198 @@ +"""单元臂执行任务测试:缓存命中/新鲜跑/INFRA/冻结跳过/题槽并发上限。""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest + +from app.harness.gate_ladder import BaselineCache +from app.harness.validate import ( + GateSpec, + _GateRun, + _QuestionSlots, + _run_unit_arm, +) +from tests.unit.test_gate_prefix import _PARAMS, _mk_unit # 复用 fixture + + +class _FakeLog: + """假 HarnessLog:query 返回预置 predictions 行。""" + + def __init__(self) -> None: + self.rows: list[dict] = [] + + def query(self, sql: str, params: tuple = ()) -> list[dict]: + """按 run_id（params[0]）过滤预置行,模拟只读 SELECT。""" + run_id = params[0] + return [r for r in self.rows if r["run_id"] == run_id] + + +def _mk_gate_run(n: int, tmp_path: Path) -> tuple[_GateRun, BaselineCache]: + """构造 n 个 single 单元的 gate 运行时状态与空基线缓存。""" + spec = GateSpec( + task_type="Action Reasoning", + target_file="action-reasoning.md", + candidate_content="cand", + base_skill_content="base", + units=tuple(_mk_unit(f"q{i}") for i in range(n)), + gate_run_prefix="r_e1_s0_gate_action-reasoning", + ) + return _GateRun.from_spec(spec), BaselineCache(tmp_path / "bc.json") + + +def _fake_run_inference(log: _FakeLog, correct: bool, stop_reason: str = "finished"): + """构造假推理:把每题结果写进 _FakeLog 并返回带 total 的结果对象。""" + + class _R: + def __init__(self, run_id: str, total: int) -> None: + self.run_id = run_id + self.total = total + + async def _run(questions, *, run_id: str, skills_dir: Path): + for q in questions: + log.rows.append( + { + "run_id": run_id, + "question_id": q.question_id, + "prediction": "A" if correct else "B", + "answer": "A", + "stop_reason": stop_reason, + "steps_json": "[]", + } + ) + return _R(run_id, len(questions)) + + return _run + + +@pytest.mark.asyncio +async def test_base_arm_cache_hit_skips_inference(tmp_path) -> None: + """base 臂缓存命中:不调推理,slot.base 直接就位,infra_denom 不增。""" + run, cache = _mk_gate_run(1, tmp_path) + cache.put("Action Reasoning", run.s_hash, "v1", "q0", True) + called = {"n": 0} + + async def _boom(questions, *, run_id, skills_dir): + called["n"] += 1 + raise AssertionError("缓存命中不应触发推理") + + slots = _QuestionSlots(4) + await _run_unit_arm( + run, + 0, + "base", + slots, + _boom, + _FakeLog(), + cache, + "v1", + Path("/nonexistent"), + Path("/nonexistent"), + _PARAMS, + 0.10, + ) + assert called["n"] == 0 and run.slots[0].base is True and run.infra_denom == 0 + + +@pytest.mark.asyncio +async def test_base_arm_fresh_run_writes_cache(tmp_path) -> None: + """base 臂 miss 新鲜跑:结果折叠入 slot 并回写缓存。""" + run, cache = _mk_gate_run(1, tmp_path) + log = _FakeLog() + slots = _QuestionSlots(4) + await _run_unit_arm( + run, + 0, + "base", + slots, + _fake_run_inference(log, correct=True), + log, + cache, + "v1", + tmp_path, + tmp_path, + _PARAMS, + 0.10, + ) + assert run.slots[0].base is True + assert cache.get("Action Reasoning", run.s_hash, "v1", "q0") is True + assert run.infra_denom == 1 + + +@pytest.mark.asyncio +async def test_infra_arm_marks_excluded_and_no_cache(tmp_path) -> None: + """INFRA 臂:标记 infra、errors+1、不写缓存。""" + run, cache = _mk_gate_run(1, tmp_path) + log = _FakeLog() + slots = _QuestionSlots(4) + await _run_unit_arm( + run, + 0, + "base", + slots, + _fake_run_inference(log, correct=False, stop_reason="error"), + log, + cache, + "v1", + tmp_path, + tmp_path, + _PARAMS, + 0.10, + ) + assert run.slots[0].base_infra and run.errors == 1 + assert cache.get("Action Reasoning", run.s_hash, "v1", "q0") is None + + +@pytest.mark.asyncio +async def test_frozen_run_skips_launch(tmp_path) -> None: + """已冻结题型的排队臂:直接返回,不占槽不推理。""" + run, cache = _mk_gate_run(1, tmp_path) + run.frozen = True + called = {"n": 0} + + async def _boom(questions, *, run_id, skills_dir): + called["n"] += 1 + + await _run_unit_arm( + run, + 0, + "cand", + _QuestionSlots(4), + _boom, + _FakeLog(), + cache, + "v1", + tmp_path, + tmp_path, + _PARAMS, + 0.10, + ) + assert called["n"] == 0 + + +@pytest.mark.asyncio +async def test_question_slots_caps_inflight() -> None: + """题槽闸:峰值在飞数严格 ≤ 宽度(多槽获取不交错死锁)。""" + slots = _QuestionSlots(2) + peak = {"cur": 0, "max": 0} + + async def _job(n: int) -> None: + await slots.acquire(n) + peak["cur"] += n + peak["max"] = max(peak["max"], peak["cur"]) + await asyncio.sleep(0.01) + peak["cur"] -= n + slots.release(n) + + await asyncio.gather(*[_job(1) for _ in range(6)], *[_job(2) for _ in range(3)]) + assert peak["max"] <= 2 + + +@pytest.mark.asyncio +async def test_question_slots_rejects_oversized_request() -> None: + """申请槽数超宽度:fail-fast ValueError 而非自死锁(Codex C2 回归锁)。""" + slots = _QuestionSlots(1) + with pytest.raises(ValueError, match="自死锁"): + await slots.acquire(2) From 30c1cf10c0bd59f083053e09dc7be4238a0a304d Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 17 Jul 2026 00:25:18 -0400 Subject: [PATCH 06/17] fix: gate slot cancel-safety + post-inference freeze discard (algo #6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex 质量审 4 项:推理后二次冻结检查(τ 后 in-flight 结果整体丢弃)、 acquire 取消回滚(半持有 permit 自动归还)、BoundedSemaphore 防静默扩容、 补取消恢复与冻结丢弃两个回归测试。 --- app/harness/validate.py | 25 +++++++++++--- tests/unit/test_gate_unit_arm.py | 59 ++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/app/harness/validate.py b/app/harness/validate.py index 0235b13..867d7ec 100644 --- a/app/harness/validate.py +++ b/app/harness/validate.py @@ -976,7 +976,8 @@ class _QuestionSlots: """ assert width > 0, f"并发宽度必须为正: {width}" self._width = width - self._sem = asyncio.Semaphore(width) + # BoundedSemaphore:多还立即 ValueError 而非静默扩容(Codex 质量审 3) + self._sem = asyncio.BoundedSemaphore(width) self._acquire_lock = asyncio.Lock() async def acquire(self, n: int) -> None: @@ -984,6 +985,8 @@ class _QuestionSlots: fail-fast:n > 宽度时任务持锁等待永不满足的槽位 → 自死锁 (AR pair 单元 2 题 + width=1 的病态配置,Codex plan 审 C2),直接报错。 + 取消安全:半持有自动回滚——逐槽获取途中被取消(或任何 BaseException) + 时,已拿到的 permit 全部归还再重抛,容量不泄漏(Codex 质量审 2)。 参数: n: 申请的题槽数(单元内题目数,single=1 / AR pair=2)。 @@ -997,8 +1000,15 @@ class _QuestionSlots: if n > self._width: raise ValueError(f"单次申请题槽 {n} 超过并发宽度 {self._width},将自死锁") async with self._acquire_lock: - for _ in range(n): - await self._sem.acquire() + got = 0 + try: + for _ in range(n): + await self._sem.acquire() + got += 1 + except BaseException: + for _ in range(got): + self._sem.release() + raise def release(self, n: int) -> None: """归还 n 个题槽。 @@ -1029,7 +1039,8 @@ async def _run_unit_arm( ) -> None: """执行一个 (单元, 臂) 任务:缓存/推理 → 到达登记 → 前缀消费推进。 - 冻结检查两次:启动时(排队任务撤销点)与获得题槽后(获槽期间被冻结)。 + 冻结检查三次:启动时(排队任务撤销点)、获得题槽后(获槽期间被冻结)、 + 推理返回后(τ 之后的 in-flight 结果不计入,整体丢弃)。 base 臂缓存命中不占题槽(零推理);INFRA 单元不写缓存(不永久污染基线快照)。 护栏在每次臂完成时检查(等价迁移自跨块累计,设计 v3 §2.3),超阈值 raise 中止整轮(与现行行为一致)。 @@ -1073,6 +1084,12 @@ async def _run_unit_arm( run_id = f"{spec.gate_run_prefix}_{arm}" skills_dir = base_skills_dir if arm == "base" else cand_dir r = await run_inference(questions, run_id=run_id, skills_dir=skills_dir) + # 推理 await 期间该题型可能已被其他任务判定冻结:设计语义是 + # "τ(冻结时刻)之后的 in-flight 结果不计入"——整体丢弃,不写 + # slot/infra_denom/errors,滞后 INFRA 也不得触发护栏 raise 掀翻 + # 整轮 gather(Codex 质量审 1)。 + if run.frozen: + return _register_arm_arrival( run=run, slot=slot, diff --git a/tests/unit/test_gate_unit_arm.py b/tests/unit/test_gate_unit_arm.py index ddb9b8b..89a789e 100644 --- a/tests/unit/test_gate_unit_arm.py +++ b/tests/unit/test_gate_unit_arm.py @@ -196,3 +196,62 @@ async def test_question_slots_rejects_oversized_request() -> None: slots = _QuestionSlots(1) with pytest.raises(ValueError, match="自死锁"): await slots.acquire(2) + + +@pytest.mark.asyncio +async def test_acquire_cancellation_restores_capacity() -> None: + """acquire 半持有时被取消:已拿 permit 自动回滚,容量完全恢复(Codex 质量审 2)。""" + slots = _QuestionSlots(2) + await slots.acquire(1) # 预占 1 槽,使 acquire(2) 卡在第二槽 + task = asyncio.ensure_future(slots.acquire(2)) + for _ in range(5): + await asyncio.sleep(0) # 让 task 拿到第 1 个 permit 并阻塞在第 2 个 + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + slots.release(1) # 归还预占 + # 半持有的 permit 若泄漏,此处 acquire(2) 将永久阻塞 → wait_for 超时暴露泄漏 + await asyncio.wait_for(slots.acquire(2), timeout=1.0) + slots.release(2) + + +@pytest.mark.asyncio +async def test_frozen_during_inference_discards_result(tmp_path) -> None: + """推理 await 期间被冻结:in-flight 结果整体丢弃(不写 slot/计数器/缓存)。 + + 设计语义:τ(冻结时刻)之后到达的结果不计入,滞后 INFRA 也不得触发护栏。 + """ + run, cache = _mk_gate_run(1, tmp_path) + log = _FakeLog() + gate_open = asyncio.Event() + + async def _slow_run(questions, *, run_id: str, skills_dir: Path): + await gate_open.wait() + return await _fake_run_inference(log, correct=True)( + questions, run_id=run_id, skills_dir=skills_dir + ) + + task = asyncio.ensure_future( + _run_unit_arm( + run, + 0, + "base", + _QuestionSlots(4), + _slow_run, + log, + cache, + "v1", + tmp_path, + tmp_path, + _PARAMS, + 0.10, + ) + ) + for _ in range(5): + await asyncio.sleep(0) # 让 task 进入推理等待 + run.frozen = True + gate_open.set() + await task + assert run.slots[0].base is None + assert run.infra_denom == 0 and run.errors == 0 + assert cache.get("Action Reasoning", run.s_hash, "v1", "q0") is None From c61a6dac845b3dd21f670a5daf6a10300b268db8 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 17 Jul 2026 00:28:25 -0400 Subject: [PATCH 07/17] docs: soften slot fairness claim, complete docstrings (algo #6) --- app/harness/validate.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/app/harness/validate.py b/app/harness/validate.py index 867d7ec..54810ad 100644 --- a/app/harness/validate.py +++ b/app/harness/validate.py @@ -964,8 +964,8 @@ class _QuestionSlots: """按题数计数的共享并发闸:峰值在飞请求恒 ≤ width(设计 v3 §2.4)。 多槽获取(AR pair 一单元两题)经内部锁串行化,防多任务半持有交错死锁。 - asyncio.Semaphore 等待队列 FIFO,任务按创建序(题型 round-robin)获得槽, - 即公平调度的实现载体(Codex I2)。 + 本类只承诺"并发上限 + 多槽获取原子性";公平性由调用方按题型 round-robin + 顺序创建任务实现(实践中 asyncio 等待队列近似先来先服务,但那不是本类契约)。 """ def __init__(self, width: int) -> None: @@ -973,6 +973,13 @@ class _QuestionSlots: 参数: width: 并发宽度(全 gate 同时在飞的题数上限),必须为正。 + + 返回: + 无。 + + 关键实现细节: + _width 供 acquire 做超宽 fail-fast;BoundedSemaphore 使多还立即 + ValueError 而非静默扩容;_acquire_lock 串行化多槽获取防交错死锁。 """ assert width > 0, f"并发宽度必须为正: {width}" self._width = width @@ -1018,6 +1025,10 @@ class _QuestionSlots: 返回: 无。 + + 关键实现细节: + 底层为 BoundedSemaphore——多还(release 数超过 acquire)立即 + ValueError 暴露调用方配对错误,属防御性设计。 """ for _ in range(n): self._sem.release() From e8b66f85ab962710261b9993a4fe0175a03181dd Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 17 Jul 2026 00:36:01 -0400 Subject: [PATCH 08/17] feat: continuous concurrent gate orchestrator (algo #6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - validate_skills_concurrent: 多题型全部臂共享题槽并发编排,发射序 = 题型 round-robin × 阶梯序(base 先 cand 后),终态统一组装 outcome, verdict None(全 INFRA)保留 RuntimeError 语义 - gate_evidence 列 block_idx → ladder_rank(阶梯序号,0-based);旧块路径 _build_evidence_rows 仅键名同步(值仍为块号)保持落库兼容 - 新增 3 项编排测试:乱序到达前缀有序性/双题型隔离/全 INFRA raise Co-Authored-By: Claude Fable 5 --- app/harness/observation.py | 7 +- app/harness/validate.py | 152 +++++++++++++++++++++- tests/unit/test_gate_concurrent.py | 167 +++++++++++++++++++++++++ tests/unit/test_harness_observation.py | 4 +- 4 files changed, 324 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_gate_concurrent.py diff --git a/app/harness/observation.py b/app/harness/observation.py index 7b1b573..b1b9d83 100644 --- a/app/harness/observation.py +++ b/app/harness/observation.py @@ -103,7 +103,7 @@ _GATE_EVIDENCE_COLS: dict[str, str] = { # question_id 列承载 unit_id（single=question_id，pair=pair_id）； # 逐题明细在 predictions 表溯源，按 pair_id join 真实 question 表会 join 不上。 "question_id": "TEXT", - "block_idx": "INTEGER", + "ladder_rank": "INTEGER", "baseline_correct": "INTEGER", "candidate_correct": "INTEGER", "e_value": "REAL", @@ -341,8 +341,9 @@ def write_gate_evidence( run_id: 训练 run ID。 epoch: 该 gate 所属的轮次（1-based）。 step: epoch 内 step 序号（0-based）。 - rows: 每 **单元** 一行，含 question_id/task_type/block_idx/baseline_correct/ - candidate_correct/e_value（该单元所在块判定后的累计 e 值）/ + rows: 每 **单元** 一行，含 question_id/task_type/ladder_rank（阶梯序号， + 0-based）/baseline_correct/ + candidate_correct/e_value（该单元判定后的累计 e 值）/ stop_reason（仅最后一单元携带最终 stop_reason，其余空串）。 question_id 字段承载 **unit_id**（single=question_id，pair=pair_id）—— 逐题明细在 predictions 表溯源，按 pair_id join 真实 question 表会 join 不上。 diff --git a/app/harness/validate.py b/app/harness/validate.py index 54810ad..bd0c1d1 100644 --- a/app/harness/validate.py +++ b/app/harness/validate.py @@ -435,7 +435,9 @@ def _build_evidence_rows( { "question_id": u.unit_id, "task_type": task_type, - "block_idx": block_idx, + # 落库列已更名 ladder_rank(阶梯序号);旧块路径此处值仍为块号, + # 仅键名对齐 gate_evidence 表结构以保持落库兼容。 + "ladder_rank": block_idx, "baseline_correct": b_units[u.unit_id], "candidate_correct": c_units[u.unit_id], "e_value": None, @@ -1172,3 +1174,151 @@ def _register_arm_arrival( ) else: slot.cand_per_q = per_q + + +def _validate_gate_specs(specs: list[GateSpec]) -> None: + """校验各题型 gate 规格,不合法直接报错(不兜底)。 + + 参数: + specs: 各题型 gate 规格。 + + 异常: + ValueError: 阶梯为空,或 gate_run_prefix 缺 "_gate_"(防泄露过滤依赖 + 该标记识别 gate run)。 + """ + for spec in specs: + if "_gate_" not in spec.gate_run_prefix: + raise ValueError(f"gate_run_prefix 必须含 '_gate_': {spec.gate_run_prefix!r}") + if not spec.units: + raise ValueError(f"task_type={spec.task_type} 阶梯为空,无法验证") + + +def _cleanup_candidate_dirs(cand_dirs: dict[str, Path]) -> None: + """尽力清理全部候选临时目录,单个失败只记 warning 不中断其余清理。 + + 参数: + cand_dirs: task_type -> 候选临时目录路径。 + + 返回: + 无。 + """ + for d in cand_dirs.values(): + try: + shutil.rmtree(d) + except OSError as e: + logger.warning("候选临时目录清理失败 {}: {}", d, e) + + +def _build_launch_order(runs: list[_GateRun]) -> list[tuple[_GateRun, int, str]]: + """构建 (run, rank, arm) 发射队列:题型 round-robin × 题型内阶梯序。 + + 交错顺序 = rank 0 各题型 → rank 1 各题型 → ...;同一 (题型, rank) 内 + base 先 cand 后。round-robin 让各题型的阶梯头部同批起跑,配合前缀消费 + 使统计推进不因某题型阶梯过长而饿死其他题型。 + + 参数: + runs: 各题型 gate 运行时状态(slots 已按阶梯序初始化)。 + + 返回: + (run, rank, arm) 三元组列表,即任务创建顺序。 + """ + order: list[tuple[_GateRun, int, str]] = [] + max_rank = max((len(r.slots) for r in runs), default=0) + for rank in range(max_rank): + for r in runs: + if rank < len(r.slots): + for arm in ("base", "cand"): + order.append((r, rank, arm)) + return order + + +async def validate_skills_concurrent( + workspace_dir: Path, + base_skills_version: str, + specs: list[GateSpec], + gate_params: GateParams, + gate_guard_err: float, + baseline_cache: BaselineCache, + prompts_version: str, + run_inference: RunInferenceFn, + log: HarnessLog, + concurrency: int, +) -> dict[str, ValidationOutcome]: + """连续并发 gate:多题型全部臂共享题槽并发,统计按阶梯序前缀有序推进。 + + 发射顺序 = 题型 round-robin × 题型内阶梯序(base 先 cand 后);题型过线即 + 冻结,其排队任务启动时自查冻结标志撤销,in-flight 结果不计入(τ 之后样本, + 合法丢弃)。全部题型判定后统一组装 ValidationOutcome。 + + 参数: + workspace_dir: workspace 根目录(候选物化用)。 + base_skills_version: 基线 skills 版本名。 + specs: 各题型 gate 规格(units 已阶梯序 + 截断 n_max)。 + gate_params: e-process 判据阈值组。 + gate_guard_err: INFRA 错误率护栏阈值。 + baseline_cache: 基线侧单元级对错缓存。 + prompts_version: 当前 prompts 版本(缓存键成分)。 + run_inference: 注入推理函数(调用方须绑定共享 HarnessLog)。 + log: HarnessLog 共享实例(推理后读预测,与 run_inference 同库)。 + concurrency: 题槽宽度(峰值在飞题数上限)。 + + 返回: + {task_type: ValidationOutcome}。 + + 异常: + RuntimeError: INFRA 护栏超阈值,或某题型全部单元被 INFRA 排除。 + ValueError: spec 校验失败(空阶梯 / run_prefix 缺 "_gate_")。 + """ + _validate_gate_specs(specs) + base_skills_dir = workspace_dir / "skills" / base_skills_version + runs = [_GateRun.from_spec(s) for s in specs] + cand_dirs = { + r.spec.task_type: materialize_candidate_skill( + workspace_dir, base_skills_version, r.spec.target_file, r.spec.candidate_content + ) + for r in runs + } + slots_gate = _QuestionSlots(concurrency) + try: + coros = [ + _run_unit_arm( + r, + rank, + arm, + slots_gate, + run_inference, + log, + baseline_cache, + prompts_version, + base_skills_dir, + cand_dirs[r.spec.task_type], + gate_params, + gate_guard_err, + ) + for r, rank, arm in _build_launch_order(runs) + ] + # gather 任一任务 raise(INFRA 护栏)即向上传播中止整轮,与现行"护栏 + # 中止训练"语义一致;finally 仍清理候选目录。 + await asyncio.gather(*coros) + finally: + _cleanup_candidate_dirs(cand_dirs) + + outcomes: dict[str, ValidationOutcome] = {} + for r in runs: + if r.verdict is None: + raise RuntimeError( + f"gate[{r.spec.task_type}] 全部 unit 被判为 INFRA 排除,无法验证(检查推理基础设施)" + ) + outcomes[r.spec.task_type] = _finalize_outcome( + verdict=r.verdict, + w=r.w, + l=r.l, + n_used=r.n_used, + n_plan=len(r.slots), + base_obs=r.base_obs, + cand_obs=r.cand_obs, + candidate_per_q=r.candidate_per_q, + evidence_rows=r.evidence_rows, + task_type=r.spec.task_type, + ) + return outcomes diff --git a/tests/unit/test_gate_concurrent.py b/tests/unit/test_gate_concurrent.py new file mode 100644 index 0000000..24bdc72 --- /dev/null +++ b/tests/unit/test_gate_concurrent.py @@ -0,0 +1,167 @@ +"""连续并发 gate 编排测试:乱序到达/多题型隔离/终态组装/全 INFRA。""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + +from app.harness.gate_ladder import BaselineCache +from app.harness.validate import GateSpec, validate_skills_concurrent +from tests.unit.test_gate_prefix import _PARAMS, _mk_unit +from tests.unit.test_gate_unit_arm import _FakeLog + + +def _mk_spec(task_type: str, slug: str, n: int) -> GateSpec: + """构造 n 个 single 单元的 gate 规格(unit_id 形如 -q)。""" + return GateSpec( + task_type=task_type, + target_file=f"{slug}.md", + candidate_content=f"cand-{slug}", + base_skill_content=f"base-{slug}", + units=tuple(_mk_unit(f"{slug}-q{i}", task_type) for i in range(n)), + gate_run_prefix=f"r_e1_s0_gate_{slug}", + ) + + +def _scripted_inference(log: _FakeLog, script: dict[str, tuple[bool, float]]): + """脚本化假推理:按 question_id+臂 决定 (对错, 延迟秒),制造乱序到达。""" + + class _R: + def __init__(self, run_id: str, total: int) -> None: + self.run_id = run_id + self.total = total + + async def _run(questions, *, run_id: str, skills_dir: Path): + arm = "cand" if run_id.endswith("_cand") else "base" + correct, delay = script[f"{questions[0].question_id}|{arm}"] + await asyncio.sleep(delay) + for q in questions: + log.rows.append( + { + "run_id": run_id, + "question_id": q.question_id, + "prediction": "A" if correct else "B", + "answer": "A", + "stop_reason": "finished", + "steps_json": "[]", + } + ) + return _R(run_id, len(questions)) + + return _run + + +@pytest.mark.asyncio +async def test_out_of_order_arrival_still_ladder_order(tmp_path, monkeypatch) -> None: + """尾部先到、头部后到:判定结果与顺序到达完全相同(前缀有序性端到端)。""" + spec = _mk_spec("Action Reasoning", "action-reasoning", 4) + log = _FakeLog() + script = {} + for i in range(4): # 头部 q0 最慢;全部翻转为 W(base 错 cand 对) + script[f"action-reasoning-q{i}|base"] = (False, 0.05 if i == 0 else 0.0) + script[f"action-reasoning-q{i}|cand"] = (True, 0.05 if i == 0 else 0.0) + monkeypatch.setattr( + "app.harness.validate.materialize_candidate_skill", + lambda *a, **k: tmp_path / "cand", + ) + outcomes = await validate_skills_concurrent( + workspace_dir=tmp_path, + base_skills_version="v1", + specs=[spec], + gate_params=_PARAMS, + gate_guard_err=0.10, + baseline_cache=BaselineCache(tmp_path / "bc.json"), + prompts_version="v1", + run_inference=_scripted_inference(log, script), + log=log, + concurrency=8, + ) + o = outcomes["Action Reasoning"] + assert o.w == 4 and o.l == 0 + assert [r["ladder_rank"] for r in o.evidence_rows] == [0, 1, 2, 3] + + +@pytest.mark.asyncio +async def test_two_types_isolated(tmp_path, monkeypatch) -> None: + """两题型并行:计数互不污染,各自独立判定。 + + A 型 4 单元全 W(题尽 accept_provisional);B 型 2 单元全平 + (futility 早停,W=L=0)——两型结果都不受对方污染。 + """ + spec_a = _mk_spec("Action Reasoning", "action-reasoning", 4) + spec_b = _mk_spec("Counting Problem", "counting-problem", 2) + log = _FakeLog() + script = {} + for i in range(4): + script[f"action-reasoning-q{i}|base"] = (False, 0.0) + script[f"action-reasoning-q{i}|cand"] = (True, 0.0) + for i in range(2): + script[f"counting-problem-q{i}|base"] = (True, 0.0) + script[f"counting-problem-q{i}|cand"] = (True, 0.0) + monkeypatch.setattr( + "app.harness.validate.materialize_candidate_skill", + lambda *a, **k: tmp_path / "cand", + ) + outcomes = await validate_skills_concurrent( + workspace_dir=tmp_path, + base_skills_version="v1", + specs=[spec_a, spec_b], + gate_params=_PARAMS, + gate_guard_err=0.10, + baseline_cache=BaselineCache(tmp_path / "bc.json"), + prompts_version="v1", + run_inference=_scripted_inference(log, script), + log=log, + concurrency=8, + ) + assert outcomes["Action Reasoning"].w == 4 + assert outcomes["Counting Problem"].w == 0 + assert outcomes["Counting Problem"].l == 0 + + +@pytest.mark.asyncio +async def test_all_infra_raises(tmp_path, monkeypatch) -> None: + """全单元 INFRA:保留现行 RuntimeError 语义(检查推理基础设施)。""" + spec = _mk_spec("Action Reasoning", "action-reasoning", 2) + log = _FakeLog() + + class _R: + def __init__(self, run_id, total): + self.run_id, self.total = run_id, total + + async def _infra_run(questions, *, run_id, skills_dir): + for q in questions: + log.rows.append( + { + "run_id": run_id, + "question_id": q.question_id, + "prediction": "", + "answer": "A", + "stop_reason": "error", + "steps_json": "[]", + } + ) + return _R(run_id, len(questions)) + + monkeypatch.setattr( + "app.harness.validate.materialize_candidate_skill", + lambda *a, **k: tmp_path / "cand", + ) + with pytest.raises(RuntimeError): + await validate_skills_concurrent( + workspace_dir=tmp_path, + base_skills_version="v1", + specs=[spec], + gate_params=_PARAMS, + gate_guard_err=0.99, # 护栏放宽,逼出全 INFRA 分支 + baseline_cache=BaselineCache(tmp_path / "bc.json"), + prompts_version="v1", + run_inference=_infra_run, + log=log, + concurrency=8, + ) diff --git a/tests/unit/test_harness_observation.py b/tests/unit/test_harness_observation.py index c6b7fd2..8b816cc 100644 --- a/tests/unit/test_harness_observation.py +++ b/tests/unit/test_harness_observation.py @@ -166,7 +166,7 @@ def test_write_read_gate_evidence(db_path: str, run_id: str) -> None: { "task_type": "temporal", "question_id": "q1", - "block_idx": 0, + "ladder_rank": 0, "baseline_correct": 1, "candidate_correct": 1, "e_value": 1.0, @@ -175,7 +175,7 @@ def test_write_read_gate_evidence(db_path: str, run_id: str) -> None: { "task_type": "temporal", "question_id": "q2", - "block_idx": 0, + "ladder_rank": 0, "baseline_correct": 0, "candidate_correct": 1, "e_value": 2.0, From 9e8a254fbb6c89afab367dbb2e5c24e56192fe89 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 17 Jul 2026 00:45:47 -0400 Subject: [PATCH 09/17] fix: gate orchestrator materialize leak + cancel-drain on abort (algo #6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex 质量审 2 Critical: - C001: 候选物化移入 try、成功一个登记一个,第 N 个题型物化失败时 finally 仍清理前 N-1 个已建目录,不泄漏 - C002: gather 首异常(护栏 raise)后显式取消其余任务并排水,确保 finally 删除候选目录时无在飞任务访问、事件循环无 pending task 警告; 护栏中止整轮语义不变 - 新增 2 测试:部分物化失败清理 / 护栏 raise 取消收束不悬挂(wait_for 5s) Co-Authored-By: Claude Fable 5 --- app/harness/validate.py | 57 ++++++++++-------- tests/unit/test_gate_concurrent.py | 92 ++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 23 deletions(-) diff --git a/app/harness/validate.py b/app/harness/validate.py index bd0c1d1..d16dbe6 100644 --- a/app/harness/validate.py +++ b/app/harness/validate.py @@ -1272,34 +1272,45 @@ async def validate_skills_concurrent( _validate_gate_specs(specs) base_skills_dir = workspace_dir / "skills" / base_skills_version runs = [_GateRun.from_spec(s) for s in specs] - cand_dirs = { - r.spec.task_type: materialize_candidate_skill( - workspace_dir, base_skills_version, r.spec.target_file, r.spec.candidate_content - ) - for r in runs - } + cand_dirs: dict[str, Path] = {} slots_gate = _QuestionSlots(concurrency) try: - coros = [ - _run_unit_arm( - r, - rank, - arm, - slots_gate, - run_inference, - log, - baseline_cache, - prompts_version, - base_skills_dir, - cand_dirs[r.spec.task_type], - gate_params, - gate_guard_err, + # 成功一个登记一个:第 N 个题型物化抛 OSError 时,已登记的前 N-1 个 + # 目录仍由 finally 统一清理,不泄漏(Codex 质量审 C001)。 + for r in runs: + cand_dirs[r.spec.task_type] = materialize_candidate_skill( + workspace_dir, base_skills_version, r.spec.target_file, r.spec.candidate_content + ) + tasks = [ + asyncio.ensure_future( + _run_unit_arm( + r, + rank, + arm, + slots_gate, + run_inference, + log, + baseline_cache, + prompts_version, + base_skills_dir, + cand_dirs[r.spec.task_type], + gate_params, + gate_guard_err, + ) ) for r, rank, arm in _build_launch_order(runs) ] - # gather 任一任务 raise(INFRA 护栏)即向上传播中止整轮,与现行"护栏 - # 中止训练"语义一致;finally 仍清理候选目录。 - await asyncio.gather(*coros) + # 护栏 raise 中止整轮的语义不变(Codex 质量审 C002):首异常先取消其余 + # 任务并排水(return_exceptions 吞取消回报),确保外层 finally 删除候选 + # 目录时已无在飞任务访问该目录、事件循环收尾无 pending task 警告; + # _run_unit_arm 的题槽获取自带取消回滚,cancel 安全。 + try: + await asyncio.gather(*tasks) + except BaseException: + for t in tasks: + t.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + raise finally: _cleanup_candidate_dirs(cand_dirs) diff --git a/tests/unit/test_gate_concurrent.py b/tests/unit/test_gate_concurrent.py index 24bdc72..38acd42 100644 --- a/tests/unit/test_gate_concurrent.py +++ b/tests/unit/test_gate_concurrent.py @@ -165,3 +165,95 @@ async def test_all_infra_raises(tmp_path, monkeypatch) -> None: log=log, concurrency=8, ) + + +@pytest.mark.asyncio +async def test_partial_materialize_failure_cleans_up(tmp_path, monkeypatch) -> None: + """第 2 个题型物化失败:OSError 传播,且第 1 个已物化目录被清理不泄漏。""" + spec_a = _mk_spec("Action Reasoning", "action-reasoning", 1) + spec_b = _mk_spec("Counting Problem", "counting-problem", 1) + made: list[Path] = [] + + def _mat(workspace_dir, base_skills_version, target_file, content): + if made: # 第 2 次调用:模拟磁盘错误 + raise OSError("第 2 个题型物化失败(模拟)") + d = tmp_path / "cand_a" + d.mkdir() + made.append(d) + return d + + monkeypatch.setattr("app.harness.validate.materialize_candidate_skill", _mat) + + async def _never_called(questions, *, run_id, skills_dir): + raise AssertionError("物化失败后不应发起任何推理") + + with pytest.raises(OSError): + await validate_skills_concurrent( + workspace_dir=tmp_path, + base_skills_version="v1", + specs=[spec_a, spec_b], + gate_params=_PARAMS, + gate_guard_err=0.10, + baseline_cache=BaselineCache(tmp_path / "bc.json"), + prompts_version="v1", + run_inference=_never_called, + log=_FakeLog(), + concurrency=8, + ) + assert len(made) == 1 + assert not made[0].exists() + + +@pytest.mark.asyncio +async def test_guard_raise_cancels_remaining_tasks(tmp_path, monkeypatch) -> None: + """护栏 raise 后其余在飞任务被取消收束:整体在超时内返回,不悬挂。 + + A 型 12 单元推理全 INFRA(stop_reason="error"),分母 ≥10 后错误率 1.0 + 超护栏 0.01 → RuntimeError;B 型推理挂在永不 set 的 Event 上,若无 + 取消收束,validate 将悬挂,wait_for 超时即为回归。 + """ + spec_a = _mk_spec("Action Reasoning", "action-reasoning", 12) + spec_b = _mk_spec("Counting Problem", "counting-problem", 2) + log = _FakeLog() + hang = asyncio.Event() # 永不 set:B 型推理只能靠取消收束 + + class _R: + def __init__(self, run_id, total): + self.run_id, self.total = run_id, total + + async def _run(questions, *, run_id, skills_dir): + if "counting-problem" in run_id: + await hang.wait() + for q in questions: + log.rows.append( + { + "run_id": run_id, + "question_id": q.question_id, + "prediction": "", + "answer": "A", + "stop_reason": "error", + "steps_json": "[]", + } + ) + return _R(run_id, len(questions)) + + monkeypatch.setattr( + "app.harness.validate.materialize_candidate_skill", + lambda *a, **k: tmp_path / "cand", + ) + with pytest.raises(RuntimeError): + await asyncio.wait_for( + validate_skills_concurrent( + workspace_dir=tmp_path, + base_skills_version="v1", + specs=[spec_a, spec_b], + gate_params=_PARAMS, + gate_guard_err=0.01, + baseline_cache=BaselineCache(tmp_path / "bc.json"), + prompts_version="v1", + run_inference=_run, + log=log, + concurrency=8, + ), + timeout=5, + ) From ea6bec542161ee478c05cef164875b143ecb8895 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 17 Jul 2026 00:57:53 -0400 Subject: [PATCH 10/17] fix: clear gate-derived rows on step rerun (idempotency) --- app/harness/runner.py | 63 ++++++++++++++++++---- tests/unit/test_step_rerun_idempotent.py | 69 ++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 9 deletions(-) create mode 100644 tests/unit/test_step_rerun_idempotent.py diff --git a/app/harness/runner.py b/app/harness/runner.py index 3c8b1a1..5b0bb7b 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -602,6 +602,52 @@ def _write_skip_report( ) +def _clear_step_rows(db_path: str, *, baseline_run_id: str, epoch: int, step: int) -> None: + """清空一个 step 的全部旧行（rollout + gate 派生），保证崩溃重跑幂等。 + + 修复前序潜伏 bug：旧实现只清 rollout run_id，gate 派生 run_id + （`{step_run_id}_gate_%`）从不清理，重跑会累积重复 predictions（HarnessLog + 无主键去重），_load_run_rows 的 dict 覆盖使结果依赖 SELECT 顺序。 + gate_evidence / quadrant_pair 以 (run_id, epoch, step) 过滤删除； + 表不存在（首个 step）时跳过。step_report 为按文件名覆盖写的 JSON，天然幂等。 + + 参数: + db_path: harness.db 路径。 + baseline_run_id: 基线 run（gate_evidence/quadrant_pair 的 run_id 维度）。 + epoch: 轮次（1-based）。 + step: epoch 内 step 序号（0-based）。 + + 返回: + 无。 + + 关键实现细节: + predictions/traces 的 gate 行按 LIKE 前缀删除，'_' 通配显式转义 + （ESCAPE）钉死字面匹配，避免 `..._s1` 误匹配 `..._s10` 类前缀陷阱。 + """ + from app.harness.inference import PREDICTIONS_SCHEMA, TRACES_SCHEMA + from app.harness.log import HarnessLog + + step_run_id = f"{baseline_run_id}_e{epoch}_s{step}" + with HarnessLog(db_path, step_run_id, register_run=False) as log: + log.create_table("predictions", PREDICTIONS_SCHEMA) + log.create_table("traces", TRACES_SCHEMA) + for table in ("predictions", "traces"): + log.execute(f"DELETE FROM {table} WHERE run_id=?", (step_run_id,)) + log.execute( + f"DELETE FROM {table} WHERE run_id LIKE ? ESCAPE '\\'", + (step_run_id.replace("_", r"\_") + r"\_gate\_%",), + ) + for table in ("gate_evidence", "quadrant_pair"): + exists = log.query( + "SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table,) + ) + if exists: + log.execute( + f"DELETE FROM {table} WHERE run_id=? AND epoch=? AND step=?", + (baseline_run_id, epoch, step), + ) + + # --------------------------------------------------------------------------- # Runner 主类 # --------------------------------------------------------------------------- @@ -1096,17 +1142,16 @@ class Runner: """单 step：rollout → correctness 增量 → 诊断 → 累加 system/tool → 按类 gate。""" run_id = f"{pools.baseline_run_id}_e{epoch}_s{step}" - from app.harness.inference import PREDICTIONS_SCHEMA, TRACES_SCHEMA from app.harness.log import HarnessLog - # 幂等：重跑同一 step 前先清旧行，避免断点续跑重复累计双计。 - # 先 CREATE TABLE IF NOT EXISTS（fresh workspace 首跑时表尚未由 run_inference 建）， - # register_run=False 避免只读清理污染 _runs 运行状态。 - with HarnessLog(str(self._paths.db_path), run_id, register_run=False) as log: - log.create_table("predictions", PREDICTIONS_SCHEMA) - log.create_table("traces", TRACES_SCHEMA) - log.execute("DELETE FROM predictions WHERE run_id=?", (run_id,)) - log.execute("DELETE FROM traces WHERE run_id=?", (run_id,)) + # 幂等：重跑同一 step 前清 rollout + 全部 gate 派生旧行（修复潜伏 bug： + # 旧实现只清 rollout，gate 行崩溃重跑会累积重复）。 + _clear_step_rows( + str(self._paths.db_path), + baseline_run_id=pools.baseline_run_id, + epoch=epoch, + step=step, + ) await self._rollout_batch(batch, run_id) diff --git a/tests/unit/test_step_rerun_idempotent.py b/tests/unit/test_step_rerun_idempotent.py new file mode 100644 index 0000000..1a966f8 --- /dev/null +++ b/tests/unit/test_step_rerun_idempotent.py @@ -0,0 +1,69 @@ +"""step 重跑幂等：gate 派生行必须随 step 清理，否则崩溃重跑累积重复。""" + +from __future__ import annotations + +import sqlite3 +from typing import TYPE_CHECKING + +from app.harness.runner import _clear_step_rows + +if TYPE_CHECKING: + from pathlib import Path + + +def _mk_db(tmp_path: Path) -> Path: + """构造含 rollout 行、gate 派生行、他 step 行与前缀陷阱行的最小 harness.db。 + + 参数: + tmp_path: pytest 临时目录。 + + 返回: + harness.db 路径。 + """ + db = tmp_path / "harness.db" + conn = sqlite3.connect(db) + conn.execute("CREATE TABLE predictions (run_id TEXT, question_id TEXT)") + conn.execute("CREATE TABLE traces (run_id TEXT, question_id TEXT)") + conn.execute("CREATE TABLE gate_evidence (run_id TEXT, epoch INTEGER, step INTEGER)") + conn.execute("CREATE TABLE quadrant_pair (run_id TEXT, epoch INTEGER, step INTEGER)") + rows = [ + ("infer_adhoc_e1_s0", "q1"), # rollout 行 + ("infer_adhoc_e1_s0_gate_action-reasoning_base", "q2"), # gate base 臂 + ("infer_adhoc_e1_s0_gate_action-reasoning_cand", "q3"), # gate cand 臂 + ("infer_adhoc_e1_s1", "q4"), # 其他 step，不许误删 + ("infer_adhoc_e1_s10_gate_x_base", "q5"), # s10 前缀陷阱，不许误删 + ] + conn.executemany("INSERT INTO predictions VALUES (?, ?)", rows) + conn.executemany("INSERT INTO traces VALUES (?, ?)", rows) + conn.execute("INSERT INTO gate_evidence VALUES ('infer_adhoc', 1, 0)") + conn.execute("INSERT INTO gate_evidence VALUES ('infer_adhoc', 1, 1)") + conn.execute("INSERT INTO quadrant_pair VALUES ('infer_adhoc', 1, 0)") + conn.commit() + conn.close() + return db + + +def test_clear_step_rows_removes_rollout_and_gate_rows(tmp_path) -> None: + """rollout 行 + 本 step 全部 gate 派生行被清；他 step 与 s10 前缀陷阱不动。""" + db = _mk_db(tmp_path) + _clear_step_rows(str(db), baseline_run_id="infer_adhoc", epoch=1, step=0) + conn = sqlite3.connect(db) + left = {r[0] for r in conn.execute("SELECT run_id FROM predictions")} + assert left == {"infer_adhoc_e1_s1", "infer_adhoc_e1_s10_gate_x_base"} + left_t = {r[0] for r in conn.execute("SELECT run_id FROM traces")} + assert left_t == left + ge = list(conn.execute("SELECT step FROM gate_evidence")) + assert ge == [(1,)] # 只剩 step=1 的行 + assert list(conn.execute("SELECT COUNT(*) FROM quadrant_pair"))[0][0] == 0 + conn.close() + + +def test_clear_step_rows_missing_tables_is_noop(tmp_path) -> None: + """gate_evidence/quadrant_pair 表尚未建（首个 step）时不报错。""" + db = tmp_path / "harness.db" + conn = sqlite3.connect(db) + conn.execute("CREATE TABLE predictions (run_id TEXT)") + conn.execute("CREATE TABLE traces (run_id TEXT)") + conn.commit() + conn.close() + _clear_step_rows(str(db), baseline_run_id="infer_adhoc", epoch=1, step=0) From 16993ed36275e2b44d9cacdf1044b49e6496b661 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 17 Jul 2026 01:13:19 -0400 Subject: [PATCH 11/17] style: consolidate inference stub, complete docstrings (algo #6) --- app/harness/observation.py | 3 +++ app/harness/validate.py | 9 ++++++--- tests/unit/test_gate_concurrent.py | 27 +++++++++++---------------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/app/harness/observation.py b/app/harness/observation.py index b1b9d83..c8e8b5a 100644 --- a/app/harness/observation.py +++ b/app/harness/observation.py @@ -348,6 +348,9 @@ def write_gate_evidence( question_id 字段承载 **unit_id**（single=question_id，pair=pair_id）—— 逐题明细在 predictions 表溯源，按 pair_id join 真实 question 表会 join 不上。 + 返回: + 无。 + 关键实现: 逐行 insert（非 insert_many），保证每行独立事务。 """ diff --git a/app/harness/validate.py b/app/harness/validate.py index d16dbe6..b2228c0 100644 --- a/app/harness/validate.py +++ b/app/harness/validate.py @@ -1246,9 +1246,12 @@ async def validate_skills_concurrent( ) -> dict[str, ValidationOutcome]: """连续并发 gate:多题型全部臂共享题槽并发,统计按阶梯序前缀有序推进。 - 发射顺序 = 题型 round-robin × 题型内阶梯序(base 先 cand 后);题型过线即 - 冻结,其排队任务启动时自查冻结标志撤销,in-flight 结果不计入(τ 之后样本, - 合法丢弃)。全部题型判定后统一组装 ValidationOutcome。 + 关键实现细节: + 发射顺序 = 题型 round-robin × 题型内阶梯序(base 先 cand 后);题型过线 + 即冻结,其排队任务启动时自查冻结标志撤销,in-flight 结果不计入(τ 之后 + 样本,合法丢弃);候选目录逐个物化即登记、统一 finally 清理(中途失败不 + 泄漏);任一任务异常先 cancel+排水其余任务再向上传播;全部题型判定后 + 统一经 _finalize_outcome 组装。 参数: workspace_dir: workspace 根目录(候选物化用)。 diff --git a/tests/unit/test_gate_concurrent.py b/tests/unit/test_gate_concurrent.py index 38acd42..cad1bda 100644 --- a/tests/unit/test_gate_concurrent.py +++ b/tests/unit/test_gate_concurrent.py @@ -16,6 +16,14 @@ from tests.unit.test_gate_prefix import _PARAMS, _mk_unit from tests.unit.test_gate_unit_arm import _FakeLog +class _FakeInferenceResult: + """推理结果桩:只承载编排器消费的 run_id 与 total 两个字段。""" + + def __init__(self, run_id: str, total: int) -> None: + self.run_id = run_id + self.total = total + + def _mk_spec(task_type: str, slug: str, n: int) -> GateSpec: """构造 n 个 single 单元的 gate 规格(unit_id 形如 -q)。""" return GateSpec( @@ -31,11 +39,6 @@ def _mk_spec(task_type: str, slug: str, n: int) -> GateSpec: def _scripted_inference(log: _FakeLog, script: dict[str, tuple[bool, float]]): """脚本化假推理:按 question_id+臂 决定 (对错, 延迟秒),制造乱序到达。""" - class _R: - def __init__(self, run_id: str, total: int) -> None: - self.run_id = run_id - self.total = total - async def _run(questions, *, run_id: str, skills_dir: Path): arm = "cand" if run_id.endswith("_cand") else "base" correct, delay = script[f"{questions[0].question_id}|{arm}"] @@ -51,7 +54,7 @@ def _scripted_inference(log: _FakeLog, script: dict[str, tuple[bool, float]]): "steps_json": "[]", } ) - return _R(run_id, len(questions)) + return _FakeInferenceResult(run_id, len(questions)) return _run @@ -130,10 +133,6 @@ async def test_all_infra_raises(tmp_path, monkeypatch) -> None: spec = _mk_spec("Action Reasoning", "action-reasoning", 2) log = _FakeLog() - class _R: - def __init__(self, run_id, total): - self.run_id, self.total = run_id, total - async def _infra_run(questions, *, run_id, skills_dir): for q in questions: log.rows.append( @@ -146,7 +145,7 @@ async def test_all_infra_raises(tmp_path, monkeypatch) -> None: "steps_json": "[]", } ) - return _R(run_id, len(questions)) + return _FakeInferenceResult(run_id, len(questions)) monkeypatch.setattr( "app.harness.validate.materialize_candidate_skill", @@ -217,10 +216,6 @@ async def test_guard_raise_cancels_remaining_tasks(tmp_path, monkeypatch) -> Non log = _FakeLog() hang = asyncio.Event() # 永不 set:B 型推理只能靠取消收束 - class _R: - def __init__(self, run_id, total): - self.run_id, self.total = run_id, total - async def _run(questions, *, run_id, skills_dir): if "counting-problem" in run_id: await hang.wait() @@ -235,7 +230,7 @@ async def test_guard_raise_cancels_remaining_tasks(tmp_path, monkeypatch) -> Non "steps_json": "[]", } ) - return _R(run_id, len(questions)) + return _FakeInferenceResult(run_id, len(questions)) monkeypatch.setattr( "app.harness.validate.materialize_candidate_skill", From b3aba7c31d11c85c8c984bc1eb6428ee52a84bfe Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 17 Jul 2026 01:17:02 -0400 Subject: [PATCH 12/17] fix: escape all LIKE specials in step-row cleanup --- app/harness/runner.py | 23 ++++++++++++++++++++--- tests/unit/test_step_rerun_idempotent.py | 23 +++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/app/harness/runner.py b/app/harness/runner.py index 5b0bb7b..0664f55 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -602,6 +602,21 @@ def _write_skip_report( ) +def _escape_sql_like(text: str) -> str: + """转义 SQL LIKE 模式中的全部特殊字符(`\\`、`%`、`_`)为字面匹配。 + + 参数: + text: 待作为 LIKE 前缀字面使用的原始字符串。 + + 返回: + 可安全拼入 `LIKE ? ESCAPE '\\'` 模式的转义串。 + + 关键实现细节: + 反斜杠必须最先转义,否则会二次转义后续替换产生的转义符。 + """ + return text.replace("\\", "\\\\").replace("%", r"\%").replace("_", r"\_") + + def _clear_step_rows(db_path: str, *, baseline_run_id: str, epoch: int, step: int) -> None: """清空一个 step 的全部旧行（rollout + gate 派生），保证崩溃重跑幂等。 @@ -621,13 +636,15 @@ def _clear_step_rows(db_path: str, *, baseline_run_id: str, epoch: int, step: in 无。 关键实现细节: - predictions/traces 的 gate 行按 LIKE 前缀删除，'_' 通配显式转义 - （ESCAPE）钉死字面匹配，避免 `..._s1` 误匹配 `..._s10` 类前缀陷阱。 + predictions/traces 的 gate 行按 LIKE 前缀删除，`\\`/`%`/`_` 三个 LIKE + 特殊字符全部显式转义（ESCAPE）钉死字面匹配，避免 `..._s1` 误匹配 + `..._s10` 类前缀陷阱，也防 run_id 含 `%`/`\\` 时通配误删他 run 行。 """ from app.harness.inference import PREDICTIONS_SCHEMA, TRACES_SCHEMA from app.harness.log import HarnessLog step_run_id = f"{baseline_run_id}_e{epoch}_s{step}" + escaped = _escape_sql_like(step_run_id) with HarnessLog(db_path, step_run_id, register_run=False) as log: log.create_table("predictions", PREDICTIONS_SCHEMA) log.create_table("traces", TRACES_SCHEMA) @@ -635,7 +652,7 @@ def _clear_step_rows(db_path: str, *, baseline_run_id: str, epoch: int, step: in log.execute(f"DELETE FROM {table} WHERE run_id=?", (step_run_id,)) log.execute( f"DELETE FROM {table} WHERE run_id LIKE ? ESCAPE '\\'", - (step_run_id.replace("_", r"\_") + r"\_gate\_%",), + (escaped + r"\_gate\_%",), ) for table in ("gate_evidence", "quadrant_pair"): exists = log.query( diff --git a/tests/unit/test_step_rerun_idempotent.py b/tests/unit/test_step_rerun_idempotent.py index 1a966f8..b19e85d 100644 --- a/tests/unit/test_step_rerun_idempotent.py +++ b/tests/unit/test_step_rerun_idempotent.py @@ -67,3 +67,26 @@ def test_clear_step_rows_missing_tables_is_noop(tmp_path) -> None: conn.commit() conn.close() _clear_step_rows(str(db), baseline_run_id="infer_adhoc", epoch=1, step=0) + + +def test_clear_step_rows_like_specials_in_run_id(tmp_path) -> None: + """run_id 含 % 与反斜杠时不通配误删他 run 行(LIKE 全特殊字符转义回归锁)。""" + db = tmp_path / "harness.db" + conn = sqlite3.connect(db) + conn.execute("CREATE TABLE predictions (run_id TEXT, question_id TEXT)") + conn.execute("CREATE TABLE traces (run_id TEXT, question_id TEXT)") + rows = [ + (r"we%ird\run_e1_s0", "q1"), # 本 step rollout + (r"we%ird\run_e1_s0_gate_x_base", "q2"), # 本 step gate 行 + (r"weXird\run_e1_s0_gate_x_base", "q3"), # % 若未转义会误匹配此行 + (r"we%irdXrun_e1_s0_gate_x_base", "q4"), # \ 若未转义会误匹配此行 + ] + conn.executemany("INSERT INTO predictions VALUES (?, ?)", rows) + conn.executemany("INSERT INTO traces VALUES (?, ?)", rows) + conn.commit() + conn.close() + _clear_step_rows(str(db), baseline_run_id=r"we%ird\run", epoch=1, step=0) + conn = sqlite3.connect(db) + left = {r[0] for r in conn.execute("SELECT run_id FROM predictions")} + conn.close() + assert left == {r"weXird\run_e1_s0_gate_x_base", r"we%irdXrun_e1_s0_gate_x_base"} From 23a64042fe56b1a78ec935aa7fe81a7f6c191c24 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 17 Jul 2026 03:32:14 -0400 Subject: [PATCH 13/17] feat: parallel evolve + continuous gate wiring in runner (algo #6) --- app/harness/runner.py | 383 +++++++++++++++++-------- tests/unit/test_gate_batch_parallel.py | 238 +++++++++++++++ 2 files changed, 508 insertions(+), 113 deletions(-) create mode 100644 tests/unit/test_gate_batch_parallel.py diff --git a/app/harness/runner.py b/app/harness/runner.py index 0664f55..013ecb2 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -12,6 +12,7 @@ from __future__ import annotations +import asyncio import json import math import random @@ -34,6 +35,7 @@ from app.harness.checkpoint import ( ) from app.harness.config import RunConfig # noqa: TC001 — 运行时 _compute_total_steps 使用 from app.harness.gate_ladder import BaselineCache, GatePools, build_or_load_gate_pools +from app.harness.log import HarnessLog from app.harness.observation import ( write_dual_metric, write_epoch_report, @@ -45,7 +47,13 @@ from app.harness.observation import ( ) from app.harness.question_units import build_units, unit_correctness_view from app.harness.store import advance_version -from app.harness.validate import Probation, ValidationOutcome +from app.harness.validate import ( + GateSpec, + Probation, + ValidationOutcome, + _ladder_units, + validate_skills_concurrent, +) from app.harness.workspace import ( ResolvedPaths, archive_workspace, @@ -602,6 +610,32 @@ def _write_skip_report( ) +def _assert_disjoint_target_files(targets_by_type: dict[str, str]) -> None: + """断言本 step 各题型进化目标文件互不相同(设计 v3 §1 fail-fast)。 + + 题型并行进化 + 并行 gate 的前提是 skill 文件不相交;两题型 fallback 到 + 同一 default-strategy.md 时并行会互相覆盖候选与 accept,必须显式中止 + 而非静默串行(当前 12 题型均有专属文件,此断言防未来配置漂移)。 + + 参数: + targets_by_type: {题型: 解析后 skill 文件名}。 + + 返回: + 无。 + + 异常: + RuntimeError: 存在两个题型映射同一文件。 + """ + seen: dict[str, str] = {} + for task_type, target in targets_by_type.items(): + if target in seen: + raise RuntimeError( + f"题型 {seen[target]!r} 与 {task_type!r} 映射同一 skill 文件 {target!r}," + "并行进化/gate 不支持共享目标文件(设计 v3 §1)" + ) + seen[target] = task_type + + def _escape_sql_like(text: str) -> str: """转义 SQL LIKE 模式中的全部特殊字符(`\\`、`%`、`_`)为字面匹配。 @@ -1199,7 +1233,7 @@ class Runner: _guard_infra_failures(result, context="rollout") # ----------------------------------------------------------------------- - # _gate_batch_skills：per task_type gate + # _gate_batch_skills：并行进化 + 连续并发 gate（四阶段） # ----------------------------------------------------------------------- async def _gate_batch_skills( @@ -1211,18 +1245,95 @@ class Runner: pools: Pools, state: _TrainState, ) -> None: - """按 task_type 独立 evolve → 局部验证 → accept/reject。""" - from app.harness.workspace import VersionedSkillStore - from core.evolution import evolve_single_skill + """按 task_type 并行 evolve → 连续并发 gate → 字母序统一落账。 + 四阶段(设计 v3 §2.1):Phase A 并行进化(cooldown/无改动照旧跳过); + Phase B 装配 GateSpec(阶梯出题 + 案例单元排除 + n_max 截断); + Phase C validate_skills_concurrent(共享题槽,统计按阶梯序前缀推进, + 只读 state);Phase D 唯一写 state 阶段——按字母序 accept/reject 落账, + 与原串行语义等价(题型 skill 文件不相交,合并顺序仅为确定性)。 + + 参数: + epoch / step / total_steps: 训练坐标。 + diagnosis: 本 step 诊断结果(skill_case_packs 按题型分组)。 + pools: 冻结三池。 + state: 训练状态(Phase D 唯一写入点)。 + + 返回: + 无。 + """ budget = edit_budget_at( global_step=state.global_step, total_steps=total_steps, start=self._config.edit_budget_start, end=self._config.edit_budget_end, ) + + # ---- Phase A: 并行进化(冷却/无真实改动照旧写 skip 后出清) ---- + records = await self._evolve_types_parallel(epoch, step, diagnosis, budget, pools, state) + if not records: + return + _assert_disjoint_target_files({t: r.target_file for t, r in records.items()}) + + # ---- Phase B: 装配 GateSpec(阶梯出题,收编原 _run_gate_validation 前半) ---- + specs = self._assemble_gate_specs(epoch, step, diagnosis, records, pools, state) + + # ---- Phase C: 连续并发 gate(只读 state) ---- + with HarnessLog(str(self._paths.db_path), f"gate_e{epoch}_s{step}") as gate_log: + outcomes = await validate_skills_concurrent( + workspace_dir=self._config.workspace_dir, + base_skills_version=self._current_version("skills"), + specs=specs, + gate_params=GateParams( + e_confirm=self._config.gate_e_confirm, + e_provisional=self._config.gate_e_provisional, + w_net_min=self._config.gate_w_net_min, + delta_min=self._config.gate_delta_min, + lambda_dir=self._config.gate_lambda_dir, + e_rollback=self._config.gate_e_rollback, + ), + gate_guard_err=self._config.gate_guard_err, + baseline_cache=state.baseline_cache, + prompts_version=self._current_version("prompts"), + run_inference=self._make_validate_run_inference_fn(gate_log), + log=gate_log, + concurrency=self._config.concurrency, + ) + + # ---- Phase D: 唯一写 state 阶段(字母序确定性落账) ---- + self._settle_gate_outcomes(epoch, step, records, outcomes, budget, pools, state) + + async def _evolve_types_parallel( + self, + epoch: int, + step: int, + diagnosis: DiagnosisResult, + budget: int, + pools: Pools, + state: _TrainState, + ) -> dict[str, EvolutionRecord]: + """Phase A:各题型进化 asyncio.gather 并行,冷却/无改动路径写 skip 出队。 + + cooldown 与"进化未产出真实改动"(rejected/skipped/内容未变)两类路径 + 与原串行实现语义一致:写 skip_report 后不进 gate。进化互相独立 + (各题型 skill 文件不相交,VersionedSkillStore 只读基线版本), + gather 并行不改变单题型结果。 + + 参数: + epoch / step: 训练坐标。 + diagnosis: 本 step 诊断结果。 + budget: 当步编辑预算。 + pools: 冻结三池。 + state: 训练状态(只读)。 + + 返回: + {题型: EvolutionRecord},仅含产出真实改动、待 gate 的题型。 + """ + from app.harness.workspace import VersionedSkillStore + from core.evolution import evolve_single_skill + + active_types: list[str] = [] for task_type in sorted(diagnosis.skill_case_packs): - # 冷却 admission control if state.gate_cooldown.get(task_type, 0) > 0: _write_skip_report( self._config.workspace_dir, @@ -1237,22 +1348,40 @@ class Runner: budget=budget, ) continue + active_types.append(task_type) + if not active_types: + return {} + evolve_prompts = self._load_evolve_prompts() + skills_version = self._current_version("skills") + + async def _evolve_one(task_type: str) -> EvolutionRecord: pack = diagnosis.skill_case_packs[task_type] skill_store = VersionedSkillStore(self._paths.skills_dir) - evolve_prompts = self._load_evolve_prompts() - record = await evolve_single_skill( + return await evolve_single_skill( self._evolve_llm, pack, skill_store, evolve_prompts, - self._current_version("skills"), + skills_version, budget, self._config.appendix_consolidate_threshold, skill_update_mode=self._config.skill_update_mode, rejected=state.rejected_buffer.get(task_type, []), ) - # 进化未产出真实改动 + + records = dict( + zip( + active_types, + await asyncio.gather(*[_evolve_one(t) for t in active_types]), + strict=True, + ) + ) + + # 无真实改动的题型照旧写 skipped 后出队 + gated: dict[str, EvolutionRecord] = {} + for task_type in active_types: + record = records[task_type] if record.status in ("rejected", "skipped") or ( record.evolved_content == record.original_content ): @@ -1270,11 +1399,107 @@ class Runner: rank_clip_triggered=bool(record.clip_info.get("triggered", False)), ) continue + gated[task_type] = record + return gated - outcome = await self._run_gate_validation( - epoch, step, task_type, pack, record, pools, state + def _assemble_gate_specs( + self, + epoch: int, + step: int, + diagnosis: DiagnosisResult, + records: dict[str, EvolutionRecord], + pools: Pools, + state: _TrainState, + ) -> list[GateSpec]: + """Phase B:为每个待 gate 题型装配 GateSpec(阶梯出题 + 截断)。 + + 案例包按 unit 排除:把每个 case 的 question_id 映射到其所属 unit_id, + 命中单元整体排除,防止只排 AR pair 半个成员而给 gate 池灌半个 pair + (下游 _ladder_units 会 fail-fast)。base_skill_content 读 step 起点 + 版本(self._paths 在 Phase D accept 前不变),保证所有题型对同一 + 基线版本验证。核心算法保真 #5。 + + 参数: + epoch / step: 训练坐标(拼 gate_run_prefix)。 + diagnosis: 本 step 诊断结果(案例排除来源)。 + records: Phase A 产出的待 gate 进化记录。 + pools: 冻结三池(baseline_run_id)。 + state: 训练状态(只读 gate_pools / gate_epoch_observed)。 + + 返回: + 与 records 键序一致的 GateSpec 列表。 + + 异常: + RuntimeError: 阶梯引用了题库中不存在的 unit_id。 + """ + specs: list[GateSpec] = [] + for task_type, record in records.items(): + pack = diagnosis.skill_case_packs[task_type] + exclude_units = { + self._gate_questions_by_id[c.question_id].unit_id + for c in pack.failure_cases + pack.success_cases + if c.question_id in self._gate_questions_by_id + } + ladder_unit_ids = state.gate_pools.ladder_for( + task_type, + exclude_units, + p_low=self._config.gate_p_low, + p_high=self._config.gate_p_high, + cold=not state.gate_epoch_observed, ) - # 观测落库 + missing = [uid for uid in ladder_unit_ids if uid not in self._gate_units_by_id] + if missing: + raise RuntimeError( + f"gate 阶梯引用未知 unit: {missing[:5]}(gate_pools.json 与题库失配)" + ) + ladder_items = [ + q for uid in ladder_unit_ids for q in self._gate_units_by_id[uid].questions + ] + slug = task_type.lower().replace(" ", "-") + specs.append( + GateSpec( + task_type=task_type, + target_file=record.target_file, + candidate_content=record.evolved_content, + base_skill_content=(self._paths.skills_dir / record.target_file).read_text( + encoding="utf-8" + ), + units=tuple(_ladder_units(ladder_items)[: self._config.gate_n_max]), + gate_run_prefix=f"{pools.baseline_run_id}_e{epoch}_s{step}_gate_{slug}", + ) + ) + return specs + + def _settle_gate_outcomes( + self, + epoch: int, + step: int, + records: dict[str, EvolutionRecord], + outcomes: dict[str, ValidationOutcome], + budget: int, + pools: Pools, + state: _TrainState, + ) -> None: + """Phase D:按字母序统一落账(观测落库 + accept/reject 写 state)。 + + 本阶段是 _gate_batch_skills 唯一写 state 的阶段。字母序仅为确定性 + (题型 skill 文件不相交,accept 串行叠加时 _accept_skill 基于最新 + manifest 版本追加各自 target_file,互不覆盖),与原串行语义等价。 + + 参数: + epoch / step: 训练坐标。 + records: Phase A 产出的进化记录。 + outcomes: Phase C 产出的 gate 判定。 + budget: 当步编辑预算(step_report 落账)。 + pools: 冻结三池。 + state: 训练状态(唯一写入点)。 + + 返回: + 无。 + """ + for task_type in sorted(outcomes): + record = records[task_type] + outcome = outcomes[task_type] write_gate_evidence( str(self._paths.db_path), run_id=pools.baseline_run_id, @@ -1313,88 +1538,6 @@ class Runner: state.rejected_buffer, task_type, record, outcome, state.global_step ) - async def _run_gate_validation( - self, - epoch: int, - step: int, - task_type: str, - pack: Any, - record: EvolutionRecord, - pools: Pools, - state: _TrainState, - ) -> ValidationOutcome: - """CE-Gate 块序贯配对验证：阶梯出题 → 基线/候选逐块配对 → e-process 四出口。 - - 参数: - epoch: 轮次。 - step: epoch 内 step。 - task_type: 待验证题型。 - pack: SkillCasePack。 - record: 进化产物。 - pools: 冻结三池。 - state: 训练状态。 - - 返回: - ValidationOutcome。 - """ - from app.harness.log import HarnessLog - from app.harness.validate import validate_skill_local - - # 案例包按 unit 排除：把每个 case 的 question_id 映射到其所属 unit_id， - # 命中单元整体排除，防止只排 AR pair 半个成员而给 gate 池灌半个 pair - # （下游 _ladder_units 会 fail-fast）。核心算法保真 #5。 - exclude_units = { - self._gate_questions_by_id[c.question_id].unit_id - for c in pack.failure_cases + pack.success_cases - if c.question_id in self._gate_questions_by_id - } - ladder_unit_ids = state.gate_pools.ladder_for( - task_type, - exclude_units, - p_low=self._config.gate_p_low, - p_high=self._config.gate_p_high, - cold=not state.gate_epoch_observed, - ) - missing = [uid for uid in ladder_unit_ids if uid not in self._gate_units_by_id] - if missing: - raise ValueError( - f"gate 阶梯[{task_type}] 含 benchmark 中不存在的单元: " - f"{missing[:5]}（gate_pools.json 与题库失配）" - ) - # 单元展开为逐题（unit 内成员顺序保持），下游 validate 再按阶梯序聚合回单元。 - ladder_items = [q for uid in ladder_unit_ids for q in self._gate_units_by_id[uid].questions] - base_skill_content = (self._paths.skills_dir / record.target_file).read_text( - encoding="utf-8" - ) - slug = task_type.lower().replace(" ", "-") - run_inference_fn = self._make_validate_run_inference_fn() - with HarnessLog(str(self._paths.db_path), f"gate_{slug}") as gate_log: - return await validate_skill_local( - workspace_dir=self._config.workspace_dir, - base_skills_version=self._current_version("skills"), - task_type=task_type, - target_file=record.target_file, - candidate_content=record.evolved_content, - base_skill_content=base_skill_content, - ladder_items=ladder_items, - gate_params=GateParams( - e_confirm=self._config.gate_e_confirm, - e_provisional=self._config.gate_e_provisional, - w_net_min=self._config.gate_w_net_min, - delta_min=self._config.gate_delta_min, - lambda_dir=self._config.gate_lambda_dir, - e_rollback=self._config.gate_e_rollback, - ), - gate_block=self._config.gate_block, - gate_n_max=self._config.gate_n_max, - gate_guard_err=self._config.gate_guard_err, - baseline_cache=state.baseline_cache, - prompts_version=self._current_version("prompts"), - run_inference=run_inference_fn, - log=gate_log, - gate_run_prefix=(f"{pools.baseline_run_id}_e{epoch}_s{step}_gate_{slug}"), - ) - # ----------------------------------------------------------------------- # accept / reject / probation # ----------------------------------------------------------------------- @@ -2475,10 +2618,23 @@ class Runner: return _noop_builder - def _make_validate_run_inference_fn(self): - """构造 validate 用的 RunInferenceFn（绑定共享依赖）。""" + def _make_validate_run_inference_fn(self, gate_log: HarnessLog): + """构造 validate 用的 RunInferenceFn(绑定共享依赖与共享 HarnessLog)。 + + 连续并发 gate 下本函数被逐单元高频并发调用:每次调用新建 HarnessLog + 连接会重现多连接争 SQLite 写锁(遥测同款教训),故复用调用方传入的 + 单一 gate_log(单连接 + threading.Lock 串行化)。_record_run 按 run_id + 去重,避免逐单元重复 upsert。 + + 参数: + gate_log: 本 step gate 阶段共享的 HarnessLog 实例。 + + 返回: + 符合 RunInferenceFn 协议的异步推理函数。 + """ from app.harness.inference import run_inference - from app.harness.log import HarnessLog + + recorded: set[str] = set() async def _run( questions: list[GeneratedQuestion], @@ -2486,21 +2642,22 @@ class Runner: run_id: str, skills_dir: Path, ) -> InferenceResult: - self._record_run(run_id) - with HarnessLog(str(self._paths.db_path), run_id) as log: - return await run_inference( - questions=questions, - llm=self._llm, - tool_dispatch_fn=self._make_tool_dispatch_fn(skills_dir=skills_dir), - prompt_builder=self._make_prompt_builder( - skills_dir=skills_dir, prompts_dir=self._paths.prompts_dir - ), - log=log, - run_id=run_id, - concurrency=self._config.concurrency, - max_steps=self._config.max_steps, - skill_mode=self._config.skill_mode, - ) + if run_id not in recorded: + recorded.add(run_id) + self._record_run(run_id) + return await run_inference( + questions=questions, + llm=self._llm, + tool_dispatch_fn=self._make_tool_dispatch_fn(skills_dir=skills_dir), + prompt_builder=self._make_prompt_builder( + skills_dir=skills_dir, prompts_dir=self._paths.prompts_dir + ), + log=gate_log, + run_id=run_id, + concurrency=self._config.concurrency, + max_steps=self._config.max_steps, + skill_mode=self._config.skill_mode, + ) return _run diff --git a/tests/unit/test_gate_batch_parallel.py b/tests/unit/test_gate_batch_parallel.py new file mode 100644 index 0000000..6ff1725 --- /dev/null +++ b/tests/unit/test_gate_batch_parallel.py @@ -0,0 +1,238 @@ +"""_gate_batch_skills 并行装配的纯逻辑护栏 + runner 级并发编排测试。""" + +from __future__ import annotations + +import asyncio +import time +from types import SimpleNamespace +from typing import TYPE_CHECKING + +import pytest + +from app.harness import runner as runner_mod +from app.harness.question_units import build_units +from app.harness.runner import Runner, _assert_disjoint_target_files +from app.harness.validate import ValidationOutcome +from core.types import GeneratedQuestion + +if TYPE_CHECKING: + from pathlib import Path + + +def test_disjoint_target_files_pass() -> None: + """各题型映射不同文件:通过。""" + _assert_disjoint_target_files( + {"Action Reasoning": "action-reasoning.md", "Counting Problem": "counting-problem.md"} + ) + + +def test_shared_target_file_fails_fast() -> None: + """两题型 fallback 到同一文件:并行进化会互相覆盖,必须 fail-fast。""" + with pytest.raises(RuntimeError, match="default-strategy.md"): + _assert_disjoint_target_files( + {"OCR Problems": "default-strategy.md", "Spatial Reasoning": "default-strategy.md"} + ) + + +# --------------------------------------------------------------------------- +# runner 级并发编排测试(Codex 计划审 I6): +# 用 Runner.__new__ 裸实例 + 假依赖驱动 _gate_batch_skills 四阶段, +# 断言 Phase A gather 并行、Phase D 字母序落账、accept/reject 正确分派。 +# --------------------------------------------------------------------------- + +_TYPE_A = "Action Reasoning" +_TYPE_C = "Counting Problem" +_TARGET_FILES = {_TYPE_A: "action-reasoning.md", _TYPE_C: "counting-problem.md"} + + +def _question(qid: str, task_type: str) -> GeneratedQuestion: + """构造一条真实结构的 single 题目(unit_id 由 __post_init__ 回填)。""" + return GeneratedQuestion( + question_id=qid, + video_id="video-001", + task_type=task_type, + question="视频中主角最先做了什么?", + options=("A. 开门", "B. 关灯", "C. 坐下", "D. 起身"), + answer="A", + source_nodes=("L3_0001",), + difficulty="medium", + ) + + +def _record(task_type: str) -> SimpleNamespace: + """构造 EvolutionRecord 替身(仅含 _gate_batch_skills 消费的属性)。""" + return SimpleNamespace( + status="accepted", + original_content="旧 skill 内容", + evolved_content=f"进化后 skill 内容({task_type})", + target_file=_TARGET_FILES[task_type], + clip_info={}, + ) + + +def _outcome(accepted: bool) -> ValidationOutcome: + """构造真实 ValidationOutcome(一 accept 一 reject 分派用)。""" + return ValidationOutcome( + action="accept_confirmed" if accepted else "reject", + accepted=accepted, + stop_reason="confirmed" if accepted else "futility", + e_value=25.0 if accepted else 0.4, + w=3, + l=0 if accepted else 3, + n_used=4, + delta_hat=0.3 if accepted else -0.2, + delta_shrunk=0.2 if accepted else -0.1, + baseline_acc=0.5, + candidate_acc=0.8 if accepted else 0.3, + evidence_rows=[{"question_id": "q", "stop_reason": "answered"}], + ) + + +class _FakeHarnessLog: + """HarnessLog no-op 替身(上下文管理器协议)。""" + + def __init__(self, *args: object, **kwargs: object) -> None: + self.args = args + + def __enter__(self) -> _FakeHarnessLog: + return self + + def __exit__(self, *exc: object) -> bool: + return False + + +def _build_runner(tmp_path: Path) -> tuple[Runner, SimpleNamespace, SimpleNamespace]: + """构造裸 Runner 实例与 state/pools 替身(不触发真实 __init__)。""" + skills_dir = tmp_path / "skills" / "v1" + skills_dir.mkdir(parents=True) + for target in _TARGET_FILES.values(): + (skills_dir / target).write_text("旧 skill 内容", encoding="utf-8") + + runner = Runner.__new__(Runner) + runner._config = SimpleNamespace( + workspace_dir=tmp_path, + edit_budget_start=4, + edit_budget_end=2, + appendix_consolidate_threshold=3, + skill_update_mode="rewrite", + gate_p_low=0.3, + gate_p_high=0.85, + gate_n_max=8, + gate_e_confirm=20.0, + gate_e_provisional=5.0, + gate_w_net_min=2, + gate_delta_min=0.05, + gate_lambda_dir=0.5, + gate_e_rollback=0.05, + gate_guard_err=0.34, + concurrency=4, + max_steps=10, + skill_mode="live", + ) + runner._paths = SimpleNamespace( + skills_dir=skills_dir, + prompts_dir=tmp_path / "prompts", + db_path=tmp_path / "harness.db", + ) + runner._llm = object() + runner._evolve_llm = object() + runner._load_evolve_prompts = lambda: None + runner._current_version = lambda kind: "v1" + runner._class_baseline_acc = lambda *a, **k: 0.5 + runner._record_run = lambda run_id: None + + questions = {t: _question(f"q-{t[:2].lower()}", t) for t in _TARGET_FILES} + units = {t: build_units([q])[0] for t, q in questions.items()} + runner._gate_questions_by_id = {q.question_id: q for q in questions.values()} + runner._gate_units_by_id = {u.unit_id: u for u in units.values()} + unit_ids_by_type = {t: [u.unit_id] for t, u in units.items()} + + state = SimpleNamespace( + gate_cooldown={}, + rejected_buffer={}, + global_step=0, + correctness={}, + gate_epoch_observed=True, + baseline_cache=object(), + gate_pools=SimpleNamespace( + ladder_for=lambda task_type, exclude, *, p_low, p_high, cold: unit_ids_by_type[ + task_type + ] + ), + ) + pools = SimpleNamespace(baseline_run_id="baseline-run", validation=[]) + return runner, state, pools + + +def test_gate_batch_parallel_evolve_and_alphabetical_settle( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Phase A 两题型进化时间窗重叠(并行),Phase D 按字母序 accept/reject 分派。""" + import core.evolution as core_evolution + + runner, state, pools = _build_runner(tmp_path) + diagnosis = SimpleNamespace( + skill_case_packs={ + # 故意逆字母序插入,验证排序不是插入序的巧合 + _TYPE_C: SimpleNamespace(task_type=_TYPE_C, failure_cases=[], success_cases=[]), + _TYPE_A: SimpleNamespace(task_type=_TYPE_A, failure_cases=[], success_cases=[]), + } + ) + records = {t: _record(t) for t in _TARGET_FILES} + outcomes = {_TYPE_A: _outcome(accepted=True), _TYPE_C: _outcome(accepted=False)} + + evolve_windows: dict[str, tuple[float, float]] = {} + + async def fake_evolve_single_skill( + llm, pack, skill_store, prompts, version, budget, threshold, **kwargs + ): + start = time.monotonic() + await asyncio.sleep(0.05) + evolve_windows[pack.task_type] = (start, time.monotonic()) + return records[pack.task_type] + + captured: dict[str, object] = {} + + async def fake_validate_skills_concurrent(**kwargs): + captured.update(kwargs) + # 逆字母序返回,验证 Phase D 落账顺序来自 sorted 而非 dict 插入序 + return { + _TYPE_C: outcomes[_TYPE_C], + _TYPE_A: outcomes[_TYPE_A], + } + + settle_calls: list[tuple[str, str]] = [] + runner._accept_skill = lambda task_type, *a: settle_calls.append(("accept", task_type)) + runner._record_rejected_skill = lambda buf, task_type, *a: settle_calls.append( + ("reject", task_type) + ) + + monkeypatch.setattr(core_evolution, "evolve_single_skill", fake_evolve_single_skill) + monkeypatch.setattr(runner_mod, "validate_skills_concurrent", fake_validate_skills_concurrent) + monkeypatch.setattr(runner_mod, "HarnessLog", _FakeHarnessLog) + monkeypatch.setattr(runner_mod, "write_gate_evidence", lambda *a, **k: None) + monkeypatch.setattr(runner_mod, "write_step_report", lambda *a, **k: None) + monkeypatch.setattr(runner_mod, "write_quadrant_pairs", lambda *a, **k: None) + monkeypatch.setattr(runner_mod, "_outcome_to_quadrant_pairs", lambda t, o: []) + monkeypatch.setattr(runner_mod, "_write_skip_report", lambda *a, **k: None) + + asyncio.run(runner._gate_batch_skills(1, 0, diagnosis, 3, pools, state)) + + # (a) 进化时间窗重叠 = gather 真并行(串行时前者 end <= 后者 start) + win_a, win_c = evolve_windows[_TYPE_A], evolve_windows[_TYPE_C] + assert win_a[0] < win_c[1] and win_c[0] < win_a[1], f"进化未并行: {evolve_windows}" + + # (b) Phase D 落账顺序 == sorted(题型),且 (c) accept/reject 分派与 outcome 一致 + assert settle_calls == [("accept", _TYPE_A), ("reject", _TYPE_C)] + + # Phase B 装配的 GateSpec 与 Phase C 共享 log 抽查 + specs = captured["specs"] + assert [s.task_type for s in specs] == sorted(_TARGET_FILES) + for spec in specs: + assert spec.target_file == _TARGET_FILES[spec.task_type] + assert spec.base_skill_content == "旧 skill 内容" + assert spec.candidate_content == records[spec.task_type].evolved_content + assert len(spec.units) == 1 + assert "_gate_" in spec.gate_run_prefix + assert isinstance(captured["log"], _FakeHarnessLog) + assert callable(captured["run_inference"]) From 0b839937dfd0d3912d2d136cc8eec202583d5bdb Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 17 Jul 2026 03:52:19 -0400 Subject: [PATCH 14/17] fix: predictions row carries arm run_id under shared gate_log; drain evolve gather on failure (algo #6) --- app/harness/inference.py | 4 ++ app/harness/runner.py | 18 +++--- tests/unit/test_gate_batch_parallel.py | 62 +++++++++++++++++++++ tests/unit/test_harness_inference.py | 16 +++--- tests/unit/test_inference_pair_aggregate.py | 8 +-- 5 files changed, 90 insertions(+), 18 deletions(-) diff --git a/app/harness/inference.py b/app/harness/inference.py index 42f58c9..a9a6136 100644 --- a/app/harness/inference.py +++ b/app/harness/inference.py @@ -409,7 +409,11 @@ async def _run_single_question( 返回: 预测结果字典（含 video_id, question_id, prediction, answer 等）。 """ + # run_id 必须显式入 record:HarnessLog.insert 缺省用**实例** run_id 填充, + # 连续并发 gate 共享单一 gate_log(实例 run_id 为 step 级)时,各臂行必须 + # 落自己的臂 run_id,否则 validate 回读 _load_run_rows(臂 run_id) 为空。 record: dict[str, Any] = { + "run_id": run_id, "video_id": qa.video_id, "question_id": qa.question_id, "task_type": qa.task_type, diff --git a/app/harness/runner.py b/app/harness/runner.py index 013ecb2..01c0c89 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -1370,13 +1370,17 @@ class Runner: rejected=state.rejected_buffer.get(task_type, []), ) - records = dict( - zip( - active_types, - await asyncio.gather(*[_evolve_one(t) for t in active_types]), - strict=True, - ) - ) + # 首异常先取消其余进化任务并排水再向上传播(与 validate_skills_concurrent + # 同款语义):避免失败后残留 in-flight LLM 任务与 pending task 警告。 + tasks = [asyncio.ensure_future(_evolve_one(t)) for t in active_types] + try: + evolved = await asyncio.gather(*tasks) + except BaseException: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + raise + records = dict(zip(active_types, evolved, strict=True)) # 无真实改动的题型照旧写 skipped 后出队 gated: dict[str, EvolutionRecord] = {} diff --git a/tests/unit/test_gate_batch_parallel.py b/tests/unit/test_gate_batch_parallel.py index 6ff1725..2708e36 100644 --- a/tests/unit/test_gate_batch_parallel.py +++ b/tests/unit/test_gate_batch_parallel.py @@ -236,3 +236,65 @@ def test_gate_batch_parallel_evolve_and_alphabetical_settle( assert "_gate_" in spec.gate_run_prefix assert isinstance(captured["log"], _FakeHarnessLog) assert callable(captured["run_inference"]) + + +# --------------------------------------------------------------------------- +# 共享 gate_log 的 run_id 契约(真 SQLite,Codex 质量审 C1): +# HarnessLog.insert 缺省用实例 run_id 填充;record 自带 run_id 必须覆盖它, +# 否则连续并发 gate 下所有臂的 predictions 会落成 step 级 run_id, +# validate 按臂 run_id 回读为空 → gate 静默废掉。 +# --------------------------------------------------------------------------- + + +def test_harness_log_insert_record_run_id_overrides_instance(tmp_path: Path) -> None: + """record 自带 run_id 覆盖实例 run_id;缺省时回落实例 run_id(锁死 enriched.update 语义)。""" + from app.harness.inference import PREDICTIONS_SCHEMA + from app.harness.log import HarnessLog + + with HarnessLog(str(tmp_path / "harness.db"), "gate_e1_s0") as log: + log.create_table("predictions", PREDICTIONS_SCHEMA) + log.insert( + "predictions", + {"run_id": "run_e1_s0_gate_a_base_u0", "question_id": "q1", "prediction": "A"}, + ) + log.insert("predictions", {"question_id": "q2", "prediction": "B"}) + rows = log.query("SELECT question_id, run_id FROM predictions ORDER BY question_id") + assert [(r["question_id"], r["run_id"]) for r in rows] == [ + ("q1", "run_e1_s0_gate_a_base_u0"), + ("q2", "gate_e1_s0"), + ] + + +def test_inference_prediction_row_carries_arm_run_id(tmp_path: Path) -> None: + """经共享 gate_log 落库的 prediction 行 run_id 必须是臂 run_id 而非实例 run_id。 + + prompt_builder 抛错走异常路径即落库,无需真实 LLM; + 该路径与成功路径共用同一 record 初始 dict,契约一致。 + """ + from app.harness.inference import PREDICTIONS_SCHEMA, _run_single_question + from app.harness.log import HarnessLog + + def _broken_prompt_builder(qa: GeneratedQuestion) -> tuple[str, str]: + raise RuntimeError("测试注入:跳过真实推理") + + async def _noop_dispatch(tool_name: str, args: dict, *, context: dict) -> str: + raise NotImplementedError + + with HarnessLog(str(tmp_path / "harness.db"), "gate_e1_s0") as gate_log: + gate_log.create_table("predictions", PREDICTIONS_SCHEMA) + asyncio.run( + _run_single_question( + _question("q-arm", _TYPE_A), + llm=object(), # prompt_builder 先抛错,不会触达 + tool_dispatch_fn=_noop_dispatch, + prompt_builder=_broken_prompt_builder, + log=gate_log, + max_steps=3, + plugins=[], + run_id="run_e1_s0_gate_action-reasoning_cand_u0", + ) + ) + rows = gate_log.query("SELECT run_id, stop_reason FROM predictions") + assert len(rows) == 1 + assert rows[0]["run_id"] == "run_e1_s0_gate_action-reasoning_cand_u0" + assert rows[0]["stop_reason"] == "error" diff --git a/tests/unit/test_harness_inference.py b/tests/unit/test_harness_inference.py index 5481cb9..c00346c 100644 --- a/tests/unit/test_harness_inference.py +++ b/tests/unit/test_harness_inference.py @@ -117,8 +117,9 @@ def harness_log(tmp_path: Any, request: Any) -> HarnessLog: """创建临时 HarnessLog 实例。 使用 test 节点名称的 hash 作为 db 文件名，避免冲突。 - run_id 固定为 "test-run"，实际 run_inference 中传入的 run_id - 由 HarnessLog.insert 自动覆盖为 HarnessLog 构造时的值。 + 实例 run_id 固定为 "test-run"；predictions 行的 run_id 由 inference + record 显式携带（run_inference 传入值），不回落实例 run_id—— + 连续并发 gate 共享单一 HarnessLog 的契约。 """ db_name = f"harness_{id(request)}.db" db_path = str(tmp_path / db_name) @@ -528,8 +529,9 @@ class TestPredictionAlwaysWritten: assert result.correct == 0 assert result.stop_reason_counts.get("error") == 1 - # 验证 DB 中的记录（HarnessLog.insert 使用构造时的 run_id） - rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("test-run",)) + # 验证 DB 中的记录（record 显式携带 run_inference 的 run_id， + # 不再回落 HarnessLog 实例 run_id——连续并发 gate 共享 log 的契约） + rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("run-error",)) assert len(rows) == 1 assert rows[0]["stop_reason"] == "error" assert rows[0]["prediction"] is None @@ -553,8 +555,8 @@ class TestPredictionAlwaysWritten: ) assert result.total == 1 - # HarnessLog.insert 使用构造时的 run_id - rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("test-run",)) + # record 显式携带 run_inference 的 run_id（共享 log 契约） + rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("run-parse-err",)) assert len(rows) == 1 assert rows[0]["prediction"] is None @@ -612,7 +614,7 @@ class TestNonScalarPrediction: ) assert result.total == 1 - rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("test-run",)) + rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("run-nonscalar",)) assert len(rows) == 1 # prediction 被 JSON 序列化为字符串，不再是 Python list assert rows[0]["prediction"] == '["B"]' diff --git a/tests/unit/test_inference_pair_aggregate.py b/tests/unit/test_inference_pair_aggregate.py index 7d07aa8..e806eda 100644 --- a/tests/unit/test_inference_pair_aggregate.py +++ b/tests/unit/test_inference_pair_aggregate.py @@ -332,8 +332,8 @@ class TestRunInferencePairEndToEnd: assert result.total == 1 assert result.correct == 1 - # 逐题溯源：predictions 表两条 record 都在 - rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("test-run",)) + # 逐题溯源：predictions 表两条 record 都在（record 显式携带传入的 run_id） + rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("run-pair-e2e",)) qids = {r["question_id"] for r in rows} assert qids == {"po", "pm"} @@ -360,6 +360,6 @@ class TestRunInferencePairEndToEnd: ) assert result.total == 1 # single 存活，孤儿剔除 - # 逐题溯源：孤儿题仍逐题落库（推理不变） - rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("test-run",)) + # 逐题溯源：孤儿题仍逐题落库（推理不变；record 显式携带传入的 run_id） + rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("run-orphan-e2e",)) assert {r["question_id"] for r in rows} == {"s1", "po"} From 8958eee11b1fb4c9e6c890802bd93da7d919cf21 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 17 Jul 2026 04:40:14 -0400 Subject: [PATCH 15/17] refactor: remove block-sequential gate path and gate_block knob (algo #6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit config/train_videomme.yaml 同时收录待入库的实验配置变更(run_id v2 / concurrency 32 / batch_size 40)。tests/integration/test_v3_contract_e2e.py 的 run_id 断言按 Task 5 显式契约同步修正(原断言依赖旧隐式实例注入)。 --- app/harness/checkpoint.py | 1 - app/harness/config.py | 13 +- app/harness/validate.py | 435 +--------------- config/default.yaml | 1 - config/question_gen_180_补.yaml | 1 - config/question_gen_360.yaml | 1 - config/train_action_recognition.yaml | 1 - config/train_ar30.yaml | 1 - config/train_videomme.yaml | 11 +- tests/integration/test_checkpoint_pair.py | 1 - tests/integration/test_v3_contract_e2e.py | 2 +- ..._block_unit.py => test_gate_unit_scope.py} | 146 +++--- tests/unit/test_harness_checkpoint.py | 2 - tests/unit/test_harness_config.py | 17 +- tests/unit/test_harness_pools.py | 3 - tests/unit/test_harness_runner.py | 1 - tests/unit/test_harness_validate.py | 475 ++++++++---------- tests/unit/test_runner_diag_tree_inject.py | 1 - 18 files changed, 322 insertions(+), 791 deletions(-) rename tests/unit/{test_gate_block_unit.py => test_gate_unit_scope.py} (72%) diff --git a/app/harness/checkpoint.py b/app/harness/checkpoint.py index c0849d6..f892962 100644 --- a/app/harness/checkpoint.py +++ b/app/harness/checkpoint.py @@ -58,7 +58,6 @@ _DECISION_KEYS = ( "gate_delta_min", "gate_lambda_dir", "gate_e_rollback", - "gate_block", "gate_n_max", "gate_p_low", "gate_p_high", diff --git a/app/harness/config.py b/app/harness/config.py index 824735d..cc24033 100644 --- a/app/harness/config.py +++ b/app/harness/config.py @@ -70,14 +70,13 @@ class RunConfig: gate_delta_min: 最小点估计效应量下限（承接旧 margin 语义）。 gate_lambda_dir: Wald 方向拒绝的对数似然比阈值（必须为负）。 gate_e_rollback: 试用期对称回滚门（回滚 e 值门槛）。 - gate_block: 块序贯验证的块大小（=推理并发度，块内跑满）。 gate_n_max: 单次 gate 消耗的题数上限。 gate_p_low: 信息量阶梯 p-hat 保留区间下界（剔除必错零信息题）。 gate_p_high: 信息量阶梯 p-hat 保留区间上界（剔除必对零信息题）。 gate_probe_quota: 冷启动探针集比例（全错题中插尾的比例）。 gate_gamma_decay: 逐题正确率估计 p-hat 的 EMA 衰减系数。 gate_cooldown_steps: 回滚后该题型跳过进化的冷却 step 数。 - gate_guard_err: gate 内跨块累计 INFRA 错误率护栏。 + gate_guard_err: gate 内累计 INFRA 错误率护栏。 skill_update_mode: skill 进化模式，"patch"（局部 edit）/ "rewrite"（整篇重写）。 appendix_consolidate_threshold: appendix note 条数达此值触发 LLM consolidation。 run_id: diagnose/evolve 模式要分析的运行 ID，默认空字符串。 @@ -125,7 +124,6 @@ class RunConfig: gate_delta_min: float gate_lambda_dir: float gate_e_rollback: float - gate_block: int gate_n_max: int gate_p_low: float gate_p_high: float @@ -361,7 +359,7 @@ def _validate_gate_thresholds(config: RunConfig) -> None: def _validate_gate_ladder(config: RunConfig) -> None: - """校验 CE-Gate 信息量阶梯与块序贯参数。 + """校验 CE-Gate 信息量阶梯参数。 参数: config: 待校验的配置实例。 @@ -369,11 +367,8 @@ def _validate_gate_ladder(config: RunConfig) -> None: 异常: ValueError: 任一阶梯参数不合法。 """ - if config.gate_block <= 0 or config.gate_n_max < config.gate_block: - raise ValueError( - f"需 0 < gate_block <= gate_n_max，" - f"实际: block={config.gate_block}, n_max={config.gate_n_max}" - ) + if config.gate_n_max <= 0: + raise ValueError(f"需 gate_n_max > 0，实际: n_max={config.gate_n_max}") if not (0 <= config.gate_p_low < config.gate_p_high <= 1): raise ValueError( f"需 0 <= gate_p_low < gate_p_high <= 1，" diff --git a/app/harness/validate.py b/app/harness/validate.py index b2228c0..f4644d8 100644 --- a/app/harness/validate.py +++ b/app/harness/validate.py @@ -1,15 +1,15 @@ -"""async 块序贯验证编排 — CE-Gate 局部验证的唯一独立子编排器。 +"""async 连续并发 gate 验证编排 — CE-Gate 局部验证的唯一独立子编排器。 -从 TRM4 core/harness/validate.py (626 行) 迁移，重大重构： -- 同步 → async（run_inference 注入为 async callable） -- _classify_quadrants → core.evolution.classify_quadrants 纯函数 -- 配对逻辑 → 复用 core.evolution.pair_block + 本地证据行组装 -- _load_run_rows / _candidate_correctness_from_db → 共享 log.query() -- materialize_candidate_skill 保持同步（纯文件操作） +多题型全部 (单元, 臂) 任务共享题槽并发（validate_skills_concurrent）， +统计推进不按到达序，而按预声明的阶梯序前缀消费（_advance_prefix）： +base 臂缓存命中瞬间返回、cand 臂必新鲜跑，两臂延迟不对称，按到达序判定 +会系统性偏向早到翻转；前缀消费把判定顺序钉回阶梯序，anytime-valid 无条件 +成立（核心算法保真 #6，语义修订：块序贯 → 阶梯序前缀逐对序贯）。 -基线与候选在同一阶梯前缀上逐块配对，只数翻转（基线错→候选对 = W， -基线对→候选错 = L），每块结束调 gate_decision 做四出口判定。 -基线侧逐题对错走 BaselineCache 内容寻址缓存，miss 才新鲜跑。 +基线与候选在同一阶梯前缀上逐单元配对，只数翻转（基线错→候选对 = W， +基线对→候选错 = L），每消费一个单元调一次 gate_decision 做四出口判定， +过线即冻结、τ 之后的 in-flight 结果整体丢弃。基线侧单元级对错走 +BaselineCache 内容寻址缓存，miss 才新鲜跑；INFRA 单元不写缓存、从配对剔除。 判定逻辑全部在 core/evolution/gate，本模块只负责推理编排与证据收集。 """ @@ -26,7 +26,7 @@ from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable from loguru import logger from app.harness.gate_ladder import BaselineCache, skill_hash -from app.harness.question_units import build_units, flatten_units, unit_correctness_view +from app.harness.question_units import build_units, unit_correctness_view from core.evolution import ( INFRA_STOP_REASONS, GateParams, @@ -68,7 +68,7 @@ class RunInferenceFn(Protocol): 调用方（runner）负责绑定 llm、tool_dispatch_fn、prompt_builder、 log、concurrency、max_steps、skill_mode 等共享依赖。 - validate 侧只传 questions、run_id、skills_dir 三个逐块变化的参数。 + validate 侧只传 questions、run_id、skills_dir 三个逐任务变化的参数。 """ async def __call__( @@ -85,21 +85,6 @@ class RunInferenceFn(Protocol): # --------------------------------------------------------------------------- -@dataclass(frozen=True) -class InferenceRunConfig: - """一次推理运行的配置三元组，把"如何跑推理"内聚成一组。 - - 字段: - concurrency: 推理并发度。 - max_steps: 单题最大推理步数。 - skill_mode: 推理 skill 模式（"auto" / "manual" / "none"）。 - """ - - concurrency: int - max_steps: int - skill_mode: str - - @dataclass class ValidationOutcome: """CE-Gate 局部验证结果：三态动作 + e-process 证据（单元口径）+ 逐题溯源对错。 @@ -260,23 +245,6 @@ def _infra_question_ids_from_db( } -def _count_infra_units(units: list[QuestionUnit], infra_qids: set[str]) -> int: - """统计含 INFRA record 的 unit 数（一个 unit 任一题 INFRA 即计 1）。 - - 使护栏分子与分母（r.total，unit 粒度）同口径：AR pair 一 unit 含两 record， - 逐 record 计数会放大分子致 gate_guard_err 误触发，破坏 unit 粒度一致性 - （核心算法保真 #5/#6）。 - - 参数: - units: 当前块的单元列表（single 或 AR pair）。 - infra_qids: 本 run 中 stop_reason 属 INFRA 故障族的 question_id 集合。 - - 返回: - 含至少一题 INFRA 的 unit 数。 - """ - return sum(1 for u in units if any(q.question_id in infra_qids for q in u.questions)) - - def _candidate_correctness_from_db( log: HarnessLog, run_id: str, @@ -296,164 +264,13 @@ def _candidate_correctness_from_db( return {q.question_id: rows.get(q.question_id, {}).get("_correct", False) for q in chunk} -# --------------------------------------------------------------------------- -# 块级 async 函数 -# --------------------------------------------------------------------------- - - -async def _resolve_baseline_block( - units: list[QuestionUnit], - task_type: str, - s_hash: str, - prompts_version: str, - baseline_cache: BaselineCache, - base_skills_dir: Path, - run_inference: RunInferenceFn, - log: HarnessLog, - run_id: str, -) -> tuple[dict[str, bool], list[QuestionUnit], int, int]: - """基线侧处理一个块：缓存优先（unit 键），miss 的单元新鲜跑基线版本并回写缓存。 - - 缓存以 unit_id 为键、存单元级对错（AR pair 双向 AND 折叠后一个布尔）。 - miss 的单元展开为逐题送推理，读回逐题预测后经 unit_correctness_view 折叠成 - 单元级对错再写缓存（核心算法保真 #5）。逐题 predictions 仍逐题落库溯源。 - - INFRA 隔离（算法 #6）：miss 单元内**任一题** stop_reason ∈ {error, parse_error} - 即判定该单元为 INFRA 故障——**不写 BaselineCache**（否则瞬时故障永久污染基线 - 快照）、**不入 b_units**、并从返回的有效单元集中剔除，避免污染 W/L 翻转与配对。 - 命中缓存的单元恒为有效（此前已成功验证过）。 - - 参数: - units: 当前块的单元列表（single 或 AR pair）。 - task_type: 当前验证题型（缓存键成分）。 - s_hash: 基线侧生效 skill 的内容哈希（缓存键成分）。 - prompts_version: 当前 prompts 版本（缓存键成分）。 - baseline_cache: 基线侧单元级对错缓存（键含 unit_id）。 - base_skills_dir: 基线 skills 版本目录。 - run_inference: 注入的 async 推理函数。 - log: HarnessLog 共享实例（推理后读预测）。 - run_id: 本块基线 run_id。 - - 返回: - (b_units, valid_units, errors_inc, denom_inc)：块内有效 unit_id -> 基线单元 - 对错、剔除 INFRA 后的有效单元列表、本块新增的 INFRA error 计数与推理题次 - 分母增量（全命中时为 0, 0）。 - """ - miss_units = [ - u - for u in units - if baseline_cache.get(task_type, s_hash, prompts_version, u.unit_id) is None - ] - errors_inc = 0 - denom_inc = 0 - infra_qids: set[str] = set() - if miss_units: - miss_questions = flatten_units(miss_units) - r_b = await run_inference(miss_questions, run_id=run_id, skills_dir=base_skills_dir) - infra_qids = _infra_question_ids_from_db(log, r_b.run_id, miss_questions) - # 护栏分子与分母（r.total，unit 粒度）同口径：含 INFRA record 的 unit 计 1， - # 避免 AR pair（一 unit 两 record）逐 record 计数放大分子致误触发；仍涵盖 - # error + parse_error（_infra_question_ids_from_db 口径），parse_error 风暴不被绕过。 - errors_inc = _count_infra_units(miss_units, infra_qids) - denom_inc = r_b.total - fresh_per_q = _candidate_correctness_from_db(log, r_b.run_id, miss_questions) - fresh_units = unit_correctness_view(miss_units, fresh_per_q) - # 只回写非 INFRA 单元；INFRA 单元不入缓存（不永久污染基线快照） - for u in miss_units: - if any(q.question_id in infra_qids for q in u.questions): - continue - baseline_cache.put(task_type, s_hash, prompts_version, u.unit_id, fresh_units[u.unit_id]) - - valid_units = [ - u for u in units if not any(q.question_id in infra_qids for q in u.questions) - ] - - b_units: dict[str, bool] = {} - for u in valid_units: - val = baseline_cache.get(task_type, s_hash, prompts_version, u.unit_id) - assert val is not None, f"基线缓存补齐后仍有 miss: unit={u.unit_id} run_id={run_id}" - b_units[u.unit_id] = val - return b_units, valid_units, errors_inc, denom_inc - - -async def _run_candidate_block( - units: list[QuestionUnit], - cand_dir: Path, - run_inference: RunInferenceFn, - log: HarnessLog, - run_id: str, -) -> tuple[dict[str, bool], int, int]: - """候选侧处理一个块：单元展开为逐题全块新鲜跑候选版本并从 db 读逐题对错。 - - 返回逐题对错映射（question_id -> bool），折叠为单元视图交由调用方完成， - 逐题结果同时用于 candidate_correctness 溯源与二轨 correctness 合并。 - - 参数: - units: 当前块的单元列表。 - cand_dir: 已物化的候选 skills 目录。 - run_inference: 注入的 async 推理函数。 - log: HarnessLog 共享实例（推理后读预测）。 - run_id: 本块候选 run_id。 - - 返回: - (c_per_q, errors_inc, denom_inc)：块内 question_id -> 候选对错。 - """ - questions = flatten_units(units) - r_c = await run_inference(questions, run_id=run_id, skills_dir=cand_dir) - c_per_q = _candidate_correctness_from_db(log, r_c.run_id, questions) - infra_qids = _infra_question_ids_from_db(log, r_c.run_id, questions) - # 护栏分子与分母（r.total，unit 粒度）同口径：含 INFRA record 的 unit 计 1 - # （见 _count_infra_units），涵盖 error + parse_error。 - errors_inc = _count_infra_units(units, infra_qids) - return c_per_q, errors_inc, r_c.total - - -def _build_evidence_rows( - units: list[QuestionUnit], - b_units: dict[str, bool], - c_units: dict[str, bool], - task_type: str, - block_idx: int, -) -> list[dict]: - """组装一个块的 gate_evidence 单元级证据行。 - - 证据行按 unit 口径（question_id 字段存 unit_id、correct 存单元级对错）， - 与 e-process 判定同粒度；逐题预测明细仍在 predictions 表逐题溯源。 - e_value 留 None 待块判定后回填，stop_reason 留空串待终态回填。 - - 参数: - units: 当前块的单元列表。 - b_units: 块内 unit_id -> 基线单元对错。 - c_units: 块内 unit_id -> 候选单元对错。 - task_type: 当前验证题型。 - block_idx: 当前块序号。 - - 返回: - 单元级证据行列表。 - """ - return [ - { - "question_id": u.unit_id, - "task_type": task_type, - # 落库列已更名 ladder_rank(阶梯序号);旧块路径此处值仍为块号, - # 仅键名对齐 gate_evidence 表结构以保持落库兼容。 - "ladder_rank": block_idx, - "baseline_correct": b_units[u.unit_id], - "candidate_correct": c_units[u.unit_id], - "e_value": None, - "stop_reason": "", - } - for u in units - ] - - # --------------------------------------------------------------------------- # INFRA 护栏 # --------------------------------------------------------------------------- def _check_infra_guard(errors: int, infra_denom: int, gate_guard_err: float) -> None: - """跨块累计 INFRA 错误率护栏：分母 >=10 且超阈值时 raise。 + """累计 INFRA 错误率护栏：分母 >=10 且超阈值时 raise。 参数: errors: 两侧累计 error 计数。 @@ -484,13 +301,13 @@ def _finalize_outcome( evidence_rows: list[dict], task_type: str, ) -> ValidationOutcome: - """将块循环终态判定组装为 ValidationOutcome。 + """将终态判定组装为 ValidationOutcome。 四象限/准确率/W/L 均按单元口径（base_obs/cand_obs 为 unit_id -> bool）， candidate_correctness 独立保留逐题溯源（供 runner 二轨合并进 state.correctness）。 参数: - verdict: 最后一块的 gate 判定结果。 + verdict: 终态 gate 判定结果。 w: 累计 W（基线错→候选对单元翻转）。 l: 累计 L（基线对→候选错单元翻转）。 n_used: 已消费的阶梯单元数。 @@ -576,228 +393,6 @@ def _ladder_units(ladder_items: list[GeneratedQuestion]) -> list[QuestionUnit]: return units -async def _run_local_validation( - workspace_dir: Path, - cand_dir: Path, - base_skills_version: str, - task_type: str, - base_skill_content: str, - units: list[QuestionUnit], - gate_params: GateParams, - gate_block: int, - gate_guard_err: float, - baseline_cache: BaselineCache, - prompts_version: str, - run_inference: RunInferenceFn, - log: HarnessLog, - gate_run_prefix: str, -) -> ValidationOutcome: - """块序贯循环主体：逐块基线（缓存优先）/候选按单元配对推理，块间 e-process 判定。 - - 按 gate_block 切**单元**前缀（AR pair 整锁在同一块，不跨块拆分），每块先补齐 - 基线侧缓存 miss（新鲜跑基线版本并按 unit_id 写 BaselineCache），再全块跑候选， - 折叠成单元视图后配对累计 W/L 调 gate_decision；非 continue 即早停。单元尽时 - 最后一块的判定即终态（n_remaining=0 走 provisional/inertia 分支），无循环外补判。 - - 参数: - workspace_dir: Workspace 根目录。 - cand_dir: 已物化的候选 skills 目录。 - base_skills_version: 基线 skills 版本名。 - task_type: 当前验证题型。 - base_skill_content: 基线侧生效 skill 全文（skill_hash 作缓存键成分）。 - units: 已截断到 gate_n_max 的阶梯单元序（single 或 AR pair）。 - gate_params: e-process 判据阈值组。 - gate_block: 块大小（单位为**单元数**）。 - gate_guard_err: 跨块累计 INFRA 错误率护栏（分母 >=10 才触发）。 - baseline_cache: 基线侧单元级对错缓存（键含 unit_id）。 - prompts_version: 当前 prompts 版本（缓存键成分）。 - run_inference: 注入的 async 推理函数。 - log: HarnessLog 共享实例。 - gate_run_prefix: 块 run_id 前缀（含 "_gate_" 标记）。 - - 返回: - ValidationOutcome。 - - 关键实现: - INFRA 护栏跨块累计基线+候选两侧的 error 计数，分母（总推理题次，仍逐题计） - >=10 且错误率超 gate_guard_err 时直接 raise，避免坏批次污染判定。 - """ - w = 0 - l = 0 # noqa: E741 - n_used = 0 - n_excluded = 0 # 累计被 INFRA 隔离剔除的单元数（从阶梯分母扣除） - errors = 0 - infra_denom = 0 - evidence_rows: list[dict] = [] - base_obs: dict[str, bool] = {} - cand_obs: dict[str, bool] = {} - candidate_per_q: dict[str, bool] = {} - s_hash = skill_hash(base_skill_content) - base_skills_dir = workspace_dir / "skills" / base_skills_version - unit_chunks = [units[i : i + gate_block] for i in range(0, len(units), gate_block)] - n_plan = len(units) - verdict: GateVerdict | None = None - - for block_idx, unit_chunk in enumerate(unit_chunks): - # Phase 1: 基线侧（缓存优先，miss 新鲜跑，INFRA 单元剔除） - b_units, valid_chunk, err_b, den_b = await _resolve_baseline_block( - units=unit_chunk, - task_type=task_type, - s_hash=s_hash, - prompts_version=prompts_version, - baseline_cache=baseline_cache, - base_skills_dir=base_skills_dir, - run_inference=run_inference, - log=log, - run_id=f"{gate_run_prefix}_b{block_idx}_base", - ) - # 本块全 INFRA：无有效单元可配对——候选无需空跑，仅把基线侧错误计入护栏后 - # 累计剔除数进入下一块（护栏仍能在整轮 INFRA 错误率超阈值时熔断）。 - n_excluded += len(unit_chunk) - len(valid_chunk) - if not valid_chunk: - errors += err_b - infra_denom += den_b - _check_infra_guard(errors, infra_denom, gate_guard_err) - continue - - # 候选侧只跑基线侧判定有效（非 INFRA）的单元，保证配对 unit_ids 两侧一致 - c_per_q, err_c, den_c = await _run_candidate_block( - units=valid_chunk, - cand_dir=cand_dir, - run_inference=run_inference, - log=log, - run_id=f"{gate_run_prefix}_b{block_idx}_cand", - ) - - # Phase 2: INFRA 护栏（跨块累计，分母 >=10 才触发）——写缓存前置于此已由 - # _resolve_baseline_block 保证 INFRA 单元不落缓存，此处仅做整轮错误率熔断。 - errors += err_b + err_c - infra_denom += den_b + den_c - _check_infra_guard(errors, infra_denom, gate_guard_err) - - # Phase 3: 折叠成单元视图 + 配对 + 证据行 + 块间判定（均用有效单元） - c_units = unit_correctness_view(valid_chunk, c_per_q) - candidate_per_q.update(c_per_q) - unit_ids = [u.unit_id for u in valid_chunk] - pair_result = pair_block(b_units, c_units, unit_ids) - for uid, (b, c) in pair_result.observed.items(): - base_obs[uid] = b - cand_obs[uid] = c - - block_rows = _build_evidence_rows(valid_chunk, b_units, c_units, task_type, block_idx) - - w += pair_result.w - l += pair_result.l # noqa: E741 - n_used += len(valid_chunk) - # 阶梯剩余按扣除 INFRA 后的有效分母计：n_remaining = (n_plan - n_excluded) - n_used - verdict = gate_decision(w, l, n_used, (n_plan - n_excluded) - n_used, params=gate_params) - - for row in block_rows: - row["e_value"] = verdict.e_value - evidence_rows.extend(block_rows) - - if verdict.decision != "continue": - break - - # verdict 仍为 None ⟺ 全部单元被 INFRA 排除（空 ladder 已在入口拒绝）。 - # 明确失败，避免落到误导性的"空阶梯"断言而无法定位为 INFRA 原因。 - if verdict is None: - raise RuntimeError("gate 阶梯所有 unit 被判为 INFRA 排除，无法验证（检查推理基础设施）") - # 最后一块判定即终态（n_remaining=0 → provisional/inertia） - return _finalize_outcome( - verdict=verdict, - w=w, - l=l, - n_used=n_used, - n_plan=n_plan, - base_obs=base_obs, - cand_obs=cand_obs, - candidate_per_q=candidate_per_q, - evidence_rows=evidence_rows, - task_type=task_type, - ) - - -async def validate_skill_local( - workspace_dir: Path, - base_skills_version: str, - task_type: str, - target_file: str, - candidate_content: str, - base_skill_content: str, - ladder_items: list[GeneratedQuestion], - gate_params: GateParams, - gate_block: int, - gate_n_max: int, - gate_guard_err: float, - baseline_cache: BaselineCache, - prompts_version: str, - run_inference: RunInferenceFn, - log: HarnessLog, - gate_run_prefix: str, -) -> ValidationOutcome: - """块序贯配对验证：阶梯出题，基线/候选逐块配对，e-process 四出口早停。 - - 参数: - workspace_dir: workspace 根目录。 - base_skills_version: 基线 skills 版本名（候选物化复制源）。 - task_type: 待验证题型。 - target_file: fallback 解析后该题型的真实生效 skill 文件名 - （record.target_file，可能是共享 default-strategy.md）； - 候选物化写此文件，与 accept 路径同源。 - candidate_content: 候选 skill 全文。 - base_skill_content: 基线侧该题型解析后生效 skill 文件全文 - （skill_hash(base_skill_content) 作 BaselineCache 键成分）。 - ladder_items: 阶梯序题目列表（已排除本 step 案例包题）。 - gate_params: e-process 判据阈值组。 - gate_block: 块大小（单位为**单元数**，AR pair 整锁不跨块拆）。 - gate_n_max: 单 gate 单元数上限（阶梯截断到此数量个单元）。 - gate_guard_err: 跨块累计 INFRA 错误率护栏（分母 >=10 才触发）。 - baseline_cache: 基线侧单元级对错缓存（键含 unit_id）。 - prompts_version: 当前 prompts 版本（缓存键成分）。 - run_inference: 注入的 async 推理函数（RunInferenceFn 协议）。 - log: HarnessLog 共享实例（供 DB 回读逐题对错）。 - gate_run_prefix: gate 内推理 run_id 前缀，必须含 "_gate_" - （防泄露过滤靠它识别）。块 run_id = f"{prefix}_b{block_idx}_{arm}"。 - - 返回: - ValidationOutcome。单元级证据记入 outcome.evidence_rows 随结果返回， - gate_evidence 落库由调用方（runner）负责。 - """ - if "_gate_" not in gate_run_prefix: - raise ValueError(f"gate_run_prefix 必须含 '_gate_'（防泄露过滤依赖）: {gate_run_prefix!r}") - if not ladder_items: - raise ValueError(f"task_type={task_type} 阶梯为空，无法验证") - - # 阶梯题序聚合为单元并按信息阶梯序截断到 gate_n_max 个单元（AR pair 整锁不拆） - units = _ladder_units(ladder_items)[:gate_n_max] - cand_dir = materialize_candidate_skill( - workspace_dir, base_skills_version, target_file, candidate_content - ) - try: - return await _run_local_validation( - workspace_dir=workspace_dir, - cand_dir=cand_dir, - base_skills_version=base_skills_version, - task_type=task_type, - base_skill_content=base_skill_content, - units=units, - gate_params=gate_params, - gate_block=gate_block, - gate_guard_err=gate_guard_err, - baseline_cache=baseline_cache, - prompts_version=prompts_version, - run_inference=run_inference, - log=log, - gate_run_prefix=gate_run_prefix, - ) - finally: - try: - shutil.rmtree(cand_dir) - except OSError as e: - logger.warning("候选临时目录清理失败 {}: {}", cand_dir, e) - - # --------------------------------------------------------------------------- # 连续并发 gate:数据结构 + 前缀消费(algo #6 语义修订:块序贯 → 阶梯序前缀逐对序贯) # --------------------------------------------------------------------------- diff --git a/config/default.yaml b/config/default.yaml index 0048741..40c0105 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -42,7 +42,6 @@ harness: gate_delta_min: 0.02 gate_lambda_dir: -0.642 gate_e_rollback: 10.0 - gate_block: 8 gate_n_max: 40 gate_p_low: 0.05 gate_p_high: 0.95 diff --git a/config/question_gen_180_补.yaml b/config/question_gen_180_补.yaml index e0bd0c6..225701c 100644 --- a/config/question_gen_180_补.yaml +++ b/config/question_gen_180_补.yaml @@ -39,7 +39,6 @@ harness: gate_delta_min: 0.02 gate_lambda_dir: -0.642 gate_e_rollback: 10.0 - gate_block: 8 gate_n_max: 40 gate_p_low: 0.05 gate_p_high: 0.95 diff --git a/config/question_gen_360.yaml b/config/question_gen_360.yaml index 1673fe5..d785dcf 100644 --- a/config/question_gen_360.yaml +++ b/config/question_gen_360.yaml @@ -42,7 +42,6 @@ harness: gate_delta_min: 0.02 gate_lambda_dir: -0.642 gate_e_rollback: 10.0 - gate_block: 8 gate_n_max: 40 gate_p_low: 0.05 gate_p_high: 0.95 diff --git a/config/train_action_recognition.yaml b/config/train_action_recognition.yaml index a80a3db..d09da4a 100644 --- a/config/train_action_recognition.yaml +++ b/config/train_action_recognition.yaml @@ -22,7 +22,6 @@ harness: gate_delta_min: 0.02 gate_lambda_dir: -0.642 gate_e_rollback: 10.0 - gate_block: 8 gate_n_max: 40 gate_p_low: 0.05 gate_p_high: 0.95 diff --git a/config/train_ar30.yaml b/config/train_ar30.yaml index a5c406e..736852b 100644 --- a/config/train_ar30.yaml +++ b/config/train_ar30.yaml @@ -23,7 +23,6 @@ harness: gate_delta_min: 0.02 gate_lambda_dir: -0.642 gate_e_rollback: 10.0 - gate_block: 8 gate_n_max: 40 gate_p_low: 0.05 gate_p_high: 0.95 diff --git a/config/train_videomme.yaml b/config/train_videomme.yaml index 8be6868..a20b165 100644 --- a/config/train_videomme.yaml +++ b/config/train_videomme.yaml @@ -10,8 +10,8 @@ harness: workspace_dir: "workspaces/train-videomme" store_dir: store mode: train - run_id: train_videomme_v1 - concurrency: 24 + run_id: train_videomme_v2 + concurrency: 32 max_steps: 40 skill_mode: auto n_samples: 0 @@ -26,7 +26,6 @@ harness: gate_delta_min: 0.02 gate_lambda_dir: -0.642 gate_e_rollback: 10.0 - gate_block: 8 gate_n_max: 40 gate_p_low: 0.05 gate_p_high: 0.95 @@ -51,8 +50,10 @@ harness: # 可训练性预检（WP3）：val 单元 < eval_min_per_class 或 非test单元 < trainable_min_units 的题型剔除 eval_min_per_class: 2 trainable_min_units: 8 - # mini-batch - batch_size: 10 + # mini-batch —— 对齐 TRM4 正式实验 batch=40（sh --batch-size 40 覆盖 yaml 15 的最终生效值）： + # 8 可训题型 × 每型约 5 题/step，保住题型级诊断信号；同时 steps/epoch 180/40≈5， + # 进化/gate 验证轮数比 batch=10 少 4 倍。 + batch_size: 40 min_class_per_batch: 2 batch_correct_ratio: 0.5 momentum_samples: 20 diff --git a/tests/integration/test_checkpoint_pair.py b/tests/integration/test_checkpoint_pair.py index c62fd9b..81e42fc 100644 --- a/tests/integration/test_checkpoint_pair.py +++ b/tests/integration/test_checkpoint_pair.py @@ -125,7 +125,6 @@ class _FakeConfig: gate_delta_min: float = 0.02 gate_lambda_dir: float = -3.0 gate_e_rollback: float = 10.0 - gate_block: int = 4 gate_n_max: int = 40 gate_p_low: float = 0.1 gate_p_high: float = 0.9 diff --git a/tests/integration/test_v3_contract_e2e.py b/tests/integration/test_v3_contract_e2e.py index 939d14f..e187220 100644 --- a/tests/integration/test_v3_contract_e2e.py +++ b/tests/integration/test_v3_contract_e2e.py @@ -338,7 +338,7 @@ class TestInferenceUnitAggregationEndToEnd: def _assert_all_persisted(self, log: HarnessLog, questions: list[GeneratedQuestion]) -> None: """逐题溯源保留：含被剔除的孤儿题在内，每题仍逐题落 predictions。""" - rows = log.query("SELECT * FROM predictions WHERE run_id = ?", ("test-run",)) + rows = log.query("SELECT * FROM predictions WHERE run_id = ?", ("run-v3-contract",)) persisted = {r["question_id"] for r in rows} assert "orphan_o" in persisted, "孤儿题未逐题落库（逐题溯源被破坏）" assert persisted == {q.question_id for q in questions}, "逐题落库题数与输入不符" diff --git a/tests/unit/test_gate_block_unit.py b/tests/unit/test_gate_unit_scope.py similarity index 72% rename from tests/unit/test_gate_block_unit.py rename to tests/unit/test_gate_unit_scope.py index b58cdd5..8795eee 100644 --- a/tests/unit/test_gate_block_unit.py +++ b/tests/unit/test_gate_unit_scope.py @@ -1,8 +1,9 @@ -"""tests/unit/test_gate_block_unit.py — gate 块实际执行路径按 unit 跑。 +"""tests/unit/test_gate_unit_scope.py — gate 真实执行路径按 unit 口径跑。 -针对 app/harness/validate.py::validate_skill_local（真实 gate 执行路径）， -断言混格阶梯下 gate 块按 unit 口径运行：baseline_cache 键含 unit_id、 -n_used 按 unit 累加、pair_block 折叠 AR pair、逐题 predictions 仍溯源。 +迁移自块序贯版 test_gate_block_unit.py（载体 validate_skill_local，Task 6 删除）： +针对 app/harness/validate.py::validate_skills_concurrent（连续并发 gate 真实路径）， +断言混格阶梯下 gate 按 unit 口径运行：baseline_cache 键含 unit_id、n_used 按 +unit 累加、pair_block 折叠 AR pair、逐题 predictions 仍溯源。 核心算法保真 #5（信息阶梯 e-process 口径从 question_id 迁至 unit_id）。 """ @@ -15,7 +16,7 @@ import pytest from app.harness.gate_ladder import BaselineCache, skill_hash from app.harness.inference import PREDICTIONS_SCHEMA, InferenceResult from app.harness.log import HarnessLog -from app.harness.validate import _ladder_units, validate_skill_local +from app.harness.validate import GateSpec, _ladder_units, validate_skills_concurrent from core.evolution import GateParams from core.types import GeneratedQuestion @@ -136,6 +137,35 @@ def _make_mock_run_inference( return mock_fn, call_log +def _mk_spec(ladder: list[GeneratedQuestion]) -> GateSpec: + """由混格阶梯题序构造单题型 GateSpec（units 经 _ladder_units 聚合）。""" + return GateSpec( + task_type="temporal", + target_file="temporal.md", + candidate_content="improved skill", + base_skill_content="baseline skill content", + units=tuple(_ladder_units(ladder)), + gate_run_prefix="step1_gate_test", + ) + + +async def _run_gate(workspace: Path, spec: GateSpec, mock_fn, log: HarnessLog, cache, params): + """跑单 spec 的 validate_skills_concurrent 并返回该题型的 outcome。""" + outcomes = await validate_skills_concurrent( + workspace_dir=workspace, + base_skills_version="v1", + specs=[spec], + gate_params=params, + gate_guard_err=0.5, + baseline_cache=cache, + prompts_version="p1", + run_inference=mock_fn, + log=log, + concurrency=8, + ) + return outcomes[spec.task_type] + + class TestLadderUnits: """_ladder_units：阶梯题序聚合为单元并保持信息阶梯序。""" @@ -177,7 +207,7 @@ class TestLadderUnits: @pytest.mark.asyncio async def test_gate_n_used_counts_units_not_questions(tmp_path: Path) -> None: - """混格阶梯（1 pair + 2 single）→ n_used=3 单元，非 4 题。""" + """混格阶梯（1 pair + 2 single）→ n_used=3 单元，非 4 题（迁移自块序贯版）。""" workspace = _setup_workspace(tmp_path) log = _make_log(workspace) cache = BaselineCache(workspace / "baseline_cache.json") @@ -186,7 +216,7 @@ async def test_gate_n_used_counts_units_not_questions(tmp_path: Path) -> None: # 基线全错、候选全对 → 3 单元齐翻 W=3 baseline = {"p1_o": False, "p1_m": False, "s0": False, "s1": False} candidate = {"p1_o": True, "p1_m": True, "s0": True, "s1": True} - mock_fn, call_log = _make_mock_run_inference(log, baseline, candidate) + mock_fn, _ = _make_mock_run_inference(log, baseline, candidate) accept_params = GateParams( e_confirm=15.0, @@ -197,30 +227,14 @@ async def test_gate_n_used_counts_units_not_questions(tmp_path: Path) -> None: e_rollback=10.0, ) try: - outcome = await validate_skill_local( - workspace_dir=workspace, - base_skills_version="v1", - task_type="temporal", - target_file="temporal.md", - candidate_content="improved skill", - base_skill_content="baseline skill content", - ladder_items=ladder, - gate_params=accept_params, - gate_block=10, - gate_n_max=20, - gate_guard_err=0.5, - baseline_cache=cache, - prompts_version="p1", - run_inference=mock_fn, - log=log, - gate_run_prefix="step1_gate_test", - ) + outcome = await _run_gate(workspace, _mk_spec(ladder), mock_fn, log, cache, accept_params) # n_used 按 unit 计（3），W 按 unit 计（3） assert outcome.n_used == 3 assert outcome.w == 3 assert outcome.l == 0 - # 证据行按 unit 口径（3 行） + # 证据行按 unit 口径（3 行），ladder_rank 沿阶梯序连续 assert len(outcome.evidence_rows) == 3 + assert [r["ladder_rank"] for r in outcome.evidence_rows] == [0, 1, 2] # baseline_cache 键含 unit_id：pair 用 pair_id、single 用 question_id s_hash = skill_hash("baseline skill content") assert cache.get("temporal", s_hash, "p1", "p1") is False @@ -235,84 +249,62 @@ async def test_gate_n_used_counts_units_not_questions(tmp_path: Path) -> None: @pytest.mark.asyncio async def test_gate_pair_partial_flip_not_counted(tmp_path: Path) -> None: - """AR pair 候选仅单向翻(T,F)→单元仍错，W 不被单题污染。""" + """AR pair 候选仅单向翻(T,F)→单元仍错，W 不被单题污染（迁移自块序贯版）。 + + 前缀逐单元判定下 2 单元小阶梯会在首单元 futility 早停，观测不到 pair 语义； + 补 2 个 single 拉长阶梯：4 单元中 3 个 single 翻转 → W=3（pair 不计入）， + candidate_acc = 3/4。 + """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) cache = BaselineCache(workspace / "baseline_cache.json") - ladder = [*_pair("p1"), _single("s0")] + ladder = [*_pair("p1"), _single("s0"), _single("s1"), _single("s2")] - baseline = {"p1_o": False, "p1_m": False, "s0": False} - # pair 只翻一半（p1_o 对、p1_m 错）→ 单元 AND 仍错；s0 翻对 - candidate = {"p1_o": True, "p1_m": False, "s0": True} + baseline = {"p1_o": False, "p1_m": False, "s0": False, "s1": False, "s2": False} + # pair 只翻一半（p1_o 对、p1_m 错）→ 单元 AND 仍错；singles 全翻对 + candidate = {"p1_o": True, "p1_m": False, "s0": True, "s1": True, "s2": True} mock_fn, _ = _make_mock_run_inference(log, baseline, candidate) try: - outcome = await validate_skill_local( - workspace_dir=workspace, - base_skills_version="v1", - task_type="temporal", - target_file="temporal.md", - candidate_content="improved skill", - base_skill_content="baseline skill content", - ladder_items=ladder, - gate_params=_DEFAULT_GATE_PARAMS, - gate_block=10, - gate_n_max=20, - gate_guard_err=0.5, - baseline_cache=cache, - prompts_version="p1", - run_inference=mock_fn, - log=log, - gate_run_prefix="step1_gate_test", + outcome = await _run_gate( + workspace, _mk_spec(ladder), mock_fn, log, cache, _DEFAULT_GATE_PARAMS ) - # 只有 s0 单元翻转，pair 单元不计 W（保真 #5：不被 P/Q 单题污染） - assert outcome.w == 1 + # 只有 single 单元翻转，pair 单元不计 W（保真 #5：不被 P/Q 单题污染） + assert outcome.w == 3 assert outcome.l == 0 - assert outcome.n_used == 2 - # candidate_acc 分母按 unit（2 单元，1 对）→ 0.5 - assert outcome.candidate_acc == 0.5 + assert outcome.n_used == 4 + # candidate_acc 分母按 unit（4 单元，1 对）→ 3/4 + assert outcome.candidate_acc == 0.75 finally: log.close() @pytest.mark.asyncio async def test_gate_baseline_cache_hit_by_unit(tmp_path: Path) -> None: - """基线缓存按 unit_id 预填充 → 基线侧全命中不发起推理。""" + """基线缓存按 unit_id 预填充 → 基线侧全命中不发起推理（迁移自块序贯版）。 + + 阶梯补长到 4 单元避免首单元 futility 早停，覆盖 pair 与 single 两种 unit 键。 + """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) cache = BaselineCache(workspace / "baseline_cache.json") - ladder = [*_pair("p1"), _single("s0")] + ladder = [*_pair("p1"), _single("s0"), _single("s1"), _single("s2")] s_hash = skill_hash("baseline skill content") # 按 unit_id 预填充（pair→pair_id，single→question_id），全错 - cache.put("temporal", s_hash, "p1", "p1", False) - cache.put("temporal", s_hash, "p1", "s0", False) + for unit_id in ("p1", "s0", "s1", "s2"): + cache.put("temporal", s_hash, "p1", unit_id, False) - baseline = {"p1_o": False, "p1_m": False, "s0": False} - candidate = {"p1_o": True, "p1_m": True, "s0": True} + baseline = {"p1_o": False, "p1_m": False, "s0": False, "s1": False, "s2": False} + candidate = {"p1_o": True, "p1_m": True, "s0": True, "s1": True, "s2": True} mock_fn, call_log = _make_mock_run_inference(log, baseline, candidate) try: - outcome = await validate_skill_local( - workspace_dir=workspace, - base_skills_version="v1", - task_type="temporal", - target_file="temporal.md", - candidate_content="improved skill", - base_skill_content="baseline skill content", - ladder_items=ladder, - gate_params=_DEFAULT_GATE_PARAMS, - gate_block=10, - gate_n_max=20, - gate_guard_err=0.5, - baseline_cache=cache, - prompts_version="p1", - run_inference=mock_fn, - log=log, - gate_run_prefix="step1_gate_test", + outcome = await _run_gate( + workspace, _mk_spec(ladder), mock_fn, log, cache, _DEFAULT_GATE_PARAMS ) base_calls = [c for c in call_log if c["run_id"].endswith("_base")] assert base_calls == [], "unit 键全命中不应发起基线推理" - assert outcome.n_used == 2 + assert outcome.n_used == 4 finally: log.close() diff --git a/tests/unit/test_harness_checkpoint.py b/tests/unit/test_harness_checkpoint.py index e1fc36f..51d4dbc 100644 --- a/tests/unit/test_harness_checkpoint.py +++ b/tests/unit/test_harness_checkpoint.py @@ -171,7 +171,6 @@ class _FakeConfig: gate_delta_min: float = 0.02 gate_lambda_dir: float = -3.0 gate_e_rollback: float = 10.0 - gate_block: int = 4 gate_n_max: int = 40 gate_p_low: float = 0.1 gate_p_high: float = 0.9 @@ -306,7 +305,6 @@ class TestFingerprintStructuralVsDecision: "gate_delta_min", "gate_lambda_dir", "gate_e_rollback", - "gate_block", "gate_n_max", "gate_p_low", "gate_p_high", diff --git a/tests/unit/test_harness_config.py b/tests/unit/test_harness_config.py index 7a689bd..a54c943 100644 --- a/tests/unit/test_harness_config.py +++ b/tests/unit/test_harness_config.py @@ -50,7 +50,6 @@ def _valid_kwargs() -> dict: "gate_delta_min": 0.02, "gate_lambda_dir": -0.642, "gate_e_rollback": 10.0, - "gate_block": 8, "gate_n_max": 40, "gate_p_low": 0.05, "gate_p_high": 0.95, @@ -378,16 +377,16 @@ class TestGateValidation: with pytest.raises(ValueError, match="gate_lambda_dir"): _validate(cfg) - def test_block_exceeds_n_max_rejected(self) -> None: - """gate_block > gate_n_max 应抛出 ValueError。""" - cfg = _make_config(gate_block=50, gate_n_max=40) - with pytest.raises(ValueError, match="gate_block"): + def test_n_max_zero_rejected(self) -> None: + """gate_n_max <= 0 应抛出 ValueError（迁移自块序贯版 gate_block 校验）。""" + cfg = _make_config(gate_n_max=0) + with pytest.raises(ValueError, match="gate_n_max"): _validate(cfg) - def test_block_zero_rejected(self) -> None: - """gate_block <= 0 应抛出 ValueError。""" - cfg = _make_config(gate_block=0) - with pytest.raises(ValueError, match="gate_block"): + def test_n_max_negative_rejected(self) -> None: + """gate_n_max 为负也应报错。""" + cfg = _make_config(gate_n_max=-1) + with pytest.raises(ValueError, match="gate_n_max"): _validate(cfg) def test_p_low_exceeds_p_high_rejected(self) -> None: diff --git a/tests/unit/test_harness_pools.py b/tests/unit/test_harness_pools.py index 48cc3d6..8c17131 100644 --- a/tests/unit/test_harness_pools.py +++ b/tests/unit/test_harness_pools.py @@ -327,7 +327,6 @@ class TestBuildOrLoadPoolsFrozen: gate_delta_min=0.02, gate_lambda_dir=-0.642, gate_e_rollback=10.0, - gate_block=8, gate_n_max=40, gate_p_low=0.05, gate_p_high=0.95, @@ -881,7 +880,6 @@ class TestRunHoldoutEvalConfig: gate_delta_min=0.02, gate_lambda_dir=-0.642, gate_e_rollback=10.0, - gate_block=8, gate_n_max=40, gate_p_low=0.05, gate_p_high=0.95, @@ -932,7 +930,6 @@ class TestRunHoldoutEvalConfig: gate_delta_min=0.02, gate_lambda_dir=-0.642, gate_e_rollback=10.0, - gate_block=8, gate_n_max=40, gate_p_low=0.05, gate_p_high=0.95, diff --git a/tests/unit/test_harness_runner.py b/tests/unit/test_harness_runner.py index 44f0711..7d88af5 100644 --- a/tests/unit/test_harness_runner.py +++ b/tests/unit/test_harness_runner.py @@ -840,7 +840,6 @@ class TestRunnerFactoryInjection: "gate_delta_min": 0.02, "gate_lambda_dir": -0.642, "gate_e_rollback": 10.0, - "gate_block": 8, "gate_n_max": 40, "gate_p_low": 0.05, "gate_p_high": 0.95, diff --git a/tests/unit/test_harness_validate.py b/tests/unit/test_harness_validate.py index 97fd936..052af8a 100644 --- a/tests/unit/test_harness_validate.py +++ b/tests/unit/test_harness_validate.py @@ -1,7 +1,9 @@ """tests/unit/test_harness_validate.py — app/harness/validate.py 的单元测试。 -覆盖：数据类型字段、materialize 物化与清理、async validate_skill_local -（accept/reject/prefix 校验/INFRA 护栏/缓存命中/最后一块终态）。 +覆盖：数据类型字段、materialize 物化与清理、async validate_skills_concurrent +（accept/reject/prefix 校验/INFRA 护栏/缓存命中/题尽终态）。async 用例迁移自 +块序贯版（validate_skill_local，Task 6 删除）：载体换连续并发 gate，语义断言 +保留；前缀逐单元判定使早停点比旧块判定更早（见各用例 docstring 的数值推导）。 """ from __future__ import annotations @@ -14,10 +16,12 @@ from app.harness.gate_ladder import BaselineCache, skill_hash from app.harness.inference import PREDICTIONS_SCHEMA, InferenceResult from app.harness.log import HarnessLog from app.harness.validate import ( + GateSpec, Probation, ValidationOutcome, + _ladder_units, materialize_candidate_skill, - validate_skill_local, + validate_skills_concurrent, ) from core.evolution import GateParams, RejectedEdit from core.types import GeneratedQuestion @@ -150,7 +154,7 @@ def _make_mock_run_inference( def _make_all_infra_mock(log: HarnessLog, stop_reason: str): - """构建基线全 INFRA 的 mock：每 record 写指定 INFRA stop_reason（error/parse_error）。 + """构建全 INFRA 的 mock：每 record 写指定 INFRA stop_reason（error/parse_error）。 与真实推理一致——per-record DB stop_reason 与汇总 stop_reason_counts 同源；护栏 分子按 unit 从 DB 读（_infra_question_ids_from_db），故须真实落 DB。total 返回 @@ -199,6 +203,48 @@ def _make_all_infra_mock(log: HarnessLog, stop_reason: str): return mock_fn, call_log +def _mk_spec( + questions: list[GeneratedQuestion], + *, + candidate_content: str = "candidate skill", + gate_run_prefix: str = "step1_gate_test", +) -> GateSpec: + """由阶梯题序构造单题型 GateSpec（units 经 _ladder_units 聚合为阶梯序单元）。""" + return GateSpec( + task_type="temporal", + target_file="temporal.md", + candidate_content=candidate_content, + base_skill_content="baseline skill content", + units=tuple(_ladder_units(questions)), + gate_run_prefix=gate_run_prefix, + ) + + +async def _run_single_spec( + workspace: Path, + spec: GateSpec, + mock_fn, + log: HarnessLog, + cache: BaselineCache, + params: GateParams, + gate_guard_err: float = 0.5, +) -> ValidationOutcome: + """跑单 spec 的 validate_skills_concurrent 并返回该题型的 outcome。""" + outcomes = await validate_skills_concurrent( + workspace_dir=workspace, + base_skills_version="v1", + specs=[spec], + gate_params=params, + gate_guard_err=gate_guard_err, + baseline_cache=cache, + prompts_version="p1", + run_inference=mock_fn, + log=log, + concurrency=8, + ) + return outcomes[spec.task_type] + + def test_infra_stop_reasons_single_source() -> None: """app 侧 INFRA_STOP_REASONS 复用 core 常量（同一对象），杜绝未来漂移（M-2）。""" from app.harness import validate @@ -331,13 +377,17 @@ class TestMaterializeCandidateSkill: # =========================================================================== -# async 验证测试 +# async 验证测试（迁移自块序贯版 validate_skill_local） # =========================================================================== @pytest.mark.asyncio -async def test_validate_skill_local_accept(tmp_path: Path) -> None: - """候选全对、基线全错 → 高 e 值 → accept_confirmed。""" +async def test_validate_concurrent_accept(tmp_path: Path) -> None: + """候选全对、基线全错 → 高 e 值 → accept_confirmed（迁移自块序贯版）。 + + 6 单元连胜：E=(2^(W+1)-1)/(W+1)，前 5 单元 E<15 且不触方向/futility， + 第 6 单元 E=18.14 ≥ e_confirm=15 → 与旧块判定同点收敛（W=6, n_used=6）。 + """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) questions = _make_questions(6) @@ -359,23 +409,13 @@ async def test_validate_skill_local_accept(tmp_path: Path) -> None: ) try: - outcome = await validate_skill_local( - workspace_dir=workspace, - base_skills_version="v1", - task_type="temporal", - target_file="temporal.md", - candidate_content="improved skill", - base_skill_content="baseline skill content", - ladder_items=questions, - gate_params=accept_params, - gate_block=6, - gate_n_max=20, - gate_guard_err=0.5, - baseline_cache=cache, - prompts_version="p1", - run_inference=mock_fn, - log=log, - gate_run_prefix="step1_gate_test", + outcome = await _run_single_spec( + workspace, + _mk_spec(questions, candidate_content="improved skill"), + mock_fn, + log, + cache, + accept_params, ) assert outcome.accepted is True @@ -387,6 +427,8 @@ async def test_validate_skill_local_accept(tmp_path: Path) -> None: assert outcome.candidate_acc == 1.0 assert outcome.baseline_acc == 0.0 assert len(outcome.evidence_rows) == 6 + # 阶梯序前缀消费：ladder_rank 连续（替代旧块边界断言） + assert [r["ladder_rank"] for r in outcome.evidence_rows] == list(range(6)) # 终态证据行携带 stop_reason assert outcome.evidence_rows[-1]["stop_reason"] == "confirmed" # 候选临时目录应被清理 @@ -398,50 +440,45 @@ async def test_validate_skill_local_accept(tmp_path: Path) -> None: @pytest.mark.asyncio -async def test_validate_skill_local_reject(tmp_path: Path) -> None: - """候选全错、基线全对 → L 高 → 方向拒绝。""" +async def test_validate_concurrent_reject_directional(tmp_path: Path) -> None: + """候选全错、基线全对 → L 高 → 方向拒绝（迁移自块序贯版）。 + + 前缀逐单元判定下早停点前移：15 单元阶梯保证 L=1..3 时 futility 不先触发 + （E(w+n_rem, l) ≥ 3），L=4 时 Wald=4·ln0.6=-2.04 ≤ lambda_dir=-2.0 → + directional 早停于第 4 单元（旧块版一次性判整块故 L=6）。 + """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) - questions = _make_questions(6) + questions = _make_questions(15) cache = BaselineCache(workspace / "baseline_cache.json") - # 基线全对，候选全错 → W=0, L=6 → 方向拒绝 - baseline_correct = {f"q{i}": True for i in range(6)} - candidate_correct = {f"q{i}": False for i in range(6)} + baseline_correct = {f"q{i}": True for i in range(15)} + candidate_correct = {f"q{i}": False for i in range(15)} mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct) try: - outcome = await validate_skill_local( - workspace_dir=workspace, - base_skills_version="v1", - task_type="temporal", - target_file="temporal.md", - candidate_content="bad skill", - base_skill_content="baseline skill content", - ladder_items=questions, - gate_params=_DEFAULT_GATE_PARAMS, - gate_block=6, - gate_n_max=20, - gate_guard_err=0.5, - baseline_cache=cache, - prompts_version="p1", - run_inference=mock_fn, - log=log, - gate_run_prefix="step1_gate_test", + outcome = await _run_single_spec( + workspace, + _mk_spec(questions, candidate_content="bad skill"), + mock_fn, + log, + cache, + _DEFAULT_GATE_PARAMS, ) assert outcome.accepted is False assert outcome.action == "reject" assert outcome.stop_reason == "directional" assert outcome.w == 0 - assert outcome.l == 6 + assert outcome.l == 4 + assert outcome.n_used == 4 finally: log.close() @pytest.mark.asyncio async def test_gate_prefix_must_contain_gate(tmp_path: Path) -> None: - """gate_run_prefix 不含 '_gate_' 时抛 ValueError。""" + """gate_run_prefix 不含 '_gate_' 时抛 ValueError（迁移自块序贯版）。""" workspace = _setup_workspace(tmp_path) log = _make_log(workspace) questions = _make_questions(4) @@ -452,23 +489,13 @@ async def test_gate_prefix_must_contain_gate(tmp_path: Path) -> None: try: with pytest.raises(ValueError, match="_gate_"): - await validate_skill_local( - workspace_dir=workspace, - base_skills_version="v1", - task_type="temporal", - target_file="temporal.md", - candidate_content="content", - base_skill_content="baseline", - ladder_items=questions, - gate_params=_DEFAULT_GATE_PARAMS, - gate_block=4, - gate_n_max=20, - gate_guard_err=0.5, - baseline_cache=cache, - prompts_version="p1", - run_inference=noop_fn, - log=log, - gate_run_prefix="step1_no_marker", + await _run_single_spec( + workspace, + _mk_spec(questions, gate_run_prefix="step1_no_marker"), + noop_fn, + log, + cache, + _DEFAULT_GATE_PARAMS, ) finally: log.close() @@ -476,34 +503,26 @@ async def test_gate_prefix_must_contain_gate(tmp_path: Path) -> None: @pytest.mark.asyncio async def test_infra_guard_threshold(tmp_path: Path) -> None: - """推理错误率超阈值时抛 RuntimeError（护栏分子/分母 unit 同粒度）。""" + """推理错误率超阈值时抛 RuntimeError（迁移自块序贯版，分子/分母 unit 同粒度）。 + + 12 个 single 双臂全 INFRA error：errors 按单元去重逐单元 +1，分母逐臂 +1， + 分母 ≥10 后错误率 >0.5 → 护栏熔断。 + """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) - # 需要 >=10 unit 分母才触发护栏：12 个 single，基线全 INFRA error。 - # 首块全 INFRA → valid_chunk 空 → errors=12/denom=12=1.0>0.5 触发护栏。 questions = _make_questions(12) cache = BaselineCache(workspace / "baseline_cache.json") mock_fn, _ = _make_all_infra_mock(log, "error") try: with pytest.raises(RuntimeError, match="错误率过高"): - await validate_skill_local( - workspace_dir=workspace, - base_skills_version="v1", - task_type="temporal", - target_file="temporal.md", - candidate_content="content", - base_skill_content="baseline skill content", - ladder_items=questions, - gate_params=_DEFAULT_GATE_PARAMS, - gate_block=12, - gate_n_max=20, - gate_guard_err=0.5, - baseline_cache=cache, - prompts_version="p1", - run_inference=mock_fn, - log=log, - gate_run_prefix="step1_gate_test", + await _run_single_spec( + workspace, + _mk_spec(questions), + mock_fn, + log, + cache, + _DEFAULT_GATE_PARAMS, ) finally: log.close() @@ -511,7 +530,10 @@ async def test_infra_guard_threshold(tmp_path: Path) -> None: @pytest.mark.asyncio async def test_baseline_cache_hit(tmp_path: Path) -> None: - """基线缓存全命中时不发起基线侧推理。""" + """基线缓存全命中时不发起基线侧推理（迁移自块序贯版）。 + + 连续并发 gate 下候选侧逐单元发臂：4 单元 → 4 次 cand 调用（旧块版整块 1 次）。 + """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) questions = _make_questions(4) @@ -528,30 +550,20 @@ async def test_baseline_cache_hit(tmp_path: Path) -> None: mock_fn, call_log = _make_mock_run_inference(log, baseline_correct, candidate_correct) try: - outcome = await validate_skill_local( - workspace_dir=workspace, - base_skills_version="v1", - task_type="temporal", - target_file="temporal.md", - candidate_content="improved skill", - base_skill_content="baseline skill content", - ladder_items=questions, - gate_params=_DEFAULT_GATE_PARAMS, - gate_block=4, - gate_n_max=20, - gate_guard_err=0.5, - baseline_cache=cache, - prompts_version="p1", - run_inference=mock_fn, - log=log, - gate_run_prefix="step1_gate_test", + outcome = await _run_single_spec( + workspace, + _mk_spec(questions, candidate_content="improved skill"), + mock_fn, + log, + cache, + _DEFAULT_GATE_PARAMS, ) # 只有候选侧调用了 run_inference（_cand），基线侧全命中不调用 base_calls = [c for c in call_log if c["run_id"].endswith("_base")] cand_calls = [c for c in call_log if c["run_id"].endswith("_cand")] assert len(base_calls) == 0, "基线缓存全命中不应发起推理" - assert len(cand_calls) == 1 + assert len(cand_calls) == 4 assert outcome.accepted is True finally: log.close() @@ -559,21 +571,21 @@ async def test_baseline_cache_hit(tmp_path: Path) -> None: @pytest.mark.asyncio async def test_baseline_infra_error_not_cached(tmp_path: Path) -> None: - """基线臂 INFRA error 的 unit 不写入 BaselineCache（不永久污染），且从有效单元排除。""" - from app.harness.gate_ladder import skill_hash - from app.harness.question_units import build_units - from app.harness.validate import _resolve_baseline_block + """基线臂 INFRA error 的 unit 不写入 BaselineCache（不永久污染），且从配对剔除。 + 迁移自块序贯版 _resolve_baseline_block 直测：改经 validate_skills_concurrent + 端到端验证同一契约——INFRA 单元不落缓存、不入配对；干净单元正常缓存并消费。 + """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) - questions = _make_questions(2) # q0 干净, q1 INFRA error - units = build_units(questions) + questions = _make_questions(2) # q0 基线 INFRA error, q1 干净 cache = BaselineCache(workspace / "baseline_cache.json") s_hash = skill_hash("baseline skill content") async def mock_fn(qs, *, run_id, skills_dir): + is_base = run_id.endswith("_base") for q in qs: - is_err = q.question_id == "q1" + is_err = is_base and q.question_id == "q0" log.insert( "predictions", { @@ -594,54 +606,49 @@ async def test_baseline_infra_error_not_cached(tmp_path: Path) -> None: ) return InferenceResult( run_id=run_id, - accuracy=0.5, - total=2, - correct=1, + accuracy=0.0, + total=len(qs), + correct=0, per_task_type={}, steps_mean=1.0, token_usage={"prompt_tokens": 10, "completion_tokens": 10}, - stop_reason_counts={"completed": 1, "error": 1}, + stop_reason_counts={}, ) try: - b_units, valid_units, _errors_inc, _denom_inc = await _resolve_baseline_block( - units=units, - task_type="temporal", - s_hash=s_hash, - prompts_version="p1", - baseline_cache=cache, - base_skills_dir=workspace / "skills" / "v1", - run_inference=mock_fn, - log=log, - run_id="step1_gate_b0_base", + outcome = await _run_single_spec( + workspace, + _mk_spec(questions), + mock_fn, + log, + cache, + _DEFAULT_GATE_PARAMS, + gate_guard_err=0.9, # 分母 <10 不触发错误率护栏 ) - # q1 是 INFRA：不写缓存、不入 b_units、不在有效单元里 - assert cache.get("temporal", s_hash, "p1", "q1") is None - assert "q1" not in b_units - assert all(u.unit_id != "q1" for u in valid_units) - # q0 干净：正常缓存并入 b_units/valid_units - assert cache.get("temporal", s_hash, "p1", "q0") is True - assert b_units["q0"] is True - assert any(u.unit_id == "q0" for u in valid_units) + # q0 是 INFRA：不写缓存、不入配对观测 + assert cache.get("temporal", s_hash, "p1", "q0") is None + assert "q0" not in outcome.improvements + outcome.regressions + # q1 干净：正常缓存并被消费（唯一有效单元） + assert cache.get("temporal", s_hash, "p1", "q1") is True + assert outcome.n_used == 1 finally: log.close() @pytest.mark.asyncio -async def test_infra_guard_counts_units_not_records(tmp_path: Path) -> None: - """护栏分子按 unit 计：AR pair 两 record 全 INFRA 只计 1 个 INFRA unit（而非 2）。 +async def test_infra_errors_counted_per_unit_not_per_record(tmp_path: Path) -> None: + """护栏分子按 unit 去重：AR pair 两 record、双臂全 INFRA 只计 1 个 error。 - 回归 I-3：分子此前用 stop_reason_counts 逐 record 计数，分母 denom_inc=r.total - 是 unit 粒度；AR pair（一 unit 两 record）致分子被放大、误触发 gate_guard_err。 - 分子改为"含 INFRA record 的 unit 数"后与分母同粒度（核心算法保真 #5/#6）。 + 迁移自块序贯版 _resolve_baseline_block 直测（回归 I-3）：分子若逐 record / + 逐臂计数会被放大（一 unit 两 record × 两臂 = 4），与 unit 粒度分母失配致 + gate_guard_err 误触发。新载体 _run_unit_arm + _register_arm_arrival 按 + slot.excluded() 去重（核心算法保真 #5/#6）。 """ - from app.harness.gate_ladder import skill_hash from app.harness.question_units import build_units - from app.harness.validate import _resolve_baseline_block + from app.harness.validate import _GateRun, _QuestionSlots, _run_unit_arm workspace = _setup_workspace(tmp_path) log = _make_log(workspace) - # 一个 AR pair（两成员共享 pair_id）→ build_units 折叠为 1 个 pair unit common = { "video_id": "vp", "task_type": "temporal", @@ -660,7 +667,17 @@ async def test_infra_guard_counts_units_not_records(tmp_path: Path) -> None: units = build_units(pair) assert len(units) == 1 # 前置：pair 折叠为 1 个 unit cache = BaselineCache(workspace / "baseline_cache.json") - s_hash = skill_hash("baseline skill content") + run = _GateRun.from_spec( + GateSpec( + task_type="temporal", + target_file="temporal.md", + candidate_content="cand", + base_skill_content="baseline skill content", + units=tuple(units), + gate_run_prefix="step1_gate_test", + ) + ) + s_hash = run.s_hash async def mock_fn(qs, *, run_id, skills_dir): # 两 record 皆 INFRA error @@ -683,7 +700,7 @@ async def test_infra_guard_counts_units_not_records(tmp_path: Path) -> None: "steps_json": "[]", }, ) - # total 为 unit 粒度（1 个 pair unit）；stop_reason_counts 为 record 粒度（2） + # total 为 unit 粒度（1 个 pair unit）；record 粒度为 2 return InferenceResult( run_id=run_id, accuracy=0.0, @@ -695,94 +712,57 @@ async def test_infra_guard_counts_units_not_records(tmp_path: Path) -> None: stop_reason_counts={"error": 2}, ) + slots = _QuestionSlots(4) try: - _b_units, valid_units, errors_inc, denom_inc = await _resolve_baseline_block( - units=units, - task_type="temporal", - s_hash=s_hash, - prompts_version="p1", - baseline_cache=cache, - base_skills_dir=workspace / "skills" / "v1", - run_inference=mock_fn, - log=log, - run_id="step1_gate_b0_base", - ) - # 分子按 unit 计：1 个 INFRA unit（不是 2 条 record）；分母同粒度 = r.total = 1 - assert errors_inc == 1 - assert denom_inc == 1 - # 整对 INFRA → 从有效单元剔除 - assert valid_units == [] + for arm in ("base", "cand"): + await _run_unit_arm( + run, + 0, + arm, + slots, + mock_fn, + log, + cache, + "p1", + workspace / "skills" / "v1", + workspace / "skills" / "v1", + _DEFAULT_GATE_PARAMS, + 0.9, + ) + # 分子按 unit 去重：双臂 × 两 record 只计 1 个 error；分母按臂 total 累计 = 2 + assert run.errors == 1 + assert run.infra_denom == 2 + assert run.slots[0].base_infra and run.slots[0].cand_infra + # INFRA 单元不写缓存 + assert cache.get("temporal", s_hash, "p1", "p1") is None finally: log.close() @pytest.mark.asyncio async def test_all_infra_ladder_raises_clear_error(tmp_path: Path) -> None: - """整个阶梯所有 unit 都被判为 INFRA 排除 → 明确 RuntimeError（非误导性空阶梯断言）。""" + """整个阶梯所有 unit 都被判为 INFRA 排除 → 明确 RuntimeError（迁移自块序贯版）。 + + 连续并发 gate 下双臂独立发射，候选臂不再依赖基线侧结果（旧版"全 INFRA 块 + 不空跑候选"的断言随块编排一并删除）。 + """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) questions = _make_questions(4) cache = BaselineCache(workspace / "baseline_cache.json") - - candidate_calls: list[str] = [] - - async def mock_fn(qs, *, run_id, skills_dir): - if run_id.endswith("_cand"): - candidate_calls.append(run_id) - # 基线臂逐题全部 INFRA error（候选臂在修复后不应被空跑） - for q in qs: - log.insert( - "predictions", - { - "run_id": run_id, - "video_id": "v0", - "question_id": q.question_id, - "task_type": "temporal", - "prediction": "", - "answer": "A", - "evidence": "", - "reasoning": "", - "steps_used": 1, - "prompt_tokens": 10, - "completion_tokens": 10, - "stop_reason": "error", - "steps_json": "[]", - }, - ) - total = len(qs) - return InferenceResult( - run_id=run_id, - accuracy=0.0, - total=total, - correct=0, - per_task_type={}, - steps_mean=1.0, - token_usage={"prompt_tokens": 10, "completion_tokens": 10}, - stop_reason_counts={"error": total}, - ) + mock_fn, _ = _make_all_infra_mock(log, "error") try: with pytest.raises(RuntimeError, match="INFRA"): - await validate_skill_local( - workspace_dir=workspace, - base_skills_version="v1", - task_type="temporal", - target_file="temporal.md", - candidate_content="content", - base_skill_content="baseline skill content", - ladder_items=questions, - gate_params=_DEFAULT_GATE_PARAMS, - gate_block=4, - gate_n_max=20, - gate_guard_err=0.9, # 高阈值：4 题 <10 分母不触发错误率护栏 - baseline_cache=cache, - prompts_version="p1", - run_inference=mock_fn, - log=log, - gate_run_prefix="step1_gate_test", + await _run_single_spec( + workspace, + _mk_spec(questions), + mock_fn, + log, + cache, + _DEFAULT_GATE_PARAMS, + gate_guard_err=0.9, # 4 单元分母 <10 不触发错误率护栏 → 逼出全排除分支 ) - # 全 INFRA 块不应触发候选空跑 - assert candidate_calls == [] finally: log.close() @@ -792,42 +772,33 @@ async def test_parse_error_counts_toward_guard(tmp_path: Path) -> None: """stop_reason=parse_error 也计入护栏错误率（与 INFRA 判定口径一致）→ 超阈值熔断。""" workspace = _setup_workspace(tmp_path) log = _make_log(workspace) - # 12 个 single，基线全 parse_error（per-record 落 DB，护栏按 unit 从 DB 读）。 - # 首块全 INFRA → errors=12/denom=12=1.0>0.5 → parse_error 亦触发护栏。 questions = _make_questions(12) cache = BaselineCache(workspace / "baseline_cache.json") mock_fn, _ = _make_all_infra_mock(log, "parse_error") try: with pytest.raises(RuntimeError, match="错误率过高"): - await validate_skill_local( - workspace_dir=workspace, - base_skills_version="v1", - task_type="temporal", - target_file="temporal.md", - candidate_content="content", - base_skill_content="baseline skill content", - ladder_items=questions, - gate_params=_DEFAULT_GATE_PARAMS, - gate_block=12, - gate_n_max=20, - gate_guard_err=0.5, - baseline_cache=cache, - prompts_version="p1", - run_inference=mock_fn, - log=log, - gate_run_prefix="step1_gate_test", + await _run_single_spec( + workspace, + _mk_spec(questions), + mock_fn, + log, + cache, + _DEFAULT_GATE_PARAMS, ) finally: log.close() @pytest.mark.asyncio -async def test_last_block_terminal(tmp_path: Path) -> None: - """单块 + n_remaining=0 → 终态判定（provisional 或 inertia），非 continue。""" +async def test_ladder_exhaustion_terminal(tmp_path: Path) -> None: + """题尽（n_remaining=0）→ 终态判定（provisional 或 inertia），非 continue。 + + 迁移自块序贯版"最后一块终态"：块边界不存在了，等价语义是阶梯耗尽时 + 第四出口兜底，终态行携带 stop_reason。 + """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) - # 4 题，gate_block=4 → 一块走完，n_remaining=0 questions = _make_questions(4) cache = BaselineCache(workspace / "baseline_cache.json") @@ -837,23 +808,13 @@ async def test_last_block_terminal(tmp_path: Path) -> None: mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct) try: - outcome = await validate_skill_local( - workspace_dir=workspace, - base_skills_version="v1", - task_type="temporal", - target_file="temporal.md", - candidate_content="candidate skill", - base_skill_content="baseline skill content", - ladder_items=questions, - gate_params=_DEFAULT_GATE_PARAMS, - gate_block=4, - gate_n_max=4, - gate_guard_err=0.5, - baseline_cache=cache, - prompts_version="p1", - run_inference=mock_fn, - log=log, - gate_run_prefix="step1_gate_test", + outcome = await _run_single_spec( + workspace, + _mk_spec(questions), + mock_fn, + log, + cache, + _DEFAULT_GATE_PARAMS, ) # n_remaining=0 → 不可能是 continue @@ -865,6 +826,8 @@ async def test_last_block_terminal(tmp_path: Path) -> None: "futility", ) assert outcome.n_used == 4 + # 阶梯序前缀消费：ladder_rank 连续 + assert [r["ladder_rank"] for r in outcome.evidence_rows] == list(range(4)) # 终态行标记 stop_reason assert outcome.evidence_rows[-1]["stop_reason"] != "" finally: diff --git a/tests/unit/test_runner_diag_tree_inject.py b/tests/unit/test_runner_diag_tree_inject.py index bc9b259..2d62e24 100644 --- a/tests/unit/test_runner_diag_tree_inject.py +++ b/tests/unit/test_runner_diag_tree_inject.py @@ -64,7 +64,6 @@ def _base_config(workspace_dir: Path, store_dir: Path) -> RunConfig: gate_delta_min=0.02, gate_lambda_dir=-0.642, gate_e_rollback=10.0, - gate_block=8, gate_n_max=40, gate_p_low=0.05, gate_p_high=0.95, From 1930ad32a48268404c7b23a460dc9895592698c6 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 17 Jul 2026 04:52:34 -0400 Subject: [PATCH 16/17] chore: raise Redis cache TTL to 7 days --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 86352a8..974b654 100644 --- a/.env.example +++ b/.env.example @@ -47,7 +47,7 @@ LLM_TTFT_TIMEOUT=30 LLM_INTER_TOKEN_TIMEOUT=15 LLM_RETRY_MAX_DELAY=30.0 # 正整数秒，禁止 0（0 会被拒绝启动）；训练场景建议 >= 单次训练时长 -REDIS_CACHE_TTL=86400 +REDIS_CACHE_TTL=604800 # 建树批量并行：全局 VLM/LLM 在途调用上限（Spec-2 工程配置） TREE_BUILD_API_CONCURRENCY=16 From eb12006d38e2255c6865057072b8458afd289f01 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 17 Jul 2026 05:29:20 -0400 Subject: [PATCH 17/17] fix: idempotent ladder_rank migration for legacy gate_evidence tables --- app/harness/observation.py | 6 +++++ tests/unit/test_harness_observation.py | 36 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/app/harness/observation.py b/app/harness/observation.py index c8e8b5a..469053b 100644 --- a/app/harness/observation.py +++ b/app/harness/observation.py @@ -358,6 +358,12 @@ def write_gate_evidence( with HarnessLog(db_path, run_id) as log: log.create_table("gate_evidence", _GATE_EVIDENCE_COLS) + # 幂等迁移(对齐 question_gen/run_store 先例):块序贯时代的旧表只有 + # block_idx 列,CREATE TABLE IF NOT EXISTS 不补列,直接插 ladder_rank + # 会 OperationalError——为旧 workspace 复用补列,新表恒为 no-op。 + cols = {r["name"] for r in log.query("PRAGMA table_info(gate_evidence)")} + if "ladder_rank" not in cols: + log.execute("ALTER TABLE gate_evidence ADD COLUMN ladder_rank INTEGER") for row in rows: log.insert("gate_evidence", {"epoch": epoch, "step": step, **row}) diff --git a/tests/unit/test_harness_observation.py b/tests/unit/test_harness_observation.py index 8b816cc..e3daadc 100644 --- a/tests/unit/test_harness_observation.py +++ b/tests/unit/test_harness_observation.py @@ -383,3 +383,39 @@ def test_write_epoch_report(tmp_path: Path) -> None: assert data["system_tool_action"] == "updated" assert data["momentum_updated_task_types"] == ["temporal", "causal"] assert data["best_val_acc"] == pytest.approx(0.88) + + +def test_write_gate_evidence_migrates_legacy_block_idx_table(tmp_path) -> None: + """旧块序贯表(含 block_idx 无 ladder_rank)复用:幂等补列后写入成功(终审 C1 回归锁)。""" + import sqlite3 + + db = tmp_path / "harness.db" + conn = sqlite3.connect(db) + conn.execute( + "CREATE TABLE gate_evidence (run_id TEXT, timestamp TEXT, epoch INTEGER," + " step INTEGER, question_id TEXT, task_type TEXT, block_idx INTEGER," + " baseline_correct INTEGER, candidate_correct INTEGER, e_value REAL," + " stop_reason TEXT)" + ) + conn.commit() + conn.close() + + write_gate_evidence( + str(db), + run_id="r1", + epoch=1, + step=0, + rows=[ + { + "question_id": "q1", + "task_type": "Action Reasoning", + "ladder_rank": 0, + "baseline_correct": False, + "candidate_correct": True, + "e_value": 1.5, + "stop_reason": "", + } + ], + ) + got = read_gate_evidence(str(db), run_id="r1") + assert len(got) == 1 and got[0]["ladder_rank"] == 0