194 lines
6.9 KiB
Python
194 lines
6.9 KiB
Python
"""连续并发 gate 的前缀消费纯逻辑测试。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from app.harness.validate import GateSpec, _advance_prefix, _GateRun
|
|
from core.evolution import GateParams
|
|
from core.types import GeneratedQuestion, QuestionUnit
|
|
|
|
|
|
def _mk_question(qid: str, task_type: str = "Action Reasoning") -> GeneratedQuestion:
|
|
"""构造最小可用的 single 题（字段以 core.types 真实定义为准，缺省值从简）。"""
|
|
return GeneratedQuestion(
|
|
question_id=qid,
|
|
video_id="v1",
|
|
task_type=task_type,
|
|
question=f"q-{qid}",
|
|
options=("A. x", "B. y"),
|
|
answer="A",
|
|
source_nodes=(),
|
|
difficulty="easy",
|
|
)
|
|
|
|
|
|
def _mk_unit(qid: str, task_type: str = "Action Reasoning") -> QuestionUnit:
|
|
"""由单条题目构造 single 单元（unit_id 回填为 question_id）。"""
|
|
return QuestionUnit.from_single(_mk_question(qid, task_type))
|
|
|
|
|
|
def _mk_run(n_units: int) -> _GateRun:
|
|
"""构造含 n_units 个 single 单元的初始 gate 运行时状态。"""
|
|
spec = GateSpec(
|
|
task_type="Action Reasoning",
|
|
target_file="action-reasoning.md",
|
|
candidate_content="cand",
|
|
base_skill_content="base",
|
|
units=tuple(_mk_unit(f"q{i}") for i in range(n_units)),
|
|
gate_run_prefix="r_e1_s0_gate_action-reasoning",
|
|
)
|
|
return _GateRun.from_spec(spec)
|
|
|
|
|
|
_PARAMS = GateParams(
|
|
e_confirm=20.0,
|
|
e_provisional=3.0,
|
|
w_net_min=2,
|
|
delta_min=0.02,
|
|
lambda_dir=-0.642,
|
|
e_rollback=10.0,
|
|
)
|
|
|
|
|
|
def test_prefix_blocks_on_unresolved_head() -> None:
|
|
"""阶梯头部单元未配齐时,即使尾部全部配齐也一个都不消费。"""
|
|
run = _mk_run(4)
|
|
for i in (1, 2, 3): # 尾部三个先到
|
|
run.slots[i].base = False
|
|
run.slots[i].cand_per_q = {f"q{i}": True}
|
|
_advance_prefix(run, _PARAMS)
|
|
assert run.n_used == 0 and run.w == 0 and run.verdict is None
|
|
|
|
|
|
def test_prefix_consumes_in_ladder_order_after_head_arrives() -> None:
|
|
"""头部补齐后一次性顺序消费到最长已配齐前缀。"""
|
|
run = _mk_run(4)
|
|
for i in (0, 1, 2):
|
|
run.slots[i].base = False
|
|
run.slots[i].cand_per_q = {f"q{i}": True}
|
|
_advance_prefix(run, _PARAMS)
|
|
assert run.n_used == 3 and run.w == 3 and run.l == 0
|
|
assert [r["ladder_rank"] for r in run.evidence_rows] == [0, 1, 2]
|
|
|
|
|
|
def test_freeze_on_terminal_verdict_stops_consumption() -> None:
|
|
"""过线即冻结,后续已配齐单元不再消费。
|
|
|
|
数值:W 连胜 L=0 时 E=(2^(W+1)-1)/(W+1),W=6→18.14<20,W=7→31.875≥20,
|
|
故 7 连胜恰好 confirmed 过线(Codex 复核)。
|
|
"""
|
|
run = _mk_run(12)
|
|
for i in range(12):
|
|
run.slots[i].base = False
|
|
run.slots[i].cand_per_q = {f"q{i}": True}
|
|
_advance_prefix(run, _PARAMS)
|
|
assert run.frozen and run.verdict is not None
|
|
assert run.verdict.decision == "accept_confirmed"
|
|
assert run.n_used == 7 # 第 7 个净胜恰好过线,早停不吃满
|
|
|
|
|
|
def test_tail_infra_reaches_terminal_not_continue() -> None:
|
|
"""尾部全 INFRA:剔除后须重判(n_remaining 归 0 → 题尽第四出口),
|
|
verdict 不得停留在 continue(Codex plan 审 C1 回归锁)。"""
|
|
run = _mk_run(4)
|
|
run.slots[0].base = False
|
|
run.slots[0].cand_per_q = {"q0": True}
|
|
run.slots[1].base = True
|
|
run.slots[1].cand_per_q = {"q1": True}
|
|
for i in (2, 3):
|
|
run.slots[i].base_infra = True
|
|
run.slots[i].cand_per_q = {f"q{i}": True}
|
|
_advance_prefix(run, _PARAMS)
|
|
assert run.verdict is not None and run.verdict.decision != "continue"
|
|
assert run.frozen
|
|
|
|
|
|
def test_infra_unit_skipped_not_counted() -> None:
|
|
"""INFRA 单元(任一臂)剔除:不入 (W,L)、计入 n_excluded、前缀继续推进。
|
|
|
|
注意 futility 出口在小 n_remaining 下很敏感,用 6 单元(首个 INFRA、其余 5 个
|
|
W 翻转)保证消费全程不提前触发 futility:题尽走 accept_provisional 终态。
|
|
"""
|
|
run = _mk_run(6)
|
|
run.slots[0].base_infra = True
|
|
run.slots[0].cand_per_q = {"q0": True}
|
|
for i in range(1, 6):
|
|
run.slots[i].base = False
|
|
run.slots[i].cand_per_q = {f"q{i}": True}
|
|
_advance_prefix(run, _PARAMS)
|
|
assert run.n_excluded == 1 and run.n_used == 5 and run.w == 5 and run.l == 0
|
|
assert run.verdict is not None and run.verdict.decision == "accept_provisional"
|
|
|
|
|
|
def test_all_infra_leaves_verdict_none() -> None:
|
|
"""全部单元被剔除:verdict 保持 None、frozen 保持 False——终态由调度编排层
|
|
(Task 3 的 validate_skills_concurrent)检测 verdict None 并 raise,本函数不管。"""
|
|
run = _mk_run(3)
|
|
for i in range(3):
|
|
run.slots[i].base_infra = True
|
|
run.slots[i].cand_per_q = {f"q{i}": True}
|
|
_advance_prefix(run, _PARAMS)
|
|
assert run.verdict is None and not run.frozen and run.n_excluded == 3 and run.prefix_ptr == 3
|
|
|
|
|
|
def test_repeated_calls_are_idempotent() -> None:
|
|
"""部分前缀消费后重复调用不改变状态;补齐剩余单元后再调用正常推进。"""
|
|
run = _mk_run(4)
|
|
for i in (0, 1):
|
|
run.slots[i].base = False
|
|
run.slots[i].cand_per_q = {f"q{i}": True}
|
|
_advance_prefix(run, _PARAMS)
|
|
snapshot = (run.n_used, run.w, run.l, run.prefix_ptr, len(run.evidence_rows))
|
|
_advance_prefix(run, _PARAMS)
|
|
_advance_prefix(run, _PARAMS)
|
|
assert (run.n_used, run.w, run.l, run.prefix_ptr, len(run.evidence_rows)) == snapshot
|
|
for i in (2, 3):
|
|
run.slots[i].base = False
|
|
run.slots[i].cand_per_q = {f"q{i}": True}
|
|
_advance_prefix(run, _PARAMS)
|
|
assert run.n_used > snapshot[0] and run.prefix_ptr > snapshot[3]
|
|
|
|
|
|
def test_frozen_run_call_is_noop() -> None:
|
|
"""frozen 后再调用是 no-op:不再消费已配齐单元、证据不再追加。"""
|
|
run = _mk_run(12)
|
|
for i in range(12):
|
|
run.slots[i].base = False
|
|
run.slots[i].cand_per_q = {f"q{i}": True}
|
|
_advance_prefix(run, _PARAMS)
|
|
assert run.frozen and run.n_used == 7
|
|
before = (
|
|
run.w,
|
|
run.l,
|
|
run.n_used,
|
|
run.prefix_ptr,
|
|
len(run.evidence_rows),
|
|
run.verdict,
|
|
run.frozen,
|
|
)
|
|
_advance_prefix(run, _PARAMS)
|
|
after = (
|
|
run.w,
|
|
run.l,
|
|
run.n_used,
|
|
run.prefix_ptr,
|
|
len(run.evidence_rows),
|
|
run.verdict,
|
|
run.frozen,
|
|
)
|
|
assert after == before
|
|
|
|
|
|
def test_ties_hit_futility_early_and_freeze() -> None:
|
|
"""全打平(无翻转对)时 futility 出口尽早触发并冻结——早停语义(数值:W=L=0
|
|
时乐观 E = E(n_remaining, 0),n 小易 <e_provisional=3)。"""
|
|
run = _mk_run(3)
|
|
for i in range(3):
|
|
run.slots[i].base = True
|
|
run.slots[i].cand_per_q = {f"q{i}": True}
|
|
_advance_prefix(run, _PARAMS)
|
|
assert run.frozen
|
|
# 精确锁定 futility 出口:首个消费后 W=L=0,n_remaining=2,
|
|
# 乐观 E=E(2,0)=(2^3-1)/3=2.33<3 → 立即 reject_futility(Codex 复核)
|
|
assert run.verdict is not None and run.verdict.decision == "reject_futility"
|
|
assert run.n_used == 1 and run.w == 0 and run.l == 0
|