feat: continuous concurrent gate orchestrator (algo #6)

- validate_skills_concurrent: 多题型全部臂共享题槽并发编排,发射序 =
  题型 round-robin × 阶梯序(base 先 cand 后),终态统一组装 outcome,
  verdict None(全 INFRA)保留 RuntimeError 语义
- gate_evidence 列 block_idx → ladder_rank(阶梯序号,0-based);旧块路径
  _build_evidence_rows 仅键名同步(值仍为块号)保持落库兼容
- 新增 3 项编排测试:乱序到达前缀有序性/双题型隔离/全 INFRA raise

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-17 00:36:01 -04:00
parent c61a6dac84
commit e8b66f85ab
4 changed files with 324 additions and 6 deletions
+167
View File
@@ -0,0 +1,167 @@
"""连续并发 gate 编排测试:乱序到达/多题型隔离/终态组装/全 INFRA。"""
from __future__ import annotations
import asyncio
from typing import TYPE_CHECKING
import pytest
if TYPE_CHECKING:
from pathlib import Path
from app.harness.gate_ladder import BaselineCache
from app.harness.validate import GateSpec, validate_skills_concurrent
from tests.unit.test_gate_prefix import _PARAMS, _mk_unit
from tests.unit.test_gate_unit_arm import _FakeLog
def _mk_spec(task_type: str, slug: str, n: int) -> GateSpec:
"""构造 n 个 single 单元的 gate 规格(unit_id 形如 <slug>-q<i>)。"""
return GateSpec(
task_type=task_type,
target_file=f"{slug}.md",
candidate_content=f"cand-{slug}",
base_skill_content=f"base-{slug}",
units=tuple(_mk_unit(f"{slug}-q{i}", task_type) for i in range(n)),
gate_run_prefix=f"r_e1_s0_gate_{slug}",
)
def _scripted_inference(log: _FakeLog, script: dict[str, tuple[bool, float]]):
"""脚本化假推理:按 question_id+臂 决定 (对错, 延迟秒),制造乱序到达。"""
class _R:
def __init__(self, run_id: str, total: int) -> None:
self.run_id = run_id
self.total = total
async def _run(questions, *, run_id: str, skills_dir: Path):
arm = "cand" if run_id.endswith("_cand") else "base"
correct, delay = script[f"{questions[0].question_id}|{arm}"]
await asyncio.sleep(delay)
for q in questions:
log.rows.append(
{
"run_id": run_id,
"question_id": q.question_id,
"prediction": "A" if correct else "B",
"answer": "A",
"stop_reason": "finished",
"steps_json": "[]",
}
)
return _R(run_id, len(questions))
return _run
@pytest.mark.asyncio
async def test_out_of_order_arrival_still_ladder_order(tmp_path, monkeypatch) -> None:
"""尾部先到、头部后到:判定结果与顺序到达完全相同(前缀有序性端到端)。"""
spec = _mk_spec("Action Reasoning", "action-reasoning", 4)
log = _FakeLog()
script = {}
for i in range(4): # 头部 q0 最慢;全部翻转为 W(base 错 cand 对)
script[f"action-reasoning-q{i}|base"] = (False, 0.05 if i == 0 else 0.0)
script[f"action-reasoning-q{i}|cand"] = (True, 0.05 if i == 0 else 0.0)
monkeypatch.setattr(
"app.harness.validate.materialize_candidate_skill",
lambda *a, **k: tmp_path / "cand",
)
outcomes = await validate_skills_concurrent(
workspace_dir=tmp_path,
base_skills_version="v1",
specs=[spec],
gate_params=_PARAMS,
gate_guard_err=0.10,
baseline_cache=BaselineCache(tmp_path / "bc.json"),
prompts_version="v1",
run_inference=_scripted_inference(log, script),
log=log,
concurrency=8,
)
o = outcomes["Action Reasoning"]
assert o.w == 4 and o.l == 0
assert [r["ladder_rank"] for r in o.evidence_rows] == [0, 1, 2, 3]
@pytest.mark.asyncio
async def test_two_types_isolated(tmp_path, monkeypatch) -> None:
"""两题型并行:计数互不污染,各自独立判定。
A 型 4 单元全 W(题尽 accept_provisional);B 型 2 单元全平
(futility 早停,W=L=0)——两型结果都不受对方污染。
"""
spec_a = _mk_spec("Action Reasoning", "action-reasoning", 4)
spec_b = _mk_spec("Counting Problem", "counting-problem", 2)
log = _FakeLog()
script = {}
for i in range(4):
script[f"action-reasoning-q{i}|base"] = (False, 0.0)
script[f"action-reasoning-q{i}|cand"] = (True, 0.0)
for i in range(2):
script[f"counting-problem-q{i}|base"] = (True, 0.0)
script[f"counting-problem-q{i}|cand"] = (True, 0.0)
monkeypatch.setattr(
"app.harness.validate.materialize_candidate_skill",
lambda *a, **k: tmp_path / "cand",
)
outcomes = await validate_skills_concurrent(
workspace_dir=tmp_path,
base_skills_version="v1",
specs=[spec_a, spec_b],
gate_params=_PARAMS,
gate_guard_err=0.10,
baseline_cache=BaselineCache(tmp_path / "bc.json"),
prompts_version="v1",
run_inference=_scripted_inference(log, script),
log=log,
concurrency=8,
)
assert outcomes["Action Reasoning"].w == 4
assert outcomes["Counting Problem"].w == 0
assert outcomes["Counting Problem"].l == 0
@pytest.mark.asyncio
async def test_all_infra_raises(tmp_path, monkeypatch) -> None:
"""全单元 INFRA:保留现行 RuntimeError 语义(检查推理基础设施)。"""
spec = _mk_spec("Action Reasoning", "action-reasoning", 2)
log = _FakeLog()
class _R:
def __init__(self, run_id, total):
self.run_id, self.total = run_id, total
async def _infra_run(questions, *, run_id, skills_dir):
for q in questions:
log.rows.append(
{
"run_id": run_id,
"question_id": q.question_id,
"prediction": "",
"answer": "A",
"stop_reason": "error",
"steps_json": "[]",
}
)
return _R(run_id, len(questions))
monkeypatch.setattr(
"app.harness.validate.materialize_candidate_skill",
lambda *a, **k: tmp_path / "cand",
)
with pytest.raises(RuntimeError):
await validate_skills_concurrent(
workspace_dir=tmp_path,
base_skills_version="v1",
specs=[spec],
gate_params=_PARAMS,
gate_guard_err=0.99, # 护栏放宽,逼出全 INFRA 分支
baseline_cache=BaselineCache(tmp_path / "bc.json"),
prompts_version="v1",
run_inference=_infra_run,
log=log,
concurrency=8,
)