301 lines
12 KiB
Python
301 lines
12 KiB
Python
"""_gate_batch_skills 并行装配的纯逻辑护栏 + runner 级并发编排测试。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import time
|
|
from types import SimpleNamespace
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
|
|
from app.harness import runner as runner_mod
|
|
from app.harness.question_units import build_units
|
|
from app.harness.runner import Runner, _assert_disjoint_target_files
|
|
from app.harness.validate import ValidationOutcome
|
|
from core.types import GeneratedQuestion
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
|
|
def test_disjoint_target_files_pass() -> None:
|
|
"""各题型映射不同文件:通过。"""
|
|
_assert_disjoint_target_files(
|
|
{"Action Reasoning": "action-reasoning.md", "Counting Problem": "counting-problem.md"}
|
|
)
|
|
|
|
|
|
def test_shared_target_file_fails_fast() -> None:
|
|
"""两题型 fallback 到同一文件:并行进化会互相覆盖,必须 fail-fast。"""
|
|
with pytest.raises(RuntimeError, match="default-strategy.md"):
|
|
_assert_disjoint_target_files(
|
|
{"OCR Problems": "default-strategy.md", "Spatial Reasoning": "default-strategy.md"}
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# runner 级并发编排测试(Codex 计划审 I6):
|
|
# 用 Runner.__new__ 裸实例 + 假依赖驱动 _gate_batch_skills 四阶段,
|
|
# 断言 Phase A gather 并行、Phase D 字母序落账、accept/reject 正确分派。
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_TYPE_A = "Action Reasoning"
|
|
_TYPE_C = "Counting Problem"
|
|
_TARGET_FILES = {_TYPE_A: "action-reasoning.md", _TYPE_C: "counting-problem.md"}
|
|
|
|
|
|
def _question(qid: str, task_type: str) -> GeneratedQuestion:
|
|
"""构造一条真实结构的 single 题目(unit_id 由 __post_init__ 回填)。"""
|
|
return GeneratedQuestion(
|
|
question_id=qid,
|
|
video_id="video-001",
|
|
task_type=task_type,
|
|
question="视频中主角最先做了什么?",
|
|
options=("A. 开门", "B. 关灯", "C. 坐下", "D. 起身"),
|
|
answer="A",
|
|
source_nodes=("L3_0001",),
|
|
difficulty="medium",
|
|
)
|
|
|
|
|
|
def _record(task_type: str) -> SimpleNamespace:
|
|
"""构造 EvolutionRecord 替身(仅含 _gate_batch_skills 消费的属性)。"""
|
|
return SimpleNamespace(
|
|
status="accepted",
|
|
original_content="旧 skill 内容",
|
|
evolved_content=f"进化后 skill 内容({task_type})",
|
|
target_file=_TARGET_FILES[task_type],
|
|
clip_info={},
|
|
)
|
|
|
|
|
|
def _outcome(accepted: bool) -> ValidationOutcome:
|
|
"""构造真实 ValidationOutcome(一 accept 一 reject 分派用)。"""
|
|
return ValidationOutcome(
|
|
action="accept_confirmed" if accepted else "reject",
|
|
accepted=accepted,
|
|
stop_reason="confirmed" if accepted else "futility",
|
|
e_value=25.0 if accepted else 0.4,
|
|
w=3,
|
|
l=0 if accepted else 3,
|
|
n_used=4,
|
|
delta_hat=0.3 if accepted else -0.2,
|
|
delta_shrunk=0.2 if accepted else -0.1,
|
|
baseline_acc=0.5,
|
|
candidate_acc=0.8 if accepted else 0.3,
|
|
evidence_rows=[{"question_id": "q", "stop_reason": "answered"}],
|
|
)
|
|
|
|
|
|
class _FakeHarnessLog:
|
|
"""HarnessLog no-op 替身(上下文管理器协议)。"""
|
|
|
|
def __init__(self, *args: object, **kwargs: object) -> None:
|
|
self.args = args
|
|
|
|
def __enter__(self) -> _FakeHarnessLog:
|
|
return self
|
|
|
|
def __exit__(self, *exc: object) -> bool:
|
|
return False
|
|
|
|
|
|
def _build_runner(tmp_path: Path) -> tuple[Runner, SimpleNamespace, SimpleNamespace]:
|
|
"""构造裸 Runner 实例与 state/pools 替身(不触发真实 __init__)。"""
|
|
skills_dir = tmp_path / "skills" / "v1"
|
|
skills_dir.mkdir(parents=True)
|
|
for target in _TARGET_FILES.values():
|
|
(skills_dir / target).write_text("旧 skill 内容", encoding="utf-8")
|
|
|
|
runner = Runner.__new__(Runner)
|
|
runner._config = SimpleNamespace(
|
|
workspace_dir=tmp_path,
|
|
edit_budget_start=4,
|
|
edit_budget_end=2,
|
|
appendix_consolidate_threshold=3,
|
|
skill_update_mode="rewrite",
|
|
gate_p_low=0.3,
|
|
gate_p_high=0.85,
|
|
gate_n_max=8,
|
|
gate_e_confirm=20.0,
|
|
gate_e_provisional=5.0,
|
|
gate_w_net_min=2,
|
|
gate_delta_min=0.05,
|
|
gate_lambda_dir=0.5,
|
|
gate_e_rollback=0.05,
|
|
gate_guard_err=0.34,
|
|
concurrency=4,
|
|
max_steps=10,
|
|
skill_mode="live",
|
|
)
|
|
runner._paths = SimpleNamespace(
|
|
skills_dir=skills_dir,
|
|
prompts_dir=tmp_path / "prompts",
|
|
db_path=tmp_path / "harness.db",
|
|
)
|
|
runner._llm = object()
|
|
runner._evolve_llm = object()
|
|
runner._load_evolve_prompts = lambda: None
|
|
runner._current_version = lambda kind: "v1"
|
|
runner._class_baseline_acc = lambda *a, **k: 0.5
|
|
runner._record_run = lambda run_id: None
|
|
|
|
questions = {t: _question(f"q-{t[:2].lower()}", t) for t in _TARGET_FILES}
|
|
units = {t: build_units([q])[0] for t, q in questions.items()}
|
|
runner._gate_questions_by_id = {q.question_id: q for q in questions.values()}
|
|
runner._gate_units_by_id = {u.unit_id: u for u in units.values()}
|
|
unit_ids_by_type = {t: [u.unit_id] for t, u in units.items()}
|
|
|
|
state = SimpleNamespace(
|
|
gate_cooldown={},
|
|
rejected_buffer={},
|
|
global_step=0,
|
|
correctness={},
|
|
gate_epoch_observed=True,
|
|
baseline_cache=object(),
|
|
gate_pools=SimpleNamespace(
|
|
ladder_for=lambda task_type, exclude, *, p_low, p_high, cold: unit_ids_by_type[
|
|
task_type
|
|
]
|
|
),
|
|
)
|
|
pools = SimpleNamespace(baseline_run_id="baseline-run", validation=[])
|
|
return runner, state, pools
|
|
|
|
|
|
def test_gate_batch_parallel_evolve_and_alphabetical_settle(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""Phase A 两题型进化时间窗重叠(并行),Phase D 按字母序 accept/reject 分派。"""
|
|
import core.evolution as core_evolution
|
|
|
|
runner, state, pools = _build_runner(tmp_path)
|
|
diagnosis = SimpleNamespace(
|
|
skill_case_packs={
|
|
# 故意逆字母序插入,验证排序不是插入序的巧合
|
|
_TYPE_C: SimpleNamespace(task_type=_TYPE_C, failure_cases=[], success_cases=[]),
|
|
_TYPE_A: SimpleNamespace(task_type=_TYPE_A, failure_cases=[], success_cases=[]),
|
|
}
|
|
)
|
|
records = {t: _record(t) for t in _TARGET_FILES}
|
|
outcomes = {_TYPE_A: _outcome(accepted=True), _TYPE_C: _outcome(accepted=False)}
|
|
|
|
evolve_windows: dict[str, tuple[float, float]] = {}
|
|
|
|
async def fake_evolve_single_skill(
|
|
llm, pack, skill_store, prompts, version, budget, threshold, **kwargs
|
|
):
|
|
start = time.monotonic()
|
|
await asyncio.sleep(0.05)
|
|
evolve_windows[pack.task_type] = (start, time.monotonic())
|
|
return records[pack.task_type]
|
|
|
|
captured: dict[str, object] = {}
|
|
|
|
async def fake_validate_skills_concurrent(**kwargs):
|
|
captured.update(kwargs)
|
|
# 逆字母序返回,验证 Phase D 落账顺序来自 sorted 而非 dict 插入序
|
|
return {
|
|
_TYPE_C: outcomes[_TYPE_C],
|
|
_TYPE_A: outcomes[_TYPE_A],
|
|
}
|
|
|
|
settle_calls: list[tuple[str, str]] = []
|
|
runner._accept_skill = lambda task_type, *a: settle_calls.append(("accept", task_type))
|
|
runner._record_rejected_skill = lambda buf, task_type, *a: settle_calls.append(
|
|
("reject", task_type)
|
|
)
|
|
|
|
monkeypatch.setattr(core_evolution, "evolve_single_skill", fake_evolve_single_skill)
|
|
monkeypatch.setattr(runner_mod, "validate_skills_concurrent", fake_validate_skills_concurrent)
|
|
monkeypatch.setattr(runner_mod, "HarnessLog", _FakeHarnessLog)
|
|
monkeypatch.setattr(runner_mod, "write_gate_evidence", lambda *a, **k: None)
|
|
monkeypatch.setattr(runner_mod, "write_step_report", lambda *a, **k: None)
|
|
monkeypatch.setattr(runner_mod, "write_quadrant_pairs", lambda *a, **k: None)
|
|
monkeypatch.setattr(runner_mod, "_outcome_to_quadrant_pairs", lambda t, o: [])
|
|
monkeypatch.setattr(runner_mod, "_write_skip_report", lambda *a, **k: None)
|
|
|
|
asyncio.run(runner._gate_batch_skills(1, 0, diagnosis, 3, pools, state))
|
|
|
|
# (a) 进化时间窗重叠 = gather 真并行(串行时前者 end <= 后者 start)
|
|
win_a, win_c = evolve_windows[_TYPE_A], evolve_windows[_TYPE_C]
|
|
assert win_a[0] < win_c[1] and win_c[0] < win_a[1], f"进化未并行: {evolve_windows}"
|
|
|
|
# (b) Phase D 落账顺序 == sorted(题型),且 (c) accept/reject 分派与 outcome 一致
|
|
assert settle_calls == [("accept", _TYPE_A), ("reject", _TYPE_C)]
|
|
|
|
# Phase B 装配的 GateSpec 与 Phase C 共享 log 抽查
|
|
specs = captured["specs"]
|
|
assert [s.task_type for s in specs] == sorted(_TARGET_FILES)
|
|
for spec in specs:
|
|
assert spec.target_file == _TARGET_FILES[spec.task_type]
|
|
assert spec.base_skill_content == "旧 skill 内容"
|
|
assert spec.candidate_content == records[spec.task_type].evolved_content
|
|
assert len(spec.units) == 1
|
|
assert "_gate_" in spec.gate_run_prefix
|
|
assert isinstance(captured["log"], _FakeHarnessLog)
|
|
assert callable(captured["run_inference"])
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 共享 gate_log 的 run_id 契约(真 SQLite,Codex 质量审 C1):
|
|
# HarnessLog.insert 缺省用实例 run_id 填充;record 自带 run_id 必须覆盖它,
|
|
# 否则连续并发 gate 下所有臂的 predictions 会落成 step 级 run_id,
|
|
# validate 按臂 run_id 回读为空 → gate 静默废掉。
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_harness_log_insert_record_run_id_overrides_instance(tmp_path: Path) -> None:
|
|
"""record 自带 run_id 覆盖实例 run_id;缺省时回落实例 run_id(锁死 enriched.update 语义)。"""
|
|
from app.harness.inference import PREDICTIONS_SCHEMA
|
|
from app.harness.log import HarnessLog
|
|
|
|
with HarnessLog(str(tmp_path / "harness.db"), "gate_e1_s0") as log:
|
|
log.create_table("predictions", PREDICTIONS_SCHEMA)
|
|
log.insert(
|
|
"predictions",
|
|
{"run_id": "run_e1_s0_gate_a_base_u0", "question_id": "q1", "prediction": "A"},
|
|
)
|
|
log.insert("predictions", {"question_id": "q2", "prediction": "B"})
|
|
rows = log.query("SELECT question_id, run_id FROM predictions ORDER BY question_id")
|
|
assert [(r["question_id"], r["run_id"]) for r in rows] == [
|
|
("q1", "run_e1_s0_gate_a_base_u0"),
|
|
("q2", "gate_e1_s0"),
|
|
]
|
|
|
|
|
|
def test_inference_prediction_row_carries_arm_run_id(tmp_path: Path) -> None:
|
|
"""经共享 gate_log 落库的 prediction 行 run_id 必须是臂 run_id 而非实例 run_id。
|
|
|
|
prompt_builder 抛错走异常路径即落库,无需真实 LLM;
|
|
该路径与成功路径共用同一 record 初始 dict,契约一致。
|
|
"""
|
|
from app.harness.inference import PREDICTIONS_SCHEMA, _run_single_question
|
|
from app.harness.log import HarnessLog
|
|
|
|
def _broken_prompt_builder(qa: GeneratedQuestion) -> tuple[str, str]:
|
|
raise RuntimeError("测试注入:跳过真实推理")
|
|
|
|
async def _noop_dispatch(tool_name: str, args: dict, *, context: dict) -> str:
|
|
raise NotImplementedError
|
|
|
|
with HarnessLog(str(tmp_path / "harness.db"), "gate_e1_s0") as gate_log:
|
|
gate_log.create_table("predictions", PREDICTIONS_SCHEMA)
|
|
asyncio.run(
|
|
_run_single_question(
|
|
_question("q-arm", _TYPE_A),
|
|
llm=object(), # prompt_builder 先抛错,不会触达
|
|
tool_dispatch_fn=_noop_dispatch,
|
|
prompt_builder=_broken_prompt_builder,
|
|
log=gate_log,
|
|
max_steps=3,
|
|
plugins=[],
|
|
run_id="run_e1_s0_gate_action-reasoning_cand_u0",
|
|
)
|
|
)
|
|
rows = gate_log.query("SELECT run_id, stop_reason FROM predictions")
|
|
assert len(rows) == 1
|
|
assert rows[0]["run_id"] == "run_e1_s0_gate_action-reasoning_cand_u0"
|
|
assert rows[0]["stop_reason"] == "error"
|