iomgaa
8958eee11b
config/train_videomme.yaml 同时收录待入库的实验配置变更(run_id v2 / concurrency 32 / batch_size 40)。tests/integration/test_v3_contract_e2e.py 的 run_id 断言按 Task 5 显式契约同步修正(原断言依赖旧隐式实例注入)。
835 lines
29 KiB
Python
835 lines
29 KiB
Python
"""tests/unit/test_harness_validate.py — app/harness/validate.py 的单元测试。
|
||
|
||
覆盖：数据类型字段、materialize 物化与清理、async validate_skills_concurrent
|
||
（accept/reject/prefix 校验/INFRA 护栏/缓存命中/题尽终态）。async 用例迁移自
|
||
块序贯版（validate_skill_local，Task 6 删除）：载体换连续并发 gate，语义断言
|
||
保留；前缀逐单元判定使早停点比旧块判定更早（见各用例 docstring 的数值推导）。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import TYPE_CHECKING, Any
|
||
|
||
import pytest
|
||
|
||
from app.harness.gate_ladder import BaselineCache, skill_hash
|
||
from app.harness.inference import PREDICTIONS_SCHEMA, InferenceResult
|
||
from app.harness.log import HarnessLog
|
||
from app.harness.validate import (
|
||
GateSpec,
|
||
Probation,
|
||
ValidationOutcome,
|
||
_ladder_units,
|
||
materialize_candidate_skill,
|
||
validate_skills_concurrent,
|
||
)
|
||
from core.evolution import GateParams, RejectedEdit
|
||
from core.types import GeneratedQuestion
|
||
|
||
if TYPE_CHECKING:
|
||
from pathlib import Path
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 辅助工具
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_DEFAULT_GATE_PARAMS = GateParams(
|
||
e_confirm=20.0,
|
||
e_provisional=3.0,
|
||
w_net_min=2,
|
||
delta_min=0.05,
|
||
lambda_dir=-2.0,
|
||
e_rollback=10.0,
|
||
)
|
||
|
||
|
||
def _make_questions(
|
||
n: int,
|
||
task_type: str = "temporal",
|
||
prefix: str = "q",
|
||
) -> list[GeneratedQuestion]:
|
||
"""生成 n 个测试用 GeneratedQuestion。"""
|
||
return [
|
||
GeneratedQuestion(
|
||
question_id=f"{prefix}{i}",
|
||
video_id=f"v{i}",
|
||
task_type=task_type,
|
||
question=f"Question {i}?",
|
||
options=("A", "B", "C", "D"),
|
||
answer="A",
|
||
source_nodes=(),
|
||
difficulty="easy",
|
||
)
|
||
for i in range(n)
|
||
]
|
||
|
||
|
||
def _setup_workspace(tmp_path: Path) -> Path:
|
||
"""在 tmp_path 下构建最小 workspace 结构。"""
|
||
skills_dir = tmp_path / "skills" / "v1"
|
||
skills_dir.mkdir(parents=True)
|
||
(skills_dir / "temporal.md").write_text("baseline skill content", encoding="utf-8")
|
||
return tmp_path
|
||
|
||
|
||
def _make_log(workspace: Path, run_id: str = "test_master") -> HarnessLog:
|
||
"""创建 HarnessLog 并初始化 predictions 表。"""
|
||
db_path = workspace / "harness.db"
|
||
log = HarnessLog(str(db_path), run_id)
|
||
log.create_table("predictions", PREDICTIONS_SCHEMA)
|
||
return log
|
||
|
||
|
||
def _insert_predictions(
|
||
log: HarnessLog,
|
||
run_id: str,
|
||
correctness: dict[str, bool],
|
||
answer: str = "A",
|
||
) -> None:
|
||
"""向 predictions 表插入指定 run_id 的逐题预测记录。
|
||
|
||
通过在 record 中显式传入 run_id 覆盖 log 的默认 run_id。
|
||
"""
|
||
for qid, correct in correctness.items():
|
||
prediction = answer if correct else "Z"
|
||
log.insert(
|
||
"predictions",
|
||
{
|
||
"run_id": run_id,
|
||
"video_id": "v0",
|
||
"question_id": qid,
|
||
"task_type": "temporal",
|
||
"prediction": prediction,
|
||
"answer": answer,
|
||
"evidence": "",
|
||
"reasoning": "",
|
||
"steps_used": 1,
|
||
"prompt_tokens": 10,
|
||
"completion_tokens": 10,
|
||
"stop_reason": "completed",
|
||
"steps_json": "[]",
|
||
},
|
||
)
|
||
|
||
|
||
def _make_mock_run_inference(
|
||
log: HarnessLog,
|
||
baseline_correctness: dict[str, bool],
|
||
candidate_correctness: dict[str, bool],
|
||
):
|
||
"""构建 mock RunInferenceFn。
|
||
|
||
根据 run_id 中的 arm 标记（_base / _cand）决定使用基线或候选对错映射，
|
||
将预测写入 log 的同一 DB，返回 InferenceResult。
|
||
"""
|
||
call_log: list[dict[str, Any]] = []
|
||
|
||
async def mock_fn(
|
||
questions: list[GeneratedQuestion],
|
||
*,
|
||
run_id: str,
|
||
skills_dir: Path,
|
||
) -> InferenceResult:
|
||
is_baseline = run_id.endswith("_base")
|
||
correctness = baseline_correctness if is_baseline else candidate_correctness
|
||
|
||
call_log.append({"run_id": run_id, "skills_dir": skills_dir, "n": len(questions)})
|
||
per_q = {q.question_id: correctness.get(q.question_id, False) for q in questions}
|
||
_insert_predictions(log, run_id, per_q)
|
||
|
||
correct = sum(per_q.values())
|
||
total = len(questions)
|
||
return InferenceResult(
|
||
run_id=run_id,
|
||
accuracy=correct / total if total else 0.0,
|
||
total=total,
|
||
correct=correct,
|
||
per_task_type={},
|
||
steps_mean=1.0,
|
||
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
|
||
stop_reason_counts={"completed": total},
|
||
)
|
||
|
||
return mock_fn, call_log
|
||
|
||
|
||
def _make_all_infra_mock(log: HarnessLog, stop_reason: str):
|
||
"""构建全 INFRA 的 mock：每 record 写指定 INFRA stop_reason（error/parse_error）。
|
||
|
||
与真实推理一致——per-record DB stop_reason 与汇总 stop_reason_counts 同源；护栏
|
||
分子按 unit 从 DB 读（_infra_question_ids_from_db），故须真实落 DB。total 返回
|
||
unit 粒度（single 时 == 题数），使护栏分子/分母同粒度。
|
||
"""
|
||
call_log: list[dict[str, Any]] = []
|
||
|
||
async def mock_fn(
|
||
questions: list[GeneratedQuestion],
|
||
*,
|
||
run_id: str,
|
||
skills_dir: Path,
|
||
) -> InferenceResult:
|
||
call_log.append({"run_id": run_id, "n": len(questions)})
|
||
for q in questions:
|
||
log.insert(
|
||
"predictions",
|
||
{
|
||
"run_id": run_id,
|
||
"video_id": "v0",
|
||
"question_id": q.question_id,
|
||
"task_type": "temporal",
|
||
"prediction": "",
|
||
"answer": "A",
|
||
"evidence": "",
|
||
"reasoning": "",
|
||
"steps_used": 1,
|
||
"prompt_tokens": 10,
|
||
"completion_tokens": 10,
|
||
"stop_reason": stop_reason,
|
||
"steps_json": "[]",
|
||
},
|
||
)
|
||
total = len(questions) # 全 single → unit 数 == 题数
|
||
return InferenceResult(
|
||
run_id=run_id,
|
||
accuracy=0.0,
|
||
total=total,
|
||
correct=0,
|
||
per_task_type={},
|
||
steps_mean=1.0,
|
||
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
|
||
stop_reason_counts={stop_reason: total},
|
||
)
|
||
|
||
return mock_fn, call_log
|
||
|
||
|
||
def _mk_spec(
|
||
questions: list[GeneratedQuestion],
|
||
*,
|
||
candidate_content: str = "candidate skill",
|
||
gate_run_prefix: str = "step1_gate_test",
|
||
) -> GateSpec:
|
||
"""由阶梯题序构造单题型 GateSpec（units 经 _ladder_units 聚合为阶梯序单元）。"""
|
||
return GateSpec(
|
||
task_type="temporal",
|
||
target_file="temporal.md",
|
||
candidate_content=candidate_content,
|
||
base_skill_content="baseline skill content",
|
||
units=tuple(_ladder_units(questions)),
|
||
gate_run_prefix=gate_run_prefix,
|
||
)
|
||
|
||
|
||
async def _run_single_spec(
|
||
workspace: Path,
|
||
spec: GateSpec,
|
||
mock_fn,
|
||
log: HarnessLog,
|
||
cache: BaselineCache,
|
||
params: GateParams,
|
||
gate_guard_err: float = 0.5,
|
||
) -> ValidationOutcome:
|
||
"""跑单 spec 的 validate_skills_concurrent 并返回该题型的 outcome。"""
|
||
outcomes = await validate_skills_concurrent(
|
||
workspace_dir=workspace,
|
||
base_skills_version="v1",
|
||
specs=[spec],
|
||
gate_params=params,
|
||
gate_guard_err=gate_guard_err,
|
||
baseline_cache=cache,
|
||
prompts_version="p1",
|
||
run_inference=mock_fn,
|
||
log=log,
|
||
concurrency=8,
|
||
)
|
||
return outcomes[spec.task_type]
|
||
|
||
|
||
def test_infra_stop_reasons_single_source() -> None:
|
||
"""app 侧 INFRA_STOP_REASONS 复用 core 常量（同一对象），杜绝未来漂移（M-2）。"""
|
||
from app.harness import validate
|
||
from core.evolution import diagnose
|
||
|
||
assert validate.INFRA_STOP_REASONS is diagnose.INFRA_STOP_REASONS
|
||
assert frozenset({"error", "parse_error"}) == diagnose.INFRA_STOP_REASONS
|
||
|
||
|
||
# ===========================================================================
|
||
# 数据类型测试
|
||
# ===========================================================================
|
||
|
||
|
||
class TestValidationOutcomeFields:
|
||
"""ValidationOutcome 数据类型字段完整性测试。"""
|
||
|
||
def test_validation_outcome_fields(self) -> None:
|
||
"""所有字段可构造、默认值合理。"""
|
||
outcome = ValidationOutcome(
|
||
action="accept_confirmed",
|
||
accepted=True,
|
||
stop_reason="confirmed",
|
||
e_value=25.0,
|
||
w=5,
|
||
l=1,
|
||
n_used=10,
|
||
delta_hat=0.4,
|
||
delta_shrunk=0.3,
|
||
baseline_acc=0.6,
|
||
candidate_acc=0.9,
|
||
)
|
||
assert outcome.action == "accept_confirmed"
|
||
assert outcome.accepted is True
|
||
assert outcome.stop_reason == "confirmed"
|
||
assert outcome.e_value == 25.0
|
||
assert outcome.w == 5
|
||
assert outcome.l == 1
|
||
assert outcome.n_used == 10
|
||
assert outcome.delta_hat == 0.4
|
||
assert outcome.delta_shrunk == 0.3
|
||
assert outcome.baseline_acc == 0.6
|
||
assert outcome.candidate_acc == 0.9
|
||
assert outcome.improvements == []
|
||
assert outcome.regressions == []
|
||
assert outcome.persistent_fails == []
|
||
assert outcome.stable_successes == []
|
||
assert outcome.candidate_correctness == {}
|
||
assert outcome.evidence_rows == []
|
||
|
||
|
||
class TestProbationFields:
|
||
"""Probation 数据类型字段完整性测试。"""
|
||
|
||
def test_probation_fields(self) -> None:
|
||
"""所有字段可构造、pending_edits 默认空列表。"""
|
||
prob = Probation(
|
||
task_type="temporal",
|
||
anchor_skills_version="v1",
|
||
target_file="temporal.md",
|
||
correctness_snapshot={"q0": True, "q1": False},
|
||
opened_step=5,
|
||
)
|
||
assert prob.task_type == "temporal"
|
||
assert prob.anchor_skills_version == "v1"
|
||
assert prob.target_file == "temporal.md"
|
||
assert prob.correctness_snapshot == {"q0": True, "q1": False}
|
||
assert prob.opened_step == 5
|
||
assert prob.pending_edits == []
|
||
|
||
def test_probation_with_pending_edits(self) -> None:
|
||
"""pending_edits 可附加 RejectedEdit。"""
|
||
edit = RejectedEdit(
|
||
target_file="temporal.md",
|
||
target_type="skill",
|
||
change_summary="bad change",
|
||
delta=-0.1,
|
||
source_version="v2",
|
||
epoch=1,
|
||
)
|
||
prob = Probation(
|
||
task_type="temporal",
|
||
anchor_skills_version="v1",
|
||
target_file="temporal.md",
|
||
correctness_snapshot={},
|
||
opened_step=3,
|
||
pending_edits=[edit],
|
||
)
|
||
assert len(prob.pending_edits) == 1
|
||
assert prob.pending_edits[0].change_summary == "bad change"
|
||
|
||
|
||
# ===========================================================================
|
||
# materialize 测试
|
||
# ===========================================================================
|
||
|
||
|
||
class TestMaterializeCandidateSkill:
|
||
"""materialize_candidate_skill 物化与清理测试。"""
|
||
|
||
def test_materialize_candidate_skill(self, tmp_path: Path) -> None:
|
||
"""正常物化：基线目录被复制，target_file 被覆写为候选内容。"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
cand_dir = materialize_candidate_skill(
|
||
workspace, "v1", "temporal.md", "candidate skill content"
|
||
)
|
||
try:
|
||
assert cand_dir.exists()
|
||
assert cand_dir.parent == workspace / ".cand_tmp"
|
||
assert (cand_dir / "temporal.md").read_text(encoding="utf-8") == (
|
||
"candidate skill content"
|
||
)
|
||
finally:
|
||
import shutil
|
||
|
||
shutil.rmtree(cand_dir)
|
||
|
||
def test_materialize_cleanup_on_failure(self, tmp_path: Path) -> None:
|
||
"""基线目录不存在时 OSError，临时目录被清理。"""
|
||
workspace = tmp_path / "ws"
|
||
workspace.mkdir()
|
||
# 不创建 skills/v1，copytree 应失败
|
||
with pytest.raises(OSError):
|
||
materialize_candidate_skill(workspace, "v1", "temporal.md", "content")
|
||
# .cand_tmp 可能存在但内部应被清理
|
||
cand_tmp = workspace / ".cand_tmp"
|
||
if cand_tmp.exists():
|
||
remaining = list(cand_tmp.iterdir())
|
||
assert remaining == [], f"临时目录未被清理: {remaining}"
|
||
|
||
|
||
# ===========================================================================
|
||
# async 验证测试（迁移自块序贯版 validate_skill_local）
|
||
# ===========================================================================
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_validate_concurrent_accept(tmp_path: Path) -> None:
|
||
"""候选全对、基线全错 → 高 e 值 → accept_confirmed（迁移自块序贯版）。
|
||
|
||
6 单元连胜：E=(2^(W+1)-1)/(W+1)，前 5 单元 E<15 且不触方向/futility，
|
||
第 6 单元 E=18.14 ≥ e_confirm=15 → 与旧块判定同点收敛（W=6, n_used=6）。
|
||
"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
questions = _make_questions(6)
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
|
||
# 基线全错，候选全对 → W=6, L=0 → E=18.14 → CONFIRMED（e_confirm=15）
|
||
baseline_correct = {f"q{i}": False for i in range(6)}
|
||
candidate_correct = {f"q{i}": True for i in range(6)}
|
||
mock_fn, call_log = _make_mock_run_inference(log, baseline_correct, candidate_correct)
|
||
|
||
# e_confirm=15 使 E=18.14 超过阈值触发 CONFIRMED
|
||
accept_params = GateParams(
|
||
e_confirm=15.0,
|
||
e_provisional=3.0,
|
||
w_net_min=2,
|
||
delta_min=0.05,
|
||
lambda_dir=-2.0,
|
||
e_rollback=10.0,
|
||
)
|
||
|
||
try:
|
||
outcome = await _run_single_spec(
|
||
workspace,
|
||
_mk_spec(questions, candidate_content="improved skill"),
|
||
mock_fn,
|
||
log,
|
||
cache,
|
||
accept_params,
|
||
)
|
||
|
||
assert outcome.accepted is True
|
||
assert outcome.action == "accept_confirmed"
|
||
assert outcome.stop_reason == "confirmed"
|
||
assert outcome.w == 6
|
||
assert outcome.l == 0
|
||
assert outcome.n_used == 6
|
||
assert outcome.candidate_acc == 1.0
|
||
assert outcome.baseline_acc == 0.0
|
||
assert len(outcome.evidence_rows) == 6
|
||
# 阶梯序前缀消费：ladder_rank 连续（替代旧块边界断言）
|
||
assert [r["ladder_rank"] for r in outcome.evidence_rows] == list(range(6))
|
||
# 终态证据行携带 stop_reason
|
||
assert outcome.evidence_rows[-1]["stop_reason"] == "confirmed"
|
||
# 候选临时目录应被清理
|
||
cand_tmp = workspace / ".cand_tmp"
|
||
if cand_tmp.exists():
|
||
assert list(cand_tmp.iterdir()) == []
|
||
finally:
|
||
log.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_validate_concurrent_reject_directional(tmp_path: Path) -> None:
|
||
"""候选全错、基线全对 → L 高 → 方向拒绝（迁移自块序贯版）。
|
||
|
||
前缀逐单元判定下早停点前移：15 单元阶梯保证 L=1..3 时 futility 不先触发
|
||
（E(w+n_rem, l) ≥ 3），L=4 时 Wald=4·ln0.6=-2.04 ≤ lambda_dir=-2.0 →
|
||
directional 早停于第 4 单元（旧块版一次性判整块故 L=6）。
|
||
"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
questions = _make_questions(15)
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
|
||
baseline_correct = {f"q{i}": True for i in range(15)}
|
||
candidate_correct = {f"q{i}": False for i in range(15)}
|
||
mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct)
|
||
|
||
try:
|
||
outcome = await _run_single_spec(
|
||
workspace,
|
||
_mk_spec(questions, candidate_content="bad skill"),
|
||
mock_fn,
|
||
log,
|
||
cache,
|
||
_DEFAULT_GATE_PARAMS,
|
||
)
|
||
|
||
assert outcome.accepted is False
|
||
assert outcome.action == "reject"
|
||
assert outcome.stop_reason == "directional"
|
||
assert outcome.w == 0
|
||
assert outcome.l == 4
|
||
assert outcome.n_used == 4
|
||
finally:
|
||
log.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_gate_prefix_must_contain_gate(tmp_path: Path) -> None:
|
||
"""gate_run_prefix 不含 '_gate_' 时抛 ValueError（迁移自块序贯版）。"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
questions = _make_questions(4)
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
|
||
async def noop_fn(questions, *, run_id, skills_dir):
|
||
raise AssertionError("不应被调用")
|
||
|
||
try:
|
||
with pytest.raises(ValueError, match="_gate_"):
|
||
await _run_single_spec(
|
||
workspace,
|
||
_mk_spec(questions, gate_run_prefix="step1_no_marker"),
|
||
noop_fn,
|
||
log,
|
||
cache,
|
||
_DEFAULT_GATE_PARAMS,
|
||
)
|
||
finally:
|
||
log.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_infra_guard_threshold(tmp_path: Path) -> None:
|
||
"""推理错误率超阈值时抛 RuntimeError（迁移自块序贯版，分子/分母 unit 同粒度）。
|
||
|
||
12 个 single 双臂全 INFRA error：errors 按单元去重逐单元 +1，分母逐臂 +1，
|
||
分母 ≥10 后错误率 >0.5 → 护栏熔断。
|
||
"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
questions = _make_questions(12)
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
mock_fn, _ = _make_all_infra_mock(log, "error")
|
||
|
||
try:
|
||
with pytest.raises(RuntimeError, match="错误率过高"):
|
||
await _run_single_spec(
|
||
workspace,
|
||
_mk_spec(questions),
|
||
mock_fn,
|
||
log,
|
||
cache,
|
||
_DEFAULT_GATE_PARAMS,
|
||
)
|
||
finally:
|
||
log.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_baseline_cache_hit(tmp_path: Path) -> None:
|
||
"""基线缓存全命中时不发起基线侧推理（迁移自块序贯版）。
|
||
|
||
连续并发 gate 下候选侧逐单元发臂：4 单元 → 4 次 cand 调用（旧块版整块 1 次）。
|
||
"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
questions = _make_questions(4)
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
|
||
s_hash = skill_hash("baseline skill content")
|
||
# 预填充缓存：全部题目基线全错
|
||
for q in questions:
|
||
cache.put("temporal", s_hash, "p1", q.question_id, False)
|
||
|
||
# 候选全对 → accept
|
||
candidate_correct = {f"q{i}": True for i in range(4)}
|
||
baseline_correct = {f"q{i}": False for i in range(4)}
|
||
mock_fn, call_log = _make_mock_run_inference(log, baseline_correct, candidate_correct)
|
||
|
||
try:
|
||
outcome = await _run_single_spec(
|
||
workspace,
|
||
_mk_spec(questions, candidate_content="improved skill"),
|
||
mock_fn,
|
||
log,
|
||
cache,
|
||
_DEFAULT_GATE_PARAMS,
|
||
)
|
||
|
||
# 只有候选侧调用了 run_inference（_cand），基线侧全命中不调用
|
||
base_calls = [c for c in call_log if c["run_id"].endswith("_base")]
|
||
cand_calls = [c for c in call_log if c["run_id"].endswith("_cand")]
|
||
assert len(base_calls) == 0, "基线缓存全命中不应发起推理"
|
||
assert len(cand_calls) == 4
|
||
assert outcome.accepted is True
|
||
finally:
|
||
log.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_baseline_infra_error_not_cached(tmp_path: Path) -> None:
|
||
"""基线臂 INFRA error 的 unit 不写入 BaselineCache（不永久污染），且从配对剔除。
|
||
|
||
迁移自块序贯版 _resolve_baseline_block 直测：改经 validate_skills_concurrent
|
||
端到端验证同一契约——INFRA 单元不落缓存、不入配对；干净单元正常缓存并消费。
|
||
"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
questions = _make_questions(2) # q0 基线 INFRA error, q1 干净
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
s_hash = skill_hash("baseline skill content")
|
||
|
||
async def mock_fn(qs, *, run_id, skills_dir):
|
||
is_base = run_id.endswith("_base")
|
||
for q in qs:
|
||
is_err = is_base and q.question_id == "q0"
|
||
log.insert(
|
||
"predictions",
|
||
{
|
||
"run_id": run_id,
|
||
"video_id": "v0",
|
||
"question_id": q.question_id,
|
||
"task_type": "temporal",
|
||
"prediction": "" if is_err else "A",
|
||
"answer": "A",
|
||
"evidence": "",
|
||
"reasoning": "",
|
||
"steps_used": 1,
|
||
"prompt_tokens": 10,
|
||
"completion_tokens": 10,
|
||
"stop_reason": "error" if is_err else "completed",
|
||
"steps_json": "[]",
|
||
},
|
||
)
|
||
return InferenceResult(
|
||
run_id=run_id,
|
||
accuracy=0.0,
|
||
total=len(qs),
|
||
correct=0,
|
||
per_task_type={},
|
||
steps_mean=1.0,
|
||
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
|
||
stop_reason_counts={},
|
||
)
|
||
|
||
try:
|
||
outcome = await _run_single_spec(
|
||
workspace,
|
||
_mk_spec(questions),
|
||
mock_fn,
|
||
log,
|
||
cache,
|
||
_DEFAULT_GATE_PARAMS,
|
||
gate_guard_err=0.9, # 分母 <10 不触发错误率护栏
|
||
)
|
||
# q0 是 INFRA：不写缓存、不入配对观测
|
||
assert cache.get("temporal", s_hash, "p1", "q0") is None
|
||
assert "q0" not in outcome.improvements + outcome.regressions
|
||
# q1 干净：正常缓存并被消费（唯一有效单元）
|
||
assert cache.get("temporal", s_hash, "p1", "q1") is True
|
||
assert outcome.n_used == 1
|
||
finally:
|
||
log.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_infra_errors_counted_per_unit_not_per_record(tmp_path: Path) -> None:
|
||
"""护栏分子按 unit 去重：AR pair 两 record、双臂全 INFRA 只计 1 个 error。
|
||
|
||
迁移自块序贯版 _resolve_baseline_block 直测（回归 I-3）：分子若逐 record /
|
||
逐臂计数会被放大（一 unit 两 record × 两臂 = 4），与 unit 粒度分母失配致
|
||
gate_guard_err 误触发。新载体 _run_unit_arm + _register_arm_arrival 按
|
||
slot.excluded() 去重（核心算法保真 #5/#6）。
|
||
"""
|
||
from app.harness.question_units import build_units
|
||
from app.harness.validate import _GateRun, _QuestionSlots, _run_unit_arm
|
||
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
common = {
|
||
"video_id": "vp",
|
||
"task_type": "temporal",
|
||
"question": "Q?",
|
||
"options": ("A", "B", "C", "D"),
|
||
"answer": "A",
|
||
"source_nodes": (),
|
||
"difficulty": "easy",
|
||
"pair_id": "p1",
|
||
"flip_axis": "before_after",
|
||
}
|
||
pair = [
|
||
GeneratedQuestion(question_id="p1_o", question_role="pair_original", **common),
|
||
GeneratedQuestion(question_id="p1_m", question_role="pair_mirror", **common),
|
||
]
|
||
units = build_units(pair)
|
||
assert len(units) == 1 # 前置：pair 折叠为 1 个 unit
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
run = _GateRun.from_spec(
|
||
GateSpec(
|
||
task_type="temporal",
|
||
target_file="temporal.md",
|
||
candidate_content="cand",
|
||
base_skill_content="baseline skill content",
|
||
units=tuple(units),
|
||
gate_run_prefix="step1_gate_test",
|
||
)
|
||
)
|
||
s_hash = run.s_hash
|
||
|
||
async def mock_fn(qs, *, run_id, skills_dir):
|
||
# 两 record 皆 INFRA error
|
||
for q in qs:
|
||
log.insert(
|
||
"predictions",
|
||
{
|
||
"run_id": run_id,
|
||
"video_id": "vp",
|
||
"question_id": q.question_id,
|
||
"task_type": "temporal",
|
||
"prediction": "",
|
||
"answer": "A",
|
||
"evidence": "",
|
||
"reasoning": "",
|
||
"steps_used": 1,
|
||
"prompt_tokens": 10,
|
||
"completion_tokens": 10,
|
||
"stop_reason": "error",
|
||
"steps_json": "[]",
|
||
},
|
||
)
|
||
# total 为 unit 粒度（1 个 pair unit）；record 粒度为 2
|
||
return InferenceResult(
|
||
run_id=run_id,
|
||
accuracy=0.0,
|
||
total=1,
|
||
correct=0,
|
||
per_task_type={},
|
||
steps_mean=1.0,
|
||
token_usage={"prompt_tokens": 20, "completion_tokens": 20},
|
||
stop_reason_counts={"error": 2},
|
||
)
|
||
|
||
slots = _QuestionSlots(4)
|
||
try:
|
||
for arm in ("base", "cand"):
|
||
await _run_unit_arm(
|
||
run,
|
||
0,
|
||
arm,
|
||
slots,
|
||
mock_fn,
|
||
log,
|
||
cache,
|
||
"p1",
|
||
workspace / "skills" / "v1",
|
||
workspace / "skills" / "v1",
|
||
_DEFAULT_GATE_PARAMS,
|
||
0.9,
|
||
)
|
||
# 分子按 unit 去重：双臂 × 两 record 只计 1 个 error；分母按臂 total 累计 = 2
|
||
assert run.errors == 1
|
||
assert run.infra_denom == 2
|
||
assert run.slots[0].base_infra and run.slots[0].cand_infra
|
||
# INFRA 单元不写缓存
|
||
assert cache.get("temporal", s_hash, "p1", "p1") is None
|
||
finally:
|
||
log.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_all_infra_ladder_raises_clear_error(tmp_path: Path) -> None:
|
||
"""整个阶梯所有 unit 都被判为 INFRA 排除 → 明确 RuntimeError（迁移自块序贯版）。
|
||
|
||
连续并发 gate 下双臂独立发射，候选臂不再依赖基线侧结果（旧版"全 INFRA 块
|
||
不空跑候选"的断言随块编排一并删除）。
|
||
"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
questions = _make_questions(4)
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
mock_fn, _ = _make_all_infra_mock(log, "error")
|
||
|
||
try:
|
||
with pytest.raises(RuntimeError, match="INFRA"):
|
||
await _run_single_spec(
|
||
workspace,
|
||
_mk_spec(questions),
|
||
mock_fn,
|
||
log,
|
||
cache,
|
||
_DEFAULT_GATE_PARAMS,
|
||
gate_guard_err=0.9, # 4 单元分母 <10 不触发错误率护栏 → 逼出全排除分支
|
||
)
|
||
finally:
|
||
log.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_parse_error_counts_toward_guard(tmp_path: Path) -> None:
|
||
"""stop_reason=parse_error 也计入护栏错误率（与 INFRA 判定口径一致）→ 超阈值熔断。"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
questions = _make_questions(12)
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
mock_fn, _ = _make_all_infra_mock(log, "parse_error")
|
||
|
||
try:
|
||
with pytest.raises(RuntimeError, match="错误率过高"):
|
||
await _run_single_spec(
|
||
workspace,
|
||
_mk_spec(questions),
|
||
mock_fn,
|
||
log,
|
||
cache,
|
||
_DEFAULT_GATE_PARAMS,
|
||
)
|
||
finally:
|
||
log.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_ladder_exhaustion_terminal(tmp_path: Path) -> None:
|
||
"""题尽（n_remaining=0）→ 终态判定（provisional 或 inertia），非 continue。
|
||
|
||
迁移自块序贯版"最后一块终态"：块边界不存在了，等价语义是阶梯耗尽时
|
||
第四出口兜底，终态行携带 stop_reason。
|
||
"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
questions = _make_questions(4)
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
|
||
# 两题翻转（W=2, L=0），但 e_confirm=20 难以达到 → provisional 或 inertia
|
||
baseline_correct = {"q0": False, "q1": False, "q2": True, "q3": True}
|
||
candidate_correct = {"q0": True, "q1": True, "q2": True, "q3": True}
|
||
mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct)
|
||
|
||
try:
|
||
outcome = await _run_single_spec(
|
||
workspace,
|
||
_mk_spec(questions),
|
||
mock_fn,
|
||
log,
|
||
cache,
|
||
_DEFAULT_GATE_PARAMS,
|
||
)
|
||
|
||
# n_remaining=0 → 不可能是 continue
|
||
assert outcome.stop_reason in (
|
||
"confirmed",
|
||
"provisional",
|
||
"inertia",
|
||
"directional",
|
||
"futility",
|
||
)
|
||
assert outcome.n_used == 4
|
||
# 阶梯序前缀消费：ladder_rank 连续
|
||
assert [r["ladder_rank"] for r in outcome.evidence_rows] == list(range(4))
|
||
# 终态行标记 stop_reason
|
||
assert outcome.evidence_rows[-1]["stop_reason"] != ""
|
||
finally:
|
||
log.close()
|