"""tests/unit/test_harness_validate.py — app/harness/validate.py 的单元测试。 覆盖:数据类型字段、materialize 物化与清理、async validate_skills_concurrent (accept/reject/prefix 校验/INFRA 护栏/缓存命中/题尽终态)。async 用例迁移自 块序贯版(validate_skill_local,Task 6 删除):载体换连续并发 gate,语义断言 保留;前缀逐单元判定使早停点比旧块判定更早(见各用例 docstring 的数值推导)。 """ from __future__ import annotations from typing import TYPE_CHECKING, Any import pytest from app.harness.gate_ladder import BaselineCache, skill_hash from app.harness.inference import PREDICTIONS_SCHEMA, InferenceResult from app.harness.log import HarnessLog from app.harness.validate import ( GateSpec, Probation, ValidationOutcome, _ladder_units, materialize_candidate_skill, validate_skills_concurrent, ) from core.evolution import GateParams, RejectedEdit from core.types import GeneratedQuestion if TYPE_CHECKING: from pathlib import Path # --------------------------------------------------------------------------- # 辅助工具 # --------------------------------------------------------------------------- _DEFAULT_GATE_PARAMS = GateParams( e_confirm=20.0, e_provisional=3.0, w_net_min=2, delta_min=0.05, lambda_dir=-2.0, e_rollback=10.0, ) def _make_questions( n: int, task_type: str = "temporal", prefix: str = "q", ) -> list[GeneratedQuestion]: """生成 n 个测试用 GeneratedQuestion。""" return [ GeneratedQuestion( question_id=f"{prefix}{i}", video_id=f"v{i}", task_type=task_type, question=f"Question {i}?", options=("A", "B", "C", "D"), answer="A", source_nodes=(), difficulty="easy", ) for i in range(n) ] def _setup_workspace(tmp_path: Path) -> Path: """在 tmp_path 下构建最小 workspace 结构。""" skills_dir = tmp_path / "skills" / "v1" skills_dir.mkdir(parents=True) (skills_dir / "temporal.md").write_text("baseline skill content", encoding="utf-8") return tmp_path def _make_log(workspace: Path, run_id: str = "test_master") -> HarnessLog: """创建 HarnessLog 并初始化 predictions 表。""" db_path = workspace / "harness.db" log = HarnessLog(str(db_path), run_id) log.create_table("predictions", PREDICTIONS_SCHEMA) return log def _insert_predictions( log: HarnessLog, run_id: str, correctness: dict[str, bool], answer: str = "A", ) -> None: """向 predictions 表插入指定 run_id 的逐题预测记录。 通过在 record 中显式传入 run_id 覆盖 log 的默认 run_id。 """ for qid, correct in correctness.items(): prediction = answer if correct else "Z" log.insert( "predictions", { "run_id": run_id, "video_id": "v0", "question_id": qid, "task_type": "temporal", "prediction": prediction, "answer": answer, "evidence": "", "reasoning": "", "steps_used": 1, "prompt_tokens": 10, "completion_tokens": 10, "stop_reason": "completed", "steps_json": "[]", }, ) def _make_mock_run_inference( log: HarnessLog, baseline_correctness: dict[str, bool], candidate_correctness: dict[str, bool], ): """构建 mock RunInferenceFn。 根据 run_id 中的 arm 标记(_base / _cand)决定使用基线或候选对错映射, 将预测写入 log 的同一 DB,返回 InferenceResult。 """ call_log: list[dict[str, Any]] = [] async def mock_fn( questions: list[GeneratedQuestion], *, run_id: str, skills_dir: Path, ) -> InferenceResult: is_baseline = run_id.endswith("_base") correctness = baseline_correctness if is_baseline else candidate_correctness call_log.append({"run_id": run_id, "skills_dir": skills_dir, "n": len(questions)}) per_q = {q.question_id: correctness.get(q.question_id, False) for q in questions} _insert_predictions(log, run_id, per_q) correct = sum(per_q.values()) total = len(questions) return InferenceResult( run_id=run_id, accuracy=correct / total if total else 0.0, total=total, correct=correct, per_task_type={}, steps_mean=1.0, token_usage={"prompt_tokens": 10, "completion_tokens": 10}, stop_reason_counts={"completed": total}, ) return mock_fn, call_log def _make_all_infra_mock(log: HarnessLog, stop_reason: str): """构建全 INFRA 的 mock:每 record 写指定 INFRA stop_reason(error/parse_error)。 与真实推理一致——per-record DB stop_reason 与汇总 stop_reason_counts 同源;护栏 分子按 unit 从 DB 读(_infra_question_ids_from_db),故须真实落 DB。total 返回 unit 粒度(single 时 == 题数),使护栏分子/分母同粒度。 """ call_log: list[dict[str, Any]] = [] async def mock_fn( questions: list[GeneratedQuestion], *, run_id: str, skills_dir: Path, ) -> InferenceResult: call_log.append({"run_id": run_id, "n": len(questions)}) for q in questions: log.insert( "predictions", { "run_id": run_id, "video_id": "v0", "question_id": q.question_id, "task_type": "temporal", "prediction": "", "answer": "A", "evidence": "", "reasoning": "", "steps_used": 1, "prompt_tokens": 10, "completion_tokens": 10, "stop_reason": stop_reason, "steps_json": "[]", }, ) total = len(questions) # 全 single → unit 数 == 题数 return InferenceResult( run_id=run_id, accuracy=0.0, total=total, correct=0, per_task_type={}, steps_mean=1.0, token_usage={"prompt_tokens": 10, "completion_tokens": 10}, stop_reason_counts={stop_reason: total}, ) return mock_fn, call_log def _mk_spec( questions: list[GeneratedQuestion], *, candidate_content: str = "candidate skill", gate_run_prefix: str = "step1_gate_test", ) -> GateSpec: """由阶梯题序构造单题型 GateSpec(units 经 _ladder_units 聚合为阶梯序单元)。""" return GateSpec( task_type="temporal", target_file="temporal.md", candidate_content=candidate_content, base_skill_content="baseline skill content", units=tuple(_ladder_units(questions)), gate_run_prefix=gate_run_prefix, ) async def _run_single_spec( workspace: Path, spec: GateSpec, mock_fn, log: HarnessLog, cache: BaselineCache, params: GateParams, gate_guard_err: float = 0.5, ) -> ValidationOutcome: """跑单 spec 的 validate_skills_concurrent 并返回该题型的 outcome。""" outcomes = await validate_skills_concurrent( workspace_dir=workspace, base_skills_version="v1", specs=[spec], gate_params=params, gate_guard_err=gate_guard_err, baseline_cache=cache, prompts_version="p1", run_inference=mock_fn, log=log, concurrency=8, ) return outcomes[spec.task_type] def test_infra_stop_reasons_single_source() -> None: """app 侧 INFRA_STOP_REASONS 复用 core 常量(同一对象),杜绝未来漂移(M-2)。""" from app.harness import validate from core.evolution import diagnose assert validate.INFRA_STOP_REASONS is diagnose.INFRA_STOP_REASONS assert frozenset({"error", "parse_error"}) == diagnose.INFRA_STOP_REASONS # =========================================================================== # 数据类型测试 # =========================================================================== class TestValidationOutcomeFields: """ValidationOutcome 数据类型字段完整性测试。""" def test_validation_outcome_fields(self) -> None: """所有字段可构造、默认值合理。""" outcome = ValidationOutcome( action="accept_confirmed", accepted=True, stop_reason="confirmed", e_value=25.0, w=5, l=1, n_used=10, delta_hat=0.4, delta_shrunk=0.3, baseline_acc=0.6, candidate_acc=0.9, ) assert outcome.action == "accept_confirmed" assert outcome.accepted is True assert outcome.stop_reason == "confirmed" assert outcome.e_value == 25.0 assert outcome.w == 5 assert outcome.l == 1 assert outcome.n_used == 10 assert outcome.delta_hat == 0.4 assert outcome.delta_shrunk == 0.3 assert outcome.baseline_acc == 0.6 assert outcome.candidate_acc == 0.9 assert outcome.improvements == [] assert outcome.regressions == [] assert outcome.persistent_fails == [] assert outcome.stable_successes == [] assert outcome.candidate_correctness == {} assert outcome.evidence_rows == [] class TestProbationFields: """Probation 数据类型字段完整性测试。""" def test_probation_fields(self) -> None: """所有字段可构造、pending_edits 默认空列表。""" prob = Probation( task_type="temporal", anchor_skills_version="v1", target_file="temporal.md", correctness_snapshot={"q0": True, "q1": False}, opened_step=5, ) assert prob.task_type == "temporal" assert prob.anchor_skills_version == "v1" assert prob.target_file == "temporal.md" assert prob.correctness_snapshot == {"q0": True, "q1": False} assert prob.opened_step == 5 assert prob.pending_edits == [] def test_probation_with_pending_edits(self) -> None: """pending_edits 可附加 RejectedEdit。""" edit = RejectedEdit( target_file="temporal.md", target_type="skill", change_summary="bad change", delta=-0.1, source_version="v2", epoch=1, ) prob = Probation( task_type="temporal", anchor_skills_version="v1", target_file="temporal.md", correctness_snapshot={}, opened_step=3, pending_edits=[edit], ) assert len(prob.pending_edits) == 1 assert prob.pending_edits[0].change_summary == "bad change" # =========================================================================== # materialize 测试 # =========================================================================== class TestMaterializeCandidateSkill: """materialize_candidate_skill 物化与清理测试。""" def test_materialize_candidate_skill(self, tmp_path: Path) -> None: """正常物化:基线目录被复制,target_file 被覆写为候选内容。""" workspace = _setup_workspace(tmp_path) cand_dir = materialize_candidate_skill( workspace, "v1", "temporal.md", "candidate skill content" ) try: assert cand_dir.exists() assert cand_dir.parent == workspace / ".cand_tmp" assert (cand_dir / "temporal.md").read_text(encoding="utf-8") == ( "candidate skill content" ) finally: import shutil shutil.rmtree(cand_dir) def test_materialize_cleanup_on_failure(self, tmp_path: Path) -> None: """基线目录不存在时 OSError,临时目录被清理。""" workspace = tmp_path / "ws" workspace.mkdir() # 不创建 skills/v1,copytree 应失败 with pytest.raises(OSError): materialize_candidate_skill(workspace, "v1", "temporal.md", "content") # .cand_tmp 可能存在但内部应被清理 cand_tmp = workspace / ".cand_tmp" if cand_tmp.exists(): remaining = list(cand_tmp.iterdir()) assert remaining == [], f"临时目录未被清理: {remaining}" # =========================================================================== # async 验证测试(迁移自块序贯版 validate_skill_local) # =========================================================================== @pytest.mark.asyncio async def test_validate_concurrent_accept(tmp_path: Path) -> None: """候选全对、基线全错 → 高 e 值 → accept_confirmed(迁移自块序贯版)。 6 单元连胜:E=(2^(W+1)-1)/(W+1),前 5 单元 E<15 且不触方向/futility, 第 6 单元 E=18.14 ≥ e_confirm=15 → 与旧块判定同点收敛(W=6, n_used=6)。 """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) questions = _make_questions(6) cache = BaselineCache(workspace / "baseline_cache.json") # 基线全错,候选全对 → W=6, L=0 → E=18.14 → CONFIRMED(e_confirm=15) baseline_correct = {f"q{i}": False for i in range(6)} candidate_correct = {f"q{i}": True for i in range(6)} mock_fn, call_log = _make_mock_run_inference(log, baseline_correct, candidate_correct) # e_confirm=15 使 E=18.14 超过阈值触发 CONFIRMED accept_params = GateParams( e_confirm=15.0, e_provisional=3.0, w_net_min=2, delta_min=0.05, lambda_dir=-2.0, e_rollback=10.0, ) try: outcome = await _run_single_spec( workspace, _mk_spec(questions, candidate_content="improved skill"), mock_fn, log, cache, accept_params, ) assert outcome.accepted is True assert outcome.action == "accept_confirmed" assert outcome.stop_reason == "confirmed" assert outcome.w == 6 assert outcome.l == 0 assert outcome.n_used == 6 assert outcome.candidate_acc == 1.0 assert outcome.baseline_acc == 0.0 assert len(outcome.evidence_rows) == 6 # 阶梯序前缀消费:ladder_rank 连续(替代旧块边界断言) assert [r["ladder_rank"] for r in outcome.evidence_rows] == list(range(6)) # 终态证据行携带 stop_reason assert outcome.evidence_rows[-1]["stop_reason"] == "confirmed" # 候选临时目录应被清理 cand_tmp = workspace / ".cand_tmp" if cand_tmp.exists(): assert list(cand_tmp.iterdir()) == [] finally: log.close() @pytest.mark.asyncio async def test_validate_concurrent_reject_directional(tmp_path: Path) -> None: """候选全错、基线全对 → L 高 → 方向拒绝(迁移自块序贯版)。 前缀逐单元判定下早停点前移:15 单元阶梯保证 L=1..3 时 futility 不先触发 (E(w+n_rem, l) ≥ 3),L=4 时 Wald=4·ln0.6=-2.04 ≤ lambda_dir=-2.0 → directional 早停于第 4 单元(旧块版一次性判整块故 L=6)。 """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) questions = _make_questions(15) cache = BaselineCache(workspace / "baseline_cache.json") baseline_correct = {f"q{i}": True for i in range(15)} candidate_correct = {f"q{i}": False for i in range(15)} mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct) try: outcome = await _run_single_spec( workspace, _mk_spec(questions, candidate_content="bad skill"), mock_fn, log, cache, _DEFAULT_GATE_PARAMS, ) assert outcome.accepted is False assert outcome.action == "reject" assert outcome.stop_reason == "directional" assert outcome.w == 0 assert outcome.l == 4 assert outcome.n_used == 4 finally: log.close() @pytest.mark.asyncio async def test_gate_prefix_must_contain_gate(tmp_path: Path) -> None: """gate_run_prefix 不含 '_gate_' 时抛 ValueError(迁移自块序贯版)。""" workspace = _setup_workspace(tmp_path) log = _make_log(workspace) questions = _make_questions(4) cache = BaselineCache(workspace / "baseline_cache.json") async def noop_fn(questions, *, run_id, skills_dir): raise AssertionError("不应被调用") try: with pytest.raises(ValueError, match="_gate_"): await _run_single_spec( workspace, _mk_spec(questions, gate_run_prefix="step1_no_marker"), noop_fn, log, cache, _DEFAULT_GATE_PARAMS, ) finally: log.close() @pytest.mark.asyncio async def test_infra_guard_threshold(tmp_path: Path) -> None: """推理错误率超阈值时抛 RuntimeError(迁移自块序贯版,分子/分母 unit 同粒度)。 12 个 single 双臂全 INFRA error:errors 按单元去重逐单元 +1,分母逐臂 +1, 分母 ≥10 后错误率 >0.5 → 护栏熔断。 """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) questions = _make_questions(12) cache = BaselineCache(workspace / "baseline_cache.json") mock_fn, _ = _make_all_infra_mock(log, "error") try: with pytest.raises(RuntimeError, match="错误率过高"): await _run_single_spec( workspace, _mk_spec(questions), mock_fn, log, cache, _DEFAULT_GATE_PARAMS, ) finally: log.close() @pytest.mark.asyncio async def test_baseline_cache_hit(tmp_path: Path) -> None: """基线缓存全命中时不发起基线侧推理(迁移自块序贯版)。 连续并发 gate 下候选侧逐单元发臂:4 单元 → 4 次 cand 调用(旧块版整块 1 次)。 """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) questions = _make_questions(4) cache = BaselineCache(workspace / "baseline_cache.json") s_hash = skill_hash("baseline skill content") # 预填充缓存:全部题目基线全错 for q in questions: cache.put("temporal", s_hash, "p1", q.question_id, False) # 候选全对 → accept candidate_correct = {f"q{i}": True for i in range(4)} baseline_correct = {f"q{i}": False for i in range(4)} mock_fn, call_log = _make_mock_run_inference(log, baseline_correct, candidate_correct) try: outcome = await _run_single_spec( workspace, _mk_spec(questions, candidate_content="improved skill"), mock_fn, log, cache, _DEFAULT_GATE_PARAMS, ) # 只有候选侧调用了 run_inference(_cand),基线侧全命中不调用 base_calls = [c for c in call_log if c["run_id"].endswith("_base")] cand_calls = [c for c in call_log if c["run_id"].endswith("_cand")] assert len(base_calls) == 0, "基线缓存全命中不应发起推理" assert len(cand_calls) == 4 assert outcome.accepted is True finally: log.close() @pytest.mark.asyncio async def test_baseline_infra_error_not_cached(tmp_path: Path) -> None: """基线臂 INFRA error 的 unit 不写入 BaselineCache(不永久污染),且从配对剔除。 迁移自块序贯版 _resolve_baseline_block 直测:改经 validate_skills_concurrent 端到端验证同一契约——INFRA 单元不落缓存、不入配对;干净单元正常缓存并消费。 """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) questions = _make_questions(2) # q0 基线 INFRA error, q1 干净 cache = BaselineCache(workspace / "baseline_cache.json") s_hash = skill_hash("baseline skill content") async def mock_fn(qs, *, run_id, skills_dir): is_base = run_id.endswith("_base") for q in qs: is_err = is_base and q.question_id == "q0" log.insert( "predictions", { "run_id": run_id, "video_id": "v0", "question_id": q.question_id, "task_type": "temporal", "prediction": "" if is_err else "A", "answer": "A", "evidence": "", "reasoning": "", "steps_used": 1, "prompt_tokens": 10, "completion_tokens": 10, "stop_reason": "error" if is_err else "completed", "steps_json": "[]", }, ) return InferenceResult( run_id=run_id, accuracy=0.0, total=len(qs), correct=0, per_task_type={}, steps_mean=1.0, token_usage={"prompt_tokens": 10, "completion_tokens": 10}, stop_reason_counts={}, ) try: outcome = await _run_single_spec( workspace, _mk_spec(questions), mock_fn, log, cache, _DEFAULT_GATE_PARAMS, gate_guard_err=0.9, # 分母 <10 不触发错误率护栏 ) # q0 是 INFRA:不写缓存、不入配对观测 assert cache.get("temporal", s_hash, "p1", "q0") is None assert "q0" not in outcome.improvements + outcome.regressions # q1 干净:正常缓存并被消费(唯一有效单元) assert cache.get("temporal", s_hash, "p1", "q1") is True assert outcome.n_used == 1 finally: log.close() @pytest.mark.asyncio async def test_infra_errors_counted_per_unit_not_per_record(tmp_path: Path) -> None: """护栏分子按 unit 去重:AR pair 两 record、双臂全 INFRA 只计 1 个 error。 迁移自块序贯版 _resolve_baseline_block 直测(回归 I-3):分子若逐 record / 逐臂计数会被放大(一 unit 两 record × 两臂 = 4),与 unit 粒度分母失配致 gate_guard_err 误触发。新载体 _run_unit_arm + _register_arm_arrival 按 slot.excluded() 去重(核心算法保真 #5/#6)。 """ from app.harness.question_units import build_units from app.harness.validate import _GateRun, _QuestionSlots, _run_unit_arm workspace = _setup_workspace(tmp_path) log = _make_log(workspace) common = { "video_id": "vp", "task_type": "temporal", "question": "Q?", "options": ("A", "B", "C", "D"), "answer": "A", "source_nodes": (), "difficulty": "easy", "pair_id": "p1", "flip_axis": "before_after", } pair = [ GeneratedQuestion(question_id="p1_o", question_role="pair_original", **common), GeneratedQuestion(question_id="p1_m", question_role="pair_mirror", **common), ] units = build_units(pair) assert len(units) == 1 # 前置:pair 折叠为 1 个 unit cache = BaselineCache(workspace / "baseline_cache.json") run = _GateRun.from_spec( GateSpec( task_type="temporal", target_file="temporal.md", candidate_content="cand", base_skill_content="baseline skill content", units=tuple(units), gate_run_prefix="step1_gate_test", ) ) s_hash = run.s_hash async def mock_fn(qs, *, run_id, skills_dir): # 两 record 皆 INFRA error for q in qs: log.insert( "predictions", { "run_id": run_id, "video_id": "vp", "question_id": q.question_id, "task_type": "temporal", "prediction": "", "answer": "A", "evidence": "", "reasoning": "", "steps_used": 1, "prompt_tokens": 10, "completion_tokens": 10, "stop_reason": "error", "steps_json": "[]", }, ) # total 为 unit 粒度(1 个 pair unit);record 粒度为 2 return InferenceResult( run_id=run_id, accuracy=0.0, total=1, correct=0, per_task_type={}, steps_mean=1.0, token_usage={"prompt_tokens": 20, "completion_tokens": 20}, stop_reason_counts={"error": 2}, ) slots = _QuestionSlots(4) try: for arm in ("base", "cand"): await _run_unit_arm( run, 0, arm, slots, mock_fn, log, cache, "p1", workspace / "skills" / "v1", workspace / "skills" / "v1", _DEFAULT_GATE_PARAMS, 0.9, ) # 分子按 unit 去重:双臂 × 两 record 只计 1 个 error;分母按臂 total 累计 = 2 assert run.errors == 1 assert run.infra_denom == 2 assert run.slots[0].base_infra and run.slots[0].cand_infra # INFRA 单元不写缓存 assert cache.get("temporal", s_hash, "p1", "p1") is None finally: log.close() @pytest.mark.asyncio async def test_all_infra_ladder_raises_clear_error(tmp_path: Path) -> None: """整个阶梯所有 unit 都被判为 INFRA 排除 → 明确 RuntimeError(迁移自块序贯版)。 连续并发 gate 下双臂独立发射,候选臂不再依赖基线侧结果(旧版"全 INFRA 块 不空跑候选"的断言随块编排一并删除)。 """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) questions = _make_questions(4) cache = BaselineCache(workspace / "baseline_cache.json") mock_fn, _ = _make_all_infra_mock(log, "error") try: with pytest.raises(RuntimeError, match="INFRA"): await _run_single_spec( workspace, _mk_spec(questions), mock_fn, log, cache, _DEFAULT_GATE_PARAMS, gate_guard_err=0.9, # 4 单元分母 <10 不触发错误率护栏 → 逼出全排除分支 ) finally: log.close() @pytest.mark.asyncio async def test_parse_error_counts_toward_guard(tmp_path: Path) -> None: """stop_reason=parse_error 也计入护栏错误率(与 INFRA 判定口径一致)→ 超阈值熔断。""" workspace = _setup_workspace(tmp_path) log = _make_log(workspace) questions = _make_questions(12) cache = BaselineCache(workspace / "baseline_cache.json") mock_fn, _ = _make_all_infra_mock(log, "parse_error") try: with pytest.raises(RuntimeError, match="错误率过高"): await _run_single_spec( workspace, _mk_spec(questions), mock_fn, log, cache, _DEFAULT_GATE_PARAMS, ) finally: log.close() @pytest.mark.asyncio async def test_ladder_exhaustion_terminal(tmp_path: Path) -> None: """题尽(n_remaining=0)→ 终态判定(provisional 或 inertia),非 continue。 迁移自块序贯版"最后一块终态":块边界不存在了,等价语义是阶梯耗尽时 第四出口兜底,终态行携带 stop_reason。 """ workspace = _setup_workspace(tmp_path) log = _make_log(workspace) questions = _make_questions(4) cache = BaselineCache(workspace / "baseline_cache.json") # 两题翻转(W=2, L=0),但 e_confirm=20 难以达到 → provisional 或 inertia baseline_correct = {"q0": False, "q1": False, "q2": True, "q3": True} candidate_correct = {"q0": True, "q1": True, "q2": True, "q3": True} mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct) try: outcome = await _run_single_spec( workspace, _mk_spec(questions), mock_fn, log, cache, _DEFAULT_GATE_PARAMS, ) # n_remaining=0 → 不可能是 continue assert outcome.stop_reason in ( "confirmed", "provisional", "inertia", "directional", "futility", ) assert outcome.n_used == 4 # 阶梯序前缀消费:ladder_rank 连续 assert [r["ladder_rank"] for r in outcome.evidence_rows] == list(range(4)) # 终态行标记 stop_reason assert outcome.evidence_rows[-1]["stop_reason"] != "" finally: log.close()