feat(harness): validate.py — async 块序贯验证编排 + Probation 统一定义

2026-07-07 13:20:43 -04:00
parent a6b816db94
commit 7bc6fc752c
3 changed files with 1220 additions and 25 deletions
@@ -0,0 +1,554 @@
+"""tests/unit/test_harness_validate.py — app/harness/validate.py 的单元测试。
+
+覆盖：数据类型字段、materialize 物化与清理、async validate_skill_local
+（accept/reject/prefix 校验/INFRA 护栏/缓存命中/最后一块终态）。
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from app.harness.gate_ladder import BaselineCache, skill_hash
+from app.harness.inference import PREDICTIONS_SCHEMA, InferenceResult
+from app.harness.log import HarnessLog
+from app.harness.validate import (
+    Probation,
+    ValidationOutcome,
+    materialize_candidate_skill,
+    validate_skill_local,
+)
+from core.evolution import GateParams, RejectedEdit
+from core.types import GeneratedQuestion
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+# ---------------------------------------------------------------------------
+# 辅助工具
+# ---------------------------------------------------------------------------
+
+_DEFAULT_GATE_PARAMS = GateParams(
+    e_confirm=20.0,
+    e_provisional=3.0,
+    w_net_min=2,
+    delta_min=0.05,
+    lambda_dir=-2.0,
+    e_rollback=10.0,
+)
+
+
+def _make_questions(
+    n: int,
+    task_type: str = "temporal",
+    prefix: str = "q",
+) -> list[GeneratedQuestion]:
+    """生成 n 个测试用 GeneratedQuestion。"""
+    return [
+        GeneratedQuestion(
+            question_id=f"{prefix}{i}",
+            video_id=f"v{i}",
+            task_type=task_type,
+            question=f"Question {i}?",
+            options=("A", "B", "C", "D"),
+            answer="A",
+            source_nodes=(),
+            difficulty="easy",
+        )
+        for i in range(n)
+    ]
+
+
+def _setup_workspace(tmp_path: Path) -> Path:
+    """在 tmp_path 下构建最小 workspace 结构。"""
+    skills_dir = tmp_path / "skills" / "v1"
+    skills_dir.mkdir(parents=True)
+    (skills_dir / "temporal.md").write_text("baseline skill content", encoding="utf-8")
+    return tmp_path
+
+
+def _make_log(workspace: Path, run_id: str = "test_master") -> HarnessLog:
+    """创建 HarnessLog 并初始化 predictions 表。"""
+    db_path = workspace / "harness.db"
+    log = HarnessLog(str(db_path), run_id)
+    log.create_table("predictions", PREDICTIONS_SCHEMA)
+    return log
+
+
+def _insert_predictions(
+    log: HarnessLog,
+    run_id: str,
+    correctness: dict[str, bool],
+    answer: str = "A",
+) -> None:
+    """向 predictions 表插入指定 run_id 的逐题预测记录。
+
+    通过在 record 中显式传入 run_id 覆盖 log 的默认 run_id。
+    """
+    for qid, correct in correctness.items():
+        prediction = answer if correct else "Z"
+        log.insert(
+            "predictions",
+            {
+                "run_id": run_id,
+                "video_id": "v0",
+                "question_id": qid,
+                "task_type": "temporal",
+                "prediction": prediction,
+                "answer": answer,
+                "evidence": "",
+                "reasoning": "",
+                "steps_used": 1,
+                "prompt_tokens": 10,
+                "completion_tokens": 10,
+                "stop_reason": "completed",
+                "steps_json": "[]",
+            },
+        )
+
+
+def _make_mock_run_inference(
+    log: HarnessLog,
+    baseline_correctness: dict[str, bool],
+    candidate_correctness: dict[str, bool],
+    error_count: int = 0,
+):
+    """构建 mock RunInferenceFn。
+
+    根据 run_id 中的 arm 标记（_base / _cand）决定使用基线或候选对错映射，
+    将预测写入 log 的同一 DB，返回 InferenceResult。
+    """
+    call_log: list[dict[str, Any]] = []
+
+    async def mock_fn(
+        questions: list[GeneratedQuestion],
+        *,
+        run_id: str,
+        skills_dir: Path,
+    ) -> InferenceResult:
+        is_baseline = run_id.endswith("_base")
+        correctness = baseline_correctness if is_baseline else candidate_correctness
+
+        call_log.append({"run_id": run_id, "skills_dir": skills_dir, "n": len(questions)})
+        per_q = {q.question_id: correctness.get(q.question_id, False) for q in questions}
+        _insert_predictions(log, run_id, per_q)
+
+        correct = sum(per_q.values())
+        total = len(questions)
+        stop_counts: dict[str, int] = {"completed": total - error_count}
+        if error_count > 0:
+            stop_counts["error"] = error_count
+        return InferenceResult(
+            run_id=run_id,
+            accuracy=correct / total if total else 0.0,
+            total=total,
+            correct=correct,
+            per_task_type={},
+            steps_mean=1.0,
+            token_usage={"prompt_tokens": 10, "completion_tokens": 10},
+            stop_reason_counts=stop_counts,
+        )
+
+    return mock_fn, call_log
+
+
+# ===========================================================================
+# 数据类型测试
+# ===========================================================================
+
+
+class TestValidationOutcomeFields:
+    """ValidationOutcome 数据类型字段完整性测试。"""
+
+    def test_validation_outcome_fields(self) -> None:
+        """所有字段可构造、默认值合理。"""
+        outcome = ValidationOutcome(
+            action="accept_confirmed",
+            accepted=True,
+            stop_reason="confirmed",
+            e_value=25.0,
+            w=5,
+            l=1,
+            n_used=10,
+            delta_hat=0.4,
+            delta_shrunk=0.3,
+            baseline_acc=0.6,
+            candidate_acc=0.9,
+        )
+        assert outcome.action == "accept_confirmed"
+        assert outcome.accepted is True
+        assert outcome.stop_reason == "confirmed"
+        assert outcome.e_value == 25.0
+        assert outcome.w == 5
+        assert outcome.l == 1
+        assert outcome.n_used == 10
+        assert outcome.delta_hat == 0.4
+        assert outcome.delta_shrunk == 0.3
+        assert outcome.baseline_acc == 0.6
+        assert outcome.candidate_acc == 0.9
+        assert outcome.improvements == []
+        assert outcome.regressions == []
+        assert outcome.persistent_fails == []
+        assert outcome.stable_successes == []
+        assert outcome.candidate_correctness == {}
+        assert outcome.evidence_rows == []
+
+
+class TestProbationFields:
+    """Probation 数据类型字段完整性测试。"""
+
+    def test_probation_fields(self) -> None:
+        """所有字段可构造、pending_edits 默认空列表。"""
+        prob = Probation(
+            task_type="temporal",
+            anchor_skills_version="v1",
+            target_file="temporal.md",
+            correctness_snapshot={"q0": True, "q1": False},
+            opened_step=5,
+        )
+        assert prob.task_type == "temporal"
+        assert prob.anchor_skills_version == "v1"
+        assert prob.target_file == "temporal.md"
+        assert prob.correctness_snapshot == {"q0": True, "q1": False}
+        assert prob.opened_step == 5
+        assert prob.pending_edits == []
+
+    def test_probation_with_pending_edits(self) -> None:
+        """pending_edits 可附加 RejectedEdit。"""
+        edit = RejectedEdit(
+            target_file="temporal.md",
+            target_type="skill",
+            change_summary="bad change",
+            delta=-0.1,
+            source_version="v2",
+            epoch=1,
+        )
+        prob = Probation(
+            task_type="temporal",
+            anchor_skills_version="v1",
+            target_file="temporal.md",
+            correctness_snapshot={},
+            opened_step=3,
+            pending_edits=[edit],
+        )
+        assert len(prob.pending_edits) == 1
+        assert prob.pending_edits[0].change_summary == "bad change"
+
+
+# ===========================================================================
+# materialize 测试
+# ===========================================================================
+
+
+class TestMaterializeCandidateSkill:
+    """materialize_candidate_skill 物化与清理测试。"""
+
+    def test_materialize_candidate_skill(self, tmp_path: Path) -> None:
+        """正常物化：基线目录被复制，target_file 被覆写为候选内容。"""
+        workspace = _setup_workspace(tmp_path)
+        cand_dir = materialize_candidate_skill(
+            workspace, "v1", "temporal.md", "candidate skill content"
+        )
+        try:
+            assert cand_dir.exists()
+            assert cand_dir.parent == workspace / ".cand_tmp"
+            assert (cand_dir / "temporal.md").read_text(encoding="utf-8") == (
+                "candidate skill content"
+            )
+        finally:
+            import shutil
+
+            shutil.rmtree(cand_dir)
+
+    def test_materialize_cleanup_on_failure(self, tmp_path: Path) -> None:
+        """基线目录不存在时 OSError，临时目录被清理。"""
+        workspace = tmp_path / "ws"
+        workspace.mkdir()
+        # 不创建 skills/v1，copytree 应失败
+        with pytest.raises(OSError):
+            materialize_candidate_skill(workspace, "v1", "temporal.md", "content")
+        # .cand_tmp 可能存在但内部应被清理
+        cand_tmp = workspace / ".cand_tmp"
+        if cand_tmp.exists():
+            remaining = list(cand_tmp.iterdir())
+            assert remaining == [], f"临时目录未被清理: {remaining}"
+
+
+# ===========================================================================
+# async 验证测试
+# ===========================================================================
+
+
+@pytest.mark.asyncio
+async def test_validate_skill_local_accept(tmp_path: Path) -> None:
+    """候选全对、基线全错 → 高 e 值 → accept_confirmed。"""
+    workspace = _setup_workspace(tmp_path)
+    log = _make_log(workspace)
+    questions = _make_questions(6)
+    cache = BaselineCache(workspace / "baseline_cache.json")
+
+    # 基线全错，候选全对 → W=6, L=0 → E=18.14 → CONFIRMED（e_confirm=15）
+    baseline_correct = {f"q{i}": False for i in range(6)}
+    candidate_correct = {f"q{i}": True for i in range(6)}
+    mock_fn, call_log = _make_mock_run_inference(log, baseline_correct, candidate_correct)
+
+    # e_confirm=15 使 E=18.14 超过阈值触发 CONFIRMED
+    accept_params = GateParams(
+        e_confirm=15.0,
+        e_provisional=3.0,
+        w_net_min=2,
+        delta_min=0.05,
+        lambda_dir=-2.0,
+        e_rollback=10.0,
+    )
+
+    try:
+        outcome = await validate_skill_local(
+            workspace_dir=workspace,
+            base_skills_version="v1",
+            task_type="temporal",
+            target_file="temporal.md",
+            candidate_content="improved skill",
+            base_skill_content="baseline skill content",
+            ladder_items=questions,
+            gate_params=accept_params,
+            gate_block=6,
+            gate_n_max=20,
+            gate_guard_err=0.5,
+            baseline_cache=cache,
+            prompts_version="p1",
+            run_inference=mock_fn,
+            log=log,
+            gate_run_prefix="step1_gate_test",
+        )
+
+        assert outcome.accepted is True
+        assert outcome.action == "accept_confirmed"
+        assert outcome.stop_reason == "confirmed"
+        assert outcome.w == 6
+        assert outcome.l == 0
+        assert outcome.n_used == 6
+        assert outcome.candidate_acc == 1.0
+        assert outcome.baseline_acc == 0.0
+        assert len(outcome.evidence_rows) == 6
+        # 终态证据行携带 stop_reason
+        assert outcome.evidence_rows[-1]["stop_reason"] == "confirmed"
+        # 候选临时目录应被清理
+        cand_tmp = workspace / ".cand_tmp"
+        if cand_tmp.exists():
+            assert list(cand_tmp.iterdir()) == []
+    finally:
+        log.close()
+
+
+@pytest.mark.asyncio
+async def test_validate_skill_local_reject(tmp_path: Path) -> None:
+    """候选全错、基线全对 → L 高 → 方向拒绝。"""
+    workspace = _setup_workspace(tmp_path)
+    log = _make_log(workspace)
+    questions = _make_questions(6)
+    cache = BaselineCache(workspace / "baseline_cache.json")
+
+    # 基线全对，候选全错 → W=0, L=6 → 方向拒绝
+    baseline_correct = {f"q{i}": True for i in range(6)}
+    candidate_correct = {f"q{i}": False for i in range(6)}
+    mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct)
+
+    try:
+        outcome = await validate_skill_local(
+            workspace_dir=workspace,
+            base_skills_version="v1",
+            task_type="temporal",
+            target_file="temporal.md",
+            candidate_content="bad skill",
+            base_skill_content="baseline skill content",
+            ladder_items=questions,
+            gate_params=_DEFAULT_GATE_PARAMS,
+            gate_block=6,
+            gate_n_max=20,
+            gate_guard_err=0.5,
+            baseline_cache=cache,
+            prompts_version="p1",
+            run_inference=mock_fn,
+            log=log,
+            gate_run_prefix="step1_gate_test",
+        )
+
+        assert outcome.accepted is False
+        assert outcome.action == "reject"
+        assert outcome.stop_reason == "directional"
+        assert outcome.w == 0
+        assert outcome.l == 6
+    finally:
+        log.close()
+
+
+@pytest.mark.asyncio
+async def test_gate_prefix_must_contain_gate(tmp_path: Path) -> None:
+    """gate_run_prefix 不含 '_gate_' 时抛 ValueError。"""
+    workspace = _setup_workspace(tmp_path)
+    log = _make_log(workspace)
+    questions = _make_questions(4)
+    cache = BaselineCache(workspace / "baseline_cache.json")
+
+    async def noop_fn(questions, *, run_id, skills_dir):
+        raise AssertionError("不应被调用")
+
+    try:
+        with pytest.raises(ValueError, match="_gate_"):
+            await validate_skill_local(
+                workspace_dir=workspace,
+                base_skills_version="v1",
+                task_type="temporal",
+                target_file="temporal.md",
+                candidate_content="content",
+                base_skill_content="baseline",
+                ladder_items=questions,
+                gate_params=_DEFAULT_GATE_PARAMS,
+                gate_block=4,
+                gate_n_max=20,
+                gate_guard_err=0.5,
+                baseline_cache=cache,
+                prompts_version="p1",
+                run_inference=noop_fn,
+                log=log,
+                gate_run_prefix="step1_no_marker",
+            )
+    finally:
+        log.close()
+
+
+@pytest.mark.asyncio
+async def test_infra_guard_threshold(tmp_path: Path) -> None:
+    """推理错误率超阈值时抛 RuntimeError。"""
+    workspace = _setup_workspace(tmp_path)
+    log = _make_log(workspace)
+    # 需要 >=10 题次才触发 INFRA 护栏
+    questions = _make_questions(6)
+    cache = BaselineCache(workspace / "baseline_cache.json")
+
+    baseline_correct = {f"q{i}": False for i in range(6)}
+    candidate_correct = {f"q{i}": False for i in range(6)}
+    # 每次 run_inference 报 error_count=5，两侧各 5 → 10/12 > 0.5
+    mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct, error_count=5)
+
+    try:
+        with pytest.raises(RuntimeError, match="错误率过高"):
+            await validate_skill_local(
+                workspace_dir=workspace,
+                base_skills_version="v1",
+                task_type="temporal",
+                target_file="temporal.md",
+                candidate_content="content",
+                base_skill_content="baseline skill content",
+                ladder_items=questions,
+                gate_params=_DEFAULT_GATE_PARAMS,
+                gate_block=6,
+                gate_n_max=20,
+                gate_guard_err=0.5,
+                baseline_cache=cache,
+                prompts_version="p1",
+                run_inference=mock_fn,
+                log=log,
+                gate_run_prefix="step1_gate_test",
+            )
+    finally:
+        log.close()
+
+
+@pytest.mark.asyncio
+async def test_baseline_cache_hit(tmp_path: Path) -> None:
+    """基线缓存全命中时不发起基线侧推理。"""
+    workspace = _setup_workspace(tmp_path)
+    log = _make_log(workspace)
+    questions = _make_questions(4)
+    cache = BaselineCache(workspace / "baseline_cache.json")
+
+    s_hash = skill_hash("baseline skill content")
+    # 预填充缓存：全部题目基线全错
+    for q in questions:
+        cache.put("temporal", s_hash, "p1", q.question_id, False)
+
+    # 候选全对 → accept
+    candidate_correct = {f"q{i}": True for i in range(4)}
+    baseline_correct = {f"q{i}": False for i in range(4)}
+    mock_fn, call_log = _make_mock_run_inference(log, baseline_correct, candidate_correct)
+
+    try:
+        outcome = await validate_skill_local(
+            workspace_dir=workspace,
+            base_skills_version="v1",
+            task_type="temporal",
+            target_file="temporal.md",
+            candidate_content="improved skill",
+            base_skill_content="baseline skill content",
+            ladder_items=questions,
+            gate_params=_DEFAULT_GATE_PARAMS,
+            gate_block=4,
+            gate_n_max=20,
+            gate_guard_err=0.5,
+            baseline_cache=cache,
+            prompts_version="p1",
+            run_inference=mock_fn,
+            log=log,
+            gate_run_prefix="step1_gate_test",
+        )
+
+        # 只有候选侧调用了 run_inference（_cand），基线侧全命中不调用
+        base_calls = [c for c in call_log if c["run_id"].endswith("_base")]
+        cand_calls = [c for c in call_log if c["run_id"].endswith("_cand")]
+        assert len(base_calls) == 0, "基线缓存全命中不应发起推理"
+        assert len(cand_calls) == 1
+        assert outcome.accepted is True
+    finally:
+        log.close()
+
+
+@pytest.mark.asyncio
+async def test_last_block_terminal(tmp_path: Path) -> None:
+    """单块 + n_remaining=0 → 终态判定（provisional 或 inertia），非 continue。"""
+    workspace = _setup_workspace(tmp_path)
+    log = _make_log(workspace)
+    # 4 题，gate_block=4 → 一块走完，n_remaining=0
+    questions = _make_questions(4)
+    cache = BaselineCache(workspace / "baseline_cache.json")
+
+    # 两题翻转（W=2, L=0），但 e_confirm=20 难以达到 → provisional 或 inertia
+    baseline_correct = {"q0": False, "q1": False, "q2": True, "q3": True}
+    candidate_correct = {"q0": True, "q1": True, "q2": True, "q3": True}
+    mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct)
+
+    try:
+        outcome = await validate_skill_local(
+            workspace_dir=workspace,
+            base_skills_version="v1",
+            task_type="temporal",
+            target_file="temporal.md",
+            candidate_content="candidate skill",
+            base_skill_content="baseline skill content",
+            ladder_items=questions,
+            gate_params=_DEFAULT_GATE_PARAMS,
+            gate_block=4,
+            gate_n_max=4,
+            gate_guard_err=0.5,
+            baseline_cache=cache,
+            prompts_version="p1",
+            run_inference=mock_fn,
+            log=log,
+            gate_run_prefix="step1_gate_test",
+        )
+
+        # n_remaining=0 → 不可能是 continue
+        assert outcome.stop_reason in (
+            "confirmed",
+            "provisional",
+            "inertia",
+            "directional",
+            "futility",
+        )
+        assert outcome.n_used == 4
+        # 终态行标记 stop_reason
+        assert outcome.evidence_rows[-1]["stop_reason"] != ""
+    finally:
+        log.close()