555 lines
19 KiB
Python
555 lines
19 KiB
Python
"""tests/unit/test_harness_validate.py — app/harness/validate.py 的单元测试。
|
||
|
||
覆盖：数据类型字段、materialize 物化与清理、async validate_skill_local
|
||
（accept/reject/prefix 校验/INFRA 护栏/缓存命中/最后一块终态）。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import TYPE_CHECKING, Any
|
||
|
||
import pytest
|
||
|
||
from app.harness.gate_ladder import BaselineCache, skill_hash
|
||
from app.harness.inference import PREDICTIONS_SCHEMA, InferenceResult
|
||
from app.harness.log import HarnessLog
|
||
from app.harness.validate import (
|
||
Probation,
|
||
ValidationOutcome,
|
||
materialize_candidate_skill,
|
||
validate_skill_local,
|
||
)
|
||
from core.evolution import GateParams, RejectedEdit
|
||
from core.types import GeneratedQuestion
|
||
|
||
if TYPE_CHECKING:
|
||
from pathlib import Path
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 辅助工具
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_DEFAULT_GATE_PARAMS = GateParams(
|
||
e_confirm=20.0,
|
||
e_provisional=3.0,
|
||
w_net_min=2,
|
||
delta_min=0.05,
|
||
lambda_dir=-2.0,
|
||
e_rollback=10.0,
|
||
)
|
||
|
||
|
||
def _make_questions(
|
||
n: int,
|
||
task_type: str = "temporal",
|
||
prefix: str = "q",
|
||
) -> list[GeneratedQuestion]:
|
||
"""生成 n 个测试用 GeneratedQuestion。"""
|
||
return [
|
||
GeneratedQuestion(
|
||
question_id=f"{prefix}{i}",
|
||
video_id=f"v{i}",
|
||
task_type=task_type,
|
||
question=f"Question {i}?",
|
||
options=("A", "B", "C", "D"),
|
||
answer="A",
|
||
source_nodes=(),
|
||
difficulty="easy",
|
||
)
|
||
for i in range(n)
|
||
]
|
||
|
||
|
||
def _setup_workspace(tmp_path: Path) -> Path:
|
||
"""在 tmp_path 下构建最小 workspace 结构。"""
|
||
skills_dir = tmp_path / "skills" / "v1"
|
||
skills_dir.mkdir(parents=True)
|
||
(skills_dir / "temporal.md").write_text("baseline skill content", encoding="utf-8")
|
||
return tmp_path
|
||
|
||
|
||
def _make_log(workspace: Path, run_id: str = "test_master") -> HarnessLog:
|
||
"""创建 HarnessLog 并初始化 predictions 表。"""
|
||
db_path = workspace / "harness.db"
|
||
log = HarnessLog(str(db_path), run_id)
|
||
log.create_table("predictions", PREDICTIONS_SCHEMA)
|
||
return log
|
||
|
||
|
||
def _insert_predictions(
|
||
log: HarnessLog,
|
||
run_id: str,
|
||
correctness: dict[str, bool],
|
||
answer: str = "A",
|
||
) -> None:
|
||
"""向 predictions 表插入指定 run_id 的逐题预测记录。
|
||
|
||
通过在 record 中显式传入 run_id 覆盖 log 的默认 run_id。
|
||
"""
|
||
for qid, correct in correctness.items():
|
||
prediction = answer if correct else "Z"
|
||
log.insert(
|
||
"predictions",
|
||
{
|
||
"run_id": run_id,
|
||
"video_id": "v0",
|
||
"question_id": qid,
|
||
"task_type": "temporal",
|
||
"prediction": prediction,
|
||
"answer": answer,
|
||
"evidence": "",
|
||
"reasoning": "",
|
||
"steps_used": 1,
|
||
"prompt_tokens": 10,
|
||
"completion_tokens": 10,
|
||
"stop_reason": "completed",
|
||
"steps_json": "[]",
|
||
},
|
||
)
|
||
|
||
|
||
def _make_mock_run_inference(
|
||
log: HarnessLog,
|
||
baseline_correctness: dict[str, bool],
|
||
candidate_correctness: dict[str, bool],
|
||
error_count: int = 0,
|
||
):
|
||
"""构建 mock RunInferenceFn。
|
||
|
||
根据 run_id 中的 arm 标记（_base / _cand）决定使用基线或候选对错映射，
|
||
将预测写入 log 的同一 DB，返回 InferenceResult。
|
||
"""
|
||
call_log: list[dict[str, Any]] = []
|
||
|
||
async def mock_fn(
|
||
questions: list[GeneratedQuestion],
|
||
*,
|
||
run_id: str,
|
||
skills_dir: Path,
|
||
) -> InferenceResult:
|
||
is_baseline = run_id.endswith("_base")
|
||
correctness = baseline_correctness if is_baseline else candidate_correctness
|
||
|
||
call_log.append({"run_id": run_id, "skills_dir": skills_dir, "n": len(questions)})
|
||
per_q = {q.question_id: correctness.get(q.question_id, False) for q in questions}
|
||
_insert_predictions(log, run_id, per_q)
|
||
|
||
correct = sum(per_q.values())
|
||
total = len(questions)
|
||
stop_counts: dict[str, int] = {"completed": total - error_count}
|
||
if error_count > 0:
|
||
stop_counts["error"] = error_count
|
||
return InferenceResult(
|
||
run_id=run_id,
|
||
accuracy=correct / total if total else 0.0,
|
||
total=total,
|
||
correct=correct,
|
||
per_task_type={},
|
||
steps_mean=1.0,
|
||
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
|
||
stop_reason_counts=stop_counts,
|
||
)
|
||
|
||
return mock_fn, call_log
|
||
|
||
|
||
# ===========================================================================
|
||
# 数据类型测试
|
||
# ===========================================================================
|
||
|
||
|
||
class TestValidationOutcomeFields:
|
||
"""ValidationOutcome 数据类型字段完整性测试。"""
|
||
|
||
def test_validation_outcome_fields(self) -> None:
|
||
"""所有字段可构造、默认值合理。"""
|
||
outcome = ValidationOutcome(
|
||
action="accept_confirmed",
|
||
accepted=True,
|
||
stop_reason="confirmed",
|
||
e_value=25.0,
|
||
w=5,
|
||
l=1,
|
||
n_used=10,
|
||
delta_hat=0.4,
|
||
delta_shrunk=0.3,
|
||
baseline_acc=0.6,
|
||
candidate_acc=0.9,
|
||
)
|
||
assert outcome.action == "accept_confirmed"
|
||
assert outcome.accepted is True
|
||
assert outcome.stop_reason == "confirmed"
|
||
assert outcome.e_value == 25.0
|
||
assert outcome.w == 5
|
||
assert outcome.l == 1
|
||
assert outcome.n_used == 10
|
||
assert outcome.delta_hat == 0.4
|
||
assert outcome.delta_shrunk == 0.3
|
||
assert outcome.baseline_acc == 0.6
|
||
assert outcome.candidate_acc == 0.9
|
||
assert outcome.improvements == []
|
||
assert outcome.regressions == []
|
||
assert outcome.persistent_fails == []
|
||
assert outcome.stable_successes == []
|
||
assert outcome.candidate_correctness == {}
|
||
assert outcome.evidence_rows == []
|
||
|
||
|
||
class TestProbationFields:
|
||
"""Probation 数据类型字段完整性测试。"""
|
||
|
||
def test_probation_fields(self) -> None:
|
||
"""所有字段可构造、pending_edits 默认空列表。"""
|
||
prob = Probation(
|
||
task_type="temporal",
|
||
anchor_skills_version="v1",
|
||
target_file="temporal.md",
|
||
correctness_snapshot={"q0": True, "q1": False},
|
||
opened_step=5,
|
||
)
|
||
assert prob.task_type == "temporal"
|
||
assert prob.anchor_skills_version == "v1"
|
||
assert prob.target_file == "temporal.md"
|
||
assert prob.correctness_snapshot == {"q0": True, "q1": False}
|
||
assert prob.opened_step == 5
|
||
assert prob.pending_edits == []
|
||
|
||
def test_probation_with_pending_edits(self) -> None:
|
||
"""pending_edits 可附加 RejectedEdit。"""
|
||
edit = RejectedEdit(
|
||
target_file="temporal.md",
|
||
target_type="skill",
|
||
change_summary="bad change",
|
||
delta=-0.1,
|
||
source_version="v2",
|
||
epoch=1,
|
||
)
|
||
prob = Probation(
|
||
task_type="temporal",
|
||
anchor_skills_version="v1",
|
||
target_file="temporal.md",
|
||
correctness_snapshot={},
|
||
opened_step=3,
|
||
pending_edits=[edit],
|
||
)
|
||
assert len(prob.pending_edits) == 1
|
||
assert prob.pending_edits[0].change_summary == "bad change"
|
||
|
||
|
||
# ===========================================================================
|
||
# materialize 测试
|
||
# ===========================================================================
|
||
|
||
|
||
class TestMaterializeCandidateSkill:
|
||
"""materialize_candidate_skill 物化与清理测试。"""
|
||
|
||
def test_materialize_candidate_skill(self, tmp_path: Path) -> None:
|
||
"""正常物化：基线目录被复制，target_file 被覆写为候选内容。"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
cand_dir = materialize_candidate_skill(
|
||
workspace, "v1", "temporal.md", "candidate skill content"
|
||
)
|
||
try:
|
||
assert cand_dir.exists()
|
||
assert cand_dir.parent == workspace / ".cand_tmp"
|
||
assert (cand_dir / "temporal.md").read_text(encoding="utf-8") == (
|
||
"candidate skill content"
|
||
)
|
||
finally:
|
||
import shutil
|
||
|
||
shutil.rmtree(cand_dir)
|
||
|
||
def test_materialize_cleanup_on_failure(self, tmp_path: Path) -> None:
|
||
"""基线目录不存在时 OSError，临时目录被清理。"""
|
||
workspace = tmp_path / "ws"
|
||
workspace.mkdir()
|
||
# 不创建 skills/v1，copytree 应失败
|
||
with pytest.raises(OSError):
|
||
materialize_candidate_skill(workspace, "v1", "temporal.md", "content")
|
||
# .cand_tmp 可能存在但内部应被清理
|
||
cand_tmp = workspace / ".cand_tmp"
|
||
if cand_tmp.exists():
|
||
remaining = list(cand_tmp.iterdir())
|
||
assert remaining == [], f"临时目录未被清理: {remaining}"
|
||
|
||
|
||
# ===========================================================================
|
||
# async 验证测试
|
||
# ===========================================================================
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_validate_skill_local_accept(tmp_path: Path) -> None:
|
||
"""候选全对、基线全错 → 高 e 值 → accept_confirmed。"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
questions = _make_questions(6)
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
|
||
# 基线全错，候选全对 → W=6, L=0 → E=18.14 → CONFIRMED（e_confirm=15）
|
||
baseline_correct = {f"q{i}": False for i in range(6)}
|
||
candidate_correct = {f"q{i}": True for i in range(6)}
|
||
mock_fn, call_log = _make_mock_run_inference(log, baseline_correct, candidate_correct)
|
||
|
||
# e_confirm=15 使 E=18.14 超过阈值触发 CONFIRMED
|
||
accept_params = GateParams(
|
||
e_confirm=15.0,
|
||
e_provisional=3.0,
|
||
w_net_min=2,
|
||
delta_min=0.05,
|
||
lambda_dir=-2.0,
|
||
e_rollback=10.0,
|
||
)
|
||
|
||
try:
|
||
outcome = await validate_skill_local(
|
||
workspace_dir=workspace,
|
||
base_skills_version="v1",
|
||
task_type="temporal",
|
||
target_file="temporal.md",
|
||
candidate_content="improved skill",
|
||
base_skill_content="baseline skill content",
|
||
ladder_items=questions,
|
||
gate_params=accept_params,
|
||
gate_block=6,
|
||
gate_n_max=20,
|
||
gate_guard_err=0.5,
|
||
baseline_cache=cache,
|
||
prompts_version="p1",
|
||
run_inference=mock_fn,
|
||
log=log,
|
||
gate_run_prefix="step1_gate_test",
|
||
)
|
||
|
||
assert outcome.accepted is True
|
||
assert outcome.action == "accept_confirmed"
|
||
assert outcome.stop_reason == "confirmed"
|
||
assert outcome.w == 6
|
||
assert outcome.l == 0
|
||
assert outcome.n_used == 6
|
||
assert outcome.candidate_acc == 1.0
|
||
assert outcome.baseline_acc == 0.0
|
||
assert len(outcome.evidence_rows) == 6
|
||
# 终态证据行携带 stop_reason
|
||
assert outcome.evidence_rows[-1]["stop_reason"] == "confirmed"
|
||
# 候选临时目录应被清理
|
||
cand_tmp = workspace / ".cand_tmp"
|
||
if cand_tmp.exists():
|
||
assert list(cand_tmp.iterdir()) == []
|
||
finally:
|
||
log.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_validate_skill_local_reject(tmp_path: Path) -> None:
|
||
"""候选全错、基线全对 → L 高 → 方向拒绝。"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
questions = _make_questions(6)
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
|
||
# 基线全对，候选全错 → W=0, L=6 → 方向拒绝
|
||
baseline_correct = {f"q{i}": True for i in range(6)}
|
||
candidate_correct = {f"q{i}": False for i in range(6)}
|
||
mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct)
|
||
|
||
try:
|
||
outcome = await validate_skill_local(
|
||
workspace_dir=workspace,
|
||
base_skills_version="v1",
|
||
task_type="temporal",
|
||
target_file="temporal.md",
|
||
candidate_content="bad skill",
|
||
base_skill_content="baseline skill content",
|
||
ladder_items=questions,
|
||
gate_params=_DEFAULT_GATE_PARAMS,
|
||
gate_block=6,
|
||
gate_n_max=20,
|
||
gate_guard_err=0.5,
|
||
baseline_cache=cache,
|
||
prompts_version="p1",
|
||
run_inference=mock_fn,
|
||
log=log,
|
||
gate_run_prefix="step1_gate_test",
|
||
)
|
||
|
||
assert outcome.accepted is False
|
||
assert outcome.action == "reject"
|
||
assert outcome.stop_reason == "directional"
|
||
assert outcome.w == 0
|
||
assert outcome.l == 6
|
||
finally:
|
||
log.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_gate_prefix_must_contain_gate(tmp_path: Path) -> None:
|
||
"""gate_run_prefix 不含 '_gate_' 时抛 ValueError。"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
questions = _make_questions(4)
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
|
||
async def noop_fn(questions, *, run_id, skills_dir):
|
||
raise AssertionError("不应被调用")
|
||
|
||
try:
|
||
with pytest.raises(ValueError, match="_gate_"):
|
||
await validate_skill_local(
|
||
workspace_dir=workspace,
|
||
base_skills_version="v1",
|
||
task_type="temporal",
|
||
target_file="temporal.md",
|
||
candidate_content="content",
|
||
base_skill_content="baseline",
|
||
ladder_items=questions,
|
||
gate_params=_DEFAULT_GATE_PARAMS,
|
||
gate_block=4,
|
||
gate_n_max=20,
|
||
gate_guard_err=0.5,
|
||
baseline_cache=cache,
|
||
prompts_version="p1",
|
||
run_inference=noop_fn,
|
||
log=log,
|
||
gate_run_prefix="step1_no_marker",
|
||
)
|
||
finally:
|
||
log.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_infra_guard_threshold(tmp_path: Path) -> None:
|
||
"""推理错误率超阈值时抛 RuntimeError。"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
# 需要 >=10 题次才触发 INFRA 护栏
|
||
questions = _make_questions(6)
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
|
||
baseline_correct = {f"q{i}": False for i in range(6)}
|
||
candidate_correct = {f"q{i}": False for i in range(6)}
|
||
# 每次 run_inference 报 error_count=5，两侧各 5 → 10/12 > 0.5
|
||
mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct, error_count=5)
|
||
|
||
try:
|
||
with pytest.raises(RuntimeError, match="错误率过高"):
|
||
await validate_skill_local(
|
||
workspace_dir=workspace,
|
||
base_skills_version="v1",
|
||
task_type="temporal",
|
||
target_file="temporal.md",
|
||
candidate_content="content",
|
||
base_skill_content="baseline skill content",
|
||
ladder_items=questions,
|
||
gate_params=_DEFAULT_GATE_PARAMS,
|
||
gate_block=6,
|
||
gate_n_max=20,
|
||
gate_guard_err=0.5,
|
||
baseline_cache=cache,
|
||
prompts_version="p1",
|
||
run_inference=mock_fn,
|
||
log=log,
|
||
gate_run_prefix="step1_gate_test",
|
||
)
|
||
finally:
|
||
log.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_baseline_cache_hit(tmp_path: Path) -> None:
|
||
"""基线缓存全命中时不发起基线侧推理。"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
questions = _make_questions(4)
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
|
||
s_hash = skill_hash("baseline skill content")
|
||
# 预填充缓存：全部题目基线全错
|
||
for q in questions:
|
||
cache.put("temporal", s_hash, "p1", q.question_id, False)
|
||
|
||
# 候选全对 → accept
|
||
candidate_correct = {f"q{i}": True for i in range(4)}
|
||
baseline_correct = {f"q{i}": False for i in range(4)}
|
||
mock_fn, call_log = _make_mock_run_inference(log, baseline_correct, candidate_correct)
|
||
|
||
try:
|
||
outcome = await validate_skill_local(
|
||
workspace_dir=workspace,
|
||
base_skills_version="v1",
|
||
task_type="temporal",
|
||
target_file="temporal.md",
|
||
candidate_content="improved skill",
|
||
base_skill_content="baseline skill content",
|
||
ladder_items=questions,
|
||
gate_params=_DEFAULT_GATE_PARAMS,
|
||
gate_block=4,
|
||
gate_n_max=20,
|
||
gate_guard_err=0.5,
|
||
baseline_cache=cache,
|
||
prompts_version="p1",
|
||
run_inference=mock_fn,
|
||
log=log,
|
||
gate_run_prefix="step1_gate_test",
|
||
)
|
||
|
||
# 只有候选侧调用了 run_inference（_cand），基线侧全命中不调用
|
||
base_calls = [c for c in call_log if c["run_id"].endswith("_base")]
|
||
cand_calls = [c for c in call_log if c["run_id"].endswith("_cand")]
|
||
assert len(base_calls) == 0, "基线缓存全命中不应发起推理"
|
||
assert len(cand_calls) == 1
|
||
assert outcome.accepted is True
|
||
finally:
|
||
log.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_last_block_terminal(tmp_path: Path) -> None:
|
||
"""单块 + n_remaining=0 → 终态判定（provisional 或 inertia），非 continue。"""
|
||
workspace = _setup_workspace(tmp_path)
|
||
log = _make_log(workspace)
|
||
# 4 题，gate_block=4 → 一块走完，n_remaining=0
|
||
questions = _make_questions(4)
|
||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||
|
||
# 两题翻转（W=2, L=0），但 e_confirm=20 难以达到 → provisional 或 inertia
|
||
baseline_correct = {"q0": False, "q1": False, "q2": True, "q3": True}
|
||
candidate_correct = {"q0": True, "q1": True, "q2": True, "q3": True}
|
||
mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct)
|
||
|
||
try:
|
||
outcome = await validate_skill_local(
|
||
workspace_dir=workspace,
|
||
base_skills_version="v1",
|
||
task_type="temporal",
|
||
target_file="temporal.md",
|
||
candidate_content="candidate skill",
|
||
base_skill_content="baseline skill content",
|
||
ladder_items=questions,
|
||
gate_params=_DEFAULT_GATE_PARAMS,
|
||
gate_block=4,
|
||
gate_n_max=4,
|
||
gate_guard_err=0.5,
|
||
baseline_cache=cache,
|
||
prompts_version="p1",
|
||
run_inference=mock_fn,
|
||
log=log,
|
||
gate_run_prefix="step1_gate_test",
|
||
)
|
||
|
||
# n_remaining=0 → 不可能是 continue
|
||
assert outcome.stop_reason in (
|
||
"confirmed",
|
||
"provisional",
|
||
"inertia",
|
||
"directional",
|
||
"futility",
|
||
)
|
||
assert outcome.n_used == 4
|
||
# 终态行标记 stop_reason
|
||
assert outcome.evidence_rows[-1]["stop_reason"] != ""
|
||
finally:
|
||
log.close()
|