feat(harness): correctness 三口径 + gate 块按 unit 跑

进化引擎与 gate e-process 从 question_id 口径迁至 unit_id 口径,AR pair
双向 AND 折叠为单元、不被 P/Q 单题计分污染;逐题 predictions 仅作溯源。

- question_units: 新增 unit_correctness_view(units, per_q)->dict[unit_id,bool]
  作为逐题→单元折叠的唯一入口(复用 unit_correctness)。
- core/evolution/validate: pair_block/compute_accuracy 参数改 unit_ids、
  分母按单元数(键即 unit_id)。
- app/harness/validate(gate 实际执行路径):阶梯题序聚合为单元并保持信息
  阶梯序(_ladder_units),gate 块按单元切分(AR pair 整锁不跨块拆);
  baseline_cache 键含 unit_id、存单元级对错;候选逐题读回后折叠成单元视图;
  n_used/W/L/四象限/准确率均按单元计;证据行按 unit 口径,candidate_correctness
  独立保留逐题对错供 runner 二轨合并。
- runner: probation 结算按 unit 折叠计 W/L(_probation_unit_flips);quadrant
  四象限 id 承载 unit_id。

核心算法保真 #5(信息阶梯 e-process):本次仅迁移 correctness 口径,不改冷启动
2:1 / gamma-EMA / 反泄漏算法本身(gate_ladder 迁移见 Task 8)。
This commit is contained in:
2026-07-15 07:31:03 -04:00
parent dee6bf4896
commit 4b6d1d8a50
6 changed files with 650 additions and 127 deletions
+131
View File
@@ -0,0 +1,131 @@
"""tests/unit/test_correctness_unit_view.py — correctness 三对象口径单元测试。
验证 unit_correctness_view(逐题 per_q → unit 折叠:AR pair 双向 AND、非 AR single
及 core.evolution 的 pair_block / compute_accuracy 消费 unit 口径时,混格池中
AR pair 折叠为单元、W/L 与准确率分母不被 P/Q 单题计分污染(核心算法保真 #5)。
"""
from __future__ import annotations
from app.harness.question_units import build_units, unit_correctness_view
from core.evolution.validate import classify_quadrants, compute_accuracy, pair_block
from core.types import GeneratedQuestion
def _single(qid: str, task_type: str = "temporal") -> GeneratedQuestion:
"""构造一条非 AR single 题(unit_id 回填为 question_id)。"""
return GeneratedQuestion(
question_id=qid,
video_id=f"v_{qid}",
task_type=task_type,
question="Q?",
options=("A", "B", "C", "D"),
answer="A",
source_nodes=(),
difficulty="easy",
)
def _pair(pair_id: str, task_type: str = "temporal") -> tuple[GeneratedQuestion, GeneratedQuestion]:
"""构造一个 AR 孪生对(original + mirror,共享 pair_id → unit_id=pair_id)。"""
common = {
"video_id": f"v_{pair_id}",
"task_type": task_type,
"question": "Q?",
"options": ("A", "B", "C", "D"),
"answer": "A",
"source_nodes": (),
"difficulty": "easy",
"pair_id": pair_id,
"flip_axis": "before_after",
}
orig = GeneratedQuestion(question_id=f"{pair_id}_o", question_role="pair_original", **common)
mirror = GeneratedQuestion(question_id=f"{pair_id}_m", question_role="pair_mirror", **common)
return orig, mirror
class TestUnitCorrectnessView:
"""unit_correctness_view:逐题对错折叠成 unit_id → bool。"""
def test_mixed_pool_and_semantics(self) -> None:
"""混格:single 直取、AR pair 双向 AND。"""
s0 = _single("s0")
s1 = _single("s1")
p1o, p1m = _pair("p1")
p2o, p2m = _pair("p2")
units = build_units([s0, s1, p1o, p1m, p2o, p2m])
per_q = {
"s0": True,
"s1": False,
"p1_o": True,
"p1_m": True,
"p2_o": True,
"p2_m": False,
}
view = unit_correctness_view(units, per_q)
# single 的 unit_id 等于 question_idpair 的 unit_id 等于 pair_id
assert view == {"s0": True, "s1": False, "p1": True, "p2": False}
def test_missing_per_q_raises(self) -> None:
"""任一题缺 per_q → KeyError(禁静默兜底)。"""
p1o, p1m = _pair("p1")
units = build_units([p1o, p1m])
import pytest
with pytest.raises(KeyError):
unit_correctness_view(units, {"p1_o": True})
class TestPairBlockUnitFold:
"""pair_block 消费 unit 口径:混格 W/L 不被 P/Q 单题计分污染。"""
def test_pair_partial_improvement_not_counted(self) -> None:
"""AR pair 基线(F,F)→候选(T,F):单元仍错,不计 W(保真 #5)。"""
p1o, p1m = _pair("p1")
s0 = _single("s0")
units = build_units([p1o, p1m, s0])
unit_ids = [u.unit_id for u in units]
baseline_per_q = {"p1_o": False, "p1_m": False, "s0": False}
candidate_per_q = {"p1_o": True, "p1_m": False, "s0": True}
b_units = unit_correctness_view(units, baseline_per_q)
c_units = unit_correctness_view(units, candidate_per_q)
result = pair_block(b_units, c_units, unit_ids)
# 只有 s0 单元发生 F→T 翻转;pair 单元双向 AND 后仍错,不计 W
assert result.w == 1
assert result.l == 0
assert result.observed["p1"] == (False, False)
assert result.observed["s0"] == (False, True)
def test_pair_full_flip_counts_once(self) -> None:
"""AR pair 两题齐翻(F,F)→(T,T):单元计 1 次 W(非 2)。"""
p1o, p1m = _pair("p1")
units = build_units([p1o, p1m])
b_units = unit_correctness_view(units, {"p1_o": False, "p1_m": False})
c_units = unit_correctness_view(units, {"p1_o": True, "p1_m": True})
result = pair_block(b_units, c_units, [u.unit_id for u in units])
assert result.w == 1
assert result.l == 0
class TestComputeAccuracyUnitDenominator:
"""compute_accuracy 分母按 unit 数(非逐题)。"""
def test_denominator_is_unit_count(self) -> None:
"""1 pair(错) + 1 single(对) → 1/2;逐题会误算 2/3。"""
p1o, p1m = _pair("p1")
s0 = _single("s0")
units = build_units([p1o, p1m, s0])
unit_ids = [u.unit_id for u in units]
view = unit_correctness_view(units, {"p1_o": True, "p1_m": False, "s0": True})
assert compute_accuracy(view, unit_ids) == 0.5
class TestClassifyQuadrantsUnitKeys:
"""classify_quadrants 按 unit_id 分桶(pair 单元只出现一次)。"""
def test_pair_unit_single_bucket(self) -> None:
"""pair 单元 F→F 落 persistent_fails,只记 unit_id 一次。"""
observed = {"p1": (False, False), "s0": (False, True)}
qc = classify_quadrants(observed)
assert qc.improvements == ["s0"]
assert qc.persistent_fails == ["p1"]
+279
View File
@@ -0,0 +1,279 @@
"""tests/unit/test_gate_block_unit.py — gate 块实际执行路径按 unit 跑。
针对 app/harness/validate.py::validate_skill_local(真实 gate 执行路径),
断言混格阶梯下 gate 块按 unit 口径运行:baseline_cache 键含 unit_id、
n_used 按 unit 累加、pair_block 折叠 AR pair、逐题 predictions 仍溯源。
核心算法保真 #5(信息阶梯 e-process 口径从 question_id 迁至 unit_id)。
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
import pytest
from app.harness.gate_ladder import BaselineCache, skill_hash
from app.harness.inference import PREDICTIONS_SCHEMA, InferenceResult
from app.harness.log import HarnessLog
from app.harness.validate import validate_skill_local
from core.evolution import GateParams
from core.types import GeneratedQuestion
if TYPE_CHECKING:
from pathlib import Path
_DEFAULT_GATE_PARAMS = GateParams(
e_confirm=20.0,
e_provisional=3.0,
w_net_min=2,
delta_min=0.05,
lambda_dir=-2.0,
e_rollback=10.0,
)
def _single(qid: str) -> GeneratedQuestion:
"""构造非 AR single 题。"""
return GeneratedQuestion(
question_id=qid,
video_id=f"v_{qid}",
task_type="temporal",
question="Q?",
options=("A", "B", "C", "D"),
answer="A",
source_nodes=(),
difficulty="easy",
)
def _pair(pair_id: str) -> list[GeneratedQuestion]:
"""构造 AR 孪生对(original + mirror,共享 pair_id)。"""
common = {
"video_id": f"v_{pair_id}",
"task_type": "temporal",
"question": "Q?",
"options": ("A", "B", "C", "D"),
"answer": "A",
"source_nodes": (),
"difficulty": "easy",
"pair_id": pair_id,
"flip_axis": "before_after",
}
return [
GeneratedQuestion(question_id=f"{pair_id}_o", question_role="pair_original", **common),
GeneratedQuestion(question_id=f"{pair_id}_m", question_role="pair_mirror", **common),
]
def _setup_workspace(tmp_path: Path) -> Path:
"""构建最小 workspaceskills/v1/temporal.md)。"""
skills_dir = tmp_path / "skills" / "v1"
skills_dir.mkdir(parents=True)
(skills_dir / "temporal.md").write_text("baseline skill content", encoding="utf-8")
return tmp_path
def _make_log(workspace: Path, run_id: str = "test_master") -> HarnessLog:
"""创建 HarnessLog 并建 predictions 表。"""
log = HarnessLog(str(workspace / "harness.db"), run_id)
log.create_table("predictions", PREDICTIONS_SCHEMA)
return log
def _insert_predictions(log: HarnessLog, run_id: str, per_q: dict[str, bool]) -> None:
"""逐题写 predictions(溯源仍逐题)。"""
for qid, correct in per_q.items():
log.insert(
"predictions",
{
"run_id": run_id,
"video_id": "v0",
"question_id": qid,
"task_type": "temporal",
"prediction": "A" if correct else "Z",
"answer": "A",
"evidence": "",
"reasoning": "",
"steps_used": 1,
"prompt_tokens": 10,
"completion_tokens": 10,
"stop_reason": "completed",
"steps_json": "[]",
},
)
def _make_mock_run_inference(
log: HarnessLog,
baseline_correct: dict[str, bool],
candidate_correct: dict[str, bool],
):
"""构建 mock RunInferenceFn,按 run_id 的 arm 后缀选基线/候选逐题对错。"""
call_log: list[dict[str, Any]] = []
async def mock_fn(
questions: list[GeneratedQuestion],
*,
run_id: str,
skills_dir: Path,
) -> InferenceResult:
src = baseline_correct if run_id.endswith("_base") else candidate_correct
per_q = {q.question_id: src.get(q.question_id, False) for q in questions}
_insert_predictions(log, run_id, per_q)
call_log.append({"run_id": run_id, "qids": [q.question_id for q in questions]})
total = len(questions)
return InferenceResult(
run_id=run_id,
accuracy=sum(per_q.values()) / total if total else 0.0,
total=total,
correct=sum(per_q.values()),
per_task_type={},
steps_mean=1.0,
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
stop_reason_counts={"completed": total},
)
return mock_fn, call_log
@pytest.mark.asyncio
async def test_gate_n_used_counts_units_not_questions(tmp_path: Path) -> None:
"""混格阶梯(1 pair + 2 single)→ n_used=3 单元,非 4 题。"""
workspace = _setup_workspace(tmp_path)
log = _make_log(workspace)
cache = BaselineCache(workspace / "baseline_cache.json")
ladder = [*_pair("p1"), _single("s0"), _single("s1")]
# 基线全错、候选全对 → 3 单元齐翻 W=3
baseline = {"p1_o": False, "p1_m": False, "s0": False, "s1": False}
candidate = {"p1_o": True, "p1_m": True, "s0": True, "s1": True}
mock_fn, call_log = _make_mock_run_inference(log, baseline, candidate)
accept_params = GateParams(
e_confirm=15.0,
e_provisional=3.0,
w_net_min=2,
delta_min=0.05,
lambda_dir=-2.0,
e_rollback=10.0,
)
try:
outcome = await validate_skill_local(
workspace_dir=workspace,
base_skills_version="v1",
task_type="temporal",
target_file="temporal.md",
candidate_content="improved skill",
base_skill_content="baseline skill content",
ladder_items=ladder,
gate_params=accept_params,
gate_block=10,
gate_n_max=20,
gate_guard_err=0.5,
baseline_cache=cache,
prompts_version="p1",
run_inference=mock_fn,
log=log,
gate_run_prefix="step1_gate_test",
)
# n_used 按 unit 计(3),W 按 unit 计(3
assert outcome.n_used == 3
assert outcome.w == 3
assert outcome.l == 0
# 证据行按 unit 口径(3 行)
assert len(outcome.evidence_rows) == 3
# baseline_cache 键含 unit_idpair 用 pair_id、single 用 question_id
s_hash = skill_hash("baseline skill content")
assert cache.get("temporal", s_hash, "p1", "p1") is False
assert cache.get("temporal", s_hash, "p1", "s0") is False
# 逐题 question_id 不作为 baseline_cache 键(pair 成员未单独缓存)
assert cache.get("temporal", s_hash, "p1", "p1_o") is None
# 逐题 predictions 仍溯源:候选 per-q 含两 pair 成员
assert set(outcome.candidate_correctness) >= {"p1_o", "p1_m", "s0", "s1"}
finally:
log.close()
@pytest.mark.asyncio
async def test_gate_pair_partial_flip_not_counted(tmp_path: Path) -> None:
"""AR pair 候选仅单向翻(T,F)→单元仍错,W 不被单题污染。"""
workspace = _setup_workspace(tmp_path)
log = _make_log(workspace)
cache = BaselineCache(workspace / "baseline_cache.json")
ladder = [*_pair("p1"), _single("s0")]
baseline = {"p1_o": False, "p1_m": False, "s0": False}
# pair 只翻一半(p1_o 对、p1_m 错)→ 单元 AND 仍错;s0 翻对
candidate = {"p1_o": True, "p1_m": False, "s0": True}
mock_fn, _ = _make_mock_run_inference(log, baseline, candidate)
try:
outcome = await validate_skill_local(
workspace_dir=workspace,
base_skills_version="v1",
task_type="temporal",
target_file="temporal.md",
candidate_content="improved skill",
base_skill_content="baseline skill content",
ladder_items=ladder,
gate_params=_DEFAULT_GATE_PARAMS,
gate_block=10,
gate_n_max=20,
gate_guard_err=0.5,
baseline_cache=cache,
prompts_version="p1",
run_inference=mock_fn,
log=log,
gate_run_prefix="step1_gate_test",
)
# 只有 s0 单元翻转,pair 单元不计 W(保真 #5:不被 P/Q 单题污染)
assert outcome.w == 1
assert outcome.l == 0
assert outcome.n_used == 2
# candidate_acc 分母按 unit2 单元,1 对)→ 0.5
assert outcome.candidate_acc == 0.5
finally:
log.close()
@pytest.mark.asyncio
async def test_gate_baseline_cache_hit_by_unit(tmp_path: Path) -> None:
"""基线缓存按 unit_id 预填充 → 基线侧全命中不发起推理。"""
workspace = _setup_workspace(tmp_path)
log = _make_log(workspace)
cache = BaselineCache(workspace / "baseline_cache.json")
ladder = [*_pair("p1"), _single("s0")]
s_hash = skill_hash("baseline skill content")
# 按 unit_id 预填充(pair→pair_idsingle→question_id),全错
cache.put("temporal", s_hash, "p1", "p1", False)
cache.put("temporal", s_hash, "p1", "s0", False)
baseline = {"p1_o": False, "p1_m": False, "s0": False}
candidate = {"p1_o": True, "p1_m": True, "s0": True}
mock_fn, call_log = _make_mock_run_inference(log, baseline, candidate)
try:
outcome = await validate_skill_local(
workspace_dir=workspace,
base_skills_version="v1",
task_type="temporal",
target_file="temporal.md",
candidate_content="improved skill",
base_skill_content="baseline skill content",
ladder_items=ladder,
gate_params=_DEFAULT_GATE_PARAMS,
gate_block=10,
gate_n_max=20,
gate_guard_err=0.5,
baseline_cache=cache,
prompts_version="p1",
run_inference=mock_fn,
log=log,
gate_run_prefix="step1_gate_test",
)
base_calls = [c for c in call_log if c["run_id"].endswith("_base")]
assert base_calls == [], "unit 键全命中不应发起基线推理"
assert outcome.n_used == 2
finally:
log.close()