375 lines
14 KiB
Python
375 lines
14 KiB
Python
"""app/harness/gate_ladder.py 单元测试。
|
||
|
||
覆盖冷启动交错、Beta(1,1) 平滑、warm 信息量排序、
|
||
GatePools 原子读写与指纹校验、BaselineCache 四维内容寻址与先盘后存、
|
||
gamma-EMA 更新、防泄露过滤等核心语义。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import TYPE_CHECKING
|
||
|
||
import pytest
|
||
|
||
from app.harness.gate_ladder import (
|
||
BaselineCache,
|
||
GatePools,
|
||
LadderEntry,
|
||
build_cold_entries,
|
||
build_or_load_gate_pools,
|
||
order_ladder,
|
||
skill_hash,
|
||
)
|
||
from core.types import GeneratedQuestion
|
||
|
||
if TYPE_CHECKING:
|
||
from pathlib import Path
|
||
|
||
|
||
# ── 工具函数 ──────────────────────────────────────────────────────────
|
||
|
||
|
||
def _make_q(qid: str, task_type: str = "AR") -> GeneratedQuestion:
|
||
"""构造最小 GeneratedQuestion 实例。"""
|
||
return GeneratedQuestion(
|
||
question_id=qid,
|
||
video_id="v1",
|
||
task_type=task_type,
|
||
question="dummy",
|
||
options=("A", "B", "C", "D"),
|
||
answer="A",
|
||
source_nodes=("n1",),
|
||
difficulty="easy",
|
||
)
|
||
|
||
|
||
# ── 冷启动 ────────────────────────────────────────────────────────────
|
||
|
||
|
||
class TestColdStart:
|
||
"""冷启动排序：2:1 交错 + 探针插尾 + Beta(1,1) 平滑。"""
|
||
|
||
def test_cold_start_interleaving(self) -> None:
|
||
"""错题:对题 = 2:1 交错顺序。
|
||
|
||
6 错 3 对（probe_quota=0 无探针）→ 交错序应为 W W R W W R W W R。
|
||
"""
|
||
wrong_ids = [f"w{i}" for i in range(6)]
|
||
right_ids = [f"r{i}" for i in range(3)]
|
||
questions = [_make_q(qid) for qid in wrong_ids + right_ids]
|
||
correctness = dict.fromkeys(wrong_ids, False)
|
||
correctness.update(dict.fromkeys(right_ids, True))
|
||
|
||
entries = build_cold_entries(questions, correctness, probe_quota=0.0, seed=42)
|
||
|
||
assert len(entries) == 9
|
||
# 验证 2:1 交错模式（seed 固定后 shuffle 结果确定）
|
||
pattern = ["W" if not correctness[e.question_id] else "R" for e in entries]
|
||
# 前 9 个交错应为 W W R W W R W W R
|
||
assert pattern == ["W", "W", "R", "W", "W", "R", "W", "W", "R"]
|
||
|
||
def test_cold_start_p_hat_beta(self) -> None:
|
||
"""p_hat 遵循 Beta(1,1) 平滑：错=1/3，对=2/3。"""
|
||
questions = [_make_q("q1"), _make_q("q2")]
|
||
correctness = {"q1": False, "q2": True}
|
||
|
||
entries = build_cold_entries(questions, correctness, probe_quota=0.0, seed=0)
|
||
|
||
p_map = {e.question_id: e.p_hat for e in entries}
|
||
assert p_map["q1"] == pytest.approx(1 / 3)
|
||
assert p_map["q2"] == pytest.approx(2 / 3)
|
||
|
||
def test_cold_start_probe_at_tail(self) -> None:
|
||
"""probe_quota > 0 时探针题追加在尾部。"""
|
||
wrong_ids = [f"w{i}" for i in range(10)]
|
||
right_ids = [f"r{i}" for i in range(2)]
|
||
questions = [_make_q(qid) for qid in wrong_ids + right_ids]
|
||
correctness = dict.fromkeys(wrong_ids, False)
|
||
correctness.update(dict.fromkeys(right_ids, True))
|
||
|
||
entries = build_cold_entries(questions, correctness, probe_quota=0.3, seed=7)
|
||
|
||
# 10 错 * 0.3 = 3 个探针在尾部
|
||
n_probe = int(10 * 0.3)
|
||
assert n_probe == 3
|
||
# 尾部 3 个都应为错题
|
||
tail = entries[-n_probe:]
|
||
for e in tail:
|
||
assert not correctness[e.question_id]
|
||
|
||
|
||
# ── warm 排序 ──────────────────────────────────────────────────────────
|
||
|
||
|
||
class TestWarmOrdering:
|
||
"""warm 阶段：信息量 p_hat(1-p_hat) 降序 + p_hat 区间过滤。"""
|
||
|
||
def test_warm_ordering_information(self) -> None:
|
||
"""p_hat=0.5 信息量最高，排在最前。"""
|
||
entries = [
|
||
LadderEntry("a", 0.1),
|
||
LadderEntry("b", 0.5),
|
||
LadderEntry("c", 0.9),
|
||
LadderEntry("d", 0.3),
|
||
]
|
||
ordered = order_ladder(entries, p_low=0.0, p_high=1.0)
|
||
assert ordered[0].question_id == "b" # 0.5*(1-0.5)=0.25 最高
|
||
# d: 0.3*0.7=0.21, a: 0.1*0.9=0.09, c: 0.9*0.1=0.09
|
||
assert ordered[1].question_id == "d"
|
||
|
||
def test_warm_filter_bounds(self) -> None:
|
||
"""p_hat 不在 [p_low, p_high] 区间的题被剔除。"""
|
||
entries = [
|
||
LadderEntry("low", 0.05),
|
||
LadderEntry("mid", 0.5),
|
||
LadderEntry("high", 0.95),
|
||
]
|
||
ordered = order_ladder(entries, p_low=0.1, p_high=0.9)
|
||
ids = [e.question_id for e in ordered]
|
||
assert "mid" in ids
|
||
assert "low" not in ids
|
||
assert "high" not in ids
|
||
|
||
|
||
# ── GatePools 持久化 ──────────────────────────────────────────────────
|
||
|
||
|
||
class TestGatePoolsPersistence:
|
||
"""GatePools.save/load 原子性与指纹校验。"""
|
||
|
||
def test_gate_pools_save_load_atomic(self, tmp_path: Path) -> None:
|
||
"""save -> load 往返保真，且使用原子写（中间 .tmp 文件不残留）。"""
|
||
entries = {
|
||
"AR": [LadderEntry("q1", 0.33), LadderEntry("q2", 0.67)],
|
||
"CR": [LadderEntry("q3", 0.5)],
|
||
}
|
||
pools = GatePools(entries=entries, seed=42, fingerprint="abc123")
|
||
path = tmp_path / "gate_pools.json"
|
||
pools.save(path)
|
||
|
||
# .tmp 文件不应残留
|
||
assert not (tmp_path / "gate_pools.json.tmp").exists()
|
||
assert path.exists()
|
||
|
||
loaded = GatePools.load(path)
|
||
assert loaded.seed == 42
|
||
assert loaded.fingerprint == "abc123"
|
||
assert len(loaded.entries["AR"]) == 2
|
||
assert loaded.entries["AR"][0].question_id == "q1"
|
||
assert loaded.entries["AR"][0].p_hat == pytest.approx(0.33)
|
||
assert loaded.entries["CR"][0].question_id == "q3"
|
||
|
||
def test_gate_pools_fingerprint_mismatch(self, tmp_path: Path) -> None:
|
||
"""指纹不一致 -> RuntimeError（不静默重建）。"""
|
||
questions = [_make_q("q1", "AR"), _make_q("q2", "AR")]
|
||
correctness = {"q1": True, "q2": False}
|
||
|
||
# 第一次构建
|
||
build_or_load_gate_pools(
|
||
workspace_dir=tmp_path,
|
||
questions=questions,
|
||
test_qids=set(),
|
||
baseline_correctness=correctness,
|
||
task_types=["AR"],
|
||
probe_quota=0.0,
|
||
seed=1,
|
||
baseline_run_id="run_001",
|
||
)
|
||
|
||
# 改 baseline_run_id 导致指纹变化 -> 应报错
|
||
with pytest.raises(RuntimeError, match="指纹不一致"):
|
||
build_or_load_gate_pools(
|
||
workspace_dir=tmp_path,
|
||
questions=questions,
|
||
test_qids=set(),
|
||
baseline_correctness=correctness,
|
||
task_types=["AR"],
|
||
probe_quota=0.0,
|
||
seed=1,
|
||
baseline_run_id="run_002",
|
||
)
|
||
|
||
|
||
# ── ladder_for ────────────────────────────────────────────────────────
|
||
|
||
|
||
class TestLadderFor:
|
||
"""ladder_for 取题序与排除逻辑。"""
|
||
|
||
def test_ladder_for_excludes_qids(self) -> None:
|
||
"""exclude_qids 中的题被排除。"""
|
||
entries = {
|
||
"AR": [
|
||
LadderEntry("q1", 0.5),
|
||
LadderEntry("q2", 0.4),
|
||
LadderEntry("q3", 0.6),
|
||
],
|
||
}
|
||
pools = GatePools(entries=entries, seed=0, fingerprint="x")
|
||
result = pools.ladder_for("AR", exclude_qids={"q2"}, p_low=0.0, p_high=1.0, cold=True)
|
||
assert "q2" not in result
|
||
assert "q1" in result
|
||
assert "q3" in result
|
||
|
||
def test_ladder_for_missing_task_type(self) -> None:
|
||
"""不存在的 task_type -> ValueError。"""
|
||
pools = GatePools(entries={}, seed=0, fingerprint="x")
|
||
with pytest.raises(ValueError, match="无阶梯"):
|
||
pools.ladder_for("MISSING", set(), 0.0, 1.0, cold=True)
|
||
|
||
def test_ladder_for_warm_uses_order_ladder(self) -> None:
|
||
"""cold=False 时走 warm 信息量排序。"""
|
||
entries = {
|
||
"AR": [
|
||
LadderEntry("low", 0.1),
|
||
LadderEntry("mid", 0.5),
|
||
LadderEntry("high", 0.9),
|
||
],
|
||
}
|
||
pools = GatePools(entries=entries, seed=0, fingerprint="x")
|
||
result = pools.ladder_for("AR", set(), p_low=0.0, p_high=1.0, cold=False)
|
||
# 信息量排序：mid(0.25) > low(0.09) = high(0.09)
|
||
assert result[0] == "mid"
|
||
|
||
|
||
# ── gamma-EMA 更新 ─────────────────────────────────────────────────────
|
||
|
||
|
||
class TestGammaEMA:
|
||
"""gamma-EMA 更新 p_hat。"""
|
||
|
||
def test_gamma_ema_update(self) -> None:
|
||
"""p_hat <- gamma * p_hat + (1-gamma) * obs。"""
|
||
entries = {"AR": [LadderEntry("q1", 0.5)]}
|
||
pools = GatePools(entries=entries, seed=0, fingerprint="x")
|
||
|
||
# 观测为正确(1.0), gamma=0.8
|
||
pools.update_probs({"q1": True}, gamma=0.8)
|
||
expected = 0.8 * 0.5 + 0.2 * 1.0 # 0.6
|
||
assert pools.entries["AR"][0].p_hat == pytest.approx(expected)
|
||
|
||
# 再次观测为错误(0.0), gamma=0.8
|
||
pools.update_probs({"q1": False}, gamma=0.8)
|
||
expected2 = 0.8 * expected + 0.2 * 0.0 # 0.48
|
||
assert pools.entries["AR"][0].p_hat == pytest.approx(expected2)
|
||
|
||
def test_update_probs_no_observation_unchanged(self) -> None:
|
||
"""无观测的题 p_hat 不变。"""
|
||
entries = {"AR": [LadderEntry("q1", 0.5), LadderEntry("q2", 0.3)]}
|
||
pools = GatePools(entries=entries, seed=0, fingerprint="x")
|
||
pools.update_probs({"q1": True}, gamma=0.9)
|
||
assert pools.entries["AR"][1].p_hat == pytest.approx(0.3)
|
||
|
||
|
||
# ── 防泄露 ─────────────────────────────────────────────────────────────
|
||
|
||
|
||
class TestLeakPrevention:
|
||
"""防泄露铁律：gate 内 rollout 永不回流 p_hat（由调用方过滤）。"""
|
||
|
||
def test_update_probs_excludes_gate_runs(self) -> None:
|
||
"""调用方须过滤 run_id 含 '_gate_' 的观测。
|
||
|
||
update_probs 本身只接收已过滤的 observations，这里验证
|
||
如果调用方正确过滤，gate run 数据不会影响 p_hat。
|
||
"""
|
||
entries = {"AR": [LadderEntry("q1", 0.5)]}
|
||
pools = GatePools(entries=entries, seed=0, fingerprint="x")
|
||
|
||
# 模拟：所有 run 的原始观测（含 gate run）
|
||
raw_observations = {
|
||
"run_normal": {"q1": True}, # 普通 run
|
||
"run_gate_01": {"q1": False}, # gate run（run_id 含 _gate_）
|
||
}
|
||
|
||
# 调用方按 run_id 过滤：排除含 "_gate_" 的 run
|
||
filtered = {}
|
||
for run_id, obs in raw_observations.items():
|
||
if "_gate_" not in run_id:
|
||
filtered.update(obs)
|
||
|
||
# 只有普通 run 的观测进入 update_probs
|
||
assert filtered == {"q1": True}
|
||
pools.update_probs(filtered, gamma=0.8)
|
||
expected = 0.8 * 0.5 + 0.2 * 1.0
|
||
assert pools.entries["AR"][0].p_hat == pytest.approx(expected)
|
||
|
||
|
||
# ── BaselineCache ──────────────────────────────────────────────────────
|
||
|
||
|
||
class TestBaselineCache:
|
||
"""BaselineCache 四维内容寻址与先盘后存。"""
|
||
|
||
def test_baseline_cache_content_addressed(self, tmp_path: Path) -> None:
|
||
"""四维键唯一寻址：任一维度变化 -> miss。"""
|
||
path = tmp_path / "baseline_cache.json"
|
||
cache = BaselineCache(path)
|
||
|
||
cache.put("AR", "hash1", "v1", "q1", True)
|
||
assert cache.get("AR", "hash1", "v1", "q1") is True
|
||
|
||
# 改 skill_hash -> miss
|
||
assert cache.get("AR", "hash2", "v1", "q1") is None
|
||
# 改 prompts_version -> miss
|
||
assert cache.get("AR", "hash1", "v2", "q1") is None
|
||
# 改 task_type -> miss
|
||
assert cache.get("CR", "hash1", "v1", "q1") is None
|
||
# 改 qid -> miss
|
||
assert cache.get("AR", "hash1", "v1", "q2") is None
|
||
|
||
def test_baseline_cache_disk_first(self, tmp_path: Path) -> None:
|
||
"""先盘后存：磁盘写成功后内存才更新，新实例可从磁盘读到。"""
|
||
path = tmp_path / "baseline_cache.json"
|
||
cache = BaselineCache(path)
|
||
|
||
cache.put("AR", "h1", "v1", "q1", True)
|
||
|
||
# 内存可读
|
||
assert cache.get("AR", "h1", "v1", "q1") is True
|
||
|
||
# 新实例从磁盘加载也能读到（证明先落盘）
|
||
cache2 = BaselineCache(path)
|
||
assert cache2.get("AR", "h1", "v1", "q1") is True
|
||
|
||
# .tmp 文件不应残留
|
||
assert not (tmp_path / "baseline_cache.json.tmp").exists()
|
||
|
||
def test_baseline_cache_empty_init(self, tmp_path: Path) -> None:
|
||
"""不存在的文件 -> 空缓存初始化。"""
|
||
path = tmp_path / "nonexistent.json"
|
||
cache = BaselineCache(path)
|
||
assert cache.get("AR", "h1", "v1", "q1") is None
|
||
|
||
def test_baseline_cache_overwrite(self, tmp_path: Path) -> None:
|
||
"""同键重复写入覆盖旧值。"""
|
||
path = tmp_path / "baseline_cache.json"
|
||
cache = BaselineCache(path)
|
||
|
||
cache.put("AR", "h1", "v1", "q1", True)
|
||
assert cache.get("AR", "h1", "v1", "q1") is True
|
||
|
||
cache.put("AR", "h1", "v1", "q1", False)
|
||
assert cache.get("AR", "h1", "v1", "q1") is False
|
||
|
||
|
||
# ── skill_hash ─────────────────────────────────────────────────────────
|
||
|
||
|
||
class TestSkillHash:
|
||
"""skill_hash SHA1 摘要。"""
|
||
|
||
def test_deterministic(self) -> None:
|
||
"""相同输入产生相同摘要。"""
|
||
assert skill_hash("hello") == skill_hash("hello")
|
||
|
||
def test_different_content(self) -> None:
|
||
"""不同输入产生不同摘要。"""
|
||
assert skill_hash("hello") != skill_hash("world")
|
||
|
||
def test_is_sha1_hex(self) -> None:
|
||
"""输出为 40 字符十六进制。"""
|
||
h = skill_hash("test")
|
||
assert len(h) == 40
|
||
assert all(c in "0123456789abcdef" for c in h)
|