feat(harness): gate_ladder.py — 信息阶梯 + BaselineCache (#6 算法保真)

2026-07-07 12:55:12 -04:00
parent e0f3ee10ec
commit bd4e438c6c
2 changed files with 709 additions and 0 deletions
@@ -0,0 +1,374 @@
+"""app/harness/gate_ladder.py 单元测试。
+
+覆盖冷启动交错、Beta(1,1) 平滑、warm 信息量排序、
+GatePools 原子读写与指纹校验、BaselineCache 四维内容寻址与先盘后存、
+gamma-EMA 更新、防泄露过滤等核心语义。
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from app.harness.gate_ladder import (
+    BaselineCache,
+    GatePools,
+    LadderEntry,
+    build_cold_entries,
+    build_or_load_gate_pools,
+    order_ladder,
+    skill_hash,
+)
+from core.types import GeneratedQuestion
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+# ── 工具函数 ──────────────────────────────────────────────────────────
+
+
+def _make_q(qid: str, task_type: str = "AR") -> GeneratedQuestion:
+    """构造最小 GeneratedQuestion 实例。"""
+    return GeneratedQuestion(
+        question_id=qid,
+        video_id="v1",
+        task_type=task_type,
+        question="dummy",
+        options=("A", "B", "C", "D"),
+        answer="A",
+        source_nodes=("n1",),
+        difficulty="easy",
+    )
+
+
+# ── 冷启动 ────────────────────────────────────────────────────────────
+
+
+class TestColdStart:
+    """冷启动排序：2:1 交错 + 探针插尾 + Beta(1,1) 平滑。"""
+
+    def test_cold_start_interleaving(self) -> None:
+        """错题:对题 = 2:1 交错顺序。
+
+        6 错 3 对（probe_quota=0 无探针）→ 交错序应为 W W R W W R W W R。
+        """
+        wrong_ids = [f"w{i}" for i in range(6)]
+        right_ids = [f"r{i}" for i in range(3)]
+        questions = [_make_q(qid) for qid in wrong_ids + right_ids]
+        correctness = dict.fromkeys(wrong_ids, False)
+        correctness.update(dict.fromkeys(right_ids, True))
+
+        entries = build_cold_entries(questions, correctness, probe_quota=0.0, seed=42)
+
+        assert len(entries) == 9
+        # 验证 2:1 交错模式（seed 固定后 shuffle 结果确定）
+        pattern = ["W" if not correctness[e.question_id] else "R" for e in entries]
+        # 前 9 个交错应为 W W R W W R W W R
+        assert pattern == ["W", "W", "R", "W", "W", "R", "W", "W", "R"]
+
+    def test_cold_start_p_hat_beta(self) -> None:
+        """p_hat 遵循 Beta(1,1) 平滑：错=1/3，对=2/3。"""
+        questions = [_make_q("q1"), _make_q("q2")]
+        correctness = {"q1": False, "q2": True}
+
+        entries = build_cold_entries(questions, correctness, probe_quota=0.0, seed=0)
+
+        p_map = {e.question_id: e.p_hat for e in entries}
+        assert p_map["q1"] == pytest.approx(1 / 3)
+        assert p_map["q2"] == pytest.approx(2 / 3)
+
+    def test_cold_start_probe_at_tail(self) -> None:
+        """probe_quota > 0 时探针题追加在尾部。"""
+        wrong_ids = [f"w{i}" for i in range(10)]
+        right_ids = [f"r{i}" for i in range(2)]
+        questions = [_make_q(qid) for qid in wrong_ids + right_ids]
+        correctness = dict.fromkeys(wrong_ids, False)
+        correctness.update(dict.fromkeys(right_ids, True))
+
+        entries = build_cold_entries(questions, correctness, probe_quota=0.3, seed=7)
+
+        # 10 错 * 0.3 = 3 个探针在尾部
+        n_probe = int(10 * 0.3)
+        assert n_probe == 3
+        # 尾部 3 个都应为错题
+        tail = entries[-n_probe:]
+        for e in tail:
+            assert not correctness[e.question_id]
+
+
+# ── warm 排序 ──────────────────────────────────────────────────────────
+
+
+class TestWarmOrdering:
+    """warm 阶段：信息量 p_hat(1-p_hat) 降序 + p_hat 区间过滤。"""
+
+    def test_warm_ordering_information(self) -> None:
+        """p_hat=0.5 信息量最高，排在最前。"""
+        entries = [
+            LadderEntry("a", 0.1),
+            LadderEntry("b", 0.5),
+            LadderEntry("c", 0.9),
+            LadderEntry("d", 0.3),
+        ]
+        ordered = order_ladder(entries, p_low=0.0, p_high=1.0)
+        assert ordered[0].question_id == "b"  # 0.5*(1-0.5)=0.25 最高
+        # d: 0.3*0.7=0.21, a: 0.1*0.9=0.09, c: 0.9*0.1=0.09
+        assert ordered[1].question_id == "d"
+
+    def test_warm_filter_bounds(self) -> None:
+        """p_hat 不在 [p_low, p_high] 区间的题被剔除。"""
+        entries = [
+            LadderEntry("low", 0.05),
+            LadderEntry("mid", 0.5),
+            LadderEntry("high", 0.95),
+        ]
+        ordered = order_ladder(entries, p_low=0.1, p_high=0.9)
+        ids = [e.question_id for e in ordered]
+        assert "mid" in ids
+        assert "low" not in ids
+        assert "high" not in ids
+
+
+# ── GatePools 持久化 ──────────────────────────────────────────────────
+
+
+class TestGatePoolsPersistence:
+    """GatePools.save/load 原子性与指纹校验。"""
+
+    def test_gate_pools_save_load_atomic(self, tmp_path: Path) -> None:
+        """save -> load 往返保真，且使用原子写（中间 .tmp 文件不残留）。"""
+        entries = {
+            "AR": [LadderEntry("q1", 0.33), LadderEntry("q2", 0.67)],
+            "CR": [LadderEntry("q3", 0.5)],
+        }
+        pools = GatePools(entries=entries, seed=42, fingerprint="abc123")
+        path = tmp_path / "gate_pools.json"
+        pools.save(path)
+
+        # .tmp 文件不应残留
+        assert not (tmp_path / "gate_pools.json.tmp").exists()
+        assert path.exists()
+
+        loaded = GatePools.load(path)
+        assert loaded.seed == 42
+        assert loaded.fingerprint == "abc123"
+        assert len(loaded.entries["AR"]) == 2
+        assert loaded.entries["AR"][0].question_id == "q1"
+        assert loaded.entries["AR"][0].p_hat == pytest.approx(0.33)
+        assert loaded.entries["CR"][0].question_id == "q3"
+
+    def test_gate_pools_fingerprint_mismatch(self, tmp_path: Path) -> None:
+        """指纹不一致 -> RuntimeError（不静默重建）。"""
+        questions = [_make_q("q1", "AR"), _make_q("q2", "AR")]
+        correctness = {"q1": True, "q2": False}
+
+        # 第一次构建
+        build_or_load_gate_pools(
+            workspace_dir=tmp_path,
+            questions=questions,
+            test_qids=set(),
+            baseline_correctness=correctness,
+            task_types=["AR"],
+            probe_quota=0.0,
+            seed=1,
+            baseline_run_id="run_001",
+        )
+
+        # 改 baseline_run_id 导致指纹变化 -> 应报错
+        with pytest.raises(RuntimeError, match="指纹不一致"):
+            build_or_load_gate_pools(
+                workspace_dir=tmp_path,
+                questions=questions,
+                test_qids=set(),
+                baseline_correctness=correctness,
+                task_types=["AR"],
+                probe_quota=0.0,
+                seed=1,
+                baseline_run_id="run_002",
+            )
+
+
+# ── ladder_for ────────────────────────────────────────────────────────
+
+
+class TestLadderFor:
+    """ladder_for 取题序与排除逻辑。"""
+
+    def test_ladder_for_excludes_qids(self) -> None:
+        """exclude_qids 中的题被排除。"""
+        entries = {
+            "AR": [
+                LadderEntry("q1", 0.5),
+                LadderEntry("q2", 0.4),
+                LadderEntry("q3", 0.6),
+            ],
+        }
+        pools = GatePools(entries=entries, seed=0, fingerprint="x")
+        result = pools.ladder_for("AR", exclude_qids={"q2"}, p_low=0.0, p_high=1.0, cold=True)
+        assert "q2" not in result
+        assert "q1" in result
+        assert "q3" in result
+
+    def test_ladder_for_missing_task_type(self) -> None:
+        """不存在的 task_type -> ValueError。"""
+        pools = GatePools(entries={}, seed=0, fingerprint="x")
+        with pytest.raises(ValueError, match="无阶梯"):
+            pools.ladder_for("MISSING", set(), 0.0, 1.0, cold=True)
+
+    def test_ladder_for_warm_uses_order_ladder(self) -> None:
+        """cold=False 时走 warm 信息量排序。"""
+        entries = {
+            "AR": [
+                LadderEntry("low", 0.1),
+                LadderEntry("mid", 0.5),
+                LadderEntry("high", 0.9),
+            ],
+        }
+        pools = GatePools(entries=entries, seed=0, fingerprint="x")
+        result = pools.ladder_for("AR", set(), p_low=0.0, p_high=1.0, cold=False)
+        # 信息量排序：mid(0.25) > low(0.09) = high(0.09)
+        assert result[0] == "mid"
+
+
+# ── gamma-EMA 更新 ─────────────────────────────────────────────────────
+
+
+class TestGammaEMA:
+    """gamma-EMA 更新 p_hat。"""
+
+    def test_gamma_ema_update(self) -> None:
+        """p_hat <- gamma * p_hat + (1-gamma) * obs。"""
+        entries = {"AR": [LadderEntry("q1", 0.5)]}
+        pools = GatePools(entries=entries, seed=0, fingerprint="x")
+
+        # 观测为正确(1.0), gamma=0.8
+        pools.update_probs({"q1": True}, gamma=0.8)
+        expected = 0.8 * 0.5 + 0.2 * 1.0  # 0.6
+        assert pools.entries["AR"][0].p_hat == pytest.approx(expected)
+
+        # 再次观测为错误(0.0), gamma=0.8
+        pools.update_probs({"q1": False}, gamma=0.8)
+        expected2 = 0.8 * expected + 0.2 * 0.0  # 0.48
+        assert pools.entries["AR"][0].p_hat == pytest.approx(expected2)
+
+    def test_update_probs_no_observation_unchanged(self) -> None:
+        """无观测的题 p_hat 不变。"""
+        entries = {"AR": [LadderEntry("q1", 0.5), LadderEntry("q2", 0.3)]}
+        pools = GatePools(entries=entries, seed=0, fingerprint="x")
+        pools.update_probs({"q1": True}, gamma=0.9)
+        assert pools.entries["AR"][1].p_hat == pytest.approx(0.3)
+
+
+# ── 防泄露 ─────────────────────────────────────────────────────────────
+
+
+class TestLeakPrevention:
+    """防泄露铁律：gate 内 rollout 永不回流 p_hat（由调用方过滤）。"""
+
+    def test_update_probs_excludes_gate_runs(self) -> None:
+        """调用方须过滤 run_id 含 '_gate_' 的观测。
+
+        update_probs 本身只接收已过滤的 observations，这里验证
+        如果调用方正确过滤，gate run 数据不会影响 p_hat。
+        """
+        entries = {"AR": [LadderEntry("q1", 0.5)]}
+        pools = GatePools(entries=entries, seed=0, fingerprint="x")
+
+        # 模拟：所有 run 的原始观测（含 gate run）
+        raw_observations = {
+            "run_normal": {"q1": True},  # 普通 run
+            "run_gate_01": {"q1": False},  # gate run（run_id 含 _gate_）
+        }
+
+        # 调用方按 run_id 过滤：排除含 "_gate_" 的 run
+        filtered = {}
+        for run_id, obs in raw_observations.items():
+            if "_gate_" not in run_id:
+                filtered.update(obs)
+
+        # 只有普通 run 的观测进入 update_probs
+        assert filtered == {"q1": True}
+        pools.update_probs(filtered, gamma=0.8)
+        expected = 0.8 * 0.5 + 0.2 * 1.0
+        assert pools.entries["AR"][0].p_hat == pytest.approx(expected)
+
+
+# ── BaselineCache ──────────────────────────────────────────────────────
+
+
+class TestBaselineCache:
+    """BaselineCache 四维内容寻址与先盘后存。"""
+
+    def test_baseline_cache_content_addressed(self, tmp_path: Path) -> None:
+        """四维键唯一寻址：任一维度变化 -> miss。"""
+        path = tmp_path / "baseline_cache.json"
+        cache = BaselineCache(path)
+
+        cache.put("AR", "hash1", "v1", "q1", True)
+        assert cache.get("AR", "hash1", "v1", "q1") is True
+
+        # 改 skill_hash -> miss
+        assert cache.get("AR", "hash2", "v1", "q1") is None
+        # 改 prompts_version -> miss
+        assert cache.get("AR", "hash1", "v2", "q1") is None
+        # 改 task_type -> miss
+        assert cache.get("CR", "hash1", "v1", "q1") is None
+        # 改 qid -> miss
+        assert cache.get("AR", "hash1", "v1", "q2") is None
+
+    def test_baseline_cache_disk_first(self, tmp_path: Path) -> None:
+        """先盘后存：磁盘写成功后内存才更新，新实例可从磁盘读到。"""
+        path = tmp_path / "baseline_cache.json"
+        cache = BaselineCache(path)
+
+        cache.put("AR", "h1", "v1", "q1", True)
+
+        # 内存可读
+        assert cache.get("AR", "h1", "v1", "q1") is True
+
+        # 新实例从磁盘加载也能读到（证明先落盘）
+        cache2 = BaselineCache(path)
+        assert cache2.get("AR", "h1", "v1", "q1") is True
+
+        # .tmp 文件不应残留
+        assert not (tmp_path / "baseline_cache.json.tmp").exists()
+
+    def test_baseline_cache_empty_init(self, tmp_path: Path) -> None:
+        """不存在的文件 -> 空缓存初始化。"""
+        path = tmp_path / "nonexistent.json"
+        cache = BaselineCache(path)
+        assert cache.get("AR", "h1", "v1", "q1") is None
+
+    def test_baseline_cache_overwrite(self, tmp_path: Path) -> None:
+        """同键重复写入覆盖旧值。"""
+        path = tmp_path / "baseline_cache.json"
+        cache = BaselineCache(path)
+
+        cache.put("AR", "h1", "v1", "q1", True)
+        assert cache.get("AR", "h1", "v1", "q1") is True
+
+        cache.put("AR", "h1", "v1", "q1", False)
+        assert cache.get("AR", "h1", "v1", "q1") is False
+
+
+# ── skill_hash ─────────────────────────────────────────────────────────
+
+
+class TestSkillHash:
+    """skill_hash SHA1 摘要。"""
+
+    def test_deterministic(self) -> None:
+        """相同输入产生相同摘要。"""
+        assert skill_hash("hello") == skill_hash("hello")
+
+    def test_different_content(self) -> None:
+        """不同输入产生不同摘要。"""
+        assert skill_hash("hello") != skill_hash("world")
+
+    def test_is_sha1_hex(self) -> None:
+        """输出为 40 字符十六进制。"""
+        h = skill_hash("test")
+        assert len(h) == 40
+        assert all(c in "0123456789abcdef" for c in h)