iomgaa/Video-Tree-TRM5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(harness): 迁移 gate 信息量阶梯到 unit 粒度 + schema_version

Browse Source 
核心算法保真#5（信息阶梯）：gate_ladder.py 从逐题迁移到 unit 粒度，
只换键 question_id→unit_id，冷启动 2:1 错优先交错、gamma-EMA 公式、
Beta(1,1) 先验、反泄漏 _gate_ 过滤的公式/比例/顺序语义一字不改。

- LadderEntry 按 unit_id 键；AR pair 折叠为一个阶梯单元
- build_cold_entries 收单元列表，unit 错 = 任一成员错（双向 AND）折叠，
  2:1 交错 + probe 探针按 unit 抽，Beta 先验 p0 不变
- ladder_for 返回 unit_id 序、exclude 迁到 unit 口径（防半 pair 灌入
  触发下游 _ladder_units fail-fast）
- update_probs 先把逐题观测折叠成单元观测再按 unit_id 匹配更新，
  半观测单元跳过（防按 qid 匹配 pair 失效致 gamma-EMA 停摆）
- GatePools.save/load 加 schema_version=2；存量无版本/旧版本 json
  加载直接报错，拒绝静默混用 qid/unit 键
- BaselineCache 第四维键改名 unit_id（与 T7 validate 路径对齐）
- build_or_load_gate_pools 先折叠单元再排除 test（抽 helper 控复杂度 B）
- runner：_init_gate_pools 建 unit 索引；gate 验证 exclude/展开、
  _refresh_gate_ladder 折叠观测走 units_by_id

反泄漏 run_id 含 _gate_ 过滤不受影响（未改）。
测试：新增 test_gate_ladder_unit_migration.py（15 例覆盖 a-e），
既有 test_harness_gate_ladder.py 迁移到 unit API。全量 1363 passed。
This commit is contained in:
iomgaa
2026-07-15 07:56:15 -04:00
parent 7e97081779
commit 273984674b
4 changed files with 499 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_harness_gate_ladder.py
+35 -26
View File
@@ -20,6 +20,7 @@ from app.harness.gate_ladder import (
order_ladder,
skill_hash,
)
from app.harness.question_units import build_units
from core.types import GeneratedQuestion
if TYPE_CHECKING:
@@ -43,6 +44,11 @@ def _make_q(qid: str, task_type: str = "AR") -> GeneratedQuestion:
)
def _units(questions: list[GeneratedQuestion]) -> list:
"""把题目列表折叠为单元列表（single 题 unit_id 等于 question_id）。"""
return build_units(questions)
# ── 冷启动 ────────────────────────────────────────────────────────────
@@ -50,52 +56,52 @@ class TestColdStart:
"""冷启动排序：2:1 交错 + 探针插尾 + Beta(1,1) 平滑。"""
def test_cold_start_interleaving(self) -> None:
"""错题:对题 = 2:1 交错顺序。
"""错 unit:对 unit = 2:1 交错顺序。
6 错 3 对（probe_quota=0 无探针）→ 交错序应为 W W R W W R W W R。
"""
wrong_ids = [f"w{i}" for i in range(6)]
right_ids = [f"r{i}" for i in range(3)]
questions = [_make_q(qid) for qid in wrong_ids + right_ids]
units = _units([_make_q(qid) for qid in wrong_ids + right_ids])
correctness = dict.fromkeys(wrong_ids, False)
correctness.update(dict.fromkeys(right_ids, True))
entries = build_cold_entries(questions, correctness, probe_quota=0.0, seed=42)
entries = build_cold_entries(units, correctness, probe_quota=0.0, seed=42)
assert len(entries) == 9
# 验证 2:1 交错模式（seed 固定后 shuffle 结果确定）
pattern = ["W" if not correctness[e.question_id] else "R" for e in entries]
pattern = ["W" if not correctness[e.unit_id] else "R" for e in entries]
# 前 9 个交错应为 W W R W W R W W R
assert pattern == ["W", "W", "R", "W", "W", "R", "W", "W", "R"]
def test_cold_start_p_hat_beta(self) -> None:
"""p_hat 遵循 Beta(1,1) 平滑：错=1/3，对=2/3。"""
questions = [_make_q("q1"), _make_q("q2")]
units = _units([_make_q("q1"), _make_q("q2")])
correctness = {"q1": False, "q2": True}
entries = build_cold_entries(questions, correctness, probe_quota=0.0, seed=0)
entries = build_cold_entries(units, correctness, probe_quota=0.0, seed=0)
p_map = {e.question_id: e.p_hat for e in entries}
p_map = {e.unit_id: e.p_hat for e in entries}
assert p_map["q1"] == pytest.approx(1 / 3)
assert p_map["q2"] == pytest.approx(2 / 3)
def test_cold_start_probe_at_tail(self) -> None:
"""probe_quota > 0 时探针题追加在尾部。"""
"""probe_quota > 0 时探针 unit 追加在尾部。"""
wrong_ids = [f"w{i}" for i in range(10)]
right_ids = [f"r{i}" for i in range(2)]
questions = [_make_q(qid) for qid in wrong_ids + right_ids]
units = _units([_make_q(qid) for qid in wrong_ids + right_ids])
correctness = dict.fromkeys(wrong_ids, False)
correctness.update(dict.fromkeys(right_ids, True))
entries = build_cold_entries(questions, correctness, probe_quota=0.3, seed=7)
entries = build_cold_entries(units, correctness, probe_quota=0.3, seed=7)
# 10 错 * 0.3 = 3 个探针在尾部
n_probe = int(10 * 0.3)
assert n_probe == 3
# 尾部 3 个都应为错题
# 尾部 3 个都应为错 unit
tail = entries[-n_probe:]
for e in tail:
assert not correctness[e.question_id]
assert not correctness[e.unit_id]
# ── warm 排序 ──────────────────────────────────────────────────────────
@@ -113,9 +119,9 @@ class TestWarmOrdering:
LadderEntry("d", 0.3),
]
ordered = order_ladder(entries, p_low=0.0, p_high=1.0)
assert ordered[0].question_id == "b" # 0.5*(1-0.5)=0.25 最高
assert ordered[0].unit_id == "b" # 0.5*(1-0.5)=0.25 最高
# d: 0.3*0.7=0.21, a: 0.1*0.9=0.09, c: 0.9*0.1=0.09
assert ordered[1].question_id == "d"
assert ordered[1].unit_id == "d"
def test_warm_filter_bounds(self) -> None:
"""p_hat 不在 [p_low, p_high] 区间的题被剔除。"""
@@ -125,7 +131,7 @@ class TestWarmOrdering:
LadderEntry("high", 0.95),
]
ordered = order_ladder(entries, p_low=0.1, p_high=0.9)
ids = [e.question_id for e in ordered]
ids = [e.unit_id for e in ordered]
assert "mid" in ids
assert "low" not in ids
assert "high" not in ids
@@ -155,9 +161,9 @@ class TestGatePoolsPersistence:
assert loaded.seed == 42
assert loaded.fingerprint == "abc123"
assert len(loaded.entries["AR"]) == 2
assert loaded.entries["AR"][0].question_id == "q1"
assert loaded.entries["AR"][0].unit_id == "q1"
assert loaded.entries["AR"][0].p_hat == pytest.approx(0.33)
assert loaded.entries["CR"][0].question_id == "q3"
assert loaded.entries["CR"][0].unit_id == "q3"
def test_gate_pools_fingerprint_mismatch(self, tmp_path: Path) -> None:
"""指纹不一致 -> RuntimeError（不静默重建）。"""
@@ -196,8 +202,8 @@ class TestGatePoolsPersistence:
class TestLadderFor:
"""ladder_for 取题序与排除逻辑。"""
def test_ladder_for_excludes_qids(self) -> None:
"""exclude_qids 中的题被排除。"""
def test_ladder_for_excludes_units(self) -> None:
"""exclude_units 中的单元被排除。"""
entries = {
"AR": [
LadderEntry("q1", 0.5),
@@ -206,7 +212,7 @@ class TestLadderFor:
],
}
pools = GatePools(entries=entries, seed=0, fingerprint="x")
result = pools.ladder_for("AR", exclude_qids={"q2"}, p_low=0.0, p_high=1.0, cold=True)
result = pools.ladder_for("AR", exclude_units={"q2"}, p_low=0.0, p_high=1.0, cold=True)
assert "q2" not in result
assert "q1" in result
assert "q3" in result
@@ -236,28 +242,30 @@ class TestLadderFor:
class TestGammaEMA:
"""gamma-EMA 更新 p_hat。"""
"""gamma-EMA 更新 p_hat（single 单元：unit_id 等于 question_id）。"""
def test_gamma_ema_update(self) -> None:
"""p_hat <- gamma * p_hat + (1-gamma) * obs。"""
entries = {"AR": [LadderEntry("q1", 0.5)]}
pools = GatePools(entries=entries, seed=0, fingerprint="x")
units_by_id = {u.unit_id: u for u in _units([_make_q("q1")])}
# 观测为正确(1.0), gamma=0.8
pools.update_probs({"q1": True}, gamma=0.8)
pools.update_probs({"q1": True}, units_by_id, gamma=0.8)
expected = 0.8 * 0.5 + 0.2 * 1.0 # 0.6
assert pools.entries["AR"][0].p_hat == pytest.approx(expected)
# 再次观测为错误(0.0), gamma=0.8
pools.update_probs({"q1": False}, gamma=0.8)
pools.update_probs({"q1": False}, units_by_id, gamma=0.8)
expected2 = 0.8 * expected + 0.2 * 0.0 # 0.48
assert pools.entries["AR"][0].p_hat == pytest.approx(expected2)
def test_update_probs_no_observation_unchanged(self) -> None:
"""无观测的题 p_hat 不变。"""
"""无观测的单元 p_hat 不变。"""
entries = {"AR": [LadderEntry("q1", 0.5), LadderEntry("q2", 0.3)]}
pools = GatePools(entries=entries, seed=0, fingerprint="x")
pools.update_probs({"q1": True}, gamma=0.9)
units_by_id = {u.unit_id: u for u in _units([_make_q("q1"), _make_q("q2")])}
pools.update_probs({"q1": True}, units_by_id, gamma=0.9)
assert pools.entries["AR"][1].p_hat == pytest.approx(0.3)
@@ -290,7 +298,8 @@ class TestLeakPrevention:
# 只有普通 run 的观测进入 update_probs
assert filtered == {"q1": True}
pools.update_probs(filtered, gamma=0.8)
units_by_id = {u.unit_id: u for u in _units([_make_q("q1")])}
pools.update_probs(filtered, units_by_id, gamma=0.8)
expected = 0.8 * 0.5 + 0.2 * 1.0
assert pools.entries["AR"][0].p_hat == pytest.approx(expected)