feat(harness): 迁移 gate 信息量阶梯到 unit 粒度 + schema_version
核心算法保真#5(信息阶梯):gate_ladder.py 从逐题迁移到 unit 粒度, 只换键 question_id→unit_id,冷启动 2:1 错优先交错、gamma-EMA 公式、 Beta(1,1) 先验、反泄漏 _gate_ 过滤的公式/比例/顺序语义一字不改。 - LadderEntry 按 unit_id 键;AR pair 折叠为一个阶梯单元 - build_cold_entries 收单元列表,unit 错 = 任一成员错(双向 AND)折叠, 2:1 交错 + probe 探针按 unit 抽,Beta 先验 p0 不变 - ladder_for 返回 unit_id 序、exclude 迁到 unit 口径(防半 pair 灌入 触发下游 _ladder_units fail-fast) - update_probs 先把逐题观测折叠成单元观测再按 unit_id 匹配更新, 半观测单元跳过(防按 qid 匹配 pair 失效致 gamma-EMA 停摆) - GatePools.save/load 加 schema_version=2;存量无版本/旧版本 json 加载直接报错,拒绝静默混用 qid/unit 键 - BaselineCache 第四维键改名 unit_id(与 T7 validate 路径对齐) - build_or_load_gate_pools 先折叠单元再排除 test(抽 helper 控复杂度 B) - runner:_init_gate_pools 建 unit 索引;gate 验证 exclude/展开、 _refresh_gate_ladder 折叠观测走 units_by_id 反泄漏 run_id 含 _gate_ 过滤不受影响(未改)。 测试:新增 test_gate_ladder_unit_migration.py(15 例覆盖 a-e), 既有 test_harness_gate_ladder.py 迁移到 unit API。全量 1363 passed。
This commit is contained in:
+23
-7
@@ -854,6 +854,11 @@ class Runner:
|
||||
self._gate_questions_by_id: dict[str, GeneratedQuestion] = {
|
||||
q.question_id: q for q in questions
|
||||
}
|
||||
# unit 索引:ladder_for 返回 unit_id、update_probs 折叠逐题观测均需按 unit_id
|
||||
# 反查成员题(核心算法保真 #5:gate 阶梯 unit 化)。不进 checkpoint、每次启动重建。
|
||||
self._gate_units_by_id: dict[str, QuestionUnit] = {
|
||||
u.unit_id: u for u in build_units(questions)
|
||||
}
|
||||
with HarnessLog(str(self._paths.db_path), pools.baseline_run_id) as log:
|
||||
rows = log.query(
|
||||
"SELECT question_id, prediction, answer FROM predictions WHERE run_id=?",
|
||||
@@ -1108,21 +1113,29 @@ class Runner:
|
||||
from app.harness.log import HarnessLog
|
||||
from app.harness.validate import validate_skill_local
|
||||
|
||||
exclude_qids = {c.question_id for c in pack.failure_cases + pack.success_cases}
|
||||
ladder_qids = state.gate_pools.ladder_for(
|
||||
# 案例包按 unit 排除:把每个 case 的 question_id 映射到其所属 unit_id,
|
||||
# 命中单元整体排除,防止只排 AR pair 半个成员而给 gate 池灌半个 pair
|
||||
# (下游 _ladder_units 会 fail-fast)。核心算法保真 #5。
|
||||
exclude_units = {
|
||||
self._gate_questions_by_id[c.question_id].unit_id
|
||||
for c in pack.failure_cases + pack.success_cases
|
||||
if c.question_id in self._gate_questions_by_id
|
||||
}
|
||||
ladder_unit_ids = state.gate_pools.ladder_for(
|
||||
task_type,
|
||||
exclude_qids,
|
||||
exclude_units,
|
||||
p_low=self._config.gate_p_low,
|
||||
p_high=self._config.gate_p_high,
|
||||
cold=not state.gate_epoch_observed,
|
||||
)
|
||||
missing = [qid for qid in ladder_qids if qid not in self._gate_questions_by_id]
|
||||
missing = [uid for uid in ladder_unit_ids if uid not in self._gate_units_by_id]
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"gate 阶梯[{task_type}] 含 benchmark 中不存在的题: "
|
||||
f"gate 阶梯[{task_type}] 含 benchmark 中不存在的单元: "
|
||||
f"{missing[:5]}(gate_pools.json 与题库失配)"
|
||||
)
|
||||
ladder_items = [self._gate_questions_by_id[qid] for qid in ladder_qids]
|
||||
# 单元展开为逐题(unit 内成员顺序保持),下游 validate 再按阶梯序聚合回单元。
|
||||
ladder_items = [q for uid in ladder_unit_ids for q in self._gate_units_by_id[uid].questions]
|
||||
base_skill_content = (self._paths.skills_dir / record.target_file).read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
@@ -1799,7 +1812,10 @@ class Runner:
|
||||
obs.update({r["question_id"]: r["prediction"] == r["answer"] for r in slow_rows})
|
||||
for extra_rows in extra_rows_lists:
|
||||
obs.update({r["question_id"]: r["prediction"] == r["answer"] for r in extra_rows})
|
||||
state.gate_pools.update_probs(obs, gamma=self._config.gate_gamma_decay)
|
||||
# 逐题观测折叠成单元观测后按 unit_id 匹配更新(防按 qid 匹配 pair 失效致 EMA 停摆)。
|
||||
state.gate_pools.update_probs(
|
||||
obs, self._gate_units_by_id, gamma=self._config.gate_gamma_decay
|
||||
)
|
||||
state.gate_pools.save(self._config.workspace_dir / "gate_pools.json")
|
||||
state.gate_epoch_observed = True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user