fix: address whole-impl review (INFRA T0 rows, reproducible manifest, evolution_target report, dead config, canonical DRY)

C-1: persist_infra_t0_rows 补 INFRA/空预测错题的 T0 信号行(不进诊断故须单独落库),run_pipeline 加 Phase 0,dry-run 用假数据走通。
C-2: CLI 加 --generated-at,真实运行默认盖真实 UTC now,可显式固定以字节级复现 manifest。
I-1: coverage_report 增 evolution_target_distribution(T2 信号按 tool/skill/system 计数)。
I-2: 删除 PoolConfig 死字段 n_trainval/floor_k/epsilon/report_floor/val_wrong_min(grep 确认无消费者,视频级切分用独立 VideoSplitConfig/SplitBuildConfig/SelectConfig)。
I-3: 抽共享 load_canonical_predictions(db_path, run_id),CLI 与 build_split 共用;消除 canonical 取行 + correct 判定重复。
M-1: build_split docstring 注明 val_wrong_min-agnostic 契约(McNemar 护栏由 CLI 冻结后执行,Task 11 契约)。
This commit is contained in:
2026-07-15 13:39:14 -04:00
parent 02b8145b7f
commit 8fef7ced42
6 changed files with 302 additions and 134 deletions
+99 -4
View File
@@ -102,10 +102,18 @@ def test_check_mcnemar_power_zero_threshold_skips():
assert cli.check_mcnemar_power(pools, val_wrong_min=0) == 0
def test_run_pipeline_orders_two_phases(monkeypatch, tmp_path):
"""run_pipeline 先 Phase 1 诊断、后 Phase 2 build_split(按序)。"""
def test_run_pipeline_orders_three_phases(monkeypatch, tmp_path):
"""run_pipeline 先补 Phase 0 INFRA T0、再 Phase 1 诊断、后 Phase 2 build_split(按序)。"""
calls: list[str] = []
class _SpyStore:
def __init__(self):
self.t0_rows: list = []
def upsert(self, row):
calls.append("t0_upsert")
self.t0_rows.append(row)
async def fake_diag(**kwargs):
calls.append("diagnosis")
assert kwargs["diag_fingerprint"] == "fp"
@@ -120,24 +128,111 @@ def test_run_pipeline_orders_two_phases(monkeypatch, tmp_path):
monkeypatch.setattr(cli, "run_baseline_diagnosis", fake_diag)
monkeypatch.setattr(cli, "build_split", fake_build_split)
# 一条 INFRA 空预测错题 → Phase 0 应补一行 T0(在诊断/切分之前)。
canonical_preds = [
{
"question_id": "q_infra",
"video_id": "v9",
"task_type": "Counting Problem",
"prediction": "",
"answer": "A",
"stop_reason": "error",
"correct": False,
}
]
store = _SpyStore()
result = asyncio.run(
cli.run_pipeline(
config=_config(),
fingerprint="fp",
diagnosis_deps=object(),
signal_store=object(),
signal_store=store,
wrong_ids=["q1"],
questions={},
canonical_preds=canonical_preds,
harness_db=tmp_path / "h.db",
questions_dir=tmp_path,
out_dir=tmp_path / "out",
generated_at="2026-07-15T00:00:00Z",
)
)
assert calls == ["diagnosis", "build_split"]
assert calls == ["t0_upsert", "diagnosis", "build_split"] # Phase 0 先于诊断与切分
assert len(store.t0_rows) == 1
assert store.t0_rows[0].tier == "T0" and store.t0_rows[0].infra is True
assert result.pools.validation == []
def test_select_diagnosable_wrong_ids_excludes_infra_and_correct():
"""可诊断错题筛选:排除 INFRA / 空预测 / 正确题,保留非空非 INFRA 错题(保序)。"""
preds = [
{"question_id": "ok", "stop_reason": "finished", "prediction": "B", "correct": True},
{"question_id": "wrong", "stop_reason": "finished", "prediction": "C", "correct": False},
{"question_id": "infra", "stop_reason": "error", "prediction": "", "correct": False},
{"question_id": "parse", "stop_reason": "parse_error", "prediction": "x", "correct": False},
{"question_id": "empty", "stop_reason": "finished", "prediction": "", "correct": False},
]
assert cli.select_diagnosable_wrong_ids(preds) == ["wrong"]
def test_persist_infra_t0_rows_persists_only_infra_or_empty_wrong(tmp_path):
"""INFRA / 空预测错题落 T0infra=Trueerror_type/target=None);正确 / 可诊断错题不落。"""
from adapters.baseline_diagnosis_store import SqliteDiagnosisSignalStore
preds = [
{
"question_id": "ok",
"video_id": "v1",
"task_type": "Counting Problem",
"prediction": "A",
"answer": "A",
"stop_reason": "finished",
"correct": True,
},
{
"question_id": "diag_wrong",
"video_id": "v2",
"task_type": "Counting Problem",
"prediction": "C",
"answer": "A",
"stop_reason": "finished",
"correct": False,
},
{
"question_id": "infra_err",
"video_id": "v3",
"task_type": "OCR Problems",
"prediction": "",
"answer": "A",
"stop_reason": "error",
"correct": False,
},
{
"question_id": "parse_err",
"video_id": "v4",
"task_type": "Counting Problem",
"prediction": "",
"answer": "A",
"stop_reason": "parse_error",
"correct": False,
},
]
store = SqliteDiagnosisSignalStore(str(tmp_path / "h.db"))
n = cli.persist_infra_t0_rows(store, preds, "infer_adhoc", "fp")
assert n == 2 # 只有两条 INFRA 空预测错题
rows = {r.question_id: r for r in store.load("infer_adhoc", "fp")}
assert set(rows) == {"infra_err", "parse_err"}
for r in rows.values():
assert r.tier == "T0" and r.infra is True
assert r.error_type is None and r.evolution_target is None and r.cause_category is None
assert r.degraded is False
# 幂等:重复调用同 PK 覆盖,行数不变。
assert cli.persist_infra_t0_rows(store, preds, "infer_adhoc", "fp") == 2
assert len(store.load("infer_adhoc", "fp")) == 2
store.close()
def test_dry_run_computes_fingerprint_without_llm(monkeypatch, tmp_path, capsys):
"""--dry-rundiag_fingerprint 被调用、Phase 1 走空错题早返回、不真调 LLM。"""
fp_calls: list[tuple[str, str, str]] = []