iomgaa/Video-Tree-TRM5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: address whole-impl review (INFRA T0 rows, reproducible manifest, evolution_target report, dead config, canonical DRY)

Browse Source 
C-1: persist_infra_t0_rows 补 INFRA/空预测错题的 T0 信号行（不进诊断故须单独落库），run_pipeline 加 Phase 0，dry-run 用假数据走通。
C-2: CLI 加 --generated-at，真实运行默认盖真实 UTC now，可显式固定以字节级复现 manifest。
I-1: coverage_report 增 evolution_target_distribution（T2 信号按 tool/skill/system 计数）。
I-2: 删除 PoolConfig 死字段 n_trainval/floor_k/epsilon/report_floor/val_wrong_min（grep 确认无消费者，视频级切分用独立 VideoSplitConfig/SplitBuildConfig/SelectConfig）。
I-3: 抽共享 load_canonical_predictions(db_path, run_id)，CLI 与 build_split 共用；消除 canonical 取行 + correct 判定重复。
M-1: build_split docstring 注明 val_wrong_min-agnostic 契约（McNemar 护栏由 CLI 冻结后执行，Task 11 契约）。
This commit is contained in:
iomgaa
2026-07-15 13:39:14 -04:00
parent 02b8145b7f
commit 8fef7ced42
6 changed files with 302 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/integration/test_build_split_e2e.py
+7
View File
@@ -153,3 +153,10 @@ def test_end_to_end_freezes_valid_pools(tmp_path: Path) -> None:
assert coverage["grid_total"] == 48
assert "tier_distribution" in coverage
assert manifest_path.exists()
# I-1：evolution_target_distribution 出现在 coverage_report，且 T2 信号按 tool/skill/system 计数。
target_dist = coverage["evolution_target_distribution"]
assert set(target_dist) <= {"tool", "skill", "system"}
# 全部错题构造为 T2（error_type 四类轮转），进化目标覆盖三层且计数为正。
assert sum(target_dist.values()) > 0
assert set(target_dist) == {"tool", "skill", "system"}
tests/unit/test_pool_config_video_split.py
+6 -53
View File
@@ -1,10 +1,12 @@
"""视频级切分科研旋钮单元测试：诊断指纹 + val_wrong_min 功效护栏 + PoolConfig 新字段。
"""视频级切分科研旋钮单元测试：诊断指纹 + val_wrong_min 功效护栏。
覆盖：
- diag_fingerprint 对 (prompt 版本 / 模型 / 代码版本) 三元组确定且敏感；
- split_by_video_assignment 的 val_wrong_min 门控 fail loud（验证信号不足即报错）；
- val_wrong_min 默认 0 时行为与 Task 11 现有调用完全一致（不回归）；
- PoolConfig 能接收视频级切分的五个新旋钮字段（纯 dataclass 装配）。
- val_wrong_min 默认 0 时行为与 Task 11 现有调用完全一致（不回归）。
注：结果驱动视频级切分不复用 PoolConfig——它有独立的 VideoSplitConfig /
SplitBuildConfig / SelectConfig，故 PoolConfig 不承载视频级切分旋钮（无死配置面）。
"""
from __future__ import annotations
@@ -13,7 +15,7 @@ import pytest
from app.harness.pools import InsufficientValSignal, split_by_video_assignment
from app.harness.split_selection import diag_fingerprint
from core.types import GeneratedQuestion, PoolConfig
from core.types import GeneratedQuestion
def _q(qid: str, vid: str, tt: str = "Counting Problem") -> GeneratedQuestion:
@@ -64,52 +66,3 @@ def test_val_wrong_min_default_zero_no_regression():
qs, assignment, correctness=correctness, val_ratio=1.0, seed=0
)
assert len(pools.validation) == 2 # 未抛异常，正常返回
def test_pool_config_accepts_video_split_knobs():
"""PoolConfig 能接收视频级切分五个新旋钮字段（默认惰性，不破坏现有构造点）。"""
cfg = PoolConfig(
task_types=None,
seed=0,
baseline_run_id="infer_adhoc",
diag_size=200,
diag_correct_ratio=0.5,
val_size=30,
val_correct_ratio=0.5,
test_size=60,
eval_min_per_class=2,
train_ratio=0.667,
test_questions_dir=None,
n_trainval=100,
floor_k={"Counting Problem": 3},
epsilon=0.1,
report_floor=27,
val_wrong_min=20,
)
assert cfg.n_trainval == 100
assert cfg.floor_k == {"Counting Problem": 3}
assert cfg.epsilon == 0.1
assert cfg.report_floor == 27
assert cfg.val_wrong_min == 20
def test_pool_config_video_split_knobs_default_inert():
"""未传视频级切分字段时默认惰性（0 / 空 dict），不破坏 GlobalPoolStrategy 现有构造。"""
cfg = PoolConfig(
task_types=None,
seed=0,
baseline_run_id="run_1",
diag_size=200,
diag_correct_ratio=0.5,
val_size=30,
val_correct_ratio=0.5,
test_size=60,
eval_min_per_class=2,
train_ratio=0.667,
test_questions_dir=None,
)
assert cfg.n_trainval == 0
assert cfg.floor_k == {}
assert cfg.epsilon == 0.0
assert cfg.report_floor == 0
assert cfg.val_wrong_min == 0
tests/unit/test_video_split_cli.py
+99 -4
View File
@@ -102,10 +102,18 @@ def test_check_mcnemar_power_zero_threshold_skips():
assert cli.check_mcnemar_power(pools, val_wrong_min=0) == 0
def test_run_pipeline_orders_two_phases(monkeypatch, tmp_path):
"""run_pipeline 先跑 Phase 1 诊断、后跑 Phase 2 build_split（按序）。"""
def test_run_pipeline_orders_three_phases(monkeypatch, tmp_path):
"""run_pipeline 先补 Phase 0 INFRA T0、再 Phase 1 诊断、后 Phase 2 build_split（按序）。"""
calls: list[str] = []
class _SpyStore:
def __init__(self):
self.t0_rows: list = []
def upsert(self, row):
calls.append("t0_upsert")
self.t0_rows.append(row)
async def fake_diag(**kwargs):
calls.append("diagnosis")
assert kwargs["diag_fingerprint"] == "fp"
@@ -120,24 +128,111 @@ def test_run_pipeline_orders_two_phases(monkeypatch, tmp_path):
monkeypatch.setattr(cli, "run_baseline_diagnosis", fake_diag)
monkeypatch.setattr(cli, "build_split", fake_build_split)
# 一条 INFRA 空预测错题 → Phase 0 应补一行 T0（在诊断/切分之前）。
canonical_preds = [
{
"question_id": "q_infra",
"video_id": "v9",
"task_type": "Counting Problem",
"prediction": "",
"answer": "A",
"stop_reason": "error",
"correct": False,
}
]
store = _SpyStore()
result = asyncio.run(
cli.run_pipeline(
config=_config(),
fingerprint="fp",
diagnosis_deps=object(),
signal_store=object(),
signal_store=store,
wrong_ids=["q1"],
questions={},
canonical_preds=canonical_preds,
harness_db=tmp_path / "h.db",
questions_dir=tmp_path,
out_dir=tmp_path / "out",
generated_at="2026-07-15T00:00:00Z",
)
)
assert calls == ["diagnosis", "build_split"]
assert calls == ["t0_upsert", "diagnosis", "build_split"] # Phase 0 先于诊断与切分
assert len(store.t0_rows) == 1
assert store.t0_rows[0].tier == "T0" and store.t0_rows[0].infra is True
assert result.pools.validation == []
def test_select_diagnosable_wrong_ids_excludes_infra_and_correct():
"""可诊断错题筛选：排除 INFRA / 空预测 / 正确题，保留非空非 INFRA 错题（保序）。"""
preds = [
{"question_id": "ok", "stop_reason": "finished", "prediction": "B", "correct": True},
{"question_id": "wrong", "stop_reason": "finished", "prediction": "C", "correct": False},
{"question_id": "infra", "stop_reason": "error", "prediction": "", "correct": False},
{"question_id": "parse", "stop_reason": "parse_error", "prediction": "x", "correct": False},
{"question_id": "empty", "stop_reason": "finished", "prediction": "", "correct": False},
]
assert cli.select_diagnosable_wrong_ids(preds) == ["wrong"]
def test_persist_infra_t0_rows_persists_only_infra_or_empty_wrong(tmp_path):
"""INFRA / 空预测错题落 T0（infra=True，error_type/target=None）；正确 / 可诊断错题不落。"""
from adapters.baseline_diagnosis_store import SqliteDiagnosisSignalStore
preds = [
{
"question_id": "ok",
"video_id": "v1",
"task_type": "Counting Problem",
"prediction": "A",
"answer": "A",
"stop_reason": "finished",
"correct": True,
},
{
"question_id": "diag_wrong",
"video_id": "v2",
"task_type": "Counting Problem",
"prediction": "C",
"answer": "A",
"stop_reason": "finished",
"correct": False,
},
{
"question_id": "infra_err",
"video_id": "v3",
"task_type": "OCR Problems",
"prediction": "",
"answer": "A",
"stop_reason": "error",
"correct": False,
},
{
"question_id": "parse_err",
"video_id": "v4",
"task_type": "Counting Problem",
"prediction": "",
"answer": "A",
"stop_reason": "parse_error",
"correct": False,
},
]
store = SqliteDiagnosisSignalStore(str(tmp_path / "h.db"))
n = cli.persist_infra_t0_rows(store, preds, "infer_adhoc", "fp")
assert n == 2 # 只有两条 INFRA 空预测错题
rows = {r.question_id: r for r in store.load("infer_adhoc", "fp")}
assert set(rows) == {"infra_err", "parse_err"}
for r in rows.values():
assert r.tier == "T0" and r.infra is True
assert r.error_type is None and r.evolution_target is None and r.cause_category is None
assert r.degraded is False
# 幂等：重复调用同 PK 覆盖，行数不变。
assert cli.persist_infra_t0_rows(store, preds, "infer_adhoc", "fp") == 2
assert len(store.load("infer_adhoc", "fp")) == 2
store.close()
def test_dry_run_computes_fingerprint_without_llm(monkeypatch, tmp_path, capsys):
"""--dry-run：diag_fingerprint 被调用、Phase 1 走空错题早返回、不真调 LLM。"""
fp_calls: list[tuple[str, str, str]] = []