refactor: self-contained two-phase video-split CLI entry

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-15 13:12:21 -04:00
parent 6a21d80313
commit 2844732126
3 changed files with 878 additions and 124 deletions
@@ -0,0 +1,168 @@
+"""video_split_cli 两阶段编排 CLI 单元测试。
+
+覆盖：
+    - 配置解析正确 + 缺关键项 fail loud（SystemExit）；
+    - 指纹计算在 main 中被调用（diag_fingerprint 收到 prompt/model/git-sha 三分量）；
+    - run_pipeline 两阶段按序触发（Phase 1 诊断 → Phase 2 build_split）；
+    - McNemar 功效护栏 fail loud；
+    - --dry-run 用假 deps 跑通装配、不真调 LLM（Phase 1 空错题早返回）。
+
+不真跑全量诊断：诊断与 build_split 均以 monkeypatch / 假 deps 替换。
+"""
+
+from __future__ import annotations
+
+import asyncio
+from types import SimpleNamespace
+
+import pytest
+
+from app.harness import video_split_cli as cli
+
+_RAW_OK = {
+    "video_split": {
+        "baseline_run_id": "infer_adhoc",
+        "n_trainval": 100,
+        "epsilon": 0.1,
+        "report_floor": 27,
+        "val_wrong_min": 20,
+        "val_ratio": 0.3,
+        "seed": 7,
+        "floor_k": {"Counting Problem": 3},
+    },
+    "diag": {"prompt_version": "diagnose_v1", "model": "deepseek-v4-pro"},
+}
+
+
+def _config(**overrides) -> cli.VideoSplitConfig:
+    """构造一个可用 VideoSplitConfig，overrides 覆盖单字段。"""
+    base = {
+        "baseline_run_id": "infer_adhoc",
+        "n_trainval": 100,
+        "epsilon": 0.1,
+        "report_floor": 27,
+        "val_wrong_min": 0,
+        "val_ratio": 0.3,
+        "seed": 7,
+        "floor_k": {"Counting Problem": 3},
+        "prompt_version": "diagnose_v1",
+        "model": "deepseek-v4-pro",
+    }
+    base.update(overrides)
+    return cli.VideoSplitConfig(**base)
+
+
+def test_parse_config_ok():
+    """完整 yaml 解析为 VideoSplitConfig，各旋钮逐一还原。"""
+    cfg = cli.parse_config(_RAW_OK)
+    assert cfg.baseline_run_id == "infer_adhoc"
+    assert cfg.n_trainval == 100
+    assert cfg.epsilon == 0.1
+    assert cfg.report_floor == 27
+    assert cfg.val_wrong_min == 20
+    assert cfg.val_ratio == 0.3
+    assert cfg.seed == 7
+    assert cfg.floor_k == {"Counting Problem": 3}
+    assert cfg.prompt_version == "diagnose_v1"
+    assert cfg.model == "deepseek-v4-pro"
+
+
+def test_parse_config_missing_video_split_key_fails_loud():
+    """video_split 段缺关键项 → SystemExit（P5 fail loud）。"""
+    raw = {"video_split": dict(_RAW_OK["video_split"]), "diag": dict(_RAW_OK["diag"])}
+    del raw["video_split"]["n_trainval"]
+    with pytest.raises(SystemExit):
+        cli.parse_config(raw)
+
+
+def test_parse_config_missing_diag_section_fails_loud():
+    """缺 diag 段 → SystemExit。"""
+    with pytest.raises(SystemExit):
+        cli.parse_config({"video_split": dict(_RAW_OK["video_split"])})
+
+
+def test_load_config_missing_file_fails_loud(tmp_path):
+    """config 文件不存在 → SystemExit。"""
+    with pytest.raises(SystemExit):
+        cli.load_config(tmp_path / "nope.yaml")
+
+
+def test_check_mcnemar_power_below_threshold_fails_loud():
+    """val 错题数 < 阈 → SystemExit（功效不足）。"""
+    q = SimpleNamespace(question_id="q1")
+    pools = SimpleNamespace(validation=[q], correctness={"q1": False})
+    with pytest.raises(SystemExit):
+        cli.check_mcnemar_power(pools, val_wrong_min=5)
+
+
+def test_check_mcnemar_power_zero_threshold_skips():
+    """val_wrong_min=0 → 不检查，返回实际错题数。"""
+    q = SimpleNamespace(question_id="q1")
+    pools = SimpleNamespace(validation=[q], correctness={"q1": True})
+    assert cli.check_mcnemar_power(pools, val_wrong_min=0) == 0
+
+
+def test_run_pipeline_orders_two_phases(monkeypatch, tmp_path):
+    """run_pipeline 先跑 Phase 1 诊断、后跑 Phase 2 build_split（按序）。"""
+    calls: list[str] = []
+
+    async def fake_diag(**kwargs):
+        calls.append("diagnosis")
+        assert kwargs["diag_fingerprint"] == "fp"
+        assert kwargs["wrong_ids"] == ["q1"]
+
+    def fake_build_split(**kwargs):
+        calls.append("build_split")
+        assert kwargs["diag_fingerprint"] == "fp"
+        pools = SimpleNamespace(validation=[], correctness={})
+        return SimpleNamespace(pools=pools, manifest={}, assignment={})
+
+    monkeypatch.setattr(cli, "run_baseline_diagnosis", fake_diag)
+    monkeypatch.setattr(cli, "build_split", fake_build_split)
+
+    result = asyncio.run(
+        cli.run_pipeline(
+            config=_config(),
+            fingerprint="fp",
+            diagnosis_deps=object(),
+            signal_store=object(),
+            wrong_ids=["q1"],
+            questions={},
+            harness_db=tmp_path / "h.db",
+            questions_dir=tmp_path,
+            out_dir=tmp_path / "out",
+            generated_at="2026-07-15T00:00:00Z",
+        )
+    )
+    assert calls == ["diagnosis", "build_split"]
+    assert result.pools.validation == []
+
+
+def test_dry_run_computes_fingerprint_without_llm(monkeypatch, tmp_path, capsys):
+    """--dry-run：diag_fingerprint 被调用、Phase 1 走空错题早返回、不真调 LLM。"""
+    fp_calls: list[tuple[str, str, str]] = []
+
+    def spy_fingerprint(prompt_version, model, code_version):
+        fp_calls.append((prompt_version, model, code_version))
+        return "dryfp"
+
+    monkeypatch.setattr(cli, "diag_fingerprint", spy_fingerprint)
+    monkeypatch.setattr(cli, "git_short_sha", lambda: "abc123")
+
+    # 用真实 config/video_split.yaml，out-dir 指向 tmp（dry-run 会建临时信号库再清理）。
+    cli.main(["--dry-run", "--config", "config/video_split.yaml", "--out-dir", str(tmp_path)])
+
+    assert len(fp_calls) == 1
+    prompt_version, model, sha = fp_calls[0]
+    assert sha == "abc123"
+    assert prompt_version == "diagnose_v1"
+    assert model == "deepseek-v4-pro"
+    # dry-run 不留下临时信号库
+    assert not (tmp_path / "_dry_run_signals.db").exists()
+
+
+def test_git_short_sha_nonempty():
+    """仓库内 git_short_sha 返回非空短 SHA。"""
+    sha = cli.git_short_sha()
+    assert sha
+    assert len(sha) >= 4