iomgaa
02b8145b7f
Codex CHANGES_REQUESTED 复审： - Critical：diag_fingerprint 用 config.diag.model，但 Phase 1 诊断 LLM 从 .env SEARCH_LLM_MODEL 构建，两者不一致会让信号以错误模型指纹落库，破坏可复现/ resume/隔离。build_diagnosis_deps 新增 expected_model 参数，Phase 1 执行前 fail loud 校验 config.model == settings.search_llm_model（附两值）。 - Minor：config/video_split.yaml diag.model 注释由 JUDGE_LLM_MODEL 更正为 SEARCH_LLM_MODEL，与实现对齐。 - 补两个单测：模型不一致 fail loud + 缺凭证 fail loud。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
205 lines
7.0 KiB
Python
205 lines
7.0 KiB
Python
"""video_split_cli 两阶段编排 CLI 单元测试。
|
||
|
||
覆盖：
|
||
- 配置解析正确 + 缺关键项 fail loud（SystemExit）；
|
||
- 指纹计算在 main 中被调用（diag_fingerprint 收到 prompt/model/git-sha 三分量）；
|
||
- run_pipeline 两阶段按序触发（Phase 1 诊断 → Phase 2 build_split）；
|
||
- McNemar 功效护栏 fail loud；
|
||
- --dry-run 用假 deps 跑通装配、不真调 LLM（Phase 1 空错题早返回）。
|
||
|
||
不真跑全量诊断：诊断与 build_split 均以 monkeypatch / 假 deps 替换。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
from types import SimpleNamespace
|
||
|
||
import pytest
|
||
|
||
from app.harness import video_split_cli as cli
|
||
|
||
_RAW_OK = {
|
||
"video_split": {
|
||
"baseline_run_id": "infer_adhoc",
|
||
"n_trainval": 100,
|
||
"epsilon": 0.1,
|
||
"report_floor": 27,
|
||
"val_wrong_min": 20,
|
||
"val_ratio": 0.3,
|
||
"seed": 7,
|
||
"floor_k": {"Counting Problem": 3},
|
||
},
|
||
"diag": {"prompt_version": "diagnose_v1", "model": "deepseek-v4-pro"},
|
||
}
|
||
|
||
|
||
def _config(**overrides) -> cli.VideoSplitConfig:
|
||
"""构造一个可用 VideoSplitConfig，overrides 覆盖单字段。"""
|
||
base = {
|
||
"baseline_run_id": "infer_adhoc",
|
||
"n_trainval": 100,
|
||
"epsilon": 0.1,
|
||
"report_floor": 27,
|
||
"val_wrong_min": 0,
|
||
"val_ratio": 0.3,
|
||
"seed": 7,
|
||
"floor_k": {"Counting Problem": 3},
|
||
"prompt_version": "diagnose_v1",
|
||
"model": "deepseek-v4-pro",
|
||
}
|
||
base.update(overrides)
|
||
return cli.VideoSplitConfig(**base)
|
||
|
||
|
||
def test_parse_config_ok():
|
||
"""完整 yaml 解析为 VideoSplitConfig，各旋钮逐一还原。"""
|
||
cfg = cli.parse_config(_RAW_OK)
|
||
assert cfg.baseline_run_id == "infer_adhoc"
|
||
assert cfg.n_trainval == 100
|
||
assert cfg.epsilon == 0.1
|
||
assert cfg.report_floor == 27
|
||
assert cfg.val_wrong_min == 20
|
||
assert cfg.val_ratio == 0.3
|
||
assert cfg.seed == 7
|
||
assert cfg.floor_k == {"Counting Problem": 3}
|
||
assert cfg.prompt_version == "diagnose_v1"
|
||
assert cfg.model == "deepseek-v4-pro"
|
||
|
||
|
||
def test_parse_config_missing_video_split_key_fails_loud():
|
||
"""video_split 段缺关键项 → SystemExit（P5 fail loud）。"""
|
||
raw = {"video_split": dict(_RAW_OK["video_split"]), "diag": dict(_RAW_OK["diag"])}
|
||
del raw["video_split"]["n_trainval"]
|
||
with pytest.raises(SystemExit):
|
||
cli.parse_config(raw)
|
||
|
||
|
||
def test_parse_config_missing_diag_section_fails_loud():
|
||
"""缺 diag 段 → SystemExit。"""
|
||
with pytest.raises(SystemExit):
|
||
cli.parse_config({"video_split": dict(_RAW_OK["video_split"])})
|
||
|
||
|
||
def test_load_config_missing_file_fails_loud(tmp_path):
|
||
"""config 文件不存在 → SystemExit。"""
|
||
with pytest.raises(SystemExit):
|
||
cli.load_config(tmp_path / "nope.yaml")
|
||
|
||
|
||
def test_check_mcnemar_power_below_threshold_fails_loud():
|
||
"""val 错题数 < 阈 → SystemExit（功效不足）。"""
|
||
q = SimpleNamespace(question_id="q1")
|
||
pools = SimpleNamespace(validation=[q], correctness={"q1": False})
|
||
with pytest.raises(SystemExit):
|
||
cli.check_mcnemar_power(pools, val_wrong_min=5)
|
||
|
||
|
||
def test_check_mcnemar_power_zero_threshold_skips():
|
||
"""val_wrong_min=0 → 不检查，返回实际错题数。"""
|
||
q = SimpleNamespace(question_id="q1")
|
||
pools = SimpleNamespace(validation=[q], correctness={"q1": True})
|
||
assert cli.check_mcnemar_power(pools, val_wrong_min=0) == 0
|
||
|
||
|
||
def test_run_pipeline_orders_two_phases(monkeypatch, tmp_path):
|
||
"""run_pipeline 先跑 Phase 1 诊断、后跑 Phase 2 build_split（按序）。"""
|
||
calls: list[str] = []
|
||
|
||
async def fake_diag(**kwargs):
|
||
calls.append("diagnosis")
|
||
assert kwargs["diag_fingerprint"] == "fp"
|
||
assert kwargs["wrong_ids"] == ["q1"]
|
||
|
||
def fake_build_split(**kwargs):
|
||
calls.append("build_split")
|
||
assert kwargs["diag_fingerprint"] == "fp"
|
||
pools = SimpleNamespace(validation=[], correctness={})
|
||
return SimpleNamespace(pools=pools, manifest={}, assignment={})
|
||
|
||
monkeypatch.setattr(cli, "run_baseline_diagnosis", fake_diag)
|
||
monkeypatch.setattr(cli, "build_split", fake_build_split)
|
||
|
||
result = asyncio.run(
|
||
cli.run_pipeline(
|
||
config=_config(),
|
||
fingerprint="fp",
|
||
diagnosis_deps=object(),
|
||
signal_store=object(),
|
||
wrong_ids=["q1"],
|
||
questions={},
|
||
harness_db=tmp_path / "h.db",
|
||
questions_dir=tmp_path,
|
||
out_dir=tmp_path / "out",
|
||
generated_at="2026-07-15T00:00:00Z",
|
||
)
|
||
)
|
||
assert calls == ["diagnosis", "build_split"]
|
||
assert result.pools.validation == []
|
||
|
||
|
||
def test_dry_run_computes_fingerprint_without_llm(monkeypatch, tmp_path, capsys):
|
||
"""--dry-run：diag_fingerprint 被调用、Phase 1 走空错题早返回、不真调 LLM。"""
|
||
fp_calls: list[tuple[str, str, str]] = []
|
||
|
||
def spy_fingerprint(prompt_version, model, code_version):
|
||
fp_calls.append((prompt_version, model, code_version))
|
||
return "dryfp"
|
||
|
||
monkeypatch.setattr(cli, "diag_fingerprint", spy_fingerprint)
|
||
monkeypatch.setattr(cli, "git_short_sha", lambda: "abc123")
|
||
|
||
# 用真实 config/video_split.yaml，out-dir 指向 tmp（dry-run 会建临时信号库再清理）。
|
||
cli.main(["--dry-run", "--config", "config/video_split.yaml", "--out-dir", str(tmp_path)])
|
||
|
||
assert len(fp_calls) == 1
|
||
prompt_version, model, sha = fp_calls[0]
|
||
assert sha == "abc123"
|
||
assert prompt_version == "diagnose_v1"
|
||
assert model == "deepseek-v4-pro"
|
||
# dry-run 不留下临时信号库
|
||
assert not (tmp_path / "_dry_run_signals.db").exists()
|
||
|
||
|
||
def test_build_diagnosis_deps_model_mismatch_fails_loud(monkeypatch, tmp_path):
|
||
"""config.diag.model 与 .env SEARCH_LLM_MODEL 不一致 → fail loud（指纹漂移防护）。"""
|
||
|
||
class _FakeSettings:
|
||
search_llm_model = "actual-model-in-env"
|
||
search_llm_base_url = "https://api.example"
|
||
search_llm_api_key = "sk-xxx"
|
||
|
||
monkeypatch.setattr(cli, "_DiagLLMSettings", lambda: _FakeSettings())
|
||
with pytest.raises(SystemExit) as exc:
|
||
cli.build_diagnosis_deps(
|
||
harness_db=tmp_path / "h.db",
|
||
concurrency=2,
|
||
expected_model="deepseek-v4-pro", # 与 env 不一致
|
||
)
|
||
# 报错须同时暴露两个值，便于人对齐
|
||
msg = str(exc.value)
|
||
assert "deepseek-v4-pro" in msg
|
||
assert "actual-model-in-env" in msg
|
||
|
||
|
||
def test_build_diagnosis_deps_missing_credentials_fails_loud(monkeypatch, tmp_path):
|
||
""".env 缺 search LLM 凭证 → fail loud（先于模型一致性校验）。"""
|
||
|
||
class _EmptySettings:
|
||
search_llm_model = ""
|
||
search_llm_base_url = ""
|
||
search_llm_api_key = ""
|
||
|
||
monkeypatch.setattr(cli, "_DiagLLMSettings", lambda: _EmptySettings())
|
||
with pytest.raises(SystemExit):
|
||
cli.build_diagnosis_deps(
|
||
harness_db=tmp_path / "h.db", concurrency=2, expected_model="deepseek-v4-pro"
|
||
)
|
||
|
||
|
||
def test_git_short_sha_nonempty():
|
||
"""仓库内 git_short_sha 返回非空短 SHA。"""
|
||
sha = cli.git_short_sha()
|
||
assert sha
|
||
assert len(sha) >= 4
|