fix: fail loud on diag model fingerprint drift vs .env SEARCH_LLM_MODEL
Codex CHANGES_REQUESTED 复审: - Critical:diag_fingerprint 用 config.diag.model,但 Phase 1 诊断 LLM 从 .env SEARCH_LLM_MODEL 构建,两者不一致会让信号以错误模型指纹落库,破坏可复现/ resume/隔离。build_diagnosis_deps 新增 expected_model 参数,Phase 1 执行前 fail loud 校验 config.model == settings.search_llm_model(附两值)。 - Minor:config/video_split.yaml diag.model 注释由 JUDGE_LLM_MODEL 更正为 SEARCH_LLM_MODEL,与实现对齐。 - 补两个单测:模型不一致 fail loud + 缺凭证 fail loud。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -245,7 +245,9 @@ def _build_redis_cache(settings: Any) -> Any | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def build_diagnosis_deps(*, harness_db: Path, concurrency: int) -> DiagnosisDeps:
|
def build_diagnosis_deps(
|
||||||
|
*, harness_db: Path, concurrency: int, expected_model: str
|
||||||
|
) -> DiagnosisDeps:
|
||||||
"""组装 Phase 1 诊断的真实依赖束(GovernedLLMClient + RunLogImpl + prompts)。
|
"""组装 Phase 1 诊断的真实依赖束(GovernedLLMClient + RunLogImpl + prompts)。
|
||||||
|
|
||||||
与 runner.py::_run_diagnosis 对齐:search LLM(thinking=True)作诊断 judge,
|
与 runner.py::_run_diagnosis 对齐:search LLM(thinking=True)作诊断 judge,
|
||||||
@@ -255,12 +257,16 @@ def build_diagnosis_deps(*, harness_db: Path, concurrency: int) -> DiagnosisDeps
|
|||||||
参数:
|
参数:
|
||||||
harness_db: harness.db 路径(诊断读预测 + 信号落库同库)。
|
harness_db: harness.db 路径(诊断读预测 + 信号落库同库)。
|
||||||
concurrency: 诊断并发上限。
|
concurrency: 诊断并发上限。
|
||||||
|
expected_model: config.diag.model(诊断口径指纹的模型分量)。必须与 .env
|
||||||
|
SEARCH_LLM_MODEL 一致——指纹里的 model 与实际诊断所用 model 不一致会让
|
||||||
|
信号以错误模型指纹落库,破坏可复现 / resume / 口径隔离,故此处 fail loud。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
DiagnosisDeps 冻结依赖束。
|
DiagnosisDeps 冻结依赖束。
|
||||||
|
|
||||||
异常:
|
异常:
|
||||||
SystemExit: .env 缺 search LLM 凭证(model / base_url / api_key 任一为空)。
|
SystemExit: .env 缺 search LLM 凭证(model / base_url / api_key 任一为空),
|
||||||
|
或 config.diag.model 与 .env SEARCH_LLM_MODEL 不一致(指纹漂移防护)。
|
||||||
"""
|
"""
|
||||||
from adapters.breaker import CircuitBreaker
|
from adapters.breaker import CircuitBreaker
|
||||||
from adapters.llm import GovernedLLMClient
|
from adapters.llm import GovernedLLMClient
|
||||||
@@ -276,6 +282,13 @@ def build_diagnosis_deps(*, harness_db: Path, concurrency: int) -> DiagnosisDeps
|
|||||||
"诊断 LLM 凭证缺失:.env 需配置 SEARCH_LLM_MODEL / SEARCH_LLM_BASE_URL / "
|
"诊断 LLM 凭证缺失:.env 需配置 SEARCH_LLM_MODEL / SEARCH_LLM_BASE_URL / "
|
||||||
"SEARCH_LLM_API_KEY(P5 fail loud,不静默兜底)"
|
"SEARCH_LLM_API_KEY(P5 fail loud,不静默兜底)"
|
||||||
)
|
)
|
||||||
|
if expected_model != settings.search_llm_model:
|
||||||
|
raise SystemExit(
|
||||||
|
"诊断模型指纹漂移:config.diag.model="
|
||||||
|
f"{expected_model!r} 与 .env SEARCH_LLM_MODEL={settings.search_llm_model!r} "
|
||||||
|
"不一致;指纹里的 model 必须等于实际诊断所用 model(P5 fail loud,"
|
||||||
|
"请对齐 config/video_split.yaml diag.model 与 .env SEARCH_LLM_MODEL)"
|
||||||
|
)
|
||||||
|
|
||||||
telemetry_db = Path("logs/telemetry.db")
|
telemetry_db = Path("logs/telemetry.db")
|
||||||
telemetry_db.parent.mkdir(parents=True, exist_ok=True)
|
telemetry_db.parent.mkdir(parents=True, exist_ok=True)
|
||||||
@@ -537,7 +550,9 @@ def _execute_real(config: VideoSplitConfig, fingerprint: str, args: argparse.Nam
|
|||||||
raise SystemExit(f"harness.db 不存在: {harness_db}(P5 fail loud)")
|
raise SystemExit(f"harness.db 不存在: {harness_db}(P5 fail loud)")
|
||||||
wrong_ids = load_diagnosable_wrong_ids(harness_db, config.baseline_run_id)
|
wrong_ids = load_diagnosable_wrong_ids(harness_db, config.baseline_run_id)
|
||||||
questions = load_questions_by_id(questions_dir)
|
questions = load_questions_by_id(questions_dir)
|
||||||
deps = build_diagnosis_deps(harness_db=harness_db, concurrency=args.concurrency)
|
deps = build_diagnosis_deps(
|
||||||
|
harness_db=harness_db, concurrency=args.concurrency, expected_model=config.model
|
||||||
|
)
|
||||||
|
|
||||||
from adapters.baseline_diagnosis_store import SqliteDiagnosisSignalStore
|
from adapters.baseline_diagnosis_store import SqliteDiagnosisSignalStore
|
||||||
|
|
||||||
|
|||||||
@@ -21,5 +21,5 @@ video_split:
|
|||||||
|
|
||||||
diag: # 诊断口径指纹三分量(隔离不同诊断配置的信号,参与主键)
|
diag: # 诊断口径指纹三分量(隔离不同诊断配置的信号,参与主键)
|
||||||
prompt_version: diagnose_v1 # 诊断 prompt 版本标识(换 prompt 即换指纹,旧记录不被覆盖)
|
prompt_version: diagnose_v1 # 诊断 prompt 版本标识(换 prompt 即换指纹,旧记录不被覆盖)
|
||||||
model: deepseek-v4-pro # 执行诊断的模型名(与 .env JUDGE_LLM_MODEL 对齐)
|
model: deepseek-v4-pro # 执行诊断的模型名(必须与 .env SEARCH_LLM_MODEL 一致,CLI 会 fail loud 校验)
|
||||||
# code_version 由 build_video_split.sh 注入 git 短 SHA,不写死在此(随代码变动)
|
# code_version 由 build_video_split.sh 注入 git 短 SHA,不写死在此(随代码变动)
|
||||||
|
|||||||
@@ -161,6 +161,42 @@ def test_dry_run_computes_fingerprint_without_llm(monkeypatch, tmp_path, capsys)
|
|||||||
assert not (tmp_path / "_dry_run_signals.db").exists()
|
assert not (tmp_path / "_dry_run_signals.db").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_diagnosis_deps_model_mismatch_fails_loud(monkeypatch, tmp_path):
|
||||||
|
"""config.diag.model 与 .env SEARCH_LLM_MODEL 不一致 → fail loud(指纹漂移防护)。"""
|
||||||
|
|
||||||
|
class _FakeSettings:
|
||||||
|
search_llm_model = "actual-model-in-env"
|
||||||
|
search_llm_base_url = "https://api.example"
|
||||||
|
search_llm_api_key = "sk-xxx"
|
||||||
|
|
||||||
|
monkeypatch.setattr(cli, "_DiagLLMSettings", lambda: _FakeSettings())
|
||||||
|
with pytest.raises(SystemExit) as exc:
|
||||||
|
cli.build_diagnosis_deps(
|
||||||
|
harness_db=tmp_path / "h.db",
|
||||||
|
concurrency=2,
|
||||||
|
expected_model="deepseek-v4-pro", # 与 env 不一致
|
||||||
|
)
|
||||||
|
# 报错须同时暴露两个值,便于人对齐
|
||||||
|
msg = str(exc.value)
|
||||||
|
assert "deepseek-v4-pro" in msg
|
||||||
|
assert "actual-model-in-env" in msg
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_diagnosis_deps_missing_credentials_fails_loud(monkeypatch, tmp_path):
|
||||||
|
""".env 缺 search LLM 凭证 → fail loud(先于模型一致性校验)。"""
|
||||||
|
|
||||||
|
class _EmptySettings:
|
||||||
|
search_llm_model = ""
|
||||||
|
search_llm_base_url = ""
|
||||||
|
search_llm_api_key = ""
|
||||||
|
|
||||||
|
monkeypatch.setattr(cli, "_DiagLLMSettings", lambda: _EmptySettings())
|
||||||
|
with pytest.raises(SystemExit):
|
||||||
|
cli.build_diagnosis_deps(
|
||||||
|
harness_db=tmp_path / "h.db", concurrency=2, expected_model="deepseek-v4-pro"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_git_short_sha_nonempty():
|
def test_git_short_sha_nonempty():
|
||||||
"""仓库内 git_short_sha 返回非空短 SHA。"""
|
"""仓库内 git_short_sha 返回非空短 SHA。"""
|
||||||
sha = cli.git_short_sha()
|
sha = cli.git_short_sha()
|
||||||
|
|||||||
Reference in New Issue
Block a user