fix: load real tree_data for offline diagnosis (video_split_cli)

This commit is contained in:
2026-07-15 22:38:23 -04:00
parent f1b6865861
commit 83056688cf
3 changed files with 71 additions and 11 deletions
+31 -10
View File
@@ -18,7 +18,8 @@
诊断配置的信号;换 prompt / 模型 / 代码实现即换指纹,旧信号不被覆盖。 诊断配置的信号;换 prompt / 模型 / 代码实现即换指纹,旧信号不被覆盖。
- 真实依赖组装参考 app/harness/runner.py::_run_diagnosisGovernedLLMClient - 真实依赖组装参考 app/harness/runner.py::_run_diagnosisGovernedLLMClient
(search llm, thinking=True) + RunLogImpl(harness.db) + VersionedSkillStore + (search llm, thinking=True) + RunLogImpl(harness.db) + VersionedSkillStore +
DiagnosePrompts(项目根 prompts/) + tree_data={}(由诊断管线内部按需加载)。 DiagnosePrompts(项目根 prompts/) + tree_data 按 wrong_ids 涉及 video 预加载
store/videos/<vid>/tree.json 展平)。
- 缺 .env / config 关键项一律 fail loud(P5),绝不静默兜底。 - 缺 .env / config 关键项一律 fail loud(P5),绝不静默兜底。
- `--dry-run` 用假 deps 跑通两阶段 wiring 不真调 LLM,打印将执行的步骤 + 指纹, - `--dry-run` 用假 deps 跑通两阶段 wiring 不真调 LLM,打印将执行的步骤 + 指纹,
用于校验装配正确性(对齐 CLAUDE.md §2.5 smoke test)。 用于校验装配正确性(对齐 CLAUDE.md §2.5 smoke test)。
@@ -65,6 +66,7 @@ _INFRA_STOP_REASONS: frozenset[str] = frozenset({"error", "parse_error"})
_DEFAULT_HARNESS_DB = Path("workspaces/default/harness.db") _DEFAULT_HARNESS_DB = Path("workspaces/default/harness.db")
_DEFAULT_QUESTIONS_DIR = Path("store/questions/benchmarks/Video-MME") _DEFAULT_QUESTIONS_DIR = Path("store/questions/benchmarks/Video-MME")
_DEFAULT_OUT_DIR = Path("workspaces/video-split") _DEFAULT_OUT_DIR = Path("workspaces/video-split")
_DEFAULT_STORE_DIR = Path("store") # tree.json 在 store/videos/<vid>/
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -260,16 +262,24 @@ def _build_redis_cache(settings: Any) -> Any | None:
def build_diagnosis_deps( def build_diagnosis_deps(
*, harness_db: Path, concurrency: int, expected_model: str *,
harness_db: Path,
store_dir: Path,
video_ids: list[str],
concurrency: int,
expected_model: str,
) -> DiagnosisDeps: ) -> DiagnosisDeps:
"""组装 Phase 1 诊断的真实依赖束(GovernedLLMClient + RunLogImpl + prompts)。 """组装 Phase 1 诊断的真实依赖束(GovernedLLMClient + RunLogImpl + prompts)。
与 runner.py::_run_diagnosis 对齐:search LLMthinking=True)作诊断 judge 与 runner.py::_run_diagnosis 对齐:search LLMthinking=True)作诊断 judge
RunLogImpl 只读读取 harness.db 的 predictions/tracesVersionedSkillStore 读技能, RunLogImpl 只读读取 harness.db 的 predictions/tracesVersionedSkillStore 读技能,
DiagnosePrompts 从项目根 prompts/ 加载,tree_data={} 由诊断管线内部按需加载。 DiagnosePrompts 从项目根 prompts/ 加载,tree_data 按 video_ids 从
store/videos/<vid>/tree.json 展平预加载(诊断需真实树,缺失即 fail-loud)。
参数: 参数:
harness_db: harness.db 路径(诊断读预测 + 信号落库同库)。 harness_db: harness.db 路径(诊断读预测 + 信号落库同库)。
store_dir: store 根目录(含 videos/<vid>/tree.json)。
video_ids: wrong_ids 涉及的 video 标识列表(可含重复,内部去重加载树)。
concurrency: 诊断并发上限。 concurrency: 诊断并发上限。
expected_model: config.diag.model(诊断口径指纹的模型分量)。必须与 .env expected_model: config.diag.model(诊断口径指纹的模型分量)。必须与 .env
SEARCH_LLM_MODEL 一致——指纹里的 model 与实际诊断所用 model 不一致会让 SEARCH_LLM_MODEL 一致——指纹里的 model 与实际诊断所用 model 不一致会让
@@ -281,6 +291,8 @@ def build_diagnosis_deps(
异常: 异常:
SystemExit: .env 缺 search LLM 凭证(model / base_url / api_key 任一为空), SystemExit: .env 缺 search LLM 凭证(model / base_url / api_key 任一为空),
或 config.diag.model 与 .env SEARCH_LLM_MODEL 不一致(指纹漂移防护)。 或 config.diag.model 与 .env SEARCH_LLM_MODEL 不一致(指纹漂移防护)。
FileNotFoundError / ValueError: video_ids 中任一 video 的 tree.json 缺失或
无效(load_tree_data_for_videos fail-loud,诊断需真实树)。
""" """
from adapters.breaker import CircuitBreaker from adapters.breaker import CircuitBreaker
from adapters.llm import GovernedLLMClient from adapters.llm import GovernedLLMClient
@@ -327,12 +339,14 @@ def build_diagnosis_deps(
retry_base_delay_s=settings.llm_retry_base_delay, retry_base_delay_s=settings.llm_retry_base_delay,
retry_max_delay_s=settings.llm_retry_max_delay, retry_max_delay_s=settings.llm_retry_max_delay,
) )
from app.harness.tree_nodes import load_tree_data_for_videos
return DiagnosisDeps( return DiagnosisDeps(
run_log=RunLogImpl(str(harness_db)), run_log=RunLogImpl(str(harness_db)),
llm=llm, llm=llm,
skill_store=VersionedSkillStore(_diagnosis_skills_dir()), skill_store=VersionedSkillStore(_diagnosis_skills_dir()),
prompts=_load_diagnose_prompts(), prompts=_load_diagnose_prompts(),
tree_data={}, tree_data=load_tree_data_for_videos(store_dir, video_ids),
concurrency=concurrency, concurrency=concurrency,
) )
@@ -579,24 +593,30 @@ async def run_pipeline(
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def _resolve_paths(args: argparse.Namespace) -> tuple[Path, Path, Path]: def _resolve_paths(args: argparse.Namespace) -> tuple[Path, Path, Path, Path]:
"""解析 harness_db / questions_dir / out_dirCLI 覆盖默认工程路径)。""" """解析 harness_db / questions_dir / out_dir / store_dirCLI 覆盖默认工程路径)。"""
harness_db = args.harness_db or _DEFAULT_HARNESS_DB harness_db = args.harness_db or _DEFAULT_HARNESS_DB
questions_dir = args.questions_dir or _DEFAULT_QUESTIONS_DIR questions_dir = args.questions_dir or _DEFAULT_QUESTIONS_DIR
out_dir = args.out_dir or _DEFAULT_OUT_DIR out_dir = args.out_dir or _DEFAULT_OUT_DIR
return harness_db, questions_dir, out_dir store_dir = args.store_dir or _DEFAULT_STORE_DIR
return harness_db, questions_dir, out_dir, store_dir
def _execute_real(config: VideoSplitConfig, fingerprint: str, args: argparse.Namespace) -> None: def _execute_real(config: VideoSplitConfig, fingerprint: str, args: argparse.Namespace) -> None:
"""真实执行两阶段管线:组装真实 deps、读错题、跑诊断 + 冻结切分。""" """真实执行两阶段管线:组装真实 deps、读错题、跑诊断 + 冻结切分。"""
harness_db, questions_dir, out_dir = _resolve_paths(args) harness_db, questions_dir, out_dir, store_dir = _resolve_paths(args)
if not harness_db.exists(): if not harness_db.exists():
raise SystemExit(f"harness.db 不存在: {harness_db}P5 fail loud") raise SystemExit(f"harness.db 不存在: {harness_db}P5 fail loud")
canonical_preds = load_canonical_predictions(harness_db, config.baseline_run_id) canonical_preds = load_canonical_predictions(harness_db, config.baseline_run_id)
wrong_ids = select_diagnosable_wrong_ids(canonical_preds) wrong_ids = select_diagnosable_wrong_ids(canonical_preds)
questions = load_questions_by_id(questions_dir) questions = load_questions_by_id(questions_dir)
video_ids = [questions[qid].video_id for qid in wrong_ids]
deps = build_diagnosis_deps( deps = build_diagnosis_deps(
harness_db=harness_db, concurrency=args.concurrency, expected_model=config.model harness_db=harness_db,
store_dir=store_dir,
video_ids=video_ids,
concurrency=args.concurrency,
expected_model=config.model,
) )
# generated_at:默认盖真实 UTC now(溯源用),--generated-at 可显式固定以复现(C-2)。 # generated_at:默认盖真实 UTC now(溯源用),--generated-at 可显式固定以复现(C-2)。
@@ -648,7 +668,7 @@ def _execute_dry_run(config: VideoSplitConfig, fingerprint: str, args: argparse.
Phase 2 build_split 需真实诊断信号方能冻结,dry-run 不真实冻结,仅打印其计划; Phase 2 build_split 需真实诊断信号方能冻结,dry-run 不真实冻结,仅打印其计划;
Phase 1 用空 wrong_ids 走 run_baseline_diagnosis 早返回路径,验证装配可调用而不触 LLM。 Phase 1 用空 wrong_ids 走 run_baseline_diagnosis 早返回路径,验证装配可调用而不触 LLM。
""" """
harness_db, questions_dir, out_dir = _resolve_paths(args) harness_db, questions_dir, out_dir, _store_dir = _resolve_paths(args)
logger.info("=== dry-run:校验两阶段装配(不真调 LLM / 不冻结产物)===") logger.info("=== dry-run:校验两阶段装配(不真调 LLM / 不冻结产物)===")
logger.info( logger.info(
"诊断口径指纹 diag_fingerprint={} (prompt={} model={})", "诊断口径指纹 diag_fingerprint={} (prompt={} model={})",
@@ -730,6 +750,7 @@ def build_arg_parser() -> argparse.ArgumentParser:
parser.add_argument("--harness-db", type=Path, default=None, dest="harness_db") parser.add_argument("--harness-db", type=Path, default=None, dest="harness_db")
parser.add_argument("--questions-dir", type=Path, default=None, dest="questions_dir") parser.add_argument("--questions-dir", type=Path, default=None, dest="questions_dir")
parser.add_argument("--out-dir", type=Path, default=None, dest="out_dir") parser.add_argument("--out-dir", type=Path, default=None, dest="out_dir")
parser.add_argument("--store-dir", type=Path, default=None, dest="store_dir")
parser.add_argument( parser.add_argument(
"--generated-at", "--generated-at",
type=str, type=str,
+7 -1
View File
@@ -268,6 +268,8 @@ def test_build_diagnosis_deps_model_mismatch_fails_loud(monkeypatch, tmp_path):
with pytest.raises(SystemExit) as exc: with pytest.raises(SystemExit) as exc:
cli.build_diagnosis_deps( cli.build_diagnosis_deps(
harness_db=tmp_path / "h.db", harness_db=tmp_path / "h.db",
store_dir=tmp_path,
video_ids=[],
concurrency=2, concurrency=2,
expected_model="deepseek-v4-pro", # 与 env 不一致 expected_model="deepseek-v4-pro", # 与 env 不一致
) )
@@ -288,7 +290,11 @@ def test_build_diagnosis_deps_missing_credentials_fails_loud(monkeypatch, tmp_pa
monkeypatch.setattr(cli, "_DiagLLMSettings", lambda: _EmptySettings()) monkeypatch.setattr(cli, "_DiagLLMSettings", lambda: _EmptySettings())
with pytest.raises(SystemExit): with pytest.raises(SystemExit):
cli.build_diagnosis_deps( cli.build_diagnosis_deps(
harness_db=tmp_path / "h.db", concurrency=2, expected_model="deepseek-v4-pro" harness_db=tmp_path / "h.db",
store_dir=tmp_path,
video_ids=[],
concurrency=2,
expected_model="deepseek-v4-pro",
) )
@@ -0,0 +1,33 @@
"""离线诊断注入:build_diagnosis_deps 按 video_ids 填充非空 tree_data。"""
from pathlib import Path
from unittest.mock import patch
from app.harness.video_split_cli import build_diagnosis_deps
def test_build_diagnosis_deps_loads_tree_for_videos():
with (
patch("app.harness.video_split_cli._DiagLLMSettings") as settings_cls,
patch("adapters.llm.GovernedLLMClient"),
patch("adapters.telemetry.SQLiteTelemetryRecorder"),
patch("app.harness.video_split_cli._build_redis_cache", return_value=None),
):
s = settings_cls.return_value
s.search_llm_model = "deepseek-v4-pro"
s.search_llm_base_url = "http://x"
s.search_llm_api_key = "k"
s.llm_circuit_breaker_threshold = 32
s.llm_circuit_breaker_cooldown = 60
s.llm_timeout = s.llm_ttft_timeout = s.llm_inter_token_timeout = 60
s.llm_max_retries = 1
s.llm_retry_base_delay = s.llm_retry_max_delay = 1
deps = build_diagnosis_deps(
harness_db=Path("workspaces/default/harness.db"),
store_dir=Path("store"),
video_ids=["0RxMZBLeqRI"],
concurrency=1,
expected_model="deepseek-v4-pro",
)
assert "0RxMZBLeqRI" in deps.tree_data
assert deps.tree_data["0RxMZBLeqRI"]["nodes"]