fix: load real tree_data for offline diagnosis (video_split_cli)
This commit is contained in:
@@ -18,7 +18,8 @@
|
||||
诊断配置的信号;换 prompt / 模型 / 代码实现即换指纹,旧信号不被覆盖。
|
||||
- 真实依赖组装参考 app/harness/runner.py::_run_diagnosis:GovernedLLMClient
|
||||
(search llm, thinking=True) + RunLogImpl(harness.db) + VersionedSkillStore +
|
||||
DiagnosePrompts(项目根 prompts/) + tree_data={}(由诊断管线内部按需加载)。
|
||||
DiagnosePrompts(项目根 prompts/) + tree_data 按 wrong_ids 涉及 video 预加载
|
||||
(store/videos/<vid>/tree.json 展平)。
|
||||
- 缺 .env / config 关键项一律 fail loud(P5),绝不静默兜底。
|
||||
- `--dry-run` 用假 deps 跑通两阶段 wiring 不真调 LLM,打印将执行的步骤 + 指纹,
|
||||
用于校验装配正确性(对齐 CLAUDE.md §2.5 smoke test)。
|
||||
@@ -65,6 +66,7 @@ _INFRA_STOP_REASONS: frozenset[str] = frozenset({"error", "parse_error"})
|
||||
_DEFAULT_HARNESS_DB = Path("workspaces/default/harness.db")
|
||||
_DEFAULT_QUESTIONS_DIR = Path("store/questions/benchmarks/Video-MME")
|
||||
_DEFAULT_OUT_DIR = Path("workspaces/video-split")
|
||||
_DEFAULT_STORE_DIR = Path("store") # tree.json 在 store/videos/<vid>/
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -260,16 +262,24 @@ def _build_redis_cache(settings: Any) -> Any | None:
|
||||
|
||||
|
||||
def build_diagnosis_deps(
|
||||
*, harness_db: Path, concurrency: int, expected_model: str
|
||||
*,
|
||||
harness_db: Path,
|
||||
store_dir: Path,
|
||||
video_ids: list[str],
|
||||
concurrency: int,
|
||||
expected_model: str,
|
||||
) -> DiagnosisDeps:
|
||||
"""组装 Phase 1 诊断的真实依赖束(GovernedLLMClient + RunLogImpl + prompts)。
|
||||
|
||||
与 runner.py::_run_diagnosis 对齐:search LLM(thinking=True)作诊断 judge,
|
||||
RunLogImpl 只读读取 harness.db 的 predictions/traces,VersionedSkillStore 读技能,
|
||||
DiagnosePrompts 从项目根 prompts/ 加载,tree_data={} 由诊断管线内部按需加载。
|
||||
DiagnosePrompts 从项目根 prompts/ 加载,tree_data 按 video_ids 从
|
||||
store/videos/<vid>/tree.json 展平预加载(诊断需真实树,缺失即 fail-loud)。
|
||||
|
||||
参数:
|
||||
harness_db: harness.db 路径(诊断读预测 + 信号落库同库)。
|
||||
store_dir: store 根目录(含 videos/<vid>/tree.json)。
|
||||
video_ids: wrong_ids 涉及的 video 标识列表(可含重复,内部去重加载树)。
|
||||
concurrency: 诊断并发上限。
|
||||
expected_model: config.diag.model(诊断口径指纹的模型分量)。必须与 .env
|
||||
SEARCH_LLM_MODEL 一致——指纹里的 model 与实际诊断所用 model 不一致会让
|
||||
@@ -281,6 +291,8 @@ def build_diagnosis_deps(
|
||||
异常:
|
||||
SystemExit: .env 缺 search LLM 凭证(model / base_url / api_key 任一为空),
|
||||
或 config.diag.model 与 .env SEARCH_LLM_MODEL 不一致(指纹漂移防护)。
|
||||
FileNotFoundError / ValueError: video_ids 中任一 video 的 tree.json 缺失或
|
||||
无效(load_tree_data_for_videos fail-loud,诊断需真实树)。
|
||||
"""
|
||||
from adapters.breaker import CircuitBreaker
|
||||
from adapters.llm import GovernedLLMClient
|
||||
@@ -327,12 +339,14 @@ def build_diagnosis_deps(
|
||||
retry_base_delay_s=settings.llm_retry_base_delay,
|
||||
retry_max_delay_s=settings.llm_retry_max_delay,
|
||||
)
|
||||
from app.harness.tree_nodes import load_tree_data_for_videos
|
||||
|
||||
return DiagnosisDeps(
|
||||
run_log=RunLogImpl(str(harness_db)),
|
||||
llm=llm,
|
||||
skill_store=VersionedSkillStore(_diagnosis_skills_dir()),
|
||||
prompts=_load_diagnose_prompts(),
|
||||
tree_data={},
|
||||
tree_data=load_tree_data_for_videos(store_dir, video_ids),
|
||||
concurrency=concurrency,
|
||||
)
|
||||
|
||||
@@ -579,24 +593,30 @@ async def run_pipeline(
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _resolve_paths(args: argparse.Namespace) -> tuple[Path, Path, Path]:
|
||||
"""解析 harness_db / questions_dir / out_dir(CLI 覆盖默认工程路径)。"""
|
||||
def _resolve_paths(args: argparse.Namespace) -> tuple[Path, Path, Path, Path]:
|
||||
"""解析 harness_db / questions_dir / out_dir / store_dir(CLI 覆盖默认工程路径)。"""
|
||||
harness_db = args.harness_db or _DEFAULT_HARNESS_DB
|
||||
questions_dir = args.questions_dir or _DEFAULT_QUESTIONS_DIR
|
||||
out_dir = args.out_dir or _DEFAULT_OUT_DIR
|
||||
return harness_db, questions_dir, out_dir
|
||||
store_dir = args.store_dir or _DEFAULT_STORE_DIR
|
||||
return harness_db, questions_dir, out_dir, store_dir
|
||||
|
||||
|
||||
def _execute_real(config: VideoSplitConfig, fingerprint: str, args: argparse.Namespace) -> None:
|
||||
"""真实执行两阶段管线:组装真实 deps、读错题、跑诊断 + 冻结切分。"""
|
||||
harness_db, questions_dir, out_dir = _resolve_paths(args)
|
||||
harness_db, questions_dir, out_dir, store_dir = _resolve_paths(args)
|
||||
if not harness_db.exists():
|
||||
raise SystemExit(f"harness.db 不存在: {harness_db}(P5 fail loud)")
|
||||
canonical_preds = load_canonical_predictions(harness_db, config.baseline_run_id)
|
||||
wrong_ids = select_diagnosable_wrong_ids(canonical_preds)
|
||||
questions = load_questions_by_id(questions_dir)
|
||||
video_ids = [questions[qid].video_id for qid in wrong_ids]
|
||||
deps = build_diagnosis_deps(
|
||||
harness_db=harness_db, concurrency=args.concurrency, expected_model=config.model
|
||||
harness_db=harness_db,
|
||||
store_dir=store_dir,
|
||||
video_ids=video_ids,
|
||||
concurrency=args.concurrency,
|
||||
expected_model=config.model,
|
||||
)
|
||||
|
||||
# generated_at:默认盖真实 UTC now(溯源用),--generated-at 可显式固定以复现(C-2)。
|
||||
@@ -648,7 +668,7 @@ def _execute_dry_run(config: VideoSplitConfig, fingerprint: str, args: argparse.
|
||||
Phase 2 build_split 需真实诊断信号方能冻结,dry-run 不真实冻结,仅打印其计划;
|
||||
Phase 1 用空 wrong_ids 走 run_baseline_diagnosis 早返回路径,验证装配可调用而不触 LLM。
|
||||
"""
|
||||
harness_db, questions_dir, out_dir = _resolve_paths(args)
|
||||
harness_db, questions_dir, out_dir, _store_dir = _resolve_paths(args)
|
||||
logger.info("=== dry-run:校验两阶段装配(不真调 LLM / 不冻结产物)===")
|
||||
logger.info(
|
||||
"诊断口径指纹 diag_fingerprint={} (prompt={} model={})",
|
||||
@@ -730,6 +750,7 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
parser.add_argument("--harness-db", type=Path, default=None, dest="harness_db")
|
||||
parser.add_argument("--questions-dir", type=Path, default=None, dest="questions_dir")
|
||||
parser.add_argument("--out-dir", type=Path, default=None, dest="out_dir")
|
||||
parser.add_argument("--store-dir", type=Path, default=None, dest="store_dir")
|
||||
parser.add_argument(
|
||||
"--generated-at",
|
||||
type=str,
|
||||
|
||||
@@ -268,6 +268,8 @@ def test_build_diagnosis_deps_model_mismatch_fails_loud(monkeypatch, tmp_path):
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
cli.build_diagnosis_deps(
|
||||
harness_db=tmp_path / "h.db",
|
||||
store_dir=tmp_path,
|
||||
video_ids=[],
|
||||
concurrency=2,
|
||||
expected_model="deepseek-v4-pro", # 与 env 不一致
|
||||
)
|
||||
@@ -288,7 +290,11 @@ def test_build_diagnosis_deps_missing_credentials_fails_loud(monkeypatch, tmp_pa
|
||||
monkeypatch.setattr(cli, "_DiagLLMSettings", lambda: _EmptySettings())
|
||||
with pytest.raises(SystemExit):
|
||||
cli.build_diagnosis_deps(
|
||||
harness_db=tmp_path / "h.db", concurrency=2, expected_model="deepseek-v4-pro"
|
||||
harness_db=tmp_path / "h.db",
|
||||
store_dir=tmp_path,
|
||||
video_ids=[],
|
||||
concurrency=2,
|
||||
expected_model="deepseek-v4-pro",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
"""离线诊断注入:build_diagnosis_deps 按 video_ids 填充非空 tree_data。"""
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.harness.video_split_cli import build_diagnosis_deps
|
||||
|
||||
|
||||
def test_build_diagnosis_deps_loads_tree_for_videos():
|
||||
with (
|
||||
patch("app.harness.video_split_cli._DiagLLMSettings") as settings_cls,
|
||||
patch("adapters.llm.GovernedLLMClient"),
|
||||
patch("adapters.telemetry.SQLiteTelemetryRecorder"),
|
||||
patch("app.harness.video_split_cli._build_redis_cache", return_value=None),
|
||||
):
|
||||
s = settings_cls.return_value
|
||||
s.search_llm_model = "deepseek-v4-pro"
|
||||
s.search_llm_base_url = "http://x"
|
||||
s.search_llm_api_key = "k"
|
||||
s.llm_circuit_breaker_threshold = 32
|
||||
s.llm_circuit_breaker_cooldown = 60
|
||||
s.llm_timeout = s.llm_ttft_timeout = s.llm_inter_token_timeout = 60
|
||||
s.llm_max_retries = 1
|
||||
s.llm_retry_base_delay = s.llm_retry_max_delay = 1
|
||||
deps = build_diagnosis_deps(
|
||||
harness_db=Path("workspaces/default/harness.db"),
|
||||
store_dir=Path("store"),
|
||||
video_ids=["0RxMZBLeqRI"],
|
||||
concurrency=1,
|
||||
expected_model="deepseek-v4-pro",
|
||||
)
|
||||
assert "0RxMZBLeqRI" in deps.tree_data
|
||||
assert deps.tree_data["0RxMZBLeqRI"]["nodes"]
|
||||
Reference in New Issue
Block a user