diff --git a/core/evolution/diagnose.py b/core/evolution/diagnose.py index 204e40a..312c74c 100644 --- a/core/evolution/diagnose.py +++ b/core/evolution/diagnose.py @@ -2142,7 +2142,14 @@ async def run_diagnosis( key = (prediction.get("video_id", ""), prediction.get("question_id", "")) traces = traces_by_question.get(key, []) vid = prediction.get("video_id", "") - td = tree_data_by_video.get(vid, {}) + if vid not in tree_data_by_video: + # P5 fail-loud:诊断需真实树,调用方须为每个诊断视频加载 tree_data; + # 静默回退空树会让 ground_truth 恒空、error_type 归因坍缩(本次修复的根因)。 + raise ValueError( + f"诊断视频树未覆盖: video_id={vid!r} 不在注入的 tree_data 中" + "(调用方须为每个诊断视频加载树,P5 fail loud)" + ) + td = tree_data_by_video[vid] skill_content = skill_cache.get(prediction.get("task_type", ""), "") try: diff --git a/tests/integration/test_baseline_diagnosis.py b/tests/integration/test_baseline_diagnosis.py index 9319d2c..dba064a 100644 --- a/tests/integration/test_baseline_diagnosis.py +++ b/tests/integration/test_baseline_diagnosis.py @@ -35,6 +35,36 @@ class _FakeRunLog: return [] +class _CoveragePredRunLog: + """RunLog 假实现:返回一条 correct=0、带 view_node 步的诊断 prediction。 + + 用于验证 core fail-loud 护栏——该 prediction 的 video_id 故意不在注入的 + tree_data 里,run_diagnosis 应 raise 而非静默回退空树。 + """ + + def __init__(self, video_id: str, question_id: str) -> None: + self._video_id = video_id + self._question_id = question_id + + async def get_predictions(self, run_id, *, question_ids=None): + return [ + { + "video_id": self._video_id, + "question_id": self._question_id, + "task_type": "Counting Problem", + "prediction": "B", # 与 answer 不同 → correct=0 + "answer": "A", + "stop_reason": "answer_found", # 非 INFRA,进 _process_question + "steps_json": [ + {"tool": "view_node", "args": {"node_id": "n0"}}, + ], + } + ] + + async def get_traces(self, run_id, *, question_ids=None): + return [] + + class _FakeLLM: ... @@ -96,6 +126,8 @@ def _deps(monkeypatch, calls): llm=_FakeLLM(), skill_store=_FakeSkillStore(), prompts=object(), + # 豁免 core 视频覆盖 fail-loud:本用例 monkeypatch 了 run_diagnosis 为 + # 忽略 tree_data 的假实现,永不进入真实 _process_question 覆盖护栏,故留空。 tree_data={}, concurrency=2, ) @@ -221,6 +253,8 @@ async def test_infra_and_degraded_projection(tmp_path, monkeypatch): llm=_FakeLLM(), skill_store=_FakeSkillStore(), prompts=object(), + # 豁免 core 视频覆盖 fail-loud:本用例 monkeypatch 了 run_diagnosis 为 + # 忽略 tree_data 的假实现,永不进入真实 _process_question 覆盖护栏,故留空。 tree_data={}, concurrency=2, ) @@ -308,6 +342,8 @@ async def test_degraded_overrides_attribution(tmp_path, monkeypatch): llm=_FakeLLM(), skill_store=_FakeSkillStore(), prompts=object(), + # 豁免 core 视频覆盖 fail-loud:本用例 monkeypatch 了 run_diagnosis 为 + # 忽略 tree_data 的假实现,永不进入真实 _process_question 覆盖护栏,故留空。 tree_data={}, concurrency=2, ) @@ -352,3 +388,49 @@ async def test_degraded_overrides_attribution(tmp_path, monkeypatch): ) assert md_path.exists() store.close() + + +class _EmptySkillStore: + """SkillStore 假实现:无 skill 文件,_resolve_skill_file 回退空串。""" + + def list_skill_files(self): + return [] + + def read_skill(self, filename): + return "" + + +@pytest.mark.asyncio +async def test_run_diagnosis_raises_when_video_tree_missing(): + """诊断视频未被 tree_data 覆盖时 fail-loud(不静默回退、不走 judge 降级)。 + + 构造一条 correct=0、带 view_node 步的 prediction,其 video_id="vMISS" + 故意不在注入的 tree_data 里。core 护栏应在取 td 处 raise ValueError, + 且该 raise 位于 compute_question_metrics 的 try 之前,不被 judge 降级吞掉。 + """ + from core.evolution.diagnose import run_diagnosis + from core.types import GeneratedQuestion + + q = GeneratedQuestion( + question_id="vMISS-1", + video_id="vMISS", + task_type="Counting Problem", + question="", + options=("A", "B", "C", "D"), + answer="A", + source_nodes=(), + difficulty="easy", + ) + + with pytest.raises(ValueError, match="诊断视频树未覆盖"): + await run_diagnosis( + run_id="infer_adhoc", + questions=[q], + tree_data={"vOTHER": {"nodes": {}}}, # 故意不含 vMISS + llm=_FakeLLM(), + run_log=_CoveragePredRunLog("vMISS", "vMISS-1"), + skill_store=_EmptySkillStore(), + prompts=object(), + concurrency=1, + question_ids=["vMISS-1"], + )