fix: fail loud when diagnosis video tree not covered (algo #7 input)
This commit is contained in:
@@ -35,6 +35,36 @@ class _FakeRunLog:
|
||||
return []
|
||||
|
||||
|
||||
class _CoveragePredRunLog:
|
||||
"""RunLog 假实现:返回一条 correct=0、带 view_node 步的诊断 prediction。
|
||||
|
||||
用于验证 core fail-loud 护栏——该 prediction 的 video_id 故意不在注入的
|
||||
tree_data 里,run_diagnosis 应 raise 而非静默回退空树。
|
||||
"""
|
||||
|
||||
def __init__(self, video_id: str, question_id: str) -> None:
|
||||
self._video_id = video_id
|
||||
self._question_id = question_id
|
||||
|
||||
async def get_predictions(self, run_id, *, question_ids=None):
|
||||
return [
|
||||
{
|
||||
"video_id": self._video_id,
|
||||
"question_id": self._question_id,
|
||||
"task_type": "Counting Problem",
|
||||
"prediction": "B", # 与 answer 不同 → correct=0
|
||||
"answer": "A",
|
||||
"stop_reason": "answer_found", # 非 INFRA,进 _process_question
|
||||
"steps_json": [
|
||||
{"tool": "view_node", "args": {"node_id": "n0"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
async def get_traces(self, run_id, *, question_ids=None):
|
||||
return []
|
||||
|
||||
|
||||
class _FakeLLM: ...
|
||||
|
||||
|
||||
@@ -96,6 +126,8 @@ def _deps(monkeypatch, calls):
|
||||
llm=_FakeLLM(),
|
||||
skill_store=_FakeSkillStore(),
|
||||
prompts=object(),
|
||||
# 豁免 core 视频覆盖 fail-loud:本用例 monkeypatch 了 run_diagnosis 为
|
||||
# 忽略 tree_data 的假实现,永不进入真实 _process_question 覆盖护栏,故留空。
|
||||
tree_data={},
|
||||
concurrency=2,
|
||||
)
|
||||
@@ -221,6 +253,8 @@ async def test_infra_and_degraded_projection(tmp_path, monkeypatch):
|
||||
llm=_FakeLLM(),
|
||||
skill_store=_FakeSkillStore(),
|
||||
prompts=object(),
|
||||
# 豁免 core 视频覆盖 fail-loud:本用例 monkeypatch 了 run_diagnosis 为
|
||||
# 忽略 tree_data 的假实现,永不进入真实 _process_question 覆盖护栏,故留空。
|
||||
tree_data={},
|
||||
concurrency=2,
|
||||
)
|
||||
@@ -308,6 +342,8 @@ async def test_degraded_overrides_attribution(tmp_path, monkeypatch):
|
||||
llm=_FakeLLM(),
|
||||
skill_store=_FakeSkillStore(),
|
||||
prompts=object(),
|
||||
# 豁免 core 视频覆盖 fail-loud:本用例 monkeypatch 了 run_diagnosis 为
|
||||
# 忽略 tree_data 的假实现,永不进入真实 _process_question 覆盖护栏,故留空。
|
||||
tree_data={},
|
||||
concurrency=2,
|
||||
)
|
||||
@@ -352,3 +388,49 @@ async def test_degraded_overrides_attribution(tmp_path, monkeypatch):
|
||||
)
|
||||
assert md_path.exists()
|
||||
store.close()
|
||||
|
||||
|
||||
class _EmptySkillStore:
|
||||
"""SkillStore 假实现:无 skill 文件,_resolve_skill_file 回退空串。"""
|
||||
|
||||
def list_skill_files(self):
|
||||
return []
|
||||
|
||||
def read_skill(self, filename):
|
||||
return ""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_diagnosis_raises_when_video_tree_missing():
|
||||
"""诊断视频未被 tree_data 覆盖时 fail-loud(不静默回退、不走 judge 降级)。
|
||||
|
||||
构造一条 correct=0、带 view_node 步的 prediction,其 video_id="vMISS"
|
||||
故意不在注入的 tree_data 里。core 护栏应在取 td 处 raise ValueError,
|
||||
且该 raise 位于 compute_question_metrics 的 try 之前,不被 judge 降级吞掉。
|
||||
"""
|
||||
from core.evolution.diagnose import run_diagnosis
|
||||
from core.types import GeneratedQuestion
|
||||
|
||||
q = GeneratedQuestion(
|
||||
question_id="vMISS-1",
|
||||
video_id="vMISS",
|
||||
task_type="Counting Problem",
|
||||
question="",
|
||||
options=("A", "B", "C", "D"),
|
||||
answer="A",
|
||||
source_nodes=(),
|
||||
difficulty="easy",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="诊断视频树未覆盖"):
|
||||
await run_diagnosis(
|
||||
run_id="infer_adhoc",
|
||||
questions=[q],
|
||||
tree_data={"vOTHER": {"nodes": {}}}, # 故意不含 vMISS
|
||||
llm=_FakeLLM(),
|
||||
run_log=_CoveragePredRunLog("vMISS", "vMISS-1"),
|
||||
skill_store=_EmptySkillStore(),
|
||||
prompts=object(),
|
||||
concurrency=1,
|
||||
question_ids=["vMISS-1"],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user