iomgaa
cf51d2de9d
Implement generator_v2.py with: - CandidateQuestion dataclass (canonical location) - _load_prompt_template: loads per-family .md from store/prompts/ - _build_v2_prompt: constructs system+user messages with material context - _parse_v2_response: JSON extraction, json_repair, field validation - generate_one_v2: async VLM call orchestration with reject_reason support Add 5 family-specific prompt templates: - retrieval.md: factual recall from visible content - reasoning.md: multi-hop inference across segments - enumeration.md: counting/listing entities and actions - visual.md: visual details requiring frame observation - spatial.md: spatial relationships between objects/people Tests: 11 unit tests covering prompt build, parse, and e2e generation. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
402 lines
13 KiB
Python
402 lines
13 KiB
Python
"""v2 生成器单元测试 — 验证 prompt 构建、响应解析、端到端生成。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
import pytest
|
|
|
|
from app.question_gen.families import RETRIEVAL_FAMILY, VISUAL_FAMILY
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
from app.question_gen.sampler_v2 import AnchorContext, MaterialContext
|
|
from app.tree.index import (
|
|
IndexMeta,
|
|
L1Card,
|
|
L1Node,
|
|
L2Card,
|
|
L2Node,
|
|
L3Card,
|
|
L3Node,
|
|
TreeIndex,
|
|
)
|
|
from core.types import LLMResponse
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixtures & Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_material(
|
|
*,
|
|
with_frames: bool = False,
|
|
with_cross_l2: bool = False,
|
|
) -> MaterialContext:
|
|
"""构造测试用 MaterialContext。"""
|
|
anchor = AnchorContext(node_id="L2_001", level=2, l2_id="L2_001")
|
|
frame_paths = ["/data/frames/f001.jpg", "/data/frames/f002.jpg"] if with_frames else []
|
|
cross_l2 = ["Person enters room and sits down."] if with_cross_l2 else []
|
|
return MaterialContext(
|
|
anchor=anchor,
|
|
source_nodes=("L2_001", "L3_001", "L3_002"),
|
|
subtitle_sentences=["He picks up the book.", "Then he starts reading."],
|
|
frame_paths=frame_paths,
|
|
cross_l2_texts=cross_l2,
|
|
)
|
|
|
|
|
|
def _make_vlm_response(content: str) -> LLMResponse:
|
|
"""构造 VLM 正常返回。"""
|
|
return LLMResponse(
|
|
content=content,
|
|
thinking="",
|
|
model="mock-vlm",
|
|
provider="mock",
|
|
prompt_tokens=50,
|
|
completion_tokens=30,
|
|
latency_ms=200,
|
|
ttft_ms=None,
|
|
max_inter_token_ms=None,
|
|
cache_hit=False,
|
|
call_id="mock-vlm-001",
|
|
)
|
|
|
|
|
|
class MockVLM:
|
|
"""可配置的 VLM mock — 记录调用并返回预设响应。"""
|
|
|
|
def __init__(self, response: LLMResponse) -> None:
|
|
self._response = response
|
|
self.calls: list[dict[str, Any]] = []
|
|
|
|
async def chat_with_images(
|
|
self,
|
|
messages: list[dict[str, Any]],
|
|
images: list[str | Path],
|
|
*,
|
|
session_id: str | None = None,
|
|
parent_call_id: str | None = None,
|
|
) -> LLMResponse:
|
|
"""记录调用并返回预设响应。"""
|
|
self.calls.append(
|
|
{
|
|
"messages": messages,
|
|
"images": images,
|
|
"session_id": session_id,
|
|
}
|
|
)
|
|
return self._response
|
|
|
|
|
|
def _make_tree() -> TreeIndex:
|
|
"""构造最小测试树。"""
|
|
l3_card = L3Card(
|
|
frame_summary="Person picks up a book from the shelf.",
|
|
visible_entities=["person", "book", "shelf"],
|
|
ongoing_actions=["picking up"],
|
|
visible_text=[],
|
|
spatial_layout="person in center",
|
|
visual_attributes={},
|
|
subtitle="He picks up the book.",
|
|
)
|
|
l3 = L3Node(id="L3_001", card=l3_card, frame_path="/data/frames/f001.jpg")
|
|
|
|
l3_card2 = L3Card(
|
|
frame_summary="Person starts reading the book.",
|
|
visible_entities=["person", "book"],
|
|
ongoing_actions=["reading"],
|
|
visible_text=[],
|
|
spatial_layout="person sitting",
|
|
visual_attributes={},
|
|
subtitle="Then he starts reading.",
|
|
)
|
|
l3_2 = L3Node(id="L3_002", card=l3_card2, frame_path="/data/frames/f002.jpg")
|
|
|
|
l2_card = L2Card(
|
|
event_description="A person picks up a book and starts reading.",
|
|
entities=["person", "book"],
|
|
actions=["pick up", "read"],
|
|
action_subjects=["person"],
|
|
visible_text=[],
|
|
spatial_relations="near shelf",
|
|
state_changes="book picked up",
|
|
subtitle="He picks up the book and reads.",
|
|
)
|
|
l2 = L2Node(id="L2_001", card=l2_card, children=[l3, l3_2])
|
|
|
|
l1_card = L1Card(
|
|
scene_summary="Library scene.",
|
|
main_setting="library",
|
|
key_entities=["person", "book"],
|
|
main_actions=["reading"],
|
|
topic_keywords=["library"],
|
|
visible_text=[],
|
|
temporal_flow="enter → read",
|
|
)
|
|
l1 = L1Node(id="L1_001", card=l1_card, children=[l2])
|
|
|
|
meta = IndexMeta(source_path="test.mp4", modality="video")
|
|
return TreeIndex(metadata=meta, roots=[l1])
|
|
|
|
|
|
_VALID_VLM_OUTPUT = json.dumps(
|
|
{
|
|
"question": "What does the person do after picking up the book?",
|
|
"options": ["A. Reads it", "B. Puts it back", "C. Throws it", "D. Burns it"],
|
|
"answer": "A",
|
|
"difficulty": "medium",
|
|
}
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# TestBuildV2Prompt
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestBuildV2Prompt:
|
|
"""验证 _build_v2_prompt 的 prompt 构造逻辑。"""
|
|
|
|
def test_includes_family_template(self) -> None:
|
|
"""prompt 中应包含家族模板的核心指令。"""
|
|
from app.question_gen.generator_v2 import _build_v2_prompt
|
|
|
|
material = _make_material()
|
|
messages, frame_paths = _build_v2_prompt(
|
|
family_spec=RETRIEVAL_FAMILY,
|
|
material=material,
|
|
task_type="Action Reasoning",
|
|
seq=1,
|
|
)
|
|
|
|
# messages 至少有 system + user
|
|
assert len(messages) >= 2
|
|
# system message 中应包含 retrieval 家族相关指令
|
|
system_content = messages[0]["content"]
|
|
assert "retrieval" in system_content.lower() or "factual" in system_content.lower()
|
|
|
|
def test_reject_reason_injected(self) -> None:
|
|
"""当 reject_reason 不为 None 时，应注入到 prompt 中。"""
|
|
from app.question_gen.generator_v2 import _build_v2_prompt
|
|
|
|
material = _make_material()
|
|
reject_msg = "The question leaked information from the answer options."
|
|
messages, _ = _build_v2_prompt(
|
|
family_spec=RETRIEVAL_FAMILY,
|
|
material=material,
|
|
task_type="Action Reasoning",
|
|
seq=2,
|
|
reject_reason=reject_msg,
|
|
)
|
|
|
|
# reject_reason 应出现在某个 message 内容中
|
|
all_content = " ".join(m["content"] for m in messages)
|
|
assert reject_msg in all_content
|
|
|
|
def test_frame_paths_from_material(self) -> None:
|
|
"""返回的 frame_paths 应来自 material.frame_paths。"""
|
|
from app.question_gen.generator_v2 import _build_v2_prompt
|
|
|
|
material = _make_material(with_frames=True)
|
|
_, frame_paths = _build_v2_prompt(
|
|
family_spec=VISUAL_FAMILY,
|
|
material=material,
|
|
task_type="Object Recognition",
|
|
seq=1,
|
|
)
|
|
|
|
assert frame_paths == ["/data/frames/f001.jpg", "/data/frames/f002.jpg"]
|
|
|
|
def test_no_frames_returns_empty(self) -> None:
|
|
"""无帧素材时 frame_paths 应为空列表。"""
|
|
from app.question_gen.generator_v2 import _build_v2_prompt
|
|
|
|
material = _make_material(with_frames=False)
|
|
_, frame_paths = _build_v2_prompt(
|
|
family_spec=RETRIEVAL_FAMILY,
|
|
material=material,
|
|
task_type="Action Reasoning",
|
|
seq=1,
|
|
)
|
|
|
|
assert frame_paths == []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# TestParseV2Response
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestParseV2Response:
|
|
"""验证 _parse_v2_response 的解析与校验逻辑。"""
|
|
|
|
def test_valid_json(self) -> None:
|
|
"""正确 JSON → 生成 CandidateQuestion。"""
|
|
from app.question_gen.generator_v2 import CandidateQuestion, _parse_v2_response
|
|
|
|
result = _parse_v2_response(
|
|
raw=_VALID_VLM_OUTPUT,
|
|
video_id="v-001",
|
|
task_type="Action Reasoning",
|
|
skill_target="M1",
|
|
seq=3,
|
|
source_nodes=("L2_001", "L3_001"),
|
|
)
|
|
|
|
assert isinstance(result, CandidateQuestion)
|
|
assert result.question_id == "v-001_Action Reasoning_0003"
|
|
assert result.question == "What does the person do after picking up the book?"
|
|
assert result.options == ("A. Reads it", "B. Puts it back", "C. Throws it", "D. Burns it")
|
|
assert result.answer == "A"
|
|
assert result.difficulty == "medium"
|
|
assert result.video_id == "v-001"
|
|
assert result.task_type == "Action Reasoning"
|
|
assert result.skill_target == "M1"
|
|
assert result.source_nodes == ("L2_001", "L3_001")
|
|
|
|
def test_missing_field_raises(self) -> None:
|
|
"""缺少必填字段 → 抛出 ValueError。"""
|
|
from app.question_gen.generator_v2 import _parse_v2_response
|
|
|
|
incomplete = json.dumps(
|
|
{
|
|
"question": "What happens?",
|
|
"options": ["A. X", "B. Y"],
|
|
# 缺少 answer 和 difficulty
|
|
}
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="answer"):
|
|
_parse_v2_response(
|
|
raw=incomplete,
|
|
video_id="v-001",
|
|
task_type="Action Reasoning",
|
|
skill_target="M1",
|
|
seq=1,
|
|
source_nodes=("L2_001",),
|
|
)
|
|
|
|
def test_invalid_answer_raises(self) -> None:
|
|
"""answer 不在 A-D 范围内 → 抛出 ValueError。"""
|
|
from app.question_gen.generator_v2 import _parse_v2_response
|
|
|
|
bad_answer = json.dumps(
|
|
{
|
|
"question": "What happens?",
|
|
"options": ["A. X", "B. Y", "C. Z", "D. W"],
|
|
"answer": "E",
|
|
"difficulty": "easy",
|
|
}
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="answer"):
|
|
_parse_v2_response(
|
|
raw=bad_answer,
|
|
video_id="v-001",
|
|
task_type="Action Reasoning",
|
|
skill_target="M1",
|
|
seq=1,
|
|
source_nodes=("L2_001",),
|
|
)
|
|
|
|
def test_json_repair_handles_trailing_comma(self) -> None:
|
|
"""json_repair 应能修复常见 JSON 错误。"""
|
|
from app.question_gen.generator_v2 import _parse_v2_response
|
|
|
|
malformed = '{"question": "What?", "options": ["A. X", "B. Y", "C. Z", "D. W",], "answer": "B", "difficulty": "easy",}'
|
|
|
|
result = _parse_v2_response(
|
|
raw=malformed,
|
|
video_id="v-001",
|
|
task_type="Action Reasoning",
|
|
skill_target="M1",
|
|
seq=1,
|
|
source_nodes=("L2_001",),
|
|
)
|
|
assert result.answer == "B"
|
|
|
|
def test_markdown_wrapped_json(self) -> None:
|
|
"""被 markdown 代码块包裹的 JSON 应正常解析。"""
|
|
from app.question_gen.generator_v2 import _parse_v2_response
|
|
|
|
wrapped = f"```json\n{_VALID_VLM_OUTPUT}\n```"
|
|
|
|
result = _parse_v2_response(
|
|
raw=wrapped,
|
|
video_id="v-001",
|
|
task_type="Action Reasoning",
|
|
skill_target="M1",
|
|
seq=5,
|
|
source_nodes=("L2_001",),
|
|
)
|
|
assert result.question_id == "v-001_Action Reasoning_0005"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# TestGenerateOneV2
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestGenerateOneV2:
|
|
"""验证 generate_one_v2 端到端流程。"""
|
|
|
|
@pytest.mark.asyncio()
|
|
async def test_happy_path(self) -> None:
|
|
"""正常流程：VLM 返回有效 JSON → 生成 CandidateQuestion。"""
|
|
from app.question_gen.generator_v2 import CandidateQuestion, generate_one_v2
|
|
|
|
mock_vlm = MockVLM(_make_vlm_response(_VALID_VLM_OUTPUT))
|
|
tree = _make_tree()
|
|
material = _make_material(with_frames=True)
|
|
|
|
result = await generate_one_v2(
|
|
vlm=mock_vlm,
|
|
tree=tree,
|
|
material=material,
|
|
family_spec=RETRIEVAL_FAMILY,
|
|
task_type="Action Reasoning",
|
|
seq=1,
|
|
video_id="v-001",
|
|
session_id="session-test-001",
|
|
)
|
|
|
|
assert isinstance(result, CandidateQuestion)
|
|
assert result.question_id == "v-001_Action Reasoning_0001"
|
|
assert result.video_id == "v-001"
|
|
assert result.skill_target == "M1"
|
|
assert result.source_nodes == ("L2_001", "L3_001", "L3_002")
|
|
assert result.subtitle_sentences == ("He picks up the book.", "Then he starts reading.")
|
|
assert result.frame_paths == ("/data/frames/f001.jpg", "/data/frames/f002.jpg")
|
|
|
|
# VLM 应被调用一次
|
|
assert len(mock_vlm.calls) == 1
|
|
assert mock_vlm.calls[0]["session_id"] == "session-test-001"
|
|
|
|
@pytest.mark.asyncio()
|
|
async def test_reject_reason_forwarded(self) -> None:
|
|
"""reject_reason 应被传入 prompt 构建。"""
|
|
from app.question_gen.generator_v2 import generate_one_v2
|
|
|
|
mock_vlm = MockVLM(_make_vlm_response(_VALID_VLM_OUTPUT))
|
|
tree = _make_tree()
|
|
material = _make_material()
|
|
|
|
await generate_one_v2(
|
|
vlm=mock_vlm,
|
|
tree=tree,
|
|
material=material,
|
|
family_spec=RETRIEVAL_FAMILY,
|
|
task_type="Action Reasoning",
|
|
seq=2,
|
|
video_id="v-001",
|
|
reject_reason="Answer was too obvious",
|
|
session_id="session-test-002",
|
|
)
|
|
|
|
# 验证 VLM 调用的 messages 中包含 reject_reason
|
|
call = mock_vlm.calls[0]
|
|
all_content = " ".join(m["content"] for m in call["messages"])
|
|
assert "Answer was too obvious" in all_content
|