feat(question_gen): add v2 generator with per-family prompt templates

Implement generator_v2.py with:
- CandidateQuestion dataclass (canonical location)
- _load_prompt_template: loads per-family .md from store/prompts/
- _build_v2_prompt: constructs system+user messages with material context
- _parse_v2_response: JSON extraction, json_repair, field validation
- generate_one_v2: async VLM call orchestration with reject_reason support

Add 5 family-specific prompt templates:
- retrieval.md: factual recall from visible content
- reasoning.md: multi-hop inference across segments
- enumeration.md: counting/listing entities and actions
- visual.md: visual details requiring frame observation
- spatial.md: spatial relationships between objects/people

Tests: 11 unit tests covering prompt build, parse, and e2e generation.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-11 23:41:36 -04:00
parent 9053233f99
commit cf51d2de9d
7 changed files with 928 additions and 0 deletions
+415
View File
@@ -0,0 +1,415 @@
"""v2 生成器 — 基于家族特化 prompt 模板的单题 VLM 出题模块。
使用 VLMProvider 接口调用视觉语言模型,结合 per-family prompt 模板
和 MaterialContext 素材上下文,生成一道四选一候选题。
典型调用路径::
candidate = await generate_one_v2(
vlm=vlm_client,
tree=tree_index,
material=material_ctx,
family_spec=RETRIEVAL_FAMILY,
task_type="Action Reasoning",
seq=1,
video_id="vid_001",
session_id="sess_001",
)
"""
from __future__ import annotations
import json
from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING
from json_repair import repair_json
from loguru import logger
if TYPE_CHECKING:
from app.question_gen.families import QuestionFamilySpec
from app.question_gen.sampler_v2 import MaterialContext
from app.tree.index import TreeIndex
from core.protocols import VLMProvider
# ---------------------------------------------------------------------------
# 常量
# ---------------------------------------------------------------------------
_PROMPTS_DIR = Path(__file__).resolve().parent.parent.parent / "store" / "prompts" / "question_gen"
_VALID_ANSWERS = frozenset({"A", "B", "C", "D"})
_VALID_DIFFICULTIES = frozenset({"easy", "medium", "hard"})
# ---------------------------------------------------------------------------
# CandidateQuestion(规范定义位置)
# ---------------------------------------------------------------------------
@dataclass(frozen=True)
class CandidateQuestion:
"""候选题目 — 出题管线生成、待门控审核的题目数据。
属性:
question_id: 题目唯一标识(格式 "{video_id}_{task_type}_{seq:04d}")。
video_id: 所属视频标识。
task_type: 题型(如 "Action Reasoning")。
skill_target: 目标失败机制编号(M1-M5)。
question: 题目文本。
options: 选项元组(如 ("A. ...", "B. ...", "C. ...", "D. ..."))。
answer: 正确答案字母(如 "A")。
source_nodes: 来源节点 ID 元组。
difficulty: 难度等级(easy/medium/hard)。
subtitle_sentences: 验证材料 — 字幕句子元组。
frame_paths: 验证材料 — 帧图片路径元组。
"""
question_id: str
video_id: str
task_type: str
skill_target: str
question: str
options: tuple[str, ...]
answer: str
source_nodes: tuple[str, ...]
difficulty: str
subtitle_sentences: tuple[str, ...] = field(default_factory=tuple)
frame_paths: tuple[str, ...] = field(default_factory=tuple)
# ---------------------------------------------------------------------------
# Prompt 模板加载
# ---------------------------------------------------------------------------
def _load_prompt_template(family_spec: QuestionFamilySpec) -> str:
"""加载家族对应的 prompt 模板文件。
参数:
family_spec: 问题家族规格(含 prompt_template 文件名)。
返回:
模板内容字符串。
异常:
FileNotFoundError: 模板文件不存在。
"""
path = _PROMPTS_DIR / family_spec.prompt_template
if not path.exists():
msg = f"家族 prompt 模板文件不存在: {path}"
raise FileNotFoundError(msg)
return path.read_text(encoding="utf-8")
# ---------------------------------------------------------------------------
# Prompt 构建
# ---------------------------------------------------------------------------
def _build_v2_prompt(
family_spec: QuestionFamilySpec,
material: MaterialContext,
task_type: str,
seq: int,
*,
reject_reason: str | None = None,
) -> tuple[list[dict[str, str]], list[str]]:
"""构建 VLM 出题调用的 messages 和帧路径列表。
参数:
family_spec: 问题家族规格。
material: 采样素材上下文。
task_type: 任务类型字符串。
seq: 当前序号。
reject_reason: 上一次被门控拒绝的原因(用于引导 VLM 避免相同错误)。
返回:
二元组:
- messages: 适配 VLMProvider 的 message 列表(system + user)。
- frame_paths: 需发送给 VLM 的帧路径列表。
"""
# Phase 1: 加载家族模板作为 system prompt
template_content = _load_prompt_template(family_spec)
system_message = template_content
# Phase 2: 构建 user prompt — 聚合素材信息
user_parts: list[str] = []
user_parts.append(f"## Task Type: {task_type}")
user_parts.append(f"## Question Family: {family_spec.name}")
user_parts.append(f"## Sequence: #{seq}")
# 字幕素材
if material.subtitle_sentences:
user_parts.append("\n## Subtitle Content:")
for i, sent in enumerate(material.subtitle_sentences, 1):
user_parts.append(f" {i}. {sent}")
# 跨 L2 上下文
if material.cross_l2_texts:
user_parts.append("\n## Cross-Segment Context:")
for text in material.cross_l2_texts:
user_parts.append(f" - {text}")
# 帧路径提示(VLM 会接收实际图像,此处仅作文本参考)
if material.frame_paths:
user_parts.append(f"\n## Visual Frames: {len(material.frame_paths)} frames attached.")
# 拒绝原因注入
if reject_reason is not None:
user_parts.append(
f"\n## IMPORTANT - Previous Attempt Rejected:\n"
f"Your previous question was rejected for the following reason:\n"
f'"{reject_reason}"\n'
f"Please generate a NEW question that avoids this issue."
)
# 输出格式指令
user_parts.append(
"\n## Output Format:\n"
"Respond with ONLY a JSON object in this exact format:\n"
"```json\n"
"{\n"
' "question": "Your question text here",\n'
' "options": ["A. ...", "B. ...", "C. ...", "D. ..."],\n'
' "answer": "A",\n'
' "difficulty": "easy|medium|hard"\n'
"}\n"
"```"
)
user_content = "\n".join(user_parts)
messages = [
{"role": "system", "content": system_message},
{"role": "user", "content": user_content},
]
# Phase 3: 帧路径
frame_paths = list(material.frame_paths)
return messages, frame_paths
# ---------------------------------------------------------------------------
# 响应解析
# ---------------------------------------------------------------------------
def _extract_json_from_text(raw: str) -> str:
"""从可能被 markdown 代码块包裹的文本中提取 JSON 部分。
参数:
raw: VLM 原始返回文本。
返回:
清理后的 JSON 字符串。
"""
content = raw.strip()
if "```" in content:
parts = content.split("```")
for part in parts:
stripped = part.strip()
if stripped.startswith("json"):
stripped = stripped[4:].strip()
if stripped.startswith("{"):
return stripped
return content
def _parse_v2_response(
raw: str,
video_id: str,
task_type: str,
skill_target: str,
seq: int,
source_nodes: tuple[str, ...],
) -> CandidateQuestion:
"""解析 VLM 返回的 JSON 响应,构造 CandidateQuestion。
流程:
1. 提取 JSON(处理 markdown 包裹)。
2. json_repair 修复常见格式错误。
3. 解析并校验必填字段。
4. 构造 CandidateQuestion 实例。
参数:
raw: VLM 原始返回文本。
video_id: 视频标识。
task_type: 任务类型。
skill_target: 目标技能编号。
seq: 当前序号。
source_nodes: 来源节点 ID 元组。
返回:
CandidateQuestion 实例。
异常:
ValueError: JSON 解析失败或缺少必填字段或字段值非法。
"""
# Phase 1: 提取 JSON 文本
json_text = _extract_json_from_text(raw)
# Phase 2: json_repair 修复
repaired = repair_json(json_text, return_objects=False)
# Phase 3: 解析
try:
data = json.loads(repaired)
except json.JSONDecodeError as e:
msg = f"VLM 响应 JSON 解析失败: {e}. 原始文本: {raw[:200]}"
raise ValueError(msg) from e
if not isinstance(data, dict):
msg = f"VLM 响应顶层不是 JSON 对象: type={type(data).__name__}"
raise ValueError(msg)
# Phase 4: 校验必填字段
missing = [f for f in ("question", "options", "answer", "difficulty") if f not in data]
if missing:
msg = f"VLM 响应缺少必填字段: {', '.join(missing)}"
raise ValueError(msg)
question_text = str(data["question"])
options_raw = data["options"]
answer = str(data["answer"]).strip().upper()
difficulty = str(data["difficulty"]).strip().lower()
# 校验 options
if not isinstance(options_raw, list) or len(options_raw) < 2:
msg = f"options 字段必须是至少 2 个选项的列表,实际: {options_raw}"
raise ValueError(msg)
options = tuple(str(o) for o in options_raw)
# 校验 answer
if answer not in _VALID_ANSWERS:
msg = f"answer 字段值 '{answer}' 非法,必须为 A/B/C/D 之一"
raise ValueError(msg)
# 校验 difficulty
if difficulty not in _VALID_DIFFICULTIES:
logger.warning(
"difficulty '{}' 不在预设范围 {},回退为 'medium'",
difficulty,
_VALID_DIFFICULTIES,
)
difficulty = "medium"
# Phase 5: 构造 CandidateQuestion
question_id = f"{video_id}_{task_type}_{seq:04d}"
return CandidateQuestion(
question_id=question_id,
video_id=video_id,
task_type=task_type,
skill_target=skill_target,
question=question_text,
options=options,
answer=answer,
source_nodes=source_nodes,
difficulty=difficulty,
)
# ---------------------------------------------------------------------------
# 主入口
# ---------------------------------------------------------------------------
async def generate_one_v2(
vlm: VLMProvider,
tree: TreeIndex,
material: MaterialContext,
family_spec: QuestionFamilySpec,
task_type: str,
seq: int,
*,
video_id: str,
reject_reason: str | None = None,
session_id: str,
) -> CandidateQuestion:
"""调用 VLM 生成一道候选题目。
流程:
1. 构建 per-family prompt + 帧路径。
2. 调用 VLMProvider.chat_with_images。
3. 解析响应为 CandidateQuestion。
4. 附加素材验证信息(subtitle_sentences、frame_paths)。
参数:
vlm: VLM 调用端口。
tree: 视频树索引(当前未直接使用,预留后续扩展)。
material: 采样素材上下文。
family_spec: 问题家族规格。
task_type: 任务类型字符串。
seq: 当前序号。
video_id: 视频标识。
reject_reason: 上一次被门控拒绝的原因。
session_id: 会话 ID(遥测关联)。
返回:
CandidateQuestion 实例(包含验证材料)。
异常:
ValueError: VLM 响应解析失败。
FileNotFoundError: 家族 prompt 模板不存在。
"""
# Phase 1: 构建 prompt
messages, frame_paths = _build_v2_prompt(
family_spec=family_spec,
material=material,
task_type=task_type,
seq=seq,
reject_reason=reject_reason,
)
# Phase 2: 调用 VLM
logger.debug(
"generate_one_v2: family={}, task_type={}, seq={}, frames={}",
family_spec.name,
task_type,
seq,
len(frame_paths),
)
response = await vlm.chat_with_images(
messages,
frame_paths,
session_id=session_id,
)
# Phase 3: 解析响应
candidate = _parse_v2_response(
raw=response.content,
video_id=video_id,
task_type=task_type,
skill_target=family_spec.skill_target,
seq=seq,
source_nodes=material.source_nodes,
)
# Phase 4: 附加验证材料(构造新实例,因 frozen=True
candidate = CandidateQuestion(
question_id=candidate.question_id,
video_id=candidate.video_id,
task_type=candidate.task_type,
skill_target=candidate.skill_target,
question=candidate.question,
options=candidate.options,
answer=candidate.answer,
source_nodes=candidate.source_nodes,
difficulty=candidate.difficulty,
subtitle_sentences=tuple(material.subtitle_sentences),
frame_paths=tuple(material.frame_paths),
)
logger.debug(
"generate_one_v2 完成: question_id={}, difficulty={}",
candidate.question_id,
candidate.difficulty,
)
return candidate
+22
View File
@@ -0,0 +1,22 @@
You are a question generator for video understanding benchmarks.
Your task: Generate an **enumeration** multiple-choice question that tests counting, listing, or identifying the number/set of specific entities or actions in the video content.
## Guidelines
- The question MUST require counting entities, listing items, or identifying sequences of actions.
- Focus on "How many...", "Which of the following are all...", "In what order..." style questions.
- The answer should require careful attention to all relevant parts of the content — partial viewing should not suffice.
- Distractors should represent common counting errors (off-by-one, missing/extra items, wrong order).
## Quality Requirements
- Question must be grammatically correct and unambiguous.
- All four options must be parallel in structure and length.
- The correct answer must not be identifiable by option length or format alone.
- Avoid trivially small counts (e.g., "How many people?" when only 1 is visible).
- Each option must begin with "A. ", "B. ", "C. ", or "D. ".
## Output
Respond with ONLY a valid JSON object. No additional text.
+22
View File
@@ -0,0 +1,22 @@
You are a question generator for video understanding benchmarks.
Your task: Generate a **multi-hop reasoning** multiple-choice question that requires connecting information from multiple segments of the video to arrive at the correct answer.
## Guidelines
- The question MUST require reasoning across at least two distinct pieces of information (temporal, causal, or logical connections).
- The answer should NOT be directly stated in any single subtitle or frame — it must be inferred by combining evidence.
- Test causal chains, temporal ordering, or logical deductions that span multiple events.
- Distractors should represent common reasoning errors (e.g., reversed causality, incorrect temporal ordering).
## Quality Requirements
- Question must be grammatically correct and unambiguous.
- All four options must be parallel in structure and length.
- The correct answer must not be identifiable from linguistic cues alone.
- The reasoning chain should be verifiable from the provided material.
- Each option must begin with "A. ", "B. ", "C. ", or "D. ".
## Output
Respond with ONLY a valid JSON object. No additional text.
+23
View File
@@ -0,0 +1,23 @@
You are a question generator for video understanding benchmarks.
Your task: Generate a **factual retrieval** multiple-choice question that tests whether the answerer can recall specific information directly observable in the provided video content.
## Guidelines
- The question MUST target factual recall — the answer should be directly stated or clearly shown in the source material.
- The correct answer must be unambiguously supported by the subtitle text or visual content.
- Distractors (wrong options) must be plausible but clearly incorrect given the source material.
- Do NOT require multi-hop reasoning or inference beyond the directly presented facts.
- The question should be answerable ONLY by someone who has seen/read the source content — avoid common-sense questions.
## Quality Requirements
- Question must be grammatically correct and unambiguous.
- All four options must be parallel in structure and length.
- The correct answer must not be identifiable from linguistic cues alone.
- Avoid negation in the question stem (e.g., "Which of the following is NOT...").
- Each option must begin with "A. ", "B. ", "C. ", or "D. ".
## Output
Respond with ONLY a valid JSON object. No additional text.
+23
View File
@@ -0,0 +1,23 @@
You are a question generator for video understanding benchmarks.
Your task: Generate a **spatial relationship** multiple-choice question that tests understanding of spatial arrangements, positions, and relationships between objects or people in the video.
## Guidelines
- The question MUST focus on spatial relationships: relative positions, directions, distances, containment, or spatial changes.
- Test understanding of "where" things are, how they relate spatially, or how spatial arrangements change over time.
- Use spatial language: "left/right of", "above/below", "between", "inside/outside", "closer/farther", "facing".
- The answer should require spatial reasoning that goes beyond simple object identification.
- Distractors should represent common spatial confusion (mirror reversals, misremembered positions).
## Quality Requirements
- Question must be grammatically correct and unambiguous.
- All four options must be parallel in structure and length.
- The correct answer must not be identifiable from linguistic cues alone.
- Spatial references must be unambiguous given the visual content.
- Each option must begin with "A. ", "B. ", "C. ", or "D. ".
## Output
Respond with ONLY a valid JSON object. No additional text.
+22
View File
@@ -0,0 +1,22 @@
You are a question generator for video understanding benchmarks.
Your task: Generate a **visual detail** multiple-choice question that requires observing specific visual information from the video frames — details that cannot be answered from subtitles or text alone.
## Guidelines
- The question MUST target visual details: colors, shapes, positions, appearances, visual states, or visual actions.
- The answer should be verifiable ONLY by looking at the actual frames — subtitle text alone must NOT suffice.
- Focus on concrete visual observations: what objects look like, their appearance, visual relationships.
- Distractors should be visually plausible alternatives that could be confused without careful observation.
## Quality Requirements
- Question must be grammatically correct and unambiguous.
- All four options must be parallel in structure and length.
- The correct answer must not be identifiable from linguistic cues alone.
- Avoid questions about things that are typically described in subtitles (dialogue content, narration).
- Each option must begin with "A. ", "B. ", "C. ", or "D. ".
## Output
Respond with ONLY a valid JSON object. No additional text.
+401
View File
@@ -0,0 +1,401 @@
"""v2 生成器单元测试 — 验证 prompt 构建、响应解析、端到端生成。"""
from __future__ import annotations
import json
from typing import TYPE_CHECKING, Any
import pytest
from app.question_gen.families import RETRIEVAL_FAMILY, VISUAL_FAMILY
if TYPE_CHECKING:
from pathlib import Path
from app.question_gen.sampler_v2 import AnchorContext, MaterialContext
from app.tree.index import (
IndexMeta,
L1Card,
L1Node,
L2Card,
L2Node,
L3Card,
L3Node,
TreeIndex,
)
from core.types import LLMResponse
# ---------------------------------------------------------------------------
# Fixtures & Helpers
# ---------------------------------------------------------------------------
def _make_material(
*,
with_frames: bool = False,
with_cross_l2: bool = False,
) -> MaterialContext:
"""构造测试用 MaterialContext。"""
anchor = AnchorContext(node_id="L2_001", level=2, l2_id="L2_001")
frame_paths = ["/data/frames/f001.jpg", "/data/frames/f002.jpg"] if with_frames else []
cross_l2 = ["Person enters room and sits down."] if with_cross_l2 else []
return MaterialContext(
anchor=anchor,
source_nodes=("L2_001", "L3_001", "L3_002"),
subtitle_sentences=["He picks up the book.", "Then he starts reading."],
frame_paths=frame_paths,
cross_l2_texts=cross_l2,
)
def _make_vlm_response(content: str) -> LLMResponse:
"""构造 VLM 正常返回。"""
return LLMResponse(
content=content,
thinking="",
model="mock-vlm",
provider="mock",
prompt_tokens=50,
completion_tokens=30,
latency_ms=200,
ttft_ms=None,
max_inter_token_ms=None,
cache_hit=False,
call_id="mock-vlm-001",
)
class MockVLM:
"""可配置的 VLM mock — 记录调用并返回预设响应。"""
def __init__(self, response: LLMResponse) -> None:
self._response = response
self.calls: list[dict[str, Any]] = []
async def chat_with_images(
self,
messages: list[dict[str, Any]],
images: list[str | Path],
*,
session_id: str | None = None,
parent_call_id: str | None = None,
) -> LLMResponse:
"""记录调用并返回预设响应。"""
self.calls.append(
{
"messages": messages,
"images": images,
"session_id": session_id,
}
)
return self._response
def _make_tree() -> TreeIndex:
"""构造最小测试树。"""
l3_card = L3Card(
frame_summary="Person picks up a book from the shelf.",
visible_entities=["person", "book", "shelf"],
ongoing_actions=["picking up"],
visible_text=[],
spatial_layout="person in center",
visual_attributes={},
subtitle="He picks up the book.",
)
l3 = L3Node(id="L3_001", card=l3_card, frame_path="/data/frames/f001.jpg")
l3_card2 = L3Card(
frame_summary="Person starts reading the book.",
visible_entities=["person", "book"],
ongoing_actions=["reading"],
visible_text=[],
spatial_layout="person sitting",
visual_attributes={},
subtitle="Then he starts reading.",
)
l3_2 = L3Node(id="L3_002", card=l3_card2, frame_path="/data/frames/f002.jpg")
l2_card = L2Card(
event_description="A person picks up a book and starts reading.",
entities=["person", "book"],
actions=["pick up", "read"],
action_subjects=["person"],
visible_text=[],
spatial_relations="near shelf",
state_changes="book picked up",
subtitle="He picks up the book and reads.",
)
l2 = L2Node(id="L2_001", card=l2_card, children=[l3, l3_2])
l1_card = L1Card(
scene_summary="Library scene.",
main_setting="library",
key_entities=["person", "book"],
main_actions=["reading"],
topic_keywords=["library"],
visible_text=[],
temporal_flow="enter → read",
)
l1 = L1Node(id="L1_001", card=l1_card, children=[l2])
meta = IndexMeta(source_path="test.mp4", modality="video")
return TreeIndex(metadata=meta, roots=[l1])
_VALID_VLM_OUTPUT = json.dumps(
{
"question": "What does the person do after picking up the book?",
"options": ["A. Reads it", "B. Puts it back", "C. Throws it", "D. Burns it"],
"answer": "A",
"difficulty": "medium",
}
)
# ---------------------------------------------------------------------------
# TestBuildV2Prompt
# ---------------------------------------------------------------------------
class TestBuildV2Prompt:
"""验证 _build_v2_prompt 的 prompt 构造逻辑。"""
def test_includes_family_template(self) -> None:
"""prompt 中应包含家族模板的核心指令。"""
from app.question_gen.generator_v2 import _build_v2_prompt
material = _make_material()
messages, frame_paths = _build_v2_prompt(
family_spec=RETRIEVAL_FAMILY,
material=material,
task_type="Action Reasoning",
seq=1,
)
# messages 至少有 system + user
assert len(messages) >= 2
# system message 中应包含 retrieval 家族相关指令
system_content = messages[0]["content"]
assert "retrieval" in system_content.lower() or "factual" in system_content.lower()
def test_reject_reason_injected(self) -> None:
"""当 reject_reason 不为 None 时,应注入到 prompt 中。"""
from app.question_gen.generator_v2 import _build_v2_prompt
material = _make_material()
reject_msg = "The question leaked information from the answer options."
messages, _ = _build_v2_prompt(
family_spec=RETRIEVAL_FAMILY,
material=material,
task_type="Action Reasoning",
seq=2,
reject_reason=reject_msg,
)
# reject_reason 应出现在某个 message 内容中
all_content = " ".join(m["content"] for m in messages)
assert reject_msg in all_content
def test_frame_paths_from_material(self) -> None:
"""返回的 frame_paths 应来自 material.frame_paths。"""
from app.question_gen.generator_v2 import _build_v2_prompt
material = _make_material(with_frames=True)
_, frame_paths = _build_v2_prompt(
family_spec=VISUAL_FAMILY,
material=material,
task_type="Object Recognition",
seq=1,
)
assert frame_paths == ["/data/frames/f001.jpg", "/data/frames/f002.jpg"]
def test_no_frames_returns_empty(self) -> None:
"""无帧素材时 frame_paths 应为空列表。"""
from app.question_gen.generator_v2 import _build_v2_prompt
material = _make_material(with_frames=False)
_, frame_paths = _build_v2_prompt(
family_spec=RETRIEVAL_FAMILY,
material=material,
task_type="Action Reasoning",
seq=1,
)
assert frame_paths == []
# ---------------------------------------------------------------------------
# TestParseV2Response
# ---------------------------------------------------------------------------
class TestParseV2Response:
"""验证 _parse_v2_response 的解析与校验逻辑。"""
def test_valid_json(self) -> None:
"""正确 JSON → 生成 CandidateQuestion。"""
from app.question_gen.generator_v2 import CandidateQuestion, _parse_v2_response
result = _parse_v2_response(
raw=_VALID_VLM_OUTPUT,
video_id="v-001",
task_type="Action Reasoning",
skill_target="M1",
seq=3,
source_nodes=("L2_001", "L3_001"),
)
assert isinstance(result, CandidateQuestion)
assert result.question_id == "v-001_Action Reasoning_0003"
assert result.question == "What does the person do after picking up the book?"
assert result.options == ("A. Reads it", "B. Puts it back", "C. Throws it", "D. Burns it")
assert result.answer == "A"
assert result.difficulty == "medium"
assert result.video_id == "v-001"
assert result.task_type == "Action Reasoning"
assert result.skill_target == "M1"
assert result.source_nodes == ("L2_001", "L3_001")
def test_missing_field_raises(self) -> None:
"""缺少必填字段 → 抛出 ValueError。"""
from app.question_gen.generator_v2 import _parse_v2_response
incomplete = json.dumps(
{
"question": "What happens?",
"options": ["A. X", "B. Y"],
# 缺少 answer 和 difficulty
}
)
with pytest.raises(ValueError, match="answer"):
_parse_v2_response(
raw=incomplete,
video_id="v-001",
task_type="Action Reasoning",
skill_target="M1",
seq=1,
source_nodes=("L2_001",),
)
def test_invalid_answer_raises(self) -> None:
"""answer 不在 A-D 范围内 → 抛出 ValueError。"""
from app.question_gen.generator_v2 import _parse_v2_response
bad_answer = json.dumps(
{
"question": "What happens?",
"options": ["A. X", "B. Y", "C. Z", "D. W"],
"answer": "E",
"difficulty": "easy",
}
)
with pytest.raises(ValueError, match="answer"):
_parse_v2_response(
raw=bad_answer,
video_id="v-001",
task_type="Action Reasoning",
skill_target="M1",
seq=1,
source_nodes=("L2_001",),
)
def test_json_repair_handles_trailing_comma(self) -> None:
"""json_repair 应能修复常见 JSON 错误。"""
from app.question_gen.generator_v2 import _parse_v2_response
malformed = '{"question": "What?", "options": ["A. X", "B. Y", "C. Z", "D. W",], "answer": "B", "difficulty": "easy",}'
result = _parse_v2_response(
raw=malformed,
video_id="v-001",
task_type="Action Reasoning",
skill_target="M1",
seq=1,
source_nodes=("L2_001",),
)
assert result.answer == "B"
def test_markdown_wrapped_json(self) -> None:
"""被 markdown 代码块包裹的 JSON 应正常解析。"""
from app.question_gen.generator_v2 import _parse_v2_response
wrapped = f"```json\n{_VALID_VLM_OUTPUT}\n```"
result = _parse_v2_response(
raw=wrapped,
video_id="v-001",
task_type="Action Reasoning",
skill_target="M1",
seq=5,
source_nodes=("L2_001",),
)
assert result.question_id == "v-001_Action Reasoning_0005"
# ---------------------------------------------------------------------------
# TestGenerateOneV2
# ---------------------------------------------------------------------------
class TestGenerateOneV2:
"""验证 generate_one_v2 端到端流程。"""
@pytest.mark.asyncio()
async def test_happy_path(self) -> None:
"""正常流程:VLM 返回有效 JSON → 生成 CandidateQuestion。"""
from app.question_gen.generator_v2 import CandidateQuestion, generate_one_v2
mock_vlm = MockVLM(_make_vlm_response(_VALID_VLM_OUTPUT))
tree = _make_tree()
material = _make_material(with_frames=True)
result = await generate_one_v2(
vlm=mock_vlm,
tree=tree,
material=material,
family_spec=RETRIEVAL_FAMILY,
task_type="Action Reasoning",
seq=1,
video_id="v-001",
session_id="session-test-001",
)
assert isinstance(result, CandidateQuestion)
assert result.question_id == "v-001_Action Reasoning_0001"
assert result.video_id == "v-001"
assert result.skill_target == "M1"
assert result.source_nodes == ("L2_001", "L3_001", "L3_002")
assert result.subtitle_sentences == ("He picks up the book.", "Then he starts reading.")
assert result.frame_paths == ("/data/frames/f001.jpg", "/data/frames/f002.jpg")
# VLM 应被调用一次
assert len(mock_vlm.calls) == 1
assert mock_vlm.calls[0]["session_id"] == "session-test-001"
@pytest.mark.asyncio()
async def test_reject_reason_forwarded(self) -> None:
"""reject_reason 应被传入 prompt 构建。"""
from app.question_gen.generator_v2 import generate_one_v2
mock_vlm = MockVLM(_make_vlm_response(_VALID_VLM_OUTPUT))
tree = _make_tree()
material = _make_material()
await generate_one_v2(
vlm=mock_vlm,
tree=tree,
material=material,
family_spec=RETRIEVAL_FAMILY,
task_type="Action Reasoning",
seq=2,
video_id="v-001",
reject_reason="Answer was too obvious",
session_id="session-test-002",
)
# 验证 VLM 调用的 messages 中包含 reject_reason
call = mock_vlm.calls[0]
all_content = " ".join(m["content"] for m in call["messages"])
assert "Answer was too obvious" in all_content