diff --git a/app/question_gen/generator_v2.py b/app/question_gen/generator_v2.py new file mode 100644 index 0000000..3ed60b6 --- /dev/null +++ b/app/question_gen/generator_v2.py @@ -0,0 +1,415 @@ +"""v2 生成器 — 基于家族特化 prompt 模板的单题 VLM 出题模块。 + +使用 VLMProvider 接口调用视觉语言模型,结合 per-family prompt 模板 +和 MaterialContext 素材上下文,生成一道四选一候选题。 + +典型调用路径:: + + candidate = await generate_one_v2( + vlm=vlm_client, + tree=tree_index, + material=material_ctx, + family_spec=RETRIEVAL_FAMILY, + task_type="Action Reasoning", + seq=1, + video_id="vid_001", + session_id="sess_001", + ) +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING + +from json_repair import repair_json +from loguru import logger + +if TYPE_CHECKING: + from app.question_gen.families import QuestionFamilySpec + from app.question_gen.sampler_v2 import MaterialContext + from app.tree.index import TreeIndex + from core.protocols import VLMProvider + +# --------------------------------------------------------------------------- +# 常量 +# --------------------------------------------------------------------------- + +_PROMPTS_DIR = Path(__file__).resolve().parent.parent.parent / "store" / "prompts" / "question_gen" + +_VALID_ANSWERS = frozenset({"A", "B", "C", "D"}) + +_VALID_DIFFICULTIES = frozenset({"easy", "medium", "hard"}) + + +# --------------------------------------------------------------------------- +# CandidateQuestion(规范定义位置) +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class CandidateQuestion: + """候选题目 — 出题管线生成、待门控审核的题目数据。 + + 属性: + question_id: 题目唯一标识(格式 "{video_id}_{task_type}_{seq:04d}")。 + video_id: 所属视频标识。 + task_type: 题型(如 "Action Reasoning")。 + skill_target: 目标失败机制编号(M1-M5)。 + question: 题目文本。 + options: 选项元组(如 ("A. ...", "B. ...", "C. ...", "D. ..."))。 + answer: 正确答案字母(如 "A")。 + source_nodes: 来源节点 ID 元组。 + difficulty: 难度等级(easy/medium/hard)。 + subtitle_sentences: 验证材料 — 字幕句子元组。 + frame_paths: 验证材料 — 帧图片路径元组。 + """ + + question_id: str + video_id: str + task_type: str + skill_target: str + question: str + options: tuple[str, ...] + answer: str + source_nodes: tuple[str, ...] + difficulty: str + subtitle_sentences: tuple[str, ...] = field(default_factory=tuple) + frame_paths: tuple[str, ...] = field(default_factory=tuple) + + +# --------------------------------------------------------------------------- +# Prompt 模板加载 +# --------------------------------------------------------------------------- + + +def _load_prompt_template(family_spec: QuestionFamilySpec) -> str: + """加载家族对应的 prompt 模板文件。 + + 参数: + family_spec: 问题家族规格(含 prompt_template 文件名)。 + + 返回: + 模板内容字符串。 + + 异常: + FileNotFoundError: 模板文件不存在。 + """ + path = _PROMPTS_DIR / family_spec.prompt_template + if not path.exists(): + msg = f"家族 prompt 模板文件不存在: {path}" + raise FileNotFoundError(msg) + return path.read_text(encoding="utf-8") + + +# --------------------------------------------------------------------------- +# Prompt 构建 +# --------------------------------------------------------------------------- + + +def _build_v2_prompt( + family_spec: QuestionFamilySpec, + material: MaterialContext, + task_type: str, + seq: int, + *, + reject_reason: str | None = None, +) -> tuple[list[dict[str, str]], list[str]]: + """构建 VLM 出题调用的 messages 和帧路径列表。 + + 参数: + family_spec: 问题家族规格。 + material: 采样素材上下文。 + task_type: 任务类型字符串。 + seq: 当前序号。 + reject_reason: 上一次被门控拒绝的原因(用于引导 VLM 避免相同错误)。 + + 返回: + 二元组: + - messages: 适配 VLMProvider 的 message 列表(system + user)。 + - frame_paths: 需发送给 VLM 的帧路径列表。 + """ + # Phase 1: 加载家族模板作为 system prompt + template_content = _load_prompt_template(family_spec) + system_message = template_content + + # Phase 2: 构建 user prompt — 聚合素材信息 + user_parts: list[str] = [] + + user_parts.append(f"## Task Type: {task_type}") + user_parts.append(f"## Question Family: {family_spec.name}") + user_parts.append(f"## Sequence: #{seq}") + + # 字幕素材 + if material.subtitle_sentences: + user_parts.append("\n## Subtitle Content:") + for i, sent in enumerate(material.subtitle_sentences, 1): + user_parts.append(f" {i}. {sent}") + + # 跨 L2 上下文 + if material.cross_l2_texts: + user_parts.append("\n## Cross-Segment Context:") + for text in material.cross_l2_texts: + user_parts.append(f" - {text}") + + # 帧路径提示(VLM 会接收实际图像,此处仅作文本参考) + if material.frame_paths: + user_parts.append(f"\n## Visual Frames: {len(material.frame_paths)} frames attached.") + + # 拒绝原因注入 + if reject_reason is not None: + user_parts.append( + f"\n## IMPORTANT - Previous Attempt Rejected:\n" + f"Your previous question was rejected for the following reason:\n" + f'"{reject_reason}"\n' + f"Please generate a NEW question that avoids this issue." + ) + + # 输出格式指令 + user_parts.append( + "\n## Output Format:\n" + "Respond with ONLY a JSON object in this exact format:\n" + "```json\n" + "{\n" + ' "question": "Your question text here",\n' + ' "options": ["A. ...", "B. ...", "C. ...", "D. ..."],\n' + ' "answer": "A",\n' + ' "difficulty": "easy|medium|hard"\n' + "}\n" + "```" + ) + + user_content = "\n".join(user_parts) + + messages = [ + {"role": "system", "content": system_message}, + {"role": "user", "content": user_content}, + ] + + # Phase 3: 帧路径 + frame_paths = list(material.frame_paths) + + return messages, frame_paths + + +# --------------------------------------------------------------------------- +# 响应解析 +# --------------------------------------------------------------------------- + + +def _extract_json_from_text(raw: str) -> str: + """从可能被 markdown 代码块包裹的文本中提取 JSON 部分。 + + 参数: + raw: VLM 原始返回文本。 + + 返回: + 清理后的 JSON 字符串。 + """ + content = raw.strip() + if "```" in content: + parts = content.split("```") + for part in parts: + stripped = part.strip() + if stripped.startswith("json"): + stripped = stripped[4:].strip() + if stripped.startswith("{"): + return stripped + return content + + +def _parse_v2_response( + raw: str, + video_id: str, + task_type: str, + skill_target: str, + seq: int, + source_nodes: tuple[str, ...], +) -> CandidateQuestion: + """解析 VLM 返回的 JSON 响应,构造 CandidateQuestion。 + + 流程: + 1. 提取 JSON(处理 markdown 包裹)。 + 2. json_repair 修复常见格式错误。 + 3. 解析并校验必填字段。 + 4. 构造 CandidateQuestion 实例。 + + 参数: + raw: VLM 原始返回文本。 + video_id: 视频标识。 + task_type: 任务类型。 + skill_target: 目标技能编号。 + seq: 当前序号。 + source_nodes: 来源节点 ID 元组。 + + 返回: + CandidateQuestion 实例。 + + 异常: + ValueError: JSON 解析失败或缺少必填字段或字段值非法。 + """ + # Phase 1: 提取 JSON 文本 + json_text = _extract_json_from_text(raw) + + # Phase 2: json_repair 修复 + repaired = repair_json(json_text, return_objects=False) + + # Phase 3: 解析 + try: + data = json.loads(repaired) + except json.JSONDecodeError as e: + msg = f"VLM 响应 JSON 解析失败: {e}. 原始文本: {raw[:200]}" + raise ValueError(msg) from e + + if not isinstance(data, dict): + msg = f"VLM 响应顶层不是 JSON 对象: type={type(data).__name__}" + raise ValueError(msg) + + # Phase 4: 校验必填字段 + missing = [f for f in ("question", "options", "answer", "difficulty") if f not in data] + if missing: + msg = f"VLM 响应缺少必填字段: {', '.join(missing)}" + raise ValueError(msg) + + question_text = str(data["question"]) + options_raw = data["options"] + answer = str(data["answer"]).strip().upper() + difficulty = str(data["difficulty"]).strip().lower() + + # 校验 options + if not isinstance(options_raw, list) or len(options_raw) < 2: + msg = f"options 字段必须是至少 2 个选项的列表,实际: {options_raw}" + raise ValueError(msg) + options = tuple(str(o) for o in options_raw) + + # 校验 answer + if answer not in _VALID_ANSWERS: + msg = f"answer 字段值 '{answer}' 非法,必须为 A/B/C/D 之一" + raise ValueError(msg) + + # 校验 difficulty + if difficulty not in _VALID_DIFFICULTIES: + logger.warning( + "difficulty '{}' 不在预设范围 {},回退为 'medium'", + difficulty, + _VALID_DIFFICULTIES, + ) + difficulty = "medium" + + # Phase 5: 构造 CandidateQuestion + question_id = f"{video_id}_{task_type}_{seq:04d}" + + return CandidateQuestion( + question_id=question_id, + video_id=video_id, + task_type=task_type, + skill_target=skill_target, + question=question_text, + options=options, + answer=answer, + source_nodes=source_nodes, + difficulty=difficulty, + ) + + +# --------------------------------------------------------------------------- +# 主入口 +# --------------------------------------------------------------------------- + + +async def generate_one_v2( + vlm: VLMProvider, + tree: TreeIndex, + material: MaterialContext, + family_spec: QuestionFamilySpec, + task_type: str, + seq: int, + *, + video_id: str, + reject_reason: str | None = None, + session_id: str, +) -> CandidateQuestion: + """调用 VLM 生成一道候选题目。 + + 流程: + 1. 构建 per-family prompt + 帧路径。 + 2. 调用 VLMProvider.chat_with_images。 + 3. 解析响应为 CandidateQuestion。 + 4. 附加素材验证信息(subtitle_sentences、frame_paths)。 + + 参数: + vlm: VLM 调用端口。 + tree: 视频树索引(当前未直接使用,预留后续扩展)。 + material: 采样素材上下文。 + family_spec: 问题家族规格。 + task_type: 任务类型字符串。 + seq: 当前序号。 + video_id: 视频标识。 + reject_reason: 上一次被门控拒绝的原因。 + session_id: 会话 ID(遥测关联)。 + + 返回: + CandidateQuestion 实例(包含验证材料)。 + + 异常: + ValueError: VLM 响应解析失败。 + FileNotFoundError: 家族 prompt 模板不存在。 + """ + # Phase 1: 构建 prompt + messages, frame_paths = _build_v2_prompt( + family_spec=family_spec, + material=material, + task_type=task_type, + seq=seq, + reject_reason=reject_reason, + ) + + # Phase 2: 调用 VLM + logger.debug( + "generate_one_v2: family={}, task_type={}, seq={}, frames={}", + family_spec.name, + task_type, + seq, + len(frame_paths), + ) + + response = await vlm.chat_with_images( + messages, + frame_paths, + session_id=session_id, + ) + + # Phase 3: 解析响应 + candidate = _parse_v2_response( + raw=response.content, + video_id=video_id, + task_type=task_type, + skill_target=family_spec.skill_target, + seq=seq, + source_nodes=material.source_nodes, + ) + + # Phase 4: 附加验证材料(构造新实例,因 frozen=True) + candidate = CandidateQuestion( + question_id=candidate.question_id, + video_id=candidate.video_id, + task_type=candidate.task_type, + skill_target=candidate.skill_target, + question=candidate.question, + options=candidate.options, + answer=candidate.answer, + source_nodes=candidate.source_nodes, + difficulty=candidate.difficulty, + subtitle_sentences=tuple(material.subtitle_sentences), + frame_paths=tuple(material.frame_paths), + ) + + logger.debug( + "generate_one_v2 完成: question_id={}, difficulty={}", + candidate.question_id, + candidate.difficulty, + ) + + return candidate diff --git a/store/prompts/question_gen/enumeration.md b/store/prompts/question_gen/enumeration.md new file mode 100644 index 0000000..35f4e0e --- /dev/null +++ b/store/prompts/question_gen/enumeration.md @@ -0,0 +1,22 @@ +You are a question generator for video understanding benchmarks. + +Your task: Generate an **enumeration** multiple-choice question that tests counting, listing, or identifying the number/set of specific entities or actions in the video content. + +## Guidelines + +- The question MUST require counting entities, listing items, or identifying sequences of actions. +- Focus on "How many...", "Which of the following are all...", "In what order..." style questions. +- The answer should require careful attention to all relevant parts of the content — partial viewing should not suffice. +- Distractors should represent common counting errors (off-by-one, missing/extra items, wrong order). + +## Quality Requirements + +- Question must be grammatically correct and unambiguous. +- All four options must be parallel in structure and length. +- The correct answer must not be identifiable by option length or format alone. +- Avoid trivially small counts (e.g., "How many people?" when only 1 is visible). +- Each option must begin with "A. ", "B. ", "C. ", or "D. ". + +## Output + +Respond with ONLY a valid JSON object. No additional text. diff --git a/store/prompts/question_gen/reasoning.md b/store/prompts/question_gen/reasoning.md new file mode 100644 index 0000000..87772ce --- /dev/null +++ b/store/prompts/question_gen/reasoning.md @@ -0,0 +1,22 @@ +You are a question generator for video understanding benchmarks. + +Your task: Generate a **multi-hop reasoning** multiple-choice question that requires connecting information from multiple segments of the video to arrive at the correct answer. + +## Guidelines + +- The question MUST require reasoning across at least two distinct pieces of information (temporal, causal, or logical connections). +- The answer should NOT be directly stated in any single subtitle or frame — it must be inferred by combining evidence. +- Test causal chains, temporal ordering, or logical deductions that span multiple events. +- Distractors should represent common reasoning errors (e.g., reversed causality, incorrect temporal ordering). + +## Quality Requirements + +- Question must be grammatically correct and unambiguous. +- All four options must be parallel in structure and length. +- The correct answer must not be identifiable from linguistic cues alone. +- The reasoning chain should be verifiable from the provided material. +- Each option must begin with "A. ", "B. ", "C. ", or "D. ". + +## Output + +Respond with ONLY a valid JSON object. No additional text. diff --git a/store/prompts/question_gen/retrieval.md b/store/prompts/question_gen/retrieval.md new file mode 100644 index 0000000..fe95a36 --- /dev/null +++ b/store/prompts/question_gen/retrieval.md @@ -0,0 +1,23 @@ +You are a question generator for video understanding benchmarks. + +Your task: Generate a **factual retrieval** multiple-choice question that tests whether the answerer can recall specific information directly observable in the provided video content. + +## Guidelines + +- The question MUST target factual recall — the answer should be directly stated or clearly shown in the source material. +- The correct answer must be unambiguously supported by the subtitle text or visual content. +- Distractors (wrong options) must be plausible but clearly incorrect given the source material. +- Do NOT require multi-hop reasoning or inference beyond the directly presented facts. +- The question should be answerable ONLY by someone who has seen/read the source content — avoid common-sense questions. + +## Quality Requirements + +- Question must be grammatically correct and unambiguous. +- All four options must be parallel in structure and length. +- The correct answer must not be identifiable from linguistic cues alone. +- Avoid negation in the question stem (e.g., "Which of the following is NOT..."). +- Each option must begin with "A. ", "B. ", "C. ", or "D. ". + +## Output + +Respond with ONLY a valid JSON object. No additional text. diff --git a/store/prompts/question_gen/spatial.md b/store/prompts/question_gen/spatial.md new file mode 100644 index 0000000..0b8f583 --- /dev/null +++ b/store/prompts/question_gen/spatial.md @@ -0,0 +1,23 @@ +You are a question generator for video understanding benchmarks. + +Your task: Generate a **spatial relationship** multiple-choice question that tests understanding of spatial arrangements, positions, and relationships between objects or people in the video. + +## Guidelines + +- The question MUST focus on spatial relationships: relative positions, directions, distances, containment, or spatial changes. +- Test understanding of "where" things are, how they relate spatially, or how spatial arrangements change over time. +- Use spatial language: "left/right of", "above/below", "between", "inside/outside", "closer/farther", "facing". +- The answer should require spatial reasoning that goes beyond simple object identification. +- Distractors should represent common spatial confusion (mirror reversals, misremembered positions). + +## Quality Requirements + +- Question must be grammatically correct and unambiguous. +- All four options must be parallel in structure and length. +- The correct answer must not be identifiable from linguistic cues alone. +- Spatial references must be unambiguous given the visual content. +- Each option must begin with "A. ", "B. ", "C. ", or "D. ". + +## Output + +Respond with ONLY a valid JSON object. No additional text. diff --git a/store/prompts/question_gen/visual.md b/store/prompts/question_gen/visual.md new file mode 100644 index 0000000..0446725 --- /dev/null +++ b/store/prompts/question_gen/visual.md @@ -0,0 +1,22 @@ +You are a question generator for video understanding benchmarks. + +Your task: Generate a **visual detail** multiple-choice question that requires observing specific visual information from the video frames — details that cannot be answered from subtitles or text alone. + +## Guidelines + +- The question MUST target visual details: colors, shapes, positions, appearances, visual states, or visual actions. +- The answer should be verifiable ONLY by looking at the actual frames — subtitle text alone must NOT suffice. +- Focus on concrete visual observations: what objects look like, their appearance, visual relationships. +- Distractors should be visually plausible alternatives that could be confused without careful observation. + +## Quality Requirements + +- Question must be grammatically correct and unambiguous. +- All four options must be parallel in structure and length. +- The correct answer must not be identifiable from linguistic cues alone. +- Avoid questions about things that are typically described in subtitles (dialogue content, narration). +- Each option must begin with "A. ", "B. ", "C. ", or "D. ". + +## Output + +Respond with ONLY a valid JSON object. No additional text. diff --git a/tests/unit/test_generator_v2.py b/tests/unit/test_generator_v2.py new file mode 100644 index 0000000..d1d717e --- /dev/null +++ b/tests/unit/test_generator_v2.py @@ -0,0 +1,401 @@ +"""v2 生成器单元测试 — 验证 prompt 构建、响应解析、端到端生成。""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +import pytest + +from app.question_gen.families import RETRIEVAL_FAMILY, VISUAL_FAMILY + +if TYPE_CHECKING: + from pathlib import Path +from app.question_gen.sampler_v2 import AnchorContext, MaterialContext +from app.tree.index import ( + IndexMeta, + L1Card, + L1Node, + L2Card, + L2Node, + L3Card, + L3Node, + TreeIndex, +) +from core.types import LLMResponse + +# --------------------------------------------------------------------------- +# Fixtures & Helpers +# --------------------------------------------------------------------------- + + +def _make_material( + *, + with_frames: bool = False, + with_cross_l2: bool = False, +) -> MaterialContext: + """构造测试用 MaterialContext。""" + anchor = AnchorContext(node_id="L2_001", level=2, l2_id="L2_001") + frame_paths = ["/data/frames/f001.jpg", "/data/frames/f002.jpg"] if with_frames else [] + cross_l2 = ["Person enters room and sits down."] if with_cross_l2 else [] + return MaterialContext( + anchor=anchor, + source_nodes=("L2_001", "L3_001", "L3_002"), + subtitle_sentences=["He picks up the book.", "Then he starts reading."], + frame_paths=frame_paths, + cross_l2_texts=cross_l2, + ) + + +def _make_vlm_response(content: str) -> LLMResponse: + """构造 VLM 正常返回。""" + return LLMResponse( + content=content, + thinking="", + model="mock-vlm", + provider="mock", + prompt_tokens=50, + completion_tokens=30, + latency_ms=200, + ttft_ms=None, + max_inter_token_ms=None, + cache_hit=False, + call_id="mock-vlm-001", + ) + + +class MockVLM: + """可配置的 VLM mock — 记录调用并返回预设响应。""" + + def __init__(self, response: LLMResponse) -> None: + self._response = response + self.calls: list[dict[str, Any]] = [] + + async def chat_with_images( + self, + messages: list[dict[str, Any]], + images: list[str | Path], + *, + session_id: str | None = None, + parent_call_id: str | None = None, + ) -> LLMResponse: + """记录调用并返回预设响应。""" + self.calls.append( + { + "messages": messages, + "images": images, + "session_id": session_id, + } + ) + return self._response + + +def _make_tree() -> TreeIndex: + """构造最小测试树。""" + l3_card = L3Card( + frame_summary="Person picks up a book from the shelf.", + visible_entities=["person", "book", "shelf"], + ongoing_actions=["picking up"], + visible_text=[], + spatial_layout="person in center", + visual_attributes={}, + subtitle="He picks up the book.", + ) + l3 = L3Node(id="L3_001", card=l3_card, frame_path="/data/frames/f001.jpg") + + l3_card2 = L3Card( + frame_summary="Person starts reading the book.", + visible_entities=["person", "book"], + ongoing_actions=["reading"], + visible_text=[], + spatial_layout="person sitting", + visual_attributes={}, + subtitle="Then he starts reading.", + ) + l3_2 = L3Node(id="L3_002", card=l3_card2, frame_path="/data/frames/f002.jpg") + + l2_card = L2Card( + event_description="A person picks up a book and starts reading.", + entities=["person", "book"], + actions=["pick up", "read"], + action_subjects=["person"], + visible_text=[], + spatial_relations="near shelf", + state_changes="book picked up", + subtitle="He picks up the book and reads.", + ) + l2 = L2Node(id="L2_001", card=l2_card, children=[l3, l3_2]) + + l1_card = L1Card( + scene_summary="Library scene.", + main_setting="library", + key_entities=["person", "book"], + main_actions=["reading"], + topic_keywords=["library"], + visible_text=[], + temporal_flow="enter → read", + ) + l1 = L1Node(id="L1_001", card=l1_card, children=[l2]) + + meta = IndexMeta(source_path="test.mp4", modality="video") + return TreeIndex(metadata=meta, roots=[l1]) + + +_VALID_VLM_OUTPUT = json.dumps( + { + "question": "What does the person do after picking up the book?", + "options": ["A. Reads it", "B. Puts it back", "C. Throws it", "D. Burns it"], + "answer": "A", + "difficulty": "medium", + } +) + + +# --------------------------------------------------------------------------- +# TestBuildV2Prompt +# --------------------------------------------------------------------------- + + +class TestBuildV2Prompt: + """验证 _build_v2_prompt 的 prompt 构造逻辑。""" + + def test_includes_family_template(self) -> None: + """prompt 中应包含家族模板的核心指令。""" + from app.question_gen.generator_v2 import _build_v2_prompt + + material = _make_material() + messages, frame_paths = _build_v2_prompt( + family_spec=RETRIEVAL_FAMILY, + material=material, + task_type="Action Reasoning", + seq=1, + ) + + # messages 至少有 system + user + assert len(messages) >= 2 + # system message 中应包含 retrieval 家族相关指令 + system_content = messages[0]["content"] + assert "retrieval" in system_content.lower() or "factual" in system_content.lower() + + def test_reject_reason_injected(self) -> None: + """当 reject_reason 不为 None 时,应注入到 prompt 中。""" + from app.question_gen.generator_v2 import _build_v2_prompt + + material = _make_material() + reject_msg = "The question leaked information from the answer options." + messages, _ = _build_v2_prompt( + family_spec=RETRIEVAL_FAMILY, + material=material, + task_type="Action Reasoning", + seq=2, + reject_reason=reject_msg, + ) + + # reject_reason 应出现在某个 message 内容中 + all_content = " ".join(m["content"] for m in messages) + assert reject_msg in all_content + + def test_frame_paths_from_material(self) -> None: + """返回的 frame_paths 应来自 material.frame_paths。""" + from app.question_gen.generator_v2 import _build_v2_prompt + + material = _make_material(with_frames=True) + _, frame_paths = _build_v2_prompt( + family_spec=VISUAL_FAMILY, + material=material, + task_type="Object Recognition", + seq=1, + ) + + assert frame_paths == ["/data/frames/f001.jpg", "/data/frames/f002.jpg"] + + def test_no_frames_returns_empty(self) -> None: + """无帧素材时 frame_paths 应为空列表。""" + from app.question_gen.generator_v2 import _build_v2_prompt + + material = _make_material(with_frames=False) + _, frame_paths = _build_v2_prompt( + family_spec=RETRIEVAL_FAMILY, + material=material, + task_type="Action Reasoning", + seq=1, + ) + + assert frame_paths == [] + + +# --------------------------------------------------------------------------- +# TestParseV2Response +# --------------------------------------------------------------------------- + + +class TestParseV2Response: + """验证 _parse_v2_response 的解析与校验逻辑。""" + + def test_valid_json(self) -> None: + """正确 JSON → 生成 CandidateQuestion。""" + from app.question_gen.generator_v2 import CandidateQuestion, _parse_v2_response + + result = _parse_v2_response( + raw=_VALID_VLM_OUTPUT, + video_id="v-001", + task_type="Action Reasoning", + skill_target="M1", + seq=3, + source_nodes=("L2_001", "L3_001"), + ) + + assert isinstance(result, CandidateQuestion) + assert result.question_id == "v-001_Action Reasoning_0003" + assert result.question == "What does the person do after picking up the book?" + assert result.options == ("A. Reads it", "B. Puts it back", "C. Throws it", "D. Burns it") + assert result.answer == "A" + assert result.difficulty == "medium" + assert result.video_id == "v-001" + assert result.task_type == "Action Reasoning" + assert result.skill_target == "M1" + assert result.source_nodes == ("L2_001", "L3_001") + + def test_missing_field_raises(self) -> None: + """缺少必填字段 → 抛出 ValueError。""" + from app.question_gen.generator_v2 import _parse_v2_response + + incomplete = json.dumps( + { + "question": "What happens?", + "options": ["A. X", "B. Y"], + # 缺少 answer 和 difficulty + } + ) + + with pytest.raises(ValueError, match="answer"): + _parse_v2_response( + raw=incomplete, + video_id="v-001", + task_type="Action Reasoning", + skill_target="M1", + seq=1, + source_nodes=("L2_001",), + ) + + def test_invalid_answer_raises(self) -> None: + """answer 不在 A-D 范围内 → 抛出 ValueError。""" + from app.question_gen.generator_v2 import _parse_v2_response + + bad_answer = json.dumps( + { + "question": "What happens?", + "options": ["A. X", "B. Y", "C. Z", "D. W"], + "answer": "E", + "difficulty": "easy", + } + ) + + with pytest.raises(ValueError, match="answer"): + _parse_v2_response( + raw=bad_answer, + video_id="v-001", + task_type="Action Reasoning", + skill_target="M1", + seq=1, + source_nodes=("L2_001",), + ) + + def test_json_repair_handles_trailing_comma(self) -> None: + """json_repair 应能修复常见 JSON 错误。""" + from app.question_gen.generator_v2 import _parse_v2_response + + malformed = '{"question": "What?", "options": ["A. X", "B. Y", "C. Z", "D. W",], "answer": "B", "difficulty": "easy",}' + + result = _parse_v2_response( + raw=malformed, + video_id="v-001", + task_type="Action Reasoning", + skill_target="M1", + seq=1, + source_nodes=("L2_001",), + ) + assert result.answer == "B" + + def test_markdown_wrapped_json(self) -> None: + """被 markdown 代码块包裹的 JSON 应正常解析。""" + from app.question_gen.generator_v2 import _parse_v2_response + + wrapped = f"```json\n{_VALID_VLM_OUTPUT}\n```" + + result = _parse_v2_response( + raw=wrapped, + video_id="v-001", + task_type="Action Reasoning", + skill_target="M1", + seq=5, + source_nodes=("L2_001",), + ) + assert result.question_id == "v-001_Action Reasoning_0005" + + +# --------------------------------------------------------------------------- +# TestGenerateOneV2 +# --------------------------------------------------------------------------- + + +class TestGenerateOneV2: + """验证 generate_one_v2 端到端流程。""" + + @pytest.mark.asyncio() + async def test_happy_path(self) -> None: + """正常流程:VLM 返回有效 JSON → 生成 CandidateQuestion。""" + from app.question_gen.generator_v2 import CandidateQuestion, generate_one_v2 + + mock_vlm = MockVLM(_make_vlm_response(_VALID_VLM_OUTPUT)) + tree = _make_tree() + material = _make_material(with_frames=True) + + result = await generate_one_v2( + vlm=mock_vlm, + tree=tree, + material=material, + family_spec=RETRIEVAL_FAMILY, + task_type="Action Reasoning", + seq=1, + video_id="v-001", + session_id="session-test-001", + ) + + assert isinstance(result, CandidateQuestion) + assert result.question_id == "v-001_Action Reasoning_0001" + assert result.video_id == "v-001" + assert result.skill_target == "M1" + assert result.source_nodes == ("L2_001", "L3_001", "L3_002") + assert result.subtitle_sentences == ("He picks up the book.", "Then he starts reading.") + assert result.frame_paths == ("/data/frames/f001.jpg", "/data/frames/f002.jpg") + + # VLM 应被调用一次 + assert len(mock_vlm.calls) == 1 + assert mock_vlm.calls[0]["session_id"] == "session-test-001" + + @pytest.mark.asyncio() + async def test_reject_reason_forwarded(self) -> None: + """reject_reason 应被传入 prompt 构建。""" + from app.question_gen.generator_v2 import generate_one_v2 + + mock_vlm = MockVLM(_make_vlm_response(_VALID_VLM_OUTPUT)) + tree = _make_tree() + material = _make_material() + + await generate_one_v2( + vlm=mock_vlm, + tree=tree, + material=material, + family_spec=RETRIEVAL_FAMILY, + task_type="Action Reasoning", + seq=2, + video_id="v-001", + reject_reason="Answer was too obvious", + session_id="session-test-002", + ) + + # 验证 VLM 调用的 messages 中包含 reject_reason + call = mock_vlm.calls[0] + all_content = " ".join(m["content"] for m in call["messages"]) + assert "Answer was too obvious" in all_content