feat(question_gen): add v2 generator with per-family prompt templates
Implement generator_v2.py with: - CandidateQuestion dataclass (canonical location) - _load_prompt_template: loads per-family .md from store/prompts/ - _build_v2_prompt: constructs system+user messages with material context - _parse_v2_response: JSON extraction, json_repair, field validation - generate_one_v2: async VLM call orchestration with reject_reason support Add 5 family-specific prompt templates: - retrieval.md: factual recall from visible content - reasoning.md: multi-hop inference across segments - enumeration.md: counting/listing entities and actions - visual.md: visual details requiring frame observation - spatial.md: spatial relationships between objects/people Tests: 11 unit tests covering prompt build, parse, and e2e generation. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,415 @@
|
|||||||
|
"""v2 生成器 — 基于家族特化 prompt 模板的单题 VLM 出题模块。
|
||||||
|
|
||||||
|
使用 VLMProvider 接口调用视觉语言模型,结合 per-family prompt 模板
|
||||||
|
和 MaterialContext 素材上下文,生成一道四选一候选题。
|
||||||
|
|
||||||
|
典型调用路径::
|
||||||
|
|
||||||
|
candidate = await generate_one_v2(
|
||||||
|
vlm=vlm_client,
|
||||||
|
tree=tree_index,
|
||||||
|
material=material_ctx,
|
||||||
|
family_spec=RETRIEVAL_FAMILY,
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
seq=1,
|
||||||
|
video_id="vid_001",
|
||||||
|
session_id="sess_001",
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from json_repair import repair_json
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from app.question_gen.families import QuestionFamilySpec
|
||||||
|
from app.question_gen.sampler_v2 import MaterialContext
|
||||||
|
from app.tree.index import TreeIndex
|
||||||
|
from core.protocols import VLMProvider
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 常量
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_PROMPTS_DIR = Path(__file__).resolve().parent.parent.parent / "store" / "prompts" / "question_gen"
|
||||||
|
|
||||||
|
_VALID_ANSWERS = frozenset({"A", "B", "C", "D"})
|
||||||
|
|
||||||
|
_VALID_DIFFICULTIES = frozenset({"easy", "medium", "hard"})
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# CandidateQuestion(规范定义位置)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class CandidateQuestion:
|
||||||
|
"""候选题目 — 出题管线生成、待门控审核的题目数据。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
question_id: 题目唯一标识(格式 "{video_id}_{task_type}_{seq:04d}")。
|
||||||
|
video_id: 所属视频标识。
|
||||||
|
task_type: 题型(如 "Action Reasoning")。
|
||||||
|
skill_target: 目标失败机制编号(M1-M5)。
|
||||||
|
question: 题目文本。
|
||||||
|
options: 选项元组(如 ("A. ...", "B. ...", "C. ...", "D. ..."))。
|
||||||
|
answer: 正确答案字母(如 "A")。
|
||||||
|
source_nodes: 来源节点 ID 元组。
|
||||||
|
difficulty: 难度等级(easy/medium/hard)。
|
||||||
|
subtitle_sentences: 验证材料 — 字幕句子元组。
|
||||||
|
frame_paths: 验证材料 — 帧图片路径元组。
|
||||||
|
"""
|
||||||
|
|
||||||
|
question_id: str
|
||||||
|
video_id: str
|
||||||
|
task_type: str
|
||||||
|
skill_target: str
|
||||||
|
question: str
|
||||||
|
options: tuple[str, ...]
|
||||||
|
answer: str
|
||||||
|
source_nodes: tuple[str, ...]
|
||||||
|
difficulty: str
|
||||||
|
subtitle_sentences: tuple[str, ...] = field(default_factory=tuple)
|
||||||
|
frame_paths: tuple[str, ...] = field(default_factory=tuple)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Prompt 模板加载
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _load_prompt_template(family_spec: QuestionFamilySpec) -> str:
|
||||||
|
"""加载家族对应的 prompt 模板文件。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
family_spec: 问题家族规格(含 prompt_template 文件名)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
模板内容字符串。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileNotFoundError: 模板文件不存在。
|
||||||
|
"""
|
||||||
|
path = _PROMPTS_DIR / family_spec.prompt_template
|
||||||
|
if not path.exists():
|
||||||
|
msg = f"家族 prompt 模板文件不存在: {path}"
|
||||||
|
raise FileNotFoundError(msg)
|
||||||
|
return path.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Prompt 构建
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _build_v2_prompt(
|
||||||
|
family_spec: QuestionFamilySpec,
|
||||||
|
material: MaterialContext,
|
||||||
|
task_type: str,
|
||||||
|
seq: int,
|
||||||
|
*,
|
||||||
|
reject_reason: str | None = None,
|
||||||
|
) -> tuple[list[dict[str, str]], list[str]]:
|
||||||
|
"""构建 VLM 出题调用的 messages 和帧路径列表。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
family_spec: 问题家族规格。
|
||||||
|
material: 采样素材上下文。
|
||||||
|
task_type: 任务类型字符串。
|
||||||
|
seq: 当前序号。
|
||||||
|
reject_reason: 上一次被门控拒绝的原因(用于引导 VLM 避免相同错误)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
二元组:
|
||||||
|
- messages: 适配 VLMProvider 的 message 列表(system + user)。
|
||||||
|
- frame_paths: 需发送给 VLM 的帧路径列表。
|
||||||
|
"""
|
||||||
|
# Phase 1: 加载家族模板作为 system prompt
|
||||||
|
template_content = _load_prompt_template(family_spec)
|
||||||
|
system_message = template_content
|
||||||
|
|
||||||
|
# Phase 2: 构建 user prompt — 聚合素材信息
|
||||||
|
user_parts: list[str] = []
|
||||||
|
|
||||||
|
user_parts.append(f"## Task Type: {task_type}")
|
||||||
|
user_parts.append(f"## Question Family: {family_spec.name}")
|
||||||
|
user_parts.append(f"## Sequence: #{seq}")
|
||||||
|
|
||||||
|
# 字幕素材
|
||||||
|
if material.subtitle_sentences:
|
||||||
|
user_parts.append("\n## Subtitle Content:")
|
||||||
|
for i, sent in enumerate(material.subtitle_sentences, 1):
|
||||||
|
user_parts.append(f" {i}. {sent}")
|
||||||
|
|
||||||
|
# 跨 L2 上下文
|
||||||
|
if material.cross_l2_texts:
|
||||||
|
user_parts.append("\n## Cross-Segment Context:")
|
||||||
|
for text in material.cross_l2_texts:
|
||||||
|
user_parts.append(f" - {text}")
|
||||||
|
|
||||||
|
# 帧路径提示(VLM 会接收实际图像,此处仅作文本参考)
|
||||||
|
if material.frame_paths:
|
||||||
|
user_parts.append(f"\n## Visual Frames: {len(material.frame_paths)} frames attached.")
|
||||||
|
|
||||||
|
# 拒绝原因注入
|
||||||
|
if reject_reason is not None:
|
||||||
|
user_parts.append(
|
||||||
|
f"\n## IMPORTANT - Previous Attempt Rejected:\n"
|
||||||
|
f"Your previous question was rejected for the following reason:\n"
|
||||||
|
f'"{reject_reason}"\n'
|
||||||
|
f"Please generate a NEW question that avoids this issue."
|
||||||
|
)
|
||||||
|
|
||||||
|
# 输出格式指令
|
||||||
|
user_parts.append(
|
||||||
|
"\n## Output Format:\n"
|
||||||
|
"Respond with ONLY a JSON object in this exact format:\n"
|
||||||
|
"```json\n"
|
||||||
|
"{\n"
|
||||||
|
' "question": "Your question text here",\n'
|
||||||
|
' "options": ["A. ...", "B. ...", "C. ...", "D. ..."],\n'
|
||||||
|
' "answer": "A",\n'
|
||||||
|
' "difficulty": "easy|medium|hard"\n'
|
||||||
|
"}\n"
|
||||||
|
"```"
|
||||||
|
)
|
||||||
|
|
||||||
|
user_content = "\n".join(user_parts)
|
||||||
|
|
||||||
|
messages = [
|
||||||
|
{"role": "system", "content": system_message},
|
||||||
|
{"role": "user", "content": user_content},
|
||||||
|
]
|
||||||
|
|
||||||
|
# Phase 3: 帧路径
|
||||||
|
frame_paths = list(material.frame_paths)
|
||||||
|
|
||||||
|
return messages, frame_paths
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 响应解析
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_json_from_text(raw: str) -> str:
|
||||||
|
"""从可能被 markdown 代码块包裹的文本中提取 JSON 部分。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
raw: VLM 原始返回文本。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
清理后的 JSON 字符串。
|
||||||
|
"""
|
||||||
|
content = raw.strip()
|
||||||
|
if "```" in content:
|
||||||
|
parts = content.split("```")
|
||||||
|
for part in parts:
|
||||||
|
stripped = part.strip()
|
||||||
|
if stripped.startswith("json"):
|
||||||
|
stripped = stripped[4:].strip()
|
||||||
|
if stripped.startswith("{"):
|
||||||
|
return stripped
|
||||||
|
return content
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_v2_response(
|
||||||
|
raw: str,
|
||||||
|
video_id: str,
|
||||||
|
task_type: str,
|
||||||
|
skill_target: str,
|
||||||
|
seq: int,
|
||||||
|
source_nodes: tuple[str, ...],
|
||||||
|
) -> CandidateQuestion:
|
||||||
|
"""解析 VLM 返回的 JSON 响应,构造 CandidateQuestion。
|
||||||
|
|
||||||
|
流程:
|
||||||
|
1. 提取 JSON(处理 markdown 包裹)。
|
||||||
|
2. json_repair 修复常见格式错误。
|
||||||
|
3. 解析并校验必填字段。
|
||||||
|
4. 构造 CandidateQuestion 实例。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
raw: VLM 原始返回文本。
|
||||||
|
video_id: 视频标识。
|
||||||
|
task_type: 任务类型。
|
||||||
|
skill_target: 目标技能编号。
|
||||||
|
seq: 当前序号。
|
||||||
|
source_nodes: 来源节点 ID 元组。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
CandidateQuestion 实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: JSON 解析失败或缺少必填字段或字段值非法。
|
||||||
|
"""
|
||||||
|
# Phase 1: 提取 JSON 文本
|
||||||
|
json_text = _extract_json_from_text(raw)
|
||||||
|
|
||||||
|
# Phase 2: json_repair 修复
|
||||||
|
repaired = repair_json(json_text, return_objects=False)
|
||||||
|
|
||||||
|
# Phase 3: 解析
|
||||||
|
try:
|
||||||
|
data = json.loads(repaired)
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
msg = f"VLM 响应 JSON 解析失败: {e}. 原始文本: {raw[:200]}"
|
||||||
|
raise ValueError(msg) from e
|
||||||
|
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
msg = f"VLM 响应顶层不是 JSON 对象: type={type(data).__name__}"
|
||||||
|
raise ValueError(msg)
|
||||||
|
|
||||||
|
# Phase 4: 校验必填字段
|
||||||
|
missing = [f for f in ("question", "options", "answer", "difficulty") if f not in data]
|
||||||
|
if missing:
|
||||||
|
msg = f"VLM 响应缺少必填字段: {', '.join(missing)}"
|
||||||
|
raise ValueError(msg)
|
||||||
|
|
||||||
|
question_text = str(data["question"])
|
||||||
|
options_raw = data["options"]
|
||||||
|
answer = str(data["answer"]).strip().upper()
|
||||||
|
difficulty = str(data["difficulty"]).strip().lower()
|
||||||
|
|
||||||
|
# 校验 options
|
||||||
|
if not isinstance(options_raw, list) or len(options_raw) < 2:
|
||||||
|
msg = f"options 字段必须是至少 2 个选项的列表,实际: {options_raw}"
|
||||||
|
raise ValueError(msg)
|
||||||
|
options = tuple(str(o) for o in options_raw)
|
||||||
|
|
||||||
|
# 校验 answer
|
||||||
|
if answer not in _VALID_ANSWERS:
|
||||||
|
msg = f"answer 字段值 '{answer}' 非法,必须为 A/B/C/D 之一"
|
||||||
|
raise ValueError(msg)
|
||||||
|
|
||||||
|
# 校验 difficulty
|
||||||
|
if difficulty not in _VALID_DIFFICULTIES:
|
||||||
|
logger.warning(
|
||||||
|
"difficulty '{}' 不在预设范围 {},回退为 'medium'",
|
||||||
|
difficulty,
|
||||||
|
_VALID_DIFFICULTIES,
|
||||||
|
)
|
||||||
|
difficulty = "medium"
|
||||||
|
|
||||||
|
# Phase 5: 构造 CandidateQuestion
|
||||||
|
question_id = f"{video_id}_{task_type}_{seq:04d}"
|
||||||
|
|
||||||
|
return CandidateQuestion(
|
||||||
|
question_id=question_id,
|
||||||
|
video_id=video_id,
|
||||||
|
task_type=task_type,
|
||||||
|
skill_target=skill_target,
|
||||||
|
question=question_text,
|
||||||
|
options=options,
|
||||||
|
answer=answer,
|
||||||
|
source_nodes=source_nodes,
|
||||||
|
difficulty=difficulty,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 主入口
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def generate_one_v2(
|
||||||
|
vlm: VLMProvider,
|
||||||
|
tree: TreeIndex,
|
||||||
|
material: MaterialContext,
|
||||||
|
family_spec: QuestionFamilySpec,
|
||||||
|
task_type: str,
|
||||||
|
seq: int,
|
||||||
|
*,
|
||||||
|
video_id: str,
|
||||||
|
reject_reason: str | None = None,
|
||||||
|
session_id: str,
|
||||||
|
) -> CandidateQuestion:
|
||||||
|
"""调用 VLM 生成一道候选题目。
|
||||||
|
|
||||||
|
流程:
|
||||||
|
1. 构建 per-family prompt + 帧路径。
|
||||||
|
2. 调用 VLMProvider.chat_with_images。
|
||||||
|
3. 解析响应为 CandidateQuestion。
|
||||||
|
4. 附加素材验证信息(subtitle_sentences、frame_paths)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
vlm: VLM 调用端口。
|
||||||
|
tree: 视频树索引(当前未直接使用,预留后续扩展)。
|
||||||
|
material: 采样素材上下文。
|
||||||
|
family_spec: 问题家族规格。
|
||||||
|
task_type: 任务类型字符串。
|
||||||
|
seq: 当前序号。
|
||||||
|
video_id: 视频标识。
|
||||||
|
reject_reason: 上一次被门控拒绝的原因。
|
||||||
|
session_id: 会话 ID(遥测关联)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
CandidateQuestion 实例(包含验证材料)。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: VLM 响应解析失败。
|
||||||
|
FileNotFoundError: 家族 prompt 模板不存在。
|
||||||
|
"""
|
||||||
|
# Phase 1: 构建 prompt
|
||||||
|
messages, frame_paths = _build_v2_prompt(
|
||||||
|
family_spec=family_spec,
|
||||||
|
material=material,
|
||||||
|
task_type=task_type,
|
||||||
|
seq=seq,
|
||||||
|
reject_reason=reject_reason,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Phase 2: 调用 VLM
|
||||||
|
logger.debug(
|
||||||
|
"generate_one_v2: family={}, task_type={}, seq={}, frames={}",
|
||||||
|
family_spec.name,
|
||||||
|
task_type,
|
||||||
|
seq,
|
||||||
|
len(frame_paths),
|
||||||
|
)
|
||||||
|
|
||||||
|
response = await vlm.chat_with_images(
|
||||||
|
messages,
|
||||||
|
frame_paths,
|
||||||
|
session_id=session_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Phase 3: 解析响应
|
||||||
|
candidate = _parse_v2_response(
|
||||||
|
raw=response.content,
|
||||||
|
video_id=video_id,
|
||||||
|
task_type=task_type,
|
||||||
|
skill_target=family_spec.skill_target,
|
||||||
|
seq=seq,
|
||||||
|
source_nodes=material.source_nodes,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Phase 4: 附加验证材料(构造新实例,因 frozen=True)
|
||||||
|
candidate = CandidateQuestion(
|
||||||
|
question_id=candidate.question_id,
|
||||||
|
video_id=candidate.video_id,
|
||||||
|
task_type=candidate.task_type,
|
||||||
|
skill_target=candidate.skill_target,
|
||||||
|
question=candidate.question,
|
||||||
|
options=candidate.options,
|
||||||
|
answer=candidate.answer,
|
||||||
|
source_nodes=candidate.source_nodes,
|
||||||
|
difficulty=candidate.difficulty,
|
||||||
|
subtitle_sentences=tuple(material.subtitle_sentences),
|
||||||
|
frame_paths=tuple(material.frame_paths),
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
"generate_one_v2 完成: question_id={}, difficulty={}",
|
||||||
|
candidate.question_id,
|
||||||
|
candidate.difficulty,
|
||||||
|
)
|
||||||
|
|
||||||
|
return candidate
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
You are a question generator for video understanding benchmarks.
|
||||||
|
|
||||||
|
Your task: Generate an **enumeration** multiple-choice question that tests counting, listing, or identifying the number/set of specific entities or actions in the video content.
|
||||||
|
|
||||||
|
## Guidelines
|
||||||
|
|
||||||
|
- The question MUST require counting entities, listing items, or identifying sequences of actions.
|
||||||
|
- Focus on "How many...", "Which of the following are all...", "In what order..." style questions.
|
||||||
|
- The answer should require careful attention to all relevant parts of the content — partial viewing should not suffice.
|
||||||
|
- Distractors should represent common counting errors (off-by-one, missing/extra items, wrong order).
|
||||||
|
|
||||||
|
## Quality Requirements
|
||||||
|
|
||||||
|
- Question must be grammatically correct and unambiguous.
|
||||||
|
- All four options must be parallel in structure and length.
|
||||||
|
- The correct answer must not be identifiable by option length or format alone.
|
||||||
|
- Avoid trivially small counts (e.g., "How many people?" when only 1 is visible).
|
||||||
|
- Each option must begin with "A. ", "B. ", "C. ", or "D. ".
|
||||||
|
|
||||||
|
## Output
|
||||||
|
|
||||||
|
Respond with ONLY a valid JSON object. No additional text.
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
You are a question generator for video understanding benchmarks.
|
||||||
|
|
||||||
|
Your task: Generate a **multi-hop reasoning** multiple-choice question that requires connecting information from multiple segments of the video to arrive at the correct answer.
|
||||||
|
|
||||||
|
## Guidelines
|
||||||
|
|
||||||
|
- The question MUST require reasoning across at least two distinct pieces of information (temporal, causal, or logical connections).
|
||||||
|
- The answer should NOT be directly stated in any single subtitle or frame — it must be inferred by combining evidence.
|
||||||
|
- Test causal chains, temporal ordering, or logical deductions that span multiple events.
|
||||||
|
- Distractors should represent common reasoning errors (e.g., reversed causality, incorrect temporal ordering).
|
||||||
|
|
||||||
|
## Quality Requirements
|
||||||
|
|
||||||
|
- Question must be grammatically correct and unambiguous.
|
||||||
|
- All four options must be parallel in structure and length.
|
||||||
|
- The correct answer must not be identifiable from linguistic cues alone.
|
||||||
|
- The reasoning chain should be verifiable from the provided material.
|
||||||
|
- Each option must begin with "A. ", "B. ", "C. ", or "D. ".
|
||||||
|
|
||||||
|
## Output
|
||||||
|
|
||||||
|
Respond with ONLY a valid JSON object. No additional text.
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
You are a question generator for video understanding benchmarks.
|
||||||
|
|
||||||
|
Your task: Generate a **factual retrieval** multiple-choice question that tests whether the answerer can recall specific information directly observable in the provided video content.
|
||||||
|
|
||||||
|
## Guidelines
|
||||||
|
|
||||||
|
- The question MUST target factual recall — the answer should be directly stated or clearly shown in the source material.
|
||||||
|
- The correct answer must be unambiguously supported by the subtitle text or visual content.
|
||||||
|
- Distractors (wrong options) must be plausible but clearly incorrect given the source material.
|
||||||
|
- Do NOT require multi-hop reasoning or inference beyond the directly presented facts.
|
||||||
|
- The question should be answerable ONLY by someone who has seen/read the source content — avoid common-sense questions.
|
||||||
|
|
||||||
|
## Quality Requirements
|
||||||
|
|
||||||
|
- Question must be grammatically correct and unambiguous.
|
||||||
|
- All four options must be parallel in structure and length.
|
||||||
|
- The correct answer must not be identifiable from linguistic cues alone.
|
||||||
|
- Avoid negation in the question stem (e.g., "Which of the following is NOT...").
|
||||||
|
- Each option must begin with "A. ", "B. ", "C. ", or "D. ".
|
||||||
|
|
||||||
|
## Output
|
||||||
|
|
||||||
|
Respond with ONLY a valid JSON object. No additional text.
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
You are a question generator for video understanding benchmarks.
|
||||||
|
|
||||||
|
Your task: Generate a **spatial relationship** multiple-choice question that tests understanding of spatial arrangements, positions, and relationships between objects or people in the video.
|
||||||
|
|
||||||
|
## Guidelines
|
||||||
|
|
||||||
|
- The question MUST focus on spatial relationships: relative positions, directions, distances, containment, or spatial changes.
|
||||||
|
- Test understanding of "where" things are, how they relate spatially, or how spatial arrangements change over time.
|
||||||
|
- Use spatial language: "left/right of", "above/below", "between", "inside/outside", "closer/farther", "facing".
|
||||||
|
- The answer should require spatial reasoning that goes beyond simple object identification.
|
||||||
|
- Distractors should represent common spatial confusion (mirror reversals, misremembered positions).
|
||||||
|
|
||||||
|
## Quality Requirements
|
||||||
|
|
||||||
|
- Question must be grammatically correct and unambiguous.
|
||||||
|
- All four options must be parallel in structure and length.
|
||||||
|
- The correct answer must not be identifiable from linguistic cues alone.
|
||||||
|
- Spatial references must be unambiguous given the visual content.
|
||||||
|
- Each option must begin with "A. ", "B. ", "C. ", or "D. ".
|
||||||
|
|
||||||
|
## Output
|
||||||
|
|
||||||
|
Respond with ONLY a valid JSON object. No additional text.
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
You are a question generator for video understanding benchmarks.
|
||||||
|
|
||||||
|
Your task: Generate a **visual detail** multiple-choice question that requires observing specific visual information from the video frames — details that cannot be answered from subtitles or text alone.
|
||||||
|
|
||||||
|
## Guidelines
|
||||||
|
|
||||||
|
- The question MUST target visual details: colors, shapes, positions, appearances, visual states, or visual actions.
|
||||||
|
- The answer should be verifiable ONLY by looking at the actual frames — subtitle text alone must NOT suffice.
|
||||||
|
- Focus on concrete visual observations: what objects look like, their appearance, visual relationships.
|
||||||
|
- Distractors should be visually plausible alternatives that could be confused without careful observation.
|
||||||
|
|
||||||
|
## Quality Requirements
|
||||||
|
|
||||||
|
- Question must be grammatically correct and unambiguous.
|
||||||
|
- All four options must be parallel in structure and length.
|
||||||
|
- The correct answer must not be identifiable from linguistic cues alone.
|
||||||
|
- Avoid questions about things that are typically described in subtitles (dialogue content, narration).
|
||||||
|
- Each option must begin with "A. ", "B. ", "C. ", or "D. ".
|
||||||
|
|
||||||
|
## Output
|
||||||
|
|
||||||
|
Respond with ONLY a valid JSON object. No additional text.
|
||||||
@@ -0,0 +1,401 @@
|
|||||||
|
"""v2 生成器单元测试 — 验证 prompt 构建、响应解析、端到端生成。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.question_gen.families import RETRIEVAL_FAMILY, VISUAL_FAMILY
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
from app.question_gen.sampler_v2 import AnchorContext, MaterialContext
|
||||||
|
from app.tree.index import (
|
||||||
|
IndexMeta,
|
||||||
|
L1Card,
|
||||||
|
L1Node,
|
||||||
|
L2Card,
|
||||||
|
L2Node,
|
||||||
|
L3Card,
|
||||||
|
L3Node,
|
||||||
|
TreeIndex,
|
||||||
|
)
|
||||||
|
from core.types import LLMResponse
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Fixtures & Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _make_material(
|
||||||
|
*,
|
||||||
|
with_frames: bool = False,
|
||||||
|
with_cross_l2: bool = False,
|
||||||
|
) -> MaterialContext:
|
||||||
|
"""构造测试用 MaterialContext。"""
|
||||||
|
anchor = AnchorContext(node_id="L2_001", level=2, l2_id="L2_001")
|
||||||
|
frame_paths = ["/data/frames/f001.jpg", "/data/frames/f002.jpg"] if with_frames else []
|
||||||
|
cross_l2 = ["Person enters room and sits down."] if with_cross_l2 else []
|
||||||
|
return MaterialContext(
|
||||||
|
anchor=anchor,
|
||||||
|
source_nodes=("L2_001", "L3_001", "L3_002"),
|
||||||
|
subtitle_sentences=["He picks up the book.", "Then he starts reading."],
|
||||||
|
frame_paths=frame_paths,
|
||||||
|
cross_l2_texts=cross_l2,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_vlm_response(content: str) -> LLMResponse:
|
||||||
|
"""构造 VLM 正常返回。"""
|
||||||
|
return LLMResponse(
|
||||||
|
content=content,
|
||||||
|
thinking="",
|
||||||
|
model="mock-vlm",
|
||||||
|
provider="mock",
|
||||||
|
prompt_tokens=50,
|
||||||
|
completion_tokens=30,
|
||||||
|
latency_ms=200,
|
||||||
|
ttft_ms=None,
|
||||||
|
max_inter_token_ms=None,
|
||||||
|
cache_hit=False,
|
||||||
|
call_id="mock-vlm-001",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class MockVLM:
|
||||||
|
"""可配置的 VLM mock — 记录调用并返回预设响应。"""
|
||||||
|
|
||||||
|
def __init__(self, response: LLMResponse) -> None:
|
||||||
|
self._response = response
|
||||||
|
self.calls: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
async def chat_with_images(
|
||||||
|
self,
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
images: list[str | Path],
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> LLMResponse:
|
||||||
|
"""记录调用并返回预设响应。"""
|
||||||
|
self.calls.append(
|
||||||
|
{
|
||||||
|
"messages": messages,
|
||||||
|
"images": images,
|
||||||
|
"session_id": session_id,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return self._response
|
||||||
|
|
||||||
|
|
||||||
|
def _make_tree() -> TreeIndex:
|
||||||
|
"""构造最小测试树。"""
|
||||||
|
l3_card = L3Card(
|
||||||
|
frame_summary="Person picks up a book from the shelf.",
|
||||||
|
visible_entities=["person", "book", "shelf"],
|
||||||
|
ongoing_actions=["picking up"],
|
||||||
|
visible_text=[],
|
||||||
|
spatial_layout="person in center",
|
||||||
|
visual_attributes={},
|
||||||
|
subtitle="He picks up the book.",
|
||||||
|
)
|
||||||
|
l3 = L3Node(id="L3_001", card=l3_card, frame_path="/data/frames/f001.jpg")
|
||||||
|
|
||||||
|
l3_card2 = L3Card(
|
||||||
|
frame_summary="Person starts reading the book.",
|
||||||
|
visible_entities=["person", "book"],
|
||||||
|
ongoing_actions=["reading"],
|
||||||
|
visible_text=[],
|
||||||
|
spatial_layout="person sitting",
|
||||||
|
visual_attributes={},
|
||||||
|
subtitle="Then he starts reading.",
|
||||||
|
)
|
||||||
|
l3_2 = L3Node(id="L3_002", card=l3_card2, frame_path="/data/frames/f002.jpg")
|
||||||
|
|
||||||
|
l2_card = L2Card(
|
||||||
|
event_description="A person picks up a book and starts reading.",
|
||||||
|
entities=["person", "book"],
|
||||||
|
actions=["pick up", "read"],
|
||||||
|
action_subjects=["person"],
|
||||||
|
visible_text=[],
|
||||||
|
spatial_relations="near shelf",
|
||||||
|
state_changes="book picked up",
|
||||||
|
subtitle="He picks up the book and reads.",
|
||||||
|
)
|
||||||
|
l2 = L2Node(id="L2_001", card=l2_card, children=[l3, l3_2])
|
||||||
|
|
||||||
|
l1_card = L1Card(
|
||||||
|
scene_summary="Library scene.",
|
||||||
|
main_setting="library",
|
||||||
|
key_entities=["person", "book"],
|
||||||
|
main_actions=["reading"],
|
||||||
|
topic_keywords=["library"],
|
||||||
|
visible_text=[],
|
||||||
|
temporal_flow="enter → read",
|
||||||
|
)
|
||||||
|
l1 = L1Node(id="L1_001", card=l1_card, children=[l2])
|
||||||
|
|
||||||
|
meta = IndexMeta(source_path="test.mp4", modality="video")
|
||||||
|
return TreeIndex(metadata=meta, roots=[l1])
|
||||||
|
|
||||||
|
|
||||||
|
_VALID_VLM_OUTPUT = json.dumps(
|
||||||
|
{
|
||||||
|
"question": "What does the person do after picking up the book?",
|
||||||
|
"options": ["A. Reads it", "B. Puts it back", "C. Throws it", "D. Burns it"],
|
||||||
|
"answer": "A",
|
||||||
|
"difficulty": "medium",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestBuildV2Prompt
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildV2Prompt:
|
||||||
|
"""验证 _build_v2_prompt 的 prompt 构造逻辑。"""
|
||||||
|
|
||||||
|
def test_includes_family_template(self) -> None:
|
||||||
|
"""prompt 中应包含家族模板的核心指令。"""
|
||||||
|
from app.question_gen.generator_v2 import _build_v2_prompt
|
||||||
|
|
||||||
|
material = _make_material()
|
||||||
|
messages, frame_paths = _build_v2_prompt(
|
||||||
|
family_spec=RETRIEVAL_FAMILY,
|
||||||
|
material=material,
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
seq=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
# messages 至少有 system + user
|
||||||
|
assert len(messages) >= 2
|
||||||
|
# system message 中应包含 retrieval 家族相关指令
|
||||||
|
system_content = messages[0]["content"]
|
||||||
|
assert "retrieval" in system_content.lower() or "factual" in system_content.lower()
|
||||||
|
|
||||||
|
def test_reject_reason_injected(self) -> None:
|
||||||
|
"""当 reject_reason 不为 None 时,应注入到 prompt 中。"""
|
||||||
|
from app.question_gen.generator_v2 import _build_v2_prompt
|
||||||
|
|
||||||
|
material = _make_material()
|
||||||
|
reject_msg = "The question leaked information from the answer options."
|
||||||
|
messages, _ = _build_v2_prompt(
|
||||||
|
family_spec=RETRIEVAL_FAMILY,
|
||||||
|
material=material,
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
seq=2,
|
||||||
|
reject_reason=reject_msg,
|
||||||
|
)
|
||||||
|
|
||||||
|
# reject_reason 应出现在某个 message 内容中
|
||||||
|
all_content = " ".join(m["content"] for m in messages)
|
||||||
|
assert reject_msg in all_content
|
||||||
|
|
||||||
|
def test_frame_paths_from_material(self) -> None:
|
||||||
|
"""返回的 frame_paths 应来自 material.frame_paths。"""
|
||||||
|
from app.question_gen.generator_v2 import _build_v2_prompt
|
||||||
|
|
||||||
|
material = _make_material(with_frames=True)
|
||||||
|
_, frame_paths = _build_v2_prompt(
|
||||||
|
family_spec=VISUAL_FAMILY,
|
||||||
|
material=material,
|
||||||
|
task_type="Object Recognition",
|
||||||
|
seq=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert frame_paths == ["/data/frames/f001.jpg", "/data/frames/f002.jpg"]
|
||||||
|
|
||||||
|
def test_no_frames_returns_empty(self) -> None:
|
||||||
|
"""无帧素材时 frame_paths 应为空列表。"""
|
||||||
|
from app.question_gen.generator_v2 import _build_v2_prompt
|
||||||
|
|
||||||
|
material = _make_material(with_frames=False)
|
||||||
|
_, frame_paths = _build_v2_prompt(
|
||||||
|
family_spec=RETRIEVAL_FAMILY,
|
||||||
|
material=material,
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
seq=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert frame_paths == []
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestParseV2Response
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseV2Response:
|
||||||
|
"""验证 _parse_v2_response 的解析与校验逻辑。"""
|
||||||
|
|
||||||
|
def test_valid_json(self) -> None:
|
||||||
|
"""正确 JSON → 生成 CandidateQuestion。"""
|
||||||
|
from app.question_gen.generator_v2 import CandidateQuestion, _parse_v2_response
|
||||||
|
|
||||||
|
result = _parse_v2_response(
|
||||||
|
raw=_VALID_VLM_OUTPUT,
|
||||||
|
video_id="v-001",
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
skill_target="M1",
|
||||||
|
seq=3,
|
||||||
|
source_nodes=("L2_001", "L3_001"),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert isinstance(result, CandidateQuestion)
|
||||||
|
assert result.question_id == "v-001_Action Reasoning_0003"
|
||||||
|
assert result.question == "What does the person do after picking up the book?"
|
||||||
|
assert result.options == ("A. Reads it", "B. Puts it back", "C. Throws it", "D. Burns it")
|
||||||
|
assert result.answer == "A"
|
||||||
|
assert result.difficulty == "medium"
|
||||||
|
assert result.video_id == "v-001"
|
||||||
|
assert result.task_type == "Action Reasoning"
|
||||||
|
assert result.skill_target == "M1"
|
||||||
|
assert result.source_nodes == ("L2_001", "L3_001")
|
||||||
|
|
||||||
|
def test_missing_field_raises(self) -> None:
|
||||||
|
"""缺少必填字段 → 抛出 ValueError。"""
|
||||||
|
from app.question_gen.generator_v2 import _parse_v2_response
|
||||||
|
|
||||||
|
incomplete = json.dumps(
|
||||||
|
{
|
||||||
|
"question": "What happens?",
|
||||||
|
"options": ["A. X", "B. Y"],
|
||||||
|
# 缺少 answer 和 difficulty
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="answer"):
|
||||||
|
_parse_v2_response(
|
||||||
|
raw=incomplete,
|
||||||
|
video_id="v-001",
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
skill_target="M1",
|
||||||
|
seq=1,
|
||||||
|
source_nodes=("L2_001",),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_invalid_answer_raises(self) -> None:
|
||||||
|
"""answer 不在 A-D 范围内 → 抛出 ValueError。"""
|
||||||
|
from app.question_gen.generator_v2 import _parse_v2_response
|
||||||
|
|
||||||
|
bad_answer = json.dumps(
|
||||||
|
{
|
||||||
|
"question": "What happens?",
|
||||||
|
"options": ["A. X", "B. Y", "C. Z", "D. W"],
|
||||||
|
"answer": "E",
|
||||||
|
"difficulty": "easy",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="answer"):
|
||||||
|
_parse_v2_response(
|
||||||
|
raw=bad_answer,
|
||||||
|
video_id="v-001",
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
skill_target="M1",
|
||||||
|
seq=1,
|
||||||
|
source_nodes=("L2_001",),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_json_repair_handles_trailing_comma(self) -> None:
|
||||||
|
"""json_repair 应能修复常见 JSON 错误。"""
|
||||||
|
from app.question_gen.generator_v2 import _parse_v2_response
|
||||||
|
|
||||||
|
malformed = '{"question": "What?", "options": ["A. X", "B. Y", "C. Z", "D. W",], "answer": "B", "difficulty": "easy",}'
|
||||||
|
|
||||||
|
result = _parse_v2_response(
|
||||||
|
raw=malformed,
|
||||||
|
video_id="v-001",
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
skill_target="M1",
|
||||||
|
seq=1,
|
||||||
|
source_nodes=("L2_001",),
|
||||||
|
)
|
||||||
|
assert result.answer == "B"
|
||||||
|
|
||||||
|
def test_markdown_wrapped_json(self) -> None:
|
||||||
|
"""被 markdown 代码块包裹的 JSON 应正常解析。"""
|
||||||
|
from app.question_gen.generator_v2 import _parse_v2_response
|
||||||
|
|
||||||
|
wrapped = f"```json\n{_VALID_VLM_OUTPUT}\n```"
|
||||||
|
|
||||||
|
result = _parse_v2_response(
|
||||||
|
raw=wrapped,
|
||||||
|
video_id="v-001",
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
skill_target="M1",
|
||||||
|
seq=5,
|
||||||
|
source_nodes=("L2_001",),
|
||||||
|
)
|
||||||
|
assert result.question_id == "v-001_Action Reasoning_0005"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestGenerateOneV2
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestGenerateOneV2:
|
||||||
|
"""验证 generate_one_v2 端到端流程。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_happy_path(self) -> None:
|
||||||
|
"""正常流程:VLM 返回有效 JSON → 生成 CandidateQuestion。"""
|
||||||
|
from app.question_gen.generator_v2 import CandidateQuestion, generate_one_v2
|
||||||
|
|
||||||
|
mock_vlm = MockVLM(_make_vlm_response(_VALID_VLM_OUTPUT))
|
||||||
|
tree = _make_tree()
|
||||||
|
material = _make_material(with_frames=True)
|
||||||
|
|
||||||
|
result = await generate_one_v2(
|
||||||
|
vlm=mock_vlm,
|
||||||
|
tree=tree,
|
||||||
|
material=material,
|
||||||
|
family_spec=RETRIEVAL_FAMILY,
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
seq=1,
|
||||||
|
video_id="v-001",
|
||||||
|
session_id="session-test-001",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert isinstance(result, CandidateQuestion)
|
||||||
|
assert result.question_id == "v-001_Action Reasoning_0001"
|
||||||
|
assert result.video_id == "v-001"
|
||||||
|
assert result.skill_target == "M1"
|
||||||
|
assert result.source_nodes == ("L2_001", "L3_001", "L3_002")
|
||||||
|
assert result.subtitle_sentences == ("He picks up the book.", "Then he starts reading.")
|
||||||
|
assert result.frame_paths == ("/data/frames/f001.jpg", "/data/frames/f002.jpg")
|
||||||
|
|
||||||
|
# VLM 应被调用一次
|
||||||
|
assert len(mock_vlm.calls) == 1
|
||||||
|
assert mock_vlm.calls[0]["session_id"] == "session-test-001"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_reject_reason_forwarded(self) -> None:
|
||||||
|
"""reject_reason 应被传入 prompt 构建。"""
|
||||||
|
from app.question_gen.generator_v2 import generate_one_v2
|
||||||
|
|
||||||
|
mock_vlm = MockVLM(_make_vlm_response(_VALID_VLM_OUTPUT))
|
||||||
|
tree = _make_tree()
|
||||||
|
material = _make_material()
|
||||||
|
|
||||||
|
await generate_one_v2(
|
||||||
|
vlm=mock_vlm,
|
||||||
|
tree=tree,
|
||||||
|
material=material,
|
||||||
|
family_spec=RETRIEVAL_FAMILY,
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
seq=2,
|
||||||
|
video_id="v-001",
|
||||||
|
reject_reason="Answer was too obvious",
|
||||||
|
session_id="session-test-002",
|
||||||
|
)
|
||||||
|
|
||||||
|
# 验证 VLM 调用的 messages 中包含 reject_reason
|
||||||
|
call = mock_vlm.calls[0]
|
||||||
|
all_content = " ".join(m["content"] for m in call["messages"])
|
||||||
|
assert "Answer was too obvious" in all_content
|
||||||
Reference in New Issue
Block a user