merge: feat/repair-concurrent — 赛题生成工具 + repair 管线完善

新增模块:
- app/question_gen/synthesizer.py — 题型映射、节点采样、prompt 构造、去重
- app/harness/factory.py — 推理依赖组装(可复用)
- tools/generate_questions.py — generate + calibrate CLI

修复:
- _VIDEO_MME_TASK_TYPE_COUNT 11→12
- repair/supplement 防御性修复
- repair/regenerator VLM 重生成器
This commit is contained in:
2026-07-09 07:54:21 -04:00
31 changed files with 6567 additions and 47 deletions
+1 -1
View File
@@ -41,7 +41,7 @@ REDIS_URL=redis://localhost:6379/0
LLM_TIMEOUT=120 LLM_TIMEOUT=120
LLM_MAX_RETRIES=3 LLM_MAX_RETRIES=3
LLM_RETRY_BASE_DELAY=2.0 LLM_RETRY_BASE_DELAY=2.0
LLM_CIRCUIT_BREAKER_THRESHOLD=5 LLM_CIRCUIT_BREAKER_THRESHOLD=5 # 实际阈值 = max(此值, concurrency*2)
LLM_CIRCUIT_BREAKER_COOLDOWN=60 LLM_CIRCUIT_BREAKER_COOLDOWN=60
LLM_TTFT_TIMEOUT=30 LLM_TTFT_TIMEOUT=30
LLM_INTER_TOKEN_TIMEOUT=15 LLM_INTER_TOKEN_TIMEOUT=15
+1 -3
View File
@@ -124,9 +124,7 @@ class SQLiteTelemetryRecorder:
finally: finally:
conn.close() conn.close()
except sqlite3.Error as exc: except sqlite3.Error as exc:
logger.warning( logger.warning("遥测写入失败(已降级),call_id={}: {}", call_id, exc)
"遥测写入失败(已降级),call_id={}: {}", call_id, exc
)
async def record_llm_call( async def record_llm_call(
self, self,
+3 -3
View File
@@ -20,8 +20,8 @@ _VALID_SKILL_MODES = {"auto", "manual", "none"}
_VALID_SKILL_UPDATE_MODES = {"patch", "rewrite"} _VALID_SKILL_UPDATE_MODES = {"patch", "rewrite"}
_PATH_FIELDS = {"workspace_dir", "store_dir"} _PATH_FIELDS = {"workspace_dir", "store_dir"}
# Video-MME 的任务类型数量:验证池每类至少保底 eval_min_per_class 题,共 11 类。 # Video-MME 的任务类型数量:验证池每类至少保底 eval_min_per_class 题,共 12 类。
_VIDEO_MME_TASK_TYPE_COUNT = 11 _VIDEO_MME_TASK_TYPE_COUNT = 12
# .env 工程配置字段映射(环境变量名 → RunConfig 字段名)。 # .env 工程配置字段映射(环境变量名 → RunConfig 字段名)。
# 仅路径类工程配置走 .env,科研实验参数走 YAML。 # 仅路径类工程配置走 .env,科研实验参数走 YAML。
@@ -267,7 +267,7 @@ def _validate_minibatch(config: RunConfig) -> None:
关键实现细节: 关键实现细节:
val_size 必须 >= eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT,保证验证池 val_size 必须 >= eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT,保证验证池
能为 Video-MME 的全部 11 个任务类型各保底 eval_min_per_class 题。 能为 Video-MME 的全部 12 个任务类型各保底 eval_min_per_class 题。
""" """
if config.batch_size <= 0: if config.batch_size <= 0:
raise ValueError(f"batch_size 必须 > 0,实际: {config.batch_size}") raise ValueError(f"batch_size 必须 > 0,实际: {config.batch_size}")
+155
View File
@@ -0,0 +1,155 @@
"""推理依赖工厂 — 组装一次推理所需的全套依赖。
将 TreeIndex 加载、TreeEnvironment 构建、SkillRegistry 发现、
SearchToolDispatcher 装配、PromptManager 初始化等步骤封装为
单一工厂函数 ``build_inference_deps``,返回不可变的 ``InferenceDeps``。
调用方(runner / inference)只需传入配置参数,无需了解内部装配逻辑。
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from loguru import logger
from app.search.prompt import PromptManager
from app.search.skills import discover_skills
from app.search.tools import SearchToolDispatcher
from app.tree.environment import TreeEnvironment
from app.tree.index import TreeIndex
if TYPE_CHECKING:
from collections.abc import Callable
from pathlib import Path
from app.ports import EmbeddingProvider, OCRProvider
from core.protocols import LLMProvider, VLMProvider
from core.types import GeneratedQuestion
@dataclass(frozen=True)
class InferenceDeps:
"""跑一次推理所需的全套依赖(不含 HarnessLog,其生命周期由调用方管理)。
属性:
llm: LLM 端口实例。
tool_dispatch_fn: SearchToolDispatcher.dispatch 的绑定方法。
prompt_builder: (GeneratedQuestion) -> (system_prompt, user_prompt)。
"""
llm: LLMProvider
tool_dispatch_fn: Callable[..., Any]
prompt_builder: Callable[[GeneratedQuestion], tuple[str, str]]
def build_inference_deps(
*,
store_dir: Path,
video_id: str,
prompts_dir: Path,
skills_dir: Path | None,
skill_mode: str,
embed_provider: EmbeddingProvider,
llm: LLMProvider,
vlm: VLMProvider,
ocr: OCRProvider | None,
verify_vision: bool,
anchor: bool,
assemble_mode: str,
) -> InferenceDeps:
"""组装一次推理所需的全套依赖。
参数:
store_dir: store 根目录(包含 videos/{video_id}/tree.json)。
video_id: 视频标识。
prompts_dir: prompt 文件目录。
skills_dir: skill 文件目录(None 则不加载 skill)。
skill_mode: skill 模式("auto"/"manual"/"none")。
embed_provider: 嵌入端口实例。
llm: LLM 端口实例。
vlm: VLM 端口实例。
ocr: OCR 端口实例(None 不启用)。
verify_vision: observe_frame 是否执行验证轮。
anchor: view_node 是否启用行号锚模式。
assemble_mode: 锚模式装配形态。
返回:
InferenceDeps 实例。
异常:
FileNotFoundError: tree.json 不存在。
"""
# Phase 1: 加载 TreeIndex
tree_path = store_dir / "videos" / video_id / "tree.json"
if not tree_path.exists():
raise FileNotFoundError(f"树索引文件不存在: {tree_path}")
tree_index = TreeIndex.load_json(str(tree_path))
logger.info("已加载 TreeIndex: video_id={}, L1 节点数={}", video_id, len(tree_index.roots))
# Phase 2: 构建 TreeEnvironment
frames_dir = store_dir / "videos" / video_id / "frames"
env = TreeEnvironment(index=tree_index, frames_dir=frames_dir)
# Phase 3: 构建 SkillRegistry
skills = None
always_skills_text = ""
task_skill_map: dict[str, str] = {}
catalog_text = ""
if skills_dir is not None:
always_skills_text, task_skill_map, catalog_text, skills = discover_skills(skills_dir)
logger.info(
"已发现 skills: always={} 字符, task_map={}",
len(always_skills_text),
len(task_skill_map),
)
# Phase 4: 构建 SearchToolDispatcher
dispatcher = SearchToolDispatcher(
env,
tool_llm=llm,
vlm=vlm,
ocr=ocr,
prompts_dir=prompts_dir,
skills=skills,
embed_fn=embed_provider.embed,
verify_vision=verify_vision,
anchor=anchor,
assemble_mode=assemble_mode,
)
# Phase 5: 构建 PromptManager + _prompt_builder 闭包
pm = PromptManager(prompts_dir)
l1_ids = [root.id for root in tree_index.roots]
def _prompt_builder(qa: GeneratedQuestion) -> tuple[str, str]:
"""为单条题目生成 (system_prompt, user_prompt)。
参数:
qa: 生成的题目实例。
返回:
(system_prompt, user_prompt) 二元组。
"""
system = pm.build_inference_prompt(
skill_mode,
qa.task_type,
always_skills_text,
task_skill_map,
catalog_text,
)
user = pm.format_user_prompt(
qa.question,
list(qa.options),
l1_ids,
qa.task_type,
)
return system, user
logger.info("InferenceDeps 组装完成: video_id={}, skill_mode={}", video_id, skill_mode)
return InferenceDeps(
llm=llm,
tool_dispatch_fn=dispatcher.dispatch,
prompt_builder=_prompt_builder,
)
+15 -2
View File
@@ -1,5 +1,18 @@
"""出题模块 — benchmark 加载分层采样。""" """出题模块 — benchmark 加载分层采样与赛题合成"""
from app.question_gen.loader import load_benchmark, stratified_sample from app.question_gen.loader import load_benchmark, stratified_sample
from app.question_gen.synthesizer import (
TASK_TYPE_LEVEL_MAP,
AnchorContext,
generate_one,
sample_anchor,
)
__all__ = ["load_benchmark", "stratified_sample"] __all__ = [
"load_benchmark",
"stratified_sample",
"TASK_TYPE_LEVEL_MAP",
"AnchorContext",
"generate_one",
"sample_anchor",
]
+756
View File
@@ -0,0 +1,756 @@
"""赛题合成核心逻辑 — 节点采样、prompt 构造、VLM 响应解析、去重。
纯函数为主,异步编排仅 generate_one。
通过 DI 接收 VLMProvider / EmbeddingProvider,不 import adapters/。
"""
from __future__ import annotations
import contextlib
import json
import re
from dataclasses import dataclass
from typing import TYPE_CHECKING
import numpy as np
from loguru import logger
if TYPE_CHECKING:
import random
from collections.abc import Callable
from app.tree.index import L2Node, L3Node, TreeIndex
from core.protocols import VLMProvider
from core.types import GeneratedQuestion
@dataclass(frozen=True)
class AnchorContext:
"""锚节点上下文——生成单道题所需的全部素材。
属性:
node_id: 锚节点 ID。
card_text: 锚节点 card 序列化文本。
frame_paths: 帧图片路径列表。
subtitle: 对应字幕(可空)。
distractor_texts: 同视频其他节点摘要(供 VLM 生成干扰项)。
"""
node_id: str
card_text: str
frame_paths: list[str]
subtitle: str
distractor_texts: list[str]
@dataclass(frozen=True)
class TaskTypeSpec:
"""题型的生成规格。
属性:
level: 锚定层级("L3" / "L2" / "L1" / "L1-L2")。
needs_frames: 是否必须提供帧图。
frame_count: 帧数范围描述(如 "1", "2-3", "0-1")。
context_fields: 需要提取的 card 字段元组。
"""
level: str
needs_frames: bool
frame_count: str
context_fields: tuple[str, ...]
# ---------------------------------------------------------------------------
# 12 种 Video-MME 题型 → 树层级 + 生成规格映射
# ---------------------------------------------------------------------------
TASK_TYPE_LEVEL_MAP: dict[str, TaskTypeSpec] = {
# --- L3 单帧题型 ---
"Object Recognition": TaskTypeSpec("L3", True, "1", ("frame_summary",)),
"Attribute Perception": TaskTypeSpec("L3", True, "1", ("frame_summary",)),
"OCR Problems": TaskTypeSpec("L3", True, "1", ("frame_summary",)),
"Spatial Reasoning": TaskTypeSpec("L3", True, "1", ("frame_summary", "spatial_layout")),
"Spatial Perception": TaskTypeSpec("L3", True, "1", ("frame_summary",)),
# --- L2 多帧 / 事件级题型 ---
"Action Recognition": TaskTypeSpec("L2", True, "2-3", ("event_description",)),
"Action Reasoning": TaskTypeSpec("L2", True, "2-3", ("event_description",)),
"Counting Problem": TaskTypeSpec("L2", True, "2-3", ("event_description",)),
"Temporal Perception": TaskTypeSpec("L2", False, "0-1", ("event_description", "time_range")),
# --- L1 / 跨层级题型 ---
"Temporal Reasoning": TaskTypeSpec("L1", True, "per-L2", ("scene_summary",)),
"Information Synopsis": TaskTypeSpec("L1", True, "per-L2", ("scene_summary",)),
"Object Reasoning": TaskTypeSpec("L1-L2", True, "per-L2", ("event_description",)),
}
# ---------------------------------------------------------------------------
# 内部辅助函数
# ---------------------------------------------------------------------------
def _serialize_l3_card(l3: L3Node, context_fields: tuple[str, ...]) -> str:
"""将 L3 节点 card 按 context_fields 序列化为可读文本。
参数:
l3: L3 节点。
context_fields: 需提取的字段名元组。
返回:
多行 "field: value" 格式的文本。
"""
parts: list[str] = []
for fld in context_fields:
val = getattr(l3.card, fld, None)
if val is not None:
parts.append(f"{fld}: {val}")
return "\n".join(parts)
def _l2_time_range_str(l2: L2Node) -> str:
"""将 L2 的 time_range 格式化为可读字符串。
参数:
l2: L2 节点。
返回:
"time_range: (start, end)" 格式,或 "time_range: unknown"
"""
if l2.time_range is not None:
return f"time_range: ({l2.time_range[0]:.2f}, {l2.time_range[1]:.2f})"
return "time_range: unknown"
def _representative_frame(l2: L2Node) -> str | None:
"""取 L2 的代表帧路径——第一个有 frame_path 的 L3 子节点。
参数:
l2: L2 节点。
返回:
帧路径字符串,或 None(无可用帧时)。
"""
for l3 in l2.children:
if l3.frame_path:
return l3.frame_path
return None
def _sort_l2_by_time(l2_nodes: list[L2Node]) -> list[L2Node]:
"""按 time_range 升序排列 L2 节点(None 排末尾)。
参数:
l2_nodes: 待排序的 L2 节点列表。
返回:
排序后的新列表(不修改原列表)。
"""
return sorted(
l2_nodes,
key=lambda n: n.time_range[0] if n.time_range is not None else float("inf"),
)
# ---------------------------------------------------------------------------
# 各层级采样策略
# ---------------------------------------------------------------------------
def _sample_l3(
tree: TreeIndex,
task_type: str,
spec: TaskTypeSpec,
used_node_ids: set[str],
rng: random.Random,
) -> AnchorContext:
"""L3 层级锚节点采样。
收集全部 L3 节点,排除已用节点,随机选取一个。
参数:
tree: 三层树索引。
task_type: 题型名称。
spec: 题型规格。
used_node_ids: 已用节点 ID 集合。
rng: 随机数生成器。
返回:
AnchorContext 实例。
异常:
ValueError: 候选 L3 节点不足。
"""
# Phase 1: 收集所有 L3 候选(必须有 frame_path
candidates: list[tuple[L3Node, L2Node]] = []
for root in tree.roots:
for l2 in root.children:
for l3 in l2.children:
if l3.id not in used_node_ids and l3.frame_path:
candidates.append((l3, l2))
if not candidates:
raise ValueError(f"锚节点不足: {task_type} 无可用 L3 节点(需具备 frame_path")
# Phase 2: 随机选取
chosen_l3, parent_l2 = rng.choice(candidates)
# Phase 3: 构造上下文(frame_path 已在候选过滤中保证非 None)
card_text = _serialize_l3_card(chosen_l3, spec.context_fields)
frame_paths = [chosen_l3.frame_path] # type: ignore[list-item]
subtitle = chosen_l3.subtitle or ""
# Phase 4: 干扰项——整棵树中其他 L3 的 frame_summary
distractor_texts = [
l3.card.frame_summary
for root in tree.roots
for l2 in root.children
for l3 in l2.children
if l3.id != chosen_l3.id
]
return AnchorContext(
node_id=chosen_l3.id,
card_text=card_text,
frame_paths=frame_paths,
subtitle=subtitle,
distractor_texts=distractor_texts,
)
def _sample_l2(
tree: TreeIndex,
task_type: str,
spec: TaskTypeSpec,
used_node_ids: set[str],
rng: random.Random,
) -> AnchorContext:
"""L2 层级锚节点采样(含 Temporal Perception 特殊处理)。
普通 L2 题型:随机选 1 个 L2,取 2-3 个子 L3 帧。
Temporal Perception0-1 帧,card_text 必含 time_range。
参数:
tree: 三层树索引。
task_type: 题型名称。
spec: 题型规格。
used_node_ids: 已用节点 ID 集合。
rng: 随机数生成器。
返回:
AnchorContext 实例。
异常:
ValueError: 候选 L2 节点不足。
"""
# Phase 1: 收集所有 L2 候选
all_l2: list[L2Node] = []
for root in tree.roots:
for l2 in root.children:
if l2.id not in used_node_ids:
all_l2.append(l2)
if not all_l2:
raise ValueError(f"锚节点不足: {task_type} 无可用 L2 节点")
# Phase 2: 随机选取
chosen_l2 = rng.choice(all_l2)
is_temporal_perception = task_type == "Temporal Perception"
# Phase 3: 帧路径
if is_temporal_perception:
# 0-1 帧:有子节点则取 1 帧,否则 0 帧
frame_paths: list[str] = []
if chosen_l2.children:
first_frame = chosen_l2.children[0].frame_path
if first_frame:
frame_paths = [first_frame]
else:
# 普通 L2:随机采样 2-3 个 L3 帧
children_with_frames = [l3 for l3 in chosen_l2.children if l3.frame_path]
if len(children_with_frames) < 2:
raise ValueError(
f"锚节点不足: {task_type} 需要 >=2 个子帧,"
f"{chosen_l2.id} 仅有 {len(children_with_frames)} 个可用帧"
)
n_frames = min(rng.randint(2, 3), len(children_with_frames))
sampled = rng.sample(children_with_frames, n_frames)
frame_paths = [l3.frame_path for l3 in sampled if l3.frame_path]
# Phase 4: card_text
card_text = f"event_description: {chosen_l2.card.event_description}"
if is_temporal_perception:
card_text += f"\n{_l2_time_range_str(chosen_l2)}"
# Phase 5: 字幕(L2 无自身字幕,取首个子 L3 字幕)
subtitle = ""
if chosen_l2.children and chosen_l2.children[0].subtitle:
subtitle = chosen_l2.children[0].subtitle
# Phase 6: 干扰项——整棵树中其他 L2 的 event_description
distractor_texts = [
l2.card.event_description
for root in tree.roots
for l2 in root.children
if l2.id != chosen_l2.id
]
return AnchorContext(
node_id=chosen_l2.id,
card_text=card_text,
frame_paths=frame_paths,
subtitle=subtitle,
distractor_texts=distractor_texts,
)
def _sample_l1(
tree: TreeIndex,
task_type: str,
spec: TaskTypeSpec,
used_node_ids: set[str],
rng: random.Random,
) -> AnchorContext:
"""L1 层级锚节点采样(Temporal Reasoning / Information Synopsis)。
Information Synopsis:使用目标 L1 下全部 L2 子节点。
Temporal Reasoning:严格要求 >=3 个 L2 子节点,不足则抛 ValueError。
L2 按 time_range 升序排列,每个 L2 取一帧代表。
参数:
tree: 三层树索引。
task_type: 题型名称。
spec: 题型规格。
used_node_ids: 已用节点 ID 集合。
rng: 随机数生成器。
返回:
AnchorContext 实例。
异常:
ValueError: 候选 L1 节点不足,或 Temporal Reasoning 的 L2 子节点 <3。
"""
# Phase 1: 收集可用 L1
candidates = [r for r in tree.roots if r.id not in used_node_ids]
if not candidates:
raise ValueError(f"锚节点不足: {task_type} 无可用 L1 节点")
# Phase 2: 随机选取
chosen_l1 = rng.choice(candidates)
# Phase 3: 选定 L2 子集
if task_type == "Information Synopsis":
# 必须使用全部 L2
selected_l2 = list(chosen_l1.children)
else:
# Temporal Reasoning:严格要求 >=3 个 L2
if len(chosen_l1.children) < 3:
raise ValueError(
f"锚节点不足: {task_type} 需要 >=3 个 L2 子节点,"
f"{chosen_l1.id} 仅有 {len(chosen_l1.children)}"
)
if len(chosen_l1.children) == 3:
selected_l2 = list(chosen_l1.children)
else:
selected_l2 = rng.sample(chosen_l1.children, rng.randint(3, len(chosen_l1.children)))
# Phase 4: 按 time_range 升序排列
selected_l2 = _sort_l2_by_time(selected_l2)
# Phase 5: card_text(场景摘要)
card_text = f"scene_summary: {chosen_l1.card.scene_summary}"
# Phase 6: 帧路径——每个 L2 取一帧代表
frame_paths: list[str] = []
for l2 in selected_l2:
rep = _representative_frame(l2)
if rep:
frame_paths.append(rep)
# Phase 7: 字幕(L1 无字幕)
subtitle = ""
# Phase 8: 干扰项——其他 L1 的 scene_summary
distractor_texts = [r.card.scene_summary for r in tree.roots if r.id != chosen_l1.id]
return AnchorContext(
node_id=chosen_l1.id,
card_text=card_text,
frame_paths=frame_paths,
subtitle=subtitle,
distractor_texts=distractor_texts,
)
def _sample_l1_l2(
tree: TreeIndex,
task_type: str,
spec: TaskTypeSpec,
used_node_ids: set[str],
rng: random.Random,
) -> AnchorContext:
"""L1-L2 跨层级锚节点采样(Object Reasoning)。
从全部 L2 中随机选 2-3 个,按 time_range 排序,
card_text 为各 L2 的 event_description 拼接。
参数:
tree: 三层树索引。
task_type: 题型名称。
spec: 题型规格。
used_node_ids: 已用节点 ID 集合。
rng: 随机数生成器。
返回:
AnchorContext 实例。
异常:
ValueError: 候选 L2 节点不足。
"""
# Phase 1: 收集全部 L2
all_l2: list[L2Node] = []
for root in tree.roots:
for l2 in root.children:
if l2.id not in used_node_ids:
all_l2.append(l2)
if len(all_l2) < 2:
raise ValueError(
f"锚节点不足: {task_type} 需要 >=2 个 L2 节点,但仅有 {len(all_l2)} 个可用"
)
# Phase 2: 随机选 2-3 个
n_pick = min(rng.randint(2, 3), len(all_l2))
selected = rng.sample(all_l2, n_pick)
# Phase 3: 按 time_range 升序排列
selected = _sort_l2_by_time(selected)
# Phase 4: card_text = 各 L2 event_description 拼接
card_text = "\n".join(f"event_description: {l2.card.event_description}" for l2 in selected)
# Phase 5: 帧路径——每个 L2 取一帧代表
frame_paths: list[str] = []
for l2 in selected:
rep = _representative_frame(l2)
if rep:
frame_paths.append(rep)
# Phase 6: 字幕
subtitle = ""
# Phase 7: 干扰项——未被选中的 L2 的 event_description
selected_ids = {l2.id for l2 in selected}
distractor_texts = [l2.card.event_description for l2 in all_l2 if l2.id not in selected_ids]
# 使用第一个被选中节点的 ID 作为锚节点 ID
anchor_id = selected[0].id
return AnchorContext(
node_id=anchor_id,
card_text=card_text,
frame_paths=frame_paths,
subtitle=subtitle,
distractor_texts=distractor_texts,
)
# ---------------------------------------------------------------------------
# 公开接口
# ---------------------------------------------------------------------------
def sample_anchor(
tree: TreeIndex,
task_type: str,
used_node_ids: set[str],
rng: random.Random,
) -> AnchorContext:
"""根据题型从视频树中采样锚节点及上下文素材。
依据 TASK_TYPE_LEVEL_MAP 中的层级规格,分发到对应的层级采样策略。
每种层级有不同的帧选取、card 序列化和干扰项收集逻辑。
参数:
tree: 三层树索引。
task_type: 12 种 Video-MME 题型之一。
used_node_ids: 本轮已用节点 ID 集合(避免重复采样)。
rng: 可控随机数生成器(保证可复现)。
返回:
AnchorContext 实例,包含锚节点 ID、card 文本、帧路径、字幕和干扰项。
异常:
KeyError: task_type 不在 TASK_TYPE_LEVEL_MAP 中。
ValueError: 候选节点不足(全部被 used_node_ids 排除)。
"""
spec = TASK_TYPE_LEVEL_MAP[task_type]
if spec.level == "L3":
return _sample_l3(tree, task_type, spec, used_node_ids, rng)
elif spec.level == "L2":
return _sample_l2(tree, task_type, spec, used_node_ids, rng)
elif spec.level == "L1":
return _sample_l1(tree, task_type, spec, used_node_ids, rng)
elif spec.level == "L1-L2":
return _sample_l1_l2(tree, task_type, spec, used_node_ids, rng)
else:
raise ValueError(f"未知层级: {spec.level}")
# ---------------------------------------------------------------------------
# Prompt 构造与 VLM 响应解析
# ---------------------------------------------------------------------------
_VALID_ANSWERS = frozenset({"A", "B", "C", "D"})
def build_generation_prompt(
task_type: str,
anchor: AnchorContext,
exemplars: list[GeneratedQuestion],
) -> tuple[list[dict[str, str]], list[str]]:
"""组装 VLM 出题 prompt。
构造 OpenAI 格式的 messages 列表和帧图片路径列表,
供 VLMProvider.chat_with_images 直接消费。
参数:
task_type: 题型名称(如 "Object Recognition")。
anchor: 锚节点上下文(card_text, subtitle, distractor_texts, frame_paths)。
exemplars: 少样本示例列表(可为空)。
返回:
(messages, image_paths) — messages 为 OpenAI 格式消息列表,
image_paths 为帧图片路径列表,直接喂给 VLMProvider.chat_with_images。
"""
# Phase 1: 构造 system message
system_parts: list[str] = [
"你是一个视频理解题目生成器。",
f"题型: {task_type}",
"约束:",
"- 题目必须基于提供的节点内容",
"- 干扰选项应来自其他节点的信息",
"- 生成风格应与示例保持一致",
'- 以 JSON 格式返回: {"question": "...", "options": ["A. ...", "B. ...", "C. ...", "D. ..."], "answer": "A/B/C/D"}',
]
# Phase 2: 加入 few-shot 示例
if exemplars:
system_parts.append("\n示例:")
for i, ex in enumerate(exemplars, 1):
system_parts.append(f" 示例 {i}:")
system_parts.append(f" question: {ex.question}")
system_parts.append(f" options: {list(ex.options)}")
system_parts.append(f" answer: {ex.answer}")
system_content = "\n".join(system_parts)
# Phase 3: 构造 user message
user_parts: list[str] = [f"节点内容:\n{anchor.card_text}"]
if anchor.subtitle:
user_parts.append(f"\n字幕:\n{anchor.subtitle}")
if anchor.distractor_texts:
user_parts.append("\n干扰项来源节点摘要:")
for dt in anchor.distractor_texts:
user_parts.append(f"- {dt}")
user_content = "\n".join(user_parts)
messages = [
{"role": "system", "content": system_content},
{"role": "user", "content": user_content},
]
return messages, list(anchor.frame_paths)
def parse_vlm_response(
raw: str,
video_id: str,
task_type: str,
seq: int,
) -> dict:
"""解析 VLM 返回的 JSON → 部分字段字典。
尝试直接解析 JSON;若失败,从 markdown 代码块中提取后重试。
校验必需字段、选项数量和答案合法性。
参数:
raw: VLM 原始返回文本。
video_id: 所属视频标识。
task_type: 题型名称(用于错误消息)。
seq: 序列号,用于生成 question_id。
返回:
{"question_id": "gen-{video_id}-{seq:03d}", "question": ..., "options": [...], "answer": ...}
调用方(generate_one)补齐 source_nodes/difficulty 后构造 GeneratedQuestion。
异常:
ValueError: JSON 解析失败、缺必需字段、options 非 4 项、answer 不在 A-D。
"""
# Phase 1: 尝试直接解析 JSON
data = None
with contextlib.suppress(json.JSONDecodeError):
data = json.loads(raw)
# Phase 2: 从 markdown 代码块提取 JSON
if data is None:
match = re.search(r"```(?:json)?\s*\n?(.*?)\n?\s*```", raw, re.DOTALL)
if match:
with contextlib.suppress(json.JSONDecodeError):
data = json.loads(match.group(1))
if data is None:
raise ValueError(f"VLM 返回无法解析为 JSON: {raw[:200]}")
# Phase 3: 校验必需字段
required = ("question", "options", "answer")
missing = [f for f in required if f not in data]
if missing:
raise ValueError(f"VLM 返回缺少必需字段 {missing}: {raw[:200]}")
# Phase 4: options 必须恰好 4 项
options = data["options"]
if not isinstance(options, list) or len(options) != 4:
raise ValueError(
f"options 必须恰好 4 项,实际 {len(options) if isinstance(options, list) else type(options).__name__}: {raw[:200]}"
)
# Phase 5: answer 必须是 A-D
answer = data["answer"]
if answer not in _VALID_ANSWERS:
raise ValueError(f"answer 必须是 A/B/C/D 之一,实际 '{answer}': {raw[:200]}")
return {
"question_id": f"gen-{video_id}-{task_type.lower().replace(' ', '_')}-{seq:03d}",
"question": data["question"],
"options": list(options),
"answer": answer,
}
# ---------------------------------------------------------------------------
# Embedding 去重
# ---------------------------------------------------------------------------
def is_duplicate(
question_text: str,
pool_embeddings: np.ndarray,
embed_fn: Callable[[str | list[str]], np.ndarray],
threshold: float,
) -> bool:
"""embedding 去重判定。
参数:
question_text: 待检查的题目文本。
pool_embeddings: 已有题目的 embedding 矩阵 [N, D]L2 归一化)。
embed_fn: 文本嵌入函数,返回 [N, D] ndarrayL2 归一化)。
threshold: 余弦相似度阈值。
返回:
True 表示与池中某题重复。空池永远返回 False。
"""
if pool_embeddings.shape[0] == 0:
return False
query = embed_fn(question_text) # [1, D]
query = query.squeeze(0) # [D]
similarities = pool_embeddings @ query # [N]
return bool(np.max(similarities) >= threshold)
# ---------------------------------------------------------------------------
# 单题生成
# ---------------------------------------------------------------------------
async def generate_one(
vlm: VLMProvider,
tree: TreeIndex,
video_id: str,
task_type: str,
seq: int,
*,
exemplars: list[GeneratedQuestion],
used_node_ids: set[str],
max_retries: int,
rng: random.Random,
session_id: str,
) -> GeneratedQuestion | None:
"""生成单道候选题(不含去重——去重在调用方汇总点原子执行)。
循环最多 max_retries 次尝试生成。每次尝试:
1. 采样锚节点
2. 构造 prompt
3. 调用 VLM
4. 解析响应
5. 构造 GeneratedQuestion
返回 None 表示耗尽重试。
参数:
vlm: VLM 调用端口。
tree: 三层树索引。
video_id: 所属视频标识。
task_type: 12 种 Video-MME 题型之一。
seq: 序列号,用于生成 question_id。
exemplars: 少样本示例列表。
used_node_ids: 已用节点 ID 集合。
max_retries: 最大重试次数。
rng: 可控随机数生成器。
session_id: 会话 ID(传递给 VLM 遥测)。
返回:
GeneratedQuestion 实例,或 None(耗尽重试)。
"""
from core.types import GeneratedQuestion as _GeneratedQuestion
for attempt in range(max_retries):
try:
# Phase 1: 采样锚节点
anchor = sample_anchor(tree, task_type, used_node_ids, rng)
# Phase 2: 构造 prompt
messages, images = build_generation_prompt(task_type, anchor, exemplars)
# Phase 3: 调用 VLM
response = await vlm.chat_with_images(
messages,
images,
session_id=session_id,
)
# Phase 4: 解析响应
parsed = parse_vlm_response(response.content, video_id, task_type, seq)
# Phase 5: 构造 GeneratedQuestion
return _GeneratedQuestion(
question_id=parsed["question_id"],
video_id=video_id,
task_type=task_type,
question=parsed["question"],
options=tuple(parsed["options"]),
answer=parsed["answer"],
source_nodes=(anchor.node_id,),
difficulty="medium",
)
except (ValueError, KeyError) as exc:
logger.warning(
"generate_one 尝试 {}/{} 失败 ({}): {}",
attempt + 1,
max_retries,
task_type,
exc,
)
continue
logger.warning(
"generate_one 耗尽 {} 次重试 (video={}, task_type={})",
max_retries,
video_id,
task_type,
)
return None
+3 -6
View File
@@ -40,7 +40,8 @@ def detect_issues(
"""扫描树,返回所有问题节点列表。 """扫描树,返回所有问题节点列表。
检查项: 检查项:
- L3: card 必填字段为空(frame_summary / visible_entities / ongoing_actions / spatial_layout - L3: card 必填字段为空(frame_summary / spatial_layout
- 注: visible_entities / ongoing_actions 为空是合法状态(静物/黑帧),不纳入检测
- L3: frame_path 对应文件不存在(需提供 frames_dir) - L3: frame_path 对应文件不存在(需提供 frames_dir)
- L2: event_description 为空 - L2: event_description 为空
- L2/L1: children 列表为空 - L2/L1: children 列表为空
@@ -108,14 +109,10 @@ def detect_issues(
continue continue
for l3 in l2.children: for l3 in l2.children:
# L3: 必填字段不为空 # L3: 核心必填字段不为空visible_entities/ongoing_actions 为空是合法状态)
empty_fields: list[str] = [] empty_fields: list[str] = []
if not l3.card.frame_summary: if not l3.card.frame_summary:
empty_fields.append("frame_summary") empty_fields.append("frame_summary")
if not l3.card.visible_entities:
empty_fields.append("visible_entities")
if not l3.card.ongoing_actions:
empty_fields.append("ongoing_actions")
if not l3.card.spatial_layout: if not l3.card.spatial_layout:
empty_fields.append("spatial_layout") empty_fields.append("spatial_layout")
if empty_fields: if empty_fields:
+513
View File
@@ -0,0 +1,513 @@
"""树修复重生成器:VLM 重新描述问题节点 + 底向上级联。
底向上修复流程:
1. 收集需修复的 L3 节点 → VLM 重新描述帧
2. 收集受影响的 L2 → LLM 从 L3 children 聚合
3. 收集受影响的 L1 → LLM 从 L2 children 聚合
仅处理 issue_type == "empty_field" 且 level == 3 的问题节点。
帧文件不存在时跳过该节点(不中断整体修复流程)。
"""
from __future__ import annotations
import json
import re
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from loguru import logger
from app.tree.index import (
L1Card,
L1Node,
L2Card,
L2Node,
L3Card,
L3Node,
TreeIndex,
)
from app.tree.subtitle import extract_subtitle_for_range
if TYPE_CHECKING:
from pathlib import Path
from app.tree.repair.detector import NodeIssue
from app.tree.subtitle import SRTEntry
from core.protocols import LLMProvider, VLMProvider
# ---------------------------------------------------------------------------
# Prompt 常量(与 VideoTreeBuilder 保持一致风格)
# ---------------------------------------------------------------------------
_L3_REPAIR_PROMPT = (
'该片段的整体内容: "{l2_description}"\n'
"用一到两句话描述这帧画面的具体内容。"
"重点关注: 动作、物体变化、文字信息、人物表情。\n"
"{subtitle_block}"
"返回 JSON 对象,包含以下字段:\n"
"- frame_summary: 画面描述\n"
"- visible_entities: 可见实体列表\n"
"- ongoing_actions: 动作列表\n"
"- visible_text: 可见文字列表\n"
"- spatial_layout: 空间布局\n"
'- visual_attributes: {{"lighting": "...", "dominant_colors": [...], "camera_angle": "..."}}\n'
"只返回 JSON 对象,不要其他内容。"
)
_L2_REGEN_PROMPT = (
"以下是一个视频片段中各帧的描述:\n{l3_texts}\n"
"用1-2句话描述该片段的核心内容。\n"
"返回 JSON 对象,包含以下字段:\n"
"- event_description: 1-2句片段描述\n"
"- entities: 可见实体列表\n"
"- actions: 动作列表\n"
"- action_subjects: 动作主体列表\n"
"- visible_text: 画面中可见文字列表\n"
"- spatial_relations: 空间关系描述\n"
"- state_changes: 状态变化描述(无则 null)\n"
"只返回 JSON 对象,不要其他内容。"
)
_L1_REGEN_PROMPT = (
"以下是一个视频段落中各片段的描述:\n{l2_texts}\n"
"用2-3句话总结该段落的整体内容,涵盖所有片段的主题。\n"
"返回 JSON 对象,包含以下字段:\n"
"- scene_summary: 2-3句段落摘要\n"
"- main_setting: 主要场景\n"
"- key_entities: 关键实体列表\n"
"- main_actions: 主要动作列表\n"
"- topic_keywords: 主题关键词列表\n"
"- visible_text: 出现的文字列表\n"
"- temporal_flow: 时间流向描述\n"
"只返回 JSON 对象,不要其他内容。"
)
# ---------------------------------------------------------------------------
# 统计数据类
# ---------------------------------------------------------------------------
@dataclass
class RepairStats:
"""修复统计信息。
属性:
l3_repaired: 修复的 L3 节点数。
l2_regenerated: 重生成的 L2 节点数。
l1_regenerated: 重生成的 L1 节点数。
"""
l3_repaired: int = 0
l2_regenerated: int = 0
l1_regenerated: int = 0
# ---------------------------------------------------------------------------
# JSON 解析辅助(复用 VideoTreeBuilder 的解析逻辑)
# ---------------------------------------------------------------------------
def _extract_json(raw: str) -> Any:
"""从 VLM/LLM 原始输出中提取 JSON(处理 markdown 代码块包裹)。
参数:
raw: 原始返回字符串。
返回:
解析后的 Python 对象(dict/list),解析失败返回 None。
"""
raw = raw.strip()
# Phase 1: 尝试提取 markdown 代码块中的 JSON
code_match = re.search(
r"```(?:json)?\s*([\[{].*?[\]}])\s*```",
raw,
re.DOTALL,
)
if code_match:
raw = code_match.group(1)
# Phase 2: 直接解析
try:
return json.loads(raw)
except json.JSONDecodeError:
pass
# Phase 3: 尝试提取裸 JSON 对象/数组
json_match = re.search(r"[\[{].*[\]}]", raw, re.DOTALL)
if json_match:
try:
return json.loads(json_match.group())
except json.JSONDecodeError:
pass
return None
def _parse_l3_card(raw: str) -> L3Card | None:
"""解析 VLM 输出为 L3Card。解析失败返回 None。
参数:
raw: VLM 原始返回字符串。
返回:
L3Card 实例或 None(解析失败时)。
"""
data = _extract_json(raw)
if isinstance(data, dict):
try:
return L3Card(
frame_summary=str(data["frame_summary"]),
visible_entities=list(data["visible_entities"]),
ongoing_actions=list(data["ongoing_actions"]),
visible_text=list(data["visible_text"]),
spatial_layout=str(data["spatial_layout"]),
visual_attributes=dict(data["visual_attributes"]),
)
except (KeyError, TypeError, ValueError):
pass
return None
def _parse_l2_card(raw: str) -> L2Card | None:
"""解析 LLM 输出为 L2Card。解析失败返回 None。
参数:
raw: LLM 原始返回字符串。
返回:
L2Card 实例或 None(解析失败时)。
"""
data = _extract_json(raw)
if isinstance(data, dict):
try:
state_changes = data.get("state_changes")
if state_changes is not None:
state_changes = str(state_changes)
return L2Card(
event_description=str(data["event_description"]),
entities=list(data["entities"]),
actions=list(data["actions"]),
action_subjects=list(data["action_subjects"]),
visible_text=list(data["visible_text"]),
spatial_relations=str(data["spatial_relations"]),
state_changes=state_changes,
)
except (KeyError, TypeError, ValueError):
pass
return None
def _parse_l1_card(raw: str) -> L1Card | None:
"""解析 LLM 输出为 L1Card。解析失败返回 None。
参数:
raw: LLM 原始返回字符串。
返回:
L1Card 实例或 None(解析失败时)。
"""
data = _extract_json(raw)
if isinstance(data, dict):
try:
return L1Card(
scene_summary=str(data["scene_summary"]),
main_setting=str(data["main_setting"]),
key_entities=list(data["key_entities"]),
main_actions=list(data["main_actions"]),
topic_keywords=list(data["topic_keywords"]),
visible_text=list(data["visible_text"]),
temporal_flow=str(data["temporal_flow"]),
)
except (KeyError, TypeError, ValueError):
pass
return None
# ---------------------------------------------------------------------------
# 节点查找辅助
# ---------------------------------------------------------------------------
def _build_node_lookup(
index: TreeIndex,
) -> tuple[
dict[str, L3Node],
dict[str, L2Node],
dict[str, L1Node],
dict[str, L2Node],
dict[str, L1Node],
]:
"""构建节点 ID 到节点的查找表 + 子节点到父节点的映射。
参数:
index: 树索引。
返回:
(l3_by_id, l2_by_id, l1_by_id, l3_parent_l2, l2_parent_l1)
- l3_by_id: L3 节点 ID → L3Node
- l2_by_id: L2 节点 ID → L2Node
- l1_by_id: L1 节点 ID → L1Node
- l3_parent_l2: L3 节点 ID → 其父 L2Node
- l2_parent_l1: L2 节点 ID → 其父 L1Node
"""
l3_by_id: dict[str, L3Node] = {}
l2_by_id: dict[str, L2Node] = {}
l1_by_id: dict[str, L1Node] = {}
l3_parent_l2: dict[str, L2Node] = {}
l2_parent_l1: dict[str, L1Node] = {}
for l1 in index.roots:
l1_by_id[l1.id] = l1
for l2 in l1.children:
l2_by_id[l2.id] = l2
l2_parent_l1[l2.id] = l1
for l3 in l2.children:
l3_by_id[l3.id] = l3
l3_parent_l2[l3.id] = l2
return l3_by_id, l2_by_id, l1_by_id, l3_parent_l2, l2_parent_l1
# ---------------------------------------------------------------------------
# 字幕辅助
# ---------------------------------------------------------------------------
def _build_subtitle_block(
srt_entries: list[SRTEntry] | None,
timestamp: float | None,
) -> str:
"""构建字幕注入文本块。
参数:
srt_entries: SRT 字幕条目列表。
timestamp: 帧时间戳(秒)。
返回:
字幕文本块字符串(无匹配时返回空字符串)。
"""
if not srt_entries or timestamp is None:
return ""
window = 2.0
start = max(0.0, timestamp - window)
end = timestamp + window
text = extract_subtitle_for_range(srt_entries, (start, end))
if not text:
return ""
return f"字幕信息:\n{text}\n"
# ---------------------------------------------------------------------------
# 主修复函数
# ---------------------------------------------------------------------------
async def repair_tree(
index: TreeIndex,
issues: list[NodeIssue],
vlm: VLMProvider,
llm: LLMProvider,
frames_dir: Path,
srt_entries: list[SRTEntry] | None = None,
) -> RepairStats:
"""修复有问题的节点,底向上级联。
流程:
1. 收集需修复的 L3 节点 → VLM 重新描述帧
2. 收集受影响的 L2 → LLM 从 L3 children 聚合
3. 收集受影响的 L1 → LLM 从 L2 children 聚合
参数:
index: 待修复的 TreeIndex(原地修改)。
issues: detect_issues() 返回的问题列表。
vlm: VLM 调用端口。
llm: LLM 调用端口。
frames_dir: 帧文件根目录。
srt_entries: 字幕条目列表(可选)。
返回:
RepairStats 统计。
"""
stats = RepairStats()
if not issues:
logger.info("无修复任务,跳过")
return stats
# 构建查找表
l3_by_id, l2_by_id, l1_by_id, l3_parent_l2, l2_parent_l1 = _build_node_lookup(index)
# Step 1: 修复 L3 节点(仅处理 empty_field + level 3
l3_issues = [
issue for issue in issues if issue.issue_type == "empty_field" and issue.level == 3
]
affected_l2_ids: set[str] = set()
for issue in l3_issues:
l3_node = l3_by_id.get(issue.node_id)
if l3_node is None:
logger.warning(
"L3 节点 ID 未在树中找到,跳过",
node_id=issue.node_id,
)
continue
# 查找帧文件
if l3_node.frame_path is None:
logger.warning(
"L3 节点无 frame_path,跳过",
node_id=issue.node_id,
)
continue
frame_file = frames_dir / l3_node.frame_path
if not frame_file.exists():
logger.warning(
"L3 帧文件不存在,跳过修复",
node_id=issue.node_id,
frame_path=str(frame_file),
)
continue
# 获取 L2 父节点描述作为上下文
parent_l2 = l3_parent_l2.get(issue.node_id)
l2_description = parent_l2.card.event_description if parent_l2 else ""
# 构建字幕块
subtitle_block = _build_subtitle_block(srt_entries, l3_node.timestamp)
# VLM 重新描述帧
prompt = _L3_REPAIR_PROMPT.format(
l2_description=l2_description,
subtitle_block=subtitle_block,
)
messages = [{"role": "user", "content": prompt}]
try:
response = await vlm.chat_with_images(messages, [str(frame_file)])
except Exception as exc:
logger.warning(
"L3 修复 VLM 调用失败,跳过: {}",
exc,
node_id=issue.node_id,
)
continue
new_card = _parse_l3_card(response.content)
if new_card is None:
logger.warning(
"L3 修复 VLM 输出解析失败,跳过",
node_id=issue.node_id,
raw_preview=response.content[:200],
)
continue
# 原地替换 cardL3Node.card 不是 frozen dataclass 的限制字段)
l3_node.card = new_card
stats.l3_repaired += 1
# 标记受影响的 L2 父节点
if parent_l2 is not None:
affected_l2_ids.add(parent_l2.id)
logger.debug(
"L3 节点修复完成",
node_id=issue.node_id,
frame_summary=new_card.frame_summary[:50],
)
# Step 2: 重生成受影响的 L2 节点
affected_l1_ids: set[str] = set()
for l2_id in affected_l2_ids:
l2_node = l2_by_id.get(l2_id)
if l2_node is None:
continue
# 从 L3 children 聚合描述
l3_texts = "\n".join(f"- {l3.card.frame_summary}" for l3 in l2_node.children)
prompt = _L2_REGEN_PROMPT.format(l3_texts=l3_texts)
messages = [{"role": "user", "content": prompt}]
try:
response = await llm.chat(messages)
except Exception as exc:
logger.warning(
"L2 重生成 LLM 调用失败,跳过: {}",
exc,
l2_id=l2_id,
)
continue
new_card = _parse_l2_card(response.content)
if new_card is None:
logger.warning(
"L2 重生成 LLM 输出解析失败,跳过",
l2_id=l2_id,
raw_preview=response.content[:200],
)
continue
l2_node.card = new_card
stats.l2_regenerated += 1
# 标记受影响的 L1 父节点
parent_l1 = l2_parent_l1.get(l2_id)
if parent_l1 is not None:
affected_l1_ids.add(parent_l1.id)
logger.debug(
"L2 节点重生成完成",
l2_id=l2_id,
event_description=new_card.event_description[:50],
)
# Step 3: 重生成受影响的 L1 节点
for l1_id in affected_l1_ids:
l1_node = l1_by_id.get(l1_id)
if l1_node is None:
continue
# 从 L2 children 聚合描述
l2_texts = "\n".join(f"- {l2.card.event_description}" for l2 in l1_node.children)
prompt = _L1_REGEN_PROMPT.format(l2_texts=l2_texts)
messages = [{"role": "user", "content": prompt}]
try:
response = await llm.chat(messages)
except Exception as exc:
logger.warning(
"L1 重生成 LLM 调用失败,跳过: {}",
exc,
l1_id=l1_id,
)
continue
new_card = _parse_l1_card(response.content)
if new_card is None:
logger.warning(
"L1 重生成 LLM 输出解析失败,跳过",
l1_id=l1_id,
raw_preview=response.content[:200],
)
continue
l1_node.card = new_card
stats.l1_regenerated += 1
logger.debug(
"L1 节点重生成完成",
l1_id=l1_id,
scene_summary=new_card.scene_summary[:50],
)
logger.info(
"树修复完成",
l3_repaired=stats.l3_repaired,
l2_regenerated=stats.l2_regenerated,
l1_regenerated=stats.l1_regenerated,
)
return stats
+4 -3
View File
@@ -84,10 +84,11 @@ def deduplicate_field(values: list[str]) -> list[str]:
seen: set[str] = set() seen: set[str] = set()
result: list[str] = [] result: list[str] = []
for v in values: for v in values:
key = v.strip().lower() s = str(v).strip()
key = s.lower()
if key and key not in seen: if key and key not in seen:
seen.add(key) seen.add(key)
result.append(v) result.append(s)
return result return result
@@ -278,7 +279,7 @@ def apply_injections(index: TreeIndex, injections: list[dict[str, Any]]) -> Supp
stats.facts_skipped += 1 stats.facts_skipped += 1
continue continue
inject_value = instr.get("inject_value", "") inject_value = str(instr.get("inject_value", "")).strip()
if not inject_value: if not inject_value:
stats.facts_skipped += 1 stats.facts_skipped += 1
continue continue
+4 -12
View File
@@ -15,9 +15,7 @@ APPENDIX_MAX_CHARS = 2000 # appendix 区软上限(守设计「长度上限+wa
MOMENTUM_START = "<!-- MOMENTUM_START -->" MOMENTUM_START = "<!-- MOMENTUM_START -->"
MOMENTUM_END = "<!-- MOMENTUM_END -->" MOMENTUM_END = "<!-- MOMENTUM_END -->"
MOMENTUM_MAX_CHARS = 2000 # momentum 区软上限(与 appendix 一致:超限 warning 不截断) MOMENTUM_MAX_CHARS = 2000 # momentum 区软上限(与 appendix 一致:超限 warning 不截断)
MOMENTUM_HEADING = ( MOMENTUM_HEADING = "## 动量指导(每轮重写,勿手改)" # replace_momentum 写入的固定标题行
"## 动量指导(每轮重写,勿手改)" # replace_momentum 写入的固定标题行
)
def momentum_region_bounds(text: str) -> tuple[int, int] | None: def momentum_region_bounds(text: str) -> tuple[int, int] | None:
@@ -303,9 +301,7 @@ def _insert_at(content: str, at: int, payload: str) -> str:
return head + "\n\n" + payload + "\n" return head + "\n\n" + payload + "\n"
def _do_append( def _do_append(content: str, payload: str, ranges: list[tuple[int, int]]) -> tuple[str, str]:
content: str, payload: str, ranges: list[tuple[int, int]]
) -> tuple[str, str]:
"""执行 append 操作,返回更新后内容与状态字符串。""" """执行 append 操作,返回更新后内容与状态字符串。"""
return _insert_at(content, _append_at(content, ranges), payload), "applied_append" return _insert_at(content, _append_at(content, ranges), payload), "applied_append"
@@ -351,9 +347,7 @@ def _do_replace_delete(
return new_content, "applied_" + op return new_content, "applied_" + op
def _apply_one( def _apply_one(content: str, edit: dict, ranges: list[tuple[int, int]]) -> tuple[str, dict]:
content: str, edit: dict, ranges: list[tuple[int, int]]
) -> tuple[str, dict]:
"""应用单条 edit,返回 (更新后内容, 状态报告)。""" """应用单条 edit,返回 (更新后内容, 状态报告)。"""
if not isinstance(edit, dict): if not isinstance(edit, dict):
return content, { return content, {
@@ -382,9 +376,7 @@ def _apply_one(
return content, report return content, report
if op in ("replace", "delete"): if op in ("replace", "delete"):
content, report["status"] = _do_replace_delete( content, report["status"] = _do_replace_delete(op, content, target, payload, ranges)
op, content, target, payload, ranges
)
return content, report return content, report
logger.warning("未知 op,跳过: {}", op) logger.warning("未知 op,跳过: {}", op)
@@ -0,0 +1,146 @@
# 论文主图设计:Self-Evolving Search Agent 推理训练闭环
**日期** 2026-07-09 · **状态** 已获用户批准(口头) · **产出物** Figma 图(文件 `xnLGUkZottqnr4dsEt9fGq` Page 1 空白区)
## 1. 目标与定位
- **用途**:论文主图(目标会议存在文档分歧:CLAUDE.md=AAAI 2026ARCHITECTURE.md 与记忆=EMNLP 2026,以用户最终决定为准,不影响本图设计),展示推理训练部分的自进化闭环;不含建树与新题生成(建树已有独立图,位于同一 Figma 文件)。
- **核心叙事**:搜索 Agent 通过 推理→诊断→进化→门控 闭环自我改进;**Frozen LLM, trainable harness**——被"训练"的不是模型权重,而是版本化的 Skills+Prompts。
- **差异化**:AVP/DVD 等相关工作画的是推理期内环(agent 怎么搜视频);本图内环只是一个面板,**训练期外环是主角**。
- **审稿人一句话记忆点**:这是一个不动模型权重的 PyTorch 式训练循环。
## 2. 已确认的关键决策
| 决策点 | 结论 |
|---|---|
| 构图 | 水平流水线 + 底部参数回流闭环(方案 A) |
| 版式 | 双栏跨页宽图,画布 2400×1050 px(≈2.3:1,缩印 180mm |
| 信息密度 | 四机制全部可见(Agent 内环 / 诊断瀑布 / patch 引擎 / CE-Gate+信息阶梯),去工程化(无熔断/缓存/遥测) |
| PyTorch 类比 | 底部独立双行对照条,与上方区域逐段对齐 |
| 示例贯穿 | 延续建树图同一 Video-MME 天文台视频,问题/诊断/patch 文本典型化设计 |
| 迭代维度 | Store 处 v1…vN 卡片堆叠 + ×N epochs 循环标记暗示,不加独立时间轴 |
| 绘制位置 | 与建树图同文件(素材直接复用),Page 1 空白区 y≥1800,不动现有图层 |
| 语言 | 图内文字全英文 |
## 3. 布局
> 下方 ASCII 草图为中文说明稿,仅示意区域关系;最终图层文字一律采用 §4/§5 的英文术语。
```
┌──────────────────────────────────────────────────────────────────┐
│ [输入] ┌──────────────┐ ┌───────────┐ ┌──────────┐ ┌──────┐ │
│ Q+缩略图→│ ① INFERENCE │→ │② DIAGNOSE │→ │③ EVOLVE │→ │④ CE- │ │
│ │ 树环境+内环 │ │ 归因瀑布 │ │ patch引擎 │ │ GATE │ │
│ └──────↑───────┘ └───────────┘ └──────────┘ └──┬───┘ │
│ │ read ┌─────────────────┐ accept│ │
│ └─────────────────│⑤ Skills+Prompts │←──────┘ │
│ │ Store v1…vN ▤▤ │ reject→保基线
│ └─────────────────┘ │
├──────────────────────────────────────────────────────────────────┤
│ DataLoader│forward()│backward()│optimizer.step()│grad clip│nn.Parameter│
└──────────────────────────────────────────────────────────────────┘
```
**数据流勘误记录**CE-Gate 位于 Evolve 之后(进化产出候选 → gate 用 e-process 验证候选 vs 基线 → accept 才写入 Store),信息阶梯为 gate 供给高信息量题序,而非控制输入难度。此顺序已与 `core/evolution/gate.py``app/harness/gate_ladder.py` 核实。
## 4. 五区域内容规格
术语均与代码核实一致(来源见 §7)。
### 4.1 输入区
- 问题卡(Q: *"What happens right after the dome opens?"* 措辞绘制时可打磨)+ 天文台视频缩略图 + 迷你树 icon,标注 "hierarchical video tree (Fig. 2)" 衔接建树图。
### 4.2 ① INFERENCEagent-controlled,图上唯一画成循环的面板)
- 树环境迷你版:L1/L2/L3 三层色带(绿/蓝/黄,复用建树图配色与缩略图)。
- Agent 内环轨迹:Thought → `search_similar``view_node``observe_frame``submit_answer`;侧边 `read_skill` 箭头(来自 ⑤,工具名与 `app/search/tools.py:56-58` 一致)。
- 示例结局:answer ✗(答错),输出 trace 流向 ②。
- 面板角标:*agent-controlled*;其余面板角标 *code-controlled*
### 4.3 ② DIAGNOSEcode-controlled
- 归因瀑布级联:`extraction failure → search failure → reasoning failure`(+ mixed 兜底),画成三级下落台阶。
- 二分岔:**defect**(改 skill 正文)vs **lapse**(记 appendix 提醒)。
- D1–D5 压缩为一排五个小 chipattribution / tool quality / search behavior / skill compliance / decision patterns。
- 示例:该题归因 `search failure` → 判 **defect**
### 4.4 ③ EVOLVE
- patch 流水线:candidate edits → **rank-and-clip** → apply patch。
- 侧边锁条带:**protected spans**appendix / momentum 区带锁图标,不可改写)。
- 示例 patch 片段:*"+ verify event boundary via L2 card before observe_frame"*。
- momentum 机制不单独出现(用户确认),仅隐含于锁条带。
### 4.5 ④ Validation · CE-GATE
- 面板标题 **Validation · CE-Gate**:块顺序验证(`validate.py` 配对翻转 W/L)作为输入喂 e-process——图上画为"candidate vs baseline 配对小图 → e 曲线"。
- e-process 小曲线:e 值随题数爬升,越过 `e_confirm` 虚线。
- 四出口:**accept (confirmed) / accept (provisional) / reject / continue**(代码中三种 reject 在图上合并,用户确认)。
- 侧挂小组件:信息阶梯(2:1 交错题序图标,标注 *info-max question ladder*),尺寸压小避免抢焦点。
- 视觉层级:accept 主路径线最粗;reject/continue 细灰次级线。
### 4.6 ⑤ Skills+Prompts Store
- v1…vN 卡片堆叠(复用建树图 Event Card 堆叠画法)+ 版本号 badge。
- accept 箭头写入 v(N+1);read 箭头回流至 ①,构成大闭环;循环标记 **×N epochs**。
## 5. PyTorch 对照条(最底部)
浅灰底横带,等宽字体,与上方区域逐段对齐:
| 上方区域 | 对照文字 |
|---|---|
| 输入 | `DataLoader` |
| ① | `model.forward()` |
| ② | `loss.backward()` |
| ③ | `optimizer.step()` |
| ④ | `grad clipping (validate)`(对应 CLAUDE.md 类比表中"进化 validation = grad clipping";④ 面板同时含 validate 配对翻转与 CE-Gate 判定) |
| ⑤ | `nn.Parameter` |
条带一侧放记忆点标语:*Frozen LLM, trainable harness*。
## 6. 视觉规范与素材复用
| 元素 | 方案 |
|---|---|
| 面板样式 | 白底、细虚线外框、顶部居中标题(沿用建树图) |
| ① | 淡绿系 · ② 淡橙红系(新增,饱和度对齐现有 pastel) · ③ 淡紫系(复用 VLM 紫) · ④ 淡蓝系 · ⑤ 白卡+badge |
| 直接复用 | 视频缩略图(candidate_a_t*)、L1/L2/L3 badge、Scene/Event/Frame Card 组件、VLM 紫块、箭头/chevron 样式 |
| 字体 | 与建树图一致(Inter);标题 24px / 正文 16-18px / 标注最小 15px 灰(2400px 画布缩印 180mm 后 15px ≈ 1.1mm12px 过小已弃用) |
| 图层组织 | 顶层 Frame 命名 `Main Figure — Self-Evolving Loop`,五区域各一个子 Group,便于后续人工微调 |
## 7. 术语出处(代码核实)
| 图上术语 | 来源 |
|---|---|
| gate 四出口 accept_confirmed / accept_provisional / reject×3 / continue | `core/evolution/gate.py:57-110` |
| 归因瀑布 extraction/search/reasoning/mixeddefect vs lapse | `core/evolution/diagnose.py:910-997` |
| D1-D5 五维聚合 | `core/evolution/diagnose.py:1095-1307`D2-D5)、`1551-1563` + `2230` + `2293-2296`D1 attribution distribution |
| 信息阶梯冷启动 2:1、信息量排序 | `app/harness/gate_ladder.py:57-117` |
| 块顺序验证配对翻转 W/L | `core/evolution/validate.py:12-69` |
| rank-and-clip、protected spans、appendix/momentum 区 | `core/evolution/evolve.py:186-594``core/evolution/patch.py:11-56` |
| agent 工具五件套(含 `read_skill` | `app/search/tools.py:33-73` |
| 版本目录 Store `store/skills/v{N}` / workspace 本地拷贝 | `app/harness/store.py:28-136``app/harness/workspace.py:103-108,150-152` |
## 8. 验收标准
1. 图在 Figma 中为独立顶层 Frame,可整体导出 PNG/SVG,缩印 180mm 宽时最小文字(12px 标注)仍可辨认。
2. 五区域 + 对照条齐全,闭环箭头(⑤→① read、④→⑤ accept)无歧义。
3. 全部术语与 §7 代码核实结果一致;无熔断/缓存/遥测等工程元素。
4. 风格与同文件建树图肉眼一致(配色、字体、面板语言、卡片组件)。
5. 不改动/移动建树图的任何现有图层。
## 9. 构图重构记录(2026-07-09 定稿后追加)
用户验收反馈:内容正确但"下半部空、无主线重点"。经方案比选(用户选 A),实施:
| 改动 | 内容 |
|---|---|
| 显式循环主干 | 面板间 chevron → 4px 黑色实心三角箭头(forward 主线);④→⑤ write 与 ⑤→① read 回流均为 4px 绿色实线带,⟳ ×N epochs 置于带上;黑/绿双色对应 forward / parameter-update 语义 |
| 底部压缩 | Store 2080px 全宽行 → 680×170 紧凑块(右缘对齐 ④,write 直指 v5);对照条 130→84px;画布 1050→**940**2.55:1 |
| 填充 | motto 24px 斜体移至左下空区;read 带起点加绿色圆点锚记 |
逐模块精修均经 Claude 自审 + Codex 独立审双 PASSQuestion/①/②/③/④/⑤+对照条/整图重构共 8 轮审核)。Codex 抓到的实质问题:④ 的 W/L 翻转数与 e 曲线出口统计不自洽(修正为序列省略号 + W=8·L=0E=56.78>e_confirm=20)、ladder 色块数与题数不符、read 线易误读为边框。
## 10. 被拒绝的备选方案
| 方案 | 拒绝原因 |
|---|---|
| B 上下双层 S 形回路 | PyTorch 对照条无法与面板逐段对齐,退化为角标 |
| C 中心辐射环形 | 2.3:1 宽幅下横向空间浪费大,机制细节难展开,与建树图直线叙事不一致 |
| 独立进化时间轴 | 占版面,与对照条拥挤;由 Store 版本堆叠 + ×N epochs 替代 |
@@ -0,0 +1,477 @@
---
id: question-gen-synth
title: 赛题生成工具设计(Question Generation Synthesis
type: design
created: 2026-07-09
status: draft
---
# 赛题生成工具设计
## 1. 目标与动机
让视频树自行生成与 Video-MME 原始赛题风格、难度近似的四选一选择题,用于自进化训练循环的 DataLoader。原始 900 道 benchmark 题保留为 held-out 最终评测集,避免"直接拿答案调"的审稿质疑。
**角色定位**:生成题 = 训练集,原始题 = 测试集。进化循环的改进效果最终由原始 benchmark 验证泛化能力。
**训练 vs 论文评测的区分**:训练循环全程使用生成题(三池切分——诊断池/验证池/test 池——均来自生成题),论文报告的 held-out 泛化指标是训练结束后,用最终 best 版本对原始 benchmark 全量 900 题单独跑推理得到的结果。两步分离,Runner 代码无需改动。
## 2. 模块结构与职责边界
### 2.1 文件布局
```
app/question_gen/
├── __init__.py ← 已有:re-export loader API
├── loader.py ← 已有:load_benchmark + stratified_sample
└── synthesizer.py ← 新增①:出题核心逻辑
app/harness/
└── factory.py ← 新增②:推理依赖组装(wiring)
tools/generate_questions.py ← 新增③:CLI 壳(generate + calibrate
```
### 2.2 职责切分
| 模块 | 职责 | 消费者 |
|------|------|--------|
| `synthesizer.py` | 题型-层级映射、锚节点采样、prompt 构造(few-shot)、embedding 去重、单题生成编排 | `tools/generate_questions.py` |
| `factory.py` | 给定 store 路径 + config → 组装 LLM/VLM/Embedding/SearchToolDispatcher/PromptManager 全套推理依赖 | `tools/generate_questions.py`(校准)、未来 `main.py`、Runner |
| `tools/generate_questions.py` | CLI 参数解析、并发编排(Semaphore)、进度日志、JSON 输出 | 用户直接运行 |
### 2.3 依赖方向
```mermaid
flowchart LR
TOOLS["tools/generate_questions.py"] --> SYN["app/question_gen/synthesizer"]
TOOLS --> FAC["app/harness/factory"]
TOOLS --> ADP["adapters/*"]
FAC --> SEARCH["app/search/*"]
FAC --> ENV["app/tree/environment"]
FAC --> ADP
SYN --> PROTO["core/protocols (VLMProvider, EmbeddingProvider via DI)"]
SYN --> TYPES["core/types (GeneratedQuestion)"]
SYN --> IDX["app/tree/index (TreeIndex)"]
```
全部合规——外层→内层,`core/` 不依赖任何外层。
### 2.4 与 QuestionGenerator Protocol 的关系
`app/ports.py` 已预留 `QuestionGenerator` Protocol。本设计**不实现该 Protocol**——出题是一次性离线工具而非运行时能力,Runner 不需要运行时出题。`synthesizer.py` 的函数式接口(`generate_one` 等纯函数 + async 编排)比 Protocol class 更适合工具脚本场景。`QuestionGenerator` Protocol 保留但标记为"预留,当前无实现",不删除——若未来需要运行时出题可基于 synthesizer 的纯函数包装实现。
### 2.5 方案选择与否决
| 方案 | 否决理由 |
|------|---------|
| A: 单体脚本(全部逻辑放 `tools/`) | 业务逻辑(题型映射、采样、prompt、去重)混在 CLI 编排中,不可独立测试;不匹配 repair 管线的 app/ + tools/ 分层惯例 |
| B: Protocol 实现 + 脚本编排(`adapters/` 实现 `QuestionGenerator`) | adapter 层语义是外部服务接口,出题逻辑是应用层业务规则,放 adapter 层语义不匹配 |
| **C: app/ 业务逻辑 + tools/ CLI 壳(采用)** | 与 repair 管线结构一致,Clean Architecture 依赖方向合规,业务逻辑可独立测试 |
## 3. synthesizer.py 核心设计
### 3.1 题型-层级映射
模块级常量,沿用 TRM4 设计文档的映射表:
| 锚定层级 | 题型 | 帧图 | 文本上下文 | 帧数 |
|---------|------|------|-----------|------|
| L3 | Object Recognition | 必须 | frame_summary | 1 |
| L3 | Attribute Perception | 必须 | frame_summary | 1 |
| L3 | OCR Problems | 必须 | frame_summary | 1 |
| L3 | Spatial Reasoning | 必须 | frame_summary + spatial_layout | 1 |
| L3 | Spatial Perception | 必须 | frame_summary | 1 |
| L2 | Action Recognition | 必须 | 事件 card | 2-3(子帧均匀采样) |
| L2 | Action Reasoning | 必须 | 事件 card | 2-3 |
| L2 | Counting Problem | 必须 | 事件 card | 2-3 |
| L2 | Temporal Perception | 可选 | 事件 card + time_range | 0-1 |
| L1 | Temporal Reasoning | 必须 | 根 card + 多个 L2 card(≥3 | 每 L2 取 1 张代表帧 |
| L1 | Information Synopsis | 必须 | 根 card + 全部 L2 card | 每 L2 取 1 张代表帧 |
| L1-L2 | Object Reasoning | 必须 | 2-3 个 L2 card | 每 L2 取 1 张代表帧 |
节点采样:每道题从全部视频树中随机选一棵,在对应层级随机选一个锚节点。同视频同题型不重复。L1 题型使用多个 L2 子节点联合输入时,按时间顺序组织节点,保持叙事连贯性。
### 3.2 AnchorContext 数据结构
```python
@dataclass(frozen=True)
class AnchorContext:
"""锚节点上下文——生成单道题所需的全部素材。"""
node_id: str # 锚节点 ID
card_text: str # 锚节点 card 序列化文本
frame_paths: list[str] # 帧图片路径
subtitle: str # 对应字幕(可空)
distractor_texts: list[str] # 同视频其他节点摘要(供 VLM 生成干扰项)
```
### 3.3 核心函数签名
```python
# 纯函数:从树中采样锚节点 + 帧 + 上下文
def sample_anchor(
tree: TreeIndex,
task_type: str,
used_node_ids: set[str],
rng: random.Random,
) -> AnchorContext
# 纯函数:组装 VLM promptsystem + user,含 few-shot exemplar
def build_generation_prompt(
task_type: str,
anchor: AnchorContext,
exemplars: list[GeneratedQuestion],
) -> tuple[list[dict], list[str]]
# 返回:(messages, image_paths) — 直接喂给 VLMProvider
# 纯函数:解析 VLM 返回的 JSON → 部分字段字典
# source_nodes 和 difficulty 由 generate_one 在 parse 后用 anchor 信息补齐
def parse_vlm_response(
raw: str,
video_id: str,
task_type: str,
seq: int,
) -> dict
# 返回:{"question_id", "question", "options", "answer"} 字典
# 调用方补齐 source_nodes/difficulty 后构造 GeneratedQuestion
# 纯函数:embedding 去重判定
def is_duplicate(
question_text: str,
pool_embeddings: np.ndarray,
embed_fn: Callable[[str | list[str]], np.ndarray],
threshold: float,
) -> bool
# 异步编排:生成单道题(含重试 + 去重循环)
async def generate_one(
vlm: VLMProvider,
embed_fn: Callable[[str | list[str]], np.ndarray],
tree: TreeIndex,
video_id: str,
task_type: str,
seq: int,
*,
exemplars: list[GeneratedQuestion],
pool_embeddings: np.ndarray,
used_node_ids: set[str],
max_retries: int,
similarity_threshold: float,
rng: random.Random,
session_id: str,
) -> GeneratedQuestion | None
```
**设计要点**
- 纯函数(sample_anchor、build_generation_prompt、parse_vlm_response、is_duplicate)可独立单测,不需要 VLM
- `generate_one` 是唯一异步函数,接收 `VLMProvider` 通过 DI
- 干扰项来自 `AnchorContext.distractor_texts`——同视频其他节点的真实信息
### 3.4 few-shot exemplar 选择
生成 prompt 包含 2-3 道同题型的原始 benchmark 题作示例,对齐风格和难度。
选择策略:
- 每题型取 `min(3, 该题型 benchmark 总量)`
- 按 seed 随机采样 + 跨视频去重(避免 exemplar 全来自同一视频)
- exemplar 是只读引用,不从 benchmark 评测集中移除
### 3.5 prompt 结构
```
System: 视频理解题目生成器,根据视频树节点内容和帧图生成 {task_type} 四选一题。
[2-3 道该题型原始 benchmark 题作示例]
约束:
- 问题必须基于给定节点内容,不能靠常识推断
- 干扰项来自同视频其他节点的真实信息(非凭空捏造)
- 难度和问法风格与示例一致
User: [锚节点 card + 字幕 + 帧图] + [同视频其他节点摘要,供干扰项素材]
```
### 3.6 去重机制
`EmbeddingProvider`nomic-embed-text-v1.5)对 question 文本做 embedding,余弦相似度检查:
| 检查对 | 阈值 | 处理 |
|--------|------|------|
| 生成题 vs 原始 benchmark 同题型题 | ≥ similarity_threshold | 丢弃,换节点重试 |
| 生成题 vs 已生成的同题型题 | ≥ similarity_threshold | 丢弃,换节点重试 |
维护 embedding 池(原始题 + 已通过的生成题),每生成一道新题即时查重。单题最多重试 `max_retries` 次。某题型连续耗尽重试配额时,脚本报错退出并输出已完成/未完成的题型统计,不静默少题。
**并发去重安全**embedding 池的"检查 + 添加"必须是原子操作。并发 `generate_one` 任务成功后,通过单线程汇总点(asyncio.Queue 或 await 后顺序提交)更新 embedding 池 + 写 JSON + 更新 progress,避免竞态导致相似题同时通过。
## 4. factory.py 推理依赖组装
### 4.1 解决的问题
目前 `Runner._make_tool_dispatch_fn()``_make_prompt_builder()` 都是 `raise NotImplementedError`,设计为"由 main.py 注入"。组装逻辑涉及 adapter 实例化 + app 组件串联,应提取为可复用的 factory 函数,避免在每个调用方(tools/ 脚本、未来 main.py)重复 wiring。
### 4.2 核心接口
```python
@dataclass(frozen=True)
class InferenceDeps:
"""跑一次推理所需的全套依赖(不含 HarnessLog,其生命周期由调用方管理)。"""
llm: LLMProvider
tool_dispatch_fn: Callable # SearchToolDispatcher.dispatch
prompt_builder: Callable # PromptManager 的偏函数
def build_inference_deps(
*,
store_dir: Path,
video_id: str,
prompts_dir: Path,
skills_dir: Path | None,
skill_mode: str,
embed_provider: EmbeddingProvider,
llm: LLMProvider,
vlm: VLMProvider,
ocr: OCRProvider | None,
verify_vision: bool,
anchor: bool,
assemble_mode: str,
) -> InferenceDeps
```
### 4.3 内部流程
```
build_inference_deps()
├── 加载 TreeIndexstore_dir/videos/{video_id}/tree.json
├── 构建 TreeEnvironmentindex=tree, frames_dir=videos/{video_id}/frames
├── 构建 SkillRegistryskills_dir,可选)
├── 构建 SearchToolDispatcherenv, tool_llm, vlm, ocr, prompts_dir,
│ skills, embed_fn, verify_vision, anchor, assemble_mode
├── 构建 PromptManagerprompts_dir)→ 偏函数化 prompt_builder(绑定 skill_mode
└── 返回 InferenceDeps
```
注意:`HarnessLog` 不放入 `InferenceDeps`——其生命周期由调用方通过 `with HarnessLog(...) as log` 管理,作为参数传给 `run_inference`
### 4.4 消费者
| 消费者 | 用法 |
|--------|------|
| `tools/generate_questions.py` calibrate | 按 video_id 分组题目,对每组调 `build_inference_deps` 构建对应视频树的依赖 → 分组 `run_inference` |
| 未来 `main.py --mode infer` | CLI 参数映射到 factory 参数 |
| `Runner` | `_make_tool_dispatch_fn` / `_make_prompt_builder` 改为委托 factory |
### 4.5 设计约束
- factory 只做**组装**,不持有状态——每次调用返回独立的 `InferenceDeps`
- adapter 实例(LLM/VLM/Embedding)由调用方创建并传入,factory 不管 adapter 生命周期
- 调用方自由决定 adapter 的复用策略(共享 vs 按需创建)
## 5. tools/generate_questions.py CLI 设计
### 5.1 子命令
```bash
# 生成
python tools/generate_questions.py generate \
--store-dir store \
--output-dir store/questions/generated/Video-MME \
--per-type 20 \
--similarity-threshold 0.85 \
--max-retries 3 \
--concurrency 8 \
--seed 42
# 校准(生成题 vs benchmark 基线对比)
python tools/generate_questions.py calibrate \
--generated-dir store/questions/generated/Video-MME \
--benchmark-dir store/questions/benchmarks/Video-MME \
--store-dir store \
--db-path results/calibrate.db \
--prompts-dir store/prompts \
--concurrency 4 \
--max-steps 15 \
--skill-mode auto \
--tolerance 0.10 \
--alpha 0.05 \
--baseline-db <可选,已有基线 DB 路径> \
--baseline-run-id <可选,已有基线 run_id>
```
除 baseline 复用参数外均必传,无默认值(CLAUDE.md §4.5)。`--baseline-db` + `--baseline-run-id` 可选但必须成对出现:有则从 DB 读 benchmark 基线,无则自动跑一次 benchmark 推理。
### 5.2 generate 流程
```
1. 加载 300 棵树的 video_id 列表
2. 加载 benchmark 题目(作为 few-shot exemplar 来源)
3. 初始化 embedding 池(benchmark 题 question text → embedding
4. 实例化 GovernedVLMClient + EmbeddingProvider
5. 检查断点续跑文件(progress.json
6. 对 12 题型 × per_type
├── 跳过已完成的(断点续跑)
├── 随机选视频 + 锚节点(同视频同题型不重复)
├── asyncio.Semaphore(concurrency) 并发调 generate_one
├── 成功 → 加入 embedding 池 + 追加到结果 + 更新 progress
└── 连续耗尽重试 → 报错退出,输出已完成/未完成统计
7. 按 video_id 分组写入 JSON
8. 全部完成后删除 progress.json
```
### 5.3 calibrate 流程
```
1. load_benchmark 加载生成题和 benchmark 题
2. 获取 benchmark 基线:
├── 有 --baseline-db + --baseline-run-id → 从 DB 读 per_task_type accuracy
└── 没有 → 按 video_id 分组 benchmark 题 → 每组 build_inference_deps
→ 分组 run_inference → 汇总存 DB
3. 按 video_id 分组生成题 → 每组 build_inference_deps → 分组 run_inference
(每组使用对应视频的 TreeEnvironment,避免跨视频树错用)
4. 汇总两组 per_task_type accuracy,对比(Fisher exact test
5. 输出对比表 + 判定结果
6. 存在 FAIL → 退出码 1
```
### 5.4 tools/ 脚本职责边界
脚本**只做**argparse、adapter 实例化(读 `.env`)、Semaphore 并发、进度日志(loguru)、JSON 写入、calibrate 时调 factory + run_inference。
脚本**不做**:prompt 构造、节点采样、去重判定(synthesizer.py)、依赖组装逻辑(factory.py)。
## 6. 校准统计方法
### 6.1 问题
benchmark 题型分布极不均匀(Spatial Perception 仅 3 道 vs Object Reasoning 240 道),固定 10% 阈值对小样本题型会产生误判——单题翻转即 33% 波动。
### 6.2 组合判定:Fisher exact test + effect size
`scipy.stats.fisher_exact` 对每个题型构造 2×2 列联表:
| | 答对 | 答错 |
|--|------|------|
| Benchmark | a | b |
| Generated | c | d |
判定规则:
| \|Δ\| > tolerance | p < α | 判定 | 含义 |
|---|---|---|---|
| ✗ | — | **PASS** | 差异在容忍范围内 |
| ✓ | ✓ | **FAIL** | 差异大且统计显著——生成题难度确实偏了 |
| ✓ | ✗ | **WARN** | 差异大但样本不足以确认——可能是噪声 |
### 6.3 优势
- 不需要 ad-hoc 的 `min_calibrate_size` 参数
- 小样本题型自动降级为 WARN——Fisher test 的 p-value 天然反映样本量不足
- CLI 只需两个语义清晰的统计参数:`--tolerance 0.10` + `--alpha 0.05`
- 退出码只看是否存在 FAIL(WARN 不阻塞)
### 6.4 检测灵敏度与 per_type 的关系
| per_type | 可检出的最小差异(大样本 benchmark 侧) |
|----------|---------------------------------------|
| 20 | ~30%(仅极大差异) |
| 50 | ~15%(中等差异) |
用户可根据需要的检测灵敏度选择 `--per-type`
### 6.5 输出格式
```
题型 | bench | gen | Δ | p-value | 判定
-------------------|--------|--------|---------|---------|--------
Spatial Perception | 66.7% | 40.0% | -26.7% | 0.590 | ⚠ WARN
Action Reasoning | 72.2% | 68.0% | -4.2% | 0.712 | ✓ PASS
Object Reasoning | 60.0% | 30.0% | -30.0% | 0.016 | ✗ FAIL
```
## 7. 断点续跑
生成 240 道题可能中断(VLM 故障、手动 Ctrl-C),沿用项目已有的 progress.json 模式:
```json
{
"completed": {
"Action Reasoning": ["gen-xyz-001", "gen-xyz-002"],
"Object Recognition": ["gen-abc-001"]
},
"output_dir": "store/questions/generated/Video-MME"
}
```
- 启动时检查 `{output_dir}/progress.json`,跳过已完成的题
- **恢复 embedding 池**:从已写出的 `{output_dir}/*.json` 重建已生成题的 embedding + `used_node_ids`,避免续跑后产生重复题
- 每道题写入 JSON 后立即更新 progress
- 全部完成后删除 progress.json
## 8. 输出格式
输出路径:`store/questions/generated/Video-MME/{video_id}.json`
```json
[
{
"question_id": "gen-{video_id}-{seq}",
"task_type": "Action Reasoning",
"question": "...",
"options": ["A. ...", "B. ...", "C. ...", "D. ..."],
"answer": "B",
"source_nodes": ["L1_000_L2_003"],
"difficulty": "medium"
}
]
```
与 loader schema 兼容(额外 `source_nodes`/`difficulty` 字段用于溯源),`load_benchmark` 零改动直接加载。
**训练集成**`--questions generated/Video-MME`
## 9. 受影响的既有接口
| 接口 | 影响 | 适配 |
|------|------|------|
| `load_benchmark` | 无 | 输出与 loader schema 兼容(额外 source_nodes/difficulty 字段用于溯源) |
| `RunConfig.questions` | 无 | 传 `generated/Video-MME` |
| `build_or_load_pools` | 无 | 三池均来自生成题 |
| `Runner._make_tool_dispatch_fn` | 改造 | 委托 factory.py |
| `Runner._make_prompt_builder` | 改造 | 委托 factory.py |
| `_VIDEO_MME_TASK_TYPE_COUNT` | **前置修复** | 从 11 改为 12`app/harness/config.py:24`),影响验证池保底下限 |
## 10. 测试策略
### 10.1 synthesizer.py
| 测试 | 覆盖点 |
|------|--------|
| `test_sample_anchor` | 各层级题型正确采锚、同视频同题型不重复、树节点不足时报错 |
| `test_build_generation_prompt` | messages 结构正确、exemplar 注入、图片路径列表、干扰项素材包含 |
| `test_parse_vlm_response` | 正常解析、格式异常(缺字段/非法 JSON)报错 |
| `test_is_duplicate` | 相似度 ≥ 阈值判重、< 阈值通过、空池不判重 |
| `test_generate_one` | mock VLMProvider,验证重试+去重循环、耗尽重试返回 None |
### 10.2 factory.py
| 测试 | 覆盖点 |
|------|--------|
| `test_build_inference_deps` | fake LLM/VLM/Embedding 验证返回各字段非 None、类型正确 |
| `test_missing_tree_file` | tree.json 不存在时报错 |
### 10.3 tools/generate_questions.py(集成级)
| 测试 | 覆盖点 |
|------|--------|
| `test_generate_smoke` | mock VLM + 1 棵真实树 + per_type=1,验证 JSON 输出格式 |
| `test_progress_resume` | 中断后重启,跳过已完成题 |
| `test_calibrate_pass_fail` | mock 两组 accuracy,验证 Fisher + tolerance 组合判定 |
真实 VLM 调用的 integration test 不在此次范围——依赖外部服务,不适合 CI。
## 11. 实现约束
- 完整类型注解 + 中文 DocstringCLAUDE.md §4.2
- 禁用 `print()`,使用 loguruCLAUDE.md §4.2
- 脚本放 `tools/`,不被其他模块 importCLAUDE.md §5
- 并发模式:`asyncio.Semaphore`CLI `--concurrency` 指定(沿用项目既有模式)
- 所有 VLM 调用经过 `GovernedLLMClient` 治理栈(CLAUDE.md §4.9
@@ -0,0 +1,17 @@
---
type: design
node_id: design:paper-main-figure
title: "论文主图:Self-Evolving Search Agent 推理训练闭环"
date: 2026-07-09
---
# 论文主图:Self-Evolving Search Agent 推理训练闭环
完整设计文档见 [2026-07-09-paper-main-figure-design.md](2026-07-09-paper-main-figure-design.md)。
**选定方案**:水平流水线 + 底部参数回流闭环(输入 → Inference → Diagnose → Evolve → CE-Gate → Store 回流),底部 PyTorch 双行对照条逐段对齐;示例贯穿延续建树图的 Video-MME 天文台视频;绘制于同一 Figma 文件以复用素材。
**关键决策理由**:阅读顺序 = 数据流,与建树图左→右语言一致;对照条只有在单行流水线下才能逐段对齐。CE-Gate 位于 Evolve 之后(与 `core/evolution/gate.py` 核实),信息阶梯为 gate 供题序。
**被拒绝方案**:上下双层 S 形回路(对照条无法对齐)、中心辐射环形(宽幅空间浪费、细节难展开)、独立进化时间轴(占版面,由版本堆叠暗示替代)。
@@ -0,0 +1,9 @@
---
type: design
node_id: design:question-gen-synth
title: 赛题生成工具设计
date: 2026-07-09
---
# 赛题生成工具设计
+22
View File
@@ -55,6 +55,21 @@
"id": "plan:tree-repair-resilience", "id": "plan:tree-repair-resilience",
"label": "建树修复管线三项改造实现计划", "label": "建树修复管线三项改造实现计划",
"type": "plan" "type": "plan"
},
{
"id": "design:paper-main-figure",
"label": "论文主图:Self-Evolving Search Agent 推理训练闭环",
"type": "design"
},
{
"id": "design:question-gen-synth",
"label": "赛题生成工具设计",
"type": "design"
},
{
"id": "plan:question-gen-synth",
"label": "赛题生成工具实现计划",
"type": "plan"
} }
], ],
"links": [ "links": [
@@ -99,6 +114,13 @@
"relation": "implements", "relation": "implements",
"evidence": "实现设计文档的三项改造:遥测加固+断点续跑+并发", "evidence": "实现设计文档的三项改造:遥测加固+断点续跑+并发",
"added": "2026-07-09T04:08:15.312470+00:00" "added": "2026-07-09T04:08:15.312470+00:00"
},
{
"source": "plan:question-gen-synth",
"target": "design:question-gen-synth",
"relation": "implements",
"evidence": "计划实现设计文档中定义的 synthesizer + factory + CLI 三模块",
"added": "2026-07-09T09:05:43.697644+00:00"
} }
] ]
} }
+9 -3
View File
@@ -1,24 +1,29 @@
# Research Wiki 索引 # Research Wiki 索引
> 自动生成,更新时间:2026-07-09 04:08 UTC > 自动生成,更新时间:2026-07-09 09:05 UTC
## design (9) ## design (13)
- [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design` - [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design`
- [2026-07-07-app-harness-design](designs/2026-07-07-app-harness-design.md) `design:2026-07-07-app-harness-design` - [2026-07-07-app-harness-design](designs/2026-07-07-app-harness-design.md) `design:2026-07-07-app-harness-design`
- [2026-07-07-core-evolution-design](designs/2026-07-07-core-evolution-design.md) `design:2026-07-07-core-evolution-design` - [2026-07-07-core-evolution-design](designs/2026-07-07-core-evolution-design.md) `design:2026-07-07-core-evolution-design`
- [2026-07-08-tree-repair-resilience-design](designs/2026-07-08-tree-repair-resilience-design.md) `design:2026-07-08-tree-repair-resilience-design` - [2026-07-08-tree-repair-resilience-design](designs/2026-07-08-tree-repair-resilience-design.md) `design:2026-07-08-tree-repair-resilience-design`
- [2026-07-09-paper-main-figure-design](designs/2026-07-09-paper-main-figure-design.md) `design:2026-07-09-paper-main-figure-design`
- [出题模块迁移设计(question_gen](designs/2026-07-07-question-gen-design.md) `design:2026-07-07-question-gen-design` - [出题模块迁移设计(question_gen](designs/2026-07-07-question-gen-design.md) `design:2026-07-07-question-gen-design`
- [建树修复管线:熔断根因修复 + 断点续跑 + 并发改造](designs/tree-repair-resilience.md) `design:tree-repair-resilience` - [建树修复管线:熔断根因修复 + 断点续跑 + 并发改造](designs/tree-repair-resilience.md) `design:tree-repair-resilience`
- [建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移](designs/2026-07-07-tree-module-design.md) `design:2026-07-07-tree-module-design` - [建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移](designs/2026-07-07-tree-module-design.md) `design:2026-07-07-tree-module-design`
- [建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移](designs/tree-module-vertical-slice.md) `design:tree-module-vertical-slice` - [建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移](designs/tree-module-vertical-slice.md) `design:tree-module-vertical-slice`
- [搜索 Agent 装配层设计(app/search/](designs/2026-07-07-search-module-design.md) `design:2026-07-07-search-module-design` - [搜索 Agent 装配层设计(app/search/](designs/2026-07-07-search-module-design.md) `design:2026-07-07-search-module-design`
- [论文主图:Self-Evolving Search Agent 推理训练闭环](designs/paper-main-figure.md) `design:paper-main-figure`
- [赛题生成工具设计](designs/question-gen-synth.md) `design:question-gen-synth`
- [赛题生成工具设计(Question Generation Synthesis](designs/2026-07-09-question-gen-synth-design.md) `design:2026-07-09-question-gen-synth-design`
## plan (13) ## plan (15)
- [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm` - [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm`
- [2026-07-07-app-harness](plans/2026-07-07-app-harness.md) `plan:2026-07-07-app-harness` - [2026-07-07-app-harness](plans/2026-07-07-app-harness.md) `plan:2026-07-07-app-harness`
- [2026-07-07-core-evolution](plans/2026-07-07-core-evolution.md) `plan:2026-07-07-core-evolution` - [2026-07-07-core-evolution](plans/2026-07-07-core-evolution.md) `plan:2026-07-07-core-evolution`
- [2026-07-07-question-gen](plans/2026-07-07-question-gen.md) `plan:2026-07-07-question-gen` - [2026-07-07-question-gen](plans/2026-07-07-question-gen.md) `plan:2026-07-07-question-gen`
- [2026-07-07-tree-module-vertical-slice](plans/2026-07-07-tree-module-vertical-slice.md) `plan:2026-07-07-tree-module-vertical-slice` - [2026-07-07-tree-module-vertical-slice](plans/2026-07-07-tree-module-vertical-slice.md) `plan:2026-07-07-tree-module-vertical-slice`
- [2026-07-09-question-gen-synth](plans/2026-07-09-question-gen-synth.md) `plan:2026-07-09-question-gen-synth`
- [2026-07-09-tree-repair-resilience](plans/2026-07-09-tree-repair-resilience.md) `plan:2026-07-09-tree-repair-resilience` - [2026-07-09-tree-repair-resilience](plans/2026-07-09-tree-repair-resilience.md) `plan:2026-07-09-tree-repair-resilience`
- [app/harness/ 训练循环编排层实现计划](plans/app-harness.md) `plan:app-harness` - [app/harness/ 训练循环编排层实现计划](plans/app-harness.md) `plan:app-harness`
- [app/search/ 搜索 Agent 装配层实现计划](plans/2026-07-07-search-module.md) `plan:2026-07-07-search-module` - [app/search/ 搜索 Agent 装配层实现计划](plans/2026-07-07-search-module.md) `plan:2026-07-07-search-module`
@@ -26,4 +31,5 @@
- [question_gen 模块实现计划](plans/question-gen.md) `plan:question-gen` - [question_gen 模块实现计划](plans/question-gen.md) `plan:question-gen`
- [建树修复管线三项改造实现计划](plans/tree-repair-resilience.md) `plan:tree-repair-resilience` - [建树修复管线三项改造实现计划](plans/tree-repair-resilience.md) `plan:tree-repair-resilience`
- [建树模块竖切实现计划](plans/tree-module-vertical-slice.md) `plan:tree-module-vertical-slice` - [建树模块竖切实现计划](plans/tree-module-vertical-slice.md) `plan:tree-module-vertical-slice`
- [赛题生成工具实现计划](plans/question-gen-synth.md) `plan:question-gen-synth`
- [项目基础设施初始化计划](plans/infrastructure-setup.md) `plan:infrastructure-setup` - [项目基础设施初始化计划](plans/infrastructure-setup.md) `plan:infrastructure-setup`
+6
View File
@@ -27,3 +27,9 @@
- [2026-07-09 04:08 UTC] 新增 plan: 建树修复管线三项改造实现计划 (plan:tree-repair-resilience) - [2026-07-09 04:08 UTC] 新增 plan: 建树修复管线三项改造实现计划 (plan:tree-repair-resilience)
- [2026-07-09 04:08 UTC] 新增边: plan:tree-repair-resilience --implements--> design:tree-repair-resilience - [2026-07-09 04:08 UTC] 新增边: plan:tree-repair-resilience --implements--> design:tree-repair-resilience
- [2026-07-09 04:08 UTC] 重建索引: 22 篇页面 - [2026-07-09 04:08 UTC] 重建索引: 22 篇页面
- [2026-07-09 04:38 UTC] 新增 design: 论文主图:Self-Evolving Search Agent 推理训练闭环 (design:paper-main-figure)
- [2026-07-09 04:38 UTC] 重建索引: 24 篇页面
- [2026-07-09 09:05 UTC] 新增 design: 赛题生成工具设计 (design:question-gen-synth)
- [2026-07-09 09:05 UTC] 新增 plan: 赛题生成工具实现计划 (plan:question-gen-synth)
- [2026-07-09 09:05 UTC] 新增边: plan:question-gen-synth --implements--> design:question-gen-synth
- [2026-07-09 09:05 UTC] 重建索引: 28 篇页面
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,9 @@
---
type: plan
node_id: plan:question-gen-synth
title: 赛题生成工具实现计划
date: 2026-07-09
---
# 赛题生成工具实现计划
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
# 树修复管线:检测 + VLM 重生成 + 校验 + Q&A 补全
#
# 用法:
# bash scripts/repair_trees.sh # 默认 16 并发续跑
# bash scripts/repair_trees.sh --dry-run # 仅检测不修复
# bash scripts/repair_trees.sh --reaggregate-all # 强制全量重聚合
# CONCURRENCY=8 bash scripts/repair_trees.sh # 自定义并发数
#
# 日志输出:
# stderr → 终端实时显示
# logs/repair_trees.log → 完整日志(自动 rotation 50MB
# logs/repair_telemetry.db → LLM/VLM 调用遥测
# logs/repair_progress.json → 断点续跑进度
set -euo pipefail
CONCURRENCY="${CONCURRENCY:-16}"
# shellcheck source=/dev/null
source "$(conda info --base)/etc/profile.d/conda.sh"
conda activate Video-Tree-TRM
python tools/repair_trees.py \
--videos-dir store/videos \
--srt-dir data/Video-MME/subtitle \
--questions-dir store/questions/benchmarks/Video-MME \
--concurrency "$CONCURRENCY" \
"$@"
+208
View File
@@ -0,0 +1,208 @@
"""app/harness/factory.py 的单元测试。
验证 build_inference_deps 的返回类型、字段连接、以及错误路径。
"""
from __future__ import annotations
import json
from typing import TYPE_CHECKING
from unittest.mock import AsyncMock, MagicMock
if TYPE_CHECKING:
from pathlib import Path
import numpy as np
import pytest
from app.harness.factory import InferenceDeps, build_inference_deps
from core.types import GeneratedQuestion
class TestBuildInferenceDeps:
"""build_inference_deps 工厂函数测试。"""
def test_returns_inference_deps(self, tmp_path: Path) -> None:
"""用 fake adapters 验证返回类型和字段非 None。"""
# 准备一棵最小树
vid_dir = tmp_path / "videos" / "test_vid"
vid_dir.mkdir(parents=True)
(vid_dir / "frames").mkdir()
minimal_tree = {
"metadata": {"source_path": "test", "modality": "video"},
"roots": [
{
"id": "L1_000",
"card": {
"scene_summary": "s",
"main_setting": "s",
"key_entities": [],
"main_actions": [],
"topic_keywords": [],
"visible_text": [],
"temporal_flow": "s",
},
"time_range": [0, 10],
"children": [],
}
],
}
(vid_dir / "tree.json").write_text(json.dumps(minimal_tree))
# prompts
prompts_dir = tmp_path / "prompts"
prompts_dir.mkdir()
(prompts_dir / "system.md").write_text("You are a search agent.")
fake_llm = AsyncMock()
fake_vlm = AsyncMock()
fake_embed = MagicMock()
fake_embed.dim = 4
fake_embed.embed = lambda t: np.zeros((1, 4), dtype=np.float32)
deps = build_inference_deps(
store_dir=tmp_path,
video_id="test_vid",
prompts_dir=prompts_dir,
skills_dir=None,
skill_mode="none",
embed_provider=fake_embed,
llm=fake_llm,
vlm=fake_vlm,
ocr=None,
verify_vision=False,
anchor=False,
assemble_mode="ids",
)
assert isinstance(deps, InferenceDeps)
assert deps.llm is fake_llm
assert callable(deps.tool_dispatch_fn)
assert callable(deps.prompt_builder)
# 验证 prompt_builder 实际可用(连接正确)
fake_q = GeneratedQuestion(
question_id="q1",
video_id="test_vid",
task_type="Object Recognition",
question="What?",
options=("A. X", "B. Y", "C. Z", "D. W"),
answer="A",
source_nodes=(),
difficulty="medium",
)
system, user = deps.prompt_builder(fake_q)
assert isinstance(system, str) and len(system) > 0
assert isinstance(user, str) and "What?" in user
def test_missing_tree_raises(self, tmp_path: Path) -> None:
"""tree.json 不存在时应抛出 FileNotFoundError。"""
prompts_dir = tmp_path / "prompts"
prompts_dir.mkdir()
(prompts_dir / "system.md").write_text("x")
vid_dir = tmp_path / "videos" / "nonexist"
vid_dir.mkdir(parents=True)
with pytest.raises(FileNotFoundError):
build_inference_deps(
store_dir=tmp_path,
video_id="nonexist",
prompts_dir=prompts_dir,
skills_dir=None,
skill_mode="none",
embed_provider=MagicMock(),
llm=AsyncMock(),
vlm=AsyncMock(),
ocr=None,
verify_vision=False,
anchor=False,
assemble_mode="ids",
)
def test_with_skills_dir(self, tmp_path: Path) -> None:
"""提供 skills_dir 时 skill 信息应正确注入到 prompt_builder 输出。"""
# 准备树
vid_dir = tmp_path / "videos" / "vid1"
vid_dir.mkdir(parents=True)
(vid_dir / "frames").mkdir()
minimal_tree = {
"metadata": {"source_path": "test", "modality": "video"},
"roots": [
{
"id": "L1_000",
"card": {
"scene_summary": "test scene",
"main_setting": "indoor",
"key_entities": [],
"main_actions": [],
"topic_keywords": [],
"visible_text": [],
"temporal_flow": "linear",
},
"time_range": [0, 5],
"children": [],
}
],
}
(vid_dir / "tree.json").write_text(json.dumps(minimal_tree))
# prompts
prompts_dir = tmp_path / "prompts"
prompts_dir.mkdir()
(prompts_dir / "system.md").write_text("Base system prompt.")
# skills
skills_dir = tmp_path / "skills"
skills_dir.mkdir()
(skills_dir / "always_nav.md").write_text(
"---\nname: always_nav\nalways: true\n---\nAlways navigate broadly."
)
(skills_dir / "action_skill.md").write_text(
"---\nname: action_skill\ntask_type: Action Reasoning\n---\nFocus on actions."
)
fake_llm = AsyncMock()
fake_vlm = AsyncMock()
fake_embed = MagicMock()
fake_embed.dim = 4
fake_embed.embed = lambda t: np.zeros((1, 4), dtype=np.float32)
deps = build_inference_deps(
store_dir=tmp_path,
video_id="vid1",
prompts_dir=prompts_dir,
skills_dir=skills_dir,
skill_mode="auto",
embed_provider=fake_embed,
llm=fake_llm,
vlm=fake_vlm,
ocr=None,
verify_vision=False,
anchor=False,
assemble_mode="ids",
)
fake_q = GeneratedQuestion(
question_id="q2",
video_id="vid1",
task_type="Action Reasoning",
question="What happened?",
options=("A. X", "B. Y", "C. Z", "D. W"),
answer="B",
source_nodes=(),
difficulty="easy",
)
system, user = deps.prompt_builder(fake_q)
# always skill 文本和 task_type skill 文本应出现在 system prompt 中
assert "Always navigate broadly" in system
assert "Focus on actions" in system
assert "What happened?" in user
def test_frozen_dataclass(self) -> None:
"""InferenceDeps 是 frozen dataclass,不可修改属性。"""
deps = InferenceDeps(
llm=AsyncMock(),
tool_dispatch_fn=lambda: None,
prompt_builder=lambda q: ("", ""),
)
with pytest.raises(AttributeError):
deps.llm = AsyncMock() # type: ignore[misc]
+417
View File
@@ -0,0 +1,417 @@
"""tools/generate_questions.py 单元测试。
覆盖断点续跑、exemplar 选取、embedding 池重建、JSON 追加写入等纯函数。
"""
from __future__ import annotations
import json
import random
import sys
from pathlib import Path
import numpy as np
import pytest
# 确保项目根目录在 sys.path 中
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from core.types import GeneratedQuestion
from tools.generate_questions import (
_append_to_json,
_calibrate_exit_code,
_judge_task_type,
_load_or_init_progress,
_rebuild_embedding_pool,
_save_progress,
_select_exemplars,
_validate_calibrate_args,
)
# ---------------------------------------------------------------------------
# 辅助工厂
# ---------------------------------------------------------------------------
def _make_question(
qid: str = "q1",
vid: str = "v1",
task_type: str = "Object Recognition",
question: str = "What is this?",
answer: str = "A",
) -> GeneratedQuestion:
"""构造测试用 GeneratedQuestion。"""
return GeneratedQuestion(
question_id=qid,
video_id=vid,
task_type=task_type,
question=question,
options=("A. X", "B. Y", "C. Z", "D. W"),
answer=answer,
source_nodes=(),
difficulty="medium",
)
# ---------------------------------------------------------------------------
# TestLoadOrInitProgress
# ---------------------------------------------------------------------------
class TestLoadOrInitProgress:
"""_load_or_init_progress 测试。"""
def test_init_fresh(self, tmp_path: Path) -> None:
"""目录为空时返回初始结构。"""
progress = _load_or_init_progress(tmp_path)
assert progress["completed"] == {}
assert progress["output_dir"] == str(tmp_path)
def test_load_existing(self, tmp_path: Path) -> None:
"""已有 progress.json 时正确加载。"""
data = {
"completed": {"Object Recognition": ["gen-x-001"]},
"output_dir": str(tmp_path),
}
(tmp_path / "progress.json").write_text(json.dumps(data))
progress = _load_or_init_progress(tmp_path)
assert "gen-x-001" in progress["completed"]["Object Recognition"]
def test_corrupted_json_reinits(self, tmp_path: Path) -> None:
"""损坏的 JSON 文件导致重新初始化。"""
(tmp_path / "progress.json").write_text("{invalid json")
progress = _load_or_init_progress(tmp_path)
assert progress["completed"] == {}
def test_invalid_completed_type_reinits(self, tmp_path: Path) -> None:
"""completed 字段类型不正确时重新初始化。"""
(tmp_path / "progress.json").write_text(
json.dumps({"completed": "not-a-dict", "output_dir": str(tmp_path)})
)
progress = _load_or_init_progress(tmp_path)
assert progress["completed"] == {}
# ---------------------------------------------------------------------------
# TestSaveProgress
# ---------------------------------------------------------------------------
class TestSaveProgress:
"""_save_progress 测试。"""
def test_atomic_write(self, tmp_path: Path) -> None:
"""原子写入 progress.json。"""
progress = {
"completed": {"Action Reasoning": ["gen-v1-001"]},
"output_dir": str(tmp_path),
}
_save_progress(tmp_path, progress)
written = json.loads((tmp_path / "progress.json").read_text())
assert written["completed"]["Action Reasoning"] == ["gen-v1-001"]
# 临时文件不应残留
assert not (tmp_path / "progress.json.tmp").exists()
# ---------------------------------------------------------------------------
# TestSelectExemplars
# ---------------------------------------------------------------------------
class TestSelectExemplars:
"""_select_exemplars 测试。"""
def test_selects_correct_type(self) -> None:
"""只选取匹配题型的示例。"""
qs = [
_make_question("q1", "v1", "Object Recognition", "Q1?"),
_make_question("q2", "v2", "Object Recognition", "Q2?"),
_make_question("q3", "v1", "Action Reasoning", "Q3?"),
]
result = _select_exemplars(qs, "Object Recognition", 3, random.Random(42))
assert all(q.task_type == "Object Recognition" for q in result)
assert len(result) == 2 # 只有 2 个可用
def test_cross_video_diversity(self) -> None:
"""优先从不同 video_id 选取示例。"""
qs = [_make_question(f"q{i}", f"v{i}", "Object Recognition", f"Q{i}?") for i in range(10)]
result = _select_exemplars(qs, "Object Recognition", 3, random.Random(42))
video_ids = {q.video_id for q in result}
assert len(video_ids) == 3 # 全部来自不同视频
def test_empty_benchmark(self) -> None:
"""benchmark 为空时返回空列表。"""
result = _select_exemplars([], "Object Recognition", 3, random.Random(42))
assert result == []
def test_no_matching_type(self) -> None:
"""无匹配题型时返回空列表。"""
qs = [_make_question("q1", "v1", "Action Reasoning", "Q1?")]
result = _select_exemplars(qs, "Object Recognition", 3, random.Random(42))
assert result == []
def test_request_more_than_available(self) -> None:
"""请求数超过可用数时返回全部。"""
qs = [
_make_question("q1", "v1", "Object Recognition", "Q1?"),
_make_question("q2", "v2", "Object Recognition", "Q2?"),
]
result = _select_exemplars(qs, "Object Recognition", 10, random.Random(42))
assert len(result) == 2
# ---------------------------------------------------------------------------
# TestProgressResume
# ---------------------------------------------------------------------------
class TestProgressResume:
"""断点续跑集成测试。"""
def test_skips_completed_and_rebuilds_pool(self, tmp_path: Path) -> None:
"""已完成的题目从 progress 加载,embedding 池从已生成 JSON 重建。"""
output_dir = tmp_path / "output"
output_dir.mkdir()
(output_dir / "test_vid.json").write_text(
json.dumps(
[
{
"question_id": "gen-test_vid-001",
"task_type": "Object Recognition",
"question": "Existing question?",
"options": ["A. X", "B. Y", "C. Z", "D. W"],
"answer": "A",
"source_nodes": ["L3_001"],
"difficulty": "medium",
}
]
)
)
progress = {
"completed": {"Object Recognition": ["gen-test_vid-001"]},
"output_dir": str(output_dir),
}
(output_dir / "progress.json").write_text(json.dumps(progress))
loaded = _load_or_init_progress(output_dir)
assert "gen-test_vid-001" in loaded["completed"]["Object Recognition"]
# ---------------------------------------------------------------------------
# TestRebuildEmbeddingPool
# ---------------------------------------------------------------------------
class TestRebuildEmbeddingPool:
"""_rebuild_embedding_pool 测试。"""
@staticmethod
def _fake_embed_fn(texts):
"""伪嵌入函数:返回固定维度的随机向量。"""
if isinstance(texts, str):
texts = [texts]
return np.random.RandomState(0).randn(len(texts), 8).astype(np.float32)
def test_empty_dir(self, tmp_path: Path) -> None:
"""空目录时所有池为空。"""
pools = _rebuild_embedding_pool(tmp_path, self._fake_embed_fn, [])
# 所有 12 种题型都应有条目
assert len(pools) >= 12
for v in pools.values():
assert v.shape[0] == 0 or v.ndim == 2
def test_with_generated_json(self, tmp_path: Path) -> None:
"""从已生成的 JSON 文件重建池。"""
(tmp_path / "vid1.json").write_text(
json.dumps(
[
{
"question_id": "gen-vid1-001",
"task_type": "Object Recognition",
"question": "Test question 1?",
"options": ["A. X", "B. Y", "C. Z", "D. W"],
"answer": "A",
"source_nodes": [],
"difficulty": "medium",
},
{
"question_id": "gen-vid1-002",
"task_type": "Object Recognition",
"question": "Test question 2?",
"options": ["A. X", "B. Y", "C. Z", "D. W"],
"answer": "B",
"source_nodes": [],
"difficulty": "medium",
},
]
)
)
pools = _rebuild_embedding_pool(tmp_path, self._fake_embed_fn, [])
assert pools["Object Recognition"].shape[0] == 2
assert pools["Object Recognition"].shape[1] == 8
def test_with_benchmark_questions(self, tmp_path: Path) -> None:
"""benchmark 题目也加入去重池。"""
benchmark = [
_make_question("bm1", "v1", "Action Reasoning", "Benchmark Q1?"),
_make_question("bm2", "v2", "Action Reasoning", "Benchmark Q2?"),
]
pools = _rebuild_embedding_pool(tmp_path, self._fake_embed_fn, benchmark)
assert pools["Action Reasoning"].shape[0] == 2
def test_progress_json_excluded(self, tmp_path: Path) -> None:
"""progress.json 不被当作题目文件。"""
(tmp_path / "progress.json").write_text(
json.dumps({"completed": {}, "output_dir": str(tmp_path)})
)
pools = _rebuild_embedding_pool(tmp_path, self._fake_embed_fn, [])
for v in pools.values():
assert v.shape[0] == 0 or v.ndim == 2
# ---------------------------------------------------------------------------
# TestAppendToJson
# ---------------------------------------------------------------------------
class TestAppendToJson:
"""_append_to_json 测试。"""
def test_create_new_file(self, tmp_path: Path) -> None:
"""文件不存在时创建新文件。"""
q = _make_question("gen-v1-001", "v1", "Object Recognition", "Q?")
_append_to_json(tmp_path, q)
written = json.loads((tmp_path / "v1.json").read_text())
assert len(written) == 1
assert written[0]["question_id"] == "gen-v1-001"
def test_append_to_existing(self, tmp_path: Path) -> None:
"""追加到已有文件。"""
existing = [
{
"question_id": "gen-v1-001",
"task_type": "Object Recognition",
"question": "Existing?",
"options": ["A. X", "B. Y", "C. Z", "D. W"],
"answer": "A",
"source_nodes": [],
"difficulty": "medium",
}
]
(tmp_path / "v1.json").write_text(json.dumps(existing))
q = _make_question("gen-v1-002", "v1", "Object Recognition", "New?")
_append_to_json(tmp_path, q)
written = json.loads((tmp_path / "v1.json").read_text())
assert len(written) == 2
assert written[1]["question_id"] == "gen-v1-002"
def test_different_video_ids(self, tmp_path: Path) -> None:
"""不同 video_id 写入不同文件。"""
q1 = _make_question("gen-v1-001", "v1", "Object Recognition", "Q1?")
q2 = _make_question("gen-v2-001", "v2", "Action Reasoning", "Q2?")
_append_to_json(tmp_path, q1)
_append_to_json(tmp_path, q2)
assert (tmp_path / "v1.json").exists()
assert (tmp_path / "v2.json").exists()
v1_data = json.loads((tmp_path / "v1.json").read_text())
v2_data = json.loads((tmp_path / "v2.json").read_text())
assert len(v1_data) == 1
assert len(v2_data) == 1
# ---------------------------------------------------------------------------
# TestCalibrateJudgment
# ---------------------------------------------------------------------------
class TestCalibrateJudgment:
"""_judge_task_type 校准判定测试。"""
def test_pass_when_delta_small(self) -> None:
"""差值在容忍范围内判定为 PASS。"""
verdict = _judge_task_type(
bench_correct=60,
bench_total=100,
gen_correct=12,
gen_total=20,
tolerance=0.10,
alpha=0.05,
)
assert verdict == "PASS"
def test_fail_when_delta_large_and_significant(self) -> None:
"""差值超阈值且统计显著判定为 FAIL。"""
verdict = _judge_task_type(
bench_correct=144,
bench_total=240,
gen_correct=6,
gen_total=20,
tolerance=0.10,
alpha=0.05,
)
assert verdict == "FAIL"
def test_warn_when_delta_large_but_not_significant(self) -> None:
"""差值超阈值但不统计显著判定为 WARN。"""
verdict = _judge_task_type(
bench_correct=2,
bench_total=3,
gen_correct=8,
gen_total=20,
tolerance=0.10,
alpha=0.05,
)
assert verdict == "WARN"
# ---------------------------------------------------------------------------
# TestCalibrateIntegration
# ---------------------------------------------------------------------------
class TestCalibrateIntegration:
"""calibrate 辅助函数集成测试。"""
def test_baseline_params_must_be_paired(self) -> None:
"""baseline 参数必须成对出现。"""
with pytest.raises(ValueError, match="成对"):
_validate_calibrate_args(baseline_db="some.db", baseline_run_id=None)
def test_baseline_params_both_none_ok(self) -> None:
"""两个参数都为 None 不报错。"""
_validate_calibrate_args(baseline_db=None, baseline_run_id=None)
def test_baseline_params_both_provided_ok(self) -> None:
"""两个参数都提供不报错。"""
_validate_calibrate_args(baseline_db="some.db", baseline_run_id="run-001")
def test_baseline_run_id_only_raises(self) -> None:
"""只提供 run_id 也报错。"""
with pytest.raises(ValueError, match="成对"):
_validate_calibrate_args(baseline_db=None, baseline_run_id="run-001")
def test_has_fail_returns_exit_code_1(self) -> None:
"""存在 FAIL 时返回退出码 1。"""
verdicts = {"Object Recognition": "PASS", "Action Reasoning": "FAIL"}
assert _calibrate_exit_code(verdicts) == 1
def test_all_pass_or_warn_returns_exit_code_0(self) -> None:
"""全部 PASS 或 WARN 时返回退出码 0。"""
verdicts = {"Object Recognition": "PASS", "Spatial Perception": "WARN"}
assert _calibrate_exit_code(verdicts) == 0
def test_all_pass_returns_exit_code_0(self) -> None:
"""全部 PASS 时返回退出码 0。"""
verdicts = {"Object Recognition": "PASS", "Action Reasoning": "PASS"}
assert _calibrate_exit_code(verdicts) == 0
def test_empty_verdicts_returns_exit_code_0(self) -> None:
"""空 verdicts 时返回退出码 0。"""
assert _calibrate_exit_code({}) == 0
+13 -6
View File
@@ -568,28 +568,35 @@ class TestValSizeFloor:
"""val_size >= eval_min_per_class * 11 的下限校验。""" """val_size >= eval_min_per_class * 11 的下限校验。"""
def test_val_size_below_floor_rejected(self) -> None: def test_val_size_below_floor_rejected(self) -> None:
"""val_size < eval_min_per_class * 11 应抛出 ValueError。 """val_size < eval_min_per_class * 12 应抛出 ValueError。
eval_min_per_class=3 → 下限 = 3 * 11 = 33val_size=30 不足。 eval_min_per_class=3 → 下限 = 3 * 12 = 36val_size=30 不足。
""" """
cfg = _make_config(eval_min_per_class=3, val_size=30) cfg = _make_config(eval_min_per_class=3, val_size=30)
with pytest.raises(ValueError, match="val_size"): with pytest.raises(ValueError, match="val_size"):
_validate(cfg) _validate(cfg)
def test_val_size_at_floor_accepted(self) -> None: def test_val_size_at_floor_accepted(self) -> None:
"""val_size == eval_min_per_class * 11 应通过。""" """val_size == eval_min_per_class * 12 应通过。"""
cfg = _make_config(eval_min_per_class=3, val_size=33) cfg = _make_config(eval_min_per_class=3, val_size=36)
_validate(cfg) _validate(cfg)
def test_val_size_above_floor_accepted(self) -> None: def test_val_size_above_floor_accepted(self) -> None:
"""val_size > eval_min_per_class * 11 应通过。""" """val_size > eval_min_per_class * 12 应通过。"""
cfg = _make_config(eval_min_per_class=2, val_size=100) cfg = _make_config(eval_min_per_class=2, val_size=100)
_validate(cfg) _validate(cfg)
def test_default_yaml_values_satisfy_floor(self) -> None: def test_default_yaml_values_satisfy_floor(self) -> None:
"""default.yaml 的默认值(val_size=30, eval_min_per_class=2)应满足下限。 """default.yaml 的默认值(val_size=30, eval_min_per_class=2)应满足下限。
下限 = 2 * 11 = 22val_size=30 >= 22,通过。 下限 = 2 * 12 = 24val_size=30 >= 24,通过。
""" """
cfg = _make_config() cfg = _make_config()
_validate(cfg) # 不应抛出异常 _validate(cfg) # 不应抛出异常
def test_video_mme_task_type_count_is_12():
"""Video-MME 实际有 12 种题型,常量必须与之一致。"""
from app.harness.config import _VIDEO_MME_TASK_TYPE_COUNT
assert _VIDEO_MME_TASK_TYPE_COUNT == 12
+8 -8
View File
@@ -119,25 +119,25 @@ class TestDetectIssues:
issues = detect_issues(index) issues = detect_issues(index)
assert any(i.issue_type == "no_children" and i.level == 1 for i in issues) assert any(i.issue_type == "no_children" and i.level == 1 for i in issues)
def test_empty_visible_entities(self) -> None: def test_empty_visible_entities_not_flagged(self) -> None:
"""visible_entities 为空触发 empty_field。""" """visible_entities 为空是合法状态(静物/黑帧),不触发 empty_field。"""
card = L3Card("正常描述", [], ["动作"], [], "居中", {}) card = L3Card("正常描述", [], ["动作"], [], "居中", {})
l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0) l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0)
l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3]) l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3])
l1 = L1Node(id="l1_0", card=_card_l1(), time_range=(0.0, 10.0), children=[l2]) l1 = L1Node(id="l1_0", card=_card_l1(), time_range=(0.0, 10.0), children=[l2])
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1]) index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
issues = detect_issues(index) issues = detect_issues(index)
assert any(i.issue_type == "empty_field" and "visible_entities" in i.details for i in issues) assert not any(i.issue_type == "empty_field" for i in issues)
def test_empty_ongoing_actions(self) -> None: def test_empty_ongoing_actions_not_flagged(self) -> None:
"""ongoing_actions 为空触发 empty_field。""" """ongoing_actions 为空是合法状态(静物/黑帧),不触发 empty_field。"""
card = L3Card("正常描述", ["实体"], [], [], "居中", {}) card = L3Card("正常描述", ["实体"], [], [], "居中", {})
l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0) l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0)
l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3]) l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3])
l1 = L1Node(id="l1_0", card=_card_l1(), time_range=(0.0, 10.0), children=[l2]) l1 = L1Node(id="l1_0", card=_card_l1(), time_range=(0.0, 10.0), children=[l2])
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1]) index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
issues = detect_issues(index) issues = detect_issues(index)
assert any(i.issue_type == "empty_field" and "ongoing_actions" in i.details for i in issues) assert not any(i.issue_type == "empty_field" for i in issues)
def test_empty_spatial_layout(self) -> None: def test_empty_spatial_layout(self) -> None:
"""spatial_layout 为空也触发 empty_field。""" """spatial_layout 为空也触发 empty_field。"""
@@ -150,7 +150,7 @@ class TestDetectIssues:
assert any(i.issue_type == "empty_field" and "spatial_layout" in i.details for i in issues) assert any(i.issue_type == "empty_field" and "spatial_layout" in i.details for i in issues)
def test_multiple_empty_fields_single_issue(self) -> None: def test_multiple_empty_fields_single_issue(self) -> None:
"""多个字段同时为空只产生一个 issue,details 列出所有空字段。""" """多个必填字段同时为空只产生一个 issue,details 列出所有空字段。"""
card = L3Card("", [], [], [], "", {}) card = L3Card("", [], [], [], "", {})
l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0) l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0)
l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3]) l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3])
@@ -159,7 +159,7 @@ class TestDetectIssues:
issues = [i for i in detect_issues(index) if i.issue_type == "empty_field"] issues = [i for i in detect_issues(index) if i.issue_type == "empty_field"]
assert len(issues) == 1 assert len(issues) == 1
assert "frame_summary" in issues[0].details assert "frame_summary" in issues[0].details
assert "visible_entities" in issues[0].details assert "spatial_layout" in issues[0].details
def test_time_gap(self) -> None: def test_time_gap(self) -> None:
l3_a = L3Node(id="l1_0_l2_0_l3_0", card=_card_l3(), timestamp=1.0) l3_a = L3Node(id="l1_0_l2_0_l3_0", card=_card_l3(), timestamp=1.0)
+71
View File
@@ -0,0 +1,71 @@
"""修复管线断点续跑 progress 管理测试。"""
from __future__ import annotations
import asyncio
import json
import pytest
def test_load_progress_missing_file(tmp_path):
"""progress 文件不存在时返回空集合。"""
from tools.repair_trees import load_progress
result = load_progress(tmp_path / "nonexistent.json")
assert result == set()
def test_load_progress_valid_file(tmp_path):
"""正常读取已有 progress 文件。"""
from tools.repair_trees import load_progress
path = tmp_path / "progress.json"
path.write_text(json.dumps({"finished_video_ids": ["vid_a", "vid_b"]}))
result = load_progress(path)
assert result == {"vid_a", "vid_b"}
def test_load_progress_corrupted_file(tmp_path):
"""损坏的 JSON 文件返回空集合(不抛异常)。"""
from tools.repair_trees import load_progress
path = tmp_path / "progress.json"
path.write_text("{invalid json")
result = load_progress(path)
assert result == set()
@pytest.mark.asyncio
async def test_save_progress_atomic(tmp_path):
"""save_progress 原子写入,并发调用不丢失更新。"""
from tools.repair_trees import save_progress
path = tmp_path / "progress.json"
lock = asyncio.Lock()
await save_progress(path, lock, "vid_a")
await save_progress(path, lock, "vid_b")
data = json.loads(path.read_text())
assert set(data["finished_video_ids"]) == {"vid_a", "vid_b"}
@pytest.mark.asyncio
async def test_save_progress_concurrent(tmp_path):
"""16 路并发 save_progress 不丢失更新。"""
from tools.repair_trees import save_progress
path = tmp_path / "progress.json"
lock = asyncio.Lock()
tasks = [save_progress(path, lock, f"vid_{i}") for i in range(16)]
await asyncio.gather(*tasks)
data = json.loads(path.read_text())
assert len(data["finished_video_ids"]) == 16
def test_should_skip_finished():
"""已在 finished 集合中的视频应跳过。"""
from tools.repair_trees import should_skip_video
finished = {"vid_a", "vid_b"}
assert should_skip_video("vid_a", finished, reaggregate_all=False) is True
assert should_skip_video("vid_c", finished, reaggregate_all=False) is False
def test_should_skip_reaggregate_all_forces_rerun():
"""--reaggregate-all 标志强制不跳过。"""
from tools.repair_trees import should_skip_video
finished = {"vid_a"}
assert should_skip_video("vid_a", finished, reaggregate_all=True) is False
+293
View File
@@ -0,0 +1,293 @@
"""修复重生成器单元测试。"""
from __future__ import annotations
import asyncio
import json
from pathlib import Path
from app.tree.index import (
IndexMeta,
L1Card,
L1Node,
L2Card,
L2Node,
L3Card,
L3Node,
TreeIndex,
)
from app.tree.repair.detector import NodeIssue
from app.tree.repair.regenerator import RepairStats, repair_tree
from core.types import LLMResponse
def _mock_response(content: str) -> LLMResponse:
"""构造模拟 LLMResponse。"""
return LLMResponse(
content=content,
thinking="",
model="mock",
provider="mock",
prompt_tokens=0,
completion_tokens=0,
latency_ms=0,
ttft_ms=None,
max_inter_token_ms=None,
cache_hit=False,
call_id="mock",
)
class MockVLM:
"""模拟 VLM 端口,返回固定的 L3Card JSON。"""
def __init__(self) -> None:
self.call_count = 0
async def chat_with_images(
self,
messages: list[dict],
images: list,
**kw: object,
) -> LLMResponse:
"""模拟 VLM 图文调用。"""
self.call_count += 1
return _mock_response(
json.dumps(
{
"frame_summary": "修复后的帧描述",
"visible_entities": ["修复实体"],
"ongoing_actions": ["修复动作"],
"visible_text": [],
"spatial_layout": "居中",
"visual_attributes": {"lighting": "明亮"},
}
)
)
class MockLLM:
"""模拟 LLM 端口,根据 prompt 内容返回 L2Card 或 L1Card JSON。"""
def __init__(self) -> None:
self.call_count = 0
async def chat(
self,
messages: list[dict],
**kw: object,
) -> LLMResponse:
"""模拟 LLM 文本调用,按 prompt 内容区分 L2/L1 响应。"""
self.call_count += 1
content = messages[-1].get("content", "")
if "段落" in content or "scene" in content.lower():
return _mock_response(
json.dumps(
{
"scene_summary": "修复后的场景",
"main_setting": "室内",
"key_entities": [],
"main_actions": [],
"topic_keywords": [],
"visible_text": [],
"temporal_flow": "",
}
)
)
return _mock_response(
json.dumps(
{
"event_description": "修复后的事件",
"entities": [],
"actions": [],
"action_subjects": [],
"visible_text": [],
"spatial_relations": "",
"state_changes": None,
}
)
)
class TestRepairTree:
"""repair_tree 核心测试。"""
def _make_broken_tree(
self,
tmp_path: Path,
) -> tuple[TreeIndex, list[NodeIssue]]:
"""构建含一个空 frame_summary 的 L3 节点的测试树。"""
frame_path = tmp_path / "frames" / "L1_000_L2_000_L3_000.jpg"
frame_path.parent.mkdir(parents=True)
frame_path.write_bytes(b"\xff\xd8\xff\xe0fake")
l3 = L3Node(
id="vid_L1_000_L2_000_L3_000",
card=L3Card("", [], [], [], "", {}),
timestamp=1.0,
frame_path="frames/L1_000_L2_000_L3_000.jpg",
)
l2 = L2Node(
id="vid_L1_000_L2_000",
card=L2Card("原始事件", [], [], [], [], "", None),
time_range=(0.0, 10.0),
children=[l3],
)
l1 = L1Node(
id="vid_L1_000",
card=L1Card("原始场景", "", [], [], [], [], ""),
time_range=(0.0, 10.0),
children=[l2],
)
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
issues = [
NodeIssue(
"vid_L1_000_L2_000_L3_000",
3,
"empty_field",
"frame_summary 为空",
)
]
return index, issues
def test_repairs_l3_and_cascades(self, tmp_path: Path) -> None:
"""修复 L3 后应级联重生成 L2 和 L1。"""
index, issues = self._make_broken_tree(tmp_path)
stats = asyncio.run(repair_tree(index, issues, MockVLM(), MockLLM(), tmp_path))
assert stats.l3_repaired == 1
assert stats.l2_regenerated == 1
assert stats.l1_regenerated == 1
assert index.roots[0].children[0].children[0].card.frame_summary == "修复后的帧描述"
assert index.roots[0].children[0].card.event_description == "修复后的事件"
assert index.roots[0].card.scene_summary == "修复后的场景"
def test_no_issues_no_changes(self) -> None:
"""无问题时不进行任何修复。"""
l3 = L3Node(
id="l1_0_l2_0_l3_0",
card=L3Card("正常", [], [], [], "", {}),
timestamp=1.0,
)
l2 = L2Node(
id="l1_0_l2_0",
card=L2Card("正常事件", [], [], [], [], "", None),
time_range=(0.0, 10.0),
children=[l3],
)
l1 = L1Node(
id="l1_0",
card=L1Card("正常场景", "", [], [], [], [], ""),
time_range=(0.0, 10.0),
children=[l2],
)
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
stats = asyncio.run(repair_tree(index, [], MockVLM(), MockLLM(), Path("/tmp")))
assert stats.l3_repaired == 0
assert stats.l2_regenerated == 0
assert stats.l1_regenerated == 0
def test_stats_dataclass(self) -> None:
"""RepairStats 数据类字段验证。"""
stats = RepairStats(l3_repaired=2, l2_regenerated=1, l1_regenerated=1)
assert stats.l3_repaired == 2
assert stats.l2_regenerated == 1
assert stats.l1_regenerated == 1
def test_skips_non_empty_field_issues(self, tmp_path: Path) -> None:
"""非 empty_field 类型的 issue 不触发 L3 修复。"""
l3 = L3Node(
id="l1_0_l2_0_l3_0",
card=L3Card("正常描述", [], [], [], "", {}),
timestamp=1.0,
)
l2 = L2Node(
id="l1_0_l2_0",
card=L2Card("原始事件", [], [], [], [], "", None),
time_range=(0.0, 10.0),
children=[l3],
)
l1 = L1Node(
id="l1_0",
card=L1Card("原始场景", "", [], [], [], [], ""),
time_range=(0.0, 10.0),
children=[l2],
)
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
issues = [NodeIssue("l1_0_l2_0_l3_0", 3, "missing_frame", "帧文件不存在")]
stats = asyncio.run(repair_tree(index, issues, MockVLM(), MockLLM(), tmp_path))
assert stats.l3_repaired == 0
assert stats.l2_regenerated == 0
def test_multiple_l3_under_same_l2(self, tmp_path: Path) -> None:
"""同一 L2 下多个 L3 修复后,L2 只重生成一次。"""
frame_dir = tmp_path / "frames"
frame_dir.mkdir(parents=True)
for i in range(2):
(frame_dir / f"f{i}.jpg").write_bytes(b"\xff\xd8\xff\xe0fake")
l3_a = L3Node(
id="l1_0_l2_0_l3_0",
card=L3Card("", [], [], [], "", {}),
timestamp=1.0,
frame_path="frames/f0.jpg",
)
l3_b = L3Node(
id="l1_0_l2_0_l3_1",
card=L3Card("", [], [], [], "", {}),
timestamp=2.0,
frame_path="frames/f1.jpg",
)
l2 = L2Node(
id="l1_0_l2_0",
card=L2Card("原始事件", [], [], [], [], "", None),
time_range=(0.0, 10.0),
children=[l3_a, l3_b],
)
l1 = L1Node(
id="l1_0",
card=L1Card("原始场景", "", [], [], [], [], ""),
time_range=(0.0, 10.0),
children=[l2],
)
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
issues = [
NodeIssue("l1_0_l2_0_l3_0", 3, "empty_field", "frame_summary 为空"),
NodeIssue("l1_0_l2_0_l3_1", 3, "empty_field", "frame_summary 为空"),
]
vlm = MockVLM()
llm = MockLLM()
stats = asyncio.run(repair_tree(index, issues, vlm, llm, tmp_path))
assert stats.l3_repaired == 2
assert stats.l2_regenerated == 1
assert stats.l1_regenerated == 1
assert vlm.call_count == 2
# LLM 应被调用 2 次:一次 L2 + 一次 L1
assert llm.call_count == 2
def test_missing_frame_file_skips_l3(self, tmp_path: Path) -> None:
"""帧文件不存在时跳过该 L3 节点的修复。"""
l3 = L3Node(
id="l1_0_l2_0_l3_0",
card=L3Card("", [], [], [], "", {}),
timestamp=1.0,
frame_path="frames/nonexistent.jpg",
)
l2 = L2Node(
id="l1_0_l2_0",
card=L2Card("原始事件", [], [], [], [], "", None),
time_range=(0.0, 10.0),
children=[l3],
)
l1 = L1Node(
id="l1_0",
card=L1Card("原始场景", "", [], [], [], [], ""),
time_range=(0.0, 10.0),
children=[l2],
)
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
issues = [NodeIssue("l1_0_l2_0_l3_0", 3, "empty_field", "frame_summary 为空")]
stats = asyncio.run(repair_tree(index, issues, MockVLM(), MockLLM(), tmp_path))
# 帧文件不存在 → 跳过 L3 修复 → 无级联
assert stats.l3_repaired == 0
assert stats.l2_regenerated == 0
assert stats.l1_regenerated == 0
+490
View File
@@ -0,0 +1,490 @@
"""synthesizer 模块单元测试 — AnchorContext + 题型映射常量 + sample_anchor + prompt 构造 + VLM 解析 + 去重 + 单题生成。"""
from __future__ import annotations
import dataclasses
import random
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock
import numpy as np
import pytest
from app.question_gen.synthesizer import (
TASK_TYPE_LEVEL_MAP,
AnchorContext,
TaskTypeSpec,
build_generation_prompt,
generate_one,
is_duplicate,
parse_vlm_response,
sample_anchor,
)
from app.tree.index import TreeIndex
from core.types import GeneratedQuestion
ALL_12_TYPES = [
"Object Recognition",
"Attribute Perception",
"OCR Problems",
"Spatial Reasoning",
"Spatial Perception",
"Action Recognition",
"Action Reasoning",
"Counting Problem",
"Temporal Perception",
"Temporal Reasoning",
"Information Synopsis",
"Object Reasoning",
]
class TestTaskTypeLevelMap:
"""TASK_TYPE_LEVEL_MAP 覆盖性与结构测试。"""
def test_covers_all_12_types(self) -> None:
"""映射表必须覆盖全部 12 种 Video-MME 题型。"""
assert set(TASK_TYPE_LEVEL_MAP.keys()) == set(ALL_12_TYPES)
def test_no_extra_types(self) -> None:
"""映射表不得包含 12 种标准题型之外的条目。"""
assert len(TASK_TYPE_LEVEL_MAP) == 12
def test_all_values_are_task_type_spec(self) -> None:
"""每个映射值必须是 TaskTypeSpec 实例。"""
for task_type, spec in TASK_TYPE_LEVEL_MAP.items():
assert isinstance(spec, TaskTypeSpec), f"{task_type} 映射值类型错误: {type(spec)}"
def test_level_values_valid(self) -> None:
"""每个 spec 的 level 必须是合法层级标识。"""
valid_levels = {"L1", "L2", "L3", "L1-L2"}
for task_type, spec in TASK_TYPE_LEVEL_MAP.items():
assert spec.level in valid_levels, (
f"{task_type} 层级 '{spec.level}' 不在 {valid_levels}"
)
def test_context_fields_non_empty(self) -> None:
"""每个 spec 的 context_fields 至少有一个字段。"""
for task_type, spec in TASK_TYPE_LEVEL_MAP.items():
assert len(spec.context_fields) >= 1, f"{task_type} 的 context_fields 为空"
class TestAnchorContext:
"""AnchorContext 数据类测试。"""
def test_frozen(self) -> None:
"""AnchorContext 是不可变的。"""
ctx = AnchorContext(
node_id="L3_001",
card_text="A person walks into a room",
frame_paths=["/data/frames/001.jpg"],
subtitle="Hello there",
distractor_texts=["A car drives by"],
)
assert ctx.node_id == "L3_001"
assert ctx.card_text == "A person walks into a room"
assert ctx.frame_paths == ["/data/frames/001.jpg"]
assert ctx.subtitle == "Hello there"
assert ctx.distractor_texts == ["A car drives by"]
def test_mutation_raises(self) -> None:
"""frozen dataclass 拒绝赋值修改。"""
ctx = AnchorContext(
node_id="L3_001",
card_text="test",
frame_paths=["a.jpg"],
subtitle="",
distractor_texts=["other node"],
)
try:
ctx.node_id = "L3_002" # type: ignore[misc]
raise AssertionError("应抛出 FrozenInstanceError")
except dataclasses.FrozenInstanceError:
pass
def test_empty_subtitle_allowed(self) -> None:
"""subtitle 可以为空字符串。"""
ctx = AnchorContext(
node_id="L2_010",
card_text="scene card",
frame_paths=[],
subtitle="",
distractor_texts=[],
)
assert ctx.subtitle == ""
def test_multiple_frame_paths(self) -> None:
"""frame_paths 可包含多个路径。"""
paths = ["/data/f1.jpg", "/data/f2.jpg", "/data/f3.jpg"]
ctx = AnchorContext(
node_id="L2_005",
card_text="multi-frame event",
frame_paths=paths,
subtitle="Dialogue line",
distractor_texts=["other1", "other2"],
)
assert len(ctx.frame_paths) == 3
class TestTaskTypeSpec:
"""TaskTypeSpec 数据类测试。"""
def test_frozen(self) -> None:
"""TaskTypeSpec 是不可变的。"""
spec = TaskTypeSpec(
level="L3",
needs_frames=True,
frame_count="1",
context_fields=("frame_summary",),
)
try:
spec.level = "L2" # type: ignore[misc]
raise AssertionError("应抛出 FrozenInstanceError")
except dataclasses.FrozenInstanceError:
pass
def test_context_fields_is_tuple(self) -> None:
"""context_fields 应为 tuple(不可变)。"""
for task_type, spec in TASK_TYPE_LEVEL_MAP.items():
assert isinstance(spec.context_fields, tuple), (
f"{task_type} 的 context_fields 不是 tuple"
)
# ---------------------------------------------------------------------------
# sample_anchor 测试
# ---------------------------------------------------------------------------
def _load_test_tree() -> tuple[TreeIndex, str]:
"""加载真实测试树(store/videos/ 下第一棵)。"""
videos_dir = Path("store/videos")
first_vid = sorted(videos_dir.iterdir())[0]
tree = TreeIndex.load_json(str(first_vid / "tree.json"))
return tree, first_vid.name
class TestSampleAnchor:
"""sample_anchor 锚节点采样测试(基于真实树数据)。"""
def test_l3_type_returns_single_frame(self) -> None:
"""L3 题型(Object Recognition)应返回恰好 1 帧。"""
tree, _vid = _load_test_tree()
ctx = sample_anchor(tree, "Object Recognition", set(), random.Random(42))
assert len(ctx.frame_paths) == 1
assert ctx.node_id.startswith("L") or "_L3_" in ctx.node_id
assert len(ctx.distractor_texts) > 0
def test_l2_type_returns_multiple_frames(self) -> None:
"""L2 题型(Action Reasoning)应返回 2-3 帧。"""
tree, _vid = _load_test_tree()
ctx = sample_anchor(tree, "Action Reasoning", set(), random.Random(42))
assert 2 <= len(ctx.frame_paths) <= 3
def test_temporal_perception_zero_or_one_frame(self) -> None:
"""Temporal Perception 特殊处理:0-1 帧,且 card_text 包含 time_range。"""
tree, _vid = _load_test_tree()
ctx = sample_anchor(tree, "Temporal Perception", set(), random.Random(42))
assert len(ctx.frame_paths) <= 1
assert "time_range" in ctx.card_text.lower() or "time" in ctx.card_text.lower()
def test_information_synopsis_uses_all_l2(self) -> None:
"""Information Synopsis 必须使用目标 L1 下所有 L2 子节点。"""
tree, _vid = _load_test_tree()
ctx = sample_anchor(tree, "Information Synopsis", set(), random.Random(42))
# 找到被选中的 L1,验证 frame_paths 数量 == 该 L1 下全部 L2 数量
chosen_l1 = next(r for r in tree.roots if r.id == ctx.node_id)
assert len(ctx.frame_paths) == len(chosen_l1.children)
def test_l1_type_l2_nodes_in_time_order(self) -> None:
"""Temporal Reasoning 应返回 >=3 帧且 card_text 有实质内容。"""
tree, _vid = _load_test_tree()
ctx = sample_anchor(tree, "Temporal Reasoning", set(), random.Random(42))
assert len(ctx.frame_paths) >= 3
assert len(ctx.card_text) > 20
def test_used_node_ids_excluded(self) -> None:
"""used_node_ids 中的节点不应被再次选中。"""
tree, _vid = _load_test_tree()
rng = random.Random(42)
ctx1 = sample_anchor(tree, "Object Recognition", set(), rng)
ctx2 = sample_anchor(tree, "Object Recognition", {ctx1.node_id}, random.Random(43))
assert ctx2.node_id != ctx1.node_id
def test_insufficient_nodes_raises(self) -> None:
"""所有候选节点均被排除时应抛出 ValueError。"""
tree, _vid = _load_test_tree()
all_l3_ids: set[str] = set()
for root in tree.roots:
for l2 in root.children:
for l3 in l2.children:
all_l3_ids.add(l3.id)
with pytest.raises(ValueError, match="锚节点不足"):
sample_anchor(tree, "Object Recognition", all_l3_ids, random.Random(42))
def test_object_reasoning_l1_l2_type(self) -> None:
"""Object Reasoning (L1-L2) 应选 2-3 个 L2 并按时间排序。"""
tree, _vid = _load_test_tree()
ctx = sample_anchor(tree, "Object Reasoning", set(), random.Random(42))
assert 2 <= len(ctx.frame_paths) <= 3
assert len(ctx.card_text) > 10
def test_spatial_reasoning_context_fields(self) -> None:
"""Spatial Reasoning 的 card_text 应包含 spatial_layout 字段。"""
tree, _vid = _load_test_tree()
ctx = sample_anchor(tree, "Spatial Reasoning", set(), random.Random(42))
# context_fields 包含 spatial_layoutcard_text 必须出现该字段名
assert "spatial_layout" in ctx.card_text
assert len(ctx.card_text) > 20
# ---------------------------------------------------------------------------
# build_generation_prompt 测试
# ---------------------------------------------------------------------------
class TestBuildGenerationPrompt:
"""build_generation_prompt 消息结构与内容测试。"""
def test_messages_structure(self) -> None:
"""带 exemplars 时,system 包含题型和示例,image_paths 来自 anchor。"""
anchor = AnchorContext(
node_id="L3_001",
card_text="A person typing on a laptop",
frame_paths=["store/videos/test/frames/L1_000_L2_000_L3_000.jpg"],
subtitle="Hello world",
distractor_texts=["Another person walking in park"],
)
exemplars = [
GeneratedQuestion(
question_id="ex-1",
video_id="v1",
task_type="Object Recognition",
question="What object?",
options=("A. Cat", "B. Dog", "C. Bird", "D. Fish"),
answer="A",
source_nodes=(),
difficulty="medium",
),
]
messages, image_paths = build_generation_prompt(
"Object Recognition",
anchor,
exemplars,
)
assert messages[0]["role"] == "system"
assert "Object Recognition" in messages[0]["content"]
assert any("What object?" in str(m) for m in messages)
assert image_paths == anchor.frame_paths
def test_distractor_in_user_message(self) -> None:
"""干扰项文本应出现在 user message 中。"""
anchor = AnchorContext(
node_id="L2_003",
card_text="Event card text",
frame_paths=["a.jpg", "b.jpg"],
subtitle="",
distractor_texts=["Distractor node summary"],
)
messages, _ = build_generation_prompt("Action Reasoning", anchor, [])
user_msg = [m for m in messages if m["role"] == "user"][0]
assert "Distractor node summary" in user_msg["content"]
def test_no_exemplars_no_crash(self) -> None:
"""exemplars 为空时不应报错,system 消息中无示例段落。"""
anchor = AnchorContext(
node_id="L3_010",
card_text="Some card text",
frame_paths=["frame.jpg"],
subtitle="",
distractor_texts=[],
)
messages, image_paths = build_generation_prompt("OCR Problems", anchor, [])
assert len(messages) >= 2
assert image_paths == ["frame.jpg"]
def test_subtitle_included_when_non_empty(self) -> None:
"""非空 subtitle 应出现在 user message 中。"""
anchor = AnchorContext(
node_id="L3_002",
card_text="Card text here",
frame_paths=["f.jpg"],
subtitle="This is a subtitle line",
distractor_texts=[],
)
messages, _ = build_generation_prompt("Attribute Perception", anchor, [])
user_msg = [m for m in messages if m["role"] == "user"][0]
assert "This is a subtitle line" in user_msg["content"]
def test_empty_subtitle_not_in_user_message(self) -> None:
"""空 subtitle 不应在 user message 中产生 subtitle 段落。"""
anchor = AnchorContext(
node_id="L3_003",
card_text="Card",
frame_paths=["f.jpg"],
subtitle="",
distractor_texts=[],
)
messages, _ = build_generation_prompt("OCR Problems", anchor, [])
user_msg = [m for m in messages if m["role"] == "user"][0]
# 不应出现空的 subtitle 标记
assert "字幕" not in user_msg["content"]
# ---------------------------------------------------------------------------
# parse_vlm_response 测试
# ---------------------------------------------------------------------------
class TestParseVlmResponse:
"""parse_vlm_response 解析与校验测试。"""
def test_valid_json(self) -> None:
"""合法 JSON 正常解析,question_id 格式正确。"""
raw = '{"question": "What?", "options": ["A. X", "B. Y", "C. Z", "D. W"], "answer": "A"}'
result = parse_vlm_response(raw, "vid1", "Object Recognition", 1)
assert result["question"] == "What?"
assert result["answer"] == "A"
assert len(result["options"]) == 4
assert result["question_id"] == "gen-vid1-object_recognition-001"
def test_json_in_code_block(self) -> None:
"""从 markdown 代码块中提取 JSON。"""
raw = '```json\n{"question": "Q?", "options": ["A. 1", "B. 2", "C. 3", "D. 4"], "answer": "B"}\n```'
result = parse_vlm_response(raw, "vid1", "Object Recognition", 2)
assert result["question"] == "Q?"
assert result["question_id"] == "gen-vid1-object_recognition-002"
def test_invalid_json_raises(self) -> None:
"""非 JSON 文本应抛出 ValueError。"""
with pytest.raises(ValueError, match="VLM 返回"):
parse_vlm_response("not json", "vid1", "Object Recognition", 1)
def test_missing_fields_raises(self) -> None:
"""缺少必需字段应抛出 ValueError。"""
raw = '{"question": "What?"}'
with pytest.raises(ValueError):
parse_vlm_response(raw, "vid1", "Object Recognition", 1)
def test_options_must_be_four(self) -> None:
"""options 非 4 项应抛出 ValueError。"""
raw = '{"question": "Q?", "options": ["A. X", "B. Y"], "answer": "A"}'
with pytest.raises(ValueError, match="4"):
parse_vlm_response(raw, "vid1", "Object Recognition", 1)
def test_answer_must_be_abcd(self) -> None:
"""answer 不在 A-D 范围应抛出 ValueError。"""
raw = '{"question": "Q?", "options": ["A. X", "B. Y", "C. Z", "D. W"], "answer": "E"}'
with pytest.raises(ValueError, match="A.*D"):
parse_vlm_response(raw, "vid1", "Object Recognition", 1)
def test_seq_zero_padded(self) -> None:
"""seq 应按 3 位零填充格式化到 question_id 中。"""
raw = '{"question": "Q?", "options": ["A. 1", "B. 2", "C. 3", "D. 4"], "answer": "C"}'
result = parse_vlm_response(raw, "video_abc", "Action Reasoning", 42)
assert result["question_id"] == "gen-video_abc-action_reasoning-042"
# ---------------------------------------------------------------------------
# is_duplicate 测试
# ---------------------------------------------------------------------------
class TestIsDuplicate:
"""is_duplicate embedding 去重判定测试。"""
@staticmethod
def _fake_embed(texts: str | list[str]) -> np.ndarray:
"""确定性 + L2 归一化的 fake embedding。"""
if isinstance(texts, str):
texts = [texts]
vecs = []
for t in texts:
rs = np.random.RandomState(hash(t) % 2**31)
v = rs.randn(4).astype(np.float32)
v /= np.linalg.norm(v)
vecs.append(v)
return np.array(vecs, dtype=np.float32)
def test_empty_pool_never_duplicate(self) -> None:
"""空池始终返回 False。"""
pool = np.zeros((0, 4), dtype=np.float32)
assert is_duplicate("anything", pool, self._fake_embed, 0.85) is False
def test_identical_text_is_duplicate(self) -> None:
"""相同文本的 embedding 与自身余弦相似度为 1,必定判重。"""
text = "What is happening in the video?"
emb = self._fake_embed(text)
pool = emb.copy()
assert is_duplicate(text, pool, self._fake_embed, 0.85) is True
def test_different_text_not_duplicate(self) -> None:
"""极高阈值下,不同文本不判重。"""
pool_texts = ["aaa", "bbb", "ccc", "ddd", "eee"]
pool = self._fake_embed(pool_texts)
assert is_duplicate("completely unique text xyz", pool, self._fake_embed, 0.99) is False
# ---------------------------------------------------------------------------
# generate_one 测试
# ---------------------------------------------------------------------------
class TestGenerateOne:
"""generate_one 单题异步生成测试。"""
@staticmethod
def _load_test_tree() -> tuple[TreeIndex, str]:
"""加载真实测试树。"""
videos_dir = Path("store/videos")
first_vid = sorted(videos_dir.iterdir())[0]
return TreeIndex.load_json(str(first_vid / "tree.json")), first_vid.name
@pytest.mark.asyncio
async def test_success_path(self) -> None:
"""mock VLM 返回合法 JSON,应成功生成 GeneratedQuestion。"""
vlm = AsyncMock()
vlm.chat_with_images.return_value = MagicMock(
content='{"question":"Q?","options":["A. 1","B. 2","C. 3","D. 4"],"answer":"A"}',
)
tree, vid = self._load_test_tree()
result = await generate_one(
vlm=vlm,
tree=tree,
video_id=vid,
task_type="Object Recognition",
seq=1,
exemplars=[],
used_node_ids=set(),
max_retries=3,
rng=random.Random(42),
session_id="test",
)
assert result is not None
assert result.question_id == f"gen-{vid}-object_recognition-001"
assert result.task_type == "Object Recognition"
assert result.source_nodes # non-empty
assert result.difficulty == "medium"
@pytest.mark.asyncio
async def test_all_retries_exhausted_returns_none(self) -> None:
"""VLM 始终返回无效 JSON,耗尽重试后返回 None。"""
vlm = AsyncMock()
vlm.chat_with_images.return_value = MagicMock(content="invalid")
tree, vid = self._load_test_tree()
result = await generate_one(
vlm=vlm,
tree=tree,
video_id=vid,
task_type="Object Recognition",
seq=1,
exemplars=[],
used_node_ids=set(),
max_retries=2,
rng=random.Random(42),
session_id="test",
)
assert result is None
assert vlm.chat_with_images.call_count == 2
File diff suppressed because it is too large Load Diff
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
# 从 TRM4.zip 迁移资产到 TRM5
# 用法: bash tools/migrate_from_trm4.sh /path/to/Video-Tree-TRM4.zip
set -euo pipefail
ZIP_PATH="${1:?用法: bash tools/migrate_from_trm4.sh /path/to/Video-Tree-TRM4.zip}"
PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
TMP_DIR=$(mktemp -d)
echo "=== TRM4 -> TRM5 迁移 ==="
echo "ZIP: $ZIP_PATH"
echo "项目根: $PROJECT_ROOT"
echo "临时目录: $TMP_DIR"
# 1. 解压
echo "[1/6] 解压 TRM4.zip..."
unzip -q "$ZIP_PATH" -d "$TMP_DIR"
SRC="$TMP_DIR/Video-Tree-TRM4"
# 2. 拷贝帧文件 (rsync --ignore-existing)
echo "[2/6] 拷贝帧文件..."
mkdir -p "$PROJECT_ROOT/store/videos"
for vid_dir in "$SRC"/store/videos/*/; do
vid=$(basename "$vid_dir")
dst="$PROJECT_ROOT/store/videos/$vid"
mkdir -p "$dst"
if [ -d "$vid_dir/frames" ]; then
rsync -a --ignore-existing "$vid_dir/frames/" "$dst/frames/"
fi
done
# 3. 拷贝 SRT 字幕
echo "[3/6] 拷贝 SRT 字幕..."
mkdir -p "$PROJECT_ROOT/data/Video-MME/subtitle"
if [ -d "$SRC/data/Video-MME/subtitle" ]; then
rsync -a --ignore-existing "$SRC/data/Video-MME/subtitle/" "$PROJECT_ROOT/data/Video-MME/subtitle/"
fi
# 4. 拷贝视频压缩包
echo "[4/6] 拷贝视频压缩包..."
mkdir -p "$PROJECT_ROOT/data/Video-MME/original_data"
if [ -d "$SRC/data/Video-MME/original_data" ]; then
rsync -a --ignore-existing "$SRC/data/Video-MME/original_data/" "$PROJECT_ROOT/data/Video-MME/original_data/"
fi
# 5. 拷贝问题 JSON
echo "[5/6] 拷贝 Benchmark 问题..."
mkdir -p "$PROJECT_ROOT/store/questions"
if [ -d "$SRC/store/questions" ]; then
rsync -a --ignore-existing "$SRC/store/questions/" "$PROJECT_ROOT/store/questions/"
fi
# 6. 格式转换
echo "[6/6] 格式转换 flat -> TreeIndex..."
conda run -n Video-Tree-TRM python "$PROJECT_ROOT/tools/convert_flat_to_treeindex.py" \
"$SRC/store/videos" "$PROJECT_ROOT/store/videos"
# 验收
echo ""
echo "=== 验收检查 ==="
VIDEO_COUNT=$(find "$PROJECT_ROOT/store/videos" -name "tree.json" | wc -l)
SRT_COUNT=$(find "$PROJECT_ROOT/data/Video-MME/subtitle" -name "*.srt" 2>/dev/null | wc -l)
echo "视频树: $VIDEO_COUNT (期望 300)"
echo "SRT 字幕: $SRT_COUNT (期望 >=290)"
# 清理
echo "清理临时目录..."
rm -rf "$TMP_DIR"
if [ "$VIDEO_COUNT" -lt 300 ]; then
echo "WARNING: 视频树数量不足 300,请检查"
exit 1
fi
echo "迁移完成"
+492
View File
@@ -0,0 +1,492 @@
#!/usr/bin/env python3
"""树修复管线:检测 + VLM 重生成 + 校验 + Q&A 反向补全。
对 store/videos/ 下所有已迁移的树执行完整修复流程:
1. detect_issues() — 扫描空字段/缺失帧
2. repair_tree() — VLM 重新描述 + 底向上级联(如有问题节点)
3. verify_tree() — 交叉校验删除幻觉
4. supplement_tree() — Q&A 反向补全注入缺失事实
5. save_json() — 覆盖保存
用法:
conda activate Video-Tree-TRM
python tools/repair_trees.py [--videos-dir store/videos] [--concurrency 4] [--dry-run]
app/core/adapters 不 import 此脚本。
"""
from __future__ import annotations
import argparse
import asyncio
import json
import os
import sys
import time
from pathlib import Path
# 确保项目根目录在 sys.path 中
PROJECT_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(PROJECT_ROOT))
from dotenv import load_dotenv
from loguru import logger
load_dotenv(PROJECT_ROOT / ".env")
from app.tree.index import TreeIndex
from app.tree.repair.detector import detect_issues
from app.tree.repair.regenerator import repair_tree
from app.tree.repair.supplement import supplement_tree
from app.tree.subtitle import parse_srt
from app.tree.verify import verify_tree
# ---------------------------------------------------------------------------
# 日志配置:不缓存,立即输出
# ---------------------------------------------------------------------------
logger.remove()
logger.add(
sys.stderr,
format="{time:HH:mm:ss} | {level:<7} | {message}",
level="DEBUG",
colorize=True,
)
logger.add(
PROJECT_ROOT / "logs" / "repair_trees.log",
format="{time:YYYY-MM-DD HH:mm:ss} | {level:<7} | {message}",
level="DEBUG",
rotation="50 MB",
)
# ---------------------------------------------------------------------------
# 断点续跑 — progress 文件管理
# ---------------------------------------------------------------------------
PROGRESS_FILE = "repair_progress.json"
def load_progress(path: Path) -> set[str]:
"""读取 progress 文件,返回已完成视频 ID 集合。
参数:
path: progress JSON 文件路径。
返回:
已完成视频 ID 集合。文件不存在或损坏时返回空集。
"""
if not path.exists():
return set()
try:
data = json.loads(path.read_text(encoding="utf-8"))
return set(data.get("finished_video_ids", []))
except (json.JSONDecodeError, KeyError, TypeError, AttributeError):
logger.warning("progress 文件损坏,忽略: {}", path)
return set()
async def save_progress(path: Path, lock: asyncio.Lock, vid: str) -> None:
"""原子追加一个视频 ID 到 progress 文件。
参数:
path: progress JSON 文件路径。
lock: asyncio.Lock,防并发读改写丢更新。
vid: 要追加的视频 ID。
"""
async with lock:
finished = load_progress(path)
finished.add(vid)
tmp = path.with_suffix(".tmp")
tmp.write_text(
json.dumps({"finished_video_ids": sorted(finished)}, ensure_ascii=False, indent=2),
encoding="utf-8",
)
os.replace(str(tmp), str(path))
def should_skip_video(vid: str, finished: set[str], *, reaggregate_all: bool) -> bool:
"""判断是否跳过该视频。
参数:
vid: 视频 ID。
finished: progress 中已完成的视频 ID 集合。
reaggregate_all: --reaggregate-all 标志。
返回:
True 表示跳过。
"""
if reaggregate_all:
return False
return vid in finished
# ---------------------------------------------------------------------------
# LLM/VLM 客户端构建
# ---------------------------------------------------------------------------
def _build_clients(concurrency: int = 16):
"""构建 GovernedLLMClientLLM + VLM)。
返回:
(llm_client, vlm_client) 元组。
"""
from adapters.breaker import CircuitBreaker
from adapters.llm import GovernedLLMClient
from adapters.telemetry import SQLiteTelemetryRecorder
from adapters.vlm import GovernedVLMClient
# 遥测记录器(GovernedLLMClient 要求非 None
(PROJECT_ROOT / "logs").mkdir(exist_ok=True)
telemetry = SQLiteTelemetryRecorder(str(PROJECT_ROOT / "logs" / "repair_telemetry.db"))
breaker_threshold = int(os.getenv("LLM_CIRCUIT_BREAKER_THRESHOLD", "5"))
breaker_threshold = max(breaker_threshold, concurrency * 2)
breaker_cooldown = int(os.getenv("LLM_CIRCUIT_BREAKER_COOLDOWN", "60"))
timeout_s = float(os.getenv("LLM_TIMEOUT", "120"))
max_retries = int(os.getenv("LLM_MAX_RETRIES", "3"))
base_delay = float(os.getenv("LLM_RETRY_BASE_DELAY", "2.0"))
max_delay = float(os.getenv("LLM_RETRY_MAX_DELAY", "30.0"))
ttft = float(os.getenv("LLM_TTFT_TIMEOUT", "30"))
inter_token = float(os.getenv("LLM_INTER_TOKEN_TIMEOUT", "15"))
# LLM 客户端(用于 supplement 和 L2/L1 重生成)
llm = GovernedLLMClient(
model=os.environ["SEARCH_LLM_MODEL"],
base_url=os.environ["SEARCH_LLM_BASE_URL"],
api_key=os.environ["SEARCH_LLM_API_KEY"],
provider="deepseek",
thinking=False,
breaker=CircuitBreaker(fail_threshold=breaker_threshold, cooldown_s=breaker_cooldown),
cache=None,
telemetry=telemetry,
timeout_s=timeout_s,
ttft_timeout_s=ttft,
inter_token_timeout_s=inter_token,
max_retries=max_retries,
retry_base_delay_s=base_delay,
retry_max_delay_s=max_delay,
)
# VLM 客户端(用于 L3 帧重新描述)
vlm_base = GovernedLLMClient(
model=os.environ["VL_LLM_MODEL"],
base_url=os.environ["VL_LLM_BASE_URL"],
api_key=os.environ["VL_LLM_API_KEY"],
provider="qwen",
thinking=False,
breaker=CircuitBreaker(fail_threshold=breaker_threshold, cooldown_s=breaker_cooldown),
cache=None,
telemetry=telemetry,
timeout_s=timeout_s,
ttft_timeout_s=ttft,
inter_token_timeout_s=inter_token,
max_retries=max_retries,
retry_base_delay_s=base_delay,
retry_max_delay_s=max_delay,
)
vlm = GovernedVLMClient(vlm_base)
return llm, vlm
# ---------------------------------------------------------------------------
# 单视频修复
# ---------------------------------------------------------------------------
async def _repair_one_video(
vid: str,
tree_path: Path,
frames_dir: Path,
srt_dir: Path,
questions_dir: Path,
llm,
vlm,
*,
dry_run: bool = False,
) -> dict:
"""修复单个视频的树。
参数:
vid: 视频 ID。
tree_path: tree.json 路径。
frames_dir: 帧文件目录。
srt_dir: SRT 字幕目录。
questions_dir: 问题 JSON 目录。
llm: LLMProvider 实例。
vlm: VLMProvider 实例。
dry_run: 仅检测不修复。
返回:
统计 dict。
"""
stats = {
"vid": vid,
"issues_found": 0,
"l3_repaired": 0,
"l2_regenerated": 0,
"l1_regenerated": 0,
"verify_removed": 0,
"facts_injected": 0,
"error": None,
}
try:
# 加载树
index = TreeIndex.load_json(str(tree_path))
# Step 1: 检测问题
issues = detect_issues(index, frames_dir=frames_dir)
stats["issues_found"] = len(issues)
if issues:
logger.info("[{}] 发现 {} 个问题", vid, len(issues))
for issue in issues[:5]:
logger.debug(" {} [L{}] {}", issue.node_id, issue.level, issue.details)
if len(issues) > 5:
logger.debug(" ... 还有 {}", len(issues) - 5)
if dry_run:
return stats
# Step 2: VLM 修复(如有 empty_field 问题)
empty_issues = [i for i in issues if i.issue_type == "empty_field"]
if empty_issues:
srt_entries = None
srt_path = srt_dir / f"{vid}.srt"
if srt_path.exists():
srt_entries = parse_srt(str(srt_path))
repair_stats = await repair_tree(index, empty_issues, vlm, llm, frames_dir, srt_entries)
stats["l3_repaired"] = repair_stats.l3_repaired
stats["l2_regenerated"] = repair_stats.l2_regenerated
stats["l1_regenerated"] = repair_stats.l1_regenerated
logger.info(
"[{}] 修复完成: L3={}, L2={}, L1={}",
vid,
repair_stats.l3_repaired,
repair_stats.l2_regenerated,
repair_stats.l1_regenerated,
)
# Step 3: 质量校验
verify_stats = verify_tree(index)
total_removed = (
verify_stats.l2_entities_removed
+ verify_stats.l2_visible_text_removed
+ verify_stats.l1_visible_text_removed
+ verify_stats.l1_key_entities_removed
)
stats["verify_removed"] = total_removed
if total_removed > 0:
logger.info("[{}] 校验删除 {} 项不可靠内容", vid, total_removed)
# Step 4: Q&A 反向补全
questions_path = questions_dir / f"{vid}.json"
if questions_path.exists():
with open(questions_path, encoding="utf-8") as f:
questions = json.load(f)
if isinstance(questions, list) and questions:
logger.info("[{}] 开始 Q&A 补全 ({} 道题)...", vid, len(questions))
srt_text = ""
srt_path = srt_dir / f"{vid}.srt"
if srt_path.exists():
srt_text = srt_path.read_text(encoding="utf-8", errors="ignore")
try:
supplement_stats = await supplement_tree(
index, questions, llm, srt_text=srt_text
)
stats["facts_injected"] = supplement_stats.facts_injected
if supplement_stats.facts_injected > 0:
logger.info("[{}] 补全注入 {} 个事实", vid, supplement_stats.facts_injected)
except Exception as exc:
logger.error("[{}] Q&A 补全失败: {}", vid, exc)
logger.info("[{}] Q&A 补全完成", vid)
# Step 5: 保存
index.save_json(str(tree_path))
logger.info("[{}] 已保存", vid)
except Exception as exc:
stats["error"] = str(exc)
logger.error("[{}] 修复失败: {}", vid, exc)
return stats
# ---------------------------------------------------------------------------
# 主流程
# ---------------------------------------------------------------------------
async def main_async(args: argparse.Namespace) -> None:
"""异步主流程:并发修复视频。"""
videos_dir = Path(args.videos_dir)
srt_dir = Path(args.srt_dir)
questions_dir = Path(args.questions_dir)
concurrency = args.concurrency
reaggregate_all = args.reaggregate_all
# 扫描所有视频
vid_dirs = sorted(d for d in videos_dir.iterdir() if d.is_dir() and (d / "tree.json").exists())
logger.info("发现 {} 个视频", len(vid_dirs))
# 加载 progress
progress_path = PROJECT_ROOT / "logs" / PROGRESS_FILE
finished = load_progress(progress_path)
if finished:
logger.info("已完成 {} 个视频(从 progress 文件加载)", len(finished))
if args.dry_run:
logger.info("=== DRY RUN 模式:仅检测不修复 ===")
# 构建客户端(dry_run 模式不需要)
llm, vlm = (None, None) if args.dry_run else _build_clients(concurrency)
# 过滤跳过的视频
pending = []
skipped_count = 0
for vid_dir in vid_dirs:
vid = vid_dir.name
if should_skip_video(vid, finished, reaggregate_all=reaggregate_all):
skipped_count += 1
continue
pending.append(vid_dir)
if skipped_count:
logger.info("跳过 {} 个已完成视频,待处理 {}", skipped_count, len(pending))
# 并发编排
sem = asyncio.Semaphore(concurrency)
progress_lock = asyncio.Lock()
all_stats: list[dict] = []
stats_lock = asyncio.Lock()
start_time = time.time()
completed = 0
async def _process(vid_dir: Path) -> None:
nonlocal completed
async with sem:
vid = vid_dir.name
tree_path = vid_dir / "tree.json"
frames_dir = vid_dir
logger.info("开始修复 {}", vid)
stats = await _repair_one_video(
vid,
tree_path,
frames_dir,
srt_dir,
questions_dir,
llm,
vlm,
dry_run=args.dry_run,
)
async with stats_lock:
all_stats.append(stats)
completed += 1
# 修复后重新检测,关键 issue 清零才记 finished
if stats["error"] is None and not args.dry_run:
index = TreeIndex.load_json(str(tree_path))
remaining = [i for i in detect_issues(index) if i.issue_type == "empty_field"]
if not remaining:
await save_progress(progress_path, progress_lock, vid)
else:
logger.warning(
"[{}] 修复后仍有 {} 个 empty_field,不计入 finished",
vid,
len(remaining),
)
# 进度日志
if completed % 10 == 0:
elapsed = time.time() - start_time
rate = completed / elapsed * 60 if elapsed > 0 else 0
logger.info(
"进度: {}/{}, 已用 {:.0f}s, 速率 {:.1f} 视频/分钟",
completed,
len(pending),
elapsed,
rate,
)
tasks = [asyncio.create_task(_process(vd)) for vd in pending]
await asyncio.gather(*tasks)
# 最终汇总
elapsed = time.time() - start_time
total_issues = sum(s["issues_found"] for s in all_stats)
total_repaired = sum(s["l3_repaired"] for s in all_stats)
total_injected = sum(s["facts_injected"] for s in all_stats)
total_errors = sum(1 for s in all_stats if s["error"])
logger.info("=" * 60)
logger.info("修复完成")
logger.info(" 视频总数: {}", len(all_stats))
logger.info(" 跳过数: {}", skipped_count)
logger.info(" 问题总数: {}", total_issues)
logger.info(" L3 修复数: {}", total_repaired)
logger.info(" 事实注入数: {}", total_injected)
logger.info(" 失败数: {}", total_errors)
logger.info(" 总耗时: {:.0f}s", elapsed)
logger.info(" 并发数: {}", concurrency)
logger.info("=" * 60)
if total_errors > 0:
logger.warning("以下视频修复失败:")
for s in all_stats:
if s["error"]:
logger.warning(" {}: {}", s["vid"], s["error"])
def parse_args() -> argparse.Namespace:
"""解析命令行参数。"""
parser = argparse.ArgumentParser(description="树修复管线")
parser.add_argument(
"--videos-dir",
default="store/videos",
help="视频目录(默认: store/videos",
)
parser.add_argument(
"--srt-dir",
default="data/Video-MME/subtitle",
help="SRT 字幕目录(默认: data/Video-MME/subtitle",
)
parser.add_argument(
"--questions-dir",
default="store/questions/benchmarks/Video-MME",
help="问题 JSON 目录(默认: store/questions/benchmarks/Video-MME",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="仅检测不修复,不调用 VLM/LLM",
)
parser.add_argument(
"--concurrency",
type=int,
default=16,
help="并发修复视频数(默认: 16",
)
parser.add_argument(
"--reaggregate-all",
action="store_true",
help="强制全量重聚合,忽略 progress 文件",
)
return parser.parse_args()
def main() -> None:
"""同步入口。"""
args = parse_args()
(PROJECT_ROOT / "logs").mkdir(exist_ok=True)
asyncio.run(main_async(args))
if __name__ == "__main__":
main()