merge: feat/repair-concurrent — 赛题生成工具 + repair 管线完善
新增模块: - app/question_gen/synthesizer.py — 题型映射、节点采样、prompt 构造、去重 - app/harness/factory.py — 推理依赖组装(可复用) - tools/generate_questions.py — generate + calibrate CLI 修复: - _VIDEO_MME_TASK_TYPE_COUNT 11→12 - repair/supplement 防御性修复 - repair/regenerator VLM 重生成器
This commit is contained in:
+1
-1
@@ -41,7 +41,7 @@ REDIS_URL=redis://localhost:6379/0
|
|||||||
LLM_TIMEOUT=120
|
LLM_TIMEOUT=120
|
||||||
LLM_MAX_RETRIES=3
|
LLM_MAX_RETRIES=3
|
||||||
LLM_RETRY_BASE_DELAY=2.0
|
LLM_RETRY_BASE_DELAY=2.0
|
||||||
LLM_CIRCUIT_BREAKER_THRESHOLD=5
|
LLM_CIRCUIT_BREAKER_THRESHOLD=5 # 实际阈值 = max(此值, concurrency*2)
|
||||||
LLM_CIRCUIT_BREAKER_COOLDOWN=60
|
LLM_CIRCUIT_BREAKER_COOLDOWN=60
|
||||||
LLM_TTFT_TIMEOUT=30
|
LLM_TTFT_TIMEOUT=30
|
||||||
LLM_INTER_TOKEN_TIMEOUT=15
|
LLM_INTER_TOKEN_TIMEOUT=15
|
||||||
|
|||||||
@@ -124,9 +124,7 @@ class SQLiteTelemetryRecorder:
|
|||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
except sqlite3.Error as exc:
|
except sqlite3.Error as exc:
|
||||||
logger.warning(
|
logger.warning("遥测写入失败(已降级),call_id={}: {}", call_id, exc)
|
||||||
"遥测写入失败(已降级),call_id={}: {}", call_id, exc
|
|
||||||
)
|
|
||||||
|
|
||||||
async def record_llm_call(
|
async def record_llm_call(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -20,8 +20,8 @@ _VALID_SKILL_MODES = {"auto", "manual", "none"}
|
|||||||
_VALID_SKILL_UPDATE_MODES = {"patch", "rewrite"}
|
_VALID_SKILL_UPDATE_MODES = {"patch", "rewrite"}
|
||||||
_PATH_FIELDS = {"workspace_dir", "store_dir"}
|
_PATH_FIELDS = {"workspace_dir", "store_dir"}
|
||||||
|
|
||||||
# Video-MME 的任务类型数量:验证池每类至少保底 eval_min_per_class 题,共 11 类。
|
# Video-MME 的任务类型数量:验证池每类至少保底 eval_min_per_class 题,共 12 类。
|
||||||
_VIDEO_MME_TASK_TYPE_COUNT = 11
|
_VIDEO_MME_TASK_TYPE_COUNT = 12
|
||||||
|
|
||||||
# .env 工程配置字段映射(环境变量名 → RunConfig 字段名)。
|
# .env 工程配置字段映射(环境变量名 → RunConfig 字段名)。
|
||||||
# 仅路径类工程配置走 .env,科研实验参数走 YAML。
|
# 仅路径类工程配置走 .env,科研实验参数走 YAML。
|
||||||
@@ -267,7 +267,7 @@ def _validate_minibatch(config: RunConfig) -> None:
|
|||||||
|
|
||||||
关键实现细节:
|
关键实现细节:
|
||||||
val_size 必须 >= eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT,保证验证池
|
val_size 必须 >= eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT,保证验证池
|
||||||
能为 Video-MME 的全部 11 个任务类型各保底 eval_min_per_class 题。
|
能为 Video-MME 的全部 12 个任务类型各保底 eval_min_per_class 题。
|
||||||
"""
|
"""
|
||||||
if config.batch_size <= 0:
|
if config.batch_size <= 0:
|
||||||
raise ValueError(f"batch_size 必须 > 0,实际: {config.batch_size}")
|
raise ValueError(f"batch_size 必须 > 0,实际: {config.batch_size}")
|
||||||
|
|||||||
@@ -0,0 +1,155 @@
|
|||||||
|
"""推理依赖工厂 — 组装一次推理所需的全套依赖。
|
||||||
|
|
||||||
|
将 TreeIndex 加载、TreeEnvironment 构建、SkillRegistry 发现、
|
||||||
|
SearchToolDispatcher 装配、PromptManager 初始化等步骤封装为
|
||||||
|
单一工厂函数 ``build_inference_deps``,返回不可变的 ``InferenceDeps``。
|
||||||
|
|
||||||
|
调用方(runner / inference)只需传入配置参数,无需了解内部装配逻辑。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from app.search.prompt import PromptManager
|
||||||
|
from app.search.skills import discover_skills
|
||||||
|
from app.search.tools import SearchToolDispatcher
|
||||||
|
from app.tree.environment import TreeEnvironment
|
||||||
|
from app.tree.index import TreeIndex
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Callable
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.ports import EmbeddingProvider, OCRProvider
|
||||||
|
from core.protocols import LLMProvider, VLMProvider
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class InferenceDeps:
|
||||||
|
"""跑一次推理所需的全套依赖(不含 HarnessLog,其生命周期由调用方管理)。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
llm: LLM 端口实例。
|
||||||
|
tool_dispatch_fn: SearchToolDispatcher.dispatch 的绑定方法。
|
||||||
|
prompt_builder: (GeneratedQuestion) -> (system_prompt, user_prompt)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
llm: LLMProvider
|
||||||
|
tool_dispatch_fn: Callable[..., Any]
|
||||||
|
prompt_builder: Callable[[GeneratedQuestion], tuple[str, str]]
|
||||||
|
|
||||||
|
|
||||||
|
def build_inference_deps(
|
||||||
|
*,
|
||||||
|
store_dir: Path,
|
||||||
|
video_id: str,
|
||||||
|
prompts_dir: Path,
|
||||||
|
skills_dir: Path | None,
|
||||||
|
skill_mode: str,
|
||||||
|
embed_provider: EmbeddingProvider,
|
||||||
|
llm: LLMProvider,
|
||||||
|
vlm: VLMProvider,
|
||||||
|
ocr: OCRProvider | None,
|
||||||
|
verify_vision: bool,
|
||||||
|
anchor: bool,
|
||||||
|
assemble_mode: str,
|
||||||
|
) -> InferenceDeps:
|
||||||
|
"""组装一次推理所需的全套依赖。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
store_dir: store 根目录(包含 videos/{video_id}/tree.json)。
|
||||||
|
video_id: 视频标识。
|
||||||
|
prompts_dir: prompt 文件目录。
|
||||||
|
skills_dir: skill 文件目录(None 则不加载 skill)。
|
||||||
|
skill_mode: skill 模式("auto"/"manual"/"none")。
|
||||||
|
embed_provider: 嵌入端口实例。
|
||||||
|
llm: LLM 端口实例。
|
||||||
|
vlm: VLM 端口实例。
|
||||||
|
ocr: OCR 端口实例(None 不启用)。
|
||||||
|
verify_vision: observe_frame 是否执行验证轮。
|
||||||
|
anchor: view_node 是否启用行号锚模式。
|
||||||
|
assemble_mode: 锚模式装配形态。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
InferenceDeps 实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileNotFoundError: tree.json 不存在。
|
||||||
|
"""
|
||||||
|
# Phase 1: 加载 TreeIndex
|
||||||
|
tree_path = store_dir / "videos" / video_id / "tree.json"
|
||||||
|
if not tree_path.exists():
|
||||||
|
raise FileNotFoundError(f"树索引文件不存在: {tree_path}")
|
||||||
|
tree_index = TreeIndex.load_json(str(tree_path))
|
||||||
|
logger.info("已加载 TreeIndex: video_id={}, L1 节点数={}", video_id, len(tree_index.roots))
|
||||||
|
|
||||||
|
# Phase 2: 构建 TreeEnvironment
|
||||||
|
frames_dir = store_dir / "videos" / video_id / "frames"
|
||||||
|
env = TreeEnvironment(index=tree_index, frames_dir=frames_dir)
|
||||||
|
|
||||||
|
# Phase 3: 构建 SkillRegistry
|
||||||
|
skills = None
|
||||||
|
always_skills_text = ""
|
||||||
|
task_skill_map: dict[str, str] = {}
|
||||||
|
catalog_text = ""
|
||||||
|
if skills_dir is not None:
|
||||||
|
always_skills_text, task_skill_map, catalog_text, skills = discover_skills(skills_dir)
|
||||||
|
logger.info(
|
||||||
|
"已发现 skills: always={} 字符, task_map={} 项",
|
||||||
|
len(always_skills_text),
|
||||||
|
len(task_skill_map),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Phase 4: 构建 SearchToolDispatcher
|
||||||
|
dispatcher = SearchToolDispatcher(
|
||||||
|
env,
|
||||||
|
tool_llm=llm,
|
||||||
|
vlm=vlm,
|
||||||
|
ocr=ocr,
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
skills=skills,
|
||||||
|
embed_fn=embed_provider.embed,
|
||||||
|
verify_vision=verify_vision,
|
||||||
|
anchor=anchor,
|
||||||
|
assemble_mode=assemble_mode,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Phase 5: 构建 PromptManager + _prompt_builder 闭包
|
||||||
|
pm = PromptManager(prompts_dir)
|
||||||
|
l1_ids = [root.id for root in tree_index.roots]
|
||||||
|
|
||||||
|
def _prompt_builder(qa: GeneratedQuestion) -> tuple[str, str]:
|
||||||
|
"""为单条题目生成 (system_prompt, user_prompt)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
qa: 生成的题目实例。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
(system_prompt, user_prompt) 二元组。
|
||||||
|
"""
|
||||||
|
system = pm.build_inference_prompt(
|
||||||
|
skill_mode,
|
||||||
|
qa.task_type,
|
||||||
|
always_skills_text,
|
||||||
|
task_skill_map,
|
||||||
|
catalog_text,
|
||||||
|
)
|
||||||
|
user = pm.format_user_prompt(
|
||||||
|
qa.question,
|
||||||
|
list(qa.options),
|
||||||
|
l1_ids,
|
||||||
|
qa.task_type,
|
||||||
|
)
|
||||||
|
return system, user
|
||||||
|
|
||||||
|
logger.info("InferenceDeps 组装完成: video_id={}, skill_mode={}", video_id, skill_mode)
|
||||||
|
return InferenceDeps(
|
||||||
|
llm=llm,
|
||||||
|
tool_dispatch_fn=dispatcher.dispatch,
|
||||||
|
prompt_builder=_prompt_builder,
|
||||||
|
)
|
||||||
@@ -1,5 +1,18 @@
|
|||||||
"""出题模块 — benchmark 加载与分层采样。"""
|
"""出题模块 — benchmark 加载、分层采样与赛题合成。"""
|
||||||
|
|
||||||
from app.question_gen.loader import load_benchmark, stratified_sample
|
from app.question_gen.loader import load_benchmark, stratified_sample
|
||||||
|
from app.question_gen.synthesizer import (
|
||||||
|
TASK_TYPE_LEVEL_MAP,
|
||||||
|
AnchorContext,
|
||||||
|
generate_one,
|
||||||
|
sample_anchor,
|
||||||
|
)
|
||||||
|
|
||||||
__all__ = ["load_benchmark", "stratified_sample"]
|
__all__ = [
|
||||||
|
"load_benchmark",
|
||||||
|
"stratified_sample",
|
||||||
|
"TASK_TYPE_LEVEL_MAP",
|
||||||
|
"AnchorContext",
|
||||||
|
"generate_one",
|
||||||
|
"sample_anchor",
|
||||||
|
]
|
||||||
|
|||||||
@@ -0,0 +1,756 @@
|
|||||||
|
"""赛题合成核心逻辑 — 节点采样、prompt 构造、VLM 响应解析、去重。
|
||||||
|
|
||||||
|
纯函数为主,异步编排仅 generate_one。
|
||||||
|
通过 DI 接收 VLMProvider / EmbeddingProvider,不 import adapters/。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
import random
|
||||||
|
from collections.abc import Callable
|
||||||
|
|
||||||
|
from app.tree.index import L2Node, L3Node, TreeIndex
|
||||||
|
from core.protocols import VLMProvider
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AnchorContext:
|
||||||
|
"""锚节点上下文——生成单道题所需的全部素材。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
node_id: 锚节点 ID。
|
||||||
|
card_text: 锚节点 card 序列化文本。
|
||||||
|
frame_paths: 帧图片路径列表。
|
||||||
|
subtitle: 对应字幕(可空)。
|
||||||
|
distractor_texts: 同视频其他节点摘要(供 VLM 生成干扰项)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
node_id: str
|
||||||
|
card_text: str
|
||||||
|
frame_paths: list[str]
|
||||||
|
subtitle: str
|
||||||
|
distractor_texts: list[str]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class TaskTypeSpec:
|
||||||
|
"""题型的生成规格。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
level: 锚定层级("L3" / "L2" / "L1" / "L1-L2")。
|
||||||
|
needs_frames: 是否必须提供帧图。
|
||||||
|
frame_count: 帧数范围描述(如 "1", "2-3", "0-1")。
|
||||||
|
context_fields: 需要提取的 card 字段元组。
|
||||||
|
"""
|
||||||
|
|
||||||
|
level: str
|
||||||
|
needs_frames: bool
|
||||||
|
frame_count: str
|
||||||
|
context_fields: tuple[str, ...]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 12 种 Video-MME 题型 → 树层级 + 生成规格映射
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
TASK_TYPE_LEVEL_MAP: dict[str, TaskTypeSpec] = {
|
||||||
|
# --- L3 单帧题型 ---
|
||||||
|
"Object Recognition": TaskTypeSpec("L3", True, "1", ("frame_summary",)),
|
||||||
|
"Attribute Perception": TaskTypeSpec("L3", True, "1", ("frame_summary",)),
|
||||||
|
"OCR Problems": TaskTypeSpec("L3", True, "1", ("frame_summary",)),
|
||||||
|
"Spatial Reasoning": TaskTypeSpec("L3", True, "1", ("frame_summary", "spatial_layout")),
|
||||||
|
"Spatial Perception": TaskTypeSpec("L3", True, "1", ("frame_summary",)),
|
||||||
|
# --- L2 多帧 / 事件级题型 ---
|
||||||
|
"Action Recognition": TaskTypeSpec("L2", True, "2-3", ("event_description",)),
|
||||||
|
"Action Reasoning": TaskTypeSpec("L2", True, "2-3", ("event_description",)),
|
||||||
|
"Counting Problem": TaskTypeSpec("L2", True, "2-3", ("event_description",)),
|
||||||
|
"Temporal Perception": TaskTypeSpec("L2", False, "0-1", ("event_description", "time_range")),
|
||||||
|
# --- L1 / 跨层级题型 ---
|
||||||
|
"Temporal Reasoning": TaskTypeSpec("L1", True, "per-L2", ("scene_summary",)),
|
||||||
|
"Information Synopsis": TaskTypeSpec("L1", True, "per-L2", ("scene_summary",)),
|
||||||
|
"Object Reasoning": TaskTypeSpec("L1-L2", True, "per-L2", ("event_description",)),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 内部辅助函数
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize_l3_card(l3: L3Node, context_fields: tuple[str, ...]) -> str:
|
||||||
|
"""将 L3 节点 card 按 context_fields 序列化为可读文本。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l3: L3 节点。
|
||||||
|
context_fields: 需提取的字段名元组。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
多行 "field: value" 格式的文本。
|
||||||
|
"""
|
||||||
|
parts: list[str] = []
|
||||||
|
for fld in context_fields:
|
||||||
|
val = getattr(l3.card, fld, None)
|
||||||
|
if val is not None:
|
||||||
|
parts.append(f"{fld}: {val}")
|
||||||
|
return "\n".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def _l2_time_range_str(l2: L2Node) -> str:
|
||||||
|
"""将 L2 的 time_range 格式化为可读字符串。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l2: L2 节点。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
"time_range: (start, end)" 格式,或 "time_range: unknown"。
|
||||||
|
"""
|
||||||
|
if l2.time_range is not None:
|
||||||
|
return f"time_range: ({l2.time_range[0]:.2f}, {l2.time_range[1]:.2f})"
|
||||||
|
return "time_range: unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def _representative_frame(l2: L2Node) -> str | None:
|
||||||
|
"""取 L2 的代表帧路径——第一个有 frame_path 的 L3 子节点。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l2: L2 节点。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
帧路径字符串,或 None(无可用帧时)。
|
||||||
|
"""
|
||||||
|
for l3 in l2.children:
|
||||||
|
if l3.frame_path:
|
||||||
|
return l3.frame_path
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _sort_l2_by_time(l2_nodes: list[L2Node]) -> list[L2Node]:
|
||||||
|
"""按 time_range 升序排列 L2 节点(None 排末尾)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l2_nodes: 待排序的 L2 节点列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
排序后的新列表(不修改原列表)。
|
||||||
|
"""
|
||||||
|
return sorted(
|
||||||
|
l2_nodes,
|
||||||
|
key=lambda n: n.time_range[0] if n.time_range is not None else float("inf"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 各层级采样策略
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _sample_l3(
|
||||||
|
tree: TreeIndex,
|
||||||
|
task_type: str,
|
||||||
|
spec: TaskTypeSpec,
|
||||||
|
used_node_ids: set[str],
|
||||||
|
rng: random.Random,
|
||||||
|
) -> AnchorContext:
|
||||||
|
"""L3 层级锚节点采样。
|
||||||
|
|
||||||
|
收集全部 L3 节点,排除已用节点,随机选取一个。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
tree: 三层树索引。
|
||||||
|
task_type: 题型名称。
|
||||||
|
spec: 题型规格。
|
||||||
|
used_node_ids: 已用节点 ID 集合。
|
||||||
|
rng: 随机数生成器。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
AnchorContext 实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 候选 L3 节点不足。
|
||||||
|
"""
|
||||||
|
# Phase 1: 收集所有 L3 候选(必须有 frame_path)
|
||||||
|
candidates: list[tuple[L3Node, L2Node]] = []
|
||||||
|
for root in tree.roots:
|
||||||
|
for l2 in root.children:
|
||||||
|
for l3 in l2.children:
|
||||||
|
if l3.id not in used_node_ids and l3.frame_path:
|
||||||
|
candidates.append((l3, l2))
|
||||||
|
|
||||||
|
if not candidates:
|
||||||
|
raise ValueError(f"锚节点不足: {task_type} 无可用 L3 节点(需具备 frame_path)")
|
||||||
|
|
||||||
|
# Phase 2: 随机选取
|
||||||
|
chosen_l3, parent_l2 = rng.choice(candidates)
|
||||||
|
|
||||||
|
# Phase 3: 构造上下文(frame_path 已在候选过滤中保证非 None)
|
||||||
|
card_text = _serialize_l3_card(chosen_l3, spec.context_fields)
|
||||||
|
frame_paths = [chosen_l3.frame_path] # type: ignore[list-item]
|
||||||
|
subtitle = chosen_l3.subtitle or ""
|
||||||
|
|
||||||
|
# Phase 4: 干扰项——整棵树中其他 L3 的 frame_summary
|
||||||
|
distractor_texts = [
|
||||||
|
l3.card.frame_summary
|
||||||
|
for root in tree.roots
|
||||||
|
for l2 in root.children
|
||||||
|
for l3 in l2.children
|
||||||
|
if l3.id != chosen_l3.id
|
||||||
|
]
|
||||||
|
|
||||||
|
return AnchorContext(
|
||||||
|
node_id=chosen_l3.id,
|
||||||
|
card_text=card_text,
|
||||||
|
frame_paths=frame_paths,
|
||||||
|
subtitle=subtitle,
|
||||||
|
distractor_texts=distractor_texts,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _sample_l2(
|
||||||
|
tree: TreeIndex,
|
||||||
|
task_type: str,
|
||||||
|
spec: TaskTypeSpec,
|
||||||
|
used_node_ids: set[str],
|
||||||
|
rng: random.Random,
|
||||||
|
) -> AnchorContext:
|
||||||
|
"""L2 层级锚节点采样(含 Temporal Perception 特殊处理)。
|
||||||
|
|
||||||
|
普通 L2 题型:随机选 1 个 L2,取 2-3 个子 L3 帧。
|
||||||
|
Temporal Perception:0-1 帧,card_text 必含 time_range。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
tree: 三层树索引。
|
||||||
|
task_type: 题型名称。
|
||||||
|
spec: 题型规格。
|
||||||
|
used_node_ids: 已用节点 ID 集合。
|
||||||
|
rng: 随机数生成器。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
AnchorContext 实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 候选 L2 节点不足。
|
||||||
|
"""
|
||||||
|
# Phase 1: 收集所有 L2 候选
|
||||||
|
all_l2: list[L2Node] = []
|
||||||
|
for root in tree.roots:
|
||||||
|
for l2 in root.children:
|
||||||
|
if l2.id not in used_node_ids:
|
||||||
|
all_l2.append(l2)
|
||||||
|
|
||||||
|
if not all_l2:
|
||||||
|
raise ValueError(f"锚节点不足: {task_type} 无可用 L2 节点")
|
||||||
|
|
||||||
|
# Phase 2: 随机选取
|
||||||
|
chosen_l2 = rng.choice(all_l2)
|
||||||
|
|
||||||
|
is_temporal_perception = task_type == "Temporal Perception"
|
||||||
|
|
||||||
|
# Phase 3: 帧路径
|
||||||
|
if is_temporal_perception:
|
||||||
|
# 0-1 帧:有子节点则取 1 帧,否则 0 帧
|
||||||
|
frame_paths: list[str] = []
|
||||||
|
if chosen_l2.children:
|
||||||
|
first_frame = chosen_l2.children[0].frame_path
|
||||||
|
if first_frame:
|
||||||
|
frame_paths = [first_frame]
|
||||||
|
else:
|
||||||
|
# 普通 L2:随机采样 2-3 个 L3 帧
|
||||||
|
children_with_frames = [l3 for l3 in chosen_l2.children if l3.frame_path]
|
||||||
|
if len(children_with_frames) < 2:
|
||||||
|
raise ValueError(
|
||||||
|
f"锚节点不足: {task_type} 需要 >=2 个子帧,"
|
||||||
|
f"但 {chosen_l2.id} 仅有 {len(children_with_frames)} 个可用帧"
|
||||||
|
)
|
||||||
|
n_frames = min(rng.randint(2, 3), len(children_with_frames))
|
||||||
|
sampled = rng.sample(children_with_frames, n_frames)
|
||||||
|
frame_paths = [l3.frame_path for l3 in sampled if l3.frame_path]
|
||||||
|
|
||||||
|
# Phase 4: card_text
|
||||||
|
card_text = f"event_description: {chosen_l2.card.event_description}"
|
||||||
|
if is_temporal_perception:
|
||||||
|
card_text += f"\n{_l2_time_range_str(chosen_l2)}"
|
||||||
|
|
||||||
|
# Phase 5: 字幕(L2 无自身字幕,取首个子 L3 字幕)
|
||||||
|
subtitle = ""
|
||||||
|
if chosen_l2.children and chosen_l2.children[0].subtitle:
|
||||||
|
subtitle = chosen_l2.children[0].subtitle
|
||||||
|
|
||||||
|
# Phase 6: 干扰项——整棵树中其他 L2 的 event_description
|
||||||
|
distractor_texts = [
|
||||||
|
l2.card.event_description
|
||||||
|
for root in tree.roots
|
||||||
|
for l2 in root.children
|
||||||
|
if l2.id != chosen_l2.id
|
||||||
|
]
|
||||||
|
|
||||||
|
return AnchorContext(
|
||||||
|
node_id=chosen_l2.id,
|
||||||
|
card_text=card_text,
|
||||||
|
frame_paths=frame_paths,
|
||||||
|
subtitle=subtitle,
|
||||||
|
distractor_texts=distractor_texts,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _sample_l1(
|
||||||
|
tree: TreeIndex,
|
||||||
|
task_type: str,
|
||||||
|
spec: TaskTypeSpec,
|
||||||
|
used_node_ids: set[str],
|
||||||
|
rng: random.Random,
|
||||||
|
) -> AnchorContext:
|
||||||
|
"""L1 层级锚节点采样(Temporal Reasoning / Information Synopsis)。
|
||||||
|
|
||||||
|
Information Synopsis:使用目标 L1 下全部 L2 子节点。
|
||||||
|
Temporal Reasoning:严格要求 >=3 个 L2 子节点,不足则抛 ValueError。
|
||||||
|
L2 按 time_range 升序排列,每个 L2 取一帧代表。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
tree: 三层树索引。
|
||||||
|
task_type: 题型名称。
|
||||||
|
spec: 题型规格。
|
||||||
|
used_node_ids: 已用节点 ID 集合。
|
||||||
|
rng: 随机数生成器。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
AnchorContext 实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 候选 L1 节点不足,或 Temporal Reasoning 的 L2 子节点 <3。
|
||||||
|
"""
|
||||||
|
# Phase 1: 收集可用 L1
|
||||||
|
candidates = [r for r in tree.roots if r.id not in used_node_ids]
|
||||||
|
if not candidates:
|
||||||
|
raise ValueError(f"锚节点不足: {task_type} 无可用 L1 节点")
|
||||||
|
|
||||||
|
# Phase 2: 随机选取
|
||||||
|
chosen_l1 = rng.choice(candidates)
|
||||||
|
|
||||||
|
# Phase 3: 选定 L2 子集
|
||||||
|
if task_type == "Information Synopsis":
|
||||||
|
# 必须使用全部 L2
|
||||||
|
selected_l2 = list(chosen_l1.children)
|
||||||
|
else:
|
||||||
|
# Temporal Reasoning:严格要求 >=3 个 L2
|
||||||
|
if len(chosen_l1.children) < 3:
|
||||||
|
raise ValueError(
|
||||||
|
f"锚节点不足: {task_type} 需要 >=3 个 L2 子节点,"
|
||||||
|
f"但 {chosen_l1.id} 仅有 {len(chosen_l1.children)} 个"
|
||||||
|
)
|
||||||
|
if len(chosen_l1.children) == 3:
|
||||||
|
selected_l2 = list(chosen_l1.children)
|
||||||
|
else:
|
||||||
|
selected_l2 = rng.sample(chosen_l1.children, rng.randint(3, len(chosen_l1.children)))
|
||||||
|
|
||||||
|
# Phase 4: 按 time_range 升序排列
|
||||||
|
selected_l2 = _sort_l2_by_time(selected_l2)
|
||||||
|
|
||||||
|
# Phase 5: card_text(场景摘要)
|
||||||
|
card_text = f"scene_summary: {chosen_l1.card.scene_summary}"
|
||||||
|
|
||||||
|
# Phase 6: 帧路径——每个 L2 取一帧代表
|
||||||
|
frame_paths: list[str] = []
|
||||||
|
for l2 in selected_l2:
|
||||||
|
rep = _representative_frame(l2)
|
||||||
|
if rep:
|
||||||
|
frame_paths.append(rep)
|
||||||
|
|
||||||
|
# Phase 7: 字幕(L1 无字幕)
|
||||||
|
subtitle = ""
|
||||||
|
|
||||||
|
# Phase 8: 干扰项——其他 L1 的 scene_summary
|
||||||
|
distractor_texts = [r.card.scene_summary for r in tree.roots if r.id != chosen_l1.id]
|
||||||
|
|
||||||
|
return AnchorContext(
|
||||||
|
node_id=chosen_l1.id,
|
||||||
|
card_text=card_text,
|
||||||
|
frame_paths=frame_paths,
|
||||||
|
subtitle=subtitle,
|
||||||
|
distractor_texts=distractor_texts,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _sample_l1_l2(
|
||||||
|
tree: TreeIndex,
|
||||||
|
task_type: str,
|
||||||
|
spec: TaskTypeSpec,
|
||||||
|
used_node_ids: set[str],
|
||||||
|
rng: random.Random,
|
||||||
|
) -> AnchorContext:
|
||||||
|
"""L1-L2 跨层级锚节点采样(Object Reasoning)。
|
||||||
|
|
||||||
|
从全部 L2 中随机选 2-3 个,按 time_range 排序,
|
||||||
|
card_text 为各 L2 的 event_description 拼接。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
tree: 三层树索引。
|
||||||
|
task_type: 题型名称。
|
||||||
|
spec: 题型规格。
|
||||||
|
used_node_ids: 已用节点 ID 集合。
|
||||||
|
rng: 随机数生成器。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
AnchorContext 实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 候选 L2 节点不足。
|
||||||
|
"""
|
||||||
|
# Phase 1: 收集全部 L2
|
||||||
|
all_l2: list[L2Node] = []
|
||||||
|
for root in tree.roots:
|
||||||
|
for l2 in root.children:
|
||||||
|
if l2.id not in used_node_ids:
|
||||||
|
all_l2.append(l2)
|
||||||
|
|
||||||
|
if len(all_l2) < 2:
|
||||||
|
raise ValueError(
|
||||||
|
f"锚节点不足: {task_type} 需要 >=2 个 L2 节点,但仅有 {len(all_l2)} 个可用"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Phase 2: 随机选 2-3 个
|
||||||
|
n_pick = min(rng.randint(2, 3), len(all_l2))
|
||||||
|
selected = rng.sample(all_l2, n_pick)
|
||||||
|
|
||||||
|
# Phase 3: 按 time_range 升序排列
|
||||||
|
selected = _sort_l2_by_time(selected)
|
||||||
|
|
||||||
|
# Phase 4: card_text = 各 L2 event_description 拼接
|
||||||
|
card_text = "\n".join(f"event_description: {l2.card.event_description}" for l2 in selected)
|
||||||
|
|
||||||
|
# Phase 5: 帧路径——每个 L2 取一帧代表
|
||||||
|
frame_paths: list[str] = []
|
||||||
|
for l2 in selected:
|
||||||
|
rep = _representative_frame(l2)
|
||||||
|
if rep:
|
||||||
|
frame_paths.append(rep)
|
||||||
|
|
||||||
|
# Phase 6: 字幕
|
||||||
|
subtitle = ""
|
||||||
|
|
||||||
|
# Phase 7: 干扰项——未被选中的 L2 的 event_description
|
||||||
|
selected_ids = {l2.id for l2 in selected}
|
||||||
|
distractor_texts = [l2.card.event_description for l2 in all_l2 if l2.id not in selected_ids]
|
||||||
|
|
||||||
|
# 使用第一个被选中节点的 ID 作为锚节点 ID
|
||||||
|
anchor_id = selected[0].id
|
||||||
|
|
||||||
|
return AnchorContext(
|
||||||
|
node_id=anchor_id,
|
||||||
|
card_text=card_text,
|
||||||
|
frame_paths=frame_paths,
|
||||||
|
subtitle=subtitle,
|
||||||
|
distractor_texts=distractor_texts,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 公开接口
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def sample_anchor(
|
||||||
|
tree: TreeIndex,
|
||||||
|
task_type: str,
|
||||||
|
used_node_ids: set[str],
|
||||||
|
rng: random.Random,
|
||||||
|
) -> AnchorContext:
|
||||||
|
"""根据题型从视频树中采样锚节点及上下文素材。
|
||||||
|
|
||||||
|
依据 TASK_TYPE_LEVEL_MAP 中的层级规格,分发到对应的层级采样策略。
|
||||||
|
每种层级有不同的帧选取、card 序列化和干扰项收集逻辑。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
tree: 三层树索引。
|
||||||
|
task_type: 12 种 Video-MME 题型之一。
|
||||||
|
used_node_ids: 本轮已用节点 ID 集合(避免重复采样)。
|
||||||
|
rng: 可控随机数生成器(保证可复现)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
AnchorContext 实例,包含锚节点 ID、card 文本、帧路径、字幕和干扰项。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
KeyError: task_type 不在 TASK_TYPE_LEVEL_MAP 中。
|
||||||
|
ValueError: 候选节点不足(全部被 used_node_ids 排除)。
|
||||||
|
"""
|
||||||
|
spec = TASK_TYPE_LEVEL_MAP[task_type]
|
||||||
|
|
||||||
|
if spec.level == "L3":
|
||||||
|
return _sample_l3(tree, task_type, spec, used_node_ids, rng)
|
||||||
|
elif spec.level == "L2":
|
||||||
|
return _sample_l2(tree, task_type, spec, used_node_ids, rng)
|
||||||
|
elif spec.level == "L1":
|
||||||
|
return _sample_l1(tree, task_type, spec, used_node_ids, rng)
|
||||||
|
elif spec.level == "L1-L2":
|
||||||
|
return _sample_l1_l2(tree, task_type, spec, used_node_ids, rng)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"未知层级: {spec.level}")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Prompt 构造与 VLM 响应解析
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_VALID_ANSWERS = frozenset({"A", "B", "C", "D"})
|
||||||
|
|
||||||
|
|
||||||
|
def build_generation_prompt(
|
||||||
|
task_type: str,
|
||||||
|
anchor: AnchorContext,
|
||||||
|
exemplars: list[GeneratedQuestion],
|
||||||
|
) -> tuple[list[dict[str, str]], list[str]]:
|
||||||
|
"""组装 VLM 出题 prompt。
|
||||||
|
|
||||||
|
构造 OpenAI 格式的 messages 列表和帧图片路径列表,
|
||||||
|
供 VLMProvider.chat_with_images 直接消费。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
task_type: 题型名称(如 "Object Recognition")。
|
||||||
|
anchor: 锚节点上下文(card_text, subtitle, distractor_texts, frame_paths)。
|
||||||
|
exemplars: 少样本示例列表(可为空)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
(messages, image_paths) — messages 为 OpenAI 格式消息列表,
|
||||||
|
image_paths 为帧图片路径列表,直接喂给 VLMProvider.chat_with_images。
|
||||||
|
"""
|
||||||
|
# Phase 1: 构造 system message
|
||||||
|
system_parts: list[str] = [
|
||||||
|
"你是一个视频理解题目生成器。",
|
||||||
|
f"题型: {task_type}",
|
||||||
|
"约束:",
|
||||||
|
"- 题目必须基于提供的节点内容",
|
||||||
|
"- 干扰选项应来自其他节点的信息",
|
||||||
|
"- 生成风格应与示例保持一致",
|
||||||
|
'- 以 JSON 格式返回: {"question": "...", "options": ["A. ...", "B. ...", "C. ...", "D. ..."], "answer": "A/B/C/D"}',
|
||||||
|
]
|
||||||
|
|
||||||
|
# Phase 2: 加入 few-shot 示例
|
||||||
|
if exemplars:
|
||||||
|
system_parts.append("\n示例:")
|
||||||
|
for i, ex in enumerate(exemplars, 1):
|
||||||
|
system_parts.append(f" 示例 {i}:")
|
||||||
|
system_parts.append(f" question: {ex.question}")
|
||||||
|
system_parts.append(f" options: {list(ex.options)}")
|
||||||
|
system_parts.append(f" answer: {ex.answer}")
|
||||||
|
|
||||||
|
system_content = "\n".join(system_parts)
|
||||||
|
|
||||||
|
# Phase 3: 构造 user message
|
||||||
|
user_parts: list[str] = [f"节点内容:\n{anchor.card_text}"]
|
||||||
|
|
||||||
|
if anchor.subtitle:
|
||||||
|
user_parts.append(f"\n字幕:\n{anchor.subtitle}")
|
||||||
|
|
||||||
|
if anchor.distractor_texts:
|
||||||
|
user_parts.append("\n干扰项来源节点摘要:")
|
||||||
|
for dt in anchor.distractor_texts:
|
||||||
|
user_parts.append(f"- {dt}")
|
||||||
|
|
||||||
|
user_content = "\n".join(user_parts)
|
||||||
|
|
||||||
|
messages = [
|
||||||
|
{"role": "system", "content": system_content},
|
||||||
|
{"role": "user", "content": user_content},
|
||||||
|
]
|
||||||
|
|
||||||
|
return messages, list(anchor.frame_paths)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_vlm_response(
|
||||||
|
raw: str,
|
||||||
|
video_id: str,
|
||||||
|
task_type: str,
|
||||||
|
seq: int,
|
||||||
|
) -> dict:
|
||||||
|
"""解析 VLM 返回的 JSON → 部分字段字典。
|
||||||
|
|
||||||
|
尝试直接解析 JSON;若失败,从 markdown 代码块中提取后重试。
|
||||||
|
校验必需字段、选项数量和答案合法性。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
raw: VLM 原始返回文本。
|
||||||
|
video_id: 所属视频标识。
|
||||||
|
task_type: 题型名称(用于错误消息)。
|
||||||
|
seq: 序列号,用于生成 question_id。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
{"question_id": "gen-{video_id}-{seq:03d}", "question": ..., "options": [...], "answer": ...}
|
||||||
|
调用方(generate_one)补齐 source_nodes/difficulty 后构造 GeneratedQuestion。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: JSON 解析失败、缺必需字段、options 非 4 项、answer 不在 A-D。
|
||||||
|
"""
|
||||||
|
# Phase 1: 尝试直接解析 JSON
|
||||||
|
data = None
|
||||||
|
with contextlib.suppress(json.JSONDecodeError):
|
||||||
|
data = json.loads(raw)
|
||||||
|
|
||||||
|
# Phase 2: 从 markdown 代码块提取 JSON
|
||||||
|
if data is None:
|
||||||
|
match = re.search(r"```(?:json)?\s*\n?(.*?)\n?\s*```", raw, re.DOTALL)
|
||||||
|
if match:
|
||||||
|
with contextlib.suppress(json.JSONDecodeError):
|
||||||
|
data = json.loads(match.group(1))
|
||||||
|
|
||||||
|
if data is None:
|
||||||
|
raise ValueError(f"VLM 返回无法解析为 JSON: {raw[:200]}")
|
||||||
|
|
||||||
|
# Phase 3: 校验必需字段
|
||||||
|
required = ("question", "options", "answer")
|
||||||
|
missing = [f for f in required if f not in data]
|
||||||
|
if missing:
|
||||||
|
raise ValueError(f"VLM 返回缺少必需字段 {missing}: {raw[:200]}")
|
||||||
|
|
||||||
|
# Phase 4: options 必须恰好 4 项
|
||||||
|
options = data["options"]
|
||||||
|
if not isinstance(options, list) or len(options) != 4:
|
||||||
|
raise ValueError(
|
||||||
|
f"options 必须恰好 4 项,实际 {len(options) if isinstance(options, list) else type(options).__name__}: {raw[:200]}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Phase 5: answer 必须是 A-D
|
||||||
|
answer = data["answer"]
|
||||||
|
if answer not in _VALID_ANSWERS:
|
||||||
|
raise ValueError(f"answer 必须是 A/B/C/D 之一,实际 '{answer}': {raw[:200]}")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"question_id": f"gen-{video_id}-{task_type.lower().replace(' ', '_')}-{seq:03d}",
|
||||||
|
"question": data["question"],
|
||||||
|
"options": list(options),
|
||||||
|
"answer": answer,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Embedding 去重
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def is_duplicate(
|
||||||
|
question_text: str,
|
||||||
|
pool_embeddings: np.ndarray,
|
||||||
|
embed_fn: Callable[[str | list[str]], np.ndarray],
|
||||||
|
threshold: float,
|
||||||
|
) -> bool:
|
||||||
|
"""embedding 去重判定。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
question_text: 待检查的题目文本。
|
||||||
|
pool_embeddings: 已有题目的 embedding 矩阵 [N, D](L2 归一化)。
|
||||||
|
embed_fn: 文本嵌入函数,返回 [N, D] ndarray(L2 归一化)。
|
||||||
|
threshold: 余弦相似度阈值。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
True 表示与池中某题重复。空池永远返回 False。
|
||||||
|
"""
|
||||||
|
if pool_embeddings.shape[0] == 0:
|
||||||
|
return False
|
||||||
|
|
||||||
|
query = embed_fn(question_text) # [1, D]
|
||||||
|
query = query.squeeze(0) # [D]
|
||||||
|
similarities = pool_embeddings @ query # [N]
|
||||||
|
return bool(np.max(similarities) >= threshold)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 单题生成
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def generate_one(
|
||||||
|
vlm: VLMProvider,
|
||||||
|
tree: TreeIndex,
|
||||||
|
video_id: str,
|
||||||
|
task_type: str,
|
||||||
|
seq: int,
|
||||||
|
*,
|
||||||
|
exemplars: list[GeneratedQuestion],
|
||||||
|
used_node_ids: set[str],
|
||||||
|
max_retries: int,
|
||||||
|
rng: random.Random,
|
||||||
|
session_id: str,
|
||||||
|
) -> GeneratedQuestion | None:
|
||||||
|
"""生成单道候选题(不含去重——去重在调用方汇总点原子执行)。
|
||||||
|
|
||||||
|
循环最多 max_retries 次尝试生成。每次尝试:
|
||||||
|
1. 采样锚节点
|
||||||
|
2. 构造 prompt
|
||||||
|
3. 调用 VLM
|
||||||
|
4. 解析响应
|
||||||
|
5. 构造 GeneratedQuestion
|
||||||
|
|
||||||
|
返回 None 表示耗尽重试。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
vlm: VLM 调用端口。
|
||||||
|
tree: 三层树索引。
|
||||||
|
video_id: 所属视频标识。
|
||||||
|
task_type: 12 种 Video-MME 题型之一。
|
||||||
|
seq: 序列号,用于生成 question_id。
|
||||||
|
exemplars: 少样本示例列表。
|
||||||
|
used_node_ids: 已用节点 ID 集合。
|
||||||
|
max_retries: 最大重试次数。
|
||||||
|
rng: 可控随机数生成器。
|
||||||
|
session_id: 会话 ID(传递给 VLM 遥测)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
GeneratedQuestion 实例,或 None(耗尽重试)。
|
||||||
|
"""
|
||||||
|
from core.types import GeneratedQuestion as _GeneratedQuestion
|
||||||
|
|
||||||
|
for attempt in range(max_retries):
|
||||||
|
try:
|
||||||
|
# Phase 1: 采样锚节点
|
||||||
|
anchor = sample_anchor(tree, task_type, used_node_ids, rng)
|
||||||
|
|
||||||
|
# Phase 2: 构造 prompt
|
||||||
|
messages, images = build_generation_prompt(task_type, anchor, exemplars)
|
||||||
|
|
||||||
|
# Phase 3: 调用 VLM
|
||||||
|
response = await vlm.chat_with_images(
|
||||||
|
messages,
|
||||||
|
images,
|
||||||
|
session_id=session_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Phase 4: 解析响应
|
||||||
|
parsed = parse_vlm_response(response.content, video_id, task_type, seq)
|
||||||
|
|
||||||
|
# Phase 5: 构造 GeneratedQuestion
|
||||||
|
return _GeneratedQuestion(
|
||||||
|
question_id=parsed["question_id"],
|
||||||
|
video_id=video_id,
|
||||||
|
task_type=task_type,
|
||||||
|
question=parsed["question"],
|
||||||
|
options=tuple(parsed["options"]),
|
||||||
|
answer=parsed["answer"],
|
||||||
|
source_nodes=(anchor.node_id,),
|
||||||
|
difficulty="medium",
|
||||||
|
)
|
||||||
|
except (ValueError, KeyError) as exc:
|
||||||
|
logger.warning(
|
||||||
|
"generate_one 尝试 {}/{} 失败 ({}): {}",
|
||||||
|
attempt + 1,
|
||||||
|
max_retries,
|
||||||
|
task_type,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
logger.warning(
|
||||||
|
"generate_one 耗尽 {} 次重试 (video={}, task_type={})",
|
||||||
|
max_retries,
|
||||||
|
video_id,
|
||||||
|
task_type,
|
||||||
|
)
|
||||||
|
return None
|
||||||
@@ -40,7 +40,8 @@ def detect_issues(
|
|||||||
"""扫描树,返回所有问题节点列表。
|
"""扫描树,返回所有问题节点列表。
|
||||||
|
|
||||||
检查项:
|
检查项:
|
||||||
- L3: card 必填字段为空(frame_summary / visible_entities / ongoing_actions / spatial_layout)
|
- L3: card 必填字段为空(frame_summary / spatial_layout)
|
||||||
|
- 注: visible_entities / ongoing_actions 为空是合法状态(静物/黑帧),不纳入检测
|
||||||
- L3: frame_path 对应文件不存在(需提供 frames_dir)
|
- L3: frame_path 对应文件不存在(需提供 frames_dir)
|
||||||
- L2: event_description 为空
|
- L2: event_description 为空
|
||||||
- L2/L1: children 列表为空
|
- L2/L1: children 列表为空
|
||||||
@@ -108,14 +109,10 @@ def detect_issues(
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
for l3 in l2.children:
|
for l3 in l2.children:
|
||||||
# L3: 各必填字段不为空
|
# L3: 核心必填字段不为空(visible_entities/ongoing_actions 为空是合法状态)
|
||||||
empty_fields: list[str] = []
|
empty_fields: list[str] = []
|
||||||
if not l3.card.frame_summary:
|
if not l3.card.frame_summary:
|
||||||
empty_fields.append("frame_summary")
|
empty_fields.append("frame_summary")
|
||||||
if not l3.card.visible_entities:
|
|
||||||
empty_fields.append("visible_entities")
|
|
||||||
if not l3.card.ongoing_actions:
|
|
||||||
empty_fields.append("ongoing_actions")
|
|
||||||
if not l3.card.spatial_layout:
|
if not l3.card.spatial_layout:
|
||||||
empty_fields.append("spatial_layout")
|
empty_fields.append("spatial_layout")
|
||||||
if empty_fields:
|
if empty_fields:
|
||||||
|
|||||||
@@ -0,0 +1,513 @@
|
|||||||
|
"""树修复重生成器:VLM 重新描述问题节点 + 底向上级联。
|
||||||
|
|
||||||
|
底向上修复流程:
|
||||||
|
1. 收集需修复的 L3 节点 → VLM 重新描述帧
|
||||||
|
2. 收集受影响的 L2 → LLM 从 L3 children 聚合
|
||||||
|
3. 收集受影响的 L1 → LLM 从 L2 children 聚合
|
||||||
|
|
||||||
|
仅处理 issue_type == "empty_field" 且 level == 3 的问题节点。
|
||||||
|
帧文件不存在时跳过该节点(不中断整体修复流程)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from app.tree.index import (
|
||||||
|
L1Card,
|
||||||
|
L1Node,
|
||||||
|
L2Card,
|
||||||
|
L2Node,
|
||||||
|
L3Card,
|
||||||
|
L3Node,
|
||||||
|
TreeIndex,
|
||||||
|
)
|
||||||
|
from app.tree.subtitle import extract_subtitle_for_range
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.tree.repair.detector import NodeIssue
|
||||||
|
from app.tree.subtitle import SRTEntry
|
||||||
|
from core.protocols import LLMProvider, VLMProvider
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Prompt 常量(与 VideoTreeBuilder 保持一致风格)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_L3_REPAIR_PROMPT = (
|
||||||
|
'该片段的整体内容: "{l2_description}"\n'
|
||||||
|
"用一到两句话描述这帧画面的具体内容。"
|
||||||
|
"重点关注: 动作、物体变化、文字信息、人物表情。\n"
|
||||||
|
"{subtitle_block}"
|
||||||
|
"返回 JSON 对象,包含以下字段:\n"
|
||||||
|
"- frame_summary: 画面描述\n"
|
||||||
|
"- visible_entities: 可见实体列表\n"
|
||||||
|
"- ongoing_actions: 动作列表\n"
|
||||||
|
"- visible_text: 可见文字列表\n"
|
||||||
|
"- spatial_layout: 空间布局\n"
|
||||||
|
'- visual_attributes: {{"lighting": "...", "dominant_colors": [...], "camera_angle": "..."}}\n'
|
||||||
|
"只返回 JSON 对象,不要其他内容。"
|
||||||
|
)
|
||||||
|
|
||||||
|
_L2_REGEN_PROMPT = (
|
||||||
|
"以下是一个视频片段中各帧的描述:\n{l3_texts}\n"
|
||||||
|
"用1-2句话描述该片段的核心内容。\n"
|
||||||
|
"返回 JSON 对象,包含以下字段:\n"
|
||||||
|
"- event_description: 1-2句片段描述\n"
|
||||||
|
"- entities: 可见实体列表\n"
|
||||||
|
"- actions: 动作列表\n"
|
||||||
|
"- action_subjects: 动作主体列表\n"
|
||||||
|
"- visible_text: 画面中可见文字列表\n"
|
||||||
|
"- spatial_relations: 空间关系描述\n"
|
||||||
|
"- state_changes: 状态变化描述(无则 null)\n"
|
||||||
|
"只返回 JSON 对象,不要其他内容。"
|
||||||
|
)
|
||||||
|
|
||||||
|
_L1_REGEN_PROMPT = (
|
||||||
|
"以下是一个视频段落中各片段的描述:\n{l2_texts}\n"
|
||||||
|
"用2-3句话总结该段落的整体内容,涵盖所有片段的主题。\n"
|
||||||
|
"返回 JSON 对象,包含以下字段:\n"
|
||||||
|
"- scene_summary: 2-3句段落摘要\n"
|
||||||
|
"- main_setting: 主要场景\n"
|
||||||
|
"- key_entities: 关键实体列表\n"
|
||||||
|
"- main_actions: 主要动作列表\n"
|
||||||
|
"- topic_keywords: 主题关键词列表\n"
|
||||||
|
"- visible_text: 出现的文字列表\n"
|
||||||
|
"- temporal_flow: 时间流向描述\n"
|
||||||
|
"只返回 JSON 对象,不要其他内容。"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 统计数据类
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class RepairStats:
|
||||||
|
"""修复统计信息。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
l3_repaired: 修复的 L3 节点数。
|
||||||
|
l2_regenerated: 重生成的 L2 节点数。
|
||||||
|
l1_regenerated: 重生成的 L1 节点数。
|
||||||
|
"""
|
||||||
|
|
||||||
|
l3_repaired: int = 0
|
||||||
|
l2_regenerated: int = 0
|
||||||
|
l1_regenerated: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# JSON 解析辅助(复用 VideoTreeBuilder 的解析逻辑)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_json(raw: str) -> Any:
|
||||||
|
"""从 VLM/LLM 原始输出中提取 JSON(处理 markdown 代码块包裹)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
raw: 原始返回字符串。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
解析后的 Python 对象(dict/list),解析失败返回 None。
|
||||||
|
"""
|
||||||
|
raw = raw.strip()
|
||||||
|
# Phase 1: 尝试提取 markdown 代码块中的 JSON
|
||||||
|
code_match = re.search(
|
||||||
|
r"```(?:json)?\s*([\[{].*?[\]}])\s*```",
|
||||||
|
raw,
|
||||||
|
re.DOTALL,
|
||||||
|
)
|
||||||
|
if code_match:
|
||||||
|
raw = code_match.group(1)
|
||||||
|
|
||||||
|
# Phase 2: 直接解析
|
||||||
|
try:
|
||||||
|
return json.loads(raw)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Phase 3: 尝试提取裸 JSON 对象/数组
|
||||||
|
json_match = re.search(r"[\[{].*[\]}]", raw, re.DOTALL)
|
||||||
|
if json_match:
|
||||||
|
try:
|
||||||
|
return json.loads(json_match.group())
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_l3_card(raw: str) -> L3Card | None:
|
||||||
|
"""解析 VLM 输出为 L3Card。解析失败返回 None。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
raw: VLM 原始返回字符串。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
L3Card 实例或 None(解析失败时)。
|
||||||
|
"""
|
||||||
|
data = _extract_json(raw)
|
||||||
|
if isinstance(data, dict):
|
||||||
|
try:
|
||||||
|
return L3Card(
|
||||||
|
frame_summary=str(data["frame_summary"]),
|
||||||
|
visible_entities=list(data["visible_entities"]),
|
||||||
|
ongoing_actions=list(data["ongoing_actions"]),
|
||||||
|
visible_text=list(data["visible_text"]),
|
||||||
|
spatial_layout=str(data["spatial_layout"]),
|
||||||
|
visual_attributes=dict(data["visual_attributes"]),
|
||||||
|
)
|
||||||
|
except (KeyError, TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_l2_card(raw: str) -> L2Card | None:
|
||||||
|
"""解析 LLM 输出为 L2Card。解析失败返回 None。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
raw: LLM 原始返回字符串。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
L2Card 实例或 None(解析失败时)。
|
||||||
|
"""
|
||||||
|
data = _extract_json(raw)
|
||||||
|
if isinstance(data, dict):
|
||||||
|
try:
|
||||||
|
state_changes = data.get("state_changes")
|
||||||
|
if state_changes is not None:
|
||||||
|
state_changes = str(state_changes)
|
||||||
|
return L2Card(
|
||||||
|
event_description=str(data["event_description"]),
|
||||||
|
entities=list(data["entities"]),
|
||||||
|
actions=list(data["actions"]),
|
||||||
|
action_subjects=list(data["action_subjects"]),
|
||||||
|
visible_text=list(data["visible_text"]),
|
||||||
|
spatial_relations=str(data["spatial_relations"]),
|
||||||
|
state_changes=state_changes,
|
||||||
|
)
|
||||||
|
except (KeyError, TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_l1_card(raw: str) -> L1Card | None:
|
||||||
|
"""解析 LLM 输出为 L1Card。解析失败返回 None。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
raw: LLM 原始返回字符串。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
L1Card 实例或 None(解析失败时)。
|
||||||
|
"""
|
||||||
|
data = _extract_json(raw)
|
||||||
|
if isinstance(data, dict):
|
||||||
|
try:
|
||||||
|
return L1Card(
|
||||||
|
scene_summary=str(data["scene_summary"]),
|
||||||
|
main_setting=str(data["main_setting"]),
|
||||||
|
key_entities=list(data["key_entities"]),
|
||||||
|
main_actions=list(data["main_actions"]),
|
||||||
|
topic_keywords=list(data["topic_keywords"]),
|
||||||
|
visible_text=list(data["visible_text"]),
|
||||||
|
temporal_flow=str(data["temporal_flow"]),
|
||||||
|
)
|
||||||
|
except (KeyError, TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 节点查找辅助
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _build_node_lookup(
|
||||||
|
index: TreeIndex,
|
||||||
|
) -> tuple[
|
||||||
|
dict[str, L3Node],
|
||||||
|
dict[str, L2Node],
|
||||||
|
dict[str, L1Node],
|
||||||
|
dict[str, L2Node],
|
||||||
|
dict[str, L1Node],
|
||||||
|
]:
|
||||||
|
"""构建节点 ID 到节点的查找表 + 子节点到父节点的映射。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
index: 树索引。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
(l3_by_id, l2_by_id, l1_by_id, l3_parent_l2, l2_parent_l1)
|
||||||
|
- l3_by_id: L3 节点 ID → L3Node
|
||||||
|
- l2_by_id: L2 节点 ID → L2Node
|
||||||
|
- l1_by_id: L1 节点 ID → L1Node
|
||||||
|
- l3_parent_l2: L3 节点 ID → 其父 L2Node
|
||||||
|
- l2_parent_l1: L2 节点 ID → 其父 L1Node
|
||||||
|
"""
|
||||||
|
l3_by_id: dict[str, L3Node] = {}
|
||||||
|
l2_by_id: dict[str, L2Node] = {}
|
||||||
|
l1_by_id: dict[str, L1Node] = {}
|
||||||
|
l3_parent_l2: dict[str, L2Node] = {}
|
||||||
|
l2_parent_l1: dict[str, L1Node] = {}
|
||||||
|
|
||||||
|
for l1 in index.roots:
|
||||||
|
l1_by_id[l1.id] = l1
|
||||||
|
for l2 in l1.children:
|
||||||
|
l2_by_id[l2.id] = l2
|
||||||
|
l2_parent_l1[l2.id] = l1
|
||||||
|
for l3 in l2.children:
|
||||||
|
l3_by_id[l3.id] = l3
|
||||||
|
l3_parent_l2[l3.id] = l2
|
||||||
|
|
||||||
|
return l3_by_id, l2_by_id, l1_by_id, l3_parent_l2, l2_parent_l1
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 字幕辅助
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _build_subtitle_block(
|
||||||
|
srt_entries: list[SRTEntry] | None,
|
||||||
|
timestamp: float | None,
|
||||||
|
) -> str:
|
||||||
|
"""构建字幕注入文本块。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
srt_entries: SRT 字幕条目列表。
|
||||||
|
timestamp: 帧时间戳(秒)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
字幕文本块字符串(无匹配时返回空字符串)。
|
||||||
|
"""
|
||||||
|
if not srt_entries or timestamp is None:
|
||||||
|
return ""
|
||||||
|
window = 2.0
|
||||||
|
start = max(0.0, timestamp - window)
|
||||||
|
end = timestamp + window
|
||||||
|
text = extract_subtitle_for_range(srt_entries, (start, end))
|
||||||
|
if not text:
|
||||||
|
return ""
|
||||||
|
return f"字幕信息:\n{text}\n"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 主修复函数
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def repair_tree(
|
||||||
|
index: TreeIndex,
|
||||||
|
issues: list[NodeIssue],
|
||||||
|
vlm: VLMProvider,
|
||||||
|
llm: LLMProvider,
|
||||||
|
frames_dir: Path,
|
||||||
|
srt_entries: list[SRTEntry] | None = None,
|
||||||
|
) -> RepairStats:
|
||||||
|
"""修复有问题的节点,底向上级联。
|
||||||
|
|
||||||
|
流程:
|
||||||
|
1. 收集需修复的 L3 节点 → VLM 重新描述帧
|
||||||
|
2. 收集受影响的 L2 → LLM 从 L3 children 聚合
|
||||||
|
3. 收集受影响的 L1 → LLM 从 L2 children 聚合
|
||||||
|
|
||||||
|
参数:
|
||||||
|
index: 待修复的 TreeIndex(原地修改)。
|
||||||
|
issues: detect_issues() 返回的问题列表。
|
||||||
|
vlm: VLM 调用端口。
|
||||||
|
llm: LLM 调用端口。
|
||||||
|
frames_dir: 帧文件根目录。
|
||||||
|
srt_entries: 字幕条目列表(可选)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
RepairStats 统计。
|
||||||
|
"""
|
||||||
|
stats = RepairStats()
|
||||||
|
|
||||||
|
if not issues:
|
||||||
|
logger.info("无修复任务,跳过")
|
||||||
|
return stats
|
||||||
|
|
||||||
|
# 构建查找表
|
||||||
|
l3_by_id, l2_by_id, l1_by_id, l3_parent_l2, l2_parent_l1 = _build_node_lookup(index)
|
||||||
|
|
||||||
|
# Step 1: 修复 L3 节点(仅处理 empty_field + level 3)
|
||||||
|
l3_issues = [
|
||||||
|
issue for issue in issues if issue.issue_type == "empty_field" and issue.level == 3
|
||||||
|
]
|
||||||
|
|
||||||
|
affected_l2_ids: set[str] = set()
|
||||||
|
|
||||||
|
for issue in l3_issues:
|
||||||
|
l3_node = l3_by_id.get(issue.node_id)
|
||||||
|
if l3_node is None:
|
||||||
|
logger.warning(
|
||||||
|
"L3 节点 ID 未在树中找到,跳过",
|
||||||
|
node_id=issue.node_id,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 查找帧文件
|
||||||
|
if l3_node.frame_path is None:
|
||||||
|
logger.warning(
|
||||||
|
"L3 节点无 frame_path,跳过",
|
||||||
|
node_id=issue.node_id,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
frame_file = frames_dir / l3_node.frame_path
|
||||||
|
if not frame_file.exists():
|
||||||
|
logger.warning(
|
||||||
|
"L3 帧文件不存在,跳过修复",
|
||||||
|
node_id=issue.node_id,
|
||||||
|
frame_path=str(frame_file),
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 获取 L2 父节点描述作为上下文
|
||||||
|
parent_l2 = l3_parent_l2.get(issue.node_id)
|
||||||
|
l2_description = parent_l2.card.event_description if parent_l2 else ""
|
||||||
|
|
||||||
|
# 构建字幕块
|
||||||
|
subtitle_block = _build_subtitle_block(srt_entries, l3_node.timestamp)
|
||||||
|
|
||||||
|
# VLM 重新描述帧
|
||||||
|
prompt = _L3_REPAIR_PROMPT.format(
|
||||||
|
l2_description=l2_description,
|
||||||
|
subtitle_block=subtitle_block,
|
||||||
|
)
|
||||||
|
messages = [{"role": "user", "content": prompt}]
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = await vlm.chat_with_images(messages, [str(frame_file)])
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
"L3 修复 VLM 调用失败,跳过: {}",
|
||||||
|
exc,
|
||||||
|
node_id=issue.node_id,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
new_card = _parse_l3_card(response.content)
|
||||||
|
if new_card is None:
|
||||||
|
logger.warning(
|
||||||
|
"L3 修复 VLM 输出解析失败,跳过",
|
||||||
|
node_id=issue.node_id,
|
||||||
|
raw_preview=response.content[:200],
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 原地替换 card(L3Node.card 不是 frozen dataclass 的限制字段)
|
||||||
|
l3_node.card = new_card
|
||||||
|
stats.l3_repaired += 1
|
||||||
|
|
||||||
|
# 标记受影响的 L2 父节点
|
||||||
|
if parent_l2 is not None:
|
||||||
|
affected_l2_ids.add(parent_l2.id)
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
"L3 节点修复完成",
|
||||||
|
node_id=issue.node_id,
|
||||||
|
frame_summary=new_card.frame_summary[:50],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 2: 重生成受影响的 L2 节点
|
||||||
|
affected_l1_ids: set[str] = set()
|
||||||
|
|
||||||
|
for l2_id in affected_l2_ids:
|
||||||
|
l2_node = l2_by_id.get(l2_id)
|
||||||
|
if l2_node is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 从 L3 children 聚合描述
|
||||||
|
l3_texts = "\n".join(f"- {l3.card.frame_summary}" for l3 in l2_node.children)
|
||||||
|
prompt = _L2_REGEN_PROMPT.format(l3_texts=l3_texts)
|
||||||
|
messages = [{"role": "user", "content": prompt}]
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = await llm.chat(messages)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
"L2 重生成 LLM 调用失败,跳过: {}",
|
||||||
|
exc,
|
||||||
|
l2_id=l2_id,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
new_card = _parse_l2_card(response.content)
|
||||||
|
if new_card is None:
|
||||||
|
logger.warning(
|
||||||
|
"L2 重生成 LLM 输出解析失败,跳过",
|
||||||
|
l2_id=l2_id,
|
||||||
|
raw_preview=response.content[:200],
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
l2_node.card = new_card
|
||||||
|
stats.l2_regenerated += 1
|
||||||
|
|
||||||
|
# 标记受影响的 L1 父节点
|
||||||
|
parent_l1 = l2_parent_l1.get(l2_id)
|
||||||
|
if parent_l1 is not None:
|
||||||
|
affected_l1_ids.add(parent_l1.id)
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
"L2 节点重生成完成",
|
||||||
|
l2_id=l2_id,
|
||||||
|
event_description=new_card.event_description[:50],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 3: 重生成受影响的 L1 节点
|
||||||
|
for l1_id in affected_l1_ids:
|
||||||
|
l1_node = l1_by_id.get(l1_id)
|
||||||
|
if l1_node is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 从 L2 children 聚合描述
|
||||||
|
l2_texts = "\n".join(f"- {l2.card.event_description}" for l2 in l1_node.children)
|
||||||
|
prompt = _L1_REGEN_PROMPT.format(l2_texts=l2_texts)
|
||||||
|
messages = [{"role": "user", "content": prompt}]
|
||||||
|
|
||||||
|
try:
|
||||||
|
response = await llm.chat(messages)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
"L1 重生成 LLM 调用失败,跳过: {}",
|
||||||
|
exc,
|
||||||
|
l1_id=l1_id,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
new_card = _parse_l1_card(response.content)
|
||||||
|
if new_card is None:
|
||||||
|
logger.warning(
|
||||||
|
"L1 重生成 LLM 输出解析失败,跳过",
|
||||||
|
l1_id=l1_id,
|
||||||
|
raw_preview=response.content[:200],
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
l1_node.card = new_card
|
||||||
|
stats.l1_regenerated += 1
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
"L1 节点重生成完成",
|
||||||
|
l1_id=l1_id,
|
||||||
|
scene_summary=new_card.scene_summary[:50],
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"树修复完成",
|
||||||
|
l3_repaired=stats.l3_repaired,
|
||||||
|
l2_regenerated=stats.l2_regenerated,
|
||||||
|
l1_regenerated=stats.l1_regenerated,
|
||||||
|
)
|
||||||
|
return stats
|
||||||
@@ -84,10 +84,11 @@ def deduplicate_field(values: list[str]) -> list[str]:
|
|||||||
seen: set[str] = set()
|
seen: set[str] = set()
|
||||||
result: list[str] = []
|
result: list[str] = []
|
||||||
for v in values:
|
for v in values:
|
||||||
key = v.strip().lower()
|
s = str(v).strip()
|
||||||
|
key = s.lower()
|
||||||
if key and key not in seen:
|
if key and key not in seen:
|
||||||
seen.add(key)
|
seen.add(key)
|
||||||
result.append(v)
|
result.append(s)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
@@ -278,7 +279,7 @@ def apply_injections(index: TreeIndex, injections: list[dict[str, Any]]) -> Supp
|
|||||||
stats.facts_skipped += 1
|
stats.facts_skipped += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
inject_value = instr.get("inject_value", "")
|
inject_value = str(instr.get("inject_value", "")).strip()
|
||||||
if not inject_value:
|
if not inject_value:
|
||||||
stats.facts_skipped += 1
|
stats.facts_skipped += 1
|
||||||
continue
|
continue
|
||||||
|
|||||||
+4
-12
@@ -15,9 +15,7 @@ APPENDIX_MAX_CHARS = 2000 # appendix 区软上限(守设计「长度上限+wa
|
|||||||
MOMENTUM_START = "<!-- MOMENTUM_START -->"
|
MOMENTUM_START = "<!-- MOMENTUM_START -->"
|
||||||
MOMENTUM_END = "<!-- MOMENTUM_END -->"
|
MOMENTUM_END = "<!-- MOMENTUM_END -->"
|
||||||
MOMENTUM_MAX_CHARS = 2000 # momentum 区软上限(与 appendix 一致:超限 warning 不截断)
|
MOMENTUM_MAX_CHARS = 2000 # momentum 区软上限(与 appendix 一致:超限 warning 不截断)
|
||||||
MOMENTUM_HEADING = (
|
MOMENTUM_HEADING = "## 动量指导(每轮重写,勿手改)" # replace_momentum 写入的固定标题行
|
||||||
"## 动量指导(每轮重写,勿手改)" # replace_momentum 写入的固定标题行
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def momentum_region_bounds(text: str) -> tuple[int, int] | None:
|
def momentum_region_bounds(text: str) -> tuple[int, int] | None:
|
||||||
@@ -303,9 +301,7 @@ def _insert_at(content: str, at: int, payload: str) -> str:
|
|||||||
return head + "\n\n" + payload + "\n"
|
return head + "\n\n" + payload + "\n"
|
||||||
|
|
||||||
|
|
||||||
def _do_append(
|
def _do_append(content: str, payload: str, ranges: list[tuple[int, int]]) -> tuple[str, str]:
|
||||||
content: str, payload: str, ranges: list[tuple[int, int]]
|
|
||||||
) -> tuple[str, str]:
|
|
||||||
"""执行 append 操作,返回更新后内容与状态字符串。"""
|
"""执行 append 操作,返回更新后内容与状态字符串。"""
|
||||||
return _insert_at(content, _append_at(content, ranges), payload), "applied_append"
|
return _insert_at(content, _append_at(content, ranges), payload), "applied_append"
|
||||||
|
|
||||||
@@ -351,9 +347,7 @@ def _do_replace_delete(
|
|||||||
return new_content, "applied_" + op
|
return new_content, "applied_" + op
|
||||||
|
|
||||||
|
|
||||||
def _apply_one(
|
def _apply_one(content: str, edit: dict, ranges: list[tuple[int, int]]) -> tuple[str, dict]:
|
||||||
content: str, edit: dict, ranges: list[tuple[int, int]]
|
|
||||||
) -> tuple[str, dict]:
|
|
||||||
"""应用单条 edit,返回 (更新后内容, 状态报告)。"""
|
"""应用单条 edit,返回 (更新后内容, 状态报告)。"""
|
||||||
if not isinstance(edit, dict):
|
if not isinstance(edit, dict):
|
||||||
return content, {
|
return content, {
|
||||||
@@ -382,9 +376,7 @@ def _apply_one(
|
|||||||
return content, report
|
return content, report
|
||||||
|
|
||||||
if op in ("replace", "delete"):
|
if op in ("replace", "delete"):
|
||||||
content, report["status"] = _do_replace_delete(
|
content, report["status"] = _do_replace_delete(op, content, target, payload, ranges)
|
||||||
op, content, target, payload, ranges
|
|
||||||
)
|
|
||||||
return content, report
|
return content, report
|
||||||
|
|
||||||
logger.warning("未知 op,跳过: {}", op)
|
logger.warning("未知 op,跳过: {}", op)
|
||||||
|
|||||||
@@ -0,0 +1,146 @@
|
|||||||
|
# 论文主图设计:Self-Evolving Search Agent 推理训练闭环
|
||||||
|
|
||||||
|
**日期** 2026-07-09 · **状态** 已获用户批准(口头) · **产出物** Figma 图(文件 `xnLGUkZottqnr4dsEt9fGq` Page 1 空白区)
|
||||||
|
|
||||||
|
## 1. 目标与定位
|
||||||
|
|
||||||
|
- **用途**:论文主图(目标会议存在文档分歧:CLAUDE.md=AAAI 2026,ARCHITECTURE.md 与记忆=EMNLP 2026,以用户最终决定为准,不影响本图设计),展示推理训练部分的自进化闭环;不含建树与新题生成(建树已有独立图,位于同一 Figma 文件)。
|
||||||
|
- **核心叙事**:搜索 Agent 通过 推理→诊断→进化→门控 闭环自我改进;**Frozen LLM, trainable harness**——被"训练"的不是模型权重,而是版本化的 Skills+Prompts。
|
||||||
|
- **差异化**:AVP/DVD 等相关工作画的是推理期内环(agent 怎么搜视频);本图内环只是一个面板,**训练期外环是主角**。
|
||||||
|
- **审稿人一句话记忆点**:这是一个不动模型权重的 PyTorch 式训练循环。
|
||||||
|
|
||||||
|
## 2. 已确认的关键决策
|
||||||
|
|
||||||
|
| 决策点 | 结论 |
|
||||||
|
|---|---|
|
||||||
|
| 构图 | 水平流水线 + 底部参数回流闭环(方案 A) |
|
||||||
|
| 版式 | 双栏跨页宽图,画布 2400×1050 px(≈2.3:1,缩印 180mm) |
|
||||||
|
| 信息密度 | 四机制全部可见(Agent 内环 / 诊断瀑布 / patch 引擎 / CE-Gate+信息阶梯),去工程化(无熔断/缓存/遥测) |
|
||||||
|
| PyTorch 类比 | 底部独立双行对照条,与上方区域逐段对齐 |
|
||||||
|
| 示例贯穿 | 延续建树图同一 Video-MME 天文台视频,问题/诊断/patch 文本典型化设计 |
|
||||||
|
| 迭代维度 | Store 处 v1…vN 卡片堆叠 + ×N epochs 循环标记暗示,不加独立时间轴 |
|
||||||
|
| 绘制位置 | 与建树图同文件(素材直接复用),Page 1 空白区 y≥1800,不动现有图层 |
|
||||||
|
| 语言 | 图内文字全英文 |
|
||||||
|
|
||||||
|
## 3. 布局
|
||||||
|
|
||||||
|
> 下方 ASCII 草图为中文说明稿,仅示意区域关系;最终图层文字一律采用 §4/§5 的英文术语。
|
||||||
|
|
||||||
|
```
|
||||||
|
┌──────────────────────────────────────────────────────────────────┐
|
||||||
|
│ [输入] ┌──────────────┐ ┌───────────┐ ┌──────────┐ ┌──────┐ │
|
||||||
|
│ Q+缩略图→│ ① INFERENCE │→ │② DIAGNOSE │→ │③ EVOLVE │→ │④ CE- │ │
|
||||||
|
│ │ 树环境+内环 │ │ 归因瀑布 │ │ patch引擎 │ │ GATE │ │
|
||||||
|
│ └──────↑───────┘ └───────────┘ └──────────┘ └──┬───┘ │
|
||||||
|
│ │ read ┌─────────────────┐ accept│ │
|
||||||
|
│ └─────────────────│⑤ Skills+Prompts │←──────┘ │
|
||||||
|
│ │ Store v1…vN ▤▤ │ reject→保基线
|
||||||
|
│ └─────────────────┘ │
|
||||||
|
├──────────────────────────────────────────────────────────────────┤
|
||||||
|
│ DataLoader│forward()│backward()│optimizer.step()│grad clip│nn.Parameter│
|
||||||
|
└──────────────────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
**数据流勘误记录**:CE-Gate 位于 Evolve 之后(进化产出候选 → gate 用 e-process 验证候选 vs 基线 → accept 才写入 Store),信息阶梯为 gate 供给高信息量题序,而非控制输入难度。此顺序已与 `core/evolution/gate.py`、`app/harness/gate_ladder.py` 核实。
|
||||||
|
|
||||||
|
## 4. 五区域内容规格
|
||||||
|
|
||||||
|
术语均与代码核实一致(来源见 §7)。
|
||||||
|
|
||||||
|
### 4.1 输入区
|
||||||
|
- 问题卡(Q: *"What happens right after the dome opens?"* 措辞绘制时可打磨)+ 天文台视频缩略图 + 迷你树 icon,标注 "hierarchical video tree (Fig. 2)" 衔接建树图。
|
||||||
|
|
||||||
|
### 4.2 ① INFERENCE(agent-controlled,图上唯一画成循环的面板)
|
||||||
|
- 树环境迷你版:L1/L2/L3 三层色带(绿/蓝/黄,复用建树图配色与缩略图)。
|
||||||
|
- Agent 内环轨迹:Thought → `search_similar` → `view_node` → `observe_frame` → `submit_answer`;侧边 `read_skill` 箭头(来自 ⑤,工具名与 `app/search/tools.py:56-58` 一致)。
|
||||||
|
- 示例结局:answer ✗(答错),输出 trace 流向 ②。
|
||||||
|
- 面板角标:*agent-controlled*;其余面板角标 *code-controlled*。
|
||||||
|
|
||||||
|
### 4.3 ② DIAGNOSE(code-controlled)
|
||||||
|
- 归因瀑布级联:`extraction failure → search failure → reasoning failure`(+ mixed 兜底),画成三级下落台阶。
|
||||||
|
- 二分岔:**defect**(改 skill 正文)vs **lapse**(记 appendix 提醒)。
|
||||||
|
- D1–D5 压缩为一排五个小 chip:attribution / tool quality / search behavior / skill compliance / decision patterns。
|
||||||
|
- 示例:该题归因 `search failure` → 判 **defect**。
|
||||||
|
|
||||||
|
### 4.4 ③ EVOLVE
|
||||||
|
- patch 流水线:candidate edits → **rank-and-clip** → apply patch。
|
||||||
|
- 侧边锁条带:**protected spans**(appendix / momentum 区带锁图标,不可改写)。
|
||||||
|
- 示例 patch 片段:*"+ verify event boundary via L2 card before observe_frame"*。
|
||||||
|
- momentum 机制不单独出现(用户确认),仅隐含于锁条带。
|
||||||
|
|
||||||
|
### 4.5 ④ Validation · CE-GATE
|
||||||
|
- 面板标题 **Validation · CE-Gate**:块顺序验证(`validate.py` 配对翻转 W/L)作为输入喂 e-process——图上画为"candidate vs baseline 配对小图 → e 曲线"。
|
||||||
|
- e-process 小曲线:e 值随题数爬升,越过 `e_confirm` 虚线。
|
||||||
|
- 四出口:**accept (confirmed) / accept (provisional) / reject / continue**(代码中三种 reject 在图上合并,用户确认)。
|
||||||
|
- 侧挂小组件:信息阶梯(2:1 交错题序图标,标注 *info-max question ladder*),尺寸压小避免抢焦点。
|
||||||
|
- 视觉层级:accept 主路径线最粗;reject/continue 细灰次级线。
|
||||||
|
|
||||||
|
### 4.6 ⑤ Skills+Prompts Store
|
||||||
|
- v1…vN 卡片堆叠(复用建树图 Event Card 堆叠画法)+ 版本号 badge。
|
||||||
|
- accept 箭头写入 v(N+1);read 箭头回流至 ①,构成大闭环;循环标记 **×N epochs**。
|
||||||
|
|
||||||
|
## 5. PyTorch 对照条(最底部)
|
||||||
|
|
||||||
|
浅灰底横带,等宽字体,与上方区域逐段对齐:
|
||||||
|
|
||||||
|
| 上方区域 | 对照文字 |
|
||||||
|
|---|---|
|
||||||
|
| 输入 | `DataLoader` |
|
||||||
|
| ① | `model.forward()` |
|
||||||
|
| ② | `loss.backward()` |
|
||||||
|
| ③ | `optimizer.step()` |
|
||||||
|
| ④ | `grad clipping (validate)`(对应 CLAUDE.md 类比表中"进化 validation = grad clipping";④ 面板同时含 validate 配对翻转与 CE-Gate 判定) |
|
||||||
|
| ⑤ | `nn.Parameter` |
|
||||||
|
|
||||||
|
条带一侧放记忆点标语:*Frozen LLM, trainable harness*。
|
||||||
|
|
||||||
|
## 6. 视觉规范与素材复用
|
||||||
|
|
||||||
|
| 元素 | 方案 |
|
||||||
|
|---|---|
|
||||||
|
| 面板样式 | 白底、细虚线外框、顶部居中标题(沿用建树图) |
|
||||||
|
| ① | 淡绿系 · ② 淡橙红系(新增,饱和度对齐现有 pastel) · ③ 淡紫系(复用 VLM 紫) · ④ 淡蓝系 · ⑤ 白卡+badge |
|
||||||
|
| 直接复用 | 视频缩略图(candidate_a_t*)、L1/L2/L3 badge、Scene/Event/Frame Card 组件、VLM 紫块、箭头/chevron 样式 |
|
||||||
|
| 字体 | 与建树图一致(Inter);标题 24px / 正文 16-18px / 标注最小 15px 灰(2400px 画布缩印 180mm 后 15px ≈ 1.1mm,12px 过小已弃用) |
|
||||||
|
| 图层组织 | 顶层 Frame 命名 `Main Figure — Self-Evolving Loop`,五区域各一个子 Group,便于后续人工微调 |
|
||||||
|
|
||||||
|
## 7. 术语出处(代码核实)
|
||||||
|
|
||||||
|
| 图上术语 | 来源 |
|
||||||
|
|---|---|
|
||||||
|
| gate 四出口 accept_confirmed / accept_provisional / reject×3 / continue | `core/evolution/gate.py:57-110` |
|
||||||
|
| 归因瀑布 extraction/search/reasoning/mixed;defect vs lapse | `core/evolution/diagnose.py:910-997` |
|
||||||
|
| D1-D5 五维聚合 | `core/evolution/diagnose.py:1095-1307`(D2-D5)、`1551-1563` + `2230` + `2293-2296`(D1 attribution distribution) |
|
||||||
|
| 信息阶梯冷启动 2:1、信息量排序 | `app/harness/gate_ladder.py:57-117` |
|
||||||
|
| 块顺序验证配对翻转 W/L | `core/evolution/validate.py:12-69` |
|
||||||
|
| rank-and-clip、protected spans、appendix/momentum 区 | `core/evolution/evolve.py:186-594`、`core/evolution/patch.py:11-56` |
|
||||||
|
| agent 工具五件套(含 `read_skill`) | `app/search/tools.py:33-73` |
|
||||||
|
| 版本目录 Store `store/skills/v{N}` / workspace 本地拷贝 | `app/harness/store.py:28-136`、`app/harness/workspace.py:103-108,150-152` |
|
||||||
|
|
||||||
|
## 8. 验收标准
|
||||||
|
|
||||||
|
1. 图在 Figma 中为独立顶层 Frame,可整体导出 PNG/SVG,缩印 180mm 宽时最小文字(12px 标注)仍可辨认。
|
||||||
|
2. 五区域 + 对照条齐全,闭环箭头(⑤→① read、④→⑤ accept)无歧义。
|
||||||
|
3. 全部术语与 §7 代码核实结果一致;无熔断/缓存/遥测等工程元素。
|
||||||
|
4. 风格与同文件建树图肉眼一致(配色、字体、面板语言、卡片组件)。
|
||||||
|
5. 不改动/移动建树图的任何现有图层。
|
||||||
|
|
||||||
|
## 9. 构图重构记录(2026-07-09 定稿后追加)
|
||||||
|
|
||||||
|
用户验收反馈:内容正确但"下半部空、无主线重点"。经方案比选(用户选 A),实施:
|
||||||
|
|
||||||
|
| 改动 | 内容 |
|
||||||
|
|---|---|
|
||||||
|
| 显式循环主干 | 面板间 chevron → 4px 黑色实心三角箭头(forward 主线);④→⑤ write 与 ⑤→① read 回流均为 4px 绿色实线带,⟳ ×N epochs 置于带上;黑/绿双色对应 forward / parameter-update 语义 |
|
||||||
|
| 底部压缩 | Store 2080px 全宽行 → 680×170 紧凑块(右缘对齐 ④,write 直指 v5);对照条 130→84px;画布 1050→**940**(2.55:1) |
|
||||||
|
| 填充 | motto 24px 斜体移至左下空区;read 带起点加绿色圆点锚记 |
|
||||||
|
|
||||||
|
逐模块精修均经 Claude 自审 + Codex 独立审双 PASS(Question/①/②/③/④/⑤+对照条/整图重构共 8 轮审核)。Codex 抓到的实质问题:④ 的 W/L 翻转数与 e 曲线出口统计不自洽(修正为序列省略号 + W=8·L=0,E=56.78>e_confirm=20)、ladder 色块数与题数不符、read 线易误读为边框。
|
||||||
|
|
||||||
|
## 10. 被拒绝的备选方案
|
||||||
|
|
||||||
|
| 方案 | 拒绝原因 |
|
||||||
|
|---|---|
|
||||||
|
| B 上下双层 S 形回路 | PyTorch 对照条无法与面板逐段对齐,退化为角标 |
|
||||||
|
| C 中心辐射环形 | 2.3:1 宽幅下横向空间浪费大,机制细节难展开,与建树图直线叙事不一致 |
|
||||||
|
| 独立进化时间轴 | 占版面,与对照条拥挤;由 Store 版本堆叠 + ×N epochs 替代 |
|
||||||
@@ -0,0 +1,477 @@
|
|||||||
|
---
|
||||||
|
id: question-gen-synth
|
||||||
|
title: 赛题生成工具设计(Question Generation Synthesis)
|
||||||
|
type: design
|
||||||
|
created: 2026-07-09
|
||||||
|
status: draft
|
||||||
|
---
|
||||||
|
|
||||||
|
# 赛题生成工具设计
|
||||||
|
|
||||||
|
## 1. 目标与动机
|
||||||
|
|
||||||
|
让视频树自行生成与 Video-MME 原始赛题风格、难度近似的四选一选择题,用于自进化训练循环的 DataLoader。原始 900 道 benchmark 题保留为 held-out 最终评测集,避免"直接拿答案调"的审稿质疑。
|
||||||
|
|
||||||
|
**角色定位**:生成题 = 训练集,原始题 = 测试集。进化循环的改进效果最终由原始 benchmark 验证泛化能力。
|
||||||
|
|
||||||
|
**训练 vs 论文评测的区分**:训练循环全程使用生成题(三池切分——诊断池/验证池/test 池——均来自生成题),论文报告的 held-out 泛化指标是训练结束后,用最终 best 版本对原始 benchmark 全量 900 题单独跑推理得到的结果。两步分离,Runner 代码无需改动。
|
||||||
|
|
||||||
|
## 2. 模块结构与职责边界
|
||||||
|
|
||||||
|
### 2.1 文件布局
|
||||||
|
|
||||||
|
```
|
||||||
|
app/question_gen/
|
||||||
|
├── __init__.py ← 已有:re-export loader API
|
||||||
|
├── loader.py ← 已有:load_benchmark + stratified_sample
|
||||||
|
└── synthesizer.py ← 新增①:出题核心逻辑
|
||||||
|
|
||||||
|
app/harness/
|
||||||
|
└── factory.py ← 新增②:推理依赖组装(wiring)
|
||||||
|
|
||||||
|
tools/generate_questions.py ← 新增③:CLI 壳(generate + calibrate)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.2 职责切分
|
||||||
|
|
||||||
|
| 模块 | 职责 | 消费者 |
|
||||||
|
|------|------|--------|
|
||||||
|
| `synthesizer.py` | 题型-层级映射、锚节点采样、prompt 构造(few-shot)、embedding 去重、单题生成编排 | `tools/generate_questions.py` |
|
||||||
|
| `factory.py` | 给定 store 路径 + config → 组装 LLM/VLM/Embedding/SearchToolDispatcher/PromptManager 全套推理依赖 | `tools/generate_questions.py`(校准)、未来 `main.py`、Runner |
|
||||||
|
| `tools/generate_questions.py` | CLI 参数解析、并发编排(Semaphore)、进度日志、JSON 输出 | 用户直接运行 |
|
||||||
|
|
||||||
|
### 2.3 依赖方向
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart LR
|
||||||
|
TOOLS["tools/generate_questions.py"] --> SYN["app/question_gen/synthesizer"]
|
||||||
|
TOOLS --> FAC["app/harness/factory"]
|
||||||
|
TOOLS --> ADP["adapters/*"]
|
||||||
|
FAC --> SEARCH["app/search/*"]
|
||||||
|
FAC --> ENV["app/tree/environment"]
|
||||||
|
FAC --> ADP
|
||||||
|
SYN --> PROTO["core/protocols (VLMProvider, EmbeddingProvider via DI)"]
|
||||||
|
SYN --> TYPES["core/types (GeneratedQuestion)"]
|
||||||
|
SYN --> IDX["app/tree/index (TreeIndex)"]
|
||||||
|
```
|
||||||
|
|
||||||
|
全部合规——外层→内层,`core/` 不依赖任何外层。
|
||||||
|
|
||||||
|
### 2.4 与 QuestionGenerator Protocol 的关系
|
||||||
|
|
||||||
|
`app/ports.py` 已预留 `QuestionGenerator` Protocol。本设计**不实现该 Protocol**——出题是一次性离线工具而非运行时能力,Runner 不需要运行时出题。`synthesizer.py` 的函数式接口(`generate_one` 等纯函数 + async 编排)比 Protocol class 更适合工具脚本场景。`QuestionGenerator` Protocol 保留但标记为"预留,当前无实现",不删除——若未来需要运行时出题可基于 synthesizer 的纯函数包装实现。
|
||||||
|
|
||||||
|
### 2.5 方案选择与否决
|
||||||
|
|
||||||
|
| 方案 | 否决理由 |
|
||||||
|
|------|---------|
|
||||||
|
| A: 单体脚本(全部逻辑放 `tools/`) | 业务逻辑(题型映射、采样、prompt、去重)混在 CLI 编排中,不可独立测试;不匹配 repair 管线的 app/ + tools/ 分层惯例 |
|
||||||
|
| B: Protocol 实现 + 脚本编排(`adapters/` 实现 `QuestionGenerator`) | adapter 层语义是外部服务接口,出题逻辑是应用层业务规则,放 adapter 层语义不匹配 |
|
||||||
|
| **C: app/ 业务逻辑 + tools/ CLI 壳(采用)** | 与 repair 管线结构一致,Clean Architecture 依赖方向合规,业务逻辑可独立测试 |
|
||||||
|
|
||||||
|
## 3. synthesizer.py 核心设计
|
||||||
|
|
||||||
|
### 3.1 题型-层级映射
|
||||||
|
|
||||||
|
模块级常量,沿用 TRM4 设计文档的映射表:
|
||||||
|
|
||||||
|
| 锚定层级 | 题型 | 帧图 | 文本上下文 | 帧数 |
|
||||||
|
|---------|------|------|-----------|------|
|
||||||
|
| L3 | Object Recognition | 必须 | frame_summary | 1 |
|
||||||
|
| L3 | Attribute Perception | 必须 | frame_summary | 1 |
|
||||||
|
| L3 | OCR Problems | 必须 | frame_summary | 1 |
|
||||||
|
| L3 | Spatial Reasoning | 必须 | frame_summary + spatial_layout | 1 |
|
||||||
|
| L3 | Spatial Perception | 必须 | frame_summary | 1 |
|
||||||
|
| L2 | Action Recognition | 必须 | 事件 card | 2-3(子帧均匀采样) |
|
||||||
|
| L2 | Action Reasoning | 必须 | 事件 card | 2-3 |
|
||||||
|
| L2 | Counting Problem | 必须 | 事件 card | 2-3 |
|
||||||
|
| L2 | Temporal Perception | 可选 | 事件 card + time_range | 0-1 |
|
||||||
|
| L1 | Temporal Reasoning | 必须 | 根 card + 多个 L2 card(≥3) | 每 L2 取 1 张代表帧 |
|
||||||
|
| L1 | Information Synopsis | 必须 | 根 card + 全部 L2 card | 每 L2 取 1 张代表帧 |
|
||||||
|
| L1-L2 | Object Reasoning | 必须 | 2-3 个 L2 card | 每 L2 取 1 张代表帧 |
|
||||||
|
|
||||||
|
节点采样:每道题从全部视频树中随机选一棵,在对应层级随机选一个锚节点。同视频同题型不重复。L1 题型使用多个 L2 子节点联合输入时,按时间顺序组织节点,保持叙事连贯性。
|
||||||
|
|
||||||
|
### 3.2 AnchorContext 数据结构
|
||||||
|
|
||||||
|
```python
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AnchorContext:
|
||||||
|
"""锚节点上下文——生成单道题所需的全部素材。"""
|
||||||
|
node_id: str # 锚节点 ID
|
||||||
|
card_text: str # 锚节点 card 序列化文本
|
||||||
|
frame_paths: list[str] # 帧图片路径
|
||||||
|
subtitle: str # 对应字幕(可空)
|
||||||
|
distractor_texts: list[str] # 同视频其他节点摘要(供 VLM 生成干扰项)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.3 核心函数签名
|
||||||
|
|
||||||
|
```python
|
||||||
|
# 纯函数:从树中采样锚节点 + 帧 + 上下文
|
||||||
|
def sample_anchor(
|
||||||
|
tree: TreeIndex,
|
||||||
|
task_type: str,
|
||||||
|
used_node_ids: set[str],
|
||||||
|
rng: random.Random,
|
||||||
|
) -> AnchorContext
|
||||||
|
|
||||||
|
# 纯函数:组装 VLM prompt(system + user,含 few-shot exemplar)
|
||||||
|
def build_generation_prompt(
|
||||||
|
task_type: str,
|
||||||
|
anchor: AnchorContext,
|
||||||
|
exemplars: list[GeneratedQuestion],
|
||||||
|
) -> tuple[list[dict], list[str]]
|
||||||
|
# 返回:(messages, image_paths) — 直接喂给 VLMProvider
|
||||||
|
|
||||||
|
# 纯函数:解析 VLM 返回的 JSON → 部分字段字典
|
||||||
|
# source_nodes 和 difficulty 由 generate_one 在 parse 后用 anchor 信息补齐
|
||||||
|
def parse_vlm_response(
|
||||||
|
raw: str,
|
||||||
|
video_id: str,
|
||||||
|
task_type: str,
|
||||||
|
seq: int,
|
||||||
|
) -> dict
|
||||||
|
# 返回:{"question_id", "question", "options", "answer"} 字典
|
||||||
|
# 调用方补齐 source_nodes/difficulty 后构造 GeneratedQuestion
|
||||||
|
|
||||||
|
# 纯函数:embedding 去重判定
|
||||||
|
def is_duplicate(
|
||||||
|
question_text: str,
|
||||||
|
pool_embeddings: np.ndarray,
|
||||||
|
embed_fn: Callable[[str | list[str]], np.ndarray],
|
||||||
|
threshold: float,
|
||||||
|
) -> bool
|
||||||
|
|
||||||
|
# 异步编排:生成单道题(含重试 + 去重循环)
|
||||||
|
async def generate_one(
|
||||||
|
vlm: VLMProvider,
|
||||||
|
embed_fn: Callable[[str | list[str]], np.ndarray],
|
||||||
|
tree: TreeIndex,
|
||||||
|
video_id: str,
|
||||||
|
task_type: str,
|
||||||
|
seq: int,
|
||||||
|
*,
|
||||||
|
exemplars: list[GeneratedQuestion],
|
||||||
|
pool_embeddings: np.ndarray,
|
||||||
|
used_node_ids: set[str],
|
||||||
|
max_retries: int,
|
||||||
|
similarity_threshold: float,
|
||||||
|
rng: random.Random,
|
||||||
|
session_id: str,
|
||||||
|
) -> GeneratedQuestion | None
|
||||||
|
```
|
||||||
|
|
||||||
|
**设计要点**:
|
||||||
|
- 纯函数(sample_anchor、build_generation_prompt、parse_vlm_response、is_duplicate)可独立单测,不需要 VLM
|
||||||
|
- `generate_one` 是唯一异步函数,接收 `VLMProvider` 通过 DI
|
||||||
|
- 干扰项来自 `AnchorContext.distractor_texts`——同视频其他节点的真实信息
|
||||||
|
|
||||||
|
### 3.4 few-shot exemplar 选择
|
||||||
|
|
||||||
|
生成 prompt 包含 2-3 道同题型的原始 benchmark 题作示例,对齐风格和难度。
|
||||||
|
|
||||||
|
选择策略:
|
||||||
|
- 每题型取 `min(3, 该题型 benchmark 总量)` 道
|
||||||
|
- 按 seed 随机采样 + 跨视频去重(避免 exemplar 全来自同一视频)
|
||||||
|
- exemplar 是只读引用,不从 benchmark 评测集中移除
|
||||||
|
|
||||||
|
### 3.5 prompt 结构
|
||||||
|
|
||||||
|
```
|
||||||
|
System: 视频理解题目生成器,根据视频树节点内容和帧图生成 {task_type} 四选一题。
|
||||||
|
|
||||||
|
[2-3 道该题型原始 benchmark 题作示例]
|
||||||
|
|
||||||
|
约束:
|
||||||
|
- 问题必须基于给定节点内容,不能靠常识推断
|
||||||
|
- 干扰项来自同视频其他节点的真实信息(非凭空捏造)
|
||||||
|
- 难度和问法风格与示例一致
|
||||||
|
|
||||||
|
User: [锚节点 card + 字幕 + 帧图] + [同视频其他节点摘要,供干扰项素材]
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.6 去重机制
|
||||||
|
|
||||||
|
用 `EmbeddingProvider`(nomic-embed-text-v1.5)对 question 文本做 embedding,余弦相似度检查:
|
||||||
|
|
||||||
|
| 检查对 | 阈值 | 处理 |
|
||||||
|
|--------|------|------|
|
||||||
|
| 生成题 vs 原始 benchmark 同题型题 | ≥ similarity_threshold | 丢弃,换节点重试 |
|
||||||
|
| 生成题 vs 已生成的同题型题 | ≥ similarity_threshold | 丢弃,换节点重试 |
|
||||||
|
|
||||||
|
维护 embedding 池(原始题 + 已通过的生成题),每生成一道新题即时查重。单题最多重试 `max_retries` 次。某题型连续耗尽重试配额时,脚本报错退出并输出已完成/未完成的题型统计,不静默少题。
|
||||||
|
|
||||||
|
**并发去重安全**:embedding 池的"检查 + 添加"必须是原子操作。并发 `generate_one` 任务成功后,通过单线程汇总点(asyncio.Queue 或 await 后顺序提交)更新 embedding 池 + 写 JSON + 更新 progress,避免竞态导致相似题同时通过。
|
||||||
|
|
||||||
|
## 4. factory.py 推理依赖组装
|
||||||
|
|
||||||
|
### 4.1 解决的问题
|
||||||
|
|
||||||
|
目前 `Runner._make_tool_dispatch_fn()` 和 `_make_prompt_builder()` 都是 `raise NotImplementedError`,设计为"由 main.py 注入"。组装逻辑涉及 adapter 实例化 + app 组件串联,应提取为可复用的 factory 函数,避免在每个调用方(tools/ 脚本、未来 main.py)重复 wiring。
|
||||||
|
|
||||||
|
### 4.2 核心接口
|
||||||
|
|
||||||
|
```python
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class InferenceDeps:
|
||||||
|
"""跑一次推理所需的全套依赖(不含 HarnessLog,其生命周期由调用方管理)。"""
|
||||||
|
llm: LLMProvider
|
||||||
|
tool_dispatch_fn: Callable # SearchToolDispatcher.dispatch
|
||||||
|
prompt_builder: Callable # PromptManager 的偏函数
|
||||||
|
|
||||||
|
def build_inference_deps(
|
||||||
|
*,
|
||||||
|
store_dir: Path,
|
||||||
|
video_id: str,
|
||||||
|
prompts_dir: Path,
|
||||||
|
skills_dir: Path | None,
|
||||||
|
skill_mode: str,
|
||||||
|
embed_provider: EmbeddingProvider,
|
||||||
|
llm: LLMProvider,
|
||||||
|
vlm: VLMProvider,
|
||||||
|
ocr: OCRProvider | None,
|
||||||
|
verify_vision: bool,
|
||||||
|
anchor: bool,
|
||||||
|
assemble_mode: str,
|
||||||
|
) -> InferenceDeps
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.3 内部流程
|
||||||
|
|
||||||
|
```
|
||||||
|
build_inference_deps()
|
||||||
|
├── 加载 TreeIndex(store_dir/videos/{video_id}/tree.json)
|
||||||
|
├── 构建 TreeEnvironment(index=tree, frames_dir=videos/{video_id}/frames)
|
||||||
|
├── 构建 SkillRegistry(skills_dir,可选)
|
||||||
|
├── 构建 SearchToolDispatcher(env, tool_llm, vlm, ocr, prompts_dir,
|
||||||
|
│ skills, embed_fn, verify_vision, anchor, assemble_mode)
|
||||||
|
├── 构建 PromptManager(prompts_dir)→ 偏函数化 prompt_builder(绑定 skill_mode)
|
||||||
|
└── 返回 InferenceDeps
|
||||||
|
```
|
||||||
|
|
||||||
|
注意:`HarnessLog` 不放入 `InferenceDeps`——其生命周期由调用方通过 `with HarnessLog(...) as log` 管理,作为参数传给 `run_inference`。
|
||||||
|
|
||||||
|
### 4.4 消费者
|
||||||
|
|
||||||
|
| 消费者 | 用法 |
|
||||||
|
|--------|------|
|
||||||
|
| `tools/generate_questions.py` calibrate | 按 video_id 分组题目,对每组调 `build_inference_deps` 构建对应视频树的依赖 → 分组 `run_inference` |
|
||||||
|
| 未来 `main.py --mode infer` | CLI 参数映射到 factory 参数 |
|
||||||
|
| `Runner` | `_make_tool_dispatch_fn` / `_make_prompt_builder` 改为委托 factory |
|
||||||
|
|
||||||
|
### 4.5 设计约束
|
||||||
|
|
||||||
|
- factory 只做**组装**,不持有状态——每次调用返回独立的 `InferenceDeps`
|
||||||
|
- adapter 实例(LLM/VLM/Embedding)由调用方创建并传入,factory 不管 adapter 生命周期
|
||||||
|
- 调用方自由决定 adapter 的复用策略(共享 vs 按需创建)
|
||||||
|
|
||||||
|
## 5. tools/generate_questions.py CLI 设计
|
||||||
|
|
||||||
|
### 5.1 子命令
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 生成
|
||||||
|
python tools/generate_questions.py generate \
|
||||||
|
--store-dir store \
|
||||||
|
--output-dir store/questions/generated/Video-MME \
|
||||||
|
--per-type 20 \
|
||||||
|
--similarity-threshold 0.85 \
|
||||||
|
--max-retries 3 \
|
||||||
|
--concurrency 8 \
|
||||||
|
--seed 42
|
||||||
|
|
||||||
|
# 校准(生成题 vs benchmark 基线对比)
|
||||||
|
python tools/generate_questions.py calibrate \
|
||||||
|
--generated-dir store/questions/generated/Video-MME \
|
||||||
|
--benchmark-dir store/questions/benchmarks/Video-MME \
|
||||||
|
--store-dir store \
|
||||||
|
--db-path results/calibrate.db \
|
||||||
|
--prompts-dir store/prompts \
|
||||||
|
--concurrency 4 \
|
||||||
|
--max-steps 15 \
|
||||||
|
--skill-mode auto \
|
||||||
|
--tolerance 0.10 \
|
||||||
|
--alpha 0.05 \
|
||||||
|
--baseline-db <可选,已有基线 DB 路径> \
|
||||||
|
--baseline-run-id <可选,已有基线 run_id>
|
||||||
|
```
|
||||||
|
|
||||||
|
除 baseline 复用参数外均必传,无默认值(CLAUDE.md §4.5)。`--baseline-db` + `--baseline-run-id` 可选但必须成对出现:有则从 DB 读 benchmark 基线,无则自动跑一次 benchmark 推理。
|
||||||
|
|
||||||
|
### 5.2 generate 流程
|
||||||
|
|
||||||
|
```
|
||||||
|
1. 加载 300 棵树的 video_id 列表
|
||||||
|
2. 加载 benchmark 题目(作为 few-shot exemplar 来源)
|
||||||
|
3. 初始化 embedding 池(benchmark 题 question text → embedding)
|
||||||
|
4. 实例化 GovernedVLMClient + EmbeddingProvider
|
||||||
|
5. 检查断点续跑文件(progress.json)
|
||||||
|
6. 对 12 题型 × per_type:
|
||||||
|
├── 跳过已完成的(断点续跑)
|
||||||
|
├── 随机选视频 + 锚节点(同视频同题型不重复)
|
||||||
|
├── asyncio.Semaphore(concurrency) 并发调 generate_one
|
||||||
|
├── 成功 → 加入 embedding 池 + 追加到结果 + 更新 progress
|
||||||
|
└── 连续耗尽重试 → 报错退出,输出已完成/未完成统计
|
||||||
|
7. 按 video_id 分组写入 JSON
|
||||||
|
8. 全部完成后删除 progress.json
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5.3 calibrate 流程
|
||||||
|
|
||||||
|
```
|
||||||
|
1. load_benchmark 加载生成题和 benchmark 题
|
||||||
|
2. 获取 benchmark 基线:
|
||||||
|
├── 有 --baseline-db + --baseline-run-id → 从 DB 读 per_task_type accuracy
|
||||||
|
└── 没有 → 按 video_id 分组 benchmark 题 → 每组 build_inference_deps
|
||||||
|
→ 分组 run_inference → 汇总存 DB
|
||||||
|
3. 按 video_id 分组生成题 → 每组 build_inference_deps → 分组 run_inference
|
||||||
|
(每组使用对应视频的 TreeEnvironment,避免跨视频树错用)
|
||||||
|
4. 汇总两组 per_task_type accuracy,对比(Fisher exact test)
|
||||||
|
5. 输出对比表 + 判定结果
|
||||||
|
6. 存在 FAIL → 退出码 1
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5.4 tools/ 脚本职责边界
|
||||||
|
|
||||||
|
脚本**只做**:argparse、adapter 实例化(读 `.env`)、Semaphore 并发、进度日志(loguru)、JSON 写入、calibrate 时调 factory + run_inference。
|
||||||
|
|
||||||
|
脚本**不做**:prompt 构造、节点采样、去重判定(synthesizer.py)、依赖组装逻辑(factory.py)。
|
||||||
|
|
||||||
|
## 6. 校准统计方法
|
||||||
|
|
||||||
|
### 6.1 问题
|
||||||
|
|
||||||
|
benchmark 题型分布极不均匀(Spatial Perception 仅 3 道 vs Object Reasoning 240 道),固定 10% 阈值对小样本题型会产生误判——单题翻转即 33% 波动。
|
||||||
|
|
||||||
|
### 6.2 组合判定:Fisher exact test + effect size
|
||||||
|
|
||||||
|
用 `scipy.stats.fisher_exact` 对每个题型构造 2×2 列联表:
|
||||||
|
|
||||||
|
| | 答对 | 答错 |
|
||||||
|
|--|------|------|
|
||||||
|
| Benchmark | a | b |
|
||||||
|
| Generated | c | d |
|
||||||
|
|
||||||
|
判定规则:
|
||||||
|
|
||||||
|
| \|Δ\| > tolerance | p < α | 判定 | 含义 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| ✗ | — | **PASS** | 差异在容忍范围内 |
|
||||||
|
| ✓ | ✓ | **FAIL** | 差异大且统计显著——生成题难度确实偏了 |
|
||||||
|
| ✓ | ✗ | **WARN** | 差异大但样本不足以确认——可能是噪声 |
|
||||||
|
|
||||||
|
### 6.3 优势
|
||||||
|
|
||||||
|
- 不需要 ad-hoc 的 `min_calibrate_size` 参数
|
||||||
|
- 小样本题型自动降级为 WARN——Fisher test 的 p-value 天然反映样本量不足
|
||||||
|
- CLI 只需两个语义清晰的统计参数:`--tolerance 0.10` + `--alpha 0.05`
|
||||||
|
- 退出码只看是否存在 FAIL(WARN 不阻塞)
|
||||||
|
|
||||||
|
### 6.4 检测灵敏度与 per_type 的关系
|
||||||
|
|
||||||
|
| per_type | 可检出的最小差异(大样本 benchmark 侧) |
|
||||||
|
|----------|---------------------------------------|
|
||||||
|
| 20 | ~30%(仅极大差异) |
|
||||||
|
| 50 | ~15%(中等差异) |
|
||||||
|
|
||||||
|
用户可根据需要的检测灵敏度选择 `--per-type`。
|
||||||
|
|
||||||
|
### 6.5 输出格式
|
||||||
|
|
||||||
|
```
|
||||||
|
题型 | bench | gen | Δ | p-value | 判定
|
||||||
|
-------------------|--------|--------|---------|---------|--------
|
||||||
|
Spatial Perception | 66.7% | 40.0% | -26.7% | 0.590 | ⚠ WARN
|
||||||
|
Action Reasoning | 72.2% | 68.0% | -4.2% | 0.712 | ✓ PASS
|
||||||
|
Object Reasoning | 60.0% | 30.0% | -30.0% | 0.016 | ✗ FAIL
|
||||||
|
```
|
||||||
|
|
||||||
|
## 7. 断点续跑
|
||||||
|
|
||||||
|
生成 240 道题可能中断(VLM 故障、手动 Ctrl-C),沿用项目已有的 progress.json 模式:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"completed": {
|
||||||
|
"Action Reasoning": ["gen-xyz-001", "gen-xyz-002"],
|
||||||
|
"Object Recognition": ["gen-abc-001"]
|
||||||
|
},
|
||||||
|
"output_dir": "store/questions/generated/Video-MME"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- 启动时检查 `{output_dir}/progress.json`,跳过已完成的题
|
||||||
|
- **恢复 embedding 池**:从已写出的 `{output_dir}/*.json` 重建已生成题的 embedding + `used_node_ids`,避免续跑后产生重复题
|
||||||
|
- 每道题写入 JSON 后立即更新 progress
|
||||||
|
- 全部完成后删除 progress.json
|
||||||
|
|
||||||
|
## 8. 输出格式
|
||||||
|
|
||||||
|
输出路径:`store/questions/generated/Video-MME/{video_id}.json`
|
||||||
|
|
||||||
|
```json
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"question_id": "gen-{video_id}-{seq}",
|
||||||
|
"task_type": "Action Reasoning",
|
||||||
|
"question": "...",
|
||||||
|
"options": ["A. ...", "B. ...", "C. ...", "D. ..."],
|
||||||
|
"answer": "B",
|
||||||
|
"source_nodes": ["L1_000_L2_003"],
|
||||||
|
"difficulty": "medium"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
与 loader schema 兼容(额外 `source_nodes`/`difficulty` 字段用于溯源),`load_benchmark` 零改动直接加载。
|
||||||
|
|
||||||
|
**训练集成**:`--questions generated/Video-MME`。
|
||||||
|
|
||||||
|
## 9. 受影响的既有接口
|
||||||
|
|
||||||
|
| 接口 | 影响 | 适配 |
|
||||||
|
|------|------|------|
|
||||||
|
| `load_benchmark` | 无 | 输出与 loader schema 兼容(额外 source_nodes/difficulty 字段用于溯源) |
|
||||||
|
| `RunConfig.questions` | 无 | 传 `generated/Video-MME` |
|
||||||
|
| `build_or_load_pools` | 无 | 三池均来自生成题 |
|
||||||
|
| `Runner._make_tool_dispatch_fn` | 改造 | 委托 factory.py |
|
||||||
|
| `Runner._make_prompt_builder` | 改造 | 委托 factory.py |
|
||||||
|
| `_VIDEO_MME_TASK_TYPE_COUNT` | **前置修复** | 从 11 改为 12(`app/harness/config.py:24`),影响验证池保底下限 |
|
||||||
|
|
||||||
|
## 10. 测试策略
|
||||||
|
|
||||||
|
### 10.1 synthesizer.py
|
||||||
|
|
||||||
|
| 测试 | 覆盖点 |
|
||||||
|
|------|--------|
|
||||||
|
| `test_sample_anchor` | 各层级题型正确采锚、同视频同题型不重复、树节点不足时报错 |
|
||||||
|
| `test_build_generation_prompt` | messages 结构正确、exemplar 注入、图片路径列表、干扰项素材包含 |
|
||||||
|
| `test_parse_vlm_response` | 正常解析、格式异常(缺字段/非法 JSON)报错 |
|
||||||
|
| `test_is_duplicate` | 相似度 ≥ 阈值判重、< 阈值通过、空池不判重 |
|
||||||
|
| `test_generate_one` | mock VLMProvider,验证重试+去重循环、耗尽重试返回 None |
|
||||||
|
|
||||||
|
### 10.2 factory.py
|
||||||
|
|
||||||
|
| 测试 | 覆盖点 |
|
||||||
|
|------|--------|
|
||||||
|
| `test_build_inference_deps` | fake LLM/VLM/Embedding 验证返回各字段非 None、类型正确 |
|
||||||
|
| `test_missing_tree_file` | tree.json 不存在时报错 |
|
||||||
|
|
||||||
|
### 10.3 tools/generate_questions.py(集成级)
|
||||||
|
|
||||||
|
| 测试 | 覆盖点 |
|
||||||
|
|------|--------|
|
||||||
|
| `test_generate_smoke` | mock VLM + 1 棵真实树 + per_type=1,验证 JSON 输出格式 |
|
||||||
|
| `test_progress_resume` | 中断后重启,跳过已完成题 |
|
||||||
|
| `test_calibrate_pass_fail` | mock 两组 accuracy,验证 Fisher + tolerance 组合判定 |
|
||||||
|
|
||||||
|
真实 VLM 调用的 integration test 不在此次范围——依赖外部服务,不适合 CI。
|
||||||
|
|
||||||
|
## 11. 实现约束
|
||||||
|
|
||||||
|
- 完整类型注解 + 中文 Docstring(CLAUDE.md §4.2)
|
||||||
|
- 禁用 `print()`,使用 loguru(CLAUDE.md §4.2)
|
||||||
|
- 脚本放 `tools/`,不被其他模块 import(CLAUDE.md §5)
|
||||||
|
- 并发模式:`asyncio.Semaphore`,CLI `--concurrency` 指定(沿用项目既有模式)
|
||||||
|
- 所有 VLM 调用经过 `GovernedLLMClient` 治理栈(CLAUDE.md §4.9)
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
---
|
||||||
|
type: design
|
||||||
|
node_id: design:paper-main-figure
|
||||||
|
title: "论文主图:Self-Evolving Search Agent 推理训练闭环"
|
||||||
|
date: 2026-07-09
|
||||||
|
---
|
||||||
|
|
||||||
|
# 论文主图:Self-Evolving Search Agent 推理训练闭环
|
||||||
|
|
||||||
|
完整设计文档见 [2026-07-09-paper-main-figure-design.md](2026-07-09-paper-main-figure-design.md)。
|
||||||
|
|
||||||
|
**选定方案**:水平流水线 + 底部参数回流闭环(输入 → Inference → Diagnose → Evolve → CE-Gate → Store 回流),底部 PyTorch 双行对照条逐段对齐;示例贯穿延续建树图的 Video-MME 天文台视频;绘制于同一 Figma 文件以复用素材。
|
||||||
|
|
||||||
|
**关键决策理由**:阅读顺序 = 数据流,与建树图左→右语言一致;对照条只有在单行流水线下才能逐段对齐。CE-Gate 位于 Evolve 之后(与 `core/evolution/gate.py` 核实),信息阶梯为 gate 供题序。
|
||||||
|
|
||||||
|
**被拒绝方案**:上下双层 S 形回路(对照条无法对齐)、中心辐射环形(宽幅空间浪费、细节难展开)、独立进化时间轴(占版面,由版本堆叠暗示替代)。
|
||||||
|
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
---
|
||||||
|
type: design
|
||||||
|
node_id: design:question-gen-synth
|
||||||
|
title: 赛题生成工具设计
|
||||||
|
date: 2026-07-09
|
||||||
|
---
|
||||||
|
|
||||||
|
# 赛题生成工具设计
|
||||||
|
|
||||||
@@ -55,6 +55,21 @@
|
|||||||
"id": "plan:tree-repair-resilience",
|
"id": "plan:tree-repair-resilience",
|
||||||
"label": "建树修复管线三项改造实现计划",
|
"label": "建树修复管线三项改造实现计划",
|
||||||
"type": "plan"
|
"type": "plan"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "design:paper-main-figure",
|
||||||
|
"label": "论文主图:Self-Evolving Search Agent 推理训练闭环",
|
||||||
|
"type": "design"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "design:question-gen-synth",
|
||||||
|
"label": "赛题生成工具设计",
|
||||||
|
"type": "design"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "plan:question-gen-synth",
|
||||||
|
"label": "赛题生成工具实现计划",
|
||||||
|
"type": "plan"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"links": [
|
"links": [
|
||||||
@@ -99,6 +114,13 @@
|
|||||||
"relation": "implements",
|
"relation": "implements",
|
||||||
"evidence": "实现设计文档的三项改造:遥测加固+断点续跑+并发",
|
"evidence": "实现设计文档的三项改造:遥测加固+断点续跑+并发",
|
||||||
"added": "2026-07-09T04:08:15.312470+00:00"
|
"added": "2026-07-09T04:08:15.312470+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source": "plan:question-gen-synth",
|
||||||
|
"target": "design:question-gen-synth",
|
||||||
|
"relation": "implements",
|
||||||
|
"evidence": "计划实现设计文档中定义的 synthesizer + factory + CLI 三模块",
|
||||||
|
"added": "2026-07-09T09:05:43.697644+00:00"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -1,24 +1,29 @@
|
|||||||
# Research Wiki 索引
|
# Research Wiki 索引
|
||||||
|
|
||||||
> 自动生成,更新时间:2026-07-09 04:08 UTC
|
> 自动生成,更新时间:2026-07-09 09:05 UTC
|
||||||
|
|
||||||
## design (9)
|
## design (13)
|
||||||
- [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design`
|
- [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design`
|
||||||
- [2026-07-07-app-harness-design](designs/2026-07-07-app-harness-design.md) `design:2026-07-07-app-harness-design`
|
- [2026-07-07-app-harness-design](designs/2026-07-07-app-harness-design.md) `design:2026-07-07-app-harness-design`
|
||||||
- [2026-07-07-core-evolution-design](designs/2026-07-07-core-evolution-design.md) `design:2026-07-07-core-evolution-design`
|
- [2026-07-07-core-evolution-design](designs/2026-07-07-core-evolution-design.md) `design:2026-07-07-core-evolution-design`
|
||||||
- [2026-07-08-tree-repair-resilience-design](designs/2026-07-08-tree-repair-resilience-design.md) `design:2026-07-08-tree-repair-resilience-design`
|
- [2026-07-08-tree-repair-resilience-design](designs/2026-07-08-tree-repair-resilience-design.md) `design:2026-07-08-tree-repair-resilience-design`
|
||||||
|
- [2026-07-09-paper-main-figure-design](designs/2026-07-09-paper-main-figure-design.md) `design:2026-07-09-paper-main-figure-design`
|
||||||
- [出题模块迁移设计(question_gen)](designs/2026-07-07-question-gen-design.md) `design:2026-07-07-question-gen-design`
|
- [出题模块迁移设计(question_gen)](designs/2026-07-07-question-gen-design.md) `design:2026-07-07-question-gen-design`
|
||||||
- [建树修复管线:熔断根因修复 + 断点续跑 + 并发改造](designs/tree-repair-resilience.md) `design:tree-repair-resilience`
|
- [建树修复管线:熔断根因修复 + 断点续跑 + 并发改造](designs/tree-repair-resilience.md) `design:tree-repair-resilience`
|
||||||
- [建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移](designs/2026-07-07-tree-module-design.md) `design:2026-07-07-tree-module-design`
|
- [建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移](designs/2026-07-07-tree-module-design.md) `design:2026-07-07-tree-module-design`
|
||||||
- [建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移](designs/tree-module-vertical-slice.md) `design:tree-module-vertical-slice`
|
- [建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移](designs/tree-module-vertical-slice.md) `design:tree-module-vertical-slice`
|
||||||
- [搜索 Agent 装配层设计(app/search/)](designs/2026-07-07-search-module-design.md) `design:2026-07-07-search-module-design`
|
- [搜索 Agent 装配层设计(app/search/)](designs/2026-07-07-search-module-design.md) `design:2026-07-07-search-module-design`
|
||||||
|
- [论文主图:Self-Evolving Search Agent 推理训练闭环](designs/paper-main-figure.md) `design:paper-main-figure`
|
||||||
|
- [赛题生成工具设计](designs/question-gen-synth.md) `design:question-gen-synth`
|
||||||
|
- [赛题生成工具设计(Question Generation Synthesis)](designs/2026-07-09-question-gen-synth-design.md) `design:2026-07-09-question-gen-synth-design`
|
||||||
|
|
||||||
## plan (13)
|
## plan (15)
|
||||||
- [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm`
|
- [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm`
|
||||||
- [2026-07-07-app-harness](plans/2026-07-07-app-harness.md) `plan:2026-07-07-app-harness`
|
- [2026-07-07-app-harness](plans/2026-07-07-app-harness.md) `plan:2026-07-07-app-harness`
|
||||||
- [2026-07-07-core-evolution](plans/2026-07-07-core-evolution.md) `plan:2026-07-07-core-evolution`
|
- [2026-07-07-core-evolution](plans/2026-07-07-core-evolution.md) `plan:2026-07-07-core-evolution`
|
||||||
- [2026-07-07-question-gen](plans/2026-07-07-question-gen.md) `plan:2026-07-07-question-gen`
|
- [2026-07-07-question-gen](plans/2026-07-07-question-gen.md) `plan:2026-07-07-question-gen`
|
||||||
- [2026-07-07-tree-module-vertical-slice](plans/2026-07-07-tree-module-vertical-slice.md) `plan:2026-07-07-tree-module-vertical-slice`
|
- [2026-07-07-tree-module-vertical-slice](plans/2026-07-07-tree-module-vertical-slice.md) `plan:2026-07-07-tree-module-vertical-slice`
|
||||||
|
- [2026-07-09-question-gen-synth](plans/2026-07-09-question-gen-synth.md) `plan:2026-07-09-question-gen-synth`
|
||||||
- [2026-07-09-tree-repair-resilience](plans/2026-07-09-tree-repair-resilience.md) `plan:2026-07-09-tree-repair-resilience`
|
- [2026-07-09-tree-repair-resilience](plans/2026-07-09-tree-repair-resilience.md) `plan:2026-07-09-tree-repair-resilience`
|
||||||
- [app/harness/ 训练循环编排层实现计划](plans/app-harness.md) `plan:app-harness`
|
- [app/harness/ 训练循环编排层实现计划](plans/app-harness.md) `plan:app-harness`
|
||||||
- [app/search/ 搜索 Agent 装配层实现计划](plans/2026-07-07-search-module.md) `plan:2026-07-07-search-module`
|
- [app/search/ 搜索 Agent 装配层实现计划](plans/2026-07-07-search-module.md) `plan:2026-07-07-search-module`
|
||||||
@@ -26,4 +31,5 @@
|
|||||||
- [question_gen 模块实现计划](plans/question-gen.md) `plan:question-gen`
|
- [question_gen 模块实现计划](plans/question-gen.md) `plan:question-gen`
|
||||||
- [建树修复管线三项改造实现计划](plans/tree-repair-resilience.md) `plan:tree-repair-resilience`
|
- [建树修复管线三项改造实现计划](plans/tree-repair-resilience.md) `plan:tree-repair-resilience`
|
||||||
- [建树模块竖切实现计划](plans/tree-module-vertical-slice.md) `plan:tree-module-vertical-slice`
|
- [建树模块竖切实现计划](plans/tree-module-vertical-slice.md) `plan:tree-module-vertical-slice`
|
||||||
|
- [赛题生成工具实现计划](plans/question-gen-synth.md) `plan:question-gen-synth`
|
||||||
- [项目基础设施初始化计划](plans/infrastructure-setup.md) `plan:infrastructure-setup`
|
- [项目基础设施初始化计划](plans/infrastructure-setup.md) `plan:infrastructure-setup`
|
||||||
|
|||||||
@@ -27,3 +27,9 @@
|
|||||||
- [2026-07-09 04:08 UTC] 新增 plan: 建树修复管线三项改造实现计划 (plan:tree-repair-resilience)
|
- [2026-07-09 04:08 UTC] 新增 plan: 建树修复管线三项改造实现计划 (plan:tree-repair-resilience)
|
||||||
- [2026-07-09 04:08 UTC] 新增边: plan:tree-repair-resilience --implements--> design:tree-repair-resilience
|
- [2026-07-09 04:08 UTC] 新增边: plan:tree-repair-resilience --implements--> design:tree-repair-resilience
|
||||||
- [2026-07-09 04:08 UTC] 重建索引: 22 篇页面
|
- [2026-07-09 04:08 UTC] 重建索引: 22 篇页面
|
||||||
|
- [2026-07-09 04:38 UTC] 新增 design: 论文主图:Self-Evolving Search Agent 推理训练闭环 (design:paper-main-figure)
|
||||||
|
- [2026-07-09 04:38 UTC] 重建索引: 24 篇页面
|
||||||
|
- [2026-07-09 09:05 UTC] 新增 design: 赛题生成工具设计 (design:question-gen-synth)
|
||||||
|
- [2026-07-09 09:05 UTC] 新增 plan: 赛题生成工具实现计划 (plan:question-gen-synth)
|
||||||
|
- [2026-07-09 09:05 UTC] 新增边: plan:question-gen-synth --implements--> design:question-gen-synth
|
||||||
|
- [2026-07-09 09:05 UTC] 重建索引: 28 篇页面
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,9 @@
|
|||||||
|
---
|
||||||
|
type: plan
|
||||||
|
node_id: plan:question-gen-synth
|
||||||
|
title: 赛题生成工具实现计划
|
||||||
|
date: 2026-07-09
|
||||||
|
---
|
||||||
|
|
||||||
|
# 赛题生成工具实现计划
|
||||||
|
|
||||||
Executable
+29
@@ -0,0 +1,29 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# 树修复管线:检测 + VLM 重生成 + 校验 + Q&A 补全
|
||||||
|
#
|
||||||
|
# 用法:
|
||||||
|
# bash scripts/repair_trees.sh # 默认 16 并发续跑
|
||||||
|
# bash scripts/repair_trees.sh --dry-run # 仅检测不修复
|
||||||
|
# bash scripts/repair_trees.sh --reaggregate-all # 强制全量重聚合
|
||||||
|
# CONCURRENCY=8 bash scripts/repair_trees.sh # 自定义并发数
|
||||||
|
#
|
||||||
|
# 日志输出:
|
||||||
|
# stderr → 终端实时显示
|
||||||
|
# logs/repair_trees.log → 完整日志(自动 rotation 50MB)
|
||||||
|
# logs/repair_telemetry.db → LLM/VLM 调用遥测
|
||||||
|
# logs/repair_progress.json → 断点续跑进度
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
CONCURRENCY="${CONCURRENCY:-16}"
|
||||||
|
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "$(conda info --base)/etc/profile.d/conda.sh"
|
||||||
|
conda activate Video-Tree-TRM
|
||||||
|
|
||||||
|
python tools/repair_trees.py \
|
||||||
|
--videos-dir store/videos \
|
||||||
|
--srt-dir data/Video-MME/subtitle \
|
||||||
|
--questions-dir store/questions/benchmarks/Video-MME \
|
||||||
|
--concurrency "$CONCURRENCY" \
|
||||||
|
"$@"
|
||||||
@@ -0,0 +1,208 @@
|
|||||||
|
"""app/harness/factory.py 的单元测试。
|
||||||
|
|
||||||
|
验证 build_inference_deps 的返回类型、字段连接、以及错误路径。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.harness.factory import InferenceDeps, build_inference_deps
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildInferenceDeps:
|
||||||
|
"""build_inference_deps 工厂函数测试。"""
|
||||||
|
|
||||||
|
def test_returns_inference_deps(self, tmp_path: Path) -> None:
|
||||||
|
"""用 fake adapters 验证返回类型和字段非 None。"""
|
||||||
|
# 准备一棵最小树
|
||||||
|
vid_dir = tmp_path / "videos" / "test_vid"
|
||||||
|
vid_dir.mkdir(parents=True)
|
||||||
|
(vid_dir / "frames").mkdir()
|
||||||
|
minimal_tree = {
|
||||||
|
"metadata": {"source_path": "test", "modality": "video"},
|
||||||
|
"roots": [
|
||||||
|
{
|
||||||
|
"id": "L1_000",
|
||||||
|
"card": {
|
||||||
|
"scene_summary": "s",
|
||||||
|
"main_setting": "s",
|
||||||
|
"key_entities": [],
|
||||||
|
"main_actions": [],
|
||||||
|
"topic_keywords": [],
|
||||||
|
"visible_text": [],
|
||||||
|
"temporal_flow": "s",
|
||||||
|
},
|
||||||
|
"time_range": [0, 10],
|
||||||
|
"children": [],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
(vid_dir / "tree.json").write_text(json.dumps(minimal_tree))
|
||||||
|
|
||||||
|
# prompts
|
||||||
|
prompts_dir = tmp_path / "prompts"
|
||||||
|
prompts_dir.mkdir()
|
||||||
|
(prompts_dir / "system.md").write_text("You are a search agent.")
|
||||||
|
|
||||||
|
fake_llm = AsyncMock()
|
||||||
|
fake_vlm = AsyncMock()
|
||||||
|
fake_embed = MagicMock()
|
||||||
|
fake_embed.dim = 4
|
||||||
|
fake_embed.embed = lambda t: np.zeros((1, 4), dtype=np.float32)
|
||||||
|
|
||||||
|
deps = build_inference_deps(
|
||||||
|
store_dir=tmp_path,
|
||||||
|
video_id="test_vid",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
skills_dir=None,
|
||||||
|
skill_mode="none",
|
||||||
|
embed_provider=fake_embed,
|
||||||
|
llm=fake_llm,
|
||||||
|
vlm=fake_vlm,
|
||||||
|
ocr=None,
|
||||||
|
verify_vision=False,
|
||||||
|
anchor=False,
|
||||||
|
assemble_mode="ids",
|
||||||
|
)
|
||||||
|
assert isinstance(deps, InferenceDeps)
|
||||||
|
assert deps.llm is fake_llm
|
||||||
|
assert callable(deps.tool_dispatch_fn)
|
||||||
|
assert callable(deps.prompt_builder)
|
||||||
|
|
||||||
|
# 验证 prompt_builder 实际可用(连接正确)
|
||||||
|
fake_q = GeneratedQuestion(
|
||||||
|
question_id="q1",
|
||||||
|
video_id="test_vid",
|
||||||
|
task_type="Object Recognition",
|
||||||
|
question="What?",
|
||||||
|
options=("A. X", "B. Y", "C. Z", "D. W"),
|
||||||
|
answer="A",
|
||||||
|
source_nodes=(),
|
||||||
|
difficulty="medium",
|
||||||
|
)
|
||||||
|
system, user = deps.prompt_builder(fake_q)
|
||||||
|
assert isinstance(system, str) and len(system) > 0
|
||||||
|
assert isinstance(user, str) and "What?" in user
|
||||||
|
|
||||||
|
def test_missing_tree_raises(self, tmp_path: Path) -> None:
|
||||||
|
"""tree.json 不存在时应抛出 FileNotFoundError。"""
|
||||||
|
prompts_dir = tmp_path / "prompts"
|
||||||
|
prompts_dir.mkdir()
|
||||||
|
(prompts_dir / "system.md").write_text("x")
|
||||||
|
vid_dir = tmp_path / "videos" / "nonexist"
|
||||||
|
vid_dir.mkdir(parents=True)
|
||||||
|
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
build_inference_deps(
|
||||||
|
store_dir=tmp_path,
|
||||||
|
video_id="nonexist",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
skills_dir=None,
|
||||||
|
skill_mode="none",
|
||||||
|
embed_provider=MagicMock(),
|
||||||
|
llm=AsyncMock(),
|
||||||
|
vlm=AsyncMock(),
|
||||||
|
ocr=None,
|
||||||
|
verify_vision=False,
|
||||||
|
anchor=False,
|
||||||
|
assemble_mode="ids",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_with_skills_dir(self, tmp_path: Path) -> None:
|
||||||
|
"""提供 skills_dir 时 skill 信息应正确注入到 prompt_builder 输出。"""
|
||||||
|
# 准备树
|
||||||
|
vid_dir = tmp_path / "videos" / "vid1"
|
||||||
|
vid_dir.mkdir(parents=True)
|
||||||
|
(vid_dir / "frames").mkdir()
|
||||||
|
minimal_tree = {
|
||||||
|
"metadata": {"source_path": "test", "modality": "video"},
|
||||||
|
"roots": [
|
||||||
|
{
|
||||||
|
"id": "L1_000",
|
||||||
|
"card": {
|
||||||
|
"scene_summary": "test scene",
|
||||||
|
"main_setting": "indoor",
|
||||||
|
"key_entities": [],
|
||||||
|
"main_actions": [],
|
||||||
|
"topic_keywords": [],
|
||||||
|
"visible_text": [],
|
||||||
|
"temporal_flow": "linear",
|
||||||
|
},
|
||||||
|
"time_range": [0, 5],
|
||||||
|
"children": [],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
(vid_dir / "tree.json").write_text(json.dumps(minimal_tree))
|
||||||
|
|
||||||
|
# prompts
|
||||||
|
prompts_dir = tmp_path / "prompts"
|
||||||
|
prompts_dir.mkdir()
|
||||||
|
(prompts_dir / "system.md").write_text("Base system prompt.")
|
||||||
|
|
||||||
|
# skills
|
||||||
|
skills_dir = tmp_path / "skills"
|
||||||
|
skills_dir.mkdir()
|
||||||
|
(skills_dir / "always_nav.md").write_text(
|
||||||
|
"---\nname: always_nav\nalways: true\n---\nAlways navigate broadly."
|
||||||
|
)
|
||||||
|
(skills_dir / "action_skill.md").write_text(
|
||||||
|
"---\nname: action_skill\ntask_type: Action Reasoning\n---\nFocus on actions."
|
||||||
|
)
|
||||||
|
|
||||||
|
fake_llm = AsyncMock()
|
||||||
|
fake_vlm = AsyncMock()
|
||||||
|
fake_embed = MagicMock()
|
||||||
|
fake_embed.dim = 4
|
||||||
|
fake_embed.embed = lambda t: np.zeros((1, 4), dtype=np.float32)
|
||||||
|
|
||||||
|
deps = build_inference_deps(
|
||||||
|
store_dir=tmp_path,
|
||||||
|
video_id="vid1",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
skills_dir=skills_dir,
|
||||||
|
skill_mode="auto",
|
||||||
|
embed_provider=fake_embed,
|
||||||
|
llm=fake_llm,
|
||||||
|
vlm=fake_vlm,
|
||||||
|
ocr=None,
|
||||||
|
verify_vision=False,
|
||||||
|
anchor=False,
|
||||||
|
assemble_mode="ids",
|
||||||
|
)
|
||||||
|
|
||||||
|
fake_q = GeneratedQuestion(
|
||||||
|
question_id="q2",
|
||||||
|
video_id="vid1",
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
question="What happened?",
|
||||||
|
options=("A. X", "B. Y", "C. Z", "D. W"),
|
||||||
|
answer="B",
|
||||||
|
source_nodes=(),
|
||||||
|
difficulty="easy",
|
||||||
|
)
|
||||||
|
system, user = deps.prompt_builder(fake_q)
|
||||||
|
# always skill 文本和 task_type skill 文本应出现在 system prompt 中
|
||||||
|
assert "Always navigate broadly" in system
|
||||||
|
assert "Focus on actions" in system
|
||||||
|
assert "What happened?" in user
|
||||||
|
|
||||||
|
def test_frozen_dataclass(self) -> None:
|
||||||
|
"""InferenceDeps 是 frozen dataclass,不可修改属性。"""
|
||||||
|
deps = InferenceDeps(
|
||||||
|
llm=AsyncMock(),
|
||||||
|
tool_dispatch_fn=lambda: None,
|
||||||
|
prompt_builder=lambda q: ("", ""),
|
||||||
|
)
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
deps.llm = AsyncMock() # type: ignore[misc]
|
||||||
@@ -0,0 +1,417 @@
|
|||||||
|
"""tools/generate_questions.py 单元测试。
|
||||||
|
|
||||||
|
覆盖断点续跑、exemplar 选取、embedding 池重建、JSON 追加写入等纯函数。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import random
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
# 确保项目根目录在 sys.path 中
|
||||||
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||||
|
if str(PROJECT_ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(PROJECT_ROOT))
|
||||||
|
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
from tools.generate_questions import (
|
||||||
|
_append_to_json,
|
||||||
|
_calibrate_exit_code,
|
||||||
|
_judge_task_type,
|
||||||
|
_load_or_init_progress,
|
||||||
|
_rebuild_embedding_pool,
|
||||||
|
_save_progress,
|
||||||
|
_select_exemplars,
|
||||||
|
_validate_calibrate_args,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 辅助工厂
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _make_question(
|
||||||
|
qid: str = "q1",
|
||||||
|
vid: str = "v1",
|
||||||
|
task_type: str = "Object Recognition",
|
||||||
|
question: str = "What is this?",
|
||||||
|
answer: str = "A",
|
||||||
|
) -> GeneratedQuestion:
|
||||||
|
"""构造测试用 GeneratedQuestion。"""
|
||||||
|
return GeneratedQuestion(
|
||||||
|
question_id=qid,
|
||||||
|
video_id=vid,
|
||||||
|
task_type=task_type,
|
||||||
|
question=question,
|
||||||
|
options=("A. X", "B. Y", "C. Z", "D. W"),
|
||||||
|
answer=answer,
|
||||||
|
source_nodes=(),
|
||||||
|
difficulty="medium",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestLoadOrInitProgress
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoadOrInitProgress:
|
||||||
|
"""_load_or_init_progress 测试。"""
|
||||||
|
|
||||||
|
def test_init_fresh(self, tmp_path: Path) -> None:
|
||||||
|
"""目录为空时返回初始结构。"""
|
||||||
|
progress = _load_or_init_progress(tmp_path)
|
||||||
|
assert progress["completed"] == {}
|
||||||
|
assert progress["output_dir"] == str(tmp_path)
|
||||||
|
|
||||||
|
def test_load_existing(self, tmp_path: Path) -> None:
|
||||||
|
"""已有 progress.json 时正确加载。"""
|
||||||
|
data = {
|
||||||
|
"completed": {"Object Recognition": ["gen-x-001"]},
|
||||||
|
"output_dir": str(tmp_path),
|
||||||
|
}
|
||||||
|
(tmp_path / "progress.json").write_text(json.dumps(data))
|
||||||
|
progress = _load_or_init_progress(tmp_path)
|
||||||
|
assert "gen-x-001" in progress["completed"]["Object Recognition"]
|
||||||
|
|
||||||
|
def test_corrupted_json_reinits(self, tmp_path: Path) -> None:
|
||||||
|
"""损坏的 JSON 文件导致重新初始化。"""
|
||||||
|
(tmp_path / "progress.json").write_text("{invalid json")
|
||||||
|
progress = _load_or_init_progress(tmp_path)
|
||||||
|
assert progress["completed"] == {}
|
||||||
|
|
||||||
|
def test_invalid_completed_type_reinits(self, tmp_path: Path) -> None:
|
||||||
|
"""completed 字段类型不正确时重新初始化。"""
|
||||||
|
(tmp_path / "progress.json").write_text(
|
||||||
|
json.dumps({"completed": "not-a-dict", "output_dir": str(tmp_path)})
|
||||||
|
)
|
||||||
|
progress = _load_or_init_progress(tmp_path)
|
||||||
|
assert progress["completed"] == {}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestSaveProgress
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestSaveProgress:
|
||||||
|
"""_save_progress 测试。"""
|
||||||
|
|
||||||
|
def test_atomic_write(self, tmp_path: Path) -> None:
|
||||||
|
"""原子写入 progress.json。"""
|
||||||
|
progress = {
|
||||||
|
"completed": {"Action Reasoning": ["gen-v1-001"]},
|
||||||
|
"output_dir": str(tmp_path),
|
||||||
|
}
|
||||||
|
_save_progress(tmp_path, progress)
|
||||||
|
|
||||||
|
written = json.loads((tmp_path / "progress.json").read_text())
|
||||||
|
assert written["completed"]["Action Reasoning"] == ["gen-v1-001"]
|
||||||
|
# 临时文件不应残留
|
||||||
|
assert not (tmp_path / "progress.json.tmp").exists()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestSelectExemplars
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestSelectExemplars:
|
||||||
|
"""_select_exemplars 测试。"""
|
||||||
|
|
||||||
|
def test_selects_correct_type(self) -> None:
|
||||||
|
"""只选取匹配题型的示例。"""
|
||||||
|
qs = [
|
||||||
|
_make_question("q1", "v1", "Object Recognition", "Q1?"),
|
||||||
|
_make_question("q2", "v2", "Object Recognition", "Q2?"),
|
||||||
|
_make_question("q3", "v1", "Action Reasoning", "Q3?"),
|
||||||
|
]
|
||||||
|
result = _select_exemplars(qs, "Object Recognition", 3, random.Random(42))
|
||||||
|
assert all(q.task_type == "Object Recognition" for q in result)
|
||||||
|
assert len(result) == 2 # 只有 2 个可用
|
||||||
|
|
||||||
|
def test_cross_video_diversity(self) -> None:
|
||||||
|
"""优先从不同 video_id 选取示例。"""
|
||||||
|
qs = [_make_question(f"q{i}", f"v{i}", "Object Recognition", f"Q{i}?") for i in range(10)]
|
||||||
|
result = _select_exemplars(qs, "Object Recognition", 3, random.Random(42))
|
||||||
|
video_ids = {q.video_id for q in result}
|
||||||
|
assert len(video_ids) == 3 # 全部来自不同视频
|
||||||
|
|
||||||
|
def test_empty_benchmark(self) -> None:
|
||||||
|
"""benchmark 为空时返回空列表。"""
|
||||||
|
result = _select_exemplars([], "Object Recognition", 3, random.Random(42))
|
||||||
|
assert result == []
|
||||||
|
|
||||||
|
def test_no_matching_type(self) -> None:
|
||||||
|
"""无匹配题型时返回空列表。"""
|
||||||
|
qs = [_make_question("q1", "v1", "Action Reasoning", "Q1?")]
|
||||||
|
result = _select_exemplars(qs, "Object Recognition", 3, random.Random(42))
|
||||||
|
assert result == []
|
||||||
|
|
||||||
|
def test_request_more_than_available(self) -> None:
|
||||||
|
"""请求数超过可用数时返回全部。"""
|
||||||
|
qs = [
|
||||||
|
_make_question("q1", "v1", "Object Recognition", "Q1?"),
|
||||||
|
_make_question("q2", "v2", "Object Recognition", "Q2?"),
|
||||||
|
]
|
||||||
|
result = _select_exemplars(qs, "Object Recognition", 10, random.Random(42))
|
||||||
|
assert len(result) == 2
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestProgressResume
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestProgressResume:
|
||||||
|
"""断点续跑集成测试。"""
|
||||||
|
|
||||||
|
def test_skips_completed_and_rebuilds_pool(self, tmp_path: Path) -> None:
|
||||||
|
"""已完成的题目从 progress 加载,embedding 池从已生成 JSON 重建。"""
|
||||||
|
output_dir = tmp_path / "output"
|
||||||
|
output_dir.mkdir()
|
||||||
|
(output_dir / "test_vid.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"question_id": "gen-test_vid-001",
|
||||||
|
"task_type": "Object Recognition",
|
||||||
|
"question": "Existing question?",
|
||||||
|
"options": ["A. X", "B. Y", "C. Z", "D. W"],
|
||||||
|
"answer": "A",
|
||||||
|
"source_nodes": ["L3_001"],
|
||||||
|
"difficulty": "medium",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
progress = {
|
||||||
|
"completed": {"Object Recognition": ["gen-test_vid-001"]},
|
||||||
|
"output_dir": str(output_dir),
|
||||||
|
}
|
||||||
|
(output_dir / "progress.json").write_text(json.dumps(progress))
|
||||||
|
|
||||||
|
loaded = _load_or_init_progress(output_dir)
|
||||||
|
assert "gen-test_vid-001" in loaded["completed"]["Object Recognition"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestRebuildEmbeddingPool
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestRebuildEmbeddingPool:
|
||||||
|
"""_rebuild_embedding_pool 测试。"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _fake_embed_fn(texts):
|
||||||
|
"""伪嵌入函数:返回固定维度的随机向量。"""
|
||||||
|
if isinstance(texts, str):
|
||||||
|
texts = [texts]
|
||||||
|
return np.random.RandomState(0).randn(len(texts), 8).astype(np.float32)
|
||||||
|
|
||||||
|
def test_empty_dir(self, tmp_path: Path) -> None:
|
||||||
|
"""空目录时所有池为空。"""
|
||||||
|
pools = _rebuild_embedding_pool(tmp_path, self._fake_embed_fn, [])
|
||||||
|
# 所有 12 种题型都应有条目
|
||||||
|
assert len(pools) >= 12
|
||||||
|
for v in pools.values():
|
||||||
|
assert v.shape[0] == 0 or v.ndim == 2
|
||||||
|
|
||||||
|
def test_with_generated_json(self, tmp_path: Path) -> None:
|
||||||
|
"""从已生成的 JSON 文件重建池。"""
|
||||||
|
(tmp_path / "vid1.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"question_id": "gen-vid1-001",
|
||||||
|
"task_type": "Object Recognition",
|
||||||
|
"question": "Test question 1?",
|
||||||
|
"options": ["A. X", "B. Y", "C. Z", "D. W"],
|
||||||
|
"answer": "A",
|
||||||
|
"source_nodes": [],
|
||||||
|
"difficulty": "medium",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question_id": "gen-vid1-002",
|
||||||
|
"task_type": "Object Recognition",
|
||||||
|
"question": "Test question 2?",
|
||||||
|
"options": ["A. X", "B. Y", "C. Z", "D. W"],
|
||||||
|
"answer": "B",
|
||||||
|
"source_nodes": [],
|
||||||
|
"difficulty": "medium",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
pools = _rebuild_embedding_pool(tmp_path, self._fake_embed_fn, [])
|
||||||
|
assert pools["Object Recognition"].shape[0] == 2
|
||||||
|
assert pools["Object Recognition"].shape[1] == 8
|
||||||
|
|
||||||
|
def test_with_benchmark_questions(self, tmp_path: Path) -> None:
|
||||||
|
"""benchmark 题目也加入去重池。"""
|
||||||
|
benchmark = [
|
||||||
|
_make_question("bm1", "v1", "Action Reasoning", "Benchmark Q1?"),
|
||||||
|
_make_question("bm2", "v2", "Action Reasoning", "Benchmark Q2?"),
|
||||||
|
]
|
||||||
|
pools = _rebuild_embedding_pool(tmp_path, self._fake_embed_fn, benchmark)
|
||||||
|
assert pools["Action Reasoning"].shape[0] == 2
|
||||||
|
|
||||||
|
def test_progress_json_excluded(self, tmp_path: Path) -> None:
|
||||||
|
"""progress.json 不被当作题目文件。"""
|
||||||
|
(tmp_path / "progress.json").write_text(
|
||||||
|
json.dumps({"completed": {}, "output_dir": str(tmp_path)})
|
||||||
|
)
|
||||||
|
pools = _rebuild_embedding_pool(tmp_path, self._fake_embed_fn, [])
|
||||||
|
for v in pools.values():
|
||||||
|
assert v.shape[0] == 0 or v.ndim == 2
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestAppendToJson
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestAppendToJson:
|
||||||
|
"""_append_to_json 测试。"""
|
||||||
|
|
||||||
|
def test_create_new_file(self, tmp_path: Path) -> None:
|
||||||
|
"""文件不存在时创建新文件。"""
|
||||||
|
q = _make_question("gen-v1-001", "v1", "Object Recognition", "Q?")
|
||||||
|
_append_to_json(tmp_path, q)
|
||||||
|
|
||||||
|
written = json.loads((tmp_path / "v1.json").read_text())
|
||||||
|
assert len(written) == 1
|
||||||
|
assert written[0]["question_id"] == "gen-v1-001"
|
||||||
|
|
||||||
|
def test_append_to_existing(self, tmp_path: Path) -> None:
|
||||||
|
"""追加到已有文件。"""
|
||||||
|
existing = [
|
||||||
|
{
|
||||||
|
"question_id": "gen-v1-001",
|
||||||
|
"task_type": "Object Recognition",
|
||||||
|
"question": "Existing?",
|
||||||
|
"options": ["A. X", "B. Y", "C. Z", "D. W"],
|
||||||
|
"answer": "A",
|
||||||
|
"source_nodes": [],
|
||||||
|
"difficulty": "medium",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
(tmp_path / "v1.json").write_text(json.dumps(existing))
|
||||||
|
|
||||||
|
q = _make_question("gen-v1-002", "v1", "Object Recognition", "New?")
|
||||||
|
_append_to_json(tmp_path, q)
|
||||||
|
|
||||||
|
written = json.loads((tmp_path / "v1.json").read_text())
|
||||||
|
assert len(written) == 2
|
||||||
|
assert written[1]["question_id"] == "gen-v1-002"
|
||||||
|
|
||||||
|
def test_different_video_ids(self, tmp_path: Path) -> None:
|
||||||
|
"""不同 video_id 写入不同文件。"""
|
||||||
|
q1 = _make_question("gen-v1-001", "v1", "Object Recognition", "Q1?")
|
||||||
|
q2 = _make_question("gen-v2-001", "v2", "Action Reasoning", "Q2?")
|
||||||
|
_append_to_json(tmp_path, q1)
|
||||||
|
_append_to_json(tmp_path, q2)
|
||||||
|
|
||||||
|
assert (tmp_path / "v1.json").exists()
|
||||||
|
assert (tmp_path / "v2.json").exists()
|
||||||
|
v1_data = json.loads((tmp_path / "v1.json").read_text())
|
||||||
|
v2_data = json.loads((tmp_path / "v2.json").read_text())
|
||||||
|
assert len(v1_data) == 1
|
||||||
|
assert len(v2_data) == 1
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestCalibrateJudgment
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestCalibrateJudgment:
|
||||||
|
"""_judge_task_type 校准判定测试。"""
|
||||||
|
|
||||||
|
def test_pass_when_delta_small(self) -> None:
|
||||||
|
"""差值在容忍范围内判定为 PASS。"""
|
||||||
|
verdict = _judge_task_type(
|
||||||
|
bench_correct=60,
|
||||||
|
bench_total=100,
|
||||||
|
gen_correct=12,
|
||||||
|
gen_total=20,
|
||||||
|
tolerance=0.10,
|
||||||
|
alpha=0.05,
|
||||||
|
)
|
||||||
|
assert verdict == "PASS"
|
||||||
|
|
||||||
|
def test_fail_when_delta_large_and_significant(self) -> None:
|
||||||
|
"""差值超阈值且统计显著判定为 FAIL。"""
|
||||||
|
verdict = _judge_task_type(
|
||||||
|
bench_correct=144,
|
||||||
|
bench_total=240,
|
||||||
|
gen_correct=6,
|
||||||
|
gen_total=20,
|
||||||
|
tolerance=0.10,
|
||||||
|
alpha=0.05,
|
||||||
|
)
|
||||||
|
assert verdict == "FAIL"
|
||||||
|
|
||||||
|
def test_warn_when_delta_large_but_not_significant(self) -> None:
|
||||||
|
"""差值超阈值但不统计显著判定为 WARN。"""
|
||||||
|
verdict = _judge_task_type(
|
||||||
|
bench_correct=2,
|
||||||
|
bench_total=3,
|
||||||
|
gen_correct=8,
|
||||||
|
gen_total=20,
|
||||||
|
tolerance=0.10,
|
||||||
|
alpha=0.05,
|
||||||
|
)
|
||||||
|
assert verdict == "WARN"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestCalibrateIntegration
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestCalibrateIntegration:
|
||||||
|
"""calibrate 辅助函数集成测试。"""
|
||||||
|
|
||||||
|
def test_baseline_params_must_be_paired(self) -> None:
|
||||||
|
"""baseline 参数必须成对出现。"""
|
||||||
|
with pytest.raises(ValueError, match="成对"):
|
||||||
|
_validate_calibrate_args(baseline_db="some.db", baseline_run_id=None)
|
||||||
|
|
||||||
|
def test_baseline_params_both_none_ok(self) -> None:
|
||||||
|
"""两个参数都为 None 不报错。"""
|
||||||
|
_validate_calibrate_args(baseline_db=None, baseline_run_id=None)
|
||||||
|
|
||||||
|
def test_baseline_params_both_provided_ok(self) -> None:
|
||||||
|
"""两个参数都提供不报错。"""
|
||||||
|
_validate_calibrate_args(baseline_db="some.db", baseline_run_id="run-001")
|
||||||
|
|
||||||
|
def test_baseline_run_id_only_raises(self) -> None:
|
||||||
|
"""只提供 run_id 也报错。"""
|
||||||
|
with pytest.raises(ValueError, match="成对"):
|
||||||
|
_validate_calibrate_args(baseline_db=None, baseline_run_id="run-001")
|
||||||
|
|
||||||
|
def test_has_fail_returns_exit_code_1(self) -> None:
|
||||||
|
"""存在 FAIL 时返回退出码 1。"""
|
||||||
|
verdicts = {"Object Recognition": "PASS", "Action Reasoning": "FAIL"}
|
||||||
|
assert _calibrate_exit_code(verdicts) == 1
|
||||||
|
|
||||||
|
def test_all_pass_or_warn_returns_exit_code_0(self) -> None:
|
||||||
|
"""全部 PASS 或 WARN 时返回退出码 0。"""
|
||||||
|
verdicts = {"Object Recognition": "PASS", "Spatial Perception": "WARN"}
|
||||||
|
assert _calibrate_exit_code(verdicts) == 0
|
||||||
|
|
||||||
|
def test_all_pass_returns_exit_code_0(self) -> None:
|
||||||
|
"""全部 PASS 时返回退出码 0。"""
|
||||||
|
verdicts = {"Object Recognition": "PASS", "Action Reasoning": "PASS"}
|
||||||
|
assert _calibrate_exit_code(verdicts) == 0
|
||||||
|
|
||||||
|
def test_empty_verdicts_returns_exit_code_0(self) -> None:
|
||||||
|
"""空 verdicts 时返回退出码 0。"""
|
||||||
|
assert _calibrate_exit_code({}) == 0
|
||||||
@@ -568,28 +568,35 @@ class TestValSizeFloor:
|
|||||||
"""val_size >= eval_min_per_class * 11 的下限校验。"""
|
"""val_size >= eval_min_per_class * 11 的下限校验。"""
|
||||||
|
|
||||||
def test_val_size_below_floor_rejected(self) -> None:
|
def test_val_size_below_floor_rejected(self) -> None:
|
||||||
"""val_size < eval_min_per_class * 11 应抛出 ValueError。
|
"""val_size < eval_min_per_class * 12 应抛出 ValueError。
|
||||||
|
|
||||||
eval_min_per_class=3 → 下限 = 3 * 11 = 33,val_size=30 不足。
|
eval_min_per_class=3 → 下限 = 3 * 12 = 36,val_size=30 不足。
|
||||||
"""
|
"""
|
||||||
cfg = _make_config(eval_min_per_class=3, val_size=30)
|
cfg = _make_config(eval_min_per_class=3, val_size=30)
|
||||||
with pytest.raises(ValueError, match="val_size"):
|
with pytest.raises(ValueError, match="val_size"):
|
||||||
_validate(cfg)
|
_validate(cfg)
|
||||||
|
|
||||||
def test_val_size_at_floor_accepted(self) -> None:
|
def test_val_size_at_floor_accepted(self) -> None:
|
||||||
"""val_size == eval_min_per_class * 11 应通过。"""
|
"""val_size == eval_min_per_class * 12 应通过。"""
|
||||||
cfg = _make_config(eval_min_per_class=3, val_size=33)
|
cfg = _make_config(eval_min_per_class=3, val_size=36)
|
||||||
_validate(cfg)
|
_validate(cfg)
|
||||||
|
|
||||||
def test_val_size_above_floor_accepted(self) -> None:
|
def test_val_size_above_floor_accepted(self) -> None:
|
||||||
"""val_size > eval_min_per_class * 11 应通过。"""
|
"""val_size > eval_min_per_class * 12 应通过。"""
|
||||||
cfg = _make_config(eval_min_per_class=2, val_size=100)
|
cfg = _make_config(eval_min_per_class=2, val_size=100)
|
||||||
_validate(cfg)
|
_validate(cfg)
|
||||||
|
|
||||||
def test_default_yaml_values_satisfy_floor(self) -> None:
|
def test_default_yaml_values_satisfy_floor(self) -> None:
|
||||||
"""default.yaml 的默认值(val_size=30, eval_min_per_class=2)应满足下限。
|
"""default.yaml 的默认值(val_size=30, eval_min_per_class=2)应满足下限。
|
||||||
|
|
||||||
下限 = 2 * 11 = 22,val_size=30 >= 22,通过。
|
下限 = 2 * 12 = 24,val_size=30 >= 24,通过。
|
||||||
"""
|
"""
|
||||||
cfg = _make_config()
|
cfg = _make_config()
|
||||||
_validate(cfg) # 不应抛出异常
|
_validate(cfg) # 不应抛出异常
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_mme_task_type_count_is_12():
|
||||||
|
"""Video-MME 实际有 12 种题型,常量必须与之一致。"""
|
||||||
|
from app.harness.config import _VIDEO_MME_TASK_TYPE_COUNT
|
||||||
|
|
||||||
|
assert _VIDEO_MME_TASK_TYPE_COUNT == 12
|
||||||
|
|||||||
@@ -119,25 +119,25 @@ class TestDetectIssues:
|
|||||||
issues = detect_issues(index)
|
issues = detect_issues(index)
|
||||||
assert any(i.issue_type == "no_children" and i.level == 1 for i in issues)
|
assert any(i.issue_type == "no_children" and i.level == 1 for i in issues)
|
||||||
|
|
||||||
def test_empty_visible_entities(self) -> None:
|
def test_empty_visible_entities_not_flagged(self) -> None:
|
||||||
"""visible_entities 为空也触发 empty_field。"""
|
"""visible_entities 为空是合法状态(静物/黑帧),不触发 empty_field。"""
|
||||||
card = L3Card("正常描述", [], ["动作"], [], "居中", {})
|
card = L3Card("正常描述", [], ["动作"], [], "居中", {})
|
||||||
l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0)
|
l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0)
|
||||||
l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3])
|
l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3])
|
||||||
l1 = L1Node(id="l1_0", card=_card_l1(), time_range=(0.0, 10.0), children=[l2])
|
l1 = L1Node(id="l1_0", card=_card_l1(), time_range=(0.0, 10.0), children=[l2])
|
||||||
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
issues = detect_issues(index)
|
issues = detect_issues(index)
|
||||||
assert any(i.issue_type == "empty_field" and "visible_entities" in i.details for i in issues)
|
assert not any(i.issue_type == "empty_field" for i in issues)
|
||||||
|
|
||||||
def test_empty_ongoing_actions(self) -> None:
|
def test_empty_ongoing_actions_not_flagged(self) -> None:
|
||||||
"""ongoing_actions 为空也触发 empty_field。"""
|
"""ongoing_actions 为空是合法状态(静物/黑帧),不触发 empty_field。"""
|
||||||
card = L3Card("正常描述", ["实体"], [], [], "居中", {})
|
card = L3Card("正常描述", ["实体"], [], [], "居中", {})
|
||||||
l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0)
|
l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0)
|
||||||
l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3])
|
l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3])
|
||||||
l1 = L1Node(id="l1_0", card=_card_l1(), time_range=(0.0, 10.0), children=[l2])
|
l1 = L1Node(id="l1_0", card=_card_l1(), time_range=(0.0, 10.0), children=[l2])
|
||||||
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
issues = detect_issues(index)
|
issues = detect_issues(index)
|
||||||
assert any(i.issue_type == "empty_field" and "ongoing_actions" in i.details for i in issues)
|
assert not any(i.issue_type == "empty_field" for i in issues)
|
||||||
|
|
||||||
def test_empty_spatial_layout(self) -> None:
|
def test_empty_spatial_layout(self) -> None:
|
||||||
"""spatial_layout 为空也触发 empty_field。"""
|
"""spatial_layout 为空也触发 empty_field。"""
|
||||||
@@ -150,7 +150,7 @@ class TestDetectIssues:
|
|||||||
assert any(i.issue_type == "empty_field" and "spatial_layout" in i.details for i in issues)
|
assert any(i.issue_type == "empty_field" and "spatial_layout" in i.details for i in issues)
|
||||||
|
|
||||||
def test_multiple_empty_fields_single_issue(self) -> None:
|
def test_multiple_empty_fields_single_issue(self) -> None:
|
||||||
"""多个字段同时为空只产生一个 issue,details 列出所有空字段。"""
|
"""多个必填字段同时为空只产生一个 issue,details 列出所有空字段。"""
|
||||||
card = L3Card("", [], [], [], "", {})
|
card = L3Card("", [], [], [], "", {})
|
||||||
l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0)
|
l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0)
|
||||||
l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3])
|
l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3])
|
||||||
@@ -159,7 +159,7 @@ class TestDetectIssues:
|
|||||||
issues = [i for i in detect_issues(index) if i.issue_type == "empty_field"]
|
issues = [i for i in detect_issues(index) if i.issue_type == "empty_field"]
|
||||||
assert len(issues) == 1
|
assert len(issues) == 1
|
||||||
assert "frame_summary" in issues[0].details
|
assert "frame_summary" in issues[0].details
|
||||||
assert "visible_entities" in issues[0].details
|
assert "spatial_layout" in issues[0].details
|
||||||
|
|
||||||
def test_time_gap(self) -> None:
|
def test_time_gap(self) -> None:
|
||||||
l3_a = L3Node(id="l1_0_l2_0_l3_0", card=_card_l3(), timestamp=1.0)
|
l3_a = L3Node(id="l1_0_l2_0_l3_0", card=_card_l3(), timestamp=1.0)
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
"""修复管线断点续跑 progress 管理测试。"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_progress_missing_file(tmp_path):
|
||||||
|
"""progress 文件不存在时返回空集合。"""
|
||||||
|
from tools.repair_trees import load_progress
|
||||||
|
result = load_progress(tmp_path / "nonexistent.json")
|
||||||
|
assert result == set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_progress_valid_file(tmp_path):
|
||||||
|
"""正常读取已有 progress 文件。"""
|
||||||
|
from tools.repair_trees import load_progress
|
||||||
|
path = tmp_path / "progress.json"
|
||||||
|
path.write_text(json.dumps({"finished_video_ids": ["vid_a", "vid_b"]}))
|
||||||
|
result = load_progress(path)
|
||||||
|
assert result == {"vid_a", "vid_b"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_progress_corrupted_file(tmp_path):
|
||||||
|
"""损坏的 JSON 文件返回空集合(不抛异常)。"""
|
||||||
|
from tools.repair_trees import load_progress
|
||||||
|
path = tmp_path / "progress.json"
|
||||||
|
path.write_text("{invalid json")
|
||||||
|
result = load_progress(path)
|
||||||
|
assert result == set()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_save_progress_atomic(tmp_path):
|
||||||
|
"""save_progress 原子写入,并发调用不丢失更新。"""
|
||||||
|
from tools.repair_trees import save_progress
|
||||||
|
path = tmp_path / "progress.json"
|
||||||
|
lock = asyncio.Lock()
|
||||||
|
await save_progress(path, lock, "vid_a")
|
||||||
|
await save_progress(path, lock, "vid_b")
|
||||||
|
data = json.loads(path.read_text())
|
||||||
|
assert set(data["finished_video_ids"]) == {"vid_a", "vid_b"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_save_progress_concurrent(tmp_path):
|
||||||
|
"""16 路并发 save_progress 不丢失更新。"""
|
||||||
|
from tools.repair_trees import save_progress
|
||||||
|
path = tmp_path / "progress.json"
|
||||||
|
lock = asyncio.Lock()
|
||||||
|
tasks = [save_progress(path, lock, f"vid_{i}") for i in range(16)]
|
||||||
|
await asyncio.gather(*tasks)
|
||||||
|
data = json.loads(path.read_text())
|
||||||
|
assert len(data["finished_video_ids"]) == 16
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_skip_finished():
|
||||||
|
"""已在 finished 集合中的视频应跳过。"""
|
||||||
|
from tools.repair_trees import should_skip_video
|
||||||
|
finished = {"vid_a", "vid_b"}
|
||||||
|
assert should_skip_video("vid_a", finished, reaggregate_all=False) is True
|
||||||
|
assert should_skip_video("vid_c", finished, reaggregate_all=False) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_skip_reaggregate_all_forces_rerun():
|
||||||
|
"""--reaggregate-all 标志强制不跳过。"""
|
||||||
|
from tools.repair_trees import should_skip_video
|
||||||
|
finished = {"vid_a"}
|
||||||
|
assert should_skip_video("vid_a", finished, reaggregate_all=True) is False
|
||||||
@@ -0,0 +1,293 @@
|
|||||||
|
"""修复重生成器单元测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.tree.index import (
|
||||||
|
IndexMeta,
|
||||||
|
L1Card,
|
||||||
|
L1Node,
|
||||||
|
L2Card,
|
||||||
|
L2Node,
|
||||||
|
L3Card,
|
||||||
|
L3Node,
|
||||||
|
TreeIndex,
|
||||||
|
)
|
||||||
|
from app.tree.repair.detector import NodeIssue
|
||||||
|
from app.tree.repair.regenerator import RepairStats, repair_tree
|
||||||
|
from core.types import LLMResponse
|
||||||
|
|
||||||
|
|
||||||
|
def _mock_response(content: str) -> LLMResponse:
|
||||||
|
"""构造模拟 LLMResponse。"""
|
||||||
|
return LLMResponse(
|
||||||
|
content=content,
|
||||||
|
thinking="",
|
||||||
|
model="mock",
|
||||||
|
provider="mock",
|
||||||
|
prompt_tokens=0,
|
||||||
|
completion_tokens=0,
|
||||||
|
latency_ms=0,
|
||||||
|
ttft_ms=None,
|
||||||
|
max_inter_token_ms=None,
|
||||||
|
cache_hit=False,
|
||||||
|
call_id="mock",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class MockVLM:
|
||||||
|
"""模拟 VLM 端口,返回固定的 L3Card JSON。"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.call_count = 0
|
||||||
|
|
||||||
|
async def chat_with_images(
|
||||||
|
self,
|
||||||
|
messages: list[dict],
|
||||||
|
images: list,
|
||||||
|
**kw: object,
|
||||||
|
) -> LLMResponse:
|
||||||
|
"""模拟 VLM 图文调用。"""
|
||||||
|
self.call_count += 1
|
||||||
|
return _mock_response(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"frame_summary": "修复后的帧描述",
|
||||||
|
"visible_entities": ["修复实体"],
|
||||||
|
"ongoing_actions": ["修复动作"],
|
||||||
|
"visible_text": [],
|
||||||
|
"spatial_layout": "居中",
|
||||||
|
"visual_attributes": {"lighting": "明亮"},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class MockLLM:
|
||||||
|
"""模拟 LLM 端口,根据 prompt 内容返回 L2Card 或 L1Card JSON。"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.call_count = 0
|
||||||
|
|
||||||
|
async def chat(
|
||||||
|
self,
|
||||||
|
messages: list[dict],
|
||||||
|
**kw: object,
|
||||||
|
) -> LLMResponse:
|
||||||
|
"""模拟 LLM 文本调用,按 prompt 内容区分 L2/L1 响应。"""
|
||||||
|
self.call_count += 1
|
||||||
|
content = messages[-1].get("content", "")
|
||||||
|
if "段落" in content or "scene" in content.lower():
|
||||||
|
return _mock_response(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"scene_summary": "修复后的场景",
|
||||||
|
"main_setting": "室内",
|
||||||
|
"key_entities": [],
|
||||||
|
"main_actions": [],
|
||||||
|
"topic_keywords": [],
|
||||||
|
"visible_text": [],
|
||||||
|
"temporal_flow": "",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return _mock_response(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"event_description": "修复后的事件",
|
||||||
|
"entities": [],
|
||||||
|
"actions": [],
|
||||||
|
"action_subjects": [],
|
||||||
|
"visible_text": [],
|
||||||
|
"spatial_relations": "",
|
||||||
|
"state_changes": None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRepairTree:
|
||||||
|
"""repair_tree 核心测试。"""
|
||||||
|
|
||||||
|
def _make_broken_tree(
|
||||||
|
self,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> tuple[TreeIndex, list[NodeIssue]]:
|
||||||
|
"""构建含一个空 frame_summary 的 L3 节点的测试树。"""
|
||||||
|
frame_path = tmp_path / "frames" / "L1_000_L2_000_L3_000.jpg"
|
||||||
|
frame_path.parent.mkdir(parents=True)
|
||||||
|
frame_path.write_bytes(b"\xff\xd8\xff\xe0fake")
|
||||||
|
|
||||||
|
l3 = L3Node(
|
||||||
|
id="vid_L1_000_L2_000_L3_000",
|
||||||
|
card=L3Card("", [], [], [], "", {}),
|
||||||
|
timestamp=1.0,
|
||||||
|
frame_path="frames/L1_000_L2_000_L3_000.jpg",
|
||||||
|
)
|
||||||
|
l2 = L2Node(
|
||||||
|
id="vid_L1_000_L2_000",
|
||||||
|
card=L2Card("原始事件", [], [], [], [], "", None),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l3],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="vid_L1_000",
|
||||||
|
card=L1Card("原始场景", "", [], [], [], [], ""),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
issues = [
|
||||||
|
NodeIssue(
|
||||||
|
"vid_L1_000_L2_000_L3_000",
|
||||||
|
3,
|
||||||
|
"empty_field",
|
||||||
|
"frame_summary 为空",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
return index, issues
|
||||||
|
|
||||||
|
def test_repairs_l3_and_cascades(self, tmp_path: Path) -> None:
|
||||||
|
"""修复 L3 后应级联重生成 L2 和 L1。"""
|
||||||
|
index, issues = self._make_broken_tree(tmp_path)
|
||||||
|
stats = asyncio.run(repair_tree(index, issues, MockVLM(), MockLLM(), tmp_path))
|
||||||
|
assert stats.l3_repaired == 1
|
||||||
|
assert stats.l2_regenerated == 1
|
||||||
|
assert stats.l1_regenerated == 1
|
||||||
|
assert index.roots[0].children[0].children[0].card.frame_summary == "修复后的帧描述"
|
||||||
|
assert index.roots[0].children[0].card.event_description == "修复后的事件"
|
||||||
|
assert index.roots[0].card.scene_summary == "修复后的场景"
|
||||||
|
|
||||||
|
def test_no_issues_no_changes(self) -> None:
|
||||||
|
"""无问题时不进行任何修复。"""
|
||||||
|
l3 = L3Node(
|
||||||
|
id="l1_0_l2_0_l3_0",
|
||||||
|
card=L3Card("正常", [], [], [], "", {}),
|
||||||
|
timestamp=1.0,
|
||||||
|
)
|
||||||
|
l2 = L2Node(
|
||||||
|
id="l1_0_l2_0",
|
||||||
|
card=L2Card("正常事件", [], [], [], [], "", None),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l3],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=L1Card("正常场景", "", [], [], [], [], ""),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
stats = asyncio.run(repair_tree(index, [], MockVLM(), MockLLM(), Path("/tmp")))
|
||||||
|
assert stats.l3_repaired == 0
|
||||||
|
assert stats.l2_regenerated == 0
|
||||||
|
assert stats.l1_regenerated == 0
|
||||||
|
|
||||||
|
def test_stats_dataclass(self) -> None:
|
||||||
|
"""RepairStats 数据类字段验证。"""
|
||||||
|
stats = RepairStats(l3_repaired=2, l2_regenerated=1, l1_regenerated=1)
|
||||||
|
assert stats.l3_repaired == 2
|
||||||
|
assert stats.l2_regenerated == 1
|
||||||
|
assert stats.l1_regenerated == 1
|
||||||
|
|
||||||
|
def test_skips_non_empty_field_issues(self, tmp_path: Path) -> None:
|
||||||
|
"""非 empty_field 类型的 issue 不触发 L3 修复。"""
|
||||||
|
l3 = L3Node(
|
||||||
|
id="l1_0_l2_0_l3_0",
|
||||||
|
card=L3Card("正常描述", [], [], [], "", {}),
|
||||||
|
timestamp=1.0,
|
||||||
|
)
|
||||||
|
l2 = L2Node(
|
||||||
|
id="l1_0_l2_0",
|
||||||
|
card=L2Card("原始事件", [], [], [], [], "", None),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l3],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=L1Card("原始场景", "", [], [], [], [], ""),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
issues = [NodeIssue("l1_0_l2_0_l3_0", 3, "missing_frame", "帧文件不存在")]
|
||||||
|
stats = asyncio.run(repair_tree(index, issues, MockVLM(), MockLLM(), tmp_path))
|
||||||
|
assert stats.l3_repaired == 0
|
||||||
|
assert stats.l2_regenerated == 0
|
||||||
|
|
||||||
|
def test_multiple_l3_under_same_l2(self, tmp_path: Path) -> None:
|
||||||
|
"""同一 L2 下多个 L3 修复后,L2 只重生成一次。"""
|
||||||
|
frame_dir = tmp_path / "frames"
|
||||||
|
frame_dir.mkdir(parents=True)
|
||||||
|
for i in range(2):
|
||||||
|
(frame_dir / f"f{i}.jpg").write_bytes(b"\xff\xd8\xff\xe0fake")
|
||||||
|
|
||||||
|
l3_a = L3Node(
|
||||||
|
id="l1_0_l2_0_l3_0",
|
||||||
|
card=L3Card("", [], [], [], "", {}),
|
||||||
|
timestamp=1.0,
|
||||||
|
frame_path="frames/f0.jpg",
|
||||||
|
)
|
||||||
|
l3_b = L3Node(
|
||||||
|
id="l1_0_l2_0_l3_1",
|
||||||
|
card=L3Card("", [], [], [], "", {}),
|
||||||
|
timestamp=2.0,
|
||||||
|
frame_path="frames/f1.jpg",
|
||||||
|
)
|
||||||
|
l2 = L2Node(
|
||||||
|
id="l1_0_l2_0",
|
||||||
|
card=L2Card("原始事件", [], [], [], [], "", None),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l3_a, l3_b],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=L1Card("原始场景", "", [], [], [], [], ""),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
issues = [
|
||||||
|
NodeIssue("l1_0_l2_0_l3_0", 3, "empty_field", "frame_summary 为空"),
|
||||||
|
NodeIssue("l1_0_l2_0_l3_1", 3, "empty_field", "frame_summary 为空"),
|
||||||
|
]
|
||||||
|
vlm = MockVLM()
|
||||||
|
llm = MockLLM()
|
||||||
|
stats = asyncio.run(repair_tree(index, issues, vlm, llm, tmp_path))
|
||||||
|
assert stats.l3_repaired == 2
|
||||||
|
assert stats.l2_regenerated == 1
|
||||||
|
assert stats.l1_regenerated == 1
|
||||||
|
assert vlm.call_count == 2
|
||||||
|
# LLM 应被调用 2 次:一次 L2 + 一次 L1
|
||||||
|
assert llm.call_count == 2
|
||||||
|
|
||||||
|
def test_missing_frame_file_skips_l3(self, tmp_path: Path) -> None:
|
||||||
|
"""帧文件不存在时跳过该 L3 节点的修复。"""
|
||||||
|
l3 = L3Node(
|
||||||
|
id="l1_0_l2_0_l3_0",
|
||||||
|
card=L3Card("", [], [], [], "", {}),
|
||||||
|
timestamp=1.0,
|
||||||
|
frame_path="frames/nonexistent.jpg",
|
||||||
|
)
|
||||||
|
l2 = L2Node(
|
||||||
|
id="l1_0_l2_0",
|
||||||
|
card=L2Card("原始事件", [], [], [], [], "", None),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l3],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=L1Card("原始场景", "", [], [], [], [], ""),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
issues = [NodeIssue("l1_0_l2_0_l3_0", 3, "empty_field", "frame_summary 为空")]
|
||||||
|
stats = asyncio.run(repair_tree(index, issues, MockVLM(), MockLLM(), tmp_path))
|
||||||
|
# 帧文件不存在 → 跳过 L3 修复 → 无级联
|
||||||
|
assert stats.l3_repaired == 0
|
||||||
|
assert stats.l2_regenerated == 0
|
||||||
|
assert stats.l1_regenerated == 0
|
||||||
@@ -0,0 +1,490 @@
|
|||||||
|
"""synthesizer 模块单元测试 — AnchorContext + 题型映射常量 + sample_anchor + prompt 构造 + VLM 解析 + 去重 + 单题生成。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import dataclasses
|
||||||
|
import random
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.question_gen.synthesizer import (
|
||||||
|
TASK_TYPE_LEVEL_MAP,
|
||||||
|
AnchorContext,
|
||||||
|
TaskTypeSpec,
|
||||||
|
build_generation_prompt,
|
||||||
|
generate_one,
|
||||||
|
is_duplicate,
|
||||||
|
parse_vlm_response,
|
||||||
|
sample_anchor,
|
||||||
|
)
|
||||||
|
from app.tree.index import TreeIndex
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
|
||||||
|
ALL_12_TYPES = [
|
||||||
|
"Object Recognition",
|
||||||
|
"Attribute Perception",
|
||||||
|
"OCR Problems",
|
||||||
|
"Spatial Reasoning",
|
||||||
|
"Spatial Perception",
|
||||||
|
"Action Recognition",
|
||||||
|
"Action Reasoning",
|
||||||
|
"Counting Problem",
|
||||||
|
"Temporal Perception",
|
||||||
|
"Temporal Reasoning",
|
||||||
|
"Information Synopsis",
|
||||||
|
"Object Reasoning",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class TestTaskTypeLevelMap:
|
||||||
|
"""TASK_TYPE_LEVEL_MAP 覆盖性与结构测试。"""
|
||||||
|
|
||||||
|
def test_covers_all_12_types(self) -> None:
|
||||||
|
"""映射表必须覆盖全部 12 种 Video-MME 题型。"""
|
||||||
|
assert set(TASK_TYPE_LEVEL_MAP.keys()) == set(ALL_12_TYPES)
|
||||||
|
|
||||||
|
def test_no_extra_types(self) -> None:
|
||||||
|
"""映射表不得包含 12 种标准题型之外的条目。"""
|
||||||
|
assert len(TASK_TYPE_LEVEL_MAP) == 12
|
||||||
|
|
||||||
|
def test_all_values_are_task_type_spec(self) -> None:
|
||||||
|
"""每个映射值必须是 TaskTypeSpec 实例。"""
|
||||||
|
for task_type, spec in TASK_TYPE_LEVEL_MAP.items():
|
||||||
|
assert isinstance(spec, TaskTypeSpec), f"{task_type} 映射值类型错误: {type(spec)}"
|
||||||
|
|
||||||
|
def test_level_values_valid(self) -> None:
|
||||||
|
"""每个 spec 的 level 必须是合法层级标识。"""
|
||||||
|
valid_levels = {"L1", "L2", "L3", "L1-L2"}
|
||||||
|
for task_type, spec in TASK_TYPE_LEVEL_MAP.items():
|
||||||
|
assert spec.level in valid_levels, (
|
||||||
|
f"{task_type} 层级 '{spec.level}' 不在 {valid_levels}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_context_fields_non_empty(self) -> None:
|
||||||
|
"""每个 spec 的 context_fields 至少有一个字段。"""
|
||||||
|
for task_type, spec in TASK_TYPE_LEVEL_MAP.items():
|
||||||
|
assert len(spec.context_fields) >= 1, f"{task_type} 的 context_fields 为空"
|
||||||
|
|
||||||
|
|
||||||
|
class TestAnchorContext:
|
||||||
|
"""AnchorContext 数据类测试。"""
|
||||||
|
|
||||||
|
def test_frozen(self) -> None:
|
||||||
|
"""AnchorContext 是不可变的。"""
|
||||||
|
ctx = AnchorContext(
|
||||||
|
node_id="L3_001",
|
||||||
|
card_text="A person walks into a room",
|
||||||
|
frame_paths=["/data/frames/001.jpg"],
|
||||||
|
subtitle="Hello there",
|
||||||
|
distractor_texts=["A car drives by"],
|
||||||
|
)
|
||||||
|
assert ctx.node_id == "L3_001"
|
||||||
|
assert ctx.card_text == "A person walks into a room"
|
||||||
|
assert ctx.frame_paths == ["/data/frames/001.jpg"]
|
||||||
|
assert ctx.subtitle == "Hello there"
|
||||||
|
assert ctx.distractor_texts == ["A car drives by"]
|
||||||
|
|
||||||
|
def test_mutation_raises(self) -> None:
|
||||||
|
"""frozen dataclass 拒绝赋值修改。"""
|
||||||
|
ctx = AnchorContext(
|
||||||
|
node_id="L3_001",
|
||||||
|
card_text="test",
|
||||||
|
frame_paths=["a.jpg"],
|
||||||
|
subtitle="",
|
||||||
|
distractor_texts=["other node"],
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
ctx.node_id = "L3_002" # type: ignore[misc]
|
||||||
|
raise AssertionError("应抛出 FrozenInstanceError")
|
||||||
|
except dataclasses.FrozenInstanceError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def test_empty_subtitle_allowed(self) -> None:
|
||||||
|
"""subtitle 可以为空字符串。"""
|
||||||
|
ctx = AnchorContext(
|
||||||
|
node_id="L2_010",
|
||||||
|
card_text="scene card",
|
||||||
|
frame_paths=[],
|
||||||
|
subtitle="",
|
||||||
|
distractor_texts=[],
|
||||||
|
)
|
||||||
|
assert ctx.subtitle == ""
|
||||||
|
|
||||||
|
def test_multiple_frame_paths(self) -> None:
|
||||||
|
"""frame_paths 可包含多个路径。"""
|
||||||
|
paths = ["/data/f1.jpg", "/data/f2.jpg", "/data/f3.jpg"]
|
||||||
|
ctx = AnchorContext(
|
||||||
|
node_id="L2_005",
|
||||||
|
card_text="multi-frame event",
|
||||||
|
frame_paths=paths,
|
||||||
|
subtitle="Dialogue line",
|
||||||
|
distractor_texts=["other1", "other2"],
|
||||||
|
)
|
||||||
|
assert len(ctx.frame_paths) == 3
|
||||||
|
|
||||||
|
|
||||||
|
class TestTaskTypeSpec:
|
||||||
|
"""TaskTypeSpec 数据类测试。"""
|
||||||
|
|
||||||
|
def test_frozen(self) -> None:
|
||||||
|
"""TaskTypeSpec 是不可变的。"""
|
||||||
|
spec = TaskTypeSpec(
|
||||||
|
level="L3",
|
||||||
|
needs_frames=True,
|
||||||
|
frame_count="1",
|
||||||
|
context_fields=("frame_summary",),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
spec.level = "L2" # type: ignore[misc]
|
||||||
|
raise AssertionError("应抛出 FrozenInstanceError")
|
||||||
|
except dataclasses.FrozenInstanceError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def test_context_fields_is_tuple(self) -> None:
|
||||||
|
"""context_fields 应为 tuple(不可变)。"""
|
||||||
|
for task_type, spec in TASK_TYPE_LEVEL_MAP.items():
|
||||||
|
assert isinstance(spec.context_fields, tuple), (
|
||||||
|
f"{task_type} 的 context_fields 不是 tuple"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# sample_anchor 测试
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _load_test_tree() -> tuple[TreeIndex, str]:
|
||||||
|
"""加载真实测试树(store/videos/ 下第一棵)。"""
|
||||||
|
videos_dir = Path("store/videos")
|
||||||
|
first_vid = sorted(videos_dir.iterdir())[0]
|
||||||
|
tree = TreeIndex.load_json(str(first_vid / "tree.json"))
|
||||||
|
return tree, first_vid.name
|
||||||
|
|
||||||
|
|
||||||
|
class TestSampleAnchor:
|
||||||
|
"""sample_anchor 锚节点采样测试(基于真实树数据)。"""
|
||||||
|
|
||||||
|
def test_l3_type_returns_single_frame(self) -> None:
|
||||||
|
"""L3 题型(Object Recognition)应返回恰好 1 帧。"""
|
||||||
|
tree, _vid = _load_test_tree()
|
||||||
|
ctx = sample_anchor(tree, "Object Recognition", set(), random.Random(42))
|
||||||
|
assert len(ctx.frame_paths) == 1
|
||||||
|
assert ctx.node_id.startswith("L") or "_L3_" in ctx.node_id
|
||||||
|
assert len(ctx.distractor_texts) > 0
|
||||||
|
|
||||||
|
def test_l2_type_returns_multiple_frames(self) -> None:
|
||||||
|
"""L2 题型(Action Reasoning)应返回 2-3 帧。"""
|
||||||
|
tree, _vid = _load_test_tree()
|
||||||
|
ctx = sample_anchor(tree, "Action Reasoning", set(), random.Random(42))
|
||||||
|
assert 2 <= len(ctx.frame_paths) <= 3
|
||||||
|
|
||||||
|
def test_temporal_perception_zero_or_one_frame(self) -> None:
|
||||||
|
"""Temporal Perception 特殊处理:0-1 帧,且 card_text 包含 time_range。"""
|
||||||
|
tree, _vid = _load_test_tree()
|
||||||
|
ctx = sample_anchor(tree, "Temporal Perception", set(), random.Random(42))
|
||||||
|
assert len(ctx.frame_paths) <= 1
|
||||||
|
assert "time_range" in ctx.card_text.lower() or "time" in ctx.card_text.lower()
|
||||||
|
|
||||||
|
def test_information_synopsis_uses_all_l2(self) -> None:
|
||||||
|
"""Information Synopsis 必须使用目标 L1 下所有 L2 子节点。"""
|
||||||
|
tree, _vid = _load_test_tree()
|
||||||
|
ctx = sample_anchor(tree, "Information Synopsis", set(), random.Random(42))
|
||||||
|
# 找到被选中的 L1,验证 frame_paths 数量 == 该 L1 下全部 L2 数量
|
||||||
|
chosen_l1 = next(r for r in tree.roots if r.id == ctx.node_id)
|
||||||
|
assert len(ctx.frame_paths) == len(chosen_l1.children)
|
||||||
|
|
||||||
|
def test_l1_type_l2_nodes_in_time_order(self) -> None:
|
||||||
|
"""Temporal Reasoning 应返回 >=3 帧且 card_text 有实质内容。"""
|
||||||
|
tree, _vid = _load_test_tree()
|
||||||
|
ctx = sample_anchor(tree, "Temporal Reasoning", set(), random.Random(42))
|
||||||
|
assert len(ctx.frame_paths) >= 3
|
||||||
|
assert len(ctx.card_text) > 20
|
||||||
|
|
||||||
|
def test_used_node_ids_excluded(self) -> None:
|
||||||
|
"""used_node_ids 中的节点不应被再次选中。"""
|
||||||
|
tree, _vid = _load_test_tree()
|
||||||
|
rng = random.Random(42)
|
||||||
|
ctx1 = sample_anchor(tree, "Object Recognition", set(), rng)
|
||||||
|
ctx2 = sample_anchor(tree, "Object Recognition", {ctx1.node_id}, random.Random(43))
|
||||||
|
assert ctx2.node_id != ctx1.node_id
|
||||||
|
|
||||||
|
def test_insufficient_nodes_raises(self) -> None:
|
||||||
|
"""所有候选节点均被排除时应抛出 ValueError。"""
|
||||||
|
tree, _vid = _load_test_tree()
|
||||||
|
all_l3_ids: set[str] = set()
|
||||||
|
for root in tree.roots:
|
||||||
|
for l2 in root.children:
|
||||||
|
for l3 in l2.children:
|
||||||
|
all_l3_ids.add(l3.id)
|
||||||
|
with pytest.raises(ValueError, match="锚节点不足"):
|
||||||
|
sample_anchor(tree, "Object Recognition", all_l3_ids, random.Random(42))
|
||||||
|
|
||||||
|
def test_object_reasoning_l1_l2_type(self) -> None:
|
||||||
|
"""Object Reasoning (L1-L2) 应选 2-3 个 L2 并按时间排序。"""
|
||||||
|
tree, _vid = _load_test_tree()
|
||||||
|
ctx = sample_anchor(tree, "Object Reasoning", set(), random.Random(42))
|
||||||
|
assert 2 <= len(ctx.frame_paths) <= 3
|
||||||
|
assert len(ctx.card_text) > 10
|
||||||
|
|
||||||
|
def test_spatial_reasoning_context_fields(self) -> None:
|
||||||
|
"""Spatial Reasoning 的 card_text 应包含 spatial_layout 字段。"""
|
||||||
|
tree, _vid = _load_test_tree()
|
||||||
|
ctx = sample_anchor(tree, "Spatial Reasoning", set(), random.Random(42))
|
||||||
|
# context_fields 包含 spatial_layout,card_text 必须出现该字段名
|
||||||
|
assert "spatial_layout" in ctx.card_text
|
||||||
|
assert len(ctx.card_text) > 20
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# build_generation_prompt 测试
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildGenerationPrompt:
|
||||||
|
"""build_generation_prompt 消息结构与内容测试。"""
|
||||||
|
|
||||||
|
def test_messages_structure(self) -> None:
|
||||||
|
"""带 exemplars 时,system 包含题型和示例,image_paths 来自 anchor。"""
|
||||||
|
anchor = AnchorContext(
|
||||||
|
node_id="L3_001",
|
||||||
|
card_text="A person typing on a laptop",
|
||||||
|
frame_paths=["store/videos/test/frames/L1_000_L2_000_L3_000.jpg"],
|
||||||
|
subtitle="Hello world",
|
||||||
|
distractor_texts=["Another person walking in park"],
|
||||||
|
)
|
||||||
|
exemplars = [
|
||||||
|
GeneratedQuestion(
|
||||||
|
question_id="ex-1",
|
||||||
|
video_id="v1",
|
||||||
|
task_type="Object Recognition",
|
||||||
|
question="What object?",
|
||||||
|
options=("A. Cat", "B. Dog", "C. Bird", "D. Fish"),
|
||||||
|
answer="A",
|
||||||
|
source_nodes=(),
|
||||||
|
difficulty="medium",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
messages, image_paths = build_generation_prompt(
|
||||||
|
"Object Recognition",
|
||||||
|
anchor,
|
||||||
|
exemplars,
|
||||||
|
)
|
||||||
|
assert messages[0]["role"] == "system"
|
||||||
|
assert "Object Recognition" in messages[0]["content"]
|
||||||
|
assert any("What object?" in str(m) for m in messages)
|
||||||
|
assert image_paths == anchor.frame_paths
|
||||||
|
|
||||||
|
def test_distractor_in_user_message(self) -> None:
|
||||||
|
"""干扰项文本应出现在 user message 中。"""
|
||||||
|
anchor = AnchorContext(
|
||||||
|
node_id="L2_003",
|
||||||
|
card_text="Event card text",
|
||||||
|
frame_paths=["a.jpg", "b.jpg"],
|
||||||
|
subtitle="",
|
||||||
|
distractor_texts=["Distractor node summary"],
|
||||||
|
)
|
||||||
|
messages, _ = build_generation_prompt("Action Reasoning", anchor, [])
|
||||||
|
user_msg = [m for m in messages if m["role"] == "user"][0]
|
||||||
|
assert "Distractor node summary" in user_msg["content"]
|
||||||
|
|
||||||
|
def test_no_exemplars_no_crash(self) -> None:
|
||||||
|
"""exemplars 为空时不应报错,system 消息中无示例段落。"""
|
||||||
|
anchor = AnchorContext(
|
||||||
|
node_id="L3_010",
|
||||||
|
card_text="Some card text",
|
||||||
|
frame_paths=["frame.jpg"],
|
||||||
|
subtitle="",
|
||||||
|
distractor_texts=[],
|
||||||
|
)
|
||||||
|
messages, image_paths = build_generation_prompt("OCR Problems", anchor, [])
|
||||||
|
assert len(messages) >= 2
|
||||||
|
assert image_paths == ["frame.jpg"]
|
||||||
|
|
||||||
|
def test_subtitle_included_when_non_empty(self) -> None:
|
||||||
|
"""非空 subtitle 应出现在 user message 中。"""
|
||||||
|
anchor = AnchorContext(
|
||||||
|
node_id="L3_002",
|
||||||
|
card_text="Card text here",
|
||||||
|
frame_paths=["f.jpg"],
|
||||||
|
subtitle="This is a subtitle line",
|
||||||
|
distractor_texts=[],
|
||||||
|
)
|
||||||
|
messages, _ = build_generation_prompt("Attribute Perception", anchor, [])
|
||||||
|
user_msg = [m for m in messages if m["role"] == "user"][0]
|
||||||
|
assert "This is a subtitle line" in user_msg["content"]
|
||||||
|
|
||||||
|
def test_empty_subtitle_not_in_user_message(self) -> None:
|
||||||
|
"""空 subtitle 不应在 user message 中产生 subtitle 段落。"""
|
||||||
|
anchor = AnchorContext(
|
||||||
|
node_id="L3_003",
|
||||||
|
card_text="Card",
|
||||||
|
frame_paths=["f.jpg"],
|
||||||
|
subtitle="",
|
||||||
|
distractor_texts=[],
|
||||||
|
)
|
||||||
|
messages, _ = build_generation_prompt("OCR Problems", anchor, [])
|
||||||
|
user_msg = [m for m in messages if m["role"] == "user"][0]
|
||||||
|
# 不应出现空的 subtitle 标记
|
||||||
|
assert "字幕" not in user_msg["content"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# parse_vlm_response 测试
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseVlmResponse:
|
||||||
|
"""parse_vlm_response 解析与校验测试。"""
|
||||||
|
|
||||||
|
def test_valid_json(self) -> None:
|
||||||
|
"""合法 JSON 正常解析,question_id 格式正确。"""
|
||||||
|
raw = '{"question": "What?", "options": ["A. X", "B. Y", "C. Z", "D. W"], "answer": "A"}'
|
||||||
|
result = parse_vlm_response(raw, "vid1", "Object Recognition", 1)
|
||||||
|
assert result["question"] == "What?"
|
||||||
|
assert result["answer"] == "A"
|
||||||
|
assert len(result["options"]) == 4
|
||||||
|
assert result["question_id"] == "gen-vid1-object_recognition-001"
|
||||||
|
|
||||||
|
def test_json_in_code_block(self) -> None:
|
||||||
|
"""从 markdown 代码块中提取 JSON。"""
|
||||||
|
raw = '```json\n{"question": "Q?", "options": ["A. 1", "B. 2", "C. 3", "D. 4"], "answer": "B"}\n```'
|
||||||
|
result = parse_vlm_response(raw, "vid1", "Object Recognition", 2)
|
||||||
|
assert result["question"] == "Q?"
|
||||||
|
assert result["question_id"] == "gen-vid1-object_recognition-002"
|
||||||
|
|
||||||
|
def test_invalid_json_raises(self) -> None:
|
||||||
|
"""非 JSON 文本应抛出 ValueError。"""
|
||||||
|
with pytest.raises(ValueError, match="VLM 返回"):
|
||||||
|
parse_vlm_response("not json", "vid1", "Object Recognition", 1)
|
||||||
|
|
||||||
|
def test_missing_fields_raises(self) -> None:
|
||||||
|
"""缺少必需字段应抛出 ValueError。"""
|
||||||
|
raw = '{"question": "What?"}'
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
parse_vlm_response(raw, "vid1", "Object Recognition", 1)
|
||||||
|
|
||||||
|
def test_options_must_be_four(self) -> None:
|
||||||
|
"""options 非 4 项应抛出 ValueError。"""
|
||||||
|
raw = '{"question": "Q?", "options": ["A. X", "B. Y"], "answer": "A"}'
|
||||||
|
with pytest.raises(ValueError, match="4"):
|
||||||
|
parse_vlm_response(raw, "vid1", "Object Recognition", 1)
|
||||||
|
|
||||||
|
def test_answer_must_be_abcd(self) -> None:
|
||||||
|
"""answer 不在 A-D 范围应抛出 ValueError。"""
|
||||||
|
raw = '{"question": "Q?", "options": ["A. X", "B. Y", "C. Z", "D. W"], "answer": "E"}'
|
||||||
|
with pytest.raises(ValueError, match="A.*D"):
|
||||||
|
parse_vlm_response(raw, "vid1", "Object Recognition", 1)
|
||||||
|
|
||||||
|
def test_seq_zero_padded(self) -> None:
|
||||||
|
"""seq 应按 3 位零填充格式化到 question_id 中。"""
|
||||||
|
raw = '{"question": "Q?", "options": ["A. 1", "B. 2", "C. 3", "D. 4"], "answer": "C"}'
|
||||||
|
result = parse_vlm_response(raw, "video_abc", "Action Reasoning", 42)
|
||||||
|
assert result["question_id"] == "gen-video_abc-action_reasoning-042"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# is_duplicate 测试
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestIsDuplicate:
|
||||||
|
"""is_duplicate embedding 去重判定测试。"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _fake_embed(texts: str | list[str]) -> np.ndarray:
|
||||||
|
"""确定性 + L2 归一化的 fake embedding。"""
|
||||||
|
if isinstance(texts, str):
|
||||||
|
texts = [texts]
|
||||||
|
vecs = []
|
||||||
|
for t in texts:
|
||||||
|
rs = np.random.RandomState(hash(t) % 2**31)
|
||||||
|
v = rs.randn(4).astype(np.float32)
|
||||||
|
v /= np.linalg.norm(v)
|
||||||
|
vecs.append(v)
|
||||||
|
return np.array(vecs, dtype=np.float32)
|
||||||
|
|
||||||
|
def test_empty_pool_never_duplicate(self) -> None:
|
||||||
|
"""空池始终返回 False。"""
|
||||||
|
pool = np.zeros((0, 4), dtype=np.float32)
|
||||||
|
assert is_duplicate("anything", pool, self._fake_embed, 0.85) is False
|
||||||
|
|
||||||
|
def test_identical_text_is_duplicate(self) -> None:
|
||||||
|
"""相同文本的 embedding 与自身余弦相似度为 1,必定判重。"""
|
||||||
|
text = "What is happening in the video?"
|
||||||
|
emb = self._fake_embed(text)
|
||||||
|
pool = emb.copy()
|
||||||
|
assert is_duplicate(text, pool, self._fake_embed, 0.85) is True
|
||||||
|
|
||||||
|
def test_different_text_not_duplicate(self) -> None:
|
||||||
|
"""极高阈值下,不同文本不判重。"""
|
||||||
|
pool_texts = ["aaa", "bbb", "ccc", "ddd", "eee"]
|
||||||
|
pool = self._fake_embed(pool_texts)
|
||||||
|
assert is_duplicate("completely unique text xyz", pool, self._fake_embed, 0.99) is False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# generate_one 测试
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestGenerateOne:
|
||||||
|
"""generate_one 单题异步生成测试。"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _load_test_tree() -> tuple[TreeIndex, str]:
|
||||||
|
"""加载真实测试树。"""
|
||||||
|
videos_dir = Path("store/videos")
|
||||||
|
first_vid = sorted(videos_dir.iterdir())[0]
|
||||||
|
return TreeIndex.load_json(str(first_vid / "tree.json")), first_vid.name
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_success_path(self) -> None:
|
||||||
|
"""mock VLM 返回合法 JSON,应成功生成 GeneratedQuestion。"""
|
||||||
|
vlm = AsyncMock()
|
||||||
|
vlm.chat_with_images.return_value = MagicMock(
|
||||||
|
content='{"question":"Q?","options":["A. 1","B. 2","C. 3","D. 4"],"answer":"A"}',
|
||||||
|
)
|
||||||
|
|
||||||
|
tree, vid = self._load_test_tree()
|
||||||
|
result = await generate_one(
|
||||||
|
vlm=vlm,
|
||||||
|
tree=tree,
|
||||||
|
video_id=vid,
|
||||||
|
task_type="Object Recognition",
|
||||||
|
seq=1,
|
||||||
|
exemplars=[],
|
||||||
|
used_node_ids=set(),
|
||||||
|
max_retries=3,
|
||||||
|
rng=random.Random(42),
|
||||||
|
session_id="test",
|
||||||
|
)
|
||||||
|
assert result is not None
|
||||||
|
assert result.question_id == f"gen-{vid}-object_recognition-001"
|
||||||
|
assert result.task_type == "Object Recognition"
|
||||||
|
assert result.source_nodes # non-empty
|
||||||
|
assert result.difficulty == "medium"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_all_retries_exhausted_returns_none(self) -> None:
|
||||||
|
"""VLM 始终返回无效 JSON,耗尽重试后返回 None。"""
|
||||||
|
vlm = AsyncMock()
|
||||||
|
vlm.chat_with_images.return_value = MagicMock(content="invalid")
|
||||||
|
|
||||||
|
tree, vid = self._load_test_tree()
|
||||||
|
result = await generate_one(
|
||||||
|
vlm=vlm,
|
||||||
|
tree=tree,
|
||||||
|
video_id=vid,
|
||||||
|
task_type="Object Recognition",
|
||||||
|
seq=1,
|
||||||
|
exemplars=[],
|
||||||
|
used_node_ids=set(),
|
||||||
|
max_retries=2,
|
||||||
|
rng=random.Random(42),
|
||||||
|
session_id="test",
|
||||||
|
)
|
||||||
|
assert result is None
|
||||||
|
assert vlm.chat_with_images.call_count == 2
|
||||||
File diff suppressed because it is too large
Load Diff
Executable
+75
@@ -0,0 +1,75 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# 从 TRM4.zip 迁移资产到 TRM5
|
||||||
|
# 用法: bash tools/migrate_from_trm4.sh /path/to/Video-Tree-TRM4.zip
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ZIP_PATH="${1:?用法: bash tools/migrate_from_trm4.sh /path/to/Video-Tree-TRM4.zip}"
|
||||||
|
PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
TMP_DIR=$(mktemp -d)
|
||||||
|
|
||||||
|
echo "=== TRM4 -> TRM5 迁移 ==="
|
||||||
|
echo "ZIP: $ZIP_PATH"
|
||||||
|
echo "项目根: $PROJECT_ROOT"
|
||||||
|
echo "临时目录: $TMP_DIR"
|
||||||
|
|
||||||
|
# 1. 解压
|
||||||
|
echo "[1/6] 解压 TRM4.zip..."
|
||||||
|
unzip -q "$ZIP_PATH" -d "$TMP_DIR"
|
||||||
|
SRC="$TMP_DIR/Video-Tree-TRM4"
|
||||||
|
|
||||||
|
# 2. 拷贝帧文件 (rsync --ignore-existing)
|
||||||
|
echo "[2/6] 拷贝帧文件..."
|
||||||
|
mkdir -p "$PROJECT_ROOT/store/videos"
|
||||||
|
for vid_dir in "$SRC"/store/videos/*/; do
|
||||||
|
vid=$(basename "$vid_dir")
|
||||||
|
dst="$PROJECT_ROOT/store/videos/$vid"
|
||||||
|
mkdir -p "$dst"
|
||||||
|
if [ -d "$vid_dir/frames" ]; then
|
||||||
|
rsync -a --ignore-existing "$vid_dir/frames/" "$dst/frames/"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# 3. 拷贝 SRT 字幕
|
||||||
|
echo "[3/6] 拷贝 SRT 字幕..."
|
||||||
|
mkdir -p "$PROJECT_ROOT/data/Video-MME/subtitle"
|
||||||
|
if [ -d "$SRC/data/Video-MME/subtitle" ]; then
|
||||||
|
rsync -a --ignore-existing "$SRC/data/Video-MME/subtitle/" "$PROJECT_ROOT/data/Video-MME/subtitle/"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 4. 拷贝视频压缩包
|
||||||
|
echo "[4/6] 拷贝视频压缩包..."
|
||||||
|
mkdir -p "$PROJECT_ROOT/data/Video-MME/original_data"
|
||||||
|
if [ -d "$SRC/data/Video-MME/original_data" ]; then
|
||||||
|
rsync -a --ignore-existing "$SRC/data/Video-MME/original_data/" "$PROJECT_ROOT/data/Video-MME/original_data/"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 5. 拷贝问题 JSON
|
||||||
|
echo "[5/6] 拷贝 Benchmark 问题..."
|
||||||
|
mkdir -p "$PROJECT_ROOT/store/questions"
|
||||||
|
if [ -d "$SRC/store/questions" ]; then
|
||||||
|
rsync -a --ignore-existing "$SRC/store/questions/" "$PROJECT_ROOT/store/questions/"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 6. 格式转换
|
||||||
|
echo "[6/6] 格式转换 flat -> TreeIndex..."
|
||||||
|
conda run -n Video-Tree-TRM python "$PROJECT_ROOT/tools/convert_flat_to_treeindex.py" \
|
||||||
|
"$SRC/store/videos" "$PROJECT_ROOT/store/videos"
|
||||||
|
|
||||||
|
# 验收
|
||||||
|
echo ""
|
||||||
|
echo "=== 验收检查 ==="
|
||||||
|
VIDEO_COUNT=$(find "$PROJECT_ROOT/store/videos" -name "tree.json" | wc -l)
|
||||||
|
SRT_COUNT=$(find "$PROJECT_ROOT/data/Video-MME/subtitle" -name "*.srt" 2>/dev/null | wc -l)
|
||||||
|
echo "视频树: $VIDEO_COUNT (期望 300)"
|
||||||
|
echo "SRT 字幕: $SRT_COUNT (期望 >=290)"
|
||||||
|
|
||||||
|
# 清理
|
||||||
|
echo "清理临时目录..."
|
||||||
|
rm -rf "$TMP_DIR"
|
||||||
|
|
||||||
|
if [ "$VIDEO_COUNT" -lt 300 ]; then
|
||||||
|
echo "WARNING: 视频树数量不足 300,请检查"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "迁移完成"
|
||||||
@@ -0,0 +1,492 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""树修复管线:检测 + VLM 重生成 + 校验 + Q&A 反向补全。
|
||||||
|
|
||||||
|
对 store/videos/ 下所有已迁移的树执行完整修复流程:
|
||||||
|
1. detect_issues() — 扫描空字段/缺失帧
|
||||||
|
2. repair_tree() — VLM 重新描述 + 底向上级联(如有问题节点)
|
||||||
|
3. verify_tree() — 交叉校验删除幻觉
|
||||||
|
4. supplement_tree() — Q&A 反向补全注入缺失事实
|
||||||
|
5. save_json() — 覆盖保存
|
||||||
|
|
||||||
|
用法:
|
||||||
|
conda activate Video-Tree-TRM
|
||||||
|
python tools/repair_trees.py [--videos-dir store/videos] [--concurrency 4] [--dry-run]
|
||||||
|
|
||||||
|
app/core/adapters 不 import 此脚本。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# 确保项目根目录在 sys.path 中
|
||||||
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
sys.path.insert(0, str(PROJECT_ROOT))
|
||||||
|
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
load_dotenv(PROJECT_ROOT / ".env")
|
||||||
|
|
||||||
|
from app.tree.index import TreeIndex
|
||||||
|
from app.tree.repair.detector import detect_issues
|
||||||
|
from app.tree.repair.regenerator import repair_tree
|
||||||
|
from app.tree.repair.supplement import supplement_tree
|
||||||
|
from app.tree.subtitle import parse_srt
|
||||||
|
from app.tree.verify import verify_tree
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 日志配置:不缓存,立即输出
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
logger.remove()
|
||||||
|
logger.add(
|
||||||
|
sys.stderr,
|
||||||
|
format="{time:HH:mm:ss} | {level:<7} | {message}",
|
||||||
|
level="DEBUG",
|
||||||
|
colorize=True,
|
||||||
|
)
|
||||||
|
logger.add(
|
||||||
|
PROJECT_ROOT / "logs" / "repair_trees.log",
|
||||||
|
format="{time:YYYY-MM-DD HH:mm:ss} | {level:<7} | {message}",
|
||||||
|
level="DEBUG",
|
||||||
|
rotation="50 MB",
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 断点续跑 — progress 文件管理
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
PROGRESS_FILE = "repair_progress.json"
|
||||||
|
|
||||||
|
|
||||||
|
def load_progress(path: Path) -> set[str]:
|
||||||
|
"""读取 progress 文件,返回已完成视频 ID 集合。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
path: progress JSON 文件路径。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
已完成视频 ID 集合。文件不存在或损坏时返回空集。
|
||||||
|
"""
|
||||||
|
if not path.exists():
|
||||||
|
return set()
|
||||||
|
try:
|
||||||
|
data = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
return set(data.get("finished_video_ids", []))
|
||||||
|
except (json.JSONDecodeError, KeyError, TypeError, AttributeError):
|
||||||
|
logger.warning("progress 文件损坏,忽略: {}", path)
|
||||||
|
return set()
|
||||||
|
|
||||||
|
|
||||||
|
async def save_progress(path: Path, lock: asyncio.Lock, vid: str) -> None:
|
||||||
|
"""原子追加一个视频 ID 到 progress 文件。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
path: progress JSON 文件路径。
|
||||||
|
lock: asyncio.Lock,防并发读改写丢更新。
|
||||||
|
vid: 要追加的视频 ID。
|
||||||
|
"""
|
||||||
|
async with lock:
|
||||||
|
finished = load_progress(path)
|
||||||
|
finished.add(vid)
|
||||||
|
tmp = path.with_suffix(".tmp")
|
||||||
|
tmp.write_text(
|
||||||
|
json.dumps({"finished_video_ids": sorted(finished)}, ensure_ascii=False, indent=2),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
os.replace(str(tmp), str(path))
|
||||||
|
|
||||||
|
|
||||||
|
def should_skip_video(vid: str, finished: set[str], *, reaggregate_all: bool) -> bool:
|
||||||
|
"""判断是否跳过该视频。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
vid: 视频 ID。
|
||||||
|
finished: progress 中已完成的视频 ID 集合。
|
||||||
|
reaggregate_all: --reaggregate-all 标志。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
True 表示跳过。
|
||||||
|
"""
|
||||||
|
if reaggregate_all:
|
||||||
|
return False
|
||||||
|
return vid in finished
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# LLM/VLM 客户端构建
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _build_clients(concurrency: int = 16):
|
||||||
|
"""构建 GovernedLLMClient(LLM + VLM)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
(llm_client, vlm_client) 元组。
|
||||||
|
"""
|
||||||
|
from adapters.breaker import CircuitBreaker
|
||||||
|
from adapters.llm import GovernedLLMClient
|
||||||
|
from adapters.telemetry import SQLiteTelemetryRecorder
|
||||||
|
from adapters.vlm import GovernedVLMClient
|
||||||
|
|
||||||
|
# 遥测记录器(GovernedLLMClient 要求非 None)
|
||||||
|
(PROJECT_ROOT / "logs").mkdir(exist_ok=True)
|
||||||
|
telemetry = SQLiteTelemetryRecorder(str(PROJECT_ROOT / "logs" / "repair_telemetry.db"))
|
||||||
|
|
||||||
|
breaker_threshold = int(os.getenv("LLM_CIRCUIT_BREAKER_THRESHOLD", "5"))
|
||||||
|
breaker_threshold = max(breaker_threshold, concurrency * 2)
|
||||||
|
breaker_cooldown = int(os.getenv("LLM_CIRCUIT_BREAKER_COOLDOWN", "60"))
|
||||||
|
timeout_s = float(os.getenv("LLM_TIMEOUT", "120"))
|
||||||
|
max_retries = int(os.getenv("LLM_MAX_RETRIES", "3"))
|
||||||
|
base_delay = float(os.getenv("LLM_RETRY_BASE_DELAY", "2.0"))
|
||||||
|
max_delay = float(os.getenv("LLM_RETRY_MAX_DELAY", "30.0"))
|
||||||
|
ttft = float(os.getenv("LLM_TTFT_TIMEOUT", "30"))
|
||||||
|
inter_token = float(os.getenv("LLM_INTER_TOKEN_TIMEOUT", "15"))
|
||||||
|
|
||||||
|
# LLM 客户端(用于 supplement 和 L2/L1 重生成)
|
||||||
|
llm = GovernedLLMClient(
|
||||||
|
model=os.environ["SEARCH_LLM_MODEL"],
|
||||||
|
base_url=os.environ["SEARCH_LLM_BASE_URL"],
|
||||||
|
api_key=os.environ["SEARCH_LLM_API_KEY"],
|
||||||
|
provider="deepseek",
|
||||||
|
thinking=False,
|
||||||
|
breaker=CircuitBreaker(fail_threshold=breaker_threshold, cooldown_s=breaker_cooldown),
|
||||||
|
cache=None,
|
||||||
|
telemetry=telemetry,
|
||||||
|
timeout_s=timeout_s,
|
||||||
|
ttft_timeout_s=ttft,
|
||||||
|
inter_token_timeout_s=inter_token,
|
||||||
|
max_retries=max_retries,
|
||||||
|
retry_base_delay_s=base_delay,
|
||||||
|
retry_max_delay_s=max_delay,
|
||||||
|
)
|
||||||
|
|
||||||
|
# VLM 客户端(用于 L3 帧重新描述)
|
||||||
|
vlm_base = GovernedLLMClient(
|
||||||
|
model=os.environ["VL_LLM_MODEL"],
|
||||||
|
base_url=os.environ["VL_LLM_BASE_URL"],
|
||||||
|
api_key=os.environ["VL_LLM_API_KEY"],
|
||||||
|
provider="qwen",
|
||||||
|
thinking=False,
|
||||||
|
breaker=CircuitBreaker(fail_threshold=breaker_threshold, cooldown_s=breaker_cooldown),
|
||||||
|
cache=None,
|
||||||
|
telemetry=telemetry,
|
||||||
|
timeout_s=timeout_s,
|
||||||
|
ttft_timeout_s=ttft,
|
||||||
|
inter_token_timeout_s=inter_token,
|
||||||
|
max_retries=max_retries,
|
||||||
|
retry_base_delay_s=base_delay,
|
||||||
|
retry_max_delay_s=max_delay,
|
||||||
|
)
|
||||||
|
vlm = GovernedVLMClient(vlm_base)
|
||||||
|
|
||||||
|
return llm, vlm
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 单视频修复
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def _repair_one_video(
|
||||||
|
vid: str,
|
||||||
|
tree_path: Path,
|
||||||
|
frames_dir: Path,
|
||||||
|
srt_dir: Path,
|
||||||
|
questions_dir: Path,
|
||||||
|
llm,
|
||||||
|
vlm,
|
||||||
|
*,
|
||||||
|
dry_run: bool = False,
|
||||||
|
) -> dict:
|
||||||
|
"""修复单个视频的树。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
vid: 视频 ID。
|
||||||
|
tree_path: tree.json 路径。
|
||||||
|
frames_dir: 帧文件目录。
|
||||||
|
srt_dir: SRT 字幕目录。
|
||||||
|
questions_dir: 问题 JSON 目录。
|
||||||
|
llm: LLMProvider 实例。
|
||||||
|
vlm: VLMProvider 实例。
|
||||||
|
dry_run: 仅检测不修复。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
统计 dict。
|
||||||
|
"""
|
||||||
|
stats = {
|
||||||
|
"vid": vid,
|
||||||
|
"issues_found": 0,
|
||||||
|
"l3_repaired": 0,
|
||||||
|
"l2_regenerated": 0,
|
||||||
|
"l1_regenerated": 0,
|
||||||
|
"verify_removed": 0,
|
||||||
|
"facts_injected": 0,
|
||||||
|
"error": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
# 加载树
|
||||||
|
index = TreeIndex.load_json(str(tree_path))
|
||||||
|
|
||||||
|
# Step 1: 检测问题
|
||||||
|
issues = detect_issues(index, frames_dir=frames_dir)
|
||||||
|
stats["issues_found"] = len(issues)
|
||||||
|
|
||||||
|
if issues:
|
||||||
|
logger.info("[{}] 发现 {} 个问题", vid, len(issues))
|
||||||
|
for issue in issues[:5]:
|
||||||
|
logger.debug(" {} [L{}] {}", issue.node_id, issue.level, issue.details)
|
||||||
|
if len(issues) > 5:
|
||||||
|
logger.debug(" ... 还有 {} 个", len(issues) - 5)
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
return stats
|
||||||
|
|
||||||
|
# Step 2: VLM 修复(如有 empty_field 问题)
|
||||||
|
empty_issues = [i for i in issues if i.issue_type == "empty_field"]
|
||||||
|
if empty_issues:
|
||||||
|
srt_entries = None
|
||||||
|
srt_path = srt_dir / f"{vid}.srt"
|
||||||
|
if srt_path.exists():
|
||||||
|
srt_entries = parse_srt(str(srt_path))
|
||||||
|
|
||||||
|
repair_stats = await repair_tree(index, empty_issues, vlm, llm, frames_dir, srt_entries)
|
||||||
|
stats["l3_repaired"] = repair_stats.l3_repaired
|
||||||
|
stats["l2_regenerated"] = repair_stats.l2_regenerated
|
||||||
|
stats["l1_regenerated"] = repair_stats.l1_regenerated
|
||||||
|
logger.info(
|
||||||
|
"[{}] 修复完成: L3={}, L2={}, L1={}",
|
||||||
|
vid,
|
||||||
|
repair_stats.l3_repaired,
|
||||||
|
repair_stats.l2_regenerated,
|
||||||
|
repair_stats.l1_regenerated,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 3: 质量校验
|
||||||
|
verify_stats = verify_tree(index)
|
||||||
|
total_removed = (
|
||||||
|
verify_stats.l2_entities_removed
|
||||||
|
+ verify_stats.l2_visible_text_removed
|
||||||
|
+ verify_stats.l1_visible_text_removed
|
||||||
|
+ verify_stats.l1_key_entities_removed
|
||||||
|
)
|
||||||
|
stats["verify_removed"] = total_removed
|
||||||
|
if total_removed > 0:
|
||||||
|
logger.info("[{}] 校验删除 {} 项不可靠内容", vid, total_removed)
|
||||||
|
|
||||||
|
# Step 4: Q&A 反向补全
|
||||||
|
questions_path = questions_dir / f"{vid}.json"
|
||||||
|
if questions_path.exists():
|
||||||
|
with open(questions_path, encoding="utf-8") as f:
|
||||||
|
questions = json.load(f)
|
||||||
|
if isinstance(questions, list) and questions:
|
||||||
|
logger.info("[{}] 开始 Q&A 补全 ({} 道题)...", vid, len(questions))
|
||||||
|
srt_text = ""
|
||||||
|
srt_path = srt_dir / f"{vid}.srt"
|
||||||
|
if srt_path.exists():
|
||||||
|
srt_text = srt_path.read_text(encoding="utf-8", errors="ignore")
|
||||||
|
|
||||||
|
try:
|
||||||
|
supplement_stats = await supplement_tree(
|
||||||
|
index, questions, llm, srt_text=srt_text
|
||||||
|
)
|
||||||
|
stats["facts_injected"] = supplement_stats.facts_injected
|
||||||
|
if supplement_stats.facts_injected > 0:
|
||||||
|
logger.info("[{}] 补全注入 {} 个事实", vid, supplement_stats.facts_injected)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error("[{}] Q&A 补全失败: {}", vid, exc)
|
||||||
|
logger.info("[{}] Q&A 补全完成", vid)
|
||||||
|
|
||||||
|
# Step 5: 保存
|
||||||
|
index.save_json(str(tree_path))
|
||||||
|
logger.info("[{}] 已保存", vid)
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
stats["error"] = str(exc)
|
||||||
|
logger.error("[{}] 修复失败: {}", vid, exc)
|
||||||
|
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 主流程
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def main_async(args: argparse.Namespace) -> None:
|
||||||
|
"""异步主流程:并发修复视频。"""
|
||||||
|
videos_dir = Path(args.videos_dir)
|
||||||
|
srt_dir = Path(args.srt_dir)
|
||||||
|
questions_dir = Path(args.questions_dir)
|
||||||
|
concurrency = args.concurrency
|
||||||
|
reaggregate_all = args.reaggregate_all
|
||||||
|
|
||||||
|
# 扫描所有视频
|
||||||
|
vid_dirs = sorted(d for d in videos_dir.iterdir() if d.is_dir() and (d / "tree.json").exists())
|
||||||
|
logger.info("发现 {} 个视频", len(vid_dirs))
|
||||||
|
|
||||||
|
# 加载 progress
|
||||||
|
progress_path = PROJECT_ROOT / "logs" / PROGRESS_FILE
|
||||||
|
finished = load_progress(progress_path)
|
||||||
|
if finished:
|
||||||
|
logger.info("已完成 {} 个视频(从 progress 文件加载)", len(finished))
|
||||||
|
|
||||||
|
if args.dry_run:
|
||||||
|
logger.info("=== DRY RUN 模式:仅检测不修复 ===")
|
||||||
|
|
||||||
|
# 构建客户端(dry_run 模式不需要)
|
||||||
|
llm, vlm = (None, None) if args.dry_run else _build_clients(concurrency)
|
||||||
|
|
||||||
|
# 过滤跳过的视频
|
||||||
|
pending = []
|
||||||
|
skipped_count = 0
|
||||||
|
for vid_dir in vid_dirs:
|
||||||
|
vid = vid_dir.name
|
||||||
|
if should_skip_video(vid, finished, reaggregate_all=reaggregate_all):
|
||||||
|
skipped_count += 1
|
||||||
|
continue
|
||||||
|
pending.append(vid_dir)
|
||||||
|
|
||||||
|
if skipped_count:
|
||||||
|
logger.info("跳过 {} 个已完成视频,待处理 {} 个", skipped_count, len(pending))
|
||||||
|
|
||||||
|
# 并发编排
|
||||||
|
sem = asyncio.Semaphore(concurrency)
|
||||||
|
progress_lock = asyncio.Lock()
|
||||||
|
all_stats: list[dict] = []
|
||||||
|
stats_lock = asyncio.Lock()
|
||||||
|
start_time = time.time()
|
||||||
|
completed = 0
|
||||||
|
|
||||||
|
async def _process(vid_dir: Path) -> None:
|
||||||
|
nonlocal completed
|
||||||
|
async with sem:
|
||||||
|
vid = vid_dir.name
|
||||||
|
tree_path = vid_dir / "tree.json"
|
||||||
|
frames_dir = vid_dir
|
||||||
|
|
||||||
|
logger.info("开始修复 {}", vid)
|
||||||
|
|
||||||
|
stats = await _repair_one_video(
|
||||||
|
vid,
|
||||||
|
tree_path,
|
||||||
|
frames_dir,
|
||||||
|
srt_dir,
|
||||||
|
questions_dir,
|
||||||
|
llm,
|
||||||
|
vlm,
|
||||||
|
dry_run=args.dry_run,
|
||||||
|
)
|
||||||
|
|
||||||
|
async with stats_lock:
|
||||||
|
all_stats.append(stats)
|
||||||
|
completed += 1
|
||||||
|
|
||||||
|
# 修复后重新检测,关键 issue 清零才记 finished
|
||||||
|
if stats["error"] is None and not args.dry_run:
|
||||||
|
index = TreeIndex.load_json(str(tree_path))
|
||||||
|
remaining = [i for i in detect_issues(index) if i.issue_type == "empty_field"]
|
||||||
|
if not remaining:
|
||||||
|
await save_progress(progress_path, progress_lock, vid)
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"[{}] 修复后仍有 {} 个 empty_field,不计入 finished",
|
||||||
|
vid,
|
||||||
|
len(remaining),
|
||||||
|
)
|
||||||
|
|
||||||
|
# 进度日志
|
||||||
|
if completed % 10 == 0:
|
||||||
|
elapsed = time.time() - start_time
|
||||||
|
rate = completed / elapsed * 60 if elapsed > 0 else 0
|
||||||
|
logger.info(
|
||||||
|
"进度: {}/{}, 已用 {:.0f}s, 速率 {:.1f} 视频/分钟",
|
||||||
|
completed,
|
||||||
|
len(pending),
|
||||||
|
elapsed,
|
||||||
|
rate,
|
||||||
|
)
|
||||||
|
|
||||||
|
tasks = [asyncio.create_task(_process(vd)) for vd in pending]
|
||||||
|
await asyncio.gather(*tasks)
|
||||||
|
|
||||||
|
# 最终汇总
|
||||||
|
elapsed = time.time() - start_time
|
||||||
|
total_issues = sum(s["issues_found"] for s in all_stats)
|
||||||
|
total_repaired = sum(s["l3_repaired"] for s in all_stats)
|
||||||
|
total_injected = sum(s["facts_injected"] for s in all_stats)
|
||||||
|
total_errors = sum(1 for s in all_stats if s["error"])
|
||||||
|
|
||||||
|
logger.info("=" * 60)
|
||||||
|
logger.info("修复完成")
|
||||||
|
logger.info(" 视频总数: {}", len(all_stats))
|
||||||
|
logger.info(" 跳过数: {}", skipped_count)
|
||||||
|
logger.info(" 问题总数: {}", total_issues)
|
||||||
|
logger.info(" L3 修复数: {}", total_repaired)
|
||||||
|
logger.info(" 事实注入数: {}", total_injected)
|
||||||
|
logger.info(" 失败数: {}", total_errors)
|
||||||
|
logger.info(" 总耗时: {:.0f}s", elapsed)
|
||||||
|
logger.info(" 并发数: {}", concurrency)
|
||||||
|
logger.info("=" * 60)
|
||||||
|
|
||||||
|
if total_errors > 0:
|
||||||
|
logger.warning("以下视频修复失败:")
|
||||||
|
for s in all_stats:
|
||||||
|
if s["error"]:
|
||||||
|
logger.warning(" {}: {}", s["vid"], s["error"])
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
"""解析命令行参数。"""
|
||||||
|
parser = argparse.ArgumentParser(description="树修复管线")
|
||||||
|
parser.add_argument(
|
||||||
|
"--videos-dir",
|
||||||
|
default="store/videos",
|
||||||
|
help="视频目录(默认: store/videos)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--srt-dir",
|
||||||
|
default="data/Video-MME/subtitle",
|
||||||
|
help="SRT 字幕目录(默认: data/Video-MME/subtitle)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--questions-dir",
|
||||||
|
default="store/questions/benchmarks/Video-MME",
|
||||||
|
help="问题 JSON 目录(默认: store/questions/benchmarks/Video-MME)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--dry-run",
|
||||||
|
action="store_true",
|
||||||
|
help="仅检测不修复,不调用 VLM/LLM",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--concurrency",
|
||||||
|
type=int,
|
||||||
|
default=16,
|
||||||
|
help="并发修复视频数(默认: 16)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--reaggregate-all",
|
||||||
|
action="store_true",
|
||||||
|
help="强制全量重聚合,忽略 progress 文件",
|
||||||
|
)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
"""同步入口。"""
|
||||||
|
args = parse_args()
|
||||||
|
(PROJECT_ROOT / "logs").mkdir(exist_ok=True)
|
||||||
|
asyncio.run(main_async(args))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user