From afe80a8b3253e958464145b307053d9b9947c9ba Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 00:23:04 -0400 Subject: [PATCH 01/21] =?UTF-8?q?feat(repair):=20=E6=96=AD=E7=82=B9?= =?UTF-8?q?=E7=BB=AD=E8=B7=91=20progress=20=E6=96=87=E4=BB=B6=E7=AE=A1?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit load_progress / save_progress(asyncio.Lock + os.replace 原子写入) / should_skip_video。支持并发安全的读改写和 --reaggregate-all 兜底。 Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/unit/test_repair_progress.py | 71 +++++ tools/repair_trees.py | 430 +++++++++++++++++++++++++++++ 2 files changed, 501 insertions(+) create mode 100644 tests/unit/test_repair_progress.py create mode 100644 tools/repair_trees.py diff --git a/tests/unit/test_repair_progress.py b/tests/unit/test_repair_progress.py new file mode 100644 index 0000000..d3a5532 --- /dev/null +++ b/tests/unit/test_repair_progress.py @@ -0,0 +1,71 @@ +"""修复管线断点续跑 progress 管理测试。""" +from __future__ import annotations + +import asyncio +import json + +import pytest + + +def test_load_progress_missing_file(tmp_path): + """progress 文件不存在时返回空集合。""" + from tools.repair_trees import load_progress + result = load_progress(tmp_path / "nonexistent.json") + assert result == set() + + +def test_load_progress_valid_file(tmp_path): + """正常读取已有 progress 文件。""" + from tools.repair_trees import load_progress + path = tmp_path / "progress.json" + path.write_text(json.dumps({"finished_video_ids": ["vid_a", "vid_b"]})) + result = load_progress(path) + assert result == {"vid_a", "vid_b"} + + +def test_load_progress_corrupted_file(tmp_path): + """损坏的 JSON 文件返回空集合（不抛异常）。""" + from tools.repair_trees import load_progress + path = tmp_path / "progress.json" + path.write_text("{invalid json") + result = load_progress(path) + assert result == set() + + +@pytest.mark.asyncio +async def test_save_progress_atomic(tmp_path): + """save_progress 原子写入，并发调用不丢失更新。""" + from tools.repair_trees import save_progress + path = tmp_path / "progress.json" + lock = asyncio.Lock() + await save_progress(path, lock, "vid_a") + await save_progress(path, lock, "vid_b") + data = json.loads(path.read_text()) + assert set(data["finished_video_ids"]) == {"vid_a", "vid_b"} + + +@pytest.mark.asyncio +async def test_save_progress_concurrent(tmp_path): + """16 路并发 save_progress 不丢失更新。""" + from tools.repair_trees import save_progress + path = tmp_path / "progress.json" + lock = asyncio.Lock() + tasks = [save_progress(path, lock, f"vid_{i}") for i in range(16)] + await asyncio.gather(*tasks) + data = json.loads(path.read_text()) + assert len(data["finished_video_ids"]) == 16 + + +def test_should_skip_finished(): + """已在 finished 集合中的视频应跳过。""" + from tools.repair_trees import should_skip_video + finished = {"vid_a", "vid_b"} + assert should_skip_video("vid_a", finished, reaggregate_all=False) is True + assert should_skip_video("vid_c", finished, reaggregate_all=False) is False + + +def test_should_skip_reaggregate_all_forces_rerun(): + """--reaggregate-all 标志强制不跳过。""" + from tools.repair_trees import should_skip_video + finished = {"vid_a"} + assert should_skip_video("vid_a", finished, reaggregate_all=True) is False diff --git a/tools/repair_trees.py b/tools/repair_trees.py new file mode 100644 index 0000000..13d287b --- /dev/null +++ b/tools/repair_trees.py @@ -0,0 +1,430 @@ +#!/usr/bin/env python3 +"""树修复管线：检测 + VLM 重生成 + 校验 + Q&A 反向补全。 + +对 store/videos/ 下所有已迁移的树执行完整修复流程： + 1. detect_issues() — 扫描空字段/缺失帧 + 2. repair_tree() — VLM 重新描述 + 底向上级联（如有问题节点） + 3. verify_tree() — 交叉校验删除幻觉 + 4. supplement_tree() — Q&A 反向补全注入缺失事实 + 5. save_json() — 覆盖保存 + +用法: + conda activate Video-Tree-TRM + python tools/repair_trees.py [--videos-dir store/videos] [--concurrency 4] [--dry-run] + +app/core/adapters 不 import 此脚本。 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +import time +from pathlib import Path + +# 确保项目根目录在 sys.path 中 +PROJECT_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + +from dotenv import load_dotenv +from loguru import logger + +load_dotenv(PROJECT_ROOT / ".env") + +from app.tree.index import TreeIndex +from app.tree.repair.detector import detect_issues +from app.tree.repair.regenerator import repair_tree +from app.tree.repair.supplement import supplement_tree +from app.tree.subtitle import SRTEntry, parse_srt +from app.tree.verify import verify_tree + +# --------------------------------------------------------------------------- +# 日志配置：不缓存，立即输出 +# --------------------------------------------------------------------------- + +logger.remove() +logger.add( + sys.stderr, + format="{time:HH:mm:ss} | {level:<7} | {message}", + level="DEBUG", + colorize=True, +) +logger.add( + PROJECT_ROOT / "logs" / "repair_trees.log", + format="{time:YYYY-MM-DD HH:mm:ss} | {level:<7} | {message}", + level="DEBUG", + rotation="50 MB", +) + +# --------------------------------------------------------------------------- +# 断点续跑 — progress 文件管理 +# --------------------------------------------------------------------------- + +PROGRESS_FILE = "repair_progress.json" + + +def load_progress(path: Path) -> set[str]: + """读取 progress 文件，返回已完成视频 ID 集合。 + + 参数: + path: progress JSON 文件路径。 + + 返回: + 已完成视频 ID 集合。文件不存在或损坏时返回空集。 + """ + if not path.exists(): + return set() + try: + data = json.loads(path.read_text(encoding="utf-8")) + return set(data.get("finished_video_ids", [])) + except (json.JSONDecodeError, KeyError, TypeError): + logger.warning("progress 文件损坏，忽略: {}", path) + return set() + + +async def save_progress(path: Path, lock: asyncio.Lock, vid: str) -> None: + """原子追加一个视频 ID 到 progress 文件。 + + 参数: + path: progress JSON 文件路径。 + lock: asyncio.Lock，防并发读改写丢更新。 + vid: 要追加的视频 ID。 + """ + async with lock: + finished = load_progress(path) + finished.add(vid) + tmp = path.with_suffix(".tmp") + tmp.write_text( + json.dumps({"finished_video_ids": sorted(finished)}, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + os.replace(str(tmp), str(path)) + + +def should_skip_video(vid: str, finished: set[str], *, reaggregate_all: bool) -> bool: + """判断是否跳过该视频。 + + 参数: + vid: 视频 ID。 + finished: progress 中已完成的视频 ID 集合。 + reaggregate_all: --reaggregate-all 标志。 + + 返回: + True 表示跳过。 + """ + if reaggregate_all: + return False + return vid in finished + + +# --------------------------------------------------------------------------- +# LLM/VLM 客户端构建 +# --------------------------------------------------------------------------- + + +def _build_clients(): + """构建 GovernedLLMClient（LLM + VLM）。 + + 返回: + (llm_client, vlm_client) 元组。 + """ + from adapters.breaker import CircuitBreaker + from adapters.llm import GovernedLLMClient + from adapters.telemetry import SQLiteTelemetryRecorder + from adapters.vlm import GovernedVLMClient + + # 遥测记录器（GovernedLLMClient 要求非 None） + (PROJECT_ROOT / "logs").mkdir(exist_ok=True) + telemetry = SQLiteTelemetryRecorder(str(PROJECT_ROOT / "logs" / "repair_telemetry.db")) + + breaker_threshold = int(os.getenv("LLM_CIRCUIT_BREAKER_THRESHOLD", "5")) + breaker_cooldown = int(os.getenv("LLM_CIRCUIT_BREAKER_COOLDOWN", "60")) + timeout_s = float(os.getenv("LLM_TIMEOUT", "120")) + max_retries = int(os.getenv("LLM_MAX_RETRIES", "3")) + base_delay = float(os.getenv("LLM_RETRY_BASE_DELAY", "2.0")) + max_delay = float(os.getenv("LLM_RETRY_MAX_DELAY", "30.0")) + ttft = float(os.getenv("LLM_TTFT_TIMEOUT", "30")) + inter_token = float(os.getenv("LLM_INTER_TOKEN_TIMEOUT", "15")) + + # LLM 客户端（用于 supplement 和 L2/L1 重生成） + llm = GovernedLLMClient( + model=os.environ["SEARCH_LLM_MODEL"], + base_url=os.environ["SEARCH_LLM_BASE_URL"], + api_key=os.environ["SEARCH_LLM_API_KEY"], + provider="deepseek", + thinking=False, + breaker=CircuitBreaker(fail_threshold=breaker_threshold, cooldown_s=breaker_cooldown), + cache=None, + telemetry=telemetry, + timeout_s=timeout_s, + ttft_timeout_s=ttft, + inter_token_timeout_s=inter_token, + max_retries=max_retries, + retry_base_delay_s=base_delay, + retry_max_delay_s=max_delay, + ) + + # VLM 客户端（用于 L3 帧重新描述） + vlm_base = GovernedLLMClient( + model=os.environ["VL_LLM_MODEL"], + base_url=os.environ["VL_LLM_BASE_URL"], + api_key=os.environ["VL_LLM_API_KEY"], + provider="qwen", + thinking=False, + breaker=CircuitBreaker(fail_threshold=breaker_threshold, cooldown_s=breaker_cooldown), + cache=None, + telemetry=telemetry, + timeout_s=timeout_s, + ttft_timeout_s=ttft, + inter_token_timeout_s=inter_token, + max_retries=max_retries, + retry_base_delay_s=base_delay, + retry_max_delay_s=max_delay, + ) + vlm = GovernedVLMClient(vlm_base) + + return llm, vlm + + +# --------------------------------------------------------------------------- +# 单视频修复 +# --------------------------------------------------------------------------- + + +async def _repair_one_video( + vid: str, + tree_path: Path, + frames_dir: Path, + srt_dir: Path, + questions_dir: Path, + llm, + vlm, + *, + dry_run: bool = False, +) -> dict: + """修复单个视频的树。 + + 参数: + vid: 视频 ID。 + tree_path: tree.json 路径。 + frames_dir: 帧文件目录。 + srt_dir: SRT 字幕目录。 + questions_dir: 问题 JSON 目录。 + llm: LLMProvider 实例。 + vlm: VLMProvider 实例。 + dry_run: 仅检测不修复。 + + 返回: + 统计 dict。 + """ + stats = { + "vid": vid, + "issues_found": 0, + "l3_repaired": 0, + "l2_regenerated": 0, + "l1_regenerated": 0, + "verify_removed": 0, + "facts_injected": 0, + "error": None, + } + + try: + # 加载树 + index = TreeIndex.load_json(str(tree_path)) + + # Step 1: 检测问题 + issues = detect_issues(index, frames_dir=frames_dir) + stats["issues_found"] = len(issues) + + if issues: + logger.info("[{}] 发现 {} 个问题", vid, len(issues)) + for issue in issues[:5]: + logger.debug(" {} [L{}] {}", issue.node_id, issue.level, issue.details) + if len(issues) > 5: + logger.debug(" ... 还有 {} 个", len(issues) - 5) + + if dry_run: + return stats + + # Step 2: VLM 修复（如有 empty_field 问题） + empty_issues = [i for i in issues if i.issue_type == "empty_field"] + if empty_issues: + srt_entries = None + srt_path = srt_dir / f"{vid}.srt" + if srt_path.exists(): + srt_entries = parse_srt(str(srt_path)) + + repair_stats = await repair_tree( + index, empty_issues, vlm, llm, frames_dir, srt_entries + ) + stats["l3_repaired"] = repair_stats.l3_repaired + stats["l2_regenerated"] = repair_stats.l2_regenerated + stats["l1_regenerated"] = repair_stats.l1_regenerated + logger.info( + "[{}] 修复完成: L3={}, L2={}, L1={}", + vid, repair_stats.l3_repaired, repair_stats.l2_regenerated, + repair_stats.l1_regenerated, + ) + + # Step 3: 质量校验 + verify_stats = verify_tree(index) + total_removed = ( + verify_stats.l2_entities_removed + + verify_stats.l2_visible_text_removed + + verify_stats.l1_visible_text_removed + + verify_stats.l1_key_entities_removed + ) + stats["verify_removed"] = total_removed + if total_removed > 0: + logger.info("[{}] 校验删除 {} 项不可靠内容", vid, total_removed) + + # Step 4: Q&A 反向补全 + questions_path = questions_dir / f"{vid}.json" + if questions_path.exists(): + with open(questions_path, encoding="utf-8") as f: + questions = json.load(f) + if isinstance(questions, list) and questions: + logger.info("[{}] 开始 Q&A 补全 ({} 道题)...", vid, len(questions)) + srt_text = "" + srt_path = srt_dir / f"{vid}.srt" + if srt_path.exists(): + srt_text = srt_path.read_text(encoding="utf-8", errors="ignore") + + try: + supplement_stats = await supplement_tree( + index, questions, llm, srt_text=srt_text + ) + stats["facts_injected"] = supplement_stats.facts_injected + if supplement_stats.facts_injected > 0: + logger.info( + "[{}] 补全注入 {} 个事实", vid, supplement_stats.facts_injected + ) + except Exception as exc: + logger.error("[{}] Q&A 补全失败: {}", vid, exc) + logger.info("[{}] Q&A 补全完成", vid) + + # Step 5: 保存 + index.save_json(str(tree_path)) + logger.info("[{}] 已保存", vid) + + except Exception as exc: + stats["error"] = str(exc) + logger.error("[{}] 修复失败: {}", vid, exc) + + return stats + + +# --------------------------------------------------------------------------- +# 主流程 +# --------------------------------------------------------------------------- + + +async def main_async(args: argparse.Namespace) -> None: + """异步主流程：遍历所有视频，逐个修复。""" + videos_dir = Path(args.videos_dir) + srt_dir = Path(args.srt_dir) + questions_dir = Path(args.questions_dir) + + # 扫描所有视频 + vid_dirs = sorted( + d for d in videos_dir.iterdir() + if d.is_dir() and (d / "tree.json").exists() + ) + logger.info("发现 {} 个视频待修复", len(vid_dirs)) + + if args.dry_run: + logger.info("=== DRY RUN 模式：仅检测不修复 ===") + + # 构建客户端（dry_run 模式不需要） + llm, vlm = (None, None) if args.dry_run else _build_clients() + + # 逐视频修复 + all_stats = [] + start_time = time.time() + for idx, vid_dir in enumerate(vid_dirs): + vid = vid_dir.name + tree_path = vid_dir / "tree.json" + frames_dir = vid_dir # frame_path 已含 "frames/" 前缀，不再嵌套 + + logger.info( + "[{}/{}] 开始修复 {}", + idx + 1, len(vid_dirs), vid, + ) + + stats = await _repair_one_video( + vid, tree_path, frames_dir, srt_dir, questions_dir, + llm, vlm, dry_run=args.dry_run, + ) + all_stats.append(stats) + + # 每 10 个视频汇总一次 + if (idx + 1) % 10 == 0: + elapsed = time.time() - start_time + rate = (idx + 1) / elapsed * 60 + logger.info( + "进度: {}/{}, 已用 {:.0f}s, 速率 {:.1f} 视频/分钟", + idx + 1, len(vid_dirs), elapsed, rate, + ) + + # 最终汇总 + elapsed = time.time() - start_time + total_issues = sum(s["issues_found"] for s in all_stats) + total_repaired = sum(s["l3_repaired"] for s in all_stats) + total_injected = sum(s["facts_injected"] for s in all_stats) + total_errors = sum(1 for s in all_stats if s["error"]) + + logger.info("=" * 60) + logger.info("修复完成") + logger.info(" 视频总数: {}", len(all_stats)) + logger.info(" 问题总数: {}", total_issues) + logger.info(" L3 修复数: {}", total_repaired) + logger.info(" 事实注入数: {}", total_injected) + logger.info(" 失败数: {}", total_errors) + logger.info(" 总耗时: {:.0f}s", elapsed) + logger.info("=" * 60) + + if total_errors > 0: + logger.warning("以下视频修复失败:") + for s in all_stats: + if s["error"]: + logger.warning(" {}: {}", s["vid"], s["error"]) + + +def parse_args() -> argparse.Namespace: + """解析命令行参数。""" + parser = argparse.ArgumentParser(description="树修复管线") + parser.add_argument( + "--videos-dir", + default="store/videos", + help="视频目录（默认: store/videos）", + ) + parser.add_argument( + "--srt-dir", + default="data/Video-MME/subtitle", + help="SRT 字幕目录（默认: data/Video-MME/subtitle）", + ) + parser.add_argument( + "--questions-dir", + default="store/questions/benchmarks/Video-MME", + help="问题 JSON 目录（默认: store/questions/benchmarks/Video-MME）", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="仅检测不修复，不调用 VLM/LLM", + ) + return parser.parse_args() + + +def main() -> None: + """同步入口。""" + args = parse_args() + (PROJECT_ROOT / "logs").mkdir(exist_ok=True) + asyncio.run(main_async(args)) + + +if __name__ == "__main__": + main() From 847def4a03669081ebbb2e5bec311a07f36ef613 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 00:23:40 -0400 Subject: [PATCH 02/21] =?UTF-8?q?feat(repair):=20asyncio.Semaphore=20?= =?UTF-8?q?=E5=B9=B6=E5=8F=91=20+=20=E6=96=AD=E7=82=B9=E7=BB=AD=E8=B7=91?= =?UTF-8?q?=20+=20CLI=20=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --concurrency 默认 16，--reaggregate-all 强制全量重聚合。 Semaphore 限视频并发数，视频内四步串行。progress 文件 asyncio.Lock + os.replace 原子写入。熔断阈值 max(.env, concurrency*2)。 Co-Authored-By: Claude Opus 4.6 (1M context) --- .env.example | 2 +- tools/repair_trees.py | 101 +++++++++++++++++++++++++++++++----------- 2 files changed, 76 insertions(+), 27 deletions(-) diff --git a/.env.example b/.env.example index 2b45c02..a0c1ce0 100644 --- a/.env.example +++ b/.env.example @@ -41,7 +41,7 @@ REDIS_URL=redis://localhost:6379/0 LLM_TIMEOUT=120 LLM_MAX_RETRIES=3 LLM_RETRY_BASE_DELAY=2.0 -LLM_CIRCUIT_BREAKER_THRESHOLD=5 +LLM_CIRCUIT_BREAKER_THRESHOLD=5 # 实际阈值 = max(此值, concurrency*2) LLM_CIRCUIT_BREAKER_COOLDOWN=60 LLM_TTFT_TIMEOUT=30 LLM_INTER_TOKEN_TIMEOUT=15 diff --git a/tools/repair_trees.py b/tools/repair_trees.py index 13d287b..e87df42 100644 --- a/tools/repair_trees.py +++ b/tools/repair_trees.py @@ -125,7 +125,7 @@ def should_skip_video(vid: str, finished: set[str], *, reaggregate_all: bool) -> # --------------------------------------------------------------------------- -def _build_clients(): +def _build_clients(concurrency: int = 16): """构建 GovernedLLMClient（LLM + VLM）。 返回: @@ -141,6 +141,7 @@ def _build_clients(): telemetry = SQLiteTelemetryRecorder(str(PROJECT_ROOT / "logs" / "repair_telemetry.db")) breaker_threshold = int(os.getenv("LLM_CIRCUIT_BREAKER_THRESHOLD", "5")) + breaker_threshold = max(breaker_threshold, concurrency * 2) breaker_cooldown = int(os.getenv("LLM_CIRCUIT_BREAKER_COOLDOWN", "60")) timeout_s = float(os.getenv("LLM_TIMEOUT", "120")) max_retries = int(os.getenv("LLM_MAX_RETRIES", "3")) @@ -323,52 +324,87 @@ async def _repair_one_video( async def main_async(args: argparse.Namespace) -> None: - """异步主流程：遍历所有视频，逐个修复。""" + """异步主流程：并发修复视频。""" videos_dir = Path(args.videos_dir) srt_dir = Path(args.srt_dir) questions_dir = Path(args.questions_dir) + concurrency = args.concurrency + reaggregate_all = args.reaggregate_all # 扫描所有视频 vid_dirs = sorted( d for d in videos_dir.iterdir() if d.is_dir() and (d / "tree.json").exists() ) - logger.info("发现 {} 个视频待修复", len(vid_dirs)) + logger.info("发现 {} 个视频", len(vid_dirs)) + + # 加载 progress + progress_path = PROJECT_ROOT / "logs" / PROGRESS_FILE + finished = load_progress(progress_path) + if finished: + logger.info("已完成 {} 个视频（从 progress 文件加载）", len(finished)) if args.dry_run: logger.info("=== DRY RUN 模式：仅检测不修复 ===") # 构建客户端（dry_run 模式不需要） - llm, vlm = (None, None) if args.dry_run else _build_clients() + llm, vlm = (None, None) if args.dry_run else _build_clients(concurrency) - # 逐视频修复 - all_stats = [] - start_time = time.time() - for idx, vid_dir in enumerate(vid_dirs): + # 过滤跳过的视频 + pending = [] + skipped_count = 0 + for vid_dir in vid_dirs: vid = vid_dir.name - tree_path = vid_dir / "tree.json" - frames_dir = vid_dir # frame_path 已含 "frames/" 前缀，不再嵌套 + if should_skip_video(vid, finished, reaggregate_all=reaggregate_all): + skipped_count += 1 + continue + pending.append(vid_dir) - logger.info( - "[{}/{}] 开始修复 {}", - idx + 1, len(vid_dirs), vid, - ) + if skipped_count: + logger.info("跳过 {} 个已完成视频，待处理 {} 个", skipped_count, len(pending)) - stats = await _repair_one_video( - vid, tree_path, frames_dir, srt_dir, questions_dir, - llm, vlm, dry_run=args.dry_run, - ) - all_stats.append(stats) + # 并发编排 + sem = asyncio.Semaphore(concurrency) + progress_lock = asyncio.Lock() + all_stats: list[dict] = [] + stats_lock = asyncio.Lock() + start_time = time.time() + completed = 0 - # 每 10 个视频汇总一次 - if (idx + 1) % 10 == 0: - elapsed = time.time() - start_time - rate = (idx + 1) / elapsed * 60 - logger.info( - "进度: {}/{}, 已用 {:.0f}s, 速率 {:.1f} 视频/分钟", - idx + 1, len(vid_dirs), elapsed, rate, + async def _process(vid_dir: Path) -> None: + nonlocal completed + async with sem: + vid = vid_dir.name + tree_path = vid_dir / "tree.json" + frames_dir = vid_dir + + logger.info("开始修复 {}", vid) + + stats = await _repair_one_video( + vid, tree_path, frames_dir, srt_dir, questions_dir, + llm, vlm, dry_run=args.dry_run, ) + async with stats_lock: + all_stats.append(stats) + completed += 1 + + # 无 error 且非 dry_run 才记 finished + if stats["error"] is None and not args.dry_run: + await save_progress(progress_path, progress_lock, vid) + + # 进度日志 + if completed % 10 == 0: + elapsed = time.time() - start_time + rate = completed / elapsed * 60 if elapsed > 0 else 0 + logger.info( + "进度: {}/{}, 已用 {:.0f}s, 速率 {:.1f} 视频/分钟", + completed, len(pending), elapsed, rate, + ) + + tasks = [asyncio.create_task(_process(vd)) for vd in pending] + await asyncio.gather(*tasks) + # 最终汇总 elapsed = time.time() - start_time total_issues = sum(s["issues_found"] for s in all_stats) @@ -379,11 +415,13 @@ async def main_async(args: argparse.Namespace) -> None: logger.info("=" * 60) logger.info("修复完成") logger.info(" 视频总数: {}", len(all_stats)) + logger.info(" 跳过数: {}", skipped_count) logger.info(" 问题总数: {}", total_issues) logger.info(" L3 修复数: {}", total_repaired) logger.info(" 事实注入数: {}", total_injected) logger.info(" 失败数: {}", total_errors) logger.info(" 总耗时: {:.0f}s", elapsed) + logger.info(" 并发数: {}", concurrency) logger.info("=" * 60) if total_errors > 0: @@ -416,6 +454,17 @@ def parse_args() -> argparse.Namespace: action="store_true", help="仅检测不修复，不调用 VLM/LLM", ) + parser.add_argument( + "--concurrency", + type=int, + default=16, + help="并发修复视频数（默认: 16）", + ) + parser.add_argument( + "--reaggregate-all", + action="store_true", + help="强制全量重聚合，忽略 progress 文件", + ) return parser.parse_args() From 4612123ec41a0ec607e5ae3d18fc64dfec6ce802 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 00:25:36 -0400 Subject: [PATCH 03/21] style: ruff format telemetry + repair_trees, remove unused SRTEntry import --- adapters/telemetry.py | 4 +--- tools/repair_trees.py | 34 +++++++++++++++++++--------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/adapters/telemetry.py b/adapters/telemetry.py index 5d24e50..ddc4f7e 100644 --- a/adapters/telemetry.py +++ b/adapters/telemetry.py @@ -124,9 +124,7 @@ class SQLiteTelemetryRecorder: finally: conn.close() except sqlite3.Error as exc: - logger.warning( - "遥测写入失败（已降级），call_id={}: {}", call_id, exc - ) + logger.warning("遥测写入失败（已降级），call_id={}: {}", call_id, exc) async def record_llm_call( self, diff --git a/tools/repair_trees.py b/tools/repair_trees.py index e87df42..c2690b7 100644 --- a/tools/repair_trees.py +++ b/tools/repair_trees.py @@ -38,7 +38,7 @@ from app.tree.index import TreeIndex from app.tree.repair.detector import detect_issues from app.tree.repair.regenerator import repair_tree from app.tree.repair.supplement import supplement_tree -from app.tree.subtitle import SRTEntry, parse_srt +from app.tree.subtitle import parse_srt from app.tree.verify import verify_tree # --------------------------------------------------------------------------- @@ -258,15 +258,15 @@ async def _repair_one_video( if srt_path.exists(): srt_entries = parse_srt(str(srt_path)) - repair_stats = await repair_tree( - index, empty_issues, vlm, llm, frames_dir, srt_entries - ) + repair_stats = await repair_tree(index, empty_issues, vlm, llm, frames_dir, srt_entries) stats["l3_repaired"] = repair_stats.l3_repaired stats["l2_regenerated"] = repair_stats.l2_regenerated stats["l1_regenerated"] = repair_stats.l1_regenerated logger.info( "[{}] 修复完成: L3={}, L2={}, L1={}", - vid, repair_stats.l3_repaired, repair_stats.l2_regenerated, + vid, + repair_stats.l3_repaired, + repair_stats.l2_regenerated, repair_stats.l1_regenerated, ) @@ -300,9 +300,7 @@ async def _repair_one_video( ) stats["facts_injected"] = supplement_stats.facts_injected if supplement_stats.facts_injected > 0: - logger.info( - "[{}] 补全注入 {} 个事实", vid, supplement_stats.facts_injected - ) + logger.info("[{}] 补全注入 {} 个事实", vid, supplement_stats.facts_injected) except Exception as exc: logger.error("[{}] Q&A 补全失败: {}", vid, exc) logger.info("[{}] Q&A 补全完成", vid) @@ -332,10 +330,7 @@ async def main_async(args: argparse.Namespace) -> None: reaggregate_all = args.reaggregate_all # 扫描所有视频 - vid_dirs = sorted( - d for d in videos_dir.iterdir() - if d.is_dir() and (d / "tree.json").exists() - ) + vid_dirs = sorted(d for d in videos_dir.iterdir() if d.is_dir() and (d / "tree.json").exists()) logger.info("发现 {} 个视频", len(vid_dirs)) # 加载 progress @@ -381,8 +376,14 @@ async def main_async(args: argparse.Namespace) -> None: logger.info("开始修复 {}", vid) stats = await _repair_one_video( - vid, tree_path, frames_dir, srt_dir, questions_dir, - llm, vlm, dry_run=args.dry_run, + vid, + tree_path, + frames_dir, + srt_dir, + questions_dir, + llm, + vlm, + dry_run=args.dry_run, ) async with stats_lock: @@ -399,7 +400,10 @@ async def main_async(args: argparse.Namespace) -> None: rate = completed / elapsed * 60 if elapsed > 0 else 0 logger.info( "进度: {}/{}, 已用 {:.0f}s, 速率 {:.1f} 视频/分钟", - completed, len(pending), elapsed, rate, + completed, + len(pending), + elapsed, + rate, ) tasks = [asyncio.create_task(_process(vd)) for vd in pending] From d6bcf413360cc42f0cbf4f3eab9f2eb689568840 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 00:30:23 -0400 Subject: [PATCH 04/21] =?UTF-8?q?fix(repair):=20Codex=20=E5=AE=A1=E6=9F=A5?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=20=E2=80=94=20progress=20=E5=AE=8C=E6=88=90?= =?UTF-8?q?=E5=88=A4=E6=8D=AE=E5=8A=A0=E4=B8=A5=20+=20load=5Fprogress=20?= =?UTF-8?q?=E9=98=B2=E5=BE=A1=E5=A2=9E=E5=BC=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 保存 progress 前重新 detect_issues，只有 empty_field 清零才记 finished （修复 L2/L1 空字段被检测但未修复仍写 finished 的问题） 2. load_progress 增加 AttributeError 捕获（防 JSON 非 dict 形态崩溃） --- tools/repair_trees.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tools/repair_trees.py b/tools/repair_trees.py index c2690b7..dac5434 100644 --- a/tools/repair_trees.py +++ b/tools/repair_trees.py @@ -80,7 +80,7 @@ def load_progress(path: Path) -> set[str]: try: data = json.loads(path.read_text(encoding="utf-8")) return set(data.get("finished_video_ids", [])) - except (json.JSONDecodeError, KeyError, TypeError): + except (json.JSONDecodeError, KeyError, TypeError, AttributeError): logger.warning("progress 文件损坏，忽略: {}", path) return set() @@ -390,9 +390,18 @@ async def main_async(args: argparse.Namespace) -> None: all_stats.append(stats) completed += 1 - # 无 error 且非 dry_run 才记 finished + # 修复后重新检测，关键 issue 清零才记 finished if stats["error"] is None and not args.dry_run: - await save_progress(progress_path, progress_lock, vid) + index = TreeIndex.load_json(str(tree_path)) + remaining = [i for i in detect_issues(index) if i.issue_type == "empty_field"] + if not remaining: + await save_progress(progress_path, progress_lock, vid) + else: + logger.warning( + "[{}] 修复后仍有 {} 个 empty_field，不计入 finished", + vid, + len(remaining), + ) # 进度日志 if completed % 10 == 0: From 8cb1158a2d3dee350fbf12fdea9f01e371f63c76 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 00:39:45 -0400 Subject: [PATCH 05/21] =?UTF-8?q?docs(wiki):=20paper=20main-figure=20desig?= =?UTF-8?q?n=20=E2=80=94=20self-evolving=20loop=20layout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2026-07-09-paper-main-figure-design.md | 130 ++++++++++++++++++ research-wiki/designs/paper-main-figure.md | 17 +++ research-wiki/graph/edges.json | 5 + research-wiki/index.md | 6 +- research-wiki/log.md | 2 + 5 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 research-wiki/designs/2026-07-09-paper-main-figure-design.md create mode 100644 research-wiki/designs/paper-main-figure.md diff --git a/research-wiki/designs/2026-07-09-paper-main-figure-design.md b/research-wiki/designs/2026-07-09-paper-main-figure-design.md new file mode 100644 index 0000000..0d6859b --- /dev/null +++ b/research-wiki/designs/2026-07-09-paper-main-figure-design.md @@ -0,0 +1,130 @@ +# 论文主图设计：Self-Evolving Search Agent 推理训练闭环 + +**日期** 2026-07-09 · **状态** 已获用户批准（口头） · **产出物** Figma 图（文件 `xnLGUkZottqnr4dsEt9fGq` Page 1 空白区） + +## 1. 目标与定位 + +- **用途**：论文（EMNLP 2026 方向）主图，展示推理训练部分的自进化闭环；不含建树与新题生成（建树已有独立图，位于同一 Figma 文件）。 +- **核心叙事**：搜索 Agent 通过 推理→诊断→进化→门控 闭环自我改进；**Frozen LLM, trainable harness**——被"训练"的不是模型权重，而是版本化的 Skills+Prompts。 +- **差异化**：AVP/DVD 等相关工作画的是推理期内环（agent 怎么搜视频）；本图内环只是一个面板，**训练期外环是主角**。 +- **审稿人一句话记忆点**：这是一个不动模型权重的 PyTorch 式训练循环。 + +## 2. 已确认的关键决策 + +| 决策点 | 结论 | +|---|---| +| 构图 | 水平流水线 + 底部参数回流闭环（方案 A） | +| 版式 | 双栏跨页宽图，画布 2400×1050 px（≈2.3:1，缩印 180mm） | +| 信息密度 | 四机制全部可见（Agent 内环 / 诊断瀑布 / patch 引擎 / CE-Gate+信息阶梯），去工程化（无熔断/缓存/遥测） | +| PyTorch 类比 | 底部独立双行对照条，与上方区域逐段对齐 | +| 示例贯穿 | 延续建树图同一 Video-MME 天文台视频，问题/诊断/patch 文本典型化设计 | +| 迭代维度 | Store 处 v1…vN 卡片堆叠 + ×N epochs 循环标记暗示，不加独立时间轴 | +| 绘制位置 | 与建树图同文件（素材直接复用），Page 1 空白区 y≥1800，不动现有图层 | +| 语言 | 图内文字全英文 | + +## 3. 布局 + +``` +┌──────────────────────────────────────────────────────────────────┐ +│ [输入] ┌──────────────┐ ┌───────────┐ ┌──────────┐ ┌──────┐ │ +│ Q+缩略图→│ ① INFERENCE │→ │② DIAGNOSE │→ │③ EVOLVE │→ │④ CE- │ │ +│ │ 树环境+内环 │ │ 归因瀑布 │ │ patch引擎 │ │ GATE │ │ +│ └──────↑───────┘ └───────────┘ └──────────┘ └──┬───┘ │ +│ │ read ┌─────────────────┐ accept│ │ +│ └─────────────────│⑤ Skills+Prompts │←──────┘ │ +│ │ Store v1…vN ▤▤ │ reject→保基线 +│ └─────────────────┘ │ +├──────────────────────────────────────────────────────────────────┤ +│ DataLoader│forward()│backward()│optimizer.step()│grad clip│nn.Parameter│ +└──────────────────────────────────────────────────────────────────┘ +``` + +**数据流勘误记录**：CE-Gate 位于 Evolve 之后（进化产出候选 → gate 用 e-process 验证候选 vs 基线 → accept 才写入 Store），信息阶梯为 gate 供给高信息量题序，而非控制输入难度。此顺序已与 `core/evolution/gate.py`、`app/harness/gate_ladder.py` 核实。 + +## 4. 五区域内容规格 + +术语均与代码核实一致（来源见 §7）。 + +### 4.1 输入区 +- 问题卡（Q: *"What happens right after the dome opens?"* 措辞绘制时可打磨）+ 天文台视频缩略图 + 迷你树 icon，标注 "hierarchical video tree (Fig. 2)" 衔接建树图。 + +### 4.2 ① INFERENCE（agent-controlled，图上唯一画成循环的面板） +- 树环境迷你版：L1/L2/L3 三层色带（绿/蓝/黄，复用建树图配色与缩略图）。 +- Agent 内环轨迹：Thought → `search_similar` → `view_node` → `observe_frame` → `submit_answer`；侧边 read-skill 箭头（来自 ⑤）。 +- 示例结局：answer ✗（答错），输出 trace 流向 ②。 +- 面板角标：*agent-controlled*；其余面板角标 *code-controlled*。 + +### 4.3 ② DIAGNOSE（code-controlled） +- 归因瀑布级联：`extraction failure → search failure → reasoning failure`（+ mixed 兜底），画成三级下落台阶。 +- 二分岔：**defect**（改 skill 正文）vs **lapse**（记 appendix 提醒）。 +- D1–D5 压缩为一排五个小 chip：attribution / tool quality / search behavior / skill compliance / decision patterns。 +- 示例：该题归因 `search failure` → 判 **defect**。 + +### 4.4 ③ EVOLVE +- patch 流水线：candidate edits → **rank-and-clip** → apply patch。 +- 侧边锁条带：**protected spans**（appendix / momentum 区带锁图标，不可改写）。 +- 示例 patch 片段：*"+ verify event boundary via L2 card before observe_frame"*。 +- momentum 机制不单独出现（用户确认），仅隐含于锁条带。 + +### 4.5 ④ CE-GATE +- 菱形门 + e-process 小曲线（e 值随题数爬升，越过 `e_confirm` 虚线）。 +- 四出口：**accept (confirmed) / accept (provisional) / reject / continue**（代码中三种 reject 在图上合并，用户确认）。 +- 侧挂小组件：信息阶梯（2:1 交错题序图标，标注 *info-max question ladder*）。 +- accept → 写入 Store 新版本；reject → 保基线（细灰箭头）。 + +### 4.6 ⑤ Skills+Prompts Store +- v1…vN 卡片堆叠（复用建树图 Event Card 堆叠画法）+ 版本号 badge。 +- accept 箭头写入 v(N+1)；read 箭头回流至 ①，构成大闭环；循环标记 **×N epochs**。 + +## 5. PyTorch 对照条（最底部） + +浅灰底横带，等宽字体，与上方区域逐段对齐： + +| 上方区域 | 对照文字 | +|---|---| +| 输入 | `DataLoader` | +| ① | `model.forward()` | +| ② | `loss.backward()` | +| ③ | `optimizer.step()` | +| ④ | `grad clipping` | +| ⑤ | `nn.Parameter` | + +条带一侧放记忆点标语：*Frozen LLM, trainable harness*。 + +## 6. 视觉规范与素材复用 + +| 元素 | 方案 | +|---|---| +| 面板样式 | 白底、细虚线外框、顶部居中标题（沿用建树图） | +| ① | 淡绿系 · ② 淡橙红系（新增，饱和度对齐现有 pastel） · ③ 淡紫系（复用 VLM 紫） · ④ 淡蓝系 · ⑤ 白卡+badge | +| 直接复用 | 视频缩略图（candidate_a_t*）、L1/L2/L3 badge、Scene/Event/Frame Card 组件、VLM 紫块、箭头/chevron 样式 | +| 字体 | 与建树图一致（Inter）；标题 22px / 正文 15-16px / 标注 12px 灰 | +| 图层组织 | 顶层 Frame 命名 `Main Figure — Self-Evolving Loop`，五区域各一个子 Group，便于后续人工微调 | + +## 7. 术语出处（代码核实） + +| 图上术语 | 来源 | +|---|---| +| gate 四出口 accept_confirmed / accept_provisional / reject×3 / continue | `core/evolution/gate.py:57-110` | +| 归因瀑布 extraction/search/reasoning/mixed；defect vs lapse | `core/evolution/diagnose.py:910-997` | +| D1-D5 五维聚合 | `core/evolution/diagnose.py:1095-1307` | +| 信息阶梯冷启动 2:1、信息量排序 | `app/harness/gate_ladder.py:57-117` | +| rank-and-clip、protected spans、appendix/momentum 区 | `core/evolution/evolve.py:186-594`、`patch.py` | +| agent 工具五件套 | `app/search/tools.py:33-73` | +| 版本目录 workspace/skills/v{N} | `app/harness/store.py:28-136` | + +## 8. 验收标准 + +1. 图在 Figma 中为独立顶层 Frame，可整体导出 PNG/SVG，缩印 180mm 宽时最小文字（12px 标注）仍可辨认。 +2. 五区域 + 对照条齐全，闭环箭头（⑤→① read、④→⑤ accept）无歧义。 +3. 全部术语与 §7 代码核实结果一致；无熔断/缓存/遥测等工程元素。 +4. 风格与同文件建树图肉眼一致（配色、字体、面板语言、卡片组件）。 +5. 不改动/移动建树图的任何现有图层。 + +## 9. 被拒绝的备选方案 + +| 方案 | 拒绝原因 | +|---|---| +| B 上下双层 S 形回路 | PyTorch 对照条无法与面板逐段对齐，退化为角标 | +| C 中心辐射环形 | 2.3:1 宽幅下横向空间浪费大，机制细节难展开，与建树图直线叙事不一致 | +| 独立进化时间轴 | 占版面，与对照条拥挤；由 Store 版本堆叠 + ×N epochs 替代 | diff --git a/research-wiki/designs/paper-main-figure.md b/research-wiki/designs/paper-main-figure.md new file mode 100644 index 0000000..561024f --- /dev/null +++ b/research-wiki/designs/paper-main-figure.md @@ -0,0 +1,17 @@ +--- +type: design +node_id: design:paper-main-figure +title: "论文主图：Self-Evolving Search Agent 推理训练闭环" +date: 2026-07-09 +--- + +# 论文主图：Self-Evolving Search Agent 推理训练闭环 + +完整设计文档见 [2026-07-09-paper-main-figure-design.md](2026-07-09-paper-main-figure-design.md)。 + +**选定方案**：水平流水线 + 底部参数回流闭环（输入 → Inference → Diagnose → Evolve → CE-Gate → Store 回流），底部 PyTorch 双行对照条逐段对齐；示例贯穿延续建树图的 Video-MME 天文台视频；绘制于同一 Figma 文件以复用素材。 + +**关键决策理由**：阅读顺序 = 数据流，与建树图左→右语言一致；对照条只有在单行流水线下才能逐段对齐。CE-Gate 位于 Evolve 之后（与 `core/evolution/gate.py` 核实），信息阶梯为 gate 供题序。 + +**被拒绝方案**：上下双层 S 形回路（对照条无法对齐）、中心辐射环形（宽幅空间浪费、细节难展开）、独立进化时间轴（占版面，由版本堆叠暗示替代）。 + diff --git a/research-wiki/graph/edges.json b/research-wiki/graph/edges.json index 4e23f0b..003afb7 100644 --- a/research-wiki/graph/edges.json +++ b/research-wiki/graph/edges.json @@ -55,6 +55,11 @@ "id": "plan:tree-repair-resilience", "label": "建树修复管线三项改造实现计划", "type": "plan" + }, + { + "id": "design:paper-main-figure", + "label": "论文主图：Self-Evolving Search Agent 推理训练闭环", + "type": "design" } ], "links": [ diff --git a/research-wiki/index.md b/research-wiki/index.md index b25d16f..4256433 100644 --- a/research-wiki/index.md +++ b/research-wiki/index.md @@ -1,17 +1,19 @@ # Research Wiki 索引 -> 自动生成，更新时间：2026-07-09 04:08 UTC +> 自动生成，更新时间：2026-07-09 04:38 UTC -## design (9) +## design (11) - [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design` - [2026-07-07-app-harness-design](designs/2026-07-07-app-harness-design.md) `design:2026-07-07-app-harness-design` - [2026-07-07-core-evolution-design](designs/2026-07-07-core-evolution-design.md) `design:2026-07-07-core-evolution-design` - [2026-07-08-tree-repair-resilience-design](designs/2026-07-08-tree-repair-resilience-design.md) `design:2026-07-08-tree-repair-resilience-design` +- [2026-07-09-paper-main-figure-design](designs/2026-07-09-paper-main-figure-design.md) `design:2026-07-09-paper-main-figure-design` - [出题模块迁移设计（question_gen）](designs/2026-07-07-question-gen-design.md) `design:2026-07-07-question-gen-design` - [建树修复管线：熔断根因修复 + 断点续跑 + 并发改造](designs/tree-repair-resilience.md) `design:tree-repair-resilience` - [建树模块竖切设计：数据结构 + 建树 + 修复 + 迁移](designs/2026-07-07-tree-module-design.md) `design:2026-07-07-tree-module-design` - [建树模块竖切设计：数据结构 + 建树 + 修复 + 迁移](designs/tree-module-vertical-slice.md) `design:tree-module-vertical-slice` - [搜索 Agent 装配层设计（app/search/）](designs/2026-07-07-search-module-design.md) `design:2026-07-07-search-module-design` +- [论文主图：Self-Evolving Search Agent 推理训练闭环](designs/paper-main-figure.md) `design:paper-main-figure` ## plan (13) - [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm` diff --git a/research-wiki/log.md b/research-wiki/log.md index b8ee52f..8526a8e 100644 --- a/research-wiki/log.md +++ b/research-wiki/log.md @@ -27,3 +27,5 @@ - [2026-07-09 04:08 UTC] 新增 plan: 建树修复管线三项改造实现计划 (plan:tree-repair-resilience) - [2026-07-09 04:08 UTC] 新增边: plan:tree-repair-resilience --implements--> design:tree-repair-resilience - [2026-07-09 04:08 UTC] 重建索引: 22 篇页面 +- [2026-07-09 04:38 UTC] 新增 design: 论文主图：Self-Evolving Search Agent 推理训练闭环 (design:paper-main-figure) +- [2026-07-09 04:38 UTC] 重建索引: 24 篇页面 From 86573735aab524f568806aab092912d98245e2ca Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 00:46:08 -0400 Subject: [PATCH 06/21] docs(wiki): apply Codex review fixes to main-figure design --- .../2026-07-09-paper-main-figure-design.md | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/research-wiki/designs/2026-07-09-paper-main-figure-design.md b/research-wiki/designs/2026-07-09-paper-main-figure-design.md index 0d6859b..7d3f9c3 100644 --- a/research-wiki/designs/2026-07-09-paper-main-figure-design.md +++ b/research-wiki/designs/2026-07-09-paper-main-figure-design.md @@ -4,7 +4,7 @@ ## 1. 目标与定位 -- **用途**：论文（EMNLP 2026 方向）主图，展示推理训练部分的自进化闭环；不含建树与新题生成（建树已有独立图，位于同一 Figma 文件）。 +- **用途**：论文主图（目标会议存在文档分歧：CLAUDE.md=AAAI 2026，ARCHITECTURE.md 与记忆=EMNLP 2026，以用户最终决定为准，不影响本图设计），展示推理训练部分的自进化闭环；不含建树与新题生成（建树已有独立图，位于同一 Figma 文件）。 - **核心叙事**：搜索 Agent 通过 推理→诊断→进化→门控 闭环自我改进；**Frozen LLM, trainable harness**——被"训练"的不是模型权重，而是版本化的 Skills+Prompts。 - **差异化**：AVP/DVD 等相关工作画的是推理期内环（agent 怎么搜视频）；本图内环只是一个面板，**训练期外环是主角**。 - **审稿人一句话记忆点**：这是一个不动模型权重的 PyTorch 式训练循环。 @@ -24,6 +24,8 @@ ## 3. 布局 +> 下方 ASCII 草图为中文说明稿，仅示意区域关系；最终图层文字一律采用 §4/§5 的英文术语。 + ``` ┌──────────────────────────────────────────────────────────────────┐ │ [输入] ┌──────────────┐ ┌───────────┐ ┌──────────┐ ┌──────┐ │ @@ -50,7 +52,7 @@ ### 4.2 ① INFERENCE（agent-controlled，图上唯一画成循环的面板） - 树环境迷你版：L1/L2/L3 三层色带（绿/蓝/黄，复用建树图配色与缩略图）。 -- Agent 内环轨迹：Thought → `search_similar` → `view_node` → `observe_frame` → `submit_answer`；侧边 read-skill 箭头（来自 ⑤）。 +- Agent 内环轨迹：Thought → `search_similar` → `view_node` → `observe_frame` → `submit_answer`；侧边 `read_skill` 箭头（来自 ⑤，工具名与 `app/search/tools.py:56-58` 一致）。 - 示例结局：answer ✗（答错），输出 trace 流向 ②。 - 面板角标：*agent-controlled*；其余面板角标 *code-controlled*。 @@ -66,11 +68,12 @@ - 示例 patch 片段：*"+ verify event boundary via L2 card before observe_frame"*。 - momentum 机制不单独出现（用户确认），仅隐含于锁条带。 -### 4.5 ④ CE-GATE -- 菱形门 + e-process 小曲线（e 值随题数爬升，越过 `e_confirm` 虚线）。 +### 4.5 ④ Validation · CE-GATE +- 面板标题 **Validation · CE-Gate**：块顺序验证（`validate.py` 配对翻转 W/L）作为输入喂 e-process——图上画为"candidate vs baseline 配对小图 → e 曲线"。 +- e-process 小曲线：e 值随题数爬升，越过 `e_confirm` 虚线。 - 四出口：**accept (confirmed) / accept (provisional) / reject / continue**（代码中三种 reject 在图上合并，用户确认）。 -- 侧挂小组件：信息阶梯（2:1 交错题序图标，标注 *info-max question ladder*）。 -- accept → 写入 Store 新版本；reject → 保基线（细灰箭头）。 +- 侧挂小组件：信息阶梯（2:1 交错题序图标，标注 *info-max question ladder*），尺寸压小避免抢焦点。 +- 视觉层级：accept 主路径线最粗；reject/continue 细灰次级线。 ### 4.6 ⑤ Skills+Prompts Store - v1…vN 卡片堆叠（复用建树图 Event Card 堆叠画法）+ 版本号 badge。 @@ -86,7 +89,7 @@ | ① | `model.forward()` | | ② | `loss.backward()` | | ③ | `optimizer.step()` | -| ④ | `grad clipping` | +| ④ | `grad clipping (validate)`（对应 CLAUDE.md 类比表中"进化 validation = grad clipping"；④ 面板同时含 validate 配对翻转与 CE-Gate 判定） | | ⑤ | `nn.Parameter` | 条带一侧放记忆点标语：*Frozen LLM, trainable harness*。 @@ -98,7 +101,7 @@ | 面板样式 | 白底、细虚线外框、顶部居中标题（沿用建树图） | | ① | 淡绿系 · ② 淡橙红系（新增，饱和度对齐现有 pastel） · ③ 淡紫系（复用 VLM 紫） · ④ 淡蓝系 · ⑤ 白卡+badge | | 直接复用 | 视频缩略图（candidate_a_t*）、L1/L2/L3 badge、Scene/Event/Frame Card 组件、VLM 紫块、箭头/chevron 样式 | -| 字体 | 与建树图一致（Inter）；标题 22px / 正文 15-16px / 标注 12px 灰 | +| 字体 | 与建树图一致（Inter）；标题 24px / 正文 16-18px / 标注最小 15px 灰（2400px 画布缩印 180mm 后 15px ≈ 1.1mm，12px 过小已弃用） | | 图层组织 | 顶层 Frame 命名 `Main Figure — Self-Evolving Loop`，五区域各一个子 Group，便于后续人工微调 | ## 7. 术语出处（代码核实） @@ -107,11 +110,12 @@ |---|---| | gate 四出口 accept_confirmed / accept_provisional / reject×3 / continue | `core/evolution/gate.py:57-110` | | 归因瀑布 extraction/search/reasoning/mixed；defect vs lapse | `core/evolution/diagnose.py:910-997` | -| D1-D5 五维聚合 | `core/evolution/diagnose.py:1095-1307` | +| D1-D5 五维聚合 | `core/evolution/diagnose.py:1095-1307`（D2-D5）、`1551-1563` + `2230` + `2293-2296`（D1 attribution distribution） | | 信息阶梯冷启动 2:1、信息量排序 | `app/harness/gate_ladder.py:57-117` | -| rank-and-clip、protected spans、appendix/momentum 区 | `core/evolution/evolve.py:186-594`、`patch.py` | -| agent 工具五件套 | `app/search/tools.py:33-73` | -| 版本目录 workspace/skills/v{N} | `app/harness/store.py:28-136` | +| 块顺序验证配对翻转 W/L | `core/evolution/validate.py:12-69` | +| rank-and-clip、protected spans、appendix/momentum 区 | `core/evolution/evolve.py:186-594`、`core/evolution/patch.py:11-56` | +| agent 工具五件套（含 `read_skill`） | `app/search/tools.py:33-73` | +| 版本目录 Store `store/skills/v{N}` / workspace 本地拷贝 | `app/harness/store.py:28-136`、`app/harness/workspace.py:103-108,150-152` | ## 8. 验收标准 From 8c383354c59d2ee5b39ded3fad0460257fbdc042 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 00:54:27 -0400 Subject: [PATCH 07/21] =?UTF-8?q?fix(detector):=20=E7=A7=BB=E9=99=A4=20ong?= =?UTF-8?q?oing=5Factions/visible=5Fentities=20=E7=A9=BA=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E6=A3=80=E6=B5=8B=20+=20=E6=96=B0=E5=A2=9E=20repair=20sh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 抽检确认这两个字段为空是合法内容状态（静物/黑帧/模糊帧）， VLM 重修也修不好，保留会导致断点续跑死循环。 L3 空字段检测缩减为 frame_summary + spatial_layout。 --- app/tree/repair/detector.py | 9 +++------ scripts/repair_trees.sh | 27 +++++++++++++++++++++++++++ tests/unit/test_repair_detector.py | 16 ++++++++-------- 3 files changed, 38 insertions(+), 14 deletions(-) create mode 100755 scripts/repair_trees.sh diff --git a/app/tree/repair/detector.py b/app/tree/repair/detector.py index aba8bb6..169cfae 100644 --- a/app/tree/repair/detector.py +++ b/app/tree/repair/detector.py @@ -40,7 +40,8 @@ def detect_issues( """扫描树，返回所有问题节点列表。 检查项: - - L3: card 必填字段为空（frame_summary / visible_entities / ongoing_actions / spatial_layout） + - L3: card 必填字段为空（frame_summary / spatial_layout） + - 注: visible_entities / ongoing_actions 为空是合法状态（静物/黑帧），不纳入检测 - L3: frame_path 对应文件不存在（需提供 frames_dir） - L2: event_description 为空 - L2/L1: children 列表为空 @@ -108,14 +109,10 @@ def detect_issues( continue for l3 in l2.children: - # L3: 各必填字段不为空 + # L3: 核心必填字段不为空（visible_entities/ongoing_actions 为空是合法状态） empty_fields: list[str] = [] if not l3.card.frame_summary: empty_fields.append("frame_summary") - if not l3.card.visible_entities: - empty_fields.append("visible_entities") - if not l3.card.ongoing_actions: - empty_fields.append("ongoing_actions") if not l3.card.spatial_layout: empty_fields.append("spatial_layout") if empty_fields: diff --git a/scripts/repair_trees.sh b/scripts/repair_trees.sh new file mode 100755 index 0000000..2f20f37 --- /dev/null +++ b/scripts/repair_trees.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# 树修复管线：检测 + VLM 重生成 + 校验 + Q&A 补全 +# +# 用法: +# bash scripts/repair_trees.sh # 默认 16 并发续跑 +# bash scripts/repair_trees.sh --dry-run # 仅检测不修复 +# bash scripts/repair_trees.sh --reaggregate-all # 强制全量重聚合 +# CONCURRENCY=8 bash scripts/repair_trees.sh # 自定义并发数 +# +# 日志输出: +# stderr → 终端实时显示 +# logs/repair_trees.log → 完整日志（自动 rotation 50MB） +# logs/repair_telemetry.db → LLM/VLM 调用遥测 +# logs/repair_progress.json → 断点续跑进度 + +set -euo pipefail + +CONCURRENCY="${CONCURRENCY:-16}" + +conda activate Video-Tree-TRM + +python tools/repair_trees.py \ + --videos-dir store/videos \ + --srt-dir data/Video-MME/subtitle \ + --questions-dir store/questions/benchmarks/Video-MME \ + --concurrency "$CONCURRENCY" \ + "$@" diff --git a/tests/unit/test_repair_detector.py b/tests/unit/test_repair_detector.py index 1780255..e81b347 100644 --- a/tests/unit/test_repair_detector.py +++ b/tests/unit/test_repair_detector.py @@ -119,25 +119,25 @@ class TestDetectIssues: issues = detect_issues(index) assert any(i.issue_type == "no_children" and i.level == 1 for i in issues) - def test_empty_visible_entities(self) -> None: - """visible_entities 为空也触发 empty_field。""" + def test_empty_visible_entities_not_flagged(self) -> None: + """visible_entities 为空是合法状态（静物/黑帧），不触发 empty_field。""" card = L3Card("正常描述", [], ["动作"], [], "居中", {}) l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0) l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3]) l1 = L1Node(id="l1_0", card=_card_l1(), time_range=(0.0, 10.0), children=[l2]) index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1]) issues = detect_issues(index) - assert any(i.issue_type == "empty_field" and "visible_entities" in i.details for i in issues) + assert not any(i.issue_type == "empty_field" for i in issues) - def test_empty_ongoing_actions(self) -> None: - """ongoing_actions 为空也触发 empty_field。""" + def test_empty_ongoing_actions_not_flagged(self) -> None: + """ongoing_actions 为空是合法状态（静物/黑帧），不触发 empty_field。""" card = L3Card("正常描述", ["实体"], [], [], "居中", {}) l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0) l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3]) l1 = L1Node(id="l1_0", card=_card_l1(), time_range=(0.0, 10.0), children=[l2]) index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1]) issues = detect_issues(index) - assert any(i.issue_type == "empty_field" and "ongoing_actions" in i.details for i in issues) + assert not any(i.issue_type == "empty_field" for i in issues) def test_empty_spatial_layout(self) -> None: """spatial_layout 为空也触发 empty_field。""" @@ -150,7 +150,7 @@ class TestDetectIssues: assert any(i.issue_type == "empty_field" and "spatial_layout" in i.details for i in issues) def test_multiple_empty_fields_single_issue(self) -> None: - """多个字段同时为空只产生一个 issue，details 列出所有空字段。""" + """多个必填字段同时为空只产生一个 issue，details 列出所有空字段。""" card = L3Card("", [], [], [], "", {}) l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0) l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3]) @@ -159,7 +159,7 @@ class TestDetectIssues: issues = [i for i in detect_issues(index) if i.issue_type == "empty_field"] assert len(issues) == 1 assert "frame_summary" in issues[0].details - assert "visible_entities" in issues[0].details + assert "spatial_layout" in issues[0].details def test_time_gap(self) -> None: l3_a = L3Node(id="l1_0_l2_0_l3_0", card=_card_l3(), timestamp=1.0) From 0b48b889e0259fc7e58305560ba4bd425edf1d81 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 05:05:57 -0400 Subject: [PATCH 08/21] =?UTF-8?q?docs(wiki):=20=E8=B5=9B=E9=A2=98=E7=94=9F?= =?UTF-8?q?=E6=88=90=E5=B7=A5=E5=85=B7=E8=AE=BE=E8=AE=A1=20+=20=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=E8=AE=A1=E5=88=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit design: synthesizer + factory + CLI 三模块架构 plan: 9 个 Task（前置修复 + synthesizer 4 步 + factory + CLI generate/calibrate + re-export） calibrate: Fisher exact test 组合判定替代固定阈值 Co-Authored-By: Claude Opus 4.6 (1M context) --- .../2026-07-09-question-gen-synth-design.md | 477 +++++++ research-wiki/designs/question-gen-synth.md | 9 + research-wiki/graph/edges.json | 17 + research-wiki/index.md | 10 +- research-wiki/log.md | 4 + .../plans/2026-07-09-question-gen-synth.md | 1183 +++++++++++++++++ research-wiki/plans/question-gen-synth.md | 9 + 7 files changed, 1706 insertions(+), 3 deletions(-) create mode 100644 research-wiki/designs/2026-07-09-question-gen-synth-design.md create mode 100644 research-wiki/designs/question-gen-synth.md create mode 100644 research-wiki/plans/2026-07-09-question-gen-synth.md create mode 100644 research-wiki/plans/question-gen-synth.md diff --git a/research-wiki/designs/2026-07-09-question-gen-synth-design.md b/research-wiki/designs/2026-07-09-question-gen-synth-design.md new file mode 100644 index 0000000..de6b39d --- /dev/null +++ b/research-wiki/designs/2026-07-09-question-gen-synth-design.md @@ -0,0 +1,477 @@ +--- +id: question-gen-synth +title: 赛题生成工具设计（Question Generation Synthesis） +type: design +created: 2026-07-09 +status: draft +--- + +# 赛题生成工具设计 + +## 1. 目标与动机 + +让视频树自行生成与 Video-MME 原始赛题风格、难度近似的四选一选择题，用于自进化训练循环的 DataLoader。原始 900 道 benchmark 题保留为 held-out 最终评测集，避免"直接拿答案调"的审稿质疑。 + +**角色定位**：生成题 = 训练集，原始题 = 测试集。进化循环的改进效果最终由原始 benchmark 验证泛化能力。 + +**训练 vs 论文评测的区分**：训练循环全程使用生成题（三池切分——诊断池/验证池/test 池——均来自生成题），论文报告的 held-out 泛化指标是训练结束后，用最终 best 版本对原始 benchmark 全量 900 题单独跑推理得到的结果。两步分离，Runner 代码无需改动。 + +## 2. 模块结构与职责边界 + +### 2.1 文件布局 + +``` +app/question_gen/ +├── __init__.py ← 已有：re-export loader API +├── loader.py ← 已有：load_benchmark + stratified_sample +└── synthesizer.py ← 新增①：出题核心逻辑 + +app/harness/ +└── factory.py ← 新增②：推理依赖组装（wiring） + +tools/generate_questions.py ← 新增③：CLI 壳（generate + calibrate） +``` + +### 2.2 职责切分 + +| 模块 | 职责 | 消费者 | +|------|------|--------| +| `synthesizer.py` | 题型-层级映射、锚节点采样、prompt 构造（few-shot）、embedding 去重、单题生成编排 | `tools/generate_questions.py` | +| `factory.py` | 给定 store 路径 + config → 组装 LLM/VLM/Embedding/SearchToolDispatcher/PromptManager 全套推理依赖 | `tools/generate_questions.py`（校准）、未来 `main.py`、Runner | +| `tools/generate_questions.py` | CLI 参数解析、并发编排（Semaphore）、进度日志、JSON 输出 | 用户直接运行 | + +### 2.3 依赖方向 + +```mermaid +flowchart LR + TOOLS["tools/generate_questions.py"] --> SYN["app/question_gen/synthesizer"] + TOOLS --> FAC["app/harness/factory"] + TOOLS --> ADP["adapters/*"] + FAC --> SEARCH["app/search/*"] + FAC --> ENV["app/tree/environment"] + FAC --> ADP + SYN --> PROTO["core/protocols (VLMProvider, EmbeddingProvider via DI)"] + SYN --> TYPES["core/types (GeneratedQuestion)"] + SYN --> IDX["app/tree/index (TreeIndex)"] +``` + +全部合规——外层→内层，`core/` 不依赖任何外层。 + +### 2.4 与 QuestionGenerator Protocol 的关系 + +`app/ports.py` 已预留 `QuestionGenerator` Protocol。本设计**不实现该 Protocol**——出题是一次性离线工具而非运行时能力，Runner 不需要运行时出题。`synthesizer.py` 的函数式接口（`generate_one` 等纯函数 + async 编排）比 Protocol class 更适合工具脚本场景。`QuestionGenerator` Protocol 保留但标记为"预留，当前无实现"，不删除——若未来需要运行时出题可基于 synthesizer 的纯函数包装实现。 + +### 2.5 方案选择与否决 + +| 方案 | 否决理由 | +|------|---------| +| A: 单体脚本（全部逻辑放 `tools/`） | 业务逻辑（题型映射、采样、prompt、去重）混在 CLI 编排中，不可独立测试；不匹配 repair 管线的 app/ + tools/ 分层惯例 | +| B: Protocol 实现 + 脚本编排（`adapters/` 实现 `QuestionGenerator`） | adapter 层语义是外部服务接口，出题逻辑是应用层业务规则，放 adapter 层语义不匹配 | +| **C: app/ 业务逻辑 + tools/ CLI 壳（采用）** | 与 repair 管线结构一致，Clean Architecture 依赖方向合规，业务逻辑可独立测试 | + +## 3. synthesizer.py 核心设计 + +### 3.1 题型-层级映射 + +模块级常量，沿用 TRM4 设计文档的映射表： + +| 锚定层级 | 题型 | 帧图 | 文本上下文 | 帧数 | +|---------|------|------|-----------|------| +| L3 | Object Recognition | 必须 | frame_summary | 1 | +| L3 | Attribute Perception | 必须 | frame_summary | 1 | +| L3 | OCR Problems | 必须 | frame_summary | 1 | +| L3 | Spatial Reasoning | 必须 | frame_summary + spatial_layout | 1 | +| L3 | Spatial Perception | 必须 | frame_summary | 1 | +| L2 | Action Recognition | 必须 | 事件 card | 2-3（子帧均匀采样） | +| L2 | Action Reasoning | 必须 | 事件 card | 2-3 | +| L2 | Counting Problem | 必须 | 事件 card | 2-3 | +| L2 | Temporal Perception | 可选 | 事件 card + time_range | 0-1 | +| L1 | Temporal Reasoning | 必须 | 根 card + 多个 L2 card（≥3） | 每 L2 取 1 张代表帧 | +| L1 | Information Synopsis | 必须 | 根 card + 全部 L2 card | 每 L2 取 1 张代表帧 | +| L1-L2 | Object Reasoning | 必须 | 2-3 个 L2 card | 每 L2 取 1 张代表帧 | + +节点采样：每道题从全部视频树中随机选一棵，在对应层级随机选一个锚节点。同视频同题型不重复。L1 题型使用多个 L2 子节点联合输入时，按时间顺序组织节点，保持叙事连贯性。 + +### 3.2 AnchorContext 数据结构 + +```python +@dataclass(frozen=True) +class AnchorContext: + """锚节点上下文——生成单道题所需的全部素材。""" + node_id: str # 锚节点 ID + card_text: str # 锚节点 card 序列化文本 + frame_paths: list[str] # 帧图片路径 + subtitle: str # 对应字幕（可空） + distractor_texts: list[str] # 同视频其他节点摘要（供 VLM 生成干扰项） +``` + +### 3.3 核心函数签名 + +```python +# 纯函数：从树中采样锚节点 + 帧 + 上下文 +def sample_anchor( + tree: TreeIndex, + task_type: str, + used_node_ids: set[str], + rng: random.Random, +) -> AnchorContext + +# 纯函数：组装 VLM prompt（system + user，含 few-shot exemplar） +def build_generation_prompt( + task_type: str, + anchor: AnchorContext, + exemplars: list[GeneratedQuestion], +) -> tuple[list[dict], list[str]] + # 返回：(messages, image_paths) — 直接喂给 VLMProvider + +# 纯函数：解析 VLM 返回的 JSON → 部分字段字典 +# source_nodes 和 difficulty 由 generate_one 在 parse 后用 anchor 信息补齐 +def parse_vlm_response( + raw: str, + video_id: str, + task_type: str, + seq: int, +) -> dict + # 返回：{"question_id", "question", "options", "answer"} 字典 + # 调用方补齐 source_nodes/difficulty 后构造 GeneratedQuestion + +# 纯函数：embedding 去重判定 +def is_duplicate( + question_text: str, + pool_embeddings: np.ndarray, + embed_fn: Callable[[str | list[str]], np.ndarray], + threshold: float, +) -> bool + +# 异步编排：生成单道题（含重试 + 去重循环） +async def generate_one( + vlm: VLMProvider, + embed_fn: Callable[[str | list[str]], np.ndarray], + tree: TreeIndex, + video_id: str, + task_type: str, + seq: int, + *, + exemplars: list[GeneratedQuestion], + pool_embeddings: np.ndarray, + used_node_ids: set[str], + max_retries: int, + similarity_threshold: float, + rng: random.Random, + session_id: str, +) -> GeneratedQuestion | None +``` + +**设计要点**： +- 纯函数（sample_anchor、build_generation_prompt、parse_vlm_response、is_duplicate）可独立单测，不需要 VLM +- `generate_one` 是唯一异步函数，接收 `VLMProvider` 通过 DI +- 干扰项来自 `AnchorContext.distractor_texts`——同视频其他节点的真实信息 + +### 3.4 few-shot exemplar 选择 + +生成 prompt 包含 2-3 道同题型的原始 benchmark 题作示例，对齐风格和难度。 + +选择策略： +- 每题型取 `min(3, 该题型 benchmark 总量)` 道 +- 按 seed 随机采样 + 跨视频去重（避免 exemplar 全来自同一视频） +- exemplar 是只读引用，不从 benchmark 评测集中移除 + +### 3.5 prompt 结构 + +``` +System: 视频理解题目生成器，根据视频树节点内容和帧图生成 {task_type} 四选一题。 + +[2-3 道该题型原始 benchmark 题作示例] + +约束： +- 问题必须基于给定节点内容，不能靠常识推断 +- 干扰项来自同视频其他节点的真实信息（非凭空捏造） +- 难度和问法风格与示例一致 + +User: [锚节点 card + 字幕 + 帧图] + [同视频其他节点摘要，供干扰项素材] +``` + +### 3.6 去重机制 + +用 `EmbeddingProvider`（nomic-embed-text-v1.5）对 question 文本做 embedding，余弦相似度检查： + +| 检查对 | 阈值 | 处理 | +|--------|------|------| +| 生成题 vs 原始 benchmark 同题型题 | ≥ similarity_threshold | 丢弃，换节点重试 | +| 生成题 vs 已生成的同题型题 | ≥ similarity_threshold | 丢弃，换节点重试 | + +维护 embedding 池（原始题 + 已通过的生成题），每生成一道新题即时查重。单题最多重试 `max_retries` 次。某题型连续耗尽重试配额时，脚本报错退出并输出已完成/未完成的题型统计，不静默少题。 + +**并发去重安全**：embedding 池的"检查 + 添加"必须是原子操作。并发 `generate_one` 任务成功后，通过单线程汇总点（asyncio.Queue 或 await 后顺序提交）更新 embedding 池 + 写 JSON + 更新 progress，避免竞态导致相似题同时通过。 + +## 4. factory.py 推理依赖组装 + +### 4.1 解决的问题 + +目前 `Runner._make_tool_dispatch_fn()` 和 `_make_prompt_builder()` 都是 `raise NotImplementedError`，设计为"由 main.py 注入"。组装逻辑涉及 adapter 实例化 + app 组件串联，应提取为可复用的 factory 函数，避免在每个调用方（tools/ 脚本、未来 main.py）重复 wiring。 + +### 4.2 核心接口 + +```python +@dataclass(frozen=True) +class InferenceDeps: + """跑一次推理所需的全套依赖（不含 HarnessLog，其生命周期由调用方管理）。""" + llm: LLMProvider + tool_dispatch_fn: Callable # SearchToolDispatcher.dispatch + prompt_builder: Callable # PromptManager 的偏函数 + +def build_inference_deps( + *, + store_dir: Path, + video_id: str, + prompts_dir: Path, + skills_dir: Path | None, + skill_mode: str, + embed_provider: EmbeddingProvider, + llm: LLMProvider, + vlm: VLMProvider, + ocr: OCRProvider | None, + verify_vision: bool, + anchor: bool, + assemble_mode: str, +) -> InferenceDeps +``` + +### 4.3 内部流程 + +``` +build_inference_deps() + ├── 加载 TreeIndex（store_dir/videos/{video_id}/tree.json） + ├── 构建 TreeEnvironment（index=tree, frames_dir=videos/{video_id}/frames） + ├── 构建 SkillRegistry（skills_dir，可选） + ├── 构建 SearchToolDispatcher（env, tool_llm, vlm, ocr, prompts_dir, + │ skills, embed_fn, verify_vision, anchor, assemble_mode） + ├── 构建 PromptManager（prompts_dir）→ 偏函数化 prompt_builder（绑定 skill_mode） + └── 返回 InferenceDeps +``` + +注意：`HarnessLog` 不放入 `InferenceDeps`——其生命周期由调用方通过 `with HarnessLog(...) as log` 管理，作为参数传给 `run_inference`。 + +### 4.4 消费者 + +| 消费者 | 用法 | +|--------|------| +| `tools/generate_questions.py` calibrate | 按 video_id 分组题目，对每组调 `build_inference_deps` 构建对应视频树的依赖 → 分组 `run_inference` | +| 未来 `main.py --mode infer` | CLI 参数映射到 factory 参数 | +| `Runner` | `_make_tool_dispatch_fn` / `_make_prompt_builder` 改为委托 factory | + +### 4.5 设计约束 + +- factory 只做**组装**，不持有状态——每次调用返回独立的 `InferenceDeps` +- adapter 实例（LLM/VLM/Embedding）由调用方创建并传入，factory 不管 adapter 生命周期 +- 调用方自由决定 adapter 的复用策略（共享 vs 按需创建） + +## 5. tools/generate_questions.py CLI 设计 + +### 5.1 子命令 + +```bash +# 生成 +python tools/generate_questions.py generate \ + --store-dir store \ + --output-dir store/questions/generated/Video-MME \ + --per-type 20 \ + --similarity-threshold 0.85 \ + --max-retries 3 \ + --concurrency 8 \ + --seed 42 + +# 校准（生成题 vs benchmark 基线对比） +python tools/generate_questions.py calibrate \ + --generated-dir store/questions/generated/Video-MME \ + --benchmark-dir store/questions/benchmarks/Video-MME \ + --store-dir store \ + --db-path results/calibrate.db \ + --prompts-dir store/prompts \ + --concurrency 4 \ + --max-steps 15 \ + --skill-mode auto \ + --tolerance 0.10 \ + --alpha 0.05 \ + --baseline-db <可选，已有基线 DB 路径> \ + --baseline-run-id <可选，已有基线 run_id> +``` + +除 baseline 复用参数外均必传，无默认值（CLAUDE.md §4.5）。`--baseline-db` + `--baseline-run-id` 可选但必须成对出现：有则从 DB 读 benchmark 基线，无则自动跑一次 benchmark 推理。 + +### 5.2 generate 流程 + +``` +1. 加载 300 棵树的 video_id 列表 +2. 加载 benchmark 题目（作为 few-shot exemplar 来源） +3. 初始化 embedding 池（benchmark 题 question text → embedding） +4. 实例化 GovernedVLMClient + EmbeddingProvider +5. 检查断点续跑文件（progress.json） +6. 对 12 题型 × per_type： + ├── 跳过已完成的（断点续跑） + ├── 随机选视频 + 锚节点（同视频同题型不重复） + ├── asyncio.Semaphore(concurrency) 并发调 generate_one + ├── 成功 → 加入 embedding 池 + 追加到结果 + 更新 progress + └── 连续耗尽重试 → 报错退出，输出已完成/未完成统计 +7. 按 video_id 分组写入 JSON +8. 全部完成后删除 progress.json +``` + +### 5.3 calibrate 流程 + +``` +1. load_benchmark 加载生成题和 benchmark 题 +2. 获取 benchmark 基线： + ├── 有 --baseline-db + --baseline-run-id → 从 DB 读 per_task_type accuracy + └── 没有 → 按 video_id 分组 benchmark 题 → 每组 build_inference_deps + → 分组 run_inference → 汇总存 DB +3. 按 video_id 分组生成题 → 每组 build_inference_deps → 分组 run_inference + （每组使用对应视频的 TreeEnvironment，避免跨视频树错用） +4. 汇总两组 per_task_type accuracy，对比（Fisher exact test） +5. 输出对比表 + 判定结果 +6. 存在 FAIL → 退出码 1 +``` + +### 5.4 tools/ 脚本职责边界 + +脚本**只做**：argparse、adapter 实例化（读 `.env`）、Semaphore 并发、进度日志（loguru）、JSON 写入、calibrate 时调 factory + run_inference。 + +脚本**不做**：prompt 构造、节点采样、去重判定（synthesizer.py）、依赖组装逻辑（factory.py）。 + +## 6. 校准统计方法 + +### 6.1 问题 + +benchmark 题型分布极不均匀（Spatial Perception 仅 3 道 vs Object Reasoning 240 道），固定 10% 阈值对小样本题型会产生误判——单题翻转即 33% 波动。 + +### 6.2 组合判定：Fisher exact test + effect size + +用 `scipy.stats.fisher_exact` 对每个题型构造 2×2 列联表： + +| | 答对 | 答错 | +|--|------|------| +| Benchmark | a | b | +| Generated | c | d | + +判定规则： + +| \|Δ\| > tolerance | p < α | 判定 | 含义 | +|---|---|---|---| +| ✗ | — | **PASS** | 差异在容忍范围内 | +| ✓ | ✓ | **FAIL** | 差异大且统计显著——生成题难度确实偏了 | +| ✓ | ✗ | **WARN** | 差异大但样本不足以确认——可能是噪声 | + +### 6.3 优势 + +- 不需要 ad-hoc 的 `min_calibrate_size` 参数 +- 小样本题型自动降级为 WARN——Fisher test 的 p-value 天然反映样本量不足 +- CLI 只需两个语义清晰的统计参数：`--tolerance 0.10` + `--alpha 0.05` +- 退出码只看是否存在 FAIL（WARN 不阻塞） + +### 6.4 检测灵敏度与 per_type 的关系 + +| per_type | 可检出的最小差异（大样本 benchmark 侧） | +|----------|---------------------------------------| +| 20 | ~30%（仅极大差异） | +| 50 | ~15%（中等差异） | + +用户可根据需要的检测灵敏度选择 `--per-type`。 + +### 6.5 输出格式 + +``` +题型 | bench | gen | Δ | p-value | 判定 +-------------------|--------|--------|---------|---------|-------- +Spatial Perception | 66.7% | 40.0% | -26.7% | 0.590 | ⚠ WARN +Action Reasoning | 72.2% | 68.0% | -4.2% | 0.712 | ✓ PASS +Object Reasoning | 60.0% | 30.0% | -30.0% | 0.016 | ✗ FAIL +``` + +## 7. 断点续跑 + +生成 240 道题可能中断（VLM 故障、手动 Ctrl-C），沿用项目已有的 progress.json 模式： + +```json +{ + "completed": { + "Action Reasoning": ["gen-xyz-001", "gen-xyz-002"], + "Object Recognition": ["gen-abc-001"] + }, + "output_dir": "store/questions/generated/Video-MME" +} +``` + +- 启动时检查 `{output_dir}/progress.json`，跳过已完成的题 +- **恢复 embedding 池**：从已写出的 `{output_dir}/*.json` 重建已生成题的 embedding + `used_node_ids`，避免续跑后产生重复题 +- 每道题写入 JSON 后立即更新 progress +- 全部完成后删除 progress.json + +## 8. 输出格式 + +输出路径：`store/questions/generated/Video-MME/{video_id}.json` + +```json +[ + { + "question_id": "gen-{video_id}-{seq}", + "task_type": "Action Reasoning", + "question": "...", + "options": ["A. ...", "B. ...", "C. ...", "D. ..."], + "answer": "B", + "source_nodes": ["L1_000_L2_003"], + "difficulty": "medium" + } +] +``` + +与 loader schema 兼容（额外 `source_nodes`/`difficulty` 字段用于溯源），`load_benchmark` 零改动直接加载。 + +**训练集成**：`--questions generated/Video-MME`。 + +## 9. 受影响的既有接口 + +| 接口 | 影响 | 适配 | +|------|------|------| +| `load_benchmark` | 无 | 输出与 loader schema 兼容（额外 source_nodes/difficulty 字段用于溯源） | +| `RunConfig.questions` | 无 | 传 `generated/Video-MME` | +| `build_or_load_pools` | 无 | 三池均来自生成题 | +| `Runner._make_tool_dispatch_fn` | 改造 | 委托 factory.py | +| `Runner._make_prompt_builder` | 改造 | 委托 factory.py | +| `_VIDEO_MME_TASK_TYPE_COUNT` | **前置修复** | 从 11 改为 12（`app/harness/config.py:24`），影响验证池保底下限 | + +## 10. 测试策略 + +### 10.1 synthesizer.py + +| 测试 | 覆盖点 | +|------|--------| +| `test_sample_anchor` | 各层级题型正确采锚、同视频同题型不重复、树节点不足时报错 | +| `test_build_generation_prompt` | messages 结构正确、exemplar 注入、图片路径列表、干扰项素材包含 | +| `test_parse_vlm_response` | 正常解析、格式异常（缺字段/非法 JSON）报错 | +| `test_is_duplicate` | 相似度 ≥ 阈值判重、< 阈值通过、空池不判重 | +| `test_generate_one` | mock VLMProvider，验证重试+去重循环、耗尽重试返回 None | + +### 10.2 factory.py + +| 测试 | 覆盖点 | +|------|--------| +| `test_build_inference_deps` | fake LLM/VLM/Embedding 验证返回各字段非 None、类型正确 | +| `test_missing_tree_file` | tree.json 不存在时报错 | + +### 10.3 tools/generate_questions.py（集成级） + +| 测试 | 覆盖点 | +|------|--------| +| `test_generate_smoke` | mock VLM + 1 棵真实树 + per_type=1，验证 JSON 输出格式 | +| `test_progress_resume` | 中断后重启，跳过已完成题 | +| `test_calibrate_pass_fail` | mock 两组 accuracy，验证 Fisher + tolerance 组合判定 | + +真实 VLM 调用的 integration test 不在此次范围——依赖外部服务，不适合 CI。 + +## 11. 实现约束 + +- 完整类型注解 + 中文 Docstring（CLAUDE.md §4.2） +- 禁用 `print()`，使用 loguru（CLAUDE.md §4.2） +- 脚本放 `tools/`，不被其他模块 import（CLAUDE.md §5） +- 并发模式：`asyncio.Semaphore`，CLI `--concurrency` 指定（沿用项目既有模式） +- 所有 VLM 调用经过 `GovernedLLMClient` 治理栈（CLAUDE.md §4.9） diff --git a/research-wiki/designs/question-gen-synth.md b/research-wiki/designs/question-gen-synth.md new file mode 100644 index 0000000..5616f86 --- /dev/null +++ b/research-wiki/designs/question-gen-synth.md @@ -0,0 +1,9 @@ +--- +type: design +node_id: design:question-gen-synth +title: 赛题生成工具设计 +date: 2026-07-09 +--- + +# 赛题生成工具设计 + diff --git a/research-wiki/graph/edges.json b/research-wiki/graph/edges.json index 003afb7..4662ea2 100644 --- a/research-wiki/graph/edges.json +++ b/research-wiki/graph/edges.json @@ -60,6 +60,16 @@ "id": "design:paper-main-figure", "label": "论文主图：Self-Evolving Search Agent 推理训练闭环", "type": "design" + }, + { + "id": "design:question-gen-synth", + "label": "赛题生成工具设计", + "type": "design" + }, + { + "id": "plan:question-gen-synth", + "label": "赛题生成工具实现计划", + "type": "plan" } ], "links": [ @@ -104,6 +114,13 @@ "relation": "implements", "evidence": "实现设计文档的三项改造：遥测加固+断点续跑+并发", "added": "2026-07-09T04:08:15.312470+00:00" + }, + { + "source": "plan:question-gen-synth", + "target": "design:question-gen-synth", + "relation": "implements", + "evidence": "计划实现设计文档中定义的 synthesizer + factory + CLI 三模块", + "added": "2026-07-09T09:05:43.697644+00:00" } ] } \ No newline at end of file diff --git a/research-wiki/index.md b/research-wiki/index.md index 4256433..5edcdaf 100644 --- a/research-wiki/index.md +++ b/research-wiki/index.md @@ -1,8 +1,8 @@ # Research Wiki 索引 -> 自动生成，更新时间：2026-07-09 04:38 UTC +> 自动生成，更新时间：2026-07-09 09:05 UTC -## design (11) +## design (13) - [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design` - [2026-07-07-app-harness-design](designs/2026-07-07-app-harness-design.md) `design:2026-07-07-app-harness-design` - [2026-07-07-core-evolution-design](designs/2026-07-07-core-evolution-design.md) `design:2026-07-07-core-evolution-design` @@ -14,13 +14,16 @@ - [建树模块竖切设计：数据结构 + 建树 + 修复 + 迁移](designs/tree-module-vertical-slice.md) `design:tree-module-vertical-slice` - [搜索 Agent 装配层设计（app/search/）](designs/2026-07-07-search-module-design.md) `design:2026-07-07-search-module-design` - [论文主图：Self-Evolving Search Agent 推理训练闭环](designs/paper-main-figure.md) `design:paper-main-figure` +- [赛题生成工具设计](designs/question-gen-synth.md) `design:question-gen-synth` +- [赛题生成工具设计（Question Generation Synthesis）](designs/2026-07-09-question-gen-synth-design.md) `design:2026-07-09-question-gen-synth-design` -## plan (13) +## plan (15) - [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm` - [2026-07-07-app-harness](plans/2026-07-07-app-harness.md) `plan:2026-07-07-app-harness` - [2026-07-07-core-evolution](plans/2026-07-07-core-evolution.md) `plan:2026-07-07-core-evolution` - [2026-07-07-question-gen](plans/2026-07-07-question-gen.md) `plan:2026-07-07-question-gen` - [2026-07-07-tree-module-vertical-slice](plans/2026-07-07-tree-module-vertical-slice.md) `plan:2026-07-07-tree-module-vertical-slice` +- [2026-07-09-question-gen-synth](plans/2026-07-09-question-gen-synth.md) `plan:2026-07-09-question-gen-synth` - [2026-07-09-tree-repair-resilience](plans/2026-07-09-tree-repair-resilience.md) `plan:2026-07-09-tree-repair-resilience` - [app/harness/ 训练循环编排层实现计划](plans/app-harness.md) `plan:app-harness` - [app/search/ 搜索 Agent 装配层实现计划](plans/2026-07-07-search-module.md) `plan:2026-07-07-search-module` @@ -28,4 +31,5 @@ - [question_gen 模块实现计划](plans/question-gen.md) `plan:question-gen` - [建树修复管线三项改造实现计划](plans/tree-repair-resilience.md) `plan:tree-repair-resilience` - [建树模块竖切实现计划](plans/tree-module-vertical-slice.md) `plan:tree-module-vertical-slice` +- [赛题生成工具实现计划](plans/question-gen-synth.md) `plan:question-gen-synth` - [项目基础设施初始化计划](plans/infrastructure-setup.md) `plan:infrastructure-setup` diff --git a/research-wiki/log.md b/research-wiki/log.md index 8526a8e..3201f2c 100644 --- a/research-wiki/log.md +++ b/research-wiki/log.md @@ -29,3 +29,7 @@ - [2026-07-09 04:08 UTC] 重建索引: 22 篇页面 - [2026-07-09 04:38 UTC] 新增 design: 论文主图：Self-Evolving Search Agent 推理训练闭环 (design:paper-main-figure) - [2026-07-09 04:38 UTC] 重建索引: 24 篇页面 +- [2026-07-09 09:05 UTC] 新增 design: 赛题生成工具设计 (design:question-gen-synth) +- [2026-07-09 09:05 UTC] 新增 plan: 赛题生成工具实现计划 (plan:question-gen-synth) +- [2026-07-09 09:05 UTC] 新增边: plan:question-gen-synth --implements--> design:question-gen-synth +- [2026-07-09 09:05 UTC] 重建索引: 28 篇页面 diff --git a/research-wiki/plans/2026-07-09-question-gen-synth.md b/research-wiki/plans/2026-07-09-question-gen-synth.md new file mode 100644 index 0000000..1491178 --- /dev/null +++ b/research-wiki/plans/2026-07-09-question-gen-synth.md @@ -0,0 +1,1183 @@ +# 赛题生成工具实现计划 + +> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 实现基于视频树的题目合成工具（generate + calibrate），含推理依赖 factory 提取。 + +**Architecture:** `app/question_gen/synthesizer.py` 承载核心业务逻辑（节点采样、prompt 构造、去重），`app/harness/factory.py` 提取推理依赖组装（TreeEnvironment + SearchToolDispatcher + PromptManager），`tools/generate_questions.py` 作为 CLI 壳编排并发和 I/O。 + +**Tech Stack:** Python 3.11, asyncio, GovernedVLMClient, EmbeddingProvider, scipy.stats.fisher_exact, loguru + +**核心算法保真校验：** 本计划不涉及核心算法迁移（13 项均已在先前 PR 完成），保真校验不适用。 + +--- + +## 文件结构总览 + +| 动作 | 文件 | 职责 | +|------|------|------| +| 修改 | `app/harness/config.py:24` | 前置修复 `_VIDEO_MME_TASK_TYPE_COUNT` 11→12 | +| 新建 | `app/question_gen/synthesizer.py` | 出题核心逻辑 | +| 新建 | `app/harness/factory.py` | 推理依赖组装 | +| 新建 | `tools/generate_questions.py` | CLI 壳 | +| 新建 | `tests/unit/test_synthesizer.py` | synthesizer 单测 | +| 新建 | `tests/unit/test_factory.py` | factory 单测 | +| 新建 | `tests/unit/test_generate_questions.py` | CLI 集成测试 | +| 修改 | `app/question_gen/__init__.py` | 追加 synthesizer re-export | + +--- + +### Task 0: 前置修复 _VIDEO_MME_TASK_TYPE_COUNT + +**Files:** +- Modify: `app/harness/config.py:24` +- Test: `tests/unit/test_harness_config.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_config.py` 中追加： + +```python +def test_video_mme_task_type_count_is_12(): + """Video-MME 实际有 12 种题型，常量必须与之一致。""" + from app.harness.config import _VIDEO_MME_TASK_TYPE_COUNT + assert _VIDEO_MME_TASK_TYPE_COUNT == 12 +``` + +- [ ] **Step 2: 运行测试确认失败** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_harness_config.py::test_video_mme_task_type_count_is_12 -v +``` + +预期：FAIL，`assert 11 == 12` + +- [ ] **Step 3: 修改常量** + +`app/harness/config.py:24`：`_VIDEO_MME_TASK_TYPE_COUNT = 11` → `_VIDEO_MME_TASK_TYPE_COUNT = 12` + +- [ ] **Step 4: 运行测试确认通过** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_harness_config.py -v +``` + +预期：全部 PASS + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/config.py tests/unit/test_harness_config.py +git commit -m "fix(config): _VIDEO_MME_TASK_TYPE_COUNT 11→12，Video-MME 实际有 12 种题型" +``` + +--- + +### Task 1: synthesizer.py — AnchorContext + 题型映射常量 + +**Files:** +- Create: `app/question_gen/synthesizer.py` +- Create: `tests/unit/test_synthesizer.py` + +- [ ] **Step 1: 写失败测试 — 题型映射完整性** + +```python +# tests/unit/test_synthesizer.py +from app.question_gen.synthesizer import TASK_TYPE_LEVEL_MAP, AnchorContext + +ALL_12_TYPES = [ + "Object Recognition", "Attribute Perception", "OCR Problems", + "Spatial Reasoning", "Spatial Perception", + "Action Recognition", "Action Reasoning", "Counting Problem", + "Temporal Perception", + "Temporal Reasoning", "Information Synopsis", + "Object Reasoning", +] + +def test_task_type_level_map_covers_all_12_types(): + """映射表必须覆盖全部 12 种 Video-MME 题型。""" + assert set(TASK_TYPE_LEVEL_MAP.keys()) == set(ALL_12_TYPES) + +def test_anchor_context_frozen(): + """AnchorContext 是不可变的。""" + ctx = AnchorContext( + node_id="L3_001", + card_text="test", + frame_paths=["a.jpg"], + subtitle="", + distractor_texts=["other node"], + ) + assert ctx.node_id == "L3_001" +``` + +- [ ] **Step 2: 运行测试确认失败** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_synthesizer.py -v +``` + +预期：ImportError + +- [ ] **Step 3: 实现 AnchorContext + TASK_TYPE_LEVEL_MAP** + +创建 `app/question_gen/synthesizer.py`： + +```python +"""赛题合成核心逻辑 — 节点采样、prompt 构造、去重。 + +纯函数为主，异步编排仅 generate_one。 +通过 DI 接收 VLMProvider / EmbeddingProvider，不 import adapters/。 +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class AnchorContext: + """锚节点上下文——生成单道题所需的全部素材。 + + 属性: + node_id: 锚节点 ID。 + card_text: 锚节点 card 序列化文本。 + frame_paths: 帧图片路径列表。 + subtitle: 对应字幕（可空）。 + distractor_texts: 同视频其他节点摘要（供 VLM 生成干扰项）。 + """ + + node_id: str + card_text: str + frame_paths: list[str] + subtitle: str + distractor_texts: list[str] + + +@dataclass(frozen=True) +class TaskTypeSpec: + """题型的生成规格。 + + 属性: + level: 锚定层级（"L3" / "L2" / "L1" / "L1-L2"）。 + needs_frames: 是否必须提供帧图。 + frame_count: 帧数范围描述（如 "1", "2-3", "0-1"）。 + context_fields: 需要提取的 card 字段元组。 + """ + + level: str + needs_frames: bool + frame_count: str + context_fields: tuple[str, ...] + + +TASK_TYPE_LEVEL_MAP: dict[str, TaskTypeSpec] = { + "Object Recognition": TaskTypeSpec("L3", True, "1", ("frame_summary",)), + "Attribute Perception": TaskTypeSpec("L3", True, "1", ("frame_summary",)), + "OCR Problems": TaskTypeSpec("L3", True, "1", ("frame_summary",)), + "Spatial Reasoning": TaskTypeSpec("L3", True, "1", ("frame_summary", "spatial_layout")), + "Spatial Perception": TaskTypeSpec("L3", True, "1", ("frame_summary",)), + "Action Recognition": TaskTypeSpec("L2", True, "2-3", ("event_description",)), + "Action Reasoning": TaskTypeSpec("L2", True, "2-3", ("event_description",)), + "Counting Problem": TaskTypeSpec("L2", True, "2-3", ("event_description",)), + "Temporal Perception": TaskTypeSpec("L2", False, "0-1", ("event_description", "time_range")), + "Temporal Reasoning": TaskTypeSpec("L1", True, "per-L2", ("scene_summary",)), + "Information Synopsis": TaskTypeSpec("L1", True, "per-L2", ("scene_summary",)), + "Object Reasoning": TaskTypeSpec("L1-L2", True, "per-L2", ("event_description",)), +} +``` + +- [ ] **Step 4: 运行测试确认通过** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_synthesizer.py -v +``` + +预期：PASS + +- [ ] **Step 5: 提交** + +```bash +git add app/question_gen/synthesizer.py tests/unit/test_synthesizer.py +git commit -m "feat(question_gen): AnchorContext + 12 题型-层级映射常量" +``` + +--- + +### Task 2: synthesizer.py — sample_anchor + +**Files:** +- Modify: `app/question_gen/synthesizer.py` +- Modify: `tests/unit/test_synthesizer.py` + +- [ ] **Step 1: 写失败测试** + +```python +import json +import random +from pathlib import Path + +from app.tree.index import TreeIndex +from app.question_gen.synthesizer import sample_anchor + + +def _load_test_tree() -> tuple[TreeIndex, str]: + """加载真实测试树（store/videos/ 下第一棵）。""" + videos_dir = Path("store/videos") + first_vid = sorted(videos_dir.iterdir())[0] + tree = TreeIndex.load_json(str(first_vid / "tree.json")) + return tree, first_vid.name + + +class TestSampleAnchor: + def test_l3_type_returns_single_frame(self): + tree, vid = _load_test_tree() + ctx = sample_anchor(tree, "Object Recognition", set(), random.Random(42)) + assert len(ctx.frame_paths) == 1 + assert ctx.node_id.startswith("L") + assert len(ctx.distractor_texts) > 0 + + def test_l2_type_returns_multiple_frames(self): + tree, vid = _load_test_tree() + ctx = sample_anchor(tree, "Action Reasoning", set(), random.Random(42)) + assert 2 <= len(ctx.frame_paths) <= 3 + + def test_temporal_perception_zero_or_one_frame(self): + """Temporal Perception 帧数 0-1，且 card_text 含 time_range。""" + tree, vid = _load_test_tree() + ctx = sample_anchor(tree, "Temporal Perception", set(), random.Random(42)) + assert len(ctx.frame_paths) <= 1 + assert "time_range" in ctx.card_text.lower() or "time" in ctx.card_text.lower() + + def test_information_synopsis_uses_all_l2(self): + """Information Synopsis 必须包含全部 L2 card（非采样子集）。""" + tree, vid = _load_test_tree() + ctx = sample_anchor(tree, "Information Synopsis", set(), random.Random(42)) + total_l2 = sum(len(r.children) for r in tree.roots) + # card_text 中应包含全部 L2 的事件描述 + assert len(ctx.frame_paths) >= min(total_l2, 1) + + def test_l1_type_l2_nodes_in_time_order(self): + """L1 题型的 L2 子节点应按时间顺序组织。""" + tree, vid = _load_test_tree() + ctx = sample_anchor(tree, "Temporal Reasoning", set(), random.Random(42)) + assert len(ctx.frame_paths) >= 1 + assert len(ctx.card_text) > 20 + + def test_used_node_ids_excluded(self): + tree, vid = _load_test_tree() + rng = random.Random(42) + ctx1 = sample_anchor(tree, "Object Recognition", set(), rng) + ctx2 = sample_anchor(tree, "Object Recognition", {ctx1.node_id}, random.Random(43)) + assert ctx2.node_id != ctx1.node_id + + def test_insufficient_nodes_raises(self): + tree, vid = _load_test_tree() + all_l3_ids = set() + for root in tree.roots: + for l2 in root.children: + for l3 in l2.children: + all_l3_ids.add(l3.id) + import pytest + with pytest.raises(ValueError, match="锚节点不足"): + sample_anchor(tree, "Object Recognition", all_l3_ids, random.Random(42)) +``` + +- [ ] **Step 2: 运行测试确认失败** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_synthesizer.py::TestSampleAnchor -v +``` + +预期：ImportError（sample_anchor 不存在） + +- [ ] **Step 3: 实现 sample_anchor** + +在 `app/question_gen/synthesizer.py` 中追加 `sample_anchor` 函数，核心逻辑： + +1. 根据 `TASK_TYPE_LEVEL_MAP[task_type].level` 确定采样层级 +2. L3 题型：从所有 L3 节点中随机选一个（排除 used_node_ids），取单帧 + card +3. L2 题型（Action Recognition / Action Reasoning / Counting Problem）：随机选一个 L2 节点，均匀采样 2-3 子帧，card 取 event_description +4. **Temporal Perception 特例**：随机选一个 L2 节点，取 0-1 帧（有子帧取 1 帧，无则 0），card_text 必须包含 event_description + time_range +5. L1 题型：取根节点 card（scene_summary）。**Information Synopsis 使用全部 L2 card，Temporal Reasoning 选 ≥3 个**。L2 子节点按 time_range 升序排列，每个 L2 取 1 张代表帧 +6. L1-L2 题型：随机选 2-3 个 L2 节点（按 time_range 排序），每个取 1 张代表帧 +7. distractor_texts：收集同树中**其他**同层级节点的摘要文本 +8. 候选不足时 `raise ValueError("锚节点不足: ...")` + +详细实现需参考 `app/tree/index.py` 中 L1Node/L2Node/L3Node 的字段结构（L3Card.frame_summary, L2Card.event_description, L1Card.scene_summary）和 frame_path 位置。 + +- [ ] **Step 4: 运行测试确认通过** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_synthesizer.py::TestSampleAnchor -v +``` + +- [ ] **Step 5: 提交** + +```bash +git add app/question_gen/synthesizer.py tests/unit/test_synthesizer.py +git commit -m "feat(question_gen): sample_anchor — 按题型层级采样锚节点" +``` + +--- + +### Task 3: synthesizer.py — build_generation_prompt + parse_vlm_response + +**Files:** +- Modify: `app/question_gen/synthesizer.py` +- Modify: `tests/unit/test_synthesizer.py` + +- [ ] **Step 1: 写失败测试** + +```python +from core.types import GeneratedQuestion +from app.question_gen.synthesizer import ( + build_generation_prompt, + parse_vlm_response, + AnchorContext, +) + + +class TestBuildGenerationPrompt: + def test_messages_structure(self): + anchor = AnchorContext( + node_id="L3_001", + card_text="A person typing on a laptop", + frame_paths=["store/videos/test/frames/L1_000_L2_000_L3_000.jpg"], + subtitle="Hello world", + distractor_texts=["Another person walking in park"], + ) + exemplars = [ + GeneratedQuestion( + question_id="ex-1", video_id="v1", task_type="Object Recognition", + question="What object?", options=("A. Cat", "B. Dog", "C. Bird", "D. Fish"), + answer="A", source_nodes=(), difficulty="medium", + ), + ] + messages, image_paths = build_generation_prompt( + "Object Recognition", anchor, exemplars, + ) + assert messages[0]["role"] == "system" + assert "Object Recognition" in messages[0]["content"] + assert any("What object?" in str(m) for m in messages) + assert image_paths == anchor.frame_paths + + def test_distractor_in_user_message(self): + anchor = AnchorContext( + node_id="L2_003", + card_text="Event card text", + frame_paths=["a.jpg", "b.jpg"], + subtitle="", + distractor_texts=["Distractor node summary"], + ) + messages, _ = build_generation_prompt("Action Reasoning", anchor, []) + user_msg = [m for m in messages if m["role"] == "user"][0] + assert "Distractor node summary" in user_msg["content"] + + +class TestParseVlmResponse: + def test_valid_json(self): + raw = '{"question": "What?", "options": ["A. X", "B. Y", "C. Z", "D. W"], "answer": "A"}' + result = parse_vlm_response(raw, "vid1", "Object Recognition", 1) + assert result["question"] == "What?" + assert result["answer"] == "A" + assert len(result["options"]) == 4 + + def test_invalid_json_raises(self): + import pytest + with pytest.raises(ValueError, match="VLM 返回"): + parse_vlm_response("not json", "vid1", "Object Recognition", 1) + + def test_missing_fields_raises(self): + import pytest + raw = '{"question": "What?"}' + with pytest.raises(ValueError): + parse_vlm_response(raw, "vid1", "Object Recognition", 1) + + def test_options_must_be_four(self): + import pytest + raw = '{"question": "Q?", "options": ["A. X", "B. Y"], "answer": "A"}' + with pytest.raises(ValueError, match="4"): + parse_vlm_response(raw, "vid1", "Object Recognition", 1) + + def test_answer_must_be_abcd(self): + import pytest + raw = '{"question": "Q?", "options": ["A. X", "B. Y", "C. Z", "D. W"], "answer": "E"}' + with pytest.raises(ValueError, match="A.*D"): + parse_vlm_response(raw, "vid1", "Object Recognition", 1) +``` + +- [ ] **Step 2: 运行测试确认失败** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_synthesizer.py::TestBuildGenerationPrompt -v +conda activate Video-Tree-TRM & pytest tests/unit/test_synthesizer.py::TestParseVlmResponse -v +``` + +预期：ImportError + +- [ ] **Step 3: 实现 build_generation_prompt + parse_vlm_response** + +`build_generation_prompt(task_type, anchor, exemplars) -> (messages, image_paths)`： +- system message：角色设定 + 题型 + exemplar 示例 + 约束（基于节点内容、干扰项来自其他节点） +- user message：锚节点 card_text + subtitle + distractor_texts +- image_paths：直接取 anchor.frame_paths + +`parse_vlm_response(raw, video_id, task_type, seq) -> dict`： +- 尝试 `json.loads(raw)`，失败时尝试从 markdown code block 提取 JSON +- 校验必需字段 question / options / answer 存在 +- 返回 `{"question_id": f"gen-{video_id}-{seq:03d}", "question": ..., "options": [...], "answer": ...}` +- 缺字段或解析失败 → `raise ValueError("VLM 返回...")` + +- [ ] **Step 4: 运行测试确认通过** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_synthesizer.py -v +``` + +- [ ] **Step 5: 提交** + +```bash +git add app/question_gen/synthesizer.py tests/unit/test_synthesizer.py +git commit -m "feat(question_gen): build_generation_prompt + parse_vlm_response" +``` + +--- + +### Task 4: synthesizer.py — is_duplicate + generate_one + +**Files:** +- Modify: `app/question_gen/synthesizer.py` +- Modify: `tests/unit/test_synthesizer.py` + +- [ ] **Step 1: 写失败测试** + +```python +import numpy as np +from unittest.mock import AsyncMock, MagicMock +from app.question_gen.synthesizer import is_duplicate, generate_one + + +class TestIsDuplicate: + @staticmethod + def _fake_embed(texts): + """确定性 + L2 归一化的 fake embedding。""" + if isinstance(texts, str): + texts = [texts] + vecs = [] + for t in texts: + rs = np.random.RandomState(hash(t) % 2**31) + v = rs.randn(4).astype(np.float32) + v /= np.linalg.norm(v) + vecs.append(v) + return np.array(vecs, dtype=np.float32) + + def test_empty_pool_never_duplicate(self): + pool = np.zeros((0, 4), dtype=np.float32) + assert is_duplicate("anything", pool, self._fake_embed, 0.85) is False + + def test_identical_text_is_duplicate(self): + text = "What is happening in the video?" + emb = self._fake_embed(text) + pool = emb.copy() + assert is_duplicate(text, pool, self._fake_embed, 0.85) is True + + def test_different_text_not_duplicate(self): + pool_texts = ["aaa", "bbb", "ccc", "ddd", "eee"] + pool = self._fake_embed(pool_texts) + assert is_duplicate("completely unique text xyz", pool, self._fake_embed, 0.99) is False + + +class TestGenerateOne: + @staticmethod + async def test_success_path(): + """mock VLM 返回合法 JSON，应成功生成。""" + vlm = AsyncMock() + vlm.chat_with_images.return_value = MagicMock( + content='{"question":"Q?","options":["A. 1","B. 2","C. 3","D. 4"],"answer":"A"}' + ) + embed_fn = lambda t: np.zeros((1, 4) if isinstance(t, str) else (len(t), 4), dtype=np.float32) + + tree, vid = _load_test_tree() + result = await generate_one( + vlm=vlm, embed_fn=embed_fn, tree=tree, video_id=vid, + task_type="Object Recognition", seq=1, + exemplars=[], pool_embeddings=np.zeros((0, 4), dtype=np.float32), + used_node_ids=set(), max_retries=3, similarity_threshold=0.85, + rng=random.Random(42), session_id="test", + ) + assert result is not None + assert result.question_id == f"gen-{vid}-001" + assert result.task_type == "Object Recognition" + + @staticmethod + async def test_all_retries_exhausted_returns_none(): + """VLM 始终返回无效 JSON，耗尽重试后返回 None。""" + vlm = AsyncMock() + vlm.chat_with_images.return_value = MagicMock(content="invalid") + + embed_fn = lambda t: np.zeros((1, 4), dtype=np.float32) + tree, vid = _load_test_tree() + + result = await generate_one( + vlm=vlm, embed_fn=embed_fn, tree=tree, video_id=vid, + task_type="Object Recognition", seq=1, + exemplars=[], pool_embeddings=np.zeros((0, 4), dtype=np.float32), + used_node_ids=set(), max_retries=2, similarity_threshold=0.85, + rng=random.Random(42), session_id="test", + ) + assert result is None +``` + +- [ ] **Step 2: 运行测试确认失败** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_synthesizer.py::TestIsDuplicate -v +conda activate Video-Tree-TRM & pytest tests/unit/test_synthesizer.py::TestGenerateOne -v +``` + +- [ ] **Step 3: 实现 is_duplicate + generate_one** + +`is_duplicate(question_text, pool_embeddings, embed_fn, threshold) -> bool`： +- `embed_fn(question_text)` → `[1, D]`，squeeze 为 `[D]` +- `pool_embeddings @ query` 余弦相似度（pool 和 query 都已 L2 归一化） +- `max(similarities) >= threshold` → True + +`generate_one(vlm, embed_fn, tree, video_id, task_type, seq, *, ...)` → `GeneratedQuestion | None`： +- 循环最多 `max_retries` 次： + 1. `sample_anchor(tree, task_type, used_node_ids, rng)` → anchor + 2. `build_generation_prompt(task_type, anchor, exemplars)` → messages, images + 3. `await vlm.chat_with_images(messages, images, session_id=session_id)` → response + 4. `parse_vlm_response(response.content, video_id, task_type, seq)` → parsed_dict（含四选一 schema 校验） + 5. 用 anchor.node_id 补齐 `source_nodes`，`difficulty="medium"` + 6. 构造并返回 `GeneratedQuestion`（**不在此处做去重**——去重在调用方的单线程汇总点原子执行） +- 全部重试失败（parse 异常）→ return None + +**并发去重安全**：`generate_one` 只负责生成候选题。调用方（tools/ CLI）在收到候选后，在单线程汇总点（async for + await）原子执行：① `is_duplicate` 检查当前题型的 embedding 池 → ② 通过则添加 embedding + 写 JSON + 更新 progress → ③ 不通过则丢弃并重试。embedding 池按 `dict[str, np.ndarray]`（key=task_type）维护，确保只在同题型内去重。 + +- [ ] **Step 4: 运行测试确认通过** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_synthesizer.py -v +``` + +- [ ] **Step 5: 提交** + +```bash +git add app/question_gen/synthesizer.py tests/unit/test_synthesizer.py +git commit -m "feat(question_gen): is_duplicate + generate_one — 去重与单题生成编排" +``` + +--- + +### Task 5: factory.py — build_inference_deps + +**Files:** +- Create: `app/harness/factory.py` +- Create: `tests/unit/test_factory.py` + +- [ ] **Step 1: 写失败测试** + +```python +# tests/unit/test_factory.py +import random +from pathlib import Path +from unittest.mock import MagicMock, AsyncMock + +import numpy as np +import pytest + +from app.harness.factory import build_inference_deps, InferenceDeps + + +class TestBuildInferenceDeps: + def test_returns_inference_deps(self, tmp_path): + """用 fake adapters 验证返回类型和字段非 None。""" + # 准备一棵最小树 + import json + vid_dir = tmp_path / "videos" / "test_vid" + vid_dir.mkdir(parents=True) + frames_dir = vid_dir / "frames" + frames_dir.mkdir() + minimal_tree = { + "metadata": {"source_path": "test", "modality": "video"}, + "roots": [{ + "id": "L1_000", "card": { + "scene_summary": "s", "main_setting": "s", + "key_entities": [], "main_actions": [], + "topic_keywords": [], "visible_text": [], + "temporal_flow": "s", + }, "time_range": [0, 10], "children": [], + }], + } + (vid_dir / "tree.json").write_text(json.dumps(minimal_tree)) + + # 准备 prompts + prompts_dir = tmp_path / "prompts" + prompts_dir.mkdir() + (prompts_dir / "system.md").write_text("You are a search agent.") + + fake_llm = AsyncMock() + fake_vlm = AsyncMock() + fake_embed = MagicMock() + fake_embed.dim = 4 + fake_embed.embed = lambda t: np.zeros((1, 4), dtype=np.float32) + + deps = build_inference_deps( + store_dir=tmp_path, + video_id="test_vid", + prompts_dir=prompts_dir, + skills_dir=None, + skill_mode="none", + embed_provider=fake_embed, + llm=fake_llm, + vlm=fake_vlm, + ocr=None, + verify_vision=False, + anchor=False, + assemble_mode="ids", + ) + assert isinstance(deps, InferenceDeps) + assert deps.llm is fake_llm + assert callable(deps.tool_dispatch_fn) + assert callable(deps.prompt_builder) + + # 验证 prompt_builder 真正可调用（wiring 正确） + from core.types import GeneratedQuestion + fake_q = GeneratedQuestion( + question_id="q1", video_id="test_vid", task_type="Object Recognition", + question="What?", options=("A. X", "B. Y", "C. Z", "D. W"), + answer="A", source_nodes=(), difficulty="medium", + ) + system, user = deps.prompt_builder(fake_q) + assert isinstance(system, str) and len(system) > 0 + assert isinstance(user, str) and "What?" in user + + def test_missing_tree_raises(self, tmp_path): + prompts_dir = tmp_path / "prompts" + prompts_dir.mkdir() + (prompts_dir / "system.md").write_text("x") + vid_dir = tmp_path / "videos" / "nonexist" + vid_dir.mkdir(parents=True) + + with pytest.raises(FileNotFoundError): + build_inference_deps( + store_dir=tmp_path, video_id="nonexist", + prompts_dir=prompts_dir, skills_dir=None, skill_mode="none", + embed_provider=MagicMock(), llm=AsyncMock(), vlm=AsyncMock(), + ocr=None, verify_vision=False, anchor=False, assemble_mode="ids", + ) +``` + +- [ ] **Step 2: 运行测试确认失败** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_factory.py -v +``` + +预期：ImportError + +- [ ] **Step 3: 实现 factory.py** + +创建 `app/harness/factory.py`： + +```python +"""推理依赖组装 — 给定 store + config 构建可工作的推理依赖集。 + +factory 只做组装，不持有状态。adapter 实例由调用方创建并传入。 +消费者：tools/generate_questions.py（校准）、未来 main.py、Runner。 +""" + +from __future__ import annotations + +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from app.search.prompt import PromptManager +from app.search.skills import SkillRegistry, discover_skills +from app.search.tools import SearchToolDispatcher +from app.tree.environment import TreeEnvironment +from app.tree.index import TreeIndex + +if TYPE_CHECKING: + from collections.abc import Callable + + from app.ports import EmbeddingProvider, OCRProvider + from core.protocols import LLMProvider, VLMProvider + from core.types import GeneratedQuestion + + +@dataclass(frozen=True) +class InferenceDeps: + """跑一次推理所需的全套依赖（不含 HarnessLog）。 + + 属性: + llm: LLM 端口实例。 + tool_dispatch_fn: SearchToolDispatcher.dispatch 的绑定方法。 + prompt_builder: (GeneratedQuestion) -> (system_prompt, user_prompt)。 + """ + + llm: LLMProvider + tool_dispatch_fn: Callable[..., Any] + prompt_builder: Callable[[GeneratedQuestion], tuple[str, str]] + + +def build_inference_deps( + *, + store_dir: Path, + video_id: str, + prompts_dir: Path, + skills_dir: Path | None, + skill_mode: str, + embed_provider: EmbeddingProvider, + llm: LLMProvider, + vlm: VLMProvider, + ocr: OCRProvider | None, + verify_vision: bool, + anchor: bool, + assemble_mode: str, +) -> InferenceDeps: + """组装单个视频的推理依赖。 + + 参数: + store_dir: store 根目录（含 videos/{video_id}/tree.json）。 + video_id: 视频标识。 + prompts_dir: prompt 文件目录（含 system.md）。 + skills_dir: skill 文件目录（None 不启用）。 + skill_mode: "auto" / "manual" / "none"。 + embed_provider: 文本嵌入端口。 + llm: LLM 端口。 + vlm: VLM 端口。 + ocr: OCR 端口（None 不启用）。 + verify_vision: observe_frame 是否执行验证轮。 + anchor: view_node 是否启用行号锚模式。 + assemble_mode: 锚模式装配形态。 + + 返回: + InferenceDeps 实例。 + + 异常: + FileNotFoundError: tree.json 不存在。 + """ + # Phase 1: 加载树 + tree_path = store_dir / "videos" / video_id / "tree.json" + if not tree_path.exists(): + raise FileNotFoundError(f"树文件不存在: {tree_path}") + tree = TreeIndex.load_json(str(tree_path)) + + frames_dir = store_dir / "videos" / video_id / "frames" + env = TreeEnvironment(tree, frames_dir if frames_dir.exists() else None) + + # Phase 2: Skills + skills: SkillRegistry | None = None + always_skills_text = "" + task_skill_map: dict[str, str] = {} + catalog_text = "" + if skills_dir and skills_dir.exists(): + always_skills_text, task_skill_map, catalog_text, skills = discover_skills(skills_dir) + + # Phase 3: SearchToolDispatcher + dispatcher = SearchToolDispatcher( + env=env, + tool_llm=llm, + vlm=vlm, + ocr=ocr, + prompts_dir=prompts_dir, + skills=skills, + embed_fn=embed_provider.embed, + verify_vision=verify_vision, + anchor=anchor, + assemble_mode=assemble_mode, + ) + + # Phase 4: PromptManager → prompt_builder 偏函数 + pm = PromptManager(prompts_dir) + l1_ids = [r.id for r in tree.roots] + + def _prompt_builder( + qa: GeneratedQuestion, + _pm: PromptManager = pm, + _skill_mode: str = skill_mode, + _always: str = always_skills_text, + _tsm: dict = task_skill_map, + _cat: str = catalog_text, + _l1_ids: list = l1_ids, + ) -> tuple[str, str]: + system = _pm.build_inference_prompt( + _skill_mode, qa.task_type, _always, _tsm, _cat, + ) + user = _pm.format_user_prompt( + qa.question, list(qa.options), _l1_ids, qa.task_type, + ) + return system, user + + return InferenceDeps( + llm=llm, + tool_dispatch_fn=dispatcher.dispatch, + prompt_builder=_prompt_builder, + ) +``` + +- [ ] **Step 4: 运行测试确认通过** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_factory.py -v +``` + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/factory.py tests/unit/test_factory.py +git commit -m "feat(harness): factory.py — 推理依赖组装，可复用于 calibrate + main.py" +``` + +--- + +### Task 6: tools/generate_questions.py — generate 子命令 + +**Files:** +- Create: `tools/generate_questions.py` +- Modify: `tests/unit/test_generate_questions.py`（新建） + +- [ ] **Step 1: 写失败测试** + +```python +# tests/unit/test_generate_questions.py +import json +import sys +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import numpy as np +import pytest + + +class TestGenerateSmoke: + def test_generate_writes_json(self, tmp_path): + """mock VLM + 1 棵真实树 + per_type=1，验证 JSON 输出格式。""" + import shutil + from unittest.mock import AsyncMock, MagicMock, patch + + # 复制一棵真实树到 tmp + src = Path("store/videos") / sorted(Path("store/videos").iterdir())[0].name + dst = tmp_path / "videos" / src.name + shutil.copytree(src, dst) + + # 准备 benchmark（至少 1 道题做 exemplar） + bench_dir = tmp_path / "questions" / "benchmarks" + bench_dir.mkdir(parents=True) + bench_file = bench_dir / f"{src.name}.json" + bench_file.write_text(json.dumps([{ + "question_id": "ex-1", "task_type": "Object Recognition", + "question": "What?", "options": ["A. X", "B. Y", "C. Z", "D. W"], + "answer": "A", + }])) + + output_dir = tmp_path / "output" + output_dir.mkdir() + + # import CLI 模块的内部函数 + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + from tools.generate_questions import _load_or_init_progress, _select_exemplars + + # 验证 progress 初始化 + progress = _load_or_init_progress(output_dir) + assert progress["completed"] == {} + + # 验证 exemplar 选择 + from app.question_gen.loader import load_benchmark + bench_qs = load_benchmark(bench_dir) + exemplars = _select_exemplars(bench_qs, "Object Recognition", 3, random.Random(42)) + assert len(exemplars) >= 1 + assert all(e.task_type == "Object Recognition" for e in exemplars) + + +class TestProgressResume: + def test_skips_completed_and_rebuilds_pool(self, tmp_path): + """progress.json 中已完成的题应被跳过，embedding 池应从已有 JSON 恢复。""" + output_dir = tmp_path / "output" + output_dir.mkdir() + + # 写一个已完成的 JSON + (output_dir / "test_vid.json").write_text(json.dumps([{ + "question_id": "gen-test_vid-001", "task_type": "Object Recognition", + "question": "Existing question?", + "options": ["A. X", "B. Y", "C. Z", "D. W"], "answer": "A", + "source_nodes": ["L3_001"], "difficulty": "medium", + }])) + + progress = { + "completed": {"Object Recognition": ["gen-test_vid-001"]}, + "output_dir": str(output_dir), + } + (output_dir / "progress.json").write_text(json.dumps(progress)) + + from tools.generate_questions import _load_or_init_progress + loaded = _load_or_init_progress(output_dir) + assert "gen-test_vid-001" in loaded["completed"]["Object Recognition"] +``` + +- [ ] **Step 2: 运行测试确认失败** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_generate_questions.py -v +``` + +- [ ] **Step 3: 实现 tools/generate_questions.py — generate 子命令** + +创建 `tools/generate_questions.py`，核心结构： + +```python +#!/usr/bin/env python3 +"""赛题生成工具：generate + calibrate。 + +用法: + conda activate Video-Tree-TRM + python tools/generate_questions.py generate --store-dir store ... + python tools/generate_questions.py calibrate --generated-dir ... --benchmark-dir ... + +app/core/adapters 不 import 此脚本。 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + +from dotenv import load_dotenv +from loguru import logger + +load_dotenv(PROJECT_ROOT / ".env") + +# generate 子命令： +# 1. 加载 video_id 列表 + benchmark exemplars + 初始化 embedding 池 +# 2. 断点续跑：读 progress.json + 恢复已生成题 embedding + used_node_ids +# 3. 实例化 GovernedVLMClient + EmbeddingProvider（从 .env 读配置） +# 4. 对 12 题型 × per_type，asyncio.Semaphore 并发调 generate_one +# 5. 单线程汇总：检查去重 → 加入 pool → 写 JSON → 更新 progress +# 6. 全部完成删除 progress.json +``` + +实现要点： +- `_load_or_init_progress(output_dir)` / `_save_progress(output_dir, progress)` 断点续跑 +- `_rebuild_embedding_pool(output_dir, embed_fn, benchmark_questions)` 续跑时恢复 embedding +- `_build_vlm_client()` / `_build_embed_provider()` 从 .env 实例化 adapters +- `_select_exemplars(benchmark, task_type, n, rng)` 跨视频采样 few-shot +- `async def _run_generate(args)` 主流程 +- 并发模型：Semaphore 限流 VLM 调用，但去重+写入在主协程中顺序执行 + +- [ ] **Step 4: 运行测试 + lint** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_generate_questions.py -v +conda activate Video-Tree-TRM & ruff check tools/generate_questions.py --fix +``` + +- [ ] **Step 5: 提交** + +```bash +git add tools/generate_questions.py tests/unit/test_generate_questions.py +git commit -m "feat(tools): generate_questions.py generate 子命令 — VLM 出题 + 去重 + 断点续跑" +``` + +--- + +### Task 7: tools/generate_questions.py — calibrate 子命令 + +**Files:** +- Modify: `tools/generate_questions.py` +- Modify: `tests/unit/test_generate_questions.py` + +- [ ] **Step 1: 写失败测试** + +```python +from scipy.stats import fisher_exact + + +class TestCalibrateJudgment: + def test_pass_when_delta_small(self): + """差异小于 tolerance → PASS。""" + from tools.generate_questions import _judge_task_type + verdict = _judge_task_type( + bench_correct=60, bench_total=100, + gen_correct=12, gen_total=20, + tolerance=0.10, alpha=0.05, + ) + assert verdict == "PASS" + + def test_fail_when_delta_large_and_significant(self): + """差异大且统计显著 → FAIL。""" + from tools.generate_questions import _judge_task_type + verdict = _judge_task_type( + bench_correct=144, bench_total=240, + gen_correct=6, gen_total=20, + tolerance=0.10, alpha=0.05, + ) + assert verdict == "FAIL" + + def test_warn_when_delta_large_but_not_significant(self): + """差异大但样本不足（p > alpha）→ WARN。""" + from tools.generate_questions import _judge_task_type + verdict = _judge_task_type( + bench_correct=2, bench_total=3, + gen_correct=8, gen_total=20, + tolerance=0.10, alpha=0.05, + ) + assert verdict == "WARN" + + +class TestCalibrateIntegration: + def test_baseline_params_must_be_paired(self): + """--baseline-db 和 --baseline-run-id 必须成对出现。""" + from tools.generate_questions import _validate_calibrate_args + import pytest + with pytest.raises(ValueError, match="成对"): + _validate_calibrate_args(baseline_db="some.db", baseline_run_id=None) + + def test_has_fail_returns_exit_code_1(self): + """存在 FAIL 判定时，_calibrate_exit_code 返回 1。""" + from tools.generate_questions import _calibrate_exit_code + verdicts = {"Object Recognition": "PASS", "Action Reasoning": "FAIL"} + assert _calibrate_exit_code(verdicts) == 1 + + def test_all_pass_or_warn_returns_exit_code_0(self): + from tools.generate_questions import _calibrate_exit_code + verdicts = {"Object Recognition": "PASS", "Spatial Perception": "WARN"} + assert _calibrate_exit_code(verdicts) == 0 +``` + +- [ ] **Step 2: 运行测试确认失败** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_generate_questions.py::TestCalibrateJudgment -v +``` + +- [ ] **Step 3: 实现 calibrate 子命令** + +在 `tools/generate_questions.py` 中追加： + +`_judge_task_type(bench_correct, bench_total, gen_correct, gen_total, tolerance, alpha) -> str`： +- `delta = abs(gen_correct/gen_total - bench_correct/bench_total)` +- `delta <= tolerance` → "PASS" +- Fisher exact test p-value：`table = [[bench_correct, bench_total-bench_correct], [gen_correct, gen_total-gen_correct]]` +- `p < alpha and delta > tolerance` → "FAIL" +- else → "WARN" + +`async def _run_calibrate(args)` 主流程： +1. `load_benchmark` 加载两组题 +2. benchmark 基线：有 `--baseline-db` 则从 DB 读，否则按 video_id 分组 → `build_inference_deps` → `run_inference` +3. 生成题同理按 video_id 分组 → 分组推理 +4. 汇总 per_task_type accuracy → `_judge_task_type` 逐题型判定 +5. 输出对比表 +6. 有 FAIL → `sys.exit(1)` + +- [ ] **Step 4: 运行测试确认通过** + +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_generate_questions.py -v +``` + +- [ ] **Step 5: 提交** + +```bash +git add tools/generate_questions.py tests/unit/test_generate_questions.py +git commit -m "feat(tools): generate_questions.py calibrate 子命令 — Fisher exact test 校准" +``` + +--- + +### Task 8: __init__.py 更新 + lint + 全量测试 + +**Files:** +- Modify: `app/question_gen/__init__.py` + +- [ ] **Step 1: 写失败测试** + +```python +# 在 tests/unit/test_synthesizer.py 中追加 +def test_public_api_importable(): + """synthesizer 的公共 API 必须可从 app.question_gen 直接 import。""" + from app.question_gen import generate_one, AnchorContext, TASK_TYPE_LEVEL_MAP, sample_anchor + assert callable(generate_one) + assert callable(sample_anchor) +``` + +运行确认失败（当前 __init__.py 不 export 这些）： +```bash +conda activate Video-Tree-TRM & pytest tests/unit/test_synthesizer.py::test_public_api_importable -v +``` + +- [ ] **Step 2: 更新 __init__.py re-export** + +```python +"""出题模块 — benchmark 加载、分层采样与赛题合成。""" + +from app.question_gen.loader import load_benchmark, stratified_sample +from app.question_gen.synthesizer import ( + TASK_TYPE_LEVEL_MAP, + AnchorContext, + generate_one, + sample_anchor, +) + +__all__ = [ + "load_benchmark", + "stratified_sample", + "TASK_TYPE_LEVEL_MAP", + "AnchorContext", + "generate_one", + "sample_anchor", +] +``` + +- [ ] **Step 2: 全量 lint** + +```bash +conda activate Video-Tree-TRM & ruff check app/question_gen/ app/harness/factory.py tools/generate_questions.py --fix +conda activate Video-Tree-TRM & ruff format app/question_gen/ app/harness/factory.py tools/generate_questions.py +``` + +- [ ] **Step 3: 全量测试** + +```bash +conda activate Video-Tree-TRM & pytest tests/ -v --tb=short +``` + +预期：全部 PASS + +- [ ] **Step 4: 提交** + +```bash +git add app/question_gen/__init__.py +git commit -m "refactor(question_gen): __init__.py 追加 synthesizer re-export" +``` + +--- + +## Self-Review 核对 + +**范围说明**：设计 §4.4 要求 `Runner._make_tool_dispatch_fn` / `_make_prompt_builder` 委托 factory，本计划不包含该改造——Runner 改造随 `main.py` 一起实施更合理。factory.py 已就绪可复用。 + +| 设计文档章节 | 对应 Task | +|-------------|-----------| +| §2 模块结构 | Task 1-5 (synthesizer) + Task 5 (factory) + Task 6-7 (CLI) | +| §3.1 题型映射 | Task 1 | +| §3.2 AnchorContext | Task 1 | +| §3.3 函数签名 | Task 2 (sample_anchor) + Task 3 (prompt/parse) + Task 4 (dedup/generate) | +| §3.4 exemplar 选择 | Task 6 (_select_exemplars) | +| §3.6 去重 + 并发安全 | Task 4 (is_duplicate) + Task 6 (单线程汇总) | +| §4 factory.py | Task 5 | +| §5 CLI 设计 | Task 6 (generate) + Task 7 (calibrate) | +| §6 Fisher 校准 | Task 7 (_judge_task_type) | +| §7 断点续跑 | Task 6 (progress + embedding 恢复) | +| §8 输出格式 | Task 6 (JSON 写入) | +| §9 前置修复 | Task 0 | +| §10 测试策略 | Task 1-7 各含测试 | diff --git a/research-wiki/plans/question-gen-synth.md b/research-wiki/plans/question-gen-synth.md new file mode 100644 index 0000000..ef38292 --- /dev/null +++ b/research-wiki/plans/question-gen-synth.md @@ -0,0 +1,9 @@ +--- +type: plan +node_id: plan:question-gen-synth +title: 赛题生成工具实现计划 +date: 2026-07-09 +--- + +# 赛题生成工具实现计划 + From eb15ab315e18faf6b372ba59b0afa1f9f3b4d3ef Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 05:08:05 -0400 Subject: [PATCH 09/21] =?UTF-8?q?fix(config):=20=5FVIDEO=5FMME=5FTASK=5FTY?= =?UTF-8?q?PE=5FCOUNT=2011=E2=86=9212=EF=BC=8CVideo-MME=20=E5=AE=9E?= =?UTF-8?q?=E9=99=85=E6=9C=89=2012=20=E7=A7=8D=E9=A2=98=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) --- app/harness/config.py | 4 ++-- tests/unit/test_harness_config.py | 19 +++++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/app/harness/config.py b/app/harness/config.py index 8fdc065..d0ba2e3 100644 --- a/app/harness/config.py +++ b/app/harness/config.py @@ -20,8 +20,8 @@ _VALID_SKILL_MODES = {"auto", "manual", "none"} _VALID_SKILL_UPDATE_MODES = {"patch", "rewrite"} _PATH_FIELDS = {"workspace_dir", "store_dir"} -# Video-MME 的任务类型数量：验证池每类至少保底 eval_min_per_class 题，共 11 类。 -_VIDEO_MME_TASK_TYPE_COUNT = 11 +# Video-MME 的任务类型数量：验证池每类至少保底 eval_min_per_class 题，共 12 类。 +_VIDEO_MME_TASK_TYPE_COUNT = 12 # .env 工程配置字段映射（环境变量名 → RunConfig 字段名）。 # 仅路径类工程配置走 .env，科研实验参数走 YAML。 diff --git a/tests/unit/test_harness_config.py b/tests/unit/test_harness_config.py index 2d01d10..555cb79 100644 --- a/tests/unit/test_harness_config.py +++ b/tests/unit/test_harness_config.py @@ -568,28 +568,35 @@ class TestValSizeFloor: """val_size >= eval_min_per_class * 11 的下限校验。""" def test_val_size_below_floor_rejected(self) -> None: - """val_size < eval_min_per_class * 11 应抛出 ValueError。 + """val_size < eval_min_per_class * 12 应抛出 ValueError。 - eval_min_per_class=3 → 下限 = 3 * 11 = 33，val_size=30 不足。 + eval_min_per_class=3 → 下限 = 3 * 12 = 36，val_size=30 不足。 """ cfg = _make_config(eval_min_per_class=3, val_size=30) with pytest.raises(ValueError, match="val_size"): _validate(cfg) def test_val_size_at_floor_accepted(self) -> None: - """val_size == eval_min_per_class * 11 应通过。""" - cfg = _make_config(eval_min_per_class=3, val_size=33) + """val_size == eval_min_per_class * 12 应通过。""" + cfg = _make_config(eval_min_per_class=3, val_size=36) _validate(cfg) def test_val_size_above_floor_accepted(self) -> None: - """val_size > eval_min_per_class * 11 应通过。""" + """val_size > eval_min_per_class * 12 应通过。""" cfg = _make_config(eval_min_per_class=2, val_size=100) _validate(cfg) def test_default_yaml_values_satisfy_floor(self) -> None: """default.yaml 的默认值（val_size=30, eval_min_per_class=2）应满足下限。 - 下限 = 2 * 11 = 22，val_size=30 >= 22，通过。 + 下限 = 2 * 12 = 24，val_size=30 >= 24，通过。 """ cfg = _make_config() _validate(cfg) # 不应抛出异常 + + +def test_video_mme_task_type_count_is_12(): + """Video-MME 实际有 12 种题型，常量必须与之一致。""" + from app.harness.config import _VIDEO_MME_TASK_TYPE_COUNT + + assert _VIDEO_MME_TASK_TYPE_COUNT == 12 From 9eb9b86954033dc6b6bf24fe047e23e94bf6f34a Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 05:10:58 -0400 Subject: [PATCH 10/21] =?UTF-8?q?feat(question=5Fgen):=20AnchorContext=20+?= =?UTF-8?q?=2012=20=E9=A2=98=E5=9E=8B-=E5=B1=82=E7=BA=A7=E6=98=A0=E5=B0=84?= =?UTF-8?q?=E5=B8=B8=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - AnchorContext frozen dataclass: 锚节点生成上下文（node_id, card_text, frame_paths, subtitle, distractor_texts） - TaskTypeSpec frozen dataclass: 题型生成规格（level, needs_frames, frame_count, context_fields） - TASK_TYPE_LEVEL_MAP: 12 种 Video-MME 题型 → 树层级 + 生成规格映射 - 11 项单元测试覆盖：映射完整性、值类型、层级合法性、frozen 不变性 Co-Authored-By: Claude Opus 4.6 (1M context) --- app/question_gen/synthesizer.py | 68 ++++++++++++++++ tests/unit/test_synthesizer.py | 134 ++++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 app/question_gen/synthesizer.py create mode 100644 tests/unit/test_synthesizer.py diff --git a/app/question_gen/synthesizer.py b/app/question_gen/synthesizer.py new file mode 100644 index 0000000..9aedb29 --- /dev/null +++ b/app/question_gen/synthesizer.py @@ -0,0 +1,68 @@ +"""赛题合成核心逻辑 — 节点采样、prompt 构造、去重。 + +纯函数为主，异步编排仅 generate_one。 +通过 DI 接收 VLMProvider / EmbeddingProvider，不 import adapters/。 +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class AnchorContext: + """锚节点上下文——生成单道题所需的全部素材。 + + 属性: + node_id: 锚节点 ID。 + card_text: 锚节点 card 序列化文本。 + frame_paths: 帧图片路径列表。 + subtitle: 对应字幕（可空）。 + distractor_texts: 同视频其他节点摘要（供 VLM 生成干扰项）。 + """ + + node_id: str + card_text: str + frame_paths: list[str] + subtitle: str + distractor_texts: list[str] + + +@dataclass(frozen=True) +class TaskTypeSpec: + """题型的生成规格。 + + 属性: + level: 锚定层级（"L3" / "L2" / "L1" / "L1-L2"）。 + needs_frames: 是否必须提供帧图。 + frame_count: 帧数范围描述（如 "1", "2-3", "0-1"）。 + context_fields: 需要提取的 card 字段元组。 + """ + + level: str + needs_frames: bool + frame_count: str + context_fields: tuple[str, ...] + + +# --------------------------------------------------------------------------- +# 12 种 Video-MME 题型 → 树层级 + 生成规格映射 +# --------------------------------------------------------------------------- + +TASK_TYPE_LEVEL_MAP: dict[str, TaskTypeSpec] = { + # --- L3 单帧题型 --- + "Object Recognition": TaskTypeSpec("L3", True, "1", ("frame_summary",)), + "Attribute Perception": TaskTypeSpec("L3", True, "1", ("frame_summary",)), + "OCR Problems": TaskTypeSpec("L3", True, "1", ("frame_summary",)), + "Spatial Reasoning": TaskTypeSpec("L3", True, "1", ("frame_summary", "spatial_layout")), + "Spatial Perception": TaskTypeSpec("L3", True, "1", ("frame_summary",)), + # --- L2 多帧 / 事件级题型 --- + "Action Recognition": TaskTypeSpec("L2", True, "2-3", ("event_description",)), + "Action Reasoning": TaskTypeSpec("L2", True, "2-3", ("event_description",)), + "Counting Problem": TaskTypeSpec("L2", True, "2-3", ("event_description",)), + "Temporal Perception": TaskTypeSpec("L2", False, "0-1", ("event_description", "time_range")), + # --- L1 / 跨层级题型 --- + "Temporal Reasoning": TaskTypeSpec("L1", True, "per-L2", ("scene_summary",)), + "Information Synopsis": TaskTypeSpec("L1", True, "per-L2", ("scene_summary",)), + "Object Reasoning": TaskTypeSpec("L1-L2", True, "per-L2", ("event_description",)), +} diff --git a/tests/unit/test_synthesizer.py b/tests/unit/test_synthesizer.py new file mode 100644 index 0000000..579b2e6 --- /dev/null +++ b/tests/unit/test_synthesizer.py @@ -0,0 +1,134 @@ +"""synthesizer 模块单元测试 — AnchorContext + 题型映射常量。""" + +from __future__ import annotations + +import dataclasses + +from app.question_gen.synthesizer import TASK_TYPE_LEVEL_MAP, AnchorContext, TaskTypeSpec + +ALL_12_TYPES = [ + "Object Recognition", + "Attribute Perception", + "OCR Problems", + "Spatial Reasoning", + "Spatial Perception", + "Action Recognition", + "Action Reasoning", + "Counting Problem", + "Temporal Perception", + "Temporal Reasoning", + "Information Synopsis", + "Object Reasoning", +] + + +class TestTaskTypeLevelMap: + """TASK_TYPE_LEVEL_MAP 覆盖性与结构测试。""" + + def test_covers_all_12_types(self) -> None: + """映射表必须覆盖全部 12 种 Video-MME 题型。""" + assert set(TASK_TYPE_LEVEL_MAP.keys()) == set(ALL_12_TYPES) + + def test_no_extra_types(self) -> None: + """映射表不得包含 12 种标准题型之外的条目。""" + assert len(TASK_TYPE_LEVEL_MAP) == 12 + + def test_all_values_are_task_type_spec(self) -> None: + """每个映射值必须是 TaskTypeSpec 实例。""" + for task_type, spec in TASK_TYPE_LEVEL_MAP.items(): + assert isinstance(spec, TaskTypeSpec), f"{task_type} 映射值类型错误: {type(spec)}" + + def test_level_values_valid(self) -> None: + """每个 spec 的 level 必须是合法层级标识。""" + valid_levels = {"L1", "L2", "L3", "L1-L2"} + for task_type, spec in TASK_TYPE_LEVEL_MAP.items(): + assert spec.level in valid_levels, ( + f"{task_type} 层级 '{spec.level}' 不在 {valid_levels}" + ) + + def test_context_fields_non_empty(self) -> None: + """每个 spec 的 context_fields 至少有一个字段。""" + for task_type, spec in TASK_TYPE_LEVEL_MAP.items(): + assert len(spec.context_fields) >= 1, f"{task_type} 的 context_fields 为空" + + +class TestAnchorContext: + """AnchorContext 数据类测试。""" + + def test_frozen(self) -> None: + """AnchorContext 是不可变的。""" + ctx = AnchorContext( + node_id="L3_001", + card_text="A person walks into a room", + frame_paths=["/data/frames/001.jpg"], + subtitle="Hello there", + distractor_texts=["A car drives by"], + ) + assert ctx.node_id == "L3_001" + assert ctx.card_text == "A person walks into a room" + assert ctx.frame_paths == ["/data/frames/001.jpg"] + assert ctx.subtitle == "Hello there" + assert ctx.distractor_texts == ["A car drives by"] + + def test_mutation_raises(self) -> None: + """frozen dataclass 拒绝赋值修改。""" + ctx = AnchorContext( + node_id="L3_001", + card_text="test", + frame_paths=["a.jpg"], + subtitle="", + distractor_texts=["other node"], + ) + try: + ctx.node_id = "L3_002" # type: ignore[misc] + raise AssertionError("应抛出 FrozenInstanceError") + except dataclasses.FrozenInstanceError: + pass + + def test_empty_subtitle_allowed(self) -> None: + """subtitle 可以为空字符串。""" + ctx = AnchorContext( + node_id="L2_010", + card_text="scene card", + frame_paths=[], + subtitle="", + distractor_texts=[], + ) + assert ctx.subtitle == "" + + def test_multiple_frame_paths(self) -> None: + """frame_paths 可包含多个路径。""" + paths = ["/data/f1.jpg", "/data/f2.jpg", "/data/f3.jpg"] + ctx = AnchorContext( + node_id="L2_005", + card_text="multi-frame event", + frame_paths=paths, + subtitle="Dialogue line", + distractor_texts=["other1", "other2"], + ) + assert len(ctx.frame_paths) == 3 + + +class TestTaskTypeSpec: + """TaskTypeSpec 数据类测试。""" + + def test_frozen(self) -> None: + """TaskTypeSpec 是不可变的。""" + spec = TaskTypeSpec( + level="L3", + needs_frames=True, + frame_count="1", + context_fields=("frame_summary",), + ) + try: + spec.level = "L2" # type: ignore[misc] + raise AssertionError("应抛出 FrozenInstanceError") + except dataclasses.FrozenInstanceError: + pass + + def test_context_fields_is_tuple(self) -> None: + """context_fields 应为 tuple（不可变）。""" + for task_type, spec in TASK_TYPE_LEVEL_MAP.items(): + assert isinstance(spec.context_fields, tuple), ( + f"{task_type} 的 context_fields 不是 tuple" + ) From a597a9f90165feb542593af0de273e346339f754 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 05:18:25 -0400 Subject: [PATCH 11/21] =?UTF-8?q?feat(question=5Fgen):=20sample=5Fanchor?= =?UTF-8?q?=20=E2=80=94=20=E6=8C=89=E9=A2=98=E5=9E=8B=E5=B1=82=E7=BA=A7?= =?UTF-8?q?=E9=87=87=E6=A0=B7=E9=94=9A=E8=8A=82=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 含 6 种层级分支：L3 单帧、L2 多帧、Temporal Perception 特例、 L1 全量/采样 L2、L1-L2 混合。时间排序 + used_node_ids 排除。 Co-Authored-By: Claude Opus 4.6 (1M context) --- app/question_gen/synthesizer.py | 399 ++++++++++++++++++++++++++++++++ tests/unit/test_synthesizer.py | 100 +++++++- 2 files changed, 497 insertions(+), 2 deletions(-) diff --git a/app/question_gen/synthesizer.py b/app/question_gen/synthesizer.py index 9aedb29..d5ee161 100644 --- a/app/question_gen/synthesizer.py +++ b/app/question_gen/synthesizer.py @@ -7,6 +7,12 @@ from __future__ import annotations from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import random + + from app.tree.index import L1Node, L2Node, L3Node, TreeIndex @dataclass(frozen=True) @@ -66,3 +72,396 @@ TASK_TYPE_LEVEL_MAP: dict[str, TaskTypeSpec] = { "Information Synopsis": TaskTypeSpec("L1", True, "per-L2", ("scene_summary",)), "Object Reasoning": TaskTypeSpec("L1-L2", True, "per-L2", ("event_description",)), } + + +# --------------------------------------------------------------------------- +# 内部辅助函数 +# --------------------------------------------------------------------------- + + +def _serialize_l3_card(l3: L3Node, context_fields: tuple[str, ...]) -> str: + """将 L3 节点 card 按 context_fields 序列化为可读文本。 + + 参数: + l3: L3 节点。 + context_fields: 需提取的字段名元组。 + + 返回: + 多行 "field: value" 格式的文本。 + """ + parts: list[str] = [] + for fld in context_fields: + val = getattr(l3.card, fld, None) + if val is not None: + parts.append(f"{fld}: {val}") + return "\n".join(parts) + + +def _l2_time_range_str(l2: L2Node) -> str: + """将 L2 的 time_range 格式化为可读字符串。 + + 参数: + l2: L2 节点。 + + 返回: + "time_range: (start, end)" 格式，或 "time_range: unknown"。 + """ + if l2.time_range is not None: + return f"time_range: ({l2.time_range[0]:.2f}, {l2.time_range[1]:.2f})" + return "time_range: unknown" + + +def _representative_frame(l2: L2Node) -> str | None: + """取 L2 的代表帧路径——第一个有 frame_path 的 L3 子节点。 + + 参数: + l2: L2 节点。 + + 返回: + 帧路径字符串，或 None（无可用帧时）。 + """ + for l3 in l2.children: + if l3.frame_path: + return l3.frame_path + return None + + +def _sort_l2_by_time(l2_nodes: list[L2Node]) -> list[L2Node]: + """按 time_range 升序排列 L2 节点（None 排末尾）。 + + 参数: + l2_nodes: 待排序的 L2 节点列表。 + + 返回: + 排序后的新列表（不修改原列表）。 + """ + return sorted( + l2_nodes, + key=lambda n: n.time_range[0] if n.time_range is not None else float("inf"), + ) + + +# --------------------------------------------------------------------------- +# 各层级采样策略 +# --------------------------------------------------------------------------- + + +def _sample_l3( + tree: TreeIndex, + task_type: str, + spec: TaskTypeSpec, + used_node_ids: set[str], + rng: random.Random, +) -> AnchorContext: + """L3 层级锚节点采样。 + + 收集全部 L3 节点，排除已用节点，随机选取一个。 + + 参数: + tree: 三层树索引。 + task_type: 题型名称。 + spec: 题型规格。 + used_node_ids: 已用节点 ID 集合。 + rng: 随机数生成器。 + + 返回: + AnchorContext 实例。 + + 异常: + ValueError: 候选 L3 节点不足。 + """ + # Phase 1: 收集所有 L3 候选 + candidates: list[tuple[L3Node, L2Node]] = [] + for root in tree.roots: + for l2 in root.children: + for l3 in l2.children: + if l3.id not in used_node_ids: + candidates.append((l3, l2)) + + if not candidates: + raise ValueError(f"锚节点不足: {task_type} 无可用 L3 节点") + + # Phase 2: 随机选取 + chosen_l3, parent_l2 = rng.choice(candidates) + + # Phase 3: 构造上下文 + card_text = _serialize_l3_card(chosen_l3, spec.context_fields) + frame_paths = [chosen_l3.frame_path] if chosen_l3.frame_path else [] + subtitle = chosen_l3.subtitle or "" + + # Phase 4: 干扰项——同 L2 下其他 L3 的 frame_summary + distractor_texts = [l3.card.frame_summary for l3 in parent_l2.children if l3.id != chosen_l3.id] + + return AnchorContext( + node_id=chosen_l3.id, + card_text=card_text, + frame_paths=frame_paths, + subtitle=subtitle, + distractor_texts=distractor_texts, + ) + + +def _sample_l2( + tree: TreeIndex, + task_type: str, + spec: TaskTypeSpec, + used_node_ids: set[str], + rng: random.Random, +) -> AnchorContext: + """L2 层级锚节点采样（含 Temporal Perception 特殊处理）。 + + 普通 L2 题型：随机选 1 个 L2，取 2-3 个子 L3 帧。 + Temporal Perception：0-1 帧，card_text 必含 time_range。 + + 参数: + tree: 三层树索引。 + task_type: 题型名称。 + spec: 题型规格。 + used_node_ids: 已用节点 ID 集合。 + rng: 随机数生成器。 + + 返回: + AnchorContext 实例。 + + 异常: + ValueError: 候选 L2 节点不足。 + """ + # Phase 1: 收集所有 L2 候选 + all_l2: list[tuple[L2Node, L1Node]] = [] + for root in tree.roots: + for l2 in root.children: + if l2.id not in used_node_ids: + all_l2.append((l2, root)) + + if not all_l2: + raise ValueError(f"锚节点不足: {task_type} 无可用 L2 节点") + + # Phase 2: 随机选取 + chosen_l2, parent_l1 = rng.choice(all_l2) + + is_temporal_perception = task_type == "Temporal Perception" + + # Phase 3: 帧路径 + if is_temporal_perception: + # 0-1 帧：有子节点则取 1 帧，否则 0 帧 + frame_paths: list[str] = [] + if chosen_l2.children: + first_frame = chosen_l2.children[0].frame_path + if first_frame: + frame_paths = [first_frame] + else: + # 普通 L2：随机采样 2-3 个 L3 帧 + children_with_frames = [l3 for l3 in chosen_l2.children if l3.frame_path] + n_frames = min(rng.randint(2, 3), len(children_with_frames)) + sampled = rng.sample(children_with_frames, n_frames) if n_frames > 0 else [] + frame_paths = [l3.frame_path for l3 in sampled if l3.frame_path] + + # Phase 4: card_text + card_text = f"event_description: {chosen_l2.card.event_description}" + if is_temporal_perception: + card_text += f"\n{_l2_time_range_str(chosen_l2)}" + + # Phase 5: 字幕（L2 无自身字幕，取首个子 L3 字幕） + subtitle = "" + if chosen_l2.children and chosen_l2.children[0].subtitle: + subtitle = chosen_l2.children[0].subtitle + + # Phase 6: 干扰项——同 L1 下其他 L2 的 event_description + distractor_texts = [ + l2.card.event_description for l2 in parent_l1.children if l2.id != chosen_l2.id + ] + + return AnchorContext( + node_id=chosen_l2.id, + card_text=card_text, + frame_paths=frame_paths, + subtitle=subtitle, + distractor_texts=distractor_texts, + ) + + +def _sample_l1( + tree: TreeIndex, + task_type: str, + spec: TaskTypeSpec, + used_node_ids: set[str], + rng: random.Random, +) -> AnchorContext: + """L1 层级锚节点采样（Temporal Reasoning / Information Synopsis）。 + + Information Synopsis：使用目标 L1 下全部 L2 子节点。 + Temporal Reasoning：使用 >=3 个 L2 子节点（不足 3 个则全部使用）。 + L2 按 time_range 升序排列，每个 L2 取一帧代表。 + + 参数: + tree: 三层树索引。 + task_type: 题型名称。 + spec: 题型规格。 + used_node_ids: 已用节点 ID 集合。 + rng: 随机数生成器。 + + 返回: + AnchorContext 实例。 + + 异常: + ValueError: 候选 L1 节点不足。 + """ + # Phase 1: 收集可用 L1 + candidates = [r for r in tree.roots if r.id not in used_node_ids] + if not candidates: + raise ValueError(f"锚节点不足: {task_type} 无可用 L1 节点") + + # Phase 2: 随机选取 + chosen_l1 = rng.choice(candidates) + + # Phase 3: 选定 L2 子集 + if task_type == "Information Synopsis": + # 必须使用全部 L2 + selected_l2 = list(chosen_l1.children) + else: + # Temporal Reasoning：>=3 个 L2（不足则全部） + if len(chosen_l1.children) <= 3: + selected_l2 = list(chosen_l1.children) + else: + selected_l2 = rng.sample(chosen_l1.children, rng.randint(3, len(chosen_l1.children))) + + # Phase 4: 按 time_range 升序排列 + selected_l2 = _sort_l2_by_time(selected_l2) + + # Phase 5: card_text（场景摘要） + card_text = f"scene_summary: {chosen_l1.card.scene_summary}" + + # Phase 6: 帧路径——每个 L2 取一帧代表 + frame_paths: list[str] = [] + for l2 in selected_l2: + rep = _representative_frame(l2) + if rep: + frame_paths.append(rep) + + # Phase 7: 字幕（L1 无字幕） + subtitle = "" + + # Phase 8: 干扰项——其他 L1 的 scene_summary + distractor_texts = [r.card.scene_summary for r in tree.roots if r.id != chosen_l1.id] + + return AnchorContext( + node_id=chosen_l1.id, + card_text=card_text, + frame_paths=frame_paths, + subtitle=subtitle, + distractor_texts=distractor_texts, + ) + + +def _sample_l1_l2( + tree: TreeIndex, + task_type: str, + spec: TaskTypeSpec, + used_node_ids: set[str], + rng: random.Random, +) -> AnchorContext: + """L1-L2 跨层级锚节点采样（Object Reasoning）。 + + 从全部 L2 中随机选 2-3 个，按 time_range 排序， + card_text 为各 L2 的 event_description 拼接。 + + 参数: + tree: 三层树索引。 + task_type: 题型名称。 + spec: 题型规格。 + used_node_ids: 已用节点 ID 集合。 + rng: 随机数生成器。 + + 返回: + AnchorContext 实例。 + + 异常: + ValueError: 候选 L2 节点不足。 + """ + # Phase 1: 收集全部 L2 + all_l2: list[L2Node] = [] + for root in tree.roots: + for l2 in root.children: + if l2.id not in used_node_ids: + all_l2.append(l2) + + if not all_l2: + raise ValueError(f"锚节点不足: {task_type} 无可用 L2 节点") + + # Phase 2: 随机选 2-3 个 + n_pick = min(rng.randint(2, 3), len(all_l2)) + selected = rng.sample(all_l2, n_pick) + + # Phase 3: 按 time_range 升序排列 + selected = _sort_l2_by_time(selected) + + # Phase 4: card_text = 各 L2 event_description 拼接 + card_text = "\n".join(f"event_description: {l2.card.event_description}" for l2 in selected) + + # Phase 5: 帧路径——每个 L2 取一帧代表 + frame_paths: list[str] = [] + for l2 in selected: + rep = _representative_frame(l2) + if rep: + frame_paths.append(rep) + + # Phase 6: 字幕 + subtitle = "" + + # Phase 7: 干扰项——未被选中的 L2 的 event_description + selected_ids = {l2.id for l2 in selected} + distractor_texts = [l2.card.event_description for l2 in all_l2 if l2.id not in selected_ids] + + # 使用第一个被选中节点的 ID 作为锚节点 ID + anchor_id = selected[0].id + + return AnchorContext( + node_id=anchor_id, + card_text=card_text, + frame_paths=frame_paths, + subtitle=subtitle, + distractor_texts=distractor_texts, + ) + + +# --------------------------------------------------------------------------- +# 公开接口 +# --------------------------------------------------------------------------- + + +def sample_anchor( + tree: TreeIndex, + task_type: str, + used_node_ids: set[str], + rng: random.Random, +) -> AnchorContext: + """根据题型从视频树中采样锚节点及上下文素材。 + + 依据 TASK_TYPE_LEVEL_MAP 中的层级规格，分发到对应的层级采样策略。 + 每种层级有不同的帧选取、card 序列化和干扰项收集逻辑。 + + 参数: + tree: 三层树索引。 + task_type: 12 种 Video-MME 题型之一。 + used_node_ids: 本轮已用节点 ID 集合（避免重复采样）。 + rng: 可控随机数生成器（保证可复现）。 + + 返回: + AnchorContext 实例，包含锚节点 ID、card 文本、帧路径、字幕和干扰项。 + + 异常: + KeyError: task_type 不在 TASK_TYPE_LEVEL_MAP 中。 + ValueError: 候选节点不足（全部被 used_node_ids 排除）。 + """ + spec = TASK_TYPE_LEVEL_MAP[task_type] + + if spec.level == "L3": + return _sample_l3(tree, task_type, spec, used_node_ids, rng) + elif spec.level == "L2": + return _sample_l2(tree, task_type, spec, used_node_ids, rng) + elif spec.level == "L1": + return _sample_l1(tree, task_type, spec, used_node_ids, rng) + elif spec.level == "L1-L2": + return _sample_l1_l2(tree, task_type, spec, used_node_ids, rng) + else: + raise ValueError(f"未知层级: {spec.level}") diff --git a/tests/unit/test_synthesizer.py b/tests/unit/test_synthesizer.py index 579b2e6..2437111 100644 --- a/tests/unit/test_synthesizer.py +++ b/tests/unit/test_synthesizer.py @@ -1,10 +1,20 @@ -"""synthesizer 模块单元测试 — AnchorContext + 题型映射常量。""" +"""synthesizer 模块单元测试 — AnchorContext + 题型映射常量 + sample_anchor。""" from __future__ import annotations import dataclasses +import random +from pathlib import Path -from app.question_gen.synthesizer import TASK_TYPE_LEVEL_MAP, AnchorContext, TaskTypeSpec +import pytest + +from app.question_gen.synthesizer import ( + TASK_TYPE_LEVEL_MAP, + AnchorContext, + TaskTypeSpec, + sample_anchor, +) +from app.tree.index import TreeIndex ALL_12_TYPES = [ "Object Recognition", @@ -132,3 +142,89 @@ class TestTaskTypeSpec: assert isinstance(spec.context_fields, tuple), ( f"{task_type} 的 context_fields 不是 tuple" ) + + +# --------------------------------------------------------------------------- +# sample_anchor 测试 +# --------------------------------------------------------------------------- + + +def _load_test_tree() -> tuple[TreeIndex, str]: + """加载真实测试树（store/videos/ 下第一棵）。""" + videos_dir = Path("store/videos") + first_vid = sorted(videos_dir.iterdir())[0] + tree = TreeIndex.load_json(str(first_vid / "tree.json")) + return tree, first_vid.name + + +class TestSampleAnchor: + """sample_anchor 锚节点采样测试（基于真实树数据）。""" + + def test_l3_type_returns_single_frame(self) -> None: + """L3 题型（Object Recognition）应返回恰好 1 帧。""" + tree, _vid = _load_test_tree() + ctx = sample_anchor(tree, "Object Recognition", set(), random.Random(42)) + assert len(ctx.frame_paths) == 1 + assert ctx.node_id.startswith("L") or "_L3_" in ctx.node_id + assert len(ctx.distractor_texts) > 0 + + def test_l2_type_returns_multiple_frames(self) -> None: + """L2 题型（Action Reasoning）应返回 2-3 帧。""" + tree, _vid = _load_test_tree() + ctx = sample_anchor(tree, "Action Reasoning", set(), random.Random(42)) + assert 2 <= len(ctx.frame_paths) <= 3 + + def test_temporal_perception_zero_or_one_frame(self) -> None: + """Temporal Perception 特殊处理：0-1 帧，且 card_text 包含 time_range。""" + tree, _vid = _load_test_tree() + ctx = sample_anchor(tree, "Temporal Perception", set(), random.Random(42)) + assert len(ctx.frame_paths) <= 1 + assert "time_range" in ctx.card_text.lower() or "time" in ctx.card_text.lower() + + def test_information_synopsis_uses_all_l2(self) -> None: + """Information Synopsis 必须使用目标 L1 下所有 L2 子节点。""" + tree, _vid = _load_test_tree() + ctx = sample_anchor(tree, "Information Synopsis", set(), random.Random(42)) + # 至少应有帧（每个 L2 取一帧代表） + total_l2 = sum(len(r.children) for r in tree.roots) + assert len(ctx.frame_paths) >= min(total_l2, 1) + + def test_l1_type_l2_nodes_in_time_order(self) -> None: + """L1 题型（Temporal Reasoning）的 card_text 应有实质内容。""" + tree, _vid = _load_test_tree() + ctx = sample_anchor(tree, "Temporal Reasoning", set(), random.Random(42)) + assert len(ctx.frame_paths) >= 1 + assert len(ctx.card_text) > 20 + + def test_used_node_ids_excluded(self) -> None: + """used_node_ids 中的节点不应被再次选中。""" + tree, _vid = _load_test_tree() + rng = random.Random(42) + ctx1 = sample_anchor(tree, "Object Recognition", set(), rng) + ctx2 = sample_anchor(tree, "Object Recognition", {ctx1.node_id}, random.Random(43)) + assert ctx2.node_id != ctx1.node_id + + def test_insufficient_nodes_raises(self) -> None: + """所有候选节点均被排除时应抛出 ValueError。""" + tree, _vid = _load_test_tree() + all_l3_ids: set[str] = set() + for root in tree.roots: + for l2 in root.children: + for l3 in l2.children: + all_l3_ids.add(l3.id) + with pytest.raises(ValueError, match="锚节点不足"): + sample_anchor(tree, "Object Recognition", all_l3_ids, random.Random(42)) + + def test_object_reasoning_l1_l2_type(self) -> None: + """Object Reasoning (L1-L2) 应选 2-3 个 L2 并按时间排序。""" + tree, _vid = _load_test_tree() + ctx = sample_anchor(tree, "Object Reasoning", set(), random.Random(42)) + assert 1 <= len(ctx.frame_paths) <= 3 + assert len(ctx.card_text) > 10 + + def test_spatial_reasoning_context_fields(self) -> None: + """Spatial Reasoning 的 card_text 应包含 spatial_layout 内容。""" + tree, _vid = _load_test_tree() + ctx = sample_anchor(tree, "Spatial Reasoning", set(), random.Random(42)) + # context_fields 包含 spatial_layout，card_text 应含有该字段内容 + assert len(ctx.card_text) > 20 From 40b04f886e4b9954f58d38d931cee16a6ae72dc4 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 05:24:30 -0400 Subject: [PATCH 12/21] =?UTF-8?q?fix(question=5Fgen):=20sample=5Fanchor=20?= =?UTF-8?q?Codex=20=E5=AE=A1=E6=9F=A5=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - C1/C2: Temporal Reasoning ≥3 L2 + Object Reasoning ≥2 L2 下限检查 - I1: L2 题型子帧不足时 ValueError - I2: L3 过滤无 frame_path 的节点 - I3/I4: distractor_texts 扩展到整棵树范围 - I5-I7: 测试补强 Information Synopsis/Temporal Reasoning/Object Reasoning - M1: Spatial Reasoning 测试断言 spatial_layout Co-Authored-By: Claude Opus 4.6 (1M context) --- app/question_gen/synthesizer.py | 61 ++++++++++++++++++++++----------- tests/unit/test_synthesizer.py | 17 ++++----- 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/app/question_gen/synthesizer.py b/app/question_gen/synthesizer.py index d5ee161..fb9ceaf 100644 --- a/app/question_gen/synthesizer.py +++ b/app/question_gen/synthesizer.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: import random - from app.tree.index import L1Node, L2Node, L3Node, TreeIndex + from app.tree.index import L2Node, L3Node, TreeIndex @dataclass(frozen=True) @@ -170,27 +170,33 @@ def _sample_l3( 异常: ValueError: 候选 L3 节点不足。 """ - # Phase 1: 收集所有 L3 候选 + # Phase 1: 收集所有 L3 候选（必须有 frame_path） candidates: list[tuple[L3Node, L2Node]] = [] for root in tree.roots: for l2 in root.children: for l3 in l2.children: - if l3.id not in used_node_ids: + if l3.id not in used_node_ids and l3.frame_path: candidates.append((l3, l2)) if not candidates: - raise ValueError(f"锚节点不足: {task_type} 无可用 L3 节点") + raise ValueError(f"锚节点不足: {task_type} 无可用 L3 节点（需具备 frame_path）") # Phase 2: 随机选取 chosen_l3, parent_l2 = rng.choice(candidates) - # Phase 3: 构造上下文 + # Phase 3: 构造上下文（frame_path 已在候选过滤中保证非 None） card_text = _serialize_l3_card(chosen_l3, spec.context_fields) - frame_paths = [chosen_l3.frame_path] if chosen_l3.frame_path else [] + frame_paths = [chosen_l3.frame_path] # type: ignore[list-item] subtitle = chosen_l3.subtitle or "" - # Phase 4: 干扰项——同 L2 下其他 L3 的 frame_summary - distractor_texts = [l3.card.frame_summary for l3 in parent_l2.children if l3.id != chosen_l3.id] + # Phase 4: 干扰项——整棵树中其他 L3 的 frame_summary + distractor_texts = [ + l3.card.frame_summary + for root in tree.roots + for l2 in root.children + for l3 in l2.children + if l3.id != chosen_l3.id + ] return AnchorContext( node_id=chosen_l3.id, @@ -227,17 +233,17 @@ def _sample_l2( ValueError: 候选 L2 节点不足。 """ # Phase 1: 收集所有 L2 候选 - all_l2: list[tuple[L2Node, L1Node]] = [] + all_l2: list[L2Node] = [] for root in tree.roots: for l2 in root.children: if l2.id not in used_node_ids: - all_l2.append((l2, root)) + all_l2.append(l2) if not all_l2: raise ValueError(f"锚节点不足: {task_type} 无可用 L2 节点") # Phase 2: 随机选取 - chosen_l2, parent_l1 = rng.choice(all_l2) + chosen_l2 = rng.choice(all_l2) is_temporal_perception = task_type == "Temporal Perception" @@ -252,8 +258,13 @@ def _sample_l2( else: # 普通 L2：随机采样 2-3 个 L3 帧 children_with_frames = [l3 for l3 in chosen_l2.children if l3.frame_path] + if len(children_with_frames) < 2: + raise ValueError( + f"锚节点不足: {task_type} 需要 >=2 个子帧，" + f"但 {chosen_l2.id} 仅有 {len(children_with_frames)} 个可用帧" + ) n_frames = min(rng.randint(2, 3), len(children_with_frames)) - sampled = rng.sample(children_with_frames, n_frames) if n_frames > 0 else [] + sampled = rng.sample(children_with_frames, n_frames) frame_paths = [l3.frame_path for l3 in sampled if l3.frame_path] # Phase 4: card_text @@ -266,9 +277,12 @@ def _sample_l2( if chosen_l2.children and chosen_l2.children[0].subtitle: subtitle = chosen_l2.children[0].subtitle - # Phase 6: 干扰项——同 L1 下其他 L2 的 event_description + # Phase 6: 干扰项——整棵树中其他 L2 的 event_description distractor_texts = [ - l2.card.event_description for l2 in parent_l1.children if l2.id != chosen_l2.id + l2.card.event_description + for root in tree.roots + for l2 in root.children + if l2.id != chosen_l2.id ] return AnchorContext( @@ -290,7 +304,7 @@ def _sample_l1( """L1 层级锚节点采样（Temporal Reasoning / Information Synopsis）。 Information Synopsis：使用目标 L1 下全部 L2 子节点。 - Temporal Reasoning：使用 >=3 个 L2 子节点（不足 3 个则全部使用）。 + Temporal Reasoning：严格要求 >=3 个 L2 子节点，不足则抛 ValueError。 L2 按 time_range 升序排列，每个 L2 取一帧代表。 参数: @@ -304,7 +318,7 @@ def _sample_l1( AnchorContext 实例。 异常: - ValueError: 候选 L1 节点不足。 + ValueError: 候选 L1 节点不足，或 Temporal Reasoning 的 L2 子节点 <3。 """ # Phase 1: 收集可用 L1 candidates = [r for r in tree.roots if r.id not in used_node_ids] @@ -319,8 +333,13 @@ def _sample_l1( # 必须使用全部 L2 selected_l2 = list(chosen_l1.children) else: - # Temporal Reasoning：>=3 个 L2（不足则全部） - if len(chosen_l1.children) <= 3: + # Temporal Reasoning：严格要求 >=3 个 L2 + if len(chosen_l1.children) < 3: + raise ValueError( + f"锚节点不足: {task_type} 需要 >=3 个 L2 子节点，" + f"但 {chosen_l1.id} 仅有 {len(chosen_l1.children)} 个" + ) + if len(chosen_l1.children) == 3: selected_l2 = list(chosen_l1.children) else: selected_l2 = rng.sample(chosen_l1.children, rng.randint(3, len(chosen_l1.children))) @@ -385,8 +404,10 @@ def _sample_l1_l2( if l2.id not in used_node_ids: all_l2.append(l2) - if not all_l2: - raise ValueError(f"锚节点不足: {task_type} 无可用 L2 节点") + if len(all_l2) < 2: + raise ValueError( + f"锚节点不足: {task_type} 需要 >=2 个 L2 节点，但仅有 {len(all_l2)} 个可用" + ) # Phase 2: 随机选 2-3 个 n_pick = min(rng.randint(2, 3), len(all_l2)) diff --git a/tests/unit/test_synthesizer.py b/tests/unit/test_synthesizer.py index 2437111..eaf1d3b 100644 --- a/tests/unit/test_synthesizer.py +++ b/tests/unit/test_synthesizer.py @@ -185,15 +185,15 @@ class TestSampleAnchor: """Information Synopsis 必须使用目标 L1 下所有 L2 子节点。""" tree, _vid = _load_test_tree() ctx = sample_anchor(tree, "Information Synopsis", set(), random.Random(42)) - # 至少应有帧（每个 L2 取一帧代表） - total_l2 = sum(len(r.children) for r in tree.roots) - assert len(ctx.frame_paths) >= min(total_l2, 1) + # 找到被选中的 L1，验证 frame_paths 数量 == 该 L1 下全部 L2 数量 + chosen_l1 = next(r for r in tree.roots if r.id == ctx.node_id) + assert len(ctx.frame_paths) == len(chosen_l1.children) def test_l1_type_l2_nodes_in_time_order(self) -> None: - """L1 题型（Temporal Reasoning）的 card_text 应有实质内容。""" + """Temporal Reasoning 应返回 >=3 帧且 card_text 有实质内容。""" tree, _vid = _load_test_tree() ctx = sample_anchor(tree, "Temporal Reasoning", set(), random.Random(42)) - assert len(ctx.frame_paths) >= 1 + assert len(ctx.frame_paths) >= 3 assert len(ctx.card_text) > 20 def test_used_node_ids_excluded(self) -> None: @@ -219,12 +219,13 @@ class TestSampleAnchor: """Object Reasoning (L1-L2) 应选 2-3 个 L2 并按时间排序。""" tree, _vid = _load_test_tree() ctx = sample_anchor(tree, "Object Reasoning", set(), random.Random(42)) - assert 1 <= len(ctx.frame_paths) <= 3 + assert 2 <= len(ctx.frame_paths) <= 3 assert len(ctx.card_text) > 10 def test_spatial_reasoning_context_fields(self) -> None: - """Spatial Reasoning 的 card_text 应包含 spatial_layout 内容。""" + """Spatial Reasoning 的 card_text 应包含 spatial_layout 字段。""" tree, _vid = _load_test_tree() ctx = sample_anchor(tree, "Spatial Reasoning", set(), random.Random(42)) - # context_fields 包含 spatial_layout，card_text 应含有该字段内容 + # context_fields 包含 spatial_layout，card_text 必须出现该字段名 + assert "spatial_layout" in ctx.card_text assert len(ctx.card_text) > 20 From 90f17e330e82b5094d87a9bf4662e7f8a6d07559 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 05:28:25 -0400 Subject: [PATCH 13/21] feat(question_gen): build_generation_prompt + parse_vlm_response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - prompt 组装：system(角色+题型+约束+few-shot) + user(card+字幕+干扰项) - VLM 响应解析：JSON 直接 + markdown code block 回退，四选一 schema 校验 Co-Authored-By: Claude Opus 4.6 (1M context) --- app/question_gen/synthesizer.py | 140 ++++++++++++++++++++++++++++- tests/unit/test_synthesizer.py | 152 +++++++++++++++++++++++++++++++- 2 files changed, 290 insertions(+), 2 deletions(-) diff --git a/app/question_gen/synthesizer.py b/app/question_gen/synthesizer.py index fb9ceaf..c070723 100644 --- a/app/question_gen/synthesizer.py +++ b/app/question_gen/synthesizer.py @@ -1,4 +1,4 @@ -"""赛题合成核心逻辑 — 节点采样、prompt 构造、去重。 +"""赛题合成核心逻辑 — 节点采样、prompt 构造、VLM 响应解析、去重。 纯函数为主，异步编排仅 generate_one。 通过 DI 接收 VLMProvider / EmbeddingProvider，不 import adapters/。 @@ -6,6 +6,9 @@ from __future__ import annotations +import contextlib +import json +import re from dataclasses import dataclass from typing import TYPE_CHECKING @@ -13,6 +16,7 @@ if TYPE_CHECKING: import random from app.tree.index import L2Node, L3Node, TreeIndex + from core.types import GeneratedQuestion @dataclass(frozen=True) @@ -486,3 +490,137 @@ def sample_anchor( return _sample_l1_l2(tree, task_type, spec, used_node_ids, rng) else: raise ValueError(f"未知层级: {spec.level}") + + +# --------------------------------------------------------------------------- +# Prompt 构造与 VLM 响应解析 +# --------------------------------------------------------------------------- + +_VALID_ANSWERS = frozenset({"A", "B", "C", "D"}) + + +def build_generation_prompt( + task_type: str, + anchor: AnchorContext, + exemplars: list[GeneratedQuestion], +) -> tuple[list[dict[str, str]], list[str]]: + """组装 VLM 出题 prompt。 + + 构造 OpenAI 格式的 messages 列表和帧图片路径列表， + 供 VLMProvider.chat_with_images 直接消费。 + + 参数: + task_type: 题型名称（如 "Object Recognition"）。 + anchor: 锚节点上下文（card_text, subtitle, distractor_texts, frame_paths）。 + exemplars: 少样本示例列表（可为空）。 + + 返回: + (messages, image_paths) — messages 为 OpenAI 格式消息列表， + image_paths 为帧图片路径列表，直接喂给 VLMProvider.chat_with_images。 + """ + # Phase 1: 构造 system message + system_parts: list[str] = [ + "你是一个视频理解题目生成器。", + f"题型: {task_type}", + "约束:", + "- 题目必须基于提供的节点内容", + "- 干扰选项应来自其他节点的信息", + "- 生成风格应与示例保持一致", + '- 以 JSON 格式返回: {"question": "...", "options": ["A. ...", "B. ...", "C. ...", "D. ..."], "answer": "A/B/C/D"}', + ] + + # Phase 2: 加入 few-shot 示例 + if exemplars: + system_parts.append("\n示例:") + for i, ex in enumerate(exemplars, 1): + system_parts.append(f" 示例 {i}:") + system_parts.append(f" question: {ex.question}") + system_parts.append(f" options: {list(ex.options)}") + system_parts.append(f" answer: {ex.answer}") + + system_content = "\n".join(system_parts) + + # Phase 3: 构造 user message + user_parts: list[str] = [f"节点内容:\n{anchor.card_text}"] + + if anchor.subtitle: + user_parts.append(f"\n字幕:\n{anchor.subtitle}") + + if anchor.distractor_texts: + user_parts.append("\n干扰项来源节点摘要:") + for dt in anchor.distractor_texts: + user_parts.append(f"- {dt}") + + user_content = "\n".join(user_parts) + + messages = [ + {"role": "system", "content": system_content}, + {"role": "user", "content": user_content}, + ] + + return messages, list(anchor.frame_paths) + + +def parse_vlm_response( + raw: str, + video_id: str, + task_type: str, + seq: int, +) -> dict: + """解析 VLM 返回的 JSON → 部分字段字典。 + + 尝试直接解析 JSON；若失败，从 markdown 代码块中提取后重试。 + 校验必需字段、选项数量和答案合法性。 + + 参数: + raw: VLM 原始返回文本。 + video_id: 所属视频标识。 + task_type: 题型名称（用于错误消息）。 + seq: 序列号，用于生成 question_id。 + + 返回: + {"question_id": "gen-{video_id}-{seq:03d}", "question": ..., "options": [...], "answer": ...} + 调用方（generate_one）补齐 source_nodes/difficulty 后构造 GeneratedQuestion。 + + 异常: + ValueError: JSON 解析失败、缺必需字段、options 非 4 项、answer 不在 A-D。 + """ + # Phase 1: 尝试直接解析 JSON + data = None + with contextlib.suppress(json.JSONDecodeError): + data = json.loads(raw) + + # Phase 2: 从 markdown 代码块提取 JSON + if data is None: + match = re.search(r"```(?:json)?\s*\n?(.*?)\n?\s*```", raw, re.DOTALL) + if match: + with contextlib.suppress(json.JSONDecodeError): + data = json.loads(match.group(1)) + + if data is None: + raise ValueError(f"VLM 返回无法解析为 JSON: {raw[:200]}") + + # Phase 3: 校验必需字段 + required = ("question", "options", "answer") + missing = [f for f in required if f not in data] + if missing: + raise ValueError(f"VLM 返回缺少必需字段 {missing}: {raw[:200]}") + + # Phase 4: options 必须恰好 4 项 + options = data["options"] + if not isinstance(options, list) or len(options) != 4: + raise ValueError( + f"options 必须恰好 4 项，实际 {len(options) if isinstance(options, list) else type(options).__name__}: {raw[:200]}" + ) + + # Phase 5: answer 必须是 A-D + answer = data["answer"] + if answer not in _VALID_ANSWERS: + raise ValueError(f"answer 必须是 A/B/C/D 之一，实际 '{answer}': {raw[:200]}") + + return { + "question_id": f"gen-{video_id}-{seq:03d}", + "question": data["question"], + "options": list(options), + "answer": answer, + } diff --git a/tests/unit/test_synthesizer.py b/tests/unit/test_synthesizer.py index eaf1d3b..691a7c4 100644 --- a/tests/unit/test_synthesizer.py +++ b/tests/unit/test_synthesizer.py @@ -1,4 +1,4 @@ -"""synthesizer 模块单元测试 — AnchorContext + 题型映射常量 + sample_anchor。""" +"""synthesizer 模块单元测试 — AnchorContext + 题型映射常量 + sample_anchor + prompt 构造 + VLM 解析。""" from __future__ import annotations @@ -12,9 +12,12 @@ from app.question_gen.synthesizer import ( TASK_TYPE_LEVEL_MAP, AnchorContext, TaskTypeSpec, + build_generation_prompt, + parse_vlm_response, sample_anchor, ) from app.tree.index import TreeIndex +from core.types import GeneratedQuestion ALL_12_TYPES = [ "Object Recognition", @@ -229,3 +232,150 @@ class TestSampleAnchor: # context_fields 包含 spatial_layout，card_text 必须出现该字段名 assert "spatial_layout" in ctx.card_text assert len(ctx.card_text) > 20 + + +# --------------------------------------------------------------------------- +# build_generation_prompt 测试 +# --------------------------------------------------------------------------- + + +class TestBuildGenerationPrompt: + """build_generation_prompt 消息结构与内容测试。""" + + def test_messages_structure(self) -> None: + """带 exemplars 时，system 包含题型和示例，image_paths 来自 anchor。""" + anchor = AnchorContext( + node_id="L3_001", + card_text="A person typing on a laptop", + frame_paths=["store/videos/test/frames/L1_000_L2_000_L3_000.jpg"], + subtitle="Hello world", + distractor_texts=["Another person walking in park"], + ) + exemplars = [ + GeneratedQuestion( + question_id="ex-1", + video_id="v1", + task_type="Object Recognition", + question="What object?", + options=("A. Cat", "B. Dog", "C. Bird", "D. Fish"), + answer="A", + source_nodes=(), + difficulty="medium", + ), + ] + messages, image_paths = build_generation_prompt( + "Object Recognition", + anchor, + exemplars, + ) + assert messages[0]["role"] == "system" + assert "Object Recognition" in messages[0]["content"] + assert any("What object?" in str(m) for m in messages) + assert image_paths == anchor.frame_paths + + def test_distractor_in_user_message(self) -> None: + """干扰项文本应出现在 user message 中。""" + anchor = AnchorContext( + node_id="L2_003", + card_text="Event card text", + frame_paths=["a.jpg", "b.jpg"], + subtitle="", + distractor_texts=["Distractor node summary"], + ) + messages, _ = build_generation_prompt("Action Reasoning", anchor, []) + user_msg = [m for m in messages if m["role"] == "user"][0] + assert "Distractor node summary" in user_msg["content"] + + def test_no_exemplars_no_crash(self) -> None: + """exemplars 为空时不应报错，system 消息中无示例段落。""" + anchor = AnchorContext( + node_id="L3_010", + card_text="Some card text", + frame_paths=["frame.jpg"], + subtitle="", + distractor_texts=[], + ) + messages, image_paths = build_generation_prompt("OCR Problems", anchor, []) + assert len(messages) >= 2 + assert image_paths == ["frame.jpg"] + + def test_subtitle_included_when_non_empty(self) -> None: + """非空 subtitle 应出现在 user message 中。""" + anchor = AnchorContext( + node_id="L3_002", + card_text="Card text here", + frame_paths=["f.jpg"], + subtitle="This is a subtitle line", + distractor_texts=[], + ) + messages, _ = build_generation_prompt("Attribute Perception", anchor, []) + user_msg = [m for m in messages if m["role"] == "user"][0] + assert "This is a subtitle line" in user_msg["content"] + + def test_empty_subtitle_not_in_user_message(self) -> None: + """空 subtitle 不应在 user message 中产生 subtitle 段落。""" + anchor = AnchorContext( + node_id="L3_003", + card_text="Card", + frame_paths=["f.jpg"], + subtitle="", + distractor_texts=[], + ) + messages, _ = build_generation_prompt("OCR Problems", anchor, []) + user_msg = [m for m in messages if m["role"] == "user"][0] + # 不应出现空的 subtitle 标记 + assert "字幕" not in user_msg["content"] + + +# --------------------------------------------------------------------------- +# parse_vlm_response 测试 +# --------------------------------------------------------------------------- + + +class TestParseVlmResponse: + """parse_vlm_response 解析与校验测试。""" + + def test_valid_json(self) -> None: + """合法 JSON 正常解析，question_id 格式正确。""" + raw = '{"question": "What?", "options": ["A. X", "B. Y", "C. Z", "D. W"], "answer": "A"}' + result = parse_vlm_response(raw, "vid1", "Object Recognition", 1) + assert result["question"] == "What?" + assert result["answer"] == "A" + assert len(result["options"]) == 4 + assert result["question_id"] == "gen-vid1-001" + + def test_json_in_code_block(self) -> None: + """从 markdown 代码块中提取 JSON。""" + raw = '```json\n{"question": "Q?", "options": ["A. 1", "B. 2", "C. 3", "D. 4"], "answer": "B"}\n```' + result = parse_vlm_response(raw, "vid1", "Object Recognition", 2) + assert result["question"] == "Q?" + assert result["question_id"] == "gen-vid1-002" + + def test_invalid_json_raises(self) -> None: + """非 JSON 文本应抛出 ValueError。""" + with pytest.raises(ValueError, match="VLM 返回"): + parse_vlm_response("not json", "vid1", "Object Recognition", 1) + + def test_missing_fields_raises(self) -> None: + """缺少必需字段应抛出 ValueError。""" + raw = '{"question": "What?"}' + with pytest.raises(ValueError): + parse_vlm_response(raw, "vid1", "Object Recognition", 1) + + def test_options_must_be_four(self) -> None: + """options 非 4 项应抛出 ValueError。""" + raw = '{"question": "Q?", "options": ["A. X", "B. Y"], "answer": "A"}' + with pytest.raises(ValueError, match="4"): + parse_vlm_response(raw, "vid1", "Object Recognition", 1) + + def test_answer_must_be_abcd(self) -> None: + """answer 不在 A-D 范围应抛出 ValueError。""" + raw = '{"question": "Q?", "options": ["A. X", "B. Y", "C. Z", "D. W"], "answer": "E"}' + with pytest.raises(ValueError, match="A.*D"): + parse_vlm_response(raw, "vid1", "Object Recognition", 1) + + def test_seq_zero_padded(self) -> None: + """seq 应按 3 位零填充格式化到 question_id 中。""" + raw = '{"question": "Q?", "options": ["A. 1", "B. 2", "C. 3", "D. 4"], "answer": "C"}' + result = parse_vlm_response(raw, "video_abc", "Action Reasoning", 42) + assert result["question_id"] == "gen-video_abc-042" From 5aa7cc48c57396cc8c0c372c01bac3f91bf16b1d Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 05:32:39 -0400 Subject: [PATCH 14/21] =?UTF-8?q?feat(question=5Fgen):=20is=5Fduplicate=20?= =?UTF-8?q?+=20generate=5Fone=20=E2=80=94=20=E5=8E=BB=E9=87=8D=E5=88=A4?= =?UTF-8?q?=E5=AE=9A=E4=B8=8E=E5=8D=95=E9=A2=98=E7=94=9F=E6=88=90=E7=BC=96?= =?UTF-8?q?=E6=8E=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - is_duplicate: 余弦相似度去重，空池短路 - generate_one: 异步重试循环，不含去重（由调用方汇总点原子执行） Co-Authored-By: Claude Opus 4.6 (1M context) --- app/question_gen/synthesizer.py | 134 ++++++++++++++++++++++++++++++++ tests/unit/test_synthesizer.py | 122 ++++++++++++++++++++++++++++- 2 files changed, 255 insertions(+), 1 deletion(-) diff --git a/app/question_gen/synthesizer.py b/app/question_gen/synthesizer.py index c070723..fbbb83e 100644 --- a/app/question_gen/synthesizer.py +++ b/app/question_gen/synthesizer.py @@ -12,10 +12,15 @@ import re from dataclasses import dataclass from typing import TYPE_CHECKING +import numpy as np +from loguru import logger + if TYPE_CHECKING: import random + from collections.abc import Callable from app.tree.index import L2Node, L3Node, TreeIndex + from core.protocols import VLMProvider from core.types import GeneratedQuestion @@ -624,3 +629,132 @@ def parse_vlm_response( "options": list(options), "answer": answer, } + + +# --------------------------------------------------------------------------- +# Embedding 去重 +# --------------------------------------------------------------------------- + + +def is_duplicate( + question_text: str, + pool_embeddings: np.ndarray, + embed_fn: Callable[[str | list[str]], np.ndarray], + threshold: float, +) -> bool: + """embedding 去重判定。 + + 参数: + question_text: 待检查的题目文本。 + pool_embeddings: 已有题目的 embedding 矩阵 [N, D]（L2 归一化）。 + embed_fn: 文本嵌入函数，返回 [N, D] ndarray（L2 归一化）。 + threshold: 余弦相似度阈值。 + + 返回: + True 表示与池中某题重复。空池永远返回 False。 + """ + if pool_embeddings.shape[0] == 0: + return False + + query = embed_fn(question_text) # [1, D] + query = query.squeeze(0) # [D] + similarities = pool_embeddings @ query # [N] + return bool(np.max(similarities) >= threshold) + + +# --------------------------------------------------------------------------- +# 单题生成 +# --------------------------------------------------------------------------- + + +async def generate_one( + vlm: VLMProvider, + embed_fn: Callable[[str | list[str]], np.ndarray], + tree: TreeIndex, + video_id: str, + task_type: str, + seq: int, + *, + exemplars: list[GeneratedQuestion], + used_node_ids: set[str], + max_retries: int, + similarity_threshold: float, + rng: random.Random, + session_id: str, +) -> GeneratedQuestion | None: + """生成单道候选题（不含去重——去重在调用方汇总点原子执行）。 + + 循环最多 max_retries 次尝试生成。每次尝试： + 1. 采样锚节点 + 2. 构造 prompt + 3. 调用 VLM + 4. 解析响应 + 5. 构造 GeneratedQuestion + + 返回 None 表示耗尽重试。 + + 参数: + vlm: VLM 调用端口。 + embed_fn: 文本嵌入函数（本函数内未使用，由调用方统一去重）。 + tree: 三层树索引。 + video_id: 所属视频标识。 + task_type: 12 种 Video-MME 题型之一。 + seq: 序列号，用于生成 question_id。 + exemplars: 少样本示例列表。 + used_node_ids: 已用节点 ID 集合。 + max_retries: 最大重试次数。 + similarity_threshold: 余弦相似度阈值（本函数内未使用）。 + rng: 可控随机数生成器。 + session_id: 会话 ID（传递给 VLM 遥测）。 + + 返回: + GeneratedQuestion 实例，或 None（耗尽重试）。 + """ + from core.types import GeneratedQuestion as _GeneratedQuestion + + for attempt in range(max_retries): + try: + # Phase 1: 采样锚节点 + anchor = sample_anchor(tree, task_type, used_node_ids, rng) + + # Phase 2: 构造 prompt + messages, images = build_generation_prompt(task_type, anchor, exemplars) + + # Phase 3: 调用 VLM + response = await vlm.chat_with_images( + messages, + images, + session_id=session_id, + ) + + # Phase 4: 解析响应 + parsed = parse_vlm_response(response.content, video_id, task_type, seq) + + # Phase 5: 构造 GeneratedQuestion + return _GeneratedQuestion( + question_id=parsed["question_id"], + video_id=video_id, + task_type=task_type, + question=parsed["question"], + options=tuple(parsed["options"]), + answer=parsed["answer"], + source_nodes=(anchor.node_id,), + difficulty="medium", + ) + except (ValueError, KeyError) as exc: + logger.warning( + "generate_one 尝试 {}/{} 失败 ({}): {}", + attempt + 1, + max_retries, + task_type, + exc, + ) + continue + + logger.warning( + "generate_one 耗尽 {} 次重试 (video={}, task_type={})", + max_retries, + video_id, + task_type, + ) + return None diff --git a/tests/unit/test_synthesizer.py b/tests/unit/test_synthesizer.py index 691a7c4..f564ae4 100644 --- a/tests/unit/test_synthesizer.py +++ b/tests/unit/test_synthesizer.py @@ -1,11 +1,13 @@ -"""synthesizer 模块单元测试 — AnchorContext + 题型映射常量 + sample_anchor + prompt 构造 + VLM 解析。""" +"""synthesizer 模块单元测试 — AnchorContext + 题型映射常量 + sample_anchor + prompt 构造 + VLM 解析 + 去重 + 单题生成。""" from __future__ import annotations import dataclasses import random from pathlib import Path +from unittest.mock import AsyncMock, MagicMock +import numpy as np import pytest from app.question_gen.synthesizer import ( @@ -13,6 +15,8 @@ from app.question_gen.synthesizer import ( AnchorContext, TaskTypeSpec, build_generation_prompt, + generate_one, + is_duplicate, parse_vlm_response, sample_anchor, ) @@ -379,3 +383,119 @@ class TestParseVlmResponse: raw = '{"question": "Q?", "options": ["A. 1", "B. 2", "C. 3", "D. 4"], "answer": "C"}' result = parse_vlm_response(raw, "video_abc", "Action Reasoning", 42) assert result["question_id"] == "gen-video_abc-042" + + +# --------------------------------------------------------------------------- +# is_duplicate 测试 +# --------------------------------------------------------------------------- + + +class TestIsDuplicate: + """is_duplicate embedding 去重判定测试。""" + + @staticmethod + def _fake_embed(texts: str | list[str]) -> np.ndarray: + """确定性 + L2 归一化的 fake embedding。""" + if isinstance(texts, str): + texts = [texts] + vecs = [] + for t in texts: + rs = np.random.RandomState(hash(t) % 2**31) + v = rs.randn(4).astype(np.float32) + v /= np.linalg.norm(v) + vecs.append(v) + return np.array(vecs, dtype=np.float32) + + def test_empty_pool_never_duplicate(self) -> None: + """空池始终返回 False。""" + pool = np.zeros((0, 4), dtype=np.float32) + assert is_duplicate("anything", pool, self._fake_embed, 0.85) is False + + def test_identical_text_is_duplicate(self) -> None: + """相同文本的 embedding 与自身余弦相似度为 1，必定判重。""" + text = "What is happening in the video?" + emb = self._fake_embed(text) + pool = emb.copy() + assert is_duplicate(text, pool, self._fake_embed, 0.85) is True + + def test_different_text_not_duplicate(self) -> None: + """极高阈值下，不同文本不判重。""" + pool_texts = ["aaa", "bbb", "ccc", "ddd", "eee"] + pool = self._fake_embed(pool_texts) + assert is_duplicate("completely unique text xyz", pool, self._fake_embed, 0.99) is False + + +# --------------------------------------------------------------------------- +# generate_one 测试 +# --------------------------------------------------------------------------- + + +class TestGenerateOne: + """generate_one 单题异步生成测试。""" + + @staticmethod + def _load_test_tree() -> tuple[TreeIndex, str]: + """加载真实测试树。""" + videos_dir = Path("store/videos") + first_vid = sorted(videos_dir.iterdir())[0] + return TreeIndex.load_json(str(first_vid / "tree.json")), first_vid.name + + @pytest.mark.asyncio + async def test_success_path(self) -> None: + """mock VLM 返回合法 JSON，应成功生成 GeneratedQuestion。""" + vlm = AsyncMock() + vlm.chat_with_images.return_value = MagicMock( + content='{"question":"Q?","options":["A. 1","B. 2","C. 3","D. 4"],"answer":"A"}', + ) + + def embed_fn(t: str | list[str]) -> np.ndarray: + shape = (1, 4) if isinstance(t, str) else (len(t), 4) + return np.zeros(shape, dtype=np.float32) + + tree, vid = self._load_test_tree() + result = await generate_one( + vlm=vlm, + embed_fn=embed_fn, + tree=tree, + video_id=vid, + task_type="Object Recognition", + seq=1, + exemplars=[], + used_node_ids=set(), + max_retries=3, + similarity_threshold=0.85, + rng=random.Random(42), + session_id="test", + ) + assert result is not None + assert result.question_id == f"gen-{vid}-001" + assert result.task_type == "Object Recognition" + assert result.source_nodes # non-empty + assert result.difficulty == "medium" + + @pytest.mark.asyncio + async def test_all_retries_exhausted_returns_none(self) -> None: + """VLM 始终返回无效 JSON，耗尽重试后返回 None。""" + vlm = AsyncMock() + vlm.chat_with_images.return_value = MagicMock(content="invalid") + + def embed_fn(t: str | list[str]) -> np.ndarray: + return np.zeros((1, 4), dtype=np.float32) + + tree, vid = self._load_test_tree() + result = await generate_one( + vlm=vlm, + embed_fn=embed_fn, + tree=tree, + video_id=vid, + task_type="Object Recognition", + seq=1, + exemplars=[], + used_node_ids=set(), + max_retries=2, + similarity_threshold=0.85, + rng=random.Random(42), + session_id="test", + ) + assert result is None + assert vlm.chat_with_images.call_count == 2 From 6e46d184b86273fca41ffe67f283799a78863996 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 05:36:07 -0400 Subject: [PATCH 15/21] =?UTF-8?q?feat(harness):=20add=20factory.py=20?= =?UTF-8?q?=E2=80=94=20InferenceDeps=20dataclass=20+=20build=5Finference?= =?UTF-8?q?=5Fdeps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 组装一次推理所需的全套依赖的工厂函数： - TreeIndex 加载（FileNotFoundError if missing） - TreeEnvironment 构建 - SkillRegistry 按需发现 - SearchToolDispatcher 装配 - PromptManager + prompt_builder 闭包 测试覆盖：正常路径、缺失树文件、skills 注入、frozen 不可变性。 Co-Authored-By: Claude Opus 4.6 (1M context) --- app/harness/factory.py | 155 +++++++++++++++++++++++++++ tests/unit/test_factory.py | 208 +++++++++++++++++++++++++++++++++++++ 2 files changed, 363 insertions(+) create mode 100644 app/harness/factory.py create mode 100644 tests/unit/test_factory.py diff --git a/app/harness/factory.py b/app/harness/factory.py new file mode 100644 index 0000000..cd856f4 --- /dev/null +++ b/app/harness/factory.py @@ -0,0 +1,155 @@ +"""推理依赖工厂 — 组装一次推理所需的全套依赖。 + +将 TreeIndex 加载、TreeEnvironment 构建、SkillRegistry 发现、 +SearchToolDispatcher 装配、PromptManager 初始化等步骤封装为 +单一工厂函数 ``build_inference_deps``，返回不可变的 ``InferenceDeps``。 + +调用方（runner / inference）只需传入配置参数，无需了解内部装配逻辑。 +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from loguru import logger + +from app.search.prompt import PromptManager +from app.search.skills import discover_skills +from app.search.tools import SearchToolDispatcher +from app.tree.environment import TreeEnvironment +from app.tree.index import TreeIndex + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from app.ports import EmbeddingProvider, OCRProvider + from core.protocols import LLMProvider, VLMProvider + from core.types import GeneratedQuestion + + +@dataclass(frozen=True) +class InferenceDeps: + """跑一次推理所需的全套依赖（不含 HarnessLog，其生命周期由调用方管理）。 + + 属性: + llm: LLM 端口实例。 + tool_dispatch_fn: SearchToolDispatcher.dispatch 的绑定方法。 + prompt_builder: (GeneratedQuestion) -> (system_prompt, user_prompt)。 + """ + + llm: LLMProvider + tool_dispatch_fn: Callable[..., Any] + prompt_builder: Callable[[GeneratedQuestion], tuple[str, str]] + + +def build_inference_deps( + *, + store_dir: Path, + video_id: str, + prompts_dir: Path, + skills_dir: Path | None, + skill_mode: str, + embed_provider: EmbeddingProvider, + llm: LLMProvider, + vlm: VLMProvider, + ocr: OCRProvider | None, + verify_vision: bool, + anchor: bool, + assemble_mode: str, +) -> InferenceDeps: + """组装一次推理所需的全套依赖。 + + 参数: + store_dir: store 根目录（包含 videos/{video_id}/tree.json）。 + video_id: 视频标识。 + prompts_dir: prompt 文件目录。 + skills_dir: skill 文件目录（None 则不加载 skill）。 + skill_mode: skill 模式（"auto"/"manual"/"none"）。 + embed_provider: 嵌入端口实例。 + llm: LLM 端口实例。 + vlm: VLM 端口实例。 + ocr: OCR 端口实例（None 不启用）。 + verify_vision: observe_frame 是否执行验证轮。 + anchor: view_node 是否启用行号锚模式。 + assemble_mode: 锚模式装配形态。 + + 返回: + InferenceDeps 实例。 + + 异常: + FileNotFoundError: tree.json 不存在。 + """ + # Phase 1: 加载 TreeIndex + tree_path = store_dir / "videos" / video_id / "tree.json" + if not tree_path.exists(): + raise FileNotFoundError(f"树索引文件不存在: {tree_path}") + tree_index = TreeIndex.load_json(str(tree_path)) + logger.info("已加载 TreeIndex: video_id={}, L1 节点数={}", video_id, len(tree_index.roots)) + + # Phase 2: 构建 TreeEnvironment + frames_dir = store_dir / "videos" / video_id / "frames" + env = TreeEnvironment(index=tree_index, frames_dir=frames_dir) + + # Phase 3: 构建 SkillRegistry + skills = None + always_skills_text = "" + task_skill_map: dict[str, str] = {} + catalog_text = "" + if skills_dir is not None: + always_skills_text, task_skill_map, catalog_text, skills = discover_skills(skills_dir) + logger.info( + "已发现 skills: always={} 字符, task_map={} 项", + len(always_skills_text), + len(task_skill_map), + ) + + # Phase 4: 构建 SearchToolDispatcher + dispatcher = SearchToolDispatcher( + env, + tool_llm=llm, + vlm=vlm, + ocr=ocr, + prompts_dir=prompts_dir, + skills=skills, + embed_fn=embed_provider.embed, + verify_vision=verify_vision, + anchor=anchor, + assemble_mode=assemble_mode, + ) + + # Phase 5: 构建 PromptManager + _prompt_builder 闭包 + pm = PromptManager(prompts_dir) + l1_ids = [root.id for root in tree_index.roots] + + def _prompt_builder(qa: GeneratedQuestion) -> tuple[str, str]: + """为单条题目生成 (system_prompt, user_prompt)。 + + 参数: + qa: 生成的题目实例。 + + 返回: + (system_prompt, user_prompt) 二元组。 + """ + system = pm.build_inference_prompt( + skill_mode, + qa.task_type, + always_skills_text, + task_skill_map, + catalog_text, + ) + user = pm.format_user_prompt( + qa.question, + list(qa.options), + l1_ids, + qa.task_type, + ) + return system, user + + logger.info("InferenceDeps 组装完成: video_id={}, skill_mode={}", video_id, skill_mode) + return InferenceDeps( + llm=llm, + tool_dispatch_fn=dispatcher.dispatch, + prompt_builder=_prompt_builder, + ) diff --git a/tests/unit/test_factory.py b/tests/unit/test_factory.py new file mode 100644 index 0000000..af18b11 --- /dev/null +++ b/tests/unit/test_factory.py @@ -0,0 +1,208 @@ +"""app/harness/factory.py 的单元测试。 + +验证 build_inference_deps 的返回类型、字段连接、以及错误路径。 +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING +from unittest.mock import AsyncMock, MagicMock + +if TYPE_CHECKING: + from pathlib import Path + +import numpy as np +import pytest + +from app.harness.factory import InferenceDeps, build_inference_deps +from core.types import GeneratedQuestion + + +class TestBuildInferenceDeps: + """build_inference_deps 工厂函数测试。""" + + def test_returns_inference_deps(self, tmp_path: Path) -> None: + """用 fake adapters 验证返回类型和字段非 None。""" + # 准备一棵最小树 + vid_dir = tmp_path / "videos" / "test_vid" + vid_dir.mkdir(parents=True) + (vid_dir / "frames").mkdir() + minimal_tree = { + "metadata": {"source_path": "test", "modality": "video"}, + "roots": [ + { + "id": "L1_000", + "card": { + "scene_summary": "s", + "main_setting": "s", + "key_entities": [], + "main_actions": [], + "topic_keywords": [], + "visible_text": [], + "temporal_flow": "s", + }, + "time_range": [0, 10], + "children": [], + } + ], + } + (vid_dir / "tree.json").write_text(json.dumps(minimal_tree)) + + # prompts + prompts_dir = tmp_path / "prompts" + prompts_dir.mkdir() + (prompts_dir / "system.md").write_text("You are a search agent.") + + fake_llm = AsyncMock() + fake_vlm = AsyncMock() + fake_embed = MagicMock() + fake_embed.dim = 4 + fake_embed.embed = lambda t: np.zeros((1, 4), dtype=np.float32) + + deps = build_inference_deps( + store_dir=tmp_path, + video_id="test_vid", + prompts_dir=prompts_dir, + skills_dir=None, + skill_mode="none", + embed_provider=fake_embed, + llm=fake_llm, + vlm=fake_vlm, + ocr=None, + verify_vision=False, + anchor=False, + assemble_mode="ids", + ) + assert isinstance(deps, InferenceDeps) + assert deps.llm is fake_llm + assert callable(deps.tool_dispatch_fn) + assert callable(deps.prompt_builder) + + # 验证 prompt_builder 实际可用（连接正确） + fake_q = GeneratedQuestion( + question_id="q1", + video_id="test_vid", + task_type="Object Recognition", + question="What?", + options=("A. X", "B. Y", "C. Z", "D. W"), + answer="A", + source_nodes=(), + difficulty="medium", + ) + system, user = deps.prompt_builder(fake_q) + assert isinstance(system, str) and len(system) > 0 + assert isinstance(user, str) and "What?" in user + + def test_missing_tree_raises(self, tmp_path: Path) -> None: + """tree.json 不存在时应抛出 FileNotFoundError。""" + prompts_dir = tmp_path / "prompts" + prompts_dir.mkdir() + (prompts_dir / "system.md").write_text("x") + vid_dir = tmp_path / "videos" / "nonexist" + vid_dir.mkdir(parents=True) + + with pytest.raises(FileNotFoundError): + build_inference_deps( + store_dir=tmp_path, + video_id="nonexist", + prompts_dir=prompts_dir, + skills_dir=None, + skill_mode="none", + embed_provider=MagicMock(), + llm=AsyncMock(), + vlm=AsyncMock(), + ocr=None, + verify_vision=False, + anchor=False, + assemble_mode="ids", + ) + + def test_with_skills_dir(self, tmp_path: Path) -> None: + """提供 skills_dir 时 skill 信息应正确注入到 prompt_builder 输出。""" + # 准备树 + vid_dir = tmp_path / "videos" / "vid1" + vid_dir.mkdir(parents=True) + (vid_dir / "frames").mkdir() + minimal_tree = { + "metadata": {"source_path": "test", "modality": "video"}, + "roots": [ + { + "id": "L1_000", + "card": { + "scene_summary": "test scene", + "main_setting": "indoor", + "key_entities": [], + "main_actions": [], + "topic_keywords": [], + "visible_text": [], + "temporal_flow": "linear", + }, + "time_range": [0, 5], + "children": [], + } + ], + } + (vid_dir / "tree.json").write_text(json.dumps(minimal_tree)) + + # prompts + prompts_dir = tmp_path / "prompts" + prompts_dir.mkdir() + (prompts_dir / "system.md").write_text("Base system prompt.") + + # skills + skills_dir = tmp_path / "skills" + skills_dir.mkdir() + (skills_dir / "always_nav.md").write_text( + "---\nname: always_nav\nalways: true\n---\nAlways navigate broadly." + ) + (skills_dir / "action_skill.md").write_text( + "---\nname: action_skill\ntask_type: Action Reasoning\n---\nFocus on actions." + ) + + fake_llm = AsyncMock() + fake_vlm = AsyncMock() + fake_embed = MagicMock() + fake_embed.dim = 4 + fake_embed.embed = lambda t: np.zeros((1, 4), dtype=np.float32) + + deps = build_inference_deps( + store_dir=tmp_path, + video_id="vid1", + prompts_dir=prompts_dir, + skills_dir=skills_dir, + skill_mode="auto", + embed_provider=fake_embed, + llm=fake_llm, + vlm=fake_vlm, + ocr=None, + verify_vision=False, + anchor=False, + assemble_mode="ids", + ) + + fake_q = GeneratedQuestion( + question_id="q2", + video_id="vid1", + task_type="Action Reasoning", + question="What happened?", + options=("A. X", "B. Y", "C. Z", "D. W"), + answer="B", + source_nodes=(), + difficulty="easy", + ) + system, user = deps.prompt_builder(fake_q) + # always skill 文本和 task_type skill 文本应出现在 system prompt 中 + assert "Always navigate broadly" in system + assert "Focus on actions" in system + assert "What happened?" in user + + def test_frozen_dataclass(self) -> None: + """InferenceDeps 是 frozen dataclass，不可修改属性。""" + deps = InferenceDeps( + llm=AsyncMock(), + tool_dispatch_fn=lambda: None, + prompt_builder=lambda q: ("", ""), + ) + with pytest.raises(AttributeError): + deps.llm = AsyncMock() # type: ignore[misc] From 11f3c9020083f77d8930b4713d7db18696389621 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 05:42:50 -0400 Subject: [PATCH 16/21] =?UTF-8?q?feat(tools):=20generate=5Fquestions.py=20?= =?UTF-8?q?generate=20=E5=AD=90=E5=91=BD=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - VLM 出题 + embedding 去重 + 断点续跑 + 并发控制 - 单线程汇总点保证去重原子性 - 18 个单元测试覆盖 progress/exemplar/pool rebuild/JSON append Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/unit/test_generate_questions.py | 322 +++++++++++++ tools/generate_questions.py | 637 ++++++++++++++++++++++++++ 2 files changed, 959 insertions(+) create mode 100644 tests/unit/test_generate_questions.py create mode 100644 tools/generate_questions.py diff --git a/tests/unit/test_generate_questions.py b/tests/unit/test_generate_questions.py new file mode 100644 index 0000000..3d608af --- /dev/null +++ b/tests/unit/test_generate_questions.py @@ -0,0 +1,322 @@ +"""tools/generate_questions.py 单元测试。 + +覆盖断点续跑、exemplar 选取、embedding 池重建、JSON 追加写入等纯函数。 +""" + +from __future__ import annotations + +import json +import random +import sys +from pathlib import Path + +import numpy as np + +# 确保项目根目录在 sys.path 中 +PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from core.types import GeneratedQuestion +from tools.generate_questions import ( + _append_to_json, + _load_or_init_progress, + _rebuild_embedding_pool, + _save_progress, + _select_exemplars, +) + +# --------------------------------------------------------------------------- +# 辅助工厂 +# --------------------------------------------------------------------------- + + +def _make_question( + qid: str = "q1", + vid: str = "v1", + task_type: str = "Object Recognition", + question: str = "What is this?", + answer: str = "A", +) -> GeneratedQuestion: + """构造测试用 GeneratedQuestion。""" + return GeneratedQuestion( + question_id=qid, + video_id=vid, + task_type=task_type, + question=question, + options=("A. X", "B. Y", "C. Z", "D. W"), + answer=answer, + source_nodes=(), + difficulty="medium", + ) + + +# --------------------------------------------------------------------------- +# TestLoadOrInitProgress +# --------------------------------------------------------------------------- + + +class TestLoadOrInitProgress: + """_load_or_init_progress 测试。""" + + def test_init_fresh(self, tmp_path: Path) -> None: + """目录为空时返回初始结构。""" + progress = _load_or_init_progress(tmp_path) + assert progress["completed"] == {} + assert progress["output_dir"] == str(tmp_path) + + def test_load_existing(self, tmp_path: Path) -> None: + """已有 progress.json 时正确加载。""" + data = { + "completed": {"Object Recognition": ["gen-x-001"]}, + "output_dir": str(tmp_path), + } + (tmp_path / "progress.json").write_text(json.dumps(data)) + progress = _load_or_init_progress(tmp_path) + assert "gen-x-001" in progress["completed"]["Object Recognition"] + + def test_corrupted_json_reinits(self, tmp_path: Path) -> None: + """损坏的 JSON 文件导致重新初始化。""" + (tmp_path / "progress.json").write_text("{invalid json") + progress = _load_or_init_progress(tmp_path) + assert progress["completed"] == {} + + def test_invalid_completed_type_reinits(self, tmp_path: Path) -> None: + """completed 字段类型不正确时重新初始化。""" + (tmp_path / "progress.json").write_text( + json.dumps({"completed": "not-a-dict", "output_dir": str(tmp_path)}) + ) + progress = _load_or_init_progress(tmp_path) + assert progress["completed"] == {} + + +# --------------------------------------------------------------------------- +# TestSaveProgress +# --------------------------------------------------------------------------- + + +class TestSaveProgress: + """_save_progress 测试。""" + + def test_atomic_write(self, tmp_path: Path) -> None: + """原子写入 progress.json。""" + progress = { + "completed": {"Action Reasoning": ["gen-v1-001"]}, + "output_dir": str(tmp_path), + } + _save_progress(tmp_path, progress) + + written = json.loads((tmp_path / "progress.json").read_text()) + assert written["completed"]["Action Reasoning"] == ["gen-v1-001"] + # 临时文件不应残留 + assert not (tmp_path / "progress.json.tmp").exists() + + +# --------------------------------------------------------------------------- +# TestSelectExemplars +# --------------------------------------------------------------------------- + + +class TestSelectExemplars: + """_select_exemplars 测试。""" + + def test_selects_correct_type(self) -> None: + """只选取匹配题型的示例。""" + qs = [ + _make_question("q1", "v1", "Object Recognition", "Q1?"), + _make_question("q2", "v2", "Object Recognition", "Q2?"), + _make_question("q3", "v1", "Action Reasoning", "Q3?"), + ] + result = _select_exemplars(qs, "Object Recognition", 3, random.Random(42)) + assert all(q.task_type == "Object Recognition" for q in result) + assert len(result) == 2 # 只有 2 个可用 + + def test_cross_video_diversity(self) -> None: + """优先从不同 video_id 选取示例。""" + qs = [_make_question(f"q{i}", f"v{i}", "Object Recognition", f"Q{i}?") for i in range(10)] + result = _select_exemplars(qs, "Object Recognition", 3, random.Random(42)) + video_ids = {q.video_id for q in result} + assert len(video_ids) == 3 # 全部来自不同视频 + + def test_empty_benchmark(self) -> None: + """benchmark 为空时返回空列表。""" + result = _select_exemplars([], "Object Recognition", 3, random.Random(42)) + assert result == [] + + def test_no_matching_type(self) -> None: + """无匹配题型时返回空列表。""" + qs = [_make_question("q1", "v1", "Action Reasoning", "Q1?")] + result = _select_exemplars(qs, "Object Recognition", 3, random.Random(42)) + assert result == [] + + def test_request_more_than_available(self) -> None: + """请求数超过可用数时返回全部。""" + qs = [ + _make_question("q1", "v1", "Object Recognition", "Q1?"), + _make_question("q2", "v2", "Object Recognition", "Q2?"), + ] + result = _select_exemplars(qs, "Object Recognition", 10, random.Random(42)) + assert len(result) == 2 + + +# --------------------------------------------------------------------------- +# TestProgressResume +# --------------------------------------------------------------------------- + + +class TestProgressResume: + """断点续跑集成测试。""" + + def test_skips_completed_and_rebuilds_pool(self, tmp_path: Path) -> None: + """已完成的题目从 progress 加载，embedding 池从已生成 JSON 重建。""" + output_dir = tmp_path / "output" + output_dir.mkdir() + (output_dir / "test_vid.json").write_text( + json.dumps( + [ + { + "question_id": "gen-test_vid-001", + "task_type": "Object Recognition", + "question": "Existing question?", + "options": ["A. X", "B. Y", "C. Z", "D. W"], + "answer": "A", + "source_nodes": ["L3_001"], + "difficulty": "medium", + } + ] + ) + ) + progress = { + "completed": {"Object Recognition": ["gen-test_vid-001"]}, + "output_dir": str(output_dir), + } + (output_dir / "progress.json").write_text(json.dumps(progress)) + + loaded = _load_or_init_progress(output_dir) + assert "gen-test_vid-001" in loaded["completed"]["Object Recognition"] + + +# --------------------------------------------------------------------------- +# TestRebuildEmbeddingPool +# --------------------------------------------------------------------------- + + +class TestRebuildEmbeddingPool: + """_rebuild_embedding_pool 测试。""" + + @staticmethod + def _fake_embed_fn(texts): + """伪嵌入函数：返回固定维度的随机向量。""" + if isinstance(texts, str): + texts = [texts] + return np.random.RandomState(0).randn(len(texts), 8).astype(np.float32) + + def test_empty_dir(self, tmp_path: Path) -> None: + """空目录时所有池为空。""" + pools = _rebuild_embedding_pool(tmp_path, self._fake_embed_fn, []) + # 所有 12 种题型都应有条目 + assert len(pools) >= 12 + for v in pools.values(): + assert v.shape[0] == 0 or v.ndim == 2 + + def test_with_generated_json(self, tmp_path: Path) -> None: + """从已生成的 JSON 文件重建池。""" + (tmp_path / "vid1.json").write_text( + json.dumps( + [ + { + "question_id": "gen-vid1-001", + "task_type": "Object Recognition", + "question": "Test question 1?", + "options": ["A. X", "B. Y", "C. Z", "D. W"], + "answer": "A", + "source_nodes": [], + "difficulty": "medium", + }, + { + "question_id": "gen-vid1-002", + "task_type": "Object Recognition", + "question": "Test question 2?", + "options": ["A. X", "B. Y", "C. Z", "D. W"], + "answer": "B", + "source_nodes": [], + "difficulty": "medium", + }, + ] + ) + ) + pools = _rebuild_embedding_pool(tmp_path, self._fake_embed_fn, []) + assert pools["Object Recognition"].shape[0] == 2 + assert pools["Object Recognition"].shape[1] == 8 + + def test_with_benchmark_questions(self, tmp_path: Path) -> None: + """benchmark 题目也加入去重池。""" + benchmark = [ + _make_question("bm1", "v1", "Action Reasoning", "Benchmark Q1?"), + _make_question("bm2", "v2", "Action Reasoning", "Benchmark Q2?"), + ] + pools = _rebuild_embedding_pool(tmp_path, self._fake_embed_fn, benchmark) + assert pools["Action Reasoning"].shape[0] == 2 + + def test_progress_json_excluded(self, tmp_path: Path) -> None: + """progress.json 不被当作题目文件。""" + (tmp_path / "progress.json").write_text( + json.dumps({"completed": {}, "output_dir": str(tmp_path)}) + ) + pools = _rebuild_embedding_pool(tmp_path, self._fake_embed_fn, []) + for v in pools.values(): + assert v.shape[0] == 0 or v.ndim == 2 + + +# --------------------------------------------------------------------------- +# TestAppendToJson +# --------------------------------------------------------------------------- + + +class TestAppendToJson: + """_append_to_json 测试。""" + + def test_create_new_file(self, tmp_path: Path) -> None: + """文件不存在时创建新文件。""" + q = _make_question("gen-v1-001", "v1", "Object Recognition", "Q?") + _append_to_json(tmp_path, q) + + written = json.loads((tmp_path / "v1.json").read_text()) + assert len(written) == 1 + assert written[0]["question_id"] == "gen-v1-001" + + def test_append_to_existing(self, tmp_path: Path) -> None: + """追加到已有文件。""" + existing = [ + { + "question_id": "gen-v1-001", + "task_type": "Object Recognition", + "question": "Existing?", + "options": ["A. X", "B. Y", "C. Z", "D. W"], + "answer": "A", + "source_nodes": [], + "difficulty": "medium", + } + ] + (tmp_path / "v1.json").write_text(json.dumps(existing)) + + q = _make_question("gen-v1-002", "v1", "Object Recognition", "New?") + _append_to_json(tmp_path, q) + + written = json.loads((tmp_path / "v1.json").read_text()) + assert len(written) == 2 + assert written[1]["question_id"] == "gen-v1-002" + + def test_different_video_ids(self, tmp_path: Path) -> None: + """不同 video_id 写入不同文件。""" + q1 = _make_question("gen-v1-001", "v1", "Object Recognition", "Q1?") + q2 = _make_question("gen-v2-001", "v2", "Action Reasoning", "Q2?") + _append_to_json(tmp_path, q1) + _append_to_json(tmp_path, q2) + + assert (tmp_path / "v1.json").exists() + assert (tmp_path / "v2.json").exists() + v1_data = json.loads((tmp_path / "v1.json").read_text()) + v2_data = json.loads((tmp_path / "v2.json").read_text()) + assert len(v1_data) == 1 + assert len(v2_data) == 1 diff --git a/tools/generate_questions.py b/tools/generate_questions.py new file mode 100644 index 0000000..741de37 --- /dev/null +++ b/tools/generate_questions.py @@ -0,0 +1,637 @@ +#!/usr/bin/env python3 +"""赛题生成工具：generate + calibrate。 + +用法: + conda activate Video-Tree-TRM + python tools/generate_questions.py generate --store-dir store ... + python tools/generate_questions.py calibrate ... + +app/core/adapters 不 import 此脚本。 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import random +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + +from dotenv import load_dotenv +from loguru import logger + +load_dotenv(PROJECT_ROOT / ".env") + +import numpy as np + +from app.question_gen.loader import load_benchmark +from app.question_gen.synthesizer import ( + TASK_TYPE_LEVEL_MAP, + generate_one, + is_duplicate, +) +from core.types import GeneratedQuestion # noqa: TCH001 — runtime use in _append_to_json + +# --------------------------------------------------------------------------- +# 日志配置：不缓存，立即输出 +# --------------------------------------------------------------------------- + +logger.remove() +logger.add( + sys.stderr, + format="{time:HH:mm:ss} | {level:<7} | {message}", + level="DEBUG", + colorize=True, +) +logger.add( + PROJECT_ROOT / "logs" / "generate_questions.log", + format="{time:YYYY-MM-DD HH:mm:ss} | {level:<7} | {message}", + level="DEBUG", + rotation="50 MB", +) + + +# --------------------------------------------------------------------------- +# 断点续跑 — progress 文件管理 +# --------------------------------------------------------------------------- + + +def _load_or_init_progress(output_dir: Path) -> dict: + """加载 progress.json，不存在则返回初始结构。 + + 参数: + output_dir: 输出目录路径。 + + 返回: + {"completed": {task_type: [question_id, ...]}, "output_dir": str}。 + 文件损坏时返回初始结构并记录警告。 + """ + progress_path = output_dir / "progress.json" + if progress_path.exists(): + try: + data = json.loads(progress_path.read_text(encoding="utf-8")) + if not isinstance(data.get("completed"), dict): + raise ValueError("completed 字段不是 dict") + return data + except (json.JSONDecodeError, ValueError, KeyError, TypeError) as exc: + logger.warning("progress.json 损坏，重新初始化: {}", exc) + return {"completed": {}, "output_dir": str(output_dir)} + + +def _save_progress(output_dir: Path, progress: dict) -> None: + """原子写入 progress.json。 + + 参数: + output_dir: 输出目录路径。 + progress: 进度数据。 + """ + tmp = output_dir / "progress.json.tmp" + tmp.write_text( + json.dumps(progress, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + os.replace(str(tmp), str(output_dir / "progress.json")) + + +# --------------------------------------------------------------------------- +# Embedding 池重建（断点续跑时从已生成 JSON 重建） +# --------------------------------------------------------------------------- + + +def _rebuild_embedding_pool( + output_dir: Path, + embed_fn, + benchmark_questions: list[GeneratedQuestion], +) -> dict[str, np.ndarray]: + """从已生成 JSON + benchmark 题目重建每个题型的 embedding 池。 + + 断点续跑时调用，确保去重池包含所有已有题目。 + + 参数: + output_dir: 包含 {video_id}.json 的输出目录。 + embed_fn: 文本嵌入函数（str | list[str] → [N, D] ndarray）。 + benchmark_questions: benchmark 题目列表（也要加入去重池）。 + + 返回: + {task_type: [N, D] ndarray}，空题型的 ndarray 为 shape (0,)。 + """ + pools: dict[str, list[str]] = {} + + # Phase 1: 收集 benchmark 题目文本 + for q in benchmark_questions: + pools.setdefault(q.task_type, []).append(q.question) + + # Phase 2: 收集已生成题目文本 + for json_path in sorted(output_dir.glob("*.json")): + if json_path.name == "progress.json": + continue + try: + items = json.loads(json_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + logger.warning("跳过损坏文件 {}: {}", json_path, exc) + continue + if not isinstance(items, list): + continue + for item in items: + task_type = item.get("task_type", "") + question = item.get("question", "") + if task_type and question: + pools.setdefault(task_type, []).append(question) + + # Phase 3: 批量嵌入 + result: dict[str, np.ndarray] = {} + for task_type, texts in pools.items(): + if texts: + result[task_type] = embed_fn(texts) + else: + result[task_type] = np.empty(0) + + # Phase 4: 确保所有 12 题型都有条目 + for task_type in TASK_TYPE_LEVEL_MAP: + if task_type not in result: + result[task_type] = np.empty(0) + + logger.info( + "embedding 池重建完成: {}", + {k: v.shape[0] if v.ndim == 2 else 0 for k, v in result.items()}, + ) + return result + + +# --------------------------------------------------------------------------- +# Exemplar 选取 +# --------------------------------------------------------------------------- + + +def _select_exemplars( + benchmark: list[GeneratedQuestion], + task_type: str, + n: int, + rng: random.Random, +) -> list[GeneratedQuestion]: + """从 benchmark 中选取同题型示例，优先跨视频多样性。 + + 参数: + benchmark: benchmark 题目全集。 + task_type: 目标题型。 + n: 期望选取数量。 + rng: 可控随机数生成器。 + + 返回: + min(n, 可用数) 个示例，尽量来自不同 video_id。 + """ + # Phase 1: 过滤同题型 + candidates = [q for q in benchmark if q.task_type == task_type] + if not candidates: + return [] + + take = min(n, len(candidates)) + + # Phase 2: 按 video_id 分桶，轮询取样保证跨视频多样性 + by_video: dict[str, list[GeneratedQuestion]] = {} + for q in candidates: + by_video.setdefault(q.video_id, []).append(q) + + # 每桶内部打乱 + for bucket in by_video.values(): + rng.shuffle(bucket) + + # 轮询选取 + video_ids = list(by_video.keys()) + rng.shuffle(video_ids) + selected: list[GeneratedQuestion] = [] + idx = 0 + while len(selected) < take: + vid = video_ids[idx % len(video_ids)] + bucket = by_video[vid] + if bucket: + selected.append(bucket.pop(0)) + else: + # 桶空了，从 video_ids 中移除 + video_ids.remove(vid) + if not video_ids: + break + # 不递增 idx，因为移除后当前位置是下一个 + continue + idx += 1 + + return selected + + +# --------------------------------------------------------------------------- +# 客户端构建（从 .env） +# --------------------------------------------------------------------------- + + +def _build_vlm_client(): + """构建 GovernedVLMClient，复用 repair_trees.py 的模式。 + + 从 .env 读取 VL_LLM_MODEL / VL_LLM_BASE_URL / VL_LLM_API_KEY + 和 LLM 韧性参数，构造治理栈。 + + 返回: + GovernedVLMClient 实例。 + """ + from adapters.breaker import CircuitBreaker + from adapters.llm import GovernedLLMClient + from adapters.telemetry import SQLiteTelemetryRecorder + from adapters.vlm import GovernedVLMClient + + (PROJECT_ROOT / "logs").mkdir(exist_ok=True) + telemetry = SQLiteTelemetryRecorder( + str(PROJECT_ROOT / "logs" / "generate_questions_telemetry.db") + ) + + breaker_threshold = int(os.getenv("LLM_CIRCUIT_BREAKER_THRESHOLD", "5")) + breaker_cooldown = int(os.getenv("LLM_CIRCUIT_BREAKER_COOLDOWN", "60")) + timeout_s = float(os.getenv("LLM_TIMEOUT", "120")) + max_retries = int(os.getenv("LLM_MAX_RETRIES", "3")) + base_delay = float(os.getenv("LLM_RETRY_BASE_DELAY", "2.0")) + max_delay = float(os.getenv("LLM_RETRY_MAX_DELAY", "30.0")) + ttft = float(os.getenv("LLM_TTFT_TIMEOUT", "30")) + inter_token = float(os.getenv("LLM_INTER_TOKEN_TIMEOUT", "15")) + + vlm_base = GovernedLLMClient( + model=os.environ["VL_LLM_MODEL"], + base_url=os.environ["VL_LLM_BASE_URL"], + api_key=os.environ["VL_LLM_API_KEY"], + provider="qwen", + thinking=False, + breaker=CircuitBreaker(fail_threshold=breaker_threshold, cooldown_s=breaker_cooldown), + cache=None, + telemetry=telemetry, + timeout_s=timeout_s, + ttft_timeout_s=ttft, + inter_token_timeout_s=inter_token, + max_retries=max_retries, + retry_base_delay_s=base_delay, + retry_max_delay_s=max_delay, + ) + return GovernedVLMClient(vlm_base) + + +def _build_embed_provider(): + """构建 EmbeddingProvider，从 .env 决定 local 或 remote。 + + 环境变量: + EMBED_API_KEY + EMBED_API_URL 都非空 → RemoteEmbeddingProvider + 否则 → LocalEmbeddingProvider + + 模型名称和维度通过 EMBED_MODEL / EMBED_DIM 环境变量配置。 + + 返回: + LocalEmbeddingProvider 或 RemoteEmbeddingProvider 实例。 + """ + from adapters.embedding import LocalEmbeddingProvider, RemoteEmbeddingProvider + + model_name = os.environ.get("EMBED_MODEL", "BAAI/bge-base-zh-v1.5") + embed_dim = int(os.environ.get("EMBED_DIM", "768")) + api_key = os.environ.get("EMBED_API_KEY", "") + api_url = os.environ.get("EMBED_API_URL", "") + + if api_key and api_url: + logger.info("使用远程嵌入: model={}, url={}", model_name, api_url) + return RemoteEmbeddingProvider( + model_name=model_name, + embed_dim=embed_dim, + api_key=api_key, + api_url=api_url, + ) + + logger.info("使用本地嵌入: model={}, dim={}", model_name, embed_dim) + device = os.environ.get("EMBED_DEVICE", "cpu") + return LocalEmbeddingProvider( + model_name=model_name, + embed_dim=embed_dim, + device=device, + ) + + +# --------------------------------------------------------------------------- +# JSON 追加写入 +# --------------------------------------------------------------------------- + + +def _append_to_json(output_dir: Path, question: GeneratedQuestion) -> None: + """将生成的题目追加到对应 video_id 的 JSON 文件。 + + 文件格式：[{...}, {...}, ...]，每个 video_id 一个文件。 + + 参数: + output_dir: 输出目录。 + question: 待写入的题目。 + """ + json_path = output_dir / f"{question.video_id}.json" + existing: list[dict] = [] + if json_path.exists(): + try: + existing = json.loads(json_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + logger.warning("读取 {} 失败，覆盖写入", json_path) + existing = [] + + entry = { + "question_id": question.question_id, + "task_type": question.task_type, + "question": question.question, + "options": list(question.options), + "answer": question.answer, + "source_nodes": list(question.source_nodes), + "difficulty": question.difficulty, + } + existing.append(entry) + + # 原子写入 + tmp = json_path.with_suffix(".json.tmp") + tmp.write_text( + json.dumps(existing, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + os.replace(str(tmp), str(json_path)) + + +# --------------------------------------------------------------------------- +# generate 主流程 +# --------------------------------------------------------------------------- + + +async def _run_generate(args: argparse.Namespace) -> None: + """generate 子命令主流程。 + + 按题型顺序生成题目，每个 slot 串行生成并去重， + 断点续跑通过 progress.json 跳过已完成 slot。 + + 参数: + args: CLI 参数（store_dir, output_dir, per_type, similarity_threshold, + max_retries, concurrency, seed）。 + """ + store_dir = Path(args.store_dir) + output_dir = Path(args.output_dir) + per_type = args.per_type + similarity_threshold = args.similarity_threshold + max_retries = args.max_retries + concurrency = args.concurrency + seed = args.seed + + output_dir.mkdir(parents=True, exist_ok=True) + + # Phase 1: 加载视频列表 + videos_dir = store_dir / "videos" + if not videos_dir.exists(): + logger.error("视频目录不存在: {}", videos_dir) + sys.exit(1) + + video_ids = sorted( + d.name for d in videos_dir.iterdir() if d.is_dir() and (d / "tree.json").exists() + ) + if not video_ids: + logger.error("未找到任何有 tree.json 的视频目录") + sys.exit(1) + logger.info("发现 {} 个视频", len(video_ids)) + + # Phase 2: 加载 benchmark 题目（用于 exemplars + 去重池初始化） + benchmark_dir = store_dir / "questions" / "benchmarks" / "Video-MME" + benchmark: list[GeneratedQuestion] = [] + if benchmark_dir.exists(): + benchmark = load_benchmark(benchmark_dir) + logger.info("加载 {} 道 benchmark 题目", len(benchmark)) + else: + logger.warning("benchmark 目录不存在: {}", benchmark_dir) + + # Phase 3: 构建客户端 + vlm = _build_vlm_client() + embed_provider = _build_embed_provider() + embed_fn = embed_provider.embed + + # Phase 4: 初始化或恢复 progress + progress = _load_or_init_progress(output_dir) + + # Phase 5: 重建 embedding 池 + pools = _rebuild_embedding_pool(output_dir, embed_fn, benchmark) + + # Phase 6: 加载视频树索引（延迟按需加载） + from app.tree.index import TreeIndex + + rng = random.Random(seed) + sem = asyncio.Semaphore(concurrency) + task_types = list(TASK_TYPE_LEVEL_MAP.keys()) + total_generated = 0 + total_failed = 0 + + async def _generate_with_sem( + vlm_client, + embed_fn_inner, + tree, + video_id, + task_type, + seq, + *, + exemplars, + used_node_ids, + max_retries_inner, + similarity_threshold_inner, + rng_inner, + session_id, + ): + """Semaphore 包装的 generate_one 调用。""" + async with sem: + return await generate_one( + vlm_client, + embed_fn_inner, + tree, + video_id, + task_type, + seq, + exemplars=exemplars, + used_node_ids=used_node_ids, + max_retries=max_retries_inner, + similarity_threshold=similarity_threshold_inner, + rng=rng_inner, + session_id=session_id, + ) + + # Phase 7: 逐题型、逐 slot 生成 + for task_type in task_types: + completed_ids = set(progress["completed"].get(task_type, [])) + start_seq = len(completed_ids) + + if start_seq >= per_type: + logger.info("题型 {} 已完成 {}/{}", task_type, start_seq, per_type) + continue + + logger.info( + "题型 {} 开始生成: 已完成 {}, 目标 {}", + task_type, + start_seq, + per_type, + ) + + # 选取 exemplars + exemplars = _select_exemplars(benchmark, task_type, 3, rng) + + for seq in range(start_seq, per_type): + # 随机选一个视频 + video_id = rng.choice(video_ids) + tree_path = videos_dir / video_id / "tree.json" + + try: + tree = TreeIndex.load_json(str(tree_path)) + except Exception as exc: + logger.warning("加载树 {} 失败: {}", tree_path, exc) + total_failed += 1 + continue + + used_node_ids: set[str] = set() + session_id = f"gen-{task_type}-{seq}" + generated = False + + for _attempt in range(max_retries): + candidate = await _generate_with_sem( + vlm, + embed_fn, + tree, + video_id, + task_type, + seq, + exemplars=exemplars, + used_node_ids=used_node_ids, + max_retries_inner=1, + similarity_threshold_inner=similarity_threshold, + rng_inner=rng, + session_id=session_id, + ) + + if candidate is None: + continue + + # 去重检查（单线程原子操作） + pool = pools.get(task_type, np.empty(0)) + if ( + pool.ndim == 2 + and pool.shape[0] > 0 + and is_duplicate(candidate.question, pool, embed_fn, similarity_threshold) + ): + logger.warning("去重: {} 与池中题目相似", candidate.question_id) + continue + + # 原子操作：更新池 + 写 JSON + 更新 progress + new_emb = embed_fn(candidate.question) # [1, D] + if pool.ndim == 2 and pool.shape[0] > 0: + pools[task_type] = np.vstack([pool, new_emb]) + else: + pools[task_type] = new_emb + + _append_to_json(output_dir, candidate) + progress["completed"].setdefault(task_type, []).append(candidate.question_id) + _save_progress(output_dir, progress) + + total_generated += 1 + generated = True + logger.debug( + "生成: {} (题型={}, 序号={})", + candidate.question_id, + task_type, + seq, + ) + break + + if not generated: + logger.error("题型 {} seq {} 耗尽 {} 次重试", task_type, seq, max_retries) + total_failed += 1 + + # Phase 8: 汇总 + logger.info("=" * 60) + logger.info("生成完成: 成功 {}, 失败 {}", total_generated, total_failed) + logger.info("=" * 60) + + if total_failed > 0: + logger.error("{} 个 slot 生成失败", total_failed) + sys.exit(1) + + # 全部完成，删除 progress.json + progress_path = output_dir / "progress.json" + if progress_path.exists(): + progress_path.unlink() + logger.info("已删除 progress.json（全部完成）") + + +# --------------------------------------------------------------------------- +# CLI 解析 +# --------------------------------------------------------------------------- + + +def _parse_args() -> argparse.Namespace: + """解析命令行参数。""" + parser = argparse.ArgumentParser(description="赛题生成工具：generate + calibrate") + subparsers = parser.add_subparsers(dest="command", required=True) + + # generate 子命令 + gen_parser = subparsers.add_parser("generate", help="生成新题目") + gen_parser.add_argument( + "--store-dir", + type=str, + required=True, + help="store 根目录（包含 videos/ 和 questions/）", + ) + gen_parser.add_argument( + "--output-dir", + type=str, + required=True, + help="输出目录（生成的 JSON 写入此处）", + ) + gen_parser.add_argument( + "--per-type", + type=int, + required=True, + help="每种题型生成数量", + ) + gen_parser.add_argument( + "--similarity-threshold", + type=float, + required=True, + help="embedding 去重阈值（余弦相似度）", + ) + gen_parser.add_argument( + "--max-retries", + type=int, + required=True, + help="每个 slot 最大重试次数", + ) + gen_parser.add_argument( + "--concurrency", + type=int, + required=True, + help="VLM 调用并发数（Semaphore 容量）", + ) + gen_parser.add_argument( + "--seed", + type=int, + required=True, + help="随机种子", + ) + + # calibrate 子命令（占位，后续任务实现） + subparsers.add_parser("calibrate", help="校准题目难度（待实现）") + + return parser.parse_args() + + +def main() -> None: + """同步入口。""" + args = _parse_args() + (PROJECT_ROOT / "logs").mkdir(exist_ok=True) + + if args.command == "generate": + asyncio.run(_run_generate(args)) + elif args.command == "calibrate": + logger.error("calibrate 子命令尚未实现") + sys.exit(1) + + +if __name__ == "__main__": + main() From 93c9be8bfa792bc73fd61d19dc4bf75a11e1a9c5 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 05:47:52 -0400 Subject: [PATCH 17/21] =?UTF-8?q?feat(tools):=20generate=5Fquestions.py=20?= =?UTF-8?q?calibrate=20=E5=AD=90=E5=91=BD=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fisher exact test + effect size 组合判定（PASS/WARN/FAIL） - 按 video_id 分组推理，避免跨视频树错用 - baseline 支持从 DB 读取或自动跑推理 - 对比表输出 + 退出码控制 Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/unit/test_generate_questions.py | 95 +++++ tools/generate_questions.py | 515 +++++++++++++++++++++++++- 2 files changed, 606 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_generate_questions.py b/tests/unit/test_generate_questions.py index 3d608af..eebd23f 100644 --- a/tests/unit/test_generate_questions.py +++ b/tests/unit/test_generate_questions.py @@ -11,6 +11,7 @@ import sys from pathlib import Path import numpy as np +import pytest # 确保项目根目录在 sys.path 中 PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent @@ -20,10 +21,13 @@ if str(PROJECT_ROOT) not in sys.path: from core.types import GeneratedQuestion from tools.generate_questions import ( _append_to_json, + _calibrate_exit_code, + _judge_task_type, _load_or_init_progress, _rebuild_embedding_pool, _save_progress, _select_exemplars, + _validate_calibrate_args, ) # --------------------------------------------------------------------------- @@ -320,3 +324,94 @@ class TestAppendToJson: v2_data = json.loads((tmp_path / "v2.json").read_text()) assert len(v1_data) == 1 assert len(v2_data) == 1 + + +# --------------------------------------------------------------------------- +# TestCalibrateJudgment +# --------------------------------------------------------------------------- + + +class TestCalibrateJudgment: + """_judge_task_type 校准判定测试。""" + + def test_pass_when_delta_small(self) -> None: + """差值在容忍范围内判定为 PASS。""" + verdict = _judge_task_type( + bench_correct=60, + bench_total=100, + gen_correct=12, + gen_total=20, + tolerance=0.10, + alpha=0.05, + ) + assert verdict == "PASS" + + def test_fail_when_delta_large_and_significant(self) -> None: + """差值超阈值且统计显著判定为 FAIL。""" + verdict = _judge_task_type( + bench_correct=144, + bench_total=240, + gen_correct=6, + gen_total=20, + tolerance=0.10, + alpha=0.05, + ) + assert verdict == "FAIL" + + def test_warn_when_delta_large_but_not_significant(self) -> None: + """差值超阈值但不统计显著判定为 WARN。""" + verdict = _judge_task_type( + bench_correct=2, + bench_total=3, + gen_correct=8, + gen_total=20, + tolerance=0.10, + alpha=0.05, + ) + assert verdict == "WARN" + + +# --------------------------------------------------------------------------- +# TestCalibrateIntegration +# --------------------------------------------------------------------------- + + +class TestCalibrateIntegration: + """calibrate 辅助函数集成测试。""" + + def test_baseline_params_must_be_paired(self) -> None: + """baseline 参数必须成对出现。""" + with pytest.raises(ValueError, match="成对"): + _validate_calibrate_args(baseline_db="some.db", baseline_run_id=None) + + def test_baseline_params_both_none_ok(self) -> None: + """两个参数都为 None 不报错。""" + _validate_calibrate_args(baseline_db=None, baseline_run_id=None) + + def test_baseline_params_both_provided_ok(self) -> None: + """两个参数都提供不报错。""" + _validate_calibrate_args(baseline_db="some.db", baseline_run_id="run-001") + + def test_baseline_run_id_only_raises(self) -> None: + """只提供 run_id 也报错。""" + with pytest.raises(ValueError, match="成对"): + _validate_calibrate_args(baseline_db=None, baseline_run_id="run-001") + + def test_has_fail_returns_exit_code_1(self) -> None: + """存在 FAIL 时返回退出码 1。""" + verdicts = {"Object Recognition": "PASS", "Action Reasoning": "FAIL"} + assert _calibrate_exit_code(verdicts) == 1 + + def test_all_pass_or_warn_returns_exit_code_0(self) -> None: + """全部 PASS 或 WARN 时返回退出码 0。""" + verdicts = {"Object Recognition": "PASS", "Spatial Perception": "WARN"} + assert _calibrate_exit_code(verdicts) == 0 + + def test_all_pass_returns_exit_code_0(self) -> None: + """全部 PASS 时返回退出码 0。""" + verdicts = {"Object Recognition": "PASS", "Action Reasoning": "PASS"} + assert _calibrate_exit_code(verdicts) == 0 + + def test_empty_verdicts_returns_exit_code_0(self) -> None: + """空 verdicts 时返回退出码 0。""" + assert _calibrate_exit_code({}) == 0 diff --git a/tools/generate_questions.py b/tools/generate_questions.py index 741de37..a646027 100644 --- a/tools/generate_questions.py +++ b/tools/generate_questions.py @@ -17,6 +17,8 @@ import json import os import random import sys +import uuid +from collections import defaultdict from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parent.parent @@ -28,7 +30,9 @@ from loguru import logger load_dotenv(PROJECT_ROOT / ".env") import numpy as np +from scipy.stats import fisher_exact +from app.harness.log import HarnessLog from app.question_gen.loader import load_benchmark from app.question_gen.synthesizer import ( TASK_TYPE_LEVEL_MAP, @@ -355,6 +359,438 @@ def _append_to_json(output_dir: Path, question: GeneratedQuestion) -> None: os.replace(str(tmp), str(json_path)) +# --------------------------------------------------------------------------- +# calibrate 辅助函数 +# --------------------------------------------------------------------------- + + +def _judge_task_type( + bench_correct: int, + bench_total: int, + gen_correct: int, + gen_total: int, + tolerance: float, + alpha: float, +) -> str: + """判定单个题型的校准结果。 + + 根据 benchmark 和生成题正确率差值 + Fisher 精确检验决定判定。 + + 参数: + bench_correct: benchmark 答对数。 + bench_total: benchmark 总题数。 + gen_correct: 生成题答对数。 + gen_total: 生成题总题数。 + tolerance: 正确率差值容忍阈值。 + alpha: Fisher 检验显著性水平。 + + 返回: + "PASS" — 差值在容忍范围内。 + "FAIL" — 差值超阈值且统计显著。 + "WARN" — 差值超阈值但不显著。 + """ + delta = abs(gen_correct / gen_total - bench_correct / bench_total) + if delta <= tolerance: + return "PASS" + + table = [ + [bench_correct, bench_total - bench_correct], + [gen_correct, gen_total - gen_correct], + ] + _, p = fisher_exact(table) + + if p < alpha and delta > tolerance: + return "FAIL" + return "WARN" + + +def _validate_calibrate_args( + baseline_db: str | None, + baseline_run_id: str | None, +) -> None: + """校验 baseline 参数必须成对出现。 + + 参数: + baseline_db: 基线数据库路径。 + baseline_run_id: 基线运行标识。 + + 异常: + ValueError: 只提供了一个而非两个参数。 + """ + has_db = baseline_db is not None + has_run_id = baseline_run_id is not None + if has_db != has_run_id: + raise ValueError("--baseline-db 和 --baseline-run-id 必须成对出现") + + +def _calibrate_exit_code(verdicts: dict[str, str]) -> int: + """根据所有题型的判定结果决定进程退出码。 + + 参数: + verdicts: {题型: "PASS"|"WARN"|"FAIL"} 映射。 + + 返回: + 存在任一 FAIL → 1，否则 → 0。 + """ + if any(v == "FAIL" for v in verdicts.values()): + return 1 + return 0 + + +def _read_baseline_per_task_type( + db_path: str, + run_id: str, +) -> dict[str, dict]: + """从已有 HarnessLog DB 中读取指定 run 的 per_task_type 正确率。 + + 参数: + db_path: SQLite 数据库路径。 + run_id: 运行标识。 + + 返回: + {task_type: {"accuracy": float, "total": int, "correct": int}}。 + + 异常: + FileNotFoundError: 数据库文件不存在。 + ValueError: 未找到指定 run_id 的预测记录。 + """ + import sqlite3 + + if not Path(db_path).exists(): + raise FileNotFoundError(f"基线数据库不存在: {db_path}") + + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + try: + rows = conn.execute( + "SELECT task_type, prediction, answer FROM predictions WHERE run_id = ?", + (run_id,), + ).fetchall() + finally: + conn.close() + + if not rows: + raise ValueError(f"未找到 run_id={run_id} 的预测记录") + + groups: dict[str, list[dict]] = defaultdict(list) + for row in rows: + groups[dict(row)["task_type"]].append(dict(row)) + + result: dict[str, dict] = {} + for task_type, records in groups.items(): + total = len(records) + correct = sum(1 for r in records if r["prediction"] == r["answer"]) + result[task_type] = { + "accuracy": correct / total, + "total": total, + "correct": correct, + } + return result + + +def _build_llm_client(): + """构建 GovernedLLMClient（推理用 LLM）。 + + 从 .env 读取 SEARCH_LLM_MODEL / SEARCH_LLM_BASE_URL / SEARCH_LLM_API_KEY + 和 LLM 韧性参数。 + + 返回: + GovernedLLMClient 实例。 + """ + from adapters.breaker import CircuitBreaker + from adapters.llm import GovernedLLMClient + from adapters.telemetry import SQLiteTelemetryRecorder + + (PROJECT_ROOT / "logs").mkdir(exist_ok=True) + telemetry = SQLiteTelemetryRecorder(str(PROJECT_ROOT / "logs" / "calibrate_telemetry.db")) + + breaker_threshold = int(os.getenv("LLM_CIRCUIT_BREAKER_THRESHOLD", "5")) + breaker_cooldown = int(os.getenv("LLM_CIRCUIT_BREAKER_COOLDOWN", "60")) + timeout_s = float(os.getenv("LLM_TIMEOUT", "120")) + max_retries = int(os.getenv("LLM_MAX_RETRIES", "3")) + base_delay = float(os.getenv("LLM_RETRY_BASE_DELAY", "2.0")) + max_delay = float(os.getenv("LLM_RETRY_MAX_DELAY", "30.0")) + ttft = float(os.getenv("LLM_TTFT_TIMEOUT", "30")) + inter_token = float(os.getenv("LLM_INTER_TOKEN_TIMEOUT", "15")) + + return GovernedLLMClient( + model=os.environ["SEARCH_LLM_MODEL"], + base_url=os.environ["SEARCH_LLM_BASE_URL"], + api_key=os.environ["SEARCH_LLM_API_KEY"], + provider="deepseek", + thinking=False, + breaker=CircuitBreaker(fail_threshold=breaker_threshold, cooldown_s=breaker_cooldown), + cache=None, + telemetry=telemetry, + timeout_s=timeout_s, + ttft_timeout_s=ttft, + inter_token_timeout_s=inter_token, + max_retries=max_retries, + retry_base_delay_s=base_delay, + retry_max_delay_s=max_delay, + ) + + +async def _run_inference_for_questions( + questions: list, + *, + store_dir: Path, + prompts_dir: Path, + db_path: str, + run_id: str, + concurrency: int, + max_steps: int, + skill_mode: str, + llm, + vlm, + embed_provider, +) -> dict[str, dict]: + """对题目列表运行推理，返回 per_task_type 指标。 + + 按 video_id 分组，逐组构建推理依赖并执行推理， + 最后合并所有组的 per_task_type 结果。 + + 参数: + questions: 待推理的题目列表。 + store_dir: store 根目录。 + prompts_dir: prompt 文件目录。 + db_path: SQLite 数据库路径。 + run_id: 运行标识。 + concurrency: 最大并发数。 + max_steps: AgentLoop 单题最大步数。 + skill_mode: skill 模式。 + llm: LLMProvider 实例。 + vlm: VLMProvider 实例。 + embed_provider: EmbeddingProvider 实例。 + + 返回: + {task_type: {"accuracy": float, "total": int, "correct": int}}。 + """ + from app.harness.factory import build_inference_deps + from app.harness.inference import run_inference + + # Phase 1: 按 video_id 分组 + by_video: dict[str, list] = defaultdict(list) + for q in questions: + by_video[q.video_id].append(q) + + # Phase 2: 逐组推理 + all_per_task: dict[str, dict] = {} + skills_dir = store_dir / "skills" + if not skills_dir.exists(): + skills_dir = None + + with HarnessLog(db_path, run_id) as log: + for video_id, group in by_video.items(): + deps = build_inference_deps( + store_dir=store_dir, + video_id=video_id, + prompts_dir=prompts_dir, + skills_dir=skills_dir, + skill_mode=skill_mode, + embed_provider=embed_provider, + llm=llm, + vlm=vlm, + ocr=None, + verify_vision=False, + anchor=False, + assemble_mode="plain", + ) + result = await run_inference( + group, + llm=deps.llm, + tool_dispatch_fn=deps.tool_dispatch_fn, + prompt_builder=deps.prompt_builder, + log=log, + run_id=run_id, + concurrency=concurrency, + max_steps=max_steps, + skill_mode=skill_mode, + ) + # Phase 3: 合并 per_task_type + for task_type, metrics in result.per_task_type.items(): + if task_type in all_per_task: + existing = all_per_task[task_type] + merged_total = existing["total"] + metrics["total"] + merged_correct = existing["correct"] + metrics["correct"] + all_per_task[task_type] = { + "accuracy": merged_correct / merged_total, + "total": merged_total, + "correct": merged_correct, + } + else: + all_per_task[task_type] = dict(metrics) + + return all_per_task + + +def _format_comparison_table( + bench_per_task: dict[str, dict], + gen_per_task: dict[str, dict], + verdicts: dict[str, str], + p_values: dict[str, float], +) -> str: + """格式化校准比较表。 + + 参数: + bench_per_task: benchmark 各题型指标。 + gen_per_task: 生成题各题型指标。 + verdicts: 各题型判定结果。 + p_values: 各题型 Fisher 检验 p 值。 + + 返回: + 格式化的比较表字符串。 + """ + verdict_symbols = {"PASS": "✓ PASS", "WARN": "⚠ WARN", "FAIL": "✗ FAIL"} + all_types = sorted(set(bench_per_task) | set(gen_per_task)) + + header = f"{'题型':<20s} | {'bench':>6s} | {'gen':>6s} | {'Δ':>7s} | {'p-value':>7s} | 判定" + sep = "-" * 19 + "-|" + "-" * 8 + "|" + "-" * 8 + "|" + "-" * 9 + "|" + "-" * 9 + "|" + "-" * 8 + lines = [header, sep] + + for task_type in all_types: + b = bench_per_task.get(task_type, {"accuracy": 0.0, "total": 0, "correct": 0}) + g = gen_per_task.get(task_type, {"accuracy": 0.0, "total": 0, "correct": 0}) + delta = g["accuracy"] - b["accuracy"] + p_val = p_values.get(task_type, float("nan")) + verdict = verdicts.get(task_type, "N/A") + symbol = verdict_symbols.get(verdict, verdict) + + lines.append( + f"{task_type:<20s} | {b['accuracy']:>5.1%} | {g['accuracy']:>5.1%} " + f"| {delta:>+6.1%} | {p_val:>7.3f} | {symbol}" + ) + + return "\n".join(lines) + + +async def _run_calibrate(args: argparse.Namespace) -> None: + """calibrate 子命令主流程。 + + 对比 benchmark 和生成题在 Agent 推理下的正确率， + 逐题型 Fisher 精确检验判定校准质量。 + + 参数: + args: CLI 参数。 + """ + _validate_calibrate_args( + getattr(args, "baseline_db", None), + getattr(args, "baseline_run_id", None), + ) + + generated_dir = Path(args.generated_dir) + benchmark_dir = Path(args.benchmark_dir) + store_dir = Path(args.store_dir) + db_path = args.db_path + prompts_dir = Path(args.prompts_dir) + concurrency = args.concurrency + max_steps = args.max_steps + skill_mode = args.skill_mode + tolerance = args.tolerance + alpha = args.alpha + + # Phase 1: 加载题目 + logger.info("加载生成题目: {}", generated_dir) + gen_questions = load_benchmark(generated_dir) + logger.info("加载 benchmark 题目: {}", benchmark_dir) + bench_questions = load_benchmark(benchmark_dir) + logger.info("生成题 {} 道, benchmark {} 道", len(gen_questions), len(bench_questions)) + + # Phase 2: 获取 benchmark baseline + baseline_db = getattr(args, "baseline_db", None) + baseline_run_id = getattr(args, "baseline_run_id", None) + + if baseline_db and baseline_run_id: + logger.info("从基线 DB 读取 benchmark 指标: db={}, run_id={}", baseline_db, baseline_run_id) + bench_per_task = _read_baseline_per_task_type(baseline_db, baseline_run_id) + else: + logger.info("运行 benchmark 推理以获取基线指标") + llm = _build_llm_client() + vlm = _build_vlm_client() + embed_provider = _build_embed_provider() + bench_run_id = f"calibrate-bench-{uuid.uuid4().hex[:8]}" + bench_per_task = await _run_inference_for_questions( + bench_questions, + store_dir=store_dir, + prompts_dir=prompts_dir, + db_path=db_path, + run_id=bench_run_id, + concurrency=concurrency, + max_steps=max_steps, + skill_mode=skill_mode, + llm=llm, + vlm=vlm, + embed_provider=embed_provider, + ) + + # Phase 3: 运行生成题推理 + logger.info("运行生成题推理") + if not baseline_db: + # 客户端已在 Phase 2 构建 + pass + else: + llm = _build_llm_client() + vlm = _build_vlm_client() + embed_provider = _build_embed_provider() + + gen_run_id = f"calibrate-gen-{uuid.uuid4().hex[:8]}" + gen_per_task = await _run_inference_for_questions( + gen_questions, + store_dir=store_dir, + prompts_dir=prompts_dir, + db_path=db_path, + run_id=gen_run_id, + concurrency=concurrency, + max_steps=max_steps, + skill_mode=skill_mode, + llm=llm, + vlm=vlm, + embed_provider=embed_provider, + ) + + # Phase 4: 逐题型判定 + all_types = sorted(set(bench_per_task) | set(gen_per_task)) + verdicts: dict[str, str] = {} + p_values: dict[str, float] = {} + + for task_type in all_types: + b = bench_per_task.get(task_type) + g = gen_per_task.get(task_type) + if b is None or g is None or b["total"] == 0 or g["total"] == 0: + verdicts[task_type] = "WARN" + p_values[task_type] = float("nan") + continue + + verdicts[task_type] = _judge_task_type( + bench_correct=b["correct"], + bench_total=b["total"], + gen_correct=g["correct"], + gen_total=g["total"], + tolerance=tolerance, + alpha=alpha, + ) + + # 计算 p-value 供表格显示 + table = [ + [b["correct"], b["total"] - b["correct"]], + [g["correct"], g["total"] - g["correct"]], + ] + _, p_val = fisher_exact(table) + p_values[task_type] = p_val + + # Phase 5: 输出比较表 + table_str = _format_comparison_table(bench_per_task, gen_per_task, verdicts, p_values) + logger.info("校准比较表:\n{}", table_str) + + # Phase 6: 退出 + exit_code = _calibrate_exit_code(verdicts) + if exit_code == 0: + logger.info("校准通过: 所有题型 PASS 或 WARN") + else: + logger.error("校准失败: 存在 FAIL 题型") + sys.exit(exit_code) + + # --------------------------------------------------------------------------- # generate 主流程 # --------------------------------------------------------------------------- @@ -615,8 +1051,80 @@ def _parse_args() -> argparse.Namespace: help="随机种子", ) - # calibrate 子命令（占位，后续任务实现） - subparsers.add_parser("calibrate", help="校准题目难度（待实现）") + # calibrate 子命令 + cal_parser = subparsers.add_parser("calibrate", help="校准生成题与 benchmark 难度一致性") + cal_parser.add_argument( + "--generated-dir", + type=str, + required=True, + help="生成题目目录", + ) + cal_parser.add_argument( + "--benchmark-dir", + type=str, + required=True, + help="benchmark 题目目录", + ) + cal_parser.add_argument( + "--store-dir", + type=str, + required=True, + help="store 根目录", + ) + cal_parser.add_argument( + "--db-path", + type=str, + required=True, + help="校准 SQLite 数据库路径", + ) + cal_parser.add_argument( + "--prompts-dir", + type=str, + required=True, + help="prompt 文件目录", + ) + cal_parser.add_argument( + "--concurrency", + type=int, + required=True, + help="推理并发数", + ) + cal_parser.add_argument( + "--max-steps", + type=int, + required=True, + help="AgentLoop 单题最大步数", + ) + cal_parser.add_argument( + "--skill-mode", + type=str, + required=True, + help="skill 模式 (auto/manual/none)", + ) + cal_parser.add_argument( + "--tolerance", + type=float, + required=True, + help="正确率差值容忍阈值", + ) + cal_parser.add_argument( + "--alpha", + type=float, + required=True, + help="Fisher 检验显著性水平", + ) + cal_parser.add_argument( + "--baseline-db", + type=str, + default=None, + help="基线数据库路径（可选，须与 --baseline-run-id 成对）", + ) + cal_parser.add_argument( + "--baseline-run-id", + type=str, + default=None, + help="基线运行标识（可选，须与 --baseline-db 成对）", + ) return parser.parse_args() @@ -629,8 +1137,7 @@ def main() -> None: if args.command == "generate": asyncio.run(_run_generate(args)) elif args.command == "calibrate": - logger.error("calibrate 子命令尚未实现") - sys.exit(1) + asyncio.run(_run_calibrate(args)) if __name__ == "__main__": From fad8147d7128b2f39589ec1d4850542e49e4a4cb Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 05:49:05 -0400 Subject: [PATCH 18/21] =?UTF-8?q?refactor(question=5Fgen):=20=5F=5Finit=5F?= =?UTF-8?q?=5F.py=20=E8=BF=BD=E5=8A=A0=20synthesizer=20re-export?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) --- app/question_gen/__init__.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/app/question_gen/__init__.py b/app/question_gen/__init__.py index 9867a7d..6dab48e 100644 --- a/app/question_gen/__init__.py +++ b/app/question_gen/__init__.py @@ -1,5 +1,18 @@ -"""出题模块 — benchmark 加载与分层采样。""" +"""出题模块 — benchmark 加载、分层采样与赛题合成。""" from app.question_gen.loader import load_benchmark, stratified_sample +from app.question_gen.synthesizer import ( + TASK_TYPE_LEVEL_MAP, + AnchorContext, + generate_one, + sample_anchor, +) -__all__ = ["load_benchmark", "stratified_sample"] +__all__ = [ + "load_benchmark", + "stratified_sample", + "TASK_TYPE_LEVEL_MAP", + "AnchorContext", + "generate_one", + "sample_anchor", +] From f57ee45dc07408d146c7cf9513c83508b0033219 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 07:40:27 -0400 Subject: [PATCH 19/21] =?UTF-8?q?fix:=20Codex=20=E5=85=A8=E9=87=8F?= =?UTF-8?q?=E5=AE=A1=E6=9F=A5=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Critical: - C1: assemble_mode 'plain' → 'ids'（合法枚举值） - C2: question_id 加入 task_type slug 避免跨题型冲突 Important/Minor: - generate_one 移除未用的 embed_fn/similarity_threshold 参数 - config.py 注释 11→12 同步 - 测试 question_id 断言更新 Co-Authored-By: Claude Opus 4.6 (1M context) --- app/harness/config.py | 2 +- app/question_gen/synthesizer.py | 6 +----- tests/unit/test_synthesizer.py | 19 ++++--------------- tools/generate_questions.py | 8 +------- 4 files changed, 7 insertions(+), 28 deletions(-) diff --git a/app/harness/config.py b/app/harness/config.py index d0ba2e3..13778a9 100644 --- a/app/harness/config.py +++ b/app/harness/config.py @@ -267,7 +267,7 @@ def _validate_minibatch(config: RunConfig) -> None: 关键实现细节: val_size 必须 >= eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT，保证验证池 - 能为 Video-MME 的全部 11 个任务类型各保底 eval_min_per_class 题。 + 能为 Video-MME 的全部 12 个任务类型各保底 eval_min_per_class 题。 """ if config.batch_size <= 0: raise ValueError(f"batch_size 必须 > 0，实际: {config.batch_size}") diff --git a/app/question_gen/synthesizer.py b/app/question_gen/synthesizer.py index fbbb83e..de6c6d7 100644 --- a/app/question_gen/synthesizer.py +++ b/app/question_gen/synthesizer.py @@ -624,7 +624,7 @@ def parse_vlm_response( raise ValueError(f"answer 必须是 A/B/C/D 之一，实际 '{answer}': {raw[:200]}") return { - "question_id": f"gen-{video_id}-{seq:03d}", + "question_id": f"gen-{video_id}-{task_type.lower().replace(' ', '_')}-{seq:03d}", "question": data["question"], "options": list(options), "answer": answer, @@ -669,7 +669,6 @@ def is_duplicate( async def generate_one( vlm: VLMProvider, - embed_fn: Callable[[str | list[str]], np.ndarray], tree: TreeIndex, video_id: str, task_type: str, @@ -678,7 +677,6 @@ async def generate_one( exemplars: list[GeneratedQuestion], used_node_ids: set[str], max_retries: int, - similarity_threshold: float, rng: random.Random, session_id: str, ) -> GeneratedQuestion | None: @@ -695,7 +693,6 @@ async def generate_one( 参数: vlm: VLM 调用端口。 - embed_fn: 文本嵌入函数（本函数内未使用，由调用方统一去重）。 tree: 三层树索引。 video_id: 所属视频标识。 task_type: 12 种 Video-MME 题型之一。 @@ -703,7 +700,6 @@ async def generate_one( exemplars: 少样本示例列表。 used_node_ids: 已用节点 ID 集合。 max_retries: 最大重试次数。 - similarity_threshold: 余弦相似度阈值（本函数内未使用）。 rng: 可控随机数生成器。 session_id: 会话 ID（传递给 VLM 遥测）。 diff --git a/tests/unit/test_synthesizer.py b/tests/unit/test_synthesizer.py index f564ae4..5040b3b 100644 --- a/tests/unit/test_synthesizer.py +++ b/tests/unit/test_synthesizer.py @@ -346,14 +346,14 @@ class TestParseVlmResponse: assert result["question"] == "What?" assert result["answer"] == "A" assert len(result["options"]) == 4 - assert result["question_id"] == "gen-vid1-001" + assert result["question_id"] == "gen-vid1-object_recognition-001" def test_json_in_code_block(self) -> None: """从 markdown 代码块中提取 JSON。""" raw = '```json\n{"question": "Q?", "options": ["A. 1", "B. 2", "C. 3", "D. 4"], "answer": "B"}\n```' result = parse_vlm_response(raw, "vid1", "Object Recognition", 2) assert result["question"] == "Q?" - assert result["question_id"] == "gen-vid1-002" + assert result["question_id"] == "gen-vid1-object_recognition-002" def test_invalid_json_raises(self) -> None: """非 JSON 文本应抛出 ValueError。""" @@ -382,7 +382,7 @@ class TestParseVlmResponse: """seq 应按 3 位零填充格式化到 question_id 中。""" raw = '{"question": "Q?", "options": ["A. 1", "B. 2", "C. 3", "D. 4"], "answer": "C"}' result = parse_vlm_response(raw, "video_abc", "Action Reasoning", 42) - assert result["question_id"] == "gen-video_abc-042" + assert result["question_id"] == "gen-video_abc-action_reasoning-042" # --------------------------------------------------------------------------- @@ -448,14 +448,9 @@ class TestGenerateOne: content='{"question":"Q?","options":["A. 1","B. 2","C. 3","D. 4"],"answer":"A"}', ) - def embed_fn(t: str | list[str]) -> np.ndarray: - shape = (1, 4) if isinstance(t, str) else (len(t), 4) - return np.zeros(shape, dtype=np.float32) - tree, vid = self._load_test_tree() result = await generate_one( vlm=vlm, - embed_fn=embed_fn, tree=tree, video_id=vid, task_type="Object Recognition", @@ -463,12 +458,11 @@ class TestGenerateOne: exemplars=[], used_node_ids=set(), max_retries=3, - similarity_threshold=0.85, rng=random.Random(42), session_id="test", ) assert result is not None - assert result.question_id == f"gen-{vid}-001" + assert result.question_id == f"gen-{vid}-object_recognition-001" assert result.task_type == "Object Recognition" assert result.source_nodes # non-empty assert result.difficulty == "medium" @@ -479,13 +473,9 @@ class TestGenerateOne: vlm = AsyncMock() vlm.chat_with_images.return_value = MagicMock(content="invalid") - def embed_fn(t: str | list[str]) -> np.ndarray: - return np.zeros((1, 4), dtype=np.float32) - tree, vid = self._load_test_tree() result = await generate_one( vlm=vlm, - embed_fn=embed_fn, tree=tree, video_id=vid, task_type="Object Recognition", @@ -493,7 +483,6 @@ class TestGenerateOne: exemplars=[], used_node_ids=set(), max_retries=2, - similarity_threshold=0.85, rng=random.Random(42), session_id="test", ) diff --git a/tools/generate_questions.py b/tools/generate_questions.py index a646027..aee5bff 100644 --- a/tools/generate_questions.py +++ b/tools/generate_questions.py @@ -594,7 +594,7 @@ async def _run_inference_for_questions( ocr=None, verify_vision=False, anchor=False, - assemble_mode="plain", + assemble_mode="ids", ) result = await run_inference( group, @@ -861,7 +861,6 @@ async def _run_generate(args: argparse.Namespace) -> None: async def _generate_with_sem( vlm_client, - embed_fn_inner, tree, video_id, task_type, @@ -870,7 +869,6 @@ async def _run_generate(args: argparse.Namespace) -> None: exemplars, used_node_ids, max_retries_inner, - similarity_threshold_inner, rng_inner, session_id, ): @@ -878,7 +876,6 @@ async def _run_generate(args: argparse.Namespace) -> None: async with sem: return await generate_one( vlm_client, - embed_fn_inner, tree, video_id, task_type, @@ -886,7 +883,6 @@ async def _run_generate(args: argparse.Namespace) -> None: exemplars=exemplars, used_node_ids=used_node_ids, max_retries=max_retries_inner, - similarity_threshold=similarity_threshold_inner, rng=rng_inner, session_id=session_id, ) @@ -929,7 +925,6 @@ async def _run_generate(args: argparse.Namespace) -> None: for _attempt in range(max_retries): candidate = await _generate_with_sem( vlm, - embed_fn, tree, video_id, task_type, @@ -937,7 +932,6 @@ async def _run_generate(args: argparse.Namespace) -> None: exemplars=exemplars, used_node_ids=used_node_ids, max_retries_inner=1, - similarity_threshold_inner=similarity_threshold, rng_inner=rng, session_id=session_id, ) From ace073854611e4a65d24ff5e045059d6c3672058 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 07:50:24 -0400 Subject: [PATCH 20/21] docs(wiki): record main-figure composition rework and review outcomes --- .../designs/2026-07-09-paper-main-figure-design.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/research-wiki/designs/2026-07-09-paper-main-figure-design.md b/research-wiki/designs/2026-07-09-paper-main-figure-design.md index 7d3f9c3..33dcf28 100644 --- a/research-wiki/designs/2026-07-09-paper-main-figure-design.md +++ b/research-wiki/designs/2026-07-09-paper-main-figure-design.md @@ -125,7 +125,19 @@ 4. 风格与同文件建树图肉眼一致（配色、字体、面板语言、卡片组件）。 5. 不改动/移动建树图的任何现有图层。 -## 9. 被拒绝的备选方案 +## 9. 构图重构记录（2026-07-09 定稿后追加） + +用户验收反馈：内容正确但"下半部空、无主线重点"。经方案比选（用户选 A），实施： + +| 改动 | 内容 | +|---|---| +| 显式循环主干 | 面板间 chevron → 4px 黑色实心三角箭头（forward 主线）；④→⑤ write 与 ⑤→① read 回流均为 4px 绿色实线带，⟳ ×N epochs 置于带上；黑/绿双色对应 forward / parameter-update 语义 | +| 底部压缩 | Store 2080px 全宽行 → 680×170 紧凑块（右缘对齐 ④，write 直指 v5）；对照条 130→84px；画布 1050→**940**（2.55:1） | +| 填充 | motto 24px 斜体移至左下空区；read 带起点加绿色圆点锚记 | + +逐模块精修均经 Claude 自审 + Codex 独立审双 PASS（Question/①/②/③/④/⑤+对照条/整图重构共 8 轮审核）。Codex 抓到的实质问题：④ 的 W/L 翻转数与 e 曲线出口统计不自洽（修正为序列省略号 + W=8·L=0，E=56.78>e_confirm=20）、ladder 色块数与题数不符、read 线易误读为边框。 + +## 10. 被拒绝的备选方案 | 方案 | 拒绝原因 | |---|---| From 45403b23b4f361ed640a03e0ca31f1be0aec994f Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 9 Jul 2026 07:53:59 -0400 Subject: [PATCH 21/21] =?UTF-8?q?feat(repair):=20regenerator=20+=20supplem?= =?UTF-8?q?ent=20=E9=98=B2=E5=BE=A1=E4=BF=AE=E5=A4=8D=20+=20=E8=BF=81?= =?UTF-8?q?=E7=A7=BB=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 app/tree/repair/regenerator.py（VLM 重生成 + 级联修复） - supplement.py: deduplicate_field str() 防御 + inject_value strip - patch.py: ruff format 格式化 - repair_trees.sh: conda source 激活修复 - 新增 migrate_from_trm4.sh 迁移工具 - enhance/__init__.py → repair/__init__.py 重命名 Co-Authored-By: Claude Opus 4.6 (1M context) --- app/tree/{enhance => repair}/__init__.py | 0 app/tree/repair/regenerator.py | 513 +++++++++++++++++++++++ app/tree/repair/supplement.py | 7 +- core/evolution/patch.py | 16 +- scripts/repair_trees.sh | 2 + tests/unit/test_repair_regenerator.py | 293 +++++++++++++ tools/migrate_from_trm4.sh | 75 ++++ 7 files changed, 891 insertions(+), 15 deletions(-) rename app/tree/{enhance => repair}/__init__.py (100%) create mode 100644 app/tree/repair/regenerator.py create mode 100644 tests/unit/test_repair_regenerator.py create mode 100755 tools/migrate_from_trm4.sh diff --git a/app/tree/enhance/__init__.py b/app/tree/repair/__init__.py similarity index 100% rename from app/tree/enhance/__init__.py rename to app/tree/repair/__init__.py diff --git a/app/tree/repair/regenerator.py b/app/tree/repair/regenerator.py new file mode 100644 index 0000000..068a669 --- /dev/null +++ b/app/tree/repair/regenerator.py @@ -0,0 +1,513 @@ +"""树修复重生成器：VLM 重新描述问题节点 + 底向上级联。 + +底向上修复流程: + 1. 收集需修复的 L3 节点 → VLM 重新描述帧 + 2. 收集受影响的 L2 → LLM 从 L3 children 聚合 + 3. 收集受影响的 L1 → LLM 从 L2 children 聚合 + +仅处理 issue_type == "empty_field" 且 level == 3 的问题节点。 +帧文件不存在时跳过该节点（不中断整体修复流程）。 +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from loguru import logger + +from app.tree.index import ( + L1Card, + L1Node, + L2Card, + L2Node, + L3Card, + L3Node, + TreeIndex, +) +from app.tree.subtitle import extract_subtitle_for_range + +if TYPE_CHECKING: + from pathlib import Path + + from app.tree.repair.detector import NodeIssue + from app.tree.subtitle import SRTEntry + from core.protocols import LLMProvider, VLMProvider + +# --------------------------------------------------------------------------- +# Prompt 常量（与 VideoTreeBuilder 保持一致风格） +# --------------------------------------------------------------------------- + +_L3_REPAIR_PROMPT = ( + '该片段的整体内容: "{l2_description}"\n' + "用一到两句话描述这帧画面的具体内容。" + "重点关注: 动作、物体变化、文字信息、人物表情。\n" + "{subtitle_block}" + "返回 JSON 对象，包含以下字段：\n" + "- frame_summary: 画面描述\n" + "- visible_entities: 可见实体列表\n" + "- ongoing_actions: 动作列表\n" + "- visible_text: 可见文字列表\n" + "- spatial_layout: 空间布局\n" + '- visual_attributes: {{"lighting": "...", "dominant_colors": [...], "camera_angle": "..."}}\n' + "只返回 JSON 对象，不要其他内容。" +) + +_L2_REGEN_PROMPT = ( + "以下是一个视频片段中各帧的描述:\n{l3_texts}\n" + "用1-2句话描述该片段的核心内容。\n" + "返回 JSON 对象，包含以下字段：\n" + "- event_description: 1-2句片段描述\n" + "- entities: 可见实体列表\n" + "- actions: 动作列表\n" + "- action_subjects: 动作主体列表\n" + "- visible_text: 画面中可见文字列表\n" + "- spatial_relations: 空间关系描述\n" + "- state_changes: 状态变化描述（无则 null）\n" + "只返回 JSON 对象，不要其他内容。" +) + +_L1_REGEN_PROMPT = ( + "以下是一个视频段落中各片段的描述:\n{l2_texts}\n" + "用2-3句话总结该段落的整体内容，涵盖所有片段的主题。\n" + "返回 JSON 对象，包含以下字段：\n" + "- scene_summary: 2-3句段落摘要\n" + "- main_setting: 主要场景\n" + "- key_entities: 关键实体列表\n" + "- main_actions: 主要动作列表\n" + "- topic_keywords: 主题关键词列表\n" + "- visible_text: 出现的文字列表\n" + "- temporal_flow: 时间流向描述\n" + "只返回 JSON 对象，不要其他内容。" +) + + +# --------------------------------------------------------------------------- +# 统计数据类 +# --------------------------------------------------------------------------- + + +@dataclass +class RepairStats: + """修复统计信息。 + + 属性: + l3_repaired: 修复的 L3 节点数。 + l2_regenerated: 重生成的 L2 节点数。 + l1_regenerated: 重生成的 L1 节点数。 + """ + + l3_repaired: int = 0 + l2_regenerated: int = 0 + l1_regenerated: int = 0 + + +# --------------------------------------------------------------------------- +# JSON 解析辅助（复用 VideoTreeBuilder 的解析逻辑） +# --------------------------------------------------------------------------- + + +def _extract_json(raw: str) -> Any: + """从 VLM/LLM 原始输出中提取 JSON（处理 markdown 代码块包裹）。 + + 参数: + raw: 原始返回字符串。 + + 返回: + 解析后的 Python 对象（dict/list），解析失败返回 None。 + """ + raw = raw.strip() + # Phase 1: 尝试提取 markdown 代码块中的 JSON + code_match = re.search( + r"```(?:json)?\s*([\[{].*?[\]}])\s*```", + raw, + re.DOTALL, + ) + if code_match: + raw = code_match.group(1) + + # Phase 2: 直接解析 + try: + return json.loads(raw) + except json.JSONDecodeError: + pass + + # Phase 3: 尝试提取裸 JSON 对象/数组 + json_match = re.search(r"[\[{].*[\]}]", raw, re.DOTALL) + if json_match: + try: + return json.loads(json_match.group()) + except json.JSONDecodeError: + pass + + return None + + +def _parse_l3_card(raw: str) -> L3Card | None: + """解析 VLM 输出为 L3Card。解析失败返回 None。 + + 参数: + raw: VLM 原始返回字符串。 + + 返回: + L3Card 实例或 None（解析失败时）。 + """ + data = _extract_json(raw) + if isinstance(data, dict): + try: + return L3Card( + frame_summary=str(data["frame_summary"]), + visible_entities=list(data["visible_entities"]), + ongoing_actions=list(data["ongoing_actions"]), + visible_text=list(data["visible_text"]), + spatial_layout=str(data["spatial_layout"]), + visual_attributes=dict(data["visual_attributes"]), + ) + except (KeyError, TypeError, ValueError): + pass + return None + + +def _parse_l2_card(raw: str) -> L2Card | None: + """解析 LLM 输出为 L2Card。解析失败返回 None。 + + 参数: + raw: LLM 原始返回字符串。 + + 返回: + L2Card 实例或 None（解析失败时）。 + """ + data = _extract_json(raw) + if isinstance(data, dict): + try: + state_changes = data.get("state_changes") + if state_changes is not None: + state_changes = str(state_changes) + return L2Card( + event_description=str(data["event_description"]), + entities=list(data["entities"]), + actions=list(data["actions"]), + action_subjects=list(data["action_subjects"]), + visible_text=list(data["visible_text"]), + spatial_relations=str(data["spatial_relations"]), + state_changes=state_changes, + ) + except (KeyError, TypeError, ValueError): + pass + return None + + +def _parse_l1_card(raw: str) -> L1Card | None: + """解析 LLM 输出为 L1Card。解析失败返回 None。 + + 参数: + raw: LLM 原始返回字符串。 + + 返回: + L1Card 实例或 None（解析失败时）。 + """ + data = _extract_json(raw) + if isinstance(data, dict): + try: + return L1Card( + scene_summary=str(data["scene_summary"]), + main_setting=str(data["main_setting"]), + key_entities=list(data["key_entities"]), + main_actions=list(data["main_actions"]), + topic_keywords=list(data["topic_keywords"]), + visible_text=list(data["visible_text"]), + temporal_flow=str(data["temporal_flow"]), + ) + except (KeyError, TypeError, ValueError): + pass + return None + + +# --------------------------------------------------------------------------- +# 节点查找辅助 +# --------------------------------------------------------------------------- + + +def _build_node_lookup( + index: TreeIndex, +) -> tuple[ + dict[str, L3Node], + dict[str, L2Node], + dict[str, L1Node], + dict[str, L2Node], + dict[str, L1Node], +]: + """构建节点 ID 到节点的查找表 + 子节点到父节点的映射。 + + 参数: + index: 树索引。 + + 返回: + (l3_by_id, l2_by_id, l1_by_id, l3_parent_l2, l2_parent_l1) + - l3_by_id: L3 节点 ID → L3Node + - l2_by_id: L2 节点 ID → L2Node + - l1_by_id: L1 节点 ID → L1Node + - l3_parent_l2: L3 节点 ID → 其父 L2Node + - l2_parent_l1: L2 节点 ID → 其父 L1Node + """ + l3_by_id: dict[str, L3Node] = {} + l2_by_id: dict[str, L2Node] = {} + l1_by_id: dict[str, L1Node] = {} + l3_parent_l2: dict[str, L2Node] = {} + l2_parent_l1: dict[str, L1Node] = {} + + for l1 in index.roots: + l1_by_id[l1.id] = l1 + for l2 in l1.children: + l2_by_id[l2.id] = l2 + l2_parent_l1[l2.id] = l1 + for l3 in l2.children: + l3_by_id[l3.id] = l3 + l3_parent_l2[l3.id] = l2 + + return l3_by_id, l2_by_id, l1_by_id, l3_parent_l2, l2_parent_l1 + + +# --------------------------------------------------------------------------- +# 字幕辅助 +# --------------------------------------------------------------------------- + + +def _build_subtitle_block( + srt_entries: list[SRTEntry] | None, + timestamp: float | None, +) -> str: + """构建字幕注入文本块。 + + 参数: + srt_entries: SRT 字幕条目列表。 + timestamp: 帧时间戳（秒）。 + + 返回: + 字幕文本块字符串（无匹配时返回空字符串）。 + """ + if not srt_entries or timestamp is None: + return "" + window = 2.0 + start = max(0.0, timestamp - window) + end = timestamp + window + text = extract_subtitle_for_range(srt_entries, (start, end)) + if not text: + return "" + return f"字幕信息：\n{text}\n" + + +# --------------------------------------------------------------------------- +# 主修复函数 +# --------------------------------------------------------------------------- + + +async def repair_tree( + index: TreeIndex, + issues: list[NodeIssue], + vlm: VLMProvider, + llm: LLMProvider, + frames_dir: Path, + srt_entries: list[SRTEntry] | None = None, +) -> RepairStats: + """修复有问题的节点，底向上级联。 + + 流程: + 1. 收集需修复的 L3 节点 → VLM 重新描述帧 + 2. 收集受影响的 L2 → LLM 从 L3 children 聚合 + 3. 收集受影响的 L1 → LLM 从 L2 children 聚合 + + 参数: + index: 待修复的 TreeIndex（原地修改）。 + issues: detect_issues() 返回的问题列表。 + vlm: VLM 调用端口。 + llm: LLM 调用端口。 + frames_dir: 帧文件根目录。 + srt_entries: 字幕条目列表（可选）。 + + 返回: + RepairStats 统计。 + """ + stats = RepairStats() + + if not issues: + logger.info("无修复任务，跳过") + return stats + + # 构建查找表 + l3_by_id, l2_by_id, l1_by_id, l3_parent_l2, l2_parent_l1 = _build_node_lookup(index) + + # Step 1: 修复 L3 节点（仅处理 empty_field + level 3） + l3_issues = [ + issue for issue in issues if issue.issue_type == "empty_field" and issue.level == 3 + ] + + affected_l2_ids: set[str] = set() + + for issue in l3_issues: + l3_node = l3_by_id.get(issue.node_id) + if l3_node is None: + logger.warning( + "L3 节点 ID 未在树中找到，跳过", + node_id=issue.node_id, + ) + continue + + # 查找帧文件 + if l3_node.frame_path is None: + logger.warning( + "L3 节点无 frame_path，跳过", + node_id=issue.node_id, + ) + continue + + frame_file = frames_dir / l3_node.frame_path + if not frame_file.exists(): + logger.warning( + "L3 帧文件不存在，跳过修复", + node_id=issue.node_id, + frame_path=str(frame_file), + ) + continue + + # 获取 L2 父节点描述作为上下文 + parent_l2 = l3_parent_l2.get(issue.node_id) + l2_description = parent_l2.card.event_description if parent_l2 else "" + + # 构建字幕块 + subtitle_block = _build_subtitle_block(srt_entries, l3_node.timestamp) + + # VLM 重新描述帧 + prompt = _L3_REPAIR_PROMPT.format( + l2_description=l2_description, + subtitle_block=subtitle_block, + ) + messages = [{"role": "user", "content": prompt}] + + try: + response = await vlm.chat_with_images(messages, [str(frame_file)]) + except Exception as exc: + logger.warning( + "L3 修复 VLM 调用失败，跳过: {}", + exc, + node_id=issue.node_id, + ) + continue + + new_card = _parse_l3_card(response.content) + if new_card is None: + logger.warning( + "L3 修复 VLM 输出解析失败，跳过", + node_id=issue.node_id, + raw_preview=response.content[:200], + ) + continue + + # 原地替换 card（L3Node.card 不是 frozen dataclass 的限制字段） + l3_node.card = new_card + stats.l3_repaired += 1 + + # 标记受影响的 L2 父节点 + if parent_l2 is not None: + affected_l2_ids.add(parent_l2.id) + + logger.debug( + "L3 节点修复完成", + node_id=issue.node_id, + frame_summary=new_card.frame_summary[:50], + ) + + # Step 2: 重生成受影响的 L2 节点 + affected_l1_ids: set[str] = set() + + for l2_id in affected_l2_ids: + l2_node = l2_by_id.get(l2_id) + if l2_node is None: + continue + + # 从 L3 children 聚合描述 + l3_texts = "\n".join(f"- {l3.card.frame_summary}" for l3 in l2_node.children) + prompt = _L2_REGEN_PROMPT.format(l3_texts=l3_texts) + messages = [{"role": "user", "content": prompt}] + + try: + response = await llm.chat(messages) + except Exception as exc: + logger.warning( + "L2 重生成 LLM 调用失败，跳过: {}", + exc, + l2_id=l2_id, + ) + continue + + new_card = _parse_l2_card(response.content) + if new_card is None: + logger.warning( + "L2 重生成 LLM 输出解析失败，跳过", + l2_id=l2_id, + raw_preview=response.content[:200], + ) + continue + + l2_node.card = new_card + stats.l2_regenerated += 1 + + # 标记受影响的 L1 父节点 + parent_l1 = l2_parent_l1.get(l2_id) + if parent_l1 is not None: + affected_l1_ids.add(parent_l1.id) + + logger.debug( + "L2 节点重生成完成", + l2_id=l2_id, + event_description=new_card.event_description[:50], + ) + + # Step 3: 重生成受影响的 L1 节点 + for l1_id in affected_l1_ids: + l1_node = l1_by_id.get(l1_id) + if l1_node is None: + continue + + # 从 L2 children 聚合描述 + l2_texts = "\n".join(f"- {l2.card.event_description}" for l2 in l1_node.children) + prompt = _L1_REGEN_PROMPT.format(l2_texts=l2_texts) + messages = [{"role": "user", "content": prompt}] + + try: + response = await llm.chat(messages) + except Exception as exc: + logger.warning( + "L1 重生成 LLM 调用失败，跳过: {}", + exc, + l1_id=l1_id, + ) + continue + + new_card = _parse_l1_card(response.content) + if new_card is None: + logger.warning( + "L1 重生成 LLM 输出解析失败，跳过", + l1_id=l1_id, + raw_preview=response.content[:200], + ) + continue + + l1_node.card = new_card + stats.l1_regenerated += 1 + + logger.debug( + "L1 节点重生成完成", + l1_id=l1_id, + scene_summary=new_card.scene_summary[:50], + ) + + logger.info( + "树修复完成", + l3_repaired=stats.l3_repaired, + l2_regenerated=stats.l2_regenerated, + l1_regenerated=stats.l1_regenerated, + ) + return stats diff --git a/app/tree/repair/supplement.py b/app/tree/repair/supplement.py index 3924896..499ec5b 100644 --- a/app/tree/repair/supplement.py +++ b/app/tree/repair/supplement.py @@ -84,10 +84,11 @@ def deduplicate_field(values: list[str]) -> list[str]: seen: set[str] = set() result: list[str] = [] for v in values: - key = v.strip().lower() + s = str(v).strip() + key = s.lower() if key and key not in seen: seen.add(key) - result.append(v) + result.append(s) return result @@ -278,7 +279,7 @@ def apply_injections(index: TreeIndex, injections: list[dict[str, Any]]) -> Supp stats.facts_skipped += 1 continue - inject_value = instr.get("inject_value", "") + inject_value = str(instr.get("inject_value", "")).strip() if not inject_value: stats.facts_skipped += 1 continue diff --git a/core/evolution/patch.py b/core/evolution/patch.py index d4cef82..87effa5 100644 --- a/core/evolution/patch.py +++ b/core/evolution/patch.py @@ -15,9 +15,7 @@ APPENDIX_MAX_CHARS = 2000 # appendix 区软上限（守设计「长度上限+wa MOMENTUM_START = "" MOMENTUM_END = "" MOMENTUM_MAX_CHARS = 2000 # momentum 区软上限（与 appendix 一致：超限 warning 不截断） -MOMENTUM_HEADING = ( - "## 动量指导（每轮重写，勿手改）" # replace_momentum 写入的固定标题行 -) +MOMENTUM_HEADING = "## 动量指导（每轮重写，勿手改）" # replace_momentum 写入的固定标题行 def momentum_region_bounds(text: str) -> tuple[int, int] | None: @@ -303,9 +301,7 @@ def _insert_at(content: str, at: int, payload: str) -> str: return head + "\n\n" + payload + "\n" -def _do_append( - content: str, payload: str, ranges: list[tuple[int, int]] -) -> tuple[str, str]: +def _do_append(content: str, payload: str, ranges: list[tuple[int, int]]) -> tuple[str, str]: """执行 append 操作，返回更新后内容与状态字符串。""" return _insert_at(content, _append_at(content, ranges), payload), "applied_append" @@ -351,9 +347,7 @@ def _do_replace_delete( return new_content, "applied_" + op -def _apply_one( - content: str, edit: dict, ranges: list[tuple[int, int]] -) -> tuple[str, dict]: +def _apply_one(content: str, edit: dict, ranges: list[tuple[int, int]]) -> tuple[str, dict]: """应用单条 edit，返回 (更新后内容, 状态报告)。""" if not isinstance(edit, dict): return content, { @@ -382,9 +376,7 @@ def _apply_one( return content, report if op in ("replace", "delete"): - content, report["status"] = _do_replace_delete( - op, content, target, payload, ranges - ) + content, report["status"] = _do_replace_delete(op, content, target, payload, ranges) return content, report logger.warning("未知 op，跳过: {}", op) diff --git a/scripts/repair_trees.sh b/scripts/repair_trees.sh index 2f20f37..b228117 100755 --- a/scripts/repair_trees.sh +++ b/scripts/repair_trees.sh @@ -17,6 +17,8 @@ set -euo pipefail CONCURRENCY="${CONCURRENCY:-16}" +# shellcheck source=/dev/null +source "$(conda info --base)/etc/profile.d/conda.sh" conda activate Video-Tree-TRM python tools/repair_trees.py \ diff --git a/tests/unit/test_repair_regenerator.py b/tests/unit/test_repair_regenerator.py new file mode 100644 index 0000000..6d58630 --- /dev/null +++ b/tests/unit/test_repair_regenerator.py @@ -0,0 +1,293 @@ +"""修复重生成器单元测试。""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +from app.tree.index import ( + IndexMeta, + L1Card, + L1Node, + L2Card, + L2Node, + L3Card, + L3Node, + TreeIndex, +) +from app.tree.repair.detector import NodeIssue +from app.tree.repair.regenerator import RepairStats, repair_tree +from core.types import LLMResponse + + +def _mock_response(content: str) -> LLMResponse: + """构造模拟 LLMResponse。""" + return LLMResponse( + content=content, + thinking="", + model="mock", + provider="mock", + prompt_tokens=0, + completion_tokens=0, + latency_ms=0, + ttft_ms=None, + max_inter_token_ms=None, + cache_hit=False, + call_id="mock", + ) + + +class MockVLM: + """模拟 VLM 端口，返回固定的 L3Card JSON。""" + + def __init__(self) -> None: + self.call_count = 0 + + async def chat_with_images( + self, + messages: list[dict], + images: list, + **kw: object, + ) -> LLMResponse: + """模拟 VLM 图文调用。""" + self.call_count += 1 + return _mock_response( + json.dumps( + { + "frame_summary": "修复后的帧描述", + "visible_entities": ["修复实体"], + "ongoing_actions": ["修复动作"], + "visible_text": [], + "spatial_layout": "居中", + "visual_attributes": {"lighting": "明亮"}, + } + ) + ) + + +class MockLLM: + """模拟 LLM 端口，根据 prompt 内容返回 L2Card 或 L1Card JSON。""" + + def __init__(self) -> None: + self.call_count = 0 + + async def chat( + self, + messages: list[dict], + **kw: object, + ) -> LLMResponse: + """模拟 LLM 文本调用，按 prompt 内容区分 L2/L1 响应。""" + self.call_count += 1 + content = messages[-1].get("content", "") + if "段落" in content or "scene" in content.lower(): + return _mock_response( + json.dumps( + { + "scene_summary": "修复后的场景", + "main_setting": "室内", + "key_entities": [], + "main_actions": [], + "topic_keywords": [], + "visible_text": [], + "temporal_flow": "", + } + ) + ) + return _mock_response( + json.dumps( + { + "event_description": "修复后的事件", + "entities": [], + "actions": [], + "action_subjects": [], + "visible_text": [], + "spatial_relations": "", + "state_changes": None, + } + ) + ) + + +class TestRepairTree: + """repair_tree 核心测试。""" + + def _make_broken_tree( + self, + tmp_path: Path, + ) -> tuple[TreeIndex, list[NodeIssue]]: + """构建含一个空 frame_summary 的 L3 节点的测试树。""" + frame_path = tmp_path / "frames" / "L1_000_L2_000_L3_000.jpg" + frame_path.parent.mkdir(parents=True) + frame_path.write_bytes(b"\xff\xd8\xff\xe0fake") + + l3 = L3Node( + id="vid_L1_000_L2_000_L3_000", + card=L3Card("", [], [], [], "", {}), + timestamp=1.0, + frame_path="frames/L1_000_L2_000_L3_000.jpg", + ) + l2 = L2Node( + id="vid_L1_000_L2_000", + card=L2Card("原始事件", [], [], [], [], "", None), + time_range=(0.0, 10.0), + children=[l3], + ) + l1 = L1Node( + id="vid_L1_000", + card=L1Card("原始场景", "", [], [], [], [], ""), + time_range=(0.0, 10.0), + children=[l2], + ) + index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1]) + issues = [ + NodeIssue( + "vid_L1_000_L2_000_L3_000", + 3, + "empty_field", + "frame_summary 为空", + ) + ] + return index, issues + + def test_repairs_l3_and_cascades(self, tmp_path: Path) -> None: + """修复 L3 后应级联重生成 L2 和 L1。""" + index, issues = self._make_broken_tree(tmp_path) + stats = asyncio.run(repair_tree(index, issues, MockVLM(), MockLLM(), tmp_path)) + assert stats.l3_repaired == 1 + assert stats.l2_regenerated == 1 + assert stats.l1_regenerated == 1 + assert index.roots[0].children[0].children[0].card.frame_summary == "修复后的帧描述" + assert index.roots[0].children[0].card.event_description == "修复后的事件" + assert index.roots[0].card.scene_summary == "修复后的场景" + + def test_no_issues_no_changes(self) -> None: + """无问题时不进行任何修复。""" + l3 = L3Node( + id="l1_0_l2_0_l3_0", + card=L3Card("正常", [], [], [], "", {}), + timestamp=1.0, + ) + l2 = L2Node( + id="l1_0_l2_0", + card=L2Card("正常事件", [], [], [], [], "", None), + time_range=(0.0, 10.0), + children=[l3], + ) + l1 = L1Node( + id="l1_0", + card=L1Card("正常场景", "", [], [], [], [], ""), + time_range=(0.0, 10.0), + children=[l2], + ) + index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1]) + stats = asyncio.run(repair_tree(index, [], MockVLM(), MockLLM(), Path("/tmp"))) + assert stats.l3_repaired == 0 + assert stats.l2_regenerated == 0 + assert stats.l1_regenerated == 0 + + def test_stats_dataclass(self) -> None: + """RepairStats 数据类字段验证。""" + stats = RepairStats(l3_repaired=2, l2_regenerated=1, l1_regenerated=1) + assert stats.l3_repaired == 2 + assert stats.l2_regenerated == 1 + assert stats.l1_regenerated == 1 + + def test_skips_non_empty_field_issues(self, tmp_path: Path) -> None: + """非 empty_field 类型的 issue 不触发 L3 修复。""" + l3 = L3Node( + id="l1_0_l2_0_l3_0", + card=L3Card("正常描述", [], [], [], "", {}), + timestamp=1.0, + ) + l2 = L2Node( + id="l1_0_l2_0", + card=L2Card("原始事件", [], [], [], [], "", None), + time_range=(0.0, 10.0), + children=[l3], + ) + l1 = L1Node( + id="l1_0", + card=L1Card("原始场景", "", [], [], [], [], ""), + time_range=(0.0, 10.0), + children=[l2], + ) + index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1]) + issues = [NodeIssue("l1_0_l2_0_l3_0", 3, "missing_frame", "帧文件不存在")] + stats = asyncio.run(repair_tree(index, issues, MockVLM(), MockLLM(), tmp_path)) + assert stats.l3_repaired == 0 + assert stats.l2_regenerated == 0 + + def test_multiple_l3_under_same_l2(self, tmp_path: Path) -> None: + """同一 L2 下多个 L3 修复后，L2 只重生成一次。""" + frame_dir = tmp_path / "frames" + frame_dir.mkdir(parents=True) + for i in range(2): + (frame_dir / f"f{i}.jpg").write_bytes(b"\xff\xd8\xff\xe0fake") + + l3_a = L3Node( + id="l1_0_l2_0_l3_0", + card=L3Card("", [], [], [], "", {}), + timestamp=1.0, + frame_path="frames/f0.jpg", + ) + l3_b = L3Node( + id="l1_0_l2_0_l3_1", + card=L3Card("", [], [], [], "", {}), + timestamp=2.0, + frame_path="frames/f1.jpg", + ) + l2 = L2Node( + id="l1_0_l2_0", + card=L2Card("原始事件", [], [], [], [], "", None), + time_range=(0.0, 10.0), + children=[l3_a, l3_b], + ) + l1 = L1Node( + id="l1_0", + card=L1Card("原始场景", "", [], [], [], [], ""), + time_range=(0.0, 10.0), + children=[l2], + ) + index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1]) + issues = [ + NodeIssue("l1_0_l2_0_l3_0", 3, "empty_field", "frame_summary 为空"), + NodeIssue("l1_0_l2_0_l3_1", 3, "empty_field", "frame_summary 为空"), + ] + vlm = MockVLM() + llm = MockLLM() + stats = asyncio.run(repair_tree(index, issues, vlm, llm, tmp_path)) + assert stats.l3_repaired == 2 + assert stats.l2_regenerated == 1 + assert stats.l1_regenerated == 1 + assert vlm.call_count == 2 + # LLM 应被调用 2 次：一次 L2 + 一次 L1 + assert llm.call_count == 2 + + def test_missing_frame_file_skips_l3(self, tmp_path: Path) -> None: + """帧文件不存在时跳过该 L3 节点的修复。""" + l3 = L3Node( + id="l1_0_l2_0_l3_0", + card=L3Card("", [], [], [], "", {}), + timestamp=1.0, + frame_path="frames/nonexistent.jpg", + ) + l2 = L2Node( + id="l1_0_l2_0", + card=L2Card("原始事件", [], [], [], [], "", None), + time_range=(0.0, 10.0), + children=[l3], + ) + l1 = L1Node( + id="l1_0", + card=L1Card("原始场景", "", [], [], [], [], ""), + time_range=(0.0, 10.0), + children=[l2], + ) + index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1]) + issues = [NodeIssue("l1_0_l2_0_l3_0", 3, "empty_field", "frame_summary 为空")] + stats = asyncio.run(repair_tree(index, issues, MockVLM(), MockLLM(), tmp_path)) + # 帧文件不存在 → 跳过 L3 修复 → 无级联 + assert stats.l3_repaired == 0 + assert stats.l2_regenerated == 0 + assert stats.l1_regenerated == 0 diff --git a/tools/migrate_from_trm4.sh b/tools/migrate_from_trm4.sh new file mode 100755 index 0000000..ec8abb3 --- /dev/null +++ b/tools/migrate_from_trm4.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# 从 TRM4.zip 迁移资产到 TRM5 +# 用法: bash tools/migrate_from_trm4.sh /path/to/Video-Tree-TRM4.zip +set -euo pipefail + +ZIP_PATH="${1:?用法: bash tools/migrate_from_trm4.sh /path/to/Video-Tree-TRM4.zip}" +PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +TMP_DIR=$(mktemp -d) + +echo "=== TRM4 -> TRM5 迁移 ===" +echo "ZIP: $ZIP_PATH" +echo "项目根: $PROJECT_ROOT" +echo "临时目录: $TMP_DIR" + +# 1. 解压 +echo "[1/6] 解压 TRM4.zip..." +unzip -q "$ZIP_PATH" -d "$TMP_DIR" +SRC="$TMP_DIR/Video-Tree-TRM4" + +# 2. 拷贝帧文件 (rsync --ignore-existing) +echo "[2/6] 拷贝帧文件..." +mkdir -p "$PROJECT_ROOT/store/videos" +for vid_dir in "$SRC"/store/videos/*/; do + vid=$(basename "$vid_dir") + dst="$PROJECT_ROOT/store/videos/$vid" + mkdir -p "$dst" + if [ -d "$vid_dir/frames" ]; then + rsync -a --ignore-existing "$vid_dir/frames/" "$dst/frames/" + fi +done + +# 3. 拷贝 SRT 字幕 +echo "[3/6] 拷贝 SRT 字幕..." +mkdir -p "$PROJECT_ROOT/data/Video-MME/subtitle" +if [ -d "$SRC/data/Video-MME/subtitle" ]; then + rsync -a --ignore-existing "$SRC/data/Video-MME/subtitle/" "$PROJECT_ROOT/data/Video-MME/subtitle/" +fi + +# 4. 拷贝视频压缩包 +echo "[4/6] 拷贝视频压缩包..." +mkdir -p "$PROJECT_ROOT/data/Video-MME/original_data" +if [ -d "$SRC/data/Video-MME/original_data" ]; then + rsync -a --ignore-existing "$SRC/data/Video-MME/original_data/" "$PROJECT_ROOT/data/Video-MME/original_data/" +fi + +# 5. 拷贝问题 JSON +echo "[5/6] 拷贝 Benchmark 问题..." +mkdir -p "$PROJECT_ROOT/store/questions" +if [ -d "$SRC/store/questions" ]; then + rsync -a --ignore-existing "$SRC/store/questions/" "$PROJECT_ROOT/store/questions/" +fi + +# 6. 格式转换 +echo "[6/6] 格式转换 flat -> TreeIndex..." +conda run -n Video-Tree-TRM python "$PROJECT_ROOT/tools/convert_flat_to_treeindex.py" \ + "$SRC/store/videos" "$PROJECT_ROOT/store/videos" + +# 验收 +echo "" +echo "=== 验收检查 ===" +VIDEO_COUNT=$(find "$PROJECT_ROOT/store/videos" -name "tree.json" | wc -l) +SRT_COUNT=$(find "$PROJECT_ROOT/data/Video-MME/subtitle" -name "*.srt" 2>/dev/null | wc -l) +echo "视频树: $VIDEO_COUNT (期望 300)" +echo "SRT 字幕: $SRT_COUNT (期望 >=290)" + +# 清理 +echo "清理临时目录..." +rm -rf "$TMP_DIR" + +if [ "$VIDEO_COUNT" -lt 300 ]; then + echo "WARNING: 视频树数量不足 300，请检查" + exit 1 +fi + +echo "迁移完成"