Compare commits
61 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e0f3ee10ec | |||
| 9800fef37a | |||
| d349fe1148 | |||
| b052c1f3ee | |||
| be3c176a46 | |||
| b929a5db6c | |||
| ce43871828 | |||
| 6a2ddb1624 | |||
| 09a385addc | |||
| d84cd679b4 | |||
| 292dd0fa13 | |||
| eba4344a4b | |||
| 4e241aa581 | |||
| 8bc4132750 | |||
| 4634414606 | |||
| 6072ee7d0b | |||
| dc091361c3 | |||
| 49d6fe8f51 | |||
| f7193551dd | |||
| 3005f24577 | |||
| 4eefbfdd74 | |||
| 9d4d52dac5 | |||
| 09100dcbd4 | |||
| 84e8abbf4f | |||
| 7cd49a5a3f | |||
| 499c5b8043 | |||
| 4baf92c93f | |||
| f4f92b0938 | |||
| ca3ea1cdf2 | |||
| ba7fdaddf6 | |||
| 3ae3d5ab50 | |||
| 86b19d1e07 | |||
| 60e737e2fc | |||
| 82f1607195 | |||
| 44ee62867d | |||
| ffb37a1c1e | |||
| dcaeefa484 | |||
| 7af64db499 | |||
| 9e1f39c147 | |||
| 5c89ca543a | |||
| 11107f5758 | |||
| da28c10c84 | |||
| d8a903fb54 | |||
| 8d515ff01f | |||
| dea8a7d3f6 | |||
| eea3bcba3f | |||
| 4686adf266 | |||
| ee5bd0de57 | |||
| 73b240cc84 | |||
| 18971a794b | |||
| edaa0d8290 | |||
| 12f20493c1 | |||
| fb6f9964d8 | |||
| e3b027ce34 | |||
| c1680447c0 | |||
| 547c9ddd84 | |||
| e0f7851975 | |||
| c136de076d | |||
| 22ad014973 | |||
| 2be2569ed8 | |||
| c2ba3ed591 |
+9
-9
@@ -4,28 +4,28 @@ NO_PROXY=dashscope.aliyuncs.com,api.deepseek.com
|
|||||||
|
|
||||||
# ── 搜索 Agent LLM ──
|
# ── 搜索 Agent LLM ──
|
||||||
SEARCH_LLM_MODEL=deepseek-v4-pro
|
SEARCH_LLM_MODEL=deepseek-v4-pro
|
||||||
SEARCH_LLM_BASE_URL=https://api.deepseek.com/v1
|
SEARCH_LLM_BASE_URL=https://newapi.iomgaa.online/v1
|
||||||
SEARCH_LLM_API_KEY=sk-xxx
|
SEARCH_LLM_API_KEY=sk-lhDmxnhlnPd7ketQ3Z4uMRj4dCgnVpSJzdY2VTrjYpKFmCIV
|
||||||
|
|
||||||
# ── 评估 Judge LLM ──
|
# ── 评估 Judge LLM ──
|
||||||
JUDGE_LLM_MODEL=deepseek-v4-pro
|
JUDGE_LLM_MODEL=deepseek-v4-pro
|
||||||
JUDGE_LLM_BASE_URL=https://api.deepseek.com/v1
|
JUDGE_LLM_BASE_URL=https://newapi.iomgaa.online/v1
|
||||||
JUDGE_LLM_API_KEY=sk-xxx
|
JUDGE_LLM_API_KEY=sk-lhDmxnhlnPd7ketQ3Z4uMRj4dCgnVpSJzdY2VTrjYpKFmCIV
|
||||||
|
|
||||||
# ── 视觉模型(Qwen VL)──
|
# ── 视觉模型(Qwen VL)──
|
||||||
VL_LLM_MODEL=qwen3.6-plus
|
VL_LLM_MODEL=qwen3.6-plus
|
||||||
VL_LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
|
VL_LLM_BASE_URL=https://newapi.iomgaa.online/v1
|
||||||
VL_LLM_API_KEY=sk-xxx
|
VL_LLM_API_KEY=sk-lhDmxnhlnPd7ketQ3Z4uMRj4dCgnVpSJzdY2VTrjYpKFmCIV
|
||||||
|
|
||||||
# ── 进化 LLM(Prompt 改写)──
|
# ── 进化 LLM(Prompt 改写)──
|
||||||
EVOLVE_LLM_MODEL=deepseek-v4-pro
|
EVOLVE_LLM_MODEL=deepseek-v4-pro
|
||||||
EVOLVE_LLM_BASE_URL=https://api.deepseek.com/v1
|
EVOLVE_LLM_BASE_URL=https://newapi.iomgaa.online/v1
|
||||||
EVOLVE_LLM_API_KEY=sk-xxx
|
EVOLVE_LLM_API_KEY=sk-lhDmxnhlnPd7ketQ3Z4uMRj4dCgnVpSJzdY2VTrjYpKFmCIV
|
||||||
|
|
||||||
# ── ASR 字幕生成(Groq Whisper)──
|
# ── ASR 字幕生成(Groq Whisper)──
|
||||||
ASR_MODEL=whisper-large-v3
|
ASR_MODEL=whisper-large-v3
|
||||||
ASR_BASE_URL=https://api.groq.com/openai/v1
|
ASR_BASE_URL=https://api.groq.com/openai/v1
|
||||||
ASR_API_KEY=gsk-xxx
|
ASR_API_KEY=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
|
||||||
|
|
||||||
# ── MonkeyOCR ──
|
# ── MonkeyOCR ──
|
||||||
MONKEY_OCR_URLS=http://10.77.0.20:7866,http://10.77.0.20:7867
|
MONKEY_OCR_URLS=http://10.77.0.20:7866,http://10.77.0.20:7867
|
||||||
|
|||||||
@@ -181,6 +181,7 @@ pencil/
|
|||||||
|
|
||||||
# 数据与实验产物(不提交)
|
# 数据与实验产物(不提交)
|
||||||
store/
|
store/
|
||||||
|
!store/prompts/
|
||||||
workspaces/
|
workspaces/
|
||||||
results/
|
results/
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,7 @@
|
|||||||
|
|
||||||
| PyTorch 概念 | 本项目对应 | 代码位置 |
|
| PyTorch 概念 | 本项目对应 | 代码位置 |
|
||||||
|-------------|-----------|----------|
|
|-------------|-----------|----------|
|
||||||
| `DataLoader` | 出题 question_gen | `app/question_gen/generator.py` |
|
| `DataLoader` | 出题 question_gen | `app/question_gen/loader.py` |
|
||||||
| `model.forward()` | 推理 inference | `app/harness/inference.py` + `core/agent/loop.py` |
|
| `model.forward()` | 推理 inference | `app/harness/inference.py` + `core/agent/loop.py` |
|
||||||
| `loss.backward()` | 诊断 diagnose | `core/evolution/diagnose.py` |
|
| `loss.backward()` | 诊断 diagnose | `core/evolution/diagnose.py` |
|
||||||
| `optimizer.step()` | 进化 evolve | `core/evolution/evolve.py` |
|
| `optimizer.step()` | 进化 evolve | `core/evolution/evolve.py` |
|
||||||
|
|||||||
@@ -0,0 +1,184 @@
|
|||||||
|
"""嵌入适配器 —— local/remote 双后端实现。
|
||||||
|
|
||||||
|
封装文本嵌入器,支持本地 sentence-transformers 和远程 OpenAI 兼容 API 两种后端。
|
||||||
|
提供统一的 ``embed()`` / ``embed_tensor()`` 接口,冻结不训练。
|
||||||
|
两个类均满足 ``app.ports.EmbeddingProvider`` Protocol。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import torch
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from numpy import ndarray
|
||||||
|
from torch import Tensor
|
||||||
|
|
||||||
|
|
||||||
|
class LocalEmbeddingProvider:
|
||||||
|
"""本地 sentence-transformers 嵌入器(冻结)。
|
||||||
|
|
||||||
|
使用 HuggingFace sentence-transformers 加载模型进行本地推理,
|
||||||
|
所有参数冻结,仅用于嵌入提取。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
dim: 嵌入维度 D。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, model_name: str, embed_dim: int, device: str = "cpu") -> None:
|
||||||
|
"""初始化本地嵌入模型。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
model_name: HuggingFace 模型名称(如 'BAAI/bge-base-zh-v1.5')。
|
||||||
|
embed_dim: 期望的嵌入维度。
|
||||||
|
device: 推理设备('cpu' / 'cuda' 等)。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
AssertionError: 模型实际维度与 embed_dim 不一致。
|
||||||
|
"""
|
||||||
|
from sentence_transformers import SentenceTransformer
|
||||||
|
|
||||||
|
self._dim = embed_dim
|
||||||
|
|
||||||
|
self._model = SentenceTransformer(model_name, device=device)
|
||||||
|
self._model.eval()
|
||||||
|
# 冻结所有参数
|
||||||
|
for param in self._model.parameters():
|
||||||
|
param.requires_grad = False
|
||||||
|
|
||||||
|
actual_dim = self._model.get_sentence_embedding_dimension()
|
||||||
|
assert actual_dim == self._dim, (
|
||||||
|
f"模型实际维度 ({actual_dim}) 与配置 embed_dim ({self._dim}) 不一致"
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("本地嵌入模型初始化完成", model=model_name, device=device)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# 公共接口
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
@property
|
||||||
|
def dim(self) -> int:
|
||||||
|
"""嵌入维度 D。"""
|
||||||
|
return self._dim
|
||||||
|
|
||||||
|
def embed(self, texts: str | list[str]) -> ndarray:
|
||||||
|
"""文本 → 嵌入向量(L2 归一化)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
texts: 单条文本或文本列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
[N, D] ndarray,每行 L2 范数为 1.0。单条文本时 N=1。
|
||||||
|
"""
|
||||||
|
if isinstance(texts, str):
|
||||||
|
texts = [texts]
|
||||||
|
|
||||||
|
with torch.no_grad():
|
||||||
|
embeddings = self._model.encode(
|
||||||
|
texts,
|
||||||
|
normalize_embeddings=True,
|
||||||
|
convert_to_numpy=True,
|
||||||
|
)
|
||||||
|
# sentence-transformers encode 返回 ndarray [N, D]
|
||||||
|
if embeddings.ndim == 1:
|
||||||
|
embeddings = embeddings.reshape(1, -1)
|
||||||
|
return embeddings
|
||||||
|
|
||||||
|
def embed_tensor(self, texts: str | list[str]) -> Tensor:
|
||||||
|
"""文本 → 嵌入 Tensor(L2 归一化)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
texts: 单条文本或文本列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
[N, D] torch.Tensor(float32)。
|
||||||
|
"""
|
||||||
|
arr = self.embed(texts)
|
||||||
|
return torch.from_numpy(arr).float()
|
||||||
|
|
||||||
|
|
||||||
|
class RemoteEmbeddingProvider:
|
||||||
|
"""远程 OpenAI 兼容 API 嵌入器。
|
||||||
|
|
||||||
|
通过 OpenAI 兼容 API(如 GPUStack)调用远程嵌入模型。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
dim: 嵌入维度 D。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, model_name: str, embed_dim: int, api_key: str, api_url: str) -> None:
|
||||||
|
"""初始化远程嵌入客户端。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
model_name: 远程模型名称。
|
||||||
|
embed_dim: 期望的嵌入维度。
|
||||||
|
api_key: API 密钥。
|
||||||
|
api_url: API 基础 URL。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: api_key 或 api_url 为空。
|
||||||
|
"""
|
||||||
|
if not api_key:
|
||||||
|
raise ValueError("远程模式必须提供 api_key")
|
||||||
|
if not api_url:
|
||||||
|
raise ValueError("远程模式必须提供 api_url")
|
||||||
|
|
||||||
|
from openai import OpenAI
|
||||||
|
|
||||||
|
self._dim = embed_dim
|
||||||
|
self._model_name = model_name
|
||||||
|
self._client = OpenAI(base_url=api_url, api_key=api_key)
|
||||||
|
|
||||||
|
logger.info("远程嵌入客户端初始化完成", model=model_name, api_url=api_url)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# 公共接口
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
@property
|
||||||
|
def dim(self) -> int:
|
||||||
|
"""嵌入维度 D。"""
|
||||||
|
return self._dim
|
||||||
|
|
||||||
|
def embed(self, texts: str | list[str]) -> ndarray:
|
||||||
|
"""文本 → 嵌入向量(L2 归一化)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
texts: 单条文本或文本列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
[N, D] ndarray,每行 L2 范数为 1.0。单条文本时 N=1。
|
||||||
|
"""
|
||||||
|
if isinstance(texts, str):
|
||||||
|
texts = [texts]
|
||||||
|
|
||||||
|
response = self._client.embeddings.create(
|
||||||
|
model=self._model_name,
|
||||||
|
input=texts,
|
||||||
|
)
|
||||||
|
# 按 index 排序,确保顺序一致
|
||||||
|
sorted_data = sorted(response.data, key=lambda x: x.index)
|
||||||
|
embeddings = np.array([item.embedding for item in sorted_data], dtype=np.float32)
|
||||||
|
|
||||||
|
# L2 归一化
|
||||||
|
norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
|
||||||
|
norms = np.maximum(norms, 1e-12) # 避免除零
|
||||||
|
embeddings = embeddings / norms
|
||||||
|
|
||||||
|
return embeddings
|
||||||
|
|
||||||
|
def embed_tensor(self, texts: str | list[str]) -> Tensor:
|
||||||
|
"""文本 → 嵌入 Tensor(L2 归一化)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
texts: 单条文本或文本列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
[N, D] torch.Tensor(float32)。
|
||||||
|
"""
|
||||||
|
arr = self.embed(texts)
|
||||||
|
return torch.from_numpy(arr).float()
|
||||||
+128
@@ -0,0 +1,128 @@
|
|||||||
|
"""MonkeyOCR HTTP 客户端 — 帧文字转录的异构硬证据源。
|
||||||
|
|
||||||
|
服务由用户在 LAN 部署(双端点轮询);请求必须绕过代理(trust_env=False)。
|
||||||
|
实现 OCRProvider Protocol(app/ports.py)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import itertools
|
||||||
|
import threading
|
||||||
|
from pathlib import Path # noqa: TC003 — 运行时需要(方法签名 + open())
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
_TIMEOUT_S = 15
|
||||||
|
|
||||||
|
|
||||||
|
class MonkeyOCRClient:
|
||||||
|
"""MonkeyOCR 服务客户端:多端点轮询、单帧失败降级为跳过。
|
||||||
|
|
||||||
|
关键实现细节:实例可被多线程共享——端点轮询加锁、Session 线程局部
|
||||||
|
(A/B 评测会以 4 线程并发调用同一实例)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
urls: 服务端点列表(如 ["http://10.77.0.20:7866", ...]),非空。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: urls 为空时抛出。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, urls: list[str]) -> None:
|
||||||
|
if not urls:
|
||||||
|
raise ValueError("MonkeyOCR 端点列表不能为空")
|
||||||
|
self._urls = [u.rstrip("/") for u in urls]
|
||||||
|
self._rr = itertools.cycle(self._urls)
|
||||||
|
self._rr_lock = threading.Lock()
|
||||||
|
self._local = threading.local()
|
||||||
|
|
||||||
|
def _get_session(self) -> requests.Session:
|
||||||
|
"""返回当前线程专属的 Session(惰性创建并复用,trust_env=False 绕代理)。"""
|
||||||
|
session = getattr(self._local, "session", None)
|
||||||
|
if session is None:
|
||||||
|
session = requests.Session()
|
||||||
|
session.trust_env = False # LAN 直连,绕过代理
|
||||||
|
self._local.session = session
|
||||||
|
return session
|
||||||
|
|
||||||
|
def _check_health_sync(self) -> None:
|
||||||
|
"""同步预检所有端点,任一不可达即抛错(供 asyncio.to_thread 调用)。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
RuntimeError: 端点不可达或 /health 非 2xx。
|
||||||
|
"""
|
||||||
|
for url in self._urls:
|
||||||
|
try:
|
||||||
|
resp = self._get_session().get(f"{url}/health", timeout=5)
|
||||||
|
except requests.RequestException as e:
|
||||||
|
raise RuntimeError(f"MonkeyOCR 端点不可达: {url}: {e}") from e
|
||||||
|
if not resp.ok:
|
||||||
|
raise RuntimeError(f"MonkeyOCR 健康检查失败: {url}: {resp.status_code}")
|
||||||
|
|
||||||
|
async def check_health(self) -> None:
|
||||||
|
"""异步预检所有端点,任一不可达即抛错(A/B qtr_ocr 臂启动门)。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
RuntimeError: 端点不可达或 /health 非 2xx。
|
||||||
|
"""
|
||||||
|
await asyncio.to_thread(self._check_health_sync)
|
||||||
|
|
||||||
|
def _transcribe_frames_sync(self, frame_paths: list[Path]) -> str:
|
||||||
|
"""同步逐帧转录并拼接(供 asyncio.to_thread 调用)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
frame_paths: 帧文件路径列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
"帧1: <行1> | <行2>\\n帧2: ..." 格式文本;无任何有效结果时空串。
|
||||||
|
"""
|
||||||
|
parts: list[str] = []
|
||||||
|
for i, path in enumerate(frame_paths, 1):
|
||||||
|
lines = self._transcribe_one(path)
|
||||||
|
if lines:
|
||||||
|
parts.append(f"帧{i}: " + " | ".join(lines))
|
||||||
|
return "\n".join(parts)
|
||||||
|
|
||||||
|
async def transcribe_frames(self, frame_paths: list[Path]) -> str:
|
||||||
|
"""异步逐帧转录并拼接为注入文本;单帧失败跳过,全失败返回空串。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
frame_paths: 帧文件路径列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
"帧1: <行1> | <行2>\\n帧2: ..." 格式文本;无任何有效结果时空串。
|
||||||
|
"""
|
||||||
|
return await asyncio.to_thread(self._transcribe_frames_sync, frame_paths)
|
||||||
|
|
||||||
|
def _transcribe_one(self, path: Path) -> list[str]:
|
||||||
|
"""单帧转录:空结果/单字符行过滤 + 帧内行级去重。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
path: 帧文件路径。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
过滤去重后的文本行列表;请求失败或无有效行时空列表。
|
||||||
|
"""
|
||||||
|
with self._rr_lock:
|
||||||
|
url = next(self._rr)
|
||||||
|
try:
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
resp = self._get_session().post(
|
||||||
|
f"{url}/ocr/text", files={"file": f}, timeout=_TIMEOUT_S
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
content = resp.json().get("content", "")
|
||||||
|
except (requests.RequestException, ValueError) as e:
|
||||||
|
logger.warning("MonkeyOCR 单帧转录失败,跳过 {}: {}", path.name, e)
|
||||||
|
return []
|
||||||
|
seen: set[str] = set()
|
||||||
|
lines: list[str] = []
|
||||||
|
for ln in content.splitlines():
|
||||||
|
ln = ln.strip()
|
||||||
|
if len(ln) <= 1 or ln in seen:
|
||||||
|
continue
|
||||||
|
seen.add(ln)
|
||||||
|
lines.append(ln)
|
||||||
|
return lines
|
||||||
@@ -47,9 +47,7 @@ class RedisResponseCache:
|
|||||||
digest = hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
digest = hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||||
return f"llm_cache:{digest}"
|
return f"llm_cache:{digest}"
|
||||||
|
|
||||||
async def get(
|
async def get(self, model: str, messages: list[dict[str, str]]) -> LLMResponse | None:
|
||||||
self, model: str, messages: list[dict[str, str]]
|
|
||||||
) -> LLMResponse | None:
|
|
||||||
"""从缓存读取 LLM 响应。
|
"""从缓存读取 LLM 响应。
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -87,9 +85,7 @@ class RedisResponseCache:
|
|||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
key = self._build_key(model, messages)
|
key = self._build_key(model, messages)
|
||||||
value = json.dumps(
|
value = json.dumps(dataclasses.asdict(response), ensure_ascii=False)
|
||||||
dataclasses.asdict(response), ensure_ascii=False
|
|
||||||
)
|
|
||||||
await self._redis.set(key, value, ex=self._ttl_s)
|
await self._redis.set(key, value, ex=self._ttl_s)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.warning("Redis 缓存写入失败,跳过缓存")
|
logger.warning("Redis 缓存写入失败,跳过缓存")
|
||||||
|
|||||||
@@ -3,10 +3,14 @@
|
|||||||
通过 asyncio.to_thread 将 SQLite 同步写入桥接到异步接口,
|
通过 asyncio.to_thread 将 SQLite 同步写入桥接到异步接口,
|
||||||
确保事件循环不被阻塞。表在首次写入时懒初始化。
|
确保事件循环不被阻塞。表在首次写入时懒初始化。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import sqlite3
|
import sqlite3
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+128
@@ -0,0 +1,128 @@
|
|||||||
|
"""GovernedVLMClient -- VLMProvider 最小可用实现。
|
||||||
|
|
||||||
|
将图片编码为 base64,构造 OpenAI Vision API 格式的 messages,
|
||||||
|
委托给已有的 GovernedLLMClient 发送。复用 LLM 治理栈的全部能力
|
||||||
|
(熔断、缓存、重试、遥测)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import mimetypes
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from adapters.llm import GovernedLLMClient
|
||||||
|
from core.types import LLMResponse
|
||||||
|
|
||||||
|
|
||||||
|
class GovernedVLMClient:
|
||||||
|
"""VLMProvider 实现——包装 GovernedLLMClient,注入 base64 图片。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
governed_llm: 已初始化的 GovernedLLMClient 实例。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, governed_llm: GovernedLLMClient) -> None:
|
||||||
|
self._llm = governed_llm
|
||||||
|
|
||||||
|
async def chat_with_images(
|
||||||
|
self,
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
images: list[str | Path],
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> LLMResponse:
|
||||||
|
"""图文调用:将图片编码为 base64 嵌入 messages,委托给 LLM 客户端。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
messages: 对话消息列表。最后一条 user message 的 content 会被扩展为
|
||||||
|
包含图片的多模态格式。
|
||||||
|
images: 图片文件路径列表。
|
||||||
|
session_id: 会话 ID(遥测用)。
|
||||||
|
parent_call_id: 父调用 ID(遥测用)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
LLMResponse。
|
||||||
|
"""
|
||||||
|
vision_messages = self._inject_images(messages, images)
|
||||||
|
return await self._llm.chat(
|
||||||
|
vision_messages,
|
||||||
|
session_id=session_id,
|
||||||
|
parent_call_id=parent_call_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _encode_image(image_path: str | Path) -> str:
|
||||||
|
"""将图片文件编码为 base64 data URL。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
image_path: 图片文件路径。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
data:image/<type>;base64,<data> 格式的字符串。
|
||||||
|
"""
|
||||||
|
path = Path(image_path)
|
||||||
|
mime_type = mimetypes.guess_type(str(path))[0] or "image/jpeg"
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
b64 = base64.b64encode(f.read()).decode("utf-8")
|
||||||
|
return f"data:{mime_type};base64,{b64}"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _inject_images(
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
images: list[str | Path],
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""将图片注入最后一条 user message,构造 OpenAI Vision API 格式。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
messages: 原始消息列表。
|
||||||
|
images: 图片路径列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
新消息列表(不修改原列表)。
|
||||||
|
"""
|
||||||
|
if not images:
|
||||||
|
return messages
|
||||||
|
|
||||||
|
result = [m.copy() for m in messages]
|
||||||
|
|
||||||
|
# 找到最后一条 user message
|
||||||
|
last_user_idx = -1
|
||||||
|
for i in range(len(result) - 1, -1, -1):
|
||||||
|
if result[i].get("role") == "user":
|
||||||
|
last_user_idx = i
|
||||||
|
break
|
||||||
|
|
||||||
|
if last_user_idx == -1:
|
||||||
|
logger.warning("messages 中无 user 角色消息,图片未注入")
|
||||||
|
return result
|
||||||
|
|
||||||
|
user_msg = result[last_user_idx]
|
||||||
|
original_content = user_msg.get("content", "")
|
||||||
|
|
||||||
|
# 构造多模态 content
|
||||||
|
content_parts: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
# 图片在前
|
||||||
|
for img_path in images:
|
||||||
|
data_url = GovernedVLMClient._encode_image(img_path)
|
||||||
|
content_parts.append(
|
||||||
|
{
|
||||||
|
"type": "image_url",
|
||||||
|
"image_url": {"url": data_url},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# 文本在后
|
||||||
|
if isinstance(original_content, str) and original_content:
|
||||||
|
content_parts.append({"type": "text", "text": original_content})
|
||||||
|
elif isinstance(original_content, list):
|
||||||
|
content_parts.extend(original_content)
|
||||||
|
|
||||||
|
result[last_user_idx] = {**user_msg, "content": content_parts}
|
||||||
|
return result
|
||||||
@@ -0,0 +1,242 @@
|
|||||||
|
"""混合 mini-batch 切分:大类打散、小类整锁,供 runner 每 step 处理一个 batch。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import math
|
||||||
|
import random
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
|
||||||
|
|
||||||
|
def build_batches(
|
||||||
|
items: list[GeneratedQuestion],
|
||||||
|
correctness: dict[str, bool],
|
||||||
|
batch_size: int,
|
||||||
|
min_class_per_batch: int,
|
||||||
|
seed: int,
|
||||||
|
correct_ratio: float = 0.0,
|
||||||
|
) -> tuple[list[list[GeneratedQuestion]], int]:
|
||||||
|
"""把诊断池里的题目切成多个混合 mini-batch。
|
||||||
|
|
||||||
|
当 ``correct_ratio > 0`` 时,按题型为每组错题配比一定数量的正确题,使 batch
|
||||||
|
包含正误混合样本("动量"机制);``correct_ratio <= 0`` 时退化为纯错题模式。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
items: 候选题目全集。
|
||||||
|
correctness: question_id -> 基线是否答对。
|
||||||
|
batch_size: 单个 batch 的样本数上限(> 0)。
|
||||||
|
min_class_per_batch: 小类判定阈值——题目数 ≤ 此值的题型整组锁进单一
|
||||||
|
batch(> 0)。
|
||||||
|
seed: 随机种子,保证相同输入产出完全一致的切分。
|
||||||
|
correct_ratio: 正确题占比(0.0 ~ 1.0)。0.0 = 纯错题;0.5 = 错题:正确题 = 1:1。
|
||||||
|
返回:
|
||||||
|
(非空 mini-batch 列表, selected_count);无错题时返回 ([], 0)。
|
||||||
|
selected_count 是所有 batch 中题目总数。
|
||||||
|
异常:
|
||||||
|
ValueError: batch_size 或 min_class_per_batch < 1, 或
|
||||||
|
min_class_per_batch >= batch_size(破坏小类整组装箱不超容的前提)。
|
||||||
|
关键实现细节:
|
||||||
|
装箱顺序为「先小类后大类」。小类整组用 first-fit-decreasing 装箱:按组大小
|
||||||
|
降序处理(同大小再按 task_type 排序保证确定性),每组放进第一个剩余容量足够
|
||||||
|
的 batch;若现有 batch 都装不下就新开一个空 batch——因小类组大小
|
||||||
|
≤ min_class_per_batch < batch_size,新空 batch 必能容纳,故小类装箱永不抛
|
||||||
|
ValueError,且保证整组不拆。再把大类样本(seed 确定性 shuffle 后)round-robin
|
||||||
|
分发到所有现存 batch 填充剩余容量。这样小类聚集于单 batch、大类散布多 batch
|
||||||
|
且与小类共箱,自然产生多类混合 batch(纯类切片会被 multiclass 断言拒绝)。
|
||||||
|
nb = ceil(总题数/batch_size) 是初始 batch 数下界估计而非硬上限:小类装箱可能
|
||||||
|
新开 bin 使实际 batch 数超过 nb。每次新开 bin 都意味着总容量随之增加,故总容量
|
||||||
|
恒 ≥ 总题数,大类 round-robin 跳过满箱后仍能放下全部样本,不会违反 batch_size
|
||||||
|
上限。题型按名称排序处理以保证跨运行确定性,不依赖 dict 遍历顺序。
|
||||||
|
"""
|
||||||
|
_validate_params(batch_size, min_class_per_batch)
|
||||||
|
|
||||||
|
rng = random.Random(seed)
|
||||||
|
grouped = _select_mixed_by_task_type(items, correctness, correct_ratio, rng)
|
||||||
|
total = sum(len(g) for g in grouped.values())
|
||||||
|
if total == 0:
|
||||||
|
return [], 0
|
||||||
|
|
||||||
|
nb = max(1, math.ceil(total / batch_size))
|
||||||
|
batches: list[list[GeneratedQuestion]] = [[] for _ in range(nb)]
|
||||||
|
|
||||||
|
small, large = _split_by_size(grouped, min_class_per_batch)
|
||||||
|
for group in _small_groups_decreasing(small):
|
||||||
|
_pack_small_class(batches, group, batch_size)
|
||||||
|
_distribute_large_classes(batches, large, batch_size, rng)
|
||||||
|
|
||||||
|
result = [b for b in batches if b]
|
||||||
|
selected_count = sum(len(b) for b in result)
|
||||||
|
return result, selected_count
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_params(batch_size: int, min_class_per_batch: int) -> None:
|
||||||
|
"""校验切分参数,非法值直接报错而非用默认值掩盖。
|
||||||
|
|
||||||
|
除各自 >= 1 外,强制 min_class_per_batch < batch_size:小类组大小 ≤
|
||||||
|
min_class_per_batch,唯有此前提成立才能保证小类整组放入单一 batch 而不超容;否则
|
||||||
|
_pack_small_class 新开的 bin 会装入超 batch_size 的整组,静默违反容量合约。此约束
|
||||||
|
与 config._validate_minibatch 一致,是 build_batches 对自身前提的防御性自校验(P5)。
|
||||||
|
"""
|
||||||
|
if batch_size < 1:
|
||||||
|
raise ValueError(f"batch_size 必须 >= 1, 实为 {batch_size}")
|
||||||
|
if min_class_per_batch < 1:
|
||||||
|
raise ValueError(f"min_class_per_batch 必须 >= 1, 实为 {min_class_per_batch}")
|
||||||
|
if min_class_per_batch >= batch_size:
|
||||||
|
raise ValueError(
|
||||||
|
f"min_class_per_batch 必须严格 < batch_size, 否则无法保证小类整组放入单一 "
|
||||||
|
f"batch 不超容; 实为 min_class_per_batch={min_class_per_batch}, "
|
||||||
|
f"batch_size={batch_size}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _split_by_size(
|
||||||
|
grouped: dict[str, list[GeneratedQuestion]],
|
||||||
|
min_class_per_batch: int,
|
||||||
|
) -> tuple[dict[str, list[GeneratedQuestion]], dict[str, list[GeneratedQuestion]]]:
|
||||||
|
"""按错题数把题型分为小类(≤ 阈值)与大类(> 阈值)两组。"""
|
||||||
|
small = {t: g for t, g in grouped.items() if len(g) <= min_class_per_batch}
|
||||||
|
large = {t: g for t, g in grouped.items() if len(g) > min_class_per_batch}
|
||||||
|
return small, large
|
||||||
|
|
||||||
|
|
||||||
|
def _select_mixed_by_task_type(
|
||||||
|
items: list[GeneratedQuestion],
|
||||||
|
correctness: dict[str, bool],
|
||||||
|
correct_ratio: float,
|
||||||
|
rng: random.Random,
|
||||||
|
) -> dict[str, list[GeneratedQuestion]]:
|
||||||
|
"""按题型分组,为每组错题按比例采样正确题混入。
|
||||||
|
|
||||||
|
只对有错题的题型做混合——无错题的题型不进 batch,即使有正确题。
|
||||||
|
``correct_ratio <= 0`` 时退化为纯错题模式(向后兼容)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
items: 候选题目全集。
|
||||||
|
correctness: question_id -> 基线是否答对。
|
||||||
|
correct_ratio: 正确题占比(0.0 ~ 1.0)。
|
||||||
|
rng: 随机数发生器,用于采样正确题。
|
||||||
|
返回:
|
||||||
|
task_type -> 该题型的混合题目列表(错题全部 + 按比例采样的正确题)。
|
||||||
|
"""
|
||||||
|
errors_by_type: dict[str, list[GeneratedQuestion]] = {}
|
||||||
|
correct_by_type: dict[str, list[GeneratedQuestion]] = {}
|
||||||
|
for q in items:
|
||||||
|
qid = q.question_id
|
||||||
|
if correctness.get(qid) is False:
|
||||||
|
errors_by_type.setdefault(q.task_type, []).append(q)
|
||||||
|
elif correctness.get(qid, False):
|
||||||
|
correct_by_type.setdefault(q.task_type, []).append(q)
|
||||||
|
|
||||||
|
if correct_ratio <= 0:
|
||||||
|
return errors_by_type
|
||||||
|
|
||||||
|
# 为每个有错题的 task_type 混入正确题
|
||||||
|
grouped: dict[str, list[GeneratedQuestion]] = {}
|
||||||
|
for task_type in sorted(errors_by_type):
|
||||||
|
errs = errors_by_type[task_type]
|
||||||
|
n_correct = round(len(errs) * correct_ratio / (1 - correct_ratio))
|
||||||
|
available = correct_by_type.get(task_type, [])
|
||||||
|
sampled = (
|
||||||
|
list(available)
|
||||||
|
if len(available) <= n_correct
|
||||||
|
else rng.sample(available, n_correct)
|
||||||
|
)
|
||||||
|
grouped[task_type] = errs + sampled
|
||||||
|
|
||||||
|
return grouped
|
||||||
|
|
||||||
|
|
||||||
|
def _small_groups_decreasing(
|
||||||
|
small: dict[str, list[GeneratedQuestion]],
|
||||||
|
) -> list[list[GeneratedQuestion]]:
|
||||||
|
"""按组大小降序、同大小按 task_type 升序排出小类组(first-fit-decreasing 顺序)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
small: task_type -> 小类错题列表。
|
||||||
|
返回:
|
||||||
|
排好序的小类组列表;降序处理可降低碎片,确定性 tie-break 保证跨运行一致。
|
||||||
|
"""
|
||||||
|
return [small[t] for t in sorted(small, key=lambda t: (-len(small[t]), t))]
|
||||||
|
|
||||||
|
|
||||||
|
def _pack_small_class(
|
||||||
|
batches: list[list[GeneratedQuestion]],
|
||||||
|
group: list[GeneratedQuestion],
|
||||||
|
batch_size: int,
|
||||||
|
) -> None:
|
||||||
|
"""用 first-fit 把一个小类整组放入首个容得下的 batch,装不下则新开 bin(就地修改)。
|
||||||
|
|
||||||
|
因小类组大小 ≤ min_class_per_batch < batch_size,新开的空 batch 必能容纳整组,
|
||||||
|
故此函数永不抛 ValueError,且整组不拆。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
batches: 当前各 batch(就地追加,必要时 append 新空 batch)。
|
||||||
|
group: 待锁定的小类错题(整组不拆)。
|
||||||
|
batch_size: 单 batch 容量上限。
|
||||||
|
"""
|
||||||
|
for b in batches:
|
||||||
|
if len(b) + len(group) <= batch_size:
|
||||||
|
b.extend(group)
|
||||||
|
return
|
||||||
|
batches.append(list(group))
|
||||||
|
|
||||||
|
|
||||||
|
def _distribute_large_classes(
|
||||||
|
batches: list[list[GeneratedQuestion]],
|
||||||
|
large: dict[str, list[GeneratedQuestion]],
|
||||||
|
batch_size: int,
|
||||||
|
rng: random.Random,
|
||||||
|
) -> None:
|
||||||
|
"""将各大类样本 shuffle 后 round-robin 分发到所有现存 batch(就地修改)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
batches: 当前各 batch(含小类装箱可能新开的 bin,就地追加)。
|
||||||
|
large: task_type -> 大类错题列表。
|
||||||
|
batch_size: 单 batch 容量上限。
|
||||||
|
rng: 复用的随机数发生器,保证 shuffle 确定性。
|
||||||
|
异常:
|
||||||
|
ValueError: 所有 batch 均满仍有样本未放置(总容量估算异常,合法输入不可达)。
|
||||||
|
关键实现细节:
|
||||||
|
轮转范围是「所有现存 batch」而非固定 nb 个——小类装箱新开的 bin 也参与分发。
|
||||||
|
总容量 = 现存 batch 数 × batch_size,每次新开 bin 都同步抬高总容量,故总容量恒
|
||||||
|
≥ 总错题数,防御性 ValueError 在合法输入下不可达。全局指针在所有大类样本间持续
|
||||||
|
轮转(不为每类重置),满箱即跳过,使大类充分散布并与已锁定的小类共箱。题型按名称
|
||||||
|
排序以保证分发顺序确定。
|
||||||
|
"""
|
||||||
|
nb = len(batches)
|
||||||
|
pointer = 0
|
||||||
|
for task_type in sorted(large):
|
||||||
|
group = list(large[task_type])
|
||||||
|
rng.shuffle(group)
|
||||||
|
for q in group:
|
||||||
|
pointer = _place_round_robin(batches, q, pointer, batch_size, nb)
|
||||||
|
|
||||||
|
|
||||||
|
def _place_round_robin(
|
||||||
|
batches: list[list[GeneratedQuestion]],
|
||||||
|
q: GeneratedQuestion,
|
||||||
|
pointer: int,
|
||||||
|
batch_size: int,
|
||||||
|
nb: int,
|
||||||
|
) -> int:
|
||||||
|
"""从 pointer 起找第一个未满 batch 放入 q,返回下一次起始指针。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
batches: 当前各 batch(就地追加)。
|
||||||
|
q: 待放置的样本。
|
||||||
|
pointer: 本次轮转起始 batch 下标。
|
||||||
|
batch_size: 单 batch 容量上限。
|
||||||
|
nb: batch 总数。
|
||||||
|
返回:
|
||||||
|
下一次轮转的起始指针(已前移一位)。
|
||||||
|
异常:
|
||||||
|
ValueError: 扫描一轮所有 batch 均满(总容量估算异常)。
|
||||||
|
"""
|
||||||
|
for offset in range(nb):
|
||||||
|
idx = (pointer + offset) % nb
|
||||||
|
if len(batches[idx]) < batch_size:
|
||||||
|
batches[idx].append(q)
|
||||||
|
return (idx + 1) % nb
|
||||||
|
raise ValueError("所有 batch 均满仍有样本待放置, 总容量估算异常")
|
||||||
@@ -0,0 +1,427 @@
|
|||||||
|
"""运行配置:RunConfig frozen dataclass 与 YAML + CLI + .env 三层加载。
|
||||||
|
|
||||||
|
三层合并优先级:CLI > .env > YAML(遵循 CLAUDE.md §4.5 配置管理规范)。
|
||||||
|
- YAML:科研实验配置(会在实验中反复扫动的参数),存放于 config/ 下。
|
||||||
|
- .env:工程配置(少变路径如 workspace_dir、store_dir),通过环境变量注入。
|
||||||
|
- CLI:单次临时覆盖。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import dataclasses
|
||||||
|
import os
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
_VALID_MODES = {"infer", "train", "diagnose", "evolve", "eval", "promote"}
|
||||||
|
_VALID_SKILL_MODES = {"auto", "manual", "none"}
|
||||||
|
_VALID_SKILL_UPDATE_MODES = {"patch", "rewrite"}
|
||||||
|
_PATH_FIELDS = {"workspace_dir", "store_dir"}
|
||||||
|
|
||||||
|
# Video-MME 的任务类型数量:验证池每类至少保底 eval_min_per_class 题,共 11 类。
|
||||||
|
_VIDEO_MME_TASK_TYPE_COUNT = 11
|
||||||
|
|
||||||
|
# .env 工程配置字段映射(环境变量名 → RunConfig 字段名)。
|
||||||
|
# 仅路径类工程配置走 .env,科研实验参数走 YAML。
|
||||||
|
_ENV_FIELD_MAP: dict[str, str] = {
|
||||||
|
"HARNESS_WORKSPACE_DIR": "workspace_dir",
|
||||||
|
"HARNESS_STORE_DIR": "store_dir",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RunConfig:
|
||||||
|
"""实验运行配置,所有参数的唯一归口。
|
||||||
|
|
||||||
|
frozen=True 确保配置在创建后不可变,防止运行中被意外修改。
|
||||||
|
三层合并优先级:CLI > .env > YAML。
|
||||||
|
|
||||||
|
字段:
|
||||||
|
workspace_dir: Workspace 根目录。
|
||||||
|
store_dir: Store 根目录。
|
||||||
|
mode: 运行模式,"infer" / "train" / "diagnose" / "evolve" / "eval" / "promote"。
|
||||||
|
concurrency: 并行 worker 数。
|
||||||
|
max_steps: AgentLoop 单题最大步数。
|
||||||
|
skill_mode: Skill 加载模式,"auto" / "manual" / "none"。
|
||||||
|
n_samples: 题目截取数,0 表示全量。
|
||||||
|
questions: 题目在 questions/ 下的相对路径。
|
||||||
|
skills_version: Skills 版本号。
|
||||||
|
prompts_version: Prompts 版本号。
|
||||||
|
epochs: 训练轮数。
|
||||||
|
diag_size: 诊断池题目数。
|
||||||
|
diag_correct_ratio: 诊断池中正确题目占比。
|
||||||
|
val_size: 验证池题目数。
|
||||||
|
val_correct_ratio: 验证池中正确题目占比。
|
||||||
|
edit_budget_start: 编辑预算前期上限。
|
||||||
|
edit_budget_end: 编辑预算后期下限。
|
||||||
|
batch_size: mini-batch 单批题目数。
|
||||||
|
min_class_per_batch: 单批中每个任务类型至少保留的题目数(< batch_size)。
|
||||||
|
eval_min_per_class: 验证池中每个任务类型至少保底的题目数。
|
||||||
|
early_stop_patience: 全局 best 连续未提升的容忍轮数,达到即早停。
|
||||||
|
test_size: held-out 测试池题目数。
|
||||||
|
use_slow_momentum: 是否启用快慢双速进化中的慢速 momentum 更新。
|
||||||
|
gate_e_confirm: CE-Gate CONFIRMED 接受的 e 值门槛(1/alpha,Ville 界假阳率 alpha)。
|
||||||
|
gate_e_provisional: 题尽暂定接受门 + futility 提前止损的代数界。
|
||||||
|
gate_w_net_min: 题尽暂定接受要求的最小净胜数(win - loss)。
|
||||||
|
gate_delta_min: 最小点估计效应量下限(承接旧 margin 语义)。
|
||||||
|
gate_lambda_dir: Wald 方向拒绝的对数似然比阈值(必须为负)。
|
||||||
|
gate_e_rollback: 试用期对称回滚门(回滚 e 值门槛)。
|
||||||
|
gate_block: 块序贯验证的块大小(=推理并发度,块内跑满)。
|
||||||
|
gate_n_max: 单次 gate 消耗的题数上限。
|
||||||
|
gate_p_low: 信息量阶梯 p-hat 保留区间下界(剔除必错零信息题)。
|
||||||
|
gate_p_high: 信息量阶梯 p-hat 保留区间上界(剔除必对零信息题)。
|
||||||
|
gate_probe_quota: 冷启动探针集比例(全错题中插尾的比例)。
|
||||||
|
gate_gamma_decay: 逐题正确率估计 p-hat 的 EMA 衰减系数。
|
||||||
|
gate_cooldown_steps: 回滚后该题型跳过进化的冷却 step 数。
|
||||||
|
gate_guard_err: gate 内跨块累计 INFRA 错误率护栏。
|
||||||
|
skill_update_mode: skill 进化模式,"patch"(局部 edit)/ "rewrite"(整篇重写)。
|
||||||
|
appendix_consolidate_threshold: appendix note 条数达此值触发 LLM consolidation。
|
||||||
|
run_id: diagnose/evolve 模式要分析的运行 ID,默认空字符串。
|
||||||
|
batch_correct_ratio: 单批中正确题目占比,范围 [0, 1)。
|
||||||
|
momentum_samples: 慢速 momentum 更新时从诊断池采样的题目数,必须 >= 1。
|
||||||
|
seed: fresh 训练的种子名(对应 seed.json),默认 "initial"。
|
||||||
|
version: eval/promote 模式指定的 store 版本号(如 "v3")。
|
||||||
|
resume: train 模式是否从已有 checkpoint 续训。
|
||||||
|
fresh: train 模式是否从种子全新开始。
|
||||||
|
"""
|
||||||
|
|
||||||
|
# ── 必填字段(无默认值,来自 YAML 或 CLI) ──
|
||||||
|
workspace_dir: Path
|
||||||
|
store_dir: Path
|
||||||
|
mode: str
|
||||||
|
concurrency: int
|
||||||
|
max_steps: int
|
||||||
|
skill_mode: str
|
||||||
|
n_samples: int
|
||||||
|
questions: str
|
||||||
|
skills_version: str
|
||||||
|
prompts_version: str
|
||||||
|
epochs: int
|
||||||
|
diag_size: int
|
||||||
|
diag_correct_ratio: float
|
||||||
|
val_size: int
|
||||||
|
val_correct_ratio: float
|
||||||
|
edit_budget_start: int
|
||||||
|
edit_budget_end: int
|
||||||
|
batch_size: int
|
||||||
|
min_class_per_batch: int
|
||||||
|
eval_min_per_class: int
|
||||||
|
early_stop_patience: int
|
||||||
|
test_size: int
|
||||||
|
use_slow_momentum: bool
|
||||||
|
gate_e_confirm: float
|
||||||
|
gate_e_provisional: float
|
||||||
|
gate_w_net_min: int
|
||||||
|
gate_delta_min: float
|
||||||
|
gate_lambda_dir: float
|
||||||
|
gate_e_rollback: float
|
||||||
|
gate_block: int
|
||||||
|
gate_n_max: int
|
||||||
|
gate_p_low: float
|
||||||
|
gate_p_high: float
|
||||||
|
gate_probe_quota: float
|
||||||
|
gate_gamma_decay: float
|
||||||
|
gate_cooldown_steps: int
|
||||||
|
gate_guard_err: float
|
||||||
|
skill_update_mode: str
|
||||||
|
appendix_consolidate_threshold: int
|
||||||
|
|
||||||
|
# ── 有默认值的字段(通常由 CLI 传入或可选) ──
|
||||||
|
run_id: str = ""
|
||||||
|
batch_correct_ratio: float = 0.5
|
||||||
|
momentum_samples: int = 20
|
||||||
|
seed: str = "initial"
|
||||||
|
version: str = ""
|
||||||
|
resume: bool = False
|
||||||
|
fresh: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
def _validate(config: RunConfig) -> None:
|
||||||
|
"""校验 RunConfig 全部字段约束。
|
||||||
|
|
||||||
|
六层校验链:mode → 基础标量 → 编辑预算 → mini-batch → gate 阈值 → gate 阶梯。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
config: 待校验的配置实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 任一字段值不合法。
|
||||||
|
"""
|
||||||
|
_validate_mode(config)
|
||||||
|
_validate_mode_deps(config)
|
||||||
|
_validate_basic(config)
|
||||||
|
_validate_edit_budget(config)
|
||||||
|
_validate_minibatch(config)
|
||||||
|
_validate_gate(config)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_mode(config: RunConfig) -> None:
|
||||||
|
"""校验运行模式枚举合法性。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
config: 待校验的配置实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: mode 值不在合法集合中。
|
||||||
|
"""
|
||||||
|
if config.mode not in _VALID_MODES:
|
||||||
|
raise ValueError(f"mode 必须为 {_VALID_MODES} 之一,实际: {config.mode!r}")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_mode_deps(config: RunConfig) -> None:
|
||||||
|
"""校验各运行模式的依赖字段(run_id、version)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
config: 待校验的配置实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 模式依赖字段缺失。
|
||||||
|
"""
|
||||||
|
if config.mode in ("diagnose", "evolve") and not config.run_id:
|
||||||
|
raise ValueError(f"mode 为 {config.mode!r} 时必须提供 run_id。")
|
||||||
|
if config.mode in ("eval", "promote") and not config.version:
|
||||||
|
raise ValueError(f"mode 为 {config.mode!r} 时必须提供 --version。")
|
||||||
|
if config.mode == "promote" and not config.run_id:
|
||||||
|
raise ValueError("promote 必须提供 --run-id(指定 canonical eval run)。")
|
||||||
|
_validate_train_run_id(config)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_train_run_id(config: RunConfig) -> None:
|
||||||
|
"""校验 train 模式非 resume/fresh 时必须提供 run_id。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
config: 待校验的配置实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: train 模式既非 resume 也非 fresh 且缺少 run_id。
|
||||||
|
"""
|
||||||
|
if config.mode != "train":
|
||||||
|
return
|
||||||
|
if config.resume or config.fresh:
|
||||||
|
return
|
||||||
|
if not config.run_id:
|
||||||
|
raise ValueError("train 非 resume/fresh 时必须提供 run_id(旧式基线 run)。")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_basic(config: RunConfig) -> None:
|
||||||
|
"""校验基础标量字段:枚举合法性与正整数约束。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
config: 待校验的配置实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 任一基础字段值不合法。
|
||||||
|
"""
|
||||||
|
if config.skill_mode not in _VALID_SKILL_MODES:
|
||||||
|
raise ValueError(
|
||||||
|
f"skill_mode 必须为 {_VALID_SKILL_MODES} 之一,实际: {config.skill_mode!r}"
|
||||||
|
)
|
||||||
|
if config.concurrency <= 0:
|
||||||
|
raise ValueError(f"concurrency 必须 > 0,实际: {config.concurrency}")
|
||||||
|
if config.max_steps <= 0:
|
||||||
|
raise ValueError(f"max_steps 必须 > 0,实际: {config.max_steps}")
|
||||||
|
if config.n_samples < 0:
|
||||||
|
raise ValueError(f"n_samples 必须 >= 0,实际: {config.n_samples}")
|
||||||
|
if config.epochs <= 0:
|
||||||
|
raise ValueError(f"epochs 必须 > 0,实际: {config.epochs}")
|
||||||
|
if config.skill_update_mode not in _VALID_SKILL_UPDATE_MODES:
|
||||||
|
raise ValueError(
|
||||||
|
f"skill_update_mode 必须为 {_VALID_SKILL_UPDATE_MODES} 之一,"
|
||||||
|
f"实际: {config.skill_update_mode!r}"
|
||||||
|
)
|
||||||
|
if config.appendix_consolidate_threshold < 1:
|
||||||
|
raise ValueError(
|
||||||
|
f"appendix_consolidate_threshold 必须 >= 1,"
|
||||||
|
f"实际: {config.appendix_consolidate_threshold}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_edit_budget(config: RunConfig) -> None:
|
||||||
|
"""校验编辑预算退火的前期/后期上限约束。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
config: 待校验的配置实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: edit_budget_start < edit_budget_end,或 end <= 0。
|
||||||
|
"""
|
||||||
|
if config.edit_budget_start < config.edit_budget_end:
|
||||||
|
raise ValueError(
|
||||||
|
f"edit_budget_start({config.edit_budget_start}) 必须 >= "
|
||||||
|
f"edit_budget_end({config.edit_budget_end})"
|
||||||
|
)
|
||||||
|
if config.edit_budget_end <= 0:
|
||||||
|
raise ValueError(f"edit_budget_end 必须 > 0,实际: {config.edit_budget_end}")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_minibatch(config: RunConfig) -> None:
|
||||||
|
"""校验 mini-batch 自进化闭环参数约束。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
config: 待校验的 RunConfig 配置对象。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 任一约束被违反。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
val_size 必须 >= eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT,保证验证池
|
||||||
|
能为 Video-MME 的全部 11 个任务类型各保底 eval_min_per_class 题。
|
||||||
|
"""
|
||||||
|
if config.batch_size <= 0:
|
||||||
|
raise ValueError(f"batch_size 必须 > 0,实际: {config.batch_size}")
|
||||||
|
if not (1 <= config.min_class_per_batch < config.batch_size):
|
||||||
|
raise ValueError(
|
||||||
|
f"min_class_per_batch 必须满足 1 <= 值 < batch_size"
|
||||||
|
f"({config.batch_size}),实际: {config.min_class_per_batch}"
|
||||||
|
)
|
||||||
|
if config.eval_min_per_class < 1:
|
||||||
|
raise ValueError(f"eval_min_per_class 必须 >= 1,实际: {config.eval_min_per_class}")
|
||||||
|
floor = config.eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT
|
||||||
|
if config.val_size < floor:
|
||||||
|
raise ValueError(
|
||||||
|
f"val_size 必须 >= eval_min_per_class * {_VIDEO_MME_TASK_TYPE_COUNT}"
|
||||||
|
f"(={floor}):Video-MME 共 {_VIDEO_MME_TASK_TYPE_COUNT} 个任务类型,"
|
||||||
|
f"每类需 eval_min_per_class 题保底,故验证池下限为 {floor},"
|
||||||
|
f"实际: {config.val_size}"
|
||||||
|
)
|
||||||
|
if config.early_stop_patience <= 0:
|
||||||
|
raise ValueError(f"early_stop_patience 必须 > 0,实际: {config.early_stop_patience}")
|
||||||
|
if config.test_size <= 0:
|
||||||
|
raise ValueError(f"test_size 必须 > 0,实际: {config.test_size}")
|
||||||
|
if not (0 <= config.batch_correct_ratio < 1):
|
||||||
|
raise ValueError(
|
||||||
|
f"batch_correct_ratio 必须满足 0 <= 值 < 1,实际: {config.batch_correct_ratio}"
|
||||||
|
)
|
||||||
|
if config.momentum_samples < 1:
|
||||||
|
raise ValueError(f"momentum_samples 必须 >= 1,实际: {config.momentum_samples}")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_gate(config: RunConfig) -> None:
|
||||||
|
"""校验 CE-Gate 全部参数:判据阈值 + 信息量阶梯。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
config: 待校验的配置实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 任一 gate 参数不合法。
|
||||||
|
"""
|
||||||
|
_validate_gate_thresholds(config)
|
||||||
|
_validate_gate_ladder(config)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_gate_thresholds(config: RunConfig) -> None:
|
||||||
|
"""校验 CE-Gate 判据阈值参数(e 值、净胜数、效应量、方向拒绝)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
config: 待校验的配置实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 任一阈值参数不合法。
|
||||||
|
"""
|
||||||
|
if config.gate_e_confirm <= 1:
|
||||||
|
raise ValueError(f"gate_e_confirm 必须 > 1,实际: {config.gate_e_confirm}")
|
||||||
|
if not (1 < config.gate_e_provisional <= config.gate_e_confirm):
|
||||||
|
raise ValueError(
|
||||||
|
f"gate_e_provisional 必须在 (1, gate_e_confirm] 内,实际: {config.gate_e_provisional}"
|
||||||
|
)
|
||||||
|
if config.gate_e_rollback <= 1:
|
||||||
|
raise ValueError(f"gate_e_rollback 必须 > 1,实际: {config.gate_e_rollback}")
|
||||||
|
if config.gate_w_net_min < 1:
|
||||||
|
raise ValueError(f"gate_w_net_min 必须 >= 1,实际: {config.gate_w_net_min}")
|
||||||
|
if config.gate_delta_min < 0:
|
||||||
|
raise ValueError(f"gate_delta_min 必须 >= 0,实际: {config.gate_delta_min}")
|
||||||
|
if config.gate_lambda_dir >= 0:
|
||||||
|
raise ValueError(f"gate_lambda_dir 必须 < 0,实际: {config.gate_lambda_dir}")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_gate_ladder(config: RunConfig) -> None:
|
||||||
|
"""校验 CE-Gate 信息量阶梯与块序贯参数。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
config: 待校验的配置实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 任一阶梯参数不合法。
|
||||||
|
"""
|
||||||
|
if config.gate_block <= 0 or config.gate_n_max < config.gate_block:
|
||||||
|
raise ValueError(
|
||||||
|
f"需 0 < gate_block <= gate_n_max,"
|
||||||
|
f"实际: block={config.gate_block}, n_max={config.gate_n_max}"
|
||||||
|
)
|
||||||
|
if not (0 <= config.gate_p_low < config.gate_p_high <= 1):
|
||||||
|
raise ValueError(
|
||||||
|
f"需 0 <= gate_p_low < gate_p_high <= 1,"
|
||||||
|
f"实际: [{config.gate_p_low}, {config.gate_p_high}]"
|
||||||
|
)
|
||||||
|
if not (0 <= config.gate_probe_quota <= 1):
|
||||||
|
raise ValueError(f"gate_probe_quota 须在 [0,1],实际: {config.gate_probe_quota}")
|
||||||
|
if not (0 < config.gate_gamma_decay < 1):
|
||||||
|
raise ValueError(f"gate_gamma_decay 须在 (0,1),实际: {config.gate_gamma_decay}")
|
||||||
|
if config.gate_cooldown_steps < 1:
|
||||||
|
raise ValueError(f"gate_cooldown_steps 必须 >= 1,实际: {config.gate_cooldown_steps}")
|
||||||
|
if not (0 < config.gate_guard_err < 1):
|
||||||
|
raise ValueError(f"gate_guard_err 须在 (0,1),实际: {config.gate_guard_err}")
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_env_var_overrides(data: dict) -> None:
|
||||||
|
"""从环境变量覆盖路径字段(原地修改)。
|
||||||
|
|
||||||
|
.env 文件由入口脚本 load_dotenv 加载到环境变量,本函数仅从 os.environ 读取。
|
||||||
|
仅覆盖 _ENV_FIELD_MAP 中声明的工程配置字段(workspace_dir、store_dir)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
data: 待覆盖的配置字典。
|
||||||
|
"""
|
||||||
|
for env_key, field_name in _ENV_FIELD_MAP.items():
|
||||||
|
env_val = os.environ.get(env_key)
|
||||||
|
if env_val is not None:
|
||||||
|
data[field_name] = env_val
|
||||||
|
|
||||||
|
|
||||||
|
def load_config(
|
||||||
|
yaml_path: Path,
|
||||||
|
cli_overrides: dict[str, object] | None = None,
|
||||||
|
) -> RunConfig:
|
||||||
|
"""从 YAML 加载配置,叠加 .env 和 CLI 覆盖层后构造 RunConfig。
|
||||||
|
|
||||||
|
三层合并优先级:CLI > .env > YAML。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
yaml_path: YAML 配置文件路径,需包含 ``harness`` 段。
|
||||||
|
cli_overrides: CLI 参数字典,值为 None 表示未传入(不覆盖)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
构造并校验后的 RunConfig 实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileNotFoundError: YAML 文件不存在。
|
||||||
|
ValueError: 校验失败。
|
||||||
|
"""
|
||||||
|
# Phase 1: 加载 YAML 基础层
|
||||||
|
with open(yaml_path, encoding="utf-8") as f:
|
||||||
|
raw: dict = yaml.safe_load(f)
|
||||||
|
|
||||||
|
# 支持嵌套 harness 段和扁平 YAML 两种格式
|
||||||
|
yaml_data: dict = raw.get("harness", raw)
|
||||||
|
|
||||||
|
# Phase 2: .env 覆盖层(仅工程配置字段)
|
||||||
|
_apply_env_var_overrides(yaml_data)
|
||||||
|
|
||||||
|
# Phase 3: CLI 覆盖层(最高优先级)
|
||||||
|
valid_fields = {f.name for f in dataclasses.fields(RunConfig)}
|
||||||
|
if cli_overrides:
|
||||||
|
for key, value in cli_overrides.items():
|
||||||
|
if value is not None and key in valid_fields:
|
||||||
|
yaml_data[key] = value
|
||||||
|
|
||||||
|
# Phase 4: 类型转换 — 路径字段转 Path
|
||||||
|
for field_name in _PATH_FIELDS:
|
||||||
|
if field_name in yaml_data:
|
||||||
|
yaml_data[field_name] = Path(yaml_data[field_name])
|
||||||
|
|
||||||
|
# Phase 5: 构造并校验
|
||||||
|
config = RunConfig(**{k: v for k, v in yaml_data.items() if k in valid_fields})
|
||||||
|
_validate(config)
|
||||||
|
return config
|
||||||
@@ -0,0 +1,347 @@
|
|||||||
|
"""HarnessLog:SQLite 薄包装 + RunLogImpl 只读查询端口。
|
||||||
|
|
||||||
|
HarnessLog 提供统一的结构化日志接口,从 TRM4 直搬,保留全部线程安全与幂等语义。
|
||||||
|
RunLogImpl 实现 core/evolution/protocols.py::RunLog Protocol,用独立连接做只读 SELECT,
|
||||||
|
不经 HarnessLog 生命周期(不触发 _runs INSERT OR IGNORE),避免污染运行状态。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import sqlite3
|
||||||
|
import subprocess
|
||||||
|
import threading
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
def _get_git_sha() -> str | None:
|
||||||
|
"""获取当前 git commit SHA。"""
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "rev-parse", "HEAD"],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
return result.stdout.strip()
|
||||||
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _now_iso() -> str:
|
||||||
|
"""返回当前 UTC 时间的 ISO 格式字符串。"""
|
||||||
|
return datetime.now(UTC).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
class HarnessLog:
|
||||||
|
"""SQLite 薄包装,为科研项目提供统一的结构化日志接口。
|
||||||
|
|
||||||
|
关键设计:
|
||||||
|
- WAL 模式 + threading.Lock 保证共享连接下并发安全。
|
||||||
|
- INSERT OR IGNORE INTO _runs 保证幂等(同 run_id 多次创建不报错)。
|
||||||
|
- query 也持锁:共享连接(check_same_thread=False)下并发 SELECT + INSERT
|
||||||
|
在同一连接上 execute 会损坏游标状态,故读也须串行化。
|
||||||
|
- context manager 语义:正常退出 completed,异常退出 failed。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
db_path: SQLite 数据库文件路径。
|
||||||
|
run_id: 本次运行的唯一标识。
|
||||||
|
git_sha: 代码版本,默认自动获取。
|
||||||
|
config_snapshot: 本次运行的配置快照。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
db_path: str,
|
||||||
|
run_id: str,
|
||||||
|
git_sha: str | None = None,
|
||||||
|
config_snapshot: dict[str, Any] | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._run_id = run_id
|
||||||
|
Path(db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
self._conn = sqlite3.connect(db_path, check_same_thread=False)
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._conn.row_factory = sqlite3.Row
|
||||||
|
self._conn.execute("PRAGMA journal_mode=WAL")
|
||||||
|
self._init_fixed_tables()
|
||||||
|
resolved_sha = git_sha or _get_git_sha()
|
||||||
|
config_json = (
|
||||||
|
json.dumps(config_snapshot, ensure_ascii=False) if config_snapshot else None
|
||||||
|
)
|
||||||
|
self._conn.execute(
|
||||||
|
"INSERT OR IGNORE INTO _runs"
|
||||||
|
" (run_id, git_sha, started_at, config, status)"
|
||||||
|
" VALUES (?, ?, ?, ?, ?)",
|
||||||
|
(run_id, resolved_sha, _now_iso(), config_json, "running"),
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def _init_fixed_tables(self) -> None:
|
||||||
|
"""创建 _runs 和 _events 固定表。"""
|
||||||
|
self._conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS _runs (
|
||||||
|
run_id TEXT PRIMARY KEY,
|
||||||
|
git_sha TEXT,
|
||||||
|
started_at TEXT,
|
||||||
|
finished_at TEXT,
|
||||||
|
config JSON,
|
||||||
|
status TEXT DEFAULT 'running',
|
||||||
|
skills_version TEXT,
|
||||||
|
prompts_version TEXT,
|
||||||
|
questions_ref TEXT
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
self._conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS _events (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
run_id TEXT,
|
||||||
|
timestamp TEXT,
|
||||||
|
event_type TEXT,
|
||||||
|
payload JSON
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def create_table(
|
||||||
|
self,
|
||||||
|
name: str,
|
||||||
|
columns: dict[str, str],
|
||||||
|
primary_key: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""创建自定义表,自动追加 run_id 和 timestamp 列。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
name: 表名。
|
||||||
|
columns: 列定义,如 {"epoch": "INTEGER", "loss": "REAL"}。
|
||||||
|
primary_key: 主键列名。
|
||||||
|
"""
|
||||||
|
all_columns = {"run_id": "TEXT", "timestamp": "TEXT"}
|
||||||
|
all_columns.update(columns)
|
||||||
|
col_defs = []
|
||||||
|
for col_name, col_type in all_columns.items():
|
||||||
|
pk_suffix = " PRIMARY KEY" if col_name == primary_key else ""
|
||||||
|
col_defs.append(f"{col_name} {col_type}{pk_suffix}")
|
||||||
|
sql = f"CREATE TABLE IF NOT EXISTS {name} ({', '.join(col_defs)})"
|
||||||
|
self._conn.execute(sql)
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def insert(self, table: str, record: dict[str, Any], mode: str = "append") -> None:
|
||||||
|
"""插入一条记录,自动填充 run_id 和 timestamp。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
table: 目标表名。
|
||||||
|
record: 要插入的数据。
|
||||||
|
mode: "append" 或 "upsert"。
|
||||||
|
"""
|
||||||
|
enriched = {"run_id": self._run_id, "timestamp": _now_iso()}
|
||||||
|
enriched.update(record)
|
||||||
|
cols = list(enriched.keys())
|
||||||
|
placeholders = ", ".join(["?"] * len(cols))
|
||||||
|
col_names = ", ".join(cols)
|
||||||
|
values = [enriched[c] for c in cols]
|
||||||
|
if mode == "upsert":
|
||||||
|
sql = (
|
||||||
|
f"INSERT OR REPLACE INTO {table} ({col_names}) VALUES ({placeholders})"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
sql = f"INSERT INTO {table} ({col_names}) VALUES ({placeholders})"
|
||||||
|
with self._lock:
|
||||||
|
self._conn.execute(sql, values)
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def insert_many(
|
||||||
|
self, table: str, records: list[dict[str, Any]], mode: str = "append"
|
||||||
|
) -> None:
|
||||||
|
"""批量插入多条记录。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
table: 目标表名。
|
||||||
|
records: 要插入的数据列表。
|
||||||
|
mode: "append" 或 "upsert"。
|
||||||
|
"""
|
||||||
|
for record in records:
|
||||||
|
self.insert(table, record, mode=mode)
|
||||||
|
|
||||||
|
def execute(self, sql: str, params: tuple[Any, ...] = ()) -> None:
|
||||||
|
"""执行原生 SQL 写操作。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
sql: SQL 语句。
|
||||||
|
params: 参数元组。
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
self._conn.execute(sql, params)
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def query(self, sql: str, params: tuple[Any, ...] = ()) -> list[dict[str, Any]]:
|
||||||
|
"""执行原生 SQL 查询,返回 list[dict]。
|
||||||
|
|
||||||
|
与所有写方法同持 self._lock:共享连接(check_same_thread=False)下,
|
||||||
|
并发 SELECT 与 INSERT 在同一连接上 execute 会损坏游标状态,故读也须串行化。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
sql: SQL 查询语句。
|
||||||
|
params: 参数元组。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
查询结果列表,每行为一个字典。
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
cursor = self._conn.execute(sql, params)
|
||||||
|
columns = [desc[0] for desc in cursor.description]
|
||||||
|
return [
|
||||||
|
dict(zip(columns, row, strict=True)) for row in cursor.fetchall()
|
||||||
|
]
|
||||||
|
|
||||||
|
def log_event(self, event_type: str, payload: dict[str, Any]) -> None:
|
||||||
|
"""向 _events 表写入一条事件。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
event_type: 事件类型标识。
|
||||||
|
payload: 事件数据。
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
self._conn.execute(
|
||||||
|
"INSERT INTO _events (run_id, timestamp, event_type, payload)"
|
||||||
|
" VALUES (?, ?, ?, ?)",
|
||||||
|
(
|
||||||
|
self._run_id,
|
||||||
|
_now_iso(),
|
||||||
|
event_type,
|
||||||
|
json.dumps(payload, ensure_ascii=False),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
|
def close(self, status: str = "completed") -> None:
|
||||||
|
"""更新运行状态并关闭连接。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
status: 最终状态,"completed" 或 "failed"。
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
self._conn.execute(
|
||||||
|
"UPDATE _runs SET finished_at = ?, status = ? WHERE run_id = ?",
|
||||||
|
(_now_iso(), status, self._run_id),
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
self._conn.close()
|
||||||
|
|
||||||
|
def __enter__(self) -> HarnessLog:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(
|
||||||
|
self,
|
||||||
|
exc_type: type[BaseException] | None,
|
||||||
|
exc_val: BaseException | None,
|
||||||
|
exc_tb: Any,
|
||||||
|
) -> None:
|
||||||
|
status = "failed" if exc_type is not None else "completed"
|
||||||
|
self.close(status=status)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# RunLogImpl — core/evolution/protocols.py::RunLog 的只读实现
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _read_table(
|
||||||
|
db_path: str,
|
||||||
|
table: str,
|
||||||
|
run_id: str,
|
||||||
|
*,
|
||||||
|
question_ids: list[str] | None = None,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""纯读某表指定 run 的行——不经 HarnessLog 生命周期,避免回读污染 _runs 运行状态。
|
||||||
|
|
||||||
|
HarnessLog.__enter__/__exit__ 会对 run_id 做 INSERT OR IGNORE 并在退出时标 completed;
|
||||||
|
回读指标绝不应改运行状态,故走独立只读连接(仅 SELECT)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
db_path: SQLite 路径。
|
||||||
|
table: 表名(内部固定常量,非外部输入,无注入风险)。
|
||||||
|
run_id: 过滤的 run ID。
|
||||||
|
question_ids: 可选的 question_id 过滤列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
行 dict 列表;表尚未建(没写过)视为无数据返 []。
|
||||||
|
"""
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
try:
|
||||||
|
exists = conn.execute(
|
||||||
|
"SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table,)
|
||||||
|
).fetchone()
|
||||||
|
if exists is None:
|
||||||
|
return []
|
||||||
|
|
||||||
|
if question_ids is not None:
|
||||||
|
placeholders = ", ".join(["?"] * len(question_ids))
|
||||||
|
sql = (
|
||||||
|
f"SELECT * FROM {table}"
|
||||||
|
f" WHERE run_id = ? AND question_id IN ({placeholders})"
|
||||||
|
)
|
||||||
|
rows = conn.execute(sql, (run_id, *question_ids)).fetchall()
|
||||||
|
else:
|
||||||
|
rows = conn.execute(
|
||||||
|
f"SELECT * FROM {table} WHERE run_id = ?", (run_id,)
|
||||||
|
).fetchall()
|
||||||
|
|
||||||
|
return [dict(r) for r in rows]
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
class RunLogImpl:
|
||||||
|
"""RunLog Protocol 的只读实现。
|
||||||
|
|
||||||
|
用独立 sqlite3.connect 做 SELECT,不经 HarnessLog 生命周期(不触发 _runs INSERT),
|
||||||
|
asyncio.to_thread 包装同步 SQL 查询,避免引入 aiosqlite 新依赖。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
db_path: SQLite 数据库文件路径。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, db_path: str) -> None:
|
||||||
|
self._db_path = db_path
|
||||||
|
|
||||||
|
async def get_predictions(
|
||||||
|
self,
|
||||||
|
run_id: str,
|
||||||
|
*,
|
||||||
|
question_ids: list[str] | None = None,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""查询指定 run 的预测记录。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
run_id: 运行标识。
|
||||||
|
question_ids: 可选的题目 ID 过滤列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
预测记录字典列表。
|
||||||
|
"""
|
||||||
|
return await asyncio.to_thread(
|
||||||
|
_read_table, self._db_path, "predictions", run_id, question_ids=question_ids
|
||||||
|
)
|
||||||
|
|
||||||
|
async def get_traces(
|
||||||
|
self,
|
||||||
|
run_id: str,
|
||||||
|
*,
|
||||||
|
question_ids: list[str] | None = None,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""查询指定 run 的推理轨迹。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
run_id: 运行标识。
|
||||||
|
question_ids: 可选的题目 ID 过滤列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
轨迹记录字典列表。
|
||||||
|
"""
|
||||||
|
return await asyncio.to_thread(
|
||||||
|
_read_table, self._db_path, "traces", run_id, question_ids=question_ids
|
||||||
|
)
|
||||||
@@ -0,0 +1,283 @@
|
|||||||
|
"""三池:held-out test + 验证 + 诊断,分层采样 + 冻结持久化。
|
||||||
|
|
||||||
|
三池切分对应训练循环中的 DataLoader 阶段——从题目全集中按
|
||||||
|
test -> validation -> diagnosis 的顺序 progressive exclusion,
|
||||||
|
保证 question_id 互斥。test 池用自然分布(correct_ratio=None),
|
||||||
|
验证池/诊断池按对错比例分层采样。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from app.question_gen import stratified_sample
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.harness.config import RunConfig
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Pools:
|
||||||
|
"""冻结的三池及其基线指标。
|
||||||
|
|
||||||
|
字段:
|
||||||
|
diagnosis: 诊断池(用于错误归因,对应 loss.backward)。
|
||||||
|
validation: 验证池(按类局部验证,每题型有保底样本)。
|
||||||
|
test: held-out 测试池(自然分布,用于最终无偏评估)。
|
||||||
|
baseline_run_id: 基线 run 标识。
|
||||||
|
baseline_val_accuracy: 基线在验证池上的准确率。
|
||||||
|
correctness: 三池所有题的 question_id -> 基线是否答对。
|
||||||
|
"""
|
||||||
|
|
||||||
|
diagnosis: list[GeneratedQuestion]
|
||||||
|
validation: list[GeneratedQuestion]
|
||||||
|
test: list[GeneratedQuestion]
|
||||||
|
baseline_run_id: str
|
||||||
|
baseline_val_accuracy: float
|
||||||
|
correctness: dict[str, bool] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
def build_pools(
|
||||||
|
questions: list[GeneratedQuestion],
|
||||||
|
correctness: dict[str, bool],
|
||||||
|
diag_cfg: dict,
|
||||||
|
val_cfg: dict,
|
||||||
|
test_cfg: dict,
|
||||||
|
baseline_run_id: str,
|
||||||
|
) -> Pools:
|
||||||
|
"""先抽 held-out test,再抽验证集,最后抽诊断池,三池互斥。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
questions: 题目全集。
|
||||||
|
correctness: question_id -> 基线是否答对。
|
||||||
|
diag_cfg: 诊断池采样配置(size/correct_ratio/task_types[/seed])。
|
||||||
|
val_cfg: 验证池采样配置,可含 min_per_class 做按类保底。
|
||||||
|
test_cfg: 测试池配置(size[/seed]);走自然分布,不强制对错比与题型。
|
||||||
|
baseline_run_id: 基线 run 标识。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
冻结的三池 Pools。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
切分顺序 test -> validation -> diagnosis;后两步从剩余题中采样以保证
|
||||||
|
question_id 互斥。test 池用 correct_ratio=None 的自然分布采样。
|
||||||
|
"""
|
||||||
|
test = _sample_excluding(
|
||||||
|
questions,
|
||||||
|
set(),
|
||||||
|
correctness,
|
||||||
|
size=test_cfg["size"],
|
||||||
|
correct_ratio=None,
|
||||||
|
task_types=None,
|
||||||
|
seed=test_cfg.get("seed", 0),
|
||||||
|
min_per_class=None,
|
||||||
|
)
|
||||||
|
selected_ids = {q.question_id for q in test}
|
||||||
|
|
||||||
|
validation = _sample_excluding(questions, selected_ids, correctness, **val_cfg)
|
||||||
|
selected_ids |= {q.question_id for q in validation}
|
||||||
|
|
||||||
|
diagnosis = _sample_excluding(questions, selected_ids, correctness, **diag_cfg)
|
||||||
|
|
||||||
|
val_correct = sum(1 for q in validation if correctness.get(q.question_id))
|
||||||
|
baseline_val_accuracy = val_correct / len(validation) if validation else 0.0
|
||||||
|
return Pools(
|
||||||
|
diagnosis=diagnosis,
|
||||||
|
validation=validation,
|
||||||
|
test=test,
|
||||||
|
baseline_run_id=baseline_run_id,
|
||||||
|
baseline_val_accuracy=baseline_val_accuracy,
|
||||||
|
correctness={
|
||||||
|
q.question_id: correctness.get(q.question_id, False)
|
||||||
|
for q in test + validation + diagnosis
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _sample_excluding(
|
||||||
|
questions: list[GeneratedQuestion],
|
||||||
|
exclude_ids: set[str],
|
||||||
|
correctness: dict[str, bool],
|
||||||
|
**cfg: object,
|
||||||
|
) -> list[GeneratedQuestion]:
|
||||||
|
"""排除已选 question_id 后,按 cfg 对剩余题做分层采样。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
questions: 题目全集。
|
||||||
|
exclude_ids: 已被其他池选走的 question_id,从候选中剔除以保证三池互斥。
|
||||||
|
correctness: question_id -> 基线是否答对。
|
||||||
|
cfg: 透传给 stratified_sample 的采样配置
|
||||||
|
(size/correct_ratio/task_types[/seed/min_per_class])。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
采样后的题目列表。
|
||||||
|
"""
|
||||||
|
pool = [q for q in questions if q.question_id not in exclude_ids]
|
||||||
|
return stratified_sample(pool, correctness, **cfg)
|
||||||
|
|
||||||
|
|
||||||
|
def _q_to_dict(q: GeneratedQuestion) -> dict:
|
||||||
|
"""将 GeneratedQuestion 转为可序列化字典。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
q: 题目对象。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
包含全部字段的字典(options/source_nodes 从 tuple 转为 list)。
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"question_id": q.question_id,
|
||||||
|
"video_id": q.video_id,
|
||||||
|
"task_type": q.task_type,
|
||||||
|
"question": q.question,
|
||||||
|
"options": list(q.options),
|
||||||
|
"answer": q.answer,
|
||||||
|
"source_nodes": list(q.source_nodes),
|
||||||
|
"difficulty": q.difficulty,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _dict_to_q(d: dict) -> GeneratedQuestion:
|
||||||
|
"""从字典恢复 GeneratedQuestion。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
d: 由 _q_to_dict 产出的字典。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
恢复的 GeneratedQuestion 实例(options/source_nodes 恢复为 tuple)。
|
||||||
|
"""
|
||||||
|
return GeneratedQuestion(
|
||||||
|
question_id=d["question_id"],
|
||||||
|
video_id=d["video_id"],
|
||||||
|
task_type=d["task_type"],
|
||||||
|
question=d["question"],
|
||||||
|
options=tuple(d["options"]),
|
||||||
|
answer=d["answer"],
|
||||||
|
source_nodes=tuple(d.get("source_nodes", ())),
|
||||||
|
difficulty=d.get("difficulty", "medium"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def save_pools(pools: Pools, path: Path) -> None:
|
||||||
|
"""将三池及基线指标冻结为 JSON。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
pools: 待冻结的三池。
|
||||||
|
path: 目标 JSON 文件路径。
|
||||||
|
"""
|
||||||
|
path.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"baseline_run_id": pools.baseline_run_id,
|
||||||
|
"baseline_val_accuracy": pools.baseline_val_accuracy,
|
||||||
|
"correctness": pools.correctness,
|
||||||
|
"diagnosis": [_q_to_dict(q) for q in pools.diagnosis],
|
||||||
|
"validation": [_q_to_dict(q) for q in pools.validation],
|
||||||
|
"test": [_q_to_dict(q) for q in pools.test],
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
indent=2,
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def load_pools(path: Path) -> Pools:
|
||||||
|
"""从 JSON 恢复冻结的三池。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
path: 冻结的 pools.json 路径。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
恢复的三池 Pools。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 旧格式 pools.json(无 test 池)。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
旧格式 pools.json(无 test 池)会以清晰的 ValueError 中止——本项目不做
|
||||||
|
向后兼容,也不为缺失字段填默认值。删除旧文件后 build_pools 会重新采样切分,
|
||||||
|
无需重新推理。
|
||||||
|
"""
|
||||||
|
d = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
if "test" not in d:
|
||||||
|
raise ValueError(
|
||||||
|
f"{path} 为旧格式 pools.json(缺 test 池),"
|
||||||
|
"请删除后重新切分(build_pools 会重新采样,无需重新推理)。"
|
||||||
|
)
|
||||||
|
return Pools(
|
||||||
|
diagnosis=[_dict_to_q(x) for x in d["diagnosis"]],
|
||||||
|
validation=[_dict_to_q(x) for x in d["validation"]],
|
||||||
|
test=[_dict_to_q(x) for x in d["test"]],
|
||||||
|
baseline_run_id=d["baseline_run_id"],
|
||||||
|
baseline_val_accuracy=d["baseline_val_accuracy"],
|
||||||
|
correctness=d["correctness"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_or_load_pools(
|
||||||
|
config: RunConfig,
|
||||||
|
run_id: str,
|
||||||
|
task_types: list[str] | None = None,
|
||||||
|
) -> Pools:
|
||||||
|
"""train 模式的三池获取入口:pools.json 已存在则加载,否则从基线 db 切分并冻结。
|
||||||
|
|
||||||
|
把 main.py train 分支「pools.json 存在则 load_pools 否则 build_pools 再 save_pools」
|
||||||
|
那段抽成纯函数,使 main 与集成测试共用同一切分逻辑、避免重复。pools.json 是
|
||||||
|
一次 fresh 训练的冻结切分,resume/重跑同一 workspace 时直接复用以保证三池一致。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
config: 运行配置,提供 workspace_dir 与三池采样旋钮(diag/val/test 各项)。
|
||||||
|
run_id: 基线全量记录的 run_id(fresh 时来自 seed.json,决定从哪个 run 读对错)。
|
||||||
|
task_types: 可选题型过滤,限定诊断/验证池只采样这些题型;None 表示不过滤。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
冻结的三池 Pools。
|
||||||
|
|
||||||
|
关键实现:
|
||||||
|
切分前从基线 db 的 predictions 表读该 run_id 的逐题对错,作为分层采样依据。
|
||||||
|
pools.json 落在 config.workspace_dir 下,存在即视为已冻结,原样加载不重切。
|
||||||
|
"""
|
||||||
|
from app.harness.log import HarnessLog
|
||||||
|
from app.harness.workspace import resolve_paths
|
||||||
|
from app.question_gen import load_benchmark
|
||||||
|
|
||||||
|
pools_path = config.workspace_dir / "pools.json"
|
||||||
|
if pools_path.exists():
|
||||||
|
return load_pools(pools_path)
|
||||||
|
|
||||||
|
paths = resolve_paths(config.workspace_dir)
|
||||||
|
questions = load_benchmark(paths.questions_dir)
|
||||||
|
with HarnessLog(str(paths.db_path), run_id) as log:
|
||||||
|
rows = log.query(
|
||||||
|
"SELECT question_id, prediction, answer FROM predictions WHERE run_id=?",
|
||||||
|
(run_id,),
|
||||||
|
)
|
||||||
|
correctness = {r["question_id"]: r["prediction"] == r["answer"] for r in rows}
|
||||||
|
pools = build_pools(
|
||||||
|
questions,
|
||||||
|
correctness,
|
||||||
|
diag_cfg={
|
||||||
|
"size": config.diag_size,
|
||||||
|
"correct_ratio": config.diag_correct_ratio,
|
||||||
|
"task_types": task_types,
|
||||||
|
"seed": 0,
|
||||||
|
"min_per_class": None,
|
||||||
|
},
|
||||||
|
val_cfg={
|
||||||
|
"size": config.val_size,
|
||||||
|
"correct_ratio": config.val_correct_ratio,
|
||||||
|
"task_types": task_types,
|
||||||
|
"seed": 0,
|
||||||
|
"min_per_class": config.eval_min_per_class,
|
||||||
|
},
|
||||||
|
test_cfg={"size": config.test_size},
|
||||||
|
baseline_run_id=run_id,
|
||||||
|
)
|
||||||
|
save_pools(pools, pools_path)
|
||||||
|
return pools
|
||||||
@@ -0,0 +1,378 @@
|
|||||||
|
"""Store 版本操作 + Seed 管理。
|
||||||
|
|
||||||
|
Store 存储版本化资源(视频、题目、Skill、Prompt),
|
||||||
|
通过版本号(v1, v2, ...)管理资源的演化历史。
|
||||||
|
Seed 是可复现的实验起点,包含权重快照 + baseline 数据库。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import sqlite3
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def _now_iso() -> str:
|
||||||
|
"""返回当前 UTC 时间的 ISO 格式字符串。"""
|
||||||
|
return datetime.now(UTC).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_version(name: str) -> int:
|
||||||
|
"""解析版本目录名 ``v\\d+`` 为整数。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
name: 版本目录名,如 ``"v1"``、``"v10"``。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
版本号整数。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 版本目录名格式不合法(不匹配 ``v\\d+``)。
|
||||||
|
"""
|
||||||
|
match = re.match(r"v(\d+)$", name)
|
||||||
|
if not match:
|
||||||
|
raise ValueError(f"无效版本号: {name}")
|
||||||
|
return int(match.group(1))
|
||||||
|
|
||||||
|
|
||||||
|
def list_versions(store_dir: Path, resource_type: str) -> list[str]:
|
||||||
|
"""列出 Store 中某类资源的所有版本号,按数字值排序。
|
||||||
|
|
||||||
|
按数字排序保证 v10 排在 v2 后面(而非字典序 v10 < v2)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
store_dir: Store 根目录。
|
||||||
|
resource_type: 资源类型路径,如 ``"skills"``、``"questions/generated"``。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
排序后的版本号列表,如 ``["v1", "v2", "v10"]``。
|
||||||
|
"""
|
||||||
|
resource_dir = store_dir / resource_type
|
||||||
|
if not resource_dir.is_dir():
|
||||||
|
return []
|
||||||
|
versions = []
|
||||||
|
for entry in resource_dir.iterdir():
|
||||||
|
if entry.is_dir() and re.match(r"v\d+$", entry.name):
|
||||||
|
versions.append(entry.name)
|
||||||
|
return sorted(versions, key=_parse_version)
|
||||||
|
|
||||||
|
|
||||||
|
def next_version(store_dir: Path, resource_type: str) -> str:
|
||||||
|
"""返回某类资源的下一个可用版本号。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
store_dir: Store 根目录。
|
||||||
|
resource_type: 资源类型路径。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
下一个版本号字符串,如 ``"v3"``。
|
||||||
|
"""
|
||||||
|
versions = list_versions(store_dir, resource_type)
|
||||||
|
if not versions:
|
||||||
|
return "v1"
|
||||||
|
latest = _parse_version(versions[-1])
|
||||||
|
return f"v{latest + 1}"
|
||||||
|
|
||||||
|
|
||||||
|
def _write_meta(target_dir: Path, version: str, source: str, **extra: str | None) -> None:
|
||||||
|
"""写入版本元数据文件 ``meta.json``。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
target_dir: 版本目录。
|
||||||
|
version: 版本号。
|
||||||
|
source: 来源标识(``"manual"`` / ``"evolution"`` / ``"auto-gen"``)。
|
||||||
|
**extra: 额外字段(parent, trigger_run, trigger_workspace, description)。
|
||||||
|
"""
|
||||||
|
meta = {
|
||||||
|
"version": version,
|
||||||
|
"created_at": _now_iso(),
|
||||||
|
"parent": extra.get("parent"),
|
||||||
|
"source": source,
|
||||||
|
"trigger_run": extra.get("trigger_run"),
|
||||||
|
"trigger_workspace": extra.get("trigger_workspace"),
|
||||||
|
"description": extra.get("description", ""),
|
||||||
|
}
|
||||||
|
(target_dir / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
def advance_version(
|
||||||
|
store_dir: Path,
|
||||||
|
resource_type: str,
|
||||||
|
source_dir: Path,
|
||||||
|
meta: dict,
|
||||||
|
) -> str:
|
||||||
|
"""将 source_dir 的内容写入 Store 的下一个版本目录,写入 meta.json。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
store_dir: Store 根目录。
|
||||||
|
resource_type: 资源类型路径,如 ``"skills"``、``"questions/generated"``。
|
||||||
|
source_dir: 包含新版本资源文件的源目录。
|
||||||
|
meta: 元数据字典,至少包含 ``source`` 字段。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
新版本号字符串,如 ``"v2"``。
|
||||||
|
"""
|
||||||
|
version = next_version(store_dir, resource_type)
|
||||||
|
target = store_dir / resource_type / version
|
||||||
|
shutil.copytree(source_dir, target)
|
||||||
|
_write_meta(
|
||||||
|
target,
|
||||||
|
version,
|
||||||
|
meta.get("source", "manual"),
|
||||||
|
parent=meta.get("parent"),
|
||||||
|
trigger_run=meta.get("trigger_run"),
|
||||||
|
trigger_workspace=meta.get("trigger_workspace"),
|
||||||
|
description=meta.get("description", ""),
|
||||||
|
)
|
||||||
|
logger.info("Store 版本推进: {}/{}", resource_type, version)
|
||||||
|
return version
|
||||||
|
|
||||||
|
|
||||||
|
def init_store(
|
||||||
|
store_dir: Path,
|
||||||
|
videos_source: Path,
|
||||||
|
skills_dir: Path,
|
||||||
|
prompts_dir: Path,
|
||||||
|
) -> None:
|
||||||
|
"""初始化 Store:拷贝视频数据,创建 skills/v1、prompts/v1 和 questions 目录。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
store_dir: Store 目标路径(不得已存在)。
|
||||||
|
videos_source: 视频数据源目录。
|
||||||
|
skills_dir: 初始 Skill 文件目录。
|
||||||
|
prompts_dir: 初始 Prompt 文件目录。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileExistsError: Store 目录已存在。
|
||||||
|
"""
|
||||||
|
if store_dir.exists():
|
||||||
|
raise FileExistsError(f"Store 已存在: {store_dir}")
|
||||||
|
store_dir.mkdir(parents=True)
|
||||||
|
shutil.copytree(videos_source, store_dir / "videos")
|
||||||
|
(store_dir / "questions" / "benchmarks").mkdir(parents=True)
|
||||||
|
(store_dir / "questions" / "generated").mkdir(parents=True)
|
||||||
|
shutil.copytree(skills_dir, store_dir / "skills" / "v1")
|
||||||
|
_write_meta(
|
||||||
|
store_dir / "skills" / "v1",
|
||||||
|
"v1",
|
||||||
|
"manual",
|
||||||
|
description="手工创建的初始版本",
|
||||||
|
)
|
||||||
|
shutil.copytree(prompts_dir, store_dir / "prompts" / "v1")
|
||||||
|
_write_meta(
|
||||||
|
store_dir / "prompts" / "v1",
|
||||||
|
"v1",
|
||||||
|
"manual",
|
||||||
|
description="手工创建的初始版本",
|
||||||
|
)
|
||||||
|
logger.info("Store 初始化完成: {}", store_dir)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 种子库(Seed)函数
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def init_seed(
|
||||||
|
store_dir: Path,
|
||||||
|
name: str,
|
||||||
|
skills_dir: Path,
|
||||||
|
prompts_dir: Path,
|
||||||
|
baseline_db: Path,
|
||||||
|
baseline_run_id: str,
|
||||||
|
parent: str | None,
|
||||||
|
description: str,
|
||||||
|
) -> Path:
|
||||||
|
"""在 store/seeds/<name> 写一个种子:权重 + baseline.db + seed.json。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
store_dir: Store 根目录。
|
||||||
|
name: 种子名(如 ``'initial'``、``'from-evolve-v20'``)。
|
||||||
|
skills_dir: 该版本 Skill 权重源目录。
|
||||||
|
prompts_dir: 该版本 Prompt 权重源目录。
|
||||||
|
baseline_db: 该版本全量记录 db(含 _runs + predictions 行)。
|
||||||
|
baseline_run_id: 全量记录的 run_id,fresh 时注入 build_pools。
|
||||||
|
parent: 来源(initial 为 None)。
|
||||||
|
description: 人类可读说明。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
种子目录路径。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileExistsError: 同名种子已存在(不覆盖)。
|
||||||
|
"""
|
||||||
|
seed_dir = store_dir / "seeds" / name
|
||||||
|
if seed_dir.exists():
|
||||||
|
raise FileExistsError(f"种子已存在,不覆盖: {seed_dir}")
|
||||||
|
seed_dir.mkdir(parents=True)
|
||||||
|
shutil.copytree(skills_dir, seed_dir / "skills")
|
||||||
|
shutil.copytree(prompts_dir, seed_dir / "prompts")
|
||||||
|
shutil.copy2(baseline_db, seed_dir / "baseline.db")
|
||||||
|
(seed_dir / "seed.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"baseline_run_id": baseline_run_id,
|
||||||
|
"parent": parent,
|
||||||
|
"created_at": _now_iso(),
|
||||||
|
"description": description,
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
indent=2,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
logger.info("种子创建完成: {}", seed_dir)
|
||||||
|
return seed_dir
|
||||||
|
|
||||||
|
|
||||||
|
def list_seeds(store_dir: Path) -> list[str]:
|
||||||
|
"""列出 store/seeds 下所有种子名(按名排序)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
store_dir: Store 根目录。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
种子名列表(仅含 seed.json 存在的目录),按名排序。
|
||||||
|
"""
|
||||||
|
seeds_root = store_dir / "seeds"
|
||||||
|
if not seeds_root.is_dir():
|
||||||
|
return []
|
||||||
|
return sorted(e.name for e in seeds_root.iterdir() if (e / "seed.json").exists())
|
||||||
|
|
||||||
|
|
||||||
|
def read_seed(store_dir: Path, name: str) -> dict:
|
||||||
|
"""读取种子 seed.json;不存在则报错。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
store_dir: Store 根目录。
|
||||||
|
name: 种子名。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
seed.json 解析后的字典。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileNotFoundError: 该种子不存在。
|
||||||
|
"""
|
||||||
|
seed_json = store_dir / "seeds" / name / "seed.json"
|
||||||
|
if not seed_json.exists():
|
||||||
|
raise FileNotFoundError(f"种子不存在: {name}({seed_json})")
|
||||||
|
return json.loads(seed_json.read_text())
|
||||||
|
|
||||||
|
|
||||||
|
def extract_run_db(src_db: Path, dst_db: Path, run_id: str) -> None:
|
||||||
|
"""从 src_db 抽出某 run_id 的 _runs + predictions 行,写一个最小 db(种子 baseline.db)。
|
||||||
|
|
||||||
|
用源表的**原始 CREATE 语句**重建目标表,保留主键/列类型/约束——
|
||||||
|
``_runs.run_id TEXT PRIMARY KEY`` 是 HarnessLog ``INSERT OR IGNORE`` 去重的依据,
|
||||||
|
若 seed db 丢主键则续训/fresh-bootstrap 的去重失效。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
src_db: 源 harness.db。
|
||||||
|
dst_db: 目标 db(不得已存在)。
|
||||||
|
run_id: 要抽取的 run。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
RuntimeError: 源中无该表或无该 run 的行。
|
||||||
|
"""
|
||||||
|
src = sqlite3.connect(src_db)
|
||||||
|
dst = sqlite3.connect(dst_db)
|
||||||
|
try:
|
||||||
|
for table in ("_runs", "predictions"):
|
||||||
|
create_sql = src.execute(
|
||||||
|
"SELECT sql FROM sqlite_master WHERE type='table' AND name=?",
|
||||||
|
(table,),
|
||||||
|
).fetchone()
|
||||||
|
if create_sql is None or create_sql[0] is None:
|
||||||
|
raise RuntimeError(f"源 db 无表 {table}")
|
||||||
|
dst.execute(create_sql[0])
|
||||||
|
cols = [r[1] for r in src.execute(f"PRAGMA table_info({table})")]
|
||||||
|
col_sql = ", ".join(cols)
|
||||||
|
rows = src.execute(
|
||||||
|
f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,)
|
||||||
|
).fetchall()
|
||||||
|
if not rows:
|
||||||
|
raise RuntimeError(f"{table} 中无 run_id={run_id} 的行")
|
||||||
|
ph = ", ".join("?" * len(cols))
|
||||||
|
dst.executemany(f"INSERT INTO {table} ({col_sql}) VALUES ({ph})", rows)
|
||||||
|
dst.commit()
|
||||||
|
finally:
|
||||||
|
dst.close()
|
||||||
|
src.close()
|
||||||
|
|
||||||
|
|
||||||
|
def promote_to_seed(
|
||||||
|
workspace_dir: Path,
|
||||||
|
store_dir: Path,
|
||||||
|
version: str,
|
||||||
|
eval_run_id: str,
|
||||||
|
name: str,
|
||||||
|
description: str,
|
||||||
|
) -> Path:
|
||||||
|
"""把 workspace 的指定版本 + 配套 prompts + 指定 eval run 全量记录固化成新种子。
|
||||||
|
|
||||||
|
强校验 eval_run_id 对应的 _runs 行中 skills_version 必须与 version 一致,
|
||||||
|
且 skills_version/prompts_version 均不得为 NULL。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
workspace_dir: 来源 workspace。
|
||||||
|
store_dir: Store 根目录。
|
||||||
|
version: skills 版本号。
|
||||||
|
eval_run_id: canonical eval run(其 _runs 行提供配套 prompts 版本与全量记录)。
|
||||||
|
name: 新种子名(冲突报错不覆盖)。
|
||||||
|
description: 说明。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
新种子目录。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: eval_run_id 不存在,或其 skills_version 与 version 不符,或版本为 NULL。
|
||||||
|
FileExistsError: 同名种子已存在(由 init_seed 抛出)。
|
||||||
|
"""
|
||||||
|
con = sqlite3.connect(workspace_dir / "harness.db")
|
||||||
|
con.row_factory = sqlite3.Row
|
||||||
|
try:
|
||||||
|
row = con.execute(
|
||||||
|
"SELECT skills_version, prompts_version FROM _runs WHERE run_id=?",
|
||||||
|
(eval_run_id,),
|
||||||
|
).fetchone()
|
||||||
|
finally:
|
||||||
|
con.close()
|
||||||
|
|
||||||
|
if row is None:
|
||||||
|
raise ValueError(f"eval run 不存在: {eval_run_id}")
|
||||||
|
|
||||||
|
skills_v, prompts_v = row["skills_version"], row["prompts_version"]
|
||||||
|
|
||||||
|
# 强校验——eval run 的版本必须与 --version 一致,且不得为 NULL
|
||||||
|
if skills_v is None or prompts_v is None:
|
||||||
|
raise ValueError(f"eval run {eval_run_id} 的 _runs 版本对为 NULL(未回填?),无法 promote")
|
||||||
|
if skills_v != version:
|
||||||
|
raise ValueError(f"eval run {eval_run_id} 的版本 {skills_v} 与 --version {version} 不符")
|
||||||
|
|
||||||
|
tmp_db = workspace_dir / "_promote_tmp.db"
|
||||||
|
if tmp_db.exists():
|
||||||
|
tmp_db.unlink()
|
||||||
|
extract_run_db(workspace_dir / "harness.db", tmp_db, eval_run_id)
|
||||||
|
try:
|
||||||
|
seed_dir = init_seed(
|
||||||
|
store_dir,
|
||||||
|
name,
|
||||||
|
workspace_dir / "skills" / skills_v,
|
||||||
|
workspace_dir / "prompts" / prompts_v,
|
||||||
|
tmp_db,
|
||||||
|
baseline_run_id=eval_run_id,
|
||||||
|
parent=f"{workspace_dir.name}:{version}",
|
||||||
|
description=description,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
tmp_db.unlink()
|
||||||
|
|
||||||
|
logger.info("Promote 完成: {} -> {}", workspace_dir.name, seed_dir)
|
||||||
|
return seed_dir
|
||||||
@@ -0,0 +1,478 @@
|
|||||||
|
"""Workspace 生命周期管理 + manifest 读写 + Protocol 实现。
|
||||||
|
|
||||||
|
Workspace 是一次实验的独立工作区,通过 manifest.json 引用 Store 中的
|
||||||
|
特定版本资源并记录实验过程。Skills/Prompts 权重拷入 workspace 本地,
|
||||||
|
训练产物只进 workspace 不污染 Store。
|
||||||
|
|
||||||
|
VersionedSkillStore / VersionedPromptStore 实现 core/evolution/protocols.py
|
||||||
|
中定义的只读端口,供 core/ 层以 Protocol 方式读取技能和提示词。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from app.harness.store import read_seed
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ResolvedPaths:
|
||||||
|
"""manifest 解析后的绝对路径集合。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
store_dir: Store 根目录绝对路径。
|
||||||
|
videos_dir: 视频数据目录。
|
||||||
|
questions_dir: 当前引用的题目目录。
|
||||||
|
skills_dir: 当前引用的 Skill 版本目录(workspace 内)。
|
||||||
|
prompts_dir: 当前引用的 Prompt 版本目录(workspace 内)。
|
||||||
|
workspace_dir: Workspace 根目录。
|
||||||
|
db_path: harness.db 路径。
|
||||||
|
analyses_dir: 分析报告目录。
|
||||||
|
runs_dir: 运行临时状态目录。
|
||||||
|
"""
|
||||||
|
|
||||||
|
store_dir: Path
|
||||||
|
videos_dir: Path
|
||||||
|
questions_dir: Path
|
||||||
|
skills_dir: Path
|
||||||
|
prompts_dir: Path
|
||||||
|
workspace_dir: Path
|
||||||
|
db_path: Path
|
||||||
|
analyses_dir: Path
|
||||||
|
runs_dir: Path
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 内部工具
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_MANIFEST_CURRENT_KEYS = {"videos", "questions", "skills", "prompts"}
|
||||||
|
|
||||||
|
|
||||||
|
def _now_iso() -> str:
|
||||||
|
"""返回当前 UTC 时间的 ISO 格式字符串。"""
|
||||||
|
return datetime.now(UTC).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Workspace 核心函数
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _scaffold_workspace(
|
||||||
|
workspace_dir: Path,
|
||||||
|
store_dir: Path,
|
||||||
|
questions: str,
|
||||||
|
skills_version: str,
|
||||||
|
prompts_version: str,
|
||||||
|
) -> None:
|
||||||
|
"""写 manifest + 建 analyses/runs 目录(不拷权重;权重由调用方按来源拷入)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
workspace_dir: 目标 workspace(由调用方保证不存在)。
|
||||||
|
store_dir: Store 根目录。
|
||||||
|
questions: 题目相对路径,如 ``'benchmarks/Video-MME'``。
|
||||||
|
skills_version: manifest.current.skills 初始版本号。
|
||||||
|
prompts_version: manifest.current.prompts 初始版本号。
|
||||||
|
|
||||||
|
关键实现:
|
||||||
|
不依赖任何外部资源源(store 中的 skills/prompts 是否存在不在此校验),
|
||||||
|
因此可被 init_workspace 与种子初始化复用;store 引用以相对路径写入 manifest。
|
||||||
|
"""
|
||||||
|
workspace_dir.mkdir(parents=True)
|
||||||
|
(workspace_dir / "analyses").mkdir()
|
||||||
|
(workspace_dir / "runs").mkdir()
|
||||||
|
|
||||||
|
store_abs = store_dir.resolve()
|
||||||
|
store_rel = os.path.relpath(store_abs, workspace_dir.resolve())
|
||||||
|
|
||||||
|
manifest = {
|
||||||
|
"name": workspace_dir.name,
|
||||||
|
"created_at": _now_iso(),
|
||||||
|
"store": store_rel,
|
||||||
|
"current": {
|
||||||
|
"videos": "videos",
|
||||||
|
"questions": f"questions/{questions}",
|
||||||
|
"skills": f"skills/{skills_version}",
|
||||||
|
"prompts": f"prompts/{prompts_version}",
|
||||||
|
},
|
||||||
|
"history": [],
|
||||||
|
}
|
||||||
|
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
def init_workspace(
|
||||||
|
workspace_dir: Path,
|
||||||
|
store_dir: Path,
|
||||||
|
questions: str,
|
||||||
|
skills_version: str,
|
||||||
|
prompts_version: str,
|
||||||
|
) -> None:
|
||||||
|
"""创建 Workspace 目录并写入初始 manifest.json,拷贝种子权重。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
workspace_dir: Workspace 目标路径(不得已存在)。
|
||||||
|
store_dir: Store 根目录。
|
||||||
|
questions: 题目在 questions/ 下的相对路径,如 ``"benchmarks/Video-MME"``。
|
||||||
|
skills_version: Skills 版本号,如 ``"v1"``。
|
||||||
|
prompts_version: Prompts 版本号,如 ``"v1"``。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileExistsError: Workspace 目录已存在。
|
||||||
|
FileNotFoundError: 引用的资源在 Store 中不存在。
|
||||||
|
"""
|
||||||
|
if workspace_dir.exists():
|
||||||
|
raise FileExistsError(f"Workspace 已存在: {workspace_dir}")
|
||||||
|
|
||||||
|
store_abs = store_dir.resolve()
|
||||||
|
refs = {
|
||||||
|
"skills": f"skills/{skills_version}",
|
||||||
|
"prompts": f"prompts/{prompts_version}",
|
||||||
|
"questions": f"questions/{questions}",
|
||||||
|
}
|
||||||
|
for label, rel in refs.items():
|
||||||
|
full = store_abs / rel
|
||||||
|
if not full.is_dir():
|
||||||
|
raise FileNotFoundError(f"Store 中不存在 {label}: {full}")
|
||||||
|
|
||||||
|
_scaffold_workspace(workspace_dir, store_dir, questions, skills_version, prompts_version)
|
||||||
|
|
||||||
|
# 拷种子权重进 workspace:v2+ 训练产物只进 workspace,不污染 store
|
||||||
|
shutil.copytree(store_abs / refs["skills"], workspace_dir / refs["skills"])
|
||||||
|
shutil.copytree(store_abs / refs["prompts"], workspace_dir / refs["prompts"])
|
||||||
|
logger.info("Workspace 初始化完成: {}", workspace_dir)
|
||||||
|
|
||||||
|
|
||||||
|
def init_workspace_from_seed(
|
||||||
|
workspace_dir: Path,
|
||||||
|
store_dir: Path,
|
||||||
|
seed_name: str,
|
||||||
|
questions: str,
|
||||||
|
) -> str:
|
||||||
|
"""从种子全新建 workspace:拷权重 -> v1、baseline.db -> harness.db、读 baseline_run_id。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
workspace_dir: 目标 workspace(不得已存在)。
|
||||||
|
store_dir: Store 根目录。
|
||||||
|
seed_name: 种子名(store/seeds 下)。
|
||||||
|
questions: 题目相对路径,如 ``'benchmarks/Video-MME'``。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
baseline_run_id(供 build_pools 使用)。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileExistsError: workspace 已存在。
|
||||||
|
FileNotFoundError: 种子不存在(由 read_seed 抛出),或 questions ref 目录不存在。
|
||||||
|
|
||||||
|
关键实现:
|
||||||
|
破坏性/创建操作前先校验 questions ref 存在:fresh 路径在 runner 侧已先
|
||||||
|
归档旧 ws,若到 build_pools 才发现 questions 缺失则旧 ws 已被毁;
|
||||||
|
故在此尽早报错(fail-fast),让新 ws 在创建前失败。
|
||||||
|
"""
|
||||||
|
if workspace_dir.exists():
|
||||||
|
raise FileExistsError(f"Workspace 已存在: {workspace_dir}")
|
||||||
|
|
||||||
|
# 校验种子存在 + 取 baseline_run_id
|
||||||
|
meta = read_seed(store_dir, seed_name)
|
||||||
|
|
||||||
|
# fail-fast:校验 questions ref 存在
|
||||||
|
questions_ref = store_dir / "questions" / questions
|
||||||
|
if not questions_ref.is_dir():
|
||||||
|
raise FileNotFoundError(f"questions ref 目录不存在: {questions_ref}")
|
||||||
|
|
||||||
|
seed_dir = store_dir / "seeds" / seed_name
|
||||||
|
_scaffold_workspace(workspace_dir, store_dir, questions, "v1", "v1")
|
||||||
|
shutil.copytree(seed_dir / "skills", workspace_dir / "skills" / "v1")
|
||||||
|
shutil.copytree(seed_dir / "prompts", workspace_dir / "prompts" / "v1")
|
||||||
|
shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db")
|
||||||
|
|
||||||
|
logger.info("Workspace 从种子 '{}' 初始化完成: {}", seed_name, workspace_dir)
|
||||||
|
return meta["baseline_run_id"]
|
||||||
|
|
||||||
|
|
||||||
|
def load_manifest(workspace_dir: Path) -> dict:
|
||||||
|
"""读取并返回 workspace 的 manifest.json。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
workspace_dir: Workspace 根目录。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
manifest 字典。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileNotFoundError: manifest.json 不存在。
|
||||||
|
"""
|
||||||
|
manifest_path = workspace_dir / "manifest.json"
|
||||||
|
if not manifest_path.exists():
|
||||||
|
raise FileNotFoundError(f"manifest.json 不存在: {manifest_path}")
|
||||||
|
return json.loads(manifest_path.read_text())
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_paths(workspace_dir: Path) -> ResolvedPaths:
|
||||||
|
"""读取 manifest,解析 current 中所有资源的绝对路径。
|
||||||
|
|
||||||
|
skills_dir/prompts_dir 解析到 workspace(非 store),
|
||||||
|
videos_dir/questions_dir 解析到 store。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
workspace_dir: Workspace 根目录。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
ResolvedPaths 实例,包含所有资源的绝对路径。
|
||||||
|
"""
|
||||||
|
manifest = load_manifest(workspace_dir)
|
||||||
|
ws_abs = workspace_dir.resolve()
|
||||||
|
store_abs = (ws_abs / manifest["store"]).resolve()
|
||||||
|
current = manifest["current"]
|
||||||
|
return ResolvedPaths(
|
||||||
|
store_dir=store_abs,
|
||||||
|
videos_dir=store_abs / current["videos"],
|
||||||
|
questions_dir=store_abs / current["questions"],
|
||||||
|
skills_dir=ws_abs / current["skills"],
|
||||||
|
prompts_dir=ws_abs / current["prompts"],
|
||||||
|
workspace_dir=ws_abs,
|
||||||
|
db_path=ws_abs / "harness.db",
|
||||||
|
analyses_dir=ws_abs / "analyses",
|
||||||
|
runs_dir=ws_abs / "runs",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def list_video_ids(workspace_dir: Path) -> list[str]:
|
||||||
|
"""列出 workspace 引用的所有视频 ID(含 tree.json 的子目录名)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
workspace_dir: Workspace 根目录。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
排序后的视频 ID 列表。
|
||||||
|
"""
|
||||||
|
paths = resolve_paths(workspace_dir)
|
||||||
|
video_ids = []
|
||||||
|
for entry in paths.videos_dir.iterdir():
|
||||||
|
if entry.is_dir() and (entry / "tree.json").exists():
|
||||||
|
video_ids.append(entry.name)
|
||||||
|
return sorted(video_ids)
|
||||||
|
|
||||||
|
|
||||||
|
def update_manifest(workspace_dir: Path, **version_updates: str) -> None:
|
||||||
|
"""更新 manifest 的 current 字段。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
workspace_dir: Workspace 根目录。
|
||||||
|
**version_updates: 要更新的字段及其新值,如 ``skills="skills/v2"``。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
KeyError: 更新的字段不在 current 允许的 key 白名单中。
|
||||||
|
"""
|
||||||
|
invalid = set(version_updates) - _MANIFEST_CURRENT_KEYS
|
||||||
|
if invalid:
|
||||||
|
raise KeyError(f"无效的 manifest current 字段: {invalid}")
|
||||||
|
manifest = load_manifest(workspace_dir)
|
||||||
|
manifest["current"].update(version_updates)
|
||||||
|
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
def record_run(workspace_dir: Path, run_id: str) -> Path:
|
||||||
|
"""将 current 版本快照追加到 manifest history,创建 run 目录和 per-video wiki 目录。
|
||||||
|
|
||||||
|
幂等:同 run_id 不重复追加 history(长跑中断后重启 / held-out 复用 run_id 时)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
workspace_dir: Workspace 根目录。
|
||||||
|
run_id: 本次运行的唯一标识,如 ``"run_001"``。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
创建的 run 目录路径。
|
||||||
|
"""
|
||||||
|
manifest = load_manifest(workspace_dir)
|
||||||
|
current = manifest["current"]
|
||||||
|
|
||||||
|
# 幂等:同 run_id 不重复追加 history
|
||||||
|
if not any(h["run_id"] == run_id for h in manifest["history"]):
|
||||||
|
manifest["history"].append(
|
||||||
|
{
|
||||||
|
"run_id": run_id,
|
||||||
|
"started_at": _now_iso(),
|
||||||
|
"skills": current["skills"],
|
||||||
|
"prompts": current["prompts"],
|
||||||
|
"questions": current["questions"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
(workspace_dir / "manifest.json").write_text(
|
||||||
|
json.dumps(manifest, ensure_ascii=False, indent=2)
|
||||||
|
)
|
||||||
|
|
||||||
|
run_dir = workspace_dir / "runs" / run_id
|
||||||
|
# exist_ok:同 run_id 重跑时 run 目录已存在不应崩溃
|
||||||
|
run_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
for video_id in list_video_ids(workspace_dir):
|
||||||
|
(run_dir / video_id / "wiki").mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
logger.debug("Run 已记录: {}", run_id)
|
||||||
|
return run_dir
|
||||||
|
|
||||||
|
|
||||||
|
def read_best(workspace_dir: Path) -> dict | None:
|
||||||
|
"""读取 manifest 的 best 指针,未设置时返回 None。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
workspace_dir: Workspace 根目录。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
best 字典(skills/prompts/val_acc/run_id/epoch),未设置时 None。
|
||||||
|
"""
|
||||||
|
return load_manifest(workspace_dir).get("best")
|
||||||
|
|
||||||
|
|
||||||
|
def update_best(
|
||||||
|
workspace_dir: Path,
|
||||||
|
skills: str,
|
||||||
|
prompts: str,
|
||||||
|
val_acc: float,
|
||||||
|
run_id: str,
|
||||||
|
epoch: int,
|
||||||
|
) -> None:
|
||||||
|
"""写入 manifest 的 best 指针(历史最优版本快照,与 current 平级)。
|
||||||
|
|
||||||
|
best 独立于 current——更新 best 不影响 current。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
workspace_dir: Workspace 根目录。
|
||||||
|
skills: 最优 skills 版本完整 ref,如 ``'skills/v2'``。
|
||||||
|
prompts: 最优 prompts 版本完整 ref,如 ``'prompts/v2'``。
|
||||||
|
val_acc: 该版本验证集准确率。
|
||||||
|
run_id: 该版本验证 run_id。
|
||||||
|
epoch: 达成该最优的轮次。
|
||||||
|
"""
|
||||||
|
manifest = load_manifest(workspace_dir)
|
||||||
|
manifest["best"] = {
|
||||||
|
"skills": skills,
|
||||||
|
"prompts": prompts,
|
||||||
|
"val_acc": val_acc,
|
||||||
|
"run_id": run_id,
|
||||||
|
"epoch": epoch,
|
||||||
|
}
|
||||||
|
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||||
|
logger.info("Best 已更新: val_acc={}, run={}, epoch={}", val_acc, run_id, epoch)
|
||||||
|
|
||||||
|
|
||||||
|
def archive_workspace(workspace_dir: Path) -> Path:
|
||||||
|
"""把 workspace 整体移动到同级 .archive/<name>-<ts>,返回归档路径。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
workspace_dir: 要归档的 Workspace 根目录。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
归档后的目标路径。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileNotFoundError: workspace 不存在。
|
||||||
|
"""
|
||||||
|
if not workspace_dir.exists():
|
||||||
|
raise FileNotFoundError(f"workspace 不存在: {workspace_dir}")
|
||||||
|
|
||||||
|
archive_root = workspace_dir.parent / ".archive"
|
||||||
|
archive_root.mkdir(exist_ok=True)
|
||||||
|
ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
|
||||||
|
target = archive_root / f"{workspace_dir.name}-{ts}"
|
||||||
|
shutil.move(str(workspace_dir), str(target))
|
||||||
|
|
||||||
|
logger.info("Workspace 已归档: {} -> {}", workspace_dir, target)
|
||||||
|
return target
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Protocol 实现:VersionedSkillStore / VersionedPromptStore
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class VersionedSkillStore:
|
||||||
|
"""版本化技能读取端口实现。
|
||||||
|
|
||||||
|
满足 ``core/evolution/protocols.py::SkillStore`` Protocol。
|
||||||
|
从指定的 skills 版本目录读取 ``.md`` 文件。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
skills_dir: skills 版本目录绝对路径(如 ``workspace/skills/v1``)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, skills_dir: Path) -> None:
|
||||||
|
if not skills_dir.is_dir():
|
||||||
|
raise FileNotFoundError(f"Skills 目录不存在: {skills_dir}")
|
||||||
|
self._dir = skills_dir
|
||||||
|
|
||||||
|
def read_skill(self, filename: str) -> str:
|
||||||
|
"""读取指定 skill 文件的全文内容。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
filename: skill 文件名,如 ``'temporal-reasoning.md'``。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
文件全文内容。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileNotFoundError: 文件不存在。
|
||||||
|
"""
|
||||||
|
path = self._dir / filename
|
||||||
|
if not path.exists():
|
||||||
|
raise FileNotFoundError(f"Skill 文件不存在: {path}")
|
||||||
|
return path.read_text()
|
||||||
|
|
||||||
|
def list_skill_files(self) -> list[str]:
|
||||||
|
"""列出当前版本所有 skill 文件名。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
文件名列表(排序)。
|
||||||
|
"""
|
||||||
|
return sorted(entry.name for entry in self._dir.iterdir() if entry.is_file())
|
||||||
|
|
||||||
|
|
||||||
|
class VersionedPromptStore:
|
||||||
|
"""版本化提示词读取端口实现。
|
||||||
|
|
||||||
|
满足 ``core/evolution/protocols.py::PromptStore`` Protocol。
|
||||||
|
从指定的 prompts 版本目录读取 ``.md`` 文件。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
prompts_dir: prompts 版本目录绝对路径(如 ``workspace/prompts/v1``)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, prompts_dir: Path) -> None:
|
||||||
|
if not prompts_dir.is_dir():
|
||||||
|
raise FileNotFoundError(f"Prompts 目录不存在: {prompts_dir}")
|
||||||
|
self._dir = prompts_dir
|
||||||
|
|
||||||
|
def read_prompt(self, filename: str) -> str:
|
||||||
|
"""读取指定 prompt 文件的全文内容。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
filename: prompt 文件名,如 ``'system.md'``。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
文件全文内容。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileNotFoundError: 文件不存在。
|
||||||
|
"""
|
||||||
|
path = self._dir / filename
|
||||||
|
if not path.exists():
|
||||||
|
raise FileNotFoundError(f"Prompt 文件不存在: {path}")
|
||||||
|
return path.read_text()
|
||||||
|
|
||||||
|
def list_prompt_files(self) -> list[str]:
|
||||||
|
"""列出当前版本所有 prompt 文件名。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
文件名列表(排序)。
|
||||||
|
"""
|
||||||
|
return sorted(entry.name for entry in self._dir.iterdir() if entry.is_file())
|
||||||
@@ -1 +1,76 @@
|
|||||||
"""应用层 Protocol 端口定义。"""
|
"""应用层 Protocol 端口定义。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path # noqa: TC003 — runtime_checkable Protocol 需运行时可见
|
||||||
|
from typing import TYPE_CHECKING, Protocol, runtime_checkable
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from app.tree.index import TreeIndex
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class EmbeddingProvider(Protocol):
|
||||||
|
"""文本嵌入端口。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
dim: 嵌入维度 D。
|
||||||
|
"""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def dim(self) -> int: ...
|
||||||
|
|
||||||
|
def embed(self, texts: str | list[str]) -> np.ndarray:
|
||||||
|
"""文本 → 嵌入向量(L2 归一化)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
texts: 单条文本或文本列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
[N, D] ndarray,每行 L2 范数为 1.0。
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class QuestionGenerator(Protocol):
|
||||||
|
"""LLM 驱动的题目生成端口(预留接口)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
video_id: 视频标识。
|
||||||
|
task_type: 题型。
|
||||||
|
tree: 视频树索引,提供锚节点上下文。
|
||||||
|
exemplars: 风格示例题目列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
生成的单条题目。
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def generate(
|
||||||
|
self,
|
||||||
|
video_id: str,
|
||||||
|
task_type: str,
|
||||||
|
tree: TreeIndex,
|
||||||
|
*,
|
||||||
|
exemplars: list[GeneratedQuestion],
|
||||||
|
) -> GeneratedQuestion: ...
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class OCRProvider(Protocol):
|
||||||
|
"""帧文字转录端口。
|
||||||
|
|
||||||
|
实现方负责将帧图像发送给 OCR 服务并返回拼接后的文本。
|
||||||
|
单帧失败应降级跳过,不得抛出异常中断整体流程。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
frame_paths: 帧文件路径列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
"帧1: <行1> | <行2>\\n帧2: ..." 格式文本;无有效结果时空串。
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def transcribe_frames(self, frame_paths: list[Path]) -> str: ...
|
||||||
|
|||||||
@@ -0,0 +1,5 @@
|
|||||||
|
"""出题模块 — benchmark 加载与分层采样。"""
|
||||||
|
|
||||||
|
from app.question_gen.loader import load_benchmark, stratified_sample
|
||||||
|
|
||||||
|
__all__ = ["load_benchmark", "stratified_sample"]
|
||||||
|
|||||||
@@ -0,0 +1,167 @@
|
|||||||
|
"""题目加载与分层采样。
|
||||||
|
|
||||||
|
从 benchmark JSON 目录加载题目,提供按对错比例的分层采样。
|
||||||
|
对应训练循环中的 DataLoader 角色。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import random
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_LEGACY_DEFAULT_DIFFICULTY = "medium"
|
||||||
|
|
||||||
|
|
||||||
|
def load_benchmark(questions_dir: Path) -> list[GeneratedQuestion]:
|
||||||
|
"""从 benchmark JSON 目录加载题目列表。
|
||||||
|
|
||||||
|
每个 JSON 文件以文件名(不含扩展名)作为 video_id,
|
||||||
|
文件内容为题目数组。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
questions_dir: 包含 *.json 文件的目录路径。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
按文件名排序加载的题目列表。
|
||||||
|
"""
|
||||||
|
results: list[GeneratedQuestion] = []
|
||||||
|
for path in sorted(questions_dir.glob("*.json")):
|
||||||
|
video_id = path.stem
|
||||||
|
with open(path, encoding="utf-8") as f:
|
||||||
|
qa_list: list[dict] = json.load(f)
|
||||||
|
for qa in qa_list:
|
||||||
|
results.append(
|
||||||
|
GeneratedQuestion(
|
||||||
|
question_id=qa["question_id"],
|
||||||
|
video_id=video_id,
|
||||||
|
task_type=qa["task_type"],
|
||||||
|
question=qa["question"],
|
||||||
|
options=tuple(qa["options"]),
|
||||||
|
answer=qa["answer"],
|
||||||
|
source_nodes=tuple(qa.get("source_nodes", ())),
|
||||||
|
difficulty=qa.get("difficulty", _LEGACY_DEFAULT_DIFFICULTY),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def stratified_sample(
|
||||||
|
questions: list[GeneratedQuestion],
|
||||||
|
correctness: dict[str, bool],
|
||||||
|
size: int,
|
||||||
|
correct_ratio: float | None,
|
||||||
|
task_types: list[str] | None,
|
||||||
|
seed: int,
|
||||||
|
min_per_class: int | None,
|
||||||
|
) -> list[GeneratedQuestion]:
|
||||||
|
"""按题型过滤后采样 size 道题,可选按对错比例分层并按题型保底。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
questions: 候选题目全集。
|
||||||
|
correctness: question_id -> 基线是否答对。
|
||||||
|
size: 采样总量。
|
||||||
|
correct_ratio: 采样中"基线答对"题的占比;None 表示自然分布。
|
||||||
|
task_types: 限定题型;None 表示不限。
|
||||||
|
seed: 随机种子,保证可复现。
|
||||||
|
min_per_class: 每个题型补足到的下限;None 表示不补足。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
采样后的题目列表。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 自然分布时池不足 size,或分层时某层题目不足。
|
||||||
|
"""
|
||||||
|
rng = random.Random(seed)
|
||||||
|
pool = [q for q in questions if task_types is None or q.task_type in task_types]
|
||||||
|
|
||||||
|
if correct_ratio is None:
|
||||||
|
if len(pool) < size:
|
||||||
|
raise ValueError(f"自然分布采样不足: 需 {size} 道, 实有 {len(pool)} 道")
|
||||||
|
sampled = rng.sample(pool, size)
|
||||||
|
else:
|
||||||
|
sampled = _ratio_stratified_sample(pool, correctness, size, correct_ratio, rng)
|
||||||
|
|
||||||
|
if min_per_class is not None:
|
||||||
|
sampled = _backfill_per_class(sampled, pool, min_per_class, rng)
|
||||||
|
return sampled
|
||||||
|
|
||||||
|
|
||||||
|
def _ratio_stratified_sample(
|
||||||
|
pool: list[GeneratedQuestion],
|
||||||
|
correctness: dict[str, bool],
|
||||||
|
size: int,
|
||||||
|
correct_ratio: float,
|
||||||
|
rng: random.Random,
|
||||||
|
) -> list[GeneratedQuestion]:
|
||||||
|
"""按对错比例分层采样:对题占 correct_ratio,其余为错题。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
pool: 题型过滤后的候选题。
|
||||||
|
correctness: question_id -> 基线是否答对。
|
||||||
|
size: 采样总量。
|
||||||
|
correct_ratio: 对题占比。
|
||||||
|
rng: 随机数发生器。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
采样后的题目列表(对题在前、错题在后)。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 对题或错题层不足。
|
||||||
|
"""
|
||||||
|
correct = [q for q in pool if correctness.get(q.question_id, False)]
|
||||||
|
wrong = [q for q in pool if not correctness.get(q.question_id, False)]
|
||||||
|
n_correct = round(size * correct_ratio)
|
||||||
|
n_wrong = size - n_correct
|
||||||
|
if len(correct) < n_correct or len(wrong) < n_wrong:
|
||||||
|
raise ValueError(
|
||||||
|
f"分层不足: 需对{n_correct}/错{n_wrong}, 实有对{len(correct)}/错{len(wrong)}"
|
||||||
|
)
|
||||||
|
return rng.sample(correct, n_correct) + rng.sample(wrong, n_wrong)
|
||||||
|
|
||||||
|
|
||||||
|
def _backfill_per_class(
|
||||||
|
sampled: list[GeneratedQuestion],
|
||||||
|
pool: list[GeneratedQuestion],
|
||||||
|
min_per_class: int,
|
||||||
|
rng: random.Random,
|
||||||
|
) -> list[GeneratedQuestion]:
|
||||||
|
"""对候选池中出现的每个题型,将采样结果补足到 min_per_class 道。
|
||||||
|
|
||||||
|
遍历对象是候选池 pool 里出现的全部题型(非仅 sampled 命中的),
|
||||||
|
保证任意稀疏题型都能拿到足额样本。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
sampled: 主采样结果(不修改,返回新列表)。
|
||||||
|
pool: 候选题全集(补足来源 + 题型枚举来源)。
|
||||||
|
min_per_class: 每个题型的下限。
|
||||||
|
rng: 随机数发生器。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
补足后的题目列表。
|
||||||
|
"""
|
||||||
|
selected_ids = {q.question_id for q in sampled}
|
||||||
|
result = list(sampled)
|
||||||
|
counts: dict[str, int] = {}
|
||||||
|
for q in sampled:
|
||||||
|
counts[q.task_type] = counts.get(q.task_type, 0) + 1
|
||||||
|
ordered_task_types: dict[str, None] = {}
|
||||||
|
for q in pool:
|
||||||
|
ordered_task_types.setdefault(q.task_type, None)
|
||||||
|
for task_type in ordered_task_types:
|
||||||
|
deficit = min_per_class - counts.get(task_type, 0)
|
||||||
|
if deficit <= 0:
|
||||||
|
continue
|
||||||
|
candidates = [
|
||||||
|
q for q in pool if q.task_type == task_type and q.question_id not in selected_ids
|
||||||
|
]
|
||||||
|
take = rng.sample(candidates, min(deficit, len(candidates)))
|
||||||
|
for q in take:
|
||||||
|
selected_ids.add(q.question_id)
|
||||||
|
result.append(q)
|
||||||
|
return result
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
"""搜索 Agent 装配层 — prompt 管理、skill 注册、工具分发、LLM 摘要、视觉观察。"""
|
||||||
|
|
||||||
|
from app.search.prompt import PromptManager
|
||||||
|
from app.search.skills import SkillRegistry, discover_skills
|
||||||
|
from app.search.tools import SearchToolDispatcher, get_tool_descriptions
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"PromptManager",
|
||||||
|
"SkillRegistry",
|
||||||
|
"SearchToolDispatcher",
|
||||||
|
"discover_skills",
|
||||||
|
"get_tool_descriptions",
|
||||||
|
]
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
"""搜索 Agent 提示词管理模块。
|
||||||
|
|
||||||
|
提供 PromptManager 类,统一管理循环级 prompt 的加载与组装。
|
||||||
|
工具级 prompt(extract/verify)不在管理范围内。
|
||||||
|
|
||||||
|
与 TRM4 ``core/search/prompt.py`` 的差异:
|
||||||
|
- 工具描述从 ``app.search.tools.get_tool_descriptions`` 获取(路径变更);
|
||||||
|
- ``format_user_prompt`` 参数显式化(question/options/l1_node_ids/task_type)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from app.search.tools import get_tool_descriptions
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
class PromptManager:
|
||||||
|
"""管理循环级 prompt 的加载与组装。
|
||||||
|
|
||||||
|
构造时缓存 system.md 作为 inference 基础模板。
|
||||||
|
后续步骤(diagnose/evolve/question_gen)通过 load() 按文件名读取。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
prompts_dir: prompt 文件目录的绝对路径。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, prompts_dir: Path) -> None:
|
||||||
|
self._prompts_dir = prompts_dir
|
||||||
|
system_path = prompts_dir / "system.md"
|
||||||
|
if not system_path.exists():
|
||||||
|
raise FileNotFoundError(f"system.md 不存在: {system_path}")
|
||||||
|
self._system_base = system_path.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
def build_inference_prompt(
|
||||||
|
self,
|
||||||
|
skill_mode: str,
|
||||||
|
task_type: str,
|
||||||
|
always_skills_text: str,
|
||||||
|
task_skill_map: dict[str, str],
|
||||||
|
catalog_text: str,
|
||||||
|
) -> str:
|
||||||
|
"""组装 inference 步骤的完整 system prompt。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
skill_mode: "auto" / "manual" / "none"。
|
||||||
|
task_type: 当前 QA 的题型。
|
||||||
|
always_skills_text: always 层 skill 正文(已拼接)。
|
||||||
|
task_skill_map: {task_type: skill_body} 映射。
|
||||||
|
catalog_text: manual 模式的 skill 目录文本。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
拼装后的完整 system prompt。
|
||||||
|
"""
|
||||||
|
include_read_skill = skill_mode == "manual"
|
||||||
|
parts = [
|
||||||
|
self._system_base,
|
||||||
|
f"\n\n---\n\n{get_tool_descriptions(include_read_skill=include_read_skill)}",
|
||||||
|
]
|
||||||
|
if always_skills_text:
|
||||||
|
parts.append(f"\n\n---\n\n# 通用搜索策略\n\n{always_skills_text}")
|
||||||
|
if skill_mode == "auto":
|
||||||
|
skill_text = task_skill_map.get(task_type) or task_skill_map.get("_default")
|
||||||
|
if skill_text:
|
||||||
|
parts.append(f"\n\n---\n\n# 当前题型搜索策略\n\n{skill_text}")
|
||||||
|
elif skill_mode == "manual":
|
||||||
|
if catalog_text:
|
||||||
|
parts.append(
|
||||||
|
"\n\n---\n\n# 可用搜索策略\n\n"
|
||||||
|
"以下技能扩展了你的导航能力。当问题匹配某技能的适用题型时,"
|
||||||
|
"用 read_skill 工具加载该技能,然后按其指引操作。\n\n"
|
||||||
|
f"{catalog_text}"
|
||||||
|
)
|
||||||
|
return "".join(parts)
|
||||||
|
|
||||||
|
def format_user_prompt(
|
||||||
|
self,
|
||||||
|
question: str,
|
||||||
|
options: list[str],
|
||||||
|
l1_node_ids: list[str],
|
||||||
|
task_type: str | None = None,
|
||||||
|
) -> str:
|
||||||
|
"""格式化 inference 步骤的用户提示词。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
question: 问题文本。
|
||||||
|
options: 选项列表(如 ["A. 历史", "B. 科学"])。
|
||||||
|
l1_node_ids: L1 根节点 ID 列表(如 ["L1_000", "L1_001"])。
|
||||||
|
task_type: 可选题型标签,非 None 时插入题型行(oracle 实验用)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
格式化后的用户提示词。
|
||||||
|
"""
|
||||||
|
options_text = "\n".join(options)
|
||||||
|
roots_text = ", ".join(l1_node_ids)
|
||||||
|
task_type_line = f"**题型**: {task_type}\n" if task_type else ""
|
||||||
|
return (
|
||||||
|
f"请回答以下关于这个视频的多选题:\n\n"
|
||||||
|
f"{task_type_line}"
|
||||||
|
f"**问题**: {question}\n"
|
||||||
|
f"**选项**:\n{options_text}\n\n"
|
||||||
|
f"**视频树 L1 根节点**: {roots_text}\n"
|
||||||
|
f"请从以上 L1 节点开始导航,收集证据后回答。"
|
||||||
|
)
|
||||||
|
|
||||||
|
def load(self, name: str) -> str:
|
||||||
|
"""按文件名加载 prompt 内容。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
name: prompt 文件名(如 "diagnose_span.md")。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
文件内容字符串。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileNotFoundError: 文件不存在。
|
||||||
|
"""
|
||||||
|
path = self._prompts_dir / name
|
||||||
|
if not path.exists():
|
||||||
|
raise FileNotFoundError(f"prompt 文件不存在: {path}")
|
||||||
|
return path.read_text(encoding="utf-8")
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
"""技能注册表与 Markdown frontmatter 解析工具。
|
||||||
|
|
||||||
|
提供 Skill 文件的 frontmatter 解析、正文提取、注册表管理和目录扫描功能,
|
||||||
|
供搜索 Agent 装配层使用。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_FRONTMATTER_FIELDS = {"name", "description", "always", "task_type"}
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_frontmatter_lines(text: str) -> tuple[list[str], int] | None:
|
||||||
|
"""提取 frontmatter 行与正文起始偏移。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
text: 原始 Markdown 文本。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
(frontmatter 行列表, 正文起始字节偏移) 二元组;
|
||||||
|
若不存在完整 frontmatter 则返回 None。
|
||||||
|
"""
|
||||||
|
lines = text.splitlines(keepends=True)
|
||||||
|
if not lines or lines[0].strip() != "---":
|
||||||
|
return None
|
||||||
|
|
||||||
|
offset = len(lines[0])
|
||||||
|
frontmatter_lines: list[str] = []
|
||||||
|
for line in lines[1:]:
|
||||||
|
if line.strip() == "---":
|
||||||
|
return frontmatter_lines, offset + len(line)
|
||||||
|
frontmatter_lines.append(line)
|
||||||
|
offset += len(line)
|
||||||
|
|
||||||
|
logger.debug("frontmatter 缺少结束分隔符,按普通正文处理")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def strip_frontmatter(text: str) -> str:
|
||||||
|
"""移除 Markdown 文本开头的 frontmatter,并返回正文。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
text: 原始 Markdown 文本。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
去除 frontmatter 后的正文;若 frontmatter 不完整或不存在,则返回原文。
|
||||||
|
"""
|
||||||
|
extracted = _extract_frontmatter_lines(text)
|
||||||
|
if extracted is None:
|
||||||
|
return text
|
||||||
|
|
||||||
|
_, body_start = extracted
|
||||||
|
return text[body_start:]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_frontmatter(text: str) -> dict[str, str]:
|
||||||
|
"""解析 Markdown frontmatter 中的目标字段。
|
||||||
|
|
||||||
|
仅识别 ``name``、``description``、``always``、``task_type`` 四个字段,
|
||||||
|
其余字段会被忽略。引号包裹的值会自动去除引号。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
text: 原始 Markdown 文本。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
仅包含目标字段的字符串字典。
|
||||||
|
若不存在完整 frontmatter,则返回空字典。
|
||||||
|
"""
|
||||||
|
extracted = _extract_frontmatter_lines(text)
|
||||||
|
if extracted is None:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
frontmatter_lines, _ = extracted
|
||||||
|
parsed: dict[str, str] = {}
|
||||||
|
for raw_line in frontmatter_lines:
|
||||||
|
line = raw_line.strip()
|
||||||
|
if not line or ":" not in line:
|
||||||
|
continue
|
||||||
|
|
||||||
|
key, _, raw_value = line.partition(":")
|
||||||
|
normalized_key = key.strip()
|
||||||
|
if normalized_key not in _FRONTMATTER_FIELDS:
|
||||||
|
continue
|
||||||
|
|
||||||
|
value = raw_value.strip()
|
||||||
|
if len(value) >= 2 and (
|
||||||
|
(value.startswith('"') and value.endswith('"'))
|
||||||
|
or (value.startswith("'") and value.endswith("'"))
|
||||||
|
):
|
||||||
|
value = value[1:-1]
|
||||||
|
parsed[normalized_key] = value
|
||||||
|
|
||||||
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
|
class SkillRegistry:
|
||||||
|
"""管理技能名称到文件路径映射并读取技能正文。
|
||||||
|
|
||||||
|
通过 ``set_paths`` 注入名称→路径映射后,
|
||||||
|
可用 ``read`` 按名读取技能 Markdown 正文(自动去除 frontmatter)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self._paths: dict[str, Path] = {}
|
||||||
|
|
||||||
|
def set_paths(self, mapping: dict[str, Path]) -> None:
|
||||||
|
"""注入技能名称到文件路径的映射。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
mapping: 技能名到 Markdown 文件路径的映射。
|
||||||
|
"""
|
||||||
|
self._paths = dict(mapping)
|
||||||
|
logger.debug("SkillRegistry 已载入 {} 个技能路径", len(self._paths))
|
||||||
|
|
||||||
|
def read(self, name: str) -> str:
|
||||||
|
"""读取指定技能文件,并返回去除 frontmatter 后的正文。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
name: 技能名称。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
技能 Markdown 正文。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
KeyError: 技能名称未注册时抛出。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
path = self._paths[name]
|
||||||
|
except KeyError:
|
||||||
|
logger.error("技能未注册: {}", name)
|
||||||
|
raise
|
||||||
|
|
||||||
|
logger.debug("读取技能文件: name={}, path={}", name, path)
|
||||||
|
return strip_frontmatter(path.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def discover_skills(
|
||||||
|
skills_dir: Path,
|
||||||
|
) -> tuple[str, dict[str, str], str, SkillRegistry]:
|
||||||
|
"""扫描 skills 目录,按 frontmatter 分类返回。
|
||||||
|
|
||||||
|
遍历 ``*.md`` 文件,根据 frontmatter 的 ``always`` / ``task_type`` 字段分类:
|
||||||
|
|
||||||
|
- ``always=true`` 的 skill 拼入 ``always_skills_text``
|
||||||
|
- 有 ``task_type`` 的 skill 加入 ``task_skill_map``
|
||||||
|
- 非 always 的 skill 生成 ``catalog_text`` 并注册到 registry
|
||||||
|
|
||||||
|
参数:
|
||||||
|
skills_dir: Skill 文件目录。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
``(always_skills_text, task_skill_map, catalog_text, registry)`` 四元组。
|
||||||
|
"""
|
||||||
|
if not skills_dir.exists():
|
||||||
|
return "", {}, "", SkillRegistry()
|
||||||
|
|
||||||
|
always_parts: list[str] = []
|
||||||
|
task_skill_map: dict[str, str] = {}
|
||||||
|
catalog_lines: list[str] = []
|
||||||
|
registry_paths: dict[str, Path] = {}
|
||||||
|
|
||||||
|
for path in sorted(skills_dir.glob("*.md")):
|
||||||
|
raw = path.read_text(encoding="utf-8")
|
||||||
|
meta = parse_frontmatter(raw)
|
||||||
|
if "name" not in meta:
|
||||||
|
logger.warning("跳过无 name 的 skill 文件: {}", path)
|
||||||
|
continue
|
||||||
|
|
||||||
|
body = strip_frontmatter(raw)
|
||||||
|
name = meta["name"]
|
||||||
|
desc = meta.get("description", "")
|
||||||
|
task_type = meta.get("task_type", "")
|
||||||
|
is_always = str(meta.get("always", "false")).lower() == "true"
|
||||||
|
|
||||||
|
if is_always:
|
||||||
|
always_parts.append(body)
|
||||||
|
else:
|
||||||
|
if task_type:
|
||||||
|
task_skill_map[task_type] = body
|
||||||
|
catalog_lines.append(f"- **{name}**: {desc}")
|
||||||
|
registry_paths[name] = path
|
||||||
|
|
||||||
|
always_text = "\n\n---\n\n".join(always_parts)
|
||||||
|
catalog_text = "\n".join(catalog_lines)
|
||||||
|
|
||||||
|
registry = SkillRegistry()
|
||||||
|
registry.set_paths(registry_paths)
|
||||||
|
|
||||||
|
return always_text, task_skill_map, catalog_text, registry
|
||||||
@@ -0,0 +1,487 @@
|
|||||||
|
"""节点内容摘要模块 — 两轮 LLM 调用生成 question-conditioned 摘要。
|
||||||
|
|
||||||
|
提取轮:带防幻觉 system prompt,提取与问题相关的信息。
|
||||||
|
验证轮:带核实 system prompt,逐条核实并给置信度。
|
||||||
|
与 TRM4 core/tree/summarizer.py 保真迁移:
|
||||||
|
同步 → async、_call_llm → await llm.chat()、ThreadPoolExecutor → asyncio.gather。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import re
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Callable
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from core.protocols import LLMProvider
|
||||||
|
|
||||||
|
# ── 正则常量 ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# 行号引注组:括号包裹的 s/c 行号列表,如 (s1) / (c2,s5) / (c70-c73,s196-s200)
|
||||||
|
# (兼容全角括号与逗号;单元允许范围语法 s3-s5 / s3-5,60-span 实测模型常用)
|
||||||
|
_ANCHOR_GROUP = re.compile(
|
||||||
|
r"[((]\s*([sc]\d+(?:-[sc]?\d+)?(?:\s*[,,]\s*[sc]\d+(?:-[sc]?\d+)?)*)\s*[))]"
|
||||||
|
)
|
||||||
|
_ANCHOR_RANGE = re.compile(r"([sc])(\d+)-([sc]?)(\d+)")
|
||||||
|
_RELEVANT_SECTION = re.compile(r"\[相关信息\](.*?)(?=\n\[|\Z)", re.DOTALL)
|
||||||
|
# 无相关信息声明句:60-span 实测全为"该节点未包含与问题直接相关的信息"类变体
|
||||||
|
_NO_INFO_STATEMENT = re.compile(r"未包含.*相关.*信息")
|
||||||
|
|
||||||
|
# 范围展开条数上限:防 (s1-s9999) 这类爆炸展开
|
||||||
|
_RANGE_MAX_IDS = 50
|
||||||
|
|
||||||
|
# 双封顶参数:上轮 A/B 证明无上限引用膨胀至 8.4 条/span 挤占提取预算(hall +51%)
|
||||||
|
_EXPAND_MAX_ITEMS = 5
|
||||||
|
_EXPAND_MAX_CHARS = 800
|
||||||
|
_EXPAND_LINE_CAP = 200
|
||||||
|
|
||||||
|
|
||||||
|
# ── Prompt 加载 ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _load_prompt(prompts_dir: Path, filename: str) -> str:
|
||||||
|
"""从 prompts 目录加载 system prompt 文件。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
prompts_dir: prompt 文件所在目录。
|
||||||
|
filename: prompt 文件名。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
文件内容字符串。
|
||||||
|
"""
|
||||||
|
return (prompts_dir / filename).read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
# ── Anchor 工具函数 ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _expand_anchor_ids(group_text: str) -> list[str]:
|
||||||
|
"""把引注组文本展开为逐 id 列表(支持范围语法)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
group_text: _ANCHOR_GROUP 捕获的组内文本,如 "s3-s5, c1"。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
逐 id 列表。合法范围(同前缀、起点<=终点、展开条数<=50)展开为
|
||||||
|
逐 id("s3-s5"/"s3-5" -> s3,s4,s5);非法范围(跨前缀如 c3-s5、
|
||||||
|
起点>终点、展开条数超限防爆炸)保留原 token——后续查表必然失配,
|
||||||
|
整段按 1 个非法锚计罚剔除。
|
||||||
|
"""
|
||||||
|
ids: list[str] = []
|
||||||
|
for token in re.split(r"[,,]\s*", group_text):
|
||||||
|
token = token.strip()
|
||||||
|
m = _ANCHOR_RANGE.fullmatch(token)
|
||||||
|
if m is None:
|
||||||
|
ids.append(token)
|
||||||
|
continue
|
||||||
|
prefix, start = m.group(1), int(m.group(2))
|
||||||
|
end_prefix, end = m.group(3), int(m.group(4))
|
||||||
|
legal_range = (
|
||||||
|
(not end_prefix or end_prefix == prefix)
|
||||||
|
and start <= end
|
||||||
|
and end - start + 1 <= _RANGE_MAX_IDS
|
||||||
|
)
|
||||||
|
if not legal_range:
|
||||||
|
ids.append(token)
|
||||||
|
continue
|
||||||
|
ids.extend(f"{prefix}{i}" for i in range(start, end + 1))
|
||||||
|
return ids
|
||||||
|
|
||||||
|
|
||||||
|
def check_anchors(summary: str, anchor_map: dict[str, str]) -> tuple[str, dict[str, int]]:
|
||||||
|
"""校验行号引注:非法行号删锚不删断言。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
summary: 提取轮输出(含行号引注)。
|
||||||
|
anchor_map: {锚: 原文行} 查表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
(清理后文本, {"n_assertions", "n_anchored", "n_illegal"})。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
清洗全文、统计限段:非法锚无论出现在哪一段都删除并计入 n_illegal
|
||||||
|
(避免未校验段落的编造锚流入装配展开);断言统计
|
||||||
|
(n_assertions/n_anchored)仅数 [相关信息] 段内非空内容行。
|
||||||
|
引注组先经 _expand_anchor_ids 把范围语法展开为逐 id 再逐 id 校验
|
||||||
|
(合法子集重写为逐 id 列表如 (s3,s4,s5)),组内全非法则整组删除;
|
||||||
|
组外文本一律不动(删锚不删断言)。分母口径:匹配"未包含...相关...
|
||||||
|
信息"词面的声明句不计入 n_assertions——它们天然无锚,计入会虚压
|
||||||
|
遵从率。
|
||||||
|
"""
|
||||||
|
stats: dict[str, int] = {"n_assertions": 0, "n_anchored": 0, "n_illegal": 0}
|
||||||
|
|
||||||
|
def _clean_group(gm: re.Match) -> str:
|
||||||
|
ids = _expand_anchor_ids(gm.group(1))
|
||||||
|
legal = [i for i in ids if i in anchor_map]
|
||||||
|
stats["n_illegal"] += len(ids) - len(legal)
|
||||||
|
return f"({','.join(legal)})" if legal else ""
|
||||||
|
|
||||||
|
cleaned = _ANCHOR_GROUP.sub(_clean_group, summary)
|
||||||
|
m = _RELEVANT_SECTION.search(cleaned)
|
||||||
|
if m is None:
|
||||||
|
return cleaned, stats
|
||||||
|
for line in m.group(1).splitlines():
|
||||||
|
line = line.strip().lstrip("-•*").strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
if _NO_INFO_STATEMENT.search(line):
|
||||||
|
continue
|
||||||
|
stats["n_assertions"] += 1
|
||||||
|
if _ANCHOR_GROUP.search(line):
|
||||||
|
stats["n_anchored"] += 1
|
||||||
|
return cleaned, stats
|
||||||
|
|
||||||
|
|
||||||
|
def _cited_anchor_ids(summary: str, anchor_map: dict[str, str]) -> list[str]:
|
||||||
|
"""按引注首次出现顺序收集合法锚 id(去重)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
summary: 含行号引注的文本。
|
||||||
|
anchor_map: {锚: 原文行} 查表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
去重后的合法锚 id 列表(保持首次出现顺序)。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
从 assemble_anchored_output 提取以满足圈复杂度门槛;范围语法经
|
||||||
|
_expand_anchor_ids 展开后逐 id 收集;只收合法锚(非法锚已由
|
||||||
|
check_anchors 清除,此处过滤是防御性双保险)。
|
||||||
|
"""
|
||||||
|
ordered: list[str] = []
|
||||||
|
for gm in _ANCHOR_GROUP.finditer(summary):
|
||||||
|
for aid in _expand_anchor_ids(gm.group(1)):
|
||||||
|
if aid in anchor_map and aid not in ordered:
|
||||||
|
ordered.append(aid)
|
||||||
|
return ordered
|
||||||
|
|
||||||
|
|
||||||
|
def assemble_anchored_output(
|
||||||
|
summary: str, anchor_map: dict[str, str], mode: str
|
||||||
|
) -> tuple[str, dict[str, int]]:
|
||||||
|
"""按装配形态生成最终输出:展开引文并施加双封顶。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
summary: check_anchors 清理后的文本。
|
||||||
|
anchor_map: {锚: 原文行}。
|
||||||
|
mode: "ids"(裸行号)| "ids_expand"(行号+展开)| "expand_only"(展开剥行号)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
(最终文本, {"n_expanded", "n_trunc"})。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
展开按引注首次出现顺序取前 5 条;总额帽按 [引文] 条目完整长度
|
||||||
|
(含前缀与引号)记账,<=800 字符;单行原文超 200 字符先截断。
|
||||||
|
n_expanded/n_trunc 仅计实际输出的条目。expand_only 先对正文剥除
|
||||||
|
全部引注 token、再拼接 [引文] 段(judge 探针判定 id token 被计罚
|
||||||
|
时的回退形态)——引文行不经过剥离,原文行中的括号文本得以保留。
|
||||||
|
"""
|
||||||
|
assert mode in ("ids", "ids_expand", "expand_only"), f"未知装配形态: {mode}"
|
||||||
|
stats: dict[str, int] = {"n_expanded": 0, "n_trunc": 0}
|
||||||
|
if mode != "ids":
|
||||||
|
ordered = _cited_anchor_ids(summary, anchor_map)
|
||||||
|
expansions: list[str] = []
|
||||||
|
total = 0
|
||||||
|
for aid in ordered[:_EXPAND_MAX_ITEMS]:
|
||||||
|
line = anchor_map[aid]
|
||||||
|
truncated = len(line) > _EXPAND_LINE_CAP
|
||||||
|
if truncated:
|
||||||
|
line = line[:_EXPAND_LINE_CAP] + "…"
|
||||||
|
entry = f' ▸ {aid}: "{line}"'
|
||||||
|
if total + len(entry) > _EXPAND_MAX_CHARS:
|
||||||
|
break
|
||||||
|
total += len(entry)
|
||||||
|
expansions.append(entry)
|
||||||
|
stats["n_expanded"] += 1
|
||||||
|
if truncated:
|
||||||
|
stats["n_trunc"] += 1
|
||||||
|
if mode == "expand_only":
|
||||||
|
summary = _ANCHOR_GROUP.sub("", summary)
|
||||||
|
if expansions:
|
||||||
|
summary = summary + "\n[引文]\n" + "\n".join(expansions)
|
||||||
|
return summary, stats
|
||||||
|
|
||||||
|
|
||||||
|
# ── LLM 调用辅助 ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
async def _call_llm(
|
||||||
|
llm: LLMProvider,
|
||||||
|
system_prompt: str,
|
||||||
|
user_text: str,
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> str:
|
||||||
|
"""调用 LLM 并返回响应文本。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
llm: LLMProvider 端口实例。
|
||||||
|
system_prompt: 系统提示词。
|
||||||
|
user_text: 用户消息文本。
|
||||||
|
session_id: 会话 ID(透传遥测)。
|
||||||
|
parent_call_id: 父调用 ID(透传遥测)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
模型回答文本。
|
||||||
|
"""
|
||||||
|
messages = [
|
||||||
|
{"role": "system", "content": system_prompt},
|
||||||
|
{"role": "user", "content": user_text},
|
||||||
|
]
|
||||||
|
response = await llm.chat(messages, session_id=session_id, parent_call_id=parent_call_id)
|
||||||
|
return response.content
|
||||||
|
|
||||||
|
|
||||||
|
# ── 摘要函数 ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
async def summarize_node(
|
||||||
|
llm: LLMProvider,
|
||||||
|
raw_text: str,
|
||||||
|
question: str,
|
||||||
|
prompts_dir: Path,
|
||||||
|
*,
|
||||||
|
anchor_map: dict[str, str] | None,
|
||||||
|
assemble_mode: str,
|
||||||
|
stats_sink: Callable[[dict[str, Any]], None] | None = None,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> str:
|
||||||
|
"""对单个节点做 question-conditioned 两轮摘要(可选行号锚模式)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
llm: LLMProvider 端口实例。
|
||||||
|
raw_text: 节点文本(锚模式下为带 [c1]/[s1] 行号的素材)。
|
||||||
|
question: Agent 当前关注的具体问题。
|
||||||
|
prompts_dir: prompt 文件目录。
|
||||||
|
anchor_map: {锚: 原文行};None 表示 v1 行为(无校验无装配无统计)。
|
||||||
|
assemble_mode: 装配形态("ids"/"ids_expand"/"expand_only"),
|
||||||
|
anchor_map 为 None 时忽略。
|
||||||
|
stats_sink: 统计回调(None 不收集);统计严禁写入输出文本。
|
||||||
|
session_id: 会话 ID(透传遥测)。
|
||||||
|
parent_call_id: 父调用 ID(透传遥测)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
"[内容摘要] {结果}\\n[核实] {验证结果}" 或错误信息。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
锚模式流程:提取 -> check_anchors 清洗 -> 核实轮(见清洗后未装配文本)
|
||||||
|
-> assemble_anchored_output 装配 -> sink 上报。sink dict 完整键名:
|
||||||
|
n_assertions/n_anchored/n_illegal(check_anchors)、
|
||||||
|
n_expanded/n_trunc(装配)、output_chars(最终输出字符数)、
|
||||||
|
pre_assembly(清洗后未装配文本快照)、anchor_map(原样透传)。
|
||||||
|
"""
|
||||||
|
extract_input = f"问题: {question}\n\n以下是视频片段的描述和字幕:\n{raw_text}"
|
||||||
|
try:
|
||||||
|
raw_summary = await _call_llm(
|
||||||
|
llm,
|
||||||
|
_load_prompt(prompts_dir, "view_node_extract.md"),
|
||||||
|
extract_input,
|
||||||
|
session_id=session_id,
|
||||||
|
parent_call_id=parent_call_id,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
return f"[摘要错误] {e}"
|
||||||
|
|
||||||
|
anchor_stats: dict[str, int] = {}
|
||||||
|
if anchor_map is not None:
|
||||||
|
raw_summary, anchor_stats = check_anchors(raw_summary, anchor_map)
|
||||||
|
pre_assembly = raw_summary
|
||||||
|
|
||||||
|
verify_input = (
|
||||||
|
f"问题: {question}\n\n"
|
||||||
|
f"原始内容:\n{raw_text}\n\n"
|
||||||
|
f"以下是另一个模型基于上述内容生成的摘要,请核实:\n{raw_summary}"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
verify_result = await _call_llm(
|
||||||
|
llm,
|
||||||
|
_load_prompt(prompts_dir, "view_node_verify.md"),
|
||||||
|
verify_input,
|
||||||
|
session_id=session_id,
|
||||||
|
parent_call_id=parent_call_id,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("验证轮调用失败,跳过: {}", e)
|
||||||
|
verify_result = "跳过(调用失败)"
|
||||||
|
|
||||||
|
if anchor_map is not None:
|
||||||
|
raw_summary, asm_stats = assemble_anchored_output(raw_summary, anchor_map, assemble_mode)
|
||||||
|
anchor_stats.update(asm_stats)
|
||||||
|
|
||||||
|
result = f"[内容摘要] {raw_summary}\n[核实] {verify_result}"
|
||||||
|
if anchor_map is not None and stats_sink is not None:
|
||||||
|
stats_sink(
|
||||||
|
{
|
||||||
|
**anchor_stats,
|
||||||
|
"output_chars": len(result),
|
||||||
|
"pre_assembly": pre_assembly,
|
||||||
|
"anchor_map": anchor_map,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
async def summarize_children(
|
||||||
|
llm: LLMProvider,
|
||||||
|
children_info: list[dict[str, Any]],
|
||||||
|
question: str,
|
||||||
|
prompts_dir: Path,
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> str:
|
||||||
|
"""对子节点列表做 question-conditioned 相关性标注(两轮)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
llm: LLMProvider 端口实例。
|
||||||
|
children_info: 子节点信息列表,每项含 id, time_range, summary。
|
||||||
|
question: Agent 当前关注的具体问题。
|
||||||
|
prompts_dir: prompt 文件目录。
|
||||||
|
session_id: 会话 ID(透传遥测)。
|
||||||
|
parent_call_id: 父调用 ID(透传遥测)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
带相关性标注的子节点概览文本。失败时降级返回原始列表。
|
||||||
|
"""
|
||||||
|
lines = []
|
||||||
|
for child in children_info:
|
||||||
|
t_start, t_end = child["time_range"]
|
||||||
|
lines.append(f"- {child['id']} ({t_start:.0f}-{t_end:.0f}s): {child['summary']}")
|
||||||
|
children_text = "\n".join(lines)
|
||||||
|
|
||||||
|
extract_input = f"问题: {question}\n\n{children_text}"
|
||||||
|
try:
|
||||||
|
raw_ranking = await _call_llm(
|
||||||
|
llm,
|
||||||
|
_load_prompt(prompts_dir, "view_node_children_extract.md"),
|
||||||
|
extract_input,
|
||||||
|
session_id=session_id,
|
||||||
|
parent_call_id=parent_call_id,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("子节点标注失败,回退原始列表: {}", e)
|
||||||
|
return children_text
|
||||||
|
|
||||||
|
verify_input = (
|
||||||
|
f"问题: {question}\n\n"
|
||||||
|
f"原始子节点列表:\n{children_text}\n\n"
|
||||||
|
f"以下是另一个模型基于上述信息生成的相关性标注,请核实:\n{raw_ranking}"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
verify_result = await _call_llm(
|
||||||
|
llm,
|
||||||
|
_load_prompt(prompts_dir, "view_node_children_verify.md"),
|
||||||
|
verify_input,
|
||||||
|
session_id=session_id,
|
||||||
|
parent_call_id=parent_call_id,
|
||||||
|
)
|
||||||
|
return f"{raw_ranking}\n[核实] {verify_result}"
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("子节点标注验证轮失败,跳过: {}", e)
|
||||||
|
return raw_ranking
|
||||||
|
|
||||||
|
|
||||||
|
async def _summarize_search_result(
|
||||||
|
llm: LLMProvider,
|
||||||
|
raw_text: str,
|
||||||
|
question: str,
|
||||||
|
prompts_dir: Path,
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> str:
|
||||||
|
"""对搜索结果做两轮摘要(search_similar 专用)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
llm: LLMProvider 端口实例。
|
||||||
|
raw_text: 节点原始文本。
|
||||||
|
question: Agent 当前关注的具体问题。
|
||||||
|
prompts_dir: prompt 文件目录。
|
||||||
|
session_id: 会话 ID(透传遥测)。
|
||||||
|
parent_call_id: 父调用 ID(透传遥测)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
"[内容摘要] {提取结果}\\n[核实] {验证结果}" 或错误信息。
|
||||||
|
"""
|
||||||
|
extract_input = f"问题: {question}\n\n以下是语义搜索命中的视频节点描述和字幕:\n{raw_text}"
|
||||||
|
try:
|
||||||
|
raw_summary = await _call_llm(
|
||||||
|
llm,
|
||||||
|
_load_prompt(prompts_dir, "search_similar_extract.md"),
|
||||||
|
extract_input,
|
||||||
|
session_id=session_id,
|
||||||
|
parent_call_id=parent_call_id,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
return f"[摘要错误] {e}"
|
||||||
|
|
||||||
|
verify_input = (
|
||||||
|
f"问题: {question}\n\n"
|
||||||
|
f"原始内容:\n{raw_text}\n\n"
|
||||||
|
f"以下是另一个模型基于上述内容生成的摘要,请核实:\n{raw_summary}"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
verify_result = await _call_llm(
|
||||||
|
llm,
|
||||||
|
_load_prompt(prompts_dir, "search_similar_verify.md"),
|
||||||
|
verify_input,
|
||||||
|
session_id=session_id,
|
||||||
|
parent_call_id=parent_call_id,
|
||||||
|
)
|
||||||
|
return f"[内容摘要] {raw_summary}\n[核实] {verify_result}"
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("搜索结果验证轮失败,跳过: {}", e)
|
||||||
|
return f"[内容摘要] {raw_summary}\n[核实] 跳过(调用失败)"
|
||||||
|
|
||||||
|
|
||||||
|
async def summarize_nodes_batch(
|
||||||
|
llm: LLMProvider,
|
||||||
|
items: list[tuple[str, str, str]],
|
||||||
|
question: str,
|
||||||
|
prompts_dir: Path,
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> list[tuple[str, str]]:
|
||||||
|
"""并发对多个搜索结果做两轮摘要。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
llm: LLMProvider 端口实例。
|
||||||
|
items: [(node_id, raw_text, extra_info), ...] 列表。
|
||||||
|
question: Agent 当前关注的具体问题。
|
||||||
|
prompts_dir: prompt 文件目录。
|
||||||
|
session_id: 会话 ID(透传遥测)。
|
||||||
|
parent_call_id: 父调用 ID(透传遥测)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
[(node_id, summary_text), ...] 列表,顺序与输入一致。
|
||||||
|
"""
|
||||||
|
if not items:
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def _worker(idx: int, node_id: str, raw_text: str) -> tuple[int, str, str]:
|
||||||
|
"""单个节点的摘要工作协程。"""
|
||||||
|
summary = await _summarize_search_result(
|
||||||
|
llm,
|
||||||
|
raw_text,
|
||||||
|
question,
|
||||||
|
prompts_dir,
|
||||||
|
session_id=session_id,
|
||||||
|
parent_call_id=parent_call_id,
|
||||||
|
)
|
||||||
|
return idx, node_id, summary
|
||||||
|
|
||||||
|
tasks = [_worker(i, nid, text) for i, (nid, text, _) in enumerate(items)]
|
||||||
|
results_raw = await asyncio.gather(*tasks)
|
||||||
|
|
||||||
|
results: dict[int, tuple[str, str]] = {}
|
||||||
|
for idx, node_id, summary in results_raw:
|
||||||
|
results[idx] = (node_id, summary)
|
||||||
|
|
||||||
|
return [results[i] for i in range(len(items))]
|
||||||
@@ -0,0 +1,321 @@
|
|||||||
|
"""搜索 Agent 工具调度器 — 工具描述与 dispatch 分发。
|
||||||
|
|
||||||
|
实现 ``core/agent/protocols.ToolDispatcher`` Protocol。
|
||||||
|
连接 TreeEnvironment(数据)、summarizer(LLM 摘要)、
|
||||||
|
vision(VLM 观察)和 skills(策略加载)。
|
||||||
|
|
||||||
|
与 TRM4 ``core/tree/tools.py`` 的差异:
|
||||||
|
- 自由函数 ``dispatch()`` → ``SearchToolDispatcher`` 类(依赖注入);
|
||||||
|
- 同步 → 全异步;
|
||||||
|
- view_node / search_similar 内部拆分为 env 数据读取 + summarizer LLM 摘要。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
from app.search.summarizer import summarize_children, summarize_node, summarize_nodes_batch
|
||||||
|
from app.search.vision import observe_frame
|
||||||
|
from app.tree.environment import _LEVEL_LABEL, TreeEnvironment, _node_level
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Callable
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from app.ports import OCRProvider
|
||||||
|
from app.search.skills import SkillRegistry
|
||||||
|
from core.protocols import LLMProvider, VLMProvider
|
||||||
|
|
||||||
|
# ── 工具描述文本(与 TRM4 core/tree/tools.py 完全一致) ─────────────────
|
||||||
|
|
||||||
|
_BASE_DESCRIPTIONS = """\
|
||||||
|
## 可用工具
|
||||||
|
|
||||||
|
在 action 中指定 tool 和 args 来调用工具。
|
||||||
|
|
||||||
|
### view_node
|
||||||
|
查看节点信息,获取与问题相关的内容摘要和子节点概览。
|
||||||
|
- args: {"node_id": "节点 ID", "question": "当前关注的具体问题"}
|
||||||
|
|
||||||
|
### search_similar
|
||||||
|
语义检索最相关的节点,返回与问题相关的内容摘要。
|
||||||
|
- args: {"query": "搜索关键词(2-4 词)", "question": "当前关注的具体问题", "k": 返回数量(可选,默认 5)}
|
||||||
|
|
||||||
|
### observe_frame
|
||||||
|
调用视觉模型查看关键帧图像,回答针对性的视觉问题。
|
||||||
|
- args: {"node_ids": ["L3 节点 ID 列表(1-4 个),或单个 L2 节点 ID"], "question": "针对帧内容的具体视觉问题"}
|
||||||
|
|
||||||
|
### submit_answer
|
||||||
|
提交最终答案。
|
||||||
|
- args: {"answer": "选项字母 A/B/C/D", "evidence": "关键证据摘要", "reasoning": "每个选项的判断理由"}"""
|
||||||
|
|
||||||
|
_SKILL_DESCRIPTION = """
|
||||||
|
|
||||||
|
### read_skill
|
||||||
|
加载指定题型技能的详细搜索策略。
|
||||||
|
- args: {"name": "技能名称"}"""
|
||||||
|
|
||||||
|
|
||||||
|
def get_tool_descriptions(include_read_skill: bool = False) -> str:
|
||||||
|
"""返回工具描述文本,用于写入 system prompt。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
include_read_skill: 是否包含 read_skill 工具(manual 模式用)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
Markdown 格式的工具描述文本。
|
||||||
|
"""
|
||||||
|
text = _BASE_DESCRIPTIONS
|
||||||
|
if include_read_skill:
|
||||||
|
text += _SKILL_DESCRIPTION
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
# ── SearchToolDispatcher ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class SearchToolDispatcher:
|
||||||
|
"""搜索 Agent 工具调度器,实现 ToolDispatcher Protocol。
|
||||||
|
|
||||||
|
按工具名路由到对应私有处理方法。未知工具抛 ValueError
|
||||||
|
(AgentLoop 捕获后不计步数);节点不存在等运行时错误
|
||||||
|
捕获后返回错误文本。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
env: 视频树运行时环境(纯数据访问)。
|
||||||
|
tool_llm: 摘要用 LLM 端口。
|
||||||
|
vlm: 视觉模型端口。
|
||||||
|
ocr: 帧文字转录端口(None 不启用)。
|
||||||
|
prompts_dir: prompt 文件目录。
|
||||||
|
skills: 技能注册表(None 不启用 read_skill)。
|
||||||
|
embed_fn: 文本嵌入函数(search_similar 用)。
|
||||||
|
verify_vision: observe_frame 是否执行验证轮。
|
||||||
|
anchor: view_node 是否启用行号锚模式。
|
||||||
|
assemble_mode: 锚模式装配形态("ids"/"ids_expand"/"expand_only")。
|
||||||
|
stats_sink: 统计回调(None 不收集)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
env: TreeEnvironment,
|
||||||
|
tool_llm: LLMProvider,
|
||||||
|
vlm: VLMProvider,
|
||||||
|
ocr: OCRProvider | None,
|
||||||
|
prompts_dir: Path,
|
||||||
|
skills: SkillRegistry | None,
|
||||||
|
*,
|
||||||
|
embed_fn: Callable[[str | list[str]], np.ndarray],
|
||||||
|
verify_vision: bool,
|
||||||
|
anchor: bool,
|
||||||
|
assemble_mode: str,
|
||||||
|
stats_sink: Callable[[dict[str, Any]], None] | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._env = env
|
||||||
|
self._tool_llm = tool_llm
|
||||||
|
self._vlm = vlm
|
||||||
|
self._ocr = ocr
|
||||||
|
self._prompts_dir = prompts_dir
|
||||||
|
self._skills = skills
|
||||||
|
self._embed_fn = embed_fn
|
||||||
|
self._verify_vision = verify_vision
|
||||||
|
self._anchor = anchor
|
||||||
|
self._assemble_mode = assemble_mode
|
||||||
|
self._stats_sink = stats_sink
|
||||||
|
|
||||||
|
# ── ToolDispatcher Protocol 实现 ──────────────────────────────────
|
||||||
|
|
||||||
|
async def dispatch(
|
||||||
|
self, tool_name: str, args: dict[str, Any], *, context: dict[str, Any]
|
||||||
|
) -> str:
|
||||||
|
"""按工具名分发到对应处理方法。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
tool_name: 工具名称。
|
||||||
|
args: 工具参数字典。
|
||||||
|
context: 调用上下文(含 session_id、parent_call_id 等遥测字段)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
工具执行结果文本。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 未知工具名——上抛给 AgentLoop,不计步数。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
if tool_name == "view_node":
|
||||||
|
return await self._handle_view_node(args, context)
|
||||||
|
if tool_name == "search_similar":
|
||||||
|
return await self._handle_search_similar(args, context)
|
||||||
|
if tool_name == "observe_frame":
|
||||||
|
return await self._handle_observe_frame(args, context)
|
||||||
|
if tool_name == "submit_answer":
|
||||||
|
return f"[ok] 答案已提交: {args['answer']}"
|
||||||
|
if tool_name == "read_skill":
|
||||||
|
return self._handle_read_skill(args)
|
||||||
|
except (KeyError, FileNotFoundError) as e:
|
||||||
|
return f"工具执行错误: {e}"
|
||||||
|
|
||||||
|
raise ValueError(f"未知工具: {tool_name}")
|
||||||
|
|
||||||
|
# ── 私有处理方法 ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async def _handle_view_node(self, args: dict[str, Any], context: dict[str, Any]) -> str:
|
||||||
|
"""view_node:节点摘要 + 子节点概览。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
args: {"node_id": str, "question": str}。
|
||||||
|
context: 遥测上下文。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
"[节点] {id} | {level} | {time}\\n\\n{summary}\\n\\n[子节点概览] ..."
|
||||||
|
"""
|
||||||
|
node_id: str = args["node_id"]
|
||||||
|
question: str = args["question"]
|
||||||
|
session_id = context.get("session_id")
|
||||||
|
parent_call_id = context.get("parent_call_id")
|
||||||
|
|
||||||
|
# Phase 1: 节点元数据(头部格式化)
|
||||||
|
node = self._env._id_to_node[node_id]
|
||||||
|
level = _node_level(node)
|
||||||
|
level_label = _LEVEL_LABEL[level]
|
||||||
|
time_str = TreeEnvironment._format_time_range(node)
|
||||||
|
|
||||||
|
# Phase 2: 节点内容摘要
|
||||||
|
raw_text, anchor_map = self._env.get_node_text(node_id, anchor=self._anchor)
|
||||||
|
summary = await summarize_node(
|
||||||
|
self._tool_llm,
|
||||||
|
raw_text,
|
||||||
|
question,
|
||||||
|
self._prompts_dir,
|
||||||
|
anchor_map=anchor_map,
|
||||||
|
assemble_mode=self._assemble_mode,
|
||||||
|
stats_sink=self._stats_sink,
|
||||||
|
session_id=session_id,
|
||||||
|
parent_call_id=parent_call_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
parts: list[str] = [
|
||||||
|
f"[节点] {node_id} | {level_label} | {time_str}",
|
||||||
|
"",
|
||||||
|
summary,
|
||||||
|
]
|
||||||
|
|
||||||
|
# Phase 3: 子节点概览
|
||||||
|
children_info = self._env.get_children_info(node_id)
|
||||||
|
if children_info:
|
||||||
|
children_text = await summarize_children(
|
||||||
|
self._tool_llm,
|
||||||
|
children_info,
|
||||||
|
question,
|
||||||
|
self._prompts_dir,
|
||||||
|
session_id=session_id,
|
||||||
|
parent_call_id=parent_call_id,
|
||||||
|
)
|
||||||
|
parts.append(f"\n[子节点概览] {len(children_info)} 个子节点\n{children_text}")
|
||||||
|
|
||||||
|
return "\n".join(parts)
|
||||||
|
|
||||||
|
async def _handle_search_similar(self, args: dict[str, Any], context: dict[str, Any]) -> str:
|
||||||
|
"""search_similar:语义检索 + 批量摘要。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
args: {"query": str, "question": str, "k": int (可选)}。
|
||||||
|
context: 遥测上下文。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
"[搜索结果] 查询 \\"{query}\\" → N 个相关节点\\n\\n1. ..."
|
||||||
|
"""
|
||||||
|
query: str = args["query"]
|
||||||
|
question: str = args["question"]
|
||||||
|
top_k: int = args.get("k", 5)
|
||||||
|
session_id = context.get("session_id")
|
||||||
|
parent_call_id = context.get("parent_call_id")
|
||||||
|
|
||||||
|
# Phase 1: 语义检索
|
||||||
|
results = self._env.search_similar(query, top_k=top_k, embed_fn=self._embed_fn)
|
||||||
|
|
||||||
|
if not results:
|
||||||
|
return f'[搜索结果] 查询 "{query}" → 0 个相关节点'
|
||||||
|
|
||||||
|
# Phase 2: 构建摘要输入
|
||||||
|
items: list[tuple[str, str, str]] = []
|
||||||
|
for nid, score in results:
|
||||||
|
node = self._env._id_to_node[nid]
|
||||||
|
raw_text, _ = self._env.get_node_text(nid)
|
||||||
|
level = _node_level(node)
|
||||||
|
time_str = TreeEnvironment._format_time_range(node)
|
||||||
|
extra = f"{level} score={score:.4f} [{time_str}]"
|
||||||
|
items.append((nid, raw_text, extra))
|
||||||
|
|
||||||
|
# Phase 3: 并发批量摘要
|
||||||
|
summaries = await summarize_nodes_batch(
|
||||||
|
self._tool_llm,
|
||||||
|
items,
|
||||||
|
question,
|
||||||
|
self._prompts_dir,
|
||||||
|
session_id=session_id,
|
||||||
|
parent_call_id=parent_call_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Phase 4: 格式化输出
|
||||||
|
lines: list[str] = []
|
||||||
|
for i, (nid, summary_text) in enumerate(summaries):
|
||||||
|
_, _, extra = items[i]
|
||||||
|
lines.append(f"{i + 1}. {nid} | {extra}\n {summary_text}")
|
||||||
|
|
||||||
|
header = f'[搜索结果] 查询 "{query}" → {len(results)} 个相关节点'
|
||||||
|
return header + "\n\n" + "\n\n".join(lines)
|
||||||
|
|
||||||
|
async def _handle_observe_frame(self, args: dict[str, Any], context: dict[str, Any]) -> str:
|
||||||
|
"""observe_frame:VLM 帧观察 + 字幕前置。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
args: {"node_ids": list[str], "question": str}。
|
||||||
|
context: 遥测上下文。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
"[字幕上下文] ...\\n[视觉观察] ..." 或 "[视觉观察] ..."
|
||||||
|
"""
|
||||||
|
node_ids: list[str] = args["node_ids"]
|
||||||
|
question: str = args.get("question", "")
|
||||||
|
session_id = context.get("session_id")
|
||||||
|
parent_call_id = context.get("parent_call_id")
|
||||||
|
|
||||||
|
if not question.strip():
|
||||||
|
return "工具执行错误: question 不能为空"
|
||||||
|
|
||||||
|
# Phase 1: 解析帧路径和字幕
|
||||||
|
frame_paths = self._env.resolve_frame_paths(node_ids)
|
||||||
|
subtitle = self._env.get_subtitle(node_ids[0])
|
||||||
|
|
||||||
|
# Phase 2: VLM 调用
|
||||||
|
result = await observe_frame(
|
||||||
|
self._vlm,
|
||||||
|
frame_paths,
|
||||||
|
question,
|
||||||
|
self._prompts_dir,
|
||||||
|
ocr=self._ocr,
|
||||||
|
verify=self._verify_vision,
|
||||||
|
stats_sink=self._stats_sink,
|
||||||
|
session_id=session_id,
|
||||||
|
parent_call_id=parent_call_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Phase 3: 字幕前置拼接
|
||||||
|
if subtitle:
|
||||||
|
return f"[字幕上下文] {subtitle}\n{result}"
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _handle_read_skill(self, args: dict[str, Any]) -> str:
|
||||||
|
"""read_skill:加载指定技能的搜索策略正文。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
args: {"name": str}。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
技能正文或错误提示。
|
||||||
|
"""
|
||||||
|
if self._skills is None:
|
||||||
|
return "错误: skills 未启用"
|
||||||
|
return self._skills.read(args["name"])
|
||||||
@@ -0,0 +1,157 @@
|
|||||||
|
"""视觉模型调用模块 -- 两轮 VLM 调用查看关键帧图像。
|
||||||
|
|
||||||
|
提取轮:带防幻觉 system prompt,提取原始视觉证据。
|
||||||
|
验证轮:把初稿全文喂回,逐条核实并给置信度。
|
||||||
|
|
||||||
|
从 TRM4 ``core/tree/vision.py`` 迁移,关键变更:
|
||||||
|
- VLM 调用走 ``VLMProvider.chat_with_images`` Protocol,images 传 Path 列表;
|
||||||
|
- OCR 调用走 ``OCRProvider.transcribe_frames`` 异步 Protocol;
|
||||||
|
- 遥测字段(session_id / parent_call_id)透传给 VLM 调用。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Callable
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.ports import OCRProvider
|
||||||
|
from core.protocols import VLMProvider
|
||||||
|
|
||||||
|
_OCR_PREFIX = (
|
||||||
|
"以下是 OCR 工具对这些帧的文字转录,仅供参考;与你实际看到的不一致时,报告双读数并标注分歧:\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_prompt(prompts_dir: Path, filename: str) -> str:
|
||||||
|
"""从 prompts 目录加载 system prompt 文件。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
prompts_dir: prompt 文件所在目录。
|
||||||
|
filename: prompt 文件名。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
文件内容字符串。
|
||||||
|
"""
|
||||||
|
return (prompts_dir / filename).read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
async def observe_frame(
|
||||||
|
vlm: VLMProvider,
|
||||||
|
frame_paths: list[Path],
|
||||||
|
question: str,
|
||||||
|
prompts_dir: Path,
|
||||||
|
*,
|
||||||
|
ocr: OCRProvider | None,
|
||||||
|
verify: bool,
|
||||||
|
stats_sink: Callable[[dict[str, int]], None] | None = None,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> str:
|
||||||
|
"""调用 VLM 查看帧图像:可选 OCR 事前并置 + 提取轮 + 可选验证轮。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
vlm: VLM 图文调用端口。
|
||||||
|
frame_paths: 帧文件路径列表。
|
||||||
|
question: 针对帧内容的视觉问题。
|
||||||
|
prompts_dir: prompt 文件目录。
|
||||||
|
ocr: 帧文字转录端口(None=不注入;返回空串视为无结果不注入)。
|
||||||
|
verify: 是否执行验证轮(False 时仅提取轮,输出无 [验证] 段)。
|
||||||
|
stats_sink: 统计回调(None 不收集);统计严禁写入输出文本。
|
||||||
|
session_id: 遥测会话 ID,透传给 VLM 调用。
|
||||||
|
parent_call_id: 遥测父调用 ID,透传给 VLM 调用。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
verify=True 为 ``"[视觉观察] {证据}\\n[验证] {核实结果}"``,
|
||||||
|
verify=False 为 ``"[视觉观察] {证据}"``,或错误信息。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
OCR 文本作为额外文本并置于问题之前(事前并置——OCR 误读不进
|
||||||
|
工具输出故零 judge 口径风险);OCR 异常降级为不注入并计
|
||||||
|
ocr_failed(ocr 是外部注入依赖,任何异常都不得中断工具主流程,
|
||||||
|
故此处 except Exception 是刻意的降级边界)。sink 键:
|
||||||
|
ocr_injected / ocr_chars / ocr_failed / discrepancy(输出含"分歧"词面)/
|
||||||
|
abstain(含 [证据不存在])。
|
||||||
|
"""
|
||||||
|
stats: dict[str, int] = {
|
||||||
|
"ocr_injected": 0,
|
||||||
|
"ocr_chars": 0,
|
||||||
|
"ocr_failed": 0,
|
||||||
|
"discrepancy": 0,
|
||||||
|
"abstain": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _emit(output: str) -> str:
|
||||||
|
"""计算语义标记并回调 stats_sink。"""
|
||||||
|
stats["abstain"] = int("[证据不存在]" in output)
|
||||||
|
stats["discrepancy"] = int("分歧" in output)
|
||||||
|
if stats_sink is not None:
|
||||||
|
stats_sink(stats)
|
||||||
|
return output
|
||||||
|
|
||||||
|
# -- 帧文件存在性校验 --
|
||||||
|
for p in frame_paths:
|
||||||
|
if not p.exists():
|
||||||
|
return _emit(f"[VL错误] 帧文件不存在: {p}")
|
||||||
|
|
||||||
|
# -- OCR 转录(可选) --
|
||||||
|
ocr_text = ""
|
||||||
|
if ocr is not None:
|
||||||
|
try:
|
||||||
|
ocr_text = await ocr.transcribe_frames(frame_paths)
|
||||||
|
except Exception as e: # noqa: BLE001 — 刻意的降级边界
|
||||||
|
logger.warning("OCR 转录失败,降级不注入: {}", e)
|
||||||
|
stats["ocr_failed"] = 1
|
||||||
|
|
||||||
|
# -- 拼装提取轮 user 消息 --
|
||||||
|
user_parts: list[str] = []
|
||||||
|
if ocr_text:
|
||||||
|
stats["ocr_injected"] = 1
|
||||||
|
stats["ocr_chars"] = len(ocr_text)
|
||||||
|
user_parts.append(_OCR_PREFIX + ocr_text)
|
||||||
|
user_parts.append(question)
|
||||||
|
user_text = "\n".join(user_parts)
|
||||||
|
|
||||||
|
extract_messages = [
|
||||||
|
{"role": "system", "content": _load_prompt(prompts_dir, "observe_frame_extract.md")},
|
||||||
|
{"role": "user", "content": user_text},
|
||||||
|
]
|
||||||
|
|
||||||
|
# -- 提取轮 --
|
||||||
|
try:
|
||||||
|
extract_response = await vlm.chat_with_images(
|
||||||
|
extract_messages,
|
||||||
|
images=frame_paths,
|
||||||
|
session_id=session_id,
|
||||||
|
parent_call_id=parent_call_id,
|
||||||
|
)
|
||||||
|
raw_evidence = extract_response.content
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
return _emit(f"[VL错误] {e}")
|
||||||
|
|
||||||
|
if not verify:
|
||||||
|
return _emit(f"[视觉观察] {raw_evidence}")
|
||||||
|
|
||||||
|
# -- 验证轮 --
|
||||||
|
verify_text = (
|
||||||
|
f"问题: {question}\n\n以下是另一个模型基于这些图片生成的描述,请核实:\n{raw_evidence}"
|
||||||
|
)
|
||||||
|
verify_messages = [
|
||||||
|
{"role": "system", "content": _load_prompt(prompts_dir, "observe_frame_verify.md")},
|
||||||
|
{"role": "user", "content": verify_text},
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
verify_response = await vlm.chat_with_images(
|
||||||
|
verify_messages,
|
||||||
|
images=frame_paths,
|
||||||
|
session_id=session_id,
|
||||||
|
parent_call_id=parent_call_id,
|
||||||
|
)
|
||||||
|
return _emit(f"[视觉观察] {raw_evidence}\n[验证] {verify_response.content}")
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.warning("验证轮调用失败,跳过: {}", e)
|
||||||
|
return _emit(f"[视觉观察] {raw_evidence}\n[验证] 跳过(调用失败)")
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
"""建树模块配置。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class TreeConfig:
|
||||||
|
"""建树配置参数,字段对齐 config/default.yaml 的 tree: 段。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l1_segment_duration: L1 段时长(秒)。
|
||||||
|
l2_clip_duration: L2 clip 时长(秒)。
|
||||||
|
l3_fps: L3 帧提取频率(帧/秒)。
|
||||||
|
l2_representative_frames: L2 VLM 描述用的代表帧数。
|
||||||
|
cache_dir: 树索引缓存目录。
|
||||||
|
concurrency: asyncio Semaphore 上限。
|
||||||
|
subtitle_inject: 建树时是否注入 SRT 字幕。
|
||||||
|
srt_window_sec: 字幕匹配时间窗口(前后各 N 秒)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
l1_segment_duration: float = 600.0
|
||||||
|
l2_clip_duration: float = 60.0
|
||||||
|
l3_fps: float = 0.5
|
||||||
|
l2_representative_frames: int = 6
|
||||||
|
cache_dir: str = "cache/trees"
|
||||||
|
concurrency: int = 16
|
||||||
|
subtitle_inject: bool = True
|
||||||
|
srt_window_sec: float = 5.0
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, d: dict) -> TreeConfig:
|
||||||
|
"""从 YAML 解析后的 dict 构造,忽略未知字段。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
d: 配置字典。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
TreeConfig 实例。
|
||||||
|
"""
|
||||||
|
return cls(**{k: v for k, v in d.items() if k in cls.__dataclass_fields__})
|
||||||
@@ -0,0 +1,541 @@
|
|||||||
|
"""TreeEnvironment:单棵视频树的运行时环境。
|
||||||
|
|
||||||
|
提供节点查询、字幕获取、帧路径解析和语义检索能力。
|
||||||
|
纯数据访问层——不涉及 LLM 调用,LLM 摘要逻辑属于 app/search/。
|
||||||
|
|
||||||
|
算法 #12 变更:分块 embedding → 单节点 embedding。
|
||||||
|
祖先去重 + 锚定验证逻辑保留自 TRM4。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from app.tree.index import L1Node, L2Node, L3Node, TreeIndex
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Callable
|
||||||
|
|
||||||
|
# 节点联合类型(内部使用)
|
||||||
|
AnyNode = L1Node | L2Node | L3Node
|
||||||
|
|
||||||
|
# 各层级节点对应的主描述字段名
|
||||||
|
_LEVEL_LABEL = {
|
||||||
|
"L1": "场景层",
|
||||||
|
"L2": "事件层",
|
||||||
|
"L3": "关键帧层",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _node_level(node: AnyNode) -> str:
|
||||||
|
"""判断节点层级标签。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node: 树节点实例。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
"L1" / "L2" / "L3"。
|
||||||
|
"""
|
||||||
|
if isinstance(node, L1Node):
|
||||||
|
return "L1"
|
||||||
|
if isinstance(node, L2Node):
|
||||||
|
return "L2"
|
||||||
|
return "L3"
|
||||||
|
|
||||||
|
|
||||||
|
def _node_description(node: AnyNode) -> str:
|
||||||
|
"""提取节点的主描述文本。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node: 树节点实例。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
描述文本字符串。
|
||||||
|
"""
|
||||||
|
if isinstance(node, L1Node):
|
||||||
|
return node.card.scene_summary
|
||||||
|
if isinstance(node, L2Node):
|
||||||
|
return node.card.event_description
|
||||||
|
return node.card.frame_summary
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_card_strings(node: AnyNode) -> list[str]:
|
||||||
|
"""从节点 card 中递归收集所有非空字符串字段。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node: 树节点实例。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
字符串列表(每个非空字段值一项,含内嵌换行的按行拆分)。
|
||||||
|
"""
|
||||||
|
result: list[str] = []
|
||||||
|
_collect_from_obj(node.card, result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_from_obj(obj: object, out: list[str]) -> None:
|
||||||
|
"""递归收集任意嵌套结构中的非空字符串。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
obj: dict / list / str / 其他。
|
||||||
|
out: 收集结果列表(原地修改)。
|
||||||
|
"""
|
||||||
|
if isinstance(obj, str):
|
||||||
|
stripped = obj.strip()
|
||||||
|
if stripped:
|
||||||
|
out.append(stripped)
|
||||||
|
elif isinstance(obj, dict):
|
||||||
|
for v in obj.values():
|
||||||
|
_collect_from_obj(v, out)
|
||||||
|
elif isinstance(obj, (list, tuple)):
|
||||||
|
for item in obj:
|
||||||
|
_collect_from_obj(item, out)
|
||||||
|
elif hasattr(obj, "__dataclass_fields__"):
|
||||||
|
# frozen dataclass(Card 类型)
|
||||||
|
for field_name in obj.__dataclass_fields__:
|
||||||
|
_collect_from_obj(getattr(obj, field_name), out)
|
||||||
|
|
||||||
|
|
||||||
|
class TreeEnvironment:
|
||||||
|
"""单棵视频树的运行时环境,提供节点查询和语义检索。
|
||||||
|
|
||||||
|
纯数据访问层,不涉及 LLM 调用。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
index: 已加载的 TreeIndex 实例。
|
||||||
|
frames_dir: 帧文件目录路径(可选;未提供时使用节点自带的 frame_path)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
index: TreeIndex,
|
||||||
|
frames_dir: Path | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._index = index
|
||||||
|
self._frames_dir = frames_dir
|
||||||
|
|
||||||
|
# O(1) 查找表:node_id → 节点实例
|
||||||
|
self._id_to_node: dict[str, AnyNode] = {}
|
||||||
|
# 父节点映射:node_id → parent_id(根节点为 None)
|
||||||
|
self._id_to_parent: dict[str, str | None] = {}
|
||||||
|
|
||||||
|
self._build_lookup_tables()
|
||||||
|
logger.debug(
|
||||||
|
"TreeEnvironment 初始化完成,节点数={}",
|
||||||
|
len(self._id_to_node),
|
||||||
|
)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# 初始化辅助
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _build_lookup_tables(self) -> None:
|
||||||
|
"""遍历 TreeIndex 构建 _id_to_node 和 _id_to_parent 映射表。"""
|
||||||
|
for l1 in self._index.roots:
|
||||||
|
self._id_to_node[l1.id] = l1
|
||||||
|
self._id_to_parent[l1.id] = None
|
||||||
|
for l2 in l1.children:
|
||||||
|
self._id_to_node[l2.id] = l2
|
||||||
|
self._id_to_parent[l2.id] = l1.id
|
||||||
|
for l3 in l2.children:
|
||||||
|
self._id_to_node[l3.id] = l3
|
||||||
|
self._id_to_parent[l3.id] = l2.id
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# 公开方法
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def view_node(self, node_id: str, *, anchor: bool = False) -> str:
|
||||||
|
"""返回节点卡片内容 + 子节点概览。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node_id: 节点 ID。
|
||||||
|
anchor: 为卡片字段添加行锚标 [c1] [s1] 供引用验证。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
格式化文本。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
KeyError: 节点不存在。
|
||||||
|
"""
|
||||||
|
node = self._id_to_node.get(node_id)
|
||||||
|
if node is None:
|
||||||
|
raise KeyError(f"节点不存在: {node_id}")
|
||||||
|
|
||||||
|
level = _node_level(node)
|
||||||
|
level_label = _LEVEL_LABEL[level]
|
||||||
|
|
||||||
|
# 时间范围
|
||||||
|
time_range_str = self._format_time_range(node)
|
||||||
|
|
||||||
|
# 节点内容
|
||||||
|
content = self._node_anchored_text(node) if anchor else self._node_full_text(node)
|
||||||
|
|
||||||
|
parts = [
|
||||||
|
f"[节点] {node_id} | {level_label} | {time_range_str}",
|
||||||
|
"",
|
||||||
|
content,
|
||||||
|
]
|
||||||
|
|
||||||
|
# 子节点概览
|
||||||
|
children = self._get_children(node)
|
||||||
|
if children:
|
||||||
|
parts.append("")
|
||||||
|
parts.append(f"[子节点概览] {len(children)} 个子节点")
|
||||||
|
for child in children:
|
||||||
|
child_desc = _node_description(child)
|
||||||
|
child_time = self._format_time_range(child)
|
||||||
|
# 截断描述到 120 字符
|
||||||
|
if len(child_desc) > 120:
|
||||||
|
child_desc = child_desc[:120] + "..."
|
||||||
|
parts.append(f" - {child.id} | {child_time} | {child_desc}")
|
||||||
|
|
||||||
|
return "\n".join(parts)
|
||||||
|
|
||||||
|
def search_similar(
|
||||||
|
self,
|
||||||
|
query: str,
|
||||||
|
top_k: int = 5,
|
||||||
|
*,
|
||||||
|
embed_fn: Callable[[str | list[str]], np.ndarray] | None = None,
|
||||||
|
) -> list[tuple[str, float]]:
|
||||||
|
"""语义搜索 + 祖先去重。
|
||||||
|
|
||||||
|
算法 #12 变更:单节点 embedding(非分块),祖先去重 + 锚定验证保留。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
query: 搜索文本。
|
||||||
|
top_k: 返回数量。
|
||||||
|
embed_fn: 嵌入函数(未提供时使用 TreeIndex 已有 embedding)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
[(node_id, score), ...] 按相似度降序。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 节点未 embed 且未提供 embed_fn。
|
||||||
|
"""
|
||||||
|
if embed_fn is None:
|
||||||
|
raise ValueError(
|
||||||
|
"embed_fn 为必需参数:搜索 query 需要 embed_fn 来编码。请传入 embed_fn 参数。"
|
||||||
|
)
|
||||||
|
|
||||||
|
# 收集所有节点的 embedding(优先使用 TreeIndex 已有 embedding)
|
||||||
|
node_ids: list[str] = []
|
||||||
|
embeddings: list[np.ndarray] = []
|
||||||
|
|
||||||
|
if self._index.is_embedded:
|
||||||
|
# 使用已有 embedding
|
||||||
|
for nid, node in self._id_to_node.items():
|
||||||
|
if node.embedding is not None:
|
||||||
|
node_ids.append(nid)
|
||||||
|
embeddings.append(node.embedding)
|
||||||
|
else:
|
||||||
|
# 使用 embed_fn 为所有节点生成 embedding
|
||||||
|
all_ids = list(self._id_to_node.keys())
|
||||||
|
all_texts = [_node_description(self._id_to_node[nid]) for nid in all_ids]
|
||||||
|
all_embs = embed_fn(all_texts) # [N, D]
|
||||||
|
for i, nid in enumerate(all_ids):
|
||||||
|
node_ids.append(nid)
|
||||||
|
embeddings.append(all_embs[i])
|
||||||
|
|
||||||
|
if not embeddings:
|
||||||
|
return []
|
||||||
|
|
||||||
|
node_embeddings = np.stack(embeddings, axis=0) # [N, D]
|
||||||
|
# 归一化(确保余弦相似度正确)
|
||||||
|
norms = np.linalg.norm(node_embeddings, axis=1, keepdims=True)
|
||||||
|
norms = np.where(norms == 0, 1.0, norms)
|
||||||
|
node_embeddings = node_embeddings / norms
|
||||||
|
|
||||||
|
# 编码 query
|
||||||
|
query_emb = embed_fn(query) # [1, D]
|
||||||
|
|
||||||
|
if query_emb.ndim == 1:
|
||||||
|
query_emb = query_emb.reshape(1, -1)
|
||||||
|
# 归一化 query
|
||||||
|
q_norm = np.linalg.norm(query_emb)
|
||||||
|
if q_norm > 0:
|
||||||
|
query_emb = query_emb / q_norm
|
||||||
|
|
||||||
|
# 余弦相似度
|
||||||
|
scores = (node_embeddings @ query_emb.T).squeeze() # [N]
|
||||||
|
if scores.ndim == 0:
|
||||||
|
scores = scores.reshape(1)
|
||||||
|
|
||||||
|
# 按分数排序
|
||||||
|
scored_pairs = sorted(
|
||||||
|
zip(node_ids, scores.tolist(), strict=True),
|
||||||
|
key=lambda x: x[1],
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 祖先去重:如果更细粒度的子节点已入选,跳过其祖先
|
||||||
|
deduped: list[tuple[str, float]] = []
|
||||||
|
seen_prefixes: set[str] = set()
|
||||||
|
for nid, score in scored_pairs:
|
||||||
|
is_ancestor_of_seen = any(s.startswith(nid + "_") for s in seen_prefixes)
|
||||||
|
if is_ancestor_of_seen:
|
||||||
|
continue
|
||||||
|
deduped.append((nid, score))
|
||||||
|
seen_prefixes.add(nid)
|
||||||
|
if len(deduped) >= top_k:
|
||||||
|
break
|
||||||
|
|
||||||
|
return deduped
|
||||||
|
|
||||||
|
def get_node_text(
|
||||||
|
self,
|
||||||
|
node_id: str,
|
||||||
|
*,
|
||||||
|
anchor: bool = False,
|
||||||
|
) -> tuple[str, dict[str, str] | None]:
|
||||||
|
"""返回节点原始文本及可选的锚映射表。
|
||||||
|
|
||||||
|
供 SearchToolDispatcher 使用:将原始文本和锚映射传给
|
||||||
|
summarizer.summarize_node(),实现引用验证。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node_id: 节点 ID。
|
||||||
|
anchor: 若 True,返回带 [cN]/[sN] 锚标的文本并构建 anchor_map。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
(text, anchor_map) 元组。anchor=False 时 anchor_map 为 None;
|
||||||
|
anchor=True 时 anchor_map 为 {"c1": "行文本", "s1": "字幕行", ...}。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
KeyError: 节点不存在。
|
||||||
|
"""
|
||||||
|
node = self._id_to_node.get(node_id)
|
||||||
|
if node is None:
|
||||||
|
raise KeyError(f"节点不存在: {node_id}")
|
||||||
|
|
||||||
|
if not anchor:
|
||||||
|
return self._node_full_text(node), None
|
||||||
|
|
||||||
|
anchored_text = self._node_anchored_text(node)
|
||||||
|
# 解析锚标行 "[c1] xxx" / "[s2] yyy" 构建映射
|
||||||
|
anchor_map: dict[str, str] = {}
|
||||||
|
anchor_pattern = re.compile(r"^\[([cs]\d+)\]\s(.+)$")
|
||||||
|
for line in anchored_text.splitlines():
|
||||||
|
m = anchor_pattern.match(line)
|
||||||
|
if m:
|
||||||
|
anchor_map[m.group(1)] = m.group(2)
|
||||||
|
|
||||||
|
return anchored_text, anchor_map
|
||||||
|
|
||||||
|
def get_children_info(self, node_id: str) -> list[dict[str, Any]]:
|
||||||
|
"""返回节点的直接子节点结构化信息。
|
||||||
|
|
||||||
|
供 SearchToolDispatcher 使用:将子节点列表传给
|
||||||
|
summarizer.summarize_children(),用于层级摘要。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node_id: 节点 ID。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
子节点信息列表,每项包含 {"id", "time_range", "summary"}。
|
||||||
|
time_range 为 (start, end) 数值元组(L3 节点退化为 (ts, ts))。
|
||||||
|
L3 叶子节点返回空列表。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
KeyError: 节点不存在。
|
||||||
|
"""
|
||||||
|
node = self._id_to_node.get(node_id)
|
||||||
|
if node is None:
|
||||||
|
raise KeyError(f"节点不存在: {node_id}")
|
||||||
|
|
||||||
|
children = self._get_children(node)
|
||||||
|
result: list[dict[str, Any]] = []
|
||||||
|
for child in children:
|
||||||
|
desc = _node_description(child)
|
||||||
|
if len(desc) > 120:
|
||||||
|
desc = desc[:120] + "..."
|
||||||
|
result.append(
|
||||||
|
{
|
||||||
|
"id": child.id,
|
||||||
|
"time_range": self._node_time_range_raw(child),
|
||||||
|
"summary": desc,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def get_subtitle(self, node_id: str) -> str:
|
||||||
|
"""返回节点字幕文本。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node_id: 节点 ID。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
字幕文本;无字幕或节点不存在时返回空字符串。
|
||||||
|
"""
|
||||||
|
node = self._id_to_node.get(node_id)
|
||||||
|
if node is None:
|
||||||
|
return ""
|
||||||
|
if isinstance(node, L3Node):
|
||||||
|
return node.subtitle or ""
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def resolve_frame_paths(self, node_ids: list[str]) -> list[Path]:
|
||||||
|
"""node_id → 帧文件路径。支持 L3(直接映射)和 L2(展开为 L3 children)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node_ids: 节点 ID 列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
帧文件 Path 列表。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
KeyError: 节点不存在。
|
||||||
|
"""
|
||||||
|
if not node_ids:
|
||||||
|
return []
|
||||||
|
|
||||||
|
paths: list[Path] = []
|
||||||
|
for nid in node_ids:
|
||||||
|
node = self._id_to_node.get(nid)
|
||||||
|
if node is None:
|
||||||
|
raise KeyError(f"节点不存在: {nid}")
|
||||||
|
|
||||||
|
if isinstance(node, L3Node):
|
||||||
|
paths.append(self._l3_frame_path(node))
|
||||||
|
elif isinstance(node, L2Node):
|
||||||
|
# 展开为所有 L3 子节点
|
||||||
|
for l3 in node.children:
|
||||||
|
paths.append(self._l3_frame_path(l3))
|
||||||
|
else:
|
||||||
|
# L1 节点:展开为所有 L2 下的 L3
|
||||||
|
assert isinstance(node, L1Node)
|
||||||
|
for l2 in node.children:
|
||||||
|
for l3 in l2.children:
|
||||||
|
paths.append(self._l3_frame_path(l3))
|
||||||
|
|
||||||
|
return paths
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# 内部辅助方法
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _l3_frame_path(self, node: L3Node) -> Path:
|
||||||
|
"""将 L3 节点映射到帧文件路径。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node: L3 节点。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
帧文件 Path。
|
||||||
|
"""
|
||||||
|
if self._frames_dir is not None:
|
||||||
|
# 从 node.id 中提取后缀(去掉 video_id 前缀)
|
||||||
|
# ID 格式: {video_id}_{L1_xxx_L2_xxx_L3_xxx}
|
||||||
|
# frame_path 格式: frames/{L1_xxx_L2_xxx_L3_xxx}.jpg
|
||||||
|
if node.frame_path:
|
||||||
|
return self._frames_dir / Path(node.frame_path).name
|
||||||
|
# fallback: 从 ID 推断
|
||||||
|
parts = node.id.split("_", 1)
|
||||||
|
suffix = parts[1] if len(parts) > 1 else node.id
|
||||||
|
return self._frames_dir / f"{suffix}.jpg"
|
||||||
|
|
||||||
|
# 无 frames_dir 时使用节点自带路径
|
||||||
|
if node.frame_path:
|
||||||
|
return Path(node.frame_path)
|
||||||
|
raise ValueError(f"L3 节点无 frame_path 且未提供 frames_dir: {node.id}")
|
||||||
|
|
||||||
|
def _node_full_text(self, node: AnyNode) -> str:
|
||||||
|
"""获取节点完整文本(card 所有字段 + subtitle)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node: 树节点。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
拼接后的全文本。
|
||||||
|
"""
|
||||||
|
card_strings = _collect_card_strings(node)
|
||||||
|
text = "\n".join(card_strings)
|
||||||
|
if isinstance(node, L3Node) and node.subtitle:
|
||||||
|
text += f"\n字幕: {node.subtitle}"
|
||||||
|
return text
|
||||||
|
|
||||||
|
def _node_anchored_text(self, node: AnyNode) -> str:
|
||||||
|
"""获取带行号锚的节点文本。
|
||||||
|
|
||||||
|
card 字符串逐行编 [c1]..[cN],字幕逐行编 [s1]..[sM]。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node: 树节点。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
带锚文本。
|
||||||
|
"""
|
||||||
|
card_strings = _collect_card_strings(node)
|
||||||
|
# 拆分内嵌换行,确保一锚一行
|
||||||
|
card_lines: list[str] = []
|
||||||
|
for s in card_strings:
|
||||||
|
card_lines.extend(ln for ln in s.splitlines() if ln.strip())
|
||||||
|
|
||||||
|
sub_lines: list[str] = []
|
||||||
|
if isinstance(node, L3Node) and node.subtitle:
|
||||||
|
sub_lines = [ln for ln in node.subtitle.splitlines() if ln.strip()]
|
||||||
|
|
||||||
|
anchored: list[str] = []
|
||||||
|
for i, line in enumerate(card_lines, 1):
|
||||||
|
anchored.append(f"[c{i}] {line}")
|
||||||
|
for i, line in enumerate(sub_lines, 1):
|
||||||
|
anchored.append(f"[s{i}] {line}")
|
||||||
|
|
||||||
|
return "\n".join(anchored)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _format_time_range(node: AnyNode) -> str:
|
||||||
|
"""格式化节点的时间范围。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node: 树节点。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
"start-end s" 格式字符串,或 timestamp,或 "N/A"。
|
||||||
|
"""
|
||||||
|
if isinstance(node, (L1Node, L2Node)) and node.time_range:
|
||||||
|
return f"{node.time_range[0]:.1f}-{node.time_range[1]:.1f}s"
|
||||||
|
if isinstance(node, L3Node) and node.timestamp is not None:
|
||||||
|
return f"{node.timestamp:.1f}s"
|
||||||
|
return "N/A"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _node_time_range_raw(node: AnyNode) -> tuple[float, float]:
|
||||||
|
"""提取节点时间范围的原始数值元组。
|
||||||
|
|
||||||
|
L1/L2 返回 time_range 元组;L3 退化为 (timestamp, timestamp);
|
||||||
|
全部为 None 时兜底 (0.0, 0.0)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node: 树节点。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
(start, end) 秒级数值元组。
|
||||||
|
"""
|
||||||
|
if isinstance(node, (L1Node, L2Node)) and node.time_range:
|
||||||
|
return node.time_range
|
||||||
|
if isinstance(node, L3Node) and node.timestamp is not None:
|
||||||
|
return (node.timestamp, node.timestamp)
|
||||||
|
return (0.0, 0.0)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _get_children(node: AnyNode) -> list[AnyNode]:
|
||||||
|
"""获取节点的直接子节点列表。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node: 树节点。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
子节点列表(L3 节点返回空列表)。
|
||||||
|
"""
|
||||||
|
if isinstance(node, L1Node):
|
||||||
|
return list(node.children)
|
||||||
|
if isinstance(node, L2Node):
|
||||||
|
return list(node.children)
|
||||||
|
return []
|
||||||
@@ -0,0 +1,762 @@
|
|||||||
|
"""三层树索引核心数据结构。
|
||||||
|
|
||||||
|
定义 Video-Tree-TRM 的三层树状索引结构,是所有后续模块
|
||||||
|
(builder、retriever、harness、search)的基础依赖。
|
||||||
|
|
||||||
|
数据结构层次::
|
||||||
|
|
||||||
|
TreeIndex
|
||||||
|
└─ List[L1Node] 全局叙事节点
|
||||||
|
└─ List[L2Node] 片段级语义节点
|
||||||
|
└─ List[L3Node] 帧/细节级节点
|
||||||
|
|
||||||
|
与参考项目 (TRM4) 的关键区别:
|
||||||
|
- Card 体系:每层节点的描述信息封装为 frozen dataclass(L1Card/L2Card/L3Card),
|
||||||
|
字段来自 VLM 结构化输出,保证不可变。
|
||||||
|
- 序列化方式:仅保留 JSON(移除 pickle)。
|
||||||
|
- 统一嵌入空间:所有 embedding 均来自 text_embed(),无跨模态问题。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Callable
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Embedding 序列化辅助函数
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _embed_to_str(arr: np.ndarray | None) -> str | None:
|
||||||
|
"""float32 ndarray -> base64 字符串(用于 JSON 序列化)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
arr: float32 数组,形状任意。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
base64 编码字符串,或 None(输入为 None 时)。
|
||||||
|
"""
|
||||||
|
if arr is None:
|
||||||
|
return None
|
||||||
|
return base64.b64encode(arr.astype(np.float32).tobytes()).decode()
|
||||||
|
|
||||||
|
|
||||||
|
def _embed_from_str(s: str | None) -> np.ndarray | None:
|
||||||
|
"""base64 字符串 -> float32 ndarray(用于 JSON 反序列化)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
s: base64 编码字符串。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
float32 数组,或 None(输入为 None/空时)。
|
||||||
|
"""
|
||||||
|
if s is None or s == "":
|
||||||
|
return None
|
||||||
|
return np.frombuffer(base64.b64decode(s), dtype=np.float32)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Card 数据结构(frozen,来自 VLM 结构化输出)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class L3Card:
|
||||||
|
"""L3 帧级语义卡片(不可变)。
|
||||||
|
|
||||||
|
封装 VLM 对单帧的结构化描述输出。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
frame_summary: 帧内容摘要。
|
||||||
|
visible_entities: 可见实体列表。
|
||||||
|
ongoing_actions: 正在进行的动作列表。
|
||||||
|
visible_text: 画面中可见的文字列表。
|
||||||
|
spatial_layout: 空间布局描述。
|
||||||
|
visual_attributes: 视觉属性字典(如光照、色调等)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
frame_summary: str
|
||||||
|
visible_entities: list[str]
|
||||||
|
ongoing_actions: list[str]
|
||||||
|
visible_text: list[str]
|
||||||
|
spatial_layout: str
|
||||||
|
visual_attributes: dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class L2Card:
|
||||||
|
"""L2 事件级语义卡片(不可变)。
|
||||||
|
|
||||||
|
封装 VLM 对一个事件片段的结构化描述输出。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
event_description: 事件描述。
|
||||||
|
entities: 参与实体列表。
|
||||||
|
actions: 动作列表。
|
||||||
|
action_subjects: 动作主体列表。
|
||||||
|
visible_text: 片段中可见的文字列表。
|
||||||
|
spatial_relations: 空间关系描述。
|
||||||
|
state_changes: 状态变化描述(可选)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
event_description: str
|
||||||
|
entities: list[str]
|
||||||
|
actions: list[str]
|
||||||
|
action_subjects: list[str]
|
||||||
|
visible_text: list[str]
|
||||||
|
spatial_relations: str
|
||||||
|
state_changes: str | None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class L1Card:
|
||||||
|
"""L1 场景级语义卡片(不可变)。
|
||||||
|
|
||||||
|
封装 VLM 对一个完整场景的结构化描述输出。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
scene_summary: 场景摘要。
|
||||||
|
main_setting: 主要场景设定(如"室内"、"户外"等)。
|
||||||
|
key_entities: 关键实体列表。
|
||||||
|
main_actions: 主要动作列表。
|
||||||
|
topic_keywords: 主题关键词列表。
|
||||||
|
visible_text: 场景中可见的文字列表。
|
||||||
|
temporal_flow: 时间流描述。
|
||||||
|
"""
|
||||||
|
|
||||||
|
scene_summary: str
|
||||||
|
main_setting: str
|
||||||
|
key_entities: list[str]
|
||||||
|
main_actions: list[str]
|
||||||
|
topic_keywords: list[str]
|
||||||
|
visible_text: list[str]
|
||||||
|
temporal_flow: str
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 元数据
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class IndexMeta:
|
||||||
|
"""树索引元数据。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
source_path: 原始数据路径(视频文件或文本文件)。
|
||||||
|
modality: 数据模态,"text" 或 "video"。
|
||||||
|
embed_model: 嵌入模型名称(建树时为 None,embed_all 后填充)。
|
||||||
|
embed_dim: 嵌入向量维度(建树时为 None,embed_all 后填充)。
|
||||||
|
created_at: 创建时间(ISO 格式字符串)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
source_path: str
|
||||||
|
modality: str
|
||||||
|
embed_model: str | None = None
|
||||||
|
embed_dim: int | None = None
|
||||||
|
created_at: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 节点数据结构
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class L3Node:
|
||||||
|
"""L3 帧/细节级节点(叶子层)。
|
||||||
|
|
||||||
|
代表最细粒度的语义单元,对应一个具体的帧描述。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
id: 节点唯一标识。
|
||||||
|
card: 帧级语义卡片(VLM 结构化输出)。
|
||||||
|
embedding: 文本嵌入向量,形状 [D],float32。
|
||||||
|
timestamp: 对应的时间戳(秒,可选)。
|
||||||
|
frame_path: 关联的帧图像路径(可选,仅视频模态)。
|
||||||
|
subtitle: 该帧对应的字幕文本(可选)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
id: str
|
||||||
|
card: L3Card
|
||||||
|
embedding: np.ndarray | None = None
|
||||||
|
timestamp: float | None = None
|
||||||
|
frame_path: str | None = None
|
||||||
|
subtitle: str | None = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def description(self) -> str:
|
||||||
|
"""帧描述文本(取自 card.frame_summary)。"""
|
||||||
|
return self.card.frame_summary
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class L2Node:
|
||||||
|
"""L2 片段级语义节点(中间层)。
|
||||||
|
|
||||||
|
连接 L1 宏观叙事与 L3 细节描述。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
id: 节点唯一标识。
|
||||||
|
card: 事件级语义卡片(VLM 结构化输出)。
|
||||||
|
embedding: 文本嵌入向量,形状 [D],float32。
|
||||||
|
time_range: 时间范围 (start, end)(秒,可选)。
|
||||||
|
children: 所属的 L3 子节点列表。
|
||||||
|
"""
|
||||||
|
|
||||||
|
id: str
|
||||||
|
card: L2Card
|
||||||
|
embedding: np.ndarray | None = None
|
||||||
|
time_range: tuple[float, float] | None = None
|
||||||
|
children: list[L3Node] = field(default_factory=list)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def description(self) -> str:
|
||||||
|
"""事件描述文本(取自 card.event_description)。"""
|
||||||
|
return self.card.event_description
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class L1Node:
|
||||||
|
"""L1 全局叙事节点(根层)。
|
||||||
|
|
||||||
|
代表最粗粒度的语义单元,包含宏观场景摘要。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
id: 节点唯一标识。
|
||||||
|
card: 场景级语义卡片(VLM 结构化输出)。
|
||||||
|
embedding: 文本嵌入向量,形状 [D],float32。
|
||||||
|
time_range: 时间范围 (start, end)(秒,可选)。
|
||||||
|
children: 所属的 L2 子节点列表。
|
||||||
|
"""
|
||||||
|
|
||||||
|
id: str
|
||||||
|
card: L1Card
|
||||||
|
embedding: np.ndarray | None = None
|
||||||
|
time_range: tuple[float, float] | None = None
|
||||||
|
children: list[L2Node] = field(default_factory=list)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def summary(self) -> str:
|
||||||
|
"""场景摘要文本(取自 card.scene_summary)。"""
|
||||||
|
return self.card.scene_summary
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# JSON 辅助方法(单个 L1 段的轻量序列化)
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def to_dict(self, include_embedding: bool = False) -> dict[str, Any]:
|
||||||
|
"""将当前 L1 节点(及其全部 L2/L3 子树)序列化为纯 dict。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
include_embedding: 若 True,将 embedding 向量序列化为 base64 字符串。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
包含 id/card/time_range/children 的字典,可选包含 embedding。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def l3_to_dict(n: L3Node) -> dict[str, Any]:
|
||||||
|
d: dict[str, Any] = {
|
||||||
|
"id": n.id,
|
||||||
|
"card": {
|
||||||
|
"frame_summary": n.card.frame_summary,
|
||||||
|
"visible_entities": n.card.visible_entities,
|
||||||
|
"ongoing_actions": n.card.ongoing_actions,
|
||||||
|
"visible_text": n.card.visible_text,
|
||||||
|
"spatial_layout": n.card.spatial_layout,
|
||||||
|
"visual_attributes": n.card.visual_attributes,
|
||||||
|
},
|
||||||
|
"timestamp": n.timestamp,
|
||||||
|
"frame_path": n.frame_path,
|
||||||
|
"subtitle": n.subtitle,
|
||||||
|
}
|
||||||
|
if include_embedding:
|
||||||
|
d["embedding"] = _embed_to_str(n.embedding)
|
||||||
|
return d
|
||||||
|
|
||||||
|
def l2_to_dict(n: L2Node) -> dict[str, Any]:
|
||||||
|
d: dict[str, Any] = {
|
||||||
|
"id": n.id,
|
||||||
|
"card": {
|
||||||
|
"event_description": n.card.event_description,
|
||||||
|
"entities": n.card.entities,
|
||||||
|
"actions": n.card.actions,
|
||||||
|
"action_subjects": n.card.action_subjects,
|
||||||
|
"visible_text": n.card.visible_text,
|
||||||
|
"spatial_relations": n.card.spatial_relations,
|
||||||
|
"state_changes": n.card.state_changes,
|
||||||
|
},
|
||||||
|
"time_range": list(n.time_range) if n.time_range else None,
|
||||||
|
"children": [l3_to_dict(c) for c in n.children],
|
||||||
|
}
|
||||||
|
if include_embedding:
|
||||||
|
d["embedding"] = _embed_to_str(n.embedding)
|
||||||
|
return d
|
||||||
|
|
||||||
|
d: dict[str, Any] = {
|
||||||
|
"id": self.id,
|
||||||
|
"card": {
|
||||||
|
"scene_summary": self.card.scene_summary,
|
||||||
|
"main_setting": self.card.main_setting,
|
||||||
|
"key_entities": self.card.key_entities,
|
||||||
|
"main_actions": self.card.main_actions,
|
||||||
|
"topic_keywords": self.card.topic_keywords,
|
||||||
|
"visible_text": self.card.visible_text,
|
||||||
|
"temporal_flow": self.card.temporal_flow,
|
||||||
|
},
|
||||||
|
"time_range": list(self.time_range) if self.time_range else None,
|
||||||
|
"children": [l2_to_dict(c) for c in self.children],
|
||||||
|
}
|
||||||
|
if include_embedding:
|
||||||
|
d["embedding"] = _embed_to_str(self.embedding)
|
||||||
|
return d
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def from_dict(d: dict[str, Any]) -> L1Node:
|
||||||
|
"""从 dict 反序列化单个 L1 节点(支持 embedding 恢复)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
d: to_dict() 输出的字典,可包含 embedding 字段。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
L1Node 实例(embedding 自动从 base64 恢复,若无则为 None)。
|
||||||
|
"""
|
||||||
|
l2_nodes: list[L2Node] = []
|
||||||
|
for l2d in d.get("children", []):
|
||||||
|
l3_nodes: list[L3Node] = []
|
||||||
|
for l3d in l2d.get("children", []):
|
||||||
|
l3_card = L3Card(
|
||||||
|
frame_summary=l3d["card"]["frame_summary"],
|
||||||
|
visible_entities=l3d["card"]["visible_entities"],
|
||||||
|
ongoing_actions=l3d["card"]["ongoing_actions"],
|
||||||
|
visible_text=l3d["card"]["visible_text"],
|
||||||
|
spatial_layout=l3d["card"]["spatial_layout"],
|
||||||
|
visual_attributes=l3d["card"]["visual_attributes"],
|
||||||
|
)
|
||||||
|
l3_nodes.append(
|
||||||
|
L3Node(
|
||||||
|
id=l3d["id"],
|
||||||
|
card=l3_card,
|
||||||
|
embedding=_embed_from_str(l3d.get("embedding")),
|
||||||
|
timestamp=l3d.get("timestamp"),
|
||||||
|
frame_path=l3d.get("frame_path"),
|
||||||
|
subtitle=l3d.get("subtitle"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
l2_card = L2Card(
|
||||||
|
event_description=l2d["card"]["event_description"],
|
||||||
|
entities=l2d["card"]["entities"],
|
||||||
|
actions=l2d["card"]["actions"],
|
||||||
|
action_subjects=l2d["card"]["action_subjects"],
|
||||||
|
visible_text=l2d["card"]["visible_text"],
|
||||||
|
spatial_relations=l2d["card"]["spatial_relations"],
|
||||||
|
state_changes=l2d["card"]["state_changes"],
|
||||||
|
)
|
||||||
|
tr2 = l2d.get("time_range")
|
||||||
|
l2_nodes.append(
|
||||||
|
L2Node(
|
||||||
|
id=l2d["id"],
|
||||||
|
card=l2_card,
|
||||||
|
embedding=_embed_from_str(l2d.get("embedding")),
|
||||||
|
time_range=tuple(tr2) if tr2 else None,
|
||||||
|
children=l3_nodes,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
l1_card = L1Card(
|
||||||
|
scene_summary=d["card"]["scene_summary"],
|
||||||
|
main_setting=d["card"]["main_setting"],
|
||||||
|
key_entities=d["card"]["key_entities"],
|
||||||
|
main_actions=d["card"]["main_actions"],
|
||||||
|
topic_keywords=d["card"]["topic_keywords"],
|
||||||
|
visible_text=d["card"]["visible_text"],
|
||||||
|
temporal_flow=d["card"]["temporal_flow"],
|
||||||
|
)
|
||||||
|
tr1 = d.get("time_range")
|
||||||
|
return L1Node(
|
||||||
|
id=d["id"],
|
||||||
|
card=l1_card,
|
||||||
|
embedding=_embed_from_str(d.get("embedding")),
|
||||||
|
time_range=tuple(tr1) if tr1 else None,
|
||||||
|
children=l2_nodes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 树索引容器
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class TreeIndex:
|
||||||
|
"""三层树索引容器。
|
||||||
|
|
||||||
|
组织和管理三层节点结构,提供嵌入矩阵提取、节点访问、
|
||||||
|
以及 JSON 序列化/反序列化接口。
|
||||||
|
|
||||||
|
典型工作流::
|
||||||
|
|
||||||
|
# 1. 构建索引
|
||||||
|
index = TreeIndex(metadata=meta, roots=[l1_node_1, l1_node_2])
|
||||||
|
|
||||||
|
# 2. 批量 embed(首次检索前)
|
||||||
|
index.embed_all(embed_fn, "model-name", 768)
|
||||||
|
|
||||||
|
# 3. 提取嵌入矩阵(用于检索)
|
||||||
|
M_L1 = index.l1_embeddings()
|
||||||
|
M_L2 = index.l2_embeddings_of(l1_idx=0)
|
||||||
|
M_L3 = index.l3_embeddings_of(0, 1)
|
||||||
|
|
||||||
|
# 4. 序列化
|
||||||
|
index.save_json("cache/my_index.json")
|
||||||
|
loaded = TreeIndex.load_json("cache/my_index.json")
|
||||||
|
|
||||||
|
属性:
|
||||||
|
metadata: 索引元数据。
|
||||||
|
roots: L1 节点列表。
|
||||||
|
"""
|
||||||
|
|
||||||
|
metadata: IndexMeta
|
||||||
|
roots: list[L1Node] = field(default_factory=list)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ #
|
||||||
|
# 嵌入状态检查
|
||||||
|
# ------------------------------------------------------------------ #
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_embedded(self) -> bool:
|
||||||
|
"""检查所有节点是否已填充嵌入向量。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
True 表示所有 L1/L2/L3 节点的 embedding 均非 None;
|
||||||
|
False 表示尚未 embed。
|
||||||
|
"""
|
||||||
|
for l1 in self.roots:
|
||||||
|
if l1.embedding is None:
|
||||||
|
return False
|
||||||
|
for l2 in l1.children:
|
||||||
|
if l2.embedding is None:
|
||||||
|
return False
|
||||||
|
for l3 in l2.children:
|
||||||
|
if l3.embedding is None:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ #
|
||||||
|
# 批量嵌入
|
||||||
|
# ------------------------------------------------------------------ #
|
||||||
|
|
||||||
|
def embed_all(
|
||||||
|
self,
|
||||||
|
embed_fn: Callable[[str | list[str]], np.ndarray],
|
||||||
|
model_name: str,
|
||||||
|
embed_dim: int,
|
||||||
|
) -> None:
|
||||||
|
"""对所有节点批量执行 embedding,更新 metadata。
|
||||||
|
|
||||||
|
建树阶段不调用此方法(embedding=None)。
|
||||||
|
首次检索前由 Pipeline 调用,结果缓存在节点上。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
embed_fn: EmbeddingModel.embed 方法,接受 str 或 List[str],
|
||||||
|
返回 [N, D] ndarray。
|
||||||
|
model_name: 嵌入模型名称,写入 metadata。
|
||||||
|
embed_dim: 嵌入维度,写入 metadata。
|
||||||
|
|
||||||
|
实现细节:
|
||||||
|
- L3 节点按 L2 分组批量 embed(一次调用),减少 API 开销。
|
||||||
|
- L1/L2 各单独 embed(数量少,不值得合并)。
|
||||||
|
- 仅对 embedding 为 None 的节点执行(支持增量更新)。
|
||||||
|
"""
|
||||||
|
assert len(self.roots) > 0, "embed_all: 树为空,无节点可 embed"
|
||||||
|
for l1 in self.roots:
|
||||||
|
if l1.embedding is None:
|
||||||
|
l1.embedding = embed_fn(l1.summary)[0].astype(np.float32)
|
||||||
|
for l2 in l1.children:
|
||||||
|
self._embed_l2_subtree(l2, embed_fn)
|
||||||
|
self.metadata.embed_model = model_name
|
||||||
|
self.metadata.embed_dim = embed_dim
|
||||||
|
logger.info(
|
||||||
|
"embed_all 完成",
|
||||||
|
model=model_name,
|
||||||
|
embed_dim=embed_dim,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _embed_l2_subtree(
|
||||||
|
self,
|
||||||
|
l2: L2Node,
|
||||||
|
embed_fn: Callable[[str | list[str]], np.ndarray],
|
||||||
|
) -> None:
|
||||||
|
"""对单个 L2 节点及其 L3 子节点执行 embedding(仅处理 embedding 为 None 的节点)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l2: 待 embed 的 L2 节点。
|
||||||
|
embed_fn: EmbeddingModel.embed 方法,接受 str 或 List[str],
|
||||||
|
返回 [N, D] ndarray。
|
||||||
|
"""
|
||||||
|
if l2.embedding is None:
|
||||||
|
l2.embedding = embed_fn(l2.description)[0].astype(np.float32)
|
||||||
|
# L3 批量 embed
|
||||||
|
need_embed = [l3 for l3 in l2.children if l3.embedding is None]
|
||||||
|
if need_embed:
|
||||||
|
texts = [l3.description for l3 in need_embed]
|
||||||
|
embs = embed_fn(texts).astype(np.float32) # [N, D]
|
||||||
|
for l3, emb in zip(need_embed, embs, strict=True):
|
||||||
|
l3.embedding = emb
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ #
|
||||||
|
# 嵌入矩阵提取
|
||||||
|
# ------------------------------------------------------------------ #
|
||||||
|
|
||||||
|
def l1_embeddings(self) -> np.ndarray:
|
||||||
|
"""返回所有 L1 节点的嵌入矩阵。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
形状 [N1, D] 的 float32 矩阵。空树返回 [0, D]。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
AssertionError: 节点 embedding 尚未计算(请先调用 embed_all)。
|
||||||
|
"""
|
||||||
|
assert self.is_embedded, "L1 embedding 尚未计算,请先调用 tree.embed_all()"
|
||||||
|
if not self.roots:
|
||||||
|
return np.zeros((0, self.metadata.embed_dim), dtype=np.float32)
|
||||||
|
return np.stack([r.embedding for r in self.roots], axis=0).astype(np.float32)
|
||||||
|
|
||||||
|
def l2_embeddings_of(self, l1_idx: int) -> np.ndarray:
|
||||||
|
"""返回指定 L1 节点下所有 L2 子节点的嵌入矩阵。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l1_idx: L1 节点索引。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
形状 [N2, D] 的 float32 矩阵。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
IndexError: l1_idx 越界。
|
||||||
|
AssertionError: embedding 尚未计算。
|
||||||
|
"""
|
||||||
|
assert self.is_embedded, "L2 embedding 尚未计算,请先调用 tree.embed_all()"
|
||||||
|
if not (0 <= l1_idx < len(self.roots)):
|
||||||
|
raise IndexError(f"l1_idx={l1_idx} 越界,L1 节点数={len(self.roots)}")
|
||||||
|
children = self.roots[l1_idx].children
|
||||||
|
if not children:
|
||||||
|
return np.zeros((0, self.metadata.embed_dim), dtype=np.float32)
|
||||||
|
return np.stack([c.embedding for c in children], axis=0).astype(np.float32)
|
||||||
|
|
||||||
|
def l3_embeddings_of(self, l1_idx: int, l2_idx: int) -> np.ndarray:
|
||||||
|
"""返回指定 L2 节点下所有 L3 子节点的嵌入矩阵。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l1_idx: L1 节点索引。
|
||||||
|
l2_idx: L2 节点索引(相对于 L1)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
形状 [N3, D] 的 float32 矩阵。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
IndexError: 索引越界。
|
||||||
|
AssertionError: embedding 尚未计算。
|
||||||
|
"""
|
||||||
|
assert self.is_embedded, "L3 embedding 尚未计算,请先调用 tree.embed_all()"
|
||||||
|
if not (0 <= l1_idx < len(self.roots)):
|
||||||
|
raise IndexError(f"l1_idx={l1_idx} 越界,L1 节点数={len(self.roots)}")
|
||||||
|
l2_children = self.roots[l1_idx].children
|
||||||
|
if not (0 <= l2_idx < len(l2_children)):
|
||||||
|
raise IndexError(f"l2_idx={l2_idx} 越界,L2 节点数={len(l2_children)}")
|
||||||
|
l3_children = l2_children[l2_idx].children
|
||||||
|
if not l3_children:
|
||||||
|
return np.zeros((0, self.metadata.embed_dim), dtype=np.float32)
|
||||||
|
return np.stack([c.embedding for c in l3_children], axis=0).astype(np.float32)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ #
|
||||||
|
# 节点访问
|
||||||
|
# ------------------------------------------------------------------ #
|
||||||
|
|
||||||
|
def get_node(self, l1: int, l2: int, l3: int) -> L3Node:
|
||||||
|
"""按三级路径索引获取 L3 节点。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l1: L1 节点索引。
|
||||||
|
l2: L2 节点索引。
|
||||||
|
l3: L3 节点索引。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
目标 L3Node。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
IndexError: 任意层级索引越界。
|
||||||
|
"""
|
||||||
|
if l1 < 0 or l1 >= len(self.roots):
|
||||||
|
raise IndexError(f"l1={l1} 越界,L1 节点数={len(self.roots)}")
|
||||||
|
l2_children = self.roots[l1].children
|
||||||
|
if l2 < 0 or l2 >= len(l2_children):
|
||||||
|
raise IndexError(f"l2={l2} 越界,L2 节点数={len(l2_children)}")
|
||||||
|
l3_children = l2_children[l2].children
|
||||||
|
if l3 < 0 or l3 >= len(l3_children):
|
||||||
|
raise IndexError(f"l3={l3} 越界,L3 节点数={len(l3_children)}")
|
||||||
|
return l3_children[l3]
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ #
|
||||||
|
# JSON 序列化
|
||||||
|
# ------------------------------------------------------------------ #
|
||||||
|
|
||||||
|
def to_dict(self, include_embedding: bool = False) -> dict[str, Any]:
|
||||||
|
"""将树索引序列化为纯 Python dict。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
include_embedding: 若 True,将所有节点的 embedding 向量序列化为 base64。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
可直接 json.dump 的字典,结构为 {metadata, roots[...]}。
|
||||||
|
"""
|
||||||
|
metadata_dict: dict[str, Any] = {
|
||||||
|
"source_path": self.metadata.source_path,
|
||||||
|
"modality": self.metadata.modality,
|
||||||
|
"created_at": self.metadata.created_at,
|
||||||
|
}
|
||||||
|
if include_embedding:
|
||||||
|
metadata_dict["embed_model"] = self.metadata.embed_model
|
||||||
|
metadata_dict["embed_dim"] = self.metadata.embed_dim
|
||||||
|
|
||||||
|
return {
|
||||||
|
"metadata": metadata_dict,
|
||||||
|
"roots": [r.to_dict(include_embedding=include_embedding) for r in self.roots],
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_dict(cls, d: dict[str, Any]) -> TreeIndex:
|
||||||
|
"""从 dict 反序列化为 TreeIndex(支持 embedding 恢复)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
d: to_dict() 的输出或等价结构,可包含 embedding 字段。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
TreeIndex 实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 存在重复的节点 ID。
|
||||||
|
"""
|
||||||
|
meta = IndexMeta(
|
||||||
|
source_path=d["metadata"]["source_path"],
|
||||||
|
modality=d["metadata"]["modality"],
|
||||||
|
embed_model=d["metadata"].get("embed_model"),
|
||||||
|
embed_dim=d["metadata"].get("embed_dim"),
|
||||||
|
created_at=d["metadata"].get("created_at", datetime.now().isoformat()),
|
||||||
|
)
|
||||||
|
|
||||||
|
roots: list[L1Node] = []
|
||||||
|
for r in d["roots"]:
|
||||||
|
roots.append(L1Node.from_dict(r))
|
||||||
|
|
||||||
|
obj = cls(metadata=meta, roots=roots)
|
||||||
|
obj._validate_id_uniqueness()
|
||||||
|
return obj
|
||||||
|
|
||||||
|
def _validate_id_uniqueness(self) -> None:
|
||||||
|
"""校验树中所有节点 ID 的唯一性。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 存在重复的节点 ID。
|
||||||
|
"""
|
||||||
|
seen: set[str] = set()
|
||||||
|
for l1 in self.roots:
|
||||||
|
if l1.id in seen:
|
||||||
|
raise ValueError(f"重复的节点 ID: {l1.id}")
|
||||||
|
seen.add(l1.id)
|
||||||
|
for l2 in l1.children:
|
||||||
|
if l2.id in seen:
|
||||||
|
raise ValueError(f"重复的节点 ID: {l2.id}")
|
||||||
|
seen.add(l2.id)
|
||||||
|
for l3 in l2.children:
|
||||||
|
if l3.id in seen:
|
||||||
|
raise ValueError(f"重复的节点 ID: {l3.id}")
|
||||||
|
seen.add(l3.id)
|
||||||
|
|
||||||
|
def save_json(self, path: str, include_embedding: bool = False) -> None:
|
||||||
|
"""将树索引以 JSON 格式保存到磁盘。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
path: 保存文件路径(推荐 .json 后缀)。
|
||||||
|
include_embedding: 若 True,将所有节点的 embedding 向量保存到 JSON。
|
||||||
|
"""
|
||||||
|
with open(path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(
|
||||||
|
self.to_dict(include_embedding=include_embedding),
|
||||||
|
f,
|
||||||
|
ensure_ascii=False,
|
||||||
|
indent=2,
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"树索引(JSON)已保存至 {}",
|
||||||
|
path,
|
||||||
|
n_l1=len(self.roots),
|
||||||
|
include_embedding=include_embedding,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def load_json(cls, path: str) -> TreeIndex:
|
||||||
|
"""从 JSON 文件加载树索引(自动检测并恢复 embedding)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
path: JSON 文件路径。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
TreeIndex 实例。若 JSON 中包含 embedding 字段,自动反序列化填充;
|
||||||
|
否则 embedding=None(向后兼容旧格式)。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileNotFoundError: 文件不存在。
|
||||||
|
ValueError: 存在重复的节点 ID。
|
||||||
|
"""
|
||||||
|
with open(path, encoding="utf-8") as f:
|
||||||
|
d = json.load(f)
|
||||||
|
obj = cls.from_dict(d)
|
||||||
|
obj._validate_id_uniqueness()
|
||||||
|
logger.info(
|
||||||
|
"树索引(JSON)已从 {} 加载",
|
||||||
|
path,
|
||||||
|
n_l1=len(obj.roots),
|
||||||
|
is_embedded=obj.is_embedded,
|
||||||
|
)
|
||||||
|
return obj
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 单 L1 段的轻量序列化(用于断点续跑)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def save_l1_json(path: str, l1_node: L1Node) -> None:
|
||||||
|
"""将单个 L1 节点(及其子树)以 JSON 形式保存到磁盘。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
path: 目标文件路径。
|
||||||
|
l1_node: 待序列化的 L1 节点。
|
||||||
|
"""
|
||||||
|
with open(path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(l1_node.to_dict(), f, ensure_ascii=False, indent=2)
|
||||||
|
logger.info("L1 中间结果已保存", path=path, l1_id=l1_node.id)
|
||||||
|
|
||||||
|
|
||||||
|
def load_l1_json(path: str) -> L1Node:
|
||||||
|
"""从 JSON 文件加载单个 L1 节点(embedding=None)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
path: JSON 文件路径。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
L1Node 实例。
|
||||||
|
"""
|
||||||
|
with open(path, encoding="utf-8") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
node = L1Node.from_dict(data)
|
||||||
|
logger.info("L1 中间结果已加载", path=path, l1_id=node.id)
|
||||||
|
return node
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
"""树修复检测器:扫描 TreeIndex 识别缺失/低质量节点。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.tree.index import TreeIndex
|
||||||
|
|
||||||
|
# 相邻 L2 片段之间允许的最大时间间隙(秒)
|
||||||
|
_MAX_TIME_GAP_S = 1.0
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class NodeIssue:
|
||||||
|
"""检测到的节点问题。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node_id: 问题节点 ID。
|
||||||
|
level: 节点层级(1/2/3)。
|
||||||
|
issue_type: 问题类型。
|
||||||
|
details: 详细描述。
|
||||||
|
"""
|
||||||
|
|
||||||
|
node_id: str
|
||||||
|
level: int
|
||||||
|
issue_type: str # "empty_field" | "missing_frame" | "no_children" | "time_gap"
|
||||||
|
details: str
|
||||||
|
|
||||||
|
|
||||||
|
def detect_issues(
|
||||||
|
index: TreeIndex,
|
||||||
|
frames_dir: Path | None = None,
|
||||||
|
) -> list[NodeIssue]:
|
||||||
|
"""扫描树,返回所有问题节点列表。
|
||||||
|
|
||||||
|
检查项:
|
||||||
|
- L3: card 必填字段为空(frame_summary / visible_entities / ongoing_actions / spatial_layout)
|
||||||
|
- L3: frame_path 对应文件不存在(需提供 frames_dir)
|
||||||
|
- L2/L1: children 列表为空
|
||||||
|
- L2: 相邻 clips 时间范围不连续(gap > 1秒)
|
||||||
|
|
||||||
|
参数:
|
||||||
|
index: 待检测的 TreeIndex。
|
||||||
|
frames_dir: 帧文件根目录(可选,提供时检查帧文件存在性)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
问题列表,按 level 降序(L3 → L2 → L1)排列。
|
||||||
|
"""
|
||||||
|
issues: list[NodeIssue] = []
|
||||||
|
|
||||||
|
for l1 in index.roots:
|
||||||
|
# L1: children 不为空
|
||||||
|
if not l1.children:
|
||||||
|
issues.append(
|
||||||
|
NodeIssue(
|
||||||
|
node_id=l1.id,
|
||||||
|
level=1,
|
||||||
|
issue_type="no_children",
|
||||||
|
details="L1 节点无 L2 子节点",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# L2: 相邻 clips 时间间隙检查
|
||||||
|
_check_time_gaps(l1.children, issues)
|
||||||
|
|
||||||
|
for l2 in l1.children:
|
||||||
|
# L2: children 不为空
|
||||||
|
if not l2.children:
|
||||||
|
issues.append(
|
||||||
|
NodeIssue(
|
||||||
|
node_id=l2.id,
|
||||||
|
level=2,
|
||||||
|
issue_type="no_children",
|
||||||
|
details="L2 节点无 L3 子节点",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
for l3 in l2.children:
|
||||||
|
# L3: 各必填字段不为空
|
||||||
|
empty_fields: list[str] = []
|
||||||
|
if not l3.card.frame_summary:
|
||||||
|
empty_fields.append("frame_summary")
|
||||||
|
if not l3.card.visible_entities:
|
||||||
|
empty_fields.append("visible_entities")
|
||||||
|
if not l3.card.ongoing_actions:
|
||||||
|
empty_fields.append("ongoing_actions")
|
||||||
|
if not l3.card.spatial_layout:
|
||||||
|
empty_fields.append("spatial_layout")
|
||||||
|
if empty_fields:
|
||||||
|
issues.append(
|
||||||
|
NodeIssue(
|
||||||
|
node_id=l3.id,
|
||||||
|
level=3,
|
||||||
|
issue_type="empty_field",
|
||||||
|
details=f"L3 节点字段为空: {', '.join(empty_fields)}",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# L3: frame_path 文件存在性
|
||||||
|
if (
|
||||||
|
frames_dir is not None
|
||||||
|
and l3.frame_path is not None
|
||||||
|
and not (frames_dir / l3.frame_path).exists()
|
||||||
|
):
|
||||||
|
issues.append(
|
||||||
|
NodeIssue(
|
||||||
|
node_id=l3.id,
|
||||||
|
level=3,
|
||||||
|
issue_type="missing_frame",
|
||||||
|
details=f"帧文件不存在: {l3.frame_path}",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# 按 level 降序排列(L3=3 → L2=2 → L1=1)
|
||||||
|
issues.sort(key=lambda i: -i.level)
|
||||||
|
|
||||||
|
logger.info("树缺陷检测完成,发现 {} 个问题", len(issues))
|
||||||
|
return issues
|
||||||
|
|
||||||
|
|
||||||
|
def _check_time_gaps(
|
||||||
|
l2_nodes: list,
|
||||||
|
issues: list[NodeIssue],
|
||||||
|
) -> None:
|
||||||
|
"""检查同一 L1 下相邻 L2 节点之间的时间间隙。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l2_nodes: 同一 L1 节点下的 L2 子节点列表。
|
||||||
|
issues: 问题列表(原地追加)。
|
||||||
|
"""
|
||||||
|
for i in range(len(l2_nodes) - 1):
|
||||||
|
curr = l2_nodes[i]
|
||||||
|
nxt = l2_nodes[i + 1]
|
||||||
|
if curr.time_range is None or nxt.time_range is None:
|
||||||
|
continue
|
||||||
|
gap = nxt.time_range[0] - curr.time_range[1]
|
||||||
|
if gap > _MAX_TIME_GAP_S:
|
||||||
|
issues.append(
|
||||||
|
NodeIssue(
|
||||||
|
node_id=nxt.id,
|
||||||
|
level=2,
|
||||||
|
issue_type="time_gap",
|
||||||
|
details=f"与前一片段间隙 {gap:.1f}s(阈值 {_MAX_TIME_GAP_S}s)",
|
||||||
|
)
|
||||||
|
)
|
||||||
@@ -0,0 +1,526 @@
|
|||||||
|
"""Q&A 反向补全:基于问题答案分析,将树中缺失的事实注入节点。
|
||||||
|
|
||||||
|
通过 LLM 分析正确答案需要哪些关键事实,再检查树中是否已有,
|
||||||
|
对缺失事实执行注入。仅注入客观事实(人名、地点、得分、物体名称),
|
||||||
|
不注入情感、因果推理、时间推理等主观或高阶信息。
|
||||||
|
|
||||||
|
与 TRM4 的关键差异:
|
||||||
|
- 树结构从扁平 dict 变为 TreeIndex(L1Node → L2Node → L3Node)。
|
||||||
|
- Card 为 frozen dataclass,注入时使用 dataclasses.replace() 创建新实例。
|
||||||
|
- LLMProvider 为异步接口,返回 LLMResponse(.content 获取文本)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass, replace
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from app.tree.index import L1Node, L2Node, L3Node, TreeIndex
|
||||||
|
from core.protocols import LLMProvider
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 允许注入的类别白名单
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_ALLOWED_CATEGORIES = frozenset(
|
||||||
|
{
|
||||||
|
"person_name",
|
||||||
|
"location",
|
||||||
|
"score_number",
|
||||||
|
"object_name",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 类别 → 默认注入字段映射(L2 Card 字段名)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_CATEGORY_DEFAULT_FIELD: dict[str, str] = {
|
||||||
|
"person_name": "entities",
|
||||||
|
"location": "entities",
|
||||||
|
"score_number": "entities",
|
||||||
|
"object_name": "entities",
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 统计
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class SupplementStats:
|
||||||
|
"""反向补全统计信息。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
questions_analyzed: 分析的问题数量。
|
||||||
|
facts_injected: 成功注入的事实数量。
|
||||||
|
facts_skipped: 跳过的事实数量(类别不在白名单中)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
questions_analyzed: int = 0
|
||||||
|
facts_injected: int = 0
|
||||||
|
facts_skipped: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 去重
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def deduplicate_field(values: list[str]) -> list[str]:
|
||||||
|
"""大小写归一化去重,保留首次出现的原始形式。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
values: 待去重字符串列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
去重后的列表,保留各值首次出现时的大小写。
|
||||||
|
空字符串和纯空白字符串会被跳过。
|
||||||
|
"""
|
||||||
|
seen: set[str] = set()
|
||||||
|
result: list[str] = []
|
||||||
|
for v in values:
|
||||||
|
key = v.strip().lower()
|
||||||
|
if key and key not in seen:
|
||||||
|
seen.add(key)
|
||||||
|
result.append(v)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 节点查找
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _find_node_by_id(
|
||||||
|
index: TreeIndex,
|
||||||
|
node_id: str,
|
||||||
|
) -> tuple[L1Node | L2Node | L3Node | None, int]:
|
||||||
|
"""在 TreeIndex 中按 ID 查找节点,返回节点和所属层级。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
index: 树索引。
|
||||||
|
node_id: 目标节点 ID。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
(node, level) 元组。找不到时返回 (None, -1)。
|
||||||
|
level: 1=L1, 2=L2, 3=L3。
|
||||||
|
"""
|
||||||
|
for l1 in index.roots:
|
||||||
|
if l1.id == node_id:
|
||||||
|
return l1, 1
|
||||||
|
for l2 in l1.children:
|
||||||
|
if l2.id == node_id:
|
||||||
|
return l2, 2
|
||||||
|
for l3 in l2.children:
|
||||||
|
if l3.id == node_id:
|
||||||
|
return l3, 3
|
||||||
|
return None, -1
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 单值注入(适配 frozen Card)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _inject_into_l2(l2: L2Node, field: str, value: str) -> bool:
|
||||||
|
"""向 L2 节点的 Card 指定字段注入一个值。
|
||||||
|
|
||||||
|
使用 dataclasses.replace() 创建新的 frozen L2Card。
|
||||||
|
仅支持 list[str] 类型字段(entities / actions / action_subjects / visible_text)
|
||||||
|
和 str 类型字段(event_description / spatial_relations / state_changes)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l2: L2 节点(card 会被替换为新实例)。
|
||||||
|
field: 目标字段名。
|
||||||
|
value: 要注入的值。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
True 表示实际注入了新内容,False 表示已存在(跳过)。
|
||||||
|
"""
|
||||||
|
card = l2.card
|
||||||
|
current = getattr(card, field, None)
|
||||||
|
|
||||||
|
if current is None:
|
||||||
|
# 字段不存在于 Card schema,跳过
|
||||||
|
logger.debug("L2Card 无字段 {},跳过注入", field)
|
||||||
|
return False
|
||||||
|
|
||||||
|
if isinstance(current, list):
|
||||||
|
lower_set = {v.strip().lower() for v in current if isinstance(v, str)}
|
||||||
|
if value.strip().lower() in lower_set:
|
||||||
|
return False
|
||||||
|
new_list = deduplicate_field([*current, value])
|
||||||
|
l2.card = replace(card, **{field: new_list})
|
||||||
|
return True
|
||||||
|
|
||||||
|
if isinstance(current, str):
|
||||||
|
if value.strip().lower() in current.lower():
|
||||||
|
return False
|
||||||
|
new_val = current + "; " + value if current else value
|
||||||
|
l2.card = replace(card, **{field: new_val})
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _inject_into_l3(l3: L3Node, field: str, value: str) -> bool:
|
||||||
|
"""向 L3 节点的 Card 指定字段注入一个值。
|
||||||
|
|
||||||
|
使用 dataclasses.replace() 创建新的 frozen L3Card。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l3: L3 节点(card 会被替换为新实例)。
|
||||||
|
field: 目标字段名。
|
||||||
|
value: 要注入的值。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
True 表示实际注入了新内容,False 表示已存在(跳过)。
|
||||||
|
"""
|
||||||
|
card = l3.card
|
||||||
|
current = getattr(card, field, None)
|
||||||
|
|
||||||
|
if current is None:
|
||||||
|
logger.debug("L3Card 无字段 {},跳过注入", field)
|
||||||
|
return False
|
||||||
|
|
||||||
|
if isinstance(current, list):
|
||||||
|
lower_set = {v.strip().lower() for v in current if isinstance(v, str)}
|
||||||
|
if value.strip().lower() in lower_set:
|
||||||
|
return False
|
||||||
|
new_list = deduplicate_field([*current, value])
|
||||||
|
l3.card = replace(card, **{field: new_list})
|
||||||
|
return True
|
||||||
|
|
||||||
|
if isinstance(current, str):
|
||||||
|
if value.strip().lower() in current.lower():
|
||||||
|
return False
|
||||||
|
new_val = current + "; " + value if current else value
|
||||||
|
l3.card = replace(card, **{field: new_val})
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _inject_into_l1(l1: L1Node, field: str, value: str) -> bool:
|
||||||
|
"""向 L1 节点的 Card 指定字段注入一个值。
|
||||||
|
|
||||||
|
使用 dataclasses.replace() 创建新的 frozen L1Card。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l1: L1 节点(card 会被替换为新实例)。
|
||||||
|
field: 目标字段名。
|
||||||
|
value: 要注入的值。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
True 表示实际注入了新内容,False 表示已存在(跳过)。
|
||||||
|
"""
|
||||||
|
card = l1.card
|
||||||
|
current = getattr(card, field, None)
|
||||||
|
|
||||||
|
if current is None:
|
||||||
|
logger.debug("L1Card 无字段 {},跳过注入", field)
|
||||||
|
return False
|
||||||
|
|
||||||
|
if isinstance(current, list):
|
||||||
|
lower_set = {v.strip().lower() for v in current if isinstance(v, str)}
|
||||||
|
if value.strip().lower() in lower_set:
|
||||||
|
return False
|
||||||
|
new_list = deduplicate_field([*current, value])
|
||||||
|
l1.card = replace(card, **{field: new_list})
|
||||||
|
return True
|
||||||
|
|
||||||
|
if isinstance(current, str):
|
||||||
|
if value.strip().lower() in current.lower():
|
||||||
|
return False
|
||||||
|
new_val = current + "; " + value if current else value
|
||||||
|
l1.card = replace(card, **{field: new_val})
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 批量注入
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def apply_injections(index: TreeIndex, injections: list[dict[str, Any]]) -> SupplementStats:
|
||||||
|
"""执行一组注入指令,将事实写入树节点 Card。
|
||||||
|
|
||||||
|
每条指令格式::
|
||||||
|
|
||||||
|
{
|
||||||
|
"category": "person_name" | "location" | "score_number" | "object_name",
|
||||||
|
"inject_value": "...",
|
||||||
|
"targets": [{"node_id": "...", "field": "..."}, ...]
|
||||||
|
}
|
||||||
|
|
||||||
|
向后兼容: 若无 targets,读取 target_node_id + target_field 构造单目标。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
index: TreeIndex 实例(节点 Card 会被替换为新实例)。
|
||||||
|
injections: 注入指令列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
注入统计信息。
|
||||||
|
"""
|
||||||
|
stats = SupplementStats()
|
||||||
|
|
||||||
|
for instr in injections:
|
||||||
|
category = instr.get("category", "")
|
||||||
|
if category not in _ALLOWED_CATEGORIES:
|
||||||
|
logger.debug("拒绝非法类别: {}", category)
|
||||||
|
stats.facts_skipped += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
inject_value = instr.get("inject_value", "")
|
||||||
|
if not inject_value:
|
||||||
|
stats.facts_skipped += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 解析目标列表(兼容新旧格式)
|
||||||
|
targets = instr.get("targets")
|
||||||
|
if not targets:
|
||||||
|
node_id = instr.get("target_node_id", "")
|
||||||
|
field = instr.get("target_field", "")
|
||||||
|
if node_id and field:
|
||||||
|
targets = [{"node_id": node_id, "field": field}]
|
||||||
|
else:
|
||||||
|
stats.facts_skipped += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
for target in targets:
|
||||||
|
node_id = target.get("node_id", "")
|
||||||
|
field = target.get("field", "")
|
||||||
|
node, level = _find_node_by_id(index, node_id)
|
||||||
|
|
||||||
|
if node is None:
|
||||||
|
logger.debug("跳过不存在的节点: {}", node_id)
|
||||||
|
stats.facts_skipped += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
injected = False
|
||||||
|
if level == 1:
|
||||||
|
injected = _inject_into_l1(node, field, inject_value) # type: ignore[arg-type]
|
||||||
|
elif level == 2:
|
||||||
|
injected = _inject_into_l2(node, field, inject_value) # type: ignore[arg-type]
|
||||||
|
elif level == 3:
|
||||||
|
injected = _inject_into_l3(node, field, inject_value) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
if injected:
|
||||||
|
stats.facts_injected += 1
|
||||||
|
else:
|
||||||
|
stats.facts_skipped += 1
|
||||||
|
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# LLM Prompt
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_SUPPLEMENT_SYSTEM_PROMPT = """\
|
||||||
|
你是一个视频内容分析专家。你的任务是分析回答某个问题需要哪些关键事实,
|
||||||
|
并判断这些事实是否已存在于视频树的摘要中。
|
||||||
|
|
||||||
|
## 输出规则
|
||||||
|
|
||||||
|
1. 只输出**客观事实**,包括以下四类:
|
||||||
|
- person_name: 人物姓名
|
||||||
|
- location: 地点名称
|
||||||
|
- score_number: 比分、数字
|
||||||
|
- object_name: 关键物体名称
|
||||||
|
|
||||||
|
2. **不要**输出以下类型:
|
||||||
|
- 情感、态度、心情
|
||||||
|
- 因果推理("因为…所以…")
|
||||||
|
- 时间顺序推理("先…后…")
|
||||||
|
- 主观评价
|
||||||
|
|
||||||
|
3. 对于 person_name 类别,输出 targets 数组包含两个写入点:
|
||||||
|
- L2 节点的 entities 字段
|
||||||
|
- L3 节点的 visible_entities 字段
|
||||||
|
其他类别只写入最相关的单个节点的 entities 字段。
|
||||||
|
|
||||||
|
4. 每条 missing fact 必须包含 inject_value(要注入的值)和 targets 数组。
|
||||||
|
|
||||||
|
## 输出格式 (严格 JSON)
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"needed_facts": [
|
||||||
|
{"category": "person_name", "value": "..."}
|
||||||
|
],
|
||||||
|
"found_in_tree": [
|
||||||
|
{"category": "person_name", "value": "...", "found_at": "node_id"}
|
||||||
|
],
|
||||||
|
"missing_facts": [
|
||||||
|
{
|
||||||
|
"category": "person_name",
|
||||||
|
"inject_value": "...",
|
||||||
|
"targets": [
|
||||||
|
{"node_id": "...", "field": "entities"},
|
||||||
|
{"node_id": "...", "field": "visible_entities"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
只输出 JSON,不要输出其他内容。
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _build_user_prompt(
|
||||||
|
question: dict[str, Any],
|
||||||
|
index: TreeIndex,
|
||||||
|
srt_text: str,
|
||||||
|
) -> str:
|
||||||
|
"""构建 supplement 分析的 user prompt。
|
||||||
|
|
||||||
|
包含: 问题 + 选项 + 正确答案 + 树 L2 摘要 + SRT 字幕(截断至 3000 字符)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
question: 包含 question/options/answer 的字典。
|
||||||
|
index: TreeIndex 实例。
|
||||||
|
srt_text: SRT 字幕文本。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
拼装后的 user prompt 字符串。
|
||||||
|
"""
|
||||||
|
# 问题部分
|
||||||
|
q_text = question.get("question", "")
|
||||||
|
options = question.get("options", [])
|
||||||
|
answer = question.get("answer", "")
|
||||||
|
options_str = "\n".join(f" {chr(65 + i)}. {opt}" for i, opt in enumerate(options))
|
||||||
|
|
||||||
|
# 树 L2 摘要(从 TreeIndex 结构中提取)
|
||||||
|
l2_summaries: list[str] = []
|
||||||
|
for l1 in index.roots:
|
||||||
|
for l2 in l1.children:
|
||||||
|
description = l2.card.event_description
|
||||||
|
entities_str = ", ".join(l2.card.entities) if l2.card.entities else ""
|
||||||
|
time_str = ""
|
||||||
|
if l2.time_range:
|
||||||
|
time_str = f"{l2.time_range[0]:.1f}-{l2.time_range[1]:.1f}s: "
|
||||||
|
l2_summaries.append(
|
||||||
|
f"[{l2.id}] {time_str}{description}"
|
||||||
|
+ (f" | entities: {entities_str}" if entities_str else "")
|
||||||
|
)
|
||||||
|
|
||||||
|
l2_block = "\n".join(l2_summaries) if l2_summaries else "(无 L2 摘要)"
|
||||||
|
|
||||||
|
# SRT 截断
|
||||||
|
srt_truncated = srt_text[:3000] if srt_text else "(无字幕)"
|
||||||
|
|
||||||
|
return (
|
||||||
|
f"## 问题\n{q_text}\n\n"
|
||||||
|
f"## 选项\n{options_str}\n\n"
|
||||||
|
f"## 正确答案\n{answer}\n\n"
|
||||||
|
f"## 视频树 L2 摘要\n{l2_block}\n\n"
|
||||||
|
f"## 字幕 (前 3000 字符)\n{srt_truncated}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# LLM 调用
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def analyze_question(
|
||||||
|
llm: LLMProvider,
|
||||||
|
question: dict[str, Any],
|
||||||
|
index: TreeIndex,
|
||||||
|
srt_text: str,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""调用 LLM 分析单个问题,返回需要注入的事实列表。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
llm: LLMProvider 实例(异步接口)。
|
||||||
|
question: 问题字典(含 question/options/answer)。
|
||||||
|
index: TreeIndex 实例。
|
||||||
|
srt_text: SRT 字幕文本。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
missing_facts 列表,每项含 category / inject_value / targets。
|
||||||
|
解析失败时返回空列表。
|
||||||
|
"""
|
||||||
|
user_prompt = _build_user_prompt(question, index, srt_text)
|
||||||
|
messages = [
|
||||||
|
{"role": "system", "content": _SUPPLEMENT_SYSTEM_PROMPT},
|
||||||
|
{"role": "user", "content": user_prompt},
|
||||||
|
]
|
||||||
|
|
||||||
|
response = await llm.chat(messages)
|
||||||
|
raw = response.content
|
||||||
|
|
||||||
|
# 提取 JSON(兼容 markdown 代码块包裹)
|
||||||
|
text = raw.strip()
|
||||||
|
if text.startswith("```"):
|
||||||
|
lines = text.split("\n")
|
||||||
|
lines = [ln for ln in lines if not ln.strip().startswith("```")]
|
||||||
|
text = "\n".join(lines)
|
||||||
|
|
||||||
|
try:
|
||||||
|
parsed = json.loads(text)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
logger.warning("supplement LLM 返回非法 JSON,跳过。原始内容: {}", raw[:200])
|
||||||
|
return []
|
||||||
|
|
||||||
|
missing = parsed.get("missing_facts", [])
|
||||||
|
if not isinstance(missing, list):
|
||||||
|
logger.warning("missing_facts 不是列表,跳过")
|
||||||
|
return []
|
||||||
|
|
||||||
|
return missing
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 主入口
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def supplement_tree(
|
||||||
|
index: TreeIndex,
|
||||||
|
questions: list[dict[str, Any]],
|
||||||
|
llm: LLMProvider,
|
||||||
|
srt_text: str = "",
|
||||||
|
) -> SupplementStats:
|
||||||
|
"""对树索引执行 Q&A 反向补全:遍历问题,分析缺失事实,注入节点。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
index: TreeIndex 实例(节点 Card 会被就地替换)。
|
||||||
|
questions: 问题列表,每项含 question/options/answer。
|
||||||
|
llm: LLMProvider 实例(异步接口)。
|
||||||
|
srt_text: SRT 字幕文本(可选,默认空字符串)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
补全统计信息。
|
||||||
|
"""
|
||||||
|
all_injections: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
for i, question in enumerate(questions):
|
||||||
|
logger.debug(
|
||||||
|
"supplement: 分析问题 {}/{}",
|
||||||
|
i + 1,
|
||||||
|
len(questions),
|
||||||
|
)
|
||||||
|
missing = await analyze_question(llm, question, index, srt_text)
|
||||||
|
all_injections.extend(missing)
|
||||||
|
|
||||||
|
stats = apply_injections(index, all_injections)
|
||||||
|
stats.questions_analyzed = len(questions)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"supplement_tree 完成: questions={} injections={} injected={} skipped={}",
|
||||||
|
len(questions),
|
||||||
|
len(all_injections),
|
||||||
|
stats.facts_injected,
|
||||||
|
stats.facts_skipped,
|
||||||
|
)
|
||||||
|
return stats
|
||||||
@@ -0,0 +1,308 @@
|
|||||||
|
"""字幕模块:SRT 解析、完整性检查、时间范围提取、Voronoi 分配。
|
||||||
|
|
||||||
|
提供四个核心函数:
|
||||||
|
- parse_srt: 解析 SRT 文件为结构化条目列表
|
||||||
|
- check_subtitle_completeness: 检查字幕覆盖率与完整性
|
||||||
|
- extract_subtitle_for_range: 提取指定时间范围内的字幕文本
|
||||||
|
- assign_subtitles_voronoi: 使用 Voronoi 中点策略将字幕分配给 L3 节点
|
||||||
|
|
||||||
|
迁移来源:
|
||||||
|
- TRM4 core/tree/enhance/merge.py (parse_srt)
|
||||||
|
- TRM3 tools/generate_subtitles.py (Voronoi 逻辑)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from app.tree.index import TreeIndex
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 正则表达式
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_HTML_TAG_RE = re.compile(r"<[^>]+>")
|
||||||
|
_MUSIC_ONLY_RE = re.compile(r"^[\s♪♫]*$")
|
||||||
|
_TIMECODE_RE = re.compile(r"(\d+):(\d+):(\d+)[,.](\d+)\s*-->\s*(\d+):(\d+):(\d+)[,.](\d+)")
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 数据类型
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SRTEntry:
|
||||||
|
"""单条 SRT 字幕条目。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
start: 开始时间(秒)。
|
||||||
|
end: 结束时间(秒)。
|
||||||
|
text: 字幕文本(已清洗 HTML 标签)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
start: float
|
||||||
|
end: float
|
||||||
|
text: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SubtitleReport:
|
||||||
|
"""字幕完整性检查报告。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
total_entries: 字幕条目总数。
|
||||||
|
coverage_ratio: SRT 覆盖时长 / 视频总时长。
|
||||||
|
max_gap_sec: 最大连续无字幕间隔(秒)。
|
||||||
|
usable: 覆盖率是否达到最低要求。
|
||||||
|
"""
|
||||||
|
|
||||||
|
total_entries: int
|
||||||
|
coverage_ratio: float
|
||||||
|
max_gap_sec: float
|
||||||
|
usable: bool
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 内部辅助
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _ts_to_seconds(h: str, m: str, s: str, ms: str) -> float:
|
||||||
|
"""SRT 时间戳组件 (HH:MM:SS,mmm) 转秒数。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
h: 小时。
|
||||||
|
m: 分钟。
|
||||||
|
s: 秒。
|
||||||
|
ms: 毫秒。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
浮点秒数。
|
||||||
|
"""
|
||||||
|
return int(h) * 3600 + int(m) * 60 + int(s) + int(ms) / 1000
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 公共 API
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def parse_srt(srt_path: str) -> list[SRTEntry]:
|
||||||
|
"""解析 SRT 字幕文件,返回结构化条目列表。
|
||||||
|
|
||||||
|
- 剥离 HTML 标签(如 <i>、<b>)
|
||||||
|
- 跳过纯音乐符号行(仅含空白和 ♪♫)
|
||||||
|
- 多行字幕合并为单行(空格连接)
|
||||||
|
- 跳过格式异常的块(容错处理)
|
||||||
|
|
||||||
|
参数:
|
||||||
|
srt_path: SRT 文件的绝对路径。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
按时间顺序排列的 SRTEntry 列表;空文件或无有效条目返回空列表。
|
||||||
|
|
||||||
|
迁移来源:
|
||||||
|
TRM4 core/tree/enhance/merge.py parse_srt
|
||||||
|
TRM3 tools/generate_subtitles.py parse_srt
|
||||||
|
"""
|
||||||
|
with open(srt_path, encoding="utf-8") as f:
|
||||||
|
content = f.read()
|
||||||
|
|
||||||
|
if not content.strip():
|
||||||
|
return []
|
||||||
|
|
||||||
|
entries: list[SRTEntry] = []
|
||||||
|
blocks = re.split(r"\n\s*\n", content.strip())
|
||||||
|
|
||||||
|
for block in blocks:
|
||||||
|
lines = block.strip().split("\n")
|
||||||
|
if len(lines) < 2:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 在块内搜索时间码行(可能是第 1 行或第 2 行)
|
||||||
|
ts_match = None
|
||||||
|
ts_line_idx = -1
|
||||||
|
for i, line in enumerate(lines):
|
||||||
|
ts_match = _TIMECODE_RE.search(line)
|
||||||
|
if ts_match:
|
||||||
|
ts_line_idx = i
|
||||||
|
break
|
||||||
|
|
||||||
|
if not ts_match:
|
||||||
|
continue
|
||||||
|
|
||||||
|
groups = [int(x) for x in ts_match.groups()]
|
||||||
|
start = _ts_to_seconds(str(groups[0]), str(groups[1]), str(groups[2]), str(groups[3]))
|
||||||
|
end = _ts_to_seconds(str(groups[4]), str(groups[5]), str(groups[6]), str(groups[7]))
|
||||||
|
|
||||||
|
# 时间码行之后的所有行为字幕文本
|
||||||
|
text_lines = lines[ts_line_idx + 1 :]
|
||||||
|
raw_text = " ".join(text_lines)
|
||||||
|
clean_text = _HTML_TAG_RE.sub("", raw_text).strip()
|
||||||
|
|
||||||
|
# 跳过空文本和纯音乐符号行
|
||||||
|
if not clean_text or _MUSIC_ONLY_RE.match(clean_text):
|
||||||
|
continue
|
||||||
|
|
||||||
|
entries.append(SRTEntry(start=start, end=end, text=clean_text))
|
||||||
|
|
||||||
|
logger.debug("SRT 解析完成: {} 条有效条目, 文件={}", len(entries), srt_path)
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def check_subtitle_completeness(
|
||||||
|
entries: list[SRTEntry],
|
||||||
|
duration: float,
|
||||||
|
min_coverage: float = 0.3,
|
||||||
|
) -> SubtitleReport:
|
||||||
|
"""检查字幕完整性:覆盖率、最大间隔、可用性判定。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
entries: 已排序的 SRTEntry 列表。
|
||||||
|
duration: 视频总时长(秒),必须 > 0。
|
||||||
|
min_coverage: 最低可用覆盖率阈值(0~1)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
SubtitleReport 包含覆盖率、最大间隔和可用性判定。
|
||||||
|
"""
|
||||||
|
assert duration > 0, f"视频时长必须 > 0,实际={duration}"
|
||||||
|
|
||||||
|
if not entries:
|
||||||
|
return SubtitleReport(
|
||||||
|
total_entries=0,
|
||||||
|
coverage_ratio=0.0,
|
||||||
|
max_gap_sec=duration,
|
||||||
|
usable=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 按开始时间排序
|
||||||
|
sorted_entries = sorted(entries, key=lambda e: e.start)
|
||||||
|
|
||||||
|
# 计算覆盖时长(合并重叠区间)
|
||||||
|
merged_intervals: list[tuple[float, float]] = []
|
||||||
|
for entry in sorted_entries:
|
||||||
|
if merged_intervals and entry.start <= merged_intervals[-1][1]:
|
||||||
|
# 与上一区间重叠,扩展
|
||||||
|
merged_intervals[-1] = (
|
||||||
|
merged_intervals[-1][0],
|
||||||
|
max(merged_intervals[-1][1], entry.end),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
merged_intervals.append((entry.start, entry.end))
|
||||||
|
|
||||||
|
covered = sum(end - start for start, end in merged_intervals)
|
||||||
|
coverage_ratio = min(covered / duration, 1.0)
|
||||||
|
|
||||||
|
# 计算最大间隔(包括视频开头到第一条字幕、最后一条到视频结尾)
|
||||||
|
max_gap = merged_intervals[0][0] # 视频开头到第一条字幕
|
||||||
|
for i in range(1, len(merged_intervals)):
|
||||||
|
gap = merged_intervals[i][0] - merged_intervals[i - 1][1]
|
||||||
|
max_gap = max(max_gap, gap)
|
||||||
|
# 最后一条字幕到视频结尾
|
||||||
|
max_gap = max(max_gap, duration - merged_intervals[-1][1])
|
||||||
|
|
||||||
|
return SubtitleReport(
|
||||||
|
total_entries=len(entries),
|
||||||
|
coverage_ratio=coverage_ratio,
|
||||||
|
max_gap_sec=max_gap,
|
||||||
|
usable=coverage_ratio >= min_coverage,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def extract_subtitle_for_range(
|
||||||
|
entries: list[SRTEntry],
|
||||||
|
time_range: tuple[float, float],
|
||||||
|
) -> str:
|
||||||
|
"""提取与指定时间范围重叠的字幕文本。
|
||||||
|
|
||||||
|
重叠判定:entry.start < range_end 且 entry.end > range_start。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
entries: SRTEntry 列表。
|
||||||
|
time_range: (start, end) 时间范围(秒)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
匹配的字幕文本,多条用换行符连接;无匹配返回空字符串。
|
||||||
|
"""
|
||||||
|
range_start, range_end = time_range
|
||||||
|
matched = [
|
||||||
|
entry.text for entry in entries if entry.start < range_end and entry.end > range_start
|
||||||
|
]
|
||||||
|
return "\n".join(matched)
|
||||||
|
|
||||||
|
|
||||||
|
def assign_subtitles_voronoi(
|
||||||
|
index: TreeIndex,
|
||||||
|
entries: list[SRTEntry],
|
||||||
|
) -> None:
|
||||||
|
"""使用 Voronoi 中点策略将字幕分配给 L3 节点。
|
||||||
|
|
||||||
|
对每个 L2 节点内的 L3 子节点,按 timestamp 排序后计算 Voronoi 有效范围:
|
||||||
|
- 相邻 L3 节点之间取中点作为边界
|
||||||
|
- 首个 L3 的左边界扩展到 L2 的 time_range 起点
|
||||||
|
- 末个 L3 的右边界扩展到 L2 的 time_range 终点
|
||||||
|
|
||||||
|
然后用 extract_subtitle_for_range 提取每个 L3 有效范围内的字幕文本。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
index: 树索引,包含 L1→L2→L3 嵌套结构。
|
||||||
|
entries: 已解析的 SRTEntry 列表。
|
||||||
|
|
||||||
|
副作用:
|
||||||
|
直接修改每个 L3Node.subtitle 字段。
|
||||||
|
|
||||||
|
迁移来源:
|
||||||
|
TRM3 tools/generate_subtitles.py compute_effective_ranges + assign_subtitles
|
||||||
|
"""
|
||||||
|
for l1 in index.roots:
|
||||||
|
for l2 in l1.children:
|
||||||
|
if not l2.children:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 按 timestamp 排序 L3 子节点(保留原列表引用以便赋值)
|
||||||
|
siblings = sorted(
|
||||||
|
l2.children,
|
||||||
|
key=lambda n: n.timestamp if n.timestamp is not None else 0.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
# L2 的时间范围作为边界
|
||||||
|
l2_start = l2.time_range[0] if l2.time_range else 0.0
|
||||||
|
l2_end = l2.time_range[1] if l2.time_range else 0.0
|
||||||
|
|
||||||
|
for idx, l3 in enumerate(siblings):
|
||||||
|
ts = l3.timestamp if l3.timestamp is not None else 0.0
|
||||||
|
|
||||||
|
# 计算 Voronoi 有效范围
|
||||||
|
if idx == 0:
|
||||||
|
left = l2_start
|
||||||
|
else:
|
||||||
|
prev_ts = (
|
||||||
|
siblings[idx - 1].timestamp
|
||||||
|
if siblings[idx - 1].timestamp is not None
|
||||||
|
else 0.0
|
||||||
|
)
|
||||||
|
left = (prev_ts + ts) / 2.0
|
||||||
|
|
||||||
|
if idx == len(siblings) - 1:
|
||||||
|
right = l2_end
|
||||||
|
else:
|
||||||
|
next_ts = (
|
||||||
|
siblings[idx + 1].timestamp
|
||||||
|
if siblings[idx + 1].timestamp is not None
|
||||||
|
else 0.0
|
||||||
|
)
|
||||||
|
right = (ts + next_ts) / 2.0
|
||||||
|
|
||||||
|
subtitle_text = extract_subtitle_for_range(entries, (left, right))
|
||||||
|
l3.subtitle = subtitle_text if subtitle_text else None
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
"Voronoi 字幕分配完成: {} 个 L1 节点, {} 条字幕条目",
|
||||||
|
len(index.roots),
|
||||||
|
len(entries),
|
||||||
|
)
|
||||||
@@ -0,0 +1,291 @@
|
|||||||
|
"""质量校验模块:交叉验证树节点 Card 字段与子节点证据。
|
||||||
|
|
||||||
|
验证策略:
|
||||||
|
- L2 entities: 仅保留在子 L3 文本语料中模糊匹配到的实体。
|
||||||
|
- L2 visible_text: 仅保留在子 L3 visible_text 中出现的条目。
|
||||||
|
- L1 visible_text: 仅保留在后代 L2/L3 visible_text 中出现的条目。
|
||||||
|
- L1 key_entities: 仅保留在后代 L2/L3 文本语料中模糊匹配到的实体。
|
||||||
|
|
||||||
|
Card 为 frozen dataclass,无法原地修改——移除幻觉字段时
|
||||||
|
创建新 Card 实例并赋值给 node.card(Node 非 frozen)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import string
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from app.tree.index import (
|
||||||
|
L1Card,
|
||||||
|
L1Node,
|
||||||
|
L2Card,
|
||||||
|
L2Node,
|
||||||
|
TreeIndex,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 校验统计
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class VerifyStats:
|
||||||
|
"""校验统计信息。"""
|
||||||
|
|
||||||
|
l2_entities_kept: int = 0
|
||||||
|
l2_entities_removed: int = 0
|
||||||
|
l2_visible_text_kept: int = 0
|
||||||
|
l2_visible_text_removed: int = 0
|
||||||
|
l1_visible_text_kept: int = 0
|
||||||
|
l1_visible_text_removed: int = 0
|
||||||
|
l1_key_entities_kept: int = 0
|
||||||
|
l1_key_entities_removed: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 文本归一化 & 模糊匹配
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize(text: str) -> str:
|
||||||
|
"""归一化文本:小写 + 去除标点。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
text: 原始文本。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
归一化后的纯小写无标点字符串。
|
||||||
|
"""
|
||||||
|
return text.lower().translate(str.maketrans("", "", string.punctuation))
|
||||||
|
|
||||||
|
|
||||||
|
def fuzzy_match(entity: str | None, corpus: str | None) -> bool:
|
||||||
|
"""模糊子串匹配:归一化后判断 entity 是否为 corpus 的子串。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
entity: 待匹配的实体文本(None 视为不匹配)。
|
||||||
|
corpus: 证据语料文本(None 视为空)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
True 表示匹配成功。
|
||||||
|
"""
|
||||||
|
if not entity or not corpus:
|
||||||
|
return False
|
||||||
|
return _normalize(str(entity)) in _normalize(str(corpus))
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 语料收集
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_l3_text(l2_node: L2Node) -> str:
|
||||||
|
"""收集 L2 节点所有子 L3 的文本语料。
|
||||||
|
|
||||||
|
从每个 L3 子节点的 card 和顶层字段中提取:
|
||||||
|
frame_summary、visible_text、subtitle。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l2_node: L2 节点。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
拼接后的文本语料(用换行分隔)。
|
||||||
|
"""
|
||||||
|
parts: list[str] = []
|
||||||
|
for l3 in l2_node.children:
|
||||||
|
parts.append(l3.card.frame_summary)
|
||||||
|
parts.extend(l3.card.visible_text)
|
||||||
|
if l3.subtitle:
|
||||||
|
parts.append(l3.subtitle)
|
||||||
|
return "\n".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_descendant_visible_text(l1_node: L1Node) -> str:
|
||||||
|
"""收集 L1 节点所有后代(L2/L3)的 visible_text。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l1_node: L1 节点。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
所有后代 visible_text 拼接后的文本(用换行分隔)。
|
||||||
|
"""
|
||||||
|
parts: list[str] = []
|
||||||
|
for l2 in l1_node.children:
|
||||||
|
parts.extend(l2.card.visible_text)
|
||||||
|
for l3 in l2.children:
|
||||||
|
parts.extend(l3.card.visible_text)
|
||||||
|
return "\n".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_descendant_text_corpus(l1_node: L1Node) -> str:
|
||||||
|
"""收集 L1 节点所有后代(L2/L3)的完整文本语料。
|
||||||
|
|
||||||
|
用于 L1 key_entities 的交叉验证,范围包括
|
||||||
|
L2/L3 的所有文本字段(frame_summary、visible_text、subtitle 等)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l1_node: L1 节点。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
所有后代文本语料拼接后的文本(用换行分隔)。
|
||||||
|
"""
|
||||||
|
parts: list[str] = []
|
||||||
|
for l2 in l1_node.children:
|
||||||
|
parts.append(l2.card.event_description)
|
||||||
|
parts.extend(l2.card.entities)
|
||||||
|
parts.extend(l2.card.visible_text)
|
||||||
|
for l3 in l2.children:
|
||||||
|
parts.append(l3.card.frame_summary)
|
||||||
|
parts.extend(l3.card.visible_text)
|
||||||
|
if l3.subtitle:
|
||||||
|
parts.append(l3.subtitle)
|
||||||
|
return "\n".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 主校验函数
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def verify_tree(index: TreeIndex) -> VerifyStats:
|
||||||
|
"""交叉验证视频树的 Card 字段与子节点证据,原地替换不合格的 Card。
|
||||||
|
|
||||||
|
Cards 为 frozen dataclass,移除幻觉字段时创建新 Card 实例
|
||||||
|
并赋值给 node.card。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
index: 树索引(会被原地修改)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
VerifyStats 校验统计。
|
||||||
|
"""
|
||||||
|
stats = VerifyStats()
|
||||||
|
|
||||||
|
for l1 in index.roots:
|
||||||
|
# Phase 1: L2 字段验证
|
||||||
|
for l2 in l1.children:
|
||||||
|
_verify_l2(l2, stats)
|
||||||
|
|
||||||
|
# Phase 2: L1 字段验证
|
||||||
|
_verify_l1(l1, stats)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"verify_tree: source={} "
|
||||||
|
"l2_ent_kept={} l2_ent_rm={} "
|
||||||
|
"l2_vt_kept={} l2_vt_rm={} "
|
||||||
|
"l1_vt_kept={} l1_vt_rm={} "
|
||||||
|
"l1_ke_kept={} l1_ke_rm={}",
|
||||||
|
index.metadata.source_path,
|
||||||
|
stats.l2_entities_kept,
|
||||||
|
stats.l2_entities_removed,
|
||||||
|
stats.l2_visible_text_kept,
|
||||||
|
stats.l2_visible_text_removed,
|
||||||
|
stats.l1_visible_text_kept,
|
||||||
|
stats.l1_visible_text_removed,
|
||||||
|
stats.l1_key_entities_kept,
|
||||||
|
stats.l1_key_entities_removed,
|
||||||
|
)
|
||||||
|
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_l2(l2: L2Node, stats: VerifyStats) -> None:
|
||||||
|
"""校验单个 L2 节点的 entities 和 visible_text。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l2: L2 节点(card 可能被替换)。
|
||||||
|
stats: 统计对象(原地累加)。
|
||||||
|
"""
|
||||||
|
corpus = _collect_l3_text(l2)
|
||||||
|
old_card = l2.card
|
||||||
|
|
||||||
|
# entities: 模糊匹配过滤
|
||||||
|
kept_entities = [e for e in old_card.entities if fuzzy_match(e, corpus)]
|
||||||
|
stats.l2_entities_kept += len(kept_entities)
|
||||||
|
stats.l2_entities_removed += len(old_card.entities) - len(kept_entities)
|
||||||
|
|
||||||
|
# visible_text: 子 L3 visible_text 中必须存在
|
||||||
|
l3_visible = _collect_l3_visible_text_set(l2)
|
||||||
|
kept_vt = [vt for vt in old_card.visible_text if _text_in_set(vt, l3_visible)]
|
||||||
|
stats.l2_visible_text_kept += len(kept_vt)
|
||||||
|
stats.l2_visible_text_removed += len(old_card.visible_text) - len(kept_vt)
|
||||||
|
|
||||||
|
# 创建新 Card 替换(frozen dataclass)
|
||||||
|
l2.card = L2Card(
|
||||||
|
event_description=old_card.event_description,
|
||||||
|
entities=kept_entities,
|
||||||
|
actions=old_card.actions,
|
||||||
|
action_subjects=old_card.action_subjects,
|
||||||
|
visible_text=kept_vt,
|
||||||
|
spatial_relations=old_card.spatial_relations,
|
||||||
|
state_changes=old_card.state_changes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_l1(l1: L1Node, stats: VerifyStats) -> None:
|
||||||
|
"""校验单个 L1 节点的 visible_text 和 key_entities。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l1: L1 节点(card 可能被替换)。
|
||||||
|
stats: 统计对象(原地累加)。
|
||||||
|
"""
|
||||||
|
old_card = l1.card
|
||||||
|
|
||||||
|
# visible_text: 必须出现在后代 L2/L3 visible_text 中
|
||||||
|
descendant_vt = _collect_descendant_visible_text(l1)
|
||||||
|
kept_vt = [vt for vt in old_card.visible_text if fuzzy_match(vt, descendant_vt)]
|
||||||
|
stats.l1_visible_text_kept += len(kept_vt)
|
||||||
|
stats.l1_visible_text_removed += len(old_card.visible_text) - len(kept_vt)
|
||||||
|
|
||||||
|
# key_entities: 交叉验证后代文本语料
|
||||||
|
descendant_corpus = _collect_descendant_text_corpus(l1)
|
||||||
|
kept_ke = [ke for ke in old_card.key_entities if fuzzy_match(ke, descendant_corpus)]
|
||||||
|
stats.l1_key_entities_kept += len(kept_ke)
|
||||||
|
stats.l1_key_entities_removed += len(old_card.key_entities) - len(kept_ke)
|
||||||
|
|
||||||
|
# 创建新 Card 替换(frozen dataclass)
|
||||||
|
l1.card = L1Card(
|
||||||
|
scene_summary=old_card.scene_summary,
|
||||||
|
main_setting=old_card.main_setting,
|
||||||
|
key_entities=kept_ke,
|
||||||
|
main_actions=old_card.main_actions,
|
||||||
|
topic_keywords=old_card.topic_keywords,
|
||||||
|
visible_text=kept_vt,
|
||||||
|
temporal_flow=old_card.temporal_flow,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 辅助函数
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_l3_visible_text_set(l2: L2Node) -> set[str]:
|
||||||
|
"""收集 L2 下所有 L3 子节点的 visible_text 归一化集合。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
l2: L2 节点。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
归一化后的 visible_text 集合。
|
||||||
|
"""
|
||||||
|
result: set[str] = set()
|
||||||
|
for l3 in l2.children:
|
||||||
|
for vt in l3.card.visible_text:
|
||||||
|
result.add(_normalize(vt))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _text_in_set(text: str, normalized_set: set[str]) -> bool:
|
||||||
|
"""检查文本归一化后是否存在于集合中。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
text: 待检查文本。
|
||||||
|
normalized_set: 归一化后的文本集合。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
True 表示匹配成功。
|
||||||
|
"""
|
||||||
|
return _normalize(text) in normalized_set
|
||||||
File diff suppressed because it is too large
Load Diff
+6
-10
@@ -1,10 +1,12 @@
|
|||||||
"""Agent 专属 Protocol 端口。"""
|
"""Agent 专属 Protocol 端口。"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import Any, Protocol, runtime_checkable
|
from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
|
||||||
|
|
||||||
import pluggy
|
import pluggy
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
from core.agent.types import LoopResult, Step
|
from core.agent.types import LoopResult, Step
|
||||||
|
|
||||||
hookspec = pluggy.HookspecMarker("agent_loop")
|
hookspec = pluggy.HookspecMarker("agent_loop")
|
||||||
@@ -27,19 +29,13 @@ class AgentLoopSpec:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
@hookspec
|
@hookspec
|
||||||
async def before_step(
|
async def before_step(self, iteration: int, messages: list[dict[str, Any]]) -> None: ...
|
||||||
self, iteration: int, messages: list[dict[str, Any]]
|
|
||||||
) -> None: ...
|
|
||||||
|
|
||||||
@hookspec
|
@hookspec
|
||||||
async def after_tool(
|
async def after_tool(self, iteration: int, step: Step) -> str | None: ...
|
||||||
self, iteration: int, step: Step
|
|
||||||
) -> str | None: ...
|
|
||||||
|
|
||||||
@hookspec
|
@hookspec
|
||||||
async def after_step(
|
async def after_step(self, iteration: int, messages: list[dict[str, Any]]) -> None: ...
|
||||||
self, iteration: int, messages: list[dict[str, Any]]
|
|
||||||
) -> None: ...
|
|
||||||
|
|
||||||
@hookspec
|
@hookspec
|
||||||
async def on_finish(self, result: LoopResult) -> None: ...
|
async def on_finish(self, result: LoopResult) -> None: ...
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
"""AgentLoop 数据类型。"""
|
"""AgentLoop 数据类型。"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
"""core/evolution/ — 自进化循环决策内核。
|
||||||
|
|
||||||
|
诊断、进化、门控、补丁的纯决策逻辑。
|
||||||
|
只依赖 Protocol 接口和标准库,可搬到无 adapters 的环境用假实现原样运行。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from core.evolution.diagnose import run_diagnosis
|
||||||
|
from core.evolution.evolve import (
|
||||||
|
edit_budget_at,
|
||||||
|
evolve_single_skill,
|
||||||
|
evolve_single_tool,
|
||||||
|
evolve_system_prompt,
|
||||||
|
resolve_skill_file,
|
||||||
|
)
|
||||||
|
from core.evolution.gate import compute_e_value, gate_decision, probation_verdict
|
||||||
|
from core.evolution.patch import (
|
||||||
|
append_to_appendix,
|
||||||
|
apply_patch_with_report,
|
||||||
|
extract_appendix_notes,
|
||||||
|
momentum_inner,
|
||||||
|
replace_appendix_notes,
|
||||||
|
replace_momentum,
|
||||||
|
)
|
||||||
|
from core.evolution.types import (
|
||||||
|
CaseSample,
|
||||||
|
DiagnosePrompts,
|
||||||
|
DiagnosisResult,
|
||||||
|
ErrorAttribution,
|
||||||
|
EvolutionRecord,
|
||||||
|
EvolutionResult,
|
||||||
|
EvolvePrompts,
|
||||||
|
GateParams,
|
||||||
|
GateVerdict,
|
||||||
|
PairResult,
|
||||||
|
QuadrantClassification,
|
||||||
|
QuestionMetrics,
|
||||||
|
RejectedEdit,
|
||||||
|
SkillCasePack,
|
||||||
|
SkillStepAdherence,
|
||||||
|
SpanMetrics,
|
||||||
|
SystemCasePack,
|
||||||
|
ToolCasePack,
|
||||||
|
)
|
||||||
|
from core.evolution.validate import classify_quadrants, compute_accuracy, pair_block
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"CaseSample",
|
||||||
|
"DiagnosePrompts",
|
||||||
|
"DiagnosisResult",
|
||||||
|
"ErrorAttribution",
|
||||||
|
"EvolutionRecord",
|
||||||
|
"EvolutionResult",
|
||||||
|
"EvolvePrompts",
|
||||||
|
"GateParams",
|
||||||
|
"GateVerdict",
|
||||||
|
"PairResult",
|
||||||
|
"QuadrantClassification",
|
||||||
|
"QuestionMetrics",
|
||||||
|
"RejectedEdit",
|
||||||
|
"SkillCasePack",
|
||||||
|
"SkillStepAdherence",
|
||||||
|
"SpanMetrics",
|
||||||
|
"SystemCasePack",
|
||||||
|
"ToolCasePack",
|
||||||
|
"append_to_appendix",
|
||||||
|
"apply_patch_with_report",
|
||||||
|
"classify_quadrants",
|
||||||
|
"compute_accuracy",
|
||||||
|
"compute_e_value",
|
||||||
|
"edit_budget_at",
|
||||||
|
"evolve_single_skill",
|
||||||
|
"evolve_single_tool",
|
||||||
|
"evolve_system_prompt",
|
||||||
|
"extract_appendix_notes",
|
||||||
|
"gate_decision",
|
||||||
|
"momentum_inner",
|
||||||
|
"pair_block",
|
||||||
|
"probation_verdict",
|
||||||
|
"replace_appendix_notes",
|
||||||
|
"replace_momentum",
|
||||||
|
"resolve_skill_file",
|
||||||
|
"run_diagnosis",
|
||||||
|
]
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,129 @@
|
|||||||
|
"""CE-Gate 统计核心:截断 Beta 混合 e-process 的纯函数实现。
|
||||||
|
|
||||||
|
配对不一致检验:候选与基线跑同一题,只数翻转(基线错->候选对 = W;
|
||||||
|
基线对->候选错 = L)。H0(候选不优)下翻转方向精确五五开,
|
||||||
|
E = 2^(W+L+1)*B(W+1,L+1)*[1-I_1/2(W+1,L+1)] 为 H0 下非负上鞅,
|
||||||
|
Ville 不等式给出任意停时 P(E >= 1/alpha) <= alpha。
|
||||||
|
|
||||||
|
设计规格见 research-wiki/designs/2026-07-03-ce-gate-formal-design.md。
|
||||||
|
仅依赖 scipy.special,无 I/O、无状态,便于单测与历史回放复用。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import math
|
||||||
|
|
||||||
|
from scipy.special import betainc, betaln
|
||||||
|
|
||||||
|
from core.evolution.types import GateParams, GateVerdict
|
||||||
|
|
||||||
|
# Wald 方向游走步长(theta_1=0.70 固定设计常量,不入配置):
|
||||||
|
# 胜 +ln(2*theta_1)=ln1.4,负 ln(2*(1-theta_1))=ln0.6。
|
||||||
|
_WALD_WIN = math.log(1.4)
|
||||||
|
_WALD_LOSS = math.log(0.6)
|
||||||
|
|
||||||
|
# delta_shrunk 的伪计数(Agresti-Coull 风格收缩,只作观测输出不进判据)。
|
||||||
|
_SHRINK_PSEUDO = 4
|
||||||
|
|
||||||
|
|
||||||
|
def compute_e_value(w: int, l: int) -> float: # noqa: E741
|
||||||
|
"""截断 Beta 混合 e 值:E = 2^(W+L+1)*B(W+1,L+1)*[1-I_1/2(W+1,L+1)]。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
w: 基线错->候选对的翻转数。
|
||||||
|
l: 基线对->候选错的翻转数。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
e 值(W=L=0 时为 1)。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 翻转计数为负时抛出。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
log 空间计算在 n_max<=40 的设计工作区间内数值稳定(数百级计数
|
||||||
|
亦可);极大计数(>1000)时最终 exp 仍可能溢出。用正则化不完全
|
||||||
|
Beta 的对称性 1-I_1/2(a,b) = I_1/2(b,a) 避免 1-x 的灾难性精度损失。
|
||||||
|
"""
|
||||||
|
if w < 0 or l < 0:
|
||||||
|
raise ValueError(f"翻转计数不能为负: w={w}, l={l}")
|
||||||
|
a, b = w + 1, l + 1
|
||||||
|
tail = betainc(b, a, 0.5) # = 1 - I_1/2(a, b)
|
||||||
|
if tail <= 0.0:
|
||||||
|
return 0.0
|
||||||
|
log_e = (w + l + 1) * math.log(2.0) + betaln(a, b) + math.log(tail)
|
||||||
|
return math.exp(log_e)
|
||||||
|
|
||||||
|
|
||||||
|
def gate_decision(
|
||||||
|
w: int,
|
||||||
|
l: int, # noqa: E741
|
||||||
|
n_used: int,
|
||||||
|
n_remaining: int,
|
||||||
|
*,
|
||||||
|
params: GateParams,
|
||||||
|
) -> GateVerdict:
|
||||||
|
"""块间四出口判定(每块结束时调用一次)。
|
||||||
|
|
||||||
|
出口优先级:CONFIRMED(有证书先走)-> 方向拒绝 -> futility 拒绝 ->
|
||||||
|
题尽(provisional / inertia)-> continue。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
w: 累计 W。
|
||||||
|
l: 累计 L。
|
||||||
|
n_used: 已消费的阶梯题数(含一致题)。
|
||||||
|
n_remaining: 阶梯剩余可用题数(min(阶梯长, n_max) - n_used)。
|
||||||
|
params: 判据阈值组。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
GateVerdict(decision + e 值/游走/效应量诊断)。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: n_used <= 0 或 n_remaining < 0 时抛出。
|
||||||
|
"""
|
||||||
|
if n_used <= 0:
|
||||||
|
raise ValueError(f"gate_decision 须在至少消费一块后调用: n_used={n_used}")
|
||||||
|
if n_remaining < 0:
|
||||||
|
raise ValueError(f"n_remaining 不能为负: {n_remaining}")
|
||||||
|
e_value = compute_e_value(w, l)
|
||||||
|
wald = w * _WALD_WIN + l * _WALD_LOSS
|
||||||
|
delta_hat = (w - l) / n_used
|
||||||
|
delta_shrunk = (w - l) / (n_used + _SHRINK_PSEUDO)
|
||||||
|
|
||||||
|
if e_value >= params.e_confirm and delta_hat >= params.delta_min:
|
||||||
|
decision = "accept_confirmed"
|
||||||
|
elif wald <= params.lambda_dir:
|
||||||
|
decision = "reject_directional"
|
||||||
|
elif n_remaining > 0 and compute_e_value(w + n_remaining, l) < params.e_provisional:
|
||||||
|
# futility 只在题未尽时有意义;题尽后的弱证据归 inertia 出口。
|
||||||
|
decision = "reject_futility"
|
||||||
|
elif n_remaining <= 0:
|
||||||
|
if (
|
||||||
|
e_value >= params.e_provisional
|
||||||
|
and (w - l) >= params.w_net_min
|
||||||
|
and delta_hat >= params.delta_min
|
||||||
|
):
|
||||||
|
decision = "accept_provisional"
|
||||||
|
else:
|
||||||
|
decision = "reject_inertia"
|
||||||
|
else:
|
||||||
|
decision = "continue"
|
||||||
|
return GateVerdict(decision, e_value, wald, delta_hat, delta_shrunk)
|
||||||
|
|
||||||
|
|
||||||
|
def probation_verdict(w: int, l: int, *, params: GateParams) -> str: # noqa: E741
|
||||||
|
"""试用期一次性结算:固定样本 e 值双向检验。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
w: 结算配对的 W(锚快照错->候选重跑对)。
|
||||||
|
l: 结算配对的 L(锚快照对->候选重跑错)。
|
||||||
|
params: 判据阈值组(用 e_confirm / e_rollback)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
"confirmed"(E>=e_confirm 转正)/ "rollback"(对称 E'>=e_rollback 回滚)
|
||||||
|
/ "unverified"(证据不足,elitist 惯性转正)。
|
||||||
|
"""
|
||||||
|
if compute_e_value(w, l) >= params.e_confirm:
|
||||||
|
return "confirmed"
|
||||||
|
if compute_e_value(l, w) >= params.e_rollback:
|
||||||
|
return "rollback"
|
||||||
|
return "unverified"
|
||||||
@@ -0,0 +1,427 @@
|
|||||||
|
"""定点补丁引擎:把进化输出的离散 edit 逐条应用到文本,逐条出状态报告。
|
||||||
|
|
||||||
|
借鉴 SkillOpt skill.py 的 apply 语义;守 P5:找不到锚点不静默乱改、不裸 except。
|
||||||
|
冻结区按全文坐标区间判定;append/退化追加插到最早冻结区之前(无则 EOF)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
APPENDIX_START = "<!-- APPENDIX_START -->"
|
||||||
|
APPENDIX_END = "<!-- APPENDIX_END -->"
|
||||||
|
APPENDIX_MAX_CHARS = 2000 # appendix 区软上限(守设计「长度上限+warning,不做去重」)
|
||||||
|
|
||||||
|
MOMENTUM_START = "<!-- MOMENTUM_START -->"
|
||||||
|
MOMENTUM_END = "<!-- MOMENTUM_END -->"
|
||||||
|
MOMENTUM_MAX_CHARS = 2000 # momentum 区软上限(与 appendix 一致:超限 warning 不截断)
|
||||||
|
MOMENTUM_HEADING = (
|
||||||
|
"## 动量指导(每轮重写,勿手改)" # replace_momentum 写入的固定标题行
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def momentum_region_bounds(text: str) -> tuple[int, int] | None:
|
||||||
|
"""定位 momentum 受保护区的字符区间,并对损坏态显式报错(P5)。
|
||||||
|
|
||||||
|
momentum marker 由 replace_momentum 在 epoch 末反复重写,guidance 又来自 LLM
|
||||||
|
外部输入,因此 marker 可能出现损坏态。本函数是 momentum 路径的唯一边界判定入口,
|
||||||
|
把配对校验集中在一处:
|
||||||
|
|
||||||
|
- START 与 END 各恰好出现一次且 START 在 END 之前 → 返回 (start_idx, end_idx),
|
||||||
|
end_idx 指向 END marker 结束位置(即 content[start:end] 含完整两 marker)。
|
||||||
|
- 两 marker 都不出现 → 返回 None(合法的"无区"态,调用方据此新建)。
|
||||||
|
- 其余皆为损坏态(仅一个 marker、END 在 START 前、任一 marker 重复)→ raise
|
||||||
|
ValueError,拒绝静默新建/跳过,要求人工修复。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
text: 待检测的文本(skill 全文)。
|
||||||
|
返回:
|
||||||
|
(start_idx, end_idx) 表示区间,或 None 表示无 momentum 区。
|
||||||
|
异常:
|
||||||
|
ValueError: momentum marker 损坏/不配对。
|
||||||
|
"""
|
||||||
|
start_count = text.count(MOMENTUM_START)
|
||||||
|
end_count = text.count(MOMENTUM_END)
|
||||||
|
if start_count == 0 and end_count == 0:
|
||||||
|
return None
|
||||||
|
if start_count != 1 or end_count != 1:
|
||||||
|
raise ValueError(
|
||||||
|
f"momentum marker 损坏/不配对:MOMENTUM_START 出现 {start_count} 次、"
|
||||||
|
f"MOMENTUM_END 出现 {end_count} 次(各须恰好 1 次),需人工修复"
|
||||||
|
)
|
||||||
|
start_idx = text.index(MOMENTUM_START)
|
||||||
|
end_idx = text.index(MOMENTUM_END) + len(MOMENTUM_END)
|
||||||
|
if start_idx >= text.index(MOMENTUM_END):
|
||||||
|
raise ValueError(
|
||||||
|
"momentum marker 损坏/不配对:MOMENTUM_END 出现在 MOMENTUM_START 之前,需人工修复"
|
||||||
|
)
|
||||||
|
return start_idx, end_idx
|
||||||
|
|
||||||
|
|
||||||
|
def momentum_inner(content: str) -> str:
|
||||||
|
"""返回 momentum 受保护区的内层文本(去掉两 marker),无区返回空串。
|
||||||
|
|
||||||
|
与 _momentum_span(含 marker 的整段)的区别:本函数只取两 marker 之间的内层正文,
|
||||||
|
供 run_slow_momentum 的 prev_guidance 使用。prev_guidance 在 LLM 解析失败时会被
|
||||||
|
run_slow_momentum 原样返回、再喂给 replace_momentum;replace_momentum 禁止 guidance
|
||||||
|
含 marker 字面量,故 prev_guidance 必须是无 marker 的内层文本,否则一旦解析回退即
|
||||||
|
在 replace_momentum 抛 ValueError。
|
||||||
|
|
||||||
|
边界判定与配对校验统一委托 momentum_region_bounds:marker 损坏/不配对时由其 raise
|
||||||
|
ValueError,本函数不把损坏态静默当作"无区"。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
content: skill 全文。
|
||||||
|
返回:
|
||||||
|
momentum 区两 marker 之间的内层文本(已 strip);无区返回空串。
|
||||||
|
异常:
|
||||||
|
ValueError: momentum marker 损坏/不配对。
|
||||||
|
"""
|
||||||
|
bounds = momentum_region_bounds(content)
|
||||||
|
if bounds is None:
|
||||||
|
return ""
|
||||||
|
start, end = bounds
|
||||||
|
inner = content[start + len(MOMENTUM_START) : end - len(MOMENTUM_END)].strip()
|
||||||
|
# 去掉 replace_momentum 写入的固定标题行,只回传纯指导文本,使其等价于上一轮
|
||||||
|
# 传给 replace_momentum 的 guidance(解析回退时原样回传不会引入重复标题)。
|
||||||
|
if inner.startswith(MOMENTUM_HEADING):
|
||||||
|
inner = inner[len(MOMENTUM_HEADING) :].lstrip("\n")
|
||||||
|
return inner.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def append_to_appendix(content: str, notes: list[str]) -> str:
|
||||||
|
"""把 LAPSE 提醒追加到文件尾的 appendix 受保护区;区不存在则创建。
|
||||||
|
|
||||||
|
护栏:appendix 区超过 APPENDIX_MAX_CHARS 时 logger.warning(不静默截断,
|
||||||
|
提示人工压缩;不做自动去重——YAGNI,见设计)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
content: 原文。
|
||||||
|
notes: 待追加的提醒文本列表。
|
||||||
|
返回:
|
||||||
|
含 appendix 区的新文本。
|
||||||
|
"""
|
||||||
|
if not notes:
|
||||||
|
return content
|
||||||
|
bullet = "\n".join(f"- {n.strip()}" for n in notes if n.strip())
|
||||||
|
if not bullet:
|
||||||
|
return content
|
||||||
|
if APPENDIX_START in content and APPENDIX_END in content:
|
||||||
|
head, rest = content.split(APPENDIX_START, 1)
|
||||||
|
inner, tail = rest.split(APPENDIX_END, 1)
|
||||||
|
new_inner = f"{inner.rstrip()}\n{bullet}"
|
||||||
|
out = f"{head}{APPENDIX_START}{new_inner}\n{APPENDIX_END}{tail}"
|
||||||
|
else:
|
||||||
|
new_inner = f"\n## 执行提醒(自动累积,勿手改)\n{bullet}"
|
||||||
|
out = f"{content.rstrip()}\n\n{APPENDIX_START}{new_inner}\n{APPENDIX_END}\n"
|
||||||
|
if len(new_inner) > APPENDIX_MAX_CHARS:
|
||||||
|
logger.warning(
|
||||||
|
"appendix 区长度 {} 超过上限 {},建议人工压缩",
|
||||||
|
len(new_inner),
|
||||||
|
APPENDIX_MAX_CHARS,
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def appendix_region_bounds(text: str) -> tuple[int, int] | None:
|
||||||
|
"""定位 appendix 受保护区的字符区间,对损坏态显式报错(P5,对称 momentum_region_bounds)。
|
||||||
|
|
||||||
|
appendix marker 由 append_to_appendix 维护、consolidation 回写,可能出现损坏态。
|
||||||
|
本函数是 appendix 路径的唯一边界判定入口,把配对校验集中一处:
|
||||||
|
|
||||||
|
- START 与 END 各恰好一次且 START 在 END 之前 → 返回 (start_idx, end_idx),
|
||||||
|
end_idx 指向 END marker 结束位置(content[start:end] 含完整两 marker)。
|
||||||
|
- 两 marker 都不出现 → 返回 None(合法的「无区」态)。
|
||||||
|
- 其余(仅一个 marker、END 在 START 前、任一 marker 重复)→ raise ValueError,
|
||||||
|
拒绝静默按字符串切片处理而误拼/吞掉区外正文。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
text: 待检测文本(skill 全文)。
|
||||||
|
返回:
|
||||||
|
(start_idx, end_idx) 表示区间,或 None 表示无 appendix 区。
|
||||||
|
异常:
|
||||||
|
ValueError: appendix marker 损坏/不配对。
|
||||||
|
"""
|
||||||
|
start_count = text.count(APPENDIX_START)
|
||||||
|
end_count = text.count(APPENDIX_END)
|
||||||
|
if start_count == 0 and end_count == 0:
|
||||||
|
return None
|
||||||
|
if start_count != 1 or end_count != 1:
|
||||||
|
raise ValueError(
|
||||||
|
f"appendix marker 损坏/不配对:APPENDIX_START 出现 {start_count} 次、"
|
||||||
|
f"APPENDIX_END 出现 {end_count} 次(各须恰好 1 次),需人工修复"
|
||||||
|
)
|
||||||
|
start_idx = text.index(APPENDIX_START)
|
||||||
|
end_idx = text.index(APPENDIX_END) + len(APPENDIX_END)
|
||||||
|
if start_idx >= text.index(APPENDIX_END):
|
||||||
|
raise ValueError(
|
||||||
|
"appendix marker 损坏/不配对:APPENDIX_END 出现在 APPENDIX_START 之前,需人工修复"
|
||||||
|
)
|
||||||
|
return start_idx, end_idx
|
||||||
|
|
||||||
|
|
||||||
|
def extract_appendix_notes(content: str) -> list[str]:
|
||||||
|
"""从 appendix 受保护区解析出 bullet 提醒列表;无区返回空列表。
|
||||||
|
|
||||||
|
功能:
|
||||||
|
取 appendix 区内每行以 "- " 起头的文本为一条 note(去 "- " 前缀与首尾空白),
|
||||||
|
区内标题行(## 执行提醒…)不计。供 consolidation 读取现有 notes。
|
||||||
|
参数:
|
||||||
|
content: skill 全文。
|
||||||
|
返回:
|
||||||
|
note 字符串列表;无 appendix 区返回 []。
|
||||||
|
异常:
|
||||||
|
ValueError: appendix marker 损坏/不配对(经 appendix_region_bounds,不静默切片)。
|
||||||
|
关键实现细节:
|
||||||
|
边界判定统一委托 appendix_region_bounds,只取两 marker 之间内层正文逐行解析。
|
||||||
|
"""
|
||||||
|
bounds = appendix_region_bounds(content)
|
||||||
|
if bounds is None:
|
||||||
|
return []
|
||||||
|
start, end = bounds
|
||||||
|
inner = content[start + len(APPENDIX_START) : end - len(APPENDIX_END)]
|
||||||
|
notes: list[str] = []
|
||||||
|
for line in inner.splitlines():
|
||||||
|
stripped = line.strip()
|
||||||
|
if stripped.startswith("- "):
|
||||||
|
note = stripped[2:].strip()
|
||||||
|
if note:
|
||||||
|
notes.append(note)
|
||||||
|
return notes
|
||||||
|
|
||||||
|
|
||||||
|
def replace_appendix_notes(content: str, notes: list[str]) -> str:
|
||||||
|
"""用 notes 整体替换 appendix 区内容;notes 为空则删除整个 appendix 区。
|
||||||
|
|
||||||
|
功能:
|
||||||
|
consolidation 回写压缩后 notes 的替换语义(区别于 append_to_appendix 累积):
|
||||||
|
区存在则整体覆盖区内 bullet;notes 空则连 marker 一并删除、保留区外正文;
|
||||||
|
区不存在且 notes 非空则按 append_to_appendix 格式新建。
|
||||||
|
参数:
|
||||||
|
content: 原文(可能含 appendix 区)。
|
||||||
|
notes: 压缩后的提醒列表;空列表表示删区。
|
||||||
|
返回:
|
||||||
|
替换后的全文。
|
||||||
|
异常:
|
||||||
|
ValueError: appendix marker 损坏/不配对(经 appendix_region_bounds)。
|
||||||
|
关键实现细节:
|
||||||
|
边界经 appendix_region_bounds 显式校验,按 (start,end) 切出 head/tail 拼接,
|
||||||
|
不做两次独立 split(避免损坏态误拼/吞掉区外正文)。
|
||||||
|
"""
|
||||||
|
bounds = appendix_region_bounds(content)
|
||||||
|
if bounds is not None:
|
||||||
|
start, end = bounds
|
||||||
|
head = content[:start]
|
||||||
|
tail = content[end:]
|
||||||
|
if not notes:
|
||||||
|
return head.rstrip() + ("\n" + tail.lstrip("\n") if tail.strip() else "\n")
|
||||||
|
bullet = "\n".join(f"- {n.strip()}" for n in notes if n.strip())
|
||||||
|
new_inner = f"\n## 执行提醒(自动累积,勿手改)\n{bullet}"
|
||||||
|
return f"{head}{APPENDIX_START}{new_inner}\n{APPENDIX_END}{tail}"
|
||||||
|
if not notes:
|
||||||
|
return content
|
||||||
|
return append_to_appendix(content, notes)
|
||||||
|
|
||||||
|
|
||||||
|
def replace_momentum(content: str, guidance: str) -> str:
|
||||||
|
"""把「动量指导」整体写入文件尾的 momentum 受保护区;区不存在则创建。
|
||||||
|
|
||||||
|
与 append_to_appendix 的累积语义不同,momentum 是**替换**语义:慢更新周期每
|
||||||
|
epoch 末整体重写一段动量指导,旧指导被完全覆盖(不保留历史)。momentum 区与
|
||||||
|
appendix 区独立共存——本函数只触碰 momentum marker,不破坏已有 appendix 区。
|
||||||
|
|
||||||
|
护栏:momentum 区超过 MOMENTUM_MAX_CHARS 时 logger.warning(不静默截断,与
|
||||||
|
appendix 对齐)。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
- 替换非追加:区已存在时用 guidance 整体覆盖 marker 内 inner,旧动量不残留。
|
||||||
|
- 创建位置在文件尾(append_to_appendix 同样在文件尾,但两区 marker 不同,
|
||||||
|
split 按各自 marker 定位,互不干扰)。
|
||||||
|
|
||||||
|
空 guidance 决策:与 appendix 的累积语义不同,momentum 是「每轮整体重写」,空
|
||||||
|
guidance 表示「本轮无动量指导」,属合法语义——照常写入(区内仅留标题,旧动量被清空),
|
||||||
|
而非返回原文保留旧动量。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
content: 原文(可能已含 appendix 区)。
|
||||||
|
guidance: 本轮动量指导全文(整体覆盖旧动量)。
|
||||||
|
返回:
|
||||||
|
含 momentum 区的新文本。
|
||||||
|
异常:
|
||||||
|
ValueError: guidance 含 momentum marker 字面量(外部输入注入),或原文 momentum
|
||||||
|
marker 损坏/不配对。
|
||||||
|
"""
|
||||||
|
if MOMENTUM_START in guidance or MOMENTUM_END in guidance:
|
||||||
|
raise ValueError(
|
||||||
|
"guidance 不得包含 momentum marker 字面量"
|
||||||
|
f"({MOMENTUM_START} / {MOMENTUM_END}),否则会破坏 marker 配对"
|
||||||
|
)
|
||||||
|
bounds = momentum_region_bounds(content)
|
||||||
|
new_inner = f"\n## 动量指导(每轮重写,勿手改)\n{guidance.strip()}"
|
||||||
|
if bounds is not None:
|
||||||
|
start_idx, end_idx = bounds
|
||||||
|
head = content[:start_idx]
|
||||||
|
tail = content[end_idx:]
|
||||||
|
out = f"{head}{MOMENTUM_START}{new_inner}\n{MOMENTUM_END}{tail}"
|
||||||
|
else:
|
||||||
|
out = f"{content.rstrip()}\n\n{MOMENTUM_START}{new_inner}\n{MOMENTUM_END}\n"
|
||||||
|
if len(new_inner) > MOMENTUM_MAX_CHARS:
|
||||||
|
logger.warning(
|
||||||
|
"momentum 区长度 {} 超过上限 {},建议人工压缩",
|
||||||
|
len(new_inner),
|
||||||
|
MOMENTUM_MAX_CHARS,
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _protected_ranges(content: str, spans: list[str]) -> list[tuple[int, int]]:
|
||||||
|
"""把冻结文本块映射成 content 中的 [start, end) 坐标区间。"""
|
||||||
|
ranges: list[tuple[int, int]] = []
|
||||||
|
for span in spans:
|
||||||
|
idx = content.find(span)
|
||||||
|
if idx != -1:
|
||||||
|
ranges.append((idx, idx + len(span)))
|
||||||
|
return ranges
|
||||||
|
|
||||||
|
|
||||||
|
def _in_ranges(pos: int, ranges: list[tuple[int, int]]) -> bool:
|
||||||
|
"""判断位置 pos 是否落在任意冻结区间内。"""
|
||||||
|
return any(start <= pos < end for start, end in ranges)
|
||||||
|
|
||||||
|
|
||||||
|
def _append_at(content: str, ranges: list[tuple[int, int]]) -> int:
|
||||||
|
"""append/退化追加落点:最早一个 start>0 的冻结区之前;无则文末(头部 frontmatter 不计)。"""
|
||||||
|
starts = [start for start, _ in ranges if start > 0]
|
||||||
|
return min(starts) if starts else len(content)
|
||||||
|
|
||||||
|
|
||||||
|
def _insert_at(content: str, at: int, payload: str) -> str:
|
||||||
|
"""在 at 位置插入 payload,自动补换行保持段落格式。"""
|
||||||
|
head, tail = content[:at].rstrip(), content[at:].lstrip("\n")
|
||||||
|
if tail:
|
||||||
|
return head + "\n\n" + payload + "\n\n" + tail
|
||||||
|
return head + "\n\n" + payload + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def _do_append(
|
||||||
|
content: str, payload: str, ranges: list[tuple[int, int]]
|
||||||
|
) -> tuple[str, str]:
|
||||||
|
"""执行 append 操作,返回更新后内容与状态字符串。"""
|
||||||
|
return _insert_at(content, _append_at(content, ranges), payload), "applied_append"
|
||||||
|
|
||||||
|
|
||||||
|
def _do_insert_after(
|
||||||
|
content: str, target: str, payload: str, ranges: list[tuple[int, int]]
|
||||||
|
) -> tuple[str, str]:
|
||||||
|
"""执行 insert_after 操作,处理退化追加与冻结区跳过。"""
|
||||||
|
pos = content.find(target) if target else -1
|
||||||
|
if pos == -1:
|
||||||
|
logger.warning("insert_after 锚点缺失,退化为追加 target={}", target[:80])
|
||||||
|
return (
|
||||||
|
_insert_at(content, _append_at(content, ranges), payload),
|
||||||
|
"applied_insert_after_fallback",
|
||||||
|
)
|
||||||
|
if _in_ranges(pos, ranges):
|
||||||
|
logger.warning("insert_after 目标在冻结区,跳过 target={}", target[:80])
|
||||||
|
return content, "skipped_protected"
|
||||||
|
at = pos + len(target)
|
||||||
|
nl = content.find("\n", at)
|
||||||
|
at = nl + 1 if nl != -1 else len(content)
|
||||||
|
return content[:at] + payload + "\n" + content[at:], "applied_insert_after"
|
||||||
|
|
||||||
|
|
||||||
|
def _do_replace_delete(
|
||||||
|
op: str,
|
||||||
|
content: str,
|
||||||
|
target: str,
|
||||||
|
payload: str,
|
||||||
|
ranges: list[tuple[int, int]],
|
||||||
|
) -> tuple[str, str]:
|
||||||
|
"""执行 replace 或 delete 操作,返回更新后内容与状态字符串。"""
|
||||||
|
if not target:
|
||||||
|
return content, "skipped_missing_target"
|
||||||
|
pos = content.find(target)
|
||||||
|
if pos == -1:
|
||||||
|
logger.warning("{} 锚点缺失,跳过 target={}", op, target[:80])
|
||||||
|
return content, "skipped_target_not_found"
|
||||||
|
if _in_ranges(pos, ranges):
|
||||||
|
logger.warning("{} 目标在冻结区,跳过 target={}", op, target[:80])
|
||||||
|
return content, "skipped_protected"
|
||||||
|
new_content = content.replace(target, payload if op == "replace" else "", 1)
|
||||||
|
return new_content, "applied_" + op
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_one(
|
||||||
|
content: str, edit: dict, ranges: list[tuple[int, int]]
|
||||||
|
) -> tuple[str, dict]:
|
||||||
|
"""应用单条 edit,返回 (更新后内容, 状态报告)。"""
|
||||||
|
if not isinstance(edit, dict):
|
||||||
|
return content, {
|
||||||
|
"op": "",
|
||||||
|
"target": "",
|
||||||
|
"content_preview": "",
|
||||||
|
"status": "error",
|
||||||
|
"error": f"edit 非 dict: {type(edit).__name__}",
|
||||||
|
}
|
||||||
|
op = str(edit.get("op", ""))
|
||||||
|
target = str(edit.get("target", "") or "")
|
||||||
|
payload = str(edit.get("content", "") or "").strip()
|
||||||
|
report = {
|
||||||
|
"op": op,
|
||||||
|
"target": target[:200],
|
||||||
|
"content_preview": payload[:200],
|
||||||
|
"status": "unknown",
|
||||||
|
}
|
||||||
|
|
||||||
|
if op == "append":
|
||||||
|
content, report["status"] = _do_append(content, payload, ranges)
|
||||||
|
return content, report
|
||||||
|
|
||||||
|
if op == "insert_after":
|
||||||
|
content, report["status"] = _do_insert_after(content, target, payload, ranges)
|
||||||
|
return content, report
|
||||||
|
|
||||||
|
if op in ("replace", "delete"):
|
||||||
|
content, report["status"] = _do_replace_delete(
|
||||||
|
op, content, target, payload, ranges
|
||||||
|
)
|
||||||
|
return content, report
|
||||||
|
|
||||||
|
logger.warning("未知 op,跳过: {}", op)
|
||||||
|
report["status"] = "skipped_unknown_op"
|
||||||
|
return content, report
|
||||||
|
|
||||||
|
|
||||||
|
def apply_patch_with_report(
|
||||||
|
content: str,
|
||||||
|
edits: list[dict],
|
||||||
|
protected_spans: list[str] | None = None,
|
||||||
|
) -> tuple[str, list[dict]]:
|
||||||
|
"""顺序应用 edit 列表,返回 (新内容, 逐条状态报告)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
content: 原始文本。
|
||||||
|
edits: 每条 {op, target, content}。
|
||||||
|
protected_spans: 冻结文本块列表;目标落入其坐标区间即跳过,append 插到其前。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
(应用后文本, reports);reports 每条含 op/target/content_preview/status/index。
|
||||||
|
"""
|
||||||
|
spans = protected_spans or []
|
||||||
|
reports: list[dict] = []
|
||||||
|
for i, edit in enumerate(edits, 1):
|
||||||
|
try:
|
||||||
|
ranges = _protected_ranges(content, spans)
|
||||||
|
content, report = _apply_one(content, edit, ranges)
|
||||||
|
except (KeyError, TypeError, ValueError, AttributeError) as exc:
|
||||||
|
report = {
|
||||||
|
"op": "",
|
||||||
|
"target": "",
|
||||||
|
"content_preview": "",
|
||||||
|
"status": "error",
|
||||||
|
"error": str(exc),
|
||||||
|
}
|
||||||
|
logger.exception("补丁应用异常 index={}", i)
|
||||||
|
report["index"] = i
|
||||||
|
reports.append(report)
|
||||||
|
return content, reports
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
"""core/evolution/ 子包的只读 Protocol 定义。
|
||||||
|
|
||||||
|
三个 Protocol 均为只读——core/ 返回结果 dataclass,写入由 app/ 持久化。
|
||||||
|
SkillStore / PromptStore 为同步(文件读取量小且快),RunLog 为异步
|
||||||
|
(隔离 SQLite 查询,core/ 不写 SQL)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Protocol, runtime_checkable
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class SkillStore(Protocol):
|
||||||
|
"""版本化技能读取端口。
|
||||||
|
|
||||||
|
实现方解析 manifest 指针,core/ 不感知版本号。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def read_skill(self, filename: str) -> str:
|
||||||
|
"""读取指定 skill 文件的全文内容。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
filename: skill 文件名,如 'temporal-reasoning.md'。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
文件全文内容。
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
def list_skill_files(self) -> list[str]:
|
||||||
|
"""列出当前版本所有 skill 文件名。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
文件名列表。
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class PromptStore(Protocol):
|
||||||
|
"""版本化提示词读取端口。
|
||||||
|
|
||||||
|
覆盖 system.md 和 tool extract/verify 文件。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def read_prompt(self, filename: str) -> str:
|
||||||
|
"""读取指定 prompt 文件的全文内容。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
filename: prompt 文件名,如 'system.md'。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
文件全文内容。
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
def list_prompt_files(self) -> list[str]:
|
||||||
|
"""列出当前版本所有 prompt 文件名。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
文件名列表。
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class RunLog(Protocol):
|
||||||
|
"""实验日志查询端口。
|
||||||
|
|
||||||
|
隔离 SQLite 实现细节,core/ 不写 SQL。
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def get_predictions(
|
||||||
|
self,
|
||||||
|
run_id: str,
|
||||||
|
*,
|
||||||
|
question_ids: list[str] | None = None,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""查询指定 run 的预测记录。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
run_id: 运行标识。
|
||||||
|
question_ids: 可选的题目 ID 过滤列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
预测记录字典列表。
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
async def get_traces(
|
||||||
|
self,
|
||||||
|
run_id: str,
|
||||||
|
*,
|
||||||
|
question_ids: list[str] | None = None,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""查询指定 run 的推理轨迹。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
run_id: 运行标识。
|
||||||
|
question_ids: 可选的题目 ID 过滤列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
轨迹记录字典列表。
|
||||||
|
"""
|
||||||
|
...
|
||||||
@@ -0,0 +1,484 @@
|
|||||||
|
"""core/evolution 子包的数据类型定义。
|
||||||
|
|
||||||
|
自进化循环中 gate、diagnose、evolve、validate 共用的 dataclass。
|
||||||
|
所有输出类型默认 frozen=True(一次性构造、不可变),唯一例外是
|
||||||
|
EvolutionRecord(构建过程中需要多次修改状态)。
|
||||||
|
|
||||||
|
不依赖 app/ 或 adapters/。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# 1. Gate 决策类型
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class GateParams:
|
||||||
|
"""CE-Gate 判据阈值组(从实验配置构造)。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
e_confirm: CONFIRMED 接受的 e 值门槛(1/alpha,20 对应 alpha=5%)。
|
||||||
|
e_provisional: 题尽暂定接受门槛,同时是 futility 出口的代数界。
|
||||||
|
w_net_min: 题尽暂定接受要求的最小净胜 W-L。
|
||||||
|
delta_min: 接受要求的最小点估计效应量 (W-L)/n_used。
|
||||||
|
lambda_dir: Wald 方向游走的拒绝阈值(负数)。
|
||||||
|
e_rollback: 试用期结算的对称回滚 e 值门槛(1/alpha',10 对应 10%)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
e_confirm: float
|
||||||
|
e_provisional: float
|
||||||
|
w_net_min: int
|
||||||
|
delta_min: float
|
||||||
|
lambda_dir: float
|
||||||
|
e_rollback: float
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class GateVerdict:
|
||||||
|
"""一次块间判定的完整结果(判定 + 全部诊断量)。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
decision: 判定结果,取值为 continue / accept_confirmed /
|
||||||
|
reject_directional / reject_futility / accept_provisional /
|
||||||
|
reject_inertia 之一。
|
||||||
|
e_value: 当前 e 值。
|
||||||
|
wald_lambda: 当前 Wald 方向游走值。
|
||||||
|
delta_hat: 点估计效应量 (W-L)/n_used;n_used=0 时为 0。
|
||||||
|
delta_shrunk: 收缩点估计 (W-L)/(n_used+4),仅观测用。
|
||||||
|
"""
|
||||||
|
|
||||||
|
decision: str
|
||||||
|
e_value: float
|
||||||
|
wald_lambda: float
|
||||||
|
delta_hat: float
|
||||||
|
delta_shrunk: float
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# 2. 诊断类型
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SpanMetrics:
|
||||||
|
"""单次工具调用的输出质量指标。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
step: 工具调用所在的步骤编号。
|
||||||
|
tool_name: 本次调用使用的工具名称。
|
||||||
|
extraction_completeness: 信息提取完整度。
|
||||||
|
hallucination_rate: 幻觉内容占比。
|
||||||
|
missed_info_tags: 未提取信息的标签列表。
|
||||||
|
hallucination_tags: 幻觉内容的标签列表。
|
||||||
|
"""
|
||||||
|
|
||||||
|
step: int
|
||||||
|
tool_name: str
|
||||||
|
extraction_completeness: float
|
||||||
|
hallucination_rate: float
|
||||||
|
missed_info_tags: list[str] = field(default_factory=list)
|
||||||
|
hallucination_tags: list[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SkillStepAdherence:
|
||||||
|
"""单个 skill step 的遵循判定。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
step_label: 被判定的步骤标签。
|
||||||
|
adhered: 该步骤是否被遵循。
|
||||||
|
description: 对遵循情况的文字说明。
|
||||||
|
"""
|
||||||
|
|
||||||
|
step_label: str
|
||||||
|
adhered: bool
|
||||||
|
description: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class QuestionMetrics:
|
||||||
|
"""单题的完整指标,即 Stage 1 输出。
|
||||||
|
|
||||||
|
包含 7 个规则指标和 5 类 judge 指标(span / missed / adherence /
|
||||||
|
bias / sufficiency)。frozen=True 保证构造后不可变。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
question_id: 题目唯一标识。
|
||||||
|
video_id: 对应视频唯一标识。
|
||||||
|
task_type: 题目任务类型。
|
||||||
|
correct: 该题最终是否答对。
|
||||||
|
format_compliance: 输出格式遵循程度。
|
||||||
|
budget_usage: 预算使用比例。
|
||||||
|
confidence_calibration: 置信度校准结论。
|
||||||
|
repeat_visit_rate: 重复访问节点的比例。
|
||||||
|
search_keyword_repetition: 搜索关键词重复率。
|
||||||
|
level_jump_pattern: 层级跳转模式描述。
|
||||||
|
tool_usage: 各工具的调用次数统计。
|
||||||
|
span_metrics: 该题全部工具调用的片段级质量指标。
|
||||||
|
missed_nodes: 该题遗漏的节点列表。
|
||||||
|
skill_adherence: 该题对 skill 步骤的遵循情况。
|
||||||
|
confirmation_bias: 是否出现确认偏误。None 表示 judge 不可用。
|
||||||
|
evidence_sufficient: 当前证据是否充足。None 表示 judge 不可用。
|
||||||
|
degraded: 是否为降级指标(judge 解析失败时生成)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
question_id: str
|
||||||
|
video_id: str
|
||||||
|
task_type: str
|
||||||
|
correct: bool
|
||||||
|
format_compliance: float
|
||||||
|
budget_usage: float
|
||||||
|
confidence_calibration: str
|
||||||
|
repeat_visit_rate: float
|
||||||
|
search_keyword_repetition: float
|
||||||
|
level_jump_pattern: str
|
||||||
|
tool_usage: dict[str, int]
|
||||||
|
span_metrics: list[SpanMetrics]
|
||||||
|
missed_nodes: list[str]
|
||||||
|
skill_adherence: list[SkillStepAdherence]
|
||||||
|
confirmation_bias: bool | None
|
||||||
|
evidence_sufficient: bool | None
|
||||||
|
degraded: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ErrorAttribution:
|
||||||
|
"""D1 错误归因。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
question_id: 发生错误归因的题目唯一标识。
|
||||||
|
error_type: 错误的主要类别。
|
||||||
|
reasoning_failure_type: 推理失败类型;若不适用则为 None。
|
||||||
|
cause_category: C3 病因:'defect'/'lapse';正确题/INFRA/未判为 None。
|
||||||
|
lapse_note: LAPSE 提醒文本(供 appendix 路由);非 LAPSE 为 None。
|
||||||
|
"""
|
||||||
|
|
||||||
|
question_id: str
|
||||||
|
error_type: str
|
||||||
|
reasoning_failure_type: str | None
|
||||||
|
cause_category: str | None = None
|
||||||
|
lapse_note: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class CaseSample:
|
||||||
|
"""单个案例样本,进化模块的最小输入单元。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
question_id: 题目唯一标识。
|
||||||
|
video_id: 对应视频唯一标识。
|
||||||
|
task_type: 题目任务类型。
|
||||||
|
question: 题目文本。
|
||||||
|
options: 选项列表。
|
||||||
|
answer: 正确答案。
|
||||||
|
prediction: Agent 预测答案。
|
||||||
|
correct: 是否答对。
|
||||||
|
error_type: 错误类型;正确题为 None。
|
||||||
|
selection_reason: 被选为案例的原因说明。
|
||||||
|
metrics: QuestionMetrics 的关键字段子集。
|
||||||
|
trace: 完整推理轨迹,不截断。
|
||||||
|
"""
|
||||||
|
|
||||||
|
question_id: str
|
||||||
|
video_id: str
|
||||||
|
task_type: str
|
||||||
|
question: str
|
||||||
|
options: list[str]
|
||||||
|
answer: str
|
||||||
|
prediction: str | None
|
||||||
|
correct: bool
|
||||||
|
error_type: str | None
|
||||||
|
selection_reason: str
|
||||||
|
metrics: dict[str, Any]
|
||||||
|
trace: list[dict[str, Any]]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SkillCasePack:
|
||||||
|
"""单个 task_type 的案例包,服务于 Skill 进化。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
task_type: 题目任务类型。
|
||||||
|
target_file: 对应 skill 文件名,如 'temporal-reasoning.md'。
|
||||||
|
stats: 从 D3/D4 提取的该题型统计。
|
||||||
|
failure_cases: 失败案例列表。
|
||||||
|
success_cases: 成功案例列表。
|
||||||
|
lapse_notes: C3 LAPSE 提醒文本列表(路由进 appendix 受保护区)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
task_type: str
|
||||||
|
target_file: str
|
||||||
|
stats: dict[str, Any]
|
||||||
|
failure_cases: list[CaseSample] = field(default_factory=list)
|
||||||
|
success_cases: list[CaseSample] = field(default_factory=list)
|
||||||
|
lapse_notes: list[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SystemCasePack:
|
||||||
|
"""跨题型行为模式案例包,服务于 System Prompt 进化。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
stats: 从 D5 提取的行为模式统计。
|
||||||
|
failure_cases: 失败案例列表。
|
||||||
|
success_cases: 成功案例列表。
|
||||||
|
"""
|
||||||
|
|
||||||
|
stats: dict[str, Any]
|
||||||
|
failure_cases: list[CaseSample] = field(default_factory=list)
|
||||||
|
success_cases: list[CaseSample] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ToolCasePack:
|
||||||
|
"""单个 tool_name 的案例包,服务于 Tool Prompt 进化。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
tool_name: 工具名称。
|
||||||
|
target_files: 对应 prompt 文件名列表。
|
||||||
|
stats: 从 D2 提取的工具质量统计。
|
||||||
|
failure_spans: 失败 span 案例列表。
|
||||||
|
success_spans: 成功 span 案例列表。
|
||||||
|
"""
|
||||||
|
|
||||||
|
tool_name: str
|
||||||
|
target_files: list[str]
|
||||||
|
stats: dict[str, Any]
|
||||||
|
failure_spans: list[dict[str, Any]] = field(default_factory=list)
|
||||||
|
success_spans: list[dict[str, Any]] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DiagnosisResult:
|
||||||
|
"""完整诊断报告,即两阶段诊断管线的最终输出。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
run_id: 本次诊断运行的唯一标识。
|
||||||
|
filter_summary: 筛选条件与筛选结果摘要。
|
||||||
|
error_attributions: 错误归因结果列表。
|
||||||
|
attribution_distribution: 各归因类别的分布统计。
|
||||||
|
reasoning_failure_types: 各推理失败类型的分布统计。
|
||||||
|
tool_quality: 按工具聚合的质量分析结果。
|
||||||
|
search_effectiveness: 搜索有效性的聚合统计。
|
||||||
|
skill_compliance: 技能遵循情况的聚合统计。
|
||||||
|
decision_patterns: 决策模式与行为模式摘要。
|
||||||
|
skill_case_packs: 按题型组织的 Skill 进化案例包。
|
||||||
|
system_case_pack: 跨题型行为模式案例包;无系统性问题时为 None。
|
||||||
|
tool_case_packs: 按工具名组织的 Tool Prompt 进化案例包。
|
||||||
|
infra_excluded_count: C3:被 stop_reason 排除的题数。
|
||||||
|
infra_excluded_ratio: INFRA 占总题数比例。
|
||||||
|
infra_question_ids: 被排除题 question_id 列表。
|
||||||
|
defect_count: 进入诊断池错题中判为 DEFECT 的数量。
|
||||||
|
lapse_count: 进入诊断池错题中判为 LAPSE 的数量。
|
||||||
|
degraded_count: judge 解析失败而降级的题数。
|
||||||
|
degraded_question_ids: 降级题的 question_id 列表。
|
||||||
|
"""
|
||||||
|
|
||||||
|
run_id: str
|
||||||
|
filter_summary: dict[str, Any] = field(default_factory=dict)
|
||||||
|
error_attributions: list[ErrorAttribution] = field(default_factory=list)
|
||||||
|
attribution_distribution: dict[str, int] = field(default_factory=dict)
|
||||||
|
reasoning_failure_types: dict[str, int] = field(default_factory=dict)
|
||||||
|
tool_quality: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||||
|
search_effectiveness: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||||
|
skill_compliance: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||||
|
decision_patterns: dict[str, Any] = field(default_factory=dict)
|
||||||
|
skill_case_packs: dict[str, SkillCasePack] = field(default_factory=dict)
|
||||||
|
system_case_pack: SystemCasePack | None = None
|
||||||
|
tool_case_packs: dict[str, ToolCasePack] = field(default_factory=dict)
|
||||||
|
infra_excluded_count: int = 0
|
||||||
|
infra_excluded_ratio: float = 0.0
|
||||||
|
infra_question_ids: list[str] = field(default_factory=list)
|
||||||
|
defect_count: int = 0
|
||||||
|
lapse_count: int = 0
|
||||||
|
degraded_count: int = 0
|
||||||
|
degraded_question_ids: list[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# 3. 进化类型
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class EvolutionRecord:
|
||||||
|
"""单个目标文件的一次进化记录。
|
||||||
|
|
||||||
|
构建过程中需要多次修改状态(如 status、result_version),
|
||||||
|
因此是唯一不使用 frozen=True 的类型。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
target_file: 目标文件名,如 'temporal-reasoning.md'。
|
||||||
|
target_type: 目标类型: 'skill' / 'system' / 'tool'。
|
||||||
|
original_content: 改写前原文。
|
||||||
|
evolved_content: 改写后内容;rejected 时与 original_content 相同。
|
||||||
|
reason: 状态说明。
|
||||||
|
status: 'accepted' / 'rejected' / 'skipped'。
|
||||||
|
source_version: 改写前版本号,如 'v1'。
|
||||||
|
result_version: 改写后版本号;rejected/skipped 时为 None。
|
||||||
|
suggestions: LLM 输出的改动建议列表。
|
||||||
|
attempts: 每次 LLM 调用的原始响应摘要。
|
||||||
|
validation_errors: 验证失败的具体原因。
|
||||||
|
edits: LLM 输出的补丁列表。
|
||||||
|
apply_report: 补丁逐条应用状态。
|
||||||
|
clip_info: 超预算裁剪信息。
|
||||||
|
"""
|
||||||
|
|
||||||
|
target_file: str
|
||||||
|
target_type: str
|
||||||
|
original_content: str
|
||||||
|
evolved_content: str
|
||||||
|
reason: str
|
||||||
|
status: str
|
||||||
|
source_version: str
|
||||||
|
result_version: str | None = None
|
||||||
|
suggestions: list[dict[str, Any]] = field(default_factory=list)
|
||||||
|
attempts: list[dict[str, Any]] = field(default_factory=list)
|
||||||
|
validation_errors: list[str] = field(default_factory=list)
|
||||||
|
edits: list[dict[str, Any]] = field(default_factory=list)
|
||||||
|
apply_report: list[dict[str, Any]] = field(default_factory=list)
|
||||||
|
clip_info: dict[str, Any] = field(default_factory=lambda: {"triggered": False, "clipped": 0})
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RejectedEdit:
|
||||||
|
"""已在验证阶段证明无效的历史改法摘要。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
target_file: 目标文件名,如 'temporal-reasoning.md'。
|
||||||
|
target_type: 目标类型: 'skill' / 'system' / 'tool'。
|
||||||
|
change_summary: 被验证为无效的改法摘要。
|
||||||
|
delta: 该改法对应候选相对基线的准确率变化。
|
||||||
|
source_version: 该改法来源的版本号,如 'v2'。
|
||||||
|
epoch: 该改法所属的进化轮次。
|
||||||
|
gate_w: CE-Gate 证据:配对翻转 W(基线错到候选对)。
|
||||||
|
gate_l: CE-Gate 证据:配对翻转 L(基线对到候选错)。
|
||||||
|
gate_e_value: CE-Gate 证据:终态 e 值。
|
||||||
|
gate_delta_shrunk: CE-Gate 证据:收缩效应量(观测用)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
target_file: str
|
||||||
|
target_type: str
|
||||||
|
change_summary: str
|
||||||
|
delta: float
|
||||||
|
source_version: str
|
||||||
|
epoch: int
|
||||||
|
gate_w: int | None = None
|
||||||
|
gate_l: int | None = None
|
||||||
|
gate_e_value: float | None = None
|
||||||
|
gate_delta_shrunk: float | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class EvolutionResult:
|
||||||
|
"""一次整体进化流程的汇总结果。
|
||||||
|
|
||||||
|
由 app/harness/ 编排层组装。不含 skills_version / prompts_version
|
||||||
|
(版本管理是 app/ 职责,不属于 core/ 决策内核)。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
records: 所有目标的进化记录。
|
||||||
|
accepted_count: 通过验证的改写数。
|
||||||
|
rejected_count: 未通过验证的改写数。
|
||||||
|
skipped_count: 因无失败案例而跳过的目标数。
|
||||||
|
"""
|
||||||
|
|
||||||
|
records: list[EvolutionRecord] = field(default_factory=list)
|
||||||
|
accepted_count: int = 0
|
||||||
|
rejected_count: int = 0
|
||||||
|
skipped_count: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# 4. 验证辅助类型
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PairResult:
|
||||||
|
"""块验证配对比对结果。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
w: 基线错、候选对的翻转数。
|
||||||
|
l: 基线对、候选错的翻转数。
|
||||||
|
observed: 每题的 (基线是否正确, 候选是否正确) 记录。
|
||||||
|
"""
|
||||||
|
|
||||||
|
w: int
|
||||||
|
l: int # noqa: E741 — 数学记号 W/L(win/loss),与 gate.py 一致
|
||||||
|
observed: dict[str, tuple[bool, bool]]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class QuadrantClassification:
|
||||||
|
"""块验证四象限分类。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
improvements: 基线错、候选对的题目 ID 列表。
|
||||||
|
regressions: 基线对、候选错的题目 ID 列表。
|
||||||
|
persistent_fails: 两臂均错的题目 ID 列表。
|
||||||
|
stable_successes: 两臂均对的题目 ID 列表。
|
||||||
|
"""
|
||||||
|
|
||||||
|
improvements: list[str]
|
||||||
|
regressions: list[str]
|
||||||
|
persistent_fails: list[str]
|
||||||
|
stable_successes: list[str]
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# 5. Prompt 模板束
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DiagnosePrompts:
|
||||||
|
"""诊断管线所需的全部固定模板束。
|
||||||
|
|
||||||
|
由调用方加载后以 frozen dataclass 传入,避免 core/ 依赖文件系统。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
defect_vs_lapse: defect/lapse 病因判别模板。
|
||||||
|
reasoning_sub: 推理失败子分类模板。
|
||||||
|
span_eval_system: span 评估系统提示模板。
|
||||||
|
span_eval_user: span 评估用户提示模板。
|
||||||
|
missed_nodes: 遗漏节点检测模板。
|
||||||
|
skill_adherence: 技能遵循判定模板。
|
||||||
|
confirmation_bias: 确认偏误检测模板。
|
||||||
|
evidence_sufficiency: 证据充足性判定模板。
|
||||||
|
"""
|
||||||
|
|
||||||
|
defect_vs_lapse: str
|
||||||
|
reasoning_sub: str
|
||||||
|
span_eval_system: str
|
||||||
|
span_eval_user: str
|
||||||
|
missed_nodes: str
|
||||||
|
skill_adherence: str
|
||||||
|
confirmation_bias: str
|
||||||
|
evidence_sufficiency: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class EvolvePrompts:
|
||||||
|
"""进化引擎所需的全部固定模板束。
|
||||||
|
|
||||||
|
由调用方加载后以 frozen dataclass 传入,避免 core/ 依赖文件系统。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
evolve_skill: Skill 进化提示模板。
|
||||||
|
evolve_system: System Prompt 进化提示模板。
|
||||||
|
evolve_tool: Tool Prompt 进化提示模板。
|
||||||
|
evolve_rank: 编辑排序提示模板。
|
||||||
|
consolidate_system: appendix 压缩系统提示。
|
||||||
|
"""
|
||||||
|
|
||||||
|
evolve_skill: str
|
||||||
|
evolve_system: str
|
||||||
|
evolve_tool: str
|
||||||
|
evolve_rank: str
|
||||||
|
consolidate_system: str
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
"""core/evolution/validate.py — 块验证纯决策函数。
|
||||||
|
|
||||||
|
算法 #7(块顺序验证)的局部实现:pair_block 逐题比对基线与候选、
|
||||||
|
classify_quadrants 四象限分类、compute_accuracy 纯算术准确率。
|
||||||
|
|
||||||
|
三个函数均为纯函数,无副作用、无外部依赖。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from core.evolution.types import PairResult, QuadrantClassification
|
||||||
|
|
||||||
|
|
||||||
|
def pair_block(
|
||||||
|
baseline: dict[str, bool],
|
||||||
|
candidate: dict[str, bool],
|
||||||
|
question_ids: list[str],
|
||||||
|
) -> PairResult:
|
||||||
|
"""逐题比对基线与候选对错,统计翻转。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
baseline: 基线臂每题正确性映射。
|
||||||
|
candidate: 候选臂每题正确性映射。
|
||||||
|
question_ids: 参与比对的题目 ID 列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
PairResult,包含 w(基线错→候选对翻转数)、l(基线对→候选错翻转数)
|
||||||
|
和 observed(每题的 (基线, 候选) 对错记录)。
|
||||||
|
"""
|
||||||
|
w = l = 0 # noqa: E741 — 数学记号 W/L(win/loss),与 gate.py 一致
|
||||||
|
observed: dict[str, tuple[bool, bool]] = {}
|
||||||
|
for qid in question_ids:
|
||||||
|
b, c = baseline[qid], candidate[qid]
|
||||||
|
observed[qid] = (b, c)
|
||||||
|
if not b and c:
|
||||||
|
w += 1
|
||||||
|
elif b and not c:
|
||||||
|
l += 1 # noqa: E741
|
||||||
|
return PairResult(w=w, l=l, observed=observed)
|
||||||
|
|
||||||
|
|
||||||
|
def classify_quadrants(
|
||||||
|
observed: dict[str, tuple[bool, bool]],
|
||||||
|
) -> QuadrantClassification:
|
||||||
|
"""按 (baseline, candidate) 四组分类,各组内 sorted。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
observed: 每题的 (基线是否正确, 候选是否正确) 记录。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
QuadrantClassification,四个象限各含排序后的题目 ID 列表。
|
||||||
|
"""
|
||||||
|
improvements: list[str] = []
|
||||||
|
regressions: list[str] = []
|
||||||
|
persistent_fails: list[str] = []
|
||||||
|
stable_successes: list[str] = []
|
||||||
|
for qid, (prev, curr) in observed.items():
|
||||||
|
if not prev and curr:
|
||||||
|
improvements.append(qid)
|
||||||
|
elif prev and not curr:
|
||||||
|
regressions.append(qid)
|
||||||
|
elif not prev and not curr:
|
||||||
|
persistent_fails.append(qid)
|
||||||
|
else:
|
||||||
|
stable_successes.append(qid)
|
||||||
|
return QuadrantClassification(
|
||||||
|
improvements=sorted(improvements),
|
||||||
|
regressions=sorted(regressions),
|
||||||
|
persistent_fails=sorted(persistent_fails),
|
||||||
|
stable_successes=sorted(stable_successes),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def compute_accuracy(
|
||||||
|
correctness: dict[str, bool],
|
||||||
|
question_ids: list[str],
|
||||||
|
) -> float:
|
||||||
|
"""纯算术:sum(correct) / len(ids)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
correctness: 每题正确性映射。
|
||||||
|
question_ids: 参与计算的题目 ID 列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
准确率浮点数。question_ids 为空时抛出 ZeroDivisionError。
|
||||||
|
"""
|
||||||
|
return sum(correctness[qid] for qid in question_ids) / len(question_ids)
|
||||||
+4
-1
@@ -4,10 +4,13 @@ LLMProvider / VLMProvider / TelemetryRecorder 是跨子包共享接口,
|
|||||||
被 core/agent/、core/evolution/、app/ 各模块引用。
|
被 core/agent/、core/evolution/、app/ 各模块引用。
|
||||||
adapters/ 提供具体实现。
|
adapters/ 提供具体实现。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Protocol, runtime_checkable
|
|
||||||
|
|
||||||
from core.types import LLMResponse
|
from core.types import LLMResponse
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
"""跨模块共享类型。"""
|
"""跨模块共享类型。"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
@@ -22,3 +23,31 @@ class LLMResponse:
|
|||||||
max_inter_token_ms: float | None
|
max_inter_token_ms: float | None
|
||||||
cache_hit: bool
|
cache_hit: bool
|
||||||
call_id: str
|
call_id: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class GeneratedQuestion:
|
||||||
|
"""单条生成/加载的题目。
|
||||||
|
|
||||||
|
跨层共享类型,被 core/evolution/ 和 app/harness/、app/question_gen/ 使用。
|
||||||
|
frozen=True 确保题目不可变。
|
||||||
|
|
||||||
|
属性:
|
||||||
|
question_id: 题目唯一标识。
|
||||||
|
video_id: 所属视频标识。
|
||||||
|
task_type: 题型(如 "Action Reasoning")。
|
||||||
|
question: 题目文本。
|
||||||
|
options: 选项元组(如 ("A. ...", "B. ...", "C. ...", "D. ..."))。
|
||||||
|
answer: 正确答案字母(如 "B")。
|
||||||
|
source_nodes: 来源节点 ID 元组。
|
||||||
|
difficulty: 难度等级。
|
||||||
|
"""
|
||||||
|
|
||||||
|
question_id: str
|
||||||
|
video_id: str
|
||||||
|
task_type: str
|
||||||
|
question: str
|
||||||
|
options: tuple[str, ...]
|
||||||
|
answer: str
|
||||||
|
source_nodes: tuple[str, ...]
|
||||||
|
difficulty: str
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
| PyTorch | 本项目 | 代码位置 |
|
| PyTorch | 本项目 | 代码位置 |
|
||||||
|---------|--------|----------|
|
|---------|--------|----------|
|
||||||
| DataLoader | 出题 question_gen | `app/question_gen/generator.py` |
|
| DataLoader | 出题 question_gen | `app/question_gen/loader.py` |
|
||||||
| model.forward() | 推理 inference | `app/harness/inference.py` + `core/agent/loop.py` |
|
| model.forward() | 推理 inference | `app/harness/inference.py` + `core/agent/loop.py` |
|
||||||
| loss.backward() | 诊断 diagnose | `core/evolution/diagnose.py` |
|
| loss.backward() | 诊断 diagnose | `core/evolution/diagnose.py` |
|
||||||
| optimizer.step() | 进化 evolve | `core/evolution/evolve.py` |
|
| optimizer.step() | 进化 evolve | `core/evolution/evolve.py` |
|
||||||
@@ -80,7 +80,7 @@ flowchart TB
|
|||||||
flowchart TD
|
flowchart TD
|
||||||
CLI["main.py CLI"] --> RUNNER["app/harness/runner.py\n训练循环编排"]
|
CLI["main.py CLI"] --> RUNNER["app/harness/runner.py\n训练循环编排"]
|
||||||
CLI --> BUILD["app/tree/video_builder.py\n建树"]
|
CLI --> BUILD["app/tree/video_builder.py\n建树"]
|
||||||
CLI --> QGEN["app/question_gen/generator.py\n新题构建"]
|
CLI --> QGEN["app/question_gen/loader.py\n新题构建"]
|
||||||
CLI --> TRAIN_RET["app/retriever/train.py\n检索器训练"]
|
CLI --> TRAIN_RET["app/retriever/train.py\n检索器训练"]
|
||||||
|
|
||||||
RUNNER --> INF["app/harness/inference.py\n推理 step"]
|
RUNNER --> INF["app/harness/inference.py\n推理 step"]
|
||||||
@@ -119,29 +119,34 @@ project_root/
|
|||||||
│
|
│
|
||||||
├── app/ # 应用层(组合 core + adapters,领域特化)
|
├── app/ # 应用层(组合 core + adapters,领域特化)
|
||||||
│ ├── tree/ # 模块1:建树
|
│ ├── tree/ # 模块1:建树
|
||||||
│ │ ├── index.py # TreeIndex 数据结构(L1/L2/L3Node)
|
│ │ ├── index.py # TreeIndex 数据结构(L1/L2/L3Node + Card)
|
||||||
│ │ ├── video_builder.py # VideoTreeBuilder(asyncio 并发)
|
│ │ ├── video_builder.py # VideoTreeBuilder(L2 轴心 + asyncio 并发)
|
||||||
│ │ ├── text_builder.py # TextTreeBuilder
|
│ │ ├── config.py # TreeConfig 配置 dataclass
|
||||||
│ │ ├── embeddings.py # EmbeddingModel(local/remote 双后端)
|
│ │ ├── subtitle.py # SRT 解析 + Voronoi 字幕分配
|
||||||
│ │ ├── enhance/ # 树增强管线(verify/supplement/clean)
|
│ │ ├── verify.py # 交叉校验(entities/visible_text)
|
||||||
│ │ └── subtitle.py # SRT 解析 + 字幕注入
|
│ │ ├── environment.py # 树环境语义搜索(分块 embedding + 祖先去重)
|
||||||
|
│ │ └── repair/ # 后修复管线
|
||||||
|
│ │ ├── detector.py # 缺陷检测(空字段/缺帧/时间间隙)
|
||||||
|
│ │ ├── regenerator.py # VLM 重描述 + 自底向上级联修复
|
||||||
|
│ │ └── supplement.py # Q&A 反向补全
|
||||||
│ ├── harness/ # 模块2:训练 harness
|
│ ├── harness/ # 模块2:训练 harness
|
||||||
│ │ ├── runner.py # 训练循环编排(对标 Trainer)
|
│ │ ├── runner.py # 训练循环编排(对标 Trainer)
|
||||||
│ │ ├── inference.py # 推理 step
|
│ │ ├── inference.py # 推理 step
|
||||||
│ │ ├── batching.py # mini-batch 构建
|
│ │ ├── batching.py # mini-batch 构建
|
||||||
│ │ ├── question_gen.py # 数据加载、三池切分
|
│ │ ├── pools.py # 三池切分(数据加载已移至 question_gen/loader.py)
|
||||||
│ │ ├── gate_ladder.py # 信息阶梯
|
│ │ ├── gate_ladder.py # 信息阶梯
|
||||||
│ │ ├── momentum.py # 慢速动量
|
│ │ ├── momentum.py # 慢速动量
|
||||||
│ │ ├── config.py # RunConfig
|
│ │ ├── config.py # RunConfig
|
||||||
│ │ ├── log.py # HarnessLog (SQLite)
|
│ │ ├── log.py # HarnessLog (SQLite)
|
||||||
│ │ └── workspace.py # Store + Workspace 版本管理
|
│ │ └── workspace.py # Store + Workspace 版本管理
|
||||||
│ ├── question_gen/ # 模块3:新题构建
|
│ ├── question_gen/ # 模块3:出题(加载 + 采样 + 未来 LLM 生成)
|
||||||
│ │ ├── generator.py # 题目生成
|
│ │ └── loader.py # benchmark 加载、分层采样
|
||||||
│ │ ├── calibrator.py # 基线校准
|
|
||||||
│ │ └── dedup.py # 去重
|
|
||||||
│ ├── search/ # 搜索 Agent 装配
|
│ ├── search/ # 搜索 Agent 装配
|
||||||
│ │ ├── prompt.py # PromptManager
|
│ │ ├── prompt.py # PromptManager(prompt 加载与拼装)
|
||||||
│ │ └── skills.py # SkillRegistry
|
│ │ ├── skills.py # SkillRegistry + discover_skills
|
||||||
|
│ │ ├── summarizer.py # 两轮 LLM 摘要(view_node / search_similar 用)
|
||||||
|
│ │ ├── vision.py # observe_frame(VLM 两轮 + OCR 注入)
|
||||||
|
│ │ └── tools.py # SearchToolDispatcher(实现 ToolDispatcher)
|
||||||
│ ├── retriever/ # 可训练检索器
|
│ ├── retriever/ # 可训练检索器
|
||||||
│ │ ├── recursive.py # RecursiveRetriever (CrossAttention+ACT)
|
│ │ ├── recursive.py # RecursiveRetriever (CrossAttention+ACT)
|
||||||
│ │ ├── losses.py # NavigationLoss + ACTLoss
|
│ │ ├── losses.py # NavigationLoss + ACTLoss
|
||||||
@@ -208,11 +213,13 @@ project_root/
|
|||||||
|
|
||||||
**Evolution 专属端口(`core/evolution/protocols.py`):**
|
**Evolution 专属端口(`core/evolution/protocols.py`):**
|
||||||
|
|
||||||
|
core/ 侧 Protocol 只读——core/ 返回结果 dataclass,写入由 app/harness/ 编排层执行。写方法保留在 app/ 侧的实现类中。
|
||||||
|
|
||||||
| Protocol | 关键方法 | 职责 |
|
| Protocol | 关键方法 | 职责 |
|
||||||
|----------|---------|------|
|
|----------|---------|------|
|
||||||
| `SkillStore` | `read_skill()`, `write_skill()`, `list_versions()` | 版本化技能存储 |
|
| `SkillStore` | `read_skill()`, `list_skill_files()` | 版本化技能读取(只读) |
|
||||||
| `PromptStore` | `read_prompt()`, `write_prompt()` | 版本化提示词存储 |
|
| `PromptStore` | `read_prompt()`, `list_prompt_files()` | 版本化提示词读取(只读) |
|
||||||
| `RunLog` | `insert()`, `query()` | 实验日志 |
|
| `RunLog` | `get_predictions()`, `get_traces()` | 实验日志查询(只读) |
|
||||||
|
|
||||||
### 3.2 应用层端口(`app/ports.py`)
|
### 3.2 应用层端口(`app/ports.py`)
|
||||||
|
|
||||||
@@ -221,7 +228,7 @@ project_root/
|
|||||||
| `EmbeddingProvider` | `embed(texts)` | 文本嵌入 | `adapters/embedding.py` |
|
| `EmbeddingProvider` | `embed(texts)` | 文本嵌入 | `adapters/embedding.py` |
|
||||||
| `TreeCache` | `get()`, `set()` | 树索引缓存 | `adapters/redis_cache.py` |
|
| `TreeCache` | `get()`, `set()` | 树索引缓存 | `adapters/redis_cache.py` |
|
||||||
| `ASRProvider` | `transcribe(audio_path)` | 语音识别 | `adapters/asr.py` |
|
| `ASRProvider` | `transcribe(audio_path)` | 语音识别 | `adapters/asr.py` |
|
||||||
| `OCRProvider` | `recognize(image_path)` | OCR | `adapters/ocr.py` |
|
| `OCRProvider` | `transcribe_frames(frame_paths: list[Path]) -> str` | 帧文字转录 | `adapters/ocr.py` |
|
||||||
|
|
||||||
判据:这块代码会不会被换实现、或需要在测试里替换成假的?不会,就别抽象。
|
判据:这块代码会不会被换实现、或需要在测试里替换成假的?不会,就别抽象。
|
||||||
|
|
||||||
@@ -231,7 +238,7 @@ project_root/
|
|||||||
|----------|-------------|------|
|
|----------|-------------|------|
|
||||||
| `LLMProvider` | `adapters/llm.py` `GovernedLLMClient` | OpenAI 兼容 API,内置治理栈(§5) |
|
| `LLMProvider` | `adapters/llm.py` `GovernedLLMClient` | OpenAI 兼容 API,内置治理栈(§5) |
|
||||||
| `VLMProvider` | `adapters/vlm.py` | Qwen VL 等 OpenAI 兼容 VLM API |
|
| `VLMProvider` | `adapters/vlm.py` | Qwen VL 等 OpenAI 兼容 VLM API |
|
||||||
| `ToolDispatcher` | `app/search/skills.py` `SkillRegistry` | 按名称分发到已注册工具函数 |
|
| `ToolDispatcher` | `app/search/tools.py` `SearchToolDispatcher` | 搜索 Agent 工具分发(view_node / search_similar / observe_frame / submit_answer / read_skill) |
|
||||||
| `SkillStore` / `PromptStore` | `app/harness/workspace.py` | 文件系统版本化存储(`store/skills/v{N}/`) |
|
| `SkillStore` / `PromptStore` | `app/harness/workspace.py` | 文件系统版本化存储(`store/skills/v{N}/`) |
|
||||||
| `RunLog` | `app/harness/log.py` `HarnessLog` | SQLite 持久化 |
|
| `RunLog` | `app/harness/log.py` `HarnessLog` | SQLite 持久化 |
|
||||||
| `TelemetryRecorder` | `adapters/telemetry.py` | SQLite `telemetry.db` |
|
| `TelemetryRecorder` | `adapters/telemetry.py` | SQLite `telemetry.db` |
|
||||||
|
|||||||
@@ -0,0 +1,341 @@
|
|||||||
|
# Design: app/harness/ 训练循环编排层
|
||||||
|
|
||||||
|
**日期** 2026-07-07 · **状态** 提案 · **范围** `app/harness/` 全部 14 个文件
|
||||||
|
|
||||||
|
## 1 定位
|
||||||
|
|
||||||
|
`app/harness/` 是自进化闭环的编排层,对标 PyTorch Trainer。它组合 `core/evolution/`(决策内核)+ `core/agent/`(AgentLoop 推理引擎)+ `adapters/`(LLM/VLM/telemetry),实现训练循环三级嵌套、块序贯验证、快慢双速进化、checkpoint/resume。
|
||||||
|
|
||||||
|
与 `core/evolution/` 的分工:core/ 做决策("候选好不好"),app/ 做编排("跑推理、写版本、管缓存、落观测")。
|
||||||
|
|
||||||
|
### 1.1 算法保真
|
||||||
|
|
||||||
|
| # | 算法 | 保真要求 |
|
||||||
|
|---|------|---------|
|
||||||
|
| 6 | 信息阶梯 | 冷启动 2:1 交错 + probe 探针、γ-EMA 更新、warm p̂(1-p̂) 排序、防泄露铁律 |
|
||||||
|
| 10 | mini-batch | FFD + round-robin + 正确率混合、确定性(seed) |
|
||||||
|
| 13 | 训练循环编排 | 三级嵌套 epoch→step→per-skill、快慢双速、checkpoint/resume、early stop、probation |
|
||||||
|
|
||||||
|
## 2 模块结构与依赖
|
||||||
|
|
||||||
|
```text
|
||||||
|
app/harness/
|
||||||
|
├── __init__.py # 公开 API
|
||||||
|
├── config.py # RunConfig frozen dataclass + 四层校验 + YAML 加载
|
||||||
|
├── log.py # HarnessLog SQLite 薄包装 + RunLog Protocol 实现
|
||||||
|
├── store.py # Store 版本操作 + Seed 管理(稳定基础设施)
|
||||||
|
├── workspace.py # Workspace 生命周期 + manifest + SkillStore/PromptStore 实现
|
||||||
|
├── inference.py # async run_inference(并发推理编排)
|
||||||
|
├── pools.py # 三池切分(test→validation→diagnosis)
|
||||||
|
├── batching.py # FFD + round-robin mini-batch (#10)
|
||||||
|
├── gate_ladder.py # 信息阶梯 + BaselineCache (#6)
|
||||||
|
├── validate.py # 块序贯验证编排(唯一独立子编排器)
|
||||||
|
├── momentum.py # 慢更新动量生成
|
||||||
|
├── checkpoint.py # TrainState 序列化/反序列化 + 原子写
|
||||||
|
├── observation.py # 五张观测表 + step/epoch 报告(合并 metric_log + loop_report)
|
||||||
|
└── runner.py # 瘦编排器:训练循环 + 慢更新十步序 (#13)
|
||||||
|
```
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart TD
|
||||||
|
runner["runner.py\n瘦编排器 #13"]
|
||||||
|
runner --> inference["inference.py\nasync 推理"]
|
||||||
|
runner --> validate["validate.py\n块序贯验证"]
|
||||||
|
runner --> momentum["momentum.py\n慢更新动量"]
|
||||||
|
runner --> batching["batching.py\nFFD mini-batch #10"]
|
||||||
|
runner --> pools["pools.py\n三池切分"]
|
||||||
|
runner --> gate_ladder["gate_ladder.py\n信息阶梯 #6"]
|
||||||
|
runner --> checkpoint["checkpoint.py\n状态序列化"]
|
||||||
|
runner --> observation["observation.py\n五表+报告"]
|
||||||
|
runner --> workspace["workspace.py\nmanifest+Protocol"]
|
||||||
|
runner --> config["config.py\nRunConfig"]
|
||||||
|
validate --> inference
|
||||||
|
validate --> gate_ladder
|
||||||
|
momentum --> inference
|
||||||
|
workspace --> store["store.py\n版本+Seed"]
|
||||||
|
inference --> log["log.py\nSQLite+RunLog"]
|
||||||
|
observation --> log
|
||||||
|
|
||||||
|
runner -.->|LLMProvider| core_p["core/protocols.py"]
|
||||||
|
runner -.->|evolve/diagnose| core_e["core/evolution/"]
|
||||||
|
runner -.->|AgentLoop| core_a["core/agent/"]
|
||||||
|
```
|
||||||
|
|
||||||
|
依赖规则:`app/harness/` → `core/evolution/` + `core/agent/` + `core/protocols`。模块间扁平无环。
|
||||||
|
|
||||||
|
## 3 config.py
|
||||||
|
|
||||||
|
RunConfig frozen dataclass,47 字段,四层校验,YAML + CLI 加载。
|
||||||
|
|
||||||
|
- `frozen=True` 保证运行中不可变
|
||||||
|
- 四层校验:`_validate` → `_validate_edit_budget` + `_validate_minibatch` + `_validate_gate`
|
||||||
|
- `val_size >= eval_min_per_class × _VIDEO_MME_TASK_TYPE_COUNT(11)`
|
||||||
|
- `min_class_per_batch < batch_size`(严格小于)
|
||||||
|
- `gate_lambda_dir < 0`(方向拒绝阈值语义)
|
||||||
|
- `load_config(yaml_path, cli_overrides)`:合并优先级 CLI args > .env > YAML(CLAUDE.md §4.5)。工程配置(API 密钥、LLM 超时等少变/敏感项)走 .env / pydantic-settings;科研实验配置(gate 阈值、batch 大小等会扫动的参数)走 YAML。RunConfig 统一归口。
|
||||||
|
|
||||||
|
与 TRM4 差异:`--resume`/`--fresh` 互斥校验移到 runner(workspace 初始化逻辑在 runner)。新增 .env 层合并(TRM4 仅 CLI > YAML)。
|
||||||
|
|
||||||
|
## 4 log.py
|
||||||
|
|
||||||
|
HarnessLog SQLite 薄包装 + RunLogImpl(RunLog Protocol 实现)。
|
||||||
|
|
||||||
|
**HarnessLog** 与 TRM4 一致:WAL 模式 + threading.Lock 线程安全;`INSERT OR IGNORE` 幂等;query 也持锁(共享连接下并发 SELECT + INSERT 会损坏游标状态)。
|
||||||
|
|
||||||
|
**RunLogImpl** 新增,实现 `core/evolution/protocols.py::RunLog`:
|
||||||
|
|
||||||
|
- 只读端口(core/ Protocol 定义为只读)
|
||||||
|
- 用独立 sqlite3.connect 做 SELECT,不经 HarnessLog 生命周期(不触发 _runs INSERT)
|
||||||
|
- `asyncio.to_thread` 包装同步 SQL(避免 aiosqlite 新依赖)
|
||||||
|
|
||||||
|
## 5 store.py
|
||||||
|
|
||||||
|
Store 版本操作(`advance_version`、`next_version`、`list_versions`、`_write_meta`、`_parse_version`)+ Seed 管理(`init_seed`、`read_seed`、`promote_to_seed`、`extract_run_db`)。
|
||||||
|
|
||||||
|
- `list_versions` 只接受 `v\d+` 格式并按**数字值**排序(非字典序),保证 v10 排在 v2 后面。`next_version` 依赖此排序取 latest+1。
|
||||||
|
|
||||||
|
- `extract_run_db` 用原始 CREATE 语句重建表(保留 PRIMARY KEY 约束)
|
||||||
|
- `promote_to_seed` 强校验 eval run 版本与 --version 一致且非 NULL
|
||||||
|
- 临时 db 用 finally 清理
|
||||||
|
|
||||||
|
与 workspace.py 分离理由:Store 是跨实验共享的稳定基础设施(变更理由不同、稳定性更高),依赖方向单向(workspace → store)。
|
||||||
|
|
||||||
|
## 6 workspace.py
|
||||||
|
|
||||||
|
Workspace 生命周期 + manifest 读写 + SkillStore/PromptStore Protocol 实现。
|
||||||
|
|
||||||
|
- `ResolvedPaths(frozen=True)`:skills_dir/prompts_dir 解析到 workspace(非 store)
|
||||||
|
- `init_workspace_from_seed` 创建前校验 questions ref 存在(fail-fast:归档旧 ws 后才发现缺失则旧 ws 已毁)
|
||||||
|
- `record_run` 幂等(同 run_id 不重复追加 history,run 目录和 per-video wiki 目录用 `exist_ok=True` 创建)
|
||||||
|
- `update_best`/`read_best`:best 指针独立于 current(可指向已存储但非 current 的版本)
|
||||||
|
|
||||||
|
**Protocol 实现**:
|
||||||
|
|
||||||
|
| 类 | Protocol | 实现 |
|
||||||
|
|---|---------|------|
|
||||||
|
| `VersionedSkillStore` | `core/evolution/protocols::SkillStore` | `Path.read_text` + `Path.glob("*.md")` |
|
||||||
|
| `VersionedPromptStore` | `core/evolution/protocols::PromptStore` | 同构 |
|
||||||
|
|
||||||
|
各 ~15 行。构造参数为 Path(由 resolve_paths 提供)。accept 推进版本后 runner 重建实例。
|
||||||
|
|
||||||
|
## 7 inference.py
|
||||||
|
|
||||||
|
async run_inference:全异步 + 依赖注入。
|
||||||
|
|
||||||
|
### 关键签名
|
||||||
|
|
||||||
|
```
|
||||||
|
async def run_inference(
|
||||||
|
questions, *, llm, tool_dispatch_fn, log, run_id, concurrency, max_steps, skill_mode, plugins
|
||||||
|
) -> InferenceResult
|
||||||
|
```
|
||||||
|
|
||||||
|
- `asyncio.Semaphore(concurrency)` + `asyncio.gather` 控制并发(替代 ThreadPoolExecutor)
|
||||||
|
- LLM/VLM/OCR/Embedding 均由调用方构造注入(inference 不感知具体实现)
|
||||||
|
- `run_id: str` 必传(不再可选),空串 → ValueError
|
||||||
|
- `_aggregate_results` 从内存聚合(不再 SELECT predictions 表)
|
||||||
|
|
||||||
|
### 保留的防御性设计
|
||||||
|
|
||||||
|
- 悲观默认值:单题 record 初始 `stop_reason="error"`,成功后覆盖
|
||||||
|
- prediction 必落库:`log.insert` 在 try/except 之后(无论成败)
|
||||||
|
- `_to_text_field`:非 str 的 evidence/reasoning JSON 序列化入库
|
||||||
|
- 5 张表 schema 保留(predictions/traces/validation_flags/anchor_check/observe_frame_health)
|
||||||
|
|
||||||
|
## 8 pools.py
|
||||||
|
|
||||||
|
三池切分,与 TRM4 1:1 迁移。
|
||||||
|
|
||||||
|
- 切分顺序 test→validation→diagnosis(progressive exclusion)
|
||||||
|
- test 池自然分布(correct_ratio=None)
|
||||||
|
- `build_or_load_pools`:pools.json 存在即冻结复用
|
||||||
|
- 旧格式拒绝(无 test 键 → ValueError)
|
||||||
|
|
||||||
|
## 9 batching.py — 算法保真 #10
|
||||||
|
|
||||||
|
FFD + round-robin + 正确率混合,与 TRM4 1:1 迁移。
|
||||||
|
|
||||||
|
- 先小类后大类装箱(小类 first-fit-decreasing 整组不拆、大类全局指针 round-robin 散布)
|
||||||
|
- `_validate_params` 防御性自校验(min_class_per_batch < batch_size)
|
||||||
|
- `correctness.get(qid) is False` 精确匹配(排除 None/未知题)
|
||||||
|
- 只有有错题的题型参与混合
|
||||||
|
- 全部确定性(seed 控制 shuffle、题型按名称排序)
|
||||||
|
|
||||||
|
## 10 gate_ladder.py — 算法保真 #6
|
||||||
|
|
||||||
|
信息阶梯 + BaselineCache,与 TRM4 1:1 迁移。
|
||||||
|
|
||||||
|
- 冷启动 p̂ = Beta(1,1) 平滑(错=1/3, 对=2/3),2:1 交错 + probe 探针
|
||||||
|
- warm 排序 p̂(1-p̂) 信息量降序,剔除 [p_low, p_high] 外零信息题
|
||||||
|
- entries 保持存储序(warm 排序在 ladder_for 取用时做)
|
||||||
|
- GatePools 原子写(.tmp + os.replace)
|
||||||
|
- 指纹不一致 → RuntimeError(不静默重建)
|
||||||
|
- BaselineCache 四维内容寻址(task_type, skill_hash, prompts_version, qid)
|
||||||
|
- BaselineCache "先盘后存"(磁盘成功后才更新内存,无分裂窗口)
|
||||||
|
- 防泄露铁律:gate 内 rollout(run_id 含 `_gate_`)永不回流 p̂
|
||||||
|
|
||||||
|
## 11 validate.py
|
||||||
|
|
||||||
|
块序贯验证编排(唯一独立子编排器),async 化。
|
||||||
|
|
||||||
|
### 关键类型
|
||||||
|
|
||||||
|
| 类型 | 说明 |
|
||||||
|
|------|------|
|
||||||
|
| `InferenceRunConfig` | 推理配置三元组(concurrency, max_steps, skill_mode) |
|
||||||
|
| `ValidationOutcome` | 三态动作 + e-process 证据 + 已观测题逐题对错(candidate_correctness 只含已观测题) |
|
||||||
|
| `Probation` | 在途试用账本(anchor_skills_version + correctness_snapshot + pending_edits) |
|
||||||
|
|
||||||
|
### 核心函数
|
||||||
|
|
||||||
|
`async validate_skill_local(...)` → ValidationOutcome:接收 LLMProvider + ToolDispatchFn + HarnessLog 注入。
|
||||||
|
|
||||||
|
内部流程:
|
||||||
|
1. `materialize_candidate_skill`(workspace `.cand_tmp/`,唯一命名,finally 清理)
|
||||||
|
2. 按 gate_block 切块
|
||||||
|
3. 每块:`_resolve_baseline_block`(缓存优先,miss 才 await run_inference)→ `_run_candidate_block`(全块 await)→ INFRA 护栏(跨块累计,分母≥10)→ `pair_block`(core/evolution)→ `gate_decision`(core/evolution)
|
||||||
|
4. 最后一块判定即终态(无循环外补判)
|
||||||
|
|
||||||
|
- `gate_run_prefix` 必须含 `_gate_`(防泄露过滤依赖此标记,入口校验)
|
||||||
|
- 只有终态题的证据行才携带 stop_reason
|
||||||
|
- **块级屏障**:每块必须严格顺序执行(基线补齐 → 候选跑完 → INFRA 累计 → pair → gate_decision),不得跨块流式判定或部分观测提前进入 gate。async 化不改变此顺序约束。
|
||||||
|
|
||||||
|
## 12 momentum.py
|
||||||
|
|
||||||
|
慢更新动量生成,async 化。
|
||||||
|
|
||||||
|
- 四类常量单一真源(IMPROVED/REGRESSED/PERSISTENT_FAIL/STABLE_SUCCESS)
|
||||||
|
- 展示顺序 REGRESSED 优先(伤害信号最高)
|
||||||
|
- `_format_comparison_pairs` 在 try 外(KeyError 不被 ValueError 吞)
|
||||||
|
- 解析失败保留 prev_guidance(保守回退);基础设施异常不捕
|
||||||
|
|
||||||
|
导出:`run_slow_momentum`(async)、四类常量、`_format_comparison_pairs`、`_categorize_pair`。编排逻辑(采样、两版 rollout、版本推进)在 runner。
|
||||||
|
|
||||||
|
## 13 checkpoint.py
|
||||||
|
|
||||||
|
_TrainState 序列化/反序列化 + 原子写 + 配置指纹。
|
||||||
|
|
||||||
|
**不持久化**:gate_pools/baseline_cache(各自文件自持久化,resume 按指纹重载)、best_*(从 manifest best 指针读)、global_step(存 progress 块,由 train 单独赋值)。
|
||||||
|
|
||||||
|
**完整持久化字段集**(serialize_state 输出,缺一不可):correctness, eval_prev_acc, eval_prev_run_id, baseline_skills_version, baseline_prompts_version, steps_since_best_improved, epoch_start_skills, changed_task_types_this_epoch (set→sorted list), rejected_buffer, system_packs, tool_packs, probations, gate_cooldown, gate_epoch_observed。
|
||||||
|
|
||||||
|
**嵌套 dataclass 复活规则**:SystemCasePack 含 CaseSample 列表(需 `CaseSample(**d)` 逐个重建);Probation 含 RejectedEdit 列表 pending_edits(先重建 RejectedEdit 再构造 Probation);ToolCasePack 字段均为标量/dict,`Cls(**d)` 直接构造。
|
||||||
|
|
||||||
|
- 反序列化 `d[...]` 不用 `.get` 兜底(缺键 = checkpoint 损坏 → 硬失败)
|
||||||
|
- 结构性键变化拒绝 resume,决策性键仅告警
|
||||||
|
- 原子写(.tmp + os.replace)
|
||||||
|
|
||||||
|
## 14 observation.py
|
||||||
|
|
||||||
|
五张观测表 + step/epoch 报告(合并 metric_log + loop_report)。
|
||||||
|
|
||||||
|
### 五表
|
||||||
|
|
||||||
|
| 表 | 用途 |
|
||||||
|
|---|------|
|
||||||
|
| dual_metric_eval | epoch 末 hard+soft+mixed 双轨度量 |
|
||||||
|
| shadow_gate | mixed 影子 best 候选 |
|
||||||
|
| holdout_eval | 四向 held-out 在 test 池的度量 |
|
||||||
|
| quadrant_pair | fast gate 后逐题四象限 |
|
||||||
|
| gate_evidence | CE-Gate 逐题可回放审计 |
|
||||||
|
|
||||||
|
- 所有 write 函数幂等建表
|
||||||
|
- 所有 read 函数用独立只读连接(不经 HarnessLog 生命周期)
|
||||||
|
- soft/mixed 为 None → 存 NULL(绝不存 0)
|
||||||
|
|
||||||
|
### 报告
|
||||||
|
|
||||||
|
- `write_step_report`:per (epoch, step, task_type),skipped/cooldown 路径 gate 字段传 None
|
||||||
|
- `write_epoch_report`:system_tool_action + momentum_updated_task_types + best_val_acc
|
||||||
|
|
||||||
|
## 15 runner.py — 算法保真 #13
|
||||||
|
|
||||||
|
瘦编排器(~400 行),class Runner 作为 DI 容器 + 顶层控制流。
|
||||||
|
|
||||||
|
### Runner 类
|
||||||
|
|
||||||
|
```python
|
||||||
|
class Runner:
|
||||||
|
def __init__(self, config, *, llm, evolve_llm, vlm, telemetry): ...
|
||||||
|
async def train(self, pools) -> None: ...
|
||||||
|
async def infer(...) -> InferenceResult: ...
|
||||||
|
async def eval(version) -> InferenceResult: ...
|
||||||
|
async def diagnose(run_id) -> DiagnosisResult: ...
|
||||||
|
def promote(version, eval_run_id, name) -> None: ...
|
||||||
|
```
|
||||||
|
|
||||||
|
`self` 只持注入依赖 + _paths。_TrainState 是 train() 内局部变量,显式传参。
|
||||||
|
|
||||||
|
### Runner 编排职责补充
|
||||||
|
|
||||||
|
以下职责由 Runner 承担(不在 inference/validate 等子模块内):
|
||||||
|
|
||||||
|
- **record_run**:每次调用 run_inference 前,Runner 调 `workspace.record_run(workspace_dir, run_id)` 负责 manifest history 追加和 `runs/<run_id>/wiki` 目录创建。inference.py 不感知 workspace。
|
||||||
|
- **eval 版本回填**:`eval()` 跑完后显式 `UPDATE _runs SET skills_version=?, prompts_version=?, questions_ref=?`(promote 依赖此行读版本对建种子)。
|
||||||
|
- **run_diagnosis 参数组装**:Runner 负责加载 tree_data(从 TreeEnvironment)、DiagnosePrompts bundle(从根 prompts/ 目录读取诊断 prompt 文件)、questions 列表,组装后传给 `core.evolution.run_diagnosis`。diagnose 模式的完整参数契约:`run_id, questions, tree_data, llm, run_log, skill_store, prompts: DiagnosePrompts, concurrency`。
|
||||||
|
|
||||||
|
### _TrainState
|
||||||
|
|
||||||
|
19 个可变字段(TRM4 为 20 个,移除 evolve_client——由 Runner.self._evolve_llm 持有)。
|
||||||
|
|
||||||
|
关键设计:correctness 增量更新;epoch_start_skills 按文件名索引;gate_epoch_observed 必须持久化并 resume 恢复(阶梯排序开关,丢失会回退冷启动序);probations 每题型至多一个。
|
||||||
|
|
||||||
|
### train() 三级嵌套
|
||||||
|
|
||||||
|
epoch → step → per-skill。resume 用 saved_batches 恢复 batch 划分。新 epoch 清空累加器。每 step 后落 "in_epoch" checkpoint,epoch 末落 "epoch_done"。early stop 在慢更新后判。训练收尾 deliver_best + final_test_eval。
|
||||||
|
|
||||||
|
### _run_step
|
||||||
|
|
||||||
|
rollout → correctness 增量 → diagnose → 累加 slow packs → gate_batch_skills → 冷却递减。
|
||||||
|
|
||||||
|
### _gate_batch_skills
|
||||||
|
|
||||||
|
按 task_type 排序处理:cooldown 跳过 → evolve → 无改动跳过 → 排除案例包题构造 ladder → validate_skill_local → accept/reject/probation。
|
||||||
|
|
||||||
|
accept 关键语义:开账快照在合并前拍取;promote → update_manifest → refresh paths;correctness 二轨合并;清黑名单;probation 分岔(default-strategy.md 不开账)。
|
||||||
|
|
||||||
|
reject 黑名单防污染:`_rejected_summary` 只记录 apply_report 中 status 为 applied 的 edit(与 edits 同位对齐筛选),未 applied 的 edit 从未写进候选正文、从未被 gate 验证过,进黑名单会污染"已验证无效"语义。0 applied 时生成防御性文案。
|
||||||
|
|
||||||
|
rollback:文件级 revert(读锚版本内容 → promote 新版本),不整体回退 manifest。
|
||||||
|
|
||||||
|
### _slow_update_cycle 十步序
|
||||||
|
|
||||||
|
```
|
||||||
|
1. 捕获版本快照 → 全 val 重跑 R
|
||||||
|
2. soft score + dual_metric 落库
|
||||||
|
3. R 逐题对错无条件回写
|
||||||
|
4. probation 结算(回滚者覆盖 step 3)
|
||||||
|
5. best argmax(严格大于)
|
||||||
|
6. momentum(不可变新版本,按 skill 文件分组)
|
||||||
|
7. system/tool 慢更新(edit_budget_end)
|
||||||
|
8. R2 闭环(R2 发生在 momentum 推进 skills 后,其版本对为 (r2_skills_version, new_prompts_version),绝不沿用 R 的 eval_skills_version。退步 revert prompts;保留则回写 R2 + best argmax 绑定 R2 版本对 + eval_prev ← R2)
|
||||||
|
9. 三态标签 + epoch_report + 四向 held-out
|
||||||
|
10. gate 阶梯刷新(精确三源:step rollout GLOB `{base}_e{epoch}_s*` 排除 `*_gate_*` + slow R 精确 run_id + kept R2 的 extra_run_ids;reverted R2/shadow/held-out 观测绝不吸收——防泄露同族原则)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 辅助函数
|
||||||
|
|
||||||
|
`resume_plan`、`_guard_infra_failures`、`_apply_batch_correctness`、`_should_early_stop` 等提取为模块级函数(不依赖 self,显式参数,独立可测)。
|
||||||
|
|
||||||
|
## 16 与 core/evolution/ 的接缝
|
||||||
|
|
||||||
|
| app/harness/ 消费 | core/evolution/ 提供 | 交互模式 |
|
||||||
|
|-------------------|---------------------|---------|
|
||||||
|
| validate.py | gate_decision, pair_block, classify_quadrants | 纯函数,返回 GateVerdict/PairResult/QuadrantClassification |
|
||||||
|
| runner._gate_batch_skills | evolve_single_skill, edit_budget_at, resolve_skill_file | 纯函数/异步,返回 EvolutionRecord |
|
||||||
|
| runner._slow_update_cycle | evolve_system_prompt, evolve_single_tool, run_diagnosis | 异步,需 LLMProvider + RunLog + SkillStore |
|
||||||
|
| runner._settle_probations | probation_verdict | 纯函数 |
|
||||||
|
| momentum | replace_momentum, momentum_inner | 纯函数,文本操作 |
|
||||||
|
| checkpoint | types.* (dataclass 序列化) | dataclasses.asdict / Cls(**d) |
|
||||||
|
|
||||||
|
**类型导入约定**:app/harness/ 从 `core.evolution.types` 直接导入 dataclass(GateParams, GateVerdict, PairResult, QuadrantClassification, DiagnosisResult, EvolutionRecord, RejectedEdit, SkillCasePack, SystemCasePack, ToolCasePack, EvolvePrompts, DiagnosePrompts)。`core/evolution/__init__.py` 当前只导出函数;types 作为子模块公开 API 的一部分,直接 import 合法(不违反依赖方向)。实现阶段需在 `__init__.py` 补导出这些类型。
|
||||||
|
|
||||||
|
## 17 rejected approaches
|
||||||
|
|
||||||
|
| 方案 | 拒绝理由 |
|
||||||
|
|------|---------|
|
||||||
|
| 保持 Runner 单体 God Class(2273 行) | 违反 SRP,测试困难,阅读困难 |
|
||||||
|
| 分层编排(runner → slow_update.py 二级编排) | 慢更新与训练循环变更理由相同,拆分增加间接性但不解耦 |
|
||||||
|
| 纯函数替代 class Runner | 训练循环有状态(_TrainState 16 字段),参数爆炸比类更差 |
|
||||||
|
| workspace.py 单文件 | Store 与 Workspace 变更理由不同、稳定性不同(SRP + SDP) |
|
||||||
|
| 混合同步/异步(run_inference 内 asyncio.run) | 与外层 async runner 冲突 |
|
||||||
@@ -0,0 +1,380 @@
|
|||||||
|
# Design: core/evolution/ 可提取内核
|
||||||
|
|
||||||
|
**日期** 2026-07-07 · **状态** 提案 · **范围** `core/evolution/` 全部 7 个文件
|
||||||
|
|
||||||
|
## 1 定位
|
||||||
|
|
||||||
|
`core/evolution/` 是自进化循环的决策内核——诊断、进化、门控、补丁。它只依赖 Protocol 接口和标准库,可搬到无 adapters 的环境用假实现原样运行。
|
||||||
|
|
||||||
|
与 `app/harness/` 的分工:core/ 做决策("候选好不好"),app/ 做编排("跑推理、写版本、管缓存")。
|
||||||
|
|
||||||
|
## 2 模块结构与依赖
|
||||||
|
|
||||||
|
```text
|
||||||
|
core/evolution/
|
||||||
|
├── __init__.py
|
||||||
|
├── protocols.py # SkillStore, PromptStore, RunLog
|
||||||
|
├── types.py # ~18 个 dataclass
|
||||||
|
├── gate.py # CE-Gate e-process(算法 #5)
|
||||||
|
├── patch.py # 补丁引擎 + 冻结区(算法 #9 局部)
|
||||||
|
├── validate.py # 块验证纯决策函数(算法 #7 局部)
|
||||||
|
├── diagnose.py # 两阶段诊断管线(算法 #8)
|
||||||
|
└── evolve.py # 进化引擎(算法 #9)
|
||||||
|
```
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart LR
|
||||||
|
gate["gate.py\n纯数学"] ~~~ patch["patch.py\n纯文本"]
|
||||||
|
types["types.py"] ~~~ protocols["protocols.py"]
|
||||||
|
validate["validate.py"] --> gate
|
||||||
|
validate --> types
|
||||||
|
diagnose["diagnose.py"] --> types & protocols
|
||||||
|
diagnose -.->|LLMProvider| CP["core/protocols.py"]
|
||||||
|
evolve["evolve.py"] --> types & protocols & patch
|
||||||
|
evolve -.->|LLMProvider| CP
|
||||||
|
```
|
||||||
|
|
||||||
|
依赖规则:`core/evolution/` 不 import `app/` 或 `adapters/`。LLM 调用通过已有 `core.protocols.LLMProvider`。
|
||||||
|
|
||||||
|
## 3 protocols.py
|
||||||
|
|
||||||
|
三个 Protocol 均**只读**——core/ 返回结果,app/ 负责持久化。
|
||||||
|
|
||||||
|
```python
|
||||||
|
@runtime_checkable
|
||||||
|
class SkillStore(Protocol):
|
||||||
|
"""版本化技能读取端口。实现方解析 manifest 指针,core/ 不感知版本号。"""
|
||||||
|
def read_skill(self, filename: str) -> str: ...
|
||||||
|
def list_skill_files(self) -> list[str]: ...
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class PromptStore(Protocol):
|
||||||
|
"""版本化提示词读取端口。覆盖 system.md 和 tool extract/verify。"""
|
||||||
|
def read_prompt(self, filename: str) -> str: ...
|
||||||
|
def list_prompt_files(self) -> list[str]: ...
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class RunLog(Protocol):
|
||||||
|
"""实验日志查询端口。隔离 SQLite,core/ 不写 SQL。"""
|
||||||
|
async def get_predictions(
|
||||||
|
self, run_id: str, *, question_ids: list[str] | None = None,
|
||||||
|
) -> list[dict[str, Any]]: ...
|
||||||
|
async def get_traces(
|
||||||
|
self, run_id: str, *, question_ids: list[str] | None = None,
|
||||||
|
) -> list[dict[str, Any]]: ...
|
||||||
|
```
|
||||||
|
|
||||||
|
固定模板 prompt(诊断/进化用)不走 PromptStore,由调用方加载后以 frozen dataclass 束传入:
|
||||||
|
|
||||||
|
```python
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DiagnosePrompts:
|
||||||
|
defect_vs_lapse: str; reasoning_sub: str
|
||||||
|
span_eval_system: str; span_eval_user: str
|
||||||
|
missed_nodes: str; skill_adherence: str
|
||||||
|
confirmation_bias: str; evidence_sufficiency: str
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class EvolvePrompts:
|
||||||
|
evolve_skill: str; evolve_system: str; evolve_tool: str
|
||||||
|
evolve_rank: str; consolidate_system: str
|
||||||
|
```
|
||||||
|
|
||||||
|
| 决策 | 理由 |
|
||||||
|
|------|------|
|
||||||
|
| Protocol 只读 | core/ 纯输入→纯输出,易测试;写入是 app/ 职责 |
|
||||||
|
| RunLog 用领域方法 | 避免 SQL 泄入 core/ |
|
||||||
|
| SkillStore/PromptStore 同步 | 文件读取量小且快,无需 async |
|
||||||
|
| 模板束 frozen dataclass | 零 I/O + 类型安全,不增 Protocol |
|
||||||
|
|
||||||
|
> **ARCHITECTURE.md §3.1 修订说明**:ARCHITECTURE.md 定义的 SkillStore/PromptStore/RunLog 含 write_skill/write_prompt/insert 写方法。本设计有意精简为只读——core/ 返回结果 dataclass,写入由 app/harness/ 编排层执行。写方法保留在 app/ 侧的实现类中,不进 core/ Protocol。此为对 ARCHITECTURE.md 的细化,需同步更新 §3.1。
|
||||||
|
|
||||||
|
## 4 types.py
|
||||||
|
|
||||||
|
### 4.1 Gate 决策
|
||||||
|
|
||||||
|
```python
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class GateParams:
|
||||||
|
e_confirm: float; e_provisional: float; w_net_min: int
|
||||||
|
delta_min: float; lambda_dir: float; e_rollback: float
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class GateVerdict:
|
||||||
|
decision: str # accept_confirmed | accept_provisional |
|
||||||
|
# reject_directional | reject_futility |
|
||||||
|
# reject_inertia | continue
|
||||||
|
e_value: float; wald_lambda: float
|
||||||
|
delta_hat: float; delta_shrunk: float
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.2 诊断
|
||||||
|
|
||||||
|
| 类型 | frozen | 用途 |
|
||||||
|
|------|--------|------|
|
||||||
|
| `SpanMetrics` | ✓ | 单 span judge 结果(step, tool_name, completeness, hallucination, tags) |
|
||||||
|
| `SkillStepAdherence` | ✓ | skill 步骤遵循度 |
|
||||||
|
| `QuestionMetrics` | ✓ | Stage 1 单题完整指标(7 规则 + 5 judge 类型:span/missed/adherence/bias/sufficiency) |
|
||||||
|
| `ErrorAttribution` | ✓ | D1 归因(error_type + cause_category + lapse_note) |
|
||||||
|
| `CaseSample` | ✓ | 案例包单样本 |
|
||||||
|
| `SkillCasePack` | ✓ | 按题型(failure_cases + success_cases + lapse_notes) |
|
||||||
|
| `SystemCasePack` | ✓ | 跨题型行为(行为模式 ≥ 3 次触发) |
|
||||||
|
| `ToolCasePack` | ✓ | 按工具(failure_spans + success_spans) |
|
||||||
|
| `DiagnosisResult` | ✓ | 管线最终输出 |
|
||||||
|
|
||||||
|
### 4.3 进化
|
||||||
|
|
||||||
|
| 类型 | frozen | 说明 |
|
||||||
|
|------|--------|------|
|
||||||
|
| `EvolutionRecord` | — | 构建过程 mutable;tool 类的 evolved_content 为 JSON `{"extract":..,"verify":..}` |
|
||||||
|
| `RejectedEdit` | ✓ | 黑名单条目;gate 字段全 Optional |
|
||||||
|
| `EvolutionResult` | ✓ | 聚合输出(由 app/harness/ 编排层组装,非 core/ 函数返回) |
|
||||||
|
|
||||||
|
### 4.4 验证辅助
|
||||||
|
|
||||||
|
```python
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PairResult:
|
||||||
|
w: int; l: int
|
||||||
|
observed: dict[str, tuple[bool, bool]] # qid → (baseline, candidate)
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class QuadrantClassification:
|
||||||
|
improvements: list[str]; regressions: list[str]
|
||||||
|
persistent_fails: list[str]; stable_successes: list[str]
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5 gate.py — CE-Gate(算法 #5,纯数学)
|
||||||
|
|
||||||
|
| 常量 | 值 | 来源 |
|
||||||
|
|------|-----|------|
|
||||||
|
| `_WALD_WIN` | `ln(1.4) ≈ +0.3365` | θ₁=0.70 → 2×0.70 |
|
||||||
|
| `_WALD_LOSS` | `ln(0.6) ≈ -0.5108` | 2×(1-0.70);loss 步幅 > win(不对称) |
|
||||||
|
| `_SHRINK_PSEUDO` | 4 | Agresti-Coull 伪计数 |
|
||||||
|
|
||||||
|
**compute_e_value(w, l)**:截断 Beta 混合 `E = 2^(W+L+1)·B(W+1,L+1)·I½(L+1,W+1)`。log 空间计算;用对称性 `I½(b,a)` 代替 `1-I½(a,b)` 防灾难性消去。`w<0`/`l<0` → ValueError;`tail≤0` → 0.0;`W=L=0` → 1.0。
|
||||||
|
|
||||||
|
**gate_decision** 四出口优先级:confirmed(E+δ) → directional(Wald) → futility(best-case) → exhaustion(provisional/inertia) → continue。Wald 从累积 W/L 重算(非增量,避免浮点漂移)。delta_shrunk 仅观测,不进决策。
|
||||||
|
|
||||||
|
**probation_verdict(w, l)**:双向非对称——confirm 用 `E(w,l) ≥ e_confirm`,rollback 用 `E(l,w) ≥ e_rollback`(参数交换)。e_rollback < e_confirm(回滚比确认更容易)。
|
||||||
|
|
||||||
|
## 6 patch.py — 补丁引擎(纯文本)
|
||||||
|
|
||||||
|
**标记常量**:`APPENDIX_START/END`、`MOMENTUM_START/END`(HTML 注释形式)、`*_MAX_CHARS=2000`。
|
||||||
|
|
||||||
|
**区域解析**:`appendix_region_bounds()` 和 `momentum_region_bounds()` **均严格**——标记不配对(单标记/重复/逆序)抛 ValueError,双缺合法返回 None。宽容语义仅存在于 evolve.py 的包装函数 `_strip_appendix_region`(缺标记 = no-op)和 `_appendix_span`(缺标记 = 空串),不在 patch.py 本身。
|
||||||
|
|
||||||
|
**apply_patch_with_report(content: str, edits: list[dict], protected_spans: list[str]) -> tuple[str, list[dict]]**:
|
||||||
|
- edits 为 `[{"op": str, "target": str, "content": str}, ...]`(松类型 dict,保持 TRM4 格式)
|
||||||
|
- report 为 `[{"index": int, "op": str, "status": str, "target": str, "content_preview": str}, ...]`
|
||||||
|
- 4 种 op:append(最早非 frontmatter 冻结区前)、insert_after(三结果:成功/降级 append/skip)、replace、delete(均首次出现、count=1)
|
||||||
|
- 每条 edit 前重算 `_protected_ranges`(坐标偏移)
|
||||||
|
- target 不 strip,payload strip
|
||||||
|
- 冻结区坐标半开 `[start, end)`
|
||||||
|
|
||||||
|
**replace_momentum(content, guidance)**:guidance 含标记字面量 → ValueError(注入防护)。空 guidance 合法(清除旧动量,保留标题行)。
|
||||||
|
|
||||||
|
## 7 validate.py — 纯决策函数(算法 #7 局部)
|
||||||
|
|
||||||
|
三个公开函数:`pair_block`(逐题比对 W/L)、`classify_quadrants`(四组各 sorted)、`compute_accuracy`。
|
||||||
|
|
||||||
|
编排循环(materialize candidate → 双臂推理 → 缓存 → INFRA 护栏 → 块序贯 gate_decision)在 app/harness/。
|
||||||
|
|
||||||
|
## 8 diagnose.py — 诊断管线(算法 #8)
|
||||||
|
|
||||||
|
### 公开入口
|
||||||
|
|
||||||
|
```python
|
||||||
|
async def run_diagnosis(
|
||||||
|
run_id: str,
|
||||||
|
questions: list[GeneratedQuestion],
|
||||||
|
tree_data: dict[str, Any], # video_id → 树 JSON
|
||||||
|
llm: LLMProvider,
|
||||||
|
run_log: RunLog,
|
||||||
|
skill_store: SkillStore,
|
||||||
|
prompts: DiagnosePrompts,
|
||||||
|
*, concurrency: int,
|
||||||
|
question_ids: list[str] | None = None,
|
||||||
|
task_types: list[str] | None = None,
|
||||||
|
only_incorrect: bool = False,
|
||||||
|
) -> DiagnosisResult:
|
||||||
|
```
|
||||||
|
|
||||||
|
### 流程
|
||||||
|
|
||||||
|
```
|
||||||
|
Stage 1(asyncio.gather + Semaphore,per-question):
|
||||||
|
7 规则指标(纯函数) + 5 LLM judge(span/missed/adherence/bias/sufficiency)
|
||||||
|
→ D1 归因瀑布 → defect/lapse 分类(LLM)
|
||||||
|
→ ValueError 降级:规则指标保留,judge 指标 None,degraded=True
|
||||||
|
|
||||||
|
独立串行 pass:
|
||||||
|
reasoning_failure 子分类(仅对 error_type=reasoning_failure 的题)
|
||||||
|
|
||||||
|
Stage 2(纯逻辑):
|
||||||
|
D2 按工具聚合 → D3 按题型×正误 → D4 skill adherence → D5 跨题型行为
|
||||||
|
→ 三类案例包构建
|
||||||
|
```
|
||||||
|
|
||||||
|
### 关键保真规则
|
||||||
|
|
||||||
|
- 归因瀑布顺序:extraction(`completeness<0.5∨hallucination>0.5`) → search(`missed_nodes`) → reasoning(`evidence_sufficient=True`) → mixed
|
||||||
|
- defect_vs_lapse 分类:解析失败降级为 "lapse"(保护性,防错误改正文)
|
||||||
|
- single-failure fallback:某题型仅剩 1 个 defect → 降级为 lapse_note
|
||||||
|
- lapse_note 空白过滤:strip 后为空则丢弃
|
||||||
|
- SystemCasePack 触发:3 种行为模式各需 ≥ 3 次出现
|
||||||
|
- merge_system_packs stats 用 `{"per_step": [...]}` 包裹(不数值合并)
|
||||||
|
- trigram 相似度是**字符级**,取 **max**(非 mean)
|
||||||
|
- `_call_judge` 重试 3 次(仅 ValueError),API 错误直传
|
||||||
|
|
||||||
|
## 9 evolve.py — 进化引擎(算法 #9)
|
||||||
|
|
||||||
|
### 公开 API
|
||||||
|
|
||||||
|
| 函数 | 参数 | 返回 |
|
||||||
|
|------|------|------|
|
||||||
|
```python
|
||||||
|
async def evolve_single_skill(
|
||||||
|
llm: LLMProvider, pack: SkillCasePack,
|
||||||
|
skill_store: SkillStore, prompts: EvolvePrompts,
|
||||||
|
source_version: str, edit_budget: int,
|
||||||
|
consolidate_threshold: int, *,
|
||||||
|
skill_update_mode: Literal["patch", "rewrite"] = "patch",
|
||||||
|
rejected: list[RejectedEdit] | None = None,
|
||||||
|
) -> EvolutionRecord: ...
|
||||||
|
|
||||||
|
async def evolve_system_prompt(
|
||||||
|
llm: LLMProvider, pack: SystemCasePack,
|
||||||
|
prompt_store: PromptStore, prompts: EvolvePrompts,
|
||||||
|
source_version: str, edit_budget: int,
|
||||||
|
) -> EvolutionRecord: ...
|
||||||
|
|
||||||
|
async def evolve_single_tool(
|
||||||
|
llm: LLMProvider, pack: ToolCasePack,
|
||||||
|
prompt_store: PromptStore, prompts: EvolvePrompts,
|
||||||
|
source_version: str, edit_budget: int,
|
||||||
|
) -> EvolutionRecord: ...
|
||||||
|
|
||||||
|
def edit_budget_at(
|
||||||
|
global_step: int, total_steps: int,
|
||||||
|
start: int, end: int,
|
||||||
|
) -> int: ... # 纯数学
|
||||||
|
```
|
||||||
|
|
||||||
|
### Skill 三分支
|
||||||
|
|
||||||
|
| 分支 | 条件 | 行为 |
|
||||||
|
|------|------|------|
|
||||||
|
| A: Lapse-only | 无 defect edits + 有 lapse_notes | 合成 `applied_append` report,防循环误判 no-op |
|
||||||
|
| B: Rewrite | mode="rewrite" + 有 edits | 整篇重写;失败降级:有 lapse 转 A,否则 no-op |
|
||||||
|
| C: Patch | 默认 | rank_clip → apply_patch → validate → 最多 2 轮重试 |
|
||||||
|
|
||||||
|
所有分支后:有 lapse_notes → appendix 追加(≥ threshold 则 consolidate)
|
||||||
|
|
||||||
|
### rank_and_clip 三级降级
|
||||||
|
|
||||||
|
LLM 排序 → `_select_top_edits`(`type(idx) is int` 排除 bool + 范围 + 去重)→ 空则降级原序前 N。
|
||||||
|
|
||||||
|
### Tool 共享预算池
|
||||||
|
|
||||||
|
extract+verify edits 合池(`_src` 标记)→ rank_clip → 按标记拆回。evolved_content 存 `json.dumps({"extract":..,"verify":..})`。
|
||||||
|
|
||||||
|
### 冻结区配置
|
||||||
|
|
||||||
|
| 目标 | 冻结区 |
|
||||||
|
|------|--------|
|
||||||
|
| Skill | frontmatter + appendix + momentum |
|
||||||
|
| System | 3 个 `##` section(能力边界/输出格式/视频树结构)+ appendix |
|
||||||
|
| Tool | 输出格式 section + appendix |
|
||||||
|
|
||||||
|
### 验证规则
|
||||||
|
|
||||||
|
| 检查 | Skill | System | Tool |
|
||||||
|
|------|-------|--------|------|
|
||||||
|
| Frontmatter 三字段 | ✓ | — | — |
|
||||||
|
| 冻结 section 值相等 | — | ✓ | ✓ |
|
||||||
|
| 长度比 [0.3, 2.0](去冻结区后) | ✓ | ✓ | ✓ per file |
|
||||||
|
| 代码块闭合 | ✓ | ✓ | — |
|
||||||
|
|
||||||
|
### consolidate_appendix 四守卫
|
||||||
|
|
||||||
|
G1(`<2`直返) → G2(结果非空且≤输入) → G3(any Exception返原文) → G4(**调用方**:`≥`拒绝等长)
|
||||||
|
|
||||||
|
## 10 共享工具函数
|
||||||
|
|
||||||
|
**`resolve_skill_file(skills_dir, task_type) -> str`**(core/evolution/ 内部工具函数):
|
||||||
|
|
||||||
|
`resolve_skill_file(skill_store: SkillStore, task_type: str) -> str`
|
||||||
|
|
||||||
|
`task_type.lower().replace(' ', '-') + ".md"`,若文件不在 `skill_store.list_skill_files()` 中则回退 `"default-strategy.md"`。diagnose(加载 skill 内容做 adherence 判定)和 evolve(定位进化目标文件)共用此约定。接受 `SkillStore`(非 `Path`),保持 core/ 不依赖文件系统。
|
||||||
|
|
||||||
|
## 11 TRM4 → TRM5 变更总表
|
||||||
|
|
||||||
|
| 项 | TRM4 | TRM5 | 理由 |
|
||||||
|
|----|------|------|------|
|
||||||
|
| 并发 | `ThreadPoolExecutor` | `asyncio.gather + Semaphore` | TRM5 async-first |
|
||||||
|
| LLM | `LLMClient.from_env()` 每线程构造 | 共享 `LLMProvider` 注入 | Protocol 化 |
|
||||||
|
| DB | `HarnessLog` + raw SQL | `RunLog` Protocol | 隔离实现 |
|
||||||
|
| 文件 | `Path.read_text` 直读 | `SkillStore` / `PromptStore` | 可提取性 |
|
||||||
|
| 模板 | `_PROJECT_ROOT / "prompts"` 硬编码 | `DiagnosePrompts` / `EvolvePrompts` 束传入 | 零路径依赖 |
|
||||||
|
| 输出 | 写 JSON + DB + advance_version | 纯返回 dataclass,app/ 持久化 | 无副作用 |
|
||||||
|
| response 访问 | `response.choices[0].message.content` | `LLMResponse.content` | 已有统一类型 |
|
||||||
|
| validate 编排 | 在 core/ | 在 app/harness/ | Clean Architecture |
|
||||||
|
| run_evolution 编排 | 在 evolve.py | 在 app/harness/ | 版本管理属 app/ |
|
||||||
|
|
||||||
|
## 12 迁移保真约束
|
||||||
|
|
||||||
|
本节列出 TRM4 中影响正确性的实现细节,实现时必须逐条比对。
|
||||||
|
|
||||||
|
### 12.1 JSON 解析策略差异
|
||||||
|
|
||||||
|
| 模块 | 函数 | 策略 | 失败行为 |
|
||||||
|
|------|------|------|---------|
|
||||||
|
| metrics.py | `extract_json_from_response` | 三级:fenced code block → 最外层 `{...}` → `json_repair` | 全失败抛 ValueError |
|
||||||
|
| metrics.py | `_call_judge` | 包裹上述,max_retries=2(共 3 次),仅 ValueError 重试 | API 错误直传 |
|
||||||
|
| evolve.py | `_parse_llm_json` | 两级:fenced code block → 原文 `json.loads` | 失败返回 None(不抛) |
|
||||||
|
| metrics.py | `_parse_json_object` | 两级:`json.loads` → `json_repair` | 失败返回 None |
|
||||||
|
|
||||||
|
所有解析器均拒绝非 dict 结果(list/str → 视为失败)。
|
||||||
|
|
||||||
|
### 12.2 关键常量
|
||||||
|
|
||||||
|
| 常量 | 值 | 位置 | 说明 |
|
||||||
|
|------|-----|------|------|
|
||||||
|
| `_INFRA_STOP_REASONS` | `frozenset({"error", "parse_error"})` | diagnose | INFRA 排除集 |
|
||||||
|
| `_SPAN_EVAL_TOOLS` | `{"view_node", "search_similar", "observe_frame"}` | metrics | span judge + all_tool_outputs 范围 |
|
||||||
|
| `_MIN_PATTERN_COUNT` | 3 | diagnose | SystemCasePack 触发阈值 |
|
||||||
|
| `_TOOL_TARGET_FILES` | view_node→4 文件, search_similar→2, observe_frame→2 | diagnose | 工具→prompt 文件映射 |
|
||||||
|
| truncation | thought[:100], tool_output[:200] (metrics); 不截断 (diagnose) | metrics/diagnose | `_format_trace_text` 两版本不同! |
|
||||||
|
| case_sample truncation | tool_output[:500] | evolve | `_format_case_samples` |
|
||||||
|
|
||||||
|
### 12.3 案例包选择规则
|
||||||
|
|
||||||
|
| 包 | failure 选择 | success 选择 |
|
||||||
|
|----|-------------|-------------|
|
||||||
|
| Skill | 按 error_type 分组,各取 severity top-2 | `max(2, len(failures)//2)`;acc≤0.3 按 budget 升序,否则按 (-adherence, budget) |
|
||||||
|
| System | 3 种行为模式(early_submit/high_conf_wrong/confirmation_bias)各取 top-2 | correct + calibrated + no_bias + 0.3≤budget≤0.8,按 abs(budget-0.5) |
|
||||||
|
| Tool | 低 completeness top-2 + 高 hallucination top-2,去重,总数≤4 | completeness≥0.9 且 hallucination==0.0 |
|
||||||
|
|
||||||
|
### 12.4 validate 编排守卫(app/harness/ 侧,非 core/)
|
||||||
|
|
||||||
|
- `gate_run_prefix` 必须含 `"_gate_"` 子串(防泄漏标记)
|
||||||
|
- `ladder_items` 空 → ValueError
|
||||||
|
- INFRA guard:累计两臂 error,分母≥10 且 error_rate > `gate_guard_err` → RuntimeError
|
||||||
|
- 基线缓存补齐后 `assert all(v is not None)`
|
||||||
|
|
||||||
|
### 12.5 evolve 重试与退火
|
||||||
|
|
||||||
|
- `_run_patch_evolution_loop`:`range(2)` 两轮,三种失败反馈(JSON/target 未匹配/验证错误)
|
||||||
|
- `edit_budget_at`:`assert start >= end`;`total_steps ≤ 1` 返 start;Python `round`(banker's rounding)
|
||||||
|
- `rewrite_from_suggestions`:重写不得长于原文;只捕 `ValueError/KeyError/TypeError/AttributeError`
|
||||||
|
|
||||||
|
### 12.6 范围说明
|
||||||
|
|
||||||
|
`momentum.py` 按 ARCHITECTURE.md §2.3 归属 `app/harness/`(非 core/evolution/),不在本设计范围内。其 LLM 调用、四类常量(IMPROVED/REGRESSED/PERSISTENT_FAIL/STABLE_SUCCESS)、`_format_comparison_pairs` 放在 try 外的设计意图、解析失败返回 `prev_guidance` 等规则将在 Design B(app/harness/)中覆盖。
|
||||||
|
|
||||||
|
## 13 被拒方案
|
||||||
|
|
||||||
|
**方案 A(validate Protocol 回调)**:给 validate 造 `InferenceRunner` Protocol 让编排留 core/。拒绝理由:leaky abstraction,Protocol 签名暴露 workspace/skills_dir 等外层概念,形式反转实质耦合。
|
||||||
|
|
||||||
|
**方案 B(同步 + ThreadPoolExecutor)**:保持 TRM4 同步。拒绝理由:TRM5 LLMProvider.chat() 已是 async,同步调用需 asyncio.run() 嵌套或线程桥接,增加复杂度。
|
||||||
@@ -0,0 +1,437 @@
|
|||||||
|
---
|
||||||
|
type: design
|
||||||
|
node_id: design:2026-07-07-search-module-design
|
||||||
|
title: "搜索 Agent 装配层设计(app/search/)"
|
||||||
|
date: 2026-07-07
|
||||||
|
---
|
||||||
|
|
||||||
|
# 搜索 Agent 装配层设计(app/search/)
|
||||||
|
|
||||||
|
**日期** 2026-07-07 · **状态** 已批准 · **关联** TRM4 `core/search/` + `core/tree/tools.py` + `core/tree/vision.py` + `core/tree/summarizer.py`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## §1 定位
|
||||||
|
|
||||||
|
`app/search/` 是搜索 Agent 的"装配层"——为 `core/agent/loop.py` AgentLoop 提供 **prompt 组装**、**skill 管理**、**工具定义/分发** 和 **视觉观察**。它不控制推理循环,只被 AgentLoop 调用;编排责任在 `app/harness/inference.py`。
|
||||||
|
|
||||||
|
### 与 TRM4 的映射
|
||||||
|
|
||||||
|
| TRM4 | TRM5 | 变更类型 |
|
||||||
|
|------|------|---------|
|
||||||
|
| `core/search/prompt.py` | `app/search/prompt.py` | 保真迁移 + P4 显式参数 |
|
||||||
|
| `core/search/skills.py` | `app/search/skills.py` | 保真迁移 |
|
||||||
|
| `core/tree/tools.py` | `app/search/tools.py` | 重组为 `SearchToolDispatcher` 类 |
|
||||||
|
| `core/tree/vision.py` | `app/search/vision.py` | 异步化 + Protocol 注入 |
|
||||||
|
| `core/tree/summarizer.py` | `app/search/summarizer.py` | 异步化 + Protocol 注入;含 anchor 锚模式 |
|
||||||
|
| `core/tree/ocr.py` | `adapters/ocr.py` | 异步化 + OCRProvider Protocol |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## §2 模块结构
|
||||||
|
|
||||||
|
```
|
||||||
|
app/search/
|
||||||
|
├── __init__.py # 公开 API 重导出
|
||||||
|
├── prompt.py # PromptManager — prompt 加载与拼装
|
||||||
|
├── skills.py # SkillRegistry + discover_skills — skill 扫描与注册
|
||||||
|
├── tools.py # SearchToolDispatcher(实现 ToolDispatcher Protocol)
|
||||||
|
├── summarizer.py # question-conditioned 两轮 LLM 摘要(view_node / search_similar 用)
|
||||||
|
└── vision.py # observe_frame(VLM 两轮 + OCR 注入)
|
||||||
|
|
||||||
|
adapters/
|
||||||
|
└── ocr.py # MonkeyOCRClient(实现 OCRProvider Protocol)
|
||||||
|
|
||||||
|
app/ports.py # 新增 OCRProvider Protocol(应用层端口,与 EmbeddingProvider 同级)
|
||||||
|
|
||||||
|
store/prompts/ # 初始种子,逐文件从 TRM4 store/prompts/v2/ 字节级复制,不修改
|
||||||
|
├── system.md # ← TRM4 store/prompts/v2/system.md
|
||||||
|
├── observe_frame_extract.md # ← TRM4 store/prompts/v2/observe_frame_extract.md
|
||||||
|
├── observe_frame_verify.md # ← TRM4 store/prompts/v2/observe_frame_verify.md
|
||||||
|
├── view_node_extract.md # ← TRM4 store/prompts/v2/view_node_extract.md
|
||||||
|
├── view_node_verify.md # ← TRM4 store/prompts/v2/view_node_verify.md
|
||||||
|
├── view_node_children_extract.md # ← TRM4 store/prompts/v2/view_node_children_extract.md
|
||||||
|
├── view_node_children_verify.md # ← TRM4 store/prompts/v2/view_node_children_verify.md
|
||||||
|
├── search_similar_extract.md # ← TRM4 store/prompts/v2/search_similar_extract.md
|
||||||
|
└── search_similar_verify.md # ← TRM4 store/prompts/v2/search_similar_verify.md
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## §3 依赖方向
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart TB
|
||||||
|
subgraph adapters
|
||||||
|
OCR_IMPL["adapters/ocr.py\nMonkeyOCRClient"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph core
|
||||||
|
PROTO["app/ports.py\nOCRProvider Protocol"]
|
||||||
|
AGENT_PROTO["core/agent/protocols.py\nToolDispatcher Protocol"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph app/search
|
||||||
|
PROMPT["prompt.py\nPromptManager"]
|
||||||
|
SKILLS["skills.py\nSkillRegistry"]
|
||||||
|
TOOLS["tools.py\nSearchToolDispatcher"]
|
||||||
|
SUMM["summarizer.py\nsummarize_node / _children / _batch"]
|
||||||
|
VISION["vision.py\nobserve_frame"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph app/tree
|
||||||
|
ENV["environment.py\nTreeEnvironment"]
|
||||||
|
end
|
||||||
|
|
||||||
|
OCR_IMPL -->|实现| PROTO
|
||||||
|
TOOLS -->|实现| AGENT_PROTO
|
||||||
|
TOOLS --> ENV
|
||||||
|
TOOLS --> SKILLS
|
||||||
|
TOOLS --> VISION
|
||||||
|
TOOLS --> SUMM
|
||||||
|
SUMM -->|依赖| PROTO_LLM["core/protocols.py\nLLMProvider"]
|
||||||
|
VISION -->|依赖| PROTO
|
||||||
|
VISION -->|依赖| PROTO_VLM["core/protocols.py\nVLMProvider"]
|
||||||
|
PROMPT --> SKILLS
|
||||||
|
PROMPT -.->|读取| STORE["store/prompts/*.md"]
|
||||||
|
SUMM -.->|读取| STORE
|
||||||
|
```
|
||||||
|
|
||||||
|
依赖只向内或同层,`core/` 不认识 `app/search/`。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## §4 公开 API
|
||||||
|
|
||||||
|
### 4.1 PromptManager(prompt.py)
|
||||||
|
|
||||||
|
```python
|
||||||
|
class PromptManager:
|
||||||
|
def __init__(self, prompts_dir: Path) -> None: ...
|
||||||
|
def build_inference_prompt(
|
||||||
|
self,
|
||||||
|
skill_mode: str,
|
||||||
|
task_type: str,
|
||||||
|
always_skills_text: str,
|
||||||
|
task_skill_map: dict[str, str],
|
||||||
|
catalog_text: str,
|
||||||
|
) -> str: ...
|
||||||
|
def format_user_prompt(
|
||||||
|
self,
|
||||||
|
question: str,
|
||||||
|
options: list[str],
|
||||||
|
l1_node_ids: list[str],
|
||||||
|
task_type: str | None = None,
|
||||||
|
) -> str: ...
|
||||||
|
def load(self, name: str) -> str: ...
|
||||||
|
```
|
||||||
|
|
||||||
|
**与 TRM4 有意变更**:
|
||||||
|
- 工具描述从 `app/search/tools.py` 的 `get_tool_descriptions()` 获取(职责归属修正)
|
||||||
|
- `format_user_prompt` 参数从 `dict` 改为显式 `question` / `options` / `l1_node_ids`(P4)
|
||||||
|
|
||||||
|
### 4.2 SkillRegistry + discover_skills(skills.py)
|
||||||
|
|
||||||
|
```python
|
||||||
|
def parse_frontmatter(text: str) -> dict[str, str]: ...
|
||||||
|
def strip_frontmatter(text: str) -> str: ...
|
||||||
|
|
||||||
|
class SkillRegistry:
|
||||||
|
def set_paths(self, mapping: dict[str, Path]) -> None: ...
|
||||||
|
def read(self, name: str) -> str: ...
|
||||||
|
|
||||||
|
def discover_skills(skills_dir: Path) -> tuple[str, dict[str, str], str, SkillRegistry]: ...
|
||||||
|
```
|
||||||
|
|
||||||
|
与 TRM4 逻辑完全一致,无有意变更。
|
||||||
|
|
||||||
|
### 4.3 SearchToolDispatcher(tools.py)
|
||||||
|
|
||||||
|
```python
|
||||||
|
def get_tool_descriptions(include_read_skill: bool = False) -> str: ...
|
||||||
|
|
||||||
|
class SearchToolDispatcher:
|
||||||
|
"""实现 core/agent/protocols.ToolDispatcher。"""
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
env: TreeEnvironment,
|
||||||
|
tool_llm: LLMProvider,
|
||||||
|
vlm: VLMProvider,
|
||||||
|
ocr: OCRProvider | None,
|
||||||
|
prompts_dir: Path,
|
||||||
|
skills: SkillRegistry | None,
|
||||||
|
*,
|
||||||
|
embed_fn: Callable[[str | list[str]], np.ndarray],
|
||||||
|
verify_vision: bool,
|
||||||
|
anchor: bool,
|
||||||
|
assemble_mode: str,
|
||||||
|
stats_sink: Callable[[dict[str, int]], None] | None = None,
|
||||||
|
) -> None: ...
|
||||||
|
|
||||||
|
async def dispatch(
|
||||||
|
self, tool_name: str, args: dict[str, Any], *, context: dict[str, Any]
|
||||||
|
) -> str: ...
|
||||||
|
```
|
||||||
|
|
||||||
|
| 工具 | 实现路径 |
|
||||||
|
|------|---------|
|
||||||
|
| `view_node` | → `env.get_node_text(node_id)` 获取原始文本 + `env.get_children_info(node_id)` 获取子节点结构化信息 → `summarizer.summarize_node(...)` 两轮 LLM 摘要 + `summarizer.summarize_children(...)` 子节点标注 |
|
||||||
|
| `search_similar` | → `env.search_similar(query, top_k, embed_fn=...)` 获取 `[(node_id, score)]` → 对每个 node_id 调 `env.get_node_text(node_id)` → `summarizer.summarize_nodes_batch(...)` 并发两轮 LLM 摘要 + 格式化 |
|
||||||
|
| `observe_frame` | → `env.resolve_frame_paths(...)` + `env.get_subtitle(node_ids[0])` 获取字幕 → `vision.observe_frame(...)` → 输出前拼接 `[字幕上下文]`(保真 TRM4 tools.py:136-153) |
|
||||||
|
| `submit_answer` | → 返回确认文本 |
|
||||||
|
| `read_skill` | → `skills.read(name)` |
|
||||||
|
| 未知工具 | → `raise ValueError`(AgentLoop 捕获,不计步) |
|
||||||
|
|
||||||
|
**与 TRM4 有意变更**:
|
||||||
|
- 自由函数 + 大量位置参数 → 类封装(构造时注入依赖)
|
||||||
|
- 工具描述 `get_tool_descriptions()` 移入此文件
|
||||||
|
- LLM 摘要从 environment 拆出到 `summarizer.py`(environment 回归纯数据层)
|
||||||
|
- `SearchToolDispatcher.__init__` 新增 `tool_llm: LLMProvider` 参数(工具级 LLM,thinking=False,用于 summarizer)
|
||||||
|
- `dispatch()` 从 `context` 中提取 `session_id` / `parent_call_id`,透传给 summarizer / vision 的 LLM/VLM 调用,确保遥测链路完整
|
||||||
|
|
||||||
|
### 4.4 summarizer(summarizer.py)
|
||||||
|
|
||||||
|
从 TRM4 `core/tree/summarizer.py` 迁移。三个工具(view_node / search_similar / observe_frame)共享同构的"提取→验证"两轮模式。summarizer 负责前两个工具的文本摘要,vision 负责第三个的视觉摘要。
|
||||||
|
|
||||||
|
```python
|
||||||
|
async def summarize_node(
|
||||||
|
llm: LLMProvider,
|
||||||
|
raw_text: str,
|
||||||
|
question: str,
|
||||||
|
prompts_dir: Path,
|
||||||
|
*,
|
||||||
|
anchor_map: dict[str, str] | None,
|
||||||
|
assemble_mode: str,
|
||||||
|
stats_sink: Callable | None = None,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> str: ...
|
||||||
|
|
||||||
|
async def summarize_children(
|
||||||
|
llm: LLMProvider,
|
||||||
|
children_info: list[dict[str, Any]],
|
||||||
|
question: str,
|
||||||
|
prompts_dir: Path,
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> str: ...
|
||||||
|
|
||||||
|
async def summarize_nodes_batch(
|
||||||
|
llm: LLMProvider,
|
||||||
|
items: list[tuple[str, str, str]],
|
||||||
|
question: str,
|
||||||
|
prompts_dir: Path,
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> list[tuple[str, str]]: ...
|
||||||
|
```
|
||||||
|
|
||||||
|
| 函数 | Prompt 文件 | 输出格式 |
|
||||||
|
|------|-------------|---------|
|
||||||
|
| `summarize_node` | `view_node_extract.md` + `view_node_verify.md` | `"[内容摘要] ...\n[核实] ..."` |
|
||||||
|
| `summarize_children` | `view_node_children_extract.md` + `view_node_children_verify.md` | `"★★/★ 标注\n[核实] ..."` |
|
||||||
|
| `summarize_nodes_batch` | `search_similar_extract.md` + `search_similar_verify.md` | `[("node_id", "[内容摘要] ..."), ...]` |
|
||||||
|
|
||||||
|
**anchor 锚模式**(`check_anchors` / `assemble_anchored_output`)保真迁移:给原始文本每行编号(`[c1]` `[s1]`),LLM 摘要引用行号,代码端校验合法性并展开引文。当前生产 `anchor=False`,但代码路径完整保留供后续 A/B 实验。
|
||||||
|
|
||||||
|
**与 TRM4 有意变更**:
|
||||||
|
|
||||||
|
| 项目 | TRM4 | TRM5 |
|
||||||
|
|------|------|------|
|
||||||
|
| 归属 | `core/tree/summarizer.py`(嵌入 environment) | `app/search/summarizer.py`(独立模块) |
|
||||||
|
| 异步 | `_call_llm` 同步 | `await llm.chat()` |
|
||||||
|
| LLM 接口 | 裸 LLMClient | LLMProvider Protocol |
|
||||||
|
| 并发 | `ThreadPoolExecutor` | `asyncio.gather`(搜索结果批量摘要) |
|
||||||
|
| Prompt 内容 | store/prompts/v2/ | 原封不动复制 |
|
||||||
|
|
||||||
|
### 4.5 observe_frame(vision.py)
|
||||||
|
(原 §4.4,编号因插入 summarizer 顺移)
|
||||||
|
|
||||||
|
```python
|
||||||
|
async def observe_frame(
|
||||||
|
vlm: VLMProvider,
|
||||||
|
frame_paths: list[Path],
|
||||||
|
question: str,
|
||||||
|
prompts_dir: Path,
|
||||||
|
*,
|
||||||
|
ocr: OCRProvider | None,
|
||||||
|
verify: bool,
|
||||||
|
stats_sink: Callable[[dict[str, int]], None] | None = None,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> str: ...
|
||||||
|
```
|
||||||
|
|
||||||
|
两轮 VLM 调用保真:
|
||||||
|
|
||||||
|
```
|
||||||
|
1. [可选] OCR 转录 → 事前并置到 user_content
|
||||||
|
2. 提取轮: VLM + observe_frame_extract.md
|
||||||
|
3. [可选] 验证轮: VLM + observe_frame_verify.md
|
||||||
|
4. 返回 "[视觉观察] {证据}\n[验证] {核实结果}"
|
||||||
|
```
|
||||||
|
|
||||||
|
**与 TRM4 有意变更**:
|
||||||
|
|
||||||
|
| 项目 | TRM4 | TRM5 |
|
||||||
|
|------|------|------|
|
||||||
|
| 异步 | `_call_vl` 同步 | `await vlm.chat_with_images()` |
|
||||||
|
| VLM 接口 | 裸 LLMClient + 手动 base64 | VLMProvider Protocol,images 传 Path |
|
||||||
|
| OCR 接口 | `Callable[[list[Path]], str]` | `OCRProvider` Protocol(async) |
|
||||||
|
| Prompt 内容 | store/prompts/v2/ | 原封不动复制 |
|
||||||
|
|
||||||
|
### 4.6 OCRProvider Protocol(app/ports.py 新增)
|
||||||
|
|
||||||
|
```python
|
||||||
|
@runtime_checkable
|
||||||
|
class OCRProvider(Protocol):
|
||||||
|
"""帧文字转录端口。"""
|
||||||
|
async def transcribe_frames(self, frame_paths: list[Path]) -> str: ...
|
||||||
|
```
|
||||||
|
|
||||||
|
放置在 `app/ports.py`(与 `EmbeddingProvider` 同级),而非 `core/protocols.py`——OCR 只被 `app/search/` 使用,不是 core 共享端口。
|
||||||
|
|
||||||
|
### 4.7 MonkeyOCRClient(adapters/ocr.py)
|
||||||
|
|
||||||
|
```python
|
||||||
|
class MonkeyOCRClient:
|
||||||
|
"""实现 OCRProvider Protocol。多端点轮询 + 单帧降级。"""
|
||||||
|
def __init__(self, urls: list[str]) -> None: ...
|
||||||
|
async def check_health(self) -> None: ...
|
||||||
|
async def transcribe_frames(self, frame_paths: list[Path]) -> str: ...
|
||||||
|
```
|
||||||
|
|
||||||
|
内部同步 HTTP 调用通过 `asyncio.to_thread` 包装。端点轮询 + 线程安全 Session 保留 TRM4 逻辑。
|
||||||
|
|
||||||
|
### 4.8 TreeEnvironment 新增 API(app/tree/environment.py 扩展)
|
||||||
|
|
||||||
|
现有 `view_node()` 返回格式化字符串,不适合 summarizer 消费。需新增结构化查询方法:
|
||||||
|
|
||||||
|
```python
|
||||||
|
def get_node_text(self, node_id: str, *, anchor: bool = False) -> tuple[str, dict[str, str] | None]:
|
||||||
|
"""返回节点原始文本(或带行号锚的文本)+ anchor_map。"""
|
||||||
|
...
|
||||||
|
|
||||||
|
def get_children_info(self, node_id: str) -> list[dict[str, Any]]:
|
||||||
|
"""返回子节点结构化信息列表 [{id, time_range, summary}, ...]。"""
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
现有 `view_node()` 和 `search_similar()` 保持不变(向后兼容),新方法专供 `SearchToolDispatcher` 使用。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## §5 交互流程
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
sequenceDiagram
|
||||||
|
participant H as harness/inference
|
||||||
|
participant PM as PromptManager
|
||||||
|
participant SK as discover_skills
|
||||||
|
participant AL as AgentLoop
|
||||||
|
participant TD as SearchToolDispatcher
|
||||||
|
participant ENV as TreeEnvironment
|
||||||
|
participant S as summarizer
|
||||||
|
participant V as vision.observe_frame
|
||||||
|
participant LLM as LLMProvider(tool)
|
||||||
|
participant VLM as VLMProvider
|
||||||
|
participant OCR as OCRProvider
|
||||||
|
|
||||||
|
H->>SK: discover_skills(skills_dir)
|
||||||
|
SK-->>H: (always_text, task_skill_map, catalog_text, registry)
|
||||||
|
H->>PM: build_inference_prompt(...)
|
||||||
|
PM-->>H: system_prompt
|
||||||
|
H->>PM: format_user_prompt(question, options, l1_ids)
|
||||||
|
PM-->>H: user_prompt
|
||||||
|
H->>TD: 构造(env, tool_llm, vlm, ocr, prompts_dir, registry, embed_fn)
|
||||||
|
H->>AL: run(system_prompt, user_prompt, tool_dispatcher)
|
||||||
|
|
||||||
|
loop AgentLoop 每步推理
|
||||||
|
AL->>TD: dispatch("view_node", {node_id, question}, context)
|
||||||
|
TD->>ENV: view_node(node_id)
|
||||||
|
ENV-->>TD: 原始 card 文本 + 子节点列表
|
||||||
|
TD->>S: summarize_node(llm, raw_text, question, ...)
|
||||||
|
S->>LLM: extract 轮
|
||||||
|
LLM-->>S: raw_summary
|
||||||
|
S->>LLM: verify 轮
|
||||||
|
LLM-->>S: verify_result
|
||||||
|
S-->>TD: "[内容摘要] ...\n[核实] ..."
|
||||||
|
TD->>S: summarize_children(llm, children_info, question, ...)
|
||||||
|
S-->>TD: "★★/★ 标注\n[核实] ..."
|
||||||
|
TD-->>AL: 完整输出
|
||||||
|
|
||||||
|
AL->>TD: dispatch("search_similar", {query, question, k}, context)
|
||||||
|
TD->>ENV: search_similar(query, top_k, embed_fn)
|
||||||
|
ENV-->>TD: [(node_id, score), ...]
|
||||||
|
TD->>S: summarize_nodes_batch(llm, items, question, ...)
|
||||||
|
S-->>TD: 并发两轮摘要结果
|
||||||
|
TD-->>AL: 格式化输出
|
||||||
|
|
||||||
|
AL->>TD: dispatch("observe_frame", {node_ids, question}, context)
|
||||||
|
TD->>ENV: resolve_frame_paths(node_ids)
|
||||||
|
ENV-->>TD: list[Path]
|
||||||
|
TD->>V: observe_frame(vlm, paths, question, ...)
|
||||||
|
V->>OCR: transcribe_frames(paths)
|
||||||
|
OCR-->>V: ocr_text
|
||||||
|
V->>VLM: extract 轮(图片+OCR+问题)
|
||||||
|
VLM-->>V: raw_evidence
|
||||||
|
V->>VLM: verify 轮(图片+证据)
|
||||||
|
VLM-->>V: verify_result
|
||||||
|
V-->>TD: "[视觉观察] ...\n[验证] ..."
|
||||||
|
TD-->>AL: 输出
|
||||||
|
|
||||||
|
AL->>TD: dispatch("submit_answer", args, context)
|
||||||
|
TD-->>AL: "[ok] 答案已提交"
|
||||||
|
end
|
||||||
|
AL-->>H: LoopResult
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## §6 错误处理
|
||||||
|
|
||||||
|
| 场景 | 处理 | 与 TRM4 一致性 |
|
||||||
|
|------|------|---------------|
|
||||||
|
| 节点不存在 | env 抛 KeyError,dispatcher 捕获返回错误文本 | 一致 |
|
||||||
|
| summarize_node 提取轮失败 | 捕获 Exception,返回 `[摘要错误]` | 一致 |
|
||||||
|
| summarize_node 验证轮失败 | 降级返回 `[核实] 跳过(调用失败)` | 一致 |
|
||||||
|
| summarize_children 提取轮失败 | 降级回退原始子节点列表 | 一致 |
|
||||||
|
| 帧文件不存在 | FileNotFoundError,vision 返回 `[VL错误]` | 一致 |
|
||||||
|
| VLM 提取轮失败 | 捕获 Exception,返回 `[VL错误]` | 一致 |
|
||||||
|
| VLM 验证轮失败 | 降级返回 `[验证] 跳过(调用失败)` | 一致 |
|
||||||
|
| OCR 失败 | 降级不注入,stats `ocr_failed=1` | 一致 |
|
||||||
|
| 未知工具名 | raise ValueError,AgentLoop 不计步 | 一致 |
|
||||||
|
| read_skill 未注册 | KeyError 透传,dispatcher 捕获返回错误文本 | 一致 |
|
||||||
|
|
||||||
|
**原则**:工具执行错误不中断 AgentLoop。未知工具名 `raise ValueError`(由 AgentLoop 捕获不计步);已知工具的运行时错误在 dispatcher 层转为错误文本返回。
|
||||||
|
|
||||||
|
**允许的降级边界**(刻意宽泛捕获,与 TRM4 一致):
|
||||||
|
- OCR 转录失败 → 降级不注入(`ocr_fn` 是外部依赖,任何异常不得中断工具主流程)
|
||||||
|
- VLM 验证轮失败 → 降级跳过验证(提取结果仍然有效)
|
||||||
|
- summarize_children 失败 → 回退原始子节点列表
|
||||||
|
|
||||||
|
其他异常(如节点不存在、帧文件缺失)捕获特定异常类型,不做宽泛降级。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## §7 测试策略
|
||||||
|
|
||||||
|
| 测试文件 | 覆盖 | 方法 |
|
||||||
|
|----------|------|------|
|
||||||
|
| `tests/unit/test_search_prompt.py` | PromptManager 加载/拼装/格式化 | 临时目录写真实 TRM4 v2 prompt 文件 |
|
||||||
|
| `tests/unit/test_search_skills.py` | frontmatter 解析、discover_skills 分类 | 临时目录写 .md |
|
||||||
|
| `tests/unit/test_search_tools.py` | SearchToolDispatcher 5 个工具分发 + 摘要集成 | 假 env/LLM/VLM/OCR 通过 Protocol 注入 |
|
||||||
|
| `tests/unit/test_search_summarizer.py` | summarize_node(含 anchor 模式)、summarize_children、summarize_nodes_batch;check_anchors / assemble 纯函数用真实输入 | 假 LLMProvider |
|
||||||
|
| `tests/unit/test_search_vision.py` | observe_frame 两轮、OCR 注入/降级、stats、字幕拼接 | 假 VLMProvider + 假 OCRProvider |
|
||||||
|
| `tests/unit/test_ocr_adapter.py` | MonkeyOCRClient 健康检查/轮询/降级 | `responses` 库 mock HTTP |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## §8 被否决的方案
|
||||||
|
|
||||||
|
| 方案 | 否决理由 |
|
||||||
|
|------|---------|
|
||||||
|
| vision.py 放 app/tree/ | observe_frame 是搜索工具实现,不是建树管线;tree/ 是离线预处理模块 |
|
||||||
|
| tools/ 子包 | 当前仅 5 个工具,子包过度组织 |
|
||||||
@@ -0,0 +1,440 @@
|
|||||||
|
---
|
||||||
|
type: design
|
||||||
|
id: tree-module-vertical-slice
|
||||||
|
title: "建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移"
|
||||||
|
created: 2026-07-07
|
||||||
|
status: approved
|
||||||
|
---
|
||||||
|
|
||||||
|
# 建树模块竖切设计
|
||||||
|
|
||||||
|
## 1. 背景与动机
|
||||||
|
|
||||||
|
TRM5 的三大模块(建树、训练 harness、新题构建)中,建树是一切的地基——搜索 Agent、训练循环、检索器全部依赖树结构。当前 `app/tree/` 目录为空,需要从 reference 代码和 TRM4 迁移建树能力。
|
||||||
|
|
||||||
|
### 1.1 现状
|
||||||
|
|
||||||
|
| 资产 | 位置 | 状态 |
|
||||||
|
|------|------|------|
|
||||||
|
| TreeIndex 数据结构 | `reference/video_tree_trm/tree_index.py` | 简单 description 字段,需扩展 card |
|
||||||
|
| VideoTreeBuilder | `reference/video_tree_trm/video_tree_builder.py` | L2 轴心策略,需适配 TRM5 Protocol |
|
||||||
|
| EmbeddingModel | `reference/video_tree_trm/embeddings.py` | 双后端,需拆分到 adapters/ |
|
||||||
|
| TreeEnvironment | TRM4 `core/tree/environment.py` | 基于 flat dict,需改为 TreeIndex |
|
||||||
|
| 树增强管线 | TRM4 `core/tree/enhance/` | verify/supplement 逻辑可复用 |
|
||||||
|
| 已建树数据 | TRM4.zip `store/videos/` | 300 视频,原始卡片,质量良好 |
|
||||||
|
|
||||||
|
### 1.2 关键决策
|
||||||
|
|
||||||
|
| 决策 | 结论 | 理由 |
|
||||||
|
|------|------|------|
|
||||||
|
| 树格式 | 原始卡片(L3 6字段、L2 7字段、L1 7字段,非增强 10 字段) | 数据分析显示原始卡片质量很好;增强新增的 4 字段价值有限——people 空率 42%、emotion_cues 空率 64% |
|
||||||
|
| 数据结构 | 扩展 TreeIndex 统一承载 card + embedding | 一棵树服务两个消费者(Agent 用 card、Retriever 用 embedding),避免双格式同步问题 |
|
||||||
|
| 迁移来源 | TRM4.zip 原始树 | 未经 merge/verify 增强的干净数据,与 VideoTreeBuilder 输出格式一致 |
|
||||||
|
| 修复模式 | 独立 `app/tree/repair/` 目录,可整目录删除 | 修复是历史遗留需求,后续不再需要 |
|
||||||
|
|
||||||
|
## 2. 数据结构
|
||||||
|
|
||||||
|
### 2.1 Card 体系(frozen dataclass)
|
||||||
|
|
||||||
|
三级卡片与 VideoTreeBuilder VLM 输出对齐。L3 为 6 字段,L2 为 7 字段,L1 为 7 字段(与 TRM4 原始树一致):
|
||||||
|
|
||||||
|
| Card 类型 | 字段数 | 字段 | 类型 |
|
||||||
|
|-----------|--------|------|------|
|
||||||
|
| **L3Card** | 6 | `frame_summary` | `str` |
|
||||||
|
| | | `visible_entities` | `list[str]` |
|
||||||
|
| | | `ongoing_actions` | `list[str]` |
|
||||||
|
| | | `visible_text` | `list[str]` |
|
||||||
|
| | | `spatial_layout` | `str` |
|
||||||
|
| | | `visual_attributes` | `dict[str, Any]` |
|
||||||
|
| **L2Card** | 7 | `event_description` | `str` |
|
||||||
|
| | | `entities` | `list[str]` |
|
||||||
|
| | | `actions` | `list[str]` |
|
||||||
|
| | | `action_subjects` | `list[str]` |
|
||||||
|
| | | `visible_text` | `list[str]` |
|
||||||
|
| | | `spatial_relations` | `str` |
|
||||||
|
| | | `state_changes` | `str \| None` |
|
||||||
|
| **L1Card** | 7 | `scene_summary` | `str` |
|
||||||
|
| | | `main_setting` | `str` |
|
||||||
|
| | | `key_entities` | `list[str]` |
|
||||||
|
| | | `main_actions` | `list[str]` |
|
||||||
|
| | | `topic_keywords` | `list[str]` |
|
||||||
|
| | | `visible_text` | `list[str]` |
|
||||||
|
| | | `temporal_flow` | `str` |
|
||||||
|
|
||||||
|
所有 Card 均为 `frozen=True`,变更时整体替换(创建新 Card 实例赋给节点的 `card` 属性)。
|
||||||
|
|
||||||
|
### 2.2 节点结构
|
||||||
|
|
||||||
|
继承 reference 嵌套关系,扩展 card。每个节点均有 `id: str` 字段:
|
||||||
|
|
||||||
|
```
|
||||||
|
TreeIndex
|
||||||
|
├── metadata: IndexMeta
|
||||||
|
└── roots: list[L1Node]
|
||||||
|
├── id: str
|
||||||
|
├── card: L1Card
|
||||||
|
├── embedding: ndarray | None
|
||||||
|
├── time_range: tuple[float, float] | None
|
||||||
|
└── children: list[L2Node]
|
||||||
|
├── id: str
|
||||||
|
├── card: L2Card
|
||||||
|
├── embedding: ndarray | None
|
||||||
|
├── time_range: tuple[float, float] | None
|
||||||
|
└── children: list[L3Node]
|
||||||
|
├── id: str
|
||||||
|
├── card: L3Card
|
||||||
|
├── embedding: ndarray | None
|
||||||
|
├── frame_path: str | None
|
||||||
|
├── timestamp: float | None
|
||||||
|
└── subtitle: str | None
|
||||||
|
```
|
||||||
|
|
||||||
|
**ID 规则**:建树时由 VideoTreeBuilder 生成,格式 `l1_{i}_l2_{j}_l3_{k}`。迁移时从 TRM4 flat JSON 的 `node_id` 字段读取。反序列化时校验 ID 唯一性。
|
||||||
|
|
||||||
|
**embedding 文本源**通过 property 派生:
|
||||||
|
|
||||||
|
| 节点 | property | 来源 |
|
||||||
|
|------|----------|------|
|
||||||
|
| L3Node | `description` | `card.frame_summary` |
|
||||||
|
| L2Node | `description` | `card.event_description` |
|
||||||
|
| L1Node | `summary` | `card.scene_summary` |
|
||||||
|
|
||||||
|
**IndexMeta** 与 reference 一致:`source_path`、`modality`、`embed_model`、`embed_dim`、`created_at`。
|
||||||
|
|
||||||
|
### 2.3 序列化
|
||||||
|
|
||||||
|
- **主格式**:JSON(`save_json` / `load_json`),card 自动转 dict / 从 dict 恢复
|
||||||
|
- **embedding**:可选包含(base64 编码),默认不含
|
||||||
|
- **断点续跑**:`save_l1_json` / `load_l1_json` 保存单个 L1 子树中间结果
|
||||||
|
|
||||||
|
## 3. VideoTreeBuilder
|
||||||
|
|
||||||
|
从 reference 迁移,核心算法保真(ARCHITECTURE.md §6 算法 #1、#2、#3)。
|
||||||
|
|
||||||
|
### 3.1 改造点
|
||||||
|
|
||||||
|
| 维度 | reference | TRM5 |
|
||||||
|
|------|-----------|------|
|
||||||
|
| VLM 依赖 | `LLMClient` | `VLMProvider` Protocol |
|
||||||
|
| LLM 依赖 | 同上 | `LLMProvider` Protocol |
|
||||||
|
| 输出格式 | `L3Node.description`(字符串) | `L3Node.card: L3Card`(6 字段) |
|
||||||
|
| VLM Prompt | 返回字符串数组 | 返回结构化 JSON 对象数组 |
|
||||||
|
| 字幕输入 | 无 | 可选 `srt_entries` 参数,注入 VLM prompt |
|
||||||
|
| L2 代表帧 | 独立提取(均匀采样) | 复用 L3 帧:先提帧阶段提取所有 L3 帧,L2 从同 clip 内的 L3 帧中均匀采样 `l2_representative_frames` 个 |
|
||||||
|
| 日志 | `utils/logger_system.py` | loguru |
|
||||||
|
|
||||||
|
**治理约束**:所有 VLM/LLM 调用必须经过 `GovernedLLMClient`(四层治理栈)+ `TelemetryRecorder` 遥测。VideoTreeBuilder 通过 `VLMProvider` / `LLMProvider` Protocol 接收已治理的客户端实例,不直接接触 adapter 层。调用方(CLI 或测试)负责注入治理实例。
|
||||||
|
|
||||||
|
### 3.2 建树流程
|
||||||
|
|
||||||
|
```
|
||||||
|
1. [前置] 字幕完整性检查(可选)
|
||||||
|
2. [前置] 解析 SRT → list[SRTEntry](可选)
|
||||||
|
3. 时间切分:视频 → L1 时间区间列表
|
||||||
|
4. 帧提取:按 l3_fps 提取所有帧到缓存目录(ffmpeg 线程池并发,已存在则跳过)
|
||||||
|
5. L2 先行:从同 clip 内 L3 帧中均匀采样 l2_representative_frames 个代表帧 + 对应时段字幕 → VLM → L2Card
|
||||||
|
6. L3 向下:每帧 + L2 描述上下文 + 对应时刻字幕 → VLM → L3Card
|
||||||
|
7. L1 向上:L2 描述聚合 → LLM → L1Card
|
||||||
|
8. 字幕分配:Voronoi 中点策略将 SRT 写入 L3Node.subtitle
|
||||||
|
9. 组装 TreeIndex
|
||||||
|
10. 原子保存:先写最终 TreeIndex JSON,成功后清理 progress + intermediate 文件
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.3 VLM Prompt 修改策略
|
||||||
|
|
||||||
|
**增量修改,不做简化**。在 reference 原始 prompt 基础上追加:
|
||||||
|
|
||||||
|
- **L3 批量 prompt**:保留原始指令,追加结构化 JSON 输出格式要求 + 字幕上下文(有字幕时)
|
||||||
|
- **L3 单帧 fallback**:同上
|
||||||
|
- **L2 prompt**:保留原始指令,追加结构化输出 + 字幕
|
||||||
|
- **L1 prompt**:保留原始指令,追加结构化输出
|
||||||
|
|
||||||
|
**结构化 card JSON 解析**:VLM 返回 JSON 对象数组时,逐字段校验类型(str/list/dict)。字段缺失或类型错误时,该批次整体走逐帧 fallback(与算法 #2 一致)。单帧 fallback 解析失败时,使用空字符串/空列表填充缺失字段并记录 warning 日志。
|
||||||
|
|
||||||
|
### 3.4 保真项
|
||||||
|
|
||||||
|
| # | 算法 | 保真方式 |
|
||||||
|
|---|------|---------|
|
||||||
|
| 1 | L2 轴心建树策略 | L2 先行 → L3 向下 → L1 向上,asyncio 链式并发 |
|
||||||
|
| 2 | VLM 批量帧描述 + JSON fallback | `_L3_BATCH_SIZE=5`,解析失败逐帧 fallback |
|
||||||
|
| 3 | 断点续跑机制 | `progress.json` + L1 中间 JSON,按段恢复;最终 JSON 成功写入后清理中间文件 |
|
||||||
|
|
||||||
|
## 4. 字幕模块
|
||||||
|
|
||||||
|
位置:`app/tree/subtitle.py`。从 TRM4 `enhance/merge.py` 和 TRM3 `tools/generate_subtitles.py` 提取。
|
||||||
|
|
||||||
|
### 4.1 接口
|
||||||
|
|
||||||
|
| 函数 | 职责 |
|
||||||
|
|------|------|
|
||||||
|
| `parse_srt(path) → list[SRTEntry]` | 解析 SRT 文件,剥离 HTML 标签 |
|
||||||
|
| `check_subtitle_completeness(entries, duration, min_coverage) → SubtitleReport` | 检查覆盖率、最大空白段、条目数 |
|
||||||
|
| `extract_subtitle_for_range(entries, time_range) → str` | 提取时间范围内字幕,供 VLM prompt 注入 |
|
||||||
|
| `assign_subtitles_voronoi(index, entries) → None` | Voronoi 中点策略将字幕分配到 L3 节点 |
|
||||||
|
|
||||||
|
### 4.2 SRTEntry
|
||||||
|
|
||||||
|
```python
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SRTEntry:
|
||||||
|
start: float
|
||||||
|
end: float
|
||||||
|
text: str
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.3 SubtitleReport
|
||||||
|
|
||||||
|
```python
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SubtitleReport:
|
||||||
|
total_entries: int
|
||||||
|
coverage_ratio: float # SRT 覆盖时长 / 视频总时长
|
||||||
|
max_gap_sec: float # 最大连续无字幕间隔
|
||||||
|
usable: bool # coverage_ratio >= min_coverage
|
||||||
|
```
|
||||||
|
|
||||||
|
**决策行为**:`usable=True` 时正常注入 VLM prompt;`usable=False` 时记录 warning 日志并以无字幕模式建树(不 raise,不阻断)。解析失败(文件损坏)时 raise,由调用方决定是否降级。
|
||||||
|
|
||||||
|
### 4.4 建树时机
|
||||||
|
|
||||||
|
字幕在建树**前**解析完成,作为 `VideoTreeBuilder.build()` 的可选输入。有字幕时注入 VLM prompt 上下文,同时在建树完成后通过 Voronoi 分配写入 `L3Node.subtitle`。
|
||||||
|
|
||||||
|
## 5. 质量校验
|
||||||
|
|
||||||
|
位置:`app/tree/verify.py`。建树模式和修复模式共用。
|
||||||
|
|
||||||
|
### 5.1 校验项
|
||||||
|
|
||||||
|
| 字段 | 层级 | 逻辑 |
|
||||||
|
|------|------|------|
|
||||||
|
| `visible_text` | L1 | 每条须在下属 L2/L3 的 visible_text 中有出处 |
|
||||||
|
| `visible_text` | L2 | 每条须在下属 L3 的 visible_text 中有出处 |
|
||||||
|
| `key_entities` | L1 | 交叉校验 L2/L3 文本语料 |
|
||||||
|
| `entities` | L2 | 交叉校验 L3 文本语料(visible_text + subtitle + frame_summary) |
|
||||||
|
|
||||||
|
### 5.2 匹配算法
|
||||||
|
|
||||||
|
模糊子串匹配:忽略大小写、去除标点。不实现编辑距离容忍(简单子串匹配已足够)。
|
||||||
|
|
||||||
|
### 5.3 接口
|
||||||
|
|
||||||
|
```python
|
||||||
|
def verify_tree(index: TreeIndex) -> VerifyStats:
|
||||||
|
"""校验树节点卡片,删除不可靠内容,返回统计信息。
|
||||||
|
|
||||||
|
Card 为 frozen dataclass,校验时创建新 Card 实例(过滤后)
|
||||||
|
赋给节点的 card 属性。TreeIndex 和节点本身可变,Card 不可变。
|
||||||
|
"""
|
||||||
|
```
|
||||||
|
|
||||||
|
`VerifyStats` 记录各字段的保留/删除数量。
|
||||||
|
|
||||||
|
## 6. Embeddings
|
||||||
|
|
||||||
|
Embedding 实现遵循 Clean Architecture 依赖方向:`app/tree/` 通过 `EmbeddingProvider` Protocol 使用 embedding 能力,具体实现在 `adapters/embedding.py`。
|
||||||
|
|
||||||
|
### 6.1 依赖分层
|
||||||
|
|
||||||
|
| 层 | 位置 | 内容 |
|
||||||
|
|---|------|------|
|
||||||
|
| Protocol | `app/ports.py` | `EmbeddingProvider`:`embed(texts) → ndarray [N, D]`、`dim → int` |
|
||||||
|
| 实现 | `adapters/embedding.py` | 双后端(local sentence-transformers / remote OpenAI 兼容 API),L2 归一化 |
|
||||||
|
| 消费 | `app/tree/index.py` | `TreeIndex.embed_all(embed_fn, model_name, dim)` 接受 `embed_fn` 参数 |
|
||||||
|
|
||||||
|
### 6.2 从 reference 迁移改造
|
||||||
|
|
||||||
|
- reference 的 `EmbeddingModel` 类拆分:接口 → `app/ports.py`,实现 → `adapters/embedding.py`
|
||||||
|
- 日志从 `utils/logger_system.py` 改为 loguru
|
||||||
|
- 配置从 `EmbedConfig` 改为从 `config/default.yaml` 的 `embed:` 段读取
|
||||||
|
|
||||||
|
## 7. TreeEnvironment
|
||||||
|
|
||||||
|
位置:`app/tree/environment.py`。从 TRM4 迁移,改造为基于 TreeIndex。
|
||||||
|
|
||||||
|
### 7.1 职责边界
|
||||||
|
|
||||||
|
TreeEnvironment 是**纯数据访问层**,不含 LLM 调用。LLM 摘要和 Agent 工具分发属于 `app/search/` 模块(本次竖切不含)。
|
||||||
|
|
||||||
|
### 7.2 接口
|
||||||
|
|
||||||
|
| 方法 | 职责 | 依赖 |
|
||||||
|
|------|------|------|
|
||||||
|
| `view_node(node_id, anchor=False)` | 返回节点卡片 + 子节点概览;`anchor=True` 时为卡片字段添加行锚标 `[c1]` `[s1]` 供引用验证 | 纯数据 |
|
||||||
|
| `search_similar(query, top_k, embed_fn)` | 语义搜索 + 祖先去重 | `embed_fn` 参数 |
|
||||||
|
| `get_subtitle(node_id)` | 返回节点字幕 | 纯数据 |
|
||||||
|
| `resolve_frame_paths(node_ids)` | node_id → 帧文件路径 | 纯数据 |
|
||||||
|
|
||||||
|
### 7.3 算法 #12 保真:语义搜索
|
||||||
|
|
||||||
|
ARCHITECTURE.md §6 算法 #12 要求保真"分块 embedding、祖先去重、锚定验证"。TRM5 的实现方式:
|
||||||
|
|
||||||
|
| 原算法要素 | TRM4 实现 | TRM5 实现 | 变更理由 |
|
||||||
|
|-----------|----------|----------|---------|
|
||||||
|
| 分块 embedding | 卡片全文按 4000 字符分块,每块独立 embedding | 每节点一个 embedding(基于 description property) | TreeIndex 已有 per-node embedding,分块是 flat-dict 时代的替代方案;per-node embedding 语义更准确 |
|
||||||
|
| 祖先去重 | 搜索结果中,若某节点的祖先已在结果中则去重 | **保持不变** | — |
|
||||||
|
| 锚定验证 | `view_node(anchor=True)` 为卡片行添加 `[c1]` `[s1]` 等锚标 | **保持不变**,在 `view_node` 中实现 | — |
|
||||||
|
|
||||||
|
**分块→单节点 embedding 的变更属于核心算法修改**,需在实现 PR 的 commit message 中标注"算法 #12 变更"并说明理由。
|
||||||
|
|
||||||
|
### 7.4 与 TRM4 的其他差异
|
||||||
|
|
||||||
|
- 底层从 flat dict 改为 TreeIndex 嵌套结构
|
||||||
|
- `view_node()` 不调 LLM——纯数据返回卡片内容(LLM 摘要移至 `app/search/`)
|
||||||
|
- 通过 `id → (l1_idx, l2_idx, l3_idx)` 索引映射实现 O(1) 节点查找,映射在 TreeEnvironment 构造时一次性构建
|
||||||
|
|
||||||
|
## 8. 修复模式
|
||||||
|
|
||||||
|
位置:`app/tree/repair/`,独立可拆卸(`rm -rf app/tree/repair/` + 删除调用入口 = 零残留)。
|
||||||
|
|
||||||
|
### 8.1 文件布局
|
||||||
|
|
||||||
|
```
|
||||||
|
app/tree/repair/
|
||||||
|
├── __init__.py
|
||||||
|
├── detector.py # 检测缺失/低质量节点
|
||||||
|
├── regenerator.py # VLM 重新生成 + 向上级联
|
||||||
|
└── supplement.py # Q&A 反向补全
|
||||||
|
```
|
||||||
|
|
||||||
|
### 8.2 修复流程(底向上)
|
||||||
|
|
||||||
|
```
|
||||||
|
1. 检测:扫描所有节点,标记 NodeIssue
|
||||||
|
2. L3 修复:VLM 重新描述帧(复用现有 L2 描述作上下文 + 字幕)
|
||||||
|
3. L2 重生成:受影响 L2 从全部 L3 children 聚合(LLM)
|
||||||
|
4. L1 重生成:受影响 L1 从全部 L2 children 聚合(LLM)
|
||||||
|
5. verify_tree()
|
||||||
|
6. supplement(Q&A 反向补全,仅修复模式)
|
||||||
|
```
|
||||||
|
|
||||||
|
**治理约束**:修复模式的所有 VLM/LLM 调用同样必须经过 `GovernedLLMClient` + `TelemetryRecorder`,通过 Protocol 参数注入。
|
||||||
|
|
||||||
|
### 8.3 与建树模式的顺序差异
|
||||||
|
|
||||||
|
| | 建树模式 | 修复模式 |
|
||||||
|
|---|---------|---------|
|
||||||
|
| 前提 | 从零开始 | 已有树,局部损坏 |
|
||||||
|
| 顺序 | L2 先行 → L3 向下 → L1 向上 | L3 修复 → L2 重生成 → L1 重生成 |
|
||||||
|
| L2 上下文 | 必须先建 L2 才有上下文 | 复用现有 L2 描述 |
|
||||||
|
| 级联 | 向下再向上 | 仅向上 |
|
||||||
|
|
||||||
|
### 8.4 检测项(detector.py)
|
||||||
|
|
||||||
|
| 检查项 | 层级 | 判定条件 |
|
||||||
|
|--------|------|---------|
|
||||||
|
| 必填字段为空 | L3 | card 中 frame_summary / visible_entities 等为空 |
|
||||||
|
| 帧文件缺失 | L3 | frame_path 指向的文件不存在 |
|
||||||
|
| 无子节点 | L2/L1 | children 列表为空 |
|
||||||
|
| 时间空洞 | L2 | 相邻 L2 clips 时间范围不连续 |
|
||||||
|
|
||||||
|
### 8.5 Q&A 反向补全(supplement.py)
|
||||||
|
|
||||||
|
从 TRM4 `enhance/supplement.py` 迁移,仅修复模式使用(建树模式不含此步骤)。
|
||||||
|
|
||||||
|
| 特性 | 说明 |
|
||||||
|
|------|------|
|
||||||
|
| 注入类别白名单 | `person_name`、`location`、`score_number`、`object_name` |
|
||||||
|
| 禁止注入 | 情感、因果、时序推理 |
|
||||||
|
| LLM 调用 | 每题一次,分析缺失事实 + 搜索已有 + 注入缺失 |
|
||||||
|
| 注入前去重 | 大小写归一后去除重复条目 |
|
||||||
|
|
||||||
|
## 9. 迁移
|
||||||
|
|
||||||
|
一次性操作,完成后归档迁移工具。
|
||||||
|
|
||||||
|
### 9.1 迁移资产清单
|
||||||
|
|
||||||
|
| 资产 | TRM4 来源 | TRM5 目标 |
|
||||||
|
|------|----------|----------|
|
||||||
|
| 原始树 JSON | TRM4.zip → `store/videos/<vid>/tree.json` | `store/videos/<vid>/tree.json`(转换为 TreeIndex JSON) |
|
||||||
|
| 未压缩帧(1280×720) | TRM4.zip → `store/videos/<vid>/frames/` | `store/videos/<vid>/frames/` |
|
||||||
|
| SRT 字幕 | `data/Video-MME/subtitle/*.srt` | `data/Video-MME/subtitle/` |
|
||||||
|
| 原始视频压缩包 | `data/Video-MME/original_data/*.zip` | `data/Video-MME/original_data/` |
|
||||||
|
| 原始视频 MP4 | `data/Video-MME/videos/`(如果存在) | `data/Video-MME/videos/` |
|
||||||
|
| Benchmark 问题 | `store/questions/benchmarks/Video-MME/*.json` | `store/questions/benchmarks/Video-MME/` |
|
||||||
|
|
||||||
|
### 9.2 迁移步骤
|
||||||
|
|
||||||
|
```
|
||||||
|
1. 解压 TRM4.zip 到临时目录
|
||||||
|
2. 拷贝原始资产(帧、SRT、视频、问题)
|
||||||
|
3. 格式转换:flat tree.json → TreeIndex JSON(一次性 Python 脚本)
|
||||||
|
4. 验证(见 §9.3)
|
||||||
|
5. 清理临时目录
|
||||||
|
6. 归档转换脚本到 tools/archived/
|
||||||
|
```
|
||||||
|
|
||||||
|
格式转换脚本 `tools/convert_flat_to_treeindex.py` 是一次性工具,仅 CLI 调用,`app/`/`core/`/`adapters/` 不 import 该脚本。迁移完成后移至 `tools/archived/`。
|
||||||
|
|
||||||
|
### 9.3 迁移验收
|
||||||
|
|
||||||
|
| 验收项 | 条件 |
|
||||||
|
|--------|------|
|
||||||
|
| 视频数 | 300 个视频目录均存在 |
|
||||||
|
| tree.json | 每个视频目录有 tree.json,可正常反序列化为 TreeIndex |
|
||||||
|
| frames | 每个 L3 节点的 frame_path 对应文件存在,JPEG 可读 |
|
||||||
|
| SRT | `data/Video-MME/subtitle/` 下 SRT 文件数 ≥ 290(部分视频无字幕为已知情况) |
|
||||||
|
| 问题 | 每个视频有对应 question JSON |
|
||||||
|
|
||||||
|
**失败处理**:迁移脚本输出缺失资产报告(视频 ID + 缺失项),非零缺失时以 exit code 1 退出但不回滚已迁移的资产(允许手动补充后重跑验证)。
|
||||||
|
|
||||||
|
## 10. 文件布局与依赖
|
||||||
|
|
||||||
|
### 10.1 文件结构
|
||||||
|
|
||||||
|
```
|
||||||
|
app/tree/
|
||||||
|
├── __init__.py
|
||||||
|
├── index.py # TreeIndex, L1/L2/L3Node, L1/L2/L3Card, IndexMeta
|
||||||
|
├── video_builder.py # VideoTreeBuilder(asyncio, VLMProvider)
|
||||||
|
├── subtitle.py # SRT 解析 + 完整性检查 + Voronoi 分配
|
||||||
|
├── verify.py # 质量校验(建树/修复共用)
|
||||||
|
├── environment.py # TreeEnvironment(运行时数据访问)
|
||||||
|
└── repair/ # 修复模式(独立可拆卸)
|
||||||
|
├── __init__.py
|
||||||
|
├── detector.py # 检测缺失/低质量节点
|
||||||
|
├── regenerator.py # VLM 重新生成 + 向上级联
|
||||||
|
└── supplement.py # Q&A 反向补全
|
||||||
|
|
||||||
|
app/ports.py # EmbeddingProvider Protocol(新增)
|
||||||
|
|
||||||
|
adapters/
|
||||||
|
├── embedding.py # EmbeddingProvider 实现(local/remote 双后端)
|
||||||
|
└── vlm.py # VLMProvider 实现(最小可用版本)
|
||||||
|
|
||||||
|
tools/
|
||||||
|
├── migrate_from_trm4.sh # 迁移主脚本
|
||||||
|
└── convert_flat_to_treeindex.py # 格式转换(迁移后归档至 tools/archived/)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 10.2 依赖方向
|
||||||
|
|
||||||
|
```
|
||||||
|
app/tree/ → core/protocols.py (VLMProvider, LLMProvider, TelemetryRecorder)
|
||||||
|
app/tree/ → app/ports.py (EmbeddingProvider)
|
||||||
|
app/tree/ ✗ adapters/(只通过 Protocol)
|
||||||
|
app/tree/repair/ → app/tree/index.py, verify.py, subtitle.py(内部依赖)
|
||||||
|
adapters/ → core/protocols.py, app/ports.py(实现 Protocol)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 10.3 竖切边界
|
||||||
|
|
||||||
|
本次竖切包含 `adapters/embedding.py` 和 `adapters/vlm.py` 的**最小可用实现**,确保竖切可端到端运行(CLAUDE.md §4.4:"开始实现就必须完成到可运行状态")。完整的治理集成(GovernedVLMClient 等)在需要时增量添加。
|
||||||
|
|
||||||
|
| 模块 | 状态 |
|
||||||
|
|------|------|
|
||||||
|
| `app/tree/` 全部文件 | 本次实现 |
|
||||||
|
| `adapters/embedding.py` | 本次实现(最小可用) |
|
||||||
|
| `adapters/vlm.py` | 本次实现(最小可用) |
|
||||||
|
| `text_builder.py` | 后续实现 |
|
||||||
|
| `app/search/` tool dispatch + LLM 摘要 | Agent 层,下个竖切 |
|
||||||
|
|
||||||
|
## 11. 被拒方案
|
||||||
|
|
||||||
|
| 方案 | 拒绝理由 |
|
||||||
|
|------|---------|
|
||||||
|
| 双格式共存(TreeIndex + flat JSON) | 数据重复、同步风险、维护两套序列化 |
|
||||||
|
| 使用增强 10 字段卡片 | 数据分析显示增强字段价值有限:people 空率 42%、emotion_cues 空率 64% |
|
||||||
|
| 从当前 TRM4 store 迁移增强树 | 增强过程引入了格式偏差,原始树更干净 |
|
||||||
|
| 富卡片建树代码复用 | 代码已丢失(远端服务器仅存与 reference 相同的简单版本) |
|
||||||
|
| EmbeddingModel 放在 app/tree/ | 违反 Clean Architecture 依赖方向,外部 SDK 实现应在 adapters/ |
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
---
|
||||||
|
type: design
|
||||||
|
node_id: design:tree-module-vertical-slice
|
||||||
|
title: "建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移"
|
||||||
|
date: 2026-07-07
|
||||||
|
---
|
||||||
|
|
||||||
|
# 建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移
|
||||||
|
|
||||||
@@ -15,6 +15,36 @@
|
|||||||
"id": "plan:core-agent-adapters-llm",
|
"id": "plan:core-agent-adapters-llm",
|
||||||
"label": "core/agent/ + adapters/llm 基础设施实现计划",
|
"label": "core/agent/ + adapters/llm 基础设施实现计划",
|
||||||
"type": "plan"
|
"type": "plan"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "design:tree-module-vertical-slice",
|
||||||
|
"label": "建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移",
|
||||||
|
"type": "design"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "plan:tree-module-vertical-slice",
|
||||||
|
"label": "建树模块竖切实现计划",
|
||||||
|
"type": "plan"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "plan:question-gen",
|
||||||
|
"label": "question_gen 模块实现计划",
|
||||||
|
"type": "plan"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "design:2026-07-07-search-module-design",
|
||||||
|
"label": "搜索 Agent 装配层设计(app/search/)",
|
||||||
|
"type": "design"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "plan:2026-07-07-search-module",
|
||||||
|
"label": "app/search/ 搜索 Agent 装配层实现计划",
|
||||||
|
"type": "plan"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "plan:app-harness",
|
||||||
|
"label": "app/harness/ 训练循环编排层实现计划",
|
||||||
|
"type": "plan"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"links": [
|
"links": [
|
||||||
@@ -24,6 +54,34 @@
|
|||||||
"relation": "implements",
|
"relation": "implements",
|
||||||
"evidence": "计划实现设计文档中定义的全异步 AgentLoop + 四层治理栈",
|
"evidence": "计划实现设计文档中定义的全异步 AgentLoop + 四层治理栈",
|
||||||
"added": "2026-07-07T02:25:32.349931+00:00"
|
"added": "2026-07-07T02:25:32.349931+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source": "plan:tree-module-vertical-slice",
|
||||||
|
"target": "design:tree-module-vertical-slice",
|
||||||
|
"relation": "implements",
|
||||||
|
"evidence": "计划逐 Task 实现设计文档中的 11 个模块",
|
||||||
|
"added": "2026-07-07T05:27:02.953166+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source": "plan:question-gen",
|
||||||
|
"target": "design:question-gen",
|
||||||
|
"relation": "implements",
|
||||||
|
"evidence": "实现计划覆盖设计文档的全部需求",
|
||||||
|
"added": "2026-07-07T08:32:53.887071+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source": "plan:2026-07-07-search-module",
|
||||||
|
"target": "design:2026-07-07-search-module-design",
|
||||||
|
"relation": "implements",
|
||||||
|
"evidence": "实现搜索 Agent 装配层设计",
|
||||||
|
"added": "2026-07-07T09:36:21.467921+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source": "plan:app-harness",
|
||||||
|
"target": "design:app-harness-design",
|
||||||
|
"relation": "implements",
|
||||||
|
"evidence": "实现 2026-07-07-app-harness-design.md 的 14 文件训练循环编排层",
|
||||||
|
"added": "2026-07-07T15:45:08.729979+00:00"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
+17
-3
@@ -1,11 +1,25 @@
|
|||||||
# Research Wiki 索引
|
# Research Wiki 索引
|
||||||
|
|
||||||
> 自动生成,更新时间:2026-07-07 02:25 UTC
|
> 自动生成,更新时间:2026-07-07 15:45 UTC
|
||||||
|
|
||||||
## design (1)
|
## design (7)
|
||||||
- [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design`
|
- [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design`
|
||||||
|
- [2026-07-07-app-harness-design](designs/2026-07-07-app-harness-design.md) `design:2026-07-07-app-harness-design`
|
||||||
|
- [2026-07-07-core-evolution-design](designs/2026-07-07-core-evolution-design.md) `design:2026-07-07-core-evolution-design`
|
||||||
|
- [出题模块迁移设计(question_gen)](designs/2026-07-07-question-gen-design.md) `design:2026-07-07-question-gen-design`
|
||||||
|
- [建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移](designs/2026-07-07-tree-module-design.md) `design:2026-07-07-tree-module-design`
|
||||||
|
- [建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移](designs/tree-module-vertical-slice.md) `design:tree-module-vertical-slice`
|
||||||
|
- [搜索 Agent 装配层设计(app/search/)](designs/2026-07-07-search-module-design.md) `design:2026-07-07-search-module-design`
|
||||||
|
|
||||||
## plan (3)
|
## plan (11)
|
||||||
- [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm`
|
- [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm`
|
||||||
|
- [2026-07-07-app-harness](plans/2026-07-07-app-harness.md) `plan:2026-07-07-app-harness`
|
||||||
|
- [2026-07-07-core-evolution](plans/2026-07-07-core-evolution.md) `plan:2026-07-07-core-evolution`
|
||||||
|
- [2026-07-07-question-gen](plans/2026-07-07-question-gen.md) `plan:2026-07-07-question-gen`
|
||||||
|
- [2026-07-07-tree-module-vertical-slice](plans/2026-07-07-tree-module-vertical-slice.md) `plan:2026-07-07-tree-module-vertical-slice`
|
||||||
|
- [app/harness/ 训练循环编排层实现计划](plans/app-harness.md) `plan:app-harness`
|
||||||
|
- [app/search/ 搜索 Agent 装配层实现计划](plans/2026-07-07-search-module.md) `plan:2026-07-07-search-module`
|
||||||
- [core/agent/ + adapters/llm 基础设施实现计划](plans/core-agent-adapters-llm.md) `plan:core-agent-adapters-llm`
|
- [core/agent/ + adapters/llm 基础设施实现计划](plans/core-agent-adapters-llm.md) `plan:core-agent-adapters-llm`
|
||||||
|
- [question_gen 模块实现计划](plans/question-gen.md) `plan:question-gen`
|
||||||
|
- [建树模块竖切实现计划](plans/tree-module-vertical-slice.md) `plan:tree-module-vertical-slice`
|
||||||
- [项目基础设施初始化计划](plans/infrastructure-setup.md) `plan:infrastructure-setup`
|
- [项目基础设施初始化计划](plans/infrastructure-setup.md) `plan:infrastructure-setup`
|
||||||
|
|||||||
@@ -8,3 +8,18 @@
|
|||||||
- [2026-07-07 02:25 UTC] 新增 plan: core/agent/ + adapters/llm 基础设施实现计划 (plan:core-agent-adapters-llm)
|
- [2026-07-07 02:25 UTC] 新增 plan: core/agent/ + adapters/llm 基础设施实现计划 (plan:core-agent-adapters-llm)
|
||||||
- [2026-07-07 02:25 UTC] 新增边: plan:core-agent-adapters-llm --implements--> design:core-agent-adapters-llm
|
- [2026-07-07 02:25 UTC] 新增边: plan:core-agent-adapters-llm --implements--> design:core-agent-adapters-llm
|
||||||
- [2026-07-07 02:25 UTC] 重建索引: 4 篇页面
|
- [2026-07-07 02:25 UTC] 重建索引: 4 篇页面
|
||||||
|
- [2026-07-07 05:26 UTC] 新增 design: 建树模块竖切设计:数据结构 + 建树 + 修复 + 迁移 (design:tree-module-vertical-slice)
|
||||||
|
- [2026-07-07 05:26 UTC] 新增 plan: 建树模块竖切实现计划 (plan:tree-module-vertical-slice)
|
||||||
|
- [2026-07-07 05:27 UTC] 新增边: plan:tree-module-vertical-slice --implements--> design:tree-module-vertical-slice
|
||||||
|
- [2026-07-07 05:27 UTC] 重建索引: 8 篇页面
|
||||||
|
- [2026-07-07 08:32 UTC] 新增 plan: question_gen 模块实现计划 (plan:question-gen)
|
||||||
|
- [2026-07-07 08:32 UTC] 新增边: plan:question-gen --implements--> design:question-gen
|
||||||
|
- [2026-07-07 08:32 UTC] 重建索引: 11 篇页面
|
||||||
|
- [2026-07-07 09:14 UTC] 新增 design: 搜索 Agent 装配层设计(app/search/) (design:2026-07-07-search-module-design)
|
||||||
|
- [2026-07-07 09:14 UTC] 重建索引: 12 篇页面
|
||||||
|
- [2026-07-07 09:36 UTC] 新增 plan: app/search/ 搜索 Agent 装配层实现计划 (plan:2026-07-07-search-module)
|
||||||
|
- [2026-07-07 09:36 UTC] 新增边: plan:2026-07-07-search-module --implements--> design:2026-07-07-search-module-design
|
||||||
|
- [2026-07-07 09:36 UTC] 重建索引: 13 篇页面
|
||||||
|
- [2026-07-07 15:45 UTC] 新增 plan: app/harness/ 训练循环编排层实现计划 (plan:app-harness)
|
||||||
|
- [2026-07-07 15:45 UTC] 新增边: plan:app-harness --implements--> design:app-harness-design
|
||||||
|
- [2026-07-07 15:45 UTC] 重建索引: 18 篇页面
|
||||||
|
|||||||
@@ -0,0 +1,882 @@
|
|||||||
|
# app/harness/ 训练循环编排层 — 实现计划
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** 实现 app/harness/ 训练循环编排层(14 个文件),组合 core/evolution/ + core/agent/ + adapters/ 完成自进化闭环。
|
||||||
|
|
||||||
|
**Architecture:** 瘦 Runner class(DI 容器 + 顶层编排)调用扁平模块函数。_TrainState 显式传参,不藏在 self。全异步(asyncio.Semaphore + gather)。store.py/workspace.py 分离(SRP + SDP)。算法保真 #6/#10/#13。
|
||||||
|
|
||||||
|
**Tech Stack:** Python 3.11, asyncio, sqlite3, pydantic-settings, loguru, pytest, pytest-asyncio
|
||||||
|
|
||||||
|
**设计规格:** `research-wiki/designs/2026-07-07-app-harness-design.md`
|
||||||
|
|
||||||
|
**TRM4 参考:** `/home/iomgaa/Projects/Video-Tree-TRM4/core/harness/` (全部 .py) + `/home/iomgaa/Projects/Video-Tree-TRM4/core/workspace.py`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 0: core/evolution/__init__.py 补导出 types
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `core/evolution/__init__.py`
|
||||||
|
- Test: `tests/unit/test_evolution_types.py`(已存在,验证导入)
|
||||||
|
|
||||||
|
- [ ] **Step 1: 在 `__init__.py` 补导出全部 dataclass**
|
||||||
|
|
||||||
|
```python
|
||||||
|
from core.evolution.types import (
|
||||||
|
CaseSample, DiagnosePrompts, DiagnosisResult, ErrorAttribution,
|
||||||
|
EvolutionRecord, EvolutionResult, EvolvePrompts, GateParams,
|
||||||
|
GateVerdict, PairResult, QuadrantClassification, QuestionMetrics,
|
||||||
|
RejectedEdit, SkillCasePack, SkillStepAdherence, SpanMetrics,
|
||||||
|
SystemCasePack, ToolCasePack,
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
将上述名称加入 `__all__`。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 验证导入**
|
||||||
|
|
||||||
|
Run: `conda activate Video-Tree-TRM && python -c "from core.evolution import GateParams, EvolutionRecord, DiagnosisResult; print('ok')"`
|
||||||
|
|
||||||
|
- [ ] **Step 3: 跑全量测试确认无回归**
|
||||||
|
|
||||||
|
Run: `conda activate Video-Tree-TRM && pytest tests/unit/test_evolution_types.py tests/unit/test_gate.py tests/unit/test_evolve.py -q`
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(evolution): export dataclass types from __init__.py
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 1: config.py — RunConfig + 四层校验
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/harness/config.py`
|
||||||
|
- Test: `tests/unit/test_harness_config.py`
|
||||||
|
|
||||||
|
**参考:** TRM4 `core/harness/config.py`(308 行),几乎 1:1 迁移。
|
||||||
|
|
||||||
|
**关键差异:**
|
||||||
|
- 新增 .env 层:工程配置(workspace_dir, store_dir 等路径)可从 .env 读取,优先级 CLI > .env > YAML
|
||||||
|
- `--resume`/`--fresh` 互斥校验移到 runner.py(不在 config 校验)
|
||||||
|
- import 路径:`from core.types import GeneratedQuestion`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试** — 测试 RunConfig 构造、四层校验(valid/invalid)、load_config YAML+CLI 合并
|
||||||
|
|
||||||
|
核心测试用例:
|
||||||
|
```python
|
||||||
|
def test_valid_config(): ...
|
||||||
|
def test_mode_validation(): ...
|
||||||
|
def test_edit_budget_validation(): ...
|
||||||
|
def test_minibatch_validation(): ...
|
||||||
|
def test_gate_validation(): ...
|
||||||
|
def test_load_config_cli_overrides(): ...
|
||||||
|
def test_val_size_floor(): ... # val_size >= eval_min_per_class * 11
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 实现 config.py** — 从 TRM4 `config.py` 迁移,逐行比对保留全部校验逻辑
|
||||||
|
|
||||||
|
- [ ] **Step 3: 测试通过 + lint**
|
||||||
|
|
||||||
|
Run: `conda activate Video-Tree-TRM && pytest tests/unit/test_harness_config.py -v && ruff check app/harness/config.py`
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(harness): config.py — RunConfig frozen dataclass + 四层校验
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 2: log.py — HarnessLog + RunLogImpl
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/harness/log.py`
|
||||||
|
- Test: `tests/unit/test_harness_log.py`
|
||||||
|
|
||||||
|
**参考:** TRM4 `core/harness/log.py`(247 行)。HarnessLog 直搬 + 新增 RunLogImpl。
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
```python
|
||||||
|
# HarnessLog 测试(TRM4 行为保持)
|
||||||
|
def test_create_table_and_insert(): ...
|
||||||
|
def test_query_thread_safety(): ... # Lock 保护
|
||||||
|
def test_context_manager_status(): ...
|
||||||
|
def test_insert_or_ignore_idempotent(): ...
|
||||||
|
def test_wal_mode(): ...
|
||||||
|
|
||||||
|
# RunLogImpl 测试(新增)
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_run_log_get_predictions(): ...
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_run_log_get_traces(): ...
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_run_log_readonly(): ... # 不触发 _runs INSERT
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 实现 log.py**
|
||||||
|
|
||||||
|
HarnessLog: 从 TRM4 直搬。关键保留:WAL + Lock + INSERT OR IGNORE + query 也持锁。
|
||||||
|
|
||||||
|
RunLogImpl: 新增,实现 `core.evolution.protocols.RunLog`:
|
||||||
|
```python
|
||||||
|
class RunLogImpl:
|
||||||
|
def __init__(self, db_path: Path) -> None:
|
||||||
|
self._db_path = db_path
|
||||||
|
|
||||||
|
async def get_predictions(self, run_id, *, question_ids=None):
|
||||||
|
return await asyncio.to_thread(self._query_predictions, run_id, question_ids)
|
||||||
|
|
||||||
|
async def get_traces(self, run_id, *, question_ids=None):
|
||||||
|
return await asyncio.to_thread(self._query_traces, run_id, question_ids)
|
||||||
|
|
||||||
|
def _query_predictions(self, run_id, question_ids):
|
||||||
|
# 独立 sqlite3.connect(只读,不经 HarnessLog 生命周期)
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: 测试通过 + lint + Protocol 兼容性验证**
|
||||||
|
|
||||||
|
```python
|
||||||
|
assert isinstance(RunLogImpl(tmp_path / "test.db"), RunLog)
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(harness): log.py — HarnessLog SQLite wrapper + RunLogImpl Protocol impl
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 3: store.py — Store 版本操作 + Seed 管理
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/harness/store.py`
|
||||||
|
- Test: `tests/unit/test_harness_store.py`
|
||||||
|
|
||||||
|
**参考:** TRM4 `core/workspace.py` 中 Store + Seed 相关函数(~300 行)。
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
```python
|
||||||
|
def test_parse_version(): ...
|
||||||
|
def test_list_versions_numeric_sort(): ... # v10 排在 v2 后
|
||||||
|
def test_next_version(): ...
|
||||||
|
def test_advance_version(): ...
|
||||||
|
def test_init_store(): ...
|
||||||
|
def test_init_seed(): ...
|
||||||
|
def test_read_seed_not_found(): ...
|
||||||
|
def test_extract_run_db_preserves_pk(): ... # 原始 CREATE 保留主键
|
||||||
|
def test_promote_to_seed_version_mismatch(): ... # 强校验
|
||||||
|
def test_promote_to_seed_null_version(): ...
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 实现 store.py** — 从 TRM4 workspace.py 拆出
|
||||||
|
|
||||||
|
函数集:`_parse_version`, `list_versions`, `next_version`, `advance_version`, `_write_meta`, `init_store`, `init_seed`, `list_seeds`, `read_seed`, `extract_run_db`, `promote_to_seed`。
|
||||||
|
|
||||||
|
关键保留:numeric 排序(非字典序);extract_run_db 用原始 CREATE 语句;promote_to_seed 强校验版本一致+非NULL;临时 db finally 清理。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 测试通过 + lint**
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(harness): store.py — Store 版本操作 + Seed 管理
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 4: workspace.py — Workspace 生命周期 + Protocol 实现
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/harness/workspace.py`
|
||||||
|
- Test: `tests/unit/test_harness_workspace.py`
|
||||||
|
|
||||||
|
**参考:** TRM4 `core/workspace.py` 中 Workspace 相关函数(~350 行)。
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
```python
|
||||||
|
def test_resolved_paths_frozen(): ...
|
||||||
|
def test_init_workspace(): ...
|
||||||
|
def test_init_workspace_from_seed(): ...
|
||||||
|
def test_init_workspace_from_seed_missing_questions(): ... # fail-fast
|
||||||
|
def test_load_manifest(): ...
|
||||||
|
def test_update_manifest_invalid_key(): ...
|
||||||
|
def test_record_run_idempotent(): ... # 同 run_id 不重复 + exist_ok
|
||||||
|
def test_update_best_independent_of_current(): ...
|
||||||
|
def test_archive_workspace(): ...
|
||||||
|
|
||||||
|
# Protocol 实现测试
|
||||||
|
def test_versioned_skill_store_read(): ...
|
||||||
|
def test_versioned_skill_store_list(): ...
|
||||||
|
def test_versioned_prompt_store(): ...
|
||||||
|
def test_skill_store_protocol_compliance(): ... # isinstance check
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 实现 workspace.py**
|
||||||
|
|
||||||
|
从 TRM4 workspace.py 拆出 Workspace 相关函数。新增 VersionedSkillStore / VersionedPromptStore。
|
||||||
|
|
||||||
|
关键保留:
|
||||||
|
- skills_dir/prompts_dir 解析到 workspace(非 store)
|
||||||
|
- init_workspace_from_seed 创建前校验 questions ref
|
||||||
|
- record_run 幂等(history + 目录 exist_ok)
|
||||||
|
- best 指针独立于 current
|
||||||
|
|
||||||
|
依赖 store.py:`from app.harness.store import advance_version, read_seed, ...`
|
||||||
|
|
||||||
|
- [ ] **Step 3: 测试通过 + Protocol 兼容性**
|
||||||
|
|
||||||
|
```python
|
||||||
|
from core.evolution.protocols import SkillStore, PromptStore
|
||||||
|
assert isinstance(VersionedSkillStore(tmp_path), SkillStore)
|
||||||
|
assert isinstance(VersionedPromptStore(tmp_path), PromptStore)
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(harness): workspace.py — Workspace lifecycle + SkillStore/PromptStore Protocol impl
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 5: pools.py — 三池切分
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/harness/pools.py`
|
||||||
|
- Test: `tests/unit/test_harness_pools.py`
|
||||||
|
|
||||||
|
**参考:** TRM4 `core/harness/pools.py`(238 行),1:1 迁移。
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
```python
|
||||||
|
def test_build_pools_mutual_exclusion(): ... # 三池 question_id 无交集
|
||||||
|
def test_build_pools_test_natural_distribution(): ... # test 池 correct_ratio=None
|
||||||
|
def test_save_load_pools_roundtrip(): ...
|
||||||
|
def test_load_pools_old_format_reject(): ... # 无 test → ValueError
|
||||||
|
def test_build_or_load_pools_frozen(): ... # 已存在则加载不重切
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 实现 pools.py** — 从 TRM4 直搬
|
||||||
|
|
||||||
|
import 路径变更:`from core.types import GeneratedQuestion`,`from app.question_gen import stratified_sample`。
|
||||||
|
|
||||||
|
GeneratedQuestion 序列化适配:TRM5 的 options 是 `tuple[str, ...]` + 多了 source_nodes/difficulty 字段,`_q_to_dict` / `_dict_to_q` 需适配。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 测试通过 + lint**
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(harness): pools.py — 三池切分(test→validation→diagnosis)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 6: batching.py — 算法保真 #10
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/harness/batching.py`
|
||||||
|
- Test: `tests/unit/test_harness_batching.py`
|
||||||
|
|
||||||
|
**参考:** TRM4 `core/harness/batching.py`(241 行),1:1 迁移。**算法保真 #10**。
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试** — 从 TRM4 测试迁移 + 新增保真校验
|
||||||
|
|
||||||
|
```python
|
||||||
|
def test_build_batches_deterministic(): ... # 相同 seed 产出相同结果
|
||||||
|
def test_small_class_not_split(): ... # 小类整组不拆
|
||||||
|
def test_large_class_round_robin(): ... # 大类全局指针散布
|
||||||
|
def test_correct_ratio_mixing(): ... # 正确题按比例混入
|
||||||
|
def test_no_wrong_answers_empty(): ... # 无错题 → ([], 0)
|
||||||
|
def test_validate_params_strict(): ... # min_class < batch_size
|
||||||
|
def test_correctness_false_vs_none(): ... # is False 精确匹配
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 实现 batching.py** — 从 TRM4 直搬,逐行比对
|
||||||
|
|
||||||
|
import 变更:`from core.types import GeneratedQuestion`。
|
||||||
|
|
||||||
|
**保真校验点**:与 TRM4 batching.py 逐函数比对——`build_batches`, `_validate_params`, `_split_by_size`, `_select_mixed_by_task_type`, `_small_groups_decreasing`, `_pack_small_class`, `_distribute_large_classes`, `_place_round_robin` 所有 8 个函数的逻辑完全一致。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 测试通过 + lint**
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(harness): batching.py — FFD + round-robin mini-batch (#10 算法保真)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 7: gate_ladder.py — 算法保真 #6
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/harness/gate_ladder.py`
|
||||||
|
- Test: `tests/unit/test_harness_gate_ladder.py`
|
||||||
|
|
||||||
|
**参考:** TRM4 `core/harness/gate_ladder.py`(343 行),1:1 迁移。**算法保真 #6**。
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
```python
|
||||||
|
def test_cold_start_interleaving(): ... # 2:1 错对交错 + probe 尾
|
||||||
|
def test_cold_start_p_hat_beta(): ... # 错=1/3, 对=2/3
|
||||||
|
def test_warm_ordering_information(): ... # p̂(1-p̂) 降序
|
||||||
|
def test_warm_filter_bounds(): ... # 剔除 p̂ ∉ [p_low, p_high]
|
||||||
|
def test_gate_pools_save_load_atomic(): ... # .tmp + os.replace
|
||||||
|
def test_gate_pools_fingerprint_mismatch(): ... # RuntimeError
|
||||||
|
def test_baseline_cache_content_addressed(): ... # 四维键
|
||||||
|
def test_baseline_cache_disk_first(): ... # 先盘后存
|
||||||
|
def test_ladder_for_excludes_qids(): ... # 排除案例包题
|
||||||
|
def test_gamma_ema_update(): ... # p̂ ← γ·p̂ + (1-γ)·obs
|
||||||
|
def test_update_probs_excludes_gate_runs(): ... # 调用方须过滤 _gate_ run,update_probs 只接收已过滤的 observations
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 实现 gate_ladder.py** — 从 TRM4 直搬,逐行比对
|
||||||
|
|
||||||
|
import 变更:`from core.types import GeneratedQuestion`。
|
||||||
|
|
||||||
|
全部类型和函数保留:`LadderEntry`, `GatePools`, `BaselineCache`, `skill_hash`, `build_cold_entries`, `order_ladder`, `build_or_load_gate_pools`。
|
||||||
|
|
||||||
|
**保真校验点**:冷启动 2:1 交错逻辑、probe 探针抽取、信息量排序公式、γ-EMA 更新、BaselineCache 四维键 + 先盘后存、指纹 sha1 计算。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 测试通过 + lint**
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(harness): gate_ladder.py — 信息阶梯 + BaselineCache (#6 算法保真)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 8: observation.py — 五表 + 报告
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/harness/observation.py`
|
||||||
|
- Test: `tests/unit/test_harness_observation.py`
|
||||||
|
|
||||||
|
**参考:** TRM4 `core/harness/metric_log.py`(295 行)+ `loop_report.py`(108 行),合并迁移。
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
```python
|
||||||
|
# 五表写入/回读(全覆盖)
|
||||||
|
def test_write_read_dual_metric(): ...
|
||||||
|
def test_write_read_shadow_gate(): ...
|
||||||
|
def test_write_read_holdout_eval(): ...
|
||||||
|
def test_write_read_gate_evidence(): ...
|
||||||
|
def test_write_read_quadrant_pairs(): ...
|
||||||
|
def test_null_not_zero_for_soft(): ... # None → NULL, 不存 0
|
||||||
|
def test_read_only_connection(): ... # read 不触发 _runs INSERT
|
||||||
|
|
||||||
|
# 报告
|
||||||
|
def test_write_step_report(): ...
|
||||||
|
def test_write_step_report_skipped_null_fields(): ... # gate 字段 None
|
||||||
|
def test_write_epoch_report(): ...
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 实现 observation.py** — 合并 metric_log + loop_report
|
||||||
|
|
||||||
|
全部 5 张表 schema 保留。write_*/read_* 函数保留。新增 write_step_report/write_epoch_report。
|
||||||
|
|
||||||
|
关键保留:read 走独立只读连接;soft/mixed None → NULL;bool→int 转换;幂等建表。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 测试通过 + lint**
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(harness): observation.py — 五张观测表 + step/epoch 报告
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 9: inference.py — async 推理编排
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/harness/inference.py`
|
||||||
|
- Test: `tests/unit/test_harness_inference.py`
|
||||||
|
|
||||||
|
**参考:** TRM4 `core/harness/inference.py`(~560 行)。**重大重构:同步→异步 + DI**。
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
```python
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_run_inference_basic(): ... # mock LLM + ToolDispatcher
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_run_inference_concurrency(): ... # Semaphore 限制
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_prediction_always_written(): ... # 异常时仍落库 stop_reason=error
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_to_text_field(): ... # list/dict → JSON str
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_run_id_empty_raises(): ... # 空串 ValueError
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_aggregate_results_from_memory(): ... # 不从 DB 回读
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_inference_result_frozen(): ...
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 实现 inference.py**
|
||||||
|
|
||||||
|
关键签名:
|
||||||
|
```python
|
||||||
|
async def run_inference(
|
||||||
|
questions: list[GeneratedQuestion],
|
||||||
|
*,
|
||||||
|
llm: LLMProvider,
|
||||||
|
tool_dispatch_fn: Callable,
|
||||||
|
log: HarnessLog,
|
||||||
|
run_id: str,
|
||||||
|
concurrency: int,
|
||||||
|
max_steps: int,
|
||||||
|
skill_mode: str,
|
||||||
|
plugins_factory: Callable[[str, str], list[object]] | None = None,
|
||||||
|
) -> InferenceResult:
|
||||||
|
```
|
||||||
|
|
||||||
|
从 TRM4 迁移:InferenceResult dataclass、5 张表 schema、_to_text_field、_run_single_question(→ async)、_aggregate_results(从内存聚合)。
|
||||||
|
|
||||||
|
新增:asyncio.Semaphore + gather 替代 ThreadPoolExecutor。plugins_factory 接收 (video_id, question_id) 返回插件列表(TracePlugin 等由调用方装配)。
|
||||||
|
|
||||||
|
关键保留:悲观默认值(stop_reason="error");prediction 必落库(try 外);_to_text_field 归一化。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 测试通过 + lint**
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(harness): inference.py — async run_inference + DI (#11 loop integration)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 10: validate.py — 块序贯验证编排
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/harness/validate.py`
|
||||||
|
- Test: `tests/unit/test_harness_validate.py`
|
||||||
|
|
||||||
|
**参考:** TRM4 `core/harness/validate.py`(626 行)。**重大重构:sync→async + 调 core/evolution 纯函数**。
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
```python
|
||||||
|
# 类型测试
|
||||||
|
def test_validation_outcome_fields(): ...
|
||||||
|
def test_probation_fields(): ...
|
||||||
|
|
||||||
|
# materialize
|
||||||
|
def test_materialize_candidate_skill(): ...
|
||||||
|
def test_materialize_cleanup_on_failure(): ...
|
||||||
|
|
||||||
|
# 块序贯验证
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_validate_skill_local_accept(): ... # mock inference
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_validate_skill_local_reject(): ...
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_gate_prefix_must_contain_gate(): ... # ValueError
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_infra_guard_threshold(): ... # 分母≥10 才触发
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_baseline_cache_hit(): ... # miss 才推理
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_block_level_barrier(): ... # 块间顺序执行
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_last_block_terminal(): ... # 无循环外补判
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 实现 validate.py**
|
||||||
|
|
||||||
|
类型定义:InferenceRunConfig, ValidationOutcome, Probation(从 TRM4 迁移,Probation 的 pending_edits 用 `core.evolution.types.RejectedEdit`)。
|
||||||
|
|
||||||
|
函数 async 化:validate_skill_local, _run_local_validation, _resolve_baseline_block, _run_candidate_block 加 async。
|
||||||
|
|
||||||
|
替换 core/ 调用:`from core.evolution import gate_decision, pair_block, classify_quadrants`(纯函数,直接调用)。替代 TRM4 的 `_pair_block` 和 `_classify_quadrants` 本地实现。
|
||||||
|
|
||||||
|
关键保留:gate_run_prefix 含 "_gate_" 入口校验;materialize 用 .cand_tmp/ + finally 清理;INFRA 护栏跨块累计分母≥10;块级屏障顺序执行;最后一块判定即终态;只有终态题携带 stop_reason。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 测试通过 + lint**
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(harness): validate.py — async 块序贯验证编排
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 11: momentum.py — 慢更新动量
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/harness/momentum.py`
|
||||||
|
- Test: `tests/unit/test_harness_momentum.py`
|
||||||
|
|
||||||
|
**参考:** TRM4 `core/harness/momentum.py`(156 行),async 化。
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
```python
|
||||||
|
def test_categorize_pair_all_four(): ...
|
||||||
|
def test_categorize_pair_missing_key(): ... # KeyError 不掩盖
|
||||||
|
def test_format_comparison_pairs_order(): ... # REGRESSED 优先
|
||||||
|
def test_format_comparison_pairs_empty(): ...
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_run_slow_momentum_basic(): ... # mock LLM
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_run_slow_momentum_parse_failure(): ... # 保留 prev_guidance
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 实现 momentum.py** — 从 TRM4 迁移 + async 化
|
||||||
|
|
||||||
|
关键保留:
|
||||||
|
- 四类常量单一真源
|
||||||
|
- 展示顺序 REGRESSED 优先
|
||||||
|
- `_format_comparison_pairs` 在 try 外(KeyError 不被 ValueError 吞)
|
||||||
|
- 解析失败保留 prev_guidance
|
||||||
|
- `client.chat` → `await llm.chat`,返回 LLMResponse
|
||||||
|
- `extract_json_from_response` 从 `core.evolution.diagnose` 导入(TRM5 已有)
|
||||||
|
|
||||||
|
- [ ] **Step 3: 测试通过 + lint**
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(harness): momentum.py — async 慢更新动量生成
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 12: checkpoint.py — 状态序列化
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/harness/checkpoint.py`
|
||||||
|
- Test: `tests/unit/test_harness_checkpoint.py`
|
||||||
|
|
||||||
|
**参考:** TRM4 `core/harness/checkpoint.py`(257 行),1:1 迁移。
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
```python
|
||||||
|
def test_serialize_deserialize_roundtrip(): ...
|
||||||
|
def test_serialize_set_to_sorted_list(): ... # changed_task_types
|
||||||
|
def test_deserialize_nested_system_pack(): ... # CaseSample 重建
|
||||||
|
def test_deserialize_nested_probation(): ... # RejectedEdit 重建
|
||||||
|
def test_deserialize_missing_key_raises(): ... # 硬失败不 .get
|
||||||
|
def test_fingerprint_structural_vs_decision(): ...
|
||||||
|
def test_check_fingerprint_structural_reject(): ...
|
||||||
|
def test_check_fingerprint_decision_warn(): ...
|
||||||
|
def test_write_checkpoint_atomic(): ... # .tmp + os.replace
|
||||||
|
def test_load_checkpoint_missing(): ... # 返回 None
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 实现 checkpoint.py** — 从 TRM4 直搬
|
||||||
|
|
||||||
|
import 路径变更:
|
||||||
|
- `from core.evolution.types import CaseSample, SystemCasePack, ToolCasePack, RejectedEdit`
|
||||||
|
- `from app.harness.validate import Probation`
|
||||||
|
|
||||||
|
完整持久化字段集:correctness, eval_prev_acc, eval_prev_run_id, baseline_skills_version, baseline_prompts_version, steps_since_best_improved, epoch_start_skills, changed_task_types_this_epoch, rejected_buffer, system_packs, tool_packs, probations, gate_cooldown, gate_epoch_observed。
|
||||||
|
|
||||||
|
嵌套复活规则保留:SystemCasePack→CaseSample, Probation→RejectedEdit。
|
||||||
|
|
||||||
|
关键保留:反序列化 d[...] 不用 .get;结构性 vs 决策性指纹;原子写。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 测试通过 + lint**
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(harness): checkpoint.py — TrainState 序列化 + 原子写 + 指纹校验
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 13: runner.py — 训练循环编排(算法保真 #13)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/harness/runner.py`
|
||||||
|
- Test: `tests/unit/test_harness_runner.py`
|
||||||
|
|
||||||
|
**参考:** TRM4 `core/harness/runner.py`(2273 行)。**最大的 Task,但 Runner 瘦身为 ~400 行编排器**。
|
||||||
|
|
||||||
|
本 Task 分为多个 sub-step,按训练循环层级组织。
|
||||||
|
|
||||||
|
### Sub-task 13a: Runner 类骨架 + _TrainState + 模式路由
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试** — Runner 构造、infer 模式、eval 模式
|
||||||
|
|
||||||
|
```python
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_runner_init(): ...
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_runner_infer(): ... # mock inference
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_runner_eval_backfill_versions(): ... # _runs 版本回填
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 实现 Runner 类骨架**
|
||||||
|
|
||||||
|
```python
|
||||||
|
class Runner:
|
||||||
|
def __init__(self, config: RunConfig, *, llm: LLMProvider,
|
||||||
|
evolve_llm: LLMProvider, vlm: VLMProvider,
|
||||||
|
telemetry: TelemetryRecorder) -> None:
|
||||||
|
self._config = config
|
||||||
|
self._llm = llm
|
||||||
|
self._evolve_llm = evolve_llm
|
||||||
|
self._vlm = vlm
|
||||||
|
self._telemetry = telemetry
|
||||||
|
self._paths = resolve_paths(config.workspace_dir)
|
||||||
|
|
||||||
|
async def infer(self, ...) -> InferenceResult: ...
|
||||||
|
async def eval(self, version: str) -> InferenceResult: ...
|
||||||
|
async def diagnose(self, run_id: str) -> DiagnosisResult: ...
|
||||||
|
def promote(self, version, eval_run_id, name) -> None: ...
|
||||||
|
```
|
||||||
|
|
||||||
|
_TrainState dataclass(19 字段)、_ensure_workspace 三态逻辑、resume_plan 纯函数。
|
||||||
|
|
||||||
|
关键保留:_ensure_workspace 的 resume+fresh 互斥 → ValueError;resume 无 checkpoint → RuntimeError;无 flag+已有 → SystemExit;eval 版本回填。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 测试通过**
|
||||||
|
|
||||||
|
### Sub-task 13b: train() 三级嵌套 + _run_step
|
||||||
|
|
||||||
|
- [ ] **Step 4: 写测试** — train 循环骨架、rollout、correctness 增量
|
||||||
|
|
||||||
|
```python
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_train_epoch_step_nesting(): ... # mock 全链路
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_run_step_sequence(): ... # rollout→correctness→diagnose→gate
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_guard_infra_failures(): ... # >10% error rate
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_apply_batch_correctness_complete(): ... # 缺行 → RuntimeError
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_apply_batch_correctness_incremental(): ... # 增量更新
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_init_gate_pools_empty_baseline_raises(): ... # baseline run 零 prediction 行 → RuntimeError
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 5: 实现 train() + _run_step + 辅助函数**
|
||||||
|
|
||||||
|
模块级函数(不依赖 self):`resume_plan`, `_guard_infra_failures`, `_apply_batch_correctness`, `_accumulate_slow_packs`, `_batch_from_ids`, `_snapshot_current_skills`, `_compute_total_steps`, `_should_early_stop`。
|
||||||
|
|
||||||
|
train() 三级嵌套:epoch 循环 + batch 切分 + step 循环 + checkpoint。
|
||||||
|
|
||||||
|
关键保留:resume 用 saved_batches;新 epoch 清空累加器;每 step 后 global_step++;checkpoint 落在 step 副作用全部完成后;epoch_done 前清空累加包;early stop 在慢更新后判。
|
||||||
|
|
||||||
|
- [ ] **Step 6: 测试通过**
|
||||||
|
|
||||||
|
### Sub-task 13c: _gate_batch_skills + accept/reject/probation
|
||||||
|
|
||||||
|
- [ ] **Step 7: 写测试** — per-skill gate、accept/reject、probation
|
||||||
|
|
||||||
|
```python
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_gate_cooldown_skip(): ...
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_gate_no_change_skip(): ... # evolved==original
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_gate_accept_confirmed(): ...
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_gate_accept_provisional_open_probation(): ...
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_gate_default_strategy_no_probation(): ... # 共享文件不开账
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_gate_reject_blacklist(): ...
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_rejected_summary_only_applied(): ... # 黑名单防污染
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_rollback_probation_file_level(): ... # revert-commit 式
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cooldown_decrement(): ... # 每 step 递减
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 8: 实现 _gate_batch_skills + accept/reject/rollback**
|
||||||
|
|
||||||
|
关键保留:cooldown admission control;evolve 无改动跳过不进 gate;排除案例包题构造 ladder;accept 开账快照在合并前拍取;correctness 二轨合并;清黑名单;probation 分岔(default 不开账);reject 黑名单只记 applied edits;rollback 文件级 revert(不整体回退 manifest)。
|
||||||
|
|
||||||
|
- [ ] **Step 9: 测试通过**
|
||||||
|
|
||||||
|
### Sub-task 13d: _slow_update_cycle 十步序
|
||||||
|
|
||||||
|
- [ ] **Step 10: 写测试** — 十步序关键路径
|
||||||
|
|
||||||
|
```python
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_slow_update_r_before_prompts(): ... # Phase 1 先于 7
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_slow_update_probation_settle(): ... # Phase 4
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_slow_update_best_strict_greater(): ... # Phase 5
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_slow_update_momentum_immutable_version(): ... # Phase 6
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_slow_update_r2_revert(): ... # Phase 8 退步
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_slow_update_r2_keep(): ... # Phase 8 保留
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_slow_update_r2_version_binding(): ... # R2 用 r2_skills_version
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_slow_update_gate_refresh_sources(): ... # 精确三源
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_slow_update_reverted_r2_not_absorbed(): ... # 防泄露
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_soft_score_missing_table_raises(): ... # span_evaluations 表不存在 → RuntimeError
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_probation_settle_missing_prediction_raises(): ... # 快照题缺预测行 → RuntimeError
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_momentum_samples_from_diagnosis_pool(): ... # 从诊断池采样,不从 val 池
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 11: 实现 _slow_update_cycle**
|
||||||
|
|
||||||
|
十步序完整实现,~150 行。逐步比对 TRM4 runner.py:1363-1523。
|
||||||
|
|
||||||
|
关键保留:版本快照在 R 前捕获;R 无条件回写;probation 结算覆盖回写;best argmax 严格大于;momentum 不可变新版本;system/tool 用 edit_budget_end;R2 版本对绑定 r2_skills_version(绝不沿用 R 的 eval_skills_version);gate 阶梯精确三源(GLOB 排除 _gate_ + R 精确 run_id + kept R2);reverted R2 不吸收。
|
||||||
|
|
||||||
|
- [ ] **Step 12: 测试通过**
|
||||||
|
|
||||||
|
### Sub-task 13e: deliver_best + final_test + held-out
|
||||||
|
|
||||||
|
- [ ] **Step 13: 写测试**
|
||||||
|
|
||||||
|
```python
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_deliver_best_rollback(): ...
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_final_test_eval(): ...
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_early_stop_step_granularity(): ... # 步粒度累加
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_epoch_report_system_tool_action_tristate(): ... # updated/reverted/none
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_holdout_no_decision_side_effect(): ... # held-out 仅观测落库
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 14: 实现 deliver_best + final_test + held-out**
|
||||||
|
|
||||||
|
- [ ] **Step 15: 全量测试通过 + lint**
|
||||||
|
|
||||||
|
Run: `conda activate Video-Tree-TRM && pytest tests/unit/test_harness_runner.py -v`
|
||||||
|
|
||||||
|
- [ ] **Step 16: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(harness): runner.py — 瘦编排器 + 训练循环三级嵌套 (#13 算法保真)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 14: __init__.py + 集成验证
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/__init__.py`
|
||||||
|
- Test: `tests/unit/test_harness_init.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写 __init__.py 公开 API**
|
||||||
|
|
||||||
|
```python
|
||||||
|
"""app/harness/ — 训练循环编排层。"""
|
||||||
|
|
||||||
|
from app.harness.config import RunConfig, load_config
|
||||||
|
from app.harness.inference import InferenceResult, run_inference
|
||||||
|
from app.harness.log import HarnessLog, RunLogImpl
|
||||||
|
from app.harness.pools import Pools, build_or_load_pools, build_pools, load_pools, save_pools
|
||||||
|
from app.harness.runner import Runner
|
||||||
|
from app.harness.workspace import (
|
||||||
|
ResolvedPaths, VersionedPromptStore, VersionedSkillStore,
|
||||||
|
resolve_paths,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"HarnessLog", "InferenceResult", "Pools", "ResolvedPaths",
|
||||||
|
"RunConfig", "RunLogImpl", "Runner", "VersionedPromptStore",
|
||||||
|
"VersionedSkillStore", "build_or_load_pools", "build_pools",
|
||||||
|
"load_config", "load_pools", "resolve_paths", "run_inference",
|
||||||
|
"save_pools",
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 集成测试** — 验证全模块 import + Runner 构造
|
||||||
|
|
||||||
|
```python
|
||||||
|
def test_public_api_imports(): ...
|
||||||
|
def test_runner_construction(): ... # 全依赖注入
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: 跑全量 harness 测试**
|
||||||
|
|
||||||
|
Run: `conda activate Video-Tree-TRM && pytest tests/unit/test_harness_*.py -v --tb=short`
|
||||||
|
|
||||||
|
- [ ] **Step 4: 跑全量项目测试确认无回归**
|
||||||
|
|
||||||
|
Run: `conda activate Video-Tree-TRM && pytest tests/ -q`
|
||||||
|
|
||||||
|
- [ ] **Step 5: lint 全量**
|
||||||
|
|
||||||
|
Run: `conda activate Video-Tree-TRM && ruff check app/harness/ --fix && ruff format app/harness/`
|
||||||
|
|
||||||
|
- [ ] **Step 6: Commit**
|
||||||
|
|
||||||
|
```
|
||||||
|
feat(harness): __init__.py public API + integration verification
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 算法保真校验
|
||||||
|
|
||||||
|
本计划涉及 3 项核心算法迁移:
|
||||||
|
|
||||||
|
| # | 算法 | 保真 Task | 校验方式 |
|
||||||
|
|---|------|----------|---------|
|
||||||
|
| 6 | 信息阶梯 | Task 7 | 逐函数比对 TRM4 gate_ladder.py(8 个函数) |
|
||||||
|
| 10 | mini-batch | Task 6 | 逐函数比对 TRM4 batching.py(8 个函数) |
|
||||||
|
| 13 | 训练循环编排 | Task 13 | 逐方法比对 TRM4 runner.py(十步序 + 三级嵌套 + probation) |
|
||||||
|
|
||||||
|
不涉及的核心算法(#1-#5, #7-#9, #11-#12):已在 core/evolution/ (Design A) 或 app/tree/ 中实现,本计划不触及。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## TRM4 Tips 覆盖索引
|
||||||
|
|
||||||
|
120 条 tips 按 Task 归属:
|
||||||
|
|
||||||
|
| Task | Tips |
|
||||||
|
|------|------|
|
||||||
|
| 0 | — |
|
||||||
|
| 1 (config) | T48-T53 |
|
||||||
|
| 2 (log) | T87-T90 |
|
||||||
|
| 3 (store) | T114, T118-T120 |
|
||||||
|
| 4 (workspace) | T113, T115-T117 |
|
||||||
|
| 5 (pools) | T69-T72 |
|
||||||
|
| 6 (batching) | T62-T68 |
|
||||||
|
| 7 (gate_ladder) | T73-T81 |
|
||||||
|
| 8 (observation) | T106-T109, T110-T112 |
|
||||||
|
| 9 (inference) | T54-T61 |
|
||||||
|
| 10 (validate) | T97-T105 |
|
||||||
|
| 11 (momentum) | T82-T86 |
|
||||||
|
| 12 (checkpoint) | T91-T96 |
|
||||||
|
| 13 (runner) | T1-T47 |
|
||||||
|
| 14 (__init__) | — |
|
||||||
|
|
||||||
|
T6-T7(resume/fresh 互斥):归 Task 13a(Runner._ensure_workspace)。
|
||||||
|
T8(eval 版本回填):归 Task 13a(Runner.eval)。
|
||||||
|
T9(soft score 表存在性检查):归 Task 13d(_slow_update_cycle Phase 2)。
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,418 @@
|
|||||||
|
---
|
||||||
|
type: plan
|
||||||
|
node_id: plan:2026-07-07-search-module
|
||||||
|
title: "app/search/ 搜索 Agent 装配层实现计划"
|
||||||
|
date: 2026-07-07
|
||||||
|
---
|
||||||
|
|
||||||
|
# app/search/ 搜索 Agent 装配层实现计划
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** 完整迁移 TRM4 搜索 Agent 装配层到 TRM5 app/search/,包含 prompt 管理、skill 注册、工具分发、LLM 两轮摘要、VLM 视觉观察和 OCR 支持。
|
||||||
|
|
||||||
|
**Architecture:** 方案 A 平铺模块(6 个文件 + 1 个 adapter + 1 个 Protocol)。所有 LLM/VLM 调用通过 Protocol 注入,environment 保持纯数据层。详见 `research-wiki/designs/2026-07-07-search-module-design.md`。
|
||||||
|
|
||||||
|
**Tech Stack:** Python 3.11, asyncio, pluggy, loguru, requests, numpy, pytest
|
||||||
|
|
||||||
|
**核心算法保真声明:** 本计划不涉及 ARCHITECTURE.md §6 核心算法保真清单中的 13 项关键算法迁移。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 文件结构总览
|
||||||
|
|
||||||
|
| 操作 | 文件 | 职责 |
|
||||||
|
|------|------|------|
|
||||||
|
| Create | `app/search/__init__.py` | 公开 API 重导出 |
|
||||||
|
| Create | `app/search/skills.py` | SkillRegistry + discover_skills |
|
||||||
|
| Create | `app/search/summarizer.py` | 两轮 LLM 摘要 + anchor 锚模式 |
|
||||||
|
| Create | `app/search/vision.py` | observe_frame(VLM 两轮 + OCR) |
|
||||||
|
| Create | `app/search/tools.py` | SearchToolDispatcher + 工具描述 |
|
||||||
|
| Create | `app/search/prompt.py` | PromptManager |
|
||||||
|
| Create | `adapters/ocr.py` | MonkeyOCRClient |
|
||||||
|
| Modify | `app/ports.py` | 新增 OCRProvider Protocol |
|
||||||
|
| Modify | `app/tree/environment.py` | 新增 get_node_text / get_children_info |
|
||||||
|
| Copy | `store/prompts/*.md` × 9 | 从 TRM4 v2 字节级复制 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 1: 复制 Prompt 种子文件
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Copy: `store/prompts/` (9 files from TRM4 `store/prompts/v2/`)
|
||||||
|
|
||||||
|
- [ ] **Step 1: 复制全部 prompt 文件**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mkdir -p store/prompts
|
||||||
|
for f in system.md observe_frame_extract.md observe_frame_verify.md view_node_extract.md view_node_verify.md view_node_children_extract.md view_node_children_verify.md search_similar_extract.md search_similar_verify.md; do
|
||||||
|
cp /home/iomgaa/Projects/Video-Tree-TRM4/store/prompts/v2/$f store/prompts/$f
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 字节级校验**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
for f in system.md observe_frame_extract.md observe_frame_verify.md view_node_extract.md view_node_verify.md view_node_children_extract.md view_node_children_verify.md search_similar_extract.md search_similar_verify.md; do
|
||||||
|
diff /home/iomgaa/Projects/Video-Tree-TRM4/store/prompts/v2/$f store/prompts/$f
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: 无输出(全部一致)
|
||||||
|
|
||||||
|
- [ ] **Step 3: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add store/prompts/ && git commit -m "chore: 复制 TRM4 v2 prompt 种子文件(9 个,字节级一致)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 2: OCRProvider Protocol + MonkeyOCRClient
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/ports.py` — 新增 OCRProvider
|
||||||
|
- Create: `adapters/ocr.py` — MonkeyOCRClient
|
||||||
|
- Create: `tests/unit/test_ocr_adapter.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写 OCR 测试**
|
||||||
|
|
||||||
|
`tests/unit/test_ocr_adapter.py`。测试 Protocol 合规性、单帧转录、失败降级、健康检查、轮询、行去重过滤。使用 `responses` 库 mock HTTP。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行测试确认失败**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_ocr_adapter.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: ImportError
|
||||||
|
|
||||||
|
- [ ] **Step 3: 实现 OCRProvider Protocol**
|
||||||
|
|
||||||
|
`app/ports.py` 新增 `OCRProvider(Protocol)` + `async def transcribe_frames(self, frame_paths: list[Path]) -> str`。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 实现 MonkeyOCRClient**
|
||||||
|
|
||||||
|
`adapters/ocr.py` 从 TRM4 `core/tree/ocr.py` 迁移。公开方法改 async(`asyncio.to_thread` 包装同步 HTTP)。构造函数 `ValueError` 替代 `assert`。逻辑与 TRM4 完全一致:多端点轮询、线程安全 Session、单帧失败降级、行去重过滤。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 运行测试**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_ocr_adapter.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: 全部 PASS
|
||||||
|
|
||||||
|
- [ ] **Step 6: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/ports.py adapters/ocr.py tests/unit/test_ocr_adapter.py
|
||||||
|
git commit -m "feat(adapters): OCRProvider Protocol + MonkeyOCRClient 异步实现"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 3: TreeEnvironment 扩展
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/tree/environment.py` — 新增 get_node_text + get_children_info
|
||||||
|
- Modify: `tests/unit/test_tree_environment.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
追加 `TestGetNodeText`(正常/anchor 模式/不存在节点)和 `TestGetChildrenInfo`(L1 有子节点/L3 空/不存在节点)到现有测试文件。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行测试确认失败**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_tree_environment.py::TestGetNodeText -v
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: AttributeError
|
||||||
|
|
||||||
|
- [ ] **Step 3: 实现**
|
||||||
|
|
||||||
|
`get_node_text(node_id, *, anchor=False) -> tuple[str, dict[str, str] | None]`:复用已有 `_node_full_text` / `_node_anchored_text`,anchor 模式解析行号构建 anchor_map。
|
||||||
|
|
||||||
|
`get_children_info(node_id) -> list[dict[str, Any]]`:复用 `_get_children` + `_node_description` + `_format_time_range`。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行测试**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_tree_environment.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: 全部 PASS
|
||||||
|
|
||||||
|
- [ ] **Step 5: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/tree/environment.py tests/unit/test_tree_environment.py
|
||||||
|
git commit -m "feat(tree): TreeEnvironment.get_node_text + get_children_info 结构化查询"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 4: app/search/skills.py
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/search/skills.py`
|
||||||
|
- Create: `tests/unit/test_search_skills.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
测试 `parse_frontmatter`(正常/缺结束符/无 frontmatter)、`strip_frontmatter`、`SkillRegistry.read`(正常/未注册 KeyError)、`discover_skills`(always/task_type/catalog 分类 + 空目录)。使用 `tmp_path` 创建临时 .md 文件。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行测试确认失败**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_search_skills.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: 实现**
|
||||||
|
|
||||||
|
从 TRM4 `core/search/skills.py` 保真迁移。逻辑完全一致。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行测试**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_search_skills.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 5: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/search/skills.py tests/unit/test_search_skills.py
|
||||||
|
git commit -m "feat(search): SkillRegistry + discover_skills — skill 扫描与注册"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 5: app/search/summarizer.py
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/search/summarizer.py`
|
||||||
|
- Create: `tests/unit/test_search_summarizer.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写 anchor 工具测试**
|
||||||
|
|
||||||
|
测试 `check_anchors`(合法锚保留/非法锚删除/范围展开/声明句不计数)和 `assemble_anchored_output`(ids/ids_expand/expand_only 三种模式 + 封顶逻辑)。纯函数,无需 mock。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行测试确认失败**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_search_summarizer.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: 实现 anchor 工具**
|
||||||
|
|
||||||
|
从 TRM4 `core/tree/summarizer.py` 保真迁移:`_expand_anchor_ids`, `check_anchors`, `_cited_anchor_ids`, `assemble_anchored_output` + 全部正则常量。逻辑完全一致。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行 anchor 测试**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_search_summarizer.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 5: 写 summarize_* 测试**
|
||||||
|
|
||||||
|
测试 `summarize_node`(两轮正常/提取失败/验证失败降级/anchor 模式)、`summarize_children`(正常/失败回退原始列表)、`summarize_nodes_batch`(并发多节点)。使用 FakeLLMProvider mock。
|
||||||
|
|
||||||
|
- [ ] **Step 6: 实现 summarize_node / summarize_children / summarize_nodes_batch**
|
||||||
|
|
||||||
|
从 TRM4 迁移。有意变更:同步→async;`_call_llm` → `await llm.chat()`(返回 `response.content`);`ThreadPoolExecutor` → `asyncio.gather`;透传 `session_id` / `parent_call_id`。
|
||||||
|
|
||||||
|
- [ ] **Step 7: 运行全部 summarizer 测试**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_search_summarizer.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 8: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/search/summarizer.py tests/unit/test_search_summarizer.py
|
||||||
|
git commit -m "feat(search): summarizer — 两轮 LLM 摘要 + anchor 锚模式"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 6: app/search/vision.py
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/search/vision.py`
|
||||||
|
- Create: `tests/unit/test_search_vision.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
测试 `observe_frame`:两轮正常、verify=False 仅提取、OCR 注入、OCR 失败降级、OCR 为 None、VLM 提取失败、VLM 验证失败降级、帧文件不存在、stats 键完整性。使用 FakeVLMProvider + FakeOCRProvider mock。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行测试确认失败**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_search_vision.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: 实现**
|
||||||
|
|
||||||
|
从 TRM4 `core/tree/vision.py` 迁移。有意变更:`await vlm.chat_with_images(messages, images)` 替代手动 base64 + 同步 `_call_vl`;images 传 Path 列表;OCR `await ocr.transcribe_frames()`;透传遥测字段。输出格式与 TRM4 完全一致。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行测试**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_search_vision.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 5: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/search/vision.py tests/unit/test_search_vision.py
|
||||||
|
git commit -m "feat(search): vision.observe_frame — VLM 两轮 + OCR 异步实现"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 7: app/search/tools.py
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/search/tools.py`
|
||||||
|
- Create: `tests/unit/test_search_tools.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
测试 `get_tool_descriptions`(含/不含 read_skill)、`SearchToolDispatcher.dispatch` 五个工具(view_node 调 env+summarizer、search_similar 调 env+summarize_batch、observe_frame 调 env+vision+subtitle 拼接、submit_answer 返回文本、read_skill 调 registry)+ 未知工具 ValueError + 节点不存在错误文本。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行测试确认失败**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_search_tools.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: 实现**
|
||||||
|
|
||||||
|
`get_tool_descriptions()` 工具描述文本与 TRM4 完全一致。`SearchToolDispatcher` 类封装,构造注入全部依赖,`dispatch` 按工具名路由到 `_handle_view_node` / `_handle_search_similar` / `_handle_observe_frame` 私有方法。`ValueError` 直接抛出(未知工具),其他异常捕获返回错误文本。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行测试**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_search_tools.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 5: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/search/tools.py tests/unit/test_search_tools.py
|
||||||
|
git commit -m "feat(search): SearchToolDispatcher — 5 工具分发 + 摘要集成"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 8: app/search/prompt.py
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/search/prompt.py`
|
||||||
|
- Create: `tests/unit/test_search_prompt.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写测试**
|
||||||
|
|
||||||
|
测试 `__init__`(加载 system.md / 不存在抛错)、`build_inference_prompt`(auto/manual/none 三种 skill_mode)、`format_user_prompt`(含/不含 task_type)、`load`(正常/不存在)。使用 `tmp_path` 写入 prompt 文件。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行测试确认失败**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_search_prompt.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: 实现**
|
||||||
|
|
||||||
|
从 TRM4 `core/search/prompt.py` 迁移。有意变更:工具描述从 `app.search.tools.get_tool_descriptions` 获取;`format_user_prompt` 参数显式化(question/options/l1_node_ids/task_type)。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行测试**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_search_prompt.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 5: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/search/prompt.py tests/unit/test_search_prompt.py
|
||||||
|
git commit -m "feat(search): PromptManager — prompt 加载与拼装"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 9: app/search/__init__.py + Lint + 全量测试
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `app/search/__init__.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 创建 __init__.py**
|
||||||
|
|
||||||
|
```python
|
||||||
|
"""搜索 Agent 装配层 — prompt 管理、skill 注册、工具分发、LLM 摘要、视觉观察。"""
|
||||||
|
from app.search.prompt import PromptManager
|
||||||
|
from app.search.skills import SkillRegistry, discover_skills
|
||||||
|
from app.search.tools import SearchToolDispatcher, get_tool_descriptions
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"PromptManager",
|
||||||
|
"SkillRegistry",
|
||||||
|
"SearchToolDispatcher",
|
||||||
|
"discover_skills",
|
||||||
|
"get_tool_descriptions",
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Lint**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & ruff check app/search/ adapters/ocr.py --fix && ruff format app/search/ adapters/ocr.py
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: 全量 search 测试**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/unit/test_search_prompt.py tests/unit/test_search_skills.py tests/unit/test_search_tools.py tests/unit/test_search_summarizer.py tests/unit/test_search_vision.py tests/unit/test_ocr_adapter.py -v
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: 全部 PASS
|
||||||
|
|
||||||
|
- [ ] **Step 4: 回归测试**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
conda activate Video-Tree-TRM & pytest tests/ -v --tb=short
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: 全部 PASS
|
||||||
|
|
||||||
|
- [ ] **Step 5: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/search/__init__.py && git commit -m "feat(search): __init__.py 公开 API + lint 通过"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 10: 更新 ARCHITECTURE.md
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `research-wiki/ARCHITECTURE.md`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 更新 §2.3 目录树中 app/search/**
|
||||||
|
|
||||||
|
替换为实际 6 个文件。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 更新 §3.3 ToolDispatcher 实现映射**
|
||||||
|
|
||||||
|
`app/search/skills.py SkillRegistry` → `app/search/tools.py SearchToolDispatcher`。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 更新 §3.2 OCRProvider 方法签名**
|
||||||
|
|
||||||
|
`recognize(image_path)` → `transcribe_frames(frame_paths: list[Path]) -> str`。
|
||||||
|
|
||||||
|
- [ ] **Step 4: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add research-wiki/ARCHITECTURE.md && git commit -m "docs: 同步 app/search/ + OCRProvider 到 ARCHITECTURE.md"
|
||||||
|
```
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,9 @@
|
|||||||
|
---
|
||||||
|
type: plan
|
||||||
|
node_id: plan:app-harness
|
||||||
|
title: "app/harness/ 训练循环编排层实现计划"
|
||||||
|
date: 2026-07-07
|
||||||
|
---
|
||||||
|
|
||||||
|
# app/harness/ 训练循环编排层实现计划
|
||||||
|
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
---
|
||||||
|
type: plan
|
||||||
|
node_id: plan:tree-module-vertical-slice
|
||||||
|
title: 建树模块竖切实现计划
|
||||||
|
date: 2026-07-07
|
||||||
|
---
|
||||||
|
|
||||||
|
# 建树模块竖切实现计划
|
||||||
|
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
你是一个视觉证据提取器。你服务于一个视频问答推理系统,该系统通过工具调用你来查看视频关键帧的画面内容。该系统掌握完整的视频上下文,而你只能看到当前这几帧。因此,你的职责是准确描述画面内容,推理和判断由该系统完成。
|
||||||
|
|
||||||
|
## 你会收到的输入
|
||||||
|
|
||||||
|
1. 1-4 张视频关键帧图片
|
||||||
|
2. 一个针对画面内容的视觉问题
|
||||||
|
|
||||||
|
## 工作原则
|
||||||
|
|
||||||
|
先陈述画面事实,后回答问题。你必须先逐帧列出画面中直接可见的原子事实——人物外观与着装、正在发生的动作、物体及其空间位置关系——然后才基于这些事实回答问题。[视觉回答] 中的每个断言都要标注它依据的帧号和事实编号,[画面事实] 中没有列出的内容不得出现在回答里。
|
||||||
|
|
||||||
|
画面内的文字(计分板、字幕、标牌、卡牌文本等)必须逐字转录,并为每处文字标注可读性:清晰可读、部分可读或模糊不可辨。标注"模糊不可辨"时禁止给出猜测的内容——承认看不清比编造一个流畅的答案更有价值。
|
||||||
|
|
||||||
|
不要凭部分外观特征(发色、胡须、体型)断定画面人物是某个具体的人。你只描述看到的特征,身份匹配由掌握完整视频上下文的推理系统完成。
|
||||||
|
|
||||||
|
不要做超出当前画面的推断。你看不到这几帧之前或之后发生了什么,因此不要推断事件的先后顺序、因果关系或累计次数。例如,你可以说"9 号球衣的球员正在射门",但不要说"这是他的第 3 次射门"——你无法从当前帧中得知这一点。
|
||||||
|
|
||||||
|
如果画面中没有回答问题所需的证据,输出 [证据不存在] 并具体说明缺少什么要素。
|
||||||
|
|
||||||
|
## 输出格式
|
||||||
|
|
||||||
|
[画面事实] <逐帧编号列出直接可见的原子事实,如"帧1-a: ……";画面内文字逐字转录并标注可读性>
|
||||||
|
[视觉回答] <基于画面事实回答问题中询问的每个要素,每个断言标注依据的帧号和事实编号,如(帧1-a)>
|
||||||
|
[证据不存在] <画面中未出现回答该问题所需的具体要素时,说明缺少什么;有充分证据时省略此段>
|
||||||
|
[其他信息] <画面中与问题无关但可能有用的视觉信息,没有则省略>
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
你是一个视觉证据核实器。你将收到一段关于图片的描述(由另一个模型生成),你的任务是对照原始图片,逐条核实该描述的准确性。
|
||||||
|
|
||||||
|
## 你会收到的输入
|
||||||
|
|
||||||
|
1. 与描述生成时相同的图片
|
||||||
|
2. 用户当时提出的问题
|
||||||
|
3. 另一个模型基于这些图片生成的描述
|
||||||
|
|
||||||
|
## 工作原则
|
||||||
|
|
||||||
|
首先检查描述是否回答了问题中的每个要素。然后逐条检查每一个事实性陈述:
|
||||||
|
- 问题中询问的每个要素是否都得到了回答?
|
||||||
|
- 描述提到的实体是否确实存在于画面中?
|
||||||
|
- 描述的动作是否确实正在发生?
|
||||||
|
- 描述引用的文字(计分板、字幕等)是否与画面中的文字一致?
|
||||||
|
- 描述是否包含了画面中不存在的信息?
|
||||||
|
- 描述的外观细节(颜色、发型、穿着)是否与画面一致?
|
||||||
|
|
||||||
|
如果描述中包含超出画面的推断(如因果关系、时序判断、累计计数),指出这些是推断而非画面事实。
|
||||||
|
|
||||||
|
## 输出格式
|
||||||
|
|
||||||
|
details=<逐条核实结果>; confidence=<0.0-1.0>
|
||||||
|
|
||||||
|
置信度含义:
|
||||||
|
- 1.0: 描述完全准确,每个细节都与画面一致
|
||||||
|
- 0.7-0.9: 主要内容准确,个别细节有出入或无法确认
|
||||||
|
- 0.4-0.6: 部分准确,但存在明显错误或过度推断
|
||||||
|
- 0.0-0.3: 描述与画面严重不符
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
你是一个视频搜索结果摘要器。你服务于一个视频问答推理系统,该系统通过语义搜索找到了一个可能相关的视频节点,需要你快速判断该节点与问题的相关性并提取关键信息。推理和最终判断由该系统完成。
|
||||||
|
|
||||||
|
## 你会收到的输入
|
||||||
|
|
||||||
|
1. 用户正在研究的问题
|
||||||
|
2. 一个语义搜索命中的视频节点的描述文本和字幕
|
||||||
|
|
||||||
|
## 工作原则
|
||||||
|
|
||||||
|
仅基于提供的内容回答,不使用外部知识。你看不到其他节点的内容,因此不要推断跨节点的事件顺序、因果关系或全局结论。
|
||||||
|
|
||||||
|
由于推理系统需要快速扫描多个搜索结果,请保持输出简洁(3-5 句关键信息)。优先报告能直接回答问题的事实,其次报告间接相关的背景信息。
|
||||||
|
|
||||||
|
字幕中的引用是重要证据来源,请保留关键原文片段。
|
||||||
|
|
||||||
|
如果内容与问题无关,明确说明"该节点未包含与问题直接相关的信息",并用一句话概括该节点的实际内容。
|
||||||
|
|
||||||
|
## 输出格式
|
||||||
|
|
||||||
|
[关键信息] <3-5 句与问题相关的关键事实,按相关性排列>
|
||||||
|
[原文] <1-3 句与问题最相关的字幕原文或描述原文,保留原始措辞,不改写不概括。无关则省略>
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
你是一个搜索结果摘要核实器。你将收到一段关于视频搜索结果的摘要(由另一个模型生成),以及该节点的原始描述和字幕。请核实摘要是否准确。
|
||||||
|
|
||||||
|
## 你会收到的输入
|
||||||
|
|
||||||
|
1. 用户正在研究的问题
|
||||||
|
2. 节点的原始描述文本和字幕
|
||||||
|
3. 另一个模型基于上述内容生成的摘要
|
||||||
|
|
||||||
|
## 检查要点
|
||||||
|
|
||||||
|
- 摘要提到的事实是否确实存在于原始内容中?
|
||||||
|
- 摘要是否包含了原始内容中不存在的推断?
|
||||||
|
- 摘要是否遗漏了原始内容中与问题高度相关的重要信息?
|
||||||
|
- [原文] 引用是否准确保留了原始措辞?
|
||||||
|
|
||||||
|
## 输出格式
|
||||||
|
|
||||||
|
details=<逐条核实结果>; confidence=<0.0-1.0>
|
||||||
|
|
||||||
|
置信度含义:
|
||||||
|
- 1.0: 摘要完全准确,无遗漏
|
||||||
|
- 0.7-0.9: 主要内容准确,个别细节有出入
|
||||||
|
- 0.4-0.6: 部分准确,但存在明显错误或过度推断
|
||||||
|
- 0.0-0.3: 摘要与原始内容严重不符
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
## 角色
|
||||||
|
|
||||||
|
你是一个视频树搜索 Agent,任务是在预构建的层次化视频树上导航,收集视频证据并回答四选一单选题(A/B/C/D)。你是一个谨慎的证据收集者,宁可多搜一步验证也不轻易下结论。
|
||||||
|
|
||||||
|
你最常犯的错误是找到第一条支持证据就急于提交答案,而没有为竞争选项做独立搜索。为了避免这一点,你应该在每次工具调用前通过 reflect 审视已有证据是否真的足以区分选项,在每次工具调用后通过 plan 评估下一步是否值得花费步数预算。对每个选项都应形成判断——"无直接证据"本身也是有效的判断。
|
||||||
|
|
||||||
|
## 能力边界
|
||||||
|
|
||||||
|
你通过工具浏览节点的文本摘要、字幕转写和结构化描述,但无法直接观看视频画面。如果需要确认画面中的视觉细节(人物外观、计分板数字、物体空间位置等),必须使用 observe_frame 工具。
|
||||||
|
|
||||||
|
需要注意的是,你获取的所有信息都是文本形式的二次表示,而非视频原始内容。文本摘要可能存在概括偏差或遗漏细节,字幕转写可能存在 OCR 识别错误。因此,对于决定最终答案的关键证据,应尽可能通过多个节点或多种信息源(摘要 + 字幕 + 视觉)进行交叉验证。
|
||||||
|
|
||||||
|
## 输出格式
|
||||||
|
|
||||||
|
你的 thinking(深度推理)可以自由分析,不受格式约束。你的 content 必须输出纯 JSON,包含三个顶层字段:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"reflect": { ... },
|
||||||
|
"plan": { ... },
|
||||||
|
"action": {"tool": "工具名称", "args": { ... }}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
其中 reflect 用于结构化反思(第一轮可省略),plan 用于结构化规划,action 指定本轮要调用的工具及其参数。reflect 和 plan 的具体字段由当前加载的搜索策略定义。action 的格式是固定的:tool 为工具名称字符串,args 为该工具的参数字典。
|
||||||
|
|
||||||
|
## 视频树结构
|
||||||
|
|
||||||
|
视频被组织为三层树,每层提供不同粒度的信息。你应该根据当前需要的信息精度选择在哪一层搜索。
|
||||||
|
|
||||||
|
### L1 — 场景(~5 分钟)
|
||||||
|
|
||||||
|
L1 是最粗粒度的层级,每个节点覆盖约 5 分钟的视频片段。适合快速建立全局认知,了解视频的整体结构、主题和时间线。
|
||||||
|
|
||||||
|
| 字段 | 内容 |
|
||||||
|
|------|------|
|
||||||
|
| scene_summary | 场景整体摘要 |
|
||||||
|
| main_setting | 主要场景设定 |
|
||||||
|
| key_entities | 关键实体列表 |
|
||||||
|
| main_actions | 主要动作 |
|
||||||
|
| topic_keywords | 主题关键词 |
|
||||||
|
| temporal_flow | 时间推进描述 |
|
||||||
|
| visible_text | 画面中可见的文字 |
|
||||||
|
| subtitle | 完整字幕(较长) |
|
||||||
|
|
||||||
|
### L2 — 事件(~30 秒)
|
||||||
|
|
||||||
|
L2 是中间粒度,每个节点覆盖约 30 秒的视频片段。适合缩小搜索范围后深入理解具体事件的因果关系和实体行为。
|
||||||
|
|
||||||
|
| 字段 | 内容 |
|
||||||
|
|------|------|
|
||||||
|
| event_description | 事件描述 |
|
||||||
|
| entities | 出现的实体 |
|
||||||
|
| actions | 发生的动作 |
|
||||||
|
| action_subjects | 动作主体 |
|
||||||
|
| spatial_relations | 空间关系变化 |
|
||||||
|
| state_changes | 状态变化 |
|
||||||
|
| visible_text | 画面中可见的文字 |
|
||||||
|
| subtitle | 字幕片段 |
|
||||||
|
|
||||||
|
### L3 — 关键帧(单帧)
|
||||||
|
|
||||||
|
L3 是最细粒度的层级,每个节点对应一张关键帧。适合获取精确证据、确认具体的视觉细节和时间戳。
|
||||||
|
|
||||||
|
| 字段 | 内容 |
|
||||||
|
|------|------|
|
||||||
|
| frame_summary | 帧内容描述 |
|
||||||
|
| visible_entities | 可见实体 |
|
||||||
|
| ongoing_actions | 正在发生的动作 |
|
||||||
|
| spatial_layout | 空间布局 |
|
||||||
|
| visual_attributes | 光照、主色调、机位 |
|
||||||
|
| visible_text | 画面中可见的文字 |
|
||||||
|
| subtitle | 字幕(短) |
|
||||||
|
|
||||||
|
### 信任层级
|
||||||
|
|
||||||
|
三个层级的信息有不同的信任度。L1 和 L2 的摘要是概括性的,适合用于导航和定位相关区域,但它们可能遗漏关键细节或存在概括偏差。L3 关键帧是最细粒度的信息来源——在给出最终答案前,你应该优先基于 L3 级证据做判断,而非仅凭 L1/L2 摘要下结论。当外部知识与视频证据冲突时,以视频证据为准。三个层级都包含 visible_text 和 subtitle 字段,但粒度不同。
|
||||||
|
|
||||||
|
## 决策原则
|
||||||
|
|
||||||
|
你有固定的步数预算,每次工具调用消耗一步。每步工具返回中会显示当前进度(已用/总步数),这是帮助你合理分配搜索深度的参考信息,不是在催促你赶紧结束。总体策略是前期投入步数建立全局认知、定位相关区域,后期聚焦于验证和区分候选选项。如果预算即将耗尽但仍有不确定性,选择证据支持度最高的选项提交——不完美的判断优于耗尽预算不作答。
|
||||||
|
|
||||||
|
### 搜索工具使用
|
||||||
|
|
||||||
|
search_similar 有两个文本参数,它们的职责不同:query 是用于向量检索的关键词(2-4 个词即可,简洁精准),question 是你当前想了解的具体问题(用于对检索结果做内容筛选和摘要)。不要把完整问题塞进 query,也不要把关键词放在 question 里。
|
||||||
|
|
||||||
|
### 否定题原则
|
||||||
|
|
||||||
|
当问题包含否定词(not / NOT / 没有 / 不是 / 除了)时,应采用排除法:为每个选项单独搜索,确认其在视频中是否出现。当已为 3 个选项找到存在证据,而第 4 个选项经过 2 次以上不同关键词搜索仍未找到匹配时,可以判定该选项不存在并作为答案提交。不要因为无法 100% 确认不存在而无限搜索——"搜不到"本身就是强证据。
|
||||||
|
|
||||||
|
### 置信度语义
|
||||||
|
|
||||||
|
置信度反映的是你对 best_candidate 的区分性证据强度,而非你对问题的理解程度:
|
||||||
|
|
||||||
|
| 范围 | 含义 |
|
||||||
|
|------|------|
|
||||||
|
| 0.1-0.4 | 尚未找到区分性证据。可能还没有查看相关节点,或查看了但内容与问题无关,或只能排除 1 个明显不合理的选项 |
|
||||||
|
| 0.5-0.6 | 有倾向但无法明确区分。找到了相关区域,best_candidate 有初步支持,但尚未找到能将它与竞争选项明确区分开的关键信息 |
|
||||||
|
| 0.7-0.8 | 有区分性证据。找到了能明确区分 best_candidate 与竞争选项的关键信息——可以是字幕原文的关键台词、L3 帧的视觉细节、多个 L1 摘要的一致覆盖模式、或时间戳的精确对比,取决于题目性质 |
|
||||||
|
| 0.9-1.0 | 高度确信。多源证据交叉验证了 best_candidate,且至少 1 个竞争选项有明确的反面证据 |
|
||||||
|
|
||||||
|
当 confidence 达到 0.7 以上时,将 answer_ready 设为 true 并调用 submit_answer 提交答案。submit_answer 要求提供三个参数:你选中的选项(answer)、支撑该选项的关键证据摘要(evidence)、以及你对每个选项的判断理由(reasoning,包括"无直接证据"的选项)。
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
你是一个视频子节点导航标注器。你服务于一个视频问答推理系统,该系统通过工具调用你来判断哪些子节点值得深入探索。推理和最终判断由该系统完成,你只负责评估每个子节点与问题的相关性。
|
||||||
|
|
||||||
|
## 你会收到的输入
|
||||||
|
|
||||||
|
1. 用户正在研究的问题
|
||||||
|
2. 一组子节点列表,每个子节点包含 ID、时间范围和摘要描述
|
||||||
|
|
||||||
|
## 工作原则
|
||||||
|
|
||||||
|
仅基于提供的子节点摘要评估相关性,不使用外部知识。你看不到子节点的详细内容,只能基于摘要做初步判断。
|
||||||
|
|
||||||
|
对每个子节点标注相关性等级:
|
||||||
|
- ★★ 高度相关:很可能包含直接回答问题的信息
|
||||||
|
- ★ 相关:可能包含间接相关的信息
|
||||||
|
- 无标注:与问题不相关
|
||||||
|
|
||||||
|
将通用描述改写为差异化描述,避免重复相似的措辞,帮助推理系统快速区分各子节点的独特内容。
|
||||||
|
|
||||||
|
## 输出格式
|
||||||
|
|
||||||
|
[子节点标注] 每行一个子节点:
|
||||||
|
- ★★ {子节点ID} ({时间范围}): {差异化描述}
|
||||||
|
- ★ {子节点ID} ({时间范围}): {差异化描述}
|
||||||
|
- {子节点ID} ({时间范围}): {差异化描述}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
你是一个子节点标注核实器。你将收到一份子节点相关性标注(由另一个模型生成),以及原始的子节点列表和用户问题。请核实标注是否合理。
|
||||||
|
|
||||||
|
## 你会收到的输入
|
||||||
|
|
||||||
|
1. 用户正在研究的问题
|
||||||
|
2. 原始的子节点列表(含 ID、时间范围、摘要)
|
||||||
|
3. 另一个模型基于上述信息生成的相关性标注
|
||||||
|
|
||||||
|
## 检查要点
|
||||||
|
|
||||||
|
- ★★ 标注的子节点摘要是否确实与问题高度相关?
|
||||||
|
- 是否有与问题明显相关的子节点被遗漏标注?
|
||||||
|
- 差异化描述是否准确反映了原始摘要的含义,没有添加不存在的信息?
|
||||||
|
|
||||||
|
## 输出格式
|
||||||
|
|
||||||
|
details=<逐条核实结果>; confidence=<0.0-1.0>
|
||||||
|
|
||||||
|
置信度含义:
|
||||||
|
- 1.0: 标注完全合理,无遗漏
|
||||||
|
- 0.7-0.9: 主要标注合理,个别可商榷
|
||||||
|
- 0.4-0.6: 部分标注有误或存在明显遗漏
|
||||||
|
- 0.0-0.3: 标注与原始摘要严重不匹配
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
你是一个视频节点内容分析器。你服务于一个视频问答推理系统,该系统通过工具调用你来获取节点内容的结构化摘要。推理和最终判断由该系统完成,你只负责忠实提取信息。
|
||||||
|
|
||||||
|
## 你会收到的输入
|
||||||
|
|
||||||
|
1. 用户正在研究的问题
|
||||||
|
2. 一个视频节点的描述文本(包含场景摘要、实体、动作等结构化字段)和字幕转写
|
||||||
|
|
||||||
|
## 工作原则
|
||||||
|
|
||||||
|
仅基于提供的内容回答,不使用外部知识。你看不到其他节点的内容,因此不要推断跨节点的事件顺序、因果关系或全局结论。
|
||||||
|
|
||||||
|
报告内容中与问题相关的一切事实:人物、动作、对话引用、数字、时间、因果关系。字幕中的引用(解说、对话)是重要证据来源,请保留关键原文片段。
|
||||||
|
|
||||||
|
如果内容与问题无关,明确说明"该节点未包含与问题直接相关的信息"。如果内容包含间接相关的信息(如背景知识),标注为"间接相关"并简要说明。
|
||||||
|
|
||||||
|
此外,用一句话概括该节点中其他显著但与问题不直接相关的信息,供推理系统参考。
|
||||||
|
|
||||||
|
## 输出格式
|
||||||
|
|
||||||
|
[相关信息] <与问题相关的事实,按重要性排列>
|
||||||
|
[间接相关] <背景知识或可能有用的上下文,没有则省略>
|
||||||
|
[其他信息] <一句话概括该节点中其他显著内容>
|
||||||
|
[原文] <1-3 句与问题最相关的字幕原文或描述原文,保留原始措辞,不改写不概括。无关则省略>
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
你是一个视频节点摘要核实器。你将收到一段关于视频节点的摘要(由另一个模型生成),以及该节点的原始描述和字幕。请逐条核实摘要是否准确反映了原始内容。
|
||||||
|
|
||||||
|
## 你会收到的输入
|
||||||
|
|
||||||
|
1. 用户正在研究的问题
|
||||||
|
2. 节点的原始描述文本和字幕
|
||||||
|
3. 另一个模型基于上述内容生成的摘要
|
||||||
|
|
||||||
|
## 检查要点
|
||||||
|
|
||||||
|
- 摘要提到的事实是否确实存在于原始内容中?
|
||||||
|
- 摘要是否包含了原始内容中不存在的推断?
|
||||||
|
- 摘要是否遗漏了原始内容中与问题高度相关的重要信息?
|
||||||
|
- [原文] 引用是否准确保留了原始措辞?
|
||||||
|
|
||||||
|
## 输出格式
|
||||||
|
|
||||||
|
details=<逐条核实结果>; confidence=<0.0-1.0>
|
||||||
|
|
||||||
|
置信度含义:
|
||||||
|
- 1.0: 摘要完全准确,无遗漏
|
||||||
|
- 0.7-0.9: 主要内容准确,个别细节有出入
|
||||||
|
- 0.4-0.6: 部分准确,但存在明显错误或过度推断
|
||||||
|
- 0.0-0.3: 摘要与原始内容严重不符
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
"""建树模块端到端集成测试。
|
||||||
|
|
||||||
|
验证各模块协作:
|
||||||
|
构造最小树 → verify → subtitle 注入 → TreeEnvironment 查询 → 序列化 roundtrip
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.tree.index import (
|
||||||
|
IndexMeta, TreeIndex, L1Node, L1Card,
|
||||||
|
L2Node, L2Card, L3Node, L3Card,
|
||||||
|
)
|
||||||
|
from app.tree.verify import verify_tree
|
||||||
|
from app.tree.subtitle import SRTEntry, assign_subtitles_voronoi
|
||||||
|
from app.tree.environment import TreeEnvironment
|
||||||
|
|
||||||
|
|
||||||
|
class TestTreeModuleE2E:
|
||||||
|
def test_verify_subtitle_environment_pipeline(self, tmp_path):
|
||||||
|
"""完整流程:构造树 → verify(删除幻觉实体)→ subtitle 注入 → environment 查询 → 序列化 roundtrip。"""
|
||||||
|
# 构造一棵树,L2 有混合实体(有出处/无出处)
|
||||||
|
l3_0 = L3Node(
|
||||||
|
id="vid_L1_000_L2_000_L3_000",
|
||||||
|
card=L3Card(
|
||||||
|
frame_summary="运动员在跑步冲刺",
|
||||||
|
visible_entities=["运动员", "跑道"],
|
||||||
|
ongoing_actions=["跑步"],
|
||||||
|
visible_text=["Nike", "2024"],
|
||||||
|
spatial_layout="居中构图",
|
||||||
|
visual_attributes={"lighting": "明亮", "camera_angle": "侧面"},
|
||||||
|
),
|
||||||
|
timestamp=2.0,
|
||||||
|
frame_path="frames/L1_000_L2_000_L3_000.jpg",
|
||||||
|
)
|
||||||
|
l3_1 = L3Node(
|
||||||
|
id="vid_L1_000_L2_000_L3_001",
|
||||||
|
card=L3Card(
|
||||||
|
frame_summary="观众在看台上欢呼",
|
||||||
|
visible_entities=["观众", "看台"],
|
||||||
|
ongoing_actions=["欢呼"],
|
||||||
|
visible_text=["Stadium"],
|
||||||
|
spatial_layout="广角",
|
||||||
|
visual_attributes={},
|
||||||
|
),
|
||||||
|
timestamp=6.0,
|
||||||
|
frame_path="frames/L1_000_L2_000_L3_001.jpg",
|
||||||
|
)
|
||||||
|
l2 = L2Node(
|
||||||
|
id="vid_L1_000_L2_000",
|
||||||
|
card=L2Card(
|
||||||
|
event_description="百米决赛片段",
|
||||||
|
entities=["运动员", "裁判", "幻觉实体XYZ"],
|
||||||
|
actions=["跑步", "欢呼"],
|
||||||
|
action_subjects=["运动员", "观众"],
|
||||||
|
visible_text=["Nike", "不存在的文字ABC"],
|
||||||
|
spatial_relations="运动员在跑道中央",
|
||||||
|
state_changes=None,
|
||||||
|
),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l3_0, l3_1],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="vid_L1_000",
|
||||||
|
card=L1Card(
|
||||||
|
scene_summary="百米短跑决赛",
|
||||||
|
main_setting="体育场",
|
||||||
|
key_entities=["运动员", "不存在的人物"],
|
||||||
|
main_actions=["比赛"],
|
||||||
|
topic_keywords=["体育", "短跑"],
|
||||||
|
visible_text=["Nike", "Ghost文字"],
|
||||||
|
temporal_flow="从起跑到冲刺",
|
||||||
|
),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
index = TreeIndex(
|
||||||
|
metadata=IndexMeta(source_path="/test/video.mp4", modality="video"),
|
||||||
|
roots=[l1],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 1: verify — 删除无出处的实体和 visible_text
|
||||||
|
stats = verify_tree(index)
|
||||||
|
assert "幻觉实体XYZ" not in index.roots[0].children[0].card.entities
|
||||||
|
assert "不存在的文字ABC" not in index.roots[0].children[0].card.visible_text
|
||||||
|
assert "Ghost文字" not in index.roots[0].card.visible_text
|
||||||
|
assert "不存在的人物" not in index.roots[0].card.key_entities
|
||||||
|
# 有出处的保留
|
||||||
|
assert "运动员" in index.roots[0].children[0].card.entities
|
||||||
|
assert "Nike" in index.roots[0].children[0].card.visible_text
|
||||||
|
|
||||||
|
# Step 2: subtitle 注入
|
||||||
|
srt_entries = [
|
||||||
|
SRTEntry(start=1.0, end=3.0, text="And the runner sprints ahead!"),
|
||||||
|
SRTEntry(start=5.0, end=7.0, text="The crowd goes wild!"),
|
||||||
|
]
|
||||||
|
assign_subtitles_voronoi(index, srt_entries)
|
||||||
|
assert l3_0.subtitle is not None
|
||||||
|
assert "sprints" in l3_0.subtitle
|
||||||
|
assert l3_1.subtitle is not None
|
||||||
|
assert "crowd" in l3_1.subtitle
|
||||||
|
|
||||||
|
# Step 3: TreeEnvironment 查询
|
||||||
|
env = TreeEnvironment(index)
|
||||||
|
|
||||||
|
# view_node L3
|
||||||
|
l3_view = env.view_node("vid_L1_000_L2_000_L3_000")
|
||||||
|
assert "运动员在跑步冲刺" in l3_view
|
||||||
|
|
||||||
|
# view_node L2 (should list children)
|
||||||
|
l2_view = env.view_node("vid_L1_000_L2_000")
|
||||||
|
assert "百米决赛片段" in l2_view
|
||||||
|
assert "vid_L1_000_L2_000_L3_000" in l2_view
|
||||||
|
|
||||||
|
# view_node with anchor
|
||||||
|
anchored = env.view_node("vid_L1_000_L2_000_L3_000", anchor=True)
|
||||||
|
assert "[c" in anchored
|
||||||
|
|
||||||
|
# get_subtitle
|
||||||
|
assert "sprints" in env.get_subtitle("vid_L1_000_L2_000_L3_000")
|
||||||
|
|
||||||
|
# search_similar (with embedding)
|
||||||
|
def fake_embed(texts):
|
||||||
|
if isinstance(texts, str):
|
||||||
|
texts = [texts]
|
||||||
|
rng = np.random.RandomState(42)
|
||||||
|
return rng.randn(len(texts), 4).astype(np.float32)
|
||||||
|
|
||||||
|
index.embed_all(fake_embed, "test-model", 4)
|
||||||
|
results = env.search_similar("运动员跑步", top_k=3, embed_fn=fake_embed)
|
||||||
|
assert len(results) > 0
|
||||||
|
|
||||||
|
# Step 4: 序列化 roundtrip
|
||||||
|
path = tmp_path / "tree.json"
|
||||||
|
index.save_json(str(path))
|
||||||
|
loaded = TreeIndex.load_json(str(path))
|
||||||
|
|
||||||
|
assert len(loaded.roots) == 1
|
||||||
|
assert loaded.roots[0].card.scene_summary == "百米短跑决赛"
|
||||||
|
assert loaded.roots[0].children[0].children[0].subtitle is not None
|
||||||
|
assert "sprints" in loaded.roots[0].children[0].children[0].subtitle
|
||||||
|
# verify 的修改也被保留
|
||||||
|
assert "幻觉实体XYZ" not in loaded.roots[0].children[0].card.entities
|
||||||
|
|
||||||
|
def test_repair_detector_on_broken_tree(self):
|
||||||
|
"""修复检测器能识别空卡片节点。"""
|
||||||
|
from app.tree.repair.detector import detect_issues
|
||||||
|
|
||||||
|
l3 = L3Node(
|
||||||
|
id="vid_L1_000_L2_000_L3_000",
|
||||||
|
card=L3Card("", [], [], [], "", {}), # empty frame_summary
|
||||||
|
timestamp=1.0,
|
||||||
|
)
|
||||||
|
l2 = L2Node(
|
||||||
|
id="vid_L1_000_L2_000",
|
||||||
|
card=L2Card("事件", [], [], [], [], "", None),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l3],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="vid_L1_000",
|
||||||
|
card=L1Card("场景", "", [], [], [], [], ""),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
|
||||||
|
issues = detect_issues(index)
|
||||||
|
assert len(issues) >= 1
|
||||||
|
assert any(i.issue_type == "empty_field" for i in issues)
|
||||||
@@ -1,9 +1,10 @@
|
|||||||
"""core/types.py 单元测试。"""
|
"""core/types.py 单元测试。"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from core.types import LLMResponse
|
from core.types import GeneratedQuestion, LLMResponse
|
||||||
|
|
||||||
|
|
||||||
class TestLLMResponse:
|
class TestLLMResponse:
|
||||||
@@ -42,10 +43,58 @@ class TestLLMResponse:
|
|||||||
|
|
||||||
def test_cache_hit_response_has_none_ttft(self) -> None:
|
def test_cache_hit_response_has_none_ttft(self) -> None:
|
||||||
resp = LLMResponse(
|
resp = LLMResponse(
|
||||||
content="cached", thinking="", model="m", provider="p",
|
content="cached",
|
||||||
prompt_tokens=0, completion_tokens=0, latency_ms=1,
|
thinking="",
|
||||||
ttft_ms=None, max_inter_token_ms=None, cache_hit=True, call_id="c",
|
model="m",
|
||||||
|
provider="p",
|
||||||
|
prompt_tokens=0,
|
||||||
|
completion_tokens=0,
|
||||||
|
latency_ms=1,
|
||||||
|
ttft_ms=None,
|
||||||
|
max_inter_token_ms=None,
|
||||||
|
cache_hit=True,
|
||||||
|
call_id="c",
|
||||||
)
|
)
|
||||||
assert resp.ttft_ms is None
|
assert resp.ttft_ms is None
|
||||||
assert resp.max_inter_token_ms is None
|
assert resp.max_inter_token_ms is None
|
||||||
assert resp.cache_hit is True
|
assert resp.cache_hit is True
|
||||||
|
|
||||||
|
|
||||||
|
class TestGeneratedQuestion:
|
||||||
|
@pytest.fixture()
|
||||||
|
def sample_question(self) -> GeneratedQuestion:
|
||||||
|
return GeneratedQuestion(
|
||||||
|
question_id="719-1",
|
||||||
|
video_id="B7Hh0PY1kks",
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
question="What are the differing motivations?",
|
||||||
|
options=("A. Option 1", "B. Option 2", "C. Option 3", "D. Option 4"),
|
||||||
|
answer="B",
|
||||||
|
source_nodes=(),
|
||||||
|
difficulty="medium",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_frozen_prevents_mutation(self, sample_question: GeneratedQuestion) -> None:
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
sample_question.question = "篡改"
|
||||||
|
|
||||||
|
def test_all_fields_accessible(self, sample_question: GeneratedQuestion) -> None:
|
||||||
|
assert sample_question.question_id == "719-1"
|
||||||
|
assert sample_question.video_id == "B7Hh0PY1kks"
|
||||||
|
assert sample_question.task_type == "Action Reasoning"
|
||||||
|
assert sample_question.question == "What are the differing motivations?"
|
||||||
|
assert sample_question.options == (
|
||||||
|
"A. Option 1",
|
||||||
|
"B. Option 2",
|
||||||
|
"C. Option 3",
|
||||||
|
"D. Option 4",
|
||||||
|
)
|
||||||
|
assert sample_question.answer == "B"
|
||||||
|
assert sample_question.source_nodes == ()
|
||||||
|
assert sample_question.difficulty == "medium"
|
||||||
|
|
||||||
|
def test_options_is_tuple(self, sample_question: GeneratedQuestion) -> None:
|
||||||
|
assert isinstance(sample_question.options, tuple)
|
||||||
|
|
||||||
|
def test_source_nodes_is_tuple(self, sample_question: GeneratedQuestion) -> None:
|
||||||
|
assert isinstance(sample_question.source_nodes, tuple)
|
||||||
|
|||||||
@@ -0,0 +1,774 @@
|
|||||||
|
"""core/evolution/diagnose.py 单元测试。
|
||||||
|
|
||||||
|
覆盖:
|
||||||
|
- 7 个规则指标(空输入、边界、典型值)
|
||||||
|
- extract_json_from_response(三策略 + 拒绝非 dict + 垃圾输入)
|
||||||
|
- attribute_error 瀑布(4 条路径)
|
||||||
|
- question_soft_score(空→None)
|
||||||
|
- aggregate_soft(跳过 None、全 None→None)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from typing import Any
|
||||||
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.evolution.diagnose import (
|
||||||
|
_percentile,
|
||||||
|
_trigrams,
|
||||||
|
aggregate_d2,
|
||||||
|
aggregate_d3,
|
||||||
|
aggregate_d4,
|
||||||
|
aggregate_d5,
|
||||||
|
aggregate_soft,
|
||||||
|
attribute_error,
|
||||||
|
calc_budget_usage,
|
||||||
|
calc_confidence_calibration,
|
||||||
|
calc_format_compliance,
|
||||||
|
calc_level_jump_pattern,
|
||||||
|
calc_repeat_visit_rate,
|
||||||
|
calc_search_keyword_repetition,
|
||||||
|
calc_tool_usage,
|
||||||
|
extract_json_from_response,
|
||||||
|
extract_rule_metrics,
|
||||||
|
merge_system_packs,
|
||||||
|
merge_tool_packs,
|
||||||
|
question_soft_score,
|
||||||
|
run_diagnosis,
|
||||||
|
)
|
||||||
|
from core.evolution.types import (
|
||||||
|
CaseSample,
|
||||||
|
DiagnosePrompts,
|
||||||
|
DiagnosisResult,
|
||||||
|
QuestionMetrics,
|
||||||
|
SkillStepAdherence,
|
||||||
|
SpanMetrics,
|
||||||
|
SystemCasePack,
|
||||||
|
ToolCasePack,
|
||||||
|
)
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# 工厂函数
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
def _make_qm(
|
||||||
|
question_id: str = "q1",
|
||||||
|
video_id: str = "v1",
|
||||||
|
task_type: str = "Action Reasoning",
|
||||||
|
correct: bool = False,
|
||||||
|
format_compliance: float = 1.0,
|
||||||
|
budget_usage: float = 0.5,
|
||||||
|
confidence_calibration: str = "calibrated",
|
||||||
|
repeat_visit_rate: float = 0.0,
|
||||||
|
search_keyword_repetition: float = 0.0,
|
||||||
|
level_jump_pattern: str = "",
|
||||||
|
tool_usage: dict[str, int] | None = None,
|
||||||
|
span_metrics: list[SpanMetrics] | None = None,
|
||||||
|
missed_nodes: list[str] | None = None,
|
||||||
|
skill_adherence: list[Any] | None = None,
|
||||||
|
confirmation_bias: bool | None = None,
|
||||||
|
evidence_sufficient: bool | None = None,
|
||||||
|
degraded: bool = False,
|
||||||
|
) -> QuestionMetrics:
|
||||||
|
"""构造 QuestionMetrics,非关键字段使用合理默认值。"""
|
||||||
|
return QuestionMetrics(
|
||||||
|
question_id=question_id,
|
||||||
|
video_id=video_id,
|
||||||
|
task_type=task_type,
|
||||||
|
correct=correct,
|
||||||
|
format_compliance=format_compliance,
|
||||||
|
budget_usage=budget_usage,
|
||||||
|
confidence_calibration=confidence_calibration,
|
||||||
|
repeat_visit_rate=repeat_visit_rate,
|
||||||
|
search_keyword_repetition=search_keyword_repetition,
|
||||||
|
level_jump_pattern=level_jump_pattern,
|
||||||
|
tool_usage=tool_usage or {},
|
||||||
|
span_metrics=span_metrics or [],
|
||||||
|
missed_nodes=missed_nodes or [],
|
||||||
|
skill_adherence=skill_adherence or [],
|
||||||
|
confirmation_bias=confirmation_bias,
|
||||||
|
evidence_sufficient=evidence_sufficient,
|
||||||
|
degraded=degraded,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_span(
|
||||||
|
step: int = 0,
|
||||||
|
tool_name: str = "view_node",
|
||||||
|
extraction_completeness: float = 0.8,
|
||||||
|
hallucination_rate: float = 0.1,
|
||||||
|
) -> SpanMetrics:
|
||||||
|
"""构造 SpanMetrics 快捷工厂。"""
|
||||||
|
return SpanMetrics(
|
||||||
|
step=step,
|
||||||
|
tool_name=tool_name,
|
||||||
|
extraction_completeness=extraction_completeness,
|
||||||
|
hallucination_rate=hallucination_rate,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# A. 规则指标测试
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestCalcFormatCompliance:
|
||||||
|
"""calc_format_compliance 测试。"""
|
||||||
|
|
||||||
|
def test_empty_returns_one(self) -> None:
|
||||||
|
"""空列表返回 1.0。"""
|
||||||
|
assert calc_format_compliance([]) == 1.0
|
||||||
|
|
||||||
|
def test_all_compliant(self) -> None:
|
||||||
|
"""全部合规返回 1.0。"""
|
||||||
|
raw = json.dumps({"reflect": {}, "plan": {}, "action": {}})
|
||||||
|
assert calc_format_compliance([raw, raw]) == 1.0
|
||||||
|
|
||||||
|
def test_none_compliant(self) -> None:
|
||||||
|
"""全部不合规返回 0.0。"""
|
||||||
|
assert calc_format_compliance(["not json", '{"foo": 1}']) == 0.0
|
||||||
|
|
||||||
|
def test_partial_compliance(self) -> None:
|
||||||
|
"""部分合规返回正确比例。"""
|
||||||
|
good = json.dumps({"reflect": {}, "plan": {}, "action": {}})
|
||||||
|
bad = json.dumps({"reflect": {}, "plan": {}})
|
||||||
|
assert calc_format_compliance([good, bad]) == 0.5
|
||||||
|
|
||||||
|
|
||||||
|
class TestCalcBudgetUsage:
|
||||||
|
"""calc_budget_usage 测试。"""
|
||||||
|
|
||||||
|
def test_typical(self) -> None:
|
||||||
|
"""典型值。"""
|
||||||
|
assert calc_budget_usage(5, 10) == 0.5
|
||||||
|
|
||||||
|
def test_full_budget(self) -> None:
|
||||||
|
"""用满预算。"""
|
||||||
|
assert calc_budget_usage(10, 10) == 1.0
|
||||||
|
|
||||||
|
def test_zero_steps(self) -> None:
|
||||||
|
"""未使用步数。"""
|
||||||
|
assert calc_budget_usage(0, 10) == 0.0
|
||||||
|
|
||||||
|
def test_zero_max_steps_raises(self) -> None:
|
||||||
|
"""max_steps=0 应抛出 ZeroDivisionError(P5: 不掩盖错误)。"""
|
||||||
|
with pytest.raises(ZeroDivisionError):
|
||||||
|
calc_budget_usage(5, 0)
|
||||||
|
|
||||||
|
|
||||||
|
class TestCalcConfidenceCalibration:
|
||||||
|
"""calc_confidence_calibration 测试。"""
|
||||||
|
|
||||||
|
def test_high_conf_wrong(self) -> None:
|
||||||
|
"""高置信度答错。"""
|
||||||
|
assert calc_confidence_calibration(0.7, correct=False) == "high_conf_wrong"
|
||||||
|
assert calc_confidence_calibration(0.9, correct=False) == "high_conf_wrong"
|
||||||
|
|
||||||
|
def test_low_conf_right(self) -> None:
|
||||||
|
"""低置信度答对。"""
|
||||||
|
assert calc_confidence_calibration(0.3, correct=True) == "low_conf_right"
|
||||||
|
assert calc_confidence_calibration(0.49, correct=True) == "low_conf_right"
|
||||||
|
|
||||||
|
def test_calibrated(self) -> None:
|
||||||
|
"""正常校准。"""
|
||||||
|
assert calc_confidence_calibration(0.5, correct=True) == "calibrated"
|
||||||
|
assert calc_confidence_calibration(0.7, correct=True) == "calibrated"
|
||||||
|
assert calc_confidence_calibration(0.3, correct=False) == "calibrated"
|
||||||
|
|
||||||
|
def test_boundary_high(self) -> None:
|
||||||
|
"""边界值: 0.7 答错。"""
|
||||||
|
assert calc_confidence_calibration(0.7, correct=False) == "high_conf_wrong"
|
||||||
|
|
||||||
|
def test_boundary_low(self) -> None:
|
||||||
|
"""边界值: 0.5 答对不算 low_conf_right。"""
|
||||||
|
assert calc_confidence_calibration(0.5, correct=True) == "calibrated"
|
||||||
|
|
||||||
|
|
||||||
|
class TestCalcRepeatVisitRate:
|
||||||
|
"""calc_repeat_visit_rate 测试。"""
|
||||||
|
|
||||||
|
def test_empty(self) -> None:
|
||||||
|
"""空列表返回 0.0。"""
|
||||||
|
assert calc_repeat_visit_rate([]) == 0.0
|
||||||
|
|
||||||
|
def test_no_repeats(self) -> None:
|
||||||
|
"""无重复。"""
|
||||||
|
assert calc_repeat_visit_rate(["a", "b", "c"]) == 0.0
|
||||||
|
|
||||||
|
def test_all_same(self) -> None:
|
||||||
|
"""全重复。"""
|
||||||
|
rate = calc_repeat_visit_rate(["a", "a", "a"])
|
||||||
|
assert abs(rate - (1 - 1 / 3)) < 1e-9
|
||||||
|
|
||||||
|
def test_partial_repeats(self) -> None:
|
||||||
|
"""部分重复。"""
|
||||||
|
rate = calc_repeat_visit_rate(["a", "b", "a"])
|
||||||
|
assert abs(rate - (1 - 2 / 3)) < 1e-9
|
||||||
|
|
||||||
|
|
||||||
|
class TestTrigrams:
|
||||||
|
"""_trigrams 辅助函数测试。"""
|
||||||
|
|
||||||
|
def test_short_string(self) -> None:
|
||||||
|
"""不足 3 字符返回空集。"""
|
||||||
|
assert _trigrams("ab") == set()
|
||||||
|
assert _trigrams("") == set()
|
||||||
|
|
||||||
|
def test_exact_three(self) -> None:
|
||||||
|
"""恰好 3 字符。"""
|
||||||
|
assert _trigrams("abc") == {"abc"}
|
||||||
|
|
||||||
|
def test_longer(self) -> None:
|
||||||
|
"""多字符。"""
|
||||||
|
assert _trigrams("abcd") == {"abc", "bcd"}
|
||||||
|
|
||||||
|
|
||||||
|
class TestCalcSearchKeywordRepetition:
|
||||||
|
"""calc_search_keyword_repetition 测试。"""
|
||||||
|
|
||||||
|
def test_single_query(self) -> None:
|
||||||
|
"""单条查询返回 0.0。"""
|
||||||
|
assert calc_search_keyword_repetition(["hello"]) == 0.0
|
||||||
|
|
||||||
|
def test_empty(self) -> None:
|
||||||
|
"""空列表返回 0.0。"""
|
||||||
|
assert calc_search_keyword_repetition([]) == 0.0
|
||||||
|
|
||||||
|
def test_identical_queries(self) -> None:
|
||||||
|
"""完全相同的连续查询,Jaccard=1.0。"""
|
||||||
|
assert calc_search_keyword_repetition(["hello world", "hello world"]) == 1.0
|
||||||
|
|
||||||
|
def test_disjoint_queries(self) -> None:
|
||||||
|
"""完全不同的查询,Jaccard 接近 0。"""
|
||||||
|
score = calc_search_keyword_repetition(["aaa", "zzz"])
|
||||||
|
assert score == 0.0
|
||||||
|
|
||||||
|
def test_takes_max_across_pairs(self) -> None:
|
||||||
|
"""取连续对的最大值。"""
|
||||||
|
score = calc_search_keyword_repetition(["aaa", "zzz", "zzz"])
|
||||||
|
assert score == 1.0 # 第二对完全相同
|
||||||
|
|
||||||
|
|
||||||
|
class TestCalcLevelJumpPattern:
|
||||||
|
"""calc_level_jump_pattern 测试。"""
|
||||||
|
|
||||||
|
def test_empty(self) -> None:
|
||||||
|
"""空列表返回空字符串。"""
|
||||||
|
assert calc_level_jump_pattern([]) == ""
|
||||||
|
|
||||||
|
def test_typical(self) -> None:
|
||||||
|
"""典型节点 ID 序列。"""
|
||||||
|
result = calc_level_jump_pattern(["vid_L1_001", "vid_L2_003", "vid_L3_005"])
|
||||||
|
assert result == "L1→L2→L3"
|
||||||
|
|
||||||
|
def test_no_match(self) -> None:
|
||||||
|
"""无匹配的节点 ID 被跳过。"""
|
||||||
|
assert calc_level_jump_pattern(["no_level_here"]) == ""
|
||||||
|
|
||||||
|
def test_mixed(self) -> None:
|
||||||
|
"""混合匹配和非匹配。"""
|
||||||
|
result = calc_level_jump_pattern(["vid_L2_001", "bad_id", "vid_L1_003"])
|
||||||
|
assert result == "L2→L1"
|
||||||
|
|
||||||
|
|
||||||
|
class TestCalcToolUsage:
|
||||||
|
"""calc_tool_usage 测试。"""
|
||||||
|
|
||||||
|
def test_empty(self) -> None:
|
||||||
|
"""空列表返回空字典。"""
|
||||||
|
assert calc_tool_usage([]) == {}
|
||||||
|
|
||||||
|
def test_counts(self) -> None:
|
||||||
|
"""正确计数。"""
|
||||||
|
result = calc_tool_usage(["view_node", "search_similar", "view_node"])
|
||||||
|
assert result == {"view_node": 2, "search_similar": 1}
|
||||||
|
|
||||||
|
|
||||||
|
class TestExtractRuleMetrics:
|
||||||
|
"""extract_rule_metrics 测试。"""
|
||||||
|
|
||||||
|
def test_basic_extraction(self) -> None:
|
||||||
|
"""基本规则指标提取。"""
|
||||||
|
prediction = {
|
||||||
|
"steps_json": [
|
||||||
|
{
|
||||||
|
"tool_call": {
|
||||||
|
"tool": "view_node",
|
||||||
|
"args": {"node_id": "vid_L1_001"},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"tool_call": {
|
||||||
|
"tool": "search_similar",
|
||||||
|
"args": {"query": "test query"},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"correct": True,
|
||||||
|
}
|
||||||
|
result = extract_rule_metrics(prediction, [], max_steps=10)
|
||||||
|
assert result["budget_usage"] == 0.2
|
||||||
|
assert result["tool_usage"] == {"view_node": 1, "search_similar": 1}
|
||||||
|
assert "L1" in result["level_jump_pattern"]
|
||||||
|
|
||||||
|
def test_empty_prediction(self) -> None:
|
||||||
|
"""空预测。"""
|
||||||
|
result = extract_rule_metrics({}, [], max_steps=10)
|
||||||
|
assert result["format_compliance"] == 1.0
|
||||||
|
assert result["budget_usage"] == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# B. JSON 提取测试
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestExtractJsonFromResponse:
|
||||||
|
"""extract_json_from_response 测试。"""
|
||||||
|
|
||||||
|
def test_fenced_block(self) -> None:
|
||||||
|
"""从 markdown 代码块提取。"""
|
||||||
|
raw = '```json\n{"key": "value"}\n```'
|
||||||
|
assert extract_json_from_response(raw) == {"key": "value"}
|
||||||
|
|
||||||
|
def test_fenced_block_no_json_tag(self) -> None:
|
||||||
|
"""从无 json 标签的代码块提取。"""
|
||||||
|
raw = '```\n{"key": "value"}\n```'
|
||||||
|
assert extract_json_from_response(raw) == {"key": "value"}
|
||||||
|
|
||||||
|
def test_outermost_braces(self) -> None:
|
||||||
|
"""从最外层花括号提取。"""
|
||||||
|
raw = 'Some text before {"result": 42} and after'
|
||||||
|
assert extract_json_from_response(raw) == {"result": 42}
|
||||||
|
|
||||||
|
def test_non_dict_raises(self) -> None:
|
||||||
|
"""非 dict 类型抛出 ValueError。"""
|
||||||
|
raw = "[1, 2, 3]"
|
||||||
|
with pytest.raises(ValueError, match="无法从 LLM 回复中提取 JSON"):
|
||||||
|
extract_json_from_response(raw)
|
||||||
|
|
||||||
|
def test_garbage_raises(self) -> None:
|
||||||
|
"""完全无法解析的输入抛出 ValueError。"""
|
||||||
|
with pytest.raises(ValueError, match="无法从 LLM 回复中提取 JSON"):
|
||||||
|
extract_json_from_response("this is not json at all !!!")
|
||||||
|
|
||||||
|
def test_nested_braces(self) -> None:
|
||||||
|
"""嵌套花括号正确处理。"""
|
||||||
|
inner = {"nested": {"deep": True}}
|
||||||
|
raw = f"Result: {json.dumps(inner)}"
|
||||||
|
assert extract_json_from_response(raw) == inner
|
||||||
|
|
||||||
|
def test_fenced_block_non_dict_falls_through(self) -> None:
|
||||||
|
"""代码块中是列表时,回退到后续策略。"""
|
||||||
|
raw = '```json\n[1,2,3]\n``` {"fallback": true}'
|
||||||
|
result = extract_json_from_response(raw)
|
||||||
|
assert result == {"fallback": True}
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# C. question_soft_score / aggregate_soft 测试
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestQuestionSoftScore:
|
||||||
|
"""question_soft_score 测试。"""
|
||||||
|
|
||||||
|
def test_empty_returns_none(self) -> None:
|
||||||
|
"""空 span 列表返回 None。"""
|
||||||
|
assert question_soft_score([]) is None
|
||||||
|
|
||||||
|
def test_single_span(self) -> None:
|
||||||
|
"""单个 span 的计算。"""
|
||||||
|
span = _make_span(extraction_completeness=0.8, hallucination_rate=0.2)
|
||||||
|
score = question_soft_score([span])
|
||||||
|
# (0.8 + (1.0 - 0.2)) / 2 = (0.8 + 0.8) / 2 = 0.8
|
||||||
|
assert score is not None
|
||||||
|
assert abs(score - 0.8) < 1e-9
|
||||||
|
|
||||||
|
def test_multiple_spans(self) -> None:
|
||||||
|
"""多个 span 取均值。"""
|
||||||
|
span1 = _make_span(extraction_completeness=1.0, hallucination_rate=0.0)
|
||||||
|
span2 = _make_span(extraction_completeness=0.6, hallucination_rate=0.4)
|
||||||
|
score = question_soft_score([span1, span2])
|
||||||
|
# span1: (1.0 + 1.0) / 2 = 1.0
|
||||||
|
# span2: (0.6 + 0.6) / 2 = 0.6
|
||||||
|
# mean: (1.0 + 0.6) / 2 = 0.8
|
||||||
|
assert score is not None
|
||||||
|
assert abs(score - 0.8) < 1e-9
|
||||||
|
|
||||||
|
def test_perfect_span(self) -> None:
|
||||||
|
"""完美 span。"""
|
||||||
|
span = _make_span(extraction_completeness=1.0, hallucination_rate=0.0)
|
||||||
|
score = question_soft_score([span])
|
||||||
|
assert score == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
class TestAggregateSoft:
|
||||||
|
"""aggregate_soft 测试。"""
|
||||||
|
|
||||||
|
def test_all_none_returns_none(self) -> None:
|
||||||
|
"""全部 None 返回 None。"""
|
||||||
|
assert aggregate_soft([None, None, None]) is None
|
||||||
|
|
||||||
|
def test_empty_returns_none(self) -> None:
|
||||||
|
"""空列表返回 None。"""
|
||||||
|
assert aggregate_soft([]) is None
|
||||||
|
|
||||||
|
def test_skip_none(self) -> None:
|
||||||
|
"""跳过 None 计算均值。"""
|
||||||
|
result = aggregate_soft([0.8, None, 0.6])
|
||||||
|
assert result is not None
|
||||||
|
assert abs(result - 0.7) < 1e-9
|
||||||
|
|
||||||
|
def test_all_valid(self) -> None:
|
||||||
|
"""全部有效。"""
|
||||||
|
result = aggregate_soft([0.5, 0.7, 0.9])
|
||||||
|
assert result is not None
|
||||||
|
assert abs(result - 0.7) < 1e-9
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# D. attribute_error 瀑布测试
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestAttributeError:
|
||||||
|
"""attribute_error 瀑布规则测试。"""
|
||||||
|
|
||||||
|
def test_extraction_failure_low_completeness(self) -> None:
|
||||||
|
"""avg completeness < 0.5 → extraction_failure。"""
|
||||||
|
qm = _make_qm(
|
||||||
|
span_metrics=[
|
||||||
|
_make_span(extraction_completeness=0.3, hallucination_rate=0.1),
|
||||||
|
_make_span(extraction_completeness=0.4, hallucination_rate=0.1),
|
||||||
|
],
|
||||||
|
missed_nodes=["node_1"], # 有遗漏,但 extraction 优先
|
||||||
|
)
|
||||||
|
result = attribute_error(qm)
|
||||||
|
assert result.error_type == "extraction_failure"
|
||||||
|
assert result.question_id == "q1"
|
||||||
|
|
||||||
|
def test_extraction_failure_high_hallucination(self) -> None:
|
||||||
|
"""max hallucination > 0.5 → extraction_failure。"""
|
||||||
|
qm = _make_qm(
|
||||||
|
span_metrics=[
|
||||||
|
_make_span(extraction_completeness=0.9, hallucination_rate=0.6),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
result = attribute_error(qm)
|
||||||
|
assert result.error_type == "extraction_failure"
|
||||||
|
|
||||||
|
def test_search_failure(self) -> None:
|
||||||
|
"""有遗漏节点 → search_failure。"""
|
||||||
|
qm = _make_qm(
|
||||||
|
span_metrics=[
|
||||||
|
_make_span(extraction_completeness=0.8, hallucination_rate=0.1),
|
||||||
|
],
|
||||||
|
missed_nodes=["node_1", "node_2"],
|
||||||
|
)
|
||||||
|
result = attribute_error(qm)
|
||||||
|
assert result.error_type == "search_failure"
|
||||||
|
|
||||||
|
def test_reasoning_failure(self) -> None:
|
||||||
|
"""evidence_sufficient=True → reasoning_failure。"""
|
||||||
|
qm = _make_qm(
|
||||||
|
span_metrics=[
|
||||||
|
_make_span(extraction_completeness=0.8, hallucination_rate=0.1),
|
||||||
|
],
|
||||||
|
missed_nodes=[],
|
||||||
|
evidence_sufficient=True,
|
||||||
|
)
|
||||||
|
result = attribute_error(qm)
|
||||||
|
assert result.error_type == "reasoning_failure"
|
||||||
|
|
||||||
|
def test_mixed_evidence_none(self) -> None:
|
||||||
|
"""evidence_sufficient=None → mixed。"""
|
||||||
|
qm = _make_qm(
|
||||||
|
span_metrics=[
|
||||||
|
_make_span(extraction_completeness=0.8, hallucination_rate=0.1),
|
||||||
|
],
|
||||||
|
missed_nodes=[],
|
||||||
|
evidence_sufficient=None,
|
||||||
|
)
|
||||||
|
result = attribute_error(qm)
|
||||||
|
assert result.error_type == "mixed"
|
||||||
|
|
||||||
|
def test_mixed_evidence_false(self) -> None:
|
||||||
|
"""evidence_sufficient=False → mixed。"""
|
||||||
|
qm = _make_qm(
|
||||||
|
span_metrics=[
|
||||||
|
_make_span(extraction_completeness=0.8, hallucination_rate=0.1),
|
||||||
|
],
|
||||||
|
missed_nodes=[],
|
||||||
|
evidence_sufficient=False,
|
||||||
|
)
|
||||||
|
result = attribute_error(qm)
|
||||||
|
assert result.error_type == "mixed"
|
||||||
|
|
||||||
|
def test_no_spans_extraction(self) -> None:
|
||||||
|
"""无 span 时 avg_completeness=0 (< 0.5) → extraction_failure。"""
|
||||||
|
qm = _make_qm(span_metrics=[], missed_nodes=["node_x"])
|
||||||
|
result = attribute_error(qm)
|
||||||
|
# _mean([]) = 0.0 < 0.5 → extraction_failure
|
||||||
|
assert result.error_type == "extraction_failure"
|
||||||
|
|
||||||
|
def test_reasoning_failure_type_is_none(self) -> None:
|
||||||
|
"""attribute_error 不设 reasoning_failure_type(由后续阶段补充)。"""
|
||||||
|
qm = _make_qm(evidence_sufficient=True)
|
||||||
|
result = attribute_error(qm)
|
||||||
|
assert result.reasoning_failure_type is None
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# E. D2-D5 聚合测试
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestPercentile:
|
||||||
|
"""_percentile 辅助函数测试。"""
|
||||||
|
|
||||||
|
def test_empty_returns_zero(self) -> None:
|
||||||
|
"""空列表返回 0.0。"""
|
||||||
|
assert _percentile([], 0.5) == 0.0
|
||||||
|
|
||||||
|
def test_single_element(self) -> None:
|
||||||
|
"""单元素返回该元素。"""
|
||||||
|
assert _percentile([42.0], 0.5) == 42.0
|
||||||
|
|
||||||
|
def test_median_two_elements(self) -> None:
|
||||||
|
"""两元素中位数。"""
|
||||||
|
assert _percentile([1.0, 3.0], 0.5) == 2.0
|
||||||
|
|
||||||
|
def test_quartiles(self) -> None:
|
||||||
|
"""四分位线性插值。"""
|
||||||
|
values = [1.0, 2.0, 3.0, 4.0, 5.0]
|
||||||
|
assert _percentile(values, 0.0) == 1.0
|
||||||
|
assert _percentile(values, 1.0) == 5.0
|
||||||
|
p25 = _percentile(values, 0.25)
|
||||||
|
assert abs(p25 - 2.0) < 1e-9
|
||||||
|
|
||||||
|
|
||||||
|
class TestAggregation:
|
||||||
|
"""D2-D5 聚合函数测试。"""
|
||||||
|
|
||||||
|
def test_d2_empty(self) -> None:
|
||||||
|
"""空输入返回空字典。"""
|
||||||
|
assert aggregate_d2([]) == {}
|
||||||
|
|
||||||
|
def test_d2_groups_by_tool(self) -> None:
|
||||||
|
"""按工具名分组聚合。"""
|
||||||
|
qm = _make_qm(
|
||||||
|
span_metrics=[
|
||||||
|
_make_span(
|
||||||
|
step=0,
|
||||||
|
tool_name="view_node",
|
||||||
|
extraction_completeness=0.8,
|
||||||
|
hallucination_rate=0.2,
|
||||||
|
),
|
||||||
|
_make_span(
|
||||||
|
step=1,
|
||||||
|
tool_name="view_node",
|
||||||
|
extraction_completeness=0.6,
|
||||||
|
hallucination_rate=0.1,
|
||||||
|
),
|
||||||
|
_make_span(
|
||||||
|
step=2,
|
||||||
|
tool_name="search_similar",
|
||||||
|
extraction_completeness=0.9,
|
||||||
|
hallucination_rate=0.0,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
result = aggregate_d2([qm])
|
||||||
|
assert "view_node" in result
|
||||||
|
assert "search_similar" in result
|
||||||
|
assert result["view_node"]["n_calls"] == 2
|
||||||
|
assert result["search_similar"]["n_calls"] == 1
|
||||||
|
assert abs(result["view_node"]["avg_completeness"] - 0.7) < 1e-9
|
||||||
|
|
||||||
|
def test_d3_empty(self) -> None:
|
||||||
|
"""空输入返回空字典。"""
|
||||||
|
assert aggregate_d3([]) == {}
|
||||||
|
|
||||||
|
def test_d3_correct_vs_incorrect(self) -> None:
|
||||||
|
"""按正误拆分。"""
|
||||||
|
qm_correct = _make_qm(correct=True, task_type="T1", budget_usage=0.5)
|
||||||
|
qm_wrong = _make_qm(correct=False, task_type="T1", budget_usage=0.8)
|
||||||
|
result = aggregate_d3([qm_correct, qm_wrong])
|
||||||
|
assert "T1" in result
|
||||||
|
assert result["T1"]["correct"]["n_questions"] == 1
|
||||||
|
assert result["T1"]["incorrect"]["n_questions"] == 1
|
||||||
|
# avg_steps 存储 budget_usage 均值
|
||||||
|
assert result["T1"]["correct"]["avg_steps"] == 0.5
|
||||||
|
assert result["T1"]["incorrect"]["avg_steps"] == 0.8
|
||||||
|
|
||||||
|
def test_d4_empty(self) -> None:
|
||||||
|
"""空输入返回空字典。"""
|
||||||
|
assert aggregate_d4([]) == {}
|
||||||
|
|
||||||
|
def test_d4_adherence_rate(self) -> None:
|
||||||
|
"""技能遵循率计算。"""
|
||||||
|
qm = _make_qm(
|
||||||
|
task_type="T1",
|
||||||
|
correct=True,
|
||||||
|
skill_adherence=[
|
||||||
|
SkillStepAdherence(step_label="S1", adhered=True, description=""),
|
||||||
|
SkillStepAdherence(step_label="S1", adhered=False, description=""),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
result = aggregate_d4([qm])
|
||||||
|
assert "T1" in result
|
||||||
|
assert result["T1"]["overall_adherence"] == 0.5
|
||||||
|
|
||||||
|
def test_d5_empty_returns_zero_structure(self) -> None:
|
||||||
|
"""空输入返回完整零结构。"""
|
||||||
|
result = aggregate_d5([])
|
||||||
|
assert "early_submit_rate" in result
|
||||||
|
assert result["early_submit_rate"] == 0.0
|
||||||
|
assert "format_compliance_rate" in result
|
||||||
|
assert "budget_usage_median" in result
|
||||||
|
assert "confirmation_bias_rate" in result
|
||||||
|
assert "per_type_bias" in result
|
||||||
|
assert result["per_type_bias"] == {}
|
||||||
|
|
||||||
|
def test_d5_with_data(self) -> None:
|
||||||
|
"""有数据时正确计算。"""
|
||||||
|
qm1 = _make_qm(
|
||||||
|
correct=True,
|
||||||
|
budget_usage=0.5,
|
||||||
|
format_compliance=1.0,
|
||||||
|
confidence_calibration="calibrated",
|
||||||
|
confirmation_bias=False,
|
||||||
|
)
|
||||||
|
qm2 = _make_qm(
|
||||||
|
correct=False,
|
||||||
|
budget_usage=0.2,
|
||||||
|
format_compliance=0.8,
|
||||||
|
confidence_calibration="high_conf_wrong",
|
||||||
|
confirmation_bias=True,
|
||||||
|
)
|
||||||
|
result = aggregate_d5([qm1, qm2])
|
||||||
|
assert result["format_compliance_rate"] == 0.9
|
||||||
|
assert result["high_conf_wrong_rate"] == 0.5
|
||||||
|
assert result["early_submit_rate"] == 1.0 # 1 wrong with budget<0.3
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# F. Merge 函数测试
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestMerge:
|
||||||
|
"""merge_system_packs / merge_tool_packs 测试。"""
|
||||||
|
|
||||||
|
def test_merge_system_packs_none_on_empty(self) -> None:
|
||||||
|
"""空列表返回 None。"""
|
||||||
|
assert merge_system_packs([]) is None
|
||||||
|
|
||||||
|
def test_merge_system_packs_wraps_stats(self) -> None:
|
||||||
|
"""stats 包裹为 per_step 列表。"""
|
||||||
|
pack = SystemCasePack(stats={"a": 1}, failure_cases=[], success_cases=[])
|
||||||
|
merged = merge_system_packs([pack, pack])
|
||||||
|
assert merged is not None
|
||||||
|
assert "per_step" in merged.stats
|
||||||
|
assert len(merged.stats["per_step"]) == 2
|
||||||
|
|
||||||
|
def test_merge_system_packs_concats_cases(self) -> None:
|
||||||
|
"""failure/success cases 拼接。"""
|
||||||
|
case = CaseSample(
|
||||||
|
question_id="q1",
|
||||||
|
video_id="v1",
|
||||||
|
task_type="T1",
|
||||||
|
question="q",
|
||||||
|
options=[],
|
||||||
|
answer="a",
|
||||||
|
prediction="b",
|
||||||
|
correct=False,
|
||||||
|
error_type="mixed",
|
||||||
|
selection_reason="test",
|
||||||
|
metrics={},
|
||||||
|
trace=[],
|
||||||
|
)
|
||||||
|
p1 = SystemCasePack(stats={}, failure_cases=[case], success_cases=[])
|
||||||
|
p2 = SystemCasePack(stats={}, failure_cases=[case], success_cases=[case])
|
||||||
|
merged = merge_system_packs([p1, p2])
|
||||||
|
assert merged is not None
|
||||||
|
assert len(merged.failure_cases) == 2
|
||||||
|
assert len(merged.success_cases) == 1
|
||||||
|
|
||||||
|
def test_merge_tool_packs_empty(self) -> None:
|
||||||
|
"""空列表返回空字典。"""
|
||||||
|
assert merge_tool_packs([]) == {}
|
||||||
|
|
||||||
|
def test_merge_tool_packs_groups_by_name(self) -> None:
|
||||||
|
"""同名工具合并。"""
|
||||||
|
p1 = ToolCasePack(
|
||||||
|
tool_name="view_node",
|
||||||
|
target_files=["f1.md"],
|
||||||
|
stats={"x": 1},
|
||||||
|
failure_spans=[{"a": 1}],
|
||||||
|
success_spans=[],
|
||||||
|
)
|
||||||
|
p2 = ToolCasePack(
|
||||||
|
tool_name="view_node",
|
||||||
|
target_files=["f1.md"],
|
||||||
|
stats={"x": 2},
|
||||||
|
failure_spans=[{"b": 2}],
|
||||||
|
success_spans=[{"c": 3}],
|
||||||
|
)
|
||||||
|
merged = merge_tool_packs([p1, p2])
|
||||||
|
assert "view_node" in merged
|
||||||
|
vn = merged["view_node"]
|
||||||
|
assert len(vn.failure_spans) == 2
|
||||||
|
assert len(vn.success_spans) == 1
|
||||||
|
assert "per_step" in vn.stats
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# G. run_diagnosis 入口测试
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestRunDiagnosis:
|
||||||
|
"""run_diagnosis 入口测试。"""
|
||||||
|
|
||||||
|
def test_empty_predictions_returns_empty_result(self) -> None:
|
||||||
|
"""无预测时返回空 DiagnosisResult。"""
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
mock_log = AsyncMock()
|
||||||
|
mock_log.get_predictions.return_value = []
|
||||||
|
mock_log.get_traces.return_value = []
|
||||||
|
mock_llm = AsyncMock()
|
||||||
|
mock_store = MagicMock()
|
||||||
|
mock_store.list_skill_files.return_value = []
|
||||||
|
prompts = DiagnosePrompts(
|
||||||
|
defect_vs_lapse="",
|
||||||
|
reasoning_sub="",
|
||||||
|
span_eval_system="",
|
||||||
|
span_eval_user="",
|
||||||
|
missed_nodes="",
|
||||||
|
skill_adherence="",
|
||||||
|
confirmation_bias="",
|
||||||
|
evidence_sufficiency="",
|
||||||
|
)
|
||||||
|
result = asyncio.run(
|
||||||
|
run_diagnosis(
|
||||||
|
"run1",
|
||||||
|
[],
|
||||||
|
{},
|
||||||
|
mock_llm,
|
||||||
|
mock_log,
|
||||||
|
mock_store,
|
||||||
|
prompts,
|
||||||
|
concurrency=1,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert isinstance(result, DiagnosisResult)
|
||||||
|
assert result.run_id == "run1"
|
||||||
|
assert result.error_attributions == []
|
||||||
|
assert result.degraded_count == 0
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
"""EmbeddingProvider 适配器单元测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from app.ports import EmbeddingProvider
|
||||||
|
|
||||||
|
|
||||||
|
class TestEmbeddingProviderProtocol:
|
||||||
|
def test_protocol_is_runtime_checkable(self):
|
||||||
|
assert (
|
||||||
|
hasattr(EmbeddingProvider, "__protocol_attrs__")
|
||||||
|
or hasattr(EmbeddingProvider, "__abstractmethods__")
|
||||||
|
or True
|
||||||
|
)
|
||||||
|
# runtime_checkable Protocols support isinstance checks
|
||||||
|
|
||||||
|
def test_mock_satisfies_protocol(self):
|
||||||
|
class FakeEmbed:
|
||||||
|
@property
|
||||||
|
def dim(self) -> int:
|
||||||
|
return 4
|
||||||
|
|
||||||
|
def embed(self, texts):
|
||||||
|
if isinstance(texts, str):
|
||||||
|
texts = [texts]
|
||||||
|
return np.random.randn(len(texts), 4).astype(np.float32)
|
||||||
|
|
||||||
|
provider = FakeEmbed()
|
||||||
|
assert isinstance(provider, EmbeddingProvider)
|
||||||
|
|
||||||
|
def test_mock_embed_shape(self):
|
||||||
|
class FakeEmbed:
|
||||||
|
@property
|
||||||
|
def dim(self) -> int:
|
||||||
|
return 8
|
||||||
|
|
||||||
|
def embed(self, texts):
|
||||||
|
if isinstance(texts, str):
|
||||||
|
texts = [texts]
|
||||||
|
return np.random.randn(len(texts), 8).astype(np.float32)
|
||||||
|
|
||||||
|
provider = FakeEmbed()
|
||||||
|
result = provider.embed(["你好", "世界"])
|
||||||
|
assert result.shape == (2, 8)
|
||||||
|
result_single = provider.embed("单条")
|
||||||
|
assert result_single.shape == (1, 8)
|
||||||
|
|
||||||
|
|
||||||
|
class TestLocalEmbeddingProviderImport:
|
||||||
|
def test_can_import(self):
|
||||||
|
from adapters.embedding import LocalEmbeddingProvider
|
||||||
|
|
||||||
|
assert LocalEmbeddingProvider is not None
|
||||||
|
|
||||||
|
def test_satisfies_protocol(self):
|
||||||
|
"""LocalEmbeddingProvider 应满足 EmbeddingProvider Protocol(不实例化,避免下载模型)。"""
|
||||||
|
from adapters.embedding import LocalEmbeddingProvider
|
||||||
|
|
||||||
|
# Check class has the required methods/properties
|
||||||
|
assert hasattr(LocalEmbeddingProvider, "embed")
|
||||||
|
assert hasattr(LocalEmbeddingProvider, "dim")
|
||||||
|
|
||||||
|
|
||||||
|
class TestRemoteEmbeddingProviderImport:
|
||||||
|
def test_can_import(self):
|
||||||
|
from adapters.embedding import RemoteEmbeddingProvider
|
||||||
|
|
||||||
|
assert RemoteEmbeddingProvider is not None
|
||||||
@@ -0,0 +1,374 @@
|
|||||||
|
"""core/evolution/types.py 的类型构造与约束测试。
|
||||||
|
|
||||||
|
验证:
|
||||||
|
- frozen 类型不可变性
|
||||||
|
- mutable 类型可修改
|
||||||
|
- 全部 18 个类型可正确构造
|
||||||
|
- 默认值正确性
|
||||||
|
- 字段完整性
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.evolution.types import (
|
||||||
|
CaseSample,
|
||||||
|
DiagnosePrompts,
|
||||||
|
DiagnosisResult,
|
||||||
|
ErrorAttribution,
|
||||||
|
EvolutionRecord,
|
||||||
|
EvolutionResult,
|
||||||
|
EvolvePrompts,
|
||||||
|
GateParams,
|
||||||
|
GateVerdict,
|
||||||
|
PairResult,
|
||||||
|
QuadrantClassification,
|
||||||
|
QuestionMetrics,
|
||||||
|
RejectedEdit,
|
||||||
|
SkillCasePack,
|
||||||
|
SkillStepAdherence,
|
||||||
|
SpanMetrics,
|
||||||
|
SystemCasePack,
|
||||||
|
ToolCasePack,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Gate 类型
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_gate_params_frozen():
|
||||||
|
"""GateParams 是 frozen dataclass,构造后不可修改。"""
|
||||||
|
p = GateParams(
|
||||||
|
e_confirm=20.0,
|
||||||
|
e_provisional=3.0,
|
||||||
|
w_net_min=2,
|
||||||
|
delta_min=0.02,
|
||||||
|
lambda_dir=-0.642,
|
||||||
|
e_rollback=10.0,
|
||||||
|
)
|
||||||
|
assert p.e_confirm == 20.0
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
p.e_confirm = 1.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_gate_verdict_frozen():
|
||||||
|
"""GateVerdict 是 frozen dataclass。"""
|
||||||
|
v = GateVerdict(
|
||||||
|
decision="accept_confirmed",
|
||||||
|
e_value=25.0,
|
||||||
|
wald_lambda=1.2,
|
||||||
|
delta_hat=0.15,
|
||||||
|
delta_shrunk=0.12,
|
||||||
|
)
|
||||||
|
assert v.decision == "accept_confirmed"
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
v.decision = "reject"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 诊断类型
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_span_metrics_frozen():
|
||||||
|
"""SpanMetrics 是 frozen dataclass,含默认空列表。"""
|
||||||
|
sm = SpanMetrics(
|
||||||
|
step=1,
|
||||||
|
tool_name="view_node",
|
||||||
|
extraction_completeness=0.9,
|
||||||
|
hallucination_rate=0.05,
|
||||||
|
)
|
||||||
|
assert sm.step == 1
|
||||||
|
assert sm.missed_info_tags == []
|
||||||
|
assert sm.hallucination_tags == []
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
sm.step = 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_skill_step_adherence_frozen():
|
||||||
|
"""SkillStepAdherence 是 frozen dataclass。"""
|
||||||
|
sa = SkillStepAdherence(
|
||||||
|
step_label="定位目标层级",
|
||||||
|
adhered=True,
|
||||||
|
description="正确遵循了定位步骤",
|
||||||
|
)
|
||||||
|
assert sa.adhered is True
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
sa.adhered = False
|
||||||
|
|
||||||
|
|
||||||
|
def test_question_metrics_frozen():
|
||||||
|
"""QuestionMetrics 是 frozen dataclass,约 17 个字段。"""
|
||||||
|
qm = QuestionMetrics(
|
||||||
|
question_id="q001",
|
||||||
|
video_id="v001",
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
correct=False,
|
||||||
|
format_compliance=1.0,
|
||||||
|
budget_usage=0.6,
|
||||||
|
confidence_calibration="calibrated",
|
||||||
|
repeat_visit_rate=0.1,
|
||||||
|
search_keyword_repetition=0.0,
|
||||||
|
level_jump_pattern="L1→L2→L3",
|
||||||
|
tool_usage={"view_node": 3, "search_similar": 1},
|
||||||
|
span_metrics=[],
|
||||||
|
missed_nodes=["L2_seg_01"],
|
||||||
|
skill_adherence=[],
|
||||||
|
confirmation_bias=None,
|
||||||
|
evidence_sufficient=True,
|
||||||
|
)
|
||||||
|
assert qm.question_id == "q001"
|
||||||
|
assert qm.degraded is False # 默认值
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
qm.correct = True
|
||||||
|
|
||||||
|
|
||||||
|
def test_error_attribution_frozen():
|
||||||
|
"""ErrorAttribution 是 frozen dataclass,含可选字段。"""
|
||||||
|
ea = ErrorAttribution(
|
||||||
|
question_id="q001",
|
||||||
|
error_type="search_failure",
|
||||||
|
reasoning_failure_type=None,
|
||||||
|
cause_category="defect",
|
||||||
|
lapse_note=None,
|
||||||
|
)
|
||||||
|
assert ea.cause_category == "defect"
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
ea.cause_category = "lapse"
|
||||||
|
|
||||||
|
|
||||||
|
def test_case_sample_frozen():
|
||||||
|
"""CaseSample 是 frozen dataclass,含完整推理轨迹。"""
|
||||||
|
cs = CaseSample(
|
||||||
|
question_id="q001",
|
||||||
|
video_id="v001",
|
||||||
|
task_type="Temporal Reasoning",
|
||||||
|
question="视频中发生了什么?",
|
||||||
|
options=["A. 跑步", "B. 走路"],
|
||||||
|
answer="A",
|
||||||
|
prediction="B",
|
||||||
|
correct=False,
|
||||||
|
error_type="reasoning_failure",
|
||||||
|
selection_reason="error_type=reasoning_failure, severity=(1, 0.6)",
|
||||||
|
metrics={"correct": False, "budget_usage": 0.6},
|
||||||
|
trace=[{"step": 1, "tool_name": "view_node", "tool_output": "..."}],
|
||||||
|
)
|
||||||
|
assert cs.prediction == "B"
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
cs.prediction = "A"
|
||||||
|
|
||||||
|
|
||||||
|
def test_skill_case_pack_frozen():
|
||||||
|
"""SkillCasePack 是 frozen dataclass,含默认空列表。"""
|
||||||
|
pack = SkillCasePack(
|
||||||
|
task_type="Action Reasoning",
|
||||||
|
target_file="action-reasoning.md",
|
||||||
|
stats={"n_total": 10, "accuracy": 0.7},
|
||||||
|
)
|
||||||
|
assert pack.failure_cases == []
|
||||||
|
assert pack.success_cases == []
|
||||||
|
assert pack.lapse_notes == []
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
pack.task_type = "other"
|
||||||
|
|
||||||
|
|
||||||
|
def test_system_case_pack_frozen():
|
||||||
|
"""SystemCasePack 是 frozen dataclass。"""
|
||||||
|
pack = SystemCasePack(stats={"early_submit_count": 5})
|
||||||
|
assert pack.failure_cases == []
|
||||||
|
assert pack.success_cases == []
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
pack.stats = {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_tool_case_pack_frozen():
|
||||||
|
"""ToolCasePack 是 frozen dataclass。"""
|
||||||
|
pack = ToolCasePack(
|
||||||
|
tool_name="view_node",
|
||||||
|
target_files=["view_node_extract.md", "view_node_verify.md"],
|
||||||
|
stats={"avg_completeness": 0.85},
|
||||||
|
)
|
||||||
|
assert pack.failure_spans == []
|
||||||
|
assert pack.success_spans == []
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
pack.tool_name = "other"
|
||||||
|
|
||||||
|
|
||||||
|
def test_diagnosis_result_frozen():
|
||||||
|
"""DiagnosisResult 是 frozen dataclass,约 18 个字段。"""
|
||||||
|
dr = DiagnosisResult(run_id="run_001")
|
||||||
|
assert dr.run_id == "run_001"
|
||||||
|
assert dr.filter_summary == {}
|
||||||
|
assert dr.error_attributions == []
|
||||||
|
assert dr.system_case_pack is None
|
||||||
|
assert dr.infra_excluded_count == 0
|
||||||
|
assert dr.infra_excluded_ratio == 0.0
|
||||||
|
assert dr.defect_count == 0
|
||||||
|
assert dr.lapse_count == 0
|
||||||
|
assert dr.degraded_count == 0
|
||||||
|
assert dr.degraded_question_ids == []
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
dr.run_id = "other"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 进化类型
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_evolution_record_mutable():
|
||||||
|
"""EvolutionRecord 是 mutable dataclass,构建过程中需修改。"""
|
||||||
|
r = EvolutionRecord(
|
||||||
|
target_file="test.md",
|
||||||
|
target_type="skill",
|
||||||
|
original_content="a",
|
||||||
|
evolved_content="b",
|
||||||
|
reason="test",
|
||||||
|
status="accepted",
|
||||||
|
source_version="v1",
|
||||||
|
suggestions=[],
|
||||||
|
edits=[],
|
||||||
|
apply_report=[],
|
||||||
|
clip_info={},
|
||||||
|
)
|
||||||
|
r.status = "rejected"
|
||||||
|
assert r.status == "rejected"
|
||||||
|
|
||||||
|
|
||||||
|
def test_evolution_record_defaults():
|
||||||
|
"""EvolutionRecord 各默认字段值正确。"""
|
||||||
|
r = EvolutionRecord(
|
||||||
|
target_file="x.md",
|
||||||
|
target_type="skill",
|
||||||
|
original_content="orig",
|
||||||
|
evolved_content="new",
|
||||||
|
reason="pass",
|
||||||
|
status="accepted",
|
||||||
|
source_version="v1",
|
||||||
|
)
|
||||||
|
assert r.result_version is None
|
||||||
|
assert r.suggestions == []
|
||||||
|
assert r.attempts == []
|
||||||
|
assert r.validation_errors == []
|
||||||
|
assert r.edits == []
|
||||||
|
assert r.apply_report == []
|
||||||
|
assert r.clip_info == {"triggered": False, "clipped": 0}
|
||||||
|
|
||||||
|
|
||||||
|
def test_rejected_edit_frozen():
|
||||||
|
"""RejectedEdit 是 frozen dataclass,含 gate 证据可选字段。"""
|
||||||
|
re_ = RejectedEdit(
|
||||||
|
target_file="temporal-reasoning.md",
|
||||||
|
target_type="skill",
|
||||||
|
change_summary="增加了时序推理步骤",
|
||||||
|
delta=-0.05,
|
||||||
|
source_version="v2",
|
||||||
|
epoch=3,
|
||||||
|
gate_w=5,
|
||||||
|
gate_l=8,
|
||||||
|
gate_e_value=0.3,
|
||||||
|
gate_delta_shrunk=-0.02,
|
||||||
|
)
|
||||||
|
assert re_.gate_w == 5
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
re_.delta = 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_rejected_edit_optional_gate_fields():
|
||||||
|
"""RejectedEdit gate 字段默认为 None。"""
|
||||||
|
re_ = RejectedEdit(
|
||||||
|
target_file="x.md",
|
||||||
|
target_type="skill",
|
||||||
|
change_summary="test",
|
||||||
|
delta=0.0,
|
||||||
|
source_version="v1",
|
||||||
|
epoch=1,
|
||||||
|
)
|
||||||
|
assert re_.gate_w is None
|
||||||
|
assert re_.gate_l is None
|
||||||
|
assert re_.gate_e_value is None
|
||||||
|
assert re_.gate_delta_shrunk is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_evolution_result_frozen():
|
||||||
|
"""EvolutionResult 是 frozen dataclass,不含 skills_version/prompts_version。"""
|
||||||
|
result = EvolutionResult(
|
||||||
|
records=[],
|
||||||
|
accepted_count=2,
|
||||||
|
rejected_count=1,
|
||||||
|
skipped_count=0,
|
||||||
|
)
|
||||||
|
assert result.accepted_count == 2
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
result.accepted_count = 0
|
||||||
|
# 确认不含 TRM4 的 skills_version/prompts_version
|
||||||
|
assert not hasattr(result, "skills_version")
|
||||||
|
assert not hasattr(result, "prompts_version")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 验证辅助类型
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_pair_result_frozen():
|
||||||
|
"""PairResult 是 frozen dataclass。"""
|
||||||
|
pr = PairResult(
|
||||||
|
w=3,
|
||||||
|
l=1,
|
||||||
|
observed={"q1": (False, True), "q2": (True, False)},
|
||||||
|
)
|
||||||
|
assert pr.w == 3
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
pr.w = 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_quadrant_classification_frozen():
|
||||||
|
"""QuadrantClassification 是 frozen dataclass,四象限分类。"""
|
||||||
|
qc = QuadrantClassification(
|
||||||
|
improvements=["q1", "q3"],
|
||||||
|
regressions=["q2"],
|
||||||
|
persistent_fails=["q4"],
|
||||||
|
stable_successes=["q5", "q6"],
|
||||||
|
)
|
||||||
|
assert len(qc.improvements) == 2
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
qc.improvements = []
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Prompt 模板束
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_diagnose_prompts_frozen():
|
||||||
|
"""DiagnosePrompts 是 frozen dataclass,8 个模板字段。"""
|
||||||
|
dp = DiagnosePrompts(
|
||||||
|
defect_vs_lapse="p1",
|
||||||
|
reasoning_sub="p2",
|
||||||
|
span_eval_system="p3",
|
||||||
|
span_eval_user="p4",
|
||||||
|
missed_nodes="p5",
|
||||||
|
skill_adherence="p6",
|
||||||
|
confirmation_bias="p7",
|
||||||
|
evidence_sufficiency="p8",
|
||||||
|
)
|
||||||
|
assert dp.defect_vs_lapse == "p1"
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
dp.defect_vs_lapse = "other"
|
||||||
|
|
||||||
|
|
||||||
|
def test_evolve_prompts_frozen():
|
||||||
|
"""EvolvePrompts 是 frozen dataclass,5 个模板字段。"""
|
||||||
|
ep = EvolvePrompts(
|
||||||
|
evolve_skill="skill_tmpl",
|
||||||
|
evolve_system="system_tmpl",
|
||||||
|
evolve_tool="tool_tmpl",
|
||||||
|
evolve_rank="rank_tmpl",
|
||||||
|
consolidate_system="consolidate_tmpl",
|
||||||
|
)
|
||||||
|
assert ep.evolve_rank == "rank_tmpl"
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
ep.evolve_skill = "other"
|
||||||
@@ -0,0 +1,777 @@
|
|||||||
|
"""core/evolution/evolve.py 单元测试。
|
||||||
|
|
||||||
|
覆盖验证函数、编辑预算退火、resolve_skill_file、内部辅助函数、
|
||||||
|
受保护区构建、JSON 解析、rank_and_clip、格式化工具。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.evolution.evolve import (
|
||||||
|
_check_code_blocks,
|
||||||
|
_check_length,
|
||||||
|
_extract_section,
|
||||||
|
_format_case_samples,
|
||||||
|
_format_rejected_edits,
|
||||||
|
_format_spans,
|
||||||
|
_parse_frontmatter,
|
||||||
|
_parse_llm_json,
|
||||||
|
_select_top_edits,
|
||||||
|
_skill_protected_spans,
|
||||||
|
_strip_appendix_region,
|
||||||
|
_strip_momentum_region,
|
||||||
|
_strip_protected_regions,
|
||||||
|
_system_protected_spans,
|
||||||
|
_tool_protected_spans,
|
||||||
|
consolidate_appendix,
|
||||||
|
edit_budget_at,
|
||||||
|
evolve_single_skill,
|
||||||
|
evolve_single_tool,
|
||||||
|
evolve_system_prompt,
|
||||||
|
rank_and_clip,
|
||||||
|
resolve_skill_file,
|
||||||
|
validate_skill,
|
||||||
|
validate_system,
|
||||||
|
validate_tool,
|
||||||
|
)
|
||||||
|
from core.evolution.patch import (
|
||||||
|
APPENDIX_END,
|
||||||
|
APPENDIX_START,
|
||||||
|
MOMENTUM_END,
|
||||||
|
MOMENTUM_START,
|
||||||
|
)
|
||||||
|
from core.evolution.types import (
|
||||||
|
EvolvePrompts,
|
||||||
|
RejectedEdit,
|
||||||
|
SkillCasePack,
|
||||||
|
SystemCasePack,
|
||||||
|
ToolCasePack,
|
||||||
|
)
|
||||||
|
from core.types import LLMResponse
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# A. 内部辅助函数
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseFrontmatter:
|
||||||
|
"""_parse_frontmatter 测试。"""
|
||||||
|
|
||||||
|
def test_valid_frontmatter(self) -> None:
|
||||||
|
text = "---\nname: test\ndescription: d\n---\nbody"
|
||||||
|
result = _parse_frontmatter(text)
|
||||||
|
assert result == {"name": "test", "description": "d"}
|
||||||
|
|
||||||
|
def test_no_frontmatter(self) -> None:
|
||||||
|
assert _parse_frontmatter("no frontmatter here") is None
|
||||||
|
|
||||||
|
def test_invalid_yaml(self) -> None:
|
||||||
|
text = "---\n: : invalid\n---\nbody"
|
||||||
|
assert _parse_frontmatter(text) is None
|
||||||
|
|
||||||
|
def test_empty_frontmatter(self) -> None:
|
||||||
|
text = "---\n\n---\nbody"
|
||||||
|
result = _parse_frontmatter(text)
|
||||||
|
assert result is None # yaml.safe_load("") returns None
|
||||||
|
|
||||||
|
|
||||||
|
class TestStripAppendixRegion:
|
||||||
|
"""_strip_appendix_region 测试。"""
|
||||||
|
|
||||||
|
def test_no_markers(self) -> None:
|
||||||
|
text = "hello world"
|
||||||
|
assert _strip_appendix_region(text) == text
|
||||||
|
|
||||||
|
def test_with_markers(self) -> None:
|
||||||
|
text = f"before\n{APPENDIX_START}\nappendix stuff\n{APPENDIX_END}\nafter"
|
||||||
|
result = _strip_appendix_region(text)
|
||||||
|
assert "appendix stuff" not in result
|
||||||
|
assert "before" in result
|
||||||
|
assert "after" in result
|
||||||
|
|
||||||
|
def test_only_start_marker(self) -> None:
|
||||||
|
text = f"before\n{APPENDIX_START}\nno end marker"
|
||||||
|
assert _strip_appendix_region(text) == text
|
||||||
|
|
||||||
|
def test_only_end_marker(self) -> None:
|
||||||
|
text = f"before\n{APPENDIX_END}\nno start marker"
|
||||||
|
assert _strip_appendix_region(text) == text
|
||||||
|
|
||||||
|
|
||||||
|
class TestStripMomentumRegion:
|
||||||
|
"""_strip_momentum_region 测试。"""
|
||||||
|
|
||||||
|
def test_no_markers(self) -> None:
|
||||||
|
text = "hello world"
|
||||||
|
assert _strip_momentum_region(text) == text
|
||||||
|
|
||||||
|
def test_with_markers(self) -> None:
|
||||||
|
text = f"before\n{MOMENTUM_START}\nmomentum stuff\n{MOMENTUM_END}\nafter"
|
||||||
|
result = _strip_momentum_region(text)
|
||||||
|
assert "momentum stuff" not in result
|
||||||
|
assert "before" in result
|
||||||
|
assert "after" in result
|
||||||
|
|
||||||
|
def test_damaged_markers_raise(self) -> None:
|
||||||
|
text = f"before\n{MOMENTUM_START}\nno end marker"
|
||||||
|
with pytest.raises(ValueError, match="momentum marker 损坏"):
|
||||||
|
_strip_momentum_region(text)
|
||||||
|
|
||||||
|
|
||||||
|
class TestStripProtectedRegions:
|
||||||
|
"""_strip_protected_regions 测试(先 appendix 后 momentum)。"""
|
||||||
|
|
||||||
|
def test_both_regions(self) -> None:
|
||||||
|
text = (
|
||||||
|
f"body\n"
|
||||||
|
f"{APPENDIX_START}\nappendix\n{APPENDIX_END}\n"
|
||||||
|
f"{MOMENTUM_START}\nmomentum\n{MOMENTUM_END}\n"
|
||||||
|
f"tail"
|
||||||
|
)
|
||||||
|
result = _strip_protected_regions(text)
|
||||||
|
assert "appendix" not in result
|
||||||
|
assert "momentum" not in result
|
||||||
|
assert "body" in result
|
||||||
|
assert "tail" in result
|
||||||
|
|
||||||
|
|
||||||
|
class TestCheckLength:
|
||||||
|
"""_check_length 测试。"""
|
||||||
|
|
||||||
|
def test_normal_ratio(self) -> None:
|
||||||
|
orig = "x" * 100
|
||||||
|
evol = "x" * 120
|
||||||
|
assert _check_length(orig, evol) == []
|
||||||
|
|
||||||
|
def test_too_long(self) -> None:
|
||||||
|
orig = "x" * 100
|
||||||
|
evol = "x" * 300
|
||||||
|
errors = _check_length(orig, evol)
|
||||||
|
assert len(errors) == 1
|
||||||
|
assert "超限" in errors[0]
|
||||||
|
|
||||||
|
def test_too_short(self) -> None:
|
||||||
|
orig = "x" * 100
|
||||||
|
evol = "x" * 10
|
||||||
|
errors = _check_length(orig, evol)
|
||||||
|
assert len(errors) == 1
|
||||||
|
assert "不足" in errors[0]
|
||||||
|
|
||||||
|
def test_orig_empty(self) -> None:
|
||||||
|
assert _check_length("", "something") == []
|
||||||
|
|
||||||
|
|
||||||
|
class TestCheckCodeBlocks:
|
||||||
|
"""_check_code_blocks 测试。"""
|
||||||
|
|
||||||
|
def test_even_count(self) -> None:
|
||||||
|
text = "```python\ncode\n```"
|
||||||
|
assert _check_code_blocks(text) == []
|
||||||
|
|
||||||
|
def test_odd_count(self) -> None:
|
||||||
|
text = "```python\ncode"
|
||||||
|
errors = _check_code_blocks(text)
|
||||||
|
assert len(errors) == 1
|
||||||
|
assert "未闭合" in errors[0]
|
||||||
|
|
||||||
|
def test_no_blocks(self) -> None:
|
||||||
|
assert _check_code_blocks("no code blocks") == []
|
||||||
|
|
||||||
|
|
||||||
|
class TestExtractSection:
|
||||||
|
"""_extract_section 测试。"""
|
||||||
|
|
||||||
|
def test_found(self) -> None:
|
||||||
|
text = "intro\n## 能力边界\nfrozen content\n## other\nmore"
|
||||||
|
result = _extract_section(text, "能力边界")
|
||||||
|
assert result is not None
|
||||||
|
assert "frozen content" in result
|
||||||
|
assert "## 能力边界" in result
|
||||||
|
|
||||||
|
def test_not_found(self) -> None:
|
||||||
|
text = "intro\n## other\nmore"
|
||||||
|
assert _extract_section(text, "不存在") is None
|
||||||
|
|
||||||
|
def test_last_section(self) -> None:
|
||||||
|
text = "intro\n## 最后段\ncontent at end"
|
||||||
|
result = _extract_section(text, "最后段")
|
||||||
|
assert result is not None
|
||||||
|
assert "content at end" in result
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# B. 受保护区构建
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestSkillProtectedSpans:
|
||||||
|
"""_skill_protected_spans 测试。"""
|
||||||
|
|
||||||
|
def test_with_frontmatter(self) -> None:
|
||||||
|
text = "---\nname: test\n---\nbody"
|
||||||
|
spans = _skill_protected_spans(text)
|
||||||
|
assert any("---" in s for s in spans)
|
||||||
|
|
||||||
|
def test_with_appendix(self) -> None:
|
||||||
|
text = f"body\n{APPENDIX_START}\nnotes\n{APPENDIX_END}"
|
||||||
|
spans = _skill_protected_spans(text)
|
||||||
|
assert any(APPENDIX_START in s for s in spans)
|
||||||
|
|
||||||
|
def test_with_momentum(self) -> None:
|
||||||
|
text = f"body\n{MOMENTUM_START}\nmomentum\n{MOMENTUM_END}"
|
||||||
|
spans = _skill_protected_spans(text)
|
||||||
|
assert any(MOMENTUM_START in s for s in spans)
|
||||||
|
|
||||||
|
def test_empty(self) -> None:
|
||||||
|
assert _skill_protected_spans("plain text") == []
|
||||||
|
|
||||||
|
|
||||||
|
class TestSystemProtectedSpans:
|
||||||
|
"""_system_protected_spans 测试。"""
|
||||||
|
|
||||||
|
def test_frozen_sections(self) -> None:
|
||||||
|
text = "intro\n## 能力边界\ncontent1\n## 输出格式\ncontent2\n## 视频树结构\ncontent3\n## other\nmore"
|
||||||
|
spans = _system_protected_spans(text)
|
||||||
|
assert len(spans) == 3
|
||||||
|
|
||||||
|
def test_with_appendix(self) -> None:
|
||||||
|
text = f"body\n{APPENDIX_START}\nnotes\n{APPENDIX_END}"
|
||||||
|
spans = _system_protected_spans(text)
|
||||||
|
assert len(spans) == 1
|
||||||
|
|
||||||
|
|
||||||
|
class TestToolProtectedSpans:
|
||||||
|
"""_tool_protected_spans 测试。"""
|
||||||
|
|
||||||
|
def test_output_format_section(self) -> None:
|
||||||
|
text = "intro\n## 输出格式\nformat\n## other\nmore"
|
||||||
|
spans = _tool_protected_spans(text)
|
||||||
|
assert any("输出格式" in s for s in spans)
|
||||||
|
|
||||||
|
def test_no_output_format(self) -> None:
|
||||||
|
text = "intro\n## other\nmore"
|
||||||
|
spans = _tool_protected_spans(text)
|
||||||
|
assert len(spans) == 0
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# C. 验证函数
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestValidateSkill:
|
||||||
|
"""validate_skill 测试。"""
|
||||||
|
|
||||||
|
def test_identical_passes(self) -> None:
|
||||||
|
content = "---\nname: test\ndescription: d\ntask_type: t\n---\nbody"
|
||||||
|
assert validate_skill(content, content).passed
|
||||||
|
|
||||||
|
def test_changed_frontmatter_fails(self) -> None:
|
||||||
|
orig = "---\nname: a\ndescription: d\ntask_type: t\n---\nbody"
|
||||||
|
evol = "---\nname: b\ndescription: d\ntask_type: t\n---\nbody"
|
||||||
|
result = validate_skill(orig, evol)
|
||||||
|
assert not result.passed
|
||||||
|
assert any("name" in e for e in result.errors)
|
||||||
|
|
||||||
|
def test_length_ratio_too_short_fails(self) -> None:
|
||||||
|
orig = "---\nname: a\ndescription: d\ntask_type: t\n---\n" + "x" * 1000
|
||||||
|
evol = "---\nname: a\ndescription: d\ntask_type: t\n---\nshort"
|
||||||
|
result = validate_skill(orig, evol)
|
||||||
|
assert not result.passed
|
||||||
|
assert any("不足" in e for e in result.errors)
|
||||||
|
|
||||||
|
def test_length_ratio_too_long_fails(self) -> None:
|
||||||
|
orig = "---\nname: a\ndescription: d\ntask_type: t\n---\nshort"
|
||||||
|
evol = "---\nname: a\ndescription: d\ntask_type: t\n---\n" + "x" * 1000
|
||||||
|
result = validate_skill(orig, evol)
|
||||||
|
assert not result.passed
|
||||||
|
assert any("超限" in e for e in result.errors)
|
||||||
|
|
||||||
|
def test_unclosed_code_block_fails(self) -> None:
|
||||||
|
orig = "---\nname: a\ndescription: d\ntask_type: t\n---\nbody"
|
||||||
|
evol = "---\nname: a\ndescription: d\ntask_type: t\n---\nbody\n```"
|
||||||
|
result = validate_skill(orig, evol)
|
||||||
|
assert not result.passed
|
||||||
|
assert any("未闭合" in e for e in result.errors)
|
||||||
|
|
||||||
|
def test_missing_original_frontmatter(self) -> None:
|
||||||
|
result = validate_skill("no frontmatter", "no frontmatter")
|
||||||
|
assert not result.passed
|
||||||
|
assert any("原文缺少" in e for e in result.errors)
|
||||||
|
|
||||||
|
def test_missing_evolved_frontmatter(self) -> None:
|
||||||
|
orig = "---\nname: a\ndescription: d\ntask_type: t\n---\nbody"
|
||||||
|
result = validate_skill(orig, "no frontmatter body")
|
||||||
|
assert not result.passed
|
||||||
|
|
||||||
|
|
||||||
|
class TestValidateSystem:
|
||||||
|
"""validate_system 测试。"""
|
||||||
|
|
||||||
|
def test_identical_passes(self) -> None:
|
||||||
|
content = "intro\n## 能力边界\nfrozen\n## 输出格式\nfrozen2\n## other\nbody"
|
||||||
|
assert validate_system(content, content).passed
|
||||||
|
|
||||||
|
def test_changed_frozen_section_fails(self) -> None:
|
||||||
|
orig = "intro\n## 能力边界\noriginal\n## other\nbody"
|
||||||
|
evol = "intro\n## 能力边界\nchanged\n## other\nbody"
|
||||||
|
result = validate_system(orig, evol)
|
||||||
|
assert not result.passed
|
||||||
|
assert any("能力边界" in e for e in result.errors)
|
||||||
|
|
||||||
|
def test_missing_frozen_section_fails(self) -> None:
|
||||||
|
orig = "intro\n## 能力边界\nfrozen\n## other\nbody"
|
||||||
|
evol = "intro\n## other\nbody that is similar length content padding"
|
||||||
|
result = validate_system(orig, evol)
|
||||||
|
assert not result.passed
|
||||||
|
assert any("缺失" in e for e in result.errors)
|
||||||
|
|
||||||
|
def test_no_frozen_sections_passes(self) -> None:
|
||||||
|
content = "intro\n## other section\nbody content"
|
||||||
|
assert validate_system(content, content).passed
|
||||||
|
|
||||||
|
def test_code_block_check(self) -> None:
|
||||||
|
content = "## 能力边界\nfrozen\n## other\nbody\n```unclosed"
|
||||||
|
result = validate_system(content, content)
|
||||||
|
assert not result.passed
|
||||||
|
|
||||||
|
|
||||||
|
class TestValidateTool:
|
||||||
|
"""validate_tool 测试。"""
|
||||||
|
|
||||||
|
def test_identical_passes(self) -> None:
|
||||||
|
extract = "## 输出格式\nfixed\n## other\nbody"
|
||||||
|
verify = "## 输出格式\nfixed2\n## other\nbody2"
|
||||||
|
assert validate_tool(extract, extract, verify, verify).passed
|
||||||
|
|
||||||
|
def test_no_code_block_check(self) -> None:
|
||||||
|
"""validate_tool 不检查代码块闭合(与 skill/system 不同)。"""
|
||||||
|
extract = "## 输出格式\nfixed\n```\nunclosed"
|
||||||
|
assert validate_tool(extract, extract, "v", "v").passed
|
||||||
|
|
||||||
|
def test_changed_output_format_fails(self) -> None:
|
||||||
|
orig = "## 输出格式\noriginal format\n## other\nbody"
|
||||||
|
evol = "## 输出格式\nchanged format\n## other\nbody"
|
||||||
|
result = validate_tool(orig, evol, "v", "v")
|
||||||
|
assert not result.passed
|
||||||
|
assert any("输出格式" in e for e in result.errors)
|
||||||
|
|
||||||
|
def test_verify_output_format_checked_too(self) -> None:
|
||||||
|
extract = "## 输出格式\nfixed\n## other\nbody"
|
||||||
|
orig_verify = "## 输出格式\nfixed_v\n## other\nbody_v"
|
||||||
|
evol_verify = "## 输出格式\nchanged_v\n## other\nbody_v"
|
||||||
|
result = validate_tool(extract, extract, orig_verify, evol_verify)
|
||||||
|
assert not result.passed
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# D. 纯数学
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestEditBudget:
|
||||||
|
"""edit_budget_at 测试。"""
|
||||||
|
|
||||||
|
def test_start_at_zero(self) -> None:
|
||||||
|
assert edit_budget_at(0, 100, 5, 2) == 5
|
||||||
|
|
||||||
|
def test_end_at_total(self) -> None:
|
||||||
|
assert edit_budget_at(100, 100, 5, 2) == 2
|
||||||
|
|
||||||
|
def test_total_steps_one(self) -> None:
|
||||||
|
assert edit_budget_at(0, 1, 5, 2) == 5
|
||||||
|
|
||||||
|
def test_start_less_than_end_asserts(self) -> None:
|
||||||
|
with pytest.raises(AssertionError):
|
||||||
|
edit_budget_at(0, 100, 2, 5)
|
||||||
|
|
||||||
|
def test_mid_step(self) -> None:
|
||||||
|
result = edit_budget_at(50, 100, 5, 2)
|
||||||
|
assert 2 <= result <= 5
|
||||||
|
|
||||||
|
def test_beyond_total_clamped(self) -> None:
|
||||||
|
"""global_step 超过 total_steps 时被钳住在 end。"""
|
||||||
|
assert edit_budget_at(200, 100, 5, 2) == 2
|
||||||
|
|
||||||
|
def test_equal_start_end(self) -> None:
|
||||||
|
assert edit_budget_at(50, 100, 3, 3) == 3
|
||||||
|
|
||||||
|
def test_total_steps_zero(self) -> None:
|
||||||
|
"""total_steps <= 1 直接返回 start。"""
|
||||||
|
assert edit_budget_at(0, 0, 5, 2) == 5
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# E. JSON 解析
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseLlmJson:
|
||||||
|
"""_parse_llm_json 测试。"""
|
||||||
|
|
||||||
|
def test_plain_json(self) -> None:
|
||||||
|
raw = '{"key": "value"}'
|
||||||
|
assert _parse_llm_json(raw) == {"key": "value"}
|
||||||
|
|
||||||
|
def test_fenced_json(self) -> None:
|
||||||
|
raw = 'text before\n```json\n{"key": "value"}\n```\ntext after'
|
||||||
|
assert _parse_llm_json(raw) == {"key": "value"}
|
||||||
|
|
||||||
|
def test_non_dict_returns_none(self) -> None:
|
||||||
|
raw = "[1, 2, 3]"
|
||||||
|
assert _parse_llm_json(raw) is None
|
||||||
|
|
||||||
|
def test_invalid_json_returns_none(self) -> None:
|
||||||
|
raw = "not json at all"
|
||||||
|
assert _parse_llm_json(raw) is None
|
||||||
|
|
||||||
|
def test_empty_string(self) -> None:
|
||||||
|
assert _parse_llm_json("") is None
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# F. rank_and_clip
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestSelectTopEdits:
|
||||||
|
"""_select_top_edits 测试。"""
|
||||||
|
|
||||||
|
def test_valid_indices(self) -> None:
|
||||||
|
edits = [{"op": "a"}, {"op": "b"}, {"op": "c"}]
|
||||||
|
result = _select_top_edits([2, 0], edits, 2)
|
||||||
|
assert result == [{"op": "c"}, {"op": "a"}]
|
||||||
|
|
||||||
|
def test_dedup(self) -> None:
|
||||||
|
edits = [{"op": "a"}, {"op": "b"}]
|
||||||
|
result = _select_top_edits([0, 0, 1], edits, 3)
|
||||||
|
assert len(result) == 2
|
||||||
|
|
||||||
|
def test_out_of_bounds_skipped(self) -> None:
|
||||||
|
edits = [{"op": "a"}, {"op": "b"}]
|
||||||
|
result = _select_top_edits([5, 0, -1], edits, 3)
|
||||||
|
assert result == [{"op": "a"}]
|
||||||
|
|
||||||
|
def test_bool_excluded(self) -> None:
|
||||||
|
"""type(True) is int 返回 True 但 type(idx) is int 排除 bool。"""
|
||||||
|
edits = [{"op": "a"}, {"op": "b"}]
|
||||||
|
result = _select_top_edits([True, False, 0], edits, 3)
|
||||||
|
assert result == [{"op": "a"}]
|
||||||
|
|
||||||
|
def test_max_edits_limit(self) -> None:
|
||||||
|
edits = [{"op": "a"}, {"op": "b"}, {"op": "c"}]
|
||||||
|
result = _select_top_edits([0, 1, 2], edits, 2)
|
||||||
|
assert len(result) == 2
|
||||||
|
|
||||||
|
|
||||||
|
class TestRankAndClip:
|
||||||
|
"""rank_and_clip 测试。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_within_budget_passthrough(self) -> None:
|
||||||
|
"""edits <= max_edits 时原样返回。"""
|
||||||
|
llm = AsyncMock()
|
||||||
|
edits = [{"op": "a"}, {"op": "b"}]
|
||||||
|
result, info = await rank_and_clip(llm, "content", edits, 5, "test")
|
||||||
|
assert result == edits
|
||||||
|
assert info["triggered"] is False
|
||||||
|
llm.chat.assert_not_called()
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_llm_rank_success(self) -> None:
|
||||||
|
"""LLM 成功返回索引时按优先级裁剪。"""
|
||||||
|
llm = AsyncMock()
|
||||||
|
response = AsyncMock()
|
||||||
|
response.content = json.dumps({"selected_indices": [2, 0]})
|
||||||
|
llm.chat.return_value = response
|
||||||
|
|
||||||
|
edits = [{"op": "a"}, {"op": "b"}, {"op": "c"}]
|
||||||
|
result, info = await rank_and_clip(llm, "content", edits, 2, "test")
|
||||||
|
assert len(result) == 2
|
||||||
|
assert result[0] == {"op": "c"}
|
||||||
|
assert info["triggered"] is True
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_llm_failure_fallback(self) -> None:
|
||||||
|
"""LLM 失败时退化为按原序取前 max_edits 条。"""
|
||||||
|
llm = AsyncMock()
|
||||||
|
llm.chat.side_effect = Exception("LLM down")
|
||||||
|
|
||||||
|
edits = [{"op": "a"}, {"op": "b"}, {"op": "c"}]
|
||||||
|
result, info = await rank_and_clip(llm, "content", edits, 2, "test")
|
||||||
|
assert len(result) == 2
|
||||||
|
assert result == [{"op": "a"}, {"op": "b"}]
|
||||||
|
assert info["triggered"] is True
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# G. resolve_skill_file
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestResolveSkillFile:
|
||||||
|
"""resolve_skill_file 测试。"""
|
||||||
|
|
||||||
|
def test_direct_match(self) -> None:
|
||||||
|
class FakeStore:
|
||||||
|
def list_skill_files(self) -> list[str]:
|
||||||
|
return ["action-reasoning.md", "default-strategy.md"]
|
||||||
|
|
||||||
|
def read_skill(self, f: str) -> str:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
assert resolve_skill_file(FakeStore(), "Action Reasoning") == "action-reasoning.md"
|
||||||
|
|
||||||
|
def test_fallback_to_default(self) -> None:
|
||||||
|
class FakeStore:
|
||||||
|
def list_skill_files(self) -> list[str]:
|
||||||
|
return ["default-strategy.md"]
|
||||||
|
|
||||||
|
def read_skill(self, f: str) -> str:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
assert resolve_skill_file(FakeStore(), "Unknown Type") == "default-strategy.md"
|
||||||
|
|
||||||
|
def test_case_insensitive(self) -> None:
|
||||||
|
class FakeStore:
|
||||||
|
def list_skill_files(self) -> list[str]:
|
||||||
|
return ["temporal-reasoning.md"]
|
||||||
|
|
||||||
|
def read_skill(self, f: str) -> str:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
assert resolve_skill_file(FakeStore(), "Temporal Reasoning") == "temporal-reasoning.md"
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# H. 格式化辅助
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestFormatCaseSamples:
|
||||||
|
"""_format_case_samples 测试。"""
|
||||||
|
|
||||||
|
def test_basic_format(self) -> None:
|
||||||
|
cases = [
|
||||||
|
{
|
||||||
|
"question_id": "q1",
|
||||||
|
"question": "What?",
|
||||||
|
"options": ["A", "B"],
|
||||||
|
"answer": "A",
|
||||||
|
"prediction": "B",
|
||||||
|
"error_type": "wrong",
|
||||||
|
"selection_reason": "test",
|
||||||
|
"trace": [],
|
||||||
|
}
|
||||||
|
]
|
||||||
|
result = _format_case_samples(cases)
|
||||||
|
assert "q1" in result
|
||||||
|
assert "What?" in result
|
||||||
|
|
||||||
|
def test_trace_truncation(self) -> None:
|
||||||
|
cases = [
|
||||||
|
{
|
||||||
|
"question_id": "q1",
|
||||||
|
"question": "Q",
|
||||||
|
"options": [],
|
||||||
|
"answer": "A",
|
||||||
|
"prediction": "B",
|
||||||
|
"error_type": "e",
|
||||||
|
"selection_reason": "s",
|
||||||
|
"trace": [
|
||||||
|
{
|
||||||
|
"step": 1,
|
||||||
|
"tool_name": "t",
|
||||||
|
"tool_args": {},
|
||||||
|
"tool_output": "x" * 600,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
]
|
||||||
|
result = _format_case_samples(cases)
|
||||||
|
assert "..." in result
|
||||||
|
|
||||||
|
|
||||||
|
class TestFormatSpans:
|
||||||
|
"""_format_spans 测试。"""
|
||||||
|
|
||||||
|
def test_basic_format(self) -> None:
|
||||||
|
spans = [
|
||||||
|
{
|
||||||
|
"step": 1,
|
||||||
|
"tool_name": "extract",
|
||||||
|
"tool_args": {"query": "test"},
|
||||||
|
"tool_output": "result",
|
||||||
|
"extraction_completeness": 0.9,
|
||||||
|
"hallucination_rate": 0.1,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
result = _format_spans(spans)
|
||||||
|
assert "extract" in result
|
||||||
|
assert "0.9" in result
|
||||||
|
|
||||||
|
def test_output_truncation(self) -> None:
|
||||||
|
spans = [
|
||||||
|
{
|
||||||
|
"step": 1,
|
||||||
|
"tool_name": "t",
|
||||||
|
"tool_args": {},
|
||||||
|
"tool_output": "x" * 600,
|
||||||
|
"extraction_completeness": 0.5,
|
||||||
|
"hallucination_rate": 0.0,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
result = _format_spans(spans)
|
||||||
|
assert "..." in result
|
||||||
|
|
||||||
|
|
||||||
|
class TestFormatRejectedEdits:
|
||||||
|
"""_format_rejected_edits 测试。"""
|
||||||
|
|
||||||
|
def test_with_gate_evidence(self) -> None:
|
||||||
|
edits = [
|
||||||
|
RejectedEdit(
|
||||||
|
target_file="skill.md",
|
||||||
|
target_type="skill",
|
||||||
|
change_summary="changed X",
|
||||||
|
delta=-0.05,
|
||||||
|
source_version="v2",
|
||||||
|
epoch=3,
|
||||||
|
gate_w=5,
|
||||||
|
gate_l=8,
|
||||||
|
gate_e_value=0.42,
|
||||||
|
gate_delta_shrunk=-0.123,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
result = _format_rejected_edits(edits)
|
||||||
|
assert "W=5" in result
|
||||||
|
assert "L=8" in result
|
||||||
|
assert "E=0.42" in result
|
||||||
|
assert "δ̂=-0.123" in result
|
||||||
|
|
||||||
|
def test_without_gate_evidence(self) -> None:
|
||||||
|
edits = [
|
||||||
|
RejectedEdit(
|
||||||
|
target_file="skill.md",
|
||||||
|
target_type="skill",
|
||||||
|
change_summary="changed X",
|
||||||
|
delta=-0.05,
|
||||||
|
source_version="v2",
|
||||||
|
epoch=3,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
result = _format_rejected_edits(edits)
|
||||||
|
assert "skill.md" in result
|
||||||
|
assert "changed X" in result
|
||||||
|
assert "W=" not in result
|
||||||
|
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# I. 进化入口函数测试(Task 8)
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
_PROMPTS = EvolvePrompts(
|
||||||
|
evolve_skill="sk",
|
||||||
|
evolve_system="sys",
|
||||||
|
evolve_tool="tool",
|
||||||
|
evolve_rank="rank",
|
||||||
|
consolidate_system="cons",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_fake_llm(response_content: str) -> AsyncMock:
|
||||||
|
"""构造返回固定 LLMResponse 的模拟 LLM。"""
|
||||||
|
mock = AsyncMock()
|
||||||
|
mock.chat.return_value = LLMResponse(
|
||||||
|
content=response_content,
|
||||||
|
thinking="",
|
||||||
|
model="test",
|
||||||
|
provider="test",
|
||||||
|
prompt_tokens=0,
|
||||||
|
completion_tokens=0,
|
||||||
|
latency_ms=0,
|
||||||
|
ttft_ms=None,
|
||||||
|
max_inter_token_ms=None,
|
||||||
|
cache_hit=False,
|
||||||
|
call_id="test-id",
|
||||||
|
)
|
||||||
|
return mock
|
||||||
|
|
||||||
|
|
||||||
|
class TestEvolveSingleSkill:
|
||||||
|
"""evolve_single_skill 测试。"""
|
||||||
|
|
||||||
|
def test_empty_pack_skipped(self) -> None:
|
||||||
|
"""空案例包(无失败、无 lapse)导致无 applied edits → rejected。"""
|
||||||
|
pack = SkillCasePack(
|
||||||
|
task_type="test",
|
||||||
|
target_file="test.md",
|
||||||
|
stats={},
|
||||||
|
failure_cases=[],
|
||||||
|
success_cases=[],
|
||||||
|
lapse_notes=[],
|
||||||
|
)
|
||||||
|
store = MagicMock()
|
||||||
|
store.read_skill.return_value = "---\nname: t\ndescription: d\ntask_type: t\n---\nbody"
|
||||||
|
store.list_skill_files.return_value = ["test.md"]
|
||||||
|
llm = _make_fake_llm('{"suggestions":[],"edits":[]}')
|
||||||
|
record = asyncio.run(evolve_single_skill(llm, pack, store, _PROMPTS, "v1", 5, 6))
|
||||||
|
assert record.status in ("rejected", "skipped")
|
||||||
|
|
||||||
|
|
||||||
|
class TestEvolveSystemPrompt:
|
||||||
|
"""evolve_system_prompt 测试。"""
|
||||||
|
|
||||||
|
def test_no_failures_returns_skipped(self) -> None:
|
||||||
|
"""空 failure_cases + 空 edits → 无 applied → rejected。"""
|
||||||
|
pack = SystemCasePack(stats={}, failure_cases=[], success_cases=[])
|
||||||
|
store = MagicMock()
|
||||||
|
store.read_prompt.return_value = (
|
||||||
|
"## 能力边界\nfixed\n## 输出格式\nfixed\n## 视频树结构\nfixed\nbody"
|
||||||
|
)
|
||||||
|
llm = _make_fake_llm('{"suggestions":[],"edits":[]}')
|
||||||
|
record = asyncio.run(evolve_system_prompt(llm, pack, store, _PROMPTS, "v1", 5))
|
||||||
|
assert record.status in ("rejected", "skipped")
|
||||||
|
|
||||||
|
|
||||||
|
class TestEvolveSingleTool:
|
||||||
|
"""evolve_single_tool 测试。"""
|
||||||
|
|
||||||
|
def test_evolved_content_is_json(self) -> None:
|
||||||
|
"""即使 rejected,evolved_content 仍是合法 JSON 含 extract/verify。"""
|
||||||
|
pack = ToolCasePack(
|
||||||
|
tool_name="view_node",
|
||||||
|
target_files=["view_node_extract.md", "view_node_verify.md"],
|
||||||
|
stats={},
|
||||||
|
failure_spans=[],
|
||||||
|
success_spans=[],
|
||||||
|
)
|
||||||
|
store = MagicMock()
|
||||||
|
store.read_prompt.return_value = "## 输出格式\nfixed\nbody"
|
||||||
|
llm = _make_fake_llm('{"suggestions":[],"edits":[]}')
|
||||||
|
record = asyncio.run(evolve_single_tool(llm, pack, store, _PROMPTS, "v1", 5))
|
||||||
|
parsed = json.loads(record.evolved_content)
|
||||||
|
assert "extract" in parsed and "verify" in parsed
|
||||||
|
|
||||||
|
|
||||||
|
class TestConsolidateAppendix:
|
||||||
|
"""consolidate_appendix 测试。"""
|
||||||
|
|
||||||
|
def test_single_note_passthrough(self) -> None:
|
||||||
|
"""G1 守卫:单条 note 直接返回,不调 LLM。"""
|
||||||
|
llm = _make_fake_llm("")
|
||||||
|
result = asyncio.run(consolidate_appendix(llm, ["note1"]))
|
||||||
|
assert result == ["note1"]
|
||||||
|
|
||||||
|
def test_exception_returns_original(self) -> None:
|
||||||
|
"""G3 守卫:LLM 异常时降级返回原 notes。"""
|
||||||
|
llm = AsyncMock()
|
||||||
|
llm.chat.side_effect = RuntimeError("boom")
|
||||||
|
result = asyncio.run(consolidate_appendix(llm, ["a", "b", "c"]))
|
||||||
|
assert result == ["a", "b", "c"]
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
"""CE-Gate e-process 纯函数单元测试。
|
||||||
|
|
||||||
|
覆盖 compute_e_value、gate_decision、probation_verdict 三个公共函数
|
||||||
|
的核心路径与边界条件。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import math
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.evolution.gate import compute_e_value, gate_decision, probation_verdict
|
||||||
|
from core.evolution.types import GateParams, GateVerdict
|
||||||
|
|
||||||
|
_PARAMS = GateParams(
|
||||||
|
e_confirm=20.0,
|
||||||
|
e_provisional=3.0,
|
||||||
|
w_net_min=2,
|
||||||
|
delta_min=0.02,
|
||||||
|
lambda_dir=-0.642,
|
||||||
|
e_rollback=10.0,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestComputeEValue:
|
||||||
|
"""compute_e_value 的数学正确性与边界校验。"""
|
||||||
|
|
||||||
|
def test_zero_zero_returns_one(self) -> None:
|
||||||
|
"""W=L=0 时 e 值应为 1(无证据,中性)。"""
|
||||||
|
assert compute_e_value(0, 0) == pytest.approx(1.0)
|
||||||
|
|
||||||
|
def test_negative_w_raises(self) -> None:
|
||||||
|
"""负 W 应立即报错。"""
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
compute_e_value(-1, 0)
|
||||||
|
|
||||||
|
def test_negative_l_raises(self) -> None:
|
||||||
|
"""负 L 应立即报错。"""
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
compute_e_value(0, -1)
|
||||||
|
|
||||||
|
def test_heavy_loss_returns_near_zero(self) -> None:
|
||||||
|
"""重度失败时 e 值趋近于零。"""
|
||||||
|
assert compute_e_value(0, 20) < 0.05
|
||||||
|
|
||||||
|
def test_heavy_win_returns_large(self) -> None:
|
||||||
|
"""重度胜利时 e 值远大于 100。"""
|
||||||
|
assert compute_e_value(10, 0) > 100
|
||||||
|
|
||||||
|
def test_symmetric(self) -> None:
|
||||||
|
"""W>L 时 e 值应大于 W<L 时。"""
|
||||||
|
assert compute_e_value(5, 3) > compute_e_value(3, 5)
|
||||||
|
|
||||||
|
|
||||||
|
class TestGateDecision:
|
||||||
|
"""gate_decision 四出口优先级链测试。"""
|
||||||
|
|
||||||
|
def test_confirmed_needs_both_e_and_delta(self) -> None:
|
||||||
|
"""高 e 值 + 足够 delta → accept_confirmed。"""
|
||||||
|
v = gate_decision(10, 0, 10, 10, params=_PARAMS)
|
||||||
|
assert v.decision == "accept_confirmed"
|
||||||
|
|
||||||
|
def test_continue_on_balanced(self) -> None:
|
||||||
|
"""平衡局面且题未尽 → continue。"""
|
||||||
|
v = gate_decision(3, 3, 6, 20, params=_PARAMS)
|
||||||
|
assert v.decision == "continue"
|
||||||
|
|
||||||
|
def test_reject_inertia_on_exhaustion(self) -> None:
|
||||||
|
"""题尽且证据不足 → reject_inertia。"""
|
||||||
|
v = gate_decision(1, 1, 2, 0, params=_PARAMS)
|
||||||
|
assert v.decision == "reject_inertia"
|
||||||
|
|
||||||
|
def test_n_used_zero_raises(self) -> None:
|
||||||
|
"""n_used=0 应立即报错。"""
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
gate_decision(0, 0, 0, 10, params=_PARAMS)
|
||||||
|
|
||||||
|
def test_n_remaining_negative_raises(self) -> None:
|
||||||
|
"""n_remaining<0 应立即报错。"""
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
gate_decision(1, 0, 1, -1, params=_PARAMS)
|
||||||
|
|
||||||
|
|
||||||
|
class TestProbationVerdict:
|
||||||
|
"""probation_verdict 试用期结算测试。"""
|
||||||
|
|
||||||
|
def test_strong_win_confirmed(self) -> None:
|
||||||
|
"""强烈胜利 → confirmed 转正。"""
|
||||||
|
assert probation_verdict(10, 0, params=_PARAMS) == "confirmed"
|
||||||
|
|
||||||
|
def test_strong_loss_rollback(self) -> None:
|
||||||
|
"""强烈失败 → rollback 回滚。"""
|
||||||
|
assert probation_verdict(0, 10, params=_PARAMS) == "rollback"
|
||||||
|
|
||||||
|
def test_balanced_unverified(self) -> None:
|
||||||
|
"""平衡局面 → unverified 惯性转正。"""
|
||||||
|
assert probation_verdict(3, 3, params=_PARAMS) == "unverified"
|
||||||
@@ -0,0 +1,267 @@
|
|||||||
|
"""app/harness/batching.py 的单元测试。
|
||||||
|
|
||||||
|
覆盖 FFD + round-robin mini-batch 构建的核心场景:
|
||||||
|
确定性、小类不拆、大类 round-robin、正确率混合、空错题、参数校验、
|
||||||
|
correctness False vs None 精确匹配。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
from app.harness.batching import (
|
||||||
|
build_batches,
|
||||||
|
_validate_params,
|
||||||
|
_select_mixed_by_task_type,
|
||||||
|
)
|
||||||
|
|
||||||
|
import random
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 辅助构造
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _make_q(
|
||||||
|
qid: str,
|
||||||
|
task_type: str = "default",
|
||||||
|
video_id: str = "v1",
|
||||||
|
) -> GeneratedQuestion:
|
||||||
|
"""构造最小 GeneratedQuestion 用于测试。"""
|
||||||
|
return GeneratedQuestion(
|
||||||
|
question_id=qid,
|
||||||
|
video_id=video_id,
|
||||||
|
task_type=task_type,
|
||||||
|
question=f"question_{qid}",
|
||||||
|
options=("A. a", "B. b", "C. c", "D. d"),
|
||||||
|
answer="A",
|
||||||
|
source_nodes=("n1",),
|
||||||
|
difficulty="medium",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# test_build_batches_deterministic
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestBuildBatchesDeterministic:
|
||||||
|
"""相同输入 + 相同 seed 产出完全一致的切分。"""
|
||||||
|
|
||||||
|
def test_same_seed_same_result(self) -> None:
|
||||||
|
items = [_make_q(f"q{i}", task_type=f"type_{i % 3}") for i in range(20)]
|
||||||
|
correctness = {f"q{i}": False for i in range(20)}
|
||||||
|
r1 = build_batches(items, correctness, batch_size=5, min_class_per_batch=2, seed=42)
|
||||||
|
r2 = build_batches(items, correctness, batch_size=5, min_class_per_batch=2, seed=42)
|
||||||
|
assert r1 == r2
|
||||||
|
|
||||||
|
def test_different_seed_may_differ(self) -> None:
|
||||||
|
"""不同 seed 结果应不同(极大概率,用大量样本保证)。"""
|
||||||
|
items = [_make_q(f"q{i}", task_type=f"type_{i % 5}") for i in range(50)]
|
||||||
|
correctness = {f"q{i}": False for i in range(50)}
|
||||||
|
r1 = build_batches(items, correctness, batch_size=10, min_class_per_batch=3, seed=1)
|
||||||
|
r2 = build_batches(items, correctness, batch_size=10, min_class_per_batch=3, seed=99)
|
||||||
|
# 至少 batch 内容不同(不比较结构,只比较 selected_count 一致性)
|
||||||
|
assert r1[1] == r2[1] # 总题数一致
|
||||||
|
# 但 batch 内部排列几乎必然不同
|
||||||
|
flat1 = [q.question_id for b in r1[0] for q in b]
|
||||||
|
flat2 = [q.question_id for b in r2[0] for q in b]
|
||||||
|
assert flat1 != flat2
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# test_small_class_not_split
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestSmallClassNotSplit:
|
||||||
|
"""小类(≤ min_class_per_batch)整组不拆,锁在同一 batch。"""
|
||||||
|
|
||||||
|
def test_small_group_stays_together(self) -> None:
|
||||||
|
# 2 道题属于 small_type(≤ min_class=3),应在同一 batch
|
||||||
|
items = [
|
||||||
|
_make_q("s1", task_type="small_type"),
|
||||||
|
_make_q("s2", task_type="small_type"),
|
||||||
|
# 大类 8 道题
|
||||||
|
*[_make_q(f"big{i}", task_type="big_type") for i in range(8)],
|
||||||
|
]
|
||||||
|
correctness = {q.question_id: False for q in items}
|
||||||
|
batches, count = build_batches(
|
||||||
|
items, correctness, batch_size=5, min_class_per_batch=3, seed=0
|
||||||
|
)
|
||||||
|
assert count == 10
|
||||||
|
# 找到包含 small_type 的 batch
|
||||||
|
small_batch = [
|
||||||
|
b for b in batches
|
||||||
|
if any(q.task_type == "small_type" for q in b)
|
||||||
|
]
|
||||||
|
assert len(small_batch) == 1 # 整组在同一个 batch
|
||||||
|
small_ids = {q.question_id for q in small_batch[0] if q.task_type == "small_type"}
|
||||||
|
assert small_ids == {"s1", "s2"}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# test_large_class_round_robin
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestLargeClassRoundRobin:
|
||||||
|
"""大类样本 round-robin 散布到多个 batch,不集中于单一 batch。"""
|
||||||
|
|
||||||
|
def test_large_group_distributed(self) -> None:
|
||||||
|
# 12 道大类题,batch_size=4,min_class=2 → 大类 > 2 → round-robin
|
||||||
|
items = [_make_q(f"q{i}", task_type="large_type") for i in range(12)]
|
||||||
|
correctness = {q.question_id: False for q in items}
|
||||||
|
batches, count = build_batches(
|
||||||
|
items, correctness, batch_size=4, min_class_per_batch=2, seed=7
|
||||||
|
)
|
||||||
|
assert count == 12
|
||||||
|
assert len(batches) >= 3 # ceil(12/4) = 3
|
||||||
|
# 每个 batch 不超过 batch_size
|
||||||
|
for b in batches:
|
||||||
|
assert len(b) <= 4
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# test_correct_ratio_mixing
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestCorrectRatioMixing:
|
||||||
|
"""correct_ratio > 0 时混入正确题。"""
|
||||||
|
|
||||||
|
def test_mixed_includes_correct(self) -> None:
|
||||||
|
items = [
|
||||||
|
_make_q("e1", task_type="t1"),
|
||||||
|
_make_q("e2", task_type="t1"),
|
||||||
|
_make_q("c1", task_type="t1"),
|
||||||
|
_make_q("c2", task_type="t1"),
|
||||||
|
_make_q("c3", task_type="t1"),
|
||||||
|
]
|
||||||
|
correctness = {"e1": False, "e2": False, "c1": True, "c2": True, "c3": True}
|
||||||
|
batches, count = build_batches(
|
||||||
|
items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
|
||||||
|
correct_ratio=0.5,
|
||||||
|
)
|
||||||
|
# correct_ratio=0.5 → 错:正 = 1:1 → 2 错 + 2 正 = 4 题
|
||||||
|
assert count == 4
|
||||||
|
all_ids = {q.question_id for b in batches for q in b}
|
||||||
|
assert {"e1", "e2"}.issubset(all_ids) # 错题全部
|
||||||
|
correct_in = all_ids - {"e1", "e2"}
|
||||||
|
assert len(correct_in) == 2 # 采样 2 个正确题
|
||||||
|
assert correct_in.issubset({"c1", "c2", "c3"})
|
||||||
|
|
||||||
|
def test_ratio_zero_pure_errors(self) -> None:
|
||||||
|
items = [
|
||||||
|
_make_q("e1", task_type="t1"),
|
||||||
|
_make_q("c1", task_type="t1"),
|
||||||
|
]
|
||||||
|
correctness = {"e1": False, "c1": True}
|
||||||
|
batches, count = build_batches(
|
||||||
|
items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
|
||||||
|
correct_ratio=0.0,
|
||||||
|
)
|
||||||
|
assert count == 1
|
||||||
|
assert batches[0][0].question_id == "e1"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# test_no_wrong_answers_empty
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestNoWrongAnswersEmpty:
|
||||||
|
"""无错题时返回空列表。"""
|
||||||
|
|
||||||
|
def test_all_correct_returns_empty(self) -> None:
|
||||||
|
items = [_make_q(f"q{i}") for i in range(5)]
|
||||||
|
correctness = {f"q{i}": True for i in range(5)}
|
||||||
|
batches, count = build_batches(
|
||||||
|
items, correctness, batch_size=3, min_class_per_batch=1, seed=0,
|
||||||
|
)
|
||||||
|
assert batches == []
|
||||||
|
assert count == 0
|
||||||
|
|
||||||
|
def test_empty_items_returns_empty(self) -> None:
|
||||||
|
batches, count = build_batches(
|
||||||
|
[], {}, batch_size=3, min_class_per_batch=1, seed=0,
|
||||||
|
)
|
||||||
|
assert batches == []
|
||||||
|
assert count == 0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# test_validate_params_strict
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestValidateParamsStrict:
|
||||||
|
"""参数校验:batch_size < 1、min_class < 1、min_class >= batch_size 都报错。"""
|
||||||
|
|
||||||
|
def test_batch_size_zero(self) -> None:
|
||||||
|
with pytest.raises(ValueError, match="batch_size 必须 >= 1"):
|
||||||
|
_validate_params(0, 1)
|
||||||
|
|
||||||
|
def test_batch_size_negative(self) -> None:
|
||||||
|
with pytest.raises(ValueError, match="batch_size 必须 >= 1"):
|
||||||
|
_validate_params(-1, 1)
|
||||||
|
|
||||||
|
def test_min_class_zero(self) -> None:
|
||||||
|
with pytest.raises(ValueError, match="min_class_per_batch 必须 >= 1"):
|
||||||
|
_validate_params(5, 0)
|
||||||
|
|
||||||
|
def test_min_class_equals_batch_size(self) -> None:
|
||||||
|
with pytest.raises(ValueError, match="min_class_per_batch 必须严格 < batch_size"):
|
||||||
|
_validate_params(5, 5)
|
||||||
|
|
||||||
|
def test_min_class_exceeds_batch_size(self) -> None:
|
||||||
|
with pytest.raises(ValueError, match="min_class_per_batch 必须严格 < batch_size"):
|
||||||
|
_validate_params(3, 5)
|
||||||
|
|
||||||
|
def test_valid_params_no_error(self) -> None:
|
||||||
|
_validate_params(5, 3) # 不抛异常
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# test_correctness_false_vs_none
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestCorrectnessFalseVsNone:
|
||||||
|
"""correctness.get(qid) is False 精确匹配:None(未知题)不算错题。"""
|
||||||
|
|
||||||
|
def test_none_excluded_from_errors(self) -> None:
|
||||||
|
items = [
|
||||||
|
_make_q("wrong", task_type="t1"),
|
||||||
|
_make_q("right", task_type="t1"),
|
||||||
|
_make_q("unknown", task_type="t1"),
|
||||||
|
]
|
||||||
|
# wrong=False(错题),right=True(正确题),unknown 不在 correctness(None)
|
||||||
|
correctness: dict[str, bool] = {"wrong": False, "right": True}
|
||||||
|
batches, count = build_batches(
|
||||||
|
items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
|
||||||
|
correct_ratio=0.0,
|
||||||
|
)
|
||||||
|
# 仅 wrong 进入 batch,unknown 不算错题
|
||||||
|
assert count == 1
|
||||||
|
assert batches[0][0].question_id == "wrong"
|
||||||
|
|
||||||
|
def test_explicit_false_only(self) -> None:
|
||||||
|
"""直接测试 _select_mixed_by_task_type 内部逻辑。"""
|
||||||
|
items = [
|
||||||
|
_make_q("f1", task_type="t1"),
|
||||||
|
_make_q("n1", task_type="t1"), # None(未知)
|
||||||
|
_make_q("t1", task_type="t1"), # True(正确)
|
||||||
|
]
|
||||||
|
correctness: dict[str, bool] = {"f1": False, "t1": True}
|
||||||
|
rng = random.Random(0)
|
||||||
|
result = _select_mixed_by_task_type(items, correctness, 0.0, rng)
|
||||||
|
assert "t1" in result
|
||||||
|
assert len(result["t1"]) == 1
|
||||||
|
assert result["t1"][0].question_id == "f1"
|
||||||
|
|
||||||
|
def test_none_not_treated_as_correct(self) -> None:
|
||||||
|
"""None(未知)不进正确组,不被 correct_ratio 采样。"""
|
||||||
|
items = [
|
||||||
|
_make_q("err", task_type="t1"),
|
||||||
|
_make_q("unk", task_type="t1"),
|
||||||
|
]
|
||||||
|
correctness: dict[str, bool] = {"err": False}
|
||||||
|
rng = random.Random(0)
|
||||||
|
result = _select_mixed_by_task_type(items, correctness, 0.5, rng)
|
||||||
|
# 只有 err 一题错题,unk 不在 correctness 中 → get 返回 None → 不进 correct 组
|
||||||
|
assert len(result["t1"]) == 1 # 只有错题,无正确题可混入
|
||||||
@@ -0,0 +1,595 @@
|
|||||||
|
"""app/harness/config.py 单元测试。
|
||||||
|
|
||||||
|
覆盖 RunConfig 构造、四层校验(_validate → 三个子校验)、
|
||||||
|
YAML + CLI + .env 三层加载优先级。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from app.harness.config import RunConfig, _validate, load_config
|
||||||
|
|
||||||
|
# ────────────────────────────── 测试数据工厂 ──────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _valid_kwargs() -> dict:
|
||||||
|
"""构造一组完整合法的 RunConfig 字段值(使用真实 default.yaml 数据)。"""
|
||||||
|
return {
|
||||||
|
"workspace_dir": Path("workspaces/default"),
|
||||||
|
"store_dir": Path("store"),
|
||||||
|
"mode": "infer",
|
||||||
|
"run_id": "",
|
||||||
|
"concurrency": 12,
|
||||||
|
"max_steps": 15,
|
||||||
|
"skill_mode": "auto",
|
||||||
|
"n_samples": 0,
|
||||||
|
"questions": "benchmarks/Video-MME",
|
||||||
|
"skills_version": "v1",
|
||||||
|
"prompts_version": "v1",
|
||||||
|
"epochs": 1,
|
||||||
|
"diag_size": 200,
|
||||||
|
"diag_correct_ratio": 0.5,
|
||||||
|
"val_size": 30,
|
||||||
|
"val_correct_ratio": 0.5,
|
||||||
|
"edit_budget_start": 5,
|
||||||
|
"edit_budget_end": 2,
|
||||||
|
"batch_size": 15,
|
||||||
|
"min_class_per_batch": 2,
|
||||||
|
"eval_min_per_class": 2,
|
||||||
|
"early_stop_patience": 8,
|
||||||
|
"test_size": 60,
|
||||||
|
"use_slow_momentum": True,
|
||||||
|
"gate_e_confirm": 20.0,
|
||||||
|
"gate_e_provisional": 3.0,
|
||||||
|
"gate_w_net_min": 2,
|
||||||
|
"gate_delta_min": 0.02,
|
||||||
|
"gate_lambda_dir": -0.642,
|
||||||
|
"gate_e_rollback": 10.0,
|
||||||
|
"gate_block": 8,
|
||||||
|
"gate_n_max": 40,
|
||||||
|
"gate_p_low": 0.05,
|
||||||
|
"gate_p_high": 0.95,
|
||||||
|
"gate_probe_quota": 0.2,
|
||||||
|
"gate_gamma_decay": 0.9,
|
||||||
|
"gate_cooldown_steps": 2,
|
||||||
|
"gate_guard_err": 0.10,
|
||||||
|
"skill_update_mode": "patch",
|
||||||
|
"appendix_consolidate_threshold": 6,
|
||||||
|
"batch_correct_ratio": 0.5,
|
||||||
|
"momentum_samples": 20,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _make_config(**overrides: object) -> RunConfig:
|
||||||
|
"""用 _valid_kwargs 构造 RunConfig,支持字段覆盖。"""
|
||||||
|
kwargs = _valid_kwargs()
|
||||||
|
kwargs.update(overrides)
|
||||||
|
return RunConfig(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def _yaml_harness_dict() -> dict:
|
||||||
|
"""构造可序列化为 YAML 的合法 harness 配置字典(路径用字符串)。"""
|
||||||
|
d = _valid_kwargs()
|
||||||
|
d["workspace_dir"] = str(d["workspace_dir"])
|
||||||
|
d["store_dir"] = str(d["store_dir"])
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
def _write_yaml(tmp_path: Path, harness_data: dict) -> Path:
|
||||||
|
"""将 harness 配置写入临时 YAML 文件,返回文件路径。"""
|
||||||
|
yaml_path = tmp_path / "experiment.yaml"
|
||||||
|
with open(yaml_path, "w", encoding="utf-8") as f:
|
||||||
|
yaml.dump({"harness": harness_data}, f)
|
||||||
|
return yaml_path
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────── test_valid_config ────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestValidConfig:
|
||||||
|
"""合法参数应能正常构造并通过校验。"""
|
||||||
|
|
||||||
|
def test_default_yaml_values_pass_validation(self) -> None:
|
||||||
|
"""使用 default.yaml 真实默认值构造的 RunConfig 应通过全部校验。"""
|
||||||
|
cfg = _make_config()
|
||||||
|
_validate(cfg)
|
||||||
|
assert cfg.mode == "infer"
|
||||||
|
assert cfg.workspace_dir == Path("workspaces/default")
|
||||||
|
assert cfg.store_dir == Path("store")
|
||||||
|
|
||||||
|
def test_frozen_immutability(self) -> None:
|
||||||
|
"""frozen=True 应禁止字段赋值。"""
|
||||||
|
cfg = _make_config()
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
cfg.mode = "train" # type: ignore[misc]
|
||||||
|
|
||||||
|
def test_default_field_values(self) -> None:
|
||||||
|
"""有默认值的字段不传时应使用默认值。"""
|
||||||
|
kwargs = _valid_kwargs()
|
||||||
|
# 不传 run_id / seed / version / resume / fresh,使用默认值
|
||||||
|
kwargs.pop("run_id", None)
|
||||||
|
cfg = RunConfig(**kwargs)
|
||||||
|
assert cfg.run_id == ""
|
||||||
|
assert cfg.seed == "initial"
|
||||||
|
assert cfg.version == ""
|
||||||
|
assert cfg.resume is False
|
||||||
|
assert cfg.fresh is False
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────── test_mode_validation ────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestModeValidation:
|
||||||
|
"""mode 字段校验。"""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("bad_mode", ["unknown", "test", "", "INFER", "Train"])
|
||||||
|
def test_invalid_mode_rejected(self, bad_mode: str) -> None:
|
||||||
|
"""非法 mode 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(mode=bad_mode)
|
||||||
|
with pytest.raises(ValueError, match="mode"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"valid_mode", ["infer", "train", "diagnose", "evolve", "eval", "promote"]
|
||||||
|
)
|
||||||
|
def test_all_valid_modes_accepted(self, valid_mode: str) -> None:
|
||||||
|
"""全部合法 mode 应通过校验(diagnose/evolve 需 run_id)。"""
|
||||||
|
overrides: dict = {"mode": valid_mode}
|
||||||
|
if valid_mode in ("diagnose", "evolve", "train"):
|
||||||
|
overrides["run_id"] = "run-001"
|
||||||
|
if valid_mode in ("eval", "promote"):
|
||||||
|
overrides["version"] = "v1"
|
||||||
|
if valid_mode == "promote":
|
||||||
|
overrides["run_id"] = "run-001"
|
||||||
|
cfg = _make_config(**overrides)
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_diagnose_requires_run_id(self) -> None:
|
||||||
|
"""diagnose 模式缺少 run_id 应报错。"""
|
||||||
|
cfg = _make_config(mode="diagnose", run_id="")
|
||||||
|
with pytest.raises(ValueError, match="run_id"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_evolve_requires_run_id(self) -> None:
|
||||||
|
"""evolve 模式缺少 run_id 应报错。"""
|
||||||
|
cfg = _make_config(mode="evolve", run_id="")
|
||||||
|
with pytest.raises(ValueError, match="run_id"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_eval_requires_version(self) -> None:
|
||||||
|
"""eval 模式缺少 version 应报错。"""
|
||||||
|
cfg = _make_config(mode="eval", version="")
|
||||||
|
with pytest.raises(ValueError, match="version"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_promote_requires_run_id_and_version(self) -> None:
|
||||||
|
"""promote 模式需同时提供 run_id 和 version。"""
|
||||||
|
cfg = _make_config(mode="promote", run_id="", version="v1")
|
||||||
|
with pytest.raises(ValueError, match="promote.*run-id"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_train_mode_requires_run_id_without_resume_fresh(self) -> None:
|
||||||
|
"""train 模式非 resume/fresh 时必须提供 run_id。"""
|
||||||
|
cfg = _make_config(mode="train", run_id="", resume=False, fresh=False)
|
||||||
|
with pytest.raises(ValueError, match="run_id"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_train_mode_resume_without_run_id_accepted(self) -> None:
|
||||||
|
"""train 模式 resume=True 时不需要 run_id。"""
|
||||||
|
cfg = _make_config(mode="train", run_id="", resume=True)
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_train_mode_fresh_without_run_id_accepted(self) -> None:
|
||||||
|
"""train 模式 fresh=True 时不需要 run_id。"""
|
||||||
|
cfg = _make_config(mode="train", run_id="", fresh=True)
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_train_mode_with_run_id_accepted(self) -> None:
|
||||||
|
"""train 模式提供 run_id 时应通过(旧式基线 run)。"""
|
||||||
|
cfg = _make_config(mode="train", run_id="baseline-001")
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_concurrency_positive(self) -> None:
|
||||||
|
"""concurrency <= 0 应报错。"""
|
||||||
|
cfg = _make_config(concurrency=0)
|
||||||
|
with pytest.raises(ValueError, match="concurrency"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_max_steps_positive(self) -> None:
|
||||||
|
"""max_steps <= 0 应报错。"""
|
||||||
|
cfg = _make_config(max_steps=-1)
|
||||||
|
with pytest.raises(ValueError, match="max_steps"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_n_samples_non_negative(self) -> None:
|
||||||
|
"""n_samples < 0 应报错。"""
|
||||||
|
cfg = _make_config(n_samples=-1)
|
||||||
|
with pytest.raises(ValueError, match="n_samples"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_epochs_positive(self) -> None:
|
||||||
|
"""epochs <= 0 应报错。"""
|
||||||
|
cfg = _make_config(epochs=0)
|
||||||
|
with pytest.raises(ValueError, match="epochs"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("bad_skill_mode", ["Auto", "disabled", ""])
|
||||||
|
def test_invalid_skill_mode(self, bad_skill_mode: str) -> None:
|
||||||
|
"""非法 skill_mode 应报错。"""
|
||||||
|
cfg = _make_config(skill_mode=bad_skill_mode)
|
||||||
|
with pytest.raises(ValueError, match="skill_mode"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("bad_update_mode", ["append", "delete", ""])
|
||||||
|
def test_invalid_skill_update_mode(self, bad_update_mode: str) -> None:
|
||||||
|
"""非法 skill_update_mode 应报错。"""
|
||||||
|
cfg = _make_config(skill_update_mode=bad_update_mode)
|
||||||
|
with pytest.raises(ValueError, match="skill_update_mode"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_appendix_consolidate_threshold_positive(self) -> None:
|
||||||
|
"""appendix_consolidate_threshold < 1 应报错。"""
|
||||||
|
cfg = _make_config(appendix_consolidate_threshold=0)
|
||||||
|
with pytest.raises(ValueError, match="appendix_consolidate_threshold"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
|
||||||
|
# ────────────────────── test_edit_budget_validation ───────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestEditBudgetValidation:
|
||||||
|
"""编辑预算退火校验(_validate_edit_budget)。"""
|
||||||
|
|
||||||
|
def test_start_less_than_end_rejected(self) -> None:
|
||||||
|
"""edit_budget_start < edit_budget_end 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(edit_budget_start=1, edit_budget_end=5)
|
||||||
|
with pytest.raises(ValueError, match="edit_budget_start"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_end_zero_rejected(self) -> None:
|
||||||
|
"""edit_budget_end <= 0 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(edit_budget_start=1, edit_budget_end=0)
|
||||||
|
with pytest.raises(ValueError, match="edit_budget_end"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_equal_values_accepted(self) -> None:
|
||||||
|
"""edit_budget_start == edit_budget_end 应通过。"""
|
||||||
|
cfg = _make_config(edit_budget_start=3, edit_budget_end=3)
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────── test_minibatch_validation ───────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestMinibatchValidation:
|
||||||
|
"""mini-batch 自进化闭环参数校验(_validate_minibatch)。"""
|
||||||
|
|
||||||
|
def test_batch_size_zero_rejected(self) -> None:
|
||||||
|
"""batch_size <= 0 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(batch_size=0)
|
||||||
|
with pytest.raises(ValueError, match="batch_size"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_min_class_per_batch_equals_batch_size_rejected(self) -> None:
|
||||||
|
"""min_class_per_batch >= batch_size 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(batch_size=5, min_class_per_batch=5)
|
||||||
|
with pytest.raises(ValueError, match="min_class_per_batch"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_min_class_per_batch_zero_rejected(self) -> None:
|
||||||
|
"""min_class_per_batch < 1 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(min_class_per_batch=0)
|
||||||
|
with pytest.raises(ValueError, match="min_class_per_batch"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_eval_min_per_class_zero_rejected(self) -> None:
|
||||||
|
"""eval_min_per_class < 1 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(eval_min_per_class=0)
|
||||||
|
with pytest.raises(ValueError, match="eval_min_per_class"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_early_stop_patience_zero_rejected(self) -> None:
|
||||||
|
"""early_stop_patience <= 0 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(early_stop_patience=0)
|
||||||
|
with pytest.raises(ValueError, match="early_stop_patience"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_test_size_zero_rejected(self) -> None:
|
||||||
|
"""test_size <= 0 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(test_size=0)
|
||||||
|
with pytest.raises(ValueError, match="test_size"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_batch_correct_ratio_one_rejected(self) -> None:
|
||||||
|
"""batch_correct_ratio >= 1 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(batch_correct_ratio=1.0)
|
||||||
|
with pytest.raises(ValueError, match="batch_correct_ratio"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_batch_correct_ratio_negative_rejected(self) -> None:
|
||||||
|
"""batch_correct_ratio < 0 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(batch_correct_ratio=-0.1)
|
||||||
|
with pytest.raises(ValueError, match="batch_correct_ratio"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_momentum_samples_zero_rejected(self) -> None:
|
||||||
|
"""momentum_samples < 1 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(momentum_samples=0)
|
||||||
|
with pytest.raises(ValueError, match="momentum_samples"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────── test_gate_validation ────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestGateValidation:
|
||||||
|
"""CE-Gate 参数校验(_validate_gate)。"""
|
||||||
|
|
||||||
|
def test_e_confirm_at_one_rejected(self) -> None:
|
||||||
|
"""gate_e_confirm <= 1 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_e_confirm=1.0, gate_e_provisional=1.0)
|
||||||
|
with pytest.raises(ValueError, match="gate_e_confirm"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_e_provisional_exceeds_confirm_rejected(self) -> None:
|
||||||
|
"""gate_e_provisional > gate_e_confirm 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_e_confirm=10.0, gate_e_provisional=15.0)
|
||||||
|
with pytest.raises(ValueError, match="gate_e_provisional"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_e_provisional_at_one_rejected(self) -> None:
|
||||||
|
"""gate_e_provisional <= 1 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_e_provisional=0.5)
|
||||||
|
with pytest.raises(ValueError, match="gate_e_provisional"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_e_rollback_at_one_rejected(self) -> None:
|
||||||
|
"""gate_e_rollback <= 1 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_e_rollback=1.0)
|
||||||
|
with pytest.raises(ValueError, match="gate_e_rollback"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_w_net_min_zero_rejected(self) -> None:
|
||||||
|
"""gate_w_net_min < 1 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_w_net_min=0)
|
||||||
|
with pytest.raises(ValueError, match="gate_w_net_min"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_delta_min_negative_rejected(self) -> None:
|
||||||
|
"""gate_delta_min < 0 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_delta_min=-0.1)
|
||||||
|
with pytest.raises(ValueError, match="gate_delta_min"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_lambda_dir_positive_rejected(self) -> None:
|
||||||
|
"""gate_lambda_dir >= 0 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_lambda_dir=0.5)
|
||||||
|
with pytest.raises(ValueError, match="gate_lambda_dir"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_lambda_dir_zero_rejected(self) -> None:
|
||||||
|
"""gate_lambda_dir == 0 也应报错。"""
|
||||||
|
cfg = _make_config(gate_lambda_dir=0.0)
|
||||||
|
with pytest.raises(ValueError, match="gate_lambda_dir"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_block_exceeds_n_max_rejected(self) -> None:
|
||||||
|
"""gate_block > gate_n_max 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_block=50, gate_n_max=40)
|
||||||
|
with pytest.raises(ValueError, match="gate_block"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_block_zero_rejected(self) -> None:
|
||||||
|
"""gate_block <= 0 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_block=0)
|
||||||
|
with pytest.raises(ValueError, match="gate_block"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_p_low_exceeds_p_high_rejected(self) -> None:
|
||||||
|
"""gate_p_low >= gate_p_high 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_p_low=0.9, gate_p_high=0.1)
|
||||||
|
with pytest.raises(ValueError, match="gate_p_low"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_probe_quota_negative_rejected(self) -> None:
|
||||||
|
"""gate_probe_quota < 0 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_probe_quota=-0.1)
|
||||||
|
with pytest.raises(ValueError, match="gate_probe_quota"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_gamma_decay_zero_rejected(self) -> None:
|
||||||
|
"""gate_gamma_decay <= 0 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_gamma_decay=0.0)
|
||||||
|
with pytest.raises(ValueError, match="gate_gamma_decay"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_gamma_decay_one_rejected(self) -> None:
|
||||||
|
"""gate_gamma_decay >= 1 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_gamma_decay=1.0)
|
||||||
|
with pytest.raises(ValueError, match="gate_gamma_decay"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_cooldown_steps_zero_rejected(self) -> None:
|
||||||
|
"""gate_cooldown_steps < 1 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_cooldown_steps=0)
|
||||||
|
with pytest.raises(ValueError, match="gate_cooldown_steps"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_guard_err_zero_rejected(self) -> None:
|
||||||
|
"""gate_guard_err <= 0 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_guard_err=0.0)
|
||||||
|
with pytest.raises(ValueError, match="gate_guard_err"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_guard_err_one_rejected(self) -> None:
|
||||||
|
"""gate_guard_err >= 1 应抛出 ValueError。"""
|
||||||
|
cfg = _make_config(gate_guard_err=1.0)
|
||||||
|
with pytest.raises(ValueError, match="gate_guard_err"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────── test_load_config_cli_overrides ──────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoadConfigCliOverrides:
|
||||||
|
"""load_config 的 CLI 覆盖层测试。"""
|
||||||
|
|
||||||
|
def test_cli_overrides_yaml_values(self, tmp_path: Path) -> None:
|
||||||
|
"""CLI 参数应覆盖 YAML 中的同名字段。"""
|
||||||
|
harness_data = _yaml_harness_dict()
|
||||||
|
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||||
|
|
||||||
|
cfg = load_config(yaml_path, cli_overrides={"concurrency": 4, "max_steps": 30})
|
||||||
|
assert cfg.concurrency == 4
|
||||||
|
assert cfg.max_steps == 30
|
||||||
|
|
||||||
|
def test_cli_none_values_ignored(self, tmp_path: Path) -> None:
|
||||||
|
"""CLI 中值为 None 的字段不应覆盖 YAML。"""
|
||||||
|
harness_data = _yaml_harness_dict()
|
||||||
|
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||||
|
|
||||||
|
cfg = load_config(yaml_path, cli_overrides={"concurrency": None, "max_steps": 20})
|
||||||
|
assert cfg.concurrency == 12 # YAML 默认值
|
||||||
|
assert cfg.max_steps == 20
|
||||||
|
|
||||||
|
def test_cli_run_id_override(self, tmp_path: Path) -> None:
|
||||||
|
"""CLI 可通过 run_id 覆盖默认空字符串。"""
|
||||||
|
harness_data = _yaml_harness_dict()
|
||||||
|
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||||
|
|
||||||
|
cfg = load_config(
|
||||||
|
yaml_path,
|
||||||
|
cli_overrides={"mode": "diagnose", "run_id": "run-abc-123"},
|
||||||
|
)
|
||||||
|
assert cfg.run_id == "run-abc-123"
|
||||||
|
assert cfg.mode == "diagnose"
|
||||||
|
|
||||||
|
def test_path_fields_converted_to_path(self, tmp_path: Path) -> None:
|
||||||
|
"""workspace_dir 和 store_dir 应被转换为 Path 对象。"""
|
||||||
|
harness_data = _yaml_harness_dict()
|
||||||
|
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||||
|
|
||||||
|
cfg = load_config(yaml_path)
|
||||||
|
assert isinstance(cfg.workspace_dir, Path)
|
||||||
|
assert isinstance(cfg.store_dir, Path)
|
||||||
|
|
||||||
|
def test_unknown_cli_keys_ignored(self, tmp_path: Path) -> None:
|
||||||
|
"""YAML 和 RunConfig 中不存在的 CLI key 应被忽略。"""
|
||||||
|
harness_data = _yaml_harness_dict()
|
||||||
|
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||||
|
|
||||||
|
cfg = load_config(yaml_path, cli_overrides={"nonexistent_field": 42})
|
||||||
|
assert cfg.concurrency == 12 # 正常字段不受影响
|
||||||
|
|
||||||
|
def test_validation_runs_after_loading(self, tmp_path: Path) -> None:
|
||||||
|
"""load_config 加载后应运行校验,非法值应报错。"""
|
||||||
|
harness_data = _yaml_harness_dict()
|
||||||
|
harness_data["mode"] = "invalid_mode"
|
||||||
|
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="mode"):
|
||||||
|
load_config(yaml_path)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────── test_load_config_env_overrides ─────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoadConfigEnvOverrides:
|
||||||
|
"""load_config 的 .env 环境变量覆盖层测试。"""
|
||||||
|
|
||||||
|
def test_env_overrides_yaml_workspace_dir(
|
||||||
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
"""环境变量 HARNESS_WORKSPACE_DIR 应覆盖 YAML 中的 workspace_dir。"""
|
||||||
|
harness_data = _yaml_harness_dict()
|
||||||
|
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||||
|
|
||||||
|
monkeypatch.setenv("HARNESS_WORKSPACE_DIR", "/custom/workspace")
|
||||||
|
cfg = load_config(yaml_path)
|
||||||
|
assert cfg.workspace_dir == Path("/custom/workspace")
|
||||||
|
|
||||||
|
def test_env_overrides_yaml_store_dir(
|
||||||
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
"""环境变量 HARNESS_STORE_DIR 应覆盖 YAML 中的 store_dir。"""
|
||||||
|
harness_data = _yaml_harness_dict()
|
||||||
|
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||||
|
|
||||||
|
monkeypatch.setenv("HARNESS_STORE_DIR", "/custom/store")
|
||||||
|
cfg = load_config(yaml_path)
|
||||||
|
assert cfg.store_dir == Path("/custom/store")
|
||||||
|
|
||||||
|
def test_cli_overrides_env(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""CLI 优先级高于 .env:CLI > .env > YAML。"""
|
||||||
|
harness_data = _yaml_harness_dict()
|
||||||
|
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||||
|
|
||||||
|
monkeypatch.setenv("HARNESS_WORKSPACE_DIR", "/env/workspace")
|
||||||
|
cfg = load_config(yaml_path, cli_overrides={"workspace_dir": "/cli/workspace"})
|
||||||
|
assert cfg.workspace_dir == Path("/cli/workspace")
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────── test_load_config_real_yaml ─────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoadConfigRealYaml:
|
||||||
|
"""用真实 config/default.yaml 验证 load_config 嵌套解析。"""
|
||||||
|
|
||||||
|
def test_load_config_real_default_yaml(self) -> None:
|
||||||
|
"""真实 config/default.yaml 的 harness 段应正确解析并通过校验。"""
|
||||||
|
cfg = load_config(Path("config/default.yaml"), {})
|
||||||
|
assert cfg.mode == "infer"
|
||||||
|
assert cfg.gate_e_confirm == 20.0
|
||||||
|
assert cfg.batch_size == 15
|
||||||
|
assert cfg.concurrency == 12
|
||||||
|
assert cfg.workspace_dir == Path("workspaces/default")
|
||||||
|
assert cfg.store_dir == Path("store")
|
||||||
|
assert cfg.skill_mode == "auto"
|
||||||
|
|
||||||
|
def test_real_yaml_cli_override(self) -> None:
|
||||||
|
"""真实 YAML + CLI 覆盖应正确合并。"""
|
||||||
|
cfg = load_config(
|
||||||
|
Path("config/default.yaml"),
|
||||||
|
{"concurrency": 4, "max_steps": 30},
|
||||||
|
)
|
||||||
|
assert cfg.concurrency == 4
|
||||||
|
assert cfg.max_steps == 30
|
||||||
|
assert cfg.mode == "infer" # 未覆盖字段保持 YAML 值
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────── test_val_size_floor ─────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestValSizeFloor:
|
||||||
|
"""val_size >= eval_min_per_class * 11 的下限校验。"""
|
||||||
|
|
||||||
|
def test_val_size_below_floor_rejected(self) -> None:
|
||||||
|
"""val_size < eval_min_per_class * 11 应抛出 ValueError。
|
||||||
|
|
||||||
|
eval_min_per_class=3 → 下限 = 3 * 11 = 33,val_size=30 不足。
|
||||||
|
"""
|
||||||
|
cfg = _make_config(eval_min_per_class=3, val_size=30)
|
||||||
|
with pytest.raises(ValueError, match="val_size"):
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_val_size_at_floor_accepted(self) -> None:
|
||||||
|
"""val_size == eval_min_per_class * 11 应通过。"""
|
||||||
|
cfg = _make_config(eval_min_per_class=3, val_size=33)
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_val_size_above_floor_accepted(self) -> None:
|
||||||
|
"""val_size > eval_min_per_class * 11 应通过。"""
|
||||||
|
cfg = _make_config(eval_min_per_class=2, val_size=100)
|
||||||
|
_validate(cfg)
|
||||||
|
|
||||||
|
def test_default_yaml_values_satisfy_floor(self) -> None:
|
||||||
|
"""default.yaml 的默认值(val_size=30, eval_min_per_class=2)应满足下限。
|
||||||
|
|
||||||
|
下限 = 2 * 11 = 22,val_size=30 >= 22,通过。
|
||||||
|
"""
|
||||||
|
cfg = _make_config()
|
||||||
|
_validate(cfg) # 不应抛出异常
|
||||||
@@ -0,0 +1,347 @@
|
|||||||
|
"""HarnessLog + RunLogImpl 单元测试。
|
||||||
|
|
||||||
|
HarnessLog: SQLite 薄包装的线程安全、幂等、context manager 语义验证。
|
||||||
|
RunLogImpl: 只读 RunLog Protocol 实现,独立连接不污染 _runs 运行状态。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlite3
|
||||||
|
import threading
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.harness.log import HarnessLog, RunLogImpl
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Fixtures
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def db_path(tmp_path: Path) -> str:
|
||||||
|
"""返回临时 SQLite 数据库路径。"""
|
||||||
|
return str(tmp_path / "test.db")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def run_id() -> str:
|
||||||
|
return "test-run-001"
|
||||||
|
|
||||||
|
|
||||||
|
# ===========================================================================
|
||||||
|
# HarnessLog 测试
|
||||||
|
# ===========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestHarnessLog:
|
||||||
|
"""HarnessLog 核心功能测试。"""
|
||||||
|
|
||||||
|
def test_create_table_and_insert(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""create_table 建表 + insert 写入 + query 读回。"""
|
||||||
|
with HarnessLog(db_path, run_id) as log:
|
||||||
|
log.create_table("metrics", {"epoch": "INTEGER", "loss": "REAL"})
|
||||||
|
log.insert("metrics", {"epoch": 1, "loss": 0.5})
|
||||||
|
rows = log.query("SELECT * FROM metrics WHERE run_id = ?", (run_id,))
|
||||||
|
|
||||||
|
assert len(rows) == 1
|
||||||
|
assert rows[0]["epoch"] == 1
|
||||||
|
assert rows[0]["loss"] == 0.5
|
||||||
|
assert rows[0]["run_id"] == run_id
|
||||||
|
# insert 自动填充 timestamp
|
||||||
|
assert rows[0]["timestamp"] is not None
|
||||||
|
|
||||||
|
def test_query_thread_safety(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""并发读写不损坏游标状态。"""
|
||||||
|
errors: list[Exception] = []
|
||||||
|
|
||||||
|
with HarnessLog(db_path, run_id) as log:
|
||||||
|
log.create_table("counter", {"value": "INTEGER"})
|
||||||
|
|
||||||
|
def writer() -> None:
|
||||||
|
try:
|
||||||
|
for i in range(50):
|
||||||
|
log.insert("counter", {"value": i})
|
||||||
|
except Exception as exc:
|
||||||
|
errors.append(exc)
|
||||||
|
|
||||||
|
def reader() -> None:
|
||||||
|
try:
|
||||||
|
for _ in range(50):
|
||||||
|
log.query("SELECT COUNT(*) as cnt FROM counter")
|
||||||
|
except Exception as exc:
|
||||||
|
errors.append(exc)
|
||||||
|
|
||||||
|
threads = [
|
||||||
|
threading.Thread(target=writer),
|
||||||
|
threading.Thread(target=reader),
|
||||||
|
threading.Thread(target=reader),
|
||||||
|
]
|
||||||
|
for t in threads:
|
||||||
|
t.start()
|
||||||
|
for t in threads:
|
||||||
|
t.join()
|
||||||
|
|
||||||
|
assert errors == [], f"并发读写产生错误: {errors}"
|
||||||
|
|
||||||
|
def test_context_manager_completed(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""正常退出时 status = completed。"""
|
||||||
|
with HarnessLog(db_path, run_id):
|
||||||
|
pass
|
||||||
|
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT status, finished_at FROM _runs WHERE run_id = ?", (run_id,)
|
||||||
|
).fetchone()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
assert row["status"] == "completed"
|
||||||
|
assert row["finished_at"] is not None
|
||||||
|
|
||||||
|
def test_context_manager_failed(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""异常退出时 status = failed。"""
|
||||||
|
with pytest.raises(ValueError, match="boom"), HarnessLog(db_path, run_id):
|
||||||
|
raise ValueError("boom")
|
||||||
|
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT status FROM _runs WHERE run_id = ?", (run_id,)
|
||||||
|
).fetchone()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
assert row["status"] == "failed"
|
||||||
|
|
||||||
|
def test_insert_or_ignore_idempotent(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""同一 run_id 多次创建 HarnessLog 不报错(INSERT OR IGNORE 幂等)。"""
|
||||||
|
with HarnessLog(db_path, run_id):
|
||||||
|
pass
|
||||||
|
|
||||||
|
# 再次用同一 run_id 打开——不应抛异常
|
||||||
|
with HarnessLog(db_path, run_id) as log:
|
||||||
|
log.create_table("t", {"x": "INTEGER"})
|
||||||
|
log.insert("t", {"x": 42})
|
||||||
|
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
count = conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM _runs WHERE run_id = ?", (run_id,)
|
||||||
|
).fetchone()[0]
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
assert count == 1, "INSERT OR IGNORE 应保证 _runs 只有一行"
|
||||||
|
|
||||||
|
def test_wal_mode(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""连接初始化后 journal_mode 应为 WAL。"""
|
||||||
|
with HarnessLog(db_path, run_id) as log:
|
||||||
|
rows = log.query("PRAGMA journal_mode")
|
||||||
|
|
||||||
|
assert rows[0]["journal_mode"].lower() == "wal"
|
||||||
|
|
||||||
|
def test_insert_many(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""insert_many 批量插入多条记录。"""
|
||||||
|
records = [{"epoch": i, "loss": float(i) * 0.1} for i in range(5)]
|
||||||
|
|
||||||
|
with HarnessLog(db_path, run_id) as log:
|
||||||
|
log.create_table("batch", {"epoch": "INTEGER", "loss": "REAL"})
|
||||||
|
log.insert_many("batch", records)
|
||||||
|
rows = log.query(
|
||||||
|
"SELECT * FROM batch WHERE run_id = ? ORDER BY epoch", (run_id,)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(rows) == 5
|
||||||
|
assert [r["epoch"] for r in rows] == [0, 1, 2, 3, 4]
|
||||||
|
|
||||||
|
def test_log_event(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""log_event 向 _events 表写入事件。"""
|
||||||
|
with HarnessLog(db_path, run_id) as log:
|
||||||
|
log.log_event("train_start", {"epoch": 1, "lr": 0.001})
|
||||||
|
log.log_event("train_end", {"epoch": 1, "loss": 0.42})
|
||||||
|
rows = log.query(
|
||||||
|
"SELECT * FROM _events WHERE run_id = ? ORDER BY id", (run_id,)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(rows) == 2
|
||||||
|
assert rows[0]["event_type"] == "train_start"
|
||||||
|
assert rows[1]["event_type"] == "train_end"
|
||||||
|
|
||||||
|
def test_execute_raw_sql(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""execute 执行原生 SQL 写操作。"""
|
||||||
|
with HarnessLog(db_path, run_id) as log:
|
||||||
|
log.create_table("raw", {"val": "INTEGER"})
|
||||||
|
log.execute(
|
||||||
|
"INSERT INTO raw (run_id, timestamp, val) VALUES (?, ?, ?)",
|
||||||
|
(run_id, "2026-01-01T00:00:00", 99),
|
||||||
|
)
|
||||||
|
rows = log.query("SELECT val FROM raw WHERE run_id = ?", (run_id,))
|
||||||
|
|
||||||
|
assert len(rows) == 1
|
||||||
|
assert rows[0]["val"] == 99
|
||||||
|
|
||||||
|
def test_upsert_mode(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""insert mode='upsert' 使用 INSERT OR REPLACE。"""
|
||||||
|
with HarnessLog(db_path, run_id) as log:
|
||||||
|
log.create_table("kv", {"key": "TEXT", "val": "TEXT"}, primary_key="key")
|
||||||
|
log.insert("kv", {"key": "a", "val": "1"}, mode="upsert")
|
||||||
|
log.insert("kv", {"key": "a", "val": "2"}, mode="upsert")
|
||||||
|
rows = log.query("SELECT val FROM kv WHERE key = 'a'")
|
||||||
|
|
||||||
|
assert len(rows) == 1
|
||||||
|
assert rows[0]["val"] == "2"
|
||||||
|
|
||||||
|
|
||||||
|
# ===========================================================================
|
||||||
|
# RunLogImpl 测试
|
||||||
|
# ===========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
def _setup_predictions_and_traces(db_path: str, run_id: str) -> None:
|
||||||
|
"""向测试数据库写入 predictions 和 traces 数据,模拟 inference 阶段产物。"""
|
||||||
|
with HarnessLog(db_path, run_id) as log:
|
||||||
|
log.create_table(
|
||||||
|
"predictions",
|
||||||
|
{
|
||||||
|
"question_id": "TEXT",
|
||||||
|
"predicted_answer": "TEXT",
|
||||||
|
"correct": "INTEGER",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
log.create_table(
|
||||||
|
"traces",
|
||||||
|
{
|
||||||
|
"question_id": "TEXT",
|
||||||
|
"step_idx": "INTEGER",
|
||||||
|
"action": "TEXT",
|
||||||
|
"observation": "TEXT",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
log.insert_many(
|
||||||
|
"predictions",
|
||||||
|
[
|
||||||
|
{"question_id": "q1", "predicted_answer": "A", "correct": 1},
|
||||||
|
{"question_id": "q2", "predicted_answer": "B", "correct": 0},
|
||||||
|
{"question_id": "q3", "predicted_answer": "C", "correct": 1},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
log.insert_many(
|
||||||
|
"traces",
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"question_id": "q1",
|
||||||
|
"step_idx": 0,
|
||||||
|
"action": "search",
|
||||||
|
"observation": "found",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question_id": "q1",
|
||||||
|
"step_idx": 1,
|
||||||
|
"action": "verify",
|
||||||
|
"observation": "ok",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question_id": "q2",
|
||||||
|
"step_idx": 0,
|
||||||
|
"action": "search",
|
||||||
|
"observation": "not found",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRunLogImpl:
|
||||||
|
"""RunLogImpl 只读查询端口测试。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_get_predictions(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""get_predictions 返回指定 run 的全部预测记录。"""
|
||||||
|
_setup_predictions_and_traces(db_path, run_id)
|
||||||
|
impl = RunLogImpl(db_path)
|
||||||
|
|
||||||
|
preds = await impl.get_predictions(run_id)
|
||||||
|
|
||||||
|
assert len(preds) == 3
|
||||||
|
q_ids = {p["question_id"] for p in preds}
|
||||||
|
assert q_ids == {"q1", "q2", "q3"}
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_get_predictions_filtered(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""get_predictions 按 question_ids 过滤。"""
|
||||||
|
_setup_predictions_and_traces(db_path, run_id)
|
||||||
|
impl = RunLogImpl(db_path)
|
||||||
|
|
||||||
|
preds = await impl.get_predictions(run_id, question_ids=["q1", "q3"])
|
||||||
|
|
||||||
|
assert len(preds) == 2
|
||||||
|
q_ids = {p["question_id"] for p in preds}
|
||||||
|
assert q_ids == {"q1", "q3"}
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_get_traces(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""get_traces 返回指定 run 的全部轨迹记录。"""
|
||||||
|
_setup_predictions_and_traces(db_path, run_id)
|
||||||
|
impl = RunLogImpl(db_path)
|
||||||
|
|
||||||
|
traces = await impl.get_traces(run_id)
|
||||||
|
|
||||||
|
assert len(traces) == 3
|
||||||
|
# q1 有 2 步,q2 有 1 步
|
||||||
|
q1_traces = [t for t in traces if t["question_id"] == "q1"]
|
||||||
|
assert len(q1_traces) == 2
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_get_traces_filtered(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""get_traces 按 question_ids 过滤。"""
|
||||||
|
_setup_predictions_and_traces(db_path, run_id)
|
||||||
|
impl = RunLogImpl(db_path)
|
||||||
|
|
||||||
|
traces = await impl.get_traces(run_id, question_ids=["q2"])
|
||||||
|
|
||||||
|
assert len(traces) == 1
|
||||||
|
assert traces[0]["question_id"] == "q2"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_readonly_no_runs_insert(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""RunLogImpl 查询不触发 _runs INSERT(不污染运行状态)。"""
|
||||||
|
_setup_predictions_and_traces(db_path, run_id)
|
||||||
|
|
||||||
|
# 用不同 run_id 查询,确保不会创建新的 _runs 行
|
||||||
|
other_run = "nonexistent-run"
|
||||||
|
impl = RunLogImpl(db_path)
|
||||||
|
preds = await impl.get_predictions(other_run)
|
||||||
|
|
||||||
|
assert preds == []
|
||||||
|
|
||||||
|
# 验证 _runs 表没有 other_run 的记录
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
count = conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM _runs WHERE run_id = ?", (other_run,)
|
||||||
|
).fetchone()[0]
|
||||||
|
conn.close()
|
||||||
|
assert count == 0, "RunLogImpl 不应向 _runs 插入记录"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_protocol_compliance(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""RunLogImpl 满足 RunLog Protocol(runtime_checkable isinstance 检查)。"""
|
||||||
|
from core.evolution.protocols import RunLog
|
||||||
|
|
||||||
|
impl = RunLogImpl(db_path)
|
||||||
|
assert isinstance(impl, RunLog)
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_missing_table_returns_empty(self, db_path: str, run_id: str) -> None:
|
||||||
|
"""查询不存在的表(predictions/traces 未建)返回空列表。"""
|
||||||
|
# 仅创建 _runs,不建 predictions/traces 表
|
||||||
|
with HarnessLog(db_path, run_id):
|
||||||
|
pass
|
||||||
|
|
||||||
|
impl = RunLogImpl(db_path)
|
||||||
|
preds = await impl.get_predictions(run_id)
|
||||||
|
traces = await impl.get_traces(run_id)
|
||||||
|
|
||||||
|
assert preds == []
|
||||||
|
assert traces == []
|
||||||
@@ -0,0 +1,290 @@
|
|||||||
|
"""三池切分单元测试。
|
||||||
|
|
||||||
|
验证:
|
||||||
|
- 三池互斥(question_id 无重叠)
|
||||||
|
- test 池自然分布(correct_ratio=None)
|
||||||
|
- save/load 往返一致
|
||||||
|
- 旧格式拒绝(无 test 键 → ValueError)
|
||||||
|
- build_or_load_pools 冻结复用(pools.json 存在时不重切)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.harness.pools import (
|
||||||
|
build_pools,
|
||||||
|
load_pools,
|
||||||
|
save_pools,
|
||||||
|
)
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def _make_question(qid: str, task_type: str = "Action Reasoning") -> GeneratedQuestion:
|
||||||
|
"""构造测试用 GeneratedQuestion。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
qid: 题目 ID。
|
||||||
|
task_type: 题型。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
GeneratedQuestion 实例。
|
||||||
|
"""
|
||||||
|
return GeneratedQuestion(
|
||||||
|
question_id=qid,
|
||||||
|
video_id="video_001",
|
||||||
|
task_type=task_type,
|
||||||
|
question=f"Question {qid}?",
|
||||||
|
options=("A. opt1", "B. opt2", "C. opt3", "D. opt4"),
|
||||||
|
answer="A",
|
||||||
|
source_nodes=("node_1",),
|
||||||
|
difficulty="medium",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_question_set(
|
||||||
|
n: int,
|
||||||
|
task_types: list[str] | None = None,
|
||||||
|
) -> list[GeneratedQuestion]:
|
||||||
|
"""构造 n 道题,交替分配题型。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
n: 题目数量。
|
||||||
|
task_types: 可选题型列表,轮转分配;None 默认 2 类。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
题目列表。
|
||||||
|
"""
|
||||||
|
types = task_types or ["Action Reasoning", "Scene Understanding"]
|
||||||
|
return [_make_question(f"q_{i:04d}", types[i % len(types)]) for i in range(n)]
|
||||||
|
|
||||||
|
|
||||||
|
def _make_correctness(
|
||||||
|
questions: list[GeneratedQuestion],
|
||||||
|
correct_ratio: float = 0.5,
|
||||||
|
) -> dict[str, bool]:
|
||||||
|
"""构造 correctness 字典,前 correct_ratio 比例标对。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
questions: 题目列表。
|
||||||
|
correct_ratio: 对题占比。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
question_id -> bool。
|
||||||
|
"""
|
||||||
|
n_correct = round(len(questions) * correct_ratio)
|
||||||
|
return {q.question_id: (i < n_correct) for i, q in enumerate(questions)}
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildPoolsMutualExclusion:
|
||||||
|
"""三池 question_id 互斥验证。"""
|
||||||
|
|
||||||
|
def test_build_pools_mutual_exclusion(self) -> None:
|
||||||
|
"""三池切分后,任意两池不共享 question_id。"""
|
||||||
|
questions = _make_question_set(200)
|
||||||
|
correctness = _make_correctness(questions, 0.5)
|
||||||
|
|
||||||
|
pools = build_pools(
|
||||||
|
questions,
|
||||||
|
correctness,
|
||||||
|
diag_cfg={
|
||||||
|
"size": 30,
|
||||||
|
"correct_ratio": 0.5,
|
||||||
|
"task_types": None,
|
||||||
|
"seed": 42,
|
||||||
|
"min_per_class": None,
|
||||||
|
},
|
||||||
|
val_cfg={
|
||||||
|
"size": 30,
|
||||||
|
"correct_ratio": 0.5,
|
||||||
|
"task_types": None,
|
||||||
|
"seed": 42,
|
||||||
|
"min_per_class": None,
|
||||||
|
},
|
||||||
|
test_cfg={"size": 30},
|
||||||
|
baseline_run_id="run_baseline",
|
||||||
|
)
|
||||||
|
|
||||||
|
diag_ids = {q.question_id for q in pools.diagnosis}
|
||||||
|
val_ids = {q.question_id for q in pools.validation}
|
||||||
|
test_ids = {q.question_id for q in pools.test}
|
||||||
|
|
||||||
|
assert diag_ids & val_ids == set(), "诊断池与验证池有重叠"
|
||||||
|
assert diag_ids & test_ids == set(), "诊断池与测试池有重叠"
|
||||||
|
assert val_ids & test_ids == set(), "验证池与测试池有重叠"
|
||||||
|
|
||||||
|
assert len(diag_ids) == 30
|
||||||
|
assert len(val_ids) == 30
|
||||||
|
assert len(test_ids) == 30
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildPoolsTestNaturalDistribution:
|
||||||
|
"""test 池使用自然分布(correct_ratio=None)。"""
|
||||||
|
|
||||||
|
def test_build_pools_test_natural_distribution(self) -> None:
|
||||||
|
"""test 池不强制对错比例,保留候选池的自然分布。
|
||||||
|
|
||||||
|
构造 correctness 为 50% 对/50% 错,diag/val 用 correct_ratio=0.3
|
||||||
|
强制裁剪,test 池走自然分布(correct_ratio=None)。验证 test 池
|
||||||
|
不受 correct_ratio 约束。
|
||||||
|
"""
|
||||||
|
questions = _make_question_set(300)
|
||||||
|
correctness = _make_correctness(questions, 0.5)
|
||||||
|
|
||||||
|
pools = build_pools(
|
||||||
|
questions,
|
||||||
|
correctness,
|
||||||
|
diag_cfg={
|
||||||
|
"size": 20,
|
||||||
|
"correct_ratio": 0.3,
|
||||||
|
"task_types": None,
|
||||||
|
"seed": 42,
|
||||||
|
"min_per_class": None,
|
||||||
|
},
|
||||||
|
val_cfg={
|
||||||
|
"size": 20,
|
||||||
|
"correct_ratio": 0.3,
|
||||||
|
"task_types": None,
|
||||||
|
"seed": 42,
|
||||||
|
"min_per_class": None,
|
||||||
|
},
|
||||||
|
test_cfg={"size": 20},
|
||||||
|
baseline_run_id="run_baseline",
|
||||||
|
)
|
||||||
|
|
||||||
|
# diag/val 被 correct_ratio=0.3 裁剪:round(20*0.3) = 6 对, 14 错
|
||||||
|
diag_correct = sum(1 for q in pools.diagnosis if correctness[q.question_id])
|
||||||
|
val_correct = sum(1 for q in pools.validation if correctness[q.question_id])
|
||||||
|
assert diag_correct == 6, "诊断池应强制 30% 对题"
|
||||||
|
assert val_correct == 6, "验证池应强制 30% 对题"
|
||||||
|
|
||||||
|
# test 池自然分布:不受 correct_ratio 约束
|
||||||
|
assert len(pools.test) == 20
|
||||||
|
|
||||||
|
|
||||||
|
class TestSaveLoadPoolsRoundtrip:
|
||||||
|
"""save/load 往返一致验证。"""
|
||||||
|
|
||||||
|
def test_save_load_pools_roundtrip(self, tmp_path: Path) -> None:
|
||||||
|
"""save_pools → load_pools 后全字段一致。"""
|
||||||
|
questions = _make_question_set(100)
|
||||||
|
correctness = _make_correctness(questions, 0.5)
|
||||||
|
|
||||||
|
original = build_pools(
|
||||||
|
questions,
|
||||||
|
correctness,
|
||||||
|
diag_cfg={
|
||||||
|
"size": 15,
|
||||||
|
"correct_ratio": 0.5,
|
||||||
|
"task_types": None,
|
||||||
|
"seed": 42,
|
||||||
|
"min_per_class": None,
|
||||||
|
},
|
||||||
|
val_cfg={
|
||||||
|
"size": 15,
|
||||||
|
"correct_ratio": 0.5,
|
||||||
|
"task_types": None,
|
||||||
|
"seed": 42,
|
||||||
|
"min_per_class": None,
|
||||||
|
},
|
||||||
|
test_cfg={"size": 15},
|
||||||
|
baseline_run_id="run_001",
|
||||||
|
)
|
||||||
|
|
||||||
|
pools_path = tmp_path / "pools.json"
|
||||||
|
save_pools(original, pools_path)
|
||||||
|
restored = load_pools(pools_path)
|
||||||
|
|
||||||
|
# 标量字段
|
||||||
|
assert restored.baseline_run_id == original.baseline_run_id
|
||||||
|
assert restored.baseline_val_accuracy == pytest.approx(original.baseline_val_accuracy)
|
||||||
|
assert restored.correctness == original.correctness
|
||||||
|
|
||||||
|
# 三池逐题比对
|
||||||
|
for pool_name in ("diagnosis", "validation", "test"):
|
||||||
|
orig_list = getattr(original, pool_name)
|
||||||
|
rest_list = getattr(restored, pool_name)
|
||||||
|
assert len(rest_list) == len(orig_list), f"{pool_name} 长度不一致"
|
||||||
|
for o, r in zip(orig_list, rest_list, strict=False):
|
||||||
|
assert o.question_id == r.question_id
|
||||||
|
assert o.video_id == r.video_id
|
||||||
|
assert o.task_type == r.task_type
|
||||||
|
assert o.question == r.question
|
||||||
|
assert o.options == r.options
|
||||||
|
assert o.answer == r.answer
|
||||||
|
assert o.source_nodes == r.source_nodes
|
||||||
|
assert o.difficulty == r.difficulty
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoadPoolsOldFormatReject:
|
||||||
|
"""旧格式 pools.json(无 test 键)→ ValueError。"""
|
||||||
|
|
||||||
|
def test_load_pools_old_format_reject(self, tmp_path: Path) -> None:
|
||||||
|
"""缺少 test 键的 pools.json 必须抛出 ValueError。"""
|
||||||
|
old_format = {
|
||||||
|
"baseline_run_id": "run_old",
|
||||||
|
"baseline_val_accuracy": 0.5,
|
||||||
|
"correctness": {},
|
||||||
|
"diagnosis": [],
|
||||||
|
"validation": [],
|
||||||
|
}
|
||||||
|
pools_path = tmp_path / "pools.json"
|
||||||
|
pools_path.write_text(json.dumps(old_format), encoding="utf-8")
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="旧格式"):
|
||||||
|
load_pools(pools_path)
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildOrLoadPoolsFrozen:
|
||||||
|
"""build_or_load_pools 冻结复用:pools.json 存在时原样加载不重切。"""
|
||||||
|
|
||||||
|
def test_build_or_load_pools_frozen(self, tmp_path: Path) -> None:
|
||||||
|
"""pools.json 已存在时,build_or_load_pools 返回冻结内容。"""
|
||||||
|
questions = _make_question_set(60)
|
||||||
|
correctness = _make_correctness(questions, 0.5)
|
||||||
|
|
||||||
|
frozen = build_pools(
|
||||||
|
questions,
|
||||||
|
correctness,
|
||||||
|
diag_cfg={
|
||||||
|
"size": 10,
|
||||||
|
"correct_ratio": 0.5,
|
||||||
|
"task_types": None,
|
||||||
|
"seed": 42,
|
||||||
|
"min_per_class": None,
|
||||||
|
},
|
||||||
|
val_cfg={
|
||||||
|
"size": 10,
|
||||||
|
"correct_ratio": 0.5,
|
||||||
|
"task_types": None,
|
||||||
|
"seed": 42,
|
||||||
|
"min_per_class": None,
|
||||||
|
},
|
||||||
|
test_cfg={"size": 10},
|
||||||
|
baseline_run_id="run_frozen",
|
||||||
|
)
|
||||||
|
|
||||||
|
pools_path = tmp_path / "pools.json"
|
||||||
|
save_pools(frozen, pools_path)
|
||||||
|
|
||||||
|
# build_or_load_pools 中 pools.json 存在 → 直接 load_pools
|
||||||
|
# 此处直接测试 load_pools 行为等价
|
||||||
|
loaded = load_pools(pools_path)
|
||||||
|
|
||||||
|
assert loaded.baseline_run_id == frozen.baseline_run_id
|
||||||
|
assert loaded.baseline_val_accuracy == pytest.approx(frozen.baseline_val_accuracy)
|
||||||
|
assert len(loaded.test) == len(frozen.test)
|
||||||
|
assert len(loaded.validation) == len(frozen.validation)
|
||||||
|
assert len(loaded.diagnosis) == len(frozen.diagnosis)
|
||||||
|
|
||||||
|
# question_id 完全一致
|
||||||
|
for pool_name in ("diagnosis", "validation", "test"):
|
||||||
|
orig_ids = [q.question_id for q in getattr(frozen, pool_name)]
|
||||||
|
load_ids = [q.question_id for q in getattr(loaded, pool_name)]
|
||||||
|
assert orig_ids == load_ids, f"{pool_name} 冻结后 ID 顺序不一致"
|
||||||
@@ -0,0 +1,460 @@
|
|||||||
|
"""Store 版本操作 + Seed 管理的单元测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.harness.store import (
|
||||||
|
_parse_version,
|
||||||
|
_write_meta,
|
||||||
|
advance_version,
|
||||||
|
extract_run_db,
|
||||||
|
init_seed,
|
||||||
|
init_store,
|
||||||
|
list_seeds,
|
||||||
|
list_versions,
|
||||||
|
next_version,
|
||||||
|
promote_to_seed,
|
||||||
|
read_seed,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# _parse_version
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseVersion:
|
||||||
|
"""_parse_version 解析 v\\d+ 格式版本号。"""
|
||||||
|
|
||||||
|
def test_parse_version_normal(self) -> None:
|
||||||
|
assert _parse_version("v1") == 1
|
||||||
|
assert _parse_version("v10") == 10
|
||||||
|
assert _parse_version("v999") == 999
|
||||||
|
|
||||||
|
def test_parse_version_invalid(self) -> None:
|
||||||
|
with pytest.raises(ValueError, match="无效版本号"):
|
||||||
|
_parse_version("abc")
|
||||||
|
with pytest.raises(ValueError, match="无效版本号"):
|
||||||
|
_parse_version("v")
|
||||||
|
with pytest.raises(ValueError, match="无效版本号"):
|
||||||
|
_parse_version("v1.0")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# list_versions — 数字排序
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestListVersions:
|
||||||
|
"""list_versions 按数字排序,v10 排在 v2 后。"""
|
||||||
|
|
||||||
|
def test_list_versions_numeric_sort(self, tmp_path: "Path") -> None:
|
||||||
|
"""v10 必须排在 v2 后面(非字典序)。"""
|
||||||
|
store = tmp_path / "store"
|
||||||
|
resource = store / "skills"
|
||||||
|
resource.mkdir(parents=True)
|
||||||
|
for v in ("v1", "v10", "v2", "v20", "v3"):
|
||||||
|
(resource / v).mkdir()
|
||||||
|
result = list_versions(store, "skills")
|
||||||
|
assert result == ["v1", "v2", "v3", "v10", "v20"]
|
||||||
|
|
||||||
|
def test_list_versions_empty(self, tmp_path: "Path") -> None:
|
||||||
|
store = tmp_path / "store"
|
||||||
|
assert list_versions(store, "skills") == []
|
||||||
|
|
||||||
|
def test_list_versions_ignores_non_version_dirs(self, tmp_path: "Path") -> None:
|
||||||
|
"""非 v\\d+ 格式的目录被忽略。"""
|
||||||
|
store = tmp_path / "store"
|
||||||
|
resource = store / "skills"
|
||||||
|
resource.mkdir(parents=True)
|
||||||
|
(resource / "v1").mkdir()
|
||||||
|
(resource / "backup").mkdir()
|
||||||
|
(resource / ".hidden").mkdir()
|
||||||
|
assert list_versions(store, "skills") == ["v1"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# next_version
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestNextVersion:
|
||||||
|
"""next_version 返回下一个可用版本号。"""
|
||||||
|
|
||||||
|
def test_next_version_empty(self, tmp_path: "Path") -> None:
|
||||||
|
store = tmp_path / "store"
|
||||||
|
assert next_version(store, "skills") == "v1"
|
||||||
|
|
||||||
|
def test_next_version_after_existing(self, tmp_path: "Path") -> None:
|
||||||
|
store = tmp_path / "store"
|
||||||
|
resource = store / "skills"
|
||||||
|
resource.mkdir(parents=True)
|
||||||
|
(resource / "v1").mkdir()
|
||||||
|
(resource / "v2").mkdir()
|
||||||
|
assert next_version(store, "skills") == "v3"
|
||||||
|
|
||||||
|
def test_next_version_with_gap(self, tmp_path: "Path") -> None:
|
||||||
|
"""v1 和 v10 之间有 gap,next 应为 v11。"""
|
||||||
|
store = tmp_path / "store"
|
||||||
|
resource = store / "skills"
|
||||||
|
resource.mkdir(parents=True)
|
||||||
|
(resource / "v1").mkdir()
|
||||||
|
(resource / "v10").mkdir()
|
||||||
|
assert next_version(store, "skills") == "v11"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# advance_version
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestAdvanceVersion:
|
||||||
|
"""advance_version copytree + _write_meta。"""
|
||||||
|
|
||||||
|
def test_advance_version(self, tmp_path: "Path") -> None:
|
||||||
|
store = tmp_path / "store"
|
||||||
|
resource = store / "skills"
|
||||||
|
resource.mkdir(parents=True)
|
||||||
|
(resource / "v1").mkdir()
|
||||||
|
|
||||||
|
source = tmp_path / "new_skills"
|
||||||
|
source.mkdir()
|
||||||
|
(source / "skill_a.md").write_text("内容A")
|
||||||
|
|
||||||
|
version = advance_version(
|
||||||
|
store,
|
||||||
|
"skills",
|
||||||
|
source,
|
||||||
|
{"source": "evolution", "description": "进化产出"},
|
||||||
|
)
|
||||||
|
assert version == "v2"
|
||||||
|
assert (resource / "v2" / "skill_a.md").read_text() == "内容A"
|
||||||
|
|
||||||
|
meta = json.loads((resource / "v2" / "meta.json").read_text())
|
||||||
|
assert meta["version"] == "v2"
|
||||||
|
assert meta["source"] == "evolution"
|
||||||
|
assert meta["description"] == "进化产出"
|
||||||
|
assert "created_at" in meta
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# init_store
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestInitStore:
|
||||||
|
"""init_store 初始化 Store 目录结构。"""
|
||||||
|
|
||||||
|
def test_init_store(self, tmp_path: "Path") -> None:
|
||||||
|
videos = tmp_path / "videos_src"
|
||||||
|
videos.mkdir()
|
||||||
|
(videos / "v001").mkdir()
|
||||||
|
(videos / "v001" / "tree.json").write_text("{}")
|
||||||
|
|
||||||
|
skills = tmp_path / "skills_src"
|
||||||
|
skills.mkdir()
|
||||||
|
(skills / "search.md").write_text("skill")
|
||||||
|
|
||||||
|
prompts = tmp_path / "prompts_src"
|
||||||
|
prompts.mkdir()
|
||||||
|
(prompts / "system.md").write_text("prompt")
|
||||||
|
|
||||||
|
store = tmp_path / "store"
|
||||||
|
init_store(store, videos, skills, prompts)
|
||||||
|
|
||||||
|
assert (store / "videos" / "v001" / "tree.json").exists()
|
||||||
|
assert (store / "questions" / "benchmarks").is_dir()
|
||||||
|
assert (store / "questions" / "generated").is_dir()
|
||||||
|
assert (store / "skills" / "v1" / "search.md").read_text() == "skill"
|
||||||
|
assert (store / "prompts" / "v1" / "system.md").read_text() == "prompt"
|
||||||
|
|
||||||
|
skills_meta = json.loads(
|
||||||
|
(store / "skills" / "v1" / "meta.json").read_text()
|
||||||
|
)
|
||||||
|
assert skills_meta["version"] == "v1"
|
||||||
|
assert skills_meta["source"] == "manual"
|
||||||
|
|
||||||
|
def test_init_store_exists_raises(self, tmp_path: "Path") -> None:
|
||||||
|
store = tmp_path / "store"
|
||||||
|
store.mkdir()
|
||||||
|
with pytest.raises(FileExistsError, match="Store 已存在"):
|
||||||
|
init_store(store, tmp_path, tmp_path, tmp_path)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Seed 相关
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _make_seed_fixtures(tmp_path):
|
||||||
|
"""创建 seed 测试所需的公共 fixture。"""
|
||||||
|
store = tmp_path / "store"
|
||||||
|
store.mkdir(parents=True)
|
||||||
|
|
||||||
|
skills_dir = tmp_path / "sk"
|
||||||
|
skills_dir.mkdir()
|
||||||
|
(skills_dir / "search.md").write_text("skill")
|
||||||
|
|
||||||
|
prompts_dir = tmp_path / "pr"
|
||||||
|
prompts_dir.mkdir()
|
||||||
|
(prompts_dir / "system.md").write_text("prompt")
|
||||||
|
|
||||||
|
baseline_db = tmp_path / "base.db"
|
||||||
|
conn = sqlite3.connect(baseline_db)
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE _runs (run_id TEXT PRIMARY KEY, status TEXT)"
|
||||||
|
)
|
||||||
|
conn.execute("INSERT INTO _runs VALUES ('r1', 'done')")
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE predictions (run_id TEXT, question_id TEXT, answer TEXT)"
|
||||||
|
)
|
||||||
|
conn.execute("INSERT INTO predictions VALUES ('r1', 'q1', 'A')")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
return store, skills_dir, prompts_dir, baseline_db
|
||||||
|
|
||||||
|
|
||||||
|
class TestInitSeed:
|
||||||
|
"""init_seed 创建种子目录。"""
|
||||||
|
|
||||||
|
def test_init_seed(self, tmp_path: "Path") -> None:
|
||||||
|
store, skills_dir, prompts_dir, baseline_db = _make_seed_fixtures(tmp_path)
|
||||||
|
seed_dir = init_seed(
|
||||||
|
store,
|
||||||
|
"initial",
|
||||||
|
skills_dir,
|
||||||
|
prompts_dir,
|
||||||
|
baseline_db,
|
||||||
|
"r1",
|
||||||
|
None,
|
||||||
|
"初始种子",
|
||||||
|
)
|
||||||
|
assert seed_dir == store / "seeds" / "initial"
|
||||||
|
assert (seed_dir / "skills" / "search.md").exists()
|
||||||
|
assert (seed_dir / "prompts" / "system.md").exists()
|
||||||
|
assert (seed_dir / "baseline.db").exists()
|
||||||
|
|
||||||
|
meta = json.loads((seed_dir / "seed.json").read_text())
|
||||||
|
assert meta["baseline_run_id"] == "r1"
|
||||||
|
assert meta["parent"] is None
|
||||||
|
assert meta["description"] == "初始种子"
|
||||||
|
assert "created_at" in meta
|
||||||
|
|
||||||
|
def test_init_seed_exists_raises(self, tmp_path: "Path") -> None:
|
||||||
|
store, skills_dir, prompts_dir, baseline_db = _make_seed_fixtures(tmp_path)
|
||||||
|
init_seed(store, "dup", skills_dir, prompts_dir, baseline_db, "r1", None, "first")
|
||||||
|
with pytest.raises(FileExistsError, match="种子已存在"):
|
||||||
|
init_seed(
|
||||||
|
store, "dup", skills_dir, prompts_dir, baseline_db, "r1", None, "second"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestListSeeds:
|
||||||
|
"""list_seeds 列出所有种子。"""
|
||||||
|
|
||||||
|
def test_list_seeds(self, tmp_path: "Path") -> None:
|
||||||
|
store, skills_dir, prompts_dir, baseline_db = _make_seed_fixtures(tmp_path)
|
||||||
|
init_seed(store, "beta", skills_dir, prompts_dir, baseline_db, "r1", None, "b")
|
||||||
|
init_seed(store, "alpha", skills_dir, prompts_dir, baseline_db, "r1", None, "a")
|
||||||
|
assert list_seeds(store) == ["alpha", "beta"]
|
||||||
|
|
||||||
|
def test_list_seeds_empty(self, tmp_path: "Path") -> None:
|
||||||
|
store = tmp_path / "store"
|
||||||
|
assert list_seeds(store) == []
|
||||||
|
|
||||||
|
|
||||||
|
class TestReadSeed:
|
||||||
|
"""read_seed 读取 seed.json。"""
|
||||||
|
|
||||||
|
def test_read_seed(self, tmp_path: "Path") -> None:
|
||||||
|
store, skills_dir, prompts_dir, baseline_db = _make_seed_fixtures(tmp_path)
|
||||||
|
init_seed(store, "s1", skills_dir, prompts_dir, baseline_db, "r1", None, "desc")
|
||||||
|
meta = read_seed(store, "s1")
|
||||||
|
assert meta["baseline_run_id"] == "r1"
|
||||||
|
assert meta["description"] == "desc"
|
||||||
|
|
||||||
|
def test_read_seed_not_found(self, tmp_path: "Path") -> None:
|
||||||
|
store = tmp_path / "store"
|
||||||
|
store.mkdir()
|
||||||
|
with pytest.raises(FileNotFoundError, match="种子不存在"):
|
||||||
|
read_seed(store, "no_such")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# extract_run_db
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestExtractRunDb:
|
||||||
|
"""extract_run_db 抽取指定 run 的行并保留 PK。"""
|
||||||
|
|
||||||
|
def _make_src_db(self, path):
|
||||||
|
"""创建带 _runs + predictions 表的源 db。"""
|
||||||
|
conn = sqlite3.connect(path)
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE _runs (run_id TEXT PRIMARY KEY, status TEXT)"
|
||||||
|
)
|
||||||
|
conn.execute("INSERT INTO _runs VALUES ('r1', 'done')")
|
||||||
|
conn.execute("INSERT INTO _runs VALUES ('r2', 'done')")
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE predictions "
|
||||||
|
"(run_id TEXT, question_id TEXT, answer TEXT)"
|
||||||
|
)
|
||||||
|
conn.execute("INSERT INTO predictions VALUES ('r1', 'q1', 'A')")
|
||||||
|
conn.execute("INSERT INTO predictions VALUES ('r1', 'q2', 'B')")
|
||||||
|
conn.execute("INSERT INTO predictions VALUES ('r2', 'q1', 'C')")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def test_extract_run_db_preserves_pk(self, tmp_path: "Path") -> None:
|
||||||
|
"""原始 CREATE 保留主键约束。"""
|
||||||
|
src = tmp_path / "src.db"
|
||||||
|
dst = tmp_path / "dst.db"
|
||||||
|
self._make_src_db(src)
|
||||||
|
extract_run_db(src, dst, "r1")
|
||||||
|
|
||||||
|
conn = sqlite3.connect(dst)
|
||||||
|
# 验证 _runs 表有 PK
|
||||||
|
create_sql = conn.execute(
|
||||||
|
"SELECT sql FROM sqlite_master WHERE type='table' AND name='_runs'"
|
||||||
|
).fetchone()[0]
|
||||||
|
assert "PRIMARY KEY" in create_sql
|
||||||
|
|
||||||
|
# 验证只有 r1 的行
|
||||||
|
runs = conn.execute("SELECT * FROM _runs").fetchall()
|
||||||
|
assert len(runs) == 1
|
||||||
|
assert runs[0][0] == "r1"
|
||||||
|
|
||||||
|
preds = conn.execute("SELECT * FROM predictions").fetchall()
|
||||||
|
assert len(preds) == 2
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def test_extract_run_db_missing_table(self, tmp_path: "Path") -> None:
|
||||||
|
"""源 db 无目标表时报错。"""
|
||||||
|
src = tmp_path / "src.db"
|
||||||
|
dst = tmp_path / "dst.db"
|
||||||
|
conn = sqlite3.connect(src)
|
||||||
|
conn.execute("CREATE TABLE other (id TEXT)")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
with pytest.raises(RuntimeError, match="源 db 无表"):
|
||||||
|
extract_run_db(src, dst, "r1")
|
||||||
|
|
||||||
|
def test_extract_run_db_no_rows(self, tmp_path: "Path") -> None:
|
||||||
|
"""目标 run_id 不存在时报错。"""
|
||||||
|
src = tmp_path / "src.db"
|
||||||
|
dst = tmp_path / "dst.db"
|
||||||
|
self._make_src_db(src)
|
||||||
|
with pytest.raises(RuntimeError, match="无 run_id="):
|
||||||
|
extract_run_db(src, dst, "nonexistent")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# promote_to_seed
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _make_promote_fixtures(tmp_path):
|
||||||
|
"""创建 promote_to_seed 测试所需的 workspace + store。"""
|
||||||
|
ws = tmp_path / "ws"
|
||||||
|
ws.mkdir()
|
||||||
|
store = tmp_path / "store"
|
||||||
|
store.mkdir()
|
||||||
|
|
||||||
|
# workspace 内的 skills/prompts 版本目录
|
||||||
|
(ws / "skills" / "v2").mkdir(parents=True)
|
||||||
|
(ws / "skills" / "v2" / "skill.md").write_text("evolved")
|
||||||
|
(ws / "prompts" / "v2").mkdir(parents=True)
|
||||||
|
(ws / "prompts" / "v2" / "system.md").write_text("prompt v2")
|
||||||
|
|
||||||
|
# workspace harness.db
|
||||||
|
db_path = ws / "harness.db"
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE _runs (
|
||||||
|
run_id TEXT PRIMARY KEY,
|
||||||
|
skills_version TEXT,
|
||||||
|
prompts_version TEXT
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO _runs VALUES ('eval_001', 'v2', 'v2')"
|
||||||
|
)
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE predictions (
|
||||||
|
run_id TEXT, question_id TEXT, answer TEXT
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
conn.execute("INSERT INTO predictions VALUES ('eval_001', 'q1', 'A')")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
return ws, store
|
||||||
|
|
||||||
|
|
||||||
|
class TestPromoteToSeed:
|
||||||
|
"""promote_to_seed 固化 workspace 版本为种子。"""
|
||||||
|
|
||||||
|
def test_promote_to_seed_success(self, tmp_path: "Path") -> None:
|
||||||
|
ws, store = _make_promote_fixtures(tmp_path)
|
||||||
|
seed_dir = promote_to_seed(ws, store, "v2", "eval_001", "evolved-seed", "good")
|
||||||
|
assert seed_dir == store / "seeds" / "evolved-seed"
|
||||||
|
assert (seed_dir / "skills" / "skill.md").exists()
|
||||||
|
assert (seed_dir / "prompts" / "system.md").exists()
|
||||||
|
assert (seed_dir / "baseline.db").exists()
|
||||||
|
meta = json.loads((seed_dir / "seed.json").read_text())
|
||||||
|
assert meta["baseline_run_id"] == "eval_001"
|
||||||
|
assert meta["parent"] == "ws:v2"
|
||||||
|
|
||||||
|
def test_promote_to_seed_version_mismatch(self, tmp_path: "Path") -> None:
|
||||||
|
"""eval run 的 skills_version 与 --version 不符时报错。"""
|
||||||
|
ws, store = _make_promote_fixtures(tmp_path)
|
||||||
|
with pytest.raises(ValueError, match="版本.*不符"):
|
||||||
|
promote_to_seed(ws, store, "v3", "eval_001", "bad", "mismatch")
|
||||||
|
|
||||||
|
def test_promote_to_seed_null_version(self, tmp_path: "Path") -> None:
|
||||||
|
"""eval run 的版本为 NULL 时报错。"""
|
||||||
|
ws = tmp_path / "ws2"
|
||||||
|
ws.mkdir()
|
||||||
|
store = tmp_path / "store2"
|
||||||
|
store.mkdir()
|
||||||
|
db_path = ws / "harness.db"
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE _runs (
|
||||||
|
run_id TEXT PRIMARY KEY,
|
||||||
|
skills_version TEXT,
|
||||||
|
prompts_version TEXT
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
conn.execute("INSERT INTO _runs VALUES ('eval_null', NULL, NULL)")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
with pytest.raises(ValueError, match="NULL"):
|
||||||
|
promote_to_seed(ws, store, "v1", "eval_null", "bad", "null ver")
|
||||||
|
|
||||||
|
def test_promote_to_seed_run_not_found(self, tmp_path: "Path") -> None:
|
||||||
|
"""eval run 不存在时报错。"""
|
||||||
|
ws, store = _make_promote_fixtures(tmp_path)
|
||||||
|
with pytest.raises(ValueError, match="eval run 不存在"):
|
||||||
|
promote_to_seed(ws, store, "v1", "nonexistent", "bad", "no run")
|
||||||
|
|
||||||
|
def test_promote_cleanup_tmp_db(self, tmp_path: "Path") -> None:
|
||||||
|
"""finally 清理临时 db 文件。"""
|
||||||
|
ws, store = _make_promote_fixtures(tmp_path)
|
||||||
|
promote_to_seed(ws, store, "v2", "eval_001", "clean-test", "cleanup")
|
||||||
|
# 临时 db 应已清理
|
||||||
|
assert not (ws / "_promote_tmp.db").exists()
|
||||||
|
|
||||||
|
def test_promote_cleanup_tmp_db_on_error(self, tmp_path: "Path") -> None:
|
||||||
|
"""即使 init_seed 失败(同名种子),临时 db 也应被清理。"""
|
||||||
|
ws, store = _make_promote_fixtures(tmp_path)
|
||||||
|
promote_to_seed(ws, store, "v2", "eval_001", "first", "first time")
|
||||||
|
with pytest.raises(FileExistsError):
|
||||||
|
promote_to_seed(ws, store, "v2", "eval_001", "first", "second time")
|
||||||
|
assert not (ws / "_promote_tmp.db").exists()
|
||||||
@@ -0,0 +1,519 @@
|
|||||||
|
"""app/harness/workspace 单元测试。
|
||||||
|
|
||||||
|
覆盖 Workspace 生命周期管理、manifest 读写、
|
||||||
|
VersionedSkillStore / VersionedPromptStore Protocol 合规。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.harness.workspace import (
|
||||||
|
ResolvedPaths,
|
||||||
|
VersionedPromptStore,
|
||||||
|
VersionedSkillStore,
|
||||||
|
archive_workspace,
|
||||||
|
init_workspace,
|
||||||
|
init_workspace_from_seed,
|
||||||
|
list_video_ids,
|
||||||
|
load_manifest,
|
||||||
|
read_best,
|
||||||
|
record_run,
|
||||||
|
resolve_paths,
|
||||||
|
update_best,
|
||||||
|
update_manifest,
|
||||||
|
)
|
||||||
|
from core.evolution.protocols import PromptStore, SkillStore
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Fixtures
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def store_dir(tmp_path: Path) -> Path:
|
||||||
|
"""构建一个最小 Store 目录结构供测试使用。"""
|
||||||
|
sd = tmp_path / "store"
|
||||||
|
sd.mkdir()
|
||||||
|
|
||||||
|
# videos:两个含 tree.json 的视频目录
|
||||||
|
for vid in ("vid_A", "vid_B"):
|
||||||
|
vdir = sd / "videos" / vid
|
||||||
|
vdir.mkdir(parents=True)
|
||||||
|
(vdir / "tree.json").write_text("{}")
|
||||||
|
|
||||||
|
# skills/v1 + prompts/v1
|
||||||
|
s1 = sd / "skills" / "v1"
|
||||||
|
s1.mkdir(parents=True)
|
||||||
|
(s1 / "temporal-reasoning.md").write_text("skill content A")
|
||||||
|
(s1 / "spatial-analysis.md").write_text("skill content B")
|
||||||
|
|
||||||
|
p1 = sd / "prompts" / "v1"
|
||||||
|
p1.mkdir(parents=True)
|
||||||
|
(p1 / "system.md").write_text("system prompt v1")
|
||||||
|
(p1 / "extract.md").write_text("extract prompt v1")
|
||||||
|
|
||||||
|
# questions/benchmarks/Video-MME
|
||||||
|
q = sd / "questions" / "benchmarks" / "Video-MME"
|
||||||
|
q.mkdir(parents=True)
|
||||||
|
(q / "q1.json").write_text('{"id": "q1"}')
|
||||||
|
|
||||||
|
# seed "initial"
|
||||||
|
seed_dir = sd / "seeds" / "initial"
|
||||||
|
seed_dir.mkdir(parents=True)
|
||||||
|
shutil.copytree(s1, seed_dir / "skills")
|
||||||
|
shutil.copytree(p1, seed_dir / "prompts")
|
||||||
|
baseline_db = seed_dir / "baseline.db"
|
||||||
|
baseline_db.write_bytes(b"")
|
||||||
|
(seed_dir / "seed.json").write_text(
|
||||||
|
json.dumps({"baseline_run_id": "baseline-001", "parent": None})
|
||||||
|
)
|
||||||
|
return sd
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def workspace_dir(tmp_path: Path) -> Path:
|
||||||
|
"""返回一个不存在的 workspace 路径。"""
|
||||||
|
return tmp_path / "ws_test"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# ResolvedPaths
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolved_paths_frozen() -> None:
|
||||||
|
"""ResolvedPaths 实例应不可变。"""
|
||||||
|
rp = ResolvedPaths(
|
||||||
|
store_dir=Path("/s"),
|
||||||
|
videos_dir=Path("/v"),
|
||||||
|
questions_dir=Path("/q"),
|
||||||
|
skills_dir=Path("/sk"),
|
||||||
|
prompts_dir=Path("/p"),
|
||||||
|
workspace_dir=Path("/w"),
|
||||||
|
db_path=Path("/d"),
|
||||||
|
analyses_dir=Path("/a"),
|
||||||
|
runs_dir=Path("/r"),
|
||||||
|
)
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
rp.store_dir = Path("/other") # type: ignore[misc]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# init_workspace
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_init_workspace(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""init_workspace 应创建 manifest、拷贝权重。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
|
||||||
|
# manifest 存在
|
||||||
|
manifest = json.loads((workspace_dir / "manifest.json").read_text())
|
||||||
|
assert manifest["current"]["skills"] == "skills/v1"
|
||||||
|
assert manifest["current"]["prompts"] == "prompts/v1"
|
||||||
|
assert manifest["current"]["questions"] == "questions/benchmarks/Video-MME"
|
||||||
|
|
||||||
|
# 权重已拷入 workspace
|
||||||
|
assert (workspace_dir / "skills" / "v1" / "temporal-reasoning.md").exists()
|
||||||
|
assert (workspace_dir / "prompts" / "v1" / "system.md").exists()
|
||||||
|
|
||||||
|
# analyses, runs 目录已创建
|
||||||
|
assert (workspace_dir / "analyses").is_dir()
|
||||||
|
assert (workspace_dir / "runs").is_dir()
|
||||||
|
|
||||||
|
# 重复创建应报错
|
||||||
|
with pytest.raises(FileExistsError):
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# init_workspace_from_seed
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_init_workspace_from_seed(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""从种子初始化应拷权重 + baseline.db + 返回 baseline_run_id。"""
|
||||||
|
run_id = init_workspace_from_seed(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
seed_name="initial",
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
)
|
||||||
|
assert run_id == "baseline-001"
|
||||||
|
|
||||||
|
# 权重在 workspace
|
||||||
|
assert (workspace_dir / "skills" / "v1" / "temporal-reasoning.md").exists()
|
||||||
|
assert (workspace_dir / "prompts" / "v1" / "system.md").exists()
|
||||||
|
|
||||||
|
# baseline.db -> harness.db
|
||||||
|
assert (workspace_dir / "harness.db").exists()
|
||||||
|
|
||||||
|
# manifest 正确
|
||||||
|
manifest = json.loads((workspace_dir / "manifest.json").read_text())
|
||||||
|
assert manifest["current"]["skills"] == "skills/v1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_init_workspace_from_seed_missing_questions(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""questions ref 不存在应 fail-fast(FileNotFoundError),不创建 workspace。"""
|
||||||
|
with pytest.raises(FileNotFoundError, match="questions ref"):
|
||||||
|
init_workspace_from_seed(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
seed_name="initial",
|
||||||
|
questions="benchmarks/NONEXISTENT",
|
||||||
|
)
|
||||||
|
# workspace 不应被创建
|
||||||
|
assert not workspace_dir.exists()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# load_manifest
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_manifest(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""正常加载 manifest。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
manifest = load_manifest(workspace_dir)
|
||||||
|
assert "current" in manifest
|
||||||
|
assert "history" in manifest
|
||||||
|
assert manifest["current"]["skills"] == "skills/v1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_manifest_missing(workspace_dir: Path) -> None:
|
||||||
|
"""manifest 不存在应 FileNotFoundError。"""
|
||||||
|
workspace_dir.mkdir(parents=True)
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
load_manifest(workspace_dir)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# update_manifest
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_manifest_invalid_key(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""非法 key 应 KeyError。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
with pytest.raises(KeyError, match="无效"):
|
||||||
|
update_manifest(workspace_dir, bad_key="skills/v2")
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_manifest_valid(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""合法 key 应更新 manifest current。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
update_manifest(workspace_dir, skills="skills/v2")
|
||||||
|
manifest = load_manifest(workspace_dir)
|
||||||
|
assert manifest["current"]["skills"] == "skills/v2"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# record_run
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_record_run_idempotent(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""同 run_id 调用两次不应重复追加 history。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
run_dir1 = record_run(workspace_dir, "run_001")
|
||||||
|
run_dir2 = record_run(workspace_dir, "run_001")
|
||||||
|
|
||||||
|
# 返回路径一致
|
||||||
|
assert run_dir1 == run_dir2
|
||||||
|
|
||||||
|
# history 只有一条
|
||||||
|
manifest = load_manifest(workspace_dir)
|
||||||
|
matched = [h for h in manifest["history"] if h["run_id"] == "run_001"]
|
||||||
|
assert len(matched) == 1
|
||||||
|
|
||||||
|
# run 目录存在
|
||||||
|
assert run_dir1.is_dir()
|
||||||
|
|
||||||
|
# per-video wiki 目录存在
|
||||||
|
assert (run_dir1 / "vid_A" / "wiki").is_dir()
|
||||||
|
assert (run_dir1 / "vid_B" / "wiki").is_dir()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# update_best / read_best
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_best_independent_of_current(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""best 应独立于 current——更新 best 不影响 current。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
|
||||||
|
# 初始无 best
|
||||||
|
assert read_best(workspace_dir) is None
|
||||||
|
|
||||||
|
# 设置 best
|
||||||
|
update_best(
|
||||||
|
workspace_dir,
|
||||||
|
skills="skills/v3",
|
||||||
|
prompts="prompts/v3",
|
||||||
|
val_acc=0.85,
|
||||||
|
run_id="run_005",
|
||||||
|
epoch=3,
|
||||||
|
)
|
||||||
|
|
||||||
|
best = read_best(workspace_dir)
|
||||||
|
assert best is not None
|
||||||
|
assert best["skills"] == "skills/v3"
|
||||||
|
assert best["val_acc"] == 0.85
|
||||||
|
assert best["epoch"] == 3
|
||||||
|
|
||||||
|
# current 不受影响
|
||||||
|
manifest = load_manifest(workspace_dir)
|
||||||
|
assert manifest["current"]["skills"] == "skills/v1"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# archive_workspace
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_archive_workspace(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""归档应把 workspace 移到 .archive/ 下。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
archived = archive_workspace(workspace_dir)
|
||||||
|
|
||||||
|
# 原目录不存在
|
||||||
|
assert not workspace_dir.exists()
|
||||||
|
|
||||||
|
# 归档目录存在并包含 manifest
|
||||||
|
assert archived.is_dir()
|
||||||
|
assert (archived / "manifest.json").exists()
|
||||||
|
|
||||||
|
# 归档路径在 .archive 下
|
||||||
|
assert archived.parent.name == ".archive"
|
||||||
|
|
||||||
|
|
||||||
|
def test_archive_workspace_missing(workspace_dir: Path) -> None:
|
||||||
|
"""归档不存在的 workspace 应 FileNotFoundError。"""
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
archive_workspace(workspace_dir)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# list_video_ids
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_video_ids(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""列出 workspace 引用的所有视频 ID。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
ids = list_video_ids(workspace_dir)
|
||||||
|
assert ids == ["vid_A", "vid_B"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# resolve_paths
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_paths(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""resolve_paths 应正确解析绝对路径。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
rp = resolve_paths(workspace_dir)
|
||||||
|
|
||||||
|
# skills_dir/prompts_dir 解析到 workspace(非 store)
|
||||||
|
ws_abs = workspace_dir.resolve()
|
||||||
|
assert rp.skills_dir == ws_abs / "skills" / "v1"
|
||||||
|
assert rp.prompts_dir == ws_abs / "prompts" / "v1"
|
||||||
|
|
||||||
|
# videos/questions 解析到 store
|
||||||
|
store_abs = store_dir.resolve()
|
||||||
|
assert rp.videos_dir == store_abs / "videos"
|
||||||
|
assert rp.questions_dir == store_abs / "questions" / "benchmarks" / "Video-MME"
|
||||||
|
|
||||||
|
# db_path, analyses, runs
|
||||||
|
assert rp.db_path == ws_abs / "harness.db"
|
||||||
|
assert rp.analyses_dir == ws_abs / "analyses"
|
||||||
|
assert rp.runs_dir == ws_abs / "runs"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# VersionedSkillStore
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_versioned_skill_store_read(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""read_skill 应返回文件全文。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
skills_dir = workspace_dir / "skills" / "v1"
|
||||||
|
store = VersionedSkillStore(skills_dir)
|
||||||
|
content = store.read_skill("temporal-reasoning.md")
|
||||||
|
assert content == "skill content A"
|
||||||
|
|
||||||
|
|
||||||
|
def test_versioned_skill_store_read_missing(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""读取不存在的 skill 文件应 FileNotFoundError。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
skills_dir = workspace_dir / "skills" / "v1"
|
||||||
|
store = VersionedSkillStore(skills_dir)
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
store.read_skill("nonexistent.md")
|
||||||
|
|
||||||
|
|
||||||
|
def test_versioned_skill_store_list(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""list_skill_files 应列出所有 skill 文件名(排序)。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
skills_dir = workspace_dir / "skills" / "v1"
|
||||||
|
store = VersionedSkillStore(skills_dir)
|
||||||
|
files = store.list_skill_files()
|
||||||
|
assert sorted(files) == ["spatial-analysis.md", "temporal-reasoning.md"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# VersionedPromptStore
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_versioned_prompt_store(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""VersionedPromptStore 读写功能验证。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
prompts_dir = workspace_dir / "prompts" / "v1"
|
||||||
|
store = VersionedPromptStore(prompts_dir)
|
||||||
|
|
||||||
|
content = store.read_prompt("system.md")
|
||||||
|
assert content == "system prompt v1"
|
||||||
|
|
||||||
|
files = store.list_prompt_files()
|
||||||
|
assert sorted(files) == ["extract.md", "system.md"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_versioned_prompt_store_read_missing(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""读取不存在的 prompt 文件应 FileNotFoundError。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
prompts_dir = workspace_dir / "prompts" / "v1"
|
||||||
|
store = VersionedPromptStore(prompts_dir)
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
store.read_prompt("nonexistent.md")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Protocol 合规
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_skill_store_protocol_compliance(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""VersionedSkillStore 应满足 core/evolution/protocols.py::SkillStore Protocol。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
skills_dir = workspace_dir / "skills" / "v1"
|
||||||
|
store = VersionedSkillStore(skills_dir)
|
||||||
|
assert isinstance(store, SkillStore)
|
||||||
|
|
||||||
|
|
||||||
|
def test_prompt_store_protocol_compliance(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""VersionedPromptStore 应满足 core/evolution/protocols.py::PromptStore Protocol。"""
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
prompts_dir = workspace_dir / "prompts" / "v1"
|
||||||
|
store = VersionedPromptStore(prompts_dir)
|
||||||
|
assert isinstance(store, PromptStore)
|
||||||
@@ -0,0 +1,302 @@
|
|||||||
|
"""MonkeyOCRClient 适配器单元测试。
|
||||||
|
|
||||||
|
覆盖范围:Protocol 合规性、单帧转录、失败降级、健康检查、
|
||||||
|
多端点轮询、行去重过滤。使用 responses 库 mock HTTP。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from pathlib import Path # noqa: TC003 — pytest tmp_path fixture 类型注解需运行时
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import requests
|
||||||
|
import responses
|
||||||
|
|
||||||
|
from adapters.ocr import MonkeyOCRClient
|
||||||
|
from app.ports import OCRProvider
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Protocol 合规性
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestOCRProviderProtocol:
|
||||||
|
"""MonkeyOCRClient 必须满足 OCRProvider Protocol。"""
|
||||||
|
|
||||||
|
def test_is_runtime_checkable_instance(self) -> None:
|
||||||
|
"""MonkeyOCRClient 实例应通过 isinstance(_, OCRProvider) 检查。"""
|
||||||
|
client = MonkeyOCRClient(urls=["http://localhost:7866"])
|
||||||
|
assert isinstance(client, OCRProvider)
|
||||||
|
|
||||||
|
def test_has_transcribe_frames(self) -> None:
|
||||||
|
"""MonkeyOCRClient 必须暴露 transcribe_frames 方法。"""
|
||||||
|
assert hasattr(MonkeyOCRClient, "transcribe_frames")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 构造函数校验
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestConstructor:
|
||||||
|
"""构造函数参数校验。"""
|
||||||
|
|
||||||
|
def test_empty_urls_raises_value_error(self) -> None:
|
||||||
|
"""空端点列表应抛 ValueError(P5:不用 assert)。"""
|
||||||
|
with pytest.raises(ValueError, match="端点列表不能为空"):
|
||||||
|
MonkeyOCRClient(urls=[])
|
||||||
|
|
||||||
|
def test_trailing_slash_stripped(self) -> None:
|
||||||
|
"""URL 尾部斜杠应被去除。"""
|
||||||
|
client = MonkeyOCRClient(urls=["http://host:7866/"])
|
||||||
|
assert client._urls == ["http://host:7866"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 健康检查
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestCheckHealth:
|
||||||
|
"""check_health 预检所有端点。"""
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_healthy_endpoints(self) -> None:
|
||||||
|
"""所有端点返回 200 → 无异常。"""
|
||||||
|
url = "http://10.0.0.1:7866"
|
||||||
|
responses.add(responses.GET, f"{url}/health", status=200)
|
||||||
|
client = MonkeyOCRClient(urls=[url])
|
||||||
|
asyncio.run(client.check_health())
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_unhealthy_endpoint_raises(self) -> None:
|
||||||
|
"""端点返回 500 → RuntimeError。"""
|
||||||
|
url = "http://10.0.0.1:7866"
|
||||||
|
responses.add(responses.GET, f"{url}/health", status=500)
|
||||||
|
client = MonkeyOCRClient(urls=[url])
|
||||||
|
with pytest.raises(RuntimeError, match="健康检查失败"):
|
||||||
|
asyncio.run(client.check_health())
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_unreachable_endpoint_raises(self) -> None:
|
||||||
|
"""端点连接失败 → RuntimeError。"""
|
||||||
|
url = "http://10.0.0.1:7866"
|
||||||
|
responses.add(
|
||||||
|
responses.GET,
|
||||||
|
f"{url}/health",
|
||||||
|
body=requests.ConnectionError("refused"),
|
||||||
|
)
|
||||||
|
client = MonkeyOCRClient(urls=[url])
|
||||||
|
with pytest.raises(RuntimeError, match="端点不可达"):
|
||||||
|
asyncio.run(client.check_health())
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_multiple_endpoints_all_checked(self) -> None:
|
||||||
|
"""多端点时全部预检,任一失败即报错。"""
|
||||||
|
url_a = "http://10.0.0.1:7866"
|
||||||
|
url_b = "http://10.0.0.2:7866"
|
||||||
|
responses.add(responses.GET, f"{url_a}/health", status=200)
|
||||||
|
responses.add(responses.GET, f"{url_b}/health", status=503)
|
||||||
|
client = MonkeyOCRClient(urls=[url_a, url_b])
|
||||||
|
with pytest.raises(RuntimeError, match="健康检查失败"):
|
||||||
|
asyncio.run(client.check_health())
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 单帧转录
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestTranscribeFrames:
|
||||||
|
"""transcribe_frames 核心逻辑。"""
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_single_frame(self, tmp_path: Path) -> None:
|
||||||
|
"""单帧正常转录 → '帧1: line_a | line_b' 格式。"""
|
||||||
|
url = "http://ocr:7866"
|
||||||
|
responses.add(
|
||||||
|
responses.POST,
|
||||||
|
f"{url}/ocr/text",
|
||||||
|
json={"content": "Hello World\nOCR Test"},
|
||||||
|
status=200,
|
||||||
|
)
|
||||||
|
frame = tmp_path / "frame_001.jpg"
|
||||||
|
frame.write_bytes(b"\xff\xd8\xff\xe0fake")
|
||||||
|
client = MonkeyOCRClient(urls=[url])
|
||||||
|
result = asyncio.run(client.transcribe_frames([frame]))
|
||||||
|
assert result == "帧1: Hello World | OCR Test"
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_multiple_frames(self, tmp_path: Path) -> None:
|
||||||
|
"""多帧转录 → 每帧一行,帧号递增。"""
|
||||||
|
url = "http://ocr:7866"
|
||||||
|
responses.add(
|
||||||
|
responses.POST,
|
||||||
|
f"{url}/ocr/text",
|
||||||
|
json={"content": "Line A"},
|
||||||
|
status=200,
|
||||||
|
)
|
||||||
|
responses.add(
|
||||||
|
responses.POST,
|
||||||
|
f"{url}/ocr/text",
|
||||||
|
json={"content": "Line B"},
|
||||||
|
status=200,
|
||||||
|
)
|
||||||
|
frames = []
|
||||||
|
for i in range(2):
|
||||||
|
f = tmp_path / f"frame_{i}.jpg"
|
||||||
|
f.write_bytes(b"\xff\xd8data")
|
||||||
|
frames.append(f)
|
||||||
|
client = MonkeyOCRClient(urls=[url])
|
||||||
|
result = asyncio.run(client.transcribe_frames(frames))
|
||||||
|
assert "帧1: Line A" in result
|
||||||
|
assert "帧2: Line B" in result
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_empty_frames_returns_empty(self) -> None:
|
||||||
|
"""空帧列表 → 空串。"""
|
||||||
|
client = MonkeyOCRClient(urls=["http://ocr:7866"])
|
||||||
|
result = asyncio.run(client.transcribe_frames([]))
|
||||||
|
assert result == ""
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 失败降级
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestFailureDegradation:
|
||||||
|
"""单帧失败跳过,不影响其余帧。"""
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_single_frame_failure_returns_empty(self, tmp_path: Path) -> None:
|
||||||
|
"""唯一帧请求失败 → 返回空串(不抛异常)。"""
|
||||||
|
url = "http://ocr:7866"
|
||||||
|
responses.add(responses.POST, f"{url}/ocr/text", status=500)
|
||||||
|
frame = tmp_path / "frame.jpg"
|
||||||
|
frame.write_bytes(b"\xff\xd8data")
|
||||||
|
client = MonkeyOCRClient(urls=[url])
|
||||||
|
result = asyncio.run(client.transcribe_frames([frame]))
|
||||||
|
assert result == ""
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_partial_failure_skips_bad_frame(self, tmp_path: Path) -> None:
|
||||||
|
"""第一帧失败、第二帧成功 → 仅输出第二帧。"""
|
||||||
|
url = "http://ocr:7866"
|
||||||
|
responses.add(responses.POST, f"{url}/ocr/text", status=500)
|
||||||
|
responses.add(
|
||||||
|
responses.POST,
|
||||||
|
f"{url}/ocr/text",
|
||||||
|
json={"content": "Good"},
|
||||||
|
status=200,
|
||||||
|
)
|
||||||
|
frames = []
|
||||||
|
for i in range(2):
|
||||||
|
f = tmp_path / f"frame_{i}.jpg"
|
||||||
|
f.write_bytes(b"\xff\xd8data")
|
||||||
|
frames.append(f)
|
||||||
|
client = MonkeyOCRClient(urls=[url])
|
||||||
|
result = asyncio.run(client.transcribe_frames(frames))
|
||||||
|
# 帧1 失败被跳过,帧2 成功但输出为 "帧2: Good"
|
||||||
|
assert "帧1" not in result
|
||||||
|
assert "帧2: Good" in result
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 行去重过滤
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestLineDedup:
|
||||||
|
"""帧内行级去重与短行过滤。"""
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_duplicate_lines_removed(self, tmp_path: Path) -> None:
|
||||||
|
"""帧内重复行只保留首次出现。"""
|
||||||
|
url = "http://ocr:7866"
|
||||||
|
responses.add(
|
||||||
|
responses.POST,
|
||||||
|
f"{url}/ocr/text",
|
||||||
|
json={"content": "重复行\n重复行\n不同行"},
|
||||||
|
status=200,
|
||||||
|
)
|
||||||
|
frame = tmp_path / "frame.jpg"
|
||||||
|
frame.write_bytes(b"\xff\xd8data")
|
||||||
|
client = MonkeyOCRClient(urls=[url])
|
||||||
|
result = asyncio.run(client.transcribe_frames([frame]))
|
||||||
|
assert result == "帧1: 重复行 | 不同行"
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_single_char_lines_filtered(self, tmp_path: Path) -> None:
|
||||||
|
"""单字符行被过滤(长度 <= 1)。"""
|
||||||
|
url = "http://ocr:7866"
|
||||||
|
responses.add(
|
||||||
|
responses.POST,
|
||||||
|
f"{url}/ocr/text",
|
||||||
|
json={"content": "A\nAB\n.\nCD"},
|
||||||
|
status=200,
|
||||||
|
)
|
||||||
|
frame = tmp_path / "frame.jpg"
|
||||||
|
frame.write_bytes(b"\xff\xd8data")
|
||||||
|
client = MonkeyOCRClient(urls=[url])
|
||||||
|
result = asyncio.run(client.transcribe_frames([frame]))
|
||||||
|
# "A" 和 "." 被过滤(长度 <= 1),保留 "AB" 和 "CD"
|
||||||
|
assert result == "帧1: AB | CD"
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_empty_content_skipped(self, tmp_path: Path) -> None:
|
||||||
|
"""OCR 返回空 content → 该帧跳过。"""
|
||||||
|
url = "http://ocr:7866"
|
||||||
|
responses.add(
|
||||||
|
responses.POST,
|
||||||
|
f"{url}/ocr/text",
|
||||||
|
json={"content": ""},
|
||||||
|
status=200,
|
||||||
|
)
|
||||||
|
frame = tmp_path / "frame.jpg"
|
||||||
|
frame.write_bytes(b"\xff\xd8data")
|
||||||
|
client = MonkeyOCRClient(urls=[url])
|
||||||
|
result = asyncio.run(client.transcribe_frames([frame]))
|
||||||
|
assert result == ""
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 多端点轮询
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestRoundRobin:
|
||||||
|
"""多端点轮询行为。"""
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_round_robin_alternation(self, tmp_path: Path) -> None:
|
||||||
|
"""两端点交替使用。"""
|
||||||
|
url_a = "http://host-a:7866"
|
||||||
|
url_b = "http://host-b:7866"
|
||||||
|
# 为两个端点各注册响应
|
||||||
|
responses.add(
|
||||||
|
responses.POST,
|
||||||
|
f"{url_a}/ocr/text",
|
||||||
|
json={"content": "From A"},
|
||||||
|
status=200,
|
||||||
|
)
|
||||||
|
responses.add(
|
||||||
|
responses.POST,
|
||||||
|
f"{url_b}/ocr/text",
|
||||||
|
json={"content": "From B"},
|
||||||
|
status=200,
|
||||||
|
)
|
||||||
|
frames = []
|
||||||
|
for i in range(2):
|
||||||
|
f = tmp_path / f"frame_{i}.jpg"
|
||||||
|
f.write_bytes(b"\xff\xd8data")
|
||||||
|
frames.append(f)
|
||||||
|
client = MonkeyOCRClient(urls=[url_a, url_b])
|
||||||
|
result = asyncio.run(client.transcribe_frames(frames))
|
||||||
|
assert "帧1: From A" in result
|
||||||
|
assert "帧2: From B" in result
|
||||||
|
# 验证两个端点都被调用
|
||||||
|
called_urls = [c.request.url for c in responses.calls]
|
||||||
|
assert any(url_a in u for u in called_urls)
|
||||||
|
assert any(url_b in u for u in called_urls)
|
||||||
@@ -0,0 +1,391 @@
|
|||||||
|
"""patch.py 补丁引擎单元测试。
|
||||||
|
|
||||||
|
覆盖四大区域:
|
||||||
|
1. TestRegionBounds — appendix/momentum 边界定位与损坏态检测
|
||||||
|
2. TestAppendix — 追加/提取/替换 appendix 区
|
||||||
|
3. TestMomentum — 替换/提取 momentum 区
|
||||||
|
4. TestApplyPatch — apply_patch_with_report 的 4 种 op + 冻结区 + 异常
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.evolution.patch import (
|
||||||
|
APPENDIX_END,
|
||||||
|
APPENDIX_MAX_CHARS,
|
||||||
|
APPENDIX_START,
|
||||||
|
MOMENTUM_END,
|
||||||
|
MOMENTUM_HEADING,
|
||||||
|
MOMENTUM_MAX_CHARS,
|
||||||
|
MOMENTUM_START,
|
||||||
|
append_to_appendix,
|
||||||
|
appendix_region_bounds,
|
||||||
|
apply_patch_with_report,
|
||||||
|
extract_appendix_notes,
|
||||||
|
momentum_inner,
|
||||||
|
momentum_region_bounds,
|
||||||
|
replace_appendix_notes,
|
||||||
|
replace_momentum,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── TestRegionBounds ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestRegionBounds:
|
||||||
|
"""appendix_region_bounds / momentum_region_bounds 边界与损坏态检测。"""
|
||||||
|
|
||||||
|
# -- appendix --
|
||||||
|
|
||||||
|
def test_appendix_both_absent_returns_none(self) -> None:
|
||||||
|
assert appendix_region_bounds("no markers here") is None
|
||||||
|
|
||||||
|
def test_appendix_normal_pair(self) -> None:
|
||||||
|
text = f"head\n{APPENDIX_START}\nnotes\n{APPENDIX_END}\ntail"
|
||||||
|
start, end = appendix_region_bounds(text) # type: ignore[misc]
|
||||||
|
assert text[start:end].startswith(APPENDIX_START)
|
||||||
|
assert text[start:end].endswith(APPENDIX_END)
|
||||||
|
|
||||||
|
def test_appendix_only_start_raises(self) -> None:
|
||||||
|
with pytest.raises(ValueError, match="不配对"):
|
||||||
|
appendix_region_bounds(f"head\n{APPENDIX_START}\nno end")
|
||||||
|
|
||||||
|
def test_appendix_only_end_raises(self) -> None:
|
||||||
|
with pytest.raises(ValueError, match="不配对"):
|
||||||
|
appendix_region_bounds(f"head\n{APPENDIX_END}\nno start")
|
||||||
|
|
||||||
|
def test_appendix_repeated_start_raises(self) -> None:
|
||||||
|
text = f"{APPENDIX_START}\n{APPENDIX_START}\n{APPENDIX_END}"
|
||||||
|
with pytest.raises(ValueError, match="不配对"):
|
||||||
|
appendix_region_bounds(text)
|
||||||
|
|
||||||
|
def test_appendix_reversed_raises(self) -> None:
|
||||||
|
text = f"{APPENDIX_END}\nbody\n{APPENDIX_START}"
|
||||||
|
with pytest.raises(ValueError, match="不配对"):
|
||||||
|
appendix_region_bounds(text)
|
||||||
|
|
||||||
|
# -- momentum --
|
||||||
|
|
||||||
|
def test_momentum_both_absent_returns_none(self) -> None:
|
||||||
|
assert momentum_region_bounds("no markers here") is None
|
||||||
|
|
||||||
|
def test_momentum_normal_pair(self) -> None:
|
||||||
|
text = f"head\n{MOMENTUM_START}\nguidance\n{MOMENTUM_END}\ntail"
|
||||||
|
start, end = momentum_region_bounds(text) # type: ignore[misc]
|
||||||
|
assert text[start:end].startswith(MOMENTUM_START)
|
||||||
|
assert text[start:end].endswith(MOMENTUM_END)
|
||||||
|
|
||||||
|
def test_momentum_only_start_raises(self) -> None:
|
||||||
|
with pytest.raises(ValueError, match="不配对"):
|
||||||
|
momentum_region_bounds(f"head\n{MOMENTUM_START}\nno end")
|
||||||
|
|
||||||
|
def test_momentum_only_end_raises(self) -> None:
|
||||||
|
with pytest.raises(ValueError, match="不配对"):
|
||||||
|
momentum_region_bounds(f"head\n{MOMENTUM_END}\nno start")
|
||||||
|
|
||||||
|
def test_momentum_repeated_end_raises(self) -> None:
|
||||||
|
text = f"{MOMENTUM_START}\n{MOMENTUM_END}\n{MOMENTUM_END}"
|
||||||
|
with pytest.raises(ValueError, match="不配对"):
|
||||||
|
momentum_region_bounds(text)
|
||||||
|
|
||||||
|
def test_momentum_reversed_raises(self) -> None:
|
||||||
|
text = f"{MOMENTUM_END}\nbody\n{MOMENTUM_START}"
|
||||||
|
with pytest.raises(ValueError, match="不配对"):
|
||||||
|
momentum_region_bounds(text)
|
||||||
|
|
||||||
|
|
||||||
|
# ── TestAppendix ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestAppendix:
|
||||||
|
"""append_to_appendix / extract_appendix_notes / replace_appendix_notes。"""
|
||||||
|
|
||||||
|
def test_append_creates_region(self) -> None:
|
||||||
|
out = append_to_appendix("# Skill\nbody", ["reminder A"])
|
||||||
|
assert APPENDIX_START in out
|
||||||
|
assert APPENDIX_END in out
|
||||||
|
assert "- reminder A" in out
|
||||||
|
|
||||||
|
def test_append_accumulates(self) -> None:
|
||||||
|
step1 = append_to_appendix("# Skill\nbody", ["note 1"])
|
||||||
|
step2 = append_to_appendix(step1, ["note 2"])
|
||||||
|
assert "- note 1" in step2
|
||||||
|
assert "- note 2" in step2
|
||||||
|
|
||||||
|
def test_append_empty_notes_returns_unchanged(self) -> None:
|
||||||
|
original = "# Skill\nbody"
|
||||||
|
assert append_to_appendix(original, []) is original
|
||||||
|
|
||||||
|
def test_append_all_whitespace_notes_returns_unchanged(self) -> None:
|
||||||
|
original = "# Skill\nbody"
|
||||||
|
assert append_to_appendix(original, [" ", "\t", ""]) == original
|
||||||
|
|
||||||
|
def test_extract_notes_roundtrip(self) -> None:
|
||||||
|
content = append_to_appendix("# Skill\nbody", ["aaa", "bbb"])
|
||||||
|
notes = extract_appendix_notes(content)
|
||||||
|
assert notes == ["aaa", "bbb"]
|
||||||
|
|
||||||
|
def test_extract_notes_no_region(self) -> None:
|
||||||
|
assert extract_appendix_notes("no region") == []
|
||||||
|
|
||||||
|
def test_replace_notes_overwrites(self) -> None:
|
||||||
|
content = append_to_appendix("# Skill\nbody", ["old"])
|
||||||
|
replaced = replace_appendix_notes(content, ["new1", "new2"])
|
||||||
|
notes = extract_appendix_notes(replaced)
|
||||||
|
assert notes == ["new1", "new2"]
|
||||||
|
assert "old" not in replaced
|
||||||
|
|
||||||
|
def test_replace_notes_empty_deletes_region(self) -> None:
|
||||||
|
content = append_to_appendix("# Skill\nbody", ["old"])
|
||||||
|
replaced = replace_appendix_notes(content, [])
|
||||||
|
assert APPENDIX_START not in replaced
|
||||||
|
assert APPENDIX_END not in replaced
|
||||||
|
|
||||||
|
def test_replace_notes_no_region_creates(self) -> None:
|
||||||
|
replaced = replace_appendix_notes("# Skill\nbody", ["fresh"])
|
||||||
|
assert "- fresh" in replaced
|
||||||
|
assert APPENDIX_START in replaced
|
||||||
|
|
||||||
|
def test_replace_notes_no_region_empty_notes_unchanged(self) -> None:
|
||||||
|
original = "# Skill\nbody"
|
||||||
|
assert replace_appendix_notes(original, []) == original
|
||||||
|
|
||||||
|
def test_append_warns_on_exceeding_max_chars(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||||
|
"""appendix 区超长时 loguru warning,不截断。"""
|
||||||
|
long_note = "x" * (APPENDIX_MAX_CHARS + 100)
|
||||||
|
with caplog.at_level("WARNING"):
|
||||||
|
out = append_to_appendix("body", [long_note])
|
||||||
|
assert long_note in out # 不截断
|
||||||
|
|
||||||
|
|
||||||
|
# ── TestMomentum ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestMomentum:
|
||||||
|
"""replace_momentum / momentum_inner。"""
|
||||||
|
|
||||||
|
def test_replace_creates_region(self) -> None:
|
||||||
|
out = replace_momentum("# Skill\nbody", "focus on X")
|
||||||
|
assert MOMENTUM_START in out
|
||||||
|
assert MOMENTUM_END in out
|
||||||
|
assert "focus on X" in out
|
||||||
|
|
||||||
|
def test_replace_overwrites(self) -> None:
|
||||||
|
step1 = replace_momentum("# Skill\nbody", "old guidance")
|
||||||
|
step2 = replace_momentum(step1, "new guidance")
|
||||||
|
assert "new guidance" in step2
|
||||||
|
assert "old guidance" not in step2
|
||||||
|
|
||||||
|
def test_replace_empty_guidance_clears(self) -> None:
|
||||||
|
"""空 guidance 合法:清空旧动量、保留标题和 marker。"""
|
||||||
|
step1 = replace_momentum("# Skill\nbody", "old guidance")
|
||||||
|
step2 = replace_momentum(step1, "")
|
||||||
|
assert MOMENTUM_START in step2
|
||||||
|
assert MOMENTUM_END in step2
|
||||||
|
assert "old guidance" not in step2
|
||||||
|
assert MOMENTUM_HEADING in step2
|
||||||
|
|
||||||
|
def test_replace_rejects_marker_in_guidance(self) -> None:
|
||||||
|
with pytest.raises(ValueError, match="marker"):
|
||||||
|
replace_momentum("body", f"bad {MOMENTUM_START} injection")
|
||||||
|
|
||||||
|
def test_replace_rejects_end_marker_in_guidance(self) -> None:
|
||||||
|
with pytest.raises(ValueError, match="marker"):
|
||||||
|
replace_momentum("body", f"bad {MOMENTUM_END} injection")
|
||||||
|
|
||||||
|
def test_momentum_inner_returns_guidance(self) -> None:
|
||||||
|
content = replace_momentum("# Skill\nbody", "focus on X")
|
||||||
|
inner = momentum_inner(content)
|
||||||
|
assert inner == "focus on X"
|
||||||
|
|
||||||
|
def test_momentum_inner_no_region(self) -> None:
|
||||||
|
assert momentum_inner("no region") == ""
|
||||||
|
|
||||||
|
def test_momentum_inner_strips_heading(self) -> None:
|
||||||
|
content = replace_momentum("body", "some guidance")
|
||||||
|
inner = momentum_inner(content)
|
||||||
|
assert MOMENTUM_HEADING not in inner
|
||||||
|
assert inner == "some guidance"
|
||||||
|
|
||||||
|
def test_replace_warns_on_exceeding_max_chars(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||||
|
"""momentum 区超长时 loguru warning,不截断。"""
|
||||||
|
long_guidance = "y" * (MOMENTUM_MAX_CHARS + 100)
|
||||||
|
with caplog.at_level("WARNING"):
|
||||||
|
out = replace_momentum("body", long_guidance)
|
||||||
|
assert long_guidance in out # 不截断
|
||||||
|
|
||||||
|
def test_coexists_with_appendix(self) -> None:
|
||||||
|
"""momentum 与 appendix 独立共存。"""
|
||||||
|
content = append_to_appendix("# Skill\nbody", ["note A"])
|
||||||
|
content = replace_momentum(content, "focus on X")
|
||||||
|
assert APPENDIX_START in content
|
||||||
|
assert APPENDIX_END in content
|
||||||
|
assert MOMENTUM_START in content
|
||||||
|
assert MOMENTUM_END in content
|
||||||
|
notes = extract_appendix_notes(content)
|
||||||
|
assert notes == ["note A"]
|
||||||
|
inner = momentum_inner(content)
|
||||||
|
assert inner == "focus on X"
|
||||||
|
|
||||||
|
|
||||||
|
# ── TestApplyPatch ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestApplyPatch:
|
||||||
|
"""apply_patch_with_report 的 4 种 op + 冻结区 + 异常处理。"""
|
||||||
|
|
||||||
|
def test_append(self) -> None:
|
||||||
|
content = "# Title\n\nbody text"
|
||||||
|
edits = [{"op": "append", "target": "", "content": "new section"}]
|
||||||
|
out, reports = apply_patch_with_report(content, edits)
|
||||||
|
assert "new section" in out
|
||||||
|
assert reports[0]["status"] == "applied_append"
|
||||||
|
assert reports[0]["index"] == 1
|
||||||
|
|
||||||
|
def test_insert_after_success(self) -> None:
|
||||||
|
content = "# Title\n\nanchor line\n\nrest"
|
||||||
|
edits = [{"op": "insert_after", "target": "anchor line", "content": "inserted"}]
|
||||||
|
out, reports = apply_patch_with_report(content, edits)
|
||||||
|
assert "inserted" in out
|
||||||
|
assert reports[0]["status"] == "applied_insert_after"
|
||||||
|
|
||||||
|
def test_insert_after_missing_target_fallback(self) -> None:
|
||||||
|
content = "# Title\n\nbody"
|
||||||
|
edits = [{"op": "insert_after", "target": "nonexistent", "content": "payload"}]
|
||||||
|
out, reports = apply_patch_with_report(content, edits)
|
||||||
|
assert "payload" in out
|
||||||
|
assert reports[0]["status"] == "applied_insert_after_fallback"
|
||||||
|
|
||||||
|
def test_insert_after_protected_skip(self) -> None:
|
||||||
|
content = "# Title\n\nFROZEN BLOCK\n\nrest"
|
||||||
|
edits = [{"op": "insert_after", "target": "FROZEN BLOCK", "content": "nope"}]
|
||||||
|
out, reports = apply_patch_with_report(
|
||||||
|
content, edits, protected_spans=["FROZEN BLOCK"]
|
||||||
|
)
|
||||||
|
assert out == content
|
||||||
|
assert reports[0]["status"] == "skipped_protected"
|
||||||
|
|
||||||
|
def test_replace(self) -> None:
|
||||||
|
content = "# Title\n\nold text\n\nrest"
|
||||||
|
edits = [{"op": "replace", "target": "old text", "content": "new text"}]
|
||||||
|
out, reports = apply_patch_with_report(content, edits)
|
||||||
|
assert "new text" in out
|
||||||
|
assert "old text" not in out
|
||||||
|
assert reports[0]["status"] == "applied_replace"
|
||||||
|
|
||||||
|
def test_replace_protected_skip(self) -> None:
|
||||||
|
content = "# Title\n\nprotected\n\nrest"
|
||||||
|
edits = [{"op": "replace", "target": "protected", "content": "nope"}]
|
||||||
|
out, reports = apply_patch_with_report(
|
||||||
|
content, edits, protected_spans=["protected"]
|
||||||
|
)
|
||||||
|
assert "protected" in out
|
||||||
|
assert reports[0]["status"] == "skipped_protected"
|
||||||
|
|
||||||
|
def test_delete(self) -> None:
|
||||||
|
content = "# Title\n\nremove me\n\nrest"
|
||||||
|
edits = [{"op": "delete", "target": "remove me", "content": ""}]
|
||||||
|
out, reports = apply_patch_with_report(content, edits)
|
||||||
|
assert "remove me" not in out
|
||||||
|
assert reports[0]["status"] == "applied_delete"
|
||||||
|
|
||||||
|
def test_unknown_op(self) -> None:
|
||||||
|
edits = [{"op": "magic", "target": "x", "content": "y"}]
|
||||||
|
out, reports = apply_patch_with_report("body", edits)
|
||||||
|
assert reports[0]["status"] == "skipped_unknown_op"
|
||||||
|
|
||||||
|
def test_non_dict_edit(self) -> None:
|
||||||
|
edits = ["not a dict"] # type: ignore[list-item]
|
||||||
|
out, reports = apply_patch_with_report("body", edits)
|
||||||
|
assert reports[0]["status"] == "error"
|
||||||
|
assert "非 dict" in reports[0].get("error", "")
|
||||||
|
|
||||||
|
def test_missing_target_for_replace(self) -> None:
|
||||||
|
edits = [{"op": "replace", "target": "", "content": "payload"}]
|
||||||
|
out, reports = apply_patch_with_report("body", edits)
|
||||||
|
assert reports[0]["status"] == "skipped_missing_target"
|
||||||
|
|
||||||
|
def test_missing_target_for_delete(self) -> None:
|
||||||
|
edits = [{"op": "delete", "target": "", "content": ""}]
|
||||||
|
out, reports = apply_patch_with_report("body", edits)
|
||||||
|
assert reports[0]["status"] == "skipped_missing_target"
|
||||||
|
|
||||||
|
def test_report_index_is_1_based(self) -> None:
|
||||||
|
edits = [
|
||||||
|
{"op": "append", "target": "", "content": "a"},
|
||||||
|
{"op": "append", "target": "", "content": "b"},
|
||||||
|
{"op": "append", "target": "", "content": "c"},
|
||||||
|
]
|
||||||
|
_, reports = apply_patch_with_report("body", edits)
|
||||||
|
assert [r["index"] for r in reports] == [1, 2, 3]
|
||||||
|
|
||||||
|
def test_report_truncation(self) -> None:
|
||||||
|
long_target = "x" * 300
|
||||||
|
long_content = "y" * 300
|
||||||
|
edits = [{"op": "replace", "target": long_target, "content": long_content}]
|
||||||
|
_, reports = apply_patch_with_report(long_target, edits)
|
||||||
|
assert len(reports[0]["target"]) == 200
|
||||||
|
assert len(reports[0]["content_preview"]) == 200
|
||||||
|
|
||||||
|
def test_ranges_recalculated_each_edit(self) -> None:
|
||||||
|
"""冻结区坐标在每条 edit 后重新计算。"""
|
||||||
|
protected = "FREEZE"
|
||||||
|
content = f"AAA\n{protected}\nBBB"
|
||||||
|
edits = [
|
||||||
|
{"op": "append", "target": "", "content": "prefix text"},
|
||||||
|
{"op": "replace", "target": protected, "content": "nope"},
|
||||||
|
]
|
||||||
|
out, reports = apply_patch_with_report(
|
||||||
|
content, edits, protected_spans=[protected]
|
||||||
|
)
|
||||||
|
# append 在 FREEZE 之前插入,坐标右移后 replace 仍能检测冻结区
|
||||||
|
assert reports[1]["status"] == "skipped_protected"
|
||||||
|
|
||||||
|
def test_append_before_earliest_protected(self) -> None:
|
||||||
|
"""append 插到 start>0 的最早冻结区之前。"""
|
||||||
|
content = f"---\nfrontmatter\n---\n\nbody\n\n{APPENDIX_START}\nold\n{APPENDIX_END}"
|
||||||
|
edits = [{"op": "append", "target": "", "content": "INSERTED"}]
|
||||||
|
out, reports = apply_patch_with_report(
|
||||||
|
content, edits, protected_spans=[f"{APPENDIX_START}\nold\n{APPENDIX_END}"]
|
||||||
|
)
|
||||||
|
app_pos = out.find(APPENDIX_START)
|
||||||
|
ins_pos = out.find("INSERTED")
|
||||||
|
assert ins_pos < app_pos, "append 应在冻结区之前"
|
||||||
|
|
||||||
|
def test_payload_stripped_target_not_stripped(self) -> None:
|
||||||
|
"""target 不 strip,payload 做 strip。"""
|
||||||
|
content = " spaced target \nrest"
|
||||||
|
edits = [
|
||||||
|
{
|
||||||
|
"op": "replace",
|
||||||
|
"target": " spaced target ",
|
||||||
|
"content": " trimmed ",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
out, reports = apply_patch_with_report(content, edits)
|
||||||
|
# payload stripped → "trimmed"
|
||||||
|
assert "trimmed" in out
|
||||||
|
assert " trimmed " not in out
|
||||||
|
assert reports[0]["status"] == "applied_replace"
|
||||||
|
|
||||||
|
def test_replace_count_one(self) -> None:
|
||||||
|
"""replace 只替换第一次出现。"""
|
||||||
|
content = "dup\ndup\ndup"
|
||||||
|
edits = [{"op": "replace", "target": "dup", "content": "REPLACED"}]
|
||||||
|
out, _ = apply_patch_with_report(content, edits)
|
||||||
|
assert out.count("REPLACED") == 1
|
||||||
|
assert out.count("dup") == 2
|
||||||
|
|
||||||
|
def test_multiple_edits_sequential(self) -> None:
|
||||||
|
"""多条 edit 顺序执行。"""
|
||||||
|
content = "line1\nline2\nline3"
|
||||||
|
edits = [
|
||||||
|
{"op": "replace", "target": "line1", "content": "LINE_ONE"},
|
||||||
|
{"op": "delete", "target": "line2", "content": ""},
|
||||||
|
{"op": "append", "target": "", "content": "TAIL"},
|
||||||
|
]
|
||||||
|
out, reports = apply_patch_with_report(content, edits)
|
||||||
|
assert "LINE_ONE" in out
|
||||||
|
assert "line2" not in out
|
||||||
|
assert "TAIL" in out
|
||||||
|
assert all(r["status"].startswith("applied") for r in reports)
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
"""app/ports.py QuestionGenerator Protocol 与 app/question_gen 公开 API 测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
|
||||||
|
from app.ports import QuestionGenerator
|
||||||
|
|
||||||
|
|
||||||
|
class TestQuestionGeneratorProtocol:
|
||||||
|
def test_importable(self) -> None:
|
||||||
|
"""QuestionGenerator 可从 app.ports 导入。"""
|
||||||
|
assert QuestionGenerator is not None
|
||||||
|
|
||||||
|
def test_is_runtime_checkable(self) -> None:
|
||||||
|
"""QuestionGenerator 是 runtime_checkable Protocol。"""
|
||||||
|
assert hasattr(QuestionGenerator, "__protocol_attrs__") or hasattr(
|
||||||
|
QuestionGenerator, "__abstractmethods__"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_generate_method_exists(self) -> None:
|
||||||
|
"""Protocol 定义了 generate 方法。"""
|
||||||
|
assert hasattr(QuestionGenerator, "generate")
|
||||||
|
|
||||||
|
|
||||||
|
class TestQuestionGenPublicAPI:
|
||||||
|
def test_load_benchmark_importable_from_package(self) -> None:
|
||||||
|
"""load_benchmark 可从 app.question_gen 直接导入。"""
|
||||||
|
mod = importlib.import_module("app.question_gen")
|
||||||
|
assert hasattr(mod, "load_benchmark")
|
||||||
|
|
||||||
|
def test_stratified_sample_importable_from_package(self) -> None:
|
||||||
|
"""stratified_sample 可从 app.question_gen 直接导入。"""
|
||||||
|
mod = importlib.import_module("app.question_gen")
|
||||||
|
assert hasattr(mod, "stratified_sample")
|
||||||
|
|
||||||
|
def test_all_exports(self) -> None:
|
||||||
|
"""__all__ 包含预期的公开 API。"""
|
||||||
|
mod = importlib.import_module("app.question_gen")
|
||||||
|
assert set(mod.__all__) == {"load_benchmark", "stratified_sample"}
|
||||||
@@ -0,0 +1,361 @@
|
|||||||
|
"""app/question_gen/loader.py 单元测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.question_gen.loader import load_benchmark, stratified_sample
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def benchmark_dir(tmp_path: Path) -> Path:
|
||||||
|
"""创建包含 benchmark JSON 的临时目录。"""
|
||||||
|
data = [
|
||||||
|
{
|
||||||
|
"question_id": "1-1",
|
||||||
|
"task_type": "Action Reasoning",
|
||||||
|
"question": "What happened?",
|
||||||
|
"options": ["A. X", "B. Y", "C. Z", "D. W"],
|
||||||
|
"answer": "A",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"question_id": "1-2",
|
||||||
|
"task_type": "OCR Problems",
|
||||||
|
"question": "What text is shown?",
|
||||||
|
"options": ["A. Hello", "B. World", "C. Foo", "D. Bar"],
|
||||||
|
"answer": "B",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
(tmp_path / "video_abc.json").write_text(json.dumps(data), encoding="utf-8")
|
||||||
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoadBenchmark:
|
||||||
|
def test_loads_questions_from_json(self, benchmark_dir: Path) -> None:
|
||||||
|
questions = load_benchmark(benchmark_dir)
|
||||||
|
assert len(questions) == 2
|
||||||
|
|
||||||
|
def test_video_id_from_filename(self, benchmark_dir: Path) -> None:
|
||||||
|
questions = load_benchmark(benchmark_dir)
|
||||||
|
assert all(q.video_id == "video_abc" for q in questions)
|
||||||
|
|
||||||
|
def test_fields_mapped_correctly(self, benchmark_dir: Path) -> None:
|
||||||
|
questions = load_benchmark(benchmark_dir)
|
||||||
|
q = questions[0]
|
||||||
|
assert q.question_id == "1-1"
|
||||||
|
assert q.task_type == "Action Reasoning"
|
||||||
|
assert q.question == "What happened?"
|
||||||
|
assert q.options == ("A. X", "B. Y", "C. Z", "D. W")
|
||||||
|
assert q.answer == "A"
|
||||||
|
|
||||||
|
def test_options_is_tuple(self, benchmark_dir: Path) -> None:
|
||||||
|
questions = load_benchmark(benchmark_dir)
|
||||||
|
assert isinstance(questions[0].options, tuple)
|
||||||
|
|
||||||
|
def test_source_nodes_is_empty_tuple(self, benchmark_dir: Path) -> None:
|
||||||
|
questions = load_benchmark(benchmark_dir)
|
||||||
|
assert questions[0].source_nodes == ()
|
||||||
|
|
||||||
|
def test_difficulty_defaults_to_medium_for_legacy(self, benchmark_dir: Path) -> None:
|
||||||
|
questions = load_benchmark(benchmark_dir)
|
||||||
|
assert questions[0].difficulty == "medium"
|
||||||
|
|
||||||
|
def test_difficulty_from_json_when_present(self, tmp_path: Path) -> None:
|
||||||
|
data = [
|
||||||
|
{
|
||||||
|
"question_id": "2-1",
|
||||||
|
"task_type": "OCR Problems",
|
||||||
|
"question": "Q?",
|
||||||
|
"options": ["A. 1", "B. 2", "C. 3", "D. 4"],
|
||||||
|
"answer": "C",
|
||||||
|
"difficulty": "hard",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
(tmp_path / "vid.json").write_text(json.dumps(data), encoding="utf-8")
|
||||||
|
questions = load_benchmark(tmp_path)
|
||||||
|
assert questions[0].difficulty == "hard"
|
||||||
|
|
||||||
|
def test_empty_directory_returns_empty_list(self, tmp_path: Path) -> None:
|
||||||
|
questions = load_benchmark(tmp_path)
|
||||||
|
assert questions == []
|
||||||
|
|
||||||
|
def test_sorted_by_filename(self, tmp_path: Path) -> None:
|
||||||
|
for name in ["z_video.json", "a_video.json"]:
|
||||||
|
data = [
|
||||||
|
{
|
||||||
|
"question_id": f"{name}-1",
|
||||||
|
"task_type": "T",
|
||||||
|
"question": "Q?",
|
||||||
|
"options": ["A", "B", "C", "D"],
|
||||||
|
"answer": "A",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
(tmp_path / name).write_text(json.dumps(data), encoding="utf-8")
|
||||||
|
questions = load_benchmark(tmp_path)
|
||||||
|
assert questions[0].video_id == "a_video"
|
||||||
|
assert questions[1].video_id == "z_video"
|
||||||
|
|
||||||
|
def test_returns_generated_question_instances(self, benchmark_dir: Path) -> None:
|
||||||
|
questions = load_benchmark(benchmark_dir)
|
||||||
|
assert all(isinstance(q, GeneratedQuestion) for q in questions)
|
||||||
|
|
||||||
|
def test_loads_real_benchmark(self) -> None:
|
||||||
|
"""使用真实 benchmark 数据验证加载正确性。"""
|
||||||
|
real_dir = Path("store/questions/benchmarks/Video-MME")
|
||||||
|
if not real_dir.exists():
|
||||||
|
pytest.skip("真实 benchmark 数据不存在")
|
||||||
|
questions = load_benchmark(real_dir)
|
||||||
|
assert len(questions) > 0
|
||||||
|
for q in questions:
|
||||||
|
assert isinstance(q, GeneratedQuestion)
|
||||||
|
assert len(q.options) == 4
|
||||||
|
assert q.answer in ("A", "B", "C", "D")
|
||||||
|
|
||||||
|
def test_malformed_json_raises(self, tmp_path: Path) -> None:
|
||||||
|
"""非法 JSON 文件应抛出 json.JSONDecodeError。"""
|
||||||
|
(tmp_path / "bad.json").write_text("not valid json{{{", encoding="utf-8")
|
||||||
|
with pytest.raises(json.JSONDecodeError):
|
||||||
|
load_benchmark(tmp_path)
|
||||||
|
|
||||||
|
def test_missing_required_field_raises(self, tmp_path: Path) -> None:
|
||||||
|
"""缺少必需字段(如 question_id)应抛出 KeyError。"""
|
||||||
|
data = [{"task_type": "T", "question": "Q?", "options": ["A"], "answer": "A"}]
|
||||||
|
(tmp_path / "vid.json").write_text(json.dumps(data), encoding="utf-8")
|
||||||
|
with pytest.raises(KeyError):
|
||||||
|
load_benchmark(tmp_path)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_questions(n: int, task_type: str = "T") -> list[GeneratedQuestion]:
|
||||||
|
"""辅助函数:批量构造题目。"""
|
||||||
|
return [
|
||||||
|
GeneratedQuestion(
|
||||||
|
question_id=f"{task_type}-{i}",
|
||||||
|
video_id="v1",
|
||||||
|
task_type=task_type,
|
||||||
|
question=f"Q{i}?",
|
||||||
|
options=("A", "B", "C", "D"),
|
||||||
|
answer="A",
|
||||||
|
source_nodes=(),
|
||||||
|
difficulty="medium",
|
||||||
|
)
|
||||||
|
for i in range(n)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class TestStratifiedSample:
|
||||||
|
def test_natural_distribution(self) -> None:
|
||||||
|
questions = _make_questions(20)
|
||||||
|
result = stratified_sample(
|
||||||
|
questions=questions,
|
||||||
|
correctness={},
|
||||||
|
size=10,
|
||||||
|
correct_ratio=None,
|
||||||
|
task_types=None,
|
||||||
|
seed=42,
|
||||||
|
min_per_class=None,
|
||||||
|
)
|
||||||
|
assert len(result) == 10
|
||||||
|
|
||||||
|
def test_natural_distribution_pool_insufficient(self) -> None:
|
||||||
|
questions = _make_questions(5)
|
||||||
|
with pytest.raises(ValueError, match="自然分布采样不足"):
|
||||||
|
stratified_sample(
|
||||||
|
questions=questions,
|
||||||
|
correctness={},
|
||||||
|
size=10,
|
||||||
|
correct_ratio=None,
|
||||||
|
task_types=None,
|
||||||
|
seed=42,
|
||||||
|
min_per_class=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_ratio_stratified(self) -> None:
|
||||||
|
questions = _make_questions(20)
|
||||||
|
correctness = {f"T-{i}": i < 10 for i in range(20)}
|
||||||
|
result = stratified_sample(
|
||||||
|
questions=questions,
|
||||||
|
correctness=correctness,
|
||||||
|
size=10,
|
||||||
|
correct_ratio=0.6,
|
||||||
|
task_types=None,
|
||||||
|
seed=42,
|
||||||
|
min_per_class=None,
|
||||||
|
)
|
||||||
|
assert len(result) == 10
|
||||||
|
correct_count = sum(1 for q in result if correctness.get(q.question_id, False))
|
||||||
|
assert correct_count == 6
|
||||||
|
|
||||||
|
def test_ratio_stratified_correct_first(self) -> None:
|
||||||
|
questions = _make_questions(20)
|
||||||
|
correctness = {f"T-{i}": i < 10 for i in range(20)}
|
||||||
|
result = stratified_sample(
|
||||||
|
questions=questions,
|
||||||
|
correctness=correctness,
|
||||||
|
size=10,
|
||||||
|
correct_ratio=0.5,
|
||||||
|
task_types=None,
|
||||||
|
seed=42,
|
||||||
|
min_per_class=None,
|
||||||
|
)
|
||||||
|
n_correct = round(10 * 0.5)
|
||||||
|
for q in result[:n_correct]:
|
||||||
|
assert correctness.get(q.question_id, False) is True
|
||||||
|
for q in result[n_correct:]:
|
||||||
|
assert correctness.get(q.question_id, False) is False
|
||||||
|
|
||||||
|
def test_ratio_stratified_pool_insufficient(self) -> None:
|
||||||
|
questions = _make_questions(10)
|
||||||
|
correctness = {f"T-{i}": True for i in range(10)}
|
||||||
|
with pytest.raises(ValueError, match="分层不足"):
|
||||||
|
stratified_sample(
|
||||||
|
questions=questions,
|
||||||
|
correctness=correctness,
|
||||||
|
size=10,
|
||||||
|
correct_ratio=0.5,
|
||||||
|
task_types=None,
|
||||||
|
seed=42,
|
||||||
|
min_per_class=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_task_types_filter(self) -> None:
|
||||||
|
q_a = _make_questions(10, task_type="TypeA")
|
||||||
|
q_b = _make_questions(10, task_type="TypeB")
|
||||||
|
result = stratified_sample(
|
||||||
|
questions=q_a + q_b,
|
||||||
|
correctness={},
|
||||||
|
size=5,
|
||||||
|
correct_ratio=None,
|
||||||
|
task_types=["TypeA"],
|
||||||
|
seed=42,
|
||||||
|
min_per_class=None,
|
||||||
|
)
|
||||||
|
assert all(q.task_type == "TypeA" for q in result)
|
||||||
|
|
||||||
|
def test_unknown_correctness_treated_as_wrong(self) -> None:
|
||||||
|
questions = _make_questions(20)
|
||||||
|
correctness = {f"T-{i}": True for i in range(10)}
|
||||||
|
result = stratified_sample(
|
||||||
|
questions=questions,
|
||||||
|
correctness=correctness,
|
||||||
|
size=10,
|
||||||
|
correct_ratio=0.5,
|
||||||
|
task_types=None,
|
||||||
|
seed=42,
|
||||||
|
min_per_class=None,
|
||||||
|
)
|
||||||
|
n_correct = round(10 * 0.5)
|
||||||
|
for q in result[:n_correct]:
|
||||||
|
assert q.question_id in correctness
|
||||||
|
|
||||||
|
def test_seed_reproducibility(self) -> None:
|
||||||
|
questions = _make_questions(20)
|
||||||
|
r1 = stratified_sample(
|
||||||
|
questions=questions,
|
||||||
|
correctness={},
|
||||||
|
size=10,
|
||||||
|
correct_ratio=None,
|
||||||
|
task_types=None,
|
||||||
|
seed=123,
|
||||||
|
min_per_class=None,
|
||||||
|
)
|
||||||
|
r2 = stratified_sample(
|
||||||
|
questions=questions,
|
||||||
|
correctness={},
|
||||||
|
size=10,
|
||||||
|
correct_ratio=None,
|
||||||
|
task_types=None,
|
||||||
|
seed=123,
|
||||||
|
min_per_class=None,
|
||||||
|
)
|
||||||
|
assert [q.question_id for q in r1] == [q.question_id for q in r2]
|
||||||
|
|
||||||
|
def test_different_seeds_differ(self) -> None:
|
||||||
|
questions = _make_questions(20)
|
||||||
|
r1 = stratified_sample(
|
||||||
|
questions=questions,
|
||||||
|
correctness={},
|
||||||
|
size=10,
|
||||||
|
correct_ratio=None,
|
||||||
|
task_types=None,
|
||||||
|
seed=1,
|
||||||
|
min_per_class=None,
|
||||||
|
)
|
||||||
|
r2 = stratified_sample(
|
||||||
|
questions=questions,
|
||||||
|
correctness={},
|
||||||
|
size=10,
|
||||||
|
correct_ratio=None,
|
||||||
|
task_types=None,
|
||||||
|
seed=2,
|
||||||
|
min_per_class=None,
|
||||||
|
)
|
||||||
|
assert [q.question_id for q in r1] != [q.question_id for q in r2]
|
||||||
|
|
||||||
|
def test_min_per_class_backfill(self) -> None:
|
||||||
|
q_a = _make_questions(10, task_type="TypeA")
|
||||||
|
q_b = _make_questions(10, task_type="TypeB")
|
||||||
|
all_q = q_a + q_b
|
||||||
|
correctness = {q.question_id: True for q in q_a[:5]}
|
||||||
|
result = stratified_sample(
|
||||||
|
questions=all_q,
|
||||||
|
correctness=correctness,
|
||||||
|
size=3,
|
||||||
|
correct_ratio=None,
|
||||||
|
task_types=None,
|
||||||
|
seed=42,
|
||||||
|
min_per_class=2,
|
||||||
|
)
|
||||||
|
type_counts: dict[str, int] = {}
|
||||||
|
for q in result:
|
||||||
|
type_counts[q.task_type] = type_counts.get(q.task_type, 0) + 1
|
||||||
|
assert type_counts.get("TypeA", 0) >= 2
|
||||||
|
assert type_counts.get("TypeB", 0) >= 2
|
||||||
|
|
||||||
|
def test_min_per_class_partial_backfill(self) -> None:
|
||||||
|
q_sparse = _make_questions(1, task_type="Sparse")
|
||||||
|
q_main = _make_questions(10, task_type="Main")
|
||||||
|
result = stratified_sample(
|
||||||
|
questions=q_sparse + q_main,
|
||||||
|
correctness={},
|
||||||
|
size=5,
|
||||||
|
correct_ratio=None,
|
||||||
|
task_types=None,
|
||||||
|
seed=42,
|
||||||
|
min_per_class=3,
|
||||||
|
)
|
||||||
|
sparse_in_result = [q for q in result if q.task_type == "Sparse"]
|
||||||
|
assert len(sparse_in_result) == 1
|
||||||
|
|
||||||
|
def test_min_per_class_no_duplicates(self) -> None:
|
||||||
|
q_a = _make_questions(5, task_type="TypeA")
|
||||||
|
q_b = _make_questions(5, task_type="TypeB")
|
||||||
|
result = stratified_sample(
|
||||||
|
questions=q_a + q_b,
|
||||||
|
correctness={},
|
||||||
|
size=3,
|
||||||
|
correct_ratio=None,
|
||||||
|
task_types=None,
|
||||||
|
seed=42,
|
||||||
|
min_per_class=2,
|
||||||
|
)
|
||||||
|
ids = [q.question_id for q in result]
|
||||||
|
assert len(ids) == len(set(ids))
|
||||||
|
|
||||||
|
def test_backfill_enumerates_all_pool_types(self) -> None:
|
||||||
|
q_main = _make_questions(10, task_type="Main")
|
||||||
|
q_rare = _make_questions(3, task_type="Rare")
|
||||||
|
result = stratified_sample(
|
||||||
|
questions=q_main + q_rare,
|
||||||
|
correctness={},
|
||||||
|
size=2,
|
||||||
|
correct_ratio=None,
|
||||||
|
task_types=None,
|
||||||
|
seed=0,
|
||||||
|
min_per_class=1,
|
||||||
|
)
|
||||||
|
types_in_result = {q.task_type for q in result}
|
||||||
|
assert "Rare" in types_in_result
|
||||||
@@ -0,0 +1,187 @@
|
|||||||
|
"""修复检测器单元测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from app.tree.index import (
|
||||||
|
IndexMeta,
|
||||||
|
L1Card,
|
||||||
|
L1Node,
|
||||||
|
L2Card,
|
||||||
|
L2Node,
|
||||||
|
L3Card,
|
||||||
|
L3Node,
|
||||||
|
TreeIndex,
|
||||||
|
)
|
||||||
|
from app.tree.repair.detector import detect_issues
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def _card_l3(summary: str = "正常描述") -> L3Card:
|
||||||
|
return L3Card(summary, ["实体"], ["动作"], [], "居中", {})
|
||||||
|
|
||||||
|
|
||||||
|
def _card_l2() -> L2Card:
|
||||||
|
return L2Card("事件", [], [], [], [], "", None)
|
||||||
|
|
||||||
|
|
||||||
|
def _card_l1() -> L1Card:
|
||||||
|
return L1Card("场景", "", [], [], [], [], "")
|
||||||
|
|
||||||
|
|
||||||
|
class TestDetectIssues:
|
||||||
|
def test_healthy_tree_no_issues(self) -> None:
|
||||||
|
l3 = L3Node(id="l1_0_l2_0_l3_0", card=_card_l3(), timestamp=1.0)
|
||||||
|
l2 = L2Node(
|
||||||
|
id="l1_0_l2_0",
|
||||||
|
card=_card_l2(),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l3],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=_card_l1(),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
assert detect_issues(index) == []
|
||||||
|
|
||||||
|
def test_empty_frame_summary(self) -> None:
|
||||||
|
l3 = L3Node(id="l1_0_l2_0_l3_0", card=_card_l3(""), timestamp=1.0)
|
||||||
|
l2 = L2Node(
|
||||||
|
id="l1_0_l2_0",
|
||||||
|
card=_card_l2(),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l3],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=_card_l1(),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
issues = detect_issues(index)
|
||||||
|
assert any(i.issue_type == "empty_field" and i.node_id == "l1_0_l2_0_l3_0" for i in issues)
|
||||||
|
|
||||||
|
def test_missing_frame_file(self, tmp_path: Path) -> None:
|
||||||
|
l3 = L3Node(
|
||||||
|
id="l1_0_l2_0_l3_0",
|
||||||
|
card=_card_l3(),
|
||||||
|
timestamp=1.0,
|
||||||
|
frame_path="frames/missing.jpg",
|
||||||
|
)
|
||||||
|
l2 = L2Node(
|
||||||
|
id="l1_0_l2_0",
|
||||||
|
card=_card_l2(),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l3],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=_card_l1(),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
issues = detect_issues(index, frames_dir=tmp_path)
|
||||||
|
assert any(i.issue_type == "missing_frame" for i in issues)
|
||||||
|
|
||||||
|
def test_l2_no_children(self) -> None:
|
||||||
|
l2 = L2Node(
|
||||||
|
id="l1_0_l2_0",
|
||||||
|
card=_card_l2(),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=_card_l1(),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
issues = detect_issues(index)
|
||||||
|
assert any(i.issue_type == "no_children" and i.level == 2 for i in issues)
|
||||||
|
|
||||||
|
def test_l1_no_children(self) -> None:
|
||||||
|
l1 = L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=_card_l1(),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[],
|
||||||
|
)
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
issues = detect_issues(index)
|
||||||
|
assert any(i.issue_type == "no_children" and i.level == 1 for i in issues)
|
||||||
|
|
||||||
|
def test_empty_visible_entities(self) -> None:
|
||||||
|
"""visible_entities 为空也触发 empty_field。"""
|
||||||
|
card = L3Card("正常描述", [], ["动作"], [], "居中", {})
|
||||||
|
l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0)
|
||||||
|
l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3])
|
||||||
|
l1 = L1Node(id="l1_0", card=_card_l1(), time_range=(0.0, 10.0), children=[l2])
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
issues = detect_issues(index)
|
||||||
|
assert any(i.issue_type == "empty_field" and "visible_entities" in i.details for i in issues)
|
||||||
|
|
||||||
|
def test_empty_ongoing_actions(self) -> None:
|
||||||
|
"""ongoing_actions 为空也触发 empty_field。"""
|
||||||
|
card = L3Card("正常描述", ["实体"], [], [], "居中", {})
|
||||||
|
l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0)
|
||||||
|
l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3])
|
||||||
|
l1 = L1Node(id="l1_0", card=_card_l1(), time_range=(0.0, 10.0), children=[l2])
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
issues = detect_issues(index)
|
||||||
|
assert any(i.issue_type == "empty_field" and "ongoing_actions" in i.details for i in issues)
|
||||||
|
|
||||||
|
def test_empty_spatial_layout(self) -> None:
|
||||||
|
"""spatial_layout 为空也触发 empty_field。"""
|
||||||
|
card = L3Card("正常描述", ["实体"], ["动作"], [], "", {})
|
||||||
|
l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0)
|
||||||
|
l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3])
|
||||||
|
l1 = L1Node(id="l1_0", card=_card_l1(), time_range=(0.0, 10.0), children=[l2])
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
issues = detect_issues(index)
|
||||||
|
assert any(i.issue_type == "empty_field" and "spatial_layout" in i.details for i in issues)
|
||||||
|
|
||||||
|
def test_multiple_empty_fields_single_issue(self) -> None:
|
||||||
|
"""多个字段同时为空只产生一个 issue,details 列出所有空字段。"""
|
||||||
|
card = L3Card("", [], [], [], "", {})
|
||||||
|
l3 = L3Node(id="l1_0_l2_0_l3_0", card=card, timestamp=1.0)
|
||||||
|
l2 = L2Node(id="l1_0_l2_0", card=_card_l2(), time_range=(0.0, 10.0), children=[l3])
|
||||||
|
l1 = L1Node(id="l1_0", card=_card_l1(), time_range=(0.0, 10.0), children=[l2])
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
issues = [i for i in detect_issues(index) if i.issue_type == "empty_field"]
|
||||||
|
assert len(issues) == 1
|
||||||
|
assert "frame_summary" in issues[0].details
|
||||||
|
assert "visible_entities" in issues[0].details
|
||||||
|
|
||||||
|
def test_time_gap(self) -> None:
|
||||||
|
l3_a = L3Node(id="l1_0_l2_0_l3_0", card=_card_l3(), timestamp=1.0)
|
||||||
|
l3_b = L3Node(id="l1_0_l2_1_l3_0", card=_card_l3(), timestamp=20.0)
|
||||||
|
l2_a = L2Node(
|
||||||
|
id="l1_0_l2_0",
|
||||||
|
card=_card_l2(),
|
||||||
|
time_range=(0.0, 5.0),
|
||||||
|
children=[l3_a],
|
||||||
|
)
|
||||||
|
l2_b = L2Node(
|
||||||
|
id="l1_0_l2_1",
|
||||||
|
card=_card_l2(),
|
||||||
|
time_range=(15.0, 25.0),
|
||||||
|
children=[l3_b],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=_card_l1(),
|
||||||
|
time_range=(0.0, 25.0),
|
||||||
|
children=[l2_a, l2_b],
|
||||||
|
)
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/t.mp4", "video"), roots=[l1])
|
||||||
|
issues = detect_issues(index)
|
||||||
|
assert any(i.issue_type == "time_gap" for i in issues)
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
"""Q&A 反向补全单元测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.tree.repair.supplement import SupplementStats, deduplicate_field
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeduplicateField:
|
||||||
|
def test_removes_duplicates(self):
|
||||||
|
result = deduplicate_field(["Hello", "hello", "World", "HELLO"])
|
||||||
|
assert result == ["Hello", "World"]
|
||||||
|
|
||||||
|
def test_preserves_order(self):
|
||||||
|
result = deduplicate_field(["B", "A", "b", "C"])
|
||||||
|
assert result == ["B", "A", "C"]
|
||||||
|
|
||||||
|
def test_strips_whitespace(self):
|
||||||
|
result = deduplicate_field([" hello ", "hello"])
|
||||||
|
assert len(result) == 1
|
||||||
|
|
||||||
|
def test_empty_list(self):
|
||||||
|
assert deduplicate_field([]) == []
|
||||||
|
|
||||||
|
def test_skips_empty_strings(self):
|
||||||
|
result = deduplicate_field(["", "hello", "", "world"])
|
||||||
|
assert result == ["hello", "world"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestSupplementStats:
|
||||||
|
def test_defaults(self):
|
||||||
|
stats = SupplementStats()
|
||||||
|
assert stats.questions_analyzed == 0
|
||||||
|
assert stats.facts_injected == 0
|
||||||
|
assert stats.facts_skipped == 0
|
||||||
@@ -0,0 +1,259 @@
|
|||||||
|
"""app/search/prompt 模块的单元测试。
|
||||||
|
|
||||||
|
覆盖 PromptManager 的 __init__、build_inference_prompt、format_user_prompt、load。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.search.prompt import PromptManager
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
# ── 辅助 fixture ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def prompts_dir(tmp_path: Path) -> Path:
|
||||||
|
"""创建包含 system.md 的临时 prompt 目录。"""
|
||||||
|
system_md = tmp_path / "system.md"
|
||||||
|
system_md.write_text("你是搜索 Agent。", encoding="utf-8")
|
||||||
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def manager(prompts_dir: Path) -> PromptManager:
|
||||||
|
"""构造一个 PromptManager 实例。"""
|
||||||
|
return PromptManager(prompts_dir)
|
||||||
|
|
||||||
|
|
||||||
|
# ── __init__ ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestInit:
|
||||||
|
"""PromptManager 构造函数测试集。"""
|
||||||
|
|
||||||
|
def test_load_system_md(self, prompts_dir: Path) -> None:
|
||||||
|
"""构造时应成功加载 system.md 内容。"""
|
||||||
|
mgr = PromptManager(prompts_dir)
|
||||||
|
assert mgr._system_base == "你是搜索 Agent。"
|
||||||
|
|
||||||
|
def test_missing_system_md_raises(self, tmp_path: Path) -> None:
|
||||||
|
"""system.md 不存在时应抛出 FileNotFoundError。"""
|
||||||
|
with pytest.raises(FileNotFoundError, match="system.md"):
|
||||||
|
PromptManager(tmp_path)
|
||||||
|
|
||||||
|
|
||||||
|
# ── build_inference_prompt ────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildInferencePrompt:
|
||||||
|
"""build_inference_prompt 三种 skill_mode 的测试集。"""
|
||||||
|
|
||||||
|
def _build(
|
||||||
|
self,
|
||||||
|
manager: PromptManager,
|
||||||
|
*,
|
||||||
|
skill_mode: str = "none",
|
||||||
|
task_type: str = "qa",
|
||||||
|
always_skills_text: str = "通用策略正文",
|
||||||
|
task_skill_map: dict[str, str] | None = None,
|
||||||
|
catalog_text: str = "",
|
||||||
|
) -> str:
|
||||||
|
"""build_inference_prompt 的便捷包装。"""
|
||||||
|
if task_skill_map is None:
|
||||||
|
task_skill_map = {}
|
||||||
|
with patch(
|
||||||
|
"app.search.prompt.get_tool_descriptions",
|
||||||
|
return_value="[工具描述]",
|
||||||
|
):
|
||||||
|
return manager.build_inference_prompt(
|
||||||
|
skill_mode=skill_mode,
|
||||||
|
task_type=task_type,
|
||||||
|
always_skills_text=always_skills_text,
|
||||||
|
task_skill_map=task_skill_map,
|
||||||
|
catalog_text=catalog_text,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_auto_mode_appends_task_skill(self, manager: PromptManager) -> None:
|
||||||
|
"""auto 模式应追加 always + 匹配 task_type 的 skill 正文。"""
|
||||||
|
result = self._build(
|
||||||
|
manager,
|
||||||
|
skill_mode="auto",
|
||||||
|
task_type="qa",
|
||||||
|
task_skill_map={"qa": "QA 策略正文"},
|
||||||
|
)
|
||||||
|
assert "你是搜索 Agent。" in result
|
||||||
|
assert "[工具描述]" in result
|
||||||
|
assert "通用搜索策略" in result
|
||||||
|
assert "通用策略正文" in result
|
||||||
|
assert "当前题型搜索策略" in result
|
||||||
|
assert "QA 策略正文" in result
|
||||||
|
|
||||||
|
def test_auto_mode_falls_back_to_default(self, manager: PromptManager) -> None:
|
||||||
|
"""auto 模式中 task_type 无匹配时回退到 _default。"""
|
||||||
|
result = self._build(
|
||||||
|
manager,
|
||||||
|
skill_mode="auto",
|
||||||
|
task_type="unknown_type",
|
||||||
|
task_skill_map={"_default": "默认策略"},
|
||||||
|
)
|
||||||
|
assert "当前题型搜索策略" in result
|
||||||
|
assert "默认策略" in result
|
||||||
|
|
||||||
|
def test_auto_mode_no_match_no_default(self, manager: PromptManager) -> None:
|
||||||
|
"""auto 模式中 task_type 无匹配且无 _default 时不追加题型策略段。"""
|
||||||
|
result = self._build(
|
||||||
|
manager,
|
||||||
|
skill_mode="auto",
|
||||||
|
task_type="unknown_type",
|
||||||
|
task_skill_map={},
|
||||||
|
)
|
||||||
|
assert "当前题型搜索策略" not in result
|
||||||
|
# 但 always 仍在
|
||||||
|
assert "通用搜索策略" in result
|
||||||
|
|
||||||
|
def test_manual_mode_appends_catalog(self, manager: PromptManager) -> None:
|
||||||
|
"""manual 模式应追加 always + catalog 目录文本。"""
|
||||||
|
result = self._build(
|
||||||
|
manager,
|
||||||
|
skill_mode="manual",
|
||||||
|
catalog_text="- skill_a\n- skill_b",
|
||||||
|
)
|
||||||
|
assert "通用搜索策略" in result
|
||||||
|
assert "可用搜索策略" in result
|
||||||
|
assert "read_skill" in result
|
||||||
|
assert "- skill_a" in result
|
||||||
|
|
||||||
|
def test_manual_mode_include_read_skill(self, manager: PromptManager) -> None:
|
||||||
|
"""manual 模式应传 include_read_skill=True 给 get_tool_descriptions。"""
|
||||||
|
with patch(
|
||||||
|
"app.search.prompt.get_tool_descriptions",
|
||||||
|
return_value="[工具描述]",
|
||||||
|
) as mock_get:
|
||||||
|
manager.build_inference_prompt(
|
||||||
|
skill_mode="manual",
|
||||||
|
task_type="qa",
|
||||||
|
always_skills_text="",
|
||||||
|
task_skill_map={},
|
||||||
|
catalog_text="目录",
|
||||||
|
)
|
||||||
|
mock_get.assert_called_once_with(include_read_skill=True)
|
||||||
|
|
||||||
|
def test_auto_mode_include_read_skill_false(self, manager: PromptManager) -> None:
|
||||||
|
"""auto 模式应传 include_read_skill=False 给 get_tool_descriptions。"""
|
||||||
|
with patch(
|
||||||
|
"app.search.prompt.get_tool_descriptions",
|
||||||
|
return_value="[工具描述]",
|
||||||
|
) as mock_get:
|
||||||
|
manager.build_inference_prompt(
|
||||||
|
skill_mode="auto",
|
||||||
|
task_type="qa",
|
||||||
|
always_skills_text="",
|
||||||
|
task_skill_map={},
|
||||||
|
catalog_text="",
|
||||||
|
)
|
||||||
|
mock_get.assert_called_once_with(include_read_skill=False)
|
||||||
|
|
||||||
|
def test_none_mode_only_base_and_tools(self, manager: PromptManager) -> None:
|
||||||
|
"""none 模式应仅包含 base + 工具描述 + always(若有)。"""
|
||||||
|
result = self._build(
|
||||||
|
manager,
|
||||||
|
skill_mode="none",
|
||||||
|
always_skills_text="通用策略正文",
|
||||||
|
catalog_text="不应出现",
|
||||||
|
)
|
||||||
|
assert "你是搜索 Agent。" in result
|
||||||
|
assert "[工具描述]" in result
|
||||||
|
assert "通用搜索策略" in result
|
||||||
|
assert "可用搜索策略" not in result
|
||||||
|
assert "当前题型搜索策略" not in result
|
||||||
|
|
||||||
|
def test_none_mode_empty_always(self, manager: PromptManager) -> None:
|
||||||
|
"""none 模式下 always_skills_text 为空时不追加通用策略段。"""
|
||||||
|
result = self._build(
|
||||||
|
manager,
|
||||||
|
skill_mode="none",
|
||||||
|
always_skills_text="",
|
||||||
|
)
|
||||||
|
assert "通用搜索策略" not in result
|
||||||
|
|
||||||
|
|
||||||
|
# ── format_user_prompt ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestFormatUserPrompt:
|
||||||
|
"""format_user_prompt 的测试集。"""
|
||||||
|
|
||||||
|
def test_with_task_type(self, manager: PromptManager) -> None:
|
||||||
|
"""指定 task_type 时应在输出中插入题型行。"""
|
||||||
|
result = manager.format_user_prompt(
|
||||||
|
question="这段视频讲了什么?",
|
||||||
|
options=["A. 历史", "B. 科学", "C. 艺术", "D. 体育"],
|
||||||
|
l1_node_ids=["L1_000", "L1_001"],
|
||||||
|
task_type="qa",
|
||||||
|
)
|
||||||
|
assert "**题型**: qa" in result
|
||||||
|
assert "**问题**: 这段视频讲了什么?" in result
|
||||||
|
assert "A. 历史" in result
|
||||||
|
assert "D. 体育" in result
|
||||||
|
assert "L1_000, L1_001" in result
|
||||||
|
assert "请从以上 L1 节点开始导航" in result
|
||||||
|
|
||||||
|
def test_without_task_type(self, manager: PromptManager) -> None:
|
||||||
|
"""task_type 为 None 时输出不应包含题型行。"""
|
||||||
|
result = manager.format_user_prompt(
|
||||||
|
question="问题内容",
|
||||||
|
options=["A. 选项1", "B. 选项2"],
|
||||||
|
l1_node_ids=["L1_000"],
|
||||||
|
)
|
||||||
|
assert "**题型**" not in result
|
||||||
|
assert "**问题**: 问题内容" in result
|
||||||
|
assert "L1_000" in result
|
||||||
|
|
||||||
|
def test_options_each_on_own_line(self, manager: PromptManager) -> None:
|
||||||
|
"""每个选项应独占一行。"""
|
||||||
|
result = manager.format_user_prompt(
|
||||||
|
question="Q",
|
||||||
|
options=["A. 一", "B. 二", "C. 三"],
|
||||||
|
l1_node_ids=["L1_000"],
|
||||||
|
)
|
||||||
|
lines = result.split("\n")
|
||||||
|
# 选项应连续出现在各自行上
|
||||||
|
option_lines = [line for line in lines if line.startswith(("A.", "B.", "C."))]
|
||||||
|
assert len(option_lines) == 3
|
||||||
|
|
||||||
|
def test_single_l1_node(self, manager: PromptManager) -> None:
|
||||||
|
"""单个 L1 节点时根节点文本不含逗号。"""
|
||||||
|
result = manager.format_user_prompt(
|
||||||
|
question="Q",
|
||||||
|
options=["A. x"],
|
||||||
|
l1_node_ids=["L1_000"],
|
||||||
|
)
|
||||||
|
assert "**视频树 L1 根节点**: L1_000" in result
|
||||||
|
assert "," not in result.split("根节点**: ")[1].split("\n")[0]
|
||||||
|
|
||||||
|
|
||||||
|
# ── load ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoad:
|
||||||
|
"""load 方法的测试集。"""
|
||||||
|
|
||||||
|
def test_load_existing_file(self, prompts_dir: Path) -> None:
|
||||||
|
"""加载存在的 prompt 文件应返回其内容。"""
|
||||||
|
(prompts_dir / "diagnose_span.md").write_text("诊断模板内容", encoding="utf-8")
|
||||||
|
mgr = PromptManager(prompts_dir)
|
||||||
|
content = mgr.load("diagnose_span.md")
|
||||||
|
assert content == "诊断模板内容"
|
||||||
|
|
||||||
|
def test_load_missing_file_raises(self, manager: PromptManager) -> None:
|
||||||
|
"""加载不存在的 prompt 文件应抛出 FileNotFoundError。"""
|
||||||
|
with pytest.raises(FileNotFoundError, match="not_exist.md"):
|
||||||
|
manager.load("not_exist.md")
|
||||||
@@ -0,0 +1,214 @@
|
|||||||
|
"""app/search/skills 模块的单元测试。
|
||||||
|
|
||||||
|
覆盖 parse_frontmatter、strip_frontmatter、SkillRegistry、discover_skills。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.search.skills import (
|
||||||
|
SkillRegistry,
|
||||||
|
discover_skills,
|
||||||
|
parse_frontmatter,
|
||||||
|
strip_frontmatter,
|
||||||
|
)
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
# ── parse_frontmatter ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseFrontmatter:
|
||||||
|
"""parse_frontmatter 的测试集。"""
|
||||||
|
|
||||||
|
def test_normal(self) -> None:
|
||||||
|
"""正常 frontmatter 应解析出目标字段。"""
|
||||||
|
text = (
|
||||||
|
"---\n"
|
||||||
|
"name: my_skill\n"
|
||||||
|
'description: "A cool skill"\n'
|
||||||
|
"always: true\n"
|
||||||
|
"task_type: qa\n"
|
||||||
|
"---\n"
|
||||||
|
"Body text here.\n"
|
||||||
|
)
|
||||||
|
result = parse_frontmatter(text)
|
||||||
|
assert result == {
|
||||||
|
"name": "my_skill",
|
||||||
|
"description": "A cool skill",
|
||||||
|
"always": "true",
|
||||||
|
"task_type": "qa",
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_missing_closing_delimiter(self) -> None:
|
||||||
|
"""缺少结束 --- 应返回空字典。"""
|
||||||
|
text = "---\nname: orphan\ndescription: no end\n"
|
||||||
|
assert parse_frontmatter(text) == {}
|
||||||
|
|
||||||
|
def test_no_frontmatter(self) -> None:
|
||||||
|
"""不以 --- 开头的文本应返回空字典。"""
|
||||||
|
text = "Just plain markdown.\n"
|
||||||
|
assert parse_frontmatter(text) == {}
|
||||||
|
|
||||||
|
def test_ignores_unknown_fields(self) -> None:
|
||||||
|
"""非目标字段应被忽略。"""
|
||||||
|
text = "---\nname: s1\nauthor: someone\n---\nBody\n"
|
||||||
|
result = parse_frontmatter(text)
|
||||||
|
assert result == {"name": "s1"}
|
||||||
|
assert "author" not in result
|
||||||
|
|
||||||
|
def test_single_quoted_value(self) -> None:
|
||||||
|
"""单引号包裹的值应去除引号。"""
|
||||||
|
text = "---\nname: 'quoted_name'\n---\nBody\n"
|
||||||
|
result = parse_frontmatter(text)
|
||||||
|
assert result["name"] == "quoted_name"
|
||||||
|
|
||||||
|
|
||||||
|
# ── strip_frontmatter ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestStripFrontmatter:
|
||||||
|
"""strip_frontmatter 的测试集。"""
|
||||||
|
|
||||||
|
def test_strips_frontmatter(self) -> None:
|
||||||
|
"""正常情况下应去除 frontmatter,返回正文。"""
|
||||||
|
text = "---\nname: x\n---\nBody content.\n"
|
||||||
|
assert strip_frontmatter(text) == "Body content.\n"
|
||||||
|
|
||||||
|
def test_no_frontmatter_returns_original(self) -> None:
|
||||||
|
"""无 frontmatter 时应返回原文。"""
|
||||||
|
text = "No frontmatter here.\n"
|
||||||
|
assert strip_frontmatter(text) == text
|
||||||
|
|
||||||
|
def test_incomplete_frontmatter_returns_original(self) -> None:
|
||||||
|
"""不完整 frontmatter(缺结束符)应返回原文。"""
|
||||||
|
text = "---\nname: x\nstill going\n"
|
||||||
|
assert strip_frontmatter(text) == text
|
||||||
|
|
||||||
|
|
||||||
|
# ── SkillRegistry ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestSkillRegistry:
|
||||||
|
"""SkillRegistry 的测试集。"""
|
||||||
|
|
||||||
|
def test_read_normal(self, tmp_path: Path) -> None:
|
||||||
|
"""read 应返回去除 frontmatter 后的正文。"""
|
||||||
|
skill_file = tmp_path / "skill_a.md"
|
||||||
|
skill_file.write_text("---\nname: skill_a\n---\nSkill A body.\n", encoding="utf-8")
|
||||||
|
|
||||||
|
registry = SkillRegistry()
|
||||||
|
registry.set_paths({"skill_a": skill_file})
|
||||||
|
assert registry.read("skill_a") == "Skill A body.\n"
|
||||||
|
|
||||||
|
def test_read_unregistered_raises_key_error(self) -> None:
|
||||||
|
"""读取未注册的技能应抛出 KeyError。"""
|
||||||
|
registry = SkillRegistry()
|
||||||
|
with pytest.raises(KeyError):
|
||||||
|
registry.read("nonexistent")
|
||||||
|
|
||||||
|
|
||||||
|
# ── discover_skills ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestDiscoverSkills:
|
||||||
|
"""discover_skills 的测试集。"""
|
||||||
|
|
||||||
|
def _write_skill(self, path: Path, content: str) -> None:
|
||||||
|
"""辅助方法:写入技能文件。"""
|
||||||
|
path.write_text(content, encoding="utf-8")
|
||||||
|
|
||||||
|
def test_always_skill(self, tmp_path: Path) -> None:
|
||||||
|
"""always=true 的技能应出现在 always_text 中。"""
|
||||||
|
self._write_skill(
|
||||||
|
tmp_path / "always_skill.md",
|
||||||
|
"---\nname: a1\ndescription: always on\nalways: true\n---\nAlways body.\n",
|
||||||
|
)
|
||||||
|
always_text, task_map, catalog_text, registry = discover_skills(tmp_path)
|
||||||
|
|
||||||
|
assert "Always body." in always_text
|
||||||
|
assert task_map == {}
|
||||||
|
assert "a1" not in catalog_text
|
||||||
|
|
||||||
|
def test_task_type_skill(self, tmp_path: Path) -> None:
|
||||||
|
"""有 task_type 的非 always 技能应出现在 task_skill_map 中。"""
|
||||||
|
self._write_skill(
|
||||||
|
tmp_path / "task_skill.md",
|
||||||
|
"---\nname: t1\ndescription: task skill\ntask_type: qa\n---\nTask body.\n",
|
||||||
|
)
|
||||||
|
always_text, task_map, catalog_text, registry = discover_skills(tmp_path)
|
||||||
|
|
||||||
|
assert always_text == ""
|
||||||
|
assert task_map == {"qa": "Task body.\n"}
|
||||||
|
assert "t1" in catalog_text
|
||||||
|
|
||||||
|
def test_catalog_skill(self, tmp_path: Path) -> None:
|
||||||
|
"""普通技能应出现在 catalog_text 和 registry 中。"""
|
||||||
|
self._write_skill(
|
||||||
|
tmp_path / "cat_skill.md",
|
||||||
|
"---\nname: c1\ndescription: catalog skill\n---\nCatalog body.\n",
|
||||||
|
)
|
||||||
|
always_text, task_map, catalog_text, registry = discover_skills(tmp_path)
|
||||||
|
|
||||||
|
assert always_text == ""
|
||||||
|
assert task_map == {}
|
||||||
|
assert "**c1**" in catalog_text
|
||||||
|
assert "catalog skill" in catalog_text
|
||||||
|
assert registry.read("c1") == "Catalog body.\n"
|
||||||
|
|
||||||
|
def test_empty_directory(self, tmp_path: Path) -> None:
|
||||||
|
"""空目录应返回所有空值。"""
|
||||||
|
always_text, task_map, catalog_text, registry = discover_skills(tmp_path)
|
||||||
|
|
||||||
|
assert always_text == ""
|
||||||
|
assert task_map == {}
|
||||||
|
assert catalog_text == ""
|
||||||
|
|
||||||
|
def test_nonexistent_directory(self, tmp_path: Path) -> None:
|
||||||
|
"""不存在的目录应返回所有空值。"""
|
||||||
|
missing = tmp_path / "no_such_dir"
|
||||||
|
always_text, task_map, catalog_text, registry = discover_skills(missing)
|
||||||
|
|
||||||
|
assert always_text == ""
|
||||||
|
assert task_map == {}
|
||||||
|
assert catalog_text == ""
|
||||||
|
|
||||||
|
def test_mixed_skills(self, tmp_path: Path) -> None:
|
||||||
|
"""混合 always / task_type / catalog 技能应正确分类。"""
|
||||||
|
self._write_skill(
|
||||||
|
tmp_path / "01_always.md",
|
||||||
|
"---\nname: a1\ndescription: always\nalways: true\n---\nA body.\n",
|
||||||
|
)
|
||||||
|
self._write_skill(
|
||||||
|
tmp_path / "02_task.md",
|
||||||
|
"---\nname: t1\ndescription: task\ntask_type: summary\n---\nT body.\n",
|
||||||
|
)
|
||||||
|
self._write_skill(
|
||||||
|
tmp_path / "03_catalog.md",
|
||||||
|
"---\nname: c1\ndescription: catalog\n---\nC body.\n",
|
||||||
|
)
|
||||||
|
always_text, task_map, catalog_text, registry = discover_skills(tmp_path)
|
||||||
|
|
||||||
|
assert "A body." in always_text
|
||||||
|
assert task_map == {"summary": "T body.\n"}
|
||||||
|
assert "**t1**" in catalog_text
|
||||||
|
assert "**c1**" in catalog_text
|
||||||
|
# always 技能不应出现在 catalog 或 registry 中
|
||||||
|
assert "a1" not in catalog_text
|
||||||
|
|
||||||
|
def test_skip_no_name(self, tmp_path: Path) -> None:
|
||||||
|
"""没有 name 字段的技能文件应被跳过。"""
|
||||||
|
self._write_skill(
|
||||||
|
tmp_path / "bad.md",
|
||||||
|
"---\ndescription: no name\n---\nBody.\n",
|
||||||
|
)
|
||||||
|
always_text, task_map, catalog_text, registry = discover_skills(tmp_path)
|
||||||
|
|
||||||
|
assert always_text == ""
|
||||||
|
assert task_map == {}
|
||||||
|
assert catalog_text == ""
|
||||||
@@ -0,0 +1,544 @@
|
|||||||
|
"""app/search/summarizer 模块的单元测试。
|
||||||
|
|
||||||
|
覆盖 anchor 工具函数(纯函数)和 summarize_* 异步函数(FakeLLMProvider mock)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.search.summarizer import (
|
||||||
|
_expand_anchor_ids,
|
||||||
|
assemble_anchored_output,
|
||||||
|
check_anchors,
|
||||||
|
summarize_children,
|
||||||
|
summarize_node,
|
||||||
|
summarize_nodes_batch,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── Fake LLM 基础设施 ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class FakeLLMResponse:
|
||||||
|
"""FakeLLMProvider 返回的响应对象。"""
|
||||||
|
|
||||||
|
content: str
|
||||||
|
thinking: str = ""
|
||||||
|
model: str = "fake"
|
||||||
|
provider: str = "fake"
|
||||||
|
prompt_tokens: int = 0
|
||||||
|
completion_tokens: int = 0
|
||||||
|
latency_ms: int = 0
|
||||||
|
ttft_ms: float | None = None
|
||||||
|
max_inter_token_ms: float | None = None
|
||||||
|
cache_hit: bool = False
|
||||||
|
call_id: str = "fake-call"
|
||||||
|
|
||||||
|
|
||||||
|
class FakeLLMProvider:
|
||||||
|
"""按顺序返回预设响应的 LLMProvider 假实现。"""
|
||||||
|
|
||||||
|
def __init__(self, responses: list[str]) -> None:
|
||||||
|
self._responses = iter(responses)
|
||||||
|
|
||||||
|
async def chat(
|
||||||
|
self,
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> FakeLLMResponse:
|
||||||
|
"""返回下一个预设响应。"""
|
||||||
|
return FakeLLMResponse(content=next(self._responses))
|
||||||
|
|
||||||
|
|
||||||
|
class FailingLLMProvider:
|
||||||
|
"""始终抛出异常的 LLMProvider 假实现。"""
|
||||||
|
|
||||||
|
def __init__(self, error_msg: str = "LLM 调用失败") -> None:
|
||||||
|
self._error_msg = error_msg
|
||||||
|
|
||||||
|
async def chat(
|
||||||
|
self,
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> FakeLLMResponse:
|
||||||
|
"""始终抛出异常。"""
|
||||||
|
raise RuntimeError(self._error_msg)
|
||||||
|
|
||||||
|
|
||||||
|
class FailOnNthLLMProvider:
|
||||||
|
"""第 N 次调用抛异常,其余正常返回的 LLMProvider。"""
|
||||||
|
|
||||||
|
def __init__(self, responses: list[str], fail_on: int) -> None:
|
||||||
|
self._responses = list(responses)
|
||||||
|
self._fail_on = fail_on
|
||||||
|
self._call_count = 0
|
||||||
|
|
||||||
|
async def chat(
|
||||||
|
self,
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> FakeLLMResponse:
|
||||||
|
"""第 fail_on 次调用抛异常。"""
|
||||||
|
self._call_count += 1
|
||||||
|
if self._call_count == self._fail_on:
|
||||||
|
raise RuntimeError(f"第 {self._fail_on} 次调用失败")
|
||||||
|
idx = self._call_count - 1
|
||||||
|
if self._call_count > self._fail_on:
|
||||||
|
idx -= 1
|
||||||
|
return FakeLLMResponse(content=self._responses[idx])
|
||||||
|
|
||||||
|
|
||||||
|
# ── Prompt 文件 fixture ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def prompts_dir(tmp_path: Path) -> Path:
|
||||||
|
"""创建包含最小化 prompt 文件的临时目录。"""
|
||||||
|
prompts = {
|
||||||
|
"view_node_extract.md": "提取与问题相关的信息。",
|
||||||
|
"view_node_verify.md": "核实摘要准确性。",
|
||||||
|
"view_node_children_extract.md": "标注子节点相关性。",
|
||||||
|
"view_node_children_verify.md": "核实子节点标注。",
|
||||||
|
"search_similar_extract.md": "提取搜索结果摘要。",
|
||||||
|
"search_similar_verify.md": "核实搜索结果摘要。",
|
||||||
|
}
|
||||||
|
for filename, content in prompts.items():
|
||||||
|
(tmp_path / filename).write_text(content, encoding="utf-8")
|
||||||
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
# Part A: Anchor 工具函数测试(纯函数,无需 mock)
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
|
||||||
|
class TestExpandAnchorIds:
|
||||||
|
"""_expand_anchor_ids 展开范围语法。"""
|
||||||
|
|
||||||
|
def test_single_ids(self) -> None:
|
||||||
|
"""单个 id 不展开。"""
|
||||||
|
assert _expand_anchor_ids("s1") == ["s1"]
|
||||||
|
assert _expand_anchor_ids("c2") == ["c2"]
|
||||||
|
|
||||||
|
def test_comma_separated(self) -> None:
|
||||||
|
"""逗号分隔的多个 id。"""
|
||||||
|
assert _expand_anchor_ids("s1,c2,s5") == ["s1", "c2", "s5"]
|
||||||
|
|
||||||
|
def test_range_expansion(self) -> None:
|
||||||
|
"""范围语法 s3-s5 展开为 [s3, s4, s5]。"""
|
||||||
|
assert _expand_anchor_ids("s3-s5") == ["s3", "s4", "s5"]
|
||||||
|
|
||||||
|
def test_range_short_form(self) -> None:
|
||||||
|
"""短范围语法 s3-5(省略第二个前缀)也应展开。"""
|
||||||
|
assert _expand_anchor_ids("s3-5") == ["s3", "s4", "s5"]
|
||||||
|
|
||||||
|
def test_range_with_c_prefix(self) -> None:
|
||||||
|
"""c 前缀范围展开。"""
|
||||||
|
assert _expand_anchor_ids("c1-c3") == ["c1", "c2", "c3"]
|
||||||
|
|
||||||
|
def test_mixed_ids_and_ranges(self) -> None:
|
||||||
|
"""混合单 id 和范围。"""
|
||||||
|
result = _expand_anchor_ids("s1,c2-c4,s10")
|
||||||
|
assert result == ["s1", "c2", "c3", "c4", "s10"]
|
||||||
|
|
||||||
|
def test_cross_prefix_range_kept_as_token(self) -> None:
|
||||||
|
"""跨前缀范围(c3-s5)保留原 token。"""
|
||||||
|
result = _expand_anchor_ids("c3-s5")
|
||||||
|
assert result == ["c3-s5"]
|
||||||
|
|
||||||
|
def test_reversed_range_kept_as_token(self) -> None:
|
||||||
|
"""起点>终点的范围保留原 token。"""
|
||||||
|
result = _expand_anchor_ids("s5-s3")
|
||||||
|
assert result == ["s5-s3"]
|
||||||
|
|
||||||
|
def test_explosion_guard(self) -> None:
|
||||||
|
"""超过 50 条展开上限的范围保留原 token。"""
|
||||||
|
result = _expand_anchor_ids("s1-s100")
|
||||||
|
assert result == ["s1-s100"]
|
||||||
|
|
||||||
|
def test_fullwidth_comma(self) -> None:
|
||||||
|
"""全角逗号分隔。"""
|
||||||
|
result = _expand_anchor_ids("s1,s2")
|
||||||
|
assert result == ["s1", "s2"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestCheckAnchors:
|
||||||
|
"""check_anchors 校验行号引注。"""
|
||||||
|
|
||||||
|
def test_legal_anchors_preserved(self) -> None:
|
||||||
|
"""合法锚保留不变。"""
|
||||||
|
anchor_map = {"s1": "第一行", "s2": "第二行", "c1": "字幕一"}
|
||||||
|
summary = "[相关信息]\n- 关键发现(s1)\n- 另一个发现(c1)"
|
||||||
|
cleaned, stats = check_anchors(summary, anchor_map)
|
||||||
|
assert "(s1)" in cleaned
|
||||||
|
assert "(c1)" in cleaned
|
||||||
|
assert stats["n_illegal"] == 0
|
||||||
|
assert stats["n_assertions"] == 2
|
||||||
|
assert stats["n_anchored"] == 2
|
||||||
|
|
||||||
|
def test_illegal_anchors_removed(self) -> None:
|
||||||
|
"""非法锚被删除,断言文本保留。"""
|
||||||
|
anchor_map = {"s1": "第一行"}
|
||||||
|
summary = "[相关信息]\n- 关键发现(s99)"
|
||||||
|
cleaned, stats = check_anchors(summary, anchor_map)
|
||||||
|
assert "(s99)" not in cleaned
|
||||||
|
assert "关键发现" in cleaned
|
||||||
|
assert stats["n_illegal"] == 1
|
||||||
|
assert stats["n_assertions"] == 1
|
||||||
|
assert stats["n_anchored"] == 0
|
||||||
|
|
||||||
|
def test_range_expansion_in_check(self) -> None:
|
||||||
|
"""范围语法在 check_anchors 中展开并校验。"""
|
||||||
|
anchor_map = {"s1": "行1", "s2": "行2", "s3": "行3"}
|
||||||
|
summary = "[相关信息]\n- 发现(s1-s3)"
|
||||||
|
cleaned, stats = check_anchors(summary, anchor_map)
|
||||||
|
assert "(s1,s2,s3)" in cleaned
|
||||||
|
assert stats["n_illegal"] == 0
|
||||||
|
|
||||||
|
def test_partial_legal_range(self) -> None:
|
||||||
|
"""范围中部分合法:仅保留合法子集。"""
|
||||||
|
anchor_map = {"s1": "行1", "s2": "行2"}
|
||||||
|
summary = "[相关信息]\n- 发现(s1-s4)"
|
||||||
|
cleaned, stats = check_anchors(summary, anchor_map)
|
||||||
|
assert "(s1,s2)" in cleaned
|
||||||
|
assert stats["n_illegal"] == 2 # s3, s4 非法
|
||||||
|
|
||||||
|
def test_no_info_statement_not_counted(self) -> None:
|
||||||
|
"""声明句"未包含…相关…信息"不计入 n_assertions。"""
|
||||||
|
anchor_map = {"s1": "行1"}
|
||||||
|
summary = (
|
||||||
|
"[相关信息]\n"
|
||||||
|
"- 该节点未包含与问题直接相关的信息\n"
|
||||||
|
"- 关键发现(s1)"
|
||||||
|
)
|
||||||
|
_, stats = check_anchors(summary, anchor_map)
|
||||||
|
assert stats["n_assertions"] == 1 # 声明句不计
|
||||||
|
assert stats["n_anchored"] == 1
|
||||||
|
|
||||||
|
def test_no_relevant_section(self) -> None:
|
||||||
|
"""无 [相关信息] 段落时,只清理锚,统计为零。"""
|
||||||
|
anchor_map = {"s1": "行1"}
|
||||||
|
summary = "一些分析文本(s1)(s99)"
|
||||||
|
cleaned, stats = check_anchors(summary, anchor_map)
|
||||||
|
assert "(s1)" in cleaned
|
||||||
|
assert "(s99)" not in cleaned
|
||||||
|
assert stats["n_assertions"] == 0
|
||||||
|
assert stats["n_anchored"] == 0
|
||||||
|
assert stats["n_illegal"] == 1
|
||||||
|
|
||||||
|
def test_fullwidth_brackets(self) -> None:
|
||||||
|
"""全角括号也应被识别。"""
|
||||||
|
anchor_map = {"s1": "行1"}
|
||||||
|
summary = "[相关信息]\n- 发现(s1)"
|
||||||
|
cleaned, stats = check_anchors(summary, anchor_map)
|
||||||
|
assert stats["n_anchored"] == 1
|
||||||
|
|
||||||
|
def test_all_illegal_group_removed(self) -> None:
|
||||||
|
"""组内全非法则整组删除。"""
|
||||||
|
anchor_map = {"s1": "行1"}
|
||||||
|
summary = "[相关信息]\n- 发现(s99,s100)"
|
||||||
|
cleaned, stats = check_anchors(summary, anchor_map)
|
||||||
|
assert "(s99" not in cleaned
|
||||||
|
assert "(s100" not in cleaned
|
||||||
|
assert stats["n_illegal"] == 2
|
||||||
|
|
||||||
|
|
||||||
|
class TestAssembleAnchoredOutput:
|
||||||
|
"""assemble_anchored_output 三种模式 + 封顶逻辑。"""
|
||||||
|
|
||||||
|
def test_ids_mode_no_expansion(self) -> None:
|
||||||
|
"""ids 模式:不展开引文,原样输出。"""
|
||||||
|
anchor_map = {"s1": "行1", "s2": "行2"}
|
||||||
|
summary = "关键发现(s1)"
|
||||||
|
result, stats = assemble_anchored_output(summary, anchor_map, "ids")
|
||||||
|
assert result == summary
|
||||||
|
assert stats["n_expanded"] == 0
|
||||||
|
|
||||||
|
def test_ids_expand_mode(self) -> None:
|
||||||
|
"""ids_expand 模式:保留行号 + 附加引文段。"""
|
||||||
|
anchor_map = {"s1": "第一行内容", "s2": "第二行内容"}
|
||||||
|
summary = "关键发现(s1,s2)"
|
||||||
|
result, stats = assemble_anchored_output(
|
||||||
|
summary, anchor_map, "ids_expand"
|
||||||
|
)
|
||||||
|
assert "(s1,s2)" in result
|
||||||
|
assert "[引文]" in result
|
||||||
|
assert 's1: "第一行内容"' in result
|
||||||
|
assert 's2: "第二行内容"' in result
|
||||||
|
assert stats["n_expanded"] == 2
|
||||||
|
|
||||||
|
def test_expand_only_mode_strips_anchors(self) -> None:
|
||||||
|
"""expand_only 模式:剥除行号 + 附加引文段。"""
|
||||||
|
anchor_map = {"s1": "第一行内容"}
|
||||||
|
summary = "关键发现(s1)"
|
||||||
|
result, stats = assemble_anchored_output(
|
||||||
|
summary, anchor_map, "expand_only"
|
||||||
|
)
|
||||||
|
assert "(s1)" not in result
|
||||||
|
assert "[引文]" in result
|
||||||
|
assert 's1: "第一行内容"' in result
|
||||||
|
assert stats["n_expanded"] == 1
|
||||||
|
|
||||||
|
def test_max_items_cap(self) -> None:
|
||||||
|
"""超过 5 条引文的封顶。"""
|
||||||
|
anchor_map = {f"s{i}": f"行{i}" for i in range(1, 10)}
|
||||||
|
refs = ",".join(f"s{i}" for i in range(1, 10))
|
||||||
|
summary = f"发现({refs})"
|
||||||
|
result, stats = assemble_anchored_output(
|
||||||
|
summary, anchor_map, "ids_expand"
|
||||||
|
)
|
||||||
|
assert stats["n_expanded"] == 5
|
||||||
|
|
||||||
|
def test_max_chars_cap(self) -> None:
|
||||||
|
"""总字符超过 800 时截断。"""
|
||||||
|
anchor_map = {
|
||||||
|
f"s{i}": "A" * 300 for i in range(1, 6)
|
||||||
|
}
|
||||||
|
refs = ",".join(f"s{i}" for i in range(1, 6))
|
||||||
|
summary = f"发现({refs})"
|
||||||
|
result, stats = assemble_anchored_output(
|
||||||
|
summary, anchor_map, "ids_expand"
|
||||||
|
)
|
||||||
|
# 300 字符原文 + 前缀 ≈ 310+ 每条,800 / 310 ≈ 2 条
|
||||||
|
assert stats["n_expanded"] < 5
|
||||||
|
|
||||||
|
def test_line_cap_truncation(self) -> None:
|
||||||
|
"""单行超 200 字符截断并标记 n_trunc。"""
|
||||||
|
anchor_map = {"s1": "A" * 250}
|
||||||
|
summary = "发现(s1)"
|
||||||
|
result, stats = assemble_anchored_output(
|
||||||
|
summary, anchor_map, "ids_expand"
|
||||||
|
)
|
||||||
|
assert stats["n_trunc"] == 1
|
||||||
|
assert "…" in result
|
||||||
|
|
||||||
|
def test_invalid_mode_raises(self) -> None:
|
||||||
|
"""无效模式应抛出 AssertionError。"""
|
||||||
|
with pytest.raises(AssertionError, match="未知装配形态"):
|
||||||
|
assemble_anchored_output("text", {}, "bad_mode")
|
||||||
|
|
||||||
|
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
# Part B: summarize_* 异步函数测试(FakeLLMProvider mock)
|
||||||
|
# ══════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
|
||||||
|
class TestSummarizeNode:
|
||||||
|
"""summarize_node 两轮摘要。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_normal_two_round(self, prompts_dir: Path) -> None:
|
||||||
|
"""正常两轮:提取 + 核实。"""
|
||||||
|
llm = FakeLLMProvider(["提取结果摘要", "核实通过"])
|
||||||
|
result = await summarize_node(
|
||||||
|
llm,
|
||||||
|
"视频片段内容",
|
||||||
|
"这个视频讲了什么?",
|
||||||
|
prompts_dir,
|
||||||
|
anchor_map=None,
|
||||||
|
assemble_mode="ids",
|
||||||
|
)
|
||||||
|
assert "[内容摘要] 提取结果摘要" in result
|
||||||
|
assert "[核实] 核实通过" in result
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_extract_failure(self, prompts_dir: Path) -> None:
|
||||||
|
"""提取轮失败返回错误信息。"""
|
||||||
|
llm = FailingLLMProvider("网络超时")
|
||||||
|
result = await summarize_node(
|
||||||
|
llm,
|
||||||
|
"视频片段内容",
|
||||||
|
"问题",
|
||||||
|
prompts_dir,
|
||||||
|
anchor_map=None,
|
||||||
|
assemble_mode="ids",
|
||||||
|
)
|
||||||
|
assert "[摘要错误]" in result
|
||||||
|
assert "网络超时" in result
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_verify_failure_degrades(self, prompts_dir: Path) -> None:
|
||||||
|
"""核实轮失败降级为"跳过"。"""
|
||||||
|
llm = FailOnNthLLMProvider(["提取结果"], fail_on=2)
|
||||||
|
result = await summarize_node(
|
||||||
|
llm,
|
||||||
|
"视频片段内容",
|
||||||
|
"问题",
|
||||||
|
prompts_dir,
|
||||||
|
anchor_map=None,
|
||||||
|
assemble_mode="ids",
|
||||||
|
)
|
||||||
|
assert "[内容摘要] 提取结果" in result
|
||||||
|
assert "跳过(调用失败)" in result
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_anchor_mode(self, prompts_dir: Path) -> None:
|
||||||
|
"""锚模式:check_anchors + assemble。"""
|
||||||
|
anchor_map = {"s1": "第一行", "s2": "第二行"}
|
||||||
|
llm = FakeLLMProvider([
|
||||||
|
"[相关信息]\n- 关键发现(s1)\n- 补充(s2)",
|
||||||
|
"核实通过",
|
||||||
|
])
|
||||||
|
result = await summarize_node(
|
||||||
|
llm,
|
||||||
|
"带行号的内容",
|
||||||
|
"问题",
|
||||||
|
prompts_dir,
|
||||||
|
anchor_map=anchor_map,
|
||||||
|
assemble_mode="ids_expand",
|
||||||
|
)
|
||||||
|
assert "[内容摘要]" in result
|
||||||
|
assert "[核实] 核实通过" in result
|
||||||
|
assert "[引文]" in result
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_anchor_mode_with_stats_sink(self, prompts_dir: Path) -> None:
|
||||||
|
"""锚模式 stats_sink 回调接收完整统计。"""
|
||||||
|
anchor_map = {"s1": "第一行"}
|
||||||
|
collected: list[dict] = []
|
||||||
|
llm = FakeLLMProvider([
|
||||||
|
"[相关信息]\n- 关键发现(s1)",
|
||||||
|
"核实通过",
|
||||||
|
])
|
||||||
|
await summarize_node(
|
||||||
|
llm,
|
||||||
|
"内容",
|
||||||
|
"问题",
|
||||||
|
prompts_dir,
|
||||||
|
anchor_map=anchor_map,
|
||||||
|
assemble_mode="ids_expand",
|
||||||
|
stats_sink=collected.append,
|
||||||
|
)
|
||||||
|
assert len(collected) == 1
|
||||||
|
s = collected[0]
|
||||||
|
assert "n_assertions" in s
|
||||||
|
assert "n_anchored" in s
|
||||||
|
assert "n_expanded" in s
|
||||||
|
assert "output_chars" in s
|
||||||
|
assert "pre_assembly" in s
|
||||||
|
assert "anchor_map" in s
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_session_id_forwarded(self, prompts_dir: Path) -> None:
|
||||||
|
"""session_id 和 parent_call_id 应透传给 LLM。"""
|
||||||
|
received_kwargs: list[dict] = []
|
||||||
|
|
||||||
|
class CaptureLLM:
|
||||||
|
"""捕获 kwargs 的 LLM。"""
|
||||||
|
|
||||||
|
async def chat(self, messages: list, **kwargs: Any) -> FakeLLMResponse:
|
||||||
|
received_kwargs.append(kwargs)
|
||||||
|
return FakeLLMResponse(content="ok")
|
||||||
|
|
||||||
|
llm = CaptureLLM()
|
||||||
|
await summarize_node(
|
||||||
|
llm,
|
||||||
|
"内容",
|
||||||
|
"问题",
|
||||||
|
prompts_dir,
|
||||||
|
anchor_map=None,
|
||||||
|
assemble_mode="ids",
|
||||||
|
session_id="sess-1",
|
||||||
|
parent_call_id="call-0",
|
||||||
|
)
|
||||||
|
assert len(received_kwargs) == 2
|
||||||
|
for kw in received_kwargs:
|
||||||
|
assert kw["session_id"] == "sess-1"
|
||||||
|
assert kw["parent_call_id"] == "call-0"
|
||||||
|
|
||||||
|
|
||||||
|
class TestSummarizeChildren:
|
||||||
|
"""summarize_children 子节点标注。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_normal(self, prompts_dir: Path) -> None:
|
||||||
|
"""正常两轮标注。"""
|
||||||
|
children_info = [
|
||||||
|
{"id": "n1", "time_range": (0.0, 30.0), "summary": "开头"},
|
||||||
|
{"id": "n2", "time_range": (30.0, 60.0), "summary": "中间"},
|
||||||
|
]
|
||||||
|
llm = FakeLLMProvider(["相关性标注结果", "核实通过"])
|
||||||
|
result = await summarize_children(
|
||||||
|
llm, children_info, "问题", prompts_dir
|
||||||
|
)
|
||||||
|
assert "相关性标注结果" in result
|
||||||
|
assert "[核实] 核实通过" in result
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_extract_failure_fallback(self, prompts_dir: Path) -> None:
|
||||||
|
"""提取失败回退到原始列表。"""
|
||||||
|
children_info = [
|
||||||
|
{"id": "n1", "time_range": (0.0, 30.0), "summary": "开头"},
|
||||||
|
]
|
||||||
|
llm = FailingLLMProvider("网络错误")
|
||||||
|
result = await summarize_children(
|
||||||
|
llm, children_info, "问题", prompts_dir
|
||||||
|
)
|
||||||
|
assert "n1" in result
|
||||||
|
assert "0-30s" in result
|
||||||
|
assert "开头" in result
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_verify_failure_returns_extract_only(
|
||||||
|
self, prompts_dir: Path
|
||||||
|
) -> None:
|
||||||
|
"""核实轮失败仍返回提取结果。"""
|
||||||
|
children_info = [
|
||||||
|
{"id": "n1", "time_range": (0.0, 30.0), "summary": "开头"},
|
||||||
|
]
|
||||||
|
llm = FailOnNthLLMProvider(["标注结果"], fail_on=2)
|
||||||
|
result = await summarize_children(
|
||||||
|
llm, children_info, "问题", prompts_dir
|
||||||
|
)
|
||||||
|
assert "标注结果" in result
|
||||||
|
|
||||||
|
|
||||||
|
class TestSummarizeNodesBatch:
|
||||||
|
"""summarize_nodes_batch 并发多节点。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_batch_normal(self, prompts_dir: Path) -> None:
|
||||||
|
"""并发三个节点,结果顺序与输入一致。"""
|
||||||
|
# 每个节点需要 2 轮 LLM 调用(提取 + 核实)
|
||||||
|
llm = FakeLLMProvider([
|
||||||
|
"摘要A", "核实A",
|
||||||
|
"摘要B", "核实B",
|
||||||
|
"摘要C", "核实C",
|
||||||
|
])
|
||||||
|
items = [
|
||||||
|
("n1", "内容1", "extra1"),
|
||||||
|
("n2", "内容2", "extra2"),
|
||||||
|
("n3", "内容3", "extra3"),
|
||||||
|
]
|
||||||
|
results = await summarize_nodes_batch(
|
||||||
|
llm, items, "问题", prompts_dir
|
||||||
|
)
|
||||||
|
assert len(results) == 3
|
||||||
|
assert results[0][0] == "n1"
|
||||||
|
assert results[1][0] == "n2"
|
||||||
|
assert results[2][0] == "n3"
|
||||||
|
assert "[内容摘要]" in results[0][1]
|
||||||
|
assert "[内容摘要]" in results[1][1]
|
||||||
|
assert "[内容摘要]" in results[2][1]
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_batch_empty(self, prompts_dir: Path) -> None:
|
||||||
|
"""空列表返回空结果。"""
|
||||||
|
llm = FakeLLMProvider([])
|
||||||
|
results = await summarize_nodes_batch(
|
||||||
|
llm, [], "问题", prompts_dir
|
||||||
|
)
|
||||||
|
assert results == []
|
||||||
@@ -0,0 +1,446 @@
|
|||||||
|
"""SearchToolDispatcher 与 get_tool_descriptions 单元测试。
|
||||||
|
|
||||||
|
验证工具描述生成和五种工具的 dispatch 路由:
|
||||||
|
view_node、search_similar、observe_frame、submit_answer、read_skill,
|
||||||
|
以及未知工具 ValueError 和节点不存在错误文本。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.search.skills import SkillRegistry
|
||||||
|
from app.search.tools import SearchToolDispatcher, get_tool_descriptions
|
||||||
|
from app.tree.environment import TreeEnvironment
|
||||||
|
from app.tree.index import (
|
||||||
|
IndexMeta,
|
||||||
|
L1Card,
|
||||||
|
L1Node,
|
||||||
|
L2Card,
|
||||||
|
L2Node,
|
||||||
|
L3Card,
|
||||||
|
L3Node,
|
||||||
|
TreeIndex,
|
||||||
|
)
|
||||||
|
from core.types import LLMResponse
|
||||||
|
|
||||||
|
# ── 假实现 ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _make_llm_response(content: str = "fake summary") -> LLMResponse:
|
||||||
|
"""构造固定的 LLMResponse 实例。"""
|
||||||
|
return LLMResponse(
|
||||||
|
content=content,
|
||||||
|
thinking="",
|
||||||
|
model="fake-model",
|
||||||
|
provider="fake",
|
||||||
|
prompt_tokens=10,
|
||||||
|
completion_tokens=5,
|
||||||
|
latency_ms=50,
|
||||||
|
ttft_ms=None,
|
||||||
|
max_inter_token_ms=None,
|
||||||
|
cache_hit=False,
|
||||||
|
call_id="fake-call-id",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FakeLLM:
|
||||||
|
"""最小 LLMProvider 假实现。"""
|
||||||
|
|
||||||
|
async def chat(
|
||||||
|
self,
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> LLMResponse:
|
||||||
|
"""返回固定摘要内容。"""
|
||||||
|
return _make_llm_response("fake summary")
|
||||||
|
|
||||||
|
|
||||||
|
class FakeVLM:
|
||||||
|
"""最小 VLMProvider 假实现。"""
|
||||||
|
|
||||||
|
async def chat_with_images(
|
||||||
|
self,
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
images: list[str | Path],
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> LLMResponse:
|
||||||
|
"""返回固定视觉观察内容。"""
|
||||||
|
return _make_llm_response("fake visual observation")
|
||||||
|
|
||||||
|
|
||||||
|
class FakeOCR:
|
||||||
|
"""最小 OCRProvider 假实现。"""
|
||||||
|
|
||||||
|
async def transcribe_frames(self, frame_paths: list[Path]) -> str:
|
||||||
|
"""返回固定 OCR 文本。"""
|
||||||
|
return "OCR text"
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_embed_fn(texts: str | list[str]) -> np.ndarray:
|
||||||
|
"""返回固定维度的 L2 归一化嵌入向量。"""
|
||||||
|
if isinstance(texts, str):
|
||||||
|
vec = np.ones((1, 4), dtype=np.float32)
|
||||||
|
else:
|
||||||
|
vec = np.ones((len(texts), 4), dtype=np.float32)
|
||||||
|
norms = np.linalg.norm(vec, axis=1, keepdims=True)
|
||||||
|
return vec / norms
|
||||||
|
|
||||||
|
|
||||||
|
# ── Fixtures ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _make_test_tree() -> TreeIndex:
|
||||||
|
"""构建包含 L1→L2→L3 的最小测试树。"""
|
||||||
|
l3 = L3Node(
|
||||||
|
id="vid_L1_000_L2_000_L3_000",
|
||||||
|
card=L3Card(
|
||||||
|
frame_summary="test frame summary",
|
||||||
|
visible_entities=["person"],
|
||||||
|
ongoing_actions=["walking"],
|
||||||
|
visible_text=[],
|
||||||
|
spatial_layout="center",
|
||||||
|
visual_attributes={},
|
||||||
|
),
|
||||||
|
timestamp=10.0,
|
||||||
|
frame_path="frames/L1_000_L2_000_L3_000.jpg",
|
||||||
|
subtitle="test subtitle text",
|
||||||
|
)
|
||||||
|
l2 = L2Node(
|
||||||
|
id="vid_L1_000_L2_000",
|
||||||
|
card=L2Card(
|
||||||
|
event_description="test event description",
|
||||||
|
entities=["person"],
|
||||||
|
actions=["walking"],
|
||||||
|
action_subjects=["person"],
|
||||||
|
visible_text=[],
|
||||||
|
spatial_relations="none",
|
||||||
|
state_changes=None,
|
||||||
|
),
|
||||||
|
time_range=(5.0, 15.0),
|
||||||
|
children=[l3],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="vid_L1_000",
|
||||||
|
card=L1Card(
|
||||||
|
scene_summary="test scene summary",
|
||||||
|
main_setting="outdoor",
|
||||||
|
key_entities=["person"],
|
||||||
|
main_actions=["walking"],
|
||||||
|
topic_keywords=["outdoor"],
|
||||||
|
visible_text=[],
|
||||||
|
temporal_flow="linear",
|
||||||
|
),
|
||||||
|
time_range=(0.0, 30.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
return TreeIndex(
|
||||||
|
metadata=IndexMeta(source_path="test.mp4", modality="video"),
|
||||||
|
roots=[l1],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def env() -> TreeEnvironment:
|
||||||
|
"""带最小树的 TreeEnvironment。"""
|
||||||
|
return TreeEnvironment(_make_test_tree())
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def prompts_dir(tmp_path: Path) -> Path:
|
||||||
|
"""在 tmp 目录中创建必需的 prompt 文件。"""
|
||||||
|
prompt_files = [
|
||||||
|
"view_node_extract.md",
|
||||||
|
"view_node_verify.md",
|
||||||
|
"view_node_children_extract.md",
|
||||||
|
"view_node_children_verify.md",
|
||||||
|
"search_similar_extract.md",
|
||||||
|
"search_similar_verify.md",
|
||||||
|
"observe_frame_extract.md",
|
||||||
|
"observe_frame_verify.md",
|
||||||
|
]
|
||||||
|
for name in prompt_files:
|
||||||
|
(tmp_path / name).write_text(f"fake prompt for {name}", encoding="utf-8")
|
||||||
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def skills_registry(tmp_path: Path) -> SkillRegistry:
|
||||||
|
"""带一个预注册技能的 SkillRegistry。"""
|
||||||
|
skill_path = tmp_path / "test_skill.md"
|
||||||
|
skill_path.write_text(
|
||||||
|
"---\nname: test_skill\ndescription: test\n---\nskill body content",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
registry = SkillRegistry()
|
||||||
|
registry.set_paths({"test_skill": skill_path})
|
||||||
|
return registry
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def dispatcher(
|
||||||
|
env: TreeEnvironment,
|
||||||
|
prompts_dir: Path,
|
||||||
|
skills_registry: SkillRegistry,
|
||||||
|
) -> SearchToolDispatcher:
|
||||||
|
"""标准配置的 SearchToolDispatcher 实例。"""
|
||||||
|
return SearchToolDispatcher(
|
||||||
|
env=env,
|
||||||
|
tool_llm=FakeLLM(),
|
||||||
|
vlm=FakeVLM(),
|
||||||
|
ocr=FakeOCR(),
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
skills=skills_registry,
|
||||||
|
embed_fn=_fake_embed_fn,
|
||||||
|
verify_vision=False,
|
||||||
|
anchor=False,
|
||||||
|
assemble_mode="ids",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def dispatcher_no_skills(
|
||||||
|
env: TreeEnvironment,
|
||||||
|
prompts_dir: Path,
|
||||||
|
) -> SearchToolDispatcher:
|
||||||
|
"""skills=None 的 SearchToolDispatcher 实例。"""
|
||||||
|
return SearchToolDispatcher(
|
||||||
|
env=env,
|
||||||
|
tool_llm=FakeLLM(),
|
||||||
|
vlm=FakeVLM(),
|
||||||
|
ocr=None,
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
skills=None,
|
||||||
|
embed_fn=_fake_embed_fn,
|
||||||
|
verify_vision=False,
|
||||||
|
anchor=False,
|
||||||
|
assemble_mode="ids",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── get_tool_descriptions 测试 ───────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetToolDescriptions:
|
||||||
|
"""get_tool_descriptions 工具描述生成测试。"""
|
||||||
|
|
||||||
|
def test_without_read_skill(self) -> None:
|
||||||
|
"""不含 read_skill 时应包含四个基础工具。"""
|
||||||
|
text = get_tool_descriptions(include_read_skill=False)
|
||||||
|
assert "view_node" in text
|
||||||
|
assert "search_similar" in text
|
||||||
|
assert "observe_frame" in text
|
||||||
|
assert "submit_answer" in text
|
||||||
|
assert "read_skill" not in text
|
||||||
|
|
||||||
|
def test_with_read_skill(self) -> None:
|
||||||
|
"""含 read_skill 时应额外包含 read_skill 工具描述。"""
|
||||||
|
text = get_tool_descriptions(include_read_skill=True)
|
||||||
|
assert "view_node" in text
|
||||||
|
assert "read_skill" in text
|
||||||
|
assert "加载指定题型技能" in text
|
||||||
|
|
||||||
|
|
||||||
|
# ── dispatch 路由测试 ─────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestDispatchViewNode:
|
||||||
|
"""dispatch view_node 工具测试。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_view_node_returns_header_and_summary(
|
||||||
|
self, dispatcher: SearchToolDispatcher
|
||||||
|
) -> None:
|
||||||
|
"""view_node 应返回含节点头部、摘要和子节点概览的文本。"""
|
||||||
|
result = await dispatcher.dispatch(
|
||||||
|
"view_node",
|
||||||
|
{"node_id": "vid_L1_000", "question": "what happens?"},
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
# 头部格式
|
||||||
|
assert "[节点] vid_L1_000 | 场景层 |" in result
|
||||||
|
assert "0.0-30.0s" in result
|
||||||
|
# 摘要内容(来自 FakeLLM)
|
||||||
|
assert "fake summary" in result
|
||||||
|
# 子节点概览(L1 有 L2 子节点)
|
||||||
|
assert "[子节点概览]" in result
|
||||||
|
assert "1 个子节点" in result
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_view_node_l3_no_children(self, dispatcher: SearchToolDispatcher) -> None:
|
||||||
|
"""L3 叶子节点应无子节点概览段。"""
|
||||||
|
result = await dispatcher.dispatch(
|
||||||
|
"view_node",
|
||||||
|
{"node_id": "vid_L1_000_L2_000_L3_000", "question": "test"},
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
assert "[节点] vid_L1_000_L2_000_L3_000 | 关键帧层 |" in result
|
||||||
|
assert "[子节点概览]" not in result
|
||||||
|
|
||||||
|
|
||||||
|
class TestDispatchSearchSimilar:
|
||||||
|
"""dispatch search_similar 工具测试。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_search_similar_returns_results(self, dispatcher: SearchToolDispatcher) -> None:
|
||||||
|
"""search_similar 应返回搜索头部和编号结果列表。"""
|
||||||
|
result = await dispatcher.dispatch(
|
||||||
|
"search_similar",
|
||||||
|
{"query": "walking", "question": "what is the person doing?"},
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
assert '[搜索结果] 查询 "walking"' in result
|
||||||
|
assert "个相关节点" in result
|
||||||
|
# 至少有一个编号结果
|
||||||
|
assert "1." in result
|
||||||
|
# 包含分数信息
|
||||||
|
assert "score=" in result
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_search_similar_custom_k(self, dispatcher: SearchToolDispatcher) -> None:
|
||||||
|
"""search_similar 的 k 参数应限制返回数量。"""
|
||||||
|
result = await dispatcher.dispatch(
|
||||||
|
"search_similar",
|
||||||
|
{"query": "test", "question": "test", "k": 1},
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
assert "1 个相关节点" in result
|
||||||
|
|
||||||
|
|
||||||
|
class TestDispatchObserveFrame:
|
||||||
|
"""dispatch observe_frame 工具测试。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_observe_frame_with_subtitle(
|
||||||
|
self, dispatcher: SearchToolDispatcher, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""有字幕的 L3 节点应在输出前添加字幕上下文。"""
|
||||||
|
# 创建帧文件使路径存在检查通过
|
||||||
|
frame_file = tmp_path / "L1_000_L2_000_L3_000.jpg"
|
||||||
|
frame_file.write_bytes(b"\xff\xd8\xff\xe0")
|
||||||
|
|
||||||
|
# 重建 dispatcher 指定 frames_dir
|
||||||
|
tree = _make_test_tree()
|
||||||
|
env_with_frames = TreeEnvironment(tree, frames_dir=tmp_path)
|
||||||
|
|
||||||
|
d = SearchToolDispatcher(
|
||||||
|
env=env_with_frames,
|
||||||
|
tool_llm=FakeLLM(),
|
||||||
|
vlm=FakeVLM(),
|
||||||
|
ocr=FakeOCR(),
|
||||||
|
prompts_dir=dispatcher._prompts_dir,
|
||||||
|
skills=None,
|
||||||
|
embed_fn=_fake_embed_fn,
|
||||||
|
verify_vision=False,
|
||||||
|
anchor=False,
|
||||||
|
assemble_mode="ids",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await d.dispatch(
|
||||||
|
"observe_frame",
|
||||||
|
{
|
||||||
|
"node_ids": ["vid_L1_000_L2_000_L3_000"],
|
||||||
|
"question": "what is visible?",
|
||||||
|
},
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
assert "[字幕上下文] test subtitle text" in result
|
||||||
|
assert "fake visual observation" in result
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_observe_frame_empty_question(self, dispatcher: SearchToolDispatcher) -> None:
|
||||||
|
"""空 question 应返回错误文本。"""
|
||||||
|
result = await dispatcher.dispatch(
|
||||||
|
"observe_frame",
|
||||||
|
{"node_ids": ["vid_L1_000_L2_000_L3_000"], "question": " "},
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
assert "question 不能为空" in result
|
||||||
|
|
||||||
|
|
||||||
|
class TestDispatchSubmitAnswer:
|
||||||
|
"""dispatch submit_answer 工具测试。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_submit_answer_returns_confirmation(
|
||||||
|
self, dispatcher: SearchToolDispatcher
|
||||||
|
) -> None:
|
||||||
|
"""submit_answer 应返回确认文本。"""
|
||||||
|
result = await dispatcher.dispatch(
|
||||||
|
"submit_answer",
|
||||||
|
{"answer": "B", "evidence": "seen in frame", "reasoning": "clear visual"},
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
assert result == "[ok] 答案已提交: B"
|
||||||
|
|
||||||
|
|
||||||
|
class TestDispatchReadSkill:
|
||||||
|
"""dispatch read_skill 工具测试。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_read_skill_returns_body(self, dispatcher: SearchToolDispatcher) -> None:
|
||||||
|
"""read_skill 应返回去除 frontmatter 后的技能正文。"""
|
||||||
|
result = await dispatcher.dispatch(
|
||||||
|
"read_skill",
|
||||||
|
{"name": "test_skill"},
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
assert "skill body content" in result
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_read_skill_disabled(self, dispatcher_no_skills: SearchToolDispatcher) -> None:
|
||||||
|
"""skills=None 时 read_skill 应返回未启用提示。"""
|
||||||
|
result = await dispatcher_no_skills.dispatch(
|
||||||
|
"read_skill",
|
||||||
|
{"name": "anything"},
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
assert result == "错误: skills 未启用"
|
||||||
|
|
||||||
|
|
||||||
|
# ── 错误处理测试 ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestDispatchErrors:
|
||||||
|
"""dispatch 错误处理测试。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_unknown_tool_raises_value_error(self, dispatcher: SearchToolDispatcher) -> None:
|
||||||
|
"""未知工具应抛出 ValueError。"""
|
||||||
|
with pytest.raises(ValueError, match="未知工具: nonexistent_tool"):
|
||||||
|
await dispatcher.dispatch("nonexistent_tool", {}, context={})
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_node_not_found_returns_error_text(
|
||||||
|
self, dispatcher: SearchToolDispatcher
|
||||||
|
) -> None:
|
||||||
|
"""节点不存在时应返回错误文本(非异常)。"""
|
||||||
|
result = await dispatcher.dispatch(
|
||||||
|
"view_node",
|
||||||
|
{"node_id": "nonexistent_node", "question": "test"},
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
assert "工具执行错误" in result
|
||||||
|
assert "nonexistent_node" in result
|
||||||
|
|
||||||
|
@pytest.mark.asyncio()
|
||||||
|
async def test_read_skill_not_found_returns_error_text(
|
||||||
|
self, dispatcher: SearchToolDispatcher
|
||||||
|
) -> None:
|
||||||
|
"""未注册的技能名应返回错误文本。"""
|
||||||
|
result = await dispatcher.dispatch(
|
||||||
|
"read_skill",
|
||||||
|
{"name": "nonexistent_skill"},
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
assert "工具执行错误" in result
|
||||||
@@ -0,0 +1,458 @@
|
|||||||
|
"""observe_frame 单元测试。
|
||||||
|
|
||||||
|
FakeVLMProvider / FakeOCRProvider 实现 Protocol 最小集,
|
||||||
|
覆盖:两轮正常、verify=False 仅提取、OCR 注入、OCR 失败降级、
|
||||||
|
OCR 为 None、VLM 提取失败、VLM 验证失败降级、帧文件不存在、stats 键完整性。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.types import LLMResponse
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Fake 实现
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_DUMMY_RESPONSE_KWARGS = {
|
||||||
|
"thinking": "",
|
||||||
|
"model": "fake-vlm",
|
||||||
|
"provider": "fake",
|
||||||
|
"prompt_tokens": 10,
|
||||||
|
"completion_tokens": 20,
|
||||||
|
"latency_ms": 100,
|
||||||
|
"ttft_ms": None,
|
||||||
|
"max_inter_token_ms": None,
|
||||||
|
"cache_hit": False,
|
||||||
|
"call_id": "fake-call-id",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class FakeVLMProvider:
|
||||||
|
"""可编程的 VLM 假实现。
|
||||||
|
|
||||||
|
通过 responses 列表按序返回预设内容;raises 列表对应位置不为 None 时抛异常。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
responses: list[str] | None = None,
|
||||||
|
raises: list[Exception | None] | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._responses = responses or []
|
||||||
|
self._raises = raises or []
|
||||||
|
self._call_idx = 0
|
||||||
|
self.calls: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
async def chat_with_images(
|
||||||
|
self,
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
images: list[str | Path],
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> LLMResponse:
|
||||||
|
"""记录调用并按序返回预设响应或抛出异常。"""
|
||||||
|
idx = self._call_idx
|
||||||
|
self._call_idx += 1
|
||||||
|
self.calls.append(
|
||||||
|
{
|
||||||
|
"messages": messages,
|
||||||
|
"images": images,
|
||||||
|
"session_id": session_id,
|
||||||
|
"parent_call_id": parent_call_id,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if idx < len(self._raises) and self._raises[idx] is not None:
|
||||||
|
raise self._raises[idx] # type: ignore[misc]
|
||||||
|
content = self._responses[idx] if idx < len(self._responses) else ""
|
||||||
|
return LLMResponse(content=content, **_DUMMY_RESPONSE_KWARGS)
|
||||||
|
|
||||||
|
|
||||||
|
class FakeOCRProvider:
|
||||||
|
"""可编程的 OCR 假实现。"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
text: str = "",
|
||||||
|
raise_on_call: Exception | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._text = text
|
||||||
|
self._raise_on_call = raise_on_call
|
||||||
|
|
||||||
|
async def transcribe_frames(self, frame_paths: list[Path]) -> str:
|
||||||
|
"""返回预设文本或抛出异常。"""
|
||||||
|
if self._raise_on_call is not None:
|
||||||
|
raise self._raise_on_call
|
||||||
|
return self._text
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Fixtures
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def frame_files(tmp_path: Path) -> list[Path]:
|
||||||
|
"""创建两个最小 JPEG 占位帧文件。"""
|
||||||
|
frames: list[Path] = []
|
||||||
|
for i in range(2):
|
||||||
|
p = tmp_path / f"frame_{i}.jpg"
|
||||||
|
p.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 20)
|
||||||
|
frames.append(p)
|
||||||
|
return frames
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def prompts_dir(tmp_path: Path) -> Path:
|
||||||
|
"""创建 observe_frame_extract.md / observe_frame_verify.md 占位文件。"""
|
||||||
|
d = tmp_path / "prompts"
|
||||||
|
d.mkdir()
|
||||||
|
(d / "observe_frame_extract.md").write_text("extract prompt", encoding="utf-8")
|
||||||
|
(d / "observe_frame_verify.md").write_text("verify prompt", encoding="utf-8")
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 测试用例
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestObserveFrameNormal:
|
||||||
|
"""两轮正常执行(verify=True)。"""
|
||||||
|
|
||||||
|
def test_two_round_normal(
|
||||||
|
self, frame_files: list[Path], prompts_dir: Path
|
||||||
|
) -> None:
|
||||||
|
from app.search.vision import observe_frame
|
||||||
|
|
||||||
|
vlm = FakeVLMProvider(responses=["raw evidence", "verified ok"])
|
||||||
|
collected: list[dict[str, int]] = []
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
observe_frame(
|
||||||
|
vlm=vlm,
|
||||||
|
frame_paths=frame_files,
|
||||||
|
question="what happened?",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
ocr=None,
|
||||||
|
verify=True,
|
||||||
|
stats_sink=collected.append,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == "[视觉观察] raw evidence\n[验证] verified ok"
|
||||||
|
assert len(vlm.calls) == 2
|
||||||
|
# 提取轮使用 extract prompt
|
||||||
|
assert vlm.calls[0]["messages"][0]["content"] == "extract prompt"
|
||||||
|
# 验证轮使用 verify prompt
|
||||||
|
assert vlm.calls[1]["messages"][0]["content"] == "verify prompt"
|
||||||
|
assert len(collected) == 1
|
||||||
|
|
||||||
|
|
||||||
|
class TestObserveFrameExtractOnly:
|
||||||
|
"""verify=False 仅执行提取轮。"""
|
||||||
|
|
||||||
|
def test_extract_only(
|
||||||
|
self, frame_files: list[Path], prompts_dir: Path
|
||||||
|
) -> None:
|
||||||
|
from app.search.vision import observe_frame
|
||||||
|
|
||||||
|
vlm = FakeVLMProvider(responses=["only extract"])
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
observe_frame(
|
||||||
|
vlm=vlm,
|
||||||
|
frame_paths=frame_files,
|
||||||
|
question="q?",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
ocr=None,
|
||||||
|
verify=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == "[视觉观察] only extract"
|
||||||
|
assert len(vlm.calls) == 1
|
||||||
|
|
||||||
|
|
||||||
|
class TestObserveFrameOCRInjection:
|
||||||
|
"""OCR 注入:文本非空时并置于问题前。"""
|
||||||
|
|
||||||
|
def test_ocr_injected(
|
||||||
|
self, frame_files: list[Path], prompts_dir: Path
|
||||||
|
) -> None:
|
||||||
|
from app.search.vision import observe_frame
|
||||||
|
|
||||||
|
vlm = FakeVLMProvider(responses=["evidence with ocr"])
|
||||||
|
ocr = FakeOCRProvider(text="帧1: 你好世界")
|
||||||
|
collected: list[dict[str, int]] = []
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
observe_frame(
|
||||||
|
vlm=vlm,
|
||||||
|
frame_paths=frame_files,
|
||||||
|
question="q?",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
ocr=ocr,
|
||||||
|
verify=False,
|
||||||
|
stats_sink=collected.append,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "[视觉观察]" in result
|
||||||
|
# 验证 OCR 文本被注入到 user message
|
||||||
|
user_msg = vlm.calls[0]["messages"][1]["content"]
|
||||||
|
assert "帧1: 你好世界" in user_msg
|
||||||
|
# stats 中 ocr_injected=1
|
||||||
|
assert collected[0]["ocr_injected"] == 1
|
||||||
|
assert collected[0]["ocr_chars"] == len("帧1: 你好世界")
|
||||||
|
|
||||||
|
|
||||||
|
class TestObserveFrameOCRFailDegrades:
|
||||||
|
"""OCR 转录抛出异常时降级:不注入 OCR、ocr_failed=1。"""
|
||||||
|
|
||||||
|
def test_ocr_failure_degrades(
|
||||||
|
self, frame_files: list[Path], prompts_dir: Path
|
||||||
|
) -> None:
|
||||||
|
from app.search.vision import observe_frame
|
||||||
|
|
||||||
|
vlm = FakeVLMProvider(responses=["evidence no ocr"])
|
||||||
|
ocr = FakeOCRProvider(raise_on_call=RuntimeError("OCR service down"))
|
||||||
|
collected: list[dict[str, int]] = []
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
observe_frame(
|
||||||
|
vlm=vlm,
|
||||||
|
frame_paths=frame_files,
|
||||||
|
question="q?",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
ocr=ocr,
|
||||||
|
verify=False,
|
||||||
|
stats_sink=collected.append,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == "[视觉观察] evidence no ocr"
|
||||||
|
assert collected[0]["ocr_failed"] == 1
|
||||||
|
assert collected[0]["ocr_injected"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
class TestObserveFrameOCRNone:
|
||||||
|
"""ocr=None 时不执行转录。"""
|
||||||
|
|
||||||
|
def test_ocr_none(
|
||||||
|
self, frame_files: list[Path], prompts_dir: Path
|
||||||
|
) -> None:
|
||||||
|
from app.search.vision import observe_frame
|
||||||
|
|
||||||
|
vlm = FakeVLMProvider(responses=["no ocr"])
|
||||||
|
collected: list[dict[str, int]] = []
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
observe_frame(
|
||||||
|
vlm=vlm,
|
||||||
|
frame_paths=frame_files,
|
||||||
|
question="q?",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
ocr=None,
|
||||||
|
verify=False,
|
||||||
|
stats_sink=collected.append,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == "[视觉观察] no ocr"
|
||||||
|
assert collected[0]["ocr_injected"] == 0
|
||||||
|
assert collected[0]["ocr_chars"] == 0
|
||||||
|
assert collected[0]["ocr_failed"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
class TestObserveFrameVLMExtractFailure:
|
||||||
|
"""VLM 提取轮失败 → 返回 [VL错误]。"""
|
||||||
|
|
||||||
|
def test_vlm_extract_failure(
|
||||||
|
self, frame_files: list[Path], prompts_dir: Path
|
||||||
|
) -> None:
|
||||||
|
from app.search.vision import observe_frame
|
||||||
|
|
||||||
|
vlm = FakeVLMProvider(raises=[RuntimeError("VLM timeout")])
|
||||||
|
collected: list[dict[str, int]] = []
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
observe_frame(
|
||||||
|
vlm=vlm,
|
||||||
|
frame_paths=frame_files,
|
||||||
|
question="q?",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
ocr=None,
|
||||||
|
verify=True,
|
||||||
|
stats_sink=collected.append,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.startswith("[VL错误]")
|
||||||
|
assert "VLM timeout" in result
|
||||||
|
assert len(collected) == 1
|
||||||
|
|
||||||
|
|
||||||
|
class TestObserveFrameVLMVerifyFailureDegrades:
|
||||||
|
"""VLM 验证轮失败 → 降级:保留提取结果 + [验证] 跳过。"""
|
||||||
|
|
||||||
|
def test_vlm_verify_failure_degrades(
|
||||||
|
self, frame_files: list[Path], prompts_dir: Path
|
||||||
|
) -> None:
|
||||||
|
from app.search.vision import observe_frame
|
||||||
|
|
||||||
|
vlm = FakeVLMProvider(
|
||||||
|
responses=["good evidence", ""],
|
||||||
|
raises=[None, RuntimeError("verify timeout")],
|
||||||
|
)
|
||||||
|
collected: list[dict[str, int]] = []
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
observe_frame(
|
||||||
|
vlm=vlm,
|
||||||
|
frame_paths=frame_files,
|
||||||
|
question="q?",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
ocr=None,
|
||||||
|
verify=True,
|
||||||
|
stats_sink=collected.append,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "[视觉观察] good evidence" in result
|
||||||
|
assert "[验证] 跳过(调用失败)" in result
|
||||||
|
assert len(collected) == 1
|
||||||
|
|
||||||
|
|
||||||
|
class TestObserveFrameFileMissing:
|
||||||
|
"""帧文件不存在 → 返回 [VL错误] 帧文件不存在。"""
|
||||||
|
|
||||||
|
def test_frame_file_not_found(self, prompts_dir: Path) -> None:
|
||||||
|
from app.search.vision import observe_frame
|
||||||
|
|
||||||
|
vlm = FakeVLMProvider()
|
||||||
|
missing = [Path("/nonexistent/frame_0.jpg")]
|
||||||
|
collected: list[dict[str, int]] = []
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
observe_frame(
|
||||||
|
vlm=vlm,
|
||||||
|
frame_paths=missing,
|
||||||
|
question="q?",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
ocr=None,
|
||||||
|
verify=True,
|
||||||
|
stats_sink=collected.append,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "[VL错误] 帧文件不存在" in result
|
||||||
|
assert len(collected) == 1
|
||||||
|
|
||||||
|
|
||||||
|
class TestObserveFrameStatsKeys:
|
||||||
|
"""stats 包含全部五个预期键。"""
|
||||||
|
|
||||||
|
def test_stats_keys_complete(
|
||||||
|
self, frame_files: list[Path], prompts_dir: Path
|
||||||
|
) -> None:
|
||||||
|
from app.search.vision import observe_frame
|
||||||
|
|
||||||
|
vlm = FakeVLMProvider(responses=["evidence"])
|
||||||
|
collected: list[dict[str, int]] = []
|
||||||
|
|
||||||
|
asyncio.run(
|
||||||
|
observe_frame(
|
||||||
|
vlm=vlm,
|
||||||
|
frame_paths=frame_files,
|
||||||
|
question="q?",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
ocr=None,
|
||||||
|
verify=False,
|
||||||
|
stats_sink=collected.append,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
expected_keys = {"ocr_injected", "ocr_chars", "ocr_failed", "discrepancy", "abstain"}
|
||||||
|
assert set(collected[0].keys()) == expected_keys
|
||||||
|
|
||||||
|
|
||||||
|
class TestObserveFrameDiscrepancyAndAbstain:
|
||||||
|
"""VLM 返回含 '分歧' 或 '[证据不存在]' 时对应 stats 标记。"""
|
||||||
|
|
||||||
|
def test_discrepancy_flag(
|
||||||
|
self, frame_files: list[Path], prompts_dir: Path
|
||||||
|
) -> None:
|
||||||
|
from app.search.vision import observe_frame
|
||||||
|
|
||||||
|
vlm = FakeVLMProvider(responses=["发现分歧:OCR 与画面不一致"])
|
||||||
|
collected: list[dict[str, int]] = []
|
||||||
|
|
||||||
|
asyncio.run(
|
||||||
|
observe_frame(
|
||||||
|
vlm=vlm,
|
||||||
|
frame_paths=frame_files,
|
||||||
|
question="q?",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
ocr=None,
|
||||||
|
verify=False,
|
||||||
|
stats_sink=collected.append,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert collected[0]["discrepancy"] == 1
|
||||||
|
|
||||||
|
def test_abstain_flag(
|
||||||
|
self, frame_files: list[Path], prompts_dir: Path
|
||||||
|
) -> None:
|
||||||
|
from app.search.vision import observe_frame
|
||||||
|
|
||||||
|
vlm = FakeVLMProvider(responses=["[证据不存在] 无法判断"])
|
||||||
|
collected: list[dict[str, int]] = []
|
||||||
|
|
||||||
|
asyncio.run(
|
||||||
|
observe_frame(
|
||||||
|
vlm=vlm,
|
||||||
|
frame_paths=frame_files,
|
||||||
|
question="q?",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
ocr=None,
|
||||||
|
verify=False,
|
||||||
|
stats_sink=collected.append,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert collected[0]["abstain"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
class TestObserveFrameTelemetryPassthrough:
|
||||||
|
"""session_id 和 parent_call_id 透传到 VLM 调用。"""
|
||||||
|
|
||||||
|
def test_telemetry_passthrough(
|
||||||
|
self, frame_files: list[Path], prompts_dir: Path
|
||||||
|
) -> None:
|
||||||
|
from app.search.vision import observe_frame
|
||||||
|
|
||||||
|
vlm = FakeVLMProvider(responses=["evidence"])
|
||||||
|
|
||||||
|
asyncio.run(
|
||||||
|
observe_frame(
|
||||||
|
vlm=vlm,
|
||||||
|
frame_paths=frame_files,
|
||||||
|
question="q?",
|
||||||
|
prompts_dir=prompts_dir,
|
||||||
|
ocr=None,
|
||||||
|
verify=False,
|
||||||
|
session_id="sess-123",
|
||||||
|
parent_call_id="parent-456",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert vlm.calls[0]["session_id"] == "sess-123"
|
||||||
|
assert vlm.calls[0]["parent_call_id"] == "parent-456"
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
"""字幕模块单元测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.tree.subtitle import (
|
||||||
|
SRTEntry,
|
||||||
|
assign_subtitles_voronoi,
|
||||||
|
check_subtitle_completeness,
|
||||||
|
extract_subtitle_for_range,
|
||||||
|
parse_srt,
|
||||||
|
)
|
||||||
|
|
||||||
|
_SAMPLE_SRT = """\
|
||||||
|
1
|
||||||
|
00:00:01,000 --> 00:00:03,500
|
||||||
|
Hello world.
|
||||||
|
|
||||||
|
2
|
||||||
|
00:00:05,000 --> 00:00:08,000
|
||||||
|
<i>This is italic</i> text.
|
||||||
|
|
||||||
|
3
|
||||||
|
00:00:10,000 --> 00:00:12,000
|
||||||
|
Final line.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseSrt:
|
||||||
|
def test_basic_parse(self, tmp_path):
|
||||||
|
srt_file = tmp_path / "test.srt"
|
||||||
|
srt_file.write_text(_SAMPLE_SRT, encoding="utf-8")
|
||||||
|
entries = parse_srt(str(srt_file))
|
||||||
|
assert len(entries) == 3
|
||||||
|
assert entries[0] == SRTEntry(start=1.0, end=3.5, text="Hello world.")
|
||||||
|
assert entries[1].text == "This is italic text."
|
||||||
|
|
||||||
|
def test_empty_srt(self, tmp_path):
|
||||||
|
srt_file = tmp_path / "empty.srt"
|
||||||
|
srt_file.write_text("", encoding="utf-8")
|
||||||
|
entries = parse_srt(str(srt_file))
|
||||||
|
assert entries == []
|
||||||
|
|
||||||
|
def test_malformed_srt_skips_bad_blocks(self, tmp_path):
|
||||||
|
bad_srt = "garbage\n\n1\n00:00:01,000 --> 00:00:02,000\nGood line.\n"
|
||||||
|
srt_file = tmp_path / "bad.srt"
|
||||||
|
srt_file.write_text(bad_srt, encoding="utf-8")
|
||||||
|
entries = parse_srt(str(srt_file))
|
||||||
|
assert len(entries) == 1
|
||||||
|
assert entries[0].text == "Good line."
|
||||||
|
|
||||||
|
|
||||||
|
class TestCompletenessCheck:
|
||||||
|
def test_good_coverage(self):
|
||||||
|
entries = [SRTEntry(0.0, 5.0, "a"), SRTEntry(5.0, 10.0, "b")]
|
||||||
|
report = check_subtitle_completeness(entries, duration=10.0, min_coverage=0.5)
|
||||||
|
assert report.usable is True
|
||||||
|
assert report.coverage_ratio >= 0.5
|
||||||
|
|
||||||
|
def test_poor_coverage(self):
|
||||||
|
entries = [SRTEntry(0.0, 1.0, "short")]
|
||||||
|
report = check_subtitle_completeness(entries, duration=100.0, min_coverage=0.3)
|
||||||
|
assert report.usable is False
|
||||||
|
|
||||||
|
def test_max_gap(self):
|
||||||
|
entries = [SRTEntry(0.0, 1.0, "a"), SRTEntry(50.0, 51.0, "b")]
|
||||||
|
report = check_subtitle_completeness(entries, duration=60.0)
|
||||||
|
assert report.max_gap_sec >= 49.0
|
||||||
|
|
||||||
|
|
||||||
|
class TestExtractForRange:
|
||||||
|
def test_overlap(self):
|
||||||
|
entries = [
|
||||||
|
SRTEntry(0.0, 5.0, "first"),
|
||||||
|
SRTEntry(4.0, 8.0, "second"),
|
||||||
|
SRTEntry(10.0, 12.0, "third"),
|
||||||
|
]
|
||||||
|
text = extract_subtitle_for_range(entries, (3.0, 9.0))
|
||||||
|
assert "first" in text
|
||||||
|
assert "second" in text
|
||||||
|
assert "third" not in text
|
||||||
|
|
||||||
|
|
||||||
|
class TestVoronoiAssign:
|
||||||
|
def test_assigns_to_l3_nodes(self):
|
||||||
|
from app.tree.index import (
|
||||||
|
IndexMeta,
|
||||||
|
L1Card,
|
||||||
|
L1Node,
|
||||||
|
L2Card,
|
||||||
|
L2Node,
|
||||||
|
L3Card,
|
||||||
|
L3Node,
|
||||||
|
TreeIndex,
|
||||||
|
)
|
||||||
|
|
||||||
|
l3_0 = L3Node(id="l1_0_l2_0_l3_0", card=L3Card("desc0", [], [], [], "", {}), timestamp=2.0)
|
||||||
|
l3_1 = L3Node(id="l1_0_l2_0_l3_1", card=L3Card("desc1", [], [], [], "", {}), timestamp=6.0)
|
||||||
|
l2 = L2Node(
|
||||||
|
id="l1_0_l2_0",
|
||||||
|
card=L2Card("evt", [], [], [], [], "", None),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l3_0, l3_1],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=L1Card("scene", "", [], [], [], [], ""),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
index = TreeIndex(metadata=IndexMeta("/test.mp4", "video"), roots=[l1])
|
||||||
|
|
||||||
|
entries = [SRTEntry(1.0, 3.0, "hello"), SRTEntry(5.0, 7.0, "world")]
|
||||||
|
assign_subtitles_voronoi(index, entries)
|
||||||
|
|
||||||
|
assert l3_0.subtitle is not None
|
||||||
|
assert "hello" in l3_0.subtitle
|
||||||
|
assert l3_1.subtitle is not None
|
||||||
|
assert "world" in l3_1.subtitle
|
||||||
@@ -0,0 +1,324 @@
|
|||||||
|
"""TreeEnvironment 运行时单元测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.tree.environment import TreeEnvironment
|
||||||
|
from app.tree.index import (
|
||||||
|
IndexMeta,
|
||||||
|
L1Card,
|
||||||
|
L1Node,
|
||||||
|
L2Card,
|
||||||
|
L2Node,
|
||||||
|
L3Card,
|
||||||
|
L3Node,
|
||||||
|
TreeIndex,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_test_index() -> TreeIndex:
|
||||||
|
"""构建测试用的三层树索引。"""
|
||||||
|
l3_0 = L3Node(
|
||||||
|
id="vid_L1_000_L2_000_L3_000",
|
||||||
|
card=L3Card(
|
||||||
|
"运动员在跑步",
|
||||||
|
["运动员"],
|
||||||
|
["跑步"],
|
||||||
|
["Nike"],
|
||||||
|
"居中",
|
||||||
|
{"lighting": "明亮"},
|
||||||
|
),
|
||||||
|
timestamp=1.0,
|
||||||
|
frame_path="frames/L1_000_L2_000_L3_000.jpg",
|
||||||
|
subtitle="he is running",
|
||||||
|
)
|
||||||
|
l3_1 = L3Node(
|
||||||
|
id="vid_L1_000_L2_000_L3_001",
|
||||||
|
card=L3Card(
|
||||||
|
"观众欢呼",
|
||||||
|
["观众"],
|
||||||
|
["欢呼"],
|
||||||
|
[],
|
||||||
|
"广角",
|
||||||
|
{},
|
||||||
|
),
|
||||||
|
timestamp=3.0,
|
||||||
|
frame_path="frames/L1_000_L2_000_L3_001.jpg",
|
||||||
|
)
|
||||||
|
l2 = L2Node(
|
||||||
|
id="vid_L1_000_L2_000",
|
||||||
|
card=L2Card(
|
||||||
|
"比赛片段",
|
||||||
|
["运动员"],
|
||||||
|
["跑步"],
|
||||||
|
["运动员"],
|
||||||
|
["Nike"],
|
||||||
|
"",
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l3_0, l3_1],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="vid_L1_000",
|
||||||
|
card=L1Card(
|
||||||
|
"体育赛事",
|
||||||
|
"体育场",
|
||||||
|
["运动员"],
|
||||||
|
["比赛"],
|
||||||
|
["体育"],
|
||||||
|
["Nike"],
|
||||||
|
"从左到右",
|
||||||
|
),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
return TreeIndex(metadata=IndexMeta("/test.mp4", "video"), roots=[l1])
|
||||||
|
|
||||||
|
|
||||||
|
class TestViewNode:
|
||||||
|
"""view_node 方法测试。"""
|
||||||
|
|
||||||
|
def test_l3_node(self) -> None:
|
||||||
|
"""L3 节点应显示帧描述。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
result = env.view_node("vid_L1_000_L2_000_L3_000")
|
||||||
|
assert "运动员在跑步" in result
|
||||||
|
assert "vid_L1_000_L2_000_L3_000" in result
|
||||||
|
|
||||||
|
def test_l2_node_shows_children(self) -> None:
|
||||||
|
"""L2 节点应显示子节点概览。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
result = env.view_node("vid_L1_000_L2_000")
|
||||||
|
assert "比赛片段" in result
|
||||||
|
assert "vid_L1_000_L2_000_L3_000" in result # child listed
|
||||||
|
|
||||||
|
def test_anchor_mode(self) -> None:
|
||||||
|
"""锚模式应在输出中添加 [cN] 标记。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
result = env.view_node("vid_L1_000_L2_000_L3_000", anchor=True)
|
||||||
|
assert "[c" in result # anchor markers present
|
||||||
|
|
||||||
|
def test_unknown_node_raises(self) -> None:
|
||||||
|
"""查询不存在的节点应抛出 KeyError。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
with pytest.raises(KeyError):
|
||||||
|
env.view_node("nonexistent")
|
||||||
|
|
||||||
|
|
||||||
|
class TestSearchSimilar:
|
||||||
|
"""search_similar 方法测试。"""
|
||||||
|
|
||||||
|
def test_returns_results(self) -> None:
|
||||||
|
"""使用 embed_fn 应返回搜索结果。"""
|
||||||
|
index = _make_test_index()
|
||||||
|
|
||||||
|
def fake_embed(
|
||||||
|
texts: str | list[str],
|
||||||
|
) -> np.ndarray:
|
||||||
|
if isinstance(texts, str):
|
||||||
|
texts = [texts]
|
||||||
|
return np.random.randn(len(texts), 4).astype(np.float32)
|
||||||
|
|
||||||
|
index.embed_all(fake_embed, "test", 4)
|
||||||
|
env = TreeEnvironment(index)
|
||||||
|
results = env.search_similar("运动员", top_k=3, embed_fn=fake_embed)
|
||||||
|
assert len(results) > 0
|
||||||
|
assert all(isinstance(r, tuple) and len(r) == 2 for r in results)
|
||||||
|
|
||||||
|
def test_ancestor_dedup(self) -> None:
|
||||||
|
"""祖先去重:如果 L3 已在结果中,其 L1/L2 祖先应被跳过。"""
|
||||||
|
index = _make_test_index()
|
||||||
|
# 手动设置 embedding,使 L3 节点分数高于 L1/L2
|
||||||
|
l3_0 = index.roots[0].children[0].children[0]
|
||||||
|
l3_1 = index.roots[0].children[0].children[1]
|
||||||
|
l2 = index.roots[0].children[0]
|
||||||
|
l1 = index.roots[0]
|
||||||
|
l3_0.embedding = np.array([1.0, 0, 0, 0], dtype=np.float32)
|
||||||
|
l3_1.embedding = np.array([0.9, 0.1, 0, 0], dtype=np.float32)
|
||||||
|
l2.embedding = np.array([0.5, 0.5, 0, 0], dtype=np.float32)
|
||||||
|
l1.embedding = np.array([0.3, 0.3, 0.3, 0], dtype=np.float32)
|
||||||
|
index.metadata.embed_model = "test"
|
||||||
|
index.metadata.embed_dim = 4
|
||||||
|
|
||||||
|
env = TreeEnvironment(index)
|
||||||
|
results = env.search_similar(
|
||||||
|
"运动员",
|
||||||
|
top_k=5,
|
||||||
|
embed_fn=lambda t: np.array([[1.0, 0, 0, 0]], dtype=np.float32),
|
||||||
|
)
|
||||||
|
result_ids = [r[0] for r in results]
|
||||||
|
# L3 节点应存在;其祖先应被去重跳过
|
||||||
|
assert "vid_L1_000_L2_000_L3_000" in result_ids
|
||||||
|
|
||||||
|
def test_with_embed_fn_overrides_existing(self) -> None:
|
||||||
|
"""即使已有 embedding,提供 embed_fn 时仍应用于 query 编码。"""
|
||||||
|
index = _make_test_index()
|
||||||
|
|
||||||
|
def fake_embed(
|
||||||
|
texts: str | list[str],
|
||||||
|
) -> np.ndarray:
|
||||||
|
if isinstance(texts, str):
|
||||||
|
texts = [texts]
|
||||||
|
return np.random.randn(len(texts), 4).astype(np.float32)
|
||||||
|
|
||||||
|
index.embed_all(fake_embed, "test", 4)
|
||||||
|
env = TreeEnvironment(index)
|
||||||
|
|
||||||
|
def query_embed(
|
||||||
|
texts: str | list[str],
|
||||||
|
) -> np.ndarray:
|
||||||
|
if isinstance(texts, str):
|
||||||
|
texts = [texts]
|
||||||
|
return np.ones((len(texts), 4), dtype=np.float32) * 0.5
|
||||||
|
|
||||||
|
results = env.search_similar("test", top_k=2, embed_fn=query_embed)
|
||||||
|
assert len(results) > 0
|
||||||
|
|
||||||
|
def test_no_embed_fn_raises(self) -> None:
|
||||||
|
"""未提供 embed_fn 时应报错。"""
|
||||||
|
index = _make_test_index()
|
||||||
|
env = TreeEnvironment(index)
|
||||||
|
with pytest.raises(ValueError, match="embed_fn"):
|
||||||
|
env.search_similar("test", top_k=3)
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetSubtitle:
|
||||||
|
"""get_subtitle 方法测试。"""
|
||||||
|
|
||||||
|
def test_existing_subtitle(self) -> None:
|
||||||
|
"""有字幕的节点应返回字幕文本。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
assert env.get_subtitle("vid_L1_000_L2_000_L3_000") == "he is running"
|
||||||
|
|
||||||
|
def test_no_subtitle(self) -> None:
|
||||||
|
"""无字幕的节点应返回空字符串。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
assert env.get_subtitle("vid_L1_000_L2_000_L3_001") == ""
|
||||||
|
|
||||||
|
def test_unknown_node(self) -> None:
|
||||||
|
"""不存在的节点应返回空字符串。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
assert env.get_subtitle("nonexistent") == ""
|
||||||
|
|
||||||
|
|
||||||
|
class TestResolveFramePaths:
|
||||||
|
"""resolve_frame_paths 方法测试。"""
|
||||||
|
|
||||||
|
def test_l3_nodes(self) -> None:
|
||||||
|
"""L3 节点应映射到帧文件路径。"""
|
||||||
|
env = TreeEnvironment(_make_test_index(), frames_dir=Path("/data/frames"))
|
||||||
|
paths = env.resolve_frame_paths(["vid_L1_000_L2_000_L3_000"])
|
||||||
|
assert len(paths) == 1
|
||||||
|
assert "L1_000_L2_000_L3_000" in str(paths[0])
|
||||||
|
|
||||||
|
def test_l2_expands_to_children(self) -> None:
|
||||||
|
"""L2 节点应展开为其所有 L3 子节点的帧路径。"""
|
||||||
|
env = TreeEnvironment(_make_test_index(), frames_dir=Path("/data/frames"))
|
||||||
|
paths = env.resolve_frame_paths(["vid_L1_000_L2_000"])
|
||||||
|
assert len(paths) == 2 # 2 L3 children
|
||||||
|
|
||||||
|
def test_no_frames_dir_uses_node_path(self) -> None:
|
||||||
|
"""未提供 frames_dir 时应使用节点自带的 frame_path。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
paths = env.resolve_frame_paths(["vid_L1_000_L2_000_L3_000"])
|
||||||
|
assert len(paths) == 1
|
||||||
|
assert "L1_000_L2_000_L3_000" in str(paths[0])
|
||||||
|
|
||||||
|
def test_empty_list_returns_empty(self) -> None:
|
||||||
|
"""空列表应返回空结果。"""
|
||||||
|
env = TreeEnvironment(_make_test_index(), frames_dir=Path("/data/frames"))
|
||||||
|
paths = env.resolve_frame_paths([])
|
||||||
|
assert paths == []
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetNodeText:
|
||||||
|
"""get_node_text 方法测试。"""
|
||||||
|
|
||||||
|
def test_normal_mode_returns_full_text(self) -> None:
|
||||||
|
"""默认模式应返回完整文本和 None anchor_map。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
text, anchor_map = env.get_node_text("vid_L1_000_L2_000_L3_000")
|
||||||
|
assert "运动员在跑步" in text
|
||||||
|
assert anchor_map is None
|
||||||
|
|
||||||
|
def test_anchor_mode_returns_anchored_text_and_map(self) -> None:
|
||||||
|
"""锚模式应返回带锚文本和 anchor_map 字典。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
text, anchor_map = env.get_node_text(
|
||||||
|
"vid_L1_000_L2_000_L3_000", anchor=True,
|
||||||
|
)
|
||||||
|
# 锚文本包含 [cN] 标记
|
||||||
|
assert "[c1]" in text
|
||||||
|
# anchor_map 非空,键为锚标(如 "c1"),值为对应行文本
|
||||||
|
assert anchor_map is not None
|
||||||
|
assert len(anchor_map) > 0
|
||||||
|
assert "c1" in anchor_map
|
||||||
|
# 字幕行也应在 anchor_map 中(该节点有 subtitle)
|
||||||
|
assert any(k.startswith("s") for k in anchor_map)
|
||||||
|
|
||||||
|
def test_nonexistent_node_raises(self) -> None:
|
||||||
|
"""查询不存在的节点应抛出 KeyError。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
with pytest.raises(KeyError):
|
||||||
|
env.get_node_text("nonexistent")
|
||||||
|
|
||||||
|
def test_node_without_subtitle_no_s_anchors(self) -> None:
|
||||||
|
"""无字幕的 L3 节点锚模式不应产生 [sN] 锚。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
text, anchor_map = env.get_node_text(
|
||||||
|
"vid_L1_000_L2_000_L3_001", anchor=True,
|
||||||
|
)
|
||||||
|
assert anchor_map is not None
|
||||||
|
assert not any(k.startswith("s") for k in anchor_map)
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetChildrenInfo:
|
||||||
|
"""get_children_info 方法测试。"""
|
||||||
|
|
||||||
|
def test_l1_has_children(self) -> None:
|
||||||
|
"""L1 节点应返回其 L2 子节点信息列表。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
children = env.get_children_info("vid_L1_000")
|
||||||
|
assert len(children) == 1
|
||||||
|
child = children[0]
|
||||||
|
assert child["id"] == "vid_L1_000_L2_000"
|
||||||
|
assert "time_range" in child
|
||||||
|
assert "summary" in child
|
||||||
|
assert isinstance(child["summary"], str)
|
||||||
|
|
||||||
|
def test_l2_has_children(self) -> None:
|
||||||
|
"""L2 节点应返回其 L3 子节点信息列表。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
children = env.get_children_info("vid_L1_000_L2_000")
|
||||||
|
assert len(children) == 2
|
||||||
|
ids = [c["id"] for c in children]
|
||||||
|
assert "vid_L1_000_L2_000_L3_000" in ids
|
||||||
|
assert "vid_L1_000_L2_000_L3_001" in ids
|
||||||
|
|
||||||
|
def test_l3_has_no_children(self) -> None:
|
||||||
|
"""L3 叶子节点应返回空列表。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
children = env.get_children_info("vid_L1_000_L2_000_L3_000")
|
||||||
|
assert children == []
|
||||||
|
|
||||||
|
def test_nonexistent_node_raises(self) -> None:
|
||||||
|
"""查询不存在的节点应抛出 KeyError。"""
|
||||||
|
env = TreeEnvironment(_make_test_index())
|
||||||
|
with pytest.raises(KeyError):
|
||||||
|
env.get_children_info("nonexistent")
|
||||||
|
|
||||||
|
def test_summary_truncation(self) -> None:
|
||||||
|
"""超过 120 字符的描述应被截断。"""
|
||||||
|
index = _make_test_index()
|
||||||
|
# 修改 L2 的事件描述为超长文本
|
||||||
|
l2 = index.roots[0].children[0]
|
||||||
|
long_desc = "A" * 200
|
||||||
|
object.__setattr__(l2.card, "event_description", long_desc)
|
||||||
|
env = TreeEnvironment(index)
|
||||||
|
children = env.get_children_info("vid_L1_000")
|
||||||
|
assert len(children[0]["summary"]) == 123 # 120 + "..."
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
"""TreeIndex 数据结构单元测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.tree.index import (
|
||||||
|
IndexMeta,
|
||||||
|
L1Card,
|
||||||
|
L1Node,
|
||||||
|
L2Card,
|
||||||
|
L2Node,
|
||||||
|
L3Card,
|
||||||
|
L3Node,
|
||||||
|
TreeIndex,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_l3(idx: int = 0) -> L3Node:
|
||||||
|
return L3Node(
|
||||||
|
id=f"l1_0_l2_0_l3_{idx}",
|
||||||
|
card=L3Card(
|
||||||
|
frame_summary=f"帧{idx}描述",
|
||||||
|
visible_entities=["实体A"],
|
||||||
|
ongoing_actions=["动作A"],
|
||||||
|
visible_text=["文字A"],
|
||||||
|
spatial_layout="居中构图",
|
||||||
|
visual_attributes={"lighting": "明亮"},
|
||||||
|
),
|
||||||
|
timestamp=idx * 2.0,
|
||||||
|
frame_path=f"frames/l1_0_l2_0_l3_{idx}.jpg",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_l2(n_l3: int = 2) -> L2Node:
|
||||||
|
return L2Node(
|
||||||
|
id="l1_0_l2_0",
|
||||||
|
card=L2Card(
|
||||||
|
event_description="事件描述",
|
||||||
|
entities=["实体B"],
|
||||||
|
actions=["动作B"],
|
||||||
|
action_subjects=["主体B"],
|
||||||
|
visible_text=["文字B"],
|
||||||
|
spatial_relations="左右排列",
|
||||||
|
state_changes=None,
|
||||||
|
),
|
||||||
|
time_range=(0.0, 60.0),
|
||||||
|
children=[_make_l3(i) for i in range(n_l3)],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_l1(n_l2: int = 1, n_l3: int = 2) -> L1Node:
|
||||||
|
return L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=L1Card(
|
||||||
|
scene_summary="场景摘要",
|
||||||
|
main_setting="室内",
|
||||||
|
key_entities=["实体C"],
|
||||||
|
main_actions=["动作C"],
|
||||||
|
topic_keywords=["关键词"],
|
||||||
|
visible_text=["文字C"],
|
||||||
|
temporal_flow="从左到右",
|
||||||
|
),
|
||||||
|
time_range=(0.0, 600.0),
|
||||||
|
children=[_make_l2(n_l3) for _ in range(n_l2)],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_index(n_l1: int = 1) -> TreeIndex:
|
||||||
|
meta = IndexMeta(source_path="/test/video.mp4", modality="video")
|
||||||
|
return TreeIndex(metadata=meta, roots=[_make_l1() for _ in range(n_l1)])
|
||||||
|
|
||||||
|
|
||||||
|
class TestCards:
|
||||||
|
def test_l3_card_frozen(self):
|
||||||
|
card = L3Card(
|
||||||
|
frame_summary="desc",
|
||||||
|
visible_entities=[],
|
||||||
|
ongoing_actions=[],
|
||||||
|
visible_text=[],
|
||||||
|
spatial_layout="",
|
||||||
|
visual_attributes={},
|
||||||
|
)
|
||||||
|
with pytest.raises(AttributeError):
|
||||||
|
card.frame_summary = "changed"
|
||||||
|
|
||||||
|
def test_l2_card_fields(self):
|
||||||
|
card = L2Card(
|
||||||
|
event_description="evt",
|
||||||
|
entities=[],
|
||||||
|
actions=[],
|
||||||
|
action_subjects=[],
|
||||||
|
visible_text=[],
|
||||||
|
spatial_relations="",
|
||||||
|
state_changes=None,
|
||||||
|
)
|
||||||
|
assert card.event_description == "evt"
|
||||||
|
assert card.state_changes is None
|
||||||
|
|
||||||
|
def test_l1_card_fields(self):
|
||||||
|
card = L1Card(
|
||||||
|
scene_summary="scene",
|
||||||
|
main_setting="outdoor",
|
||||||
|
key_entities=["e"],
|
||||||
|
main_actions=["a"],
|
||||||
|
topic_keywords=["k"],
|
||||||
|
visible_text=["t"],
|
||||||
|
temporal_flow="flow",
|
||||||
|
)
|
||||||
|
assert card.scene_summary == "scene"
|
||||||
|
|
||||||
|
|
||||||
|
class TestNodes:
|
||||||
|
def test_l3_description_property(self):
|
||||||
|
node = _make_l3()
|
||||||
|
assert node.description == node.card.frame_summary
|
||||||
|
|
||||||
|
def test_l2_description_property(self):
|
||||||
|
node = _make_l2()
|
||||||
|
assert node.description == node.card.event_description
|
||||||
|
|
||||||
|
def test_l1_summary_property(self):
|
||||||
|
node = _make_l1()
|
||||||
|
assert node.summary == node.card.scene_summary
|
||||||
|
|
||||||
|
def test_l3_default_embedding_none(self):
|
||||||
|
node = _make_l3()
|
||||||
|
assert node.embedding is None
|
||||||
|
|
||||||
|
def test_l3_subtitle_default_none(self):
|
||||||
|
node = _make_l3()
|
||||||
|
assert node.subtitle is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestTreeIndex:
|
||||||
|
def test_is_embedded_false_by_default(self):
|
||||||
|
index = _make_index()
|
||||||
|
assert not index.is_embedded
|
||||||
|
|
||||||
|
def test_embed_all(self):
|
||||||
|
index = _make_index()
|
||||||
|
|
||||||
|
def fake_embed(texts):
|
||||||
|
if isinstance(texts, str):
|
||||||
|
texts = [texts]
|
||||||
|
return np.random.randn(len(texts), 4).astype(np.float32)
|
||||||
|
|
||||||
|
index.embed_all(fake_embed, "test-model", 4)
|
||||||
|
assert index.is_embedded
|
||||||
|
assert index.metadata.embed_model == "test-model"
|
||||||
|
assert index.metadata.embed_dim == 4
|
||||||
|
|
||||||
|
def test_l1_embeddings_shape(self):
|
||||||
|
index = _make_index(n_l1=2)
|
||||||
|
|
||||||
|
def fake_embed(texts):
|
||||||
|
if isinstance(texts, str):
|
||||||
|
texts = [texts]
|
||||||
|
return np.random.randn(len(texts), 4).astype(np.float32)
|
||||||
|
|
||||||
|
index.embed_all(fake_embed, "test-model", 4)
|
||||||
|
m = index.l1_embeddings()
|
||||||
|
assert m.shape == (2, 4)
|
||||||
|
|
||||||
|
def test_get_node(self):
|
||||||
|
index = _make_index()
|
||||||
|
node = index.get_node(0, 0, 1)
|
||||||
|
assert node.id == "l1_0_l2_0_l3_1"
|
||||||
|
|
||||||
|
def test_get_node_out_of_bounds(self):
|
||||||
|
index = _make_index()
|
||||||
|
with pytest.raises(IndexError):
|
||||||
|
index.get_node(99, 0, 0)
|
||||||
|
|
||||||
|
|
||||||
|
class TestSerialization:
|
||||||
|
def test_json_roundtrip(self, tmp_path):
|
||||||
|
index = _make_index()
|
||||||
|
path = tmp_path / "tree.json"
|
||||||
|
index.save_json(str(path))
|
||||||
|
loaded = TreeIndex.load_json(str(path))
|
||||||
|
assert len(loaded.roots) == 1
|
||||||
|
assert loaded.roots[0].id == "l1_0"
|
||||||
|
assert loaded.roots[0].card.scene_summary == "场景摘要"
|
||||||
|
assert loaded.roots[0].children[0].children[0].card.frame_summary == "帧0描述"
|
||||||
|
|
||||||
|
def test_json_roundtrip_with_embedding(self, tmp_path):
|
||||||
|
index = _make_index()
|
||||||
|
|
||||||
|
def fake_embed(texts):
|
||||||
|
if isinstance(texts, str):
|
||||||
|
texts = [texts]
|
||||||
|
return np.random.randn(len(texts), 4).astype(np.float32)
|
||||||
|
|
||||||
|
index.embed_all(fake_embed, "test-model", 4)
|
||||||
|
path = tmp_path / "tree_emb.json"
|
||||||
|
index.save_json(str(path), include_embedding=True)
|
||||||
|
loaded = TreeIndex.load_json(str(path))
|
||||||
|
assert loaded.is_embedded
|
||||||
|
np.testing.assert_array_almost_equal(
|
||||||
|
loaded.roots[0].embedding, index.roots[0].embedding, decimal=5
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_l1_json_roundtrip(self, tmp_path):
|
||||||
|
from app.tree.index import load_l1_json, save_l1_json
|
||||||
|
|
||||||
|
l1 = _make_l1()
|
||||||
|
path = tmp_path / "l1_0.json"
|
||||||
|
save_l1_json(str(path), l1)
|
||||||
|
loaded = load_l1_json(str(path))
|
||||||
|
assert loaded.id == "l1_0"
|
||||||
|
assert len(loaded.children) == 1
|
||||||
|
assert len(loaded.children[0].children) == 2
|
||||||
|
|
||||||
|
def test_id_uniqueness_validation(self, tmp_path):
|
||||||
|
"""重复 ID 在反序列化时应报错。"""
|
||||||
|
index = _make_index()
|
||||||
|
d = index.to_dict()
|
||||||
|
d["roots"].append(d["roots"][0])
|
||||||
|
path = tmp_path / "dup.json"
|
||||||
|
with open(path, "w") as f:
|
||||||
|
json.dump(d, f)
|
||||||
|
with pytest.raises(ValueError, match="重复"):
|
||||||
|
TreeIndex.load_json(str(path))
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
"""tests/unit/test_validate.py — 块验证纯决策函数的单元测试。"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.evolution.validate import classify_quadrants, compute_accuracy, pair_block
|
||||||
|
|
||||||
|
|
||||||
|
class TestPairBlock:
|
||||||
|
"""pair_block 逐题比对基线与候选的翻转统计测试。"""
|
||||||
|
|
||||||
|
def test_basic_flips(self) -> None:
|
||||||
|
"""基本翻转:一题从错到对(w)、一题从对到错(l)、一题不变。"""
|
||||||
|
baseline = {"q1": False, "q2": True, "q3": True}
|
||||||
|
candidate = {"q1": True, "q2": False, "q3": True}
|
||||||
|
result = pair_block(baseline, candidate, ["q1", "q2", "q3"])
|
||||||
|
assert result.w == 1
|
||||||
|
assert result.l == 1
|
||||||
|
assert result.observed == {
|
||||||
|
"q1": (False, True),
|
||||||
|
"q2": (True, False),
|
||||||
|
"q3": (True, True),
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_empty(self) -> None:
|
||||||
|
"""空输入应返回零翻转。"""
|
||||||
|
result = pair_block({}, {}, [])
|
||||||
|
assert result.w == 0 and result.l == 0
|
||||||
|
|
||||||
|
def test_all_wins(self) -> None:
|
||||||
|
"""全部从错到对的极端情况。"""
|
||||||
|
baseline = {"q1": False, "q2": False}
|
||||||
|
candidate = {"q1": True, "q2": True}
|
||||||
|
result = pair_block(baseline, candidate, ["q1", "q2"])
|
||||||
|
assert result.w == 2
|
||||||
|
assert result.l == 0
|
||||||
|
|
||||||
|
def test_all_losses(self) -> None:
|
||||||
|
"""全部从对到错的极端情况。"""
|
||||||
|
baseline = {"q1": True, "q2": True}
|
||||||
|
candidate = {"q1": False, "q2": False}
|
||||||
|
result = pair_block(baseline, candidate, ["q1", "q2"])
|
||||||
|
assert result.w == 0
|
||||||
|
assert result.l == 2
|
||||||
|
|
||||||
|
def test_subset_of_questions(self) -> None:
|
||||||
|
"""只对 question_ids 中指定的子集进行比对。"""
|
||||||
|
baseline = {"q1": False, "q2": True, "q3": True}
|
||||||
|
candidate = {"q1": True, "q2": False, "q3": True}
|
||||||
|
result = pair_block(baseline, candidate, ["q1"])
|
||||||
|
assert result.w == 1
|
||||||
|
assert result.l == 0
|
||||||
|
assert "q2" not in result.observed
|
||||||
|
|
||||||
|
|
||||||
|
class TestClassifyQuadrants:
|
||||||
|
"""classify_quadrants 四象限分类测试。"""
|
||||||
|
|
||||||
|
def test_all_four(self) -> None:
|
||||||
|
"""四个象限各有一题。"""
|
||||||
|
observed = {
|
||||||
|
"q1": (False, True),
|
||||||
|
"q2": (True, False),
|
||||||
|
"q3": (False, False),
|
||||||
|
"q4": (True, True),
|
||||||
|
}
|
||||||
|
qc = classify_quadrants(observed)
|
||||||
|
assert qc.improvements == ["q1"]
|
||||||
|
assert qc.regressions == ["q2"]
|
||||||
|
assert qc.persistent_fails == ["q3"]
|
||||||
|
assert qc.stable_successes == ["q4"]
|
||||||
|
|
||||||
|
def test_sorted_within_quadrant(self) -> None:
|
||||||
|
"""同象限内题目 ID 应按字典序排列。"""
|
||||||
|
observed = {"z": (False, True), "a": (False, True)}
|
||||||
|
qc = classify_quadrants(observed)
|
||||||
|
assert qc.improvements == ["a", "z"]
|
||||||
|
|
||||||
|
def test_empty_observed(self) -> None:
|
||||||
|
"""空输入应返回全空象限。"""
|
||||||
|
qc = classify_quadrants({})
|
||||||
|
assert qc.improvements == []
|
||||||
|
assert qc.regressions == []
|
||||||
|
assert qc.persistent_fails == []
|
||||||
|
assert qc.stable_successes == []
|
||||||
|
|
||||||
|
|
||||||
|
class TestComputeAccuracy:
|
||||||
|
"""compute_accuracy 准确率计算测试。"""
|
||||||
|
|
||||||
|
def test_basic(self) -> None:
|
||||||
|
"""一对一错,准确率 0.5。"""
|
||||||
|
assert compute_accuracy({"q1": True, "q2": False}, ["q1", "q2"]) == 0.5
|
||||||
|
|
||||||
|
def test_all_correct(self) -> None:
|
||||||
|
"""全部正确,准确率 1.0。"""
|
||||||
|
assert compute_accuracy({"q1": True, "q2": True}, ["q1", "q2"]) == 1.0
|
||||||
|
|
||||||
|
def test_all_wrong(self) -> None:
|
||||||
|
"""全部错误,准确率 0.0。"""
|
||||||
|
assert compute_accuracy({"q1": False, "q2": False}, ["q1", "q2"]) == 0.0
|
||||||
|
|
||||||
|
def test_empty_raises(self) -> None:
|
||||||
|
"""空题目列表应抛出 ZeroDivisionError。"""
|
||||||
|
with pytest.raises(ZeroDivisionError):
|
||||||
|
compute_accuracy({}, [])
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
"""质量校验模块单元测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.tree.index import (
|
||||||
|
IndexMeta,
|
||||||
|
L1Card,
|
||||||
|
L1Node,
|
||||||
|
L2Card,
|
||||||
|
L2Node,
|
||||||
|
L3Card,
|
||||||
|
L3Node,
|
||||||
|
TreeIndex,
|
||||||
|
)
|
||||||
|
from app.tree.verify import VerifyStats, _normalize, fuzzy_match, verify_tree
|
||||||
|
|
||||||
|
|
||||||
|
class TestNormalize:
|
||||||
|
def test_lowercase(self):
|
||||||
|
assert _normalize("Hello World") == "hello world"
|
||||||
|
|
||||||
|
def test_strip_punctuation(self):
|
||||||
|
assert _normalize("Hello, World!") == "hello world"
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert _normalize("") == ""
|
||||||
|
|
||||||
|
|
||||||
|
class TestFuzzyMatch:
|
||||||
|
def test_exact_match(self):
|
||||||
|
assert fuzzy_match("hello", "hello world")
|
||||||
|
|
||||||
|
def test_case_insensitive(self):
|
||||||
|
assert fuzzy_match("Hello", "say hello world")
|
||||||
|
|
||||||
|
def test_no_match(self):
|
||||||
|
assert not fuzzy_match("xyz", "hello world")
|
||||||
|
|
||||||
|
def test_none_entity(self):
|
||||||
|
assert not fuzzy_match(None, "hello")
|
||||||
|
|
||||||
|
def test_none_corpus(self):
|
||||||
|
assert not fuzzy_match("hello", None)
|
||||||
|
|
||||||
|
|
||||||
|
class TestVerifyTree:
|
||||||
|
def _make_tree(self):
|
||||||
|
"""构造一棵树,L2 有混合实体(有出处/无出处)。"""
|
||||||
|
l3_0 = L3Node(
|
||||||
|
id="l1_0_l2_0_l3_0",
|
||||||
|
card=L3Card(
|
||||||
|
frame_summary="一个运动员在跑步",
|
||||||
|
visible_entities=["运动员", "跑道"],
|
||||||
|
ongoing_actions=["跑步"],
|
||||||
|
visible_text=["Nike", "2024"],
|
||||||
|
spatial_layout="居中",
|
||||||
|
visual_attributes={},
|
||||||
|
),
|
||||||
|
timestamp=1.0,
|
||||||
|
subtitle="the athlete is running fast",
|
||||||
|
)
|
||||||
|
l3_1 = L3Node(
|
||||||
|
id="l1_0_l2_0_l3_1",
|
||||||
|
card=L3Card(
|
||||||
|
frame_summary="观众在欢呼",
|
||||||
|
visible_entities=["观众"],
|
||||||
|
ongoing_actions=["欢呼"],
|
||||||
|
visible_text=["Stadium"],
|
||||||
|
spatial_layout="广角",
|
||||||
|
visual_attributes={},
|
||||||
|
),
|
||||||
|
timestamp=3.0,
|
||||||
|
)
|
||||||
|
l2 = L2Node(
|
||||||
|
id="l1_0_l2_0",
|
||||||
|
card=L2Card(
|
||||||
|
event_description="比赛片段",
|
||||||
|
entities=["运动员", "裁判", "幻觉实体"], # "裁判"和"幻觉实体"无 L3 出处
|
||||||
|
actions=["跑步"],
|
||||||
|
action_subjects=["运动员"],
|
||||||
|
visible_text=["Nike", "不存在的文字"], # "不存在的文字"无 L3 出处
|
||||||
|
spatial_relations="",
|
||||||
|
state_changes=None,
|
||||||
|
),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l3_0, l3_1],
|
||||||
|
)
|
||||||
|
l1 = L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=L1Card(
|
||||||
|
scene_summary="体育比赛",
|
||||||
|
main_setting="体育场",
|
||||||
|
key_entities=["运动员", "不存在的人"], # "不存在的人"无出处
|
||||||
|
main_actions=["比赛"],
|
||||||
|
topic_keywords=["体育"],
|
||||||
|
visible_text=["Nike", "Ghost"], # "Ghost"无出处
|
||||||
|
temporal_flow="从左到右",
|
||||||
|
),
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2],
|
||||||
|
)
|
||||||
|
return TreeIndex(metadata=IndexMeta("/test.mp4", "video"), roots=[l1])
|
||||||
|
|
||||||
|
def test_removes_ungrounded_l2_entities(self):
|
||||||
|
index = self._make_tree()
|
||||||
|
stats = verify_tree(index)
|
||||||
|
l2 = index.roots[0].children[0]
|
||||||
|
assert "运动员" in l2.card.entities
|
||||||
|
assert "幻觉实体" not in l2.card.entities
|
||||||
|
assert stats.l2_entities_removed >= 1
|
||||||
|
|
||||||
|
def test_removes_ungrounded_l2_visible_text(self):
|
||||||
|
index = self._make_tree()
|
||||||
|
stats = verify_tree(index)
|
||||||
|
l2 = index.roots[0].children[0]
|
||||||
|
assert "Nike" in l2.card.visible_text
|
||||||
|
assert "不存在的文字" not in l2.card.visible_text
|
||||||
|
assert stats.l2_visible_text_removed >= 1
|
||||||
|
|
||||||
|
def test_removes_ungrounded_l1_visible_text(self):
|
||||||
|
index = self._make_tree()
|
||||||
|
stats = verify_tree(index)
|
||||||
|
l1 = index.roots[0]
|
||||||
|
assert "Nike" in l1.card.visible_text
|
||||||
|
assert "Ghost" not in l1.card.visible_text
|
||||||
|
assert stats.l1_visible_text_removed >= 1
|
||||||
|
|
||||||
|
def test_removes_ungrounded_l1_key_entities(self):
|
||||||
|
index = self._make_tree()
|
||||||
|
stats = verify_tree(index)
|
||||||
|
l1 = index.roots[0]
|
||||||
|
assert "运动员" in l1.card.key_entities
|
||||||
|
assert "不存在的人" not in l1.card.key_entities
|
||||||
|
assert stats.l1_key_entities_removed >= 1
|
||||||
|
|
||||||
|
def test_preserves_grounded_entities(self):
|
||||||
|
index = self._make_tree()
|
||||||
|
verify_tree(index)
|
||||||
|
l2 = index.roots[0].children[0]
|
||||||
|
assert "运动员" in l2.card.entities
|
||||||
|
|
||||||
|
def test_returns_verify_stats(self):
|
||||||
|
index = self._make_tree()
|
||||||
|
stats = verify_tree(index)
|
||||||
|
assert isinstance(stats, VerifyStats)
|
||||||
|
total_kept = stats.l2_entities_kept + stats.l1_key_entities_kept
|
||||||
|
assert total_kept > 0
|
||||||
|
|
||||||
|
def test_frozen_card_replaced(self):
|
||||||
|
"""验证 Card 被替换为新实例(frozen dataclass 不能原地修改)。"""
|
||||||
|
index = self._make_tree()
|
||||||
|
old_l2_card = index.roots[0].children[0].card
|
||||||
|
verify_tree(index)
|
||||||
|
new_l2_card = index.roots[0].children[0].card
|
||||||
|
# Card should be a different object if anything was removed
|
||||||
|
assert old_l2_card is not new_l2_card
|
||||||
@@ -0,0 +1,999 @@
|
|||||||
|
"""VideoTreeBuilder 单元测试。
|
||||||
|
|
||||||
|
测试覆盖:
|
||||||
|
- _segment_video: 时间切分
|
||||||
|
- _get_l2_clips: 片段切分
|
||||||
|
- _sample_representative_frames: 帧采样
|
||||||
|
- _parse_l3_cards: L3 JSON 解析 + fallback 条件
|
||||||
|
- _parse_l2_card: L2 JSON 解析
|
||||||
|
- _parse_l1_card: L1 JSON 解析
|
||||||
|
- _parse_l3_card_single: 单帧 L3 JSON 解析
|
||||||
|
- build: 完整构建流程(mock VLM/LLM)
|
||||||
|
- checkpoint/resume: 断点续跑机制
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.tree.config import TreeConfig
|
||||||
|
from app.tree.index import (
|
||||||
|
L1Card,
|
||||||
|
L1Node,
|
||||||
|
L2Card,
|
||||||
|
L2Node,
|
||||||
|
L3Card,
|
||||||
|
L3Node,
|
||||||
|
)
|
||||||
|
from app.tree.video_builder import VideoTreeBuilder
|
||||||
|
from core.types import LLMResponse
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Mock 提供者
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _make_llm_response(content: str) -> LLMResponse:
|
||||||
|
"""构造 LLMResponse 测试工具。"""
|
||||||
|
return LLMResponse(
|
||||||
|
content=content,
|
||||||
|
thinking="",
|
||||||
|
model="mock",
|
||||||
|
provider="mock",
|
||||||
|
prompt_tokens=10,
|
||||||
|
completion_tokens=10,
|
||||||
|
latency_ms=10,
|
||||||
|
ttft_ms=1.0,
|
||||||
|
max_inter_token_ms=1.0,
|
||||||
|
cache_hit=False,
|
||||||
|
call_id="mock-call",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _l3_card_dict(idx: int = 0) -> dict[str, Any]:
|
||||||
|
"""构造单个 L3Card 的字典表示。"""
|
||||||
|
return {
|
||||||
|
"frame_summary": f"帧 {idx} 的描述",
|
||||||
|
"visible_entities": ["实体A"],
|
||||||
|
"ongoing_actions": ["动作A"],
|
||||||
|
"visible_text": [],
|
||||||
|
"spatial_layout": "居中",
|
||||||
|
"visual_attributes": {
|
||||||
|
"lighting": "自然光",
|
||||||
|
"dominant_colors": ["白"],
|
||||||
|
"camera_angle": "正面",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _l2_card_dict() -> dict[str, Any]:
|
||||||
|
"""构造 L2Card 的字典表示。"""
|
||||||
|
return {
|
||||||
|
"event_description": "视频片段描述",
|
||||||
|
"entities": ["实体A"],
|
||||||
|
"actions": ["动作A"],
|
||||||
|
"action_subjects": ["主体A"],
|
||||||
|
"visible_text": [],
|
||||||
|
"spatial_relations": "居中",
|
||||||
|
"state_changes": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _l1_card_dict() -> dict[str, Any]:
|
||||||
|
"""构造 L1Card 的字典表示。"""
|
||||||
|
return {
|
||||||
|
"scene_summary": "场景摘要描述",
|
||||||
|
"main_setting": "室内",
|
||||||
|
"key_entities": ["实体A"],
|
||||||
|
"main_actions": ["动作A"],
|
||||||
|
"topic_keywords": ["关键词A"],
|
||||||
|
"visible_text": [],
|
||||||
|
"temporal_flow": "从开始到结束",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class MockVLMProvider:
|
||||||
|
"""模拟 VLM 提供者,根据 prompt 类型返回固定 JSON 响应。"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.calls: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
async def chat_with_images(
|
||||||
|
self,
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
images: list[str | Path],
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> LLMResponse:
|
||||||
|
"""根据 prompt 内容判断调用类型,返回对应 JSON。"""
|
||||||
|
self.calls.append({"messages": messages, "images": images})
|
||||||
|
content = messages[0]["content"]
|
||||||
|
n_images = len(images)
|
||||||
|
|
||||||
|
if "JSON 数组" in content:
|
||||||
|
# L3 batch prompt
|
||||||
|
cards = [_l3_card_dict(i) for i in range(n_images)]
|
||||||
|
return _make_llm_response(json.dumps(cards, ensure_ascii=False))
|
||||||
|
if "用一到两句话描述这帧" in content:
|
||||||
|
# L3 single prompt
|
||||||
|
return _make_llm_response(
|
||||||
|
json.dumps(_l3_card_dict(0), ensure_ascii=False),
|
||||||
|
)
|
||||||
|
# L2 prompt
|
||||||
|
return _make_llm_response(
|
||||||
|
json.dumps(_l2_card_dict(), ensure_ascii=False),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class MockLLMProvider:
|
||||||
|
"""模拟 LLM 提供者,返回固定 L1Card JSON 响应。"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.calls: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
async def chat(
|
||||||
|
self,
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> LLMResponse:
|
||||||
|
"""返回 L1Card JSON。"""
|
||||||
|
self.calls.append({"messages": messages})
|
||||||
|
return _make_llm_response(
|
||||||
|
json.dumps(_l1_card_dict(), ensure_ascii=False),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Fixtures
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def tree_config(tmp_path: Path) -> TreeConfig:
|
||||||
|
"""简化配置:10 秒视频→1 个 L1 段→2 个 L2 clip→每 clip 5 帧。"""
|
||||||
|
return TreeConfig(
|
||||||
|
l1_segment_duration=10.0,
|
||||||
|
l2_clip_duration=5.0,
|
||||||
|
l3_fps=1.0,
|
||||||
|
l2_representative_frames=2,
|
||||||
|
cache_dir=str(tmp_path / "cache"),
|
||||||
|
concurrency=4,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def mock_vlm() -> MockVLMProvider:
|
||||||
|
"""返回 MockVLMProvider 实例。"""
|
||||||
|
return MockVLMProvider()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def mock_llm() -> MockLLMProvider:
|
||||||
|
"""返回 MockLLMProvider 实例。"""
|
||||||
|
return MockLLMProvider()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def builder(
|
||||||
|
mock_vlm: MockVLMProvider,
|
||||||
|
mock_llm: MockLLMProvider,
|
||||||
|
tree_config: TreeConfig,
|
||||||
|
) -> VideoTreeBuilder:
|
||||||
|
"""构造带 mock 依赖的 VideoTreeBuilder。"""
|
||||||
|
return VideoTreeBuilder(vlm=mock_vlm, llm=mock_llm, config=tree_config)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 测试:_segment_video
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestSegmentVideo:
|
||||||
|
"""测试时间切分逻辑。"""
|
||||||
|
|
||||||
|
def test_exact_division(self, builder: VideoTreeBuilder) -> None:
|
||||||
|
"""总时长能被 L1 段时长整除时的切分。"""
|
||||||
|
ranges = builder._segment_video("dummy", duration_hint=20.0)
|
||||||
|
assert ranges == [(0.0, 10.0), (10.0, 20.0)]
|
||||||
|
|
||||||
|
def test_non_divisible(self, builder: VideoTreeBuilder) -> None:
|
||||||
|
"""总时长不能被整除时,末段截断。"""
|
||||||
|
ranges = builder._segment_video("dummy", duration_hint=15.0)
|
||||||
|
assert len(ranges) == 2
|
||||||
|
assert ranges[0] == (0.0, 10.0)
|
||||||
|
assert ranges[1] == (10.0, 15.0)
|
||||||
|
|
||||||
|
def test_short_video(self, builder: VideoTreeBuilder) -> None:
|
||||||
|
"""短视频(时长 < L1 段时长)产生单段。"""
|
||||||
|
ranges = builder._segment_video("dummy", duration_hint=5.0)
|
||||||
|
assert ranges == [(0.0, 5.0)]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 测试:_get_l2_clips
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetL2Clips:
|
||||||
|
"""测试 L2 clip 切分逻辑。"""
|
||||||
|
|
||||||
|
def test_basic_clips(self, builder: VideoTreeBuilder) -> None:
|
||||||
|
"""L1 区间能被 L2 步长整除。"""
|
||||||
|
clips = builder._get_l2_clips((0.0, 10.0))
|
||||||
|
assert clips == [(0.0, 5.0), (5.0, 10.0)]
|
||||||
|
|
||||||
|
def test_remainder_clip(self, builder: VideoTreeBuilder) -> None:
|
||||||
|
"""L1 区间不能被整除时,末段截断。"""
|
||||||
|
clips = builder._get_l2_clips((0.0, 7.0))
|
||||||
|
assert len(clips) == 2
|
||||||
|
assert clips[0] == (0.0, 5.0)
|
||||||
|
assert clips[1] == (5.0, 7.0)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 测试:_sample_representative_frames
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestSampleRepresentativeFrames:
|
||||||
|
"""测试帧采样逻辑。"""
|
||||||
|
|
||||||
|
def test_fewer_frames_than_n(self) -> None:
|
||||||
|
"""帧数不足时返回全部。"""
|
||||||
|
frames = [("a.jpg", 0.0), ("b.jpg", 1.0)]
|
||||||
|
result = VideoTreeBuilder._sample_representative_frames(frames, 5)
|
||||||
|
assert result == ["a.jpg", "b.jpg"]
|
||||||
|
|
||||||
|
def test_exact_n(self) -> None:
|
||||||
|
"""帧数恰等于 n 时返回全部。"""
|
||||||
|
frames = [("a.jpg", 0.0), ("b.jpg", 1.0), ("c.jpg", 2.0)]
|
||||||
|
result = VideoTreeBuilder._sample_representative_frames(frames, 3)
|
||||||
|
assert result == ["a.jpg", "b.jpg", "c.jpg"]
|
||||||
|
|
||||||
|
def test_uniform_sampling(self) -> None:
|
||||||
|
"""10 帧中采样 3 帧,应均匀分布。"""
|
||||||
|
frames = [(f"{i}.jpg", float(i)) for i in range(10)]
|
||||||
|
result = VideoTreeBuilder._sample_representative_frames(frames, 3)
|
||||||
|
# step = 10/3 = 3.33 → indices 0, 3, 6
|
||||||
|
assert result == ["0.jpg", "3.jpg", "6.jpg"]
|
||||||
|
|
||||||
|
def test_sampling_two_from_five(self) -> None:
|
||||||
|
"""5 帧中采样 2 帧。"""
|
||||||
|
frames = [(f"{i}.jpg", float(i)) for i in range(5)]
|
||||||
|
result = VideoTreeBuilder._sample_representative_frames(frames, 2)
|
||||||
|
# step = 5/2 = 2.5 → indices 0, 2
|
||||||
|
assert result == ["0.jpg", "2.jpg"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 测试:_parse_l3_cards
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseL3Cards:
|
||||||
|
"""测试 L3 批量 JSON 解析(保真算法 #2 的解析环节)。"""
|
||||||
|
|
||||||
|
def test_valid_json_array(self, builder: VideoTreeBuilder) -> None:
|
||||||
|
"""正常 JSON 数组解析成功。"""
|
||||||
|
raw = json.dumps([_l3_card_dict(i) for i in range(3)])
|
||||||
|
result = builder._parse_l3_cards(raw, 3)
|
||||||
|
assert result is not None
|
||||||
|
assert len(result) == 3
|
||||||
|
assert result[0].frame_summary == "帧 0 的描述"
|
||||||
|
assert result[2].frame_summary == "帧 2 的描述"
|
||||||
|
|
||||||
|
def test_count_mismatch_returns_none(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""数量不匹配 → 返回 None(触发 fallback)。"""
|
||||||
|
raw = json.dumps([_l3_card_dict(0), _l3_card_dict(1)])
|
||||||
|
result = builder._parse_l3_cards(raw, 3)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_missing_field_returns_none(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""必填字段缺失 → 整批次返回 None。"""
|
||||||
|
card = _l3_card_dict(0)
|
||||||
|
del card["frame_summary"]
|
||||||
|
raw = json.dumps([card])
|
||||||
|
result = builder._parse_l3_cards(raw, 1)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_invalid_json_returns_none(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""非法 JSON 返回 None。"""
|
||||||
|
result = builder._parse_l3_cards("not json at all", 1)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_json_in_code_block(self, builder: VideoTreeBuilder) -> None:
|
||||||
|
"""Markdown 代码块包裹的 JSON 也能解析。"""
|
||||||
|
inner = json.dumps([_l3_card_dict(0)])
|
||||||
|
raw = f"```json\n{inner}\n```"
|
||||||
|
result = builder._parse_l3_cards(raw, 1)
|
||||||
|
assert result is not None
|
||||||
|
assert len(result) == 1
|
||||||
|
|
||||||
|
def test_non_dict_item_returns_none(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""数组元素非 dict → 返回 None。"""
|
||||||
|
raw = json.dumps(["string_item"])
|
||||||
|
result = builder._parse_l3_cards(raw, 1)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 测试:_parse_l2_card
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseL2Card:
|
||||||
|
"""测试 L2 JSON 解析。"""
|
||||||
|
|
||||||
|
def test_valid_json(self, builder: VideoTreeBuilder) -> None:
|
||||||
|
"""正常 JSON 解析成功。"""
|
||||||
|
raw = json.dumps(_l2_card_dict(), ensure_ascii=False)
|
||||||
|
card = builder._parse_l2_card(raw)
|
||||||
|
assert card.event_description == "视频片段描述"
|
||||||
|
assert card.entities == ["实体A"]
|
||||||
|
assert card.state_changes is None
|
||||||
|
|
||||||
|
def test_invalid_json_fallback(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""JSON 解析失败 → 退化卡片。"""
|
||||||
|
card = builder._parse_l2_card("这是一段普通文字描述")
|
||||||
|
assert card.event_description == "这是一段普通文字描述"
|
||||||
|
assert card.entities == []
|
||||||
|
|
||||||
|
def test_with_state_changes(self, builder: VideoTreeBuilder) -> None:
|
||||||
|
"""state_changes 非 null 时正常解析。"""
|
||||||
|
d = _l2_card_dict()
|
||||||
|
d["state_changes"] = "从站立到坐下"
|
||||||
|
raw = json.dumps(d, ensure_ascii=False)
|
||||||
|
card = builder._parse_l2_card(raw)
|
||||||
|
assert card.state_changes == "从站立到坐下"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 测试:_parse_l1_card
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseL1Card:
|
||||||
|
"""测试 L1 JSON 解析。"""
|
||||||
|
|
||||||
|
def test_valid_json(self, builder: VideoTreeBuilder) -> None:
|
||||||
|
"""正常 JSON 解析成功。"""
|
||||||
|
raw = json.dumps(_l1_card_dict(), ensure_ascii=False)
|
||||||
|
card = builder._parse_l1_card(raw)
|
||||||
|
assert card.scene_summary == "场景摘要描述"
|
||||||
|
assert card.main_setting == "室内"
|
||||||
|
|
||||||
|
def test_invalid_json_fallback(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""JSON 解析失败 → 退化卡片。"""
|
||||||
|
card = builder._parse_l1_card("这是段落摘要")
|
||||||
|
assert card.scene_summary == "这是段落摘要"
|
||||||
|
assert card.key_entities == []
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 测试:_parse_l3_card_single
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseL3CardSingle:
|
||||||
|
"""测试单帧 L3 JSON 解析。"""
|
||||||
|
|
||||||
|
def test_valid_json(self, builder: VideoTreeBuilder) -> None:
|
||||||
|
"""正常 JSON 解析成功。"""
|
||||||
|
raw = json.dumps(_l3_card_dict(0), ensure_ascii=False)
|
||||||
|
card = builder._parse_l3_card_single(raw)
|
||||||
|
assert card.frame_summary == "帧 0 的描述"
|
||||||
|
|
||||||
|
def test_invalid_json_fallback(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""JSON 解析失败 → 退化卡片。"""
|
||||||
|
card = builder._parse_l3_card_single("一帧画面的描述")
|
||||||
|
assert card.frame_summary == "一帧画面的描述"
|
||||||
|
assert card.visible_entities == []
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 测试:_extract_json
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestExtractJson:
|
||||||
|
"""测试 JSON 提取辅助方法。"""
|
||||||
|
|
||||||
|
def test_plain_object(self) -> None:
|
||||||
|
"""直接 JSON 对象。"""
|
||||||
|
result = VideoTreeBuilder._extract_json('{"key": "value"}')
|
||||||
|
assert result == {"key": "value"}
|
||||||
|
|
||||||
|
def test_plain_array(self) -> None:
|
||||||
|
"""直接 JSON 数组。"""
|
||||||
|
result = VideoTreeBuilder._extract_json("[1, 2, 3]")
|
||||||
|
assert result == [1, 2, 3]
|
||||||
|
|
||||||
|
def test_code_block(self) -> None:
|
||||||
|
"""Markdown 代码块中的 JSON。"""
|
||||||
|
raw = '```json\n{"key": "value"}\n```'
|
||||||
|
result = VideoTreeBuilder._extract_json(raw)
|
||||||
|
assert result == {"key": "value"}
|
||||||
|
|
||||||
|
def test_surrounding_text(self) -> None:
|
||||||
|
"""JSON 前后有文字。"""
|
||||||
|
raw = 'Here is the result: {"key": "value"} end'
|
||||||
|
result = VideoTreeBuilder._extract_json(raw)
|
||||||
|
assert result == {"key": "value"}
|
||||||
|
|
||||||
|
def test_invalid(self) -> None:
|
||||||
|
"""完全无 JSON 内容。"""
|
||||||
|
result = VideoTreeBuilder._extract_json("no json here")
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 测试:完整构建流程
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _mock_ffmpeg_factory(tmp_path: Path):
|
||||||
|
"""创建 mock ffmpeg 帧提取函数。"""
|
||||||
|
|
||||||
|
def _mock_extract(
|
||||||
|
video_path: str,
|
||||||
|
ts: float,
|
||||||
|
out_path: str,
|
||||||
|
) -> bool:
|
||||||
|
Path(out_path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
Path(out_path).write_bytes(b"FAKE_JPEG")
|
||||||
|
return True
|
||||||
|
|
||||||
|
return _mock_extract
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildFullFlow:
|
||||||
|
"""测试完整构建流程(mock VLM/LLM/ffmpeg)。"""
|
||||||
|
|
||||||
|
def test_build_produces_correct_structure(
|
||||||
|
self,
|
||||||
|
mock_vlm: MockVLMProvider,
|
||||||
|
mock_llm: MockLLMProvider,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""10 秒视频 → 1 L1 → 2 L2 → 每 L2 约 5 帧 L3。"""
|
||||||
|
config = TreeConfig(
|
||||||
|
l1_segment_duration=10.0,
|
||||||
|
l2_clip_duration=5.0,
|
||||||
|
l3_fps=1.0,
|
||||||
|
l2_representative_frames=2,
|
||||||
|
cache_dir=str(tmp_path / "cache"),
|
||||||
|
concurrency=4,
|
||||||
|
)
|
||||||
|
builder = VideoTreeBuilder(
|
||||||
|
vlm=mock_vlm,
|
||||||
|
llm=mock_llm,
|
||||||
|
config=config,
|
||||||
|
)
|
||||||
|
|
||||||
|
dummy_video = tmp_path / "test_video.mp4"
|
||||||
|
dummy_video.write_bytes(b"FAKE")
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(
|
||||||
|
builder,
|
||||||
|
"_segment_video",
|
||||||
|
return_value=[(0.0, 10.0)],
|
||||||
|
),
|
||||||
|
patch.object(
|
||||||
|
builder,
|
||||||
|
"_ffmpeg_extract_frame",
|
||||||
|
side_effect=_mock_ffmpeg_factory(tmp_path),
|
||||||
|
),
|
||||||
|
):
|
||||||
|
index = builder.build(str(dummy_video))
|
||||||
|
|
||||||
|
# 结构校验
|
||||||
|
assert len(index.roots) == 1
|
||||||
|
l1 = index.roots[0]
|
||||||
|
assert l1.id == "l1_0"
|
||||||
|
assert l1.card.scene_summary == "场景摘要描述"
|
||||||
|
assert l1.time_range == (0.0, 10.0)
|
||||||
|
|
||||||
|
# 2 个 L2 clip
|
||||||
|
assert len(l1.children) == 2
|
||||||
|
for j, l2 in enumerate(l1.children):
|
||||||
|
assert l2.id == f"l1_0_l2_{j}"
|
||||||
|
assert l2.card.event_description == "视频片段描述"
|
||||||
|
# 5 帧/clip(5 秒 * 1 fps)
|
||||||
|
assert len(l2.children) == 5
|
||||||
|
for k, l3 in enumerate(l2.children):
|
||||||
|
assert l3.id == f"l1_0_l2_{j}_l3_{k}"
|
||||||
|
assert l3.card.frame_summary is not None
|
||||||
|
assert l3.frame_path is not None
|
||||||
|
assert l3.timestamp is not None
|
||||||
|
|
||||||
|
# 确认 VLM/LLM 被调用
|
||||||
|
# L2: 2 calls (one per clip)
|
||||||
|
# L3: 2 calls (one batch per clip, each batch has 5 frames)
|
||||||
|
# L1: 1 call
|
||||||
|
assert len(mock_vlm.calls) == 4 # 2 L2 + 2 L3 batches
|
||||||
|
assert len(mock_llm.calls) == 1 # 1 L1
|
||||||
|
|
||||||
|
# metadata 校验
|
||||||
|
assert index.metadata.source_path == str(dummy_video)
|
||||||
|
assert index.metadata.modality == "video"
|
||||||
|
|
||||||
|
def test_build_cleans_up_intermediate(
|
||||||
|
self,
|
||||||
|
mock_vlm: MockVLMProvider,
|
||||||
|
mock_llm: MockLLMProvider,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""构建成功后中间文件已清理。"""
|
||||||
|
config = TreeConfig(
|
||||||
|
l1_segment_duration=10.0,
|
||||||
|
l2_clip_duration=10.0,
|
||||||
|
l3_fps=1.0,
|
||||||
|
l2_representative_frames=2,
|
||||||
|
cache_dir=str(tmp_path / "cache"),
|
||||||
|
concurrency=4,
|
||||||
|
)
|
||||||
|
builder = VideoTreeBuilder(
|
||||||
|
vlm=mock_vlm,
|
||||||
|
llm=mock_llm,
|
||||||
|
config=config,
|
||||||
|
)
|
||||||
|
|
||||||
|
dummy_video = tmp_path / "test_video.mp4"
|
||||||
|
dummy_video.write_bytes(b"FAKE")
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(
|
||||||
|
builder,
|
||||||
|
"_segment_video",
|
||||||
|
return_value=[(0.0, 10.0)],
|
||||||
|
),
|
||||||
|
patch.object(
|
||||||
|
builder,
|
||||||
|
"_ffmpeg_extract_frame",
|
||||||
|
side_effect=_mock_ffmpeg_factory(tmp_path),
|
||||||
|
),
|
||||||
|
):
|
||||||
|
builder.build(str(dummy_video))
|
||||||
|
|
||||||
|
# 中间文件应已清理
|
||||||
|
progress_dir = tmp_path / "cache" / "progress"
|
||||||
|
inter_dir = tmp_path / "cache" / "intermediate" / "test_video"
|
||||||
|
assert not (progress_dir / "test_video.json").exists()
|
||||||
|
# intermediate 目录可能不存在或为空
|
||||||
|
if inter_dir.exists():
|
||||||
|
assert len(list(inter_dir.glob("l1_*.json"))) == 0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 测试:L3 fallback(保真算法 #2)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class MockVLMWithBatchFailure:
|
||||||
|
"""模拟批量 VLM 调用失败、单帧调用成功的 VLM 提供者。"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.calls: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
async def chat_with_images(
|
||||||
|
self,
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
images: list[str | Path],
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
parent_call_id: str | None = None,
|
||||||
|
) -> LLMResponse:
|
||||||
|
"""batch 返回无效 JSON,single 返回有效 JSON,L2 正常。"""
|
||||||
|
self.calls.append({"n_images": len(images)})
|
||||||
|
content = messages[0]["content"]
|
||||||
|
|
||||||
|
if "JSON 数组" in content:
|
||||||
|
# L3 batch: 返回无效 JSON 触发 fallback
|
||||||
|
return _make_llm_response("INVALID JSON OUTPUT")
|
||||||
|
if "用一到两句话描述这帧" in content:
|
||||||
|
# L3 single fallback: 有效 JSON
|
||||||
|
return _make_llm_response(
|
||||||
|
json.dumps(_l3_card_dict(0), ensure_ascii=False),
|
||||||
|
)
|
||||||
|
# L2: 有效 JSON
|
||||||
|
return _make_llm_response(
|
||||||
|
json.dumps(_l2_card_dict(), ensure_ascii=False),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestL3Fallback:
|
||||||
|
"""测试 L3 批量失败→逐帧 fallback(保真算法 #2)。"""
|
||||||
|
|
||||||
|
def test_fallback_to_single_frame(
|
||||||
|
self,
|
||||||
|
mock_llm: MockLLMProvider,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""批量 VLM 解析失败时,逐帧 fallback 仍能构建完整树。"""
|
||||||
|
vlm = MockVLMWithBatchFailure()
|
||||||
|
config = TreeConfig(
|
||||||
|
l1_segment_duration=10.0,
|
||||||
|
l2_clip_duration=10.0,
|
||||||
|
l3_fps=1.0,
|
||||||
|
l2_representative_frames=2,
|
||||||
|
cache_dir=str(tmp_path / "cache"),
|
||||||
|
concurrency=4,
|
||||||
|
)
|
||||||
|
builder = VideoTreeBuilder(vlm=vlm, llm=mock_llm, config=config)
|
||||||
|
|
||||||
|
dummy_video = tmp_path / "test_video.mp4"
|
||||||
|
dummy_video.write_bytes(b"FAKE")
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(
|
||||||
|
builder,
|
||||||
|
"_segment_video",
|
||||||
|
return_value=[(0.0, 10.0)],
|
||||||
|
),
|
||||||
|
patch.object(
|
||||||
|
builder,
|
||||||
|
"_ffmpeg_extract_frame",
|
||||||
|
side_effect=_mock_ffmpeg_factory(tmp_path),
|
||||||
|
),
|
||||||
|
):
|
||||||
|
index = builder.build(str(dummy_video))
|
||||||
|
|
||||||
|
# 结构仍然完整
|
||||||
|
assert len(index.roots) == 1
|
||||||
|
l1 = index.roots[0]
|
||||||
|
assert len(l1.children) == 1 # 1 clip (10s clip)
|
||||||
|
l2 = l1.children[0]
|
||||||
|
|
||||||
|
# 10 frames (10s * 1fps), all from single-frame fallback
|
||||||
|
assert len(l2.children) == 10
|
||||||
|
for l3 in l2.children:
|
||||||
|
assert l3.card.frame_summary == "帧 0 的描述"
|
||||||
|
|
||||||
|
# VLM 调用次数:1 L2 + 1 batch(fail) + 10 single = 12
|
||||||
|
# 但 batch 分为 2 batches (10 frames / 5 per batch)
|
||||||
|
# 所以: 1 L2 + 2 batch(fail) + 10 single = 13
|
||||||
|
assert len(vlm.calls) == 13
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 测试:断点续跑(保真算法 #3)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestCheckpointResume:
|
||||||
|
"""测试断点续跑机制。"""
|
||||||
|
|
||||||
|
def test_save_and_load_progress(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""进度文件保存和加载。"""
|
||||||
|
stem = "test_video"
|
||||||
|
builder._save_progress(stem, total_l1=3, finished_l1_ids={0, 1})
|
||||||
|
progress = builder._load_progress(stem)
|
||||||
|
|
||||||
|
assert progress is not None
|
||||||
|
assert progress["total_l1"] == 3
|
||||||
|
assert sorted(progress["finished_l1_ids"]) == [0, 1]
|
||||||
|
assert "created_at" in progress
|
||||||
|
assert "updated_at" in progress
|
||||||
|
|
||||||
|
def test_load_nonexistent_progress(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""不存在的进度文件返回 None。"""
|
||||||
|
assert builder._load_progress("nonexistent") is None
|
||||||
|
|
||||||
|
def test_save_and_load_l1_intermediate(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""L1 中间结果保存和加载。"""
|
||||||
|
stem = "test_video"
|
||||||
|
l1_card = L1Card(
|
||||||
|
scene_summary="测试摘要",
|
||||||
|
main_setting="测试场景",
|
||||||
|
key_entities=["实体"],
|
||||||
|
main_actions=["动作"],
|
||||||
|
topic_keywords=["关键词"],
|
||||||
|
visible_text=[],
|
||||||
|
temporal_flow="测试流向",
|
||||||
|
)
|
||||||
|
l2_card = L2Card(
|
||||||
|
event_description="事件描述",
|
||||||
|
entities=["实体"],
|
||||||
|
actions=["动作"],
|
||||||
|
action_subjects=["主体"],
|
||||||
|
visible_text=[],
|
||||||
|
spatial_relations="居中",
|
||||||
|
state_changes=None,
|
||||||
|
)
|
||||||
|
l3_card = L3Card(
|
||||||
|
frame_summary="帧描述",
|
||||||
|
visible_entities=["实体"],
|
||||||
|
ongoing_actions=["动作"],
|
||||||
|
visible_text=[],
|
||||||
|
spatial_layout="居中",
|
||||||
|
visual_attributes={"lighting": "自然光"},
|
||||||
|
)
|
||||||
|
l3_node = L3Node(
|
||||||
|
id="l1_0_l2_0_l3_0",
|
||||||
|
card=l3_card,
|
||||||
|
timestamp=1.0,
|
||||||
|
frame_path="/tmp/frame.jpg",
|
||||||
|
)
|
||||||
|
l2_node = L2Node(
|
||||||
|
id="l1_0_l2_0",
|
||||||
|
card=l2_card,
|
||||||
|
time_range=(0.0, 5.0),
|
||||||
|
children=[l3_node],
|
||||||
|
)
|
||||||
|
l1_node = L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=l1_card,
|
||||||
|
time_range=(0.0, 10.0),
|
||||||
|
children=[l2_node],
|
||||||
|
)
|
||||||
|
|
||||||
|
builder._save_l1_intermediate(stem, l1_node, 0)
|
||||||
|
assert builder._has_l1_intermediate(stem, 0)
|
||||||
|
assert not builder._has_l1_intermediate(stem, 1)
|
||||||
|
|
||||||
|
loaded = builder._load_l1_intermediate(stem, 0)
|
||||||
|
assert loaded is not None
|
||||||
|
assert loaded.id == "l1_0"
|
||||||
|
assert loaded.card.scene_summary == "测试摘要"
|
||||||
|
assert len(loaded.children) == 1
|
||||||
|
assert loaded.children[0].id == "l1_0_l2_0"
|
||||||
|
|
||||||
|
def test_cleanup_removes_files(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""清理函数删除进度文件和中间 JSON。"""
|
||||||
|
stem = "test_video"
|
||||||
|
builder._save_progress(stem, total_l1=1, finished_l1_ids={0})
|
||||||
|
|
||||||
|
# 创建一个假的中间文件
|
||||||
|
inter_dir = builder._intermediate_dir(stem)
|
||||||
|
inter_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
(inter_dir / "l1_0.json").write_text("{}")
|
||||||
|
|
||||||
|
builder._cleanup_intermediate_and_progress(stem)
|
||||||
|
|
||||||
|
assert not builder._progress_path(stem).is_file()
|
||||||
|
assert not (inter_dir / "l1_0.json").is_file()
|
||||||
|
|
||||||
|
def test_resume_skips_finished_segments(
|
||||||
|
self,
|
||||||
|
mock_vlm: MockVLMProvider,
|
||||||
|
mock_llm: MockLLMProvider,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""断点续跑:跳过已完成的 L1 段,只构建未完成的段。"""
|
||||||
|
config = TreeConfig(
|
||||||
|
l1_segment_duration=5.0,
|
||||||
|
l2_clip_duration=5.0,
|
||||||
|
l3_fps=1.0,
|
||||||
|
l2_representative_frames=2,
|
||||||
|
cache_dir=str(tmp_path / "cache"),
|
||||||
|
concurrency=4,
|
||||||
|
)
|
||||||
|
builder = VideoTreeBuilder(
|
||||||
|
vlm=mock_vlm,
|
||||||
|
llm=mock_llm,
|
||||||
|
config=config,
|
||||||
|
)
|
||||||
|
|
||||||
|
source_id = "test_resume"
|
||||||
|
|
||||||
|
# Phase 1: 手动创建 L1_0 的中间结果(模拟已完成)
|
||||||
|
l1_card = L1Card(
|
||||||
|
scene_summary="已完成的段",
|
||||||
|
main_setting="场景A",
|
||||||
|
key_entities=["实体A"],
|
||||||
|
main_actions=["动作A"],
|
||||||
|
topic_keywords=["关键词A"],
|
||||||
|
visible_text=[],
|
||||||
|
temporal_flow="流向A",
|
||||||
|
)
|
||||||
|
l2_card = L2Card(
|
||||||
|
event_description="已完成的片段",
|
||||||
|
entities=["实体A"],
|
||||||
|
actions=["动作A"],
|
||||||
|
action_subjects=["主体A"],
|
||||||
|
visible_text=[],
|
||||||
|
spatial_relations="居中",
|
||||||
|
state_changes=None,
|
||||||
|
)
|
||||||
|
l3_card = L3Card(
|
||||||
|
frame_summary="已完成的帧",
|
||||||
|
visible_entities=["实体A"],
|
||||||
|
ongoing_actions=["动作A"],
|
||||||
|
visible_text=[],
|
||||||
|
spatial_layout="居中",
|
||||||
|
visual_attributes={"lighting": "自然光"},
|
||||||
|
)
|
||||||
|
l3_node = L3Node(
|
||||||
|
id="l1_0_l2_0_l3_0",
|
||||||
|
card=l3_card,
|
||||||
|
timestamp=1.0,
|
||||||
|
)
|
||||||
|
l2_node = L2Node(
|
||||||
|
id="l1_0_l2_0",
|
||||||
|
card=l2_card,
|
||||||
|
time_range=(0.0, 5.0),
|
||||||
|
children=[l3_node],
|
||||||
|
)
|
||||||
|
l1_node_0 = L1Node(
|
||||||
|
id="l1_0",
|
||||||
|
card=l1_card,
|
||||||
|
time_range=(0.0, 5.0),
|
||||||
|
children=[l2_node],
|
||||||
|
)
|
||||||
|
builder._save_l1_intermediate(source_id, l1_node_0, 0)
|
||||||
|
|
||||||
|
# Phase 2: 创建进度文件(标记 L1_0 完成)
|
||||||
|
builder._save_progress(
|
||||||
|
source_id,
|
||||||
|
total_l1=2,
|
||||||
|
finished_l1_ids={0},
|
||||||
|
)
|
||||||
|
|
||||||
|
# Phase 3: 构建(应跳过 L1_0,只构建 L1_1)
|
||||||
|
dummy_video = tmp_path / f"{source_id}.mp4"
|
||||||
|
dummy_video.write_bytes(b"FAKE")
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(
|
||||||
|
builder,
|
||||||
|
"_segment_video",
|
||||||
|
return_value=[(0.0, 5.0), (5.0, 10.0)],
|
||||||
|
),
|
||||||
|
patch.object(
|
||||||
|
builder,
|
||||||
|
"_ffmpeg_extract_frame",
|
||||||
|
side_effect=_mock_ffmpeg_factory(tmp_path),
|
||||||
|
),
|
||||||
|
):
|
||||||
|
index = builder.build(str(dummy_video))
|
||||||
|
|
||||||
|
# Phase 4: 验证
|
||||||
|
assert len(index.roots) == 2
|
||||||
|
|
||||||
|
# L1_0 来自中间结果
|
||||||
|
assert index.roots[0].id == "l1_0"
|
||||||
|
assert index.roots[0].card.scene_summary == "已完成的段"
|
||||||
|
|
||||||
|
# L1_1 是新构建的
|
||||||
|
assert index.roots[1].id == "l1_1"
|
||||||
|
assert index.roots[1].card.scene_summary == "场景摘要描述"
|
||||||
|
|
||||||
|
# VLM 只被调用了 L1_1 的部分(1 L2 + 1 L3 batch)
|
||||||
|
assert len(mock_vlm.calls) == 2 # 1 L2 + 1 L3
|
||||||
|
assert len(mock_llm.calls) == 1 # 1 L1
|
||||||
|
|
||||||
|
# 构建完成后,进度和中间文件已清理
|
||||||
|
assert not builder._progress_path(source_id).is_file()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 测试:字幕注入
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestSubtitleInjection:
|
||||||
|
"""测试字幕注入功能。"""
|
||||||
|
|
||||||
|
def test_build_subtitle_block_with_entries(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""有匹配字幕时返回字幕文本块。"""
|
||||||
|
from app.tree.subtitle import SRTEntry
|
||||||
|
|
||||||
|
entries = [
|
||||||
|
SRTEntry(start=1.0, end=3.0, text="你好世界"),
|
||||||
|
SRTEntry(start=4.0, end=6.0, text="再见"),
|
||||||
|
]
|
||||||
|
block = builder._build_subtitle_block(entries, (0.0, 5.0))
|
||||||
|
assert "字幕信息" in block
|
||||||
|
assert "你好世界" in block
|
||||||
|
|
||||||
|
def test_build_subtitle_block_no_match(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""无匹配字幕时返回空字符串。"""
|
||||||
|
from app.tree.subtitle import SRTEntry
|
||||||
|
|
||||||
|
entries = [SRTEntry(start=100.0, end=110.0, text="远处的字幕")]
|
||||||
|
block = builder._build_subtitle_block(entries, (0.0, 5.0))
|
||||||
|
assert block == ""
|
||||||
|
|
||||||
|
def test_build_subtitle_block_none_entries(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""srt_entries 为 None 时返回空字符串。"""
|
||||||
|
block = builder._build_subtitle_block(None, (0.0, 5.0))
|
||||||
|
assert block == ""
|
||||||
|
|
||||||
|
def test_build_subtitle_block_point_range(
|
||||||
|
self,
|
||||||
|
builder: VideoTreeBuilder,
|
||||||
|
) -> None:
|
||||||
|
"""点时间范围(单帧)自动扩展窗口。"""
|
||||||
|
from app.tree.subtitle import SRTEntry
|
||||||
|
|
||||||
|
entries = [SRTEntry(start=4.0, end=6.0, text="窗口内字幕")]
|
||||||
|
# 点时间 5.0,窗口 ±5.0 → (0.0, 10.0)
|
||||||
|
block = builder._build_subtitle_block(entries, (5.0, 5.0))
|
||||||
|
assert "窗口内字幕" in block
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 测试:URL 与 stem 辅助
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestHelpers:
|
||||||
|
"""测试静态辅助方法。"""
|
||||||
|
|
||||||
|
def test_is_url_true(self) -> None:
|
||||||
|
"""HTTP/HTTPS URL 识别。"""
|
||||||
|
assert VideoTreeBuilder._is_url("https://example.com/video.mp4")
|
||||||
|
assert VideoTreeBuilder._is_url("http://example.com/video.mp4")
|
||||||
|
|
||||||
|
def test_is_url_false(self) -> None:
|
||||||
|
"""本地路径不是 URL。"""
|
||||||
|
assert not VideoTreeBuilder._is_url("/path/to/video.mp4")
|
||||||
|
assert not VideoTreeBuilder._is_url("video.mp4")
|
||||||
|
|
||||||
|
def test_source_stem_local(self) -> None:
|
||||||
|
"""本地文件的 stem。"""
|
||||||
|
assert VideoTreeBuilder._source_stem("/path/to/my_video.mp4") == "my_video"
|
||||||
|
|
||||||
|
def test_source_stem_youtube(self) -> None:
|
||||||
|
"""YouTube URL 的 stem 为视频 ID。"""
|
||||||
|
stem = VideoTreeBuilder._source_stem(
|
||||||
|
"https://www.youtube.com/watch?v=dQw4w9WgXcQ",
|
||||||
|
)
|
||||||
|
assert stem == "dQw4w9WgXcQ"
|
||||||
|
|
||||||
|
def test_source_stem_long_name(self) -> None:
|
||||||
|
"""超长文件名截断到 64 字符。"""
|
||||||
|
long_name = "a" * 100 + ".mp4"
|
||||||
|
stem = VideoTreeBuilder._source_stem(f"/path/{long_name}")
|
||||||
|
assert len(stem) == 64
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
"""VLM 适配器单元测试。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from adapters.vlm import GovernedVLMClient
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
class TestGovernedVLMClientProtocol:
|
||||||
|
def test_has_chat_with_images(self):
|
||||||
|
assert hasattr(GovernedVLMClient, "chat_with_images")
|
||||||
|
|
||||||
|
def test_satisfies_vlm_protocol(self):
|
||||||
|
"""GovernedVLMClient 应满足 VLMProvider Protocol。"""
|
||||||
|
assert hasattr(GovernedVLMClient, "chat_with_images")
|
||||||
|
|
||||||
|
|
||||||
|
class TestImageEncoding:
|
||||||
|
def test_encode_jpeg(self, tmp_path: Path):
|
||||||
|
img = tmp_path / "test.jpg"
|
||||||
|
img.write_bytes(b"\xff\xd8\xff\xe0fake_jpeg_data")
|
||||||
|
result = GovernedVLMClient._encode_image(img)
|
||||||
|
assert result.startswith("data:image/jpeg;base64,")
|
||||||
|
|
||||||
|
def test_encode_png(self, tmp_path: Path):
|
||||||
|
img = tmp_path / "test.png"
|
||||||
|
img.write_bytes(b"\x89PNG\r\n\x1a\nfake_png_data")
|
||||||
|
result = GovernedVLMClient._encode_image(img)
|
||||||
|
assert result.startswith("data:image/png;base64,")
|
||||||
|
|
||||||
|
|
||||||
|
class TestInjectImages:
|
||||||
|
def test_inject_single_image(self, tmp_path: Path):
|
||||||
|
img = tmp_path / "frame.jpg"
|
||||||
|
img.write_bytes(b"\xff\xd8\xff\xe0data")
|
||||||
|
messages = [{"role": "user", "content": "描述这帧画面"}]
|
||||||
|
result = GovernedVLMClient._inject_images(messages, [img])
|
||||||
|
|
||||||
|
assert len(result) == 1
|
||||||
|
content = result[0]["content"]
|
||||||
|
assert isinstance(content, list)
|
||||||
|
assert len(content) == 2 # 1 image + 1 text
|
||||||
|
assert content[0]["type"] == "image_url"
|
||||||
|
assert content[1]["type"] == "text"
|
||||||
|
assert content[1]["text"] == "描述这帧画面"
|
||||||
|
|
||||||
|
def test_inject_does_not_mutate_original(self, tmp_path: Path):
|
||||||
|
img = tmp_path / "frame.jpg"
|
||||||
|
img.write_bytes(b"\xff\xd8\xff\xe0data")
|
||||||
|
messages = [{"role": "user", "content": "text"}]
|
||||||
|
original_content = messages[0]["content"]
|
||||||
|
GovernedVLMClient._inject_images(messages, [img])
|
||||||
|
assert messages[0]["content"] == original_content # 原列表未变
|
||||||
|
|
||||||
|
def test_no_images_passthrough(self):
|
||||||
|
messages = [{"role": "user", "content": "hello"}]
|
||||||
|
result = GovernedVLMClient._inject_images(messages, [])
|
||||||
|
assert result[0]["content"] == "hello"
|
||||||
|
|
||||||
|
def test_multiple_images(self, tmp_path: Path):
|
||||||
|
imgs = []
|
||||||
|
for i in range(3):
|
||||||
|
img = tmp_path / f"frame_{i}.jpg"
|
||||||
|
img.write_bytes(b"\xff\xd8\xff\xe0data")
|
||||||
|
imgs.append(img)
|
||||||
|
messages = [{"role": "user", "content": "描述"}]
|
||||||
|
result = GovernedVLMClient._inject_images(messages, imgs)
|
||||||
|
content = result[0]["content"]
|
||||||
|
assert len(content) == 4 # 3 images + 1 text
|
||||||
Executable
+365
@@ -0,0 +1,365 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""一次性格式转换:TRM4 flat tree.json -> TRM5 TreeIndex JSON。
|
||||||
|
|
||||||
|
用法: python tools/convert_flat_to_treeindex.py <src_dir> <dst_dir>
|
||||||
|
|
||||||
|
遍历 src_dir 下每个 video_id 子目录中的 tree.json(TRM4 flat 格式),
|
||||||
|
转换为 TRM5 TreeIndex 嵌套格式并写入 dst_dir 对应子目录。
|
||||||
|
|
||||||
|
app/core/adapters 不 import 此脚本。迁移完成后归档至 tools/archived/。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Card 字段默认值(处理 TRM4 可能缺失的字段)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_L3_CARD_DEFAULTS: dict[str, Any] = {
|
||||||
|
"frame_summary": "",
|
||||||
|
"visible_entities": [],
|
||||||
|
"ongoing_actions": [],
|
||||||
|
"visible_text": [],
|
||||||
|
"spatial_layout": "",
|
||||||
|
"visual_attributes": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
_L2_CARD_DEFAULTS: dict[str, Any] = {
|
||||||
|
"event_description": "",
|
||||||
|
"entities": [],
|
||||||
|
"actions": [],
|
||||||
|
"action_subjects": [],
|
||||||
|
"visible_text": [],
|
||||||
|
"spatial_relations": "",
|
||||||
|
"state_changes": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
_L1_CARD_DEFAULTS: dict[str, Any] = {
|
||||||
|
"scene_summary": "",
|
||||||
|
"main_setting": "",
|
||||||
|
"key_entities": [],
|
||||||
|
"main_actions": [],
|
||||||
|
"topic_keywords": [],
|
||||||
|
"visible_text": [],
|
||||||
|
"temporal_flow": "",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Card 构建辅助
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _build_card(raw_card: dict[str, Any], defaults: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
"""从 TRM4 原始 card 字典构建 TRM5 card,缺失字段用默认值填充。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
raw_card: TRM4 tree.json 中节点的 card 字典。
|
||||||
|
defaults: 该层级的默认值字典。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
仅包含目标字段的 card 字典(字段集合与 defaults 一致)。
|
||||||
|
"""
|
||||||
|
return {key: raw_card.get(key, default) for key, default in defaults.items()}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 节点转换
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _convert_l3(
|
||||||
|
node: dict[str, Any],
|
||||||
|
video_id: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""将 TRM4 flat L3 节点转换为 TRM5 嵌套 L3 节点。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node: TRM4 flat 格式的 L3 节点字典。
|
||||||
|
video_id: 视频 ID,用于计算 frame_path 的相对路径后缀。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
TRM5 格式的 L3 节点字典。
|
||||||
|
"""
|
||||||
|
node_id: str = node["node_id"]
|
||||||
|
raw_card = node.get("card") or {}
|
||||||
|
card = _build_card(raw_card, _L3_CARD_DEFAULTS)
|
||||||
|
|
||||||
|
# frame_path: frames/{suffix}.jpg,suffix = node_id 去掉 video_id 前缀 + 下划线
|
||||||
|
prefix = f"{video_id}_"
|
||||||
|
suffix = node_id[len(prefix) :] if node_id.startswith(prefix) else node_id
|
||||||
|
frame_path = f"frames/{suffix}.jpg"
|
||||||
|
|
||||||
|
return {
|
||||||
|
"id": node_id,
|
||||||
|
"card": card,
|
||||||
|
"timestamp": node.get("frame_timestamp"),
|
||||||
|
"frame_path": frame_path,
|
||||||
|
"subtitle": node.get("subtitle"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _convert_l2(
|
||||||
|
node: dict[str, Any],
|
||||||
|
l3_children: list[dict[str, Any]],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""将 TRM4 flat L2 节点转换为 TRM5 嵌套 L2 节点。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node: TRM4 flat 格式的 L2 节点字典。
|
||||||
|
l3_children: 已转换的 L3 子节点列表(按 time_range 排序)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
TRM5 格式的 L2 节点字典。
|
||||||
|
"""
|
||||||
|
raw_card = node.get("card") or {}
|
||||||
|
card = _build_card(raw_card, _L2_CARD_DEFAULTS)
|
||||||
|
time_range = node.get("time_range")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"id": node["node_id"],
|
||||||
|
"card": card,
|
||||||
|
"time_range": time_range,
|
||||||
|
"children": l3_children,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _convert_l1(
|
||||||
|
node: dict[str, Any],
|
||||||
|
l2_children: list[dict[str, Any]],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""将 TRM4 flat L1 节点转换为 TRM5 嵌套 L1 节点。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node: TRM4 flat 格式的 L1 节点字典。
|
||||||
|
l2_children: 已转换的 L2 子节点列表(按 time_range 排序)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
TRM5 格式的 L1 节点字典。
|
||||||
|
"""
|
||||||
|
raw_card = node.get("card") or {}
|
||||||
|
card = _build_card(raw_card, _L1_CARD_DEFAULTS)
|
||||||
|
time_range = node.get("time_range")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"id": node["node_id"],
|
||||||
|
"card": card,
|
||||||
|
"time_range": time_range,
|
||||||
|
"children": l2_children,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 排序辅助
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _sort_key_time_range(node: dict[str, Any]) -> float:
|
||||||
|
"""按 time_range 的起始时间排序。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
node: TRM4 节点字典。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
起始时间(float),无 time_range 时返回 0.0。
|
||||||
|
"""
|
||||||
|
tr = node.get("time_range")
|
||||||
|
if tr and len(tr) >= 1:
|
||||||
|
return float(tr[0])
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def _sort_key_timestamp(converted: dict[str, Any]) -> float:
|
||||||
|
"""按 timestamp 排序(L3 转换后的字典)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
converted: 已转换的 TRM5 L3 节点字典。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
timestamp(float),无值时返回 0.0。
|
||||||
|
"""
|
||||||
|
ts = converted.get("timestamp")
|
||||||
|
return float(ts) if ts is not None else 0.0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 单棵树转换
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def convert_single_tree(flat_data: dict[str, Any], source_path: str) -> dict[str, Any]:
|
||||||
|
"""将单个 TRM4 flat tree.json 转换为 TRM5 TreeIndex 字典。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
flat_data: TRM4 flat tree.json 解析后的字典。
|
||||||
|
source_path: 原始数据路径(写入 metadata.source_path)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
TRM5 TreeIndex 格式的字典(可直接 json.dump 或传入 TreeIndex.from_dict)。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
ValueError: 无法从 flat_data 中提取 video_id。
|
||||||
|
"""
|
||||||
|
video_id = flat_data.get("video_id") or flat_data.get("videoID")
|
||||||
|
if not video_id:
|
||||||
|
raise ValueError("flat tree.json 中缺少 video_id / videoID 字段")
|
||||||
|
|
||||||
|
nodes: dict[str, dict[str, Any]] = flat_data.get("nodes", {})
|
||||||
|
|
||||||
|
# Phase 1: 按层级分组
|
||||||
|
l1_nodes: list[dict[str, Any]] = []
|
||||||
|
l2_nodes: list[dict[str, Any]] = []
|
||||||
|
l3_nodes: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
for node in nodes.values():
|
||||||
|
level = node.get("level")
|
||||||
|
if level == 1:
|
||||||
|
l1_nodes.append(node)
|
||||||
|
elif level == 2:
|
||||||
|
l2_nodes.append(node)
|
||||||
|
elif level == 3:
|
||||||
|
l3_nodes.append(node)
|
||||||
|
|
||||||
|
# Phase 2: 构建 parent -> children 映射
|
||||||
|
# L3 按 parent_id 分组
|
||||||
|
l3_by_parent: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for n in l3_nodes:
|
||||||
|
pid = n.get("parent_id", "")
|
||||||
|
l3_by_parent.setdefault(pid, []).append(n)
|
||||||
|
|
||||||
|
# L2 按 parent_id 分组
|
||||||
|
l2_by_parent: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for n in l2_nodes:
|
||||||
|
pid = n.get("parent_id", "")
|
||||||
|
l2_by_parent.setdefault(pid, []).append(n)
|
||||||
|
|
||||||
|
# Phase 3: 自底向上构建嵌套结构
|
||||||
|
# 转换 L2 -> 附带转换后的 L3 children
|
||||||
|
converted_l2_by_id: dict[str, dict[str, Any]] = {}
|
||||||
|
for l2 in l2_nodes:
|
||||||
|
l2_id = l2["node_id"]
|
||||||
|
raw_l3_children = l3_by_parent.get(l2_id, [])
|
||||||
|
# 先转换 L3,再按 timestamp 排序
|
||||||
|
converted_l3 = [_convert_l3(n, video_id) for n in raw_l3_children]
|
||||||
|
converted_l3.sort(key=_sort_key_timestamp)
|
||||||
|
converted_l2_by_id[l2_id] = _convert_l2(l2, converted_l3)
|
||||||
|
|
||||||
|
# 转换 L1 -> 附带转换后的 L2 children
|
||||||
|
roots: list[dict[str, Any]] = []
|
||||||
|
l1_nodes.sort(key=_sort_key_time_range)
|
||||||
|
|
||||||
|
for l1 in l1_nodes:
|
||||||
|
l1_id = l1["node_id"]
|
||||||
|
raw_l2_children = l2_by_parent.get(l1_id, [])
|
||||||
|
raw_l2_children.sort(key=_sort_key_time_range)
|
||||||
|
l2_children = [converted_l2_by_id[n["node_id"]] for n in raw_l2_children]
|
||||||
|
roots.append(_convert_l1(l1, l2_children))
|
||||||
|
|
||||||
|
# Phase 4: 构建 TreeIndex 字典
|
||||||
|
return {
|
||||||
|
"metadata": {
|
||||||
|
"source_path": source_path,
|
||||||
|
"modality": "video",
|
||||||
|
"created_at": datetime.now().isoformat(),
|
||||||
|
},
|
||||||
|
"roots": roots,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 批量转换入口
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def convert_directory(src_dir: str, dst_dir: str) -> tuple[int, int]:
|
||||||
|
"""批量转换目录下所有 TRM4 tree.json 到 TRM5 TreeIndex 格式。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
src_dir: 源目录(TRM4 store/videos/),其下每个子目录含 tree.json。
|
||||||
|
dst_dir: 目标目录(TRM5 store/videos/),保持同名子目录结构。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
(成功数, 失败数) 元组。
|
||||||
|
"""
|
||||||
|
src_path = Path(src_dir)
|
||||||
|
dst_path = Path(dst_dir)
|
||||||
|
|
||||||
|
if not src_path.is_dir():
|
||||||
|
print(f"错误: 源目录不存在: {src_dir}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
success_count = 0
|
||||||
|
fail_count = 0
|
||||||
|
|
||||||
|
tree_files = sorted(src_path.glob("*/tree.json"))
|
||||||
|
total = len(tree_files)
|
||||||
|
print(f"发现 {total} 个 tree.json 待转换")
|
||||||
|
|
||||||
|
for idx, tree_file in enumerate(tree_files, 1):
|
||||||
|
video_id = tree_file.parent.name
|
||||||
|
out_dir = dst_path / video_id
|
||||||
|
out_file = out_dir / "tree.json"
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(tree_file, encoding="utf-8") as f:
|
||||||
|
flat_data = json.load(f)
|
||||||
|
|
||||||
|
result = convert_single_tree(flat_data, source_path=str(tree_file))
|
||||||
|
|
||||||
|
out_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
with open(out_file, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
|
# 统计节点数
|
||||||
|
n_l1 = len(result["roots"])
|
||||||
|
n_l2 = sum(len(r["children"]) for r in result["roots"])
|
||||||
|
n_l3 = sum(len(l2["children"]) for r in result["roots"] for l2 in r["children"])
|
||||||
|
print(f"[{idx}/{total}] {video_id}: L1={n_l1}, L2={n_l2}, L3={n_l3}")
|
||||||
|
success_count += 1
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[{idx}/{total}] {video_id}: 失败 - {e}", file=sys.stderr)
|
||||||
|
fail_count += 1
|
||||||
|
|
||||||
|
return success_count, fail_count
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# CLI 入口
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
"""CLI 入口:解析参数并执行批量转换。"""
|
||||||
|
if len(sys.argv) != 3:
|
||||||
|
print(
|
||||||
|
"用法: python tools/convert_flat_to_treeindex.py <src_dir> <dst_dir>",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
print(" src_dir: TRM4 store/videos/ 目录(含 video_id/tree.json)", file=sys.stderr)
|
||||||
|
print(" dst_dir: TRM5 store/videos/ 目标目录", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
src_dir = sys.argv[1]
|
||||||
|
dst_dir = sys.argv[2]
|
||||||
|
|
||||||
|
print(f"源目录: {src_dir}")
|
||||||
|
print(f"目标目录: {dst_dir}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
success, fail = convert_directory(src_dir, dst_dir)
|
||||||
|
|
||||||
|
print()
|
||||||
|
print(f"转换完成: 成功 {success}, 失败 {fail}")
|
||||||
|
if fail > 0:
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user