feat(harness): workspace.py — Workspace lifecycle + VersionedSkillStore/PromptStore
- ResolvedPaths frozen dataclass: store_dir, videos_dir, questions_dir, skills_dir, prompts_dir, workspace_dir, db_path, analyses_dir, runs_dir - init_workspace: create ws + copy seed weights from store - init_workspace_from_seed: create from seed with fail-fast questions check - load_manifest / resolve_paths: manifest I/O + path resolution (skills/prompts resolve to workspace, videos/questions to store) - update_manifest: key whitelist validation - record_run: idempotent history append + per-video wiki dirs - read_best / update_best: best pointer independent of current - list_video_ids: videos with tree.json - archive_workspace: move to .archive/<name>-<ts> - VersionedSkillStore: implements core/evolution/protocols.py::SkillStore - VersionedPromptStore: implements core/evolution/protocols.py::PromptStore - 21 tests all passing (incl. Protocol compliance checks) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,478 @@
|
||||
"""Workspace 生命周期管理 + manifest 读写 + Protocol 实现。
|
||||
|
||||
Workspace 是一次实验的独立工作区,通过 manifest.json 引用 Store 中的
|
||||
特定版本资源并记录实验过程。Skills/Prompts 权重拷入 workspace 本地,
|
||||
训练产物只进 workspace 不污染 Store。
|
||||
|
||||
VersionedSkillStore / VersionedPromptStore 实现 core/evolution/protocols.py
|
||||
中定义的只读端口,供 core/ 层以 Protocol 方式读取技能和提示词。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from app.harness.store import read_seed
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedPaths:
|
||||
"""manifest 解析后的绝对路径集合。
|
||||
|
||||
属性:
|
||||
store_dir: Store 根目录绝对路径。
|
||||
videos_dir: 视频数据目录。
|
||||
questions_dir: 当前引用的题目目录。
|
||||
skills_dir: 当前引用的 Skill 版本目录(workspace 内)。
|
||||
prompts_dir: 当前引用的 Prompt 版本目录(workspace 内)。
|
||||
workspace_dir: Workspace 根目录。
|
||||
db_path: harness.db 路径。
|
||||
analyses_dir: 分析报告目录。
|
||||
runs_dir: 运行临时状态目录。
|
||||
"""
|
||||
|
||||
store_dir: Path
|
||||
videos_dir: Path
|
||||
questions_dir: Path
|
||||
skills_dir: Path
|
||||
prompts_dir: Path
|
||||
workspace_dir: Path
|
||||
db_path: Path
|
||||
analyses_dir: Path
|
||||
runs_dir: Path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 内部工具
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MANIFEST_CURRENT_KEYS = {"videos", "questions", "skills", "prompts"}
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
"""返回当前 UTC 时间的 ISO 格式字符串。"""
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Workspace 核心函数
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _scaffold_workspace(
|
||||
workspace_dir: Path,
|
||||
store_dir: Path,
|
||||
questions: str,
|
||||
skills_version: str,
|
||||
prompts_version: str,
|
||||
) -> None:
|
||||
"""写 manifest + 建 analyses/runs 目录(不拷权重;权重由调用方按来源拷入)。
|
||||
|
||||
参数:
|
||||
workspace_dir: 目标 workspace(由调用方保证不存在)。
|
||||
store_dir: Store 根目录。
|
||||
questions: 题目相对路径,如 ``'benchmarks/Video-MME'``。
|
||||
skills_version: manifest.current.skills 初始版本号。
|
||||
prompts_version: manifest.current.prompts 初始版本号。
|
||||
|
||||
关键实现:
|
||||
不依赖任何外部资源源(store 中的 skills/prompts 是否存在不在此校验),
|
||||
因此可被 init_workspace 与种子初始化复用;store 引用以相对路径写入 manifest。
|
||||
"""
|
||||
workspace_dir.mkdir(parents=True)
|
||||
(workspace_dir / "analyses").mkdir()
|
||||
(workspace_dir / "runs").mkdir()
|
||||
|
||||
store_abs = store_dir.resolve()
|
||||
store_rel = os.path.relpath(store_abs, workspace_dir.resolve())
|
||||
|
||||
manifest = {
|
||||
"name": workspace_dir.name,
|
||||
"created_at": _now_iso(),
|
||||
"store": store_rel,
|
||||
"current": {
|
||||
"videos": "videos",
|
||||
"questions": f"questions/{questions}",
|
||||
"skills": f"skills/{skills_version}",
|
||||
"prompts": f"prompts/{prompts_version}",
|
||||
},
|
||||
"history": [],
|
||||
}
|
||||
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
def init_workspace(
|
||||
workspace_dir: Path,
|
||||
store_dir: Path,
|
||||
questions: str,
|
||||
skills_version: str,
|
||||
prompts_version: str,
|
||||
) -> None:
|
||||
"""创建 Workspace 目录并写入初始 manifest.json,拷贝种子权重。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 目标路径(不得已存在)。
|
||||
store_dir: Store 根目录。
|
||||
questions: 题目在 questions/ 下的相对路径,如 ``"benchmarks/Video-MME"``。
|
||||
skills_version: Skills 版本号,如 ``"v1"``。
|
||||
prompts_version: Prompts 版本号,如 ``"v1"``。
|
||||
|
||||
异常:
|
||||
FileExistsError: Workspace 目录已存在。
|
||||
FileNotFoundError: 引用的资源在 Store 中不存在。
|
||||
"""
|
||||
if workspace_dir.exists():
|
||||
raise FileExistsError(f"Workspace 已存在: {workspace_dir}")
|
||||
|
||||
store_abs = store_dir.resolve()
|
||||
refs = {
|
||||
"skills": f"skills/{skills_version}",
|
||||
"prompts": f"prompts/{prompts_version}",
|
||||
"questions": f"questions/{questions}",
|
||||
}
|
||||
for label, rel in refs.items():
|
||||
full = store_abs / rel
|
||||
if not full.is_dir():
|
||||
raise FileNotFoundError(f"Store 中不存在 {label}: {full}")
|
||||
|
||||
_scaffold_workspace(workspace_dir, store_dir, questions, skills_version, prompts_version)
|
||||
|
||||
# 拷种子权重进 workspace:v2+ 训练产物只进 workspace,不污染 store
|
||||
shutil.copytree(store_abs / refs["skills"], workspace_dir / refs["skills"])
|
||||
shutil.copytree(store_abs / refs["prompts"], workspace_dir / refs["prompts"])
|
||||
logger.info("Workspace 初始化完成: {}", workspace_dir)
|
||||
|
||||
|
||||
def init_workspace_from_seed(
|
||||
workspace_dir: Path,
|
||||
store_dir: Path,
|
||||
seed_name: str,
|
||||
questions: str,
|
||||
) -> str:
|
||||
"""从种子全新建 workspace:拷权重 -> v1、baseline.db -> harness.db、读 baseline_run_id。
|
||||
|
||||
参数:
|
||||
workspace_dir: 目标 workspace(不得已存在)。
|
||||
store_dir: Store 根目录。
|
||||
seed_name: 种子名(store/seeds 下)。
|
||||
questions: 题目相对路径,如 ``'benchmarks/Video-MME'``。
|
||||
|
||||
返回:
|
||||
baseline_run_id(供 build_pools 使用)。
|
||||
|
||||
异常:
|
||||
FileExistsError: workspace 已存在。
|
||||
FileNotFoundError: 种子不存在(由 read_seed 抛出),或 questions ref 目录不存在。
|
||||
|
||||
关键实现:
|
||||
破坏性/创建操作前先校验 questions ref 存在:fresh 路径在 runner 侧已先
|
||||
归档旧 ws,若到 build_pools 才发现 questions 缺失则旧 ws 已被毁;
|
||||
故在此尽早报错(fail-fast),让新 ws 在创建前失败。
|
||||
"""
|
||||
if workspace_dir.exists():
|
||||
raise FileExistsError(f"Workspace 已存在: {workspace_dir}")
|
||||
|
||||
# 校验种子存在 + 取 baseline_run_id
|
||||
meta = read_seed(store_dir, seed_name)
|
||||
|
||||
# fail-fast:校验 questions ref 存在
|
||||
questions_ref = store_dir / "questions" / questions
|
||||
if not questions_ref.is_dir():
|
||||
raise FileNotFoundError(f"questions ref 目录不存在: {questions_ref}")
|
||||
|
||||
seed_dir = store_dir / "seeds" / seed_name
|
||||
_scaffold_workspace(workspace_dir, store_dir, questions, "v1", "v1")
|
||||
shutil.copytree(seed_dir / "skills", workspace_dir / "skills" / "v1")
|
||||
shutil.copytree(seed_dir / "prompts", workspace_dir / "prompts" / "v1")
|
||||
shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db")
|
||||
|
||||
logger.info("Workspace 从种子 '{}' 初始化完成: {}", seed_name, workspace_dir)
|
||||
return meta["baseline_run_id"]
|
||||
|
||||
|
||||
def load_manifest(workspace_dir: Path) -> dict:
|
||||
"""读取并返回 workspace 的 manifest.json。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
|
||||
返回:
|
||||
manifest 字典。
|
||||
|
||||
异常:
|
||||
FileNotFoundError: manifest.json 不存在。
|
||||
"""
|
||||
manifest_path = workspace_dir / "manifest.json"
|
||||
if not manifest_path.exists():
|
||||
raise FileNotFoundError(f"manifest.json 不存在: {manifest_path}")
|
||||
return json.loads(manifest_path.read_text())
|
||||
|
||||
|
||||
def resolve_paths(workspace_dir: Path) -> ResolvedPaths:
|
||||
"""读取 manifest,解析 current 中所有资源的绝对路径。
|
||||
|
||||
skills_dir/prompts_dir 解析到 workspace(非 store),
|
||||
videos_dir/questions_dir 解析到 store。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
|
||||
返回:
|
||||
ResolvedPaths 实例,包含所有资源的绝对路径。
|
||||
"""
|
||||
manifest = load_manifest(workspace_dir)
|
||||
ws_abs = workspace_dir.resolve()
|
||||
store_abs = (ws_abs / manifest["store"]).resolve()
|
||||
current = manifest["current"]
|
||||
return ResolvedPaths(
|
||||
store_dir=store_abs,
|
||||
videos_dir=store_abs / current["videos"],
|
||||
questions_dir=store_abs / current["questions"],
|
||||
skills_dir=ws_abs / current["skills"],
|
||||
prompts_dir=ws_abs / current["prompts"],
|
||||
workspace_dir=ws_abs,
|
||||
db_path=ws_abs / "harness.db",
|
||||
analyses_dir=ws_abs / "analyses",
|
||||
runs_dir=ws_abs / "runs",
|
||||
)
|
||||
|
||||
|
||||
def list_video_ids(workspace_dir: Path) -> list[str]:
|
||||
"""列出 workspace 引用的所有视频 ID(含 tree.json 的子目录名)。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
|
||||
返回:
|
||||
排序后的视频 ID 列表。
|
||||
"""
|
||||
paths = resolve_paths(workspace_dir)
|
||||
video_ids = []
|
||||
for entry in paths.videos_dir.iterdir():
|
||||
if entry.is_dir() and (entry / "tree.json").exists():
|
||||
video_ids.append(entry.name)
|
||||
return sorted(video_ids)
|
||||
|
||||
|
||||
def update_manifest(workspace_dir: Path, **version_updates: str) -> None:
|
||||
"""更新 manifest 的 current 字段。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
**version_updates: 要更新的字段及其新值,如 ``skills="skills/v2"``。
|
||||
|
||||
异常:
|
||||
KeyError: 更新的字段不在 current 允许的 key 白名单中。
|
||||
"""
|
||||
invalid = set(version_updates) - _MANIFEST_CURRENT_KEYS
|
||||
if invalid:
|
||||
raise KeyError(f"无效的 manifest current 字段: {invalid}")
|
||||
manifest = load_manifest(workspace_dir)
|
||||
manifest["current"].update(version_updates)
|
||||
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
def record_run(workspace_dir: Path, run_id: str) -> Path:
|
||||
"""将 current 版本快照追加到 manifest history,创建 run 目录和 per-video wiki 目录。
|
||||
|
||||
幂等:同 run_id 不重复追加 history(长跑中断后重启 / held-out 复用 run_id 时)。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
run_id: 本次运行的唯一标识,如 ``"run_001"``。
|
||||
|
||||
返回:
|
||||
创建的 run 目录路径。
|
||||
"""
|
||||
manifest = load_manifest(workspace_dir)
|
||||
current = manifest["current"]
|
||||
|
||||
# 幂等:同 run_id 不重复追加 history
|
||||
if not any(h["run_id"] == run_id for h in manifest["history"]):
|
||||
manifest["history"].append(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"started_at": _now_iso(),
|
||||
"skills": current["skills"],
|
||||
"prompts": current["prompts"],
|
||||
"questions": current["questions"],
|
||||
}
|
||||
)
|
||||
(workspace_dir / "manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2)
|
||||
)
|
||||
|
||||
run_dir = workspace_dir / "runs" / run_id
|
||||
# exist_ok:同 run_id 重跑时 run 目录已存在不应崩溃
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
for video_id in list_video_ids(workspace_dir):
|
||||
(run_dir / video_id / "wiki").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.debug("Run 已记录: {}", run_id)
|
||||
return run_dir
|
||||
|
||||
|
||||
def read_best(workspace_dir: Path) -> dict | None:
|
||||
"""读取 manifest 的 best 指针,未设置时返回 None。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
|
||||
返回:
|
||||
best 字典(skills/prompts/val_acc/run_id/epoch),未设置时 None。
|
||||
"""
|
||||
return load_manifest(workspace_dir).get("best")
|
||||
|
||||
|
||||
def update_best(
|
||||
workspace_dir: Path,
|
||||
skills: str,
|
||||
prompts: str,
|
||||
val_acc: float,
|
||||
run_id: str,
|
||||
epoch: int,
|
||||
) -> None:
|
||||
"""写入 manifest 的 best 指针(历史最优版本快照,与 current 平级)。
|
||||
|
||||
best 独立于 current——更新 best 不影响 current。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
skills: 最优 skills 版本完整 ref,如 ``'skills/v2'``。
|
||||
prompts: 最优 prompts 版本完整 ref,如 ``'prompts/v2'``。
|
||||
val_acc: 该版本验证集准确率。
|
||||
run_id: 该版本验证 run_id。
|
||||
epoch: 达成该最优的轮次。
|
||||
"""
|
||||
manifest = load_manifest(workspace_dir)
|
||||
manifest["best"] = {
|
||||
"skills": skills,
|
||||
"prompts": prompts,
|
||||
"val_acc": val_acc,
|
||||
"run_id": run_id,
|
||||
"epoch": epoch,
|
||||
}
|
||||
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
logger.info("Best 已更新: val_acc={}, run={}, epoch={}", val_acc, run_id, epoch)
|
||||
|
||||
|
||||
def archive_workspace(workspace_dir: Path) -> Path:
|
||||
"""把 workspace 整体移动到同级 .archive/<name>-<ts>,返回归档路径。
|
||||
|
||||
参数:
|
||||
workspace_dir: 要归档的 Workspace 根目录。
|
||||
|
||||
返回:
|
||||
归档后的目标路径。
|
||||
|
||||
异常:
|
||||
FileNotFoundError: workspace 不存在。
|
||||
"""
|
||||
if not workspace_dir.exists():
|
||||
raise FileNotFoundError(f"workspace 不存在: {workspace_dir}")
|
||||
|
||||
archive_root = workspace_dir.parent / ".archive"
|
||||
archive_root.mkdir(exist_ok=True)
|
||||
ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
|
||||
target = archive_root / f"{workspace_dir.name}-{ts}"
|
||||
shutil.move(str(workspace_dir), str(target))
|
||||
|
||||
logger.info("Workspace 已归档: {} -> {}", workspace_dir, target)
|
||||
return target
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Protocol 实现:VersionedSkillStore / VersionedPromptStore
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VersionedSkillStore:
|
||||
"""版本化技能读取端口实现。
|
||||
|
||||
满足 ``core/evolution/protocols.py::SkillStore`` Protocol。
|
||||
从指定的 skills 版本目录读取 ``.md`` 文件。
|
||||
|
||||
参数:
|
||||
skills_dir: skills 版本目录绝对路径(如 ``workspace/skills/v1``)。
|
||||
"""
|
||||
|
||||
def __init__(self, skills_dir: Path) -> None:
|
||||
if not skills_dir.is_dir():
|
||||
raise FileNotFoundError(f"Skills 目录不存在: {skills_dir}")
|
||||
self._dir = skills_dir
|
||||
|
||||
def read_skill(self, filename: str) -> str:
|
||||
"""读取指定 skill 文件的全文内容。
|
||||
|
||||
参数:
|
||||
filename: skill 文件名,如 ``'temporal-reasoning.md'``。
|
||||
|
||||
返回:
|
||||
文件全文内容。
|
||||
|
||||
异常:
|
||||
FileNotFoundError: 文件不存在。
|
||||
"""
|
||||
path = self._dir / filename
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Skill 文件不存在: {path}")
|
||||
return path.read_text()
|
||||
|
||||
def list_skill_files(self) -> list[str]:
|
||||
"""列出当前版本所有 skill 文件名。
|
||||
|
||||
返回:
|
||||
文件名列表(排序)。
|
||||
"""
|
||||
return sorted(entry.name for entry in self._dir.iterdir() if entry.is_file())
|
||||
|
||||
|
||||
class VersionedPromptStore:
|
||||
"""版本化提示词读取端口实现。
|
||||
|
||||
满足 ``core/evolution/protocols.py::PromptStore`` Protocol。
|
||||
从指定的 prompts 版本目录读取 ``.md`` 文件。
|
||||
|
||||
参数:
|
||||
prompts_dir: prompts 版本目录绝对路径(如 ``workspace/prompts/v1``)。
|
||||
"""
|
||||
|
||||
def __init__(self, prompts_dir: Path) -> None:
|
||||
if not prompts_dir.is_dir():
|
||||
raise FileNotFoundError(f"Prompts 目录不存在: {prompts_dir}")
|
||||
self._dir = prompts_dir
|
||||
|
||||
def read_prompt(self, filename: str) -> str:
|
||||
"""读取指定 prompt 文件的全文内容。
|
||||
|
||||
参数:
|
||||
filename: prompt 文件名,如 ``'system.md'``。
|
||||
|
||||
返回:
|
||||
文件全文内容。
|
||||
|
||||
异常:
|
||||
FileNotFoundError: 文件不存在。
|
||||
"""
|
||||
path = self._dir / filename
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Prompt 文件不存在: {path}")
|
||||
return path.read_text()
|
||||
|
||||
def list_prompt_files(self) -> list[str]:
|
||||
"""列出当前版本所有 prompt 文件名。
|
||||
|
||||
返回:
|
||||
文件名列表(排序)。
|
||||
"""
|
||||
return sorted(entry.name for entry in self._dir.iterdir() if entry.is_file())
|
||||
@@ -0,0 +1,519 @@
|
||||
"""app/harness/workspace 单元测试。
|
||||
|
||||
覆盖 Workspace 生命周期管理、manifest 读写、
|
||||
VersionedSkillStore / VersionedPromptStore Protocol 合规。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.harness.workspace import (
|
||||
ResolvedPaths,
|
||||
VersionedPromptStore,
|
||||
VersionedSkillStore,
|
||||
archive_workspace,
|
||||
init_workspace,
|
||||
init_workspace_from_seed,
|
||||
list_video_ids,
|
||||
load_manifest,
|
||||
read_best,
|
||||
record_run,
|
||||
resolve_paths,
|
||||
update_best,
|
||||
update_manifest,
|
||||
)
|
||||
from core.evolution.protocols import PromptStore, SkillStore
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def store_dir(tmp_path: Path) -> Path:
|
||||
"""构建一个最小 Store 目录结构供测试使用。"""
|
||||
sd = tmp_path / "store"
|
||||
sd.mkdir()
|
||||
|
||||
# videos:两个含 tree.json 的视频目录
|
||||
for vid in ("vid_A", "vid_B"):
|
||||
vdir = sd / "videos" / vid
|
||||
vdir.mkdir(parents=True)
|
||||
(vdir / "tree.json").write_text("{}")
|
||||
|
||||
# skills/v1 + prompts/v1
|
||||
s1 = sd / "skills" / "v1"
|
||||
s1.mkdir(parents=True)
|
||||
(s1 / "temporal-reasoning.md").write_text("skill content A")
|
||||
(s1 / "spatial-analysis.md").write_text("skill content B")
|
||||
|
||||
p1 = sd / "prompts" / "v1"
|
||||
p1.mkdir(parents=True)
|
||||
(p1 / "system.md").write_text("system prompt v1")
|
||||
(p1 / "extract.md").write_text("extract prompt v1")
|
||||
|
||||
# questions/benchmarks/Video-MME
|
||||
q = sd / "questions" / "benchmarks" / "Video-MME"
|
||||
q.mkdir(parents=True)
|
||||
(q / "q1.json").write_text('{"id": "q1"}')
|
||||
|
||||
# seed "initial"
|
||||
seed_dir = sd / "seeds" / "initial"
|
||||
seed_dir.mkdir(parents=True)
|
||||
shutil.copytree(s1, seed_dir / "skills")
|
||||
shutil.copytree(p1, seed_dir / "prompts")
|
||||
baseline_db = seed_dir / "baseline.db"
|
||||
baseline_db.write_bytes(b"")
|
||||
(seed_dir / "seed.json").write_text(
|
||||
json.dumps({"baseline_run_id": "baseline-001", "parent": None})
|
||||
)
|
||||
return sd
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def workspace_dir(tmp_path: Path) -> Path:
|
||||
"""返回一个不存在的 workspace 路径。"""
|
||||
return tmp_path / "ws_test"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ResolvedPaths
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolved_paths_frozen() -> None:
|
||||
"""ResolvedPaths 实例应不可变。"""
|
||||
rp = ResolvedPaths(
|
||||
store_dir=Path("/s"),
|
||||
videos_dir=Path("/v"),
|
||||
questions_dir=Path("/q"),
|
||||
skills_dir=Path("/sk"),
|
||||
prompts_dir=Path("/p"),
|
||||
workspace_dir=Path("/w"),
|
||||
db_path=Path("/d"),
|
||||
analyses_dir=Path("/a"),
|
||||
runs_dir=Path("/r"),
|
||||
)
|
||||
with pytest.raises(AttributeError):
|
||||
rp.store_dir = Path("/other") # type: ignore[misc]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# init_workspace
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_init_workspace(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""init_workspace 应创建 manifest、拷贝权重。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
|
||||
# manifest 存在
|
||||
manifest = json.loads((workspace_dir / "manifest.json").read_text())
|
||||
assert manifest["current"]["skills"] == "skills/v1"
|
||||
assert manifest["current"]["prompts"] == "prompts/v1"
|
||||
assert manifest["current"]["questions"] == "questions/benchmarks/Video-MME"
|
||||
|
||||
# 权重已拷入 workspace
|
||||
assert (workspace_dir / "skills" / "v1" / "temporal-reasoning.md").exists()
|
||||
assert (workspace_dir / "prompts" / "v1" / "system.md").exists()
|
||||
|
||||
# analyses, runs 目录已创建
|
||||
assert (workspace_dir / "analyses").is_dir()
|
||||
assert (workspace_dir / "runs").is_dir()
|
||||
|
||||
# 重复创建应报错
|
||||
with pytest.raises(FileExistsError):
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# init_workspace_from_seed
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_init_workspace_from_seed(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""从种子初始化应拷权重 + baseline.db + 返回 baseline_run_id。"""
|
||||
run_id = init_workspace_from_seed(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
seed_name="initial",
|
||||
questions="benchmarks/Video-MME",
|
||||
)
|
||||
assert run_id == "baseline-001"
|
||||
|
||||
# 权重在 workspace
|
||||
assert (workspace_dir / "skills" / "v1" / "temporal-reasoning.md").exists()
|
||||
assert (workspace_dir / "prompts" / "v1" / "system.md").exists()
|
||||
|
||||
# baseline.db -> harness.db
|
||||
assert (workspace_dir / "harness.db").exists()
|
||||
|
||||
# manifest 正确
|
||||
manifest = json.loads((workspace_dir / "manifest.json").read_text())
|
||||
assert manifest["current"]["skills"] == "skills/v1"
|
||||
|
||||
|
||||
def test_init_workspace_from_seed_missing_questions(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""questions ref 不存在应 fail-fast(FileNotFoundError),不创建 workspace。"""
|
||||
with pytest.raises(FileNotFoundError, match="questions ref"):
|
||||
init_workspace_from_seed(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
seed_name="initial",
|
||||
questions="benchmarks/NONEXISTENT",
|
||||
)
|
||||
# workspace 不应被创建
|
||||
assert not workspace_dir.exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# load_manifest
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_load_manifest(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""正常加载 manifest。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
manifest = load_manifest(workspace_dir)
|
||||
assert "current" in manifest
|
||||
assert "history" in manifest
|
||||
assert manifest["current"]["skills"] == "skills/v1"
|
||||
|
||||
|
||||
def test_load_manifest_missing(workspace_dir: Path) -> None:
|
||||
"""manifest 不存在应 FileNotFoundError。"""
|
||||
workspace_dir.mkdir(parents=True)
|
||||
with pytest.raises(FileNotFoundError):
|
||||
load_manifest(workspace_dir)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# update_manifest
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_update_manifest_invalid_key(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""非法 key 应 KeyError。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
with pytest.raises(KeyError, match="无效"):
|
||||
update_manifest(workspace_dir, bad_key="skills/v2")
|
||||
|
||||
|
||||
def test_update_manifest_valid(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""合法 key 应更新 manifest current。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
update_manifest(workspace_dir, skills="skills/v2")
|
||||
manifest = load_manifest(workspace_dir)
|
||||
assert manifest["current"]["skills"] == "skills/v2"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# record_run
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_record_run_idempotent(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""同 run_id 调用两次不应重复追加 history。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
run_dir1 = record_run(workspace_dir, "run_001")
|
||||
run_dir2 = record_run(workspace_dir, "run_001")
|
||||
|
||||
# 返回路径一致
|
||||
assert run_dir1 == run_dir2
|
||||
|
||||
# history 只有一条
|
||||
manifest = load_manifest(workspace_dir)
|
||||
matched = [h for h in manifest["history"] if h["run_id"] == "run_001"]
|
||||
assert len(matched) == 1
|
||||
|
||||
# run 目录存在
|
||||
assert run_dir1.is_dir()
|
||||
|
||||
# per-video wiki 目录存在
|
||||
assert (run_dir1 / "vid_A" / "wiki").is_dir()
|
||||
assert (run_dir1 / "vid_B" / "wiki").is_dir()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# update_best / read_best
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_update_best_independent_of_current(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""best 应独立于 current——更新 best 不影响 current。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
|
||||
# 初始无 best
|
||||
assert read_best(workspace_dir) is None
|
||||
|
||||
# 设置 best
|
||||
update_best(
|
||||
workspace_dir,
|
||||
skills="skills/v3",
|
||||
prompts="prompts/v3",
|
||||
val_acc=0.85,
|
||||
run_id="run_005",
|
||||
epoch=3,
|
||||
)
|
||||
|
||||
best = read_best(workspace_dir)
|
||||
assert best is not None
|
||||
assert best["skills"] == "skills/v3"
|
||||
assert best["val_acc"] == 0.85
|
||||
assert best["epoch"] == 3
|
||||
|
||||
# current 不受影响
|
||||
manifest = load_manifest(workspace_dir)
|
||||
assert manifest["current"]["skills"] == "skills/v1"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# archive_workspace
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_archive_workspace(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""归档应把 workspace 移到 .archive/ 下。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
archived = archive_workspace(workspace_dir)
|
||||
|
||||
# 原目录不存在
|
||||
assert not workspace_dir.exists()
|
||||
|
||||
# 归档目录存在并包含 manifest
|
||||
assert archived.is_dir()
|
||||
assert (archived / "manifest.json").exists()
|
||||
|
||||
# 归档路径在 .archive 下
|
||||
assert archived.parent.name == ".archive"
|
||||
|
||||
|
||||
def test_archive_workspace_missing(workspace_dir: Path) -> None:
|
||||
"""归档不存在的 workspace 应 FileNotFoundError。"""
|
||||
with pytest.raises(FileNotFoundError):
|
||||
archive_workspace(workspace_dir)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# list_video_ids
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_list_video_ids(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""列出 workspace 引用的所有视频 ID。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
ids = list_video_ids(workspace_dir)
|
||||
assert ids == ["vid_A", "vid_B"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_paths
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolve_paths(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""resolve_paths 应正确解析绝对路径。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
rp = resolve_paths(workspace_dir)
|
||||
|
||||
# skills_dir/prompts_dir 解析到 workspace(非 store)
|
||||
ws_abs = workspace_dir.resolve()
|
||||
assert rp.skills_dir == ws_abs / "skills" / "v1"
|
||||
assert rp.prompts_dir == ws_abs / "prompts" / "v1"
|
||||
|
||||
# videos/questions 解析到 store
|
||||
store_abs = store_dir.resolve()
|
||||
assert rp.videos_dir == store_abs / "videos"
|
||||
assert rp.questions_dir == store_abs / "questions" / "benchmarks" / "Video-MME"
|
||||
|
||||
# db_path, analyses, runs
|
||||
assert rp.db_path == ws_abs / "harness.db"
|
||||
assert rp.analyses_dir == ws_abs / "analyses"
|
||||
assert rp.runs_dir == ws_abs / "runs"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VersionedSkillStore
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_versioned_skill_store_read(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""read_skill 应返回文件全文。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
skills_dir = workspace_dir / "skills" / "v1"
|
||||
store = VersionedSkillStore(skills_dir)
|
||||
content = store.read_skill("temporal-reasoning.md")
|
||||
assert content == "skill content A"
|
||||
|
||||
|
||||
def test_versioned_skill_store_read_missing(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""读取不存在的 skill 文件应 FileNotFoundError。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
skills_dir = workspace_dir / "skills" / "v1"
|
||||
store = VersionedSkillStore(skills_dir)
|
||||
with pytest.raises(FileNotFoundError):
|
||||
store.read_skill("nonexistent.md")
|
||||
|
||||
|
||||
def test_versioned_skill_store_list(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""list_skill_files 应列出所有 skill 文件名(排序)。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
skills_dir = workspace_dir / "skills" / "v1"
|
||||
store = VersionedSkillStore(skills_dir)
|
||||
files = store.list_skill_files()
|
||||
assert sorted(files) == ["spatial-analysis.md", "temporal-reasoning.md"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VersionedPromptStore
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_versioned_prompt_store(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""VersionedPromptStore 读写功能验证。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
prompts_dir = workspace_dir / "prompts" / "v1"
|
||||
store = VersionedPromptStore(prompts_dir)
|
||||
|
||||
content = store.read_prompt("system.md")
|
||||
assert content == "system prompt v1"
|
||||
|
||||
files = store.list_prompt_files()
|
||||
assert sorted(files) == ["extract.md", "system.md"]
|
||||
|
||||
|
||||
def test_versioned_prompt_store_read_missing(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""读取不存在的 prompt 文件应 FileNotFoundError。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
prompts_dir = workspace_dir / "prompts" / "v1"
|
||||
store = VersionedPromptStore(prompts_dir)
|
||||
with pytest.raises(FileNotFoundError):
|
||||
store.read_prompt("nonexistent.md")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Protocol 合规
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_skill_store_protocol_compliance(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""VersionedSkillStore 应满足 core/evolution/protocols.py::SkillStore Protocol。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
skills_dir = workspace_dir / "skills" / "v1"
|
||||
store = VersionedSkillStore(skills_dir)
|
||||
assert isinstance(store, SkillStore)
|
||||
|
||||
|
||||
def test_prompt_store_protocol_compliance(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""VersionedPromptStore 应满足 core/evolution/protocols.py::PromptStore Protocol。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
prompts_dir = workspace_dir / "prompts" / "v1"
|
||||
store = VersionedPromptStore(prompts_dir)
|
||||
assert isinstance(store, PromptStore)
|
||||
Reference in New Issue
Block a user