feat(harness): workspace.py — Workspace lifecycle + VersionedSkillStore/PromptStore

- ResolvedPaths frozen dataclass: store_dir, videos_dir, questions_dir,
  skills_dir, prompts_dir, workspace_dir, db_path, analyses_dir, runs_dir
- init_workspace: create ws + copy seed weights from store
- init_workspace_from_seed: create from seed with fail-fast questions check
- load_manifest / resolve_paths: manifest I/O + path resolution
  (skills/prompts resolve to workspace, videos/questions to store)
- update_manifest: key whitelist validation
- record_run: idempotent history append + per-video wiki dirs
- read_best / update_best: best pointer independent of current
- list_video_ids: videos with tree.json
- archive_workspace: move to .archive/<name>-<ts>
- VersionedSkillStore: implements core/evolution/protocols.py::SkillStore
- VersionedPromptStore: implements core/evolution/protocols.py::PromptStore
- 21 tests all passing (incl. Protocol compliance checks)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-07 12:41:59 -04:00
parent b052c1f3ee
commit d349fe1148
2 changed files with 997 additions and 0 deletions
+478
View File
@@ -0,0 +1,478 @@
"""Workspace 生命周期管理 + manifest 读写 + Protocol 实现。
Workspace 是一次实验的独立工作区,通过 manifest.json 引用 Store 中的
特定版本资源并记录实验过程。Skills/Prompts 权重拷入 workspace 本地,
训练产物只进 workspace 不污染 Store。
VersionedSkillStore / VersionedPromptStore 实现 core/evolution/protocols.py
中定义的只读端口,供 core/ 层以 Protocol 方式读取技能和提示词。
"""
from __future__ import annotations
import json
import os
import shutil
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import TYPE_CHECKING
from loguru import logger
from app.harness.store import read_seed
if TYPE_CHECKING:
from pathlib import Path
@dataclass(frozen=True)
class ResolvedPaths:
"""manifest 解析后的绝对路径集合。
属性:
store_dir: Store 根目录绝对路径。
videos_dir: 视频数据目录。
questions_dir: 当前引用的题目目录。
skills_dir: 当前引用的 Skill 版本目录(workspace 内)。
prompts_dir: 当前引用的 Prompt 版本目录(workspace 内)。
workspace_dir: Workspace 根目录。
db_path: harness.db 路径。
analyses_dir: 分析报告目录。
runs_dir: 运行临时状态目录。
"""
store_dir: Path
videos_dir: Path
questions_dir: Path
skills_dir: Path
prompts_dir: Path
workspace_dir: Path
db_path: Path
analyses_dir: Path
runs_dir: Path
# ---------------------------------------------------------------------------
# 内部工具
# ---------------------------------------------------------------------------
_MANIFEST_CURRENT_KEYS = {"videos", "questions", "skills", "prompts"}
def _now_iso() -> str:
"""返回当前 UTC 时间的 ISO 格式字符串。"""
return datetime.now(UTC).isoformat()
# ---------------------------------------------------------------------------
# Workspace 核心函数
# ---------------------------------------------------------------------------
def _scaffold_workspace(
workspace_dir: Path,
store_dir: Path,
questions: str,
skills_version: str,
prompts_version: str,
) -> None:
"""写 manifest + 建 analyses/runs 目录(不拷权重;权重由调用方按来源拷入)。
参数:
workspace_dir: 目标 workspace(由调用方保证不存在)。
store_dir: Store 根目录。
questions: 题目相对路径,如 ``'benchmarks/Video-MME'``。
skills_version: manifest.current.skills 初始版本号。
prompts_version: manifest.current.prompts 初始版本号。
关键实现:
不依赖任何外部资源源(store 中的 skills/prompts 是否存在不在此校验),
因此可被 init_workspace 与种子初始化复用;store 引用以相对路径写入 manifest。
"""
workspace_dir.mkdir(parents=True)
(workspace_dir / "analyses").mkdir()
(workspace_dir / "runs").mkdir()
store_abs = store_dir.resolve()
store_rel = os.path.relpath(store_abs, workspace_dir.resolve())
manifest = {
"name": workspace_dir.name,
"created_at": _now_iso(),
"store": store_rel,
"current": {
"videos": "videos",
"questions": f"questions/{questions}",
"skills": f"skills/{skills_version}",
"prompts": f"prompts/{prompts_version}",
},
"history": [],
}
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
def init_workspace(
workspace_dir: Path,
store_dir: Path,
questions: str,
skills_version: str,
prompts_version: str,
) -> None:
"""创建 Workspace 目录并写入初始 manifest.json,拷贝种子权重。
参数:
workspace_dir: Workspace 目标路径(不得已存在)。
store_dir: Store 根目录。
questions: 题目在 questions/ 下的相对路径,如 ``"benchmarks/Video-MME"``。
skills_version: Skills 版本号,如 ``"v1"``。
prompts_version: Prompts 版本号,如 ``"v1"``。
异常:
FileExistsError: Workspace 目录已存在。
FileNotFoundError: 引用的资源在 Store 中不存在。
"""
if workspace_dir.exists():
raise FileExistsError(f"Workspace 已存在: {workspace_dir}")
store_abs = store_dir.resolve()
refs = {
"skills": f"skills/{skills_version}",
"prompts": f"prompts/{prompts_version}",
"questions": f"questions/{questions}",
}
for label, rel in refs.items():
full = store_abs / rel
if not full.is_dir():
raise FileNotFoundError(f"Store 中不存在 {label}: {full}")
_scaffold_workspace(workspace_dir, store_dir, questions, skills_version, prompts_version)
# 拷种子权重进 workspacev2+ 训练产物只进 workspace,不污染 store
shutil.copytree(store_abs / refs["skills"], workspace_dir / refs["skills"])
shutil.copytree(store_abs / refs["prompts"], workspace_dir / refs["prompts"])
logger.info("Workspace 初始化完成: {}", workspace_dir)
def init_workspace_from_seed(
workspace_dir: Path,
store_dir: Path,
seed_name: str,
questions: str,
) -> str:
"""从种子全新建 workspace:拷权重 -> v1、baseline.db -> harness.db、读 baseline_run_id。
参数:
workspace_dir: 目标 workspace(不得已存在)。
store_dir: Store 根目录。
seed_name: 种子名(store/seeds 下)。
questions: 题目相对路径,如 ``'benchmarks/Video-MME'``。
返回:
baseline_run_id(供 build_pools 使用)。
异常:
FileExistsError: workspace 已存在。
FileNotFoundError: 种子不存在(由 read_seed 抛出),或 questions ref 目录不存在。
关键实现:
破坏性/创建操作前先校验 questions ref 存在:fresh 路径在 runner 侧已先
归档旧 ws,若到 build_pools 才发现 questions 缺失则旧 ws 已被毁;
故在此尽早报错(fail-fast),让新 ws 在创建前失败。
"""
if workspace_dir.exists():
raise FileExistsError(f"Workspace 已存在: {workspace_dir}")
# 校验种子存在 + 取 baseline_run_id
meta = read_seed(store_dir, seed_name)
# fail-fast:校验 questions ref 存在
questions_ref = store_dir / "questions" / questions
if not questions_ref.is_dir():
raise FileNotFoundError(f"questions ref 目录不存在: {questions_ref}")
seed_dir = store_dir / "seeds" / seed_name
_scaffold_workspace(workspace_dir, store_dir, questions, "v1", "v1")
shutil.copytree(seed_dir / "skills", workspace_dir / "skills" / "v1")
shutil.copytree(seed_dir / "prompts", workspace_dir / "prompts" / "v1")
shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db")
logger.info("Workspace 从种子 '{}' 初始化完成: {}", seed_name, workspace_dir)
return meta["baseline_run_id"]
def load_manifest(workspace_dir: Path) -> dict:
"""读取并返回 workspace 的 manifest.json。
参数:
workspace_dir: Workspace 根目录。
返回:
manifest 字典。
异常:
FileNotFoundError: manifest.json 不存在。
"""
manifest_path = workspace_dir / "manifest.json"
if not manifest_path.exists():
raise FileNotFoundError(f"manifest.json 不存在: {manifest_path}")
return json.loads(manifest_path.read_text())
def resolve_paths(workspace_dir: Path) -> ResolvedPaths:
"""读取 manifest,解析 current 中所有资源的绝对路径。
skills_dir/prompts_dir 解析到 workspace(非 store),
videos_dir/questions_dir 解析到 store。
参数:
workspace_dir: Workspace 根目录。
返回:
ResolvedPaths 实例,包含所有资源的绝对路径。
"""
manifest = load_manifest(workspace_dir)
ws_abs = workspace_dir.resolve()
store_abs = (ws_abs / manifest["store"]).resolve()
current = manifest["current"]
return ResolvedPaths(
store_dir=store_abs,
videos_dir=store_abs / current["videos"],
questions_dir=store_abs / current["questions"],
skills_dir=ws_abs / current["skills"],
prompts_dir=ws_abs / current["prompts"],
workspace_dir=ws_abs,
db_path=ws_abs / "harness.db",
analyses_dir=ws_abs / "analyses",
runs_dir=ws_abs / "runs",
)
def list_video_ids(workspace_dir: Path) -> list[str]:
"""列出 workspace 引用的所有视频 ID(含 tree.json 的子目录名)。
参数:
workspace_dir: Workspace 根目录。
返回:
排序后的视频 ID 列表。
"""
paths = resolve_paths(workspace_dir)
video_ids = []
for entry in paths.videos_dir.iterdir():
if entry.is_dir() and (entry / "tree.json").exists():
video_ids.append(entry.name)
return sorted(video_ids)
def update_manifest(workspace_dir: Path, **version_updates: str) -> None:
"""更新 manifest 的 current 字段。
参数:
workspace_dir: Workspace 根目录。
**version_updates: 要更新的字段及其新值,如 ``skills="skills/v2"``。
异常:
KeyError: 更新的字段不在 current 允许的 key 白名单中。
"""
invalid = set(version_updates) - _MANIFEST_CURRENT_KEYS
if invalid:
raise KeyError(f"无效的 manifest current 字段: {invalid}")
manifest = load_manifest(workspace_dir)
manifest["current"].update(version_updates)
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
def record_run(workspace_dir: Path, run_id: str) -> Path:
"""将 current 版本快照追加到 manifest history,创建 run 目录和 per-video wiki 目录。
幂等:同 run_id 不重复追加 history(长跑中断后重启 / held-out 复用 run_id 时)。
参数:
workspace_dir: Workspace 根目录。
run_id: 本次运行的唯一标识,如 ``"run_001"``。
返回:
创建的 run 目录路径。
"""
manifest = load_manifest(workspace_dir)
current = manifest["current"]
# 幂等:同 run_id 不重复追加 history
if not any(h["run_id"] == run_id for h in manifest["history"]):
manifest["history"].append(
{
"run_id": run_id,
"started_at": _now_iso(),
"skills": current["skills"],
"prompts": current["prompts"],
"questions": current["questions"],
}
)
(workspace_dir / "manifest.json").write_text(
json.dumps(manifest, ensure_ascii=False, indent=2)
)
run_dir = workspace_dir / "runs" / run_id
# exist_ok:同 run_id 重跑时 run 目录已存在不应崩溃
run_dir.mkdir(parents=True, exist_ok=True)
for video_id in list_video_ids(workspace_dir):
(run_dir / video_id / "wiki").mkdir(parents=True, exist_ok=True)
logger.debug("Run 已记录: {}", run_id)
return run_dir
def read_best(workspace_dir: Path) -> dict | None:
"""读取 manifest 的 best 指针,未设置时返回 None。
参数:
workspace_dir: Workspace 根目录。
返回:
best 字典(skills/prompts/val_acc/run_id/epoch),未设置时 None。
"""
return load_manifest(workspace_dir).get("best")
def update_best(
workspace_dir: Path,
skills: str,
prompts: str,
val_acc: float,
run_id: str,
epoch: int,
) -> None:
"""写入 manifest 的 best 指针(历史最优版本快照,与 current 平级)。
best 独立于 current——更新 best 不影响 current。
参数:
workspace_dir: Workspace 根目录。
skills: 最优 skills 版本完整 ref,如 ``'skills/v2'``。
prompts: 最优 prompts 版本完整 ref,如 ``'prompts/v2'``。
val_acc: 该版本验证集准确率。
run_id: 该版本验证 run_id。
epoch: 达成该最优的轮次。
"""
manifest = load_manifest(workspace_dir)
manifest["best"] = {
"skills": skills,
"prompts": prompts,
"val_acc": val_acc,
"run_id": run_id,
"epoch": epoch,
}
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
logger.info("Best 已更新: val_acc={}, run={}, epoch={}", val_acc, run_id, epoch)
def archive_workspace(workspace_dir: Path) -> Path:
"""把 workspace 整体移动到同级 .archive/<name>-<ts>,返回归档路径。
参数:
workspace_dir: 要归档的 Workspace 根目录。
返回:
归档后的目标路径。
异常:
FileNotFoundError: workspace 不存在。
"""
if not workspace_dir.exists():
raise FileNotFoundError(f"workspace 不存在: {workspace_dir}")
archive_root = workspace_dir.parent / ".archive"
archive_root.mkdir(exist_ok=True)
ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
target = archive_root / f"{workspace_dir.name}-{ts}"
shutil.move(str(workspace_dir), str(target))
logger.info("Workspace 已归档: {} -> {}", workspace_dir, target)
return target
# ---------------------------------------------------------------------------
# Protocol 实现:VersionedSkillStore / VersionedPromptStore
# ---------------------------------------------------------------------------
class VersionedSkillStore:
"""版本化技能读取端口实现。
满足 ``core/evolution/protocols.py::SkillStore`` Protocol。
从指定的 skills 版本目录读取 ``.md`` 文件。
参数:
skills_dir: skills 版本目录绝对路径(如 ``workspace/skills/v1``)。
"""
def __init__(self, skills_dir: Path) -> None:
if not skills_dir.is_dir():
raise FileNotFoundError(f"Skills 目录不存在: {skills_dir}")
self._dir = skills_dir
def read_skill(self, filename: str) -> str:
"""读取指定 skill 文件的全文内容。
参数:
filename: skill 文件名,如 ``'temporal-reasoning.md'``。
返回:
文件全文内容。
异常:
FileNotFoundError: 文件不存在。
"""
path = self._dir / filename
if not path.exists():
raise FileNotFoundError(f"Skill 文件不存在: {path}")
return path.read_text()
def list_skill_files(self) -> list[str]:
"""列出当前版本所有 skill 文件名。
返回:
文件名列表(排序)。
"""
return sorted(entry.name for entry in self._dir.iterdir() if entry.is_file())
class VersionedPromptStore:
"""版本化提示词读取端口实现。
满足 ``core/evolution/protocols.py::PromptStore`` Protocol。
从指定的 prompts 版本目录读取 ``.md`` 文件。
参数:
prompts_dir: prompts 版本目录绝对路径(如 ``workspace/prompts/v1``)。
"""
def __init__(self, prompts_dir: Path) -> None:
if not prompts_dir.is_dir():
raise FileNotFoundError(f"Prompts 目录不存在: {prompts_dir}")
self._dir = prompts_dir
def read_prompt(self, filename: str) -> str:
"""读取指定 prompt 文件的全文内容。
参数:
filename: prompt 文件名,如 ``'system.md'``。
返回:
文件全文内容。
异常:
FileNotFoundError: 文件不存在。
"""
path = self._dir / filename
if not path.exists():
raise FileNotFoundError(f"Prompt 文件不存在: {path}")
return path.read_text()
def list_prompt_files(self) -> list[str]:
"""列出当前版本所有 prompt 文件名。
返回:
文件名列表(排序)。
"""
return sorted(entry.name for entry in self._dir.iterdir() if entry.is_file())
+519
View File
@@ -0,0 +1,519 @@
"""app/harness/workspace 单元测试。
覆盖 Workspace 生命周期管理、manifest 读写、
VersionedSkillStore / VersionedPromptStore Protocol 合规。
"""
from __future__ import annotations
import json
import shutil
from pathlib import Path
import pytest
from app.harness.workspace import (
ResolvedPaths,
VersionedPromptStore,
VersionedSkillStore,
archive_workspace,
init_workspace,
init_workspace_from_seed,
list_video_ids,
load_manifest,
read_best,
record_run,
resolve_paths,
update_best,
update_manifest,
)
from core.evolution.protocols import PromptStore, SkillStore
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture()
def store_dir(tmp_path: Path) -> Path:
"""构建一个最小 Store 目录结构供测试使用。"""
sd = tmp_path / "store"
sd.mkdir()
# videos:两个含 tree.json 的视频目录
for vid in ("vid_A", "vid_B"):
vdir = sd / "videos" / vid
vdir.mkdir(parents=True)
(vdir / "tree.json").write_text("{}")
# skills/v1 + prompts/v1
s1 = sd / "skills" / "v1"
s1.mkdir(parents=True)
(s1 / "temporal-reasoning.md").write_text("skill content A")
(s1 / "spatial-analysis.md").write_text("skill content B")
p1 = sd / "prompts" / "v1"
p1.mkdir(parents=True)
(p1 / "system.md").write_text("system prompt v1")
(p1 / "extract.md").write_text("extract prompt v1")
# questions/benchmarks/Video-MME
q = sd / "questions" / "benchmarks" / "Video-MME"
q.mkdir(parents=True)
(q / "q1.json").write_text('{"id": "q1"}')
# seed "initial"
seed_dir = sd / "seeds" / "initial"
seed_dir.mkdir(parents=True)
shutil.copytree(s1, seed_dir / "skills")
shutil.copytree(p1, seed_dir / "prompts")
baseline_db = seed_dir / "baseline.db"
baseline_db.write_bytes(b"")
(seed_dir / "seed.json").write_text(
json.dumps({"baseline_run_id": "baseline-001", "parent": None})
)
return sd
@pytest.fixture()
def workspace_dir(tmp_path: Path) -> Path:
"""返回一个不存在的 workspace 路径。"""
return tmp_path / "ws_test"
# ---------------------------------------------------------------------------
# ResolvedPaths
# ---------------------------------------------------------------------------
def test_resolved_paths_frozen() -> None:
"""ResolvedPaths 实例应不可变。"""
rp = ResolvedPaths(
store_dir=Path("/s"),
videos_dir=Path("/v"),
questions_dir=Path("/q"),
skills_dir=Path("/sk"),
prompts_dir=Path("/p"),
workspace_dir=Path("/w"),
db_path=Path("/d"),
analyses_dir=Path("/a"),
runs_dir=Path("/r"),
)
with pytest.raises(AttributeError):
rp.store_dir = Path("/other") # type: ignore[misc]
# ---------------------------------------------------------------------------
# init_workspace
# ---------------------------------------------------------------------------
def test_init_workspace(store_dir: Path, workspace_dir: Path) -> None:
"""init_workspace 应创建 manifest、拷贝权重。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
# manifest 存在
manifest = json.loads((workspace_dir / "manifest.json").read_text())
assert manifest["current"]["skills"] == "skills/v1"
assert manifest["current"]["prompts"] == "prompts/v1"
assert manifest["current"]["questions"] == "questions/benchmarks/Video-MME"
# 权重已拷入 workspace
assert (workspace_dir / "skills" / "v1" / "temporal-reasoning.md").exists()
assert (workspace_dir / "prompts" / "v1" / "system.md").exists()
# analyses, runs 目录已创建
assert (workspace_dir / "analyses").is_dir()
assert (workspace_dir / "runs").is_dir()
# 重复创建应报错
with pytest.raises(FileExistsError):
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
# ---------------------------------------------------------------------------
# init_workspace_from_seed
# ---------------------------------------------------------------------------
def test_init_workspace_from_seed(store_dir: Path, workspace_dir: Path) -> None:
"""从种子初始化应拷权重 + baseline.db + 返回 baseline_run_id。"""
run_id = init_workspace_from_seed(
workspace_dir,
store_dir,
seed_name="initial",
questions="benchmarks/Video-MME",
)
assert run_id == "baseline-001"
# 权重在 workspace
assert (workspace_dir / "skills" / "v1" / "temporal-reasoning.md").exists()
assert (workspace_dir / "prompts" / "v1" / "system.md").exists()
# baseline.db -> harness.db
assert (workspace_dir / "harness.db").exists()
# manifest 正确
manifest = json.loads((workspace_dir / "manifest.json").read_text())
assert manifest["current"]["skills"] == "skills/v1"
def test_init_workspace_from_seed_missing_questions(store_dir: Path, workspace_dir: Path) -> None:
"""questions ref 不存在应 fail-fastFileNotFoundError),不创建 workspace。"""
with pytest.raises(FileNotFoundError, match="questions ref"):
init_workspace_from_seed(
workspace_dir,
store_dir,
seed_name="initial",
questions="benchmarks/NONEXISTENT",
)
# workspace 不应被创建
assert not workspace_dir.exists()
# ---------------------------------------------------------------------------
# load_manifest
# ---------------------------------------------------------------------------
def test_load_manifest(store_dir: Path, workspace_dir: Path) -> None:
"""正常加载 manifest。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
manifest = load_manifest(workspace_dir)
assert "current" in manifest
assert "history" in manifest
assert manifest["current"]["skills"] == "skills/v1"
def test_load_manifest_missing(workspace_dir: Path) -> None:
"""manifest 不存在应 FileNotFoundError。"""
workspace_dir.mkdir(parents=True)
with pytest.raises(FileNotFoundError):
load_manifest(workspace_dir)
# ---------------------------------------------------------------------------
# update_manifest
# ---------------------------------------------------------------------------
def test_update_manifest_invalid_key(store_dir: Path, workspace_dir: Path) -> None:
"""非法 key 应 KeyError。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
with pytest.raises(KeyError, match="无效"):
update_manifest(workspace_dir, bad_key="skills/v2")
def test_update_manifest_valid(store_dir: Path, workspace_dir: Path) -> None:
"""合法 key 应更新 manifest current。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
update_manifest(workspace_dir, skills="skills/v2")
manifest = load_manifest(workspace_dir)
assert manifest["current"]["skills"] == "skills/v2"
# ---------------------------------------------------------------------------
# record_run
# ---------------------------------------------------------------------------
def test_record_run_idempotent(store_dir: Path, workspace_dir: Path) -> None:
"""同 run_id 调用两次不应重复追加 history。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
run_dir1 = record_run(workspace_dir, "run_001")
run_dir2 = record_run(workspace_dir, "run_001")
# 返回路径一致
assert run_dir1 == run_dir2
# history 只有一条
manifest = load_manifest(workspace_dir)
matched = [h for h in manifest["history"] if h["run_id"] == "run_001"]
assert len(matched) == 1
# run 目录存在
assert run_dir1.is_dir()
# per-video wiki 目录存在
assert (run_dir1 / "vid_A" / "wiki").is_dir()
assert (run_dir1 / "vid_B" / "wiki").is_dir()
# ---------------------------------------------------------------------------
# update_best / read_best
# ---------------------------------------------------------------------------
def test_update_best_independent_of_current(store_dir: Path, workspace_dir: Path) -> None:
"""best 应独立于 current——更新 best 不影响 current。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
# 初始无 best
assert read_best(workspace_dir) is None
# 设置 best
update_best(
workspace_dir,
skills="skills/v3",
prompts="prompts/v3",
val_acc=0.85,
run_id="run_005",
epoch=3,
)
best = read_best(workspace_dir)
assert best is not None
assert best["skills"] == "skills/v3"
assert best["val_acc"] == 0.85
assert best["epoch"] == 3
# current 不受影响
manifest = load_manifest(workspace_dir)
assert manifest["current"]["skills"] == "skills/v1"
# ---------------------------------------------------------------------------
# archive_workspace
# ---------------------------------------------------------------------------
def test_archive_workspace(store_dir: Path, workspace_dir: Path) -> None:
"""归档应把 workspace 移到 .archive/ 下。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
archived = archive_workspace(workspace_dir)
# 原目录不存在
assert not workspace_dir.exists()
# 归档目录存在并包含 manifest
assert archived.is_dir()
assert (archived / "manifest.json").exists()
# 归档路径在 .archive 下
assert archived.parent.name == ".archive"
def test_archive_workspace_missing(workspace_dir: Path) -> None:
"""归档不存在的 workspace 应 FileNotFoundError。"""
with pytest.raises(FileNotFoundError):
archive_workspace(workspace_dir)
# ---------------------------------------------------------------------------
# list_video_ids
# ---------------------------------------------------------------------------
def test_list_video_ids(store_dir: Path, workspace_dir: Path) -> None:
"""列出 workspace 引用的所有视频 ID。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
ids = list_video_ids(workspace_dir)
assert ids == ["vid_A", "vid_B"]
# ---------------------------------------------------------------------------
# resolve_paths
# ---------------------------------------------------------------------------
def test_resolve_paths(store_dir: Path, workspace_dir: Path) -> None:
"""resolve_paths 应正确解析绝对路径。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
rp = resolve_paths(workspace_dir)
# skills_dir/prompts_dir 解析到 workspace(非 store
ws_abs = workspace_dir.resolve()
assert rp.skills_dir == ws_abs / "skills" / "v1"
assert rp.prompts_dir == ws_abs / "prompts" / "v1"
# videos/questions 解析到 store
store_abs = store_dir.resolve()
assert rp.videos_dir == store_abs / "videos"
assert rp.questions_dir == store_abs / "questions" / "benchmarks" / "Video-MME"
# db_path, analyses, runs
assert rp.db_path == ws_abs / "harness.db"
assert rp.analyses_dir == ws_abs / "analyses"
assert rp.runs_dir == ws_abs / "runs"
# ---------------------------------------------------------------------------
# VersionedSkillStore
# ---------------------------------------------------------------------------
def test_versioned_skill_store_read(store_dir: Path, workspace_dir: Path) -> None:
"""read_skill 应返回文件全文。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
skills_dir = workspace_dir / "skills" / "v1"
store = VersionedSkillStore(skills_dir)
content = store.read_skill("temporal-reasoning.md")
assert content == "skill content A"
def test_versioned_skill_store_read_missing(store_dir: Path, workspace_dir: Path) -> None:
"""读取不存在的 skill 文件应 FileNotFoundError。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
skills_dir = workspace_dir / "skills" / "v1"
store = VersionedSkillStore(skills_dir)
with pytest.raises(FileNotFoundError):
store.read_skill("nonexistent.md")
def test_versioned_skill_store_list(store_dir: Path, workspace_dir: Path) -> None:
"""list_skill_files 应列出所有 skill 文件名(排序)。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
skills_dir = workspace_dir / "skills" / "v1"
store = VersionedSkillStore(skills_dir)
files = store.list_skill_files()
assert sorted(files) == ["spatial-analysis.md", "temporal-reasoning.md"]
# ---------------------------------------------------------------------------
# VersionedPromptStore
# ---------------------------------------------------------------------------
def test_versioned_prompt_store(store_dir: Path, workspace_dir: Path) -> None:
"""VersionedPromptStore 读写功能验证。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
prompts_dir = workspace_dir / "prompts" / "v1"
store = VersionedPromptStore(prompts_dir)
content = store.read_prompt("system.md")
assert content == "system prompt v1"
files = store.list_prompt_files()
assert sorted(files) == ["extract.md", "system.md"]
def test_versioned_prompt_store_read_missing(store_dir: Path, workspace_dir: Path) -> None:
"""读取不存在的 prompt 文件应 FileNotFoundError。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
prompts_dir = workspace_dir / "prompts" / "v1"
store = VersionedPromptStore(prompts_dir)
with pytest.raises(FileNotFoundError):
store.read_prompt("nonexistent.md")
# ---------------------------------------------------------------------------
# Protocol 合规
# ---------------------------------------------------------------------------
def test_skill_store_protocol_compliance(store_dir: Path, workspace_dir: Path) -> None:
"""VersionedSkillStore 应满足 core/evolution/protocols.py::SkillStore Protocol。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
skills_dir = workspace_dir / "skills" / "v1"
store = VersionedSkillStore(skills_dir)
assert isinstance(store, SkillStore)
def test_prompt_store_protocol_compliance(store_dir: Path, workspace_dir: Path) -> None:
"""VersionedPromptStore 应满足 core/evolution/protocols.py::PromptStore Protocol。"""
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
prompts_dir = workspace_dir / "prompts" / "v1"
store = VersionedPromptStore(prompts_dir)
assert isinstance(store, PromptStore)