diff --git a/app/harness/workspace.py b/app/harness/workspace.py new file mode 100644 index 0000000..6c13452 --- /dev/null +++ b/app/harness/workspace.py @@ -0,0 +1,478 @@ +"""Workspace 生命周期管理 + manifest 读写 + Protocol 实现。 + +Workspace 是一次实验的独立工作区,通过 manifest.json 引用 Store 中的 +特定版本资源并记录实验过程。Skills/Prompts 权重拷入 workspace 本地, +训练产物只进 workspace 不污染 Store。 + +VersionedSkillStore / VersionedPromptStore 实现 core/evolution/protocols.py +中定义的只读端口,供 core/ 层以 Protocol 方式读取技能和提示词。 +""" + +from __future__ import annotations + +import json +import os +import shutil +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +from loguru import logger + +from app.harness.store import read_seed + +if TYPE_CHECKING: + from pathlib import Path + + +@dataclass(frozen=True) +class ResolvedPaths: + """manifest 解析后的绝对路径集合。 + + 属性: + store_dir: Store 根目录绝对路径。 + videos_dir: 视频数据目录。 + questions_dir: 当前引用的题目目录。 + skills_dir: 当前引用的 Skill 版本目录(workspace 内)。 + prompts_dir: 当前引用的 Prompt 版本目录(workspace 内)。 + workspace_dir: Workspace 根目录。 + db_path: harness.db 路径。 + analyses_dir: 分析报告目录。 + runs_dir: 运行临时状态目录。 + """ + + store_dir: Path + videos_dir: Path + questions_dir: Path + skills_dir: Path + prompts_dir: Path + workspace_dir: Path + db_path: Path + analyses_dir: Path + runs_dir: Path + + +# --------------------------------------------------------------------------- +# 内部工具 +# --------------------------------------------------------------------------- + +_MANIFEST_CURRENT_KEYS = {"videos", "questions", "skills", "prompts"} + + +def _now_iso() -> str: + """返回当前 UTC 时间的 ISO 格式字符串。""" + return datetime.now(UTC).isoformat() + + +# --------------------------------------------------------------------------- +# Workspace 核心函数 +# --------------------------------------------------------------------------- + + +def _scaffold_workspace( + workspace_dir: Path, + store_dir: Path, + questions: str, + skills_version: str, + prompts_version: str, +) -> None: + """写 manifest + 建 analyses/runs 目录(不拷权重;权重由调用方按来源拷入)。 + + 参数: + workspace_dir: 目标 workspace(由调用方保证不存在)。 + store_dir: Store 根目录。 + questions: 题目相对路径,如 ``'benchmarks/Video-MME'``。 + skills_version: manifest.current.skills 初始版本号。 + prompts_version: manifest.current.prompts 初始版本号。 + + 关键实现: + 不依赖任何外部资源源(store 中的 skills/prompts 是否存在不在此校验), + 因此可被 init_workspace 与种子初始化复用;store 引用以相对路径写入 manifest。 + """ + workspace_dir.mkdir(parents=True) + (workspace_dir / "analyses").mkdir() + (workspace_dir / "runs").mkdir() + + store_abs = store_dir.resolve() + store_rel = os.path.relpath(store_abs, workspace_dir.resolve()) + + manifest = { + "name": workspace_dir.name, + "created_at": _now_iso(), + "store": store_rel, + "current": { + "videos": "videos", + "questions": f"questions/{questions}", + "skills": f"skills/{skills_version}", + "prompts": f"prompts/{prompts_version}", + }, + "history": [], + } + (workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) + + +def init_workspace( + workspace_dir: Path, + store_dir: Path, + questions: str, + skills_version: str, + prompts_version: str, +) -> None: + """创建 Workspace 目录并写入初始 manifest.json,拷贝种子权重。 + + 参数: + workspace_dir: Workspace 目标路径(不得已存在)。 + store_dir: Store 根目录。 + questions: 题目在 questions/ 下的相对路径,如 ``"benchmarks/Video-MME"``。 + skills_version: Skills 版本号,如 ``"v1"``。 + prompts_version: Prompts 版本号,如 ``"v1"``。 + + 异常: + FileExistsError: Workspace 目录已存在。 + FileNotFoundError: 引用的资源在 Store 中不存在。 + """ + if workspace_dir.exists(): + raise FileExistsError(f"Workspace 已存在: {workspace_dir}") + + store_abs = store_dir.resolve() + refs = { + "skills": f"skills/{skills_version}", + "prompts": f"prompts/{prompts_version}", + "questions": f"questions/{questions}", + } + for label, rel in refs.items(): + full = store_abs / rel + if not full.is_dir(): + raise FileNotFoundError(f"Store 中不存在 {label}: {full}") + + _scaffold_workspace(workspace_dir, store_dir, questions, skills_version, prompts_version) + + # 拷种子权重进 workspace:v2+ 训练产物只进 workspace,不污染 store + shutil.copytree(store_abs / refs["skills"], workspace_dir / refs["skills"]) + shutil.copytree(store_abs / refs["prompts"], workspace_dir / refs["prompts"]) + logger.info("Workspace 初始化完成: {}", workspace_dir) + + +def init_workspace_from_seed( + workspace_dir: Path, + store_dir: Path, + seed_name: str, + questions: str, +) -> str: + """从种子全新建 workspace:拷权重 -> v1、baseline.db -> harness.db、读 baseline_run_id。 + + 参数: + workspace_dir: 目标 workspace(不得已存在)。 + store_dir: Store 根目录。 + seed_name: 种子名(store/seeds 下)。 + questions: 题目相对路径,如 ``'benchmarks/Video-MME'``。 + + 返回: + baseline_run_id(供 build_pools 使用)。 + + 异常: + FileExistsError: workspace 已存在。 + FileNotFoundError: 种子不存在(由 read_seed 抛出),或 questions ref 目录不存在。 + + 关键实现: + 破坏性/创建操作前先校验 questions ref 存在:fresh 路径在 runner 侧已先 + 归档旧 ws,若到 build_pools 才发现 questions 缺失则旧 ws 已被毁; + 故在此尽早报错(fail-fast),让新 ws 在创建前失败。 + """ + if workspace_dir.exists(): + raise FileExistsError(f"Workspace 已存在: {workspace_dir}") + + # 校验种子存在 + 取 baseline_run_id + meta = read_seed(store_dir, seed_name) + + # fail-fast:校验 questions ref 存在 + questions_ref = store_dir / "questions" / questions + if not questions_ref.is_dir(): + raise FileNotFoundError(f"questions ref 目录不存在: {questions_ref}") + + seed_dir = store_dir / "seeds" / seed_name + _scaffold_workspace(workspace_dir, store_dir, questions, "v1", "v1") + shutil.copytree(seed_dir / "skills", workspace_dir / "skills" / "v1") + shutil.copytree(seed_dir / "prompts", workspace_dir / "prompts" / "v1") + shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db") + + logger.info("Workspace 从种子 '{}' 初始化完成: {}", seed_name, workspace_dir) + return meta["baseline_run_id"] + + +def load_manifest(workspace_dir: Path) -> dict: + """读取并返回 workspace 的 manifest.json。 + + 参数: + workspace_dir: Workspace 根目录。 + + 返回: + manifest 字典。 + + 异常: + FileNotFoundError: manifest.json 不存在。 + """ + manifest_path = workspace_dir / "manifest.json" + if not manifest_path.exists(): + raise FileNotFoundError(f"manifest.json 不存在: {manifest_path}") + return json.loads(manifest_path.read_text()) + + +def resolve_paths(workspace_dir: Path) -> ResolvedPaths: + """读取 manifest,解析 current 中所有资源的绝对路径。 + + skills_dir/prompts_dir 解析到 workspace(非 store), + videos_dir/questions_dir 解析到 store。 + + 参数: + workspace_dir: Workspace 根目录。 + + 返回: + ResolvedPaths 实例,包含所有资源的绝对路径。 + """ + manifest = load_manifest(workspace_dir) + ws_abs = workspace_dir.resolve() + store_abs = (ws_abs / manifest["store"]).resolve() + current = manifest["current"] + return ResolvedPaths( + store_dir=store_abs, + videos_dir=store_abs / current["videos"], + questions_dir=store_abs / current["questions"], + skills_dir=ws_abs / current["skills"], + prompts_dir=ws_abs / current["prompts"], + workspace_dir=ws_abs, + db_path=ws_abs / "harness.db", + analyses_dir=ws_abs / "analyses", + runs_dir=ws_abs / "runs", + ) + + +def list_video_ids(workspace_dir: Path) -> list[str]: + """列出 workspace 引用的所有视频 ID(含 tree.json 的子目录名)。 + + 参数: + workspace_dir: Workspace 根目录。 + + 返回: + 排序后的视频 ID 列表。 + """ + paths = resolve_paths(workspace_dir) + video_ids = [] + for entry in paths.videos_dir.iterdir(): + if entry.is_dir() and (entry / "tree.json").exists(): + video_ids.append(entry.name) + return sorted(video_ids) + + +def update_manifest(workspace_dir: Path, **version_updates: str) -> None: + """更新 manifest 的 current 字段。 + + 参数: + workspace_dir: Workspace 根目录。 + **version_updates: 要更新的字段及其新值,如 ``skills="skills/v2"``。 + + 异常: + KeyError: 更新的字段不在 current 允许的 key 白名单中。 + """ + invalid = set(version_updates) - _MANIFEST_CURRENT_KEYS + if invalid: + raise KeyError(f"无效的 manifest current 字段: {invalid}") + manifest = load_manifest(workspace_dir) + manifest["current"].update(version_updates) + (workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) + + +def record_run(workspace_dir: Path, run_id: str) -> Path: + """将 current 版本快照追加到 manifest history,创建 run 目录和 per-video wiki 目录。 + + 幂等:同 run_id 不重复追加 history(长跑中断后重启 / held-out 复用 run_id 时)。 + + 参数: + workspace_dir: Workspace 根目录。 + run_id: 本次运行的唯一标识,如 ``"run_001"``。 + + 返回: + 创建的 run 目录路径。 + """ + manifest = load_manifest(workspace_dir) + current = manifest["current"] + + # 幂等:同 run_id 不重复追加 history + if not any(h["run_id"] == run_id for h in manifest["history"]): + manifest["history"].append( + { + "run_id": run_id, + "started_at": _now_iso(), + "skills": current["skills"], + "prompts": current["prompts"], + "questions": current["questions"], + } + ) + (workspace_dir / "manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2) + ) + + run_dir = workspace_dir / "runs" / run_id + # exist_ok:同 run_id 重跑时 run 目录已存在不应崩溃 + run_dir.mkdir(parents=True, exist_ok=True) + for video_id in list_video_ids(workspace_dir): + (run_dir / video_id / "wiki").mkdir(parents=True, exist_ok=True) + + logger.debug("Run 已记录: {}", run_id) + return run_dir + + +def read_best(workspace_dir: Path) -> dict | None: + """读取 manifest 的 best 指针,未设置时返回 None。 + + 参数: + workspace_dir: Workspace 根目录。 + + 返回: + best 字典(skills/prompts/val_acc/run_id/epoch),未设置时 None。 + """ + return load_manifest(workspace_dir).get("best") + + +def update_best( + workspace_dir: Path, + skills: str, + prompts: str, + val_acc: float, + run_id: str, + epoch: int, +) -> None: + """写入 manifest 的 best 指针(历史最优版本快照,与 current 平级)。 + + best 独立于 current——更新 best 不影响 current。 + + 参数: + workspace_dir: Workspace 根目录。 + skills: 最优 skills 版本完整 ref,如 ``'skills/v2'``。 + prompts: 最优 prompts 版本完整 ref,如 ``'prompts/v2'``。 + val_acc: 该版本验证集准确率。 + run_id: 该版本验证 run_id。 + epoch: 达成该最优的轮次。 + """ + manifest = load_manifest(workspace_dir) + manifest["best"] = { + "skills": skills, + "prompts": prompts, + "val_acc": val_acc, + "run_id": run_id, + "epoch": epoch, + } + (workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) + logger.info("Best 已更新: val_acc={}, run={}, epoch={}", val_acc, run_id, epoch) + + +def archive_workspace(workspace_dir: Path) -> Path: + """把 workspace 整体移动到同级 .archive/-,返回归档路径。 + + 参数: + workspace_dir: 要归档的 Workspace 根目录。 + + 返回: + 归档后的目标路径。 + + 异常: + FileNotFoundError: workspace 不存在。 + """ + if not workspace_dir.exists(): + raise FileNotFoundError(f"workspace 不存在: {workspace_dir}") + + archive_root = workspace_dir.parent / ".archive" + archive_root.mkdir(exist_ok=True) + ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S") + target = archive_root / f"{workspace_dir.name}-{ts}" + shutil.move(str(workspace_dir), str(target)) + + logger.info("Workspace 已归档: {} -> {}", workspace_dir, target) + return target + + +# --------------------------------------------------------------------------- +# Protocol 实现:VersionedSkillStore / VersionedPromptStore +# --------------------------------------------------------------------------- + + +class VersionedSkillStore: + """版本化技能读取端口实现。 + + 满足 ``core/evolution/protocols.py::SkillStore`` Protocol。 + 从指定的 skills 版本目录读取 ``.md`` 文件。 + + 参数: + skills_dir: skills 版本目录绝对路径(如 ``workspace/skills/v1``)。 + """ + + def __init__(self, skills_dir: Path) -> None: + if not skills_dir.is_dir(): + raise FileNotFoundError(f"Skills 目录不存在: {skills_dir}") + self._dir = skills_dir + + def read_skill(self, filename: str) -> str: + """读取指定 skill 文件的全文内容。 + + 参数: + filename: skill 文件名,如 ``'temporal-reasoning.md'``。 + + 返回: + 文件全文内容。 + + 异常: + FileNotFoundError: 文件不存在。 + """ + path = self._dir / filename + if not path.exists(): + raise FileNotFoundError(f"Skill 文件不存在: {path}") + return path.read_text() + + def list_skill_files(self) -> list[str]: + """列出当前版本所有 skill 文件名。 + + 返回: + 文件名列表(排序)。 + """ + return sorted(entry.name for entry in self._dir.iterdir() if entry.is_file()) + + +class VersionedPromptStore: + """版本化提示词读取端口实现。 + + 满足 ``core/evolution/protocols.py::PromptStore`` Protocol。 + 从指定的 prompts 版本目录读取 ``.md`` 文件。 + + 参数: + prompts_dir: prompts 版本目录绝对路径(如 ``workspace/prompts/v1``)。 + """ + + def __init__(self, prompts_dir: Path) -> None: + if not prompts_dir.is_dir(): + raise FileNotFoundError(f"Prompts 目录不存在: {prompts_dir}") + self._dir = prompts_dir + + def read_prompt(self, filename: str) -> str: + """读取指定 prompt 文件的全文内容。 + + 参数: + filename: prompt 文件名,如 ``'system.md'``。 + + 返回: + 文件全文内容。 + + 异常: + FileNotFoundError: 文件不存在。 + """ + path = self._dir / filename + if not path.exists(): + raise FileNotFoundError(f"Prompt 文件不存在: {path}") + return path.read_text() + + def list_prompt_files(self) -> list[str]: + """列出当前版本所有 prompt 文件名。 + + 返回: + 文件名列表(排序)。 + """ + return sorted(entry.name for entry in self._dir.iterdir() if entry.is_file()) diff --git a/tests/unit/test_harness_workspace.py b/tests/unit/test_harness_workspace.py new file mode 100644 index 0000000..f11fb6d --- /dev/null +++ b/tests/unit/test_harness_workspace.py @@ -0,0 +1,519 @@ +"""app/harness/workspace 单元测试。 + +覆盖 Workspace 生命周期管理、manifest 读写、 +VersionedSkillStore / VersionedPromptStore Protocol 合规。 +""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pytest + +from app.harness.workspace import ( + ResolvedPaths, + VersionedPromptStore, + VersionedSkillStore, + archive_workspace, + init_workspace, + init_workspace_from_seed, + list_video_ids, + load_manifest, + read_best, + record_run, + resolve_paths, + update_best, + update_manifest, +) +from core.evolution.protocols import PromptStore, SkillStore + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def store_dir(tmp_path: Path) -> Path: + """构建一个最小 Store 目录结构供测试使用。""" + sd = tmp_path / "store" + sd.mkdir() + + # videos:两个含 tree.json 的视频目录 + for vid in ("vid_A", "vid_B"): + vdir = sd / "videos" / vid + vdir.mkdir(parents=True) + (vdir / "tree.json").write_text("{}") + + # skills/v1 + prompts/v1 + s1 = sd / "skills" / "v1" + s1.mkdir(parents=True) + (s1 / "temporal-reasoning.md").write_text("skill content A") + (s1 / "spatial-analysis.md").write_text("skill content B") + + p1 = sd / "prompts" / "v1" + p1.mkdir(parents=True) + (p1 / "system.md").write_text("system prompt v1") + (p1 / "extract.md").write_text("extract prompt v1") + + # questions/benchmarks/Video-MME + q = sd / "questions" / "benchmarks" / "Video-MME" + q.mkdir(parents=True) + (q / "q1.json").write_text('{"id": "q1"}') + + # seed "initial" + seed_dir = sd / "seeds" / "initial" + seed_dir.mkdir(parents=True) + shutil.copytree(s1, seed_dir / "skills") + shutil.copytree(p1, seed_dir / "prompts") + baseline_db = seed_dir / "baseline.db" + baseline_db.write_bytes(b"") + (seed_dir / "seed.json").write_text( + json.dumps({"baseline_run_id": "baseline-001", "parent": None}) + ) + return sd + + +@pytest.fixture() +def workspace_dir(tmp_path: Path) -> Path: + """返回一个不存在的 workspace 路径。""" + return tmp_path / "ws_test" + + +# --------------------------------------------------------------------------- +# ResolvedPaths +# --------------------------------------------------------------------------- + + +def test_resolved_paths_frozen() -> None: + """ResolvedPaths 实例应不可变。""" + rp = ResolvedPaths( + store_dir=Path("/s"), + videos_dir=Path("/v"), + questions_dir=Path("/q"), + skills_dir=Path("/sk"), + prompts_dir=Path("/p"), + workspace_dir=Path("/w"), + db_path=Path("/d"), + analyses_dir=Path("/a"), + runs_dir=Path("/r"), + ) + with pytest.raises(AttributeError): + rp.store_dir = Path("/other") # type: ignore[misc] + + +# --------------------------------------------------------------------------- +# init_workspace +# --------------------------------------------------------------------------- + + +def test_init_workspace(store_dir: Path, workspace_dir: Path) -> None: + """init_workspace 应创建 manifest、拷贝权重。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + + # manifest 存在 + manifest = json.loads((workspace_dir / "manifest.json").read_text()) + assert manifest["current"]["skills"] == "skills/v1" + assert manifest["current"]["prompts"] == "prompts/v1" + assert manifest["current"]["questions"] == "questions/benchmarks/Video-MME" + + # 权重已拷入 workspace + assert (workspace_dir / "skills" / "v1" / "temporal-reasoning.md").exists() + assert (workspace_dir / "prompts" / "v1" / "system.md").exists() + + # analyses, runs 目录已创建 + assert (workspace_dir / "analyses").is_dir() + assert (workspace_dir / "runs").is_dir() + + # 重复创建应报错 + with pytest.raises(FileExistsError): + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + + +# --------------------------------------------------------------------------- +# init_workspace_from_seed +# --------------------------------------------------------------------------- + + +def test_init_workspace_from_seed(store_dir: Path, workspace_dir: Path) -> None: + """从种子初始化应拷权重 + baseline.db + 返回 baseline_run_id。""" + run_id = init_workspace_from_seed( + workspace_dir, + store_dir, + seed_name="initial", + questions="benchmarks/Video-MME", + ) + assert run_id == "baseline-001" + + # 权重在 workspace + assert (workspace_dir / "skills" / "v1" / "temporal-reasoning.md").exists() + assert (workspace_dir / "prompts" / "v1" / "system.md").exists() + + # baseline.db -> harness.db + assert (workspace_dir / "harness.db").exists() + + # manifest 正确 + manifest = json.loads((workspace_dir / "manifest.json").read_text()) + assert manifest["current"]["skills"] == "skills/v1" + + +def test_init_workspace_from_seed_missing_questions(store_dir: Path, workspace_dir: Path) -> None: + """questions ref 不存在应 fail-fast(FileNotFoundError),不创建 workspace。""" + with pytest.raises(FileNotFoundError, match="questions ref"): + init_workspace_from_seed( + workspace_dir, + store_dir, + seed_name="initial", + questions="benchmarks/NONEXISTENT", + ) + # workspace 不应被创建 + assert not workspace_dir.exists() + + +# --------------------------------------------------------------------------- +# load_manifest +# --------------------------------------------------------------------------- + + +def test_load_manifest(store_dir: Path, workspace_dir: Path) -> None: + """正常加载 manifest。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + manifest = load_manifest(workspace_dir) + assert "current" in manifest + assert "history" in manifest + assert manifest["current"]["skills"] == "skills/v1" + + +def test_load_manifest_missing(workspace_dir: Path) -> None: + """manifest 不存在应 FileNotFoundError。""" + workspace_dir.mkdir(parents=True) + with pytest.raises(FileNotFoundError): + load_manifest(workspace_dir) + + +# --------------------------------------------------------------------------- +# update_manifest +# --------------------------------------------------------------------------- + + +def test_update_manifest_invalid_key(store_dir: Path, workspace_dir: Path) -> None: + """非法 key 应 KeyError。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + with pytest.raises(KeyError, match="无效"): + update_manifest(workspace_dir, bad_key="skills/v2") + + +def test_update_manifest_valid(store_dir: Path, workspace_dir: Path) -> None: + """合法 key 应更新 manifest current。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + update_manifest(workspace_dir, skills="skills/v2") + manifest = load_manifest(workspace_dir) + assert manifest["current"]["skills"] == "skills/v2" + + +# --------------------------------------------------------------------------- +# record_run +# --------------------------------------------------------------------------- + + +def test_record_run_idempotent(store_dir: Path, workspace_dir: Path) -> None: + """同 run_id 调用两次不应重复追加 history。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + run_dir1 = record_run(workspace_dir, "run_001") + run_dir2 = record_run(workspace_dir, "run_001") + + # 返回路径一致 + assert run_dir1 == run_dir2 + + # history 只有一条 + manifest = load_manifest(workspace_dir) + matched = [h for h in manifest["history"] if h["run_id"] == "run_001"] + assert len(matched) == 1 + + # run 目录存在 + assert run_dir1.is_dir() + + # per-video wiki 目录存在 + assert (run_dir1 / "vid_A" / "wiki").is_dir() + assert (run_dir1 / "vid_B" / "wiki").is_dir() + + +# --------------------------------------------------------------------------- +# update_best / read_best +# --------------------------------------------------------------------------- + + +def test_update_best_independent_of_current(store_dir: Path, workspace_dir: Path) -> None: + """best 应独立于 current——更新 best 不影响 current。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + + # 初始无 best + assert read_best(workspace_dir) is None + + # 设置 best + update_best( + workspace_dir, + skills="skills/v3", + prompts="prompts/v3", + val_acc=0.85, + run_id="run_005", + epoch=3, + ) + + best = read_best(workspace_dir) + assert best is not None + assert best["skills"] == "skills/v3" + assert best["val_acc"] == 0.85 + assert best["epoch"] == 3 + + # current 不受影响 + manifest = load_manifest(workspace_dir) + assert manifest["current"]["skills"] == "skills/v1" + + +# --------------------------------------------------------------------------- +# archive_workspace +# --------------------------------------------------------------------------- + + +def test_archive_workspace(store_dir: Path, workspace_dir: Path) -> None: + """归档应把 workspace 移到 .archive/ 下。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + archived = archive_workspace(workspace_dir) + + # 原目录不存在 + assert not workspace_dir.exists() + + # 归档目录存在并包含 manifest + assert archived.is_dir() + assert (archived / "manifest.json").exists() + + # 归档路径在 .archive 下 + assert archived.parent.name == ".archive" + + +def test_archive_workspace_missing(workspace_dir: Path) -> None: + """归档不存在的 workspace 应 FileNotFoundError。""" + with pytest.raises(FileNotFoundError): + archive_workspace(workspace_dir) + + +# --------------------------------------------------------------------------- +# list_video_ids +# --------------------------------------------------------------------------- + + +def test_list_video_ids(store_dir: Path, workspace_dir: Path) -> None: + """列出 workspace 引用的所有视频 ID。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + ids = list_video_ids(workspace_dir) + assert ids == ["vid_A", "vid_B"] + + +# --------------------------------------------------------------------------- +# resolve_paths +# --------------------------------------------------------------------------- + + +def test_resolve_paths(store_dir: Path, workspace_dir: Path) -> None: + """resolve_paths 应正确解析绝对路径。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + rp = resolve_paths(workspace_dir) + + # skills_dir/prompts_dir 解析到 workspace(非 store) + ws_abs = workspace_dir.resolve() + assert rp.skills_dir == ws_abs / "skills" / "v1" + assert rp.prompts_dir == ws_abs / "prompts" / "v1" + + # videos/questions 解析到 store + store_abs = store_dir.resolve() + assert rp.videos_dir == store_abs / "videos" + assert rp.questions_dir == store_abs / "questions" / "benchmarks" / "Video-MME" + + # db_path, analyses, runs + assert rp.db_path == ws_abs / "harness.db" + assert rp.analyses_dir == ws_abs / "analyses" + assert rp.runs_dir == ws_abs / "runs" + + +# --------------------------------------------------------------------------- +# VersionedSkillStore +# --------------------------------------------------------------------------- + + +def test_versioned_skill_store_read(store_dir: Path, workspace_dir: Path) -> None: + """read_skill 应返回文件全文。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + skills_dir = workspace_dir / "skills" / "v1" + store = VersionedSkillStore(skills_dir) + content = store.read_skill("temporal-reasoning.md") + assert content == "skill content A" + + +def test_versioned_skill_store_read_missing(store_dir: Path, workspace_dir: Path) -> None: + """读取不存在的 skill 文件应 FileNotFoundError。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + skills_dir = workspace_dir / "skills" / "v1" + store = VersionedSkillStore(skills_dir) + with pytest.raises(FileNotFoundError): + store.read_skill("nonexistent.md") + + +def test_versioned_skill_store_list(store_dir: Path, workspace_dir: Path) -> None: + """list_skill_files 应列出所有 skill 文件名(排序)。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + skills_dir = workspace_dir / "skills" / "v1" + store = VersionedSkillStore(skills_dir) + files = store.list_skill_files() + assert sorted(files) == ["spatial-analysis.md", "temporal-reasoning.md"] + + +# --------------------------------------------------------------------------- +# VersionedPromptStore +# --------------------------------------------------------------------------- + + +def test_versioned_prompt_store(store_dir: Path, workspace_dir: Path) -> None: + """VersionedPromptStore 读写功能验证。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + prompts_dir = workspace_dir / "prompts" / "v1" + store = VersionedPromptStore(prompts_dir) + + content = store.read_prompt("system.md") + assert content == "system prompt v1" + + files = store.list_prompt_files() + assert sorted(files) == ["extract.md", "system.md"] + + +def test_versioned_prompt_store_read_missing(store_dir: Path, workspace_dir: Path) -> None: + """读取不存在的 prompt 文件应 FileNotFoundError。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + prompts_dir = workspace_dir / "prompts" / "v1" + store = VersionedPromptStore(prompts_dir) + with pytest.raises(FileNotFoundError): + store.read_prompt("nonexistent.md") + + +# --------------------------------------------------------------------------- +# Protocol 合规 +# --------------------------------------------------------------------------- + + +def test_skill_store_protocol_compliance(store_dir: Path, workspace_dir: Path) -> None: + """VersionedSkillStore 应满足 core/evolution/protocols.py::SkillStore Protocol。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + skills_dir = workspace_dir / "skills" / "v1" + store = VersionedSkillStore(skills_dir) + assert isinstance(store, SkillStore) + + +def test_prompt_store_protocol_compliance(store_dir: Path, workspace_dir: Path) -> None: + """VersionedPromptStore 应满足 core/evolution/protocols.py::PromptStore Protocol。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + prompts_dir = workspace_dir / "prompts" / "v1" + store = VersionedPromptStore(prompts_dir) + assert isinstance(store, PromptStore)