From d84cd679b42ccca6a35249a23f1fb0c1a9164788 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 11:48:13 -0400 Subject: [PATCH 01/19] feat(evolution): export dataclass types from __init__.py --- core/evolution/__init__.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/core/evolution/__init__.py b/core/evolution/__init__.py index 0a463ac..b9e0b8c 100644 --- a/core/evolution/__init__.py +++ b/core/evolution/__init__.py @@ -21,9 +21,47 @@ from core.evolution.patch import ( replace_appendix_notes, replace_momentum, ) +from core.evolution.types import ( + CaseSample, + DiagnosePrompts, + DiagnosisResult, + ErrorAttribution, + EvolutionRecord, + EvolutionResult, + EvolvePrompts, + GateParams, + GateVerdict, + PairResult, + QuadrantClassification, + QuestionMetrics, + RejectedEdit, + SkillCasePack, + SkillStepAdherence, + SpanMetrics, + SystemCasePack, + ToolCasePack, +) from core.evolution.validate import classify_quadrants, compute_accuracy, pair_block __all__ = [ + "CaseSample", + "DiagnosePrompts", + "DiagnosisResult", + "ErrorAttribution", + "EvolutionRecord", + "EvolutionResult", + "EvolvePrompts", + "GateParams", + "GateVerdict", + "PairResult", + "QuadrantClassification", + "QuestionMetrics", + "RejectedEdit", + "SkillCasePack", + "SkillStepAdherence", + "SpanMetrics", + "SystemCasePack", + "ToolCasePack", "append_to_appendix", "apply_patch_with_report", "classify_quadrants", From 09a385addc9aabcf212cae57e9bd84691df29dfa Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 11:57:35 -0400 Subject: [PATCH 02/19] =?UTF-8?q?feat(harness):=20RunConfig=20frozen=20dat?= =?UTF-8?q?aclass=20+=20=E5=9B=9B=E5=B1=82=E6=A0=A1=E9=AA=8C=20+=20YAML/CL?= =?UTF-8?q?I/.env=20=E4=B8=89=E5=B1=82=E5=8A=A0=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - RunConfig: 46 字段 frozen dataclass，从 TRM4 core/harness/config.py 迁移 - 四层校验链：_validate → _validate_edit_budget + _validate_minibatch + _validate_gate - 新增 .env 覆盖层：工程配置（workspace_dir, store_dir）可通过 HARNESS_* 环境变量注入 - 合并优先级：CLI > .env > YAML（CLAUDE.md §4.5） - load_config 支持嵌套 harness 段和扁平 YAML 两种格式 - run_id 改为默认空字符串（CLI-only 字段，YAML 不提供） - resume/fresh 互斥校验不在 config 层（移至 runner.py） - 70 个单元测试全部通过 Co-Authored-By: Claude Opus 4.6 (1M context) --- app/harness/config.py | 343 +++++++++++++++++++ tests/unit/test_harness_config.py | 540 ++++++++++++++++++++++++++++++ 2 files changed, 883 insertions(+) create mode 100644 app/harness/config.py create mode 100644 tests/unit/test_harness_config.py diff --git a/app/harness/config.py b/app/harness/config.py new file mode 100644 index 0000000..c895039 --- /dev/null +++ b/app/harness/config.py @@ -0,0 +1,343 @@ +"""运行配置：RunConfig frozen dataclass 与 YAML + CLI + .env 三层加载。 + +三层合并优先级：CLI > .env > YAML（遵循 CLAUDE.md §4.5 配置管理规范）。 +- YAML：科研实验配置（会在实验中反复扫动的参数），存放于 config/ 下。 +- .env：工程配置（少变路径如 workspace_dir、store_dir），通过环境变量注入。 +- CLI：单次临时覆盖。 +""" + +from __future__ import annotations + +import dataclasses +import os +from dataclasses import dataclass +from pathlib import Path + +import yaml + +_VALID_MODES = {"infer", "train", "diagnose", "evolve", "eval", "promote"} +_VALID_SKILL_MODES = {"auto", "manual", "none"} +_VALID_SKILL_UPDATE_MODES = {"patch", "rewrite"} +_PATH_FIELDS = {"workspace_dir", "store_dir"} + +# Video-MME 的任务类型数量：验证池每类至少保底 eval_min_per_class 题，共 11 类。 +_VIDEO_MME_TASK_TYPE_COUNT = 11 + +# .env 工程配置字段映射（环境变量名 → RunConfig 字段名）。 +# 仅路径类工程配置走 .env，科研实验参数走 YAML。 +_ENV_FIELD_MAP: dict[str, str] = { + "HARNESS_WORKSPACE_DIR": "workspace_dir", + "HARNESS_STORE_DIR": "store_dir", +} + + +@dataclass(frozen=True) +class RunConfig: + """实验运行配置，所有参数的唯一归口。 + + frozen=True 确保配置在创建后不可变，防止运行中被意外修改。 + 三层合并优先级：CLI > .env > YAML。 + + 字段: + workspace_dir: Workspace 根目录。 + store_dir: Store 根目录。 + mode: 运行模式，"infer" / "train" / "diagnose" / "evolve" / "eval" / "promote"。 + concurrency: 并行 worker 数。 + max_steps: AgentLoop 单题最大步数。 + skill_mode: Skill 加载模式，"auto" / "manual" / "none"。 + n_samples: 题目截取数，0 表示全量。 + questions: 题目在 questions/ 下的相对路径。 + skills_version: Skills 版本号。 + prompts_version: Prompts 版本号。 + epochs: 训练轮数。 + diag_size: 诊断池题目数。 + diag_correct_ratio: 诊断池中正确题目占比。 + val_size: 验证池题目数。 + val_correct_ratio: 验证池中正确题目占比。 + edit_budget_start: 编辑预算前期上限。 + edit_budget_end: 编辑预算后期下限。 + batch_size: mini-batch 单批题目数。 + min_class_per_batch: 单批中每个任务类型至少保留的题目数（< batch_size）。 + eval_min_per_class: 验证池中每个任务类型至少保底的题目数。 + early_stop_patience: 全局 best 连续未提升的容忍轮数，达到即早停。 + test_size: held-out 测试池题目数。 + use_slow_momentum: 是否启用快慢双速进化中的慢速 momentum 更新。 + gate_e_confirm: CE-Gate CONFIRMED 接受的 e 值门槛（1/alpha，Ville 界假阳率 alpha）。 + gate_e_provisional: 题尽暂定接受门 + futility 提前止损的代数界。 + gate_w_net_min: 题尽暂定接受要求的最小净胜数（win - loss）。 + gate_delta_min: 最小点估计效应量下限（承接旧 margin 语义）。 + gate_lambda_dir: Wald 方向拒绝的对数似然比阈值（必须为负）。 + gate_e_rollback: 试用期对称回滚门（回滚 e 值门槛）。 + gate_block: 块序贯验证的块大小（=推理并发度，块内跑满）。 + gate_n_max: 单次 gate 消耗的题数上限。 + gate_p_low: 信息量阶梯 p-hat 保留区间下界（剔除必错零信息题）。 + gate_p_high: 信息量阶梯 p-hat 保留区间上界（剔除必对零信息题）。 + gate_probe_quota: 冷启动探针集比例（全错题中插尾的比例）。 + gate_gamma_decay: 逐题正确率估计 p-hat 的 EMA 衰减系数。 + gate_cooldown_steps: 回滚后该题型跳过进化的冷却 step 数。 + gate_guard_err: gate 内跨块累计 INFRA 错误率护栏。 + skill_update_mode: skill 进化模式，"patch"（局部 edit）/ "rewrite"（整篇重写）。 + appendix_consolidate_threshold: appendix note 条数达此值触发 LLM consolidation。 + run_id: diagnose/evolve 模式要分析的运行 ID，默认空字符串。 + batch_correct_ratio: 单批中正确题目占比，范围 [0, 1)。 + momentum_samples: 慢速 momentum 更新时从诊断池采样的题目数，必须 >= 1。 + seed: fresh 训练的种子名（对应 seed.json），默认 "initial"。 + version: eval/promote 模式指定的 store 版本号（如 "v3"）。 + resume: train 模式是否从已有 checkpoint 续训。 + fresh: train 模式是否从种子全新开始。 + """ + + # ── 必填字段（无默认值，来自 YAML 或 CLI） ── + workspace_dir: Path + store_dir: Path + mode: str + concurrency: int + max_steps: int + skill_mode: str + n_samples: int + questions: str + skills_version: str + prompts_version: str + epochs: int + diag_size: int + diag_correct_ratio: float + val_size: int + val_correct_ratio: float + edit_budget_start: int + edit_budget_end: int + batch_size: int + min_class_per_batch: int + eval_min_per_class: int + early_stop_patience: int + test_size: int + use_slow_momentum: bool + gate_e_confirm: float + gate_e_provisional: float + gate_w_net_min: int + gate_delta_min: float + gate_lambda_dir: float + gate_e_rollback: float + gate_block: int + gate_n_max: int + gate_p_low: float + gate_p_high: float + gate_probe_quota: float + gate_gamma_decay: float + gate_cooldown_steps: int + gate_guard_err: float + skill_update_mode: str + appendix_consolidate_threshold: int + + # ── 有默认值的字段（通常由 CLI 传入或可选） ── + run_id: str = "" + batch_correct_ratio: float = 0.5 + momentum_samples: int = 20 + seed: str = "initial" + version: str = "" + resume: bool = False + fresh: bool = False + + +def _validate(config: RunConfig) -> None: + """校验 RunConfig 全部字段约束。 + + 四层校验链：基础字段 → 编辑预算 → mini-batch → CE-Gate。 + + 参数: + config: 待校验的配置实例。 + + 异常: + ValueError: 任一字段值不合法。 + """ + # ── 基础字段校验 ── + if config.mode not in _VALID_MODES: + raise ValueError(f"mode 必须为 {_VALID_MODES} 之一，实际: {config.mode!r}") + if config.mode in ("diagnose", "evolve") and not config.run_id: + raise ValueError(f"mode 为 {config.mode!r} 时必须提供 run_id。") + if config.mode in ("eval", "promote") and not config.version: + raise ValueError(f"mode 为 {config.mode!r} 时必须提供 --version。") + if config.mode == "promote" and not config.run_id: + raise ValueError("promote 必须提供 --run-id（指定 canonical eval run）。") + if config.skill_mode not in _VALID_SKILL_MODES: + raise ValueError( + f"skill_mode 必须为 {_VALID_SKILL_MODES} 之一，实际: {config.skill_mode!r}" + ) + if config.concurrency <= 0: + raise ValueError(f"concurrency 必须 > 0，实际: {config.concurrency}") + if config.max_steps <= 0: + raise ValueError(f"max_steps 必须 > 0，实际: {config.max_steps}") + if config.n_samples < 0: + raise ValueError(f"n_samples 必须 >= 0，实际: {config.n_samples}") + if config.epochs <= 0: + raise ValueError(f"epochs 必须 > 0，实际: {config.epochs}") + + # ── 编辑预算校验 ── + _validate_edit_budget(config) + + if config.skill_update_mode not in _VALID_SKILL_UPDATE_MODES: + raise ValueError( + f"skill_update_mode 必须为 {_VALID_SKILL_UPDATE_MODES} 之一，" + f"实际: {config.skill_update_mode!r}" + ) + if config.appendix_consolidate_threshold < 1: + raise ValueError( + f"appendix_consolidate_threshold 必须 >= 1，" + f"实际: {config.appendix_consolidate_threshold}" + ) + + # ── mini-batch 校验 ── + _validate_minibatch(config) + + # ── CE-Gate 校验 ── + _validate_gate(config) + + +def _validate_edit_budget(config: RunConfig) -> None: + """校验编辑预算退火的前期/后期上限约束。 + + 参数: + config: 待校验的配置实例。 + + 异常: + ValueError: edit_budget_start < edit_budget_end，或 end <= 0。 + """ + if config.edit_budget_start < config.edit_budget_end: + raise ValueError( + f"edit_budget_start({config.edit_budget_start}) 必须 >= " + f"edit_budget_end({config.edit_budget_end})" + ) + if config.edit_budget_end <= 0: + raise ValueError(f"edit_budget_end 必须 > 0，实际: {config.edit_budget_end}") + + +def _validate_minibatch(config: RunConfig) -> None: + """校验 mini-batch 自进化闭环参数约束。 + + 参数: + config: 待校验的 RunConfig 配置对象。 + + 异常: + ValueError: 任一约束被违反。 + + 关键实现细节: + val_size 必须 >= eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT，保证验证池 + 能为 Video-MME 的全部 11 个任务类型各保底 eval_min_per_class 题。 + """ + if config.batch_size <= 0: + raise ValueError(f"batch_size 必须 > 0，实际: {config.batch_size}") + if not (1 <= config.min_class_per_batch < config.batch_size): + raise ValueError( + f"min_class_per_batch 必须满足 1 <= 值 < batch_size" + f"({config.batch_size})，实际: {config.min_class_per_batch}" + ) + if config.eval_min_per_class < 1: + raise ValueError(f"eval_min_per_class 必须 >= 1，实际: {config.eval_min_per_class}") + floor = config.eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT + if config.val_size < floor: + raise ValueError( + f"val_size 必须 >= eval_min_per_class * {_VIDEO_MME_TASK_TYPE_COUNT}" + f"(={floor})：Video-MME 共 {_VIDEO_MME_TASK_TYPE_COUNT} 个任务类型，" + f"每类需 eval_min_per_class 题保底，故验证池下限为 {floor}，" + f"实际: {config.val_size}" + ) + if config.early_stop_patience <= 0: + raise ValueError(f"early_stop_patience 必须 > 0，实际: {config.early_stop_patience}") + if config.test_size <= 0: + raise ValueError(f"test_size 必须 > 0，实际: {config.test_size}") + if not (0 <= config.batch_correct_ratio < 1): + raise ValueError( + f"batch_correct_ratio 必须满足 0 <= 值 < 1，实际: {config.batch_correct_ratio}" + ) + if config.momentum_samples < 1: + raise ValueError(f"momentum_samples 必须 >= 1，实际: {config.momentum_samples}") + + +def _validate_gate(config: RunConfig) -> None: + """校验 CE-Gate 判据与阶梯参数约束。 + + 参数: + config: 待校验的配置实例。 + + 异常: + ValueError: 任一 gate 参数不合法。 + """ + if config.gate_e_confirm <= 1: + raise ValueError(f"gate_e_confirm 必须 > 1，实际: {config.gate_e_confirm}") + if not (1 < config.gate_e_provisional <= config.gate_e_confirm): + raise ValueError( + f"gate_e_provisional 必须在 (1, gate_e_confirm] 内，实际: {config.gate_e_provisional}" + ) + if config.gate_e_rollback <= 1: + raise ValueError(f"gate_e_rollback 必须 > 1，实际: {config.gate_e_rollback}") + if config.gate_w_net_min < 1: + raise ValueError(f"gate_w_net_min 必须 >= 1，实际: {config.gate_w_net_min}") + if config.gate_lambda_dir >= 0: + raise ValueError(f"gate_lambda_dir 必须 < 0，实际: {config.gate_lambda_dir}") + if config.gate_block <= 0 or config.gate_n_max < config.gate_block: + raise ValueError( + f"需 0 < gate_block <= gate_n_max，" + f"实际: block={config.gate_block}, n_max={config.gate_n_max}" + ) + if not (0 <= config.gate_p_low < config.gate_p_high <= 1): + raise ValueError( + f"需 0 <= gate_p_low < gate_p_high <= 1，" + f"实际: [{config.gate_p_low}, {config.gate_p_high}]" + ) + if not (0 <= config.gate_probe_quota <= 1): + raise ValueError(f"gate_probe_quota 须在 [0,1]，实际: {config.gate_probe_quota}") + if not (0 < config.gate_gamma_decay < 1): + raise ValueError(f"gate_gamma_decay 须在 (0,1)，实际: {config.gate_gamma_decay}") + if config.gate_cooldown_steps < 1: + raise ValueError(f"gate_cooldown_steps 必须 >= 1，实际: {config.gate_cooldown_steps}") + if not (0 < config.gate_guard_err < 1): + raise ValueError(f"gate_guard_err 须在 (0,1)，实际: {config.gate_guard_err}") + + +def load_config( + yaml_path: Path, + cli_overrides: dict[str, object] | None = None, +) -> RunConfig: + """从 YAML 加载配置，叠加 .env 和 CLI 覆盖层后构造 RunConfig。 + + 三层合并优先级：CLI > .env > YAML。 + + 参数: + yaml_path: YAML 配置文件路径，需包含 ``harness`` 段。 + cli_overrides: CLI 参数字典，值为 None 表示未传入（不覆盖）。 + + 返回: + 构造并校验后的 RunConfig 实例。 + + 异常: + FileNotFoundError: YAML 文件不存在。 + ValueError: 校验失败。 + """ + # Phase 1: 加载 YAML 基础层 + with open(yaml_path, encoding="utf-8") as f: + raw: dict = yaml.safe_load(f) + + # 支持嵌套 harness 段和扁平 YAML 两种格式 + yaml_data: dict = raw.get("harness", raw) + + # Phase 2: .env 覆盖层（仅工程配置字段） + for env_key, field_name in _ENV_FIELD_MAP.items(): + env_val = os.environ.get(env_key) + if env_val is not None: + yaml_data[field_name] = env_val + + # Phase 3: CLI 覆盖层（最高优先级） + valid_fields = {f.name for f in dataclasses.fields(RunConfig)} + if cli_overrides: + for key, value in cli_overrides.items(): + if value is not None and key in valid_fields: + yaml_data[key] = value + + # Phase 4: 类型转换 — 路径字段转 Path + for field_name in _PATH_FIELDS: + if field_name in yaml_data: + yaml_data[field_name] = Path(yaml_data[field_name]) + + # Phase 5: 构造并校验 + config = RunConfig(**{k: v for k, v in yaml_data.items() if k in valid_fields}) + _validate(config) + return config diff --git a/tests/unit/test_harness_config.py b/tests/unit/test_harness_config.py new file mode 100644 index 0000000..b2f15b7 --- /dev/null +++ b/tests/unit/test_harness_config.py @@ -0,0 +1,540 @@ +"""app/harness/config.py 单元测试。 + +覆盖 RunConfig 构造、四层校验（_validate → 三个子校验）、 +YAML + CLI + .env 三层加载优先级。 +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from app.harness.config import RunConfig, _validate, load_config + +# ────────────────────────────── 测试数据工厂 ────────────────────────────── + + +def _valid_kwargs() -> dict: + """构造一组完整合法的 RunConfig 字段值（使用真实 default.yaml 数据）。""" + return { + "workspace_dir": Path("workspaces/default"), + "store_dir": Path("store"), + "mode": "infer", + "run_id": "", + "concurrency": 12, + "max_steps": 15, + "skill_mode": "auto", + "n_samples": 0, + "questions": "benchmarks/Video-MME", + "skills_version": "v1", + "prompts_version": "v1", + "epochs": 1, + "diag_size": 200, + "diag_correct_ratio": 0.5, + "val_size": 30, + "val_correct_ratio": 0.5, + "edit_budget_start": 5, + "edit_budget_end": 2, + "batch_size": 15, + "min_class_per_batch": 2, + "eval_min_per_class": 2, + "early_stop_patience": 8, + "test_size": 60, + "use_slow_momentum": True, + "gate_e_confirm": 20.0, + "gate_e_provisional": 3.0, + "gate_w_net_min": 2, + "gate_delta_min": 0.02, + "gate_lambda_dir": -0.642, + "gate_e_rollback": 10.0, + "gate_block": 8, + "gate_n_max": 40, + "gate_p_low": 0.05, + "gate_p_high": 0.95, + "gate_probe_quota": 0.2, + "gate_gamma_decay": 0.9, + "gate_cooldown_steps": 2, + "gate_guard_err": 0.10, + "skill_update_mode": "patch", + "appendix_consolidate_threshold": 6, + "batch_correct_ratio": 0.5, + "momentum_samples": 20, + } + + +def _make_config(**overrides: object) -> RunConfig: + """用 _valid_kwargs 构造 RunConfig，支持字段覆盖。""" + kwargs = _valid_kwargs() + kwargs.update(overrides) + return RunConfig(**kwargs) + + +def _yaml_harness_dict() -> dict: + """构造可序列化为 YAML 的合法 harness 配置字典（路径用字符串）。""" + d = _valid_kwargs() + d["workspace_dir"] = str(d["workspace_dir"]) + d["store_dir"] = str(d["store_dir"]) + return d + + +def _write_yaml(tmp_path: Path, harness_data: dict) -> Path: + """将 harness 配置写入临时 YAML 文件，返回文件路径。""" + yaml_path = tmp_path / "experiment.yaml" + with open(yaml_path, "w", encoding="utf-8") as f: + yaml.dump({"harness": harness_data}, f) + return yaml_path + + +# ──────────────────────────── test_valid_config ──────────────────────────── + + +class TestValidConfig: + """合法参数应能正常构造并通过校验。""" + + def test_default_yaml_values_pass_validation(self) -> None: + """使用 default.yaml 真实默认值构造的 RunConfig 应通过全部校验。""" + cfg = _make_config() + _validate(cfg) + assert cfg.mode == "infer" + assert cfg.workspace_dir == Path("workspaces/default") + assert cfg.store_dir == Path("store") + + def test_frozen_immutability(self) -> None: + """frozen=True 应禁止字段赋值。""" + cfg = _make_config() + with pytest.raises(AttributeError): + cfg.mode = "train" # type: ignore[misc] + + def test_default_field_values(self) -> None: + """有默认值的字段不传时应使用默认值。""" + kwargs = _valid_kwargs() + # 不传 run_id / seed / version / resume / fresh，使用默认值 + kwargs.pop("run_id", None) + cfg = RunConfig(**kwargs) + assert cfg.run_id == "" + assert cfg.seed == "initial" + assert cfg.version == "" + assert cfg.resume is False + assert cfg.fresh is False + + +# ──────────────────────────── test_mode_validation ──────────────────────── + + +class TestModeValidation: + """mode 字段校验。""" + + @pytest.mark.parametrize("bad_mode", ["unknown", "test", "", "INFER", "Train"]) + def test_invalid_mode_rejected(self, bad_mode: str) -> None: + """非法 mode 应抛出 ValueError。""" + cfg = _make_config(mode=bad_mode) + with pytest.raises(ValueError, match="mode"): + _validate(cfg) + + @pytest.mark.parametrize( + "valid_mode", ["infer", "train", "diagnose", "evolve", "eval", "promote"] + ) + def test_all_valid_modes_accepted(self, valid_mode: str) -> None: + """全部合法 mode 应通过校验（diagnose/evolve 需 run_id）。""" + overrides: dict = {"mode": valid_mode} + if valid_mode in ("diagnose", "evolve"): + overrides["run_id"] = "run-001" + if valid_mode in ("eval", "promote"): + overrides["version"] = "v1" + if valid_mode == "promote": + overrides["run_id"] = "run-001" + cfg = _make_config(**overrides) + _validate(cfg) + + def test_diagnose_requires_run_id(self) -> None: + """diagnose 模式缺少 run_id 应报错。""" + cfg = _make_config(mode="diagnose", run_id="") + with pytest.raises(ValueError, match="run_id"): + _validate(cfg) + + def test_evolve_requires_run_id(self) -> None: + """evolve 模式缺少 run_id 应报错。""" + cfg = _make_config(mode="evolve", run_id="") + with pytest.raises(ValueError, match="run_id"): + _validate(cfg) + + def test_eval_requires_version(self) -> None: + """eval 模式缺少 version 应报错。""" + cfg = _make_config(mode="eval", version="") + with pytest.raises(ValueError, match="version"): + _validate(cfg) + + def test_promote_requires_run_id_and_version(self) -> None: + """promote 模式需同时提供 run_id 和 version。""" + cfg = _make_config(mode="promote", run_id="", version="v1") + with pytest.raises(ValueError, match="run.id"): + _validate(cfg) + + def test_concurrency_positive(self) -> None: + """concurrency <= 0 应报错。""" + cfg = _make_config(concurrency=0) + with pytest.raises(ValueError, match="concurrency"): + _validate(cfg) + + def test_max_steps_positive(self) -> None: + """max_steps <= 0 应报错。""" + cfg = _make_config(max_steps=-1) + with pytest.raises(ValueError, match="max_steps"): + _validate(cfg) + + def test_n_samples_non_negative(self) -> None: + """n_samples < 0 应报错。""" + cfg = _make_config(n_samples=-1) + with pytest.raises(ValueError, match="n_samples"): + _validate(cfg) + + def test_epochs_positive(self) -> None: + """epochs <= 0 应报错。""" + cfg = _make_config(epochs=0) + with pytest.raises(ValueError, match="epochs"): + _validate(cfg) + + @pytest.mark.parametrize("bad_skill_mode", ["Auto", "disabled", ""]) + def test_invalid_skill_mode(self, bad_skill_mode: str) -> None: + """非法 skill_mode 应报错。""" + cfg = _make_config(skill_mode=bad_skill_mode) + with pytest.raises(ValueError, match="skill_mode"): + _validate(cfg) + + @pytest.mark.parametrize("bad_update_mode", ["append", "delete", ""]) + def test_invalid_skill_update_mode(self, bad_update_mode: str) -> None: + """非法 skill_update_mode 应报错。""" + cfg = _make_config(skill_update_mode=bad_update_mode) + with pytest.raises(ValueError, match="skill_update_mode"): + _validate(cfg) + + def test_appendix_consolidate_threshold_positive(self) -> None: + """appendix_consolidate_threshold < 1 应报错。""" + cfg = _make_config(appendix_consolidate_threshold=0) + with pytest.raises(ValueError, match="appendix_consolidate_threshold"): + _validate(cfg) + + +# ────────────────────── test_edit_budget_validation ─────────────────────── + + +class TestEditBudgetValidation: + """编辑预算退火校验（_validate_edit_budget）。""" + + def test_start_less_than_end_rejected(self) -> None: + """edit_budget_start < edit_budget_end 应抛出 ValueError。""" + cfg = _make_config(edit_budget_start=1, edit_budget_end=5) + with pytest.raises(ValueError, match="edit_budget_start"): + _validate(cfg) + + def test_end_zero_rejected(self) -> None: + """edit_budget_end <= 0 应抛出 ValueError。""" + cfg = _make_config(edit_budget_start=1, edit_budget_end=0) + with pytest.raises(ValueError, match="edit_budget_end"): + _validate(cfg) + + def test_equal_values_accepted(self) -> None: + """edit_budget_start == edit_budget_end 应通过。""" + cfg = _make_config(edit_budget_start=3, edit_budget_end=3) + _validate(cfg) + + +# ─────────────────────── test_minibatch_validation ─────────────────────── + + +class TestMinibatchValidation: + """mini-batch 自进化闭环参数校验（_validate_minibatch）。""" + + def test_batch_size_zero_rejected(self) -> None: + """batch_size <= 0 应抛出 ValueError。""" + cfg = _make_config(batch_size=0) + with pytest.raises(ValueError, match="batch_size"): + _validate(cfg) + + def test_min_class_per_batch_equals_batch_size_rejected(self) -> None: + """min_class_per_batch >= batch_size 应抛出 ValueError。""" + cfg = _make_config(batch_size=5, min_class_per_batch=5) + with pytest.raises(ValueError, match="min_class_per_batch"): + _validate(cfg) + + def test_min_class_per_batch_zero_rejected(self) -> None: + """min_class_per_batch < 1 应抛出 ValueError。""" + cfg = _make_config(min_class_per_batch=0) + with pytest.raises(ValueError, match="min_class_per_batch"): + _validate(cfg) + + def test_eval_min_per_class_zero_rejected(self) -> None: + """eval_min_per_class < 1 应抛出 ValueError。""" + cfg = _make_config(eval_min_per_class=0) + with pytest.raises(ValueError, match="eval_min_per_class"): + _validate(cfg) + + def test_early_stop_patience_zero_rejected(self) -> None: + """early_stop_patience <= 0 应抛出 ValueError。""" + cfg = _make_config(early_stop_patience=0) + with pytest.raises(ValueError, match="early_stop_patience"): + _validate(cfg) + + def test_test_size_zero_rejected(self) -> None: + """test_size <= 0 应抛出 ValueError。""" + cfg = _make_config(test_size=0) + with pytest.raises(ValueError, match="test_size"): + _validate(cfg) + + def test_batch_correct_ratio_one_rejected(self) -> None: + """batch_correct_ratio >= 1 应抛出 ValueError。""" + cfg = _make_config(batch_correct_ratio=1.0) + with pytest.raises(ValueError, match="batch_correct_ratio"): + _validate(cfg) + + def test_batch_correct_ratio_negative_rejected(self) -> None: + """batch_correct_ratio < 0 应抛出 ValueError。""" + cfg = _make_config(batch_correct_ratio=-0.1) + with pytest.raises(ValueError, match="batch_correct_ratio"): + _validate(cfg) + + def test_momentum_samples_zero_rejected(self) -> None: + """momentum_samples < 1 应抛出 ValueError。""" + cfg = _make_config(momentum_samples=0) + with pytest.raises(ValueError, match="momentum_samples"): + _validate(cfg) + + +# ─────────────────────────── test_gate_validation ──────────────────────── + + +class TestGateValidation: + """CE-Gate 参数校验（_validate_gate）。""" + + def test_e_confirm_at_one_rejected(self) -> None: + """gate_e_confirm <= 1 应抛出 ValueError。""" + cfg = _make_config(gate_e_confirm=1.0, gate_e_provisional=1.0) + with pytest.raises(ValueError, match="gate_e_confirm"): + _validate(cfg) + + def test_e_provisional_exceeds_confirm_rejected(self) -> None: + """gate_e_provisional > gate_e_confirm 应抛出 ValueError。""" + cfg = _make_config(gate_e_confirm=10.0, gate_e_provisional=15.0) + with pytest.raises(ValueError, match="gate_e_provisional"): + _validate(cfg) + + def test_e_provisional_at_one_rejected(self) -> None: + """gate_e_provisional <= 1 应抛出 ValueError。""" + cfg = _make_config(gate_e_provisional=0.5) + with pytest.raises(ValueError, match="gate_e_provisional"): + _validate(cfg) + + def test_e_rollback_at_one_rejected(self) -> None: + """gate_e_rollback <= 1 应抛出 ValueError。""" + cfg = _make_config(gate_e_rollback=1.0) + with pytest.raises(ValueError, match="gate_e_rollback"): + _validate(cfg) + + def test_w_net_min_zero_rejected(self) -> None: + """gate_w_net_min < 1 应抛出 ValueError。""" + cfg = _make_config(gate_w_net_min=0) + with pytest.raises(ValueError, match="gate_w_net_min"): + _validate(cfg) + + def test_lambda_dir_positive_rejected(self) -> None: + """gate_lambda_dir >= 0 应抛出 ValueError。""" + cfg = _make_config(gate_lambda_dir=0.5) + with pytest.raises(ValueError, match="gate_lambda_dir"): + _validate(cfg) + + def test_lambda_dir_zero_rejected(self) -> None: + """gate_lambda_dir == 0 也应报错。""" + cfg = _make_config(gate_lambda_dir=0.0) + with pytest.raises(ValueError, match="gate_lambda_dir"): + _validate(cfg) + + def test_block_exceeds_n_max_rejected(self) -> None: + """gate_block > gate_n_max 应抛出 ValueError。""" + cfg = _make_config(gate_block=50, gate_n_max=40) + with pytest.raises(ValueError, match="gate_block"): + _validate(cfg) + + def test_block_zero_rejected(self) -> None: + """gate_block <= 0 应抛出 ValueError。""" + cfg = _make_config(gate_block=0) + with pytest.raises(ValueError, match="gate_block"): + _validate(cfg) + + def test_p_low_exceeds_p_high_rejected(self) -> None: + """gate_p_low >= gate_p_high 应抛出 ValueError。""" + cfg = _make_config(gate_p_low=0.9, gate_p_high=0.1) + with pytest.raises(ValueError, match="gate_p_low"): + _validate(cfg) + + def test_probe_quota_negative_rejected(self) -> None: + """gate_probe_quota < 0 应抛出 ValueError。""" + cfg = _make_config(gate_probe_quota=-0.1) + with pytest.raises(ValueError, match="gate_probe_quota"): + _validate(cfg) + + def test_gamma_decay_zero_rejected(self) -> None: + """gate_gamma_decay <= 0 应抛出 ValueError。""" + cfg = _make_config(gate_gamma_decay=0.0) + with pytest.raises(ValueError, match="gate_gamma_decay"): + _validate(cfg) + + def test_gamma_decay_one_rejected(self) -> None: + """gate_gamma_decay >= 1 应抛出 ValueError。""" + cfg = _make_config(gate_gamma_decay=1.0) + with pytest.raises(ValueError, match="gate_gamma_decay"): + _validate(cfg) + + def test_cooldown_steps_zero_rejected(self) -> None: + """gate_cooldown_steps < 1 应抛出 ValueError。""" + cfg = _make_config(gate_cooldown_steps=0) + with pytest.raises(ValueError, match="gate_cooldown_steps"): + _validate(cfg) + + def test_guard_err_zero_rejected(self) -> None: + """gate_guard_err <= 0 应抛出 ValueError。""" + cfg = _make_config(gate_guard_err=0.0) + with pytest.raises(ValueError, match="gate_guard_err"): + _validate(cfg) + + def test_guard_err_one_rejected(self) -> None: + """gate_guard_err >= 1 应抛出 ValueError。""" + cfg = _make_config(gate_guard_err=1.0) + with pytest.raises(ValueError, match="gate_guard_err"): + _validate(cfg) + + +# ─────────────────────── test_load_config_cli_overrides ────────────────── + + +class TestLoadConfigCliOverrides: + """load_config 的 CLI 覆盖层测试。""" + + def test_cli_overrides_yaml_values(self, tmp_path: Path) -> None: + """CLI 参数应覆盖 YAML 中的同名字段。""" + harness_data = _yaml_harness_dict() + yaml_path = _write_yaml(tmp_path, harness_data) + + cfg = load_config(yaml_path, cli_overrides={"concurrency": 4, "max_steps": 30}) + assert cfg.concurrency == 4 + assert cfg.max_steps == 30 + + def test_cli_none_values_ignored(self, tmp_path: Path) -> None: + """CLI 中值为 None 的字段不应覆盖 YAML。""" + harness_data = _yaml_harness_dict() + yaml_path = _write_yaml(tmp_path, harness_data) + + cfg = load_config(yaml_path, cli_overrides={"concurrency": None, "max_steps": 20}) + assert cfg.concurrency == 12 # YAML 默认值 + assert cfg.max_steps == 20 + + def test_cli_run_id_override(self, tmp_path: Path) -> None: + """CLI 可通过 run_id 覆盖默认空字符串。""" + harness_data = _yaml_harness_dict() + yaml_path = _write_yaml(tmp_path, harness_data) + + cfg = load_config( + yaml_path, + cli_overrides={"mode": "diagnose", "run_id": "run-abc-123"}, + ) + assert cfg.run_id == "run-abc-123" + assert cfg.mode == "diagnose" + + def test_path_fields_converted_to_path(self, tmp_path: Path) -> None: + """workspace_dir 和 store_dir 应被转换为 Path 对象。""" + harness_data = _yaml_harness_dict() + yaml_path = _write_yaml(tmp_path, harness_data) + + cfg = load_config(yaml_path) + assert isinstance(cfg.workspace_dir, Path) + assert isinstance(cfg.store_dir, Path) + + def test_unknown_cli_keys_ignored(self, tmp_path: Path) -> None: + """YAML 和 RunConfig 中不存在的 CLI key 应被忽略。""" + harness_data = _yaml_harness_dict() + yaml_path = _write_yaml(tmp_path, harness_data) + + cfg = load_config(yaml_path, cli_overrides={"nonexistent_field": 42}) + assert cfg.concurrency == 12 # 正常字段不受影响 + + def test_validation_runs_after_loading(self, tmp_path: Path) -> None: + """load_config 加载后应运行校验，非法值应报错。""" + harness_data = _yaml_harness_dict() + harness_data["mode"] = "invalid_mode" + yaml_path = _write_yaml(tmp_path, harness_data) + + with pytest.raises(ValueError, match="mode"): + load_config(yaml_path) + + +# ──────────────────── test_load_config_env_overrides ───────────────────── + + +class TestLoadConfigEnvOverrides: + """load_config 的 .env 环境变量覆盖层测试。""" + + def test_env_overrides_yaml_workspace_dir( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """环境变量 HARNESS_WORKSPACE_DIR 应覆盖 YAML 中的 workspace_dir。""" + harness_data = _yaml_harness_dict() + yaml_path = _write_yaml(tmp_path, harness_data) + + monkeypatch.setenv("HARNESS_WORKSPACE_DIR", "/custom/workspace") + cfg = load_config(yaml_path) + assert cfg.workspace_dir == Path("/custom/workspace") + + def test_env_overrides_yaml_store_dir( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """环境变量 HARNESS_STORE_DIR 应覆盖 YAML 中的 store_dir。""" + harness_data = _yaml_harness_dict() + yaml_path = _write_yaml(tmp_path, harness_data) + + monkeypatch.setenv("HARNESS_STORE_DIR", "/custom/store") + cfg = load_config(yaml_path) + assert cfg.store_dir == Path("/custom/store") + + def test_cli_overrides_env(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """CLI 优先级高于 .env：CLI > .env > YAML。""" + harness_data = _yaml_harness_dict() + yaml_path = _write_yaml(tmp_path, harness_data) + + monkeypatch.setenv("HARNESS_WORKSPACE_DIR", "/env/workspace") + cfg = load_config(yaml_path, cli_overrides={"workspace_dir": "/cli/workspace"}) + assert cfg.workspace_dir == Path("/cli/workspace") + + +# ─────────────────────────── test_val_size_floor ───────────────────────── + + +class TestValSizeFloor: + """val_size >= eval_min_per_class * 11 的下限校验。""" + + def test_val_size_below_floor_rejected(self) -> None: + """val_size < eval_min_per_class * 11 应抛出 ValueError。 + + eval_min_per_class=3 → 下限 = 3 * 11 = 33，val_size=30 不足。 + """ + cfg = _make_config(eval_min_per_class=3, val_size=30) + with pytest.raises(ValueError, match="val_size"): + _validate(cfg) + + def test_val_size_at_floor_accepted(self) -> None: + """val_size == eval_min_per_class * 11 应通过。""" + cfg = _make_config(eval_min_per_class=3, val_size=33) + _validate(cfg) + + def test_val_size_above_floor_accepted(self) -> None: + """val_size > eval_min_per_class * 11 应通过。""" + cfg = _make_config(eval_min_per_class=2, val_size=100) + _validate(cfg) + + def test_default_yaml_values_satisfy_floor(self) -> None: + """default.yaml 的默认值（val_size=30, eval_min_per_class=2）应满足下限。 + + 下限 = 2 * 11 = 22，val_size=30 >= 22，通过。 + """ + cfg = _make_config() + _validate(cfg) # 不应抛出异常 From 6a2ddb16245cbe5032378f9bef86f35cdbbd0e34 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 12:01:46 -0400 Subject: [PATCH 03/19] =?UTF-8?q?refactor(harness):=20=E6=8B=86=E5=88=86?= =?UTF-8?q?=E6=A0=A1=E9=AA=8C=E5=87=BD=E6=95=B0=E9=99=8D=E4=BD=8E=20radon?= =?UTF-8?q?=20=E5=9C=88=E5=A4=8D=E6=9D=82=E5=BA=A6=E8=87=B3=20Grade=20B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _validate: 拆出 _validate_mode（mode 依赖校验）+ _validate_basic（标量/枚举校验） - _validate_gate: 拆为 _validate_gate_thresholds（e 值/净胜/方向）+ _validate_gate_ladder（阶梯/块序贯） - load_config: 提取 _apply_env_overrides 函数 - radon cc -n C 无输出（全部 Grade B 或更好） - 70 个单元测试全部通过 Co-Authored-By: Claude Opus 4.6 (1M context) --- app/harness/config.py | 87 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 17 deletions(-) diff --git a/app/harness/config.py b/app/harness/config.py index c895039..a87d4b4 100644 --- a/app/harness/config.py +++ b/app/harness/config.py @@ -141,7 +141,7 @@ class RunConfig: def _validate(config: RunConfig) -> None: """校验 RunConfig 全部字段约束。 - 四层校验链：基础字段 → 编辑预算 → mini-batch → CE-Gate。 + 六层校验链：mode → 基础标量 → 编辑预算 → mini-batch → gate 阈值 → gate 阶梯。 参数: config: 待校验的配置实例。 @@ -149,7 +149,22 @@ def _validate(config: RunConfig) -> None: 异常: ValueError: 任一字段值不合法。 """ - # ── 基础字段校验 ── + _validate_mode(config) + _validate_basic(config) + _validate_edit_budget(config) + _validate_minibatch(config) + _validate_gate(config) + + +def _validate_mode(config: RunConfig) -> None: + """校验运行模式及其依赖字段（run_id、version）。 + + 参数: + config: 待校验的配置实例。 + + 异常: + ValueError: mode 非法或模式依赖字段缺失。 + """ if config.mode not in _VALID_MODES: raise ValueError(f"mode 必须为 {_VALID_MODES} 之一，实际: {config.mode!r}") if config.mode in ("diagnose", "evolve") and not config.run_id: @@ -158,6 +173,17 @@ def _validate(config: RunConfig) -> None: raise ValueError(f"mode 为 {config.mode!r} 时必须提供 --version。") if config.mode == "promote" and not config.run_id: raise ValueError("promote 必须提供 --run-id（指定 canonical eval run）。") + + +def _validate_basic(config: RunConfig) -> None: + """校验基础标量字段：枚举合法性与正整数约束。 + + 参数: + config: 待校验的配置实例。 + + 异常: + ValueError: 任一基础字段值不合法。 + """ if config.skill_mode not in _VALID_SKILL_MODES: raise ValueError( f"skill_mode 必须为 {_VALID_SKILL_MODES} 之一，实际: {config.skill_mode!r}" @@ -170,10 +196,6 @@ def _validate(config: RunConfig) -> None: raise ValueError(f"n_samples 必须 >= 0，实际: {config.n_samples}") if config.epochs <= 0: raise ValueError(f"epochs 必须 > 0，实际: {config.epochs}") - - # ── 编辑预算校验 ── - _validate_edit_budget(config) - if config.skill_update_mode not in _VALID_SKILL_UPDATE_MODES: raise ValueError( f"skill_update_mode 必须为 {_VALID_SKILL_UPDATE_MODES} 之一，" @@ -185,12 +207,6 @@ def _validate(config: RunConfig) -> None: f"实际: {config.appendix_consolidate_threshold}" ) - # ── mini-batch 校验 ── - _validate_minibatch(config) - - # ── CE-Gate 校验 ── - _validate_gate(config) - def _validate_edit_budget(config: RunConfig) -> None: """校验编辑预算退火的前期/后期上限约束。 @@ -253,7 +269,7 @@ def _validate_minibatch(config: RunConfig) -> None: def _validate_gate(config: RunConfig) -> None: - """校验 CE-Gate 判据与阶梯参数约束。 + """校验 CE-Gate 全部参数：判据阈值 + 信息量阶梯。 参数: config: 待校验的配置实例。 @@ -261,6 +277,19 @@ def _validate_gate(config: RunConfig) -> None: 异常: ValueError: 任一 gate 参数不合法。 """ + _validate_gate_thresholds(config) + _validate_gate_ladder(config) + + +def _validate_gate_thresholds(config: RunConfig) -> None: + """校验 CE-Gate 判据阈值参数（e 值、净胜数、效应量、方向拒绝）。 + + 参数: + config: 待校验的配置实例。 + + 异常: + ValueError: 任一阈值参数不合法。 + """ if config.gate_e_confirm <= 1: raise ValueError(f"gate_e_confirm 必须 > 1，实际: {config.gate_e_confirm}") if not (1 < config.gate_e_provisional <= config.gate_e_confirm): @@ -271,8 +300,21 @@ def _validate_gate(config: RunConfig) -> None: raise ValueError(f"gate_e_rollback 必须 > 1，实际: {config.gate_e_rollback}") if config.gate_w_net_min < 1: raise ValueError(f"gate_w_net_min 必须 >= 1，实际: {config.gate_w_net_min}") + if config.gate_delta_min < 0: + raise ValueError(f"gate_delta_min 必须 >= 0，实际: {config.gate_delta_min}") if config.gate_lambda_dir >= 0: raise ValueError(f"gate_lambda_dir 必须 < 0，实际: {config.gate_lambda_dir}") + + +def _validate_gate_ladder(config: RunConfig) -> None: + """校验 CE-Gate 信息量阶梯与块序贯参数。 + + 参数: + config: 待校验的配置实例。 + + 异常: + ValueError: 任一阶梯参数不合法。 + """ if config.gate_block <= 0 or config.gate_n_max < config.gate_block: raise ValueError( f"需 0 < gate_block <= gate_n_max，" @@ -293,6 +335,20 @@ def _validate_gate(config: RunConfig) -> None: raise ValueError(f"gate_guard_err 须在 (0,1)，实际: {config.gate_guard_err}") +def _apply_env_overrides(data: dict) -> None: + """将 .env 工程配置环境变量覆盖到配置字典中（原地修改）。 + + 仅覆盖 _ENV_FIELD_MAP 中声明的工程配置字段（workspace_dir、store_dir）。 + + 参数: + data: 待覆盖的配置字典。 + """ + for env_key, field_name in _ENV_FIELD_MAP.items(): + env_val = os.environ.get(env_key) + if env_val is not None: + data[field_name] = env_val + + def load_config( yaml_path: Path, cli_overrides: dict[str, object] | None = None, @@ -320,10 +376,7 @@ def load_config( yaml_data: dict = raw.get("harness", raw) # Phase 2: .env 覆盖层（仅工程配置字段） - for env_key, field_name in _ENV_FIELD_MAP.items(): - env_val = os.environ.get(env_key) - if env_val is not None: - yaml_data[field_name] = env_val + _apply_env_overrides(yaml_data) # Phase 3: CLI 覆盖层（最高优先级） valid_fields = {f.name for f in dataclasses.fields(RunConfig)} From ce438718280ff726df84470f36a3355679f5e7c3 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 12:14:28 -0400 Subject: [PATCH 04/19] =?UTF-8?q?fix(harness):=20=E8=A1=A5=E5=85=85=20trai?= =?UTF-8?q?n=20=E6=A8=A1=E5=BC=8F=20run=5Fid=20=E6=A0=A1=E9=AA=8C=20+=20?= =?UTF-8?q?=E6=8B=86=E5=88=86=E5=87=BD=E6=95=B0=E4=BF=9D=E6=8C=81=20radon?= =?UTF-8?q?=20Grade=20B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _validate_mode_deps: 恢复 train 非 resume/fresh 时必须提供 run_id 校验 - 提取 _validate_train_run_id 用 early return 展平条件，避免 radon Grade C - 合并 promote run_id 检查到 diagnose/evolve/promote 统一检查 - 新增 4 个测试：train+run_id / train+resume / train+fresh / train+baseline - radon cc -n C 无输出（全部 Grade B 或更好） - 74 个单元测试全部通过 Co-Authored-By: Claude Opus 4.6 (1M context) --- app/harness/config.py | 38 +++++++++++++++++++++++++++---- tests/unit/test_harness_config.py | 23 ++++++++++++++++++- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/app/harness/config.py b/app/harness/config.py index a87d4b4..188d6b1 100644 --- a/app/harness/config.py +++ b/app/harness/config.py @@ -150,6 +150,7 @@ def _validate(config: RunConfig) -> None: ValueError: 任一字段值不合法。 """ _validate_mode(config) + _validate_mode_deps(config) _validate_basic(config) _validate_edit_budget(config) _validate_minibatch(config) @@ -157,22 +158,49 @@ def _validate(config: RunConfig) -> None: def _validate_mode(config: RunConfig) -> None: - """校验运行模式及其依赖字段（run_id、version）。 + """校验运行模式枚举合法性。 参数: config: 待校验的配置实例。 异常: - ValueError: mode 非法或模式依赖字段缺失。 + ValueError: mode 值不在合法集合中。 """ if config.mode not in _VALID_MODES: raise ValueError(f"mode 必须为 {_VALID_MODES} 之一，实际: {config.mode!r}") - if config.mode in ("diagnose", "evolve") and not config.run_id: + + +def _validate_mode_deps(config: RunConfig) -> None: + """校验各运行模式的依赖字段（run_id、version）。 + + 参数: + config: 待校验的配置实例。 + + 异常: + ValueError: 模式依赖字段缺失。 + """ + if config.mode in ("diagnose", "evolve", "promote") and not config.run_id: raise ValueError(f"mode 为 {config.mode!r} 时必须提供 run_id。") if config.mode in ("eval", "promote") and not config.version: raise ValueError(f"mode 为 {config.mode!r} 时必须提供 --version。") - if config.mode == "promote" and not config.run_id: - raise ValueError("promote 必须提供 --run-id（指定 canonical eval run）。") + _validate_train_run_id(config) + + +def _validate_train_run_id(config: RunConfig) -> None: + """校验 train 模式非 resume/fresh 时必须提供 run_id。 + + 参数: + config: 待校验的配置实例。 + + 异常: + ValueError: train 模式既非 resume 也非 fresh 且缺少 run_id。 + """ + if config.mode != "train": + return + if config.resume or config.fresh: + return + if not config.run_id: + raise ValueError("train 非 resume/fresh 时必须提供 run_id（旧式基线 run）。") def _validate_basic(config: RunConfig) -> None: diff --git a/tests/unit/test_harness_config.py b/tests/unit/test_harness_config.py index b2f15b7..2d78c76 100644 --- a/tests/unit/test_harness_config.py +++ b/tests/unit/test_harness_config.py @@ -139,7 +139,7 @@ class TestModeValidation: def test_all_valid_modes_accepted(self, valid_mode: str) -> None: """全部合法 mode 应通过校验（diagnose/evolve 需 run_id）。""" overrides: dict = {"mode": valid_mode} - if valid_mode in ("diagnose", "evolve"): + if valid_mode in ("diagnose", "evolve", "train"): overrides["run_id"] = "run-001" if valid_mode in ("eval", "promote"): overrides["version"] = "v1" @@ -172,6 +172,27 @@ class TestModeValidation: with pytest.raises(ValueError, match="run.id"): _validate(cfg) + def test_train_mode_requires_run_id_without_resume_fresh(self) -> None: + """train 模式非 resume/fresh 时必须提供 run_id。""" + cfg = _make_config(mode="train", run_id="", resume=False, fresh=False) + with pytest.raises(ValueError, match="run_id"): + _validate(cfg) + + def test_train_mode_resume_without_run_id_accepted(self) -> None: + """train 模式 resume=True 时不需要 run_id。""" + cfg = _make_config(mode="train", run_id="", resume=True) + _validate(cfg) + + def test_train_mode_fresh_without_run_id_accepted(self) -> None: + """train 模式 fresh=True 时不需要 run_id。""" + cfg = _make_config(mode="train", run_id="", fresh=True) + _validate(cfg) + + def test_train_mode_with_run_id_accepted(self) -> None: + """train 模式提供 run_id 时应通过（旧式基线 run）。""" + cfg = _make_config(mode="train", run_id="baseline-001") + _validate(cfg) + def test_concurrency_positive(self) -> None: """concurrency <= 0 应报错。""" cfg = _make_config(concurrency=0) From b929a5db6c3b02a2f833c5f91bede08cfbcda97a Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 12:23:27 -0400 Subject: [PATCH 05/19] =?UTF-8?q?fix(harness):=20Codex=20functional=20revi?= =?UTF-8?q?ew=20=E4=BF=AE=E5=A4=8D=20=E2=80=94=20=E5=91=BD=E5=90=8D/?= =?UTF-8?q?=E9=9B=86=E6=88=90=E6=B5=8B=E8=AF=95/delta=5Fmin/promote=20?= =?UTF-8?q?=E6=B6=88=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _apply_env_overrides → _apply_env_var_overrides，docstring 明确从 os.environ 读取 - 新增 TestLoadConfigRealYaml：用真实 config/default.yaml 验证嵌套 harness 解析 - 新增 test_delta_min_negative_rejected：覆盖 gate_delta_min >= 0 校验 - 恢复 promote 模式独立错误消息（从合并分支分离回 TRM4 原始提示） - 77 个单元测试全部通过，radon 全部 Grade B 或更好 Co-Authored-By: Claude Opus 4.6 (1M context) --- app/harness/config.py | 11 ++++++---- tests/unit/test_harness_config.py | 36 ++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/app/harness/config.py b/app/harness/config.py index 188d6b1..8fdc065 100644 --- a/app/harness/config.py +++ b/app/harness/config.py @@ -179,10 +179,12 @@ def _validate_mode_deps(config: RunConfig) -> None: 异常: ValueError: 模式依赖字段缺失。 """ - if config.mode in ("diagnose", "evolve", "promote") and not config.run_id: + if config.mode in ("diagnose", "evolve") and not config.run_id: raise ValueError(f"mode 为 {config.mode!r} 时必须提供 run_id。") if config.mode in ("eval", "promote") and not config.version: raise ValueError(f"mode 为 {config.mode!r} 时必须提供 --version。") + if config.mode == "promote" and not config.run_id: + raise ValueError("promote 必须提供 --run-id（指定 canonical eval run）。") _validate_train_run_id(config) @@ -363,9 +365,10 @@ def _validate_gate_ladder(config: RunConfig) -> None: raise ValueError(f"gate_guard_err 须在 (0,1)，实际: {config.gate_guard_err}") -def _apply_env_overrides(data: dict) -> None: - """将 .env 工程配置环境变量覆盖到配置字典中（原地修改）。 +def _apply_env_var_overrides(data: dict) -> None: + """从环境变量覆盖路径字段（原地修改）。 + .env 文件由入口脚本 load_dotenv 加载到环境变量，本函数仅从 os.environ 读取。 仅覆盖 _ENV_FIELD_MAP 中声明的工程配置字段（workspace_dir、store_dir）。 参数: @@ -404,7 +407,7 @@ def load_config( yaml_data: dict = raw.get("harness", raw) # Phase 2: .env 覆盖层（仅工程配置字段） - _apply_env_overrides(yaml_data) + _apply_env_var_overrides(yaml_data) # Phase 3: CLI 覆盖层（最高优先级） valid_fields = {f.name for f in dataclasses.fields(RunConfig)} diff --git a/tests/unit/test_harness_config.py b/tests/unit/test_harness_config.py index 2d78c76..2d01d10 100644 --- a/tests/unit/test_harness_config.py +++ b/tests/unit/test_harness_config.py @@ -169,7 +169,7 @@ class TestModeValidation: def test_promote_requires_run_id_and_version(self) -> None: """promote 模式需同时提供 run_id 和 version。""" cfg = _make_config(mode="promote", run_id="", version="v1") - with pytest.raises(ValueError, match="run.id"): + with pytest.raises(ValueError, match="promote.*run-id"): _validate(cfg) def test_train_mode_requires_run_id_without_resume_fresh(self) -> None: @@ -359,6 +359,12 @@ class TestGateValidation: with pytest.raises(ValueError, match="gate_w_net_min"): _validate(cfg) + def test_delta_min_negative_rejected(self) -> None: + """gate_delta_min < 0 应抛出 ValueError。""" + cfg = _make_config(gate_delta_min=-0.1) + with pytest.raises(ValueError, match="gate_delta_min"): + _validate(cfg) + def test_lambda_dir_positive_rejected(self) -> None: """gate_lambda_dir >= 0 应抛出 ValueError。""" cfg = _make_config(gate_lambda_dir=0.5) @@ -527,6 +533,34 @@ class TestLoadConfigEnvOverrides: assert cfg.workspace_dir == Path("/cli/workspace") +# ──────────────────── test_load_config_real_yaml ───────────────────────── + + +class TestLoadConfigRealYaml: + """用真实 config/default.yaml 验证 load_config 嵌套解析。""" + + def test_load_config_real_default_yaml(self) -> None: + """真实 config/default.yaml 的 harness 段应正确解析并通过校验。""" + cfg = load_config(Path("config/default.yaml"), {}) + assert cfg.mode == "infer" + assert cfg.gate_e_confirm == 20.0 + assert cfg.batch_size == 15 + assert cfg.concurrency == 12 + assert cfg.workspace_dir == Path("workspaces/default") + assert cfg.store_dir == Path("store") + assert cfg.skill_mode == "auto" + + def test_real_yaml_cli_override(self) -> None: + """真实 YAML + CLI 覆盖应正确合并。""" + cfg = load_config( + Path("config/default.yaml"), + {"concurrency": 4, "max_steps": 30}, + ) + assert cfg.concurrency == 4 + assert cfg.max_steps == 30 + assert cfg.mode == "infer" # 未覆盖字段保持 YAML 值 + + # ─────────────────────────── test_val_size_floor ───────────────────────── From be3c176a46099d247f5bd7dd06f4e3602bbc4e47 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 12:29:29 -0400 Subject: [PATCH 06/19] feat(harness): HarnessLog SQLite wrapper + RunLogImpl readonly port - HarnessLog: TRM4 direct port with WAL mode, threading.Lock, INSERT OR IGNORE idempotent _runs, context manager (completed/failed), create_table with auto run_id+timestamp, insert/insert_many/execute/query/log_event - RunLogImpl: implements core/evolution/protocols.py::RunLog Protocol with independent sqlite3.connect for read-only SELECT (no _runs pollution), asyncio.to_thread wrapping for async interface - _read_table: shared readonly helper with optional question_ids filtering, graceful empty-list return for missing tables - Tests: 17 cases covering thread safety, idempotent inserts, context manager status, WAL mode, protocol compliance, readonly isolation Co-Authored-By: Claude Opus 4.6 (1M context) --- app/harness/log.py | 347 +++++++++++++++++++++++++++++++++ tests/unit/test_harness_log.py | 347 +++++++++++++++++++++++++++++++++ 2 files changed, 694 insertions(+) create mode 100644 app/harness/log.py create mode 100644 tests/unit/test_harness_log.py diff --git a/app/harness/log.py b/app/harness/log.py new file mode 100644 index 0000000..8d9f970 --- /dev/null +++ b/app/harness/log.py @@ -0,0 +1,347 @@ +"""HarnessLog：SQLite 薄包装 + RunLogImpl 只读查询端口。 + +HarnessLog 提供统一的结构化日志接口，从 TRM4 直搬，保留全部线程安全与幂等语义。 +RunLogImpl 实现 core/evolution/protocols.py::RunLog Protocol，用独立连接做只读 SELECT， +不经 HarnessLog 生命周期（不触发 _runs INSERT OR IGNORE），避免污染运行状态。 +""" + +from __future__ import annotations + +import asyncio +import json +import sqlite3 +import subprocess +import threading +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + + +def _get_git_sha() -> str | None: + """获取当前 git commit SHA。""" + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError): + return None + + +def _now_iso() -> str: + """返回当前 UTC 时间的 ISO 格式字符串。""" + return datetime.now(UTC).isoformat() + + +class HarnessLog: + """SQLite 薄包装，为科研项目提供统一的结构化日志接口。 + + 关键设计： + - WAL 模式 + threading.Lock 保证共享连接下并发安全。 + - INSERT OR IGNORE INTO _runs 保证幂等（同 run_id 多次创建不报错）。 + - query 也持锁：共享连接（check_same_thread=False）下并发 SELECT + INSERT + 在同一连接上 execute 会损坏游标状态，故读也须串行化。 + - context manager 语义：正常退出 completed，异常退出 failed。 + + 参数: + db_path: SQLite 数据库文件路径。 + run_id: 本次运行的唯一标识。 + git_sha: 代码版本，默认自动获取。 + config_snapshot: 本次运行的配置快照。 + """ + + def __init__( + self, + db_path: str, + run_id: str, + git_sha: str | None = None, + config_snapshot: dict[str, Any] | None = None, + ) -> None: + self._run_id = run_id + Path(db_path).parent.mkdir(parents=True, exist_ok=True) + self._conn = sqlite3.connect(db_path, check_same_thread=False) + self._lock = threading.Lock() + self._conn.row_factory = sqlite3.Row + self._conn.execute("PRAGMA journal_mode=WAL") + self._init_fixed_tables() + resolved_sha = git_sha or _get_git_sha() + config_json = ( + json.dumps(config_snapshot, ensure_ascii=False) if config_snapshot else None + ) + self._conn.execute( + "INSERT OR IGNORE INTO _runs" + " (run_id, git_sha, started_at, config, status)" + " VALUES (?, ?, ?, ?, ?)", + (run_id, resolved_sha, _now_iso(), config_json, "running"), + ) + self._conn.commit() + + def _init_fixed_tables(self) -> None: + """创建 _runs 和 _events 固定表。""" + self._conn.execute(""" + CREATE TABLE IF NOT EXISTS _runs ( + run_id TEXT PRIMARY KEY, + git_sha TEXT, + started_at TEXT, + finished_at TEXT, + config JSON, + status TEXT DEFAULT 'running', + skills_version TEXT, + prompts_version TEXT, + questions_ref TEXT + ) + """) + self._conn.execute(""" + CREATE TABLE IF NOT EXISTS _events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT, + timestamp TEXT, + event_type TEXT, + payload JSON + ) + """) + self._conn.commit() + + def create_table( + self, + name: str, + columns: dict[str, str], + primary_key: str | None = None, + ) -> None: + """创建自定义表，自动追加 run_id 和 timestamp 列。 + + 参数: + name: 表名。 + columns: 列定义，如 {"epoch": "INTEGER", "loss": "REAL"}。 + primary_key: 主键列名。 + """ + all_columns = {"run_id": "TEXT", "timestamp": "TEXT"} + all_columns.update(columns) + col_defs = [] + for col_name, col_type in all_columns.items(): + pk_suffix = " PRIMARY KEY" if col_name == primary_key else "" + col_defs.append(f"{col_name} {col_type}{pk_suffix}") + sql = f"CREATE TABLE IF NOT EXISTS {name} ({', '.join(col_defs)})" + self._conn.execute(sql) + self._conn.commit() + + def insert(self, table: str, record: dict[str, Any], mode: str = "append") -> None: + """插入一条记录，自动填充 run_id 和 timestamp。 + + 参数: + table: 目标表名。 + record: 要插入的数据。 + mode: "append" 或 "upsert"。 + """ + enriched = {"run_id": self._run_id, "timestamp": _now_iso()} + enriched.update(record) + cols = list(enriched.keys()) + placeholders = ", ".join(["?"] * len(cols)) + col_names = ", ".join(cols) + values = [enriched[c] for c in cols] + if mode == "upsert": + sql = ( + f"INSERT OR REPLACE INTO {table} ({col_names}) VALUES ({placeholders})" + ) + else: + sql = f"INSERT INTO {table} ({col_names}) VALUES ({placeholders})" + with self._lock: + self._conn.execute(sql, values) + self._conn.commit() + + def insert_many( + self, table: str, records: list[dict[str, Any]], mode: str = "append" + ) -> None: + """批量插入多条记录。 + + 参数: + table: 目标表名。 + records: 要插入的数据列表。 + mode: "append" 或 "upsert"。 + """ + for record in records: + self.insert(table, record, mode=mode) + + def execute(self, sql: str, params: tuple[Any, ...] = ()) -> None: + """执行原生 SQL 写操作。 + + 参数: + sql: SQL 语句。 + params: 参数元组。 + """ + with self._lock: + self._conn.execute(sql, params) + self._conn.commit() + + def query(self, sql: str, params: tuple[Any, ...] = ()) -> list[dict[str, Any]]: + """执行原生 SQL 查询，返回 list[dict]。 + + 与所有写方法同持 self._lock：共享连接（check_same_thread=False）下， + 并发 SELECT 与 INSERT 在同一连接上 execute 会损坏游标状态，故读也须串行化。 + + 参数: + sql: SQL 查询语句。 + params: 参数元组。 + + 返回: + 查询结果列表，每行为一个字典。 + """ + with self._lock: + cursor = self._conn.execute(sql, params) + columns = [desc[0] for desc in cursor.description] + return [ + dict(zip(columns, row, strict=True)) for row in cursor.fetchall() + ] + + def log_event(self, event_type: str, payload: dict[str, Any]) -> None: + """向 _events 表写入一条事件。 + + 参数: + event_type: 事件类型标识。 + payload: 事件数据。 + """ + with self._lock: + self._conn.execute( + "INSERT INTO _events (run_id, timestamp, event_type, payload)" + " VALUES (?, ?, ?, ?)", + ( + self._run_id, + _now_iso(), + event_type, + json.dumps(payload, ensure_ascii=False), + ), + ) + self._conn.commit() + + def close(self, status: str = "completed") -> None: + """更新运行状态并关闭连接。 + + 参数: + status: 最终状态，"completed" 或 "failed"。 + """ + with self._lock: + self._conn.execute( + "UPDATE _runs SET finished_at = ?, status = ? WHERE run_id = ?", + (_now_iso(), status, self._run_id), + ) + self._conn.commit() + self._conn.close() + + def __enter__(self) -> HarnessLog: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: Any, + ) -> None: + status = "failed" if exc_type is not None else "completed" + self.close(status=status) + + +# --------------------------------------------------------------------------- +# RunLogImpl — core/evolution/protocols.py::RunLog 的只读实现 +# --------------------------------------------------------------------------- + + +def _read_table( + db_path: str, + table: str, + run_id: str, + *, + question_ids: list[str] | None = None, +) -> list[dict[str, Any]]: + """纯读某表指定 run 的行——不经 HarnessLog 生命周期，避免回读污染 _runs 运行状态。 + + HarnessLog.__enter__/__exit__ 会对 run_id 做 INSERT OR IGNORE 并在退出时标 completed； + 回读指标绝不应改运行状态，故走独立只读连接（仅 SELECT）。 + + 参数: + db_path: SQLite 路径。 + table: 表名（内部固定常量，非外部输入，无注入风险）。 + run_id: 过滤的 run ID。 + question_ids: 可选的 question_id 过滤列表。 + + 返回: + 行 dict 列表；表尚未建（没写过）视为无数据返 []。 + """ + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + try: + exists = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table,) + ).fetchone() + if exists is None: + return [] + + if question_ids is not None: + placeholders = ", ".join(["?"] * len(question_ids)) + sql = ( + f"SELECT * FROM {table}" + f" WHERE run_id = ? AND question_id IN ({placeholders})" + ) + rows = conn.execute(sql, (run_id, *question_ids)).fetchall() + else: + rows = conn.execute( + f"SELECT * FROM {table} WHERE run_id = ?", (run_id,) + ).fetchall() + + return [dict(r) for r in rows] + finally: + conn.close() + + +class RunLogImpl: + """RunLog Protocol 的只读实现。 + + 用独立 sqlite3.connect 做 SELECT，不经 HarnessLog 生命周期（不触发 _runs INSERT）， + asyncio.to_thread 包装同步 SQL 查询，避免引入 aiosqlite 新依赖。 + + 参数: + db_path: SQLite 数据库文件路径。 + """ + + def __init__(self, db_path: str) -> None: + self._db_path = db_path + + async def get_predictions( + self, + run_id: str, + *, + question_ids: list[str] | None = None, + ) -> list[dict[str, Any]]: + """查询指定 run 的预测记录。 + + 参数: + run_id: 运行标识。 + question_ids: 可选的题目 ID 过滤列表。 + + 返回: + 预测记录字典列表。 + """ + return await asyncio.to_thread( + _read_table, self._db_path, "predictions", run_id, question_ids=question_ids + ) + + async def get_traces( + self, + run_id: str, + *, + question_ids: list[str] | None = None, + ) -> list[dict[str, Any]]: + """查询指定 run 的推理轨迹。 + + 参数: + run_id: 运行标识。 + question_ids: 可选的题目 ID 过滤列表。 + + 返回: + 轨迹记录字典列表。 + """ + return await asyncio.to_thread( + _read_table, self._db_path, "traces", run_id, question_ids=question_ids + ) diff --git a/tests/unit/test_harness_log.py b/tests/unit/test_harness_log.py new file mode 100644 index 0000000..ed86871 --- /dev/null +++ b/tests/unit/test_harness_log.py @@ -0,0 +1,347 @@ +"""HarnessLog + RunLogImpl 单元测试。 + +HarnessLog: SQLite 薄包装的线程安全、幂等、context manager 语义验证。 +RunLogImpl: 只读 RunLog Protocol 实现，独立连接不污染 _runs 运行状态。 +""" + +from __future__ import annotations + +import sqlite3 +import threading +from typing import TYPE_CHECKING + +import pytest + +from app.harness.log import HarnessLog, RunLogImpl + +if TYPE_CHECKING: + from pathlib import Path + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def db_path(tmp_path: Path) -> str: + """返回临时 SQLite 数据库路径。""" + return str(tmp_path / "test.db") + + +@pytest.fixture() +def run_id() -> str: + return "test-run-001" + + +# =========================================================================== +# HarnessLog 测试 +# =========================================================================== + + +class TestHarnessLog: + """HarnessLog 核心功能测试。""" + + def test_create_table_and_insert(self, db_path: str, run_id: str) -> None: + """create_table 建表 + insert 写入 + query 读回。""" + with HarnessLog(db_path, run_id) as log: + log.create_table("metrics", {"epoch": "INTEGER", "loss": "REAL"}) + log.insert("metrics", {"epoch": 1, "loss": 0.5}) + rows = log.query("SELECT * FROM metrics WHERE run_id = ?", (run_id,)) + + assert len(rows) == 1 + assert rows[0]["epoch"] == 1 + assert rows[0]["loss"] == 0.5 + assert rows[0]["run_id"] == run_id + # insert 自动填充 timestamp + assert rows[0]["timestamp"] is not None + + def test_query_thread_safety(self, db_path: str, run_id: str) -> None: + """并发读写不损坏游标状态。""" + errors: list[Exception] = [] + + with HarnessLog(db_path, run_id) as log: + log.create_table("counter", {"value": "INTEGER"}) + + def writer() -> None: + try: + for i in range(50): + log.insert("counter", {"value": i}) + except Exception as exc: + errors.append(exc) + + def reader() -> None: + try: + for _ in range(50): + log.query("SELECT COUNT(*) as cnt FROM counter") + except Exception as exc: + errors.append(exc) + + threads = [ + threading.Thread(target=writer), + threading.Thread(target=reader), + threading.Thread(target=reader), + ] + for t in threads: + t.start() + for t in threads: + t.join() + + assert errors == [], f"并发读写产生错误: {errors}" + + def test_context_manager_completed(self, db_path: str, run_id: str) -> None: + """正常退出时 status = completed。""" + with HarnessLog(db_path, run_id): + pass + + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + row = conn.execute( + "SELECT status, finished_at FROM _runs WHERE run_id = ?", (run_id,) + ).fetchone() + conn.close() + + assert row["status"] == "completed" + assert row["finished_at"] is not None + + def test_context_manager_failed(self, db_path: str, run_id: str) -> None: + """异常退出时 status = failed。""" + with pytest.raises(ValueError, match="boom"), HarnessLog(db_path, run_id): + raise ValueError("boom") + + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + row = conn.execute( + "SELECT status FROM _runs WHERE run_id = ?", (run_id,) + ).fetchone() + conn.close() + + assert row["status"] == "failed" + + def test_insert_or_ignore_idempotent(self, db_path: str, run_id: str) -> None: + """同一 run_id 多次创建 HarnessLog 不报错（INSERT OR IGNORE 幂等）。""" + with HarnessLog(db_path, run_id): + pass + + # 再次用同一 run_id 打开——不应抛异常 + with HarnessLog(db_path, run_id) as log: + log.create_table("t", {"x": "INTEGER"}) + log.insert("t", {"x": 42}) + + conn = sqlite3.connect(db_path) + count = conn.execute( + "SELECT COUNT(*) FROM _runs WHERE run_id = ?", (run_id,) + ).fetchone()[0] + conn.close() + + assert count == 1, "INSERT OR IGNORE 应保证 _runs 只有一行" + + def test_wal_mode(self, db_path: str, run_id: str) -> None: + """连接初始化后 journal_mode 应为 WAL。""" + with HarnessLog(db_path, run_id) as log: + rows = log.query("PRAGMA journal_mode") + + assert rows[0]["journal_mode"].lower() == "wal" + + def test_insert_many(self, db_path: str, run_id: str) -> None: + """insert_many 批量插入多条记录。""" + records = [{"epoch": i, "loss": float(i) * 0.1} for i in range(5)] + + with HarnessLog(db_path, run_id) as log: + log.create_table("batch", {"epoch": "INTEGER", "loss": "REAL"}) + log.insert_many("batch", records) + rows = log.query( + "SELECT * FROM batch WHERE run_id = ? ORDER BY epoch", (run_id,) + ) + + assert len(rows) == 5 + assert [r["epoch"] for r in rows] == [0, 1, 2, 3, 4] + + def test_log_event(self, db_path: str, run_id: str) -> None: + """log_event 向 _events 表写入事件。""" + with HarnessLog(db_path, run_id) as log: + log.log_event("train_start", {"epoch": 1, "lr": 0.001}) + log.log_event("train_end", {"epoch": 1, "loss": 0.42}) + rows = log.query( + "SELECT * FROM _events WHERE run_id = ? ORDER BY id", (run_id,) + ) + + assert len(rows) == 2 + assert rows[0]["event_type"] == "train_start" + assert rows[1]["event_type"] == "train_end" + + def test_execute_raw_sql(self, db_path: str, run_id: str) -> None: + """execute 执行原生 SQL 写操作。""" + with HarnessLog(db_path, run_id) as log: + log.create_table("raw", {"val": "INTEGER"}) + log.execute( + "INSERT INTO raw (run_id, timestamp, val) VALUES (?, ?, ?)", + (run_id, "2026-01-01T00:00:00", 99), + ) + rows = log.query("SELECT val FROM raw WHERE run_id = ?", (run_id,)) + + assert len(rows) == 1 + assert rows[0]["val"] == 99 + + def test_upsert_mode(self, db_path: str, run_id: str) -> None: + """insert mode='upsert' 使用 INSERT OR REPLACE。""" + with HarnessLog(db_path, run_id) as log: + log.create_table("kv", {"key": "TEXT", "val": "TEXT"}, primary_key="key") + log.insert("kv", {"key": "a", "val": "1"}, mode="upsert") + log.insert("kv", {"key": "a", "val": "2"}, mode="upsert") + rows = log.query("SELECT val FROM kv WHERE key = 'a'") + + assert len(rows) == 1 + assert rows[0]["val"] == "2" + + +# =========================================================================== +# RunLogImpl 测试 +# =========================================================================== + + +def _setup_predictions_and_traces(db_path: str, run_id: str) -> None: + """向测试数据库写入 predictions 和 traces 数据，模拟 inference 阶段产物。""" + with HarnessLog(db_path, run_id) as log: + log.create_table( + "predictions", + { + "question_id": "TEXT", + "predicted_answer": "TEXT", + "correct": "INTEGER", + }, + ) + log.create_table( + "traces", + { + "question_id": "TEXT", + "step_idx": "INTEGER", + "action": "TEXT", + "observation": "TEXT", + }, + ) + log.insert_many( + "predictions", + [ + {"question_id": "q1", "predicted_answer": "A", "correct": 1}, + {"question_id": "q2", "predicted_answer": "B", "correct": 0}, + {"question_id": "q3", "predicted_answer": "C", "correct": 1}, + ], + ) + log.insert_many( + "traces", + [ + { + "question_id": "q1", + "step_idx": 0, + "action": "search", + "observation": "found", + }, + { + "question_id": "q1", + "step_idx": 1, + "action": "verify", + "observation": "ok", + }, + { + "question_id": "q2", + "step_idx": 0, + "action": "search", + "observation": "not found", + }, + ], + ) + + +class TestRunLogImpl: + """RunLogImpl 只读查询端口测试。""" + + @pytest.mark.asyncio + async def test_get_predictions(self, db_path: str, run_id: str) -> None: + """get_predictions 返回指定 run 的全部预测记录。""" + _setup_predictions_and_traces(db_path, run_id) + impl = RunLogImpl(db_path) + + preds = await impl.get_predictions(run_id) + + assert len(preds) == 3 + q_ids = {p["question_id"] for p in preds} + assert q_ids == {"q1", "q2", "q3"} + + @pytest.mark.asyncio + async def test_get_predictions_filtered(self, db_path: str, run_id: str) -> None: + """get_predictions 按 question_ids 过滤。""" + _setup_predictions_and_traces(db_path, run_id) + impl = RunLogImpl(db_path) + + preds = await impl.get_predictions(run_id, question_ids=["q1", "q3"]) + + assert len(preds) == 2 + q_ids = {p["question_id"] for p in preds} + assert q_ids == {"q1", "q3"} + + @pytest.mark.asyncio + async def test_get_traces(self, db_path: str, run_id: str) -> None: + """get_traces 返回指定 run 的全部轨迹记录。""" + _setup_predictions_and_traces(db_path, run_id) + impl = RunLogImpl(db_path) + + traces = await impl.get_traces(run_id) + + assert len(traces) == 3 + # q1 有 2 步，q2 有 1 步 + q1_traces = [t for t in traces if t["question_id"] == "q1"] + assert len(q1_traces) == 2 + + @pytest.mark.asyncio + async def test_get_traces_filtered(self, db_path: str, run_id: str) -> None: + """get_traces 按 question_ids 过滤。""" + _setup_predictions_and_traces(db_path, run_id) + impl = RunLogImpl(db_path) + + traces = await impl.get_traces(run_id, question_ids=["q2"]) + + assert len(traces) == 1 + assert traces[0]["question_id"] == "q2" + + @pytest.mark.asyncio + async def test_readonly_no_runs_insert(self, db_path: str, run_id: str) -> None: + """RunLogImpl 查询不触发 _runs INSERT（不污染运行状态）。""" + _setup_predictions_and_traces(db_path, run_id) + + # 用不同 run_id 查询，确保不会创建新的 _runs 行 + other_run = "nonexistent-run" + impl = RunLogImpl(db_path) + preds = await impl.get_predictions(other_run) + + assert preds == [] + + # 验证 _runs 表没有 other_run 的记录 + conn = sqlite3.connect(db_path) + count = conn.execute( + "SELECT COUNT(*) FROM _runs WHERE run_id = ?", (other_run,) + ).fetchone()[0] + conn.close() + assert count == 0, "RunLogImpl 不应向 _runs 插入记录" + + @pytest.mark.asyncio + async def test_protocol_compliance(self, db_path: str, run_id: str) -> None: + """RunLogImpl 满足 RunLog Protocol（runtime_checkable isinstance 检查）。""" + from core.evolution.protocols import RunLog + + impl = RunLogImpl(db_path) + assert isinstance(impl, RunLog) + + @pytest.mark.asyncio + async def test_missing_table_returns_empty(self, db_path: str, run_id: str) -> None: + """查询不存在的表（predictions/traces 未建）返回空列表。""" + # 仅创建 _runs，不建 predictions/traces 表 + with HarnessLog(db_path, run_id): + pass + + impl = RunLogImpl(db_path) + preds = await impl.get_predictions(run_id) + traces = await impl.get_traces(run_id) + + assert preds == [] + assert traces == [] From b052c1f3ee4df93f0480cf8654998c446d3afbf4 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 12:36:36 -0400 Subject: [PATCH 07/19] =?UTF-8?q?feat(harness):=20store.py=20=E2=80=94=20S?= =?UTF-8?q?tore=20=E7=89=88=E6=9C=AC=E6=93=8D=E4=BD=9C=20+=20Seed=20?= =?UTF-8?q?=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 从 TRM4 core/workspace.py 拆出 Store + Seed 相关函数： - _parse_version / list_versions / next_version / advance_version - _write_meta / init_store - init_seed / list_seeds / read_seed - extract_run_db（保留原始 CREATE 语句重建主键约束） - promote_to_seed（强校验版本一致 + 非 NULL + finally 清理） 26 个测试全部通过，radon 复杂度 A (2.83)。 Co-Authored-By: Claude Opus 4.6 (1M context) --- app/harness/store.py | 378 +++++++++++++++++++++++++ tests/unit/test_harness_store.py | 460 +++++++++++++++++++++++++++++++ 2 files changed, 838 insertions(+) create mode 100644 app/harness/store.py create mode 100644 tests/unit/test_harness_store.py diff --git a/app/harness/store.py b/app/harness/store.py new file mode 100644 index 0000000..49509d9 --- /dev/null +++ b/app/harness/store.py @@ -0,0 +1,378 @@ +"""Store 版本操作 + Seed 管理。 + +Store 存储版本化资源（视频、题目、Skill、Prompt）， +通过版本号（v1, v2, ...）管理资源的演化历史。 +Seed 是可复现的实验起点，包含权重快照 + baseline 数据库。 +""" + +from __future__ import annotations + +import json +import re +import shutil +import sqlite3 +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +from loguru import logger + +if TYPE_CHECKING: + from pathlib import Path + + +def _now_iso() -> str: + """返回当前 UTC 时间的 ISO 格式字符串。""" + return datetime.now(UTC).isoformat() + + +def _parse_version(name: str) -> int: + """解析版本目录名 ``v\\d+`` 为整数。 + + 参数: + name: 版本目录名，如 ``"v1"``、``"v10"``。 + + 返回: + 版本号整数。 + + 异常: + ValueError: 版本目录名格式不合法（不匹配 ``v\\d+``）。 + """ + match = re.match(r"v(\d+)$", name) + if not match: + raise ValueError(f"无效版本号: {name}") + return int(match.group(1)) + + +def list_versions(store_dir: Path, resource_type: str) -> list[str]: + """列出 Store 中某类资源的所有版本号，按数字值排序。 + + 按数字排序保证 v10 排在 v2 后面（而非字典序 v10 < v2）。 + + 参数: + store_dir: Store 根目录。 + resource_type: 资源类型路径，如 ``"skills"``、``"questions/generated"``。 + + 返回: + 排序后的版本号列表，如 ``["v1", "v2", "v10"]``。 + """ + resource_dir = store_dir / resource_type + if not resource_dir.is_dir(): + return [] + versions = [] + for entry in resource_dir.iterdir(): + if entry.is_dir() and re.match(r"v\d+$", entry.name): + versions.append(entry.name) + return sorted(versions, key=_parse_version) + + +def next_version(store_dir: Path, resource_type: str) -> str: + """返回某类资源的下一个可用版本号。 + + 参数: + store_dir: Store 根目录。 + resource_type: 资源类型路径。 + + 返回: + 下一个版本号字符串，如 ``"v3"``。 + """ + versions = list_versions(store_dir, resource_type) + if not versions: + return "v1" + latest = _parse_version(versions[-1]) + return f"v{latest + 1}" + + +def _write_meta(target_dir: Path, version: str, source: str, **extra: str | None) -> None: + """写入版本元数据文件 ``meta.json``。 + + 参数: + target_dir: 版本目录。 + version: 版本号。 + source: 来源标识（``"manual"`` / ``"evolution"`` / ``"auto-gen"``）。 + **extra: 额外字段（parent, trigger_run, trigger_workspace, description）。 + """ + meta = { + "version": version, + "created_at": _now_iso(), + "parent": extra.get("parent"), + "source": source, + "trigger_run": extra.get("trigger_run"), + "trigger_workspace": extra.get("trigger_workspace"), + "description": extra.get("description", ""), + } + (target_dir / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2)) + + +def advance_version( + store_dir: Path, + resource_type: str, + source_dir: Path, + meta: dict, +) -> str: + """将 source_dir 的内容写入 Store 的下一个版本目录，写入 meta.json。 + + 参数: + store_dir: Store 根目录。 + resource_type: 资源类型路径，如 ``"skills"``、``"questions/generated"``。 + source_dir: 包含新版本资源文件的源目录。 + meta: 元数据字典，至少包含 ``source`` 字段。 + + 返回: + 新版本号字符串，如 ``"v2"``。 + """ + version = next_version(store_dir, resource_type) + target = store_dir / resource_type / version + shutil.copytree(source_dir, target) + _write_meta( + target, + version, + meta.get("source", "manual"), + parent=meta.get("parent"), + trigger_run=meta.get("trigger_run"), + trigger_workspace=meta.get("trigger_workspace"), + description=meta.get("description", ""), + ) + logger.info("Store 版本推进: {}/{}", resource_type, version) + return version + + +def init_store( + store_dir: Path, + videos_source: Path, + skills_dir: Path, + prompts_dir: Path, +) -> None: + """初始化 Store：拷贝视频数据，创建 skills/v1、prompts/v1 和 questions 目录。 + + 参数: + store_dir: Store 目标路径（不得已存在）。 + videos_source: 视频数据源目录。 + skills_dir: 初始 Skill 文件目录。 + prompts_dir: 初始 Prompt 文件目录。 + + 异常: + FileExistsError: Store 目录已存在。 + """ + if store_dir.exists(): + raise FileExistsError(f"Store 已存在: {store_dir}") + store_dir.mkdir(parents=True) + shutil.copytree(videos_source, store_dir / "videos") + (store_dir / "questions" / "benchmarks").mkdir(parents=True) + (store_dir / "questions" / "generated").mkdir(parents=True) + shutil.copytree(skills_dir, store_dir / "skills" / "v1") + _write_meta( + store_dir / "skills" / "v1", + "v1", + "manual", + description="手工创建的初始版本", + ) + shutil.copytree(prompts_dir, store_dir / "prompts" / "v1") + _write_meta( + store_dir / "prompts" / "v1", + "v1", + "manual", + description="手工创建的初始版本", + ) + logger.info("Store 初始化完成: {}", store_dir) + + +# --------------------------------------------------------------------------- +# 种子库（Seed）函数 +# --------------------------------------------------------------------------- + + +def init_seed( + store_dir: Path, + name: str, + skills_dir: Path, + prompts_dir: Path, + baseline_db: Path, + baseline_run_id: str, + parent: str | None, + description: str, +) -> Path: + """在 store/seeds/ 写一个种子：权重 + baseline.db + seed.json。 + + 参数: + store_dir: Store 根目录。 + name: 种子名（如 ``'initial'``、``'from-evolve-v20'``）。 + skills_dir: 该版本 Skill 权重源目录。 + prompts_dir: 该版本 Prompt 权重源目录。 + baseline_db: 该版本全量记录 db（含 _runs + predictions 行）。 + baseline_run_id: 全量记录的 run_id，fresh 时注入 build_pools。 + parent: 来源（initial 为 None）。 + description: 人类可读说明。 + + 返回: + 种子目录路径。 + + 异常: + FileExistsError: 同名种子已存在（不覆盖）。 + """ + seed_dir = store_dir / "seeds" / name + if seed_dir.exists(): + raise FileExistsError(f"种子已存在，不覆盖: {seed_dir}") + seed_dir.mkdir(parents=True) + shutil.copytree(skills_dir, seed_dir / "skills") + shutil.copytree(prompts_dir, seed_dir / "prompts") + shutil.copy2(baseline_db, seed_dir / "baseline.db") + (seed_dir / "seed.json").write_text( + json.dumps( + { + "baseline_run_id": baseline_run_id, + "parent": parent, + "created_at": _now_iso(), + "description": description, + }, + ensure_ascii=False, + indent=2, + ) + ) + logger.info("种子创建完成: {}", seed_dir) + return seed_dir + + +def list_seeds(store_dir: Path) -> list[str]: + """列出 store/seeds 下所有种子名（按名排序）。 + + 参数: + store_dir: Store 根目录。 + + 返回: + 种子名列表（仅含 seed.json 存在的目录），按名排序。 + """ + seeds_root = store_dir / "seeds" + if not seeds_root.is_dir(): + return [] + return sorted(e.name for e in seeds_root.iterdir() if (e / "seed.json").exists()) + + +def read_seed(store_dir: Path, name: str) -> dict: + """读取种子 seed.json；不存在则报错。 + + 参数: + store_dir: Store 根目录。 + name: 种子名。 + + 返回: + seed.json 解析后的字典。 + + 异常: + FileNotFoundError: 该种子不存在。 + """ + seed_json = store_dir / "seeds" / name / "seed.json" + if not seed_json.exists(): + raise FileNotFoundError(f"种子不存在: {name}（{seed_json}）") + return json.loads(seed_json.read_text()) + + +def extract_run_db(src_db: Path, dst_db: Path, run_id: str) -> None: + """从 src_db 抽出某 run_id 的 _runs + predictions 行，写一个最小 db（种子 baseline.db）。 + + 用源表的**原始 CREATE 语句**重建目标表，保留主键/列类型/约束—— + ``_runs.run_id TEXT PRIMARY KEY`` 是 HarnessLog ``INSERT OR IGNORE`` 去重的依据， + 若 seed db 丢主键则续训/fresh-bootstrap 的去重失效。 + + 参数: + src_db: 源 harness.db。 + dst_db: 目标 db（不得已存在）。 + run_id: 要抽取的 run。 + + 异常: + RuntimeError: 源中无该表或无该 run 的行。 + """ + src = sqlite3.connect(src_db) + dst = sqlite3.connect(dst_db) + try: + for table in ("_runs", "predictions"): + create_sql = src.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name=?", + (table,), + ).fetchone() + if create_sql is None or create_sql[0] is None: + raise RuntimeError(f"源 db 无表 {table}") + dst.execute(create_sql[0]) + cols = [r[1] for r in src.execute(f"PRAGMA table_info({table})")] + col_sql = ", ".join(cols) + rows = src.execute( + f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,) + ).fetchall() + if not rows: + raise RuntimeError(f"{table} 中无 run_id={run_id} 的行") + ph = ", ".join("?" * len(cols)) + dst.executemany(f"INSERT INTO {table} ({col_sql}) VALUES ({ph})", rows) + dst.commit() + finally: + dst.close() + src.close() + + +def promote_to_seed( + workspace_dir: Path, + store_dir: Path, + version: str, + eval_run_id: str, + name: str, + description: str, +) -> Path: + """把 workspace 的指定版本 + 配套 prompts + 指定 eval run 全量记录固化成新种子。 + + 强校验 eval_run_id 对应的 _runs 行中 skills_version 必须与 version 一致， + 且 skills_version/prompts_version 均不得为 NULL。 + + 参数: + workspace_dir: 来源 workspace。 + store_dir: Store 根目录。 + version: skills 版本号。 + eval_run_id: canonical eval run（其 _runs 行提供配套 prompts 版本与全量记录）。 + name: 新种子名（冲突报错不覆盖）。 + description: 说明。 + + 返回: + 新种子目录。 + + 异常: + ValueError: eval_run_id 不存在，或其 skills_version 与 version 不符，或版本为 NULL。 + FileExistsError: 同名种子已存在（由 init_seed 抛出）。 + """ + con = sqlite3.connect(workspace_dir / "harness.db") + con.row_factory = sqlite3.Row + try: + row = con.execute( + "SELECT skills_version, prompts_version FROM _runs WHERE run_id=?", + (eval_run_id,), + ).fetchone() + finally: + con.close() + + if row is None: + raise ValueError(f"eval run 不存在: {eval_run_id}") + + skills_v, prompts_v = row["skills_version"], row["prompts_version"] + + # 强校验——eval run 的版本必须与 --version 一致，且不得为 NULL + if skills_v is None or prompts_v is None: + raise ValueError(f"eval run {eval_run_id} 的 _runs 版本对为 NULL（未回填？），无法 promote") + if skills_v != version: + raise ValueError(f"eval run {eval_run_id} 的版本 {skills_v} 与 --version {version} 不符") + + tmp_db = workspace_dir / "_promote_tmp.db" + if tmp_db.exists(): + tmp_db.unlink() + extract_run_db(workspace_dir / "harness.db", tmp_db, eval_run_id) + try: + seed_dir = init_seed( + store_dir, + name, + workspace_dir / "skills" / skills_v, + workspace_dir / "prompts" / prompts_v, + tmp_db, + baseline_run_id=eval_run_id, + parent=f"{workspace_dir.name}:{version}", + description=description, + ) + finally: + tmp_db.unlink() + + logger.info("Promote 完成: {} -> {}", workspace_dir.name, seed_dir) + return seed_dir diff --git a/tests/unit/test_harness_store.py b/tests/unit/test_harness_store.py new file mode 100644 index 0000000..31ae713 --- /dev/null +++ b/tests/unit/test_harness_store.py @@ -0,0 +1,460 @@ +"""Store 版本操作 + Seed 管理的单元测试。""" + +from __future__ import annotations + +import json +import sqlite3 + +import pytest + +from app.harness.store import ( + _parse_version, + _write_meta, + advance_version, + extract_run_db, + init_seed, + init_store, + list_seeds, + list_versions, + next_version, + promote_to_seed, + read_seed, +) + + +# --------------------------------------------------------------------------- +# _parse_version +# --------------------------------------------------------------------------- + + +class TestParseVersion: + """_parse_version 解析 v\\d+ 格式版本号。""" + + def test_parse_version_normal(self) -> None: + assert _parse_version("v1") == 1 + assert _parse_version("v10") == 10 + assert _parse_version("v999") == 999 + + def test_parse_version_invalid(self) -> None: + with pytest.raises(ValueError, match="无效版本号"): + _parse_version("abc") + with pytest.raises(ValueError, match="无效版本号"): + _parse_version("v") + with pytest.raises(ValueError, match="无效版本号"): + _parse_version("v1.0") + + +# --------------------------------------------------------------------------- +# list_versions — 数字排序 +# --------------------------------------------------------------------------- + + +class TestListVersions: + """list_versions 按数字排序，v10 排在 v2 后。""" + + def test_list_versions_numeric_sort(self, tmp_path: "Path") -> None: + """v10 必须排在 v2 后面（非字典序）。""" + store = tmp_path / "store" + resource = store / "skills" + resource.mkdir(parents=True) + for v in ("v1", "v10", "v2", "v20", "v3"): + (resource / v).mkdir() + result = list_versions(store, "skills") + assert result == ["v1", "v2", "v3", "v10", "v20"] + + def test_list_versions_empty(self, tmp_path: "Path") -> None: + store = tmp_path / "store" + assert list_versions(store, "skills") == [] + + def test_list_versions_ignores_non_version_dirs(self, tmp_path: "Path") -> None: + """非 v\\d+ 格式的目录被忽略。""" + store = tmp_path / "store" + resource = store / "skills" + resource.mkdir(parents=True) + (resource / "v1").mkdir() + (resource / "backup").mkdir() + (resource / ".hidden").mkdir() + assert list_versions(store, "skills") == ["v1"] + + +# --------------------------------------------------------------------------- +# next_version +# --------------------------------------------------------------------------- + + +class TestNextVersion: + """next_version 返回下一个可用版本号。""" + + def test_next_version_empty(self, tmp_path: "Path") -> None: + store = tmp_path / "store" + assert next_version(store, "skills") == "v1" + + def test_next_version_after_existing(self, tmp_path: "Path") -> None: + store = tmp_path / "store" + resource = store / "skills" + resource.mkdir(parents=True) + (resource / "v1").mkdir() + (resource / "v2").mkdir() + assert next_version(store, "skills") == "v3" + + def test_next_version_with_gap(self, tmp_path: "Path") -> None: + """v1 和 v10 之间有 gap，next 应为 v11。""" + store = tmp_path / "store" + resource = store / "skills" + resource.mkdir(parents=True) + (resource / "v1").mkdir() + (resource / "v10").mkdir() + assert next_version(store, "skills") == "v11" + + +# --------------------------------------------------------------------------- +# advance_version +# --------------------------------------------------------------------------- + + +class TestAdvanceVersion: + """advance_version copytree + _write_meta。""" + + def test_advance_version(self, tmp_path: "Path") -> None: + store = tmp_path / "store" + resource = store / "skills" + resource.mkdir(parents=True) + (resource / "v1").mkdir() + + source = tmp_path / "new_skills" + source.mkdir() + (source / "skill_a.md").write_text("内容A") + + version = advance_version( + store, + "skills", + source, + {"source": "evolution", "description": "进化产出"}, + ) + assert version == "v2" + assert (resource / "v2" / "skill_a.md").read_text() == "内容A" + + meta = json.loads((resource / "v2" / "meta.json").read_text()) + assert meta["version"] == "v2" + assert meta["source"] == "evolution" + assert meta["description"] == "进化产出" + assert "created_at" in meta + + +# --------------------------------------------------------------------------- +# init_store +# --------------------------------------------------------------------------- + + +class TestInitStore: + """init_store 初始化 Store 目录结构。""" + + def test_init_store(self, tmp_path: "Path") -> None: + videos = tmp_path / "videos_src" + videos.mkdir() + (videos / "v001").mkdir() + (videos / "v001" / "tree.json").write_text("{}") + + skills = tmp_path / "skills_src" + skills.mkdir() + (skills / "search.md").write_text("skill") + + prompts = tmp_path / "prompts_src" + prompts.mkdir() + (prompts / "system.md").write_text("prompt") + + store = tmp_path / "store" + init_store(store, videos, skills, prompts) + + assert (store / "videos" / "v001" / "tree.json").exists() + assert (store / "questions" / "benchmarks").is_dir() + assert (store / "questions" / "generated").is_dir() + assert (store / "skills" / "v1" / "search.md").read_text() == "skill" + assert (store / "prompts" / "v1" / "system.md").read_text() == "prompt" + + skills_meta = json.loads( + (store / "skills" / "v1" / "meta.json").read_text() + ) + assert skills_meta["version"] == "v1" + assert skills_meta["source"] == "manual" + + def test_init_store_exists_raises(self, tmp_path: "Path") -> None: + store = tmp_path / "store" + store.mkdir() + with pytest.raises(FileExistsError, match="Store 已存在"): + init_store(store, tmp_path, tmp_path, tmp_path) + + +# --------------------------------------------------------------------------- +# Seed 相关 +# --------------------------------------------------------------------------- + + +def _make_seed_fixtures(tmp_path): + """创建 seed 测试所需的公共 fixture。""" + store = tmp_path / "store" + store.mkdir(parents=True) + + skills_dir = tmp_path / "sk" + skills_dir.mkdir() + (skills_dir / "search.md").write_text("skill") + + prompts_dir = tmp_path / "pr" + prompts_dir.mkdir() + (prompts_dir / "system.md").write_text("prompt") + + baseline_db = tmp_path / "base.db" + conn = sqlite3.connect(baseline_db) + conn.execute( + "CREATE TABLE _runs (run_id TEXT PRIMARY KEY, status TEXT)" + ) + conn.execute("INSERT INTO _runs VALUES ('r1', 'done')") + conn.execute( + "CREATE TABLE predictions (run_id TEXT, question_id TEXT, answer TEXT)" + ) + conn.execute("INSERT INTO predictions VALUES ('r1', 'q1', 'A')") + conn.commit() + conn.close() + + return store, skills_dir, prompts_dir, baseline_db + + +class TestInitSeed: + """init_seed 创建种子目录。""" + + def test_init_seed(self, tmp_path: "Path") -> None: + store, skills_dir, prompts_dir, baseline_db = _make_seed_fixtures(tmp_path) + seed_dir = init_seed( + store, + "initial", + skills_dir, + prompts_dir, + baseline_db, + "r1", + None, + "初始种子", + ) + assert seed_dir == store / "seeds" / "initial" + assert (seed_dir / "skills" / "search.md").exists() + assert (seed_dir / "prompts" / "system.md").exists() + assert (seed_dir / "baseline.db").exists() + + meta = json.loads((seed_dir / "seed.json").read_text()) + assert meta["baseline_run_id"] == "r1" + assert meta["parent"] is None + assert meta["description"] == "初始种子" + assert "created_at" in meta + + def test_init_seed_exists_raises(self, tmp_path: "Path") -> None: + store, skills_dir, prompts_dir, baseline_db = _make_seed_fixtures(tmp_path) + init_seed(store, "dup", skills_dir, prompts_dir, baseline_db, "r1", None, "first") + with pytest.raises(FileExistsError, match="种子已存在"): + init_seed( + store, "dup", skills_dir, prompts_dir, baseline_db, "r1", None, "second" + ) + + +class TestListSeeds: + """list_seeds 列出所有种子。""" + + def test_list_seeds(self, tmp_path: "Path") -> None: + store, skills_dir, prompts_dir, baseline_db = _make_seed_fixtures(tmp_path) + init_seed(store, "beta", skills_dir, prompts_dir, baseline_db, "r1", None, "b") + init_seed(store, "alpha", skills_dir, prompts_dir, baseline_db, "r1", None, "a") + assert list_seeds(store) == ["alpha", "beta"] + + def test_list_seeds_empty(self, tmp_path: "Path") -> None: + store = tmp_path / "store" + assert list_seeds(store) == [] + + +class TestReadSeed: + """read_seed 读取 seed.json。""" + + def test_read_seed(self, tmp_path: "Path") -> None: + store, skills_dir, prompts_dir, baseline_db = _make_seed_fixtures(tmp_path) + init_seed(store, "s1", skills_dir, prompts_dir, baseline_db, "r1", None, "desc") + meta = read_seed(store, "s1") + assert meta["baseline_run_id"] == "r1" + assert meta["description"] == "desc" + + def test_read_seed_not_found(self, tmp_path: "Path") -> None: + store = tmp_path / "store" + store.mkdir() + with pytest.raises(FileNotFoundError, match="种子不存在"): + read_seed(store, "no_such") + + +# --------------------------------------------------------------------------- +# extract_run_db +# --------------------------------------------------------------------------- + + +class TestExtractRunDb: + """extract_run_db 抽取指定 run 的行并保留 PK。""" + + def _make_src_db(self, path): + """创建带 _runs + predictions 表的源 db。""" + conn = sqlite3.connect(path) + conn.execute( + "CREATE TABLE _runs (run_id TEXT PRIMARY KEY, status TEXT)" + ) + conn.execute("INSERT INTO _runs VALUES ('r1', 'done')") + conn.execute("INSERT INTO _runs VALUES ('r2', 'done')") + conn.execute( + "CREATE TABLE predictions " + "(run_id TEXT, question_id TEXT, answer TEXT)" + ) + conn.execute("INSERT INTO predictions VALUES ('r1', 'q1', 'A')") + conn.execute("INSERT INTO predictions VALUES ('r1', 'q2', 'B')") + conn.execute("INSERT INTO predictions VALUES ('r2', 'q1', 'C')") + conn.commit() + conn.close() + + def test_extract_run_db_preserves_pk(self, tmp_path: "Path") -> None: + """原始 CREATE 保留主键约束。""" + src = tmp_path / "src.db" + dst = tmp_path / "dst.db" + self._make_src_db(src) + extract_run_db(src, dst, "r1") + + conn = sqlite3.connect(dst) + # 验证 _runs 表有 PK + create_sql = conn.execute( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='_runs'" + ).fetchone()[0] + assert "PRIMARY KEY" in create_sql + + # 验证只有 r1 的行 + runs = conn.execute("SELECT * FROM _runs").fetchall() + assert len(runs) == 1 + assert runs[0][0] == "r1" + + preds = conn.execute("SELECT * FROM predictions").fetchall() + assert len(preds) == 2 + conn.close() + + def test_extract_run_db_missing_table(self, tmp_path: "Path") -> None: + """源 db 无目标表时报错。""" + src = tmp_path / "src.db" + dst = tmp_path / "dst.db" + conn = sqlite3.connect(src) + conn.execute("CREATE TABLE other (id TEXT)") + conn.commit() + conn.close() + with pytest.raises(RuntimeError, match="源 db 无表"): + extract_run_db(src, dst, "r1") + + def test_extract_run_db_no_rows(self, tmp_path: "Path") -> None: + """目标 run_id 不存在时报错。""" + src = tmp_path / "src.db" + dst = tmp_path / "dst.db" + self._make_src_db(src) + with pytest.raises(RuntimeError, match="无 run_id="): + extract_run_db(src, dst, "nonexistent") + + +# --------------------------------------------------------------------------- +# promote_to_seed +# --------------------------------------------------------------------------- + + +def _make_promote_fixtures(tmp_path): + """创建 promote_to_seed 测试所需的 workspace + store。""" + ws = tmp_path / "ws" + ws.mkdir() + store = tmp_path / "store" + store.mkdir() + + # workspace 内的 skills/prompts 版本目录 + (ws / "skills" / "v2").mkdir(parents=True) + (ws / "skills" / "v2" / "skill.md").write_text("evolved") + (ws / "prompts" / "v2").mkdir(parents=True) + (ws / "prompts" / "v2" / "system.md").write_text("prompt v2") + + # workspace harness.db + db_path = ws / "harness.db" + conn = sqlite3.connect(db_path) + conn.execute(""" + CREATE TABLE _runs ( + run_id TEXT PRIMARY KEY, + skills_version TEXT, + prompts_version TEXT + ) + """) + conn.execute( + "INSERT INTO _runs VALUES ('eval_001', 'v2', 'v2')" + ) + conn.execute(""" + CREATE TABLE predictions ( + run_id TEXT, question_id TEXT, answer TEXT + ) + """) + conn.execute("INSERT INTO predictions VALUES ('eval_001', 'q1', 'A')") + conn.commit() + conn.close() + + return ws, store + + +class TestPromoteToSeed: + """promote_to_seed 固化 workspace 版本为种子。""" + + def test_promote_to_seed_success(self, tmp_path: "Path") -> None: + ws, store = _make_promote_fixtures(tmp_path) + seed_dir = promote_to_seed(ws, store, "v2", "eval_001", "evolved-seed", "good") + assert seed_dir == store / "seeds" / "evolved-seed" + assert (seed_dir / "skills" / "skill.md").exists() + assert (seed_dir / "prompts" / "system.md").exists() + assert (seed_dir / "baseline.db").exists() + meta = json.loads((seed_dir / "seed.json").read_text()) + assert meta["baseline_run_id"] == "eval_001" + assert meta["parent"] == "ws:v2" + + def test_promote_to_seed_version_mismatch(self, tmp_path: "Path") -> None: + """eval run 的 skills_version 与 --version 不符时报错。""" + ws, store = _make_promote_fixtures(tmp_path) + with pytest.raises(ValueError, match="版本.*不符"): + promote_to_seed(ws, store, "v3", "eval_001", "bad", "mismatch") + + def test_promote_to_seed_null_version(self, tmp_path: "Path") -> None: + """eval run 的版本为 NULL 时报错。""" + ws = tmp_path / "ws2" + ws.mkdir() + store = tmp_path / "store2" + store.mkdir() + db_path = ws / "harness.db" + conn = sqlite3.connect(db_path) + conn.execute(""" + CREATE TABLE _runs ( + run_id TEXT PRIMARY KEY, + skills_version TEXT, + prompts_version TEXT + ) + """) + conn.execute("INSERT INTO _runs VALUES ('eval_null', NULL, NULL)") + conn.commit() + conn.close() + with pytest.raises(ValueError, match="NULL"): + promote_to_seed(ws, store, "v1", "eval_null", "bad", "null ver") + + def test_promote_to_seed_run_not_found(self, tmp_path: "Path") -> None: + """eval run 不存在时报错。""" + ws, store = _make_promote_fixtures(tmp_path) + with pytest.raises(ValueError, match="eval run 不存在"): + promote_to_seed(ws, store, "v1", "nonexistent", "bad", "no run") + + def test_promote_cleanup_tmp_db(self, tmp_path: "Path") -> None: + """finally 清理临时 db 文件。""" + ws, store = _make_promote_fixtures(tmp_path) + promote_to_seed(ws, store, "v2", "eval_001", "clean-test", "cleanup") + # 临时 db 应已清理 + assert not (ws / "_promote_tmp.db").exists() + + def test_promote_cleanup_tmp_db_on_error(self, tmp_path: "Path") -> None: + """即使 init_seed 失败（同名种子），临时 db 也应被清理。""" + ws, store = _make_promote_fixtures(tmp_path) + promote_to_seed(ws, store, "v2", "eval_001", "first", "first time") + with pytest.raises(FileExistsError): + promote_to_seed(ws, store, "v2", "eval_001", "first", "second time") + assert not (ws / "_promote_tmp.db").exists() From d349fe114857918cfde3c95f183f58bb2a7fbfa1 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 12:41:59 -0400 Subject: [PATCH 08/19] =?UTF-8?q?feat(harness):=20workspace.py=20=E2=80=94?= =?UTF-8?q?=20Workspace=20lifecycle=20+=20VersionedSkillStore/PromptStore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ResolvedPaths frozen dataclass: store_dir, videos_dir, questions_dir, skills_dir, prompts_dir, workspace_dir, db_path, analyses_dir, runs_dir - init_workspace: create ws + copy seed weights from store - init_workspace_from_seed: create from seed with fail-fast questions check - load_manifest / resolve_paths: manifest I/O + path resolution (skills/prompts resolve to workspace, videos/questions to store) - update_manifest: key whitelist validation - record_run: idempotent history append + per-video wiki dirs - read_best / update_best: best pointer independent of current - list_video_ids: videos with tree.json - archive_workspace: move to .archive/- - VersionedSkillStore: implements core/evolution/protocols.py::SkillStore - VersionedPromptStore: implements core/evolution/protocols.py::PromptStore - 21 tests all passing (incl. Protocol compliance checks) Co-Authored-By: Claude Opus 4.6 (1M context) --- app/harness/workspace.py | 478 ++++++++++++++++++++++++ tests/unit/test_harness_workspace.py | 519 +++++++++++++++++++++++++++ 2 files changed, 997 insertions(+) create mode 100644 app/harness/workspace.py create mode 100644 tests/unit/test_harness_workspace.py diff --git a/app/harness/workspace.py b/app/harness/workspace.py new file mode 100644 index 0000000..6c13452 --- /dev/null +++ b/app/harness/workspace.py @@ -0,0 +1,478 @@ +"""Workspace 生命周期管理 + manifest 读写 + Protocol 实现。 + +Workspace 是一次实验的独立工作区，通过 manifest.json 引用 Store 中的 +特定版本资源并记录实验过程。Skills/Prompts 权重拷入 workspace 本地， +训练产物只进 workspace 不污染 Store。 + +VersionedSkillStore / VersionedPromptStore 实现 core/evolution/protocols.py +中定义的只读端口，供 core/ 层以 Protocol 方式读取技能和提示词。 +""" + +from __future__ import annotations + +import json +import os +import shutil +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +from loguru import logger + +from app.harness.store import read_seed + +if TYPE_CHECKING: + from pathlib import Path + + +@dataclass(frozen=True) +class ResolvedPaths: + """manifest 解析后的绝对路径集合。 + + 属性: + store_dir: Store 根目录绝对路径。 + videos_dir: 视频数据目录。 + questions_dir: 当前引用的题目目录。 + skills_dir: 当前引用的 Skill 版本目录（workspace 内）。 + prompts_dir: 当前引用的 Prompt 版本目录（workspace 内）。 + workspace_dir: Workspace 根目录。 + db_path: harness.db 路径。 + analyses_dir: 分析报告目录。 + runs_dir: 运行临时状态目录。 + """ + + store_dir: Path + videos_dir: Path + questions_dir: Path + skills_dir: Path + prompts_dir: Path + workspace_dir: Path + db_path: Path + analyses_dir: Path + runs_dir: Path + + +# --------------------------------------------------------------------------- +# 内部工具 +# --------------------------------------------------------------------------- + +_MANIFEST_CURRENT_KEYS = {"videos", "questions", "skills", "prompts"} + + +def _now_iso() -> str: + """返回当前 UTC 时间的 ISO 格式字符串。""" + return datetime.now(UTC).isoformat() + + +# --------------------------------------------------------------------------- +# Workspace 核心函数 +# --------------------------------------------------------------------------- + + +def _scaffold_workspace( + workspace_dir: Path, + store_dir: Path, + questions: str, + skills_version: str, + prompts_version: str, +) -> None: + """写 manifest + 建 analyses/runs 目录（不拷权重；权重由调用方按来源拷入）。 + + 参数: + workspace_dir: 目标 workspace（由调用方保证不存在）。 + store_dir: Store 根目录。 + questions: 题目相对路径，如 ``'benchmarks/Video-MME'``。 + skills_version: manifest.current.skills 初始版本号。 + prompts_version: manifest.current.prompts 初始版本号。 + + 关键实现: + 不依赖任何外部资源源（store 中的 skills/prompts 是否存在不在此校验）， + 因此可被 init_workspace 与种子初始化复用；store 引用以相对路径写入 manifest。 + """ + workspace_dir.mkdir(parents=True) + (workspace_dir / "analyses").mkdir() + (workspace_dir / "runs").mkdir() + + store_abs = store_dir.resolve() + store_rel = os.path.relpath(store_abs, workspace_dir.resolve()) + + manifest = { + "name": workspace_dir.name, + "created_at": _now_iso(), + "store": store_rel, + "current": { + "videos": "videos", + "questions": f"questions/{questions}", + "skills": f"skills/{skills_version}", + "prompts": f"prompts/{prompts_version}", + }, + "history": [], + } + (workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) + + +def init_workspace( + workspace_dir: Path, + store_dir: Path, + questions: str, + skills_version: str, + prompts_version: str, +) -> None: + """创建 Workspace 目录并写入初始 manifest.json，拷贝种子权重。 + + 参数: + workspace_dir: Workspace 目标路径（不得已存在）。 + store_dir: Store 根目录。 + questions: 题目在 questions/ 下的相对路径，如 ``"benchmarks/Video-MME"``。 + skills_version: Skills 版本号，如 ``"v1"``。 + prompts_version: Prompts 版本号，如 ``"v1"``。 + + 异常: + FileExistsError: Workspace 目录已存在。 + FileNotFoundError: 引用的资源在 Store 中不存在。 + """ + if workspace_dir.exists(): + raise FileExistsError(f"Workspace 已存在: {workspace_dir}") + + store_abs = store_dir.resolve() + refs = { + "skills": f"skills/{skills_version}", + "prompts": f"prompts/{prompts_version}", + "questions": f"questions/{questions}", + } + for label, rel in refs.items(): + full = store_abs / rel + if not full.is_dir(): + raise FileNotFoundError(f"Store 中不存在 {label}: {full}") + + _scaffold_workspace(workspace_dir, store_dir, questions, skills_version, prompts_version) + + # 拷种子权重进 workspace：v2+ 训练产物只进 workspace，不污染 store + shutil.copytree(store_abs / refs["skills"], workspace_dir / refs["skills"]) + shutil.copytree(store_abs / refs["prompts"], workspace_dir / refs["prompts"]) + logger.info("Workspace 初始化完成: {}", workspace_dir) + + +def init_workspace_from_seed( + workspace_dir: Path, + store_dir: Path, + seed_name: str, + questions: str, +) -> str: + """从种子全新建 workspace：拷权重 -> v1、baseline.db -> harness.db、读 baseline_run_id。 + + 参数: + workspace_dir: 目标 workspace（不得已存在）。 + store_dir: Store 根目录。 + seed_name: 种子名（store/seeds 下）。 + questions: 题目相对路径，如 ``'benchmarks/Video-MME'``。 + + 返回: + baseline_run_id（供 build_pools 使用）。 + + 异常: + FileExistsError: workspace 已存在。 + FileNotFoundError: 种子不存在（由 read_seed 抛出），或 questions ref 目录不存在。 + + 关键实现: + 破坏性/创建操作前先校验 questions ref 存在：fresh 路径在 runner 侧已先 + 归档旧 ws，若到 build_pools 才发现 questions 缺失则旧 ws 已被毁； + 故在此尽早报错（fail-fast），让新 ws 在创建前失败。 + """ + if workspace_dir.exists(): + raise FileExistsError(f"Workspace 已存在: {workspace_dir}") + + # 校验种子存在 + 取 baseline_run_id + meta = read_seed(store_dir, seed_name) + + # fail-fast：校验 questions ref 存在 + questions_ref = store_dir / "questions" / questions + if not questions_ref.is_dir(): + raise FileNotFoundError(f"questions ref 目录不存在: {questions_ref}") + + seed_dir = store_dir / "seeds" / seed_name + _scaffold_workspace(workspace_dir, store_dir, questions, "v1", "v1") + shutil.copytree(seed_dir / "skills", workspace_dir / "skills" / "v1") + shutil.copytree(seed_dir / "prompts", workspace_dir / "prompts" / "v1") + shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db") + + logger.info("Workspace 从种子 '{}' 初始化完成: {}", seed_name, workspace_dir) + return meta["baseline_run_id"] + + +def load_manifest(workspace_dir: Path) -> dict: + """读取并返回 workspace 的 manifest.json。 + + 参数: + workspace_dir: Workspace 根目录。 + + 返回: + manifest 字典。 + + 异常: + FileNotFoundError: manifest.json 不存在。 + """ + manifest_path = workspace_dir / "manifest.json" + if not manifest_path.exists(): + raise FileNotFoundError(f"manifest.json 不存在: {manifest_path}") + return json.loads(manifest_path.read_text()) + + +def resolve_paths(workspace_dir: Path) -> ResolvedPaths: + """读取 manifest，解析 current 中所有资源的绝对路径。 + + skills_dir/prompts_dir 解析到 workspace（非 store）， + videos_dir/questions_dir 解析到 store。 + + 参数: + workspace_dir: Workspace 根目录。 + + 返回: + ResolvedPaths 实例，包含所有资源的绝对路径。 + """ + manifest = load_manifest(workspace_dir) + ws_abs = workspace_dir.resolve() + store_abs = (ws_abs / manifest["store"]).resolve() + current = manifest["current"] + return ResolvedPaths( + store_dir=store_abs, + videos_dir=store_abs / current["videos"], + questions_dir=store_abs / current["questions"], + skills_dir=ws_abs / current["skills"], + prompts_dir=ws_abs / current["prompts"], + workspace_dir=ws_abs, + db_path=ws_abs / "harness.db", + analyses_dir=ws_abs / "analyses", + runs_dir=ws_abs / "runs", + ) + + +def list_video_ids(workspace_dir: Path) -> list[str]: + """列出 workspace 引用的所有视频 ID（含 tree.json 的子目录名）。 + + 参数: + workspace_dir: Workspace 根目录。 + + 返回: + 排序后的视频 ID 列表。 + """ + paths = resolve_paths(workspace_dir) + video_ids = [] + for entry in paths.videos_dir.iterdir(): + if entry.is_dir() and (entry / "tree.json").exists(): + video_ids.append(entry.name) + return sorted(video_ids) + + +def update_manifest(workspace_dir: Path, **version_updates: str) -> None: + """更新 manifest 的 current 字段。 + + 参数: + workspace_dir: Workspace 根目录。 + **version_updates: 要更新的字段及其新值，如 ``skills="skills/v2"``。 + + 异常: + KeyError: 更新的字段不在 current 允许的 key 白名单中。 + """ + invalid = set(version_updates) - _MANIFEST_CURRENT_KEYS + if invalid: + raise KeyError(f"无效的 manifest current 字段: {invalid}") + manifest = load_manifest(workspace_dir) + manifest["current"].update(version_updates) + (workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) + + +def record_run(workspace_dir: Path, run_id: str) -> Path: + """将 current 版本快照追加到 manifest history，创建 run 目录和 per-video wiki 目录。 + + 幂等：同 run_id 不重复追加 history（长跑中断后重启 / held-out 复用 run_id 时）。 + + 参数: + workspace_dir: Workspace 根目录。 + run_id: 本次运行的唯一标识，如 ``"run_001"``。 + + 返回: + 创建的 run 目录路径。 + """ + manifest = load_manifest(workspace_dir) + current = manifest["current"] + + # 幂等：同 run_id 不重复追加 history + if not any(h["run_id"] == run_id for h in manifest["history"]): + manifest["history"].append( + { + "run_id": run_id, + "started_at": _now_iso(), + "skills": current["skills"], + "prompts": current["prompts"], + "questions": current["questions"], + } + ) + (workspace_dir / "manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2) + ) + + run_dir = workspace_dir / "runs" / run_id + # exist_ok：同 run_id 重跑时 run 目录已存在不应崩溃 + run_dir.mkdir(parents=True, exist_ok=True) + for video_id in list_video_ids(workspace_dir): + (run_dir / video_id / "wiki").mkdir(parents=True, exist_ok=True) + + logger.debug("Run 已记录: {}", run_id) + return run_dir + + +def read_best(workspace_dir: Path) -> dict | None: + """读取 manifest 的 best 指针，未设置时返回 None。 + + 参数: + workspace_dir: Workspace 根目录。 + + 返回: + best 字典（skills/prompts/val_acc/run_id/epoch），未设置时 None。 + """ + return load_manifest(workspace_dir).get("best") + + +def update_best( + workspace_dir: Path, + skills: str, + prompts: str, + val_acc: float, + run_id: str, + epoch: int, +) -> None: + """写入 manifest 的 best 指针（历史最优版本快照，与 current 平级）。 + + best 独立于 current——更新 best 不影响 current。 + + 参数: + workspace_dir: Workspace 根目录。 + skills: 最优 skills 版本完整 ref，如 ``'skills/v2'``。 + prompts: 最优 prompts 版本完整 ref，如 ``'prompts/v2'``。 + val_acc: 该版本验证集准确率。 + run_id: 该版本验证 run_id。 + epoch: 达成该最优的轮次。 + """ + manifest = load_manifest(workspace_dir) + manifest["best"] = { + "skills": skills, + "prompts": prompts, + "val_acc": val_acc, + "run_id": run_id, + "epoch": epoch, + } + (workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) + logger.info("Best 已更新: val_acc={}, run={}, epoch={}", val_acc, run_id, epoch) + + +def archive_workspace(workspace_dir: Path) -> Path: + """把 workspace 整体移动到同级 .archive/-，返回归档路径。 + + 参数: + workspace_dir: 要归档的 Workspace 根目录。 + + 返回: + 归档后的目标路径。 + + 异常: + FileNotFoundError: workspace 不存在。 + """ + if not workspace_dir.exists(): + raise FileNotFoundError(f"workspace 不存在: {workspace_dir}") + + archive_root = workspace_dir.parent / ".archive" + archive_root.mkdir(exist_ok=True) + ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S") + target = archive_root / f"{workspace_dir.name}-{ts}" + shutil.move(str(workspace_dir), str(target)) + + logger.info("Workspace 已归档: {} -> {}", workspace_dir, target) + return target + + +# --------------------------------------------------------------------------- +# Protocol 实现：VersionedSkillStore / VersionedPromptStore +# --------------------------------------------------------------------------- + + +class VersionedSkillStore: + """版本化技能读取端口实现。 + + 满足 ``core/evolution/protocols.py::SkillStore`` Protocol。 + 从指定的 skills 版本目录读取 ``.md`` 文件。 + + 参数: + skills_dir: skills 版本目录绝对路径（如 ``workspace/skills/v1``）。 + """ + + def __init__(self, skills_dir: Path) -> None: + if not skills_dir.is_dir(): + raise FileNotFoundError(f"Skills 目录不存在: {skills_dir}") + self._dir = skills_dir + + def read_skill(self, filename: str) -> str: + """读取指定 skill 文件的全文内容。 + + 参数: + filename: skill 文件名，如 ``'temporal-reasoning.md'``。 + + 返回: + 文件全文内容。 + + 异常: + FileNotFoundError: 文件不存在。 + """ + path = self._dir / filename + if not path.exists(): + raise FileNotFoundError(f"Skill 文件不存在: {path}") + return path.read_text() + + def list_skill_files(self) -> list[str]: + """列出当前版本所有 skill 文件名。 + + 返回: + 文件名列表（排序）。 + """ + return sorted(entry.name for entry in self._dir.iterdir() if entry.is_file()) + + +class VersionedPromptStore: + """版本化提示词读取端口实现。 + + 满足 ``core/evolution/protocols.py::PromptStore`` Protocol。 + 从指定的 prompts 版本目录读取 ``.md`` 文件。 + + 参数: + prompts_dir: prompts 版本目录绝对路径（如 ``workspace/prompts/v1``）。 + """ + + def __init__(self, prompts_dir: Path) -> None: + if not prompts_dir.is_dir(): + raise FileNotFoundError(f"Prompts 目录不存在: {prompts_dir}") + self._dir = prompts_dir + + def read_prompt(self, filename: str) -> str: + """读取指定 prompt 文件的全文内容。 + + 参数: + filename: prompt 文件名，如 ``'system.md'``。 + + 返回: + 文件全文内容。 + + 异常: + FileNotFoundError: 文件不存在。 + """ + path = self._dir / filename + if not path.exists(): + raise FileNotFoundError(f"Prompt 文件不存在: {path}") + return path.read_text() + + def list_prompt_files(self) -> list[str]: + """列出当前版本所有 prompt 文件名。 + + 返回: + 文件名列表（排序）。 + """ + return sorted(entry.name for entry in self._dir.iterdir() if entry.is_file()) diff --git a/tests/unit/test_harness_workspace.py b/tests/unit/test_harness_workspace.py new file mode 100644 index 0000000..f11fb6d --- /dev/null +++ b/tests/unit/test_harness_workspace.py @@ -0,0 +1,519 @@ +"""app/harness/workspace 单元测试。 + +覆盖 Workspace 生命周期管理、manifest 读写、 +VersionedSkillStore / VersionedPromptStore Protocol 合规。 +""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pytest + +from app.harness.workspace import ( + ResolvedPaths, + VersionedPromptStore, + VersionedSkillStore, + archive_workspace, + init_workspace, + init_workspace_from_seed, + list_video_ids, + load_manifest, + read_best, + record_run, + resolve_paths, + update_best, + update_manifest, +) +from core.evolution.protocols import PromptStore, SkillStore + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def store_dir(tmp_path: Path) -> Path: + """构建一个最小 Store 目录结构供测试使用。""" + sd = tmp_path / "store" + sd.mkdir() + + # videos：两个含 tree.json 的视频目录 + for vid in ("vid_A", "vid_B"): + vdir = sd / "videos" / vid + vdir.mkdir(parents=True) + (vdir / "tree.json").write_text("{}") + + # skills/v1 + prompts/v1 + s1 = sd / "skills" / "v1" + s1.mkdir(parents=True) + (s1 / "temporal-reasoning.md").write_text("skill content A") + (s1 / "spatial-analysis.md").write_text("skill content B") + + p1 = sd / "prompts" / "v1" + p1.mkdir(parents=True) + (p1 / "system.md").write_text("system prompt v1") + (p1 / "extract.md").write_text("extract prompt v1") + + # questions/benchmarks/Video-MME + q = sd / "questions" / "benchmarks" / "Video-MME" + q.mkdir(parents=True) + (q / "q1.json").write_text('{"id": "q1"}') + + # seed "initial" + seed_dir = sd / "seeds" / "initial" + seed_dir.mkdir(parents=True) + shutil.copytree(s1, seed_dir / "skills") + shutil.copytree(p1, seed_dir / "prompts") + baseline_db = seed_dir / "baseline.db" + baseline_db.write_bytes(b"") + (seed_dir / "seed.json").write_text( + json.dumps({"baseline_run_id": "baseline-001", "parent": None}) + ) + return sd + + +@pytest.fixture() +def workspace_dir(tmp_path: Path) -> Path: + """返回一个不存在的 workspace 路径。""" + return tmp_path / "ws_test" + + +# --------------------------------------------------------------------------- +# ResolvedPaths +# --------------------------------------------------------------------------- + + +def test_resolved_paths_frozen() -> None: + """ResolvedPaths 实例应不可变。""" + rp = ResolvedPaths( + store_dir=Path("/s"), + videos_dir=Path("/v"), + questions_dir=Path("/q"), + skills_dir=Path("/sk"), + prompts_dir=Path("/p"), + workspace_dir=Path("/w"), + db_path=Path("/d"), + analyses_dir=Path("/a"), + runs_dir=Path("/r"), + ) + with pytest.raises(AttributeError): + rp.store_dir = Path("/other") # type: ignore[misc] + + +# --------------------------------------------------------------------------- +# init_workspace +# --------------------------------------------------------------------------- + + +def test_init_workspace(store_dir: Path, workspace_dir: Path) -> None: + """init_workspace 应创建 manifest、拷贝权重。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + + # manifest 存在 + manifest = json.loads((workspace_dir / "manifest.json").read_text()) + assert manifest["current"]["skills"] == "skills/v1" + assert manifest["current"]["prompts"] == "prompts/v1" + assert manifest["current"]["questions"] == "questions/benchmarks/Video-MME" + + # 权重已拷入 workspace + assert (workspace_dir / "skills" / "v1" / "temporal-reasoning.md").exists() + assert (workspace_dir / "prompts" / "v1" / "system.md").exists() + + # analyses, runs 目录已创建 + assert (workspace_dir / "analyses").is_dir() + assert (workspace_dir / "runs").is_dir() + + # 重复创建应报错 + with pytest.raises(FileExistsError): + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + + +# --------------------------------------------------------------------------- +# init_workspace_from_seed +# --------------------------------------------------------------------------- + + +def test_init_workspace_from_seed(store_dir: Path, workspace_dir: Path) -> None: + """从种子初始化应拷权重 + baseline.db + 返回 baseline_run_id。""" + run_id = init_workspace_from_seed( + workspace_dir, + store_dir, + seed_name="initial", + questions="benchmarks/Video-MME", + ) + assert run_id == "baseline-001" + + # 权重在 workspace + assert (workspace_dir / "skills" / "v1" / "temporal-reasoning.md").exists() + assert (workspace_dir / "prompts" / "v1" / "system.md").exists() + + # baseline.db -> harness.db + assert (workspace_dir / "harness.db").exists() + + # manifest 正确 + manifest = json.loads((workspace_dir / "manifest.json").read_text()) + assert manifest["current"]["skills"] == "skills/v1" + + +def test_init_workspace_from_seed_missing_questions(store_dir: Path, workspace_dir: Path) -> None: + """questions ref 不存在应 fail-fast（FileNotFoundError），不创建 workspace。""" + with pytest.raises(FileNotFoundError, match="questions ref"): + init_workspace_from_seed( + workspace_dir, + store_dir, + seed_name="initial", + questions="benchmarks/NONEXISTENT", + ) + # workspace 不应被创建 + assert not workspace_dir.exists() + + +# --------------------------------------------------------------------------- +# load_manifest +# --------------------------------------------------------------------------- + + +def test_load_manifest(store_dir: Path, workspace_dir: Path) -> None: + """正常加载 manifest。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + manifest = load_manifest(workspace_dir) + assert "current" in manifest + assert "history" in manifest + assert manifest["current"]["skills"] == "skills/v1" + + +def test_load_manifest_missing(workspace_dir: Path) -> None: + """manifest 不存在应 FileNotFoundError。""" + workspace_dir.mkdir(parents=True) + with pytest.raises(FileNotFoundError): + load_manifest(workspace_dir) + + +# --------------------------------------------------------------------------- +# update_manifest +# --------------------------------------------------------------------------- + + +def test_update_manifest_invalid_key(store_dir: Path, workspace_dir: Path) -> None: + """非法 key 应 KeyError。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + with pytest.raises(KeyError, match="无效"): + update_manifest(workspace_dir, bad_key="skills/v2") + + +def test_update_manifest_valid(store_dir: Path, workspace_dir: Path) -> None: + """合法 key 应更新 manifest current。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + update_manifest(workspace_dir, skills="skills/v2") + manifest = load_manifest(workspace_dir) + assert manifest["current"]["skills"] == "skills/v2" + + +# --------------------------------------------------------------------------- +# record_run +# --------------------------------------------------------------------------- + + +def test_record_run_idempotent(store_dir: Path, workspace_dir: Path) -> None: + """同 run_id 调用两次不应重复追加 history。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + run_dir1 = record_run(workspace_dir, "run_001") + run_dir2 = record_run(workspace_dir, "run_001") + + # 返回路径一致 + assert run_dir1 == run_dir2 + + # history 只有一条 + manifest = load_manifest(workspace_dir) + matched = [h for h in manifest["history"] if h["run_id"] == "run_001"] + assert len(matched) == 1 + + # run 目录存在 + assert run_dir1.is_dir() + + # per-video wiki 目录存在 + assert (run_dir1 / "vid_A" / "wiki").is_dir() + assert (run_dir1 / "vid_B" / "wiki").is_dir() + + +# --------------------------------------------------------------------------- +# update_best / read_best +# --------------------------------------------------------------------------- + + +def test_update_best_independent_of_current(store_dir: Path, workspace_dir: Path) -> None: + """best 应独立于 current——更新 best 不影响 current。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + + # 初始无 best + assert read_best(workspace_dir) is None + + # 设置 best + update_best( + workspace_dir, + skills="skills/v3", + prompts="prompts/v3", + val_acc=0.85, + run_id="run_005", + epoch=3, + ) + + best = read_best(workspace_dir) + assert best is not None + assert best["skills"] == "skills/v3" + assert best["val_acc"] == 0.85 + assert best["epoch"] == 3 + + # current 不受影响 + manifest = load_manifest(workspace_dir) + assert manifest["current"]["skills"] == "skills/v1" + + +# --------------------------------------------------------------------------- +# archive_workspace +# --------------------------------------------------------------------------- + + +def test_archive_workspace(store_dir: Path, workspace_dir: Path) -> None: + """归档应把 workspace 移到 .archive/ 下。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + archived = archive_workspace(workspace_dir) + + # 原目录不存在 + assert not workspace_dir.exists() + + # 归档目录存在并包含 manifest + assert archived.is_dir() + assert (archived / "manifest.json").exists() + + # 归档路径在 .archive 下 + assert archived.parent.name == ".archive" + + +def test_archive_workspace_missing(workspace_dir: Path) -> None: + """归档不存在的 workspace 应 FileNotFoundError。""" + with pytest.raises(FileNotFoundError): + archive_workspace(workspace_dir) + + +# --------------------------------------------------------------------------- +# list_video_ids +# --------------------------------------------------------------------------- + + +def test_list_video_ids(store_dir: Path, workspace_dir: Path) -> None: + """列出 workspace 引用的所有视频 ID。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + ids = list_video_ids(workspace_dir) + assert ids == ["vid_A", "vid_B"] + + +# --------------------------------------------------------------------------- +# resolve_paths +# --------------------------------------------------------------------------- + + +def test_resolve_paths(store_dir: Path, workspace_dir: Path) -> None: + """resolve_paths 应正确解析绝对路径。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + rp = resolve_paths(workspace_dir) + + # skills_dir/prompts_dir 解析到 workspace（非 store） + ws_abs = workspace_dir.resolve() + assert rp.skills_dir == ws_abs / "skills" / "v1" + assert rp.prompts_dir == ws_abs / "prompts" / "v1" + + # videos/questions 解析到 store + store_abs = store_dir.resolve() + assert rp.videos_dir == store_abs / "videos" + assert rp.questions_dir == store_abs / "questions" / "benchmarks" / "Video-MME" + + # db_path, analyses, runs + assert rp.db_path == ws_abs / "harness.db" + assert rp.analyses_dir == ws_abs / "analyses" + assert rp.runs_dir == ws_abs / "runs" + + +# --------------------------------------------------------------------------- +# VersionedSkillStore +# --------------------------------------------------------------------------- + + +def test_versioned_skill_store_read(store_dir: Path, workspace_dir: Path) -> None: + """read_skill 应返回文件全文。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + skills_dir = workspace_dir / "skills" / "v1" + store = VersionedSkillStore(skills_dir) + content = store.read_skill("temporal-reasoning.md") + assert content == "skill content A" + + +def test_versioned_skill_store_read_missing(store_dir: Path, workspace_dir: Path) -> None: + """读取不存在的 skill 文件应 FileNotFoundError。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + skills_dir = workspace_dir / "skills" / "v1" + store = VersionedSkillStore(skills_dir) + with pytest.raises(FileNotFoundError): + store.read_skill("nonexistent.md") + + +def test_versioned_skill_store_list(store_dir: Path, workspace_dir: Path) -> None: + """list_skill_files 应列出所有 skill 文件名（排序）。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + skills_dir = workspace_dir / "skills" / "v1" + store = VersionedSkillStore(skills_dir) + files = store.list_skill_files() + assert sorted(files) == ["spatial-analysis.md", "temporal-reasoning.md"] + + +# --------------------------------------------------------------------------- +# VersionedPromptStore +# --------------------------------------------------------------------------- + + +def test_versioned_prompt_store(store_dir: Path, workspace_dir: Path) -> None: + """VersionedPromptStore 读写功能验证。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + prompts_dir = workspace_dir / "prompts" / "v1" + store = VersionedPromptStore(prompts_dir) + + content = store.read_prompt("system.md") + assert content == "system prompt v1" + + files = store.list_prompt_files() + assert sorted(files) == ["extract.md", "system.md"] + + +def test_versioned_prompt_store_read_missing(store_dir: Path, workspace_dir: Path) -> None: + """读取不存在的 prompt 文件应 FileNotFoundError。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + prompts_dir = workspace_dir / "prompts" / "v1" + store = VersionedPromptStore(prompts_dir) + with pytest.raises(FileNotFoundError): + store.read_prompt("nonexistent.md") + + +# --------------------------------------------------------------------------- +# Protocol 合规 +# --------------------------------------------------------------------------- + + +def test_skill_store_protocol_compliance(store_dir: Path, workspace_dir: Path) -> None: + """VersionedSkillStore 应满足 core/evolution/protocols.py::SkillStore Protocol。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + skills_dir = workspace_dir / "skills" / "v1" + store = VersionedSkillStore(skills_dir) + assert isinstance(store, SkillStore) + + +def test_prompt_store_protocol_compliance(store_dir: Path, workspace_dir: Path) -> None: + """VersionedPromptStore 应满足 core/evolution/protocols.py::PromptStore Protocol。""" + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + prompts_dir = workspace_dir / "prompts" / "v1" + store = VersionedPromptStore(prompts_dir) + assert isinstance(store, PromptStore) From 9800fef37a6215ccc766e8622937d467a5f65584 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 12:47:33 -0400 Subject: [PATCH 09/19] =?UTF-8?q?feat(harness):=20batching.py=20=E2=80=94?= =?UTF-8?q?=20FFD=20+=20round-robin=20mini-batch=20(#10=20=E7=AE=97?= =?UTF-8?q?=E6=B3=95=E4=BF=9D=E7=9C=9F)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/harness/batching.py | 242 +++++++++++++++++++++++++ tests/unit/test_harness_batching.py | 267 ++++++++++++++++++++++++++++ 2 files changed, 509 insertions(+) create mode 100644 app/harness/batching.py create mode 100644 tests/unit/test_harness_batching.py diff --git a/app/harness/batching.py b/app/harness/batching.py new file mode 100644 index 0000000..5e24d3e --- /dev/null +++ b/app/harness/batching.py @@ -0,0 +1,242 @@ +"""混合 mini-batch 切分：大类打散、小类整锁，供 runner 每 step 处理一个 batch。""" + +from __future__ import annotations + +import math +import random +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from core.types import GeneratedQuestion + + +def build_batches( + items: list[GeneratedQuestion], + correctness: dict[str, bool], + batch_size: int, + min_class_per_batch: int, + seed: int, + correct_ratio: float = 0.0, +) -> tuple[list[list[GeneratedQuestion]], int]: + """把诊断池里的题目切成多个混合 mini-batch。 + + 当 ``correct_ratio > 0`` 时，按题型为每组错题配比一定数量的正确题，使 batch + 包含正误混合样本（"动量"机制）；``correct_ratio <= 0`` 时退化为纯错题模式。 + + 参数: + items: 候选题目全集。 + correctness: question_id -> 基线是否答对。 + batch_size: 单个 batch 的样本数上限（> 0）。 + min_class_per_batch: 小类判定阈值——题目数 ≤ 此值的题型整组锁进单一 + batch（> 0）。 + seed: 随机种子，保证相同输入产出完全一致的切分。 + correct_ratio: 正确题占比（0.0 ~ 1.0）。0.0 = 纯错题；0.5 = 错题:正确题 = 1:1。 + 返回: + (非空 mini-batch 列表, selected_count)；无错题时返回 ([], 0)。 + selected_count 是所有 batch 中题目总数。 + 异常: + ValueError: batch_size 或 min_class_per_batch < 1, 或 + min_class_per_batch >= batch_size（破坏小类整组装箱不超容的前提）。 + 关键实现细节: + 装箱顺序为「先小类后大类」。小类整组用 first-fit-decreasing 装箱：按组大小 + 降序处理（同大小再按 task_type 排序保证确定性），每组放进第一个剩余容量足够 + 的 batch；若现有 batch 都装不下就新开一个空 batch——因小类组大小 + ≤ min_class_per_batch < batch_size，新空 batch 必能容纳，故小类装箱永不抛 + ValueError，且保证整组不拆。再把大类样本（seed 确定性 shuffle 后）round-robin + 分发到所有现存 batch 填充剩余容量。这样小类聚集于单 batch、大类散布多 batch + 且与小类共箱，自然产生多类混合 batch（纯类切片会被 multiclass 断言拒绝）。 + nb = ceil(总题数/batch_size) 是初始 batch 数下界估计而非硬上限：小类装箱可能 + 新开 bin 使实际 batch 数超过 nb。每次新开 bin 都意味着总容量随之增加，故总容量 + 恒 ≥ 总题数，大类 round-robin 跳过满箱后仍能放下全部样本，不会违反 batch_size + 上限。题型按名称排序处理以保证跨运行确定性，不依赖 dict 遍历顺序。 + """ + _validate_params(batch_size, min_class_per_batch) + + rng = random.Random(seed) + grouped = _select_mixed_by_task_type(items, correctness, correct_ratio, rng) + total = sum(len(g) for g in grouped.values()) + if total == 0: + return [], 0 + + nb = max(1, math.ceil(total / batch_size)) + batches: list[list[GeneratedQuestion]] = [[] for _ in range(nb)] + + small, large = _split_by_size(grouped, min_class_per_batch) + for group in _small_groups_decreasing(small): + _pack_small_class(batches, group, batch_size) + _distribute_large_classes(batches, large, batch_size, rng) + + result = [b for b in batches if b] + selected_count = sum(len(b) for b in result) + return result, selected_count + + +def _validate_params(batch_size: int, min_class_per_batch: int) -> None: + """校验切分参数，非法值直接报错而非用默认值掩盖。 + + 除各自 >= 1 外，强制 min_class_per_batch < batch_size：小类组大小 ≤ + min_class_per_batch，唯有此前提成立才能保证小类整组放入单一 batch 而不超容；否则 + _pack_small_class 新开的 bin 会装入超 batch_size 的整组，静默违反容量合约。此约束 + 与 config._validate_minibatch 一致，是 build_batches 对自身前提的防御性自校验（P5）。 + """ + if batch_size < 1: + raise ValueError(f"batch_size 必须 >= 1, 实为 {batch_size}") + if min_class_per_batch < 1: + raise ValueError(f"min_class_per_batch 必须 >= 1, 实为 {min_class_per_batch}") + if min_class_per_batch >= batch_size: + raise ValueError( + f"min_class_per_batch 必须严格 < batch_size, 否则无法保证小类整组放入单一 " + f"batch 不超容; 实为 min_class_per_batch={min_class_per_batch}, " + f"batch_size={batch_size}" + ) + + +def _split_by_size( + grouped: dict[str, list[GeneratedQuestion]], + min_class_per_batch: int, +) -> tuple[dict[str, list[GeneratedQuestion]], dict[str, list[GeneratedQuestion]]]: + """按错题数把题型分为小类（≤ 阈值）与大类（> 阈值）两组。""" + small = {t: g for t, g in grouped.items() if len(g) <= min_class_per_batch} + large = {t: g for t, g in grouped.items() if len(g) > min_class_per_batch} + return small, large + + +def _select_mixed_by_task_type( + items: list[GeneratedQuestion], + correctness: dict[str, bool], + correct_ratio: float, + rng: random.Random, +) -> dict[str, list[GeneratedQuestion]]: + """按题型分组，为每组错题按比例采样正确题混入。 + + 只对有错题的题型做混合——无错题的题型不进 batch，即使有正确题。 + ``correct_ratio <= 0`` 时退化为纯错题模式（向后兼容）。 + + 参数: + items: 候选题目全集。 + correctness: question_id -> 基线是否答对。 + correct_ratio: 正确题占比（0.0 ~ 1.0）。 + rng: 随机数发生器，用于采样正确题。 + 返回: + task_type -> 该题型的混合题目列表（错题全部 + 按比例采样的正确题）。 + """ + errors_by_type: dict[str, list[GeneratedQuestion]] = {} + correct_by_type: dict[str, list[GeneratedQuestion]] = {} + for q in items: + qid = q.question_id + if correctness.get(qid) is False: + errors_by_type.setdefault(q.task_type, []).append(q) + elif correctness.get(qid, False): + correct_by_type.setdefault(q.task_type, []).append(q) + + if correct_ratio <= 0: + return errors_by_type + + # 为每个有错题的 task_type 混入正确题 + grouped: dict[str, list[GeneratedQuestion]] = {} + for task_type in sorted(errors_by_type): + errs = errors_by_type[task_type] + n_correct = round(len(errs) * correct_ratio / (1 - correct_ratio)) + available = correct_by_type.get(task_type, []) + sampled = ( + list(available) + if len(available) <= n_correct + else rng.sample(available, n_correct) + ) + grouped[task_type] = errs + sampled + + return grouped + + +def _small_groups_decreasing( + small: dict[str, list[GeneratedQuestion]], +) -> list[list[GeneratedQuestion]]: + """按组大小降序、同大小按 task_type 升序排出小类组（first-fit-decreasing 顺序）。 + + 参数: + small: task_type -> 小类错题列表。 + 返回: + 排好序的小类组列表；降序处理可降低碎片，确定性 tie-break 保证跨运行一致。 + """ + return [small[t] for t in sorted(small, key=lambda t: (-len(small[t]), t))] + + +def _pack_small_class( + batches: list[list[GeneratedQuestion]], + group: list[GeneratedQuestion], + batch_size: int, +) -> None: + """用 first-fit 把一个小类整组放入首个容得下的 batch，装不下则新开 bin（就地修改）。 + + 因小类组大小 ≤ min_class_per_batch < batch_size，新开的空 batch 必能容纳整组， + 故此函数永不抛 ValueError，且整组不拆。 + + 参数: + batches: 当前各 batch（就地追加，必要时 append 新空 batch）。 + group: 待锁定的小类错题（整组不拆）。 + batch_size: 单 batch 容量上限。 + """ + for b in batches: + if len(b) + len(group) <= batch_size: + b.extend(group) + return + batches.append(list(group)) + + +def _distribute_large_classes( + batches: list[list[GeneratedQuestion]], + large: dict[str, list[GeneratedQuestion]], + batch_size: int, + rng: random.Random, +) -> None: + """将各大类样本 shuffle 后 round-robin 分发到所有现存 batch（就地修改）。 + + 参数: + batches: 当前各 batch（含小类装箱可能新开的 bin，就地追加）。 + large: task_type -> 大类错题列表。 + batch_size: 单 batch 容量上限。 + rng: 复用的随机数发生器，保证 shuffle 确定性。 + 异常: + ValueError: 所有 batch 均满仍有样本未放置（总容量估算异常，合法输入不可达）。 + 关键实现细节: + 轮转范围是「所有现存 batch」而非固定 nb 个——小类装箱新开的 bin 也参与分发。 + 总容量 = 现存 batch 数 × batch_size，每次新开 bin 都同步抬高总容量，故总容量恒 + ≥ 总错题数，防御性 ValueError 在合法输入下不可达。全局指针在所有大类样本间持续 + 轮转（不为每类重置），满箱即跳过，使大类充分散布并与已锁定的小类共箱。题型按名称 + 排序以保证分发顺序确定。 + """ + nb = len(batches) + pointer = 0 + for task_type in sorted(large): + group = list(large[task_type]) + rng.shuffle(group) + for q in group: + pointer = _place_round_robin(batches, q, pointer, batch_size, nb) + + +def _place_round_robin( + batches: list[list[GeneratedQuestion]], + q: GeneratedQuestion, + pointer: int, + batch_size: int, + nb: int, +) -> int: + """从 pointer 起找第一个未满 batch 放入 q，返回下一次起始指针。 + + 参数: + batches: 当前各 batch（就地追加）。 + q: 待放置的样本。 + pointer: 本次轮转起始 batch 下标。 + batch_size: 单 batch 容量上限。 + nb: batch 总数。 + 返回: + 下一次轮转的起始指针（已前移一位）。 + 异常: + ValueError: 扫描一轮所有 batch 均满（总容量估算异常）。 + """ + for offset in range(nb): + idx = (pointer + offset) % nb + if len(batches[idx]) < batch_size: + batches[idx].append(q) + return (idx + 1) % nb + raise ValueError("所有 batch 均满仍有样本待放置, 总容量估算异常") diff --git a/tests/unit/test_harness_batching.py b/tests/unit/test_harness_batching.py new file mode 100644 index 0000000..ae956f2 --- /dev/null +++ b/tests/unit/test_harness_batching.py @@ -0,0 +1,267 @@ +"""app/harness/batching.py 的单元测试。 + +覆盖 FFD + round-robin mini-batch 构建的核心场景： +确定性、小类不拆、大类 round-robin、正确率混合、空错题、参数校验、 +correctness False vs None 精确匹配。 +""" + +from __future__ import annotations + +import pytest + +from core.types import GeneratedQuestion +from app.harness.batching import ( + build_batches, + _validate_params, + _select_mixed_by_task_type, +) + +import random + + +# --------------------------------------------------------------------------- +# 辅助构造 +# --------------------------------------------------------------------------- + +def _make_q( + qid: str, + task_type: str = "default", + video_id: str = "v1", +) -> GeneratedQuestion: + """构造最小 GeneratedQuestion 用于测试。""" + return GeneratedQuestion( + question_id=qid, + video_id=video_id, + task_type=task_type, + question=f"question_{qid}", + options=("A. a", "B. b", "C. c", "D. d"), + answer="A", + source_nodes=("n1",), + difficulty="medium", + ) + + +# --------------------------------------------------------------------------- +# test_build_batches_deterministic +# --------------------------------------------------------------------------- + +class TestBuildBatchesDeterministic: + """相同输入 + 相同 seed 产出完全一致的切分。""" + + def test_same_seed_same_result(self) -> None: + items = [_make_q(f"q{i}", task_type=f"type_{i % 3}") for i in range(20)] + correctness = {f"q{i}": False for i in range(20)} + r1 = build_batches(items, correctness, batch_size=5, min_class_per_batch=2, seed=42) + r2 = build_batches(items, correctness, batch_size=5, min_class_per_batch=2, seed=42) + assert r1 == r2 + + def test_different_seed_may_differ(self) -> None: + """不同 seed 结果应不同（极大概率，用大量样本保证）。""" + items = [_make_q(f"q{i}", task_type=f"type_{i % 5}") for i in range(50)] + correctness = {f"q{i}": False for i in range(50)} + r1 = build_batches(items, correctness, batch_size=10, min_class_per_batch=3, seed=1) + r2 = build_batches(items, correctness, batch_size=10, min_class_per_batch=3, seed=99) + # 至少 batch 内容不同（不比较结构，只比较 selected_count 一致性） + assert r1[1] == r2[1] # 总题数一致 + # 但 batch 内部排列几乎必然不同 + flat1 = [q.question_id for b in r1[0] for q in b] + flat2 = [q.question_id for b in r2[0] for q in b] + assert flat1 != flat2 + + +# --------------------------------------------------------------------------- +# test_small_class_not_split +# --------------------------------------------------------------------------- + +class TestSmallClassNotSplit: + """小类（≤ min_class_per_batch）整组不拆，锁在同一 batch。""" + + def test_small_group_stays_together(self) -> None: + # 2 道题属于 small_type（≤ min_class=3），应在同一 batch + items = [ + _make_q("s1", task_type="small_type"), + _make_q("s2", task_type="small_type"), + # 大类 8 道题 + *[_make_q(f"big{i}", task_type="big_type") for i in range(8)], + ] + correctness = {q.question_id: False for q in items} + batches, count = build_batches( + items, correctness, batch_size=5, min_class_per_batch=3, seed=0 + ) + assert count == 10 + # 找到包含 small_type 的 batch + small_batch = [ + b for b in batches + if any(q.task_type == "small_type" for q in b) + ] + assert len(small_batch) == 1 # 整组在同一个 batch + small_ids = {q.question_id for q in small_batch[0] if q.task_type == "small_type"} + assert small_ids == {"s1", "s2"} + + +# --------------------------------------------------------------------------- +# test_large_class_round_robin +# --------------------------------------------------------------------------- + +class TestLargeClassRoundRobin: + """大类样本 round-robin 散布到多个 batch，不集中于单一 batch。""" + + def test_large_group_distributed(self) -> None: + # 12 道大类题，batch_size=4，min_class=2 → 大类 > 2 → round-robin + items = [_make_q(f"q{i}", task_type="large_type") for i in range(12)] + correctness = {q.question_id: False for q in items} + batches, count = build_batches( + items, correctness, batch_size=4, min_class_per_batch=2, seed=7 + ) + assert count == 12 + assert len(batches) >= 3 # ceil(12/4) = 3 + # 每个 batch 不超过 batch_size + for b in batches: + assert len(b) <= 4 + + +# --------------------------------------------------------------------------- +# test_correct_ratio_mixing +# --------------------------------------------------------------------------- + +class TestCorrectRatioMixing: + """correct_ratio > 0 时混入正确题。""" + + def test_mixed_includes_correct(self) -> None: + items = [ + _make_q("e1", task_type="t1"), + _make_q("e2", task_type="t1"), + _make_q("c1", task_type="t1"), + _make_q("c2", task_type="t1"), + _make_q("c3", task_type="t1"), + ] + correctness = {"e1": False, "e2": False, "c1": True, "c2": True, "c3": True} + batches, count = build_batches( + items, correctness, batch_size=10, min_class_per_batch=2, seed=0, + correct_ratio=0.5, + ) + # correct_ratio=0.5 → 错:正 = 1:1 → 2 错 + 2 正 = 4 题 + assert count == 4 + all_ids = {q.question_id for b in batches for q in b} + assert {"e1", "e2"}.issubset(all_ids) # 错题全部 + correct_in = all_ids - {"e1", "e2"} + assert len(correct_in) == 2 # 采样 2 个正确题 + assert correct_in.issubset({"c1", "c2", "c3"}) + + def test_ratio_zero_pure_errors(self) -> None: + items = [ + _make_q("e1", task_type="t1"), + _make_q("c1", task_type="t1"), + ] + correctness = {"e1": False, "c1": True} + batches, count = build_batches( + items, correctness, batch_size=10, min_class_per_batch=2, seed=0, + correct_ratio=0.0, + ) + assert count == 1 + assert batches[0][0].question_id == "e1" + + +# --------------------------------------------------------------------------- +# test_no_wrong_answers_empty +# --------------------------------------------------------------------------- + +class TestNoWrongAnswersEmpty: + """无错题时返回空列表。""" + + def test_all_correct_returns_empty(self) -> None: + items = [_make_q(f"q{i}") for i in range(5)] + correctness = {f"q{i}": True for i in range(5)} + batches, count = build_batches( + items, correctness, batch_size=3, min_class_per_batch=1, seed=0, + ) + assert batches == [] + assert count == 0 + + def test_empty_items_returns_empty(self) -> None: + batches, count = build_batches( + [], {}, batch_size=3, min_class_per_batch=1, seed=0, + ) + assert batches == [] + assert count == 0 + + +# --------------------------------------------------------------------------- +# test_validate_params_strict +# --------------------------------------------------------------------------- + +class TestValidateParamsStrict: + """参数校验：batch_size < 1、min_class < 1、min_class >= batch_size 都报错。""" + + def test_batch_size_zero(self) -> None: + with pytest.raises(ValueError, match="batch_size 必须 >= 1"): + _validate_params(0, 1) + + def test_batch_size_negative(self) -> None: + with pytest.raises(ValueError, match="batch_size 必须 >= 1"): + _validate_params(-1, 1) + + def test_min_class_zero(self) -> None: + with pytest.raises(ValueError, match="min_class_per_batch 必须 >= 1"): + _validate_params(5, 0) + + def test_min_class_equals_batch_size(self) -> None: + with pytest.raises(ValueError, match="min_class_per_batch 必须严格 < batch_size"): + _validate_params(5, 5) + + def test_min_class_exceeds_batch_size(self) -> None: + with pytest.raises(ValueError, match="min_class_per_batch 必须严格 < batch_size"): + _validate_params(3, 5) + + def test_valid_params_no_error(self) -> None: + _validate_params(5, 3) # 不抛异常 + + +# --------------------------------------------------------------------------- +# test_correctness_false_vs_none +# --------------------------------------------------------------------------- + +class TestCorrectnessFalseVsNone: + """correctness.get(qid) is False 精确匹配：None（未知题）不算错题。""" + + def test_none_excluded_from_errors(self) -> None: + items = [ + _make_q("wrong", task_type="t1"), + _make_q("right", task_type="t1"), + _make_q("unknown", task_type="t1"), + ] + # wrong=False（错题），right=True（正确题），unknown 不在 correctness（None） + correctness: dict[str, bool] = {"wrong": False, "right": True} + batches, count = build_batches( + items, correctness, batch_size=10, min_class_per_batch=2, seed=0, + correct_ratio=0.0, + ) + # 仅 wrong 进入 batch，unknown 不算错题 + assert count == 1 + assert batches[0][0].question_id == "wrong" + + def test_explicit_false_only(self) -> None: + """直接测试 _select_mixed_by_task_type 内部逻辑。""" + items = [ + _make_q("f1", task_type="t1"), + _make_q("n1", task_type="t1"), # None（未知） + _make_q("t1", task_type="t1"), # True（正确） + ] + correctness: dict[str, bool] = {"f1": False, "t1": True} + rng = random.Random(0) + result = _select_mixed_by_task_type(items, correctness, 0.0, rng) + assert "t1" in result + assert len(result["t1"]) == 1 + assert result["t1"][0].question_id == "f1" + + def test_none_not_treated_as_correct(self) -> None: + """None（未知）不进正确组，不被 correct_ratio 采样。""" + items = [ + _make_q("err", task_type="t1"), + _make_q("unk", task_type="t1"), + ] + correctness: dict[str, bool] = {"err": False} + rng = random.Random(0) + result = _select_mixed_by_task_type(items, correctness, 0.5, rng) + # 只有 err 一题错题，unk 不在 correctness 中 → get 返回 None → 不进 correct 组 + assert len(result["t1"]) == 1 # 只有错题，无正确题可混入 From e0f3ee10ec69b895d712226dc6cc69a7a995b158 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 12:48:54 -0400 Subject: [PATCH 10/19] =?UTF-8?q?feat(harness):=20pools.py=20=E2=80=94=20?= =?UTF-8?q?=E4=B8=89=E6=B1=A0=E5=88=87=E5=88=86=EF=BC=88test=E2=86=92valid?= =?UTF-8?q?ation=E2=86=92diagnosis=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/harness/pools.py | 283 ++++++++++++++++++++++++++++++ tests/unit/test_harness_pools.py | 290 +++++++++++++++++++++++++++++++ 2 files changed, 573 insertions(+) create mode 100644 app/harness/pools.py create mode 100644 tests/unit/test_harness_pools.py diff --git a/app/harness/pools.py b/app/harness/pools.py new file mode 100644 index 0000000..809f01b --- /dev/null +++ b/app/harness/pools.py @@ -0,0 +1,283 @@ +"""三池：held-out test + 验证 + 诊断，分层采样 + 冻结持久化。 + +三池切分对应训练循环中的 DataLoader 阶段——从题目全集中按 +test -> validation -> diagnosis 的顺序 progressive exclusion， +保证 question_id 互斥。test 池用自然分布（correct_ratio=None）， +验证池/诊断池按对错比例分层采样。 +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from app.question_gen import stratified_sample +from core.types import GeneratedQuestion + +if TYPE_CHECKING: + from pathlib import Path + + from app.harness.config import RunConfig + + +@dataclass +class Pools: + """冻结的三池及其基线指标。 + + 字段: + diagnosis: 诊断池（用于错误归因，对应 loss.backward）。 + validation: 验证池（按类局部验证，每题型有保底样本）。 + test: held-out 测试池（自然分布，用于最终无偏评估）。 + baseline_run_id: 基线 run 标识。 + baseline_val_accuracy: 基线在验证池上的准确率。 + correctness: 三池所有题的 question_id -> 基线是否答对。 + """ + + diagnosis: list[GeneratedQuestion] + validation: list[GeneratedQuestion] + test: list[GeneratedQuestion] + baseline_run_id: str + baseline_val_accuracy: float + correctness: dict[str, bool] = field(default_factory=dict) + + +def build_pools( + questions: list[GeneratedQuestion], + correctness: dict[str, bool], + diag_cfg: dict, + val_cfg: dict, + test_cfg: dict, + baseline_run_id: str, +) -> Pools: + """先抽 held-out test，再抽验证集，最后抽诊断池，三池互斥。 + + 参数: + questions: 题目全集。 + correctness: question_id -> 基线是否答对。 + diag_cfg: 诊断池采样配置（size/correct_ratio/task_types[/seed]）。 + val_cfg: 验证池采样配置，可含 min_per_class 做按类保底。 + test_cfg: 测试池配置（size[/seed]）；走自然分布，不强制对错比与题型。 + baseline_run_id: 基线 run 标识。 + + 返回: + 冻结的三池 Pools。 + + 关键实现细节: + 切分顺序 test -> validation -> diagnosis；后两步从剩余题中采样以保证 + question_id 互斥。test 池用 correct_ratio=None 的自然分布采样。 + """ + test = _sample_excluding( + questions, + set(), + correctness, + size=test_cfg["size"], + correct_ratio=None, + task_types=None, + seed=test_cfg.get("seed", 0), + min_per_class=None, + ) + selected_ids = {q.question_id for q in test} + + validation = _sample_excluding(questions, selected_ids, correctness, **val_cfg) + selected_ids |= {q.question_id for q in validation} + + diagnosis = _sample_excluding(questions, selected_ids, correctness, **diag_cfg) + + val_correct = sum(1 for q in validation if correctness.get(q.question_id)) + baseline_val_accuracy = val_correct / len(validation) if validation else 0.0 + return Pools( + diagnosis=diagnosis, + validation=validation, + test=test, + baseline_run_id=baseline_run_id, + baseline_val_accuracy=baseline_val_accuracy, + correctness={ + q.question_id: correctness.get(q.question_id, False) + for q in test + validation + diagnosis + }, + ) + + +def _sample_excluding( + questions: list[GeneratedQuestion], + exclude_ids: set[str], + correctness: dict[str, bool], + **cfg: object, +) -> list[GeneratedQuestion]: + """排除已选 question_id 后，按 cfg 对剩余题做分层采样。 + + 参数: + questions: 题目全集。 + exclude_ids: 已被其他池选走的 question_id，从候选中剔除以保证三池互斥。 + correctness: question_id -> 基线是否答对。 + cfg: 透传给 stratified_sample 的采样配置 + （size/correct_ratio/task_types[/seed/min_per_class]）。 + + 返回: + 采样后的题目列表。 + """ + pool = [q for q in questions if q.question_id not in exclude_ids] + return stratified_sample(pool, correctness, **cfg) + + +def _q_to_dict(q: GeneratedQuestion) -> dict: + """将 GeneratedQuestion 转为可序列化字典。 + + 参数: + q: 题目对象。 + + 返回: + 包含全部字段的字典（options/source_nodes 从 tuple 转为 list）。 + """ + return { + "question_id": q.question_id, + "video_id": q.video_id, + "task_type": q.task_type, + "question": q.question, + "options": list(q.options), + "answer": q.answer, + "source_nodes": list(q.source_nodes), + "difficulty": q.difficulty, + } + + +def _dict_to_q(d: dict) -> GeneratedQuestion: + """从字典恢复 GeneratedQuestion。 + + 参数: + d: 由 _q_to_dict 产出的字典。 + + 返回: + 恢复的 GeneratedQuestion 实例（options/source_nodes 恢复为 tuple）。 + """ + return GeneratedQuestion( + question_id=d["question_id"], + video_id=d["video_id"], + task_type=d["task_type"], + question=d["question"], + options=tuple(d["options"]), + answer=d["answer"], + source_nodes=tuple(d.get("source_nodes", ())), + difficulty=d.get("difficulty", "medium"), + ) + + +def save_pools(pools: Pools, path: Path) -> None: + """将三池及基线指标冻结为 JSON。 + + 参数: + pools: 待冻结的三池。 + path: 目标 JSON 文件路径。 + """ + path.write_text( + json.dumps( + { + "baseline_run_id": pools.baseline_run_id, + "baseline_val_accuracy": pools.baseline_val_accuracy, + "correctness": pools.correctness, + "diagnosis": [_q_to_dict(q) for q in pools.diagnosis], + "validation": [_q_to_dict(q) for q in pools.validation], + "test": [_q_to_dict(q) for q in pools.test], + }, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + +def load_pools(path: Path) -> Pools: + """从 JSON 恢复冻结的三池。 + + 参数: + path: 冻结的 pools.json 路径。 + + 返回: + 恢复的三池 Pools。 + + 异常: + ValueError: 旧格式 pools.json（无 test 池）。 + + 关键实现细节: + 旧格式 pools.json（无 test 池）会以清晰的 ValueError 中止——本项目不做 + 向后兼容，也不为缺失字段填默认值。删除旧文件后 build_pools 会重新采样切分， + 无需重新推理。 + """ + d = json.loads(path.read_text(encoding="utf-8")) + if "test" not in d: + raise ValueError( + f"{path} 为旧格式 pools.json（缺 test 池），" + "请删除后重新切分（build_pools 会重新采样，无需重新推理）。" + ) + return Pools( + diagnosis=[_dict_to_q(x) for x in d["diagnosis"]], + validation=[_dict_to_q(x) for x in d["validation"]], + test=[_dict_to_q(x) for x in d["test"]], + baseline_run_id=d["baseline_run_id"], + baseline_val_accuracy=d["baseline_val_accuracy"], + correctness=d["correctness"], + ) + + +def build_or_load_pools( + config: RunConfig, + run_id: str, + task_types: list[str] | None = None, +) -> Pools: + """train 模式的三池获取入口：pools.json 已存在则加载，否则从基线 db 切分并冻结。 + + 把 main.py train 分支「pools.json 存在则 load_pools 否则 build_pools 再 save_pools」 + 那段抽成纯函数，使 main 与集成测试共用同一切分逻辑、避免重复。pools.json 是 + 一次 fresh 训练的冻结切分，resume/重跑同一 workspace 时直接复用以保证三池一致。 + + 参数: + config: 运行配置，提供 workspace_dir 与三池采样旋钮（diag/val/test 各项）。 + run_id: 基线全量记录的 run_id（fresh 时来自 seed.json，决定从哪个 run 读对错）。 + task_types: 可选题型过滤，限定诊断/验证池只采样这些题型；None 表示不过滤。 + + 返回: + 冻结的三池 Pools。 + + 关键实现: + 切分前从基线 db 的 predictions 表读该 run_id 的逐题对错，作为分层采样依据。 + pools.json 落在 config.workspace_dir 下，存在即视为已冻结，原样加载不重切。 + """ + from app.harness.log import HarnessLog + from app.harness.workspace import resolve_paths + from app.question_gen import load_benchmark + + pools_path = config.workspace_dir / "pools.json" + if pools_path.exists(): + return load_pools(pools_path) + + paths = resolve_paths(config.workspace_dir) + questions = load_benchmark(paths.questions_dir) + with HarnessLog(str(paths.db_path), run_id) as log: + rows = log.query( + "SELECT question_id, prediction, answer FROM predictions WHERE run_id=?", + (run_id,), + ) + correctness = {r["question_id"]: r["prediction"] == r["answer"] for r in rows} + pools = build_pools( + questions, + correctness, + diag_cfg={ + "size": config.diag_size, + "correct_ratio": config.diag_correct_ratio, + "task_types": task_types, + "seed": 0, + "min_per_class": None, + }, + val_cfg={ + "size": config.val_size, + "correct_ratio": config.val_correct_ratio, + "task_types": task_types, + "seed": 0, + "min_per_class": config.eval_min_per_class, + }, + test_cfg={"size": config.test_size}, + baseline_run_id=run_id, + ) + save_pools(pools, pools_path) + return pools diff --git a/tests/unit/test_harness_pools.py b/tests/unit/test_harness_pools.py new file mode 100644 index 0000000..c148a40 --- /dev/null +++ b/tests/unit/test_harness_pools.py @@ -0,0 +1,290 @@ +"""三池切分单元测试。 + +验证: +- 三池互斥（question_id 无重叠） +- test 池自然分布（correct_ratio=None） +- save/load 往返一致 +- 旧格式拒绝（无 test 键 → ValueError） +- build_or_load_pools 冻结复用（pools.json 存在时不重切） +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest + +from app.harness.pools import ( + build_pools, + load_pools, + save_pools, +) +from core.types import GeneratedQuestion + +if TYPE_CHECKING: + from pathlib import Path + + +def _make_question(qid: str, task_type: str = "Action Reasoning") -> GeneratedQuestion: + """构造测试用 GeneratedQuestion。 + + 参数: + qid: 题目 ID。 + task_type: 题型。 + + 返回: + GeneratedQuestion 实例。 + """ + return GeneratedQuestion( + question_id=qid, + video_id="video_001", + task_type=task_type, + question=f"Question {qid}?", + options=("A. opt1", "B. opt2", "C. opt3", "D. opt4"), + answer="A", + source_nodes=("node_1",), + difficulty="medium", + ) + + +def _make_question_set( + n: int, + task_types: list[str] | None = None, +) -> list[GeneratedQuestion]: + """构造 n 道题，交替分配题型。 + + 参数: + n: 题目数量。 + task_types: 可选题型列表，轮转分配；None 默认 2 类。 + + 返回: + 题目列表。 + """ + types = task_types or ["Action Reasoning", "Scene Understanding"] + return [_make_question(f"q_{i:04d}", types[i % len(types)]) for i in range(n)] + + +def _make_correctness( + questions: list[GeneratedQuestion], + correct_ratio: float = 0.5, +) -> dict[str, bool]: + """构造 correctness 字典，前 correct_ratio 比例标对。 + + 参数: + questions: 题目列表。 + correct_ratio: 对题占比。 + + 返回: + question_id -> bool。 + """ + n_correct = round(len(questions) * correct_ratio) + return {q.question_id: (i < n_correct) for i, q in enumerate(questions)} + + +class TestBuildPoolsMutualExclusion: + """三池 question_id 互斥验证。""" + + def test_build_pools_mutual_exclusion(self) -> None: + """三池切分后，任意两池不共享 question_id。""" + questions = _make_question_set(200) + correctness = _make_correctness(questions, 0.5) + + pools = build_pools( + questions, + correctness, + diag_cfg={ + "size": 30, + "correct_ratio": 0.5, + "task_types": None, + "seed": 42, + "min_per_class": None, + }, + val_cfg={ + "size": 30, + "correct_ratio": 0.5, + "task_types": None, + "seed": 42, + "min_per_class": None, + }, + test_cfg={"size": 30}, + baseline_run_id="run_baseline", + ) + + diag_ids = {q.question_id for q in pools.diagnosis} + val_ids = {q.question_id for q in pools.validation} + test_ids = {q.question_id for q in pools.test} + + assert diag_ids & val_ids == set(), "诊断池与验证池有重叠" + assert diag_ids & test_ids == set(), "诊断池与测试池有重叠" + assert val_ids & test_ids == set(), "验证池与测试池有重叠" + + assert len(diag_ids) == 30 + assert len(val_ids) == 30 + assert len(test_ids) == 30 + + +class TestBuildPoolsTestNaturalDistribution: + """test 池使用自然分布（correct_ratio=None）。""" + + def test_build_pools_test_natural_distribution(self) -> None: + """test 池不强制对错比例，保留候选池的自然分布。 + + 构造 correctness 为 50% 对/50% 错，diag/val 用 correct_ratio=0.3 + 强制裁剪，test 池走自然分布（correct_ratio=None）。验证 test 池 + 不受 correct_ratio 约束。 + """ + questions = _make_question_set(300) + correctness = _make_correctness(questions, 0.5) + + pools = build_pools( + questions, + correctness, + diag_cfg={ + "size": 20, + "correct_ratio": 0.3, + "task_types": None, + "seed": 42, + "min_per_class": None, + }, + val_cfg={ + "size": 20, + "correct_ratio": 0.3, + "task_types": None, + "seed": 42, + "min_per_class": None, + }, + test_cfg={"size": 20}, + baseline_run_id="run_baseline", + ) + + # diag/val 被 correct_ratio=0.3 裁剪：round(20*0.3) = 6 对, 14 错 + diag_correct = sum(1 for q in pools.diagnosis if correctness[q.question_id]) + val_correct = sum(1 for q in pools.validation if correctness[q.question_id]) + assert diag_correct == 6, "诊断池应强制 30% 对题" + assert val_correct == 6, "验证池应强制 30% 对题" + + # test 池自然分布：不受 correct_ratio 约束 + assert len(pools.test) == 20 + + +class TestSaveLoadPoolsRoundtrip: + """save/load 往返一致验证。""" + + def test_save_load_pools_roundtrip(self, tmp_path: Path) -> None: + """save_pools → load_pools 后全字段一致。""" + questions = _make_question_set(100) + correctness = _make_correctness(questions, 0.5) + + original = build_pools( + questions, + correctness, + diag_cfg={ + "size": 15, + "correct_ratio": 0.5, + "task_types": None, + "seed": 42, + "min_per_class": None, + }, + val_cfg={ + "size": 15, + "correct_ratio": 0.5, + "task_types": None, + "seed": 42, + "min_per_class": None, + }, + test_cfg={"size": 15}, + baseline_run_id="run_001", + ) + + pools_path = tmp_path / "pools.json" + save_pools(original, pools_path) + restored = load_pools(pools_path) + + # 标量字段 + assert restored.baseline_run_id == original.baseline_run_id + assert restored.baseline_val_accuracy == pytest.approx(original.baseline_val_accuracy) + assert restored.correctness == original.correctness + + # 三池逐题比对 + for pool_name in ("diagnosis", "validation", "test"): + orig_list = getattr(original, pool_name) + rest_list = getattr(restored, pool_name) + assert len(rest_list) == len(orig_list), f"{pool_name} 长度不一致" + for o, r in zip(orig_list, rest_list, strict=False): + assert o.question_id == r.question_id + assert o.video_id == r.video_id + assert o.task_type == r.task_type + assert o.question == r.question + assert o.options == r.options + assert o.answer == r.answer + assert o.source_nodes == r.source_nodes + assert o.difficulty == r.difficulty + + +class TestLoadPoolsOldFormatReject: + """旧格式 pools.json（无 test 键）→ ValueError。""" + + def test_load_pools_old_format_reject(self, tmp_path: Path) -> None: + """缺少 test 键的 pools.json 必须抛出 ValueError。""" + old_format = { + "baseline_run_id": "run_old", + "baseline_val_accuracy": 0.5, + "correctness": {}, + "diagnosis": [], + "validation": [], + } + pools_path = tmp_path / "pools.json" + pools_path.write_text(json.dumps(old_format), encoding="utf-8") + + with pytest.raises(ValueError, match="旧格式"): + load_pools(pools_path) + + +class TestBuildOrLoadPoolsFrozen: + """build_or_load_pools 冻结复用：pools.json 存在时原样加载不重切。""" + + def test_build_or_load_pools_frozen(self, tmp_path: Path) -> None: + """pools.json 已存在时，build_or_load_pools 返回冻结内容。""" + questions = _make_question_set(60) + correctness = _make_correctness(questions, 0.5) + + frozen = build_pools( + questions, + correctness, + diag_cfg={ + "size": 10, + "correct_ratio": 0.5, + "task_types": None, + "seed": 42, + "min_per_class": None, + }, + val_cfg={ + "size": 10, + "correct_ratio": 0.5, + "task_types": None, + "seed": 42, + "min_per_class": None, + }, + test_cfg={"size": 10}, + baseline_run_id="run_frozen", + ) + + pools_path = tmp_path / "pools.json" + save_pools(frozen, pools_path) + + # build_or_load_pools 中 pools.json 存在 → 直接 load_pools + # 此处直接测试 load_pools 行为等价 + loaded = load_pools(pools_path) + + assert loaded.baseline_run_id == frozen.baseline_run_id + assert loaded.baseline_val_accuracy == pytest.approx(frozen.baseline_val_accuracy) + assert len(loaded.test) == len(frozen.test) + assert len(loaded.validation) == len(frozen.validation) + assert len(loaded.diagnosis) == len(frozen.diagnosis) + + # question_id 完全一致 + for pool_name in ("diagnosis", "validation", "test"): + orig_ids = [q.question_id for q in getattr(frozen, pool_name)] + load_ids = [q.question_id for q in getattr(loaded, pool_name)] + assert orig_ids == load_ids, f"{pool_name} 冻结后 ID 顺序不一致" From bd4e438c6c2f19ce0adddf7dfce7ddc283bebd13 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 12:55:12 -0400 Subject: [PATCH 11/19] =?UTF-8?q?feat(harness):=20gate=5Fladder.py=20?= =?UTF-8?q?=E2=80=94=20=E4=BF=A1=E6=81=AF=E9=98=B6=E6=A2=AF=20+=20Baseline?= =?UTF-8?q?Cache=20(#6=20=E7=AE=97=E6=B3=95=E4=BF=9D=E7=9C=9F)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/harness/gate_ladder.py | 335 ++++++++++++++++++++++ tests/unit/test_harness_gate_ladder.py | 374 +++++++++++++++++++++++++ 2 files changed, 709 insertions(+) create mode 100644 app/harness/gate_ladder.py create mode 100644 tests/unit/test_harness_gate_ladder.py diff --git a/app/harness/gate_ladder.py b/app/harness/gate_ladder.py new file mode 100644 index 0000000..aad589d --- /dev/null +++ b/app/harness/gate_ladder.py @@ -0,0 +1,335 @@ +"""CE-Gate 信息量阶梯与基线缓存。 + +阶梯（每题型一条）：gate 的出题顺序表。冷启动（FRESH）用种子基线对错 +两档粗排（错题高优先 2:1 交错 + 全错题 probe_quota 探针插尾）； +epoch >=1 用非 gate run 观测做 gamma-EMA 更新 p_hat，按信息量 p_hat(1-p_hat) 降序、 +剔 p_hat 不在 [p_low, p_high]。防泄露铁律：gate 内 rollout 永不回流 p_hat +（调用方以 run_id 含 "_gate_" 过滤观测源）。 + +BaselineCache：基线侧逐题对错缓存，键 = (task_type, skill_hash, +prompts_version, qid) 内容寻址、无显式失效。JSON 持久化到 workspace， +供 resume 后合法复用已冻结阶梯上的新鲜 draw。 +""" + +from __future__ import annotations + +import hashlib +import json +import os +import random +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from loguru import logger + +if TYPE_CHECKING: + from pathlib import Path + + from core.types import GeneratedQuestion + + +def skill_hash(content: str) -> str: + """对 skill 正文取 sha1 摘要，作缓存键的内容维度。 + + 参数: + content: skill 文件全文（基线侧为解析后生效文件的正文）。 + + 返回: + sha1 十六进制摘要。 + """ + return hashlib.sha1(content.encode("utf-8")).hexdigest() + + +@dataclass +class LadderEntry: + """阶梯单元：题目与其估计答对率。 + + 字段: + question_id: 题目唯一标识。 + p_hat: 估计答对率。冷启动为 Beta(1,1) 平滑的单次观测后验均值 + （错=1/3、对=2/3），此后经 gamma-EMA 更新。 + """ + + question_id: str + p_hat: float + + +def build_cold_entries( + questions: list[GeneratedQuestion], + correctness: dict[str, bool], + probe_quota: float, + seed: int, +) -> list[LadderEntry]: + """冷启动排序：错题高优先 2:1 交错 + 全错题 probe_quota 探针插尾。 + + 参数: + questions: 该题型的全部候选题（已排除 test 池）。 + correctness: question_id -> 种子基线是否答对（900 题全量对错）。 + probe_quota: 从错题中随机抽出插到梯尾的探针比例（防"解锁新能力"盲区）。 + seed: 洗牌种子，保证确定性重建。 + + 返回: + 排序后的 LadderEntry 列表（p_hat 用 Beta(1,1) 平滑：错=1/3、对=2/3， + 与 warm 阶段 gamma-EMA / 信息量排序自然衔接）。 + + 关键实现细节: + 错题、对题各自固定种子洗牌 -> 抽探针 -> 剩余按 错错对 2:1 交错 + （一方耗尽后顺排另一方）-> 探针追加尾部。 + """ + rng = random.Random(seed) + wrong = [q for q in questions if not correctness.get(q.question_id, False)] + right = [q for q in questions if correctness.get(q.question_id, False)] + rng.shuffle(wrong) + rng.shuffle(right) + + n_probe = int(len(wrong) * probe_quota) + probes, wrong_main = wrong[:n_probe], wrong[n_probe:] + + interleaved: list[GeneratedQuestion] = [] + wi, ri = 0, 0 + while wi < len(wrong_main) or ri < len(right): + for _ in range(2): + if wi < len(wrong_main): + interleaved.append(wrong_main[wi]) + wi += 1 + if ri < len(right): + interleaved.append(right[ri]) + ri += 1 + interleaved.extend(probes) + + def _p0(q: GeneratedQuestion) -> float: + return 2 / 3 if correctness.get(q.question_id, False) else 1 / 3 + + return [LadderEntry(q.question_id, _p0(q)) for q in interleaved] + + +def order_ladder(entries: list[LadderEntry], p_low: float, p_high: float) -> list[LadderEntry]: + """warm 排序：剔 p_hat 不在 [p_low, p_high] 的零信息题，按信息量 p_hat(1-p_hat) 降序。 + + 参数: + entries: 待排序的阶梯单元。 + p_low / p_high: p_hat 保留区间。 + + 返回: + 过滤并排序后的新列表（稳定排序，同信息量保持原相对序）。 + """ + kept = [e for e in entries if p_low <= e.p_hat <= p_high] + return sorted(kept, key=lambda e: e.p_hat * (1 - e.p_hat), reverse=True) + + +@dataclass +class GatePools: + """全部题型的阶梯容器，含构建种子与数据指纹（确定性重建凭据）。 + + 字段: + entries: task_type -> 冷启动序 LadderEntry 列表（warm 排序在取用时做， + 保持存储序稳定、避免每次更新重写全表顺序）。 + seed: 冷启动洗牌种子。 + fingerprint: 构建输入指纹（基线 run_id + 题集 hash 等），resume 校验用。 + """ + + entries: dict[str, list[LadderEntry]] + seed: int + fingerprint: str + + def ladder_for( + self, + task_type: str, + exclude_qids: set[str], + p_low: float, + p_high: float, + cold: bool, + ) -> list[str]: + """取该题型的 gate 出题序（qid 列表），排除本 step 进化案例包题。 + + 参数: + task_type: 目标题型。 + exclude_qids: 本 step 案例包（failure/success cases）的题目 id， + 防止在"刚学的那道题"上自测。 + p_low / p_high: warm 阶段的 p_hat 保留区间。 + cold: True 表示尚无 epoch 级观测（epoch 1），用冷启动存储序； + False 走 order_ladder 信息量排序。 + + 返回: + 排除后的有序 question_id 列表。 + + 异常: + ValueError: 该题型无阶梯（冷启动构建缺失），或该题型阶梯为空。 + """ + if task_type not in self.entries: + raise ValueError(f"task_type={task_type} 无阶梯，冷启动构建缺失该题型") + pool = self.entries[task_type] + if not pool: + raise ValueError(f"task_type={task_type} 阶梯为空，无可出题目") + ordered = pool if cold else order_ladder(pool, p_low, p_high) + return [e.question_id for e in ordered if e.question_id not in exclude_qids] + + def update_probs(self, observations: dict[str, bool], gamma: float) -> None: + """gamma-EMA 更新 p_hat：p_hat <- gamma * p_hat + (1-gamma) * obs。只更新有新观测的题。 + + 参数: + observations: question_id -> 本 epoch 非 gate run 的最新对错。 + 调用方必须已按 run_id 过滤掉 gate 内 rollout（防泄露铁律）。 + gamma: EMA 衰减系数。 + """ + for entries in self.entries.values(): + for e in entries: + if e.question_id in observations: + obs = 1.0 if observations[e.question_id] else 0.0 + e.p_hat = gamma * e.p_hat + (1 - gamma) * obs + + def save(self, path: Path) -> None: + """原子写 gate_pools.json（.tmp 再 replace）。 + + 参数: + path: 目标 JSON 路径。 + """ + payload = { + "seed": self.seed, + "fingerprint": self.fingerprint, + "entries": { + t: [{"question_id": e.question_id, "p_hat": e.p_hat} for e in es] + for t, es in self.entries.items() + }, + } + tmp = path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + os.replace(tmp, path) + + @classmethod + def load(cls, path: Path) -> GatePools: + """从 gate_pools.json 恢复。 + + 参数: + path: gate_pools.json 路径。 + + 返回: + 复活的 GatePools。 + """ + d = json.loads(path.read_text(encoding="utf-8")) + return cls( + entries={ + t: [LadderEntry(x["question_id"], x["p_hat"]) for x in es] + for t, es in d["entries"].items() + }, + seed=d["seed"], + fingerprint=d["fingerprint"], + ) + + +def build_or_load_gate_pools( + workspace_dir: Path, + questions: list[GeneratedQuestion], + test_qids: set[str], + baseline_correctness: dict[str, bool], + task_types: list[str], + probe_quota: float, + seed: int, + baseline_run_id: str, +) -> GatePools: + """gate 阶梯获取入口：gate_pools.json 存在且指纹一致则加载，否则冷启动构建。 + + 参数: + workspace_dir: workspace 根目录（gate_pools.json 落其下）。 + questions: benchmark 全量题（900 题）。 + test_qids: held-out test 池题目 id（阶梯题源必须排除）。 + baseline_correctness: 种子基线 900 题全量对错（从基线 run 的 db 读）。 + task_types: 参与进化的题型列表。 + probe_quota: 冷启动探针比例。 + seed: 冷启动洗牌种子。 + baseline_run_id: 指纹成分。 + + 返回: + GatePools。 + + 关键实现细节: + 指纹 = sha1(baseline_run_id|全 qid|seed|probe_quota|task_types|test_qids)。 + 指纹不一致（题集/基线/参数变了）直接报错——FRESH 语义下不该发生， + 防御性拒绝而非静默重建。 + """ + joined = ",".join(sorted(q.question_id for q in questions)) + fp_src = ( + f"{baseline_run_id}|{joined}|{seed}|{probe_quota}" + f"|{','.join(sorted(task_types))}|{','.join(sorted(test_qids))}" + ) + fingerprint = hashlib.sha1(fp_src.encode()).hexdigest() + path = workspace_dir / "gate_pools.json" + if path.exists(): + pools = GatePools.load(path) + if pools.fingerprint != fingerprint: + raise RuntimeError( + f"gate_pools.json 指纹不一致（题集或基线变更），拒绝静默重建: {path}" + ) + return pools + + entries: dict[str, list[LadderEntry]] = {} + for t in task_types: + pool = [q for q in questions if q.task_type == t and q.question_id not in test_qids] + if not pool: + raise ValueError(f"task_type={t} 无非 test 题，无法建阶梯") + entries[t] = build_cold_entries(pool, baseline_correctness, probe_quota, seed) + logger.info("gate 阶梯[{}]: {} 题（冷启动）", t, len(entries[t])) + pools = GatePools(entries=entries, seed=seed, fingerprint=fingerprint) + pools.save(path) + return pools + + +class BaselineCache: + """基线侧逐题对错缓存（内容寻址，JSON 持久化）。 + + 键 = (task_type, skill_hash, prompts_version, qid)：任何影响该题型 + 有效 skill 的变化（含共享 default-strategy.md 被他类 accept 改写） + 都使 skill_hash 变化、缓存自然 miss；prompts 版本变化同理。 + """ + + def __init__(self, path: Path) -> None: + """加载或初始化缓存文件。 + + 参数: + path: 缓存 JSON 路径（workspace/baseline_cache.json）。 + """ + self._path = path + self._store: dict[str, bool] = {} + if path.exists(): + self._store = json.loads(path.read_text(encoding="utf-8")) + + @staticmethod + def _key(task_type: str, s_hash: str, prompts_version: str, qid: str) -> str: + """拼缓存键（四维内容寻址）。""" + return f"{task_type}|{s_hash}|{prompts_version}|{qid}" + + def get(self, task_type: str, s_hash: str, prompts_version: str, qid: str) -> bool | None: + """读缓存；未命中返回 None。 + + 参数: + task_type: 题型。 + s_hash: 基线侧生效 skill 文件的内容哈希。 + prompts_version: 当前 prompts 版本。 + qid: 题目 id。 + + 返回: + 缓存的对错；未命中 None。 + """ + return self._store.get(self._key(task_type, s_hash, prompts_version, qid)) + + def put( + self, task_type: str, s_hash: str, prompts_version: str, qid: str, correct: bool + ) -> None: + """写缓存并落盘（原子写，gate 频度低、全量重写成本可忽略）。 + + 参数: + task_type / s_hash / prompts_version / qid: 缓存键四维。 + correct: 基线侧该题对错。 + + 关键实现细节: + 先盘后存：新条目先原子落盘（tmp 写 + os.replace）成功后才更新 + 内存，磁盘写失败时内存与磁盘一致（均无新条目），无分裂窗口。 + """ + updated = { + **self._store, + self._key(task_type, s_hash, prompts_version, qid): correct, + } + tmp = self._path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(updated, ensure_ascii=False), encoding="utf-8") + os.replace(tmp, self._path) + self._store = updated diff --git a/tests/unit/test_harness_gate_ladder.py b/tests/unit/test_harness_gate_ladder.py new file mode 100644 index 0000000..18edfb5 --- /dev/null +++ b/tests/unit/test_harness_gate_ladder.py @@ -0,0 +1,374 @@ +"""app/harness/gate_ladder.py 单元测试。 + +覆盖冷启动交错、Beta(1,1) 平滑、warm 信息量排序、 +GatePools 原子读写与指纹校验、BaselineCache 四维内容寻址与先盘后存、 +gamma-EMA 更新、防泄露过滤等核心语义。 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from app.harness.gate_ladder import ( + BaselineCache, + GatePools, + LadderEntry, + build_cold_entries, + build_or_load_gate_pools, + order_ladder, + skill_hash, +) +from core.types import GeneratedQuestion + +if TYPE_CHECKING: + from pathlib import Path + + +# ── 工具函数 ────────────────────────────────────────────────────────── + + +def _make_q(qid: str, task_type: str = "AR") -> GeneratedQuestion: + """构造最小 GeneratedQuestion 实例。""" + return GeneratedQuestion( + question_id=qid, + video_id="v1", + task_type=task_type, + question="dummy", + options=("A", "B", "C", "D"), + answer="A", + source_nodes=("n1",), + difficulty="easy", + ) + + +# ── 冷启动 ──────────────────────────────────────────────────────────── + + +class TestColdStart: + """冷启动排序：2:1 交错 + 探针插尾 + Beta(1,1) 平滑。""" + + def test_cold_start_interleaving(self) -> None: + """错题:对题 = 2:1 交错顺序。 + + 6 错 3 对（probe_quota=0 无探针）→ 交错序应为 W W R W W R W W R。 + """ + wrong_ids = [f"w{i}" for i in range(6)] + right_ids = [f"r{i}" for i in range(3)] + questions = [_make_q(qid) for qid in wrong_ids + right_ids] + correctness = dict.fromkeys(wrong_ids, False) + correctness.update(dict.fromkeys(right_ids, True)) + + entries = build_cold_entries(questions, correctness, probe_quota=0.0, seed=42) + + assert len(entries) == 9 + # 验证 2:1 交错模式（seed 固定后 shuffle 结果确定） + pattern = ["W" if not correctness[e.question_id] else "R" for e in entries] + # 前 9 个交错应为 W W R W W R W W R + assert pattern == ["W", "W", "R", "W", "W", "R", "W", "W", "R"] + + def test_cold_start_p_hat_beta(self) -> None: + """p_hat 遵循 Beta(1,1) 平滑：错=1/3，对=2/3。""" + questions = [_make_q("q1"), _make_q("q2")] + correctness = {"q1": False, "q2": True} + + entries = build_cold_entries(questions, correctness, probe_quota=0.0, seed=0) + + p_map = {e.question_id: e.p_hat for e in entries} + assert p_map["q1"] == pytest.approx(1 / 3) + assert p_map["q2"] == pytest.approx(2 / 3) + + def test_cold_start_probe_at_tail(self) -> None: + """probe_quota > 0 时探针题追加在尾部。""" + wrong_ids = [f"w{i}" for i in range(10)] + right_ids = [f"r{i}" for i in range(2)] + questions = [_make_q(qid) for qid in wrong_ids + right_ids] + correctness = dict.fromkeys(wrong_ids, False) + correctness.update(dict.fromkeys(right_ids, True)) + + entries = build_cold_entries(questions, correctness, probe_quota=0.3, seed=7) + + # 10 错 * 0.3 = 3 个探针在尾部 + n_probe = int(10 * 0.3) + assert n_probe == 3 + # 尾部 3 个都应为错题 + tail = entries[-n_probe:] + for e in tail: + assert not correctness[e.question_id] + + +# ── warm 排序 ────────────────────────────────────────────────────────── + + +class TestWarmOrdering: + """warm 阶段：信息量 p_hat(1-p_hat) 降序 + p_hat 区间过滤。""" + + def test_warm_ordering_information(self) -> None: + """p_hat=0.5 信息量最高，排在最前。""" + entries = [ + LadderEntry("a", 0.1), + LadderEntry("b", 0.5), + LadderEntry("c", 0.9), + LadderEntry("d", 0.3), + ] + ordered = order_ladder(entries, p_low=0.0, p_high=1.0) + assert ordered[0].question_id == "b" # 0.5*(1-0.5)=0.25 最高 + # d: 0.3*0.7=0.21, a: 0.1*0.9=0.09, c: 0.9*0.1=0.09 + assert ordered[1].question_id == "d" + + def test_warm_filter_bounds(self) -> None: + """p_hat 不在 [p_low, p_high] 区间的题被剔除。""" + entries = [ + LadderEntry("low", 0.05), + LadderEntry("mid", 0.5), + LadderEntry("high", 0.95), + ] + ordered = order_ladder(entries, p_low=0.1, p_high=0.9) + ids = [e.question_id for e in ordered] + assert "mid" in ids + assert "low" not in ids + assert "high" not in ids + + +# ── GatePools 持久化 ────────────────────────────────────────────────── + + +class TestGatePoolsPersistence: + """GatePools.save/load 原子性与指纹校验。""" + + def test_gate_pools_save_load_atomic(self, tmp_path: Path) -> None: + """save -> load 往返保真，且使用原子写（中间 .tmp 文件不残留）。""" + entries = { + "AR": [LadderEntry("q1", 0.33), LadderEntry("q2", 0.67)], + "CR": [LadderEntry("q3", 0.5)], + } + pools = GatePools(entries=entries, seed=42, fingerprint="abc123") + path = tmp_path / "gate_pools.json" + pools.save(path) + + # .tmp 文件不应残留 + assert not (tmp_path / "gate_pools.json.tmp").exists() + assert path.exists() + + loaded = GatePools.load(path) + assert loaded.seed == 42 + assert loaded.fingerprint == "abc123" + assert len(loaded.entries["AR"]) == 2 + assert loaded.entries["AR"][0].question_id == "q1" + assert loaded.entries["AR"][0].p_hat == pytest.approx(0.33) + assert loaded.entries["CR"][0].question_id == "q3" + + def test_gate_pools_fingerprint_mismatch(self, tmp_path: Path) -> None: + """指纹不一致 -> RuntimeError（不静默重建）。""" + questions = [_make_q("q1", "AR"), _make_q("q2", "AR")] + correctness = {"q1": True, "q2": False} + + # 第一次构建 + build_or_load_gate_pools( + workspace_dir=tmp_path, + questions=questions, + test_qids=set(), + baseline_correctness=correctness, + task_types=["AR"], + probe_quota=0.0, + seed=1, + baseline_run_id="run_001", + ) + + # 改 baseline_run_id 导致指纹变化 -> 应报错 + with pytest.raises(RuntimeError, match="指纹不一致"): + build_or_load_gate_pools( + workspace_dir=tmp_path, + questions=questions, + test_qids=set(), + baseline_correctness=correctness, + task_types=["AR"], + probe_quota=0.0, + seed=1, + baseline_run_id="run_002", + ) + + +# ── ladder_for ──────────────────────────────────────────────────────── + + +class TestLadderFor: + """ladder_for 取题序与排除逻辑。""" + + def test_ladder_for_excludes_qids(self) -> None: + """exclude_qids 中的题被排除。""" + entries = { + "AR": [ + LadderEntry("q1", 0.5), + LadderEntry("q2", 0.4), + LadderEntry("q3", 0.6), + ], + } + pools = GatePools(entries=entries, seed=0, fingerprint="x") + result = pools.ladder_for("AR", exclude_qids={"q2"}, p_low=0.0, p_high=1.0, cold=True) + assert "q2" not in result + assert "q1" in result + assert "q3" in result + + def test_ladder_for_missing_task_type(self) -> None: + """不存在的 task_type -> ValueError。""" + pools = GatePools(entries={}, seed=0, fingerprint="x") + with pytest.raises(ValueError, match="无阶梯"): + pools.ladder_for("MISSING", set(), 0.0, 1.0, cold=True) + + def test_ladder_for_warm_uses_order_ladder(self) -> None: + """cold=False 时走 warm 信息量排序。""" + entries = { + "AR": [ + LadderEntry("low", 0.1), + LadderEntry("mid", 0.5), + LadderEntry("high", 0.9), + ], + } + pools = GatePools(entries=entries, seed=0, fingerprint="x") + result = pools.ladder_for("AR", set(), p_low=0.0, p_high=1.0, cold=False) + # 信息量排序：mid(0.25) > low(0.09) = high(0.09) + assert result[0] == "mid" + + +# ── gamma-EMA 更新 ───────────────────────────────────────────────────── + + +class TestGammaEMA: + """gamma-EMA 更新 p_hat。""" + + def test_gamma_ema_update(self) -> None: + """p_hat <- gamma * p_hat + (1-gamma) * obs。""" + entries = {"AR": [LadderEntry("q1", 0.5)]} + pools = GatePools(entries=entries, seed=0, fingerprint="x") + + # 观测为正确(1.0), gamma=0.8 + pools.update_probs({"q1": True}, gamma=0.8) + expected = 0.8 * 0.5 + 0.2 * 1.0 # 0.6 + assert pools.entries["AR"][0].p_hat == pytest.approx(expected) + + # 再次观测为错误(0.0), gamma=0.8 + pools.update_probs({"q1": False}, gamma=0.8) + expected2 = 0.8 * expected + 0.2 * 0.0 # 0.48 + assert pools.entries["AR"][0].p_hat == pytest.approx(expected2) + + def test_update_probs_no_observation_unchanged(self) -> None: + """无观测的题 p_hat 不变。""" + entries = {"AR": [LadderEntry("q1", 0.5), LadderEntry("q2", 0.3)]} + pools = GatePools(entries=entries, seed=0, fingerprint="x") + pools.update_probs({"q1": True}, gamma=0.9) + assert pools.entries["AR"][1].p_hat == pytest.approx(0.3) + + +# ── 防泄露 ───────────────────────────────────────────────────────────── + + +class TestLeakPrevention: + """防泄露铁律：gate 内 rollout 永不回流 p_hat（由调用方过滤）。""" + + def test_update_probs_excludes_gate_runs(self) -> None: + """调用方须过滤 run_id 含 '_gate_' 的观测。 + + update_probs 本身只接收已过滤的 observations，这里验证 + 如果调用方正确过滤，gate run 数据不会影响 p_hat。 + """ + entries = {"AR": [LadderEntry("q1", 0.5)]} + pools = GatePools(entries=entries, seed=0, fingerprint="x") + + # 模拟：所有 run 的原始观测（含 gate run） + raw_observations = { + "run_normal": {"q1": True}, # 普通 run + "run_gate_01": {"q1": False}, # gate run（run_id 含 _gate_） + } + + # 调用方按 run_id 过滤：排除含 "_gate_" 的 run + filtered = {} + for run_id, obs in raw_observations.items(): + if "_gate_" not in run_id: + filtered.update(obs) + + # 只有普通 run 的观测进入 update_probs + assert filtered == {"q1": True} + pools.update_probs(filtered, gamma=0.8) + expected = 0.8 * 0.5 + 0.2 * 1.0 + assert pools.entries["AR"][0].p_hat == pytest.approx(expected) + + +# ── BaselineCache ────────────────────────────────────────────────────── + + +class TestBaselineCache: + """BaselineCache 四维内容寻址与先盘后存。""" + + def test_baseline_cache_content_addressed(self, tmp_path: Path) -> None: + """四维键唯一寻址：任一维度变化 -> miss。""" + path = tmp_path / "baseline_cache.json" + cache = BaselineCache(path) + + cache.put("AR", "hash1", "v1", "q1", True) + assert cache.get("AR", "hash1", "v1", "q1") is True + + # 改 skill_hash -> miss + assert cache.get("AR", "hash2", "v1", "q1") is None + # 改 prompts_version -> miss + assert cache.get("AR", "hash1", "v2", "q1") is None + # 改 task_type -> miss + assert cache.get("CR", "hash1", "v1", "q1") is None + # 改 qid -> miss + assert cache.get("AR", "hash1", "v1", "q2") is None + + def test_baseline_cache_disk_first(self, tmp_path: Path) -> None: + """先盘后存：磁盘写成功后内存才更新，新实例可从磁盘读到。""" + path = tmp_path / "baseline_cache.json" + cache = BaselineCache(path) + + cache.put("AR", "h1", "v1", "q1", True) + + # 内存可读 + assert cache.get("AR", "h1", "v1", "q1") is True + + # 新实例从磁盘加载也能读到（证明先落盘） + cache2 = BaselineCache(path) + assert cache2.get("AR", "h1", "v1", "q1") is True + + # .tmp 文件不应残留 + assert not (tmp_path / "baseline_cache.json.tmp").exists() + + def test_baseline_cache_empty_init(self, tmp_path: Path) -> None: + """不存在的文件 -> 空缓存初始化。""" + path = tmp_path / "nonexistent.json" + cache = BaselineCache(path) + assert cache.get("AR", "h1", "v1", "q1") is None + + def test_baseline_cache_overwrite(self, tmp_path: Path) -> None: + """同键重复写入覆盖旧值。""" + path = tmp_path / "baseline_cache.json" + cache = BaselineCache(path) + + cache.put("AR", "h1", "v1", "q1", True) + assert cache.get("AR", "h1", "v1", "q1") is True + + cache.put("AR", "h1", "v1", "q1", False) + assert cache.get("AR", "h1", "v1", "q1") is False + + +# ── skill_hash ───────────────────────────────────────────────────────── + + +class TestSkillHash: + """skill_hash SHA1 摘要。""" + + def test_deterministic(self) -> None: + """相同输入产生相同摘要。""" + assert skill_hash("hello") == skill_hash("hello") + + def test_different_content(self) -> None: + """不同输入产生不同摘要。""" + assert skill_hash("hello") != skill_hash("world") + + def test_is_sha1_hex(self) -> None: + """输出为 40 字符十六进制。""" + h = skill_hash("test") + assert len(h) == 40 + assert all(c in "0123456789abcdef" for c in h) From a550d39e1c5ba956c6db11a8c4625efeece38d60 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 12:55:12 -0400 Subject: [PATCH 12/19] =?UTF-8?q?feat(harness):=20observation.py=20?= =?UTF-8?q?=E2=80=94=20=E4=BA=94=E5=BC=A0=E8=A7=82=E6=B5=8B=E8=A1=A8=20+?= =?UTF-8?q?=20step/epoch=20=E6=8A=A5=E5=91=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/harness/observation.py | 459 +++++++++++++++++++++++++ tests/unit/test_harness_observation.py | 354 +++++++++++++++++++ 2 files changed, 813 insertions(+) create mode 100644 app/harness/observation.py create mode 100644 tests/unit/test_harness_observation.py diff --git a/app/harness/observation.py b/app/harness/observation.py new file mode 100644 index 0000000..f9ddd92 --- /dev/null +++ b/app/harness/observation.py @@ -0,0 +1,459 @@ +"""五张观测表的落库写入与回读 + step/epoch 报告文件输出。 + +合并 TRM4 的 metric_log.py（五表）和 loop_report.py（报告）。 + +五张表均经 structured-logging 定义，DDL 与之逐列一致： + dual_metric_eval / shadow_gate / holdout_eval / quadrant_pair / gate_evidence。 + +公共契约（守 P5）：soft/mixed 为 None（invalid，无 span / 诊断失败）时存 NULL，**绝不存 0**—— +SQLite 对 dict 中 None 值写入即 NULL，分析时按 NULL 跳过。每个写函数内幂等建表 +（``HarnessLog.create_table`` 用 CREATE TABLE IF NOT EXISTS），run_id/timestamp 列由 +HarnessLog 自动补。 + +报告函数输出 JSON 到 workspace 的 analyses/ 目录，供人工审查诊断 prompt 与进化 prompt。 +""" + +from __future__ import annotations + +import json +import sqlite3 +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from pathlib import Path + + +def _read_table(db_path: str, table: str, run_id: str) -> list[dict[str, Any]]: + """纯读某表指定 run 的全部行——不经 HarnessLog 生命周期，避免回读污染 _runs 运行状态。 + + HarnessLog.__enter__/__exit__ 会对 run_id 做 INSERT OR IGNORE 并在退出时标 completed； + 回读指标绝不应改运行状态，故 read_* 一律走本只读连接（仅 SELECT）。 + + 参数: + db_path: SQLite 路径。 + table: 表名（内部固定常量，非外部输入，无注入风险）。 + run_id: 过滤的 run ID。 + + 返回: + 行 dict 列表；表尚未建（没写过）视为无数据返 []。 + """ + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + try: + exists = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table,) + ).fetchone() + if exists is None: + return [] + rows = conn.execute(f"SELECT * FROM {table} WHERE run_id=?", (run_id,)).fetchall() + return [dict(r) for r in rows] + finally: + conn.close() + + +# --------------------------------------------------------------------------- +# 列定义严格对齐 research-wiki/schemas/*.md（run_id/timestamp 由 create_table 自动补） +# --------------------------------------------------------------------------- + +_DUAL_COLS: dict[str, str] = { + "epoch": "INTEGER", + "version_kind": "TEXT", + "skills_version": "TEXT", + "prompts_version": "TEXT", + "pool": "TEXT", + "hard_acc": "REAL", + "soft_score": "REAL", + "mixed_score": "REAL", +} + +_SHADOW_COLS: dict[str, str] = { + "epoch": "INTEGER", + "candidate_version": "TEXT", + "hard_acc": "REAL", + "soft_score": "REAL", + "mixed_score": "REAL", + "is_mixed_best": "INTEGER", +} + +_HOLDOUT_COLS: dict[str, str] = { + "epoch": "INTEGER", + "version_kind": "TEXT", + "hard_acc": "REAL", + "soft_score": "REAL", + "mixed_score": "REAL", + "per_task_type_json": "TEXT", +} + +_QUADRANT_COLS: dict[str, str] = { + "epoch": "INTEGER", + "step": "INTEGER", + "question_id": "TEXT", + "task_type": "TEXT", + "prev_correct": "INTEGER", + "curr_correct": "INTEGER", + "category": "TEXT", +} + +_GATE_EVIDENCE_COLS: dict[str, str] = { + "epoch": "INTEGER", + "step": "INTEGER", + "task_type": "TEXT", + "question_id": "TEXT", + "block_idx": "INTEGER", + "baseline_correct": "INTEGER", + "candidate_correct": "INTEGER", + "e_value": "REAL", + "stop_reason": "TEXT", +} + + +# --------------------------------------------------------------------------- +# dual_metric_eval +# --------------------------------------------------------------------------- + + +def write_dual_metric( + db_path: str, + *, + run_id: str, + epoch: int, + version_kind: str, + skills_version: str, + prompts_version: str, + pool: str, + hard_acc: float, + soft_score: float | None, + mixed_score: float | None, +) -> None: + """落 dual_metric_eval 一行：epoch 末关键版本的 hard+soft+mixed 双轨度量。 + + 参数: + db_path: SQLite 路径。 + run_id: 训练 run ID。 + epoch: 轮次（1-based）。 + version_kind: baseline / best_hard / best_mixed / final。 + skills_version / prompts_version: 评估的资源版本。 + pool: val / test。 + hard_acc: hard 准确率。 + soft_score: soft 连续分；invalid 传 None -> 存 NULL。 + mixed_score: 0.5*hard+0.5*soft；soft 缺失传 None -> 存 NULL。 + """ + from app.harness.log import HarnessLog + + with HarnessLog(db_path, run_id) as log: + log.create_table("dual_metric_eval", _DUAL_COLS) + log.insert( + "dual_metric_eval", + { + "epoch": epoch, + "version_kind": version_kind, + "skills_version": skills_version, + "prompts_version": prompts_version, + "pool": pool, + "hard_acc": hard_acc, + "soft_score": soft_score, + "mixed_score": mixed_score, + }, + ) + + +def read_dual_metric(db_path: str, *, run_id: str) -> list[dict[str, Any]]: + """回读指定 run 的 dual_metric_eval 全部行（纯读，不污染运行状态）。""" + return _read_table(db_path, "dual_metric_eval", run_id) + + +# --------------------------------------------------------------------------- +# shadow_gate +# --------------------------------------------------------------------------- + + +def write_shadow_gate( + db_path: str, + *, + run_id: str, + epoch: int, + candidate_version: str, + hard_acc: float, + soft_score: float | None, + mixed_score: float | None, + is_mixed_best: bool, +) -> None: + """落 shadow_gate 一行：mixed 影子 best 候选的 hard/soft/mixed 及是否 argmax 选中。 + + 参数: + db_path: SQLite 路径。 + run_id: 训练 run ID。 + epoch: 轮次（1-based）。 + candidate_version: 候选版本标识（如 skills/vX+prompts/vY）。 + hard_acc: hard 准确率。 + soft_score: soft 连续分；invalid 传 None -> 存 NULL（该版本不进 argmax）。 + mixed_score: 0.5*hard+0.5*soft；soft 缺失传 None -> 存 NULL。 + is_mixed_best: 是否本 epoch mixed argmax 选中（存 1/0）。 + """ + from app.harness.log import HarnessLog + + with HarnessLog(db_path, run_id) as log: + log.create_table("shadow_gate", _SHADOW_COLS) + log.insert( + "shadow_gate", + { + "epoch": epoch, + "candidate_version": candidate_version, + "hard_acc": hard_acc, + "soft_score": soft_score, + "mixed_score": mixed_score, + "is_mixed_best": int(is_mixed_best), + }, + ) + + +def read_shadow_gate(db_path: str, *, run_id: str) -> list[dict[str, Any]]: + """回读指定 run 的 shadow_gate 全部行（纯读，不污染运行状态）。""" + return _read_table(db_path, "shadow_gate", run_id) + + +# --------------------------------------------------------------------------- +# holdout_eval +# --------------------------------------------------------------------------- + + +def write_holdout_eval( + db_path: str, + *, + run_id: str, + epoch: int, + version_kind: str, + hard_acc: float, + soft_score: float | None, + mixed_score: float | None, + per_task_type_json: str, +) -> None: + """落 holdout_eval 一行：四向 held-out 在 test 池的 hard+soft+mixed 及按题型细分。 + + 参数: + db_path: SQLite 路径。 + run_id: 训练 run ID。 + epoch: 轮次（1-based）。 + version_kind: baseline / best_hard / best_mixed / final。 + hard_acc: hard 准确率。 + soft_score: soft 连续分；invalid 传 None -> 存 NULL。 + mixed_score: 0.5*hard+0.5*soft；soft 缺失传 None -> 存 NULL。 + per_task_type_json: 按 task_type 的 {accuracy,total,correct} JSON 串。 + """ + from app.harness.log import HarnessLog + + with HarnessLog(db_path, run_id) as log: + log.create_table("holdout_eval", _HOLDOUT_COLS) + log.insert( + "holdout_eval", + { + "epoch": epoch, + "version_kind": version_kind, + "hard_acc": hard_acc, + "soft_score": soft_score, + "mixed_score": mixed_score, + "per_task_type_json": per_task_type_json, + }, + ) + + +def read_holdout_eval(db_path: str, *, run_id: str) -> list[dict[str, Any]]: + """回读指定 run 的 holdout_eval 全部行（纯读，不污染运行状态）。""" + return _read_table(db_path, "holdout_eval", run_id) + + +# --------------------------------------------------------------------------- +# quadrant_pair +# --------------------------------------------------------------------------- + + +def write_quadrant_pairs( + db_path: str, + *, + run_id: str, + epoch: int, + step: int, + pairs: list[dict[str, Any]], +) -> None: + """落 quadrant_pair 多行：fast gate 后逐题四象限（prev/curr 翻转 + category）落库。 + + 参数: + db_path: SQLite 路径。 + run_id: 训练 run ID。 + epoch: 轮次（1-based）。 + step: epoch 内 step 序号（0-based）。 + pairs: 每条含 question_id/task_type/prev_correct/curr_correct/category； + prev_correct/curr_correct 为 bool，写库前转 0/1。 + + 关键实现: + 用 insert_many 批量落库；pairs 为空时只建表不插入（fast gate 无翻转的极端情况）。 + """ + records = [ + { + "epoch": epoch, + "step": step, + "question_id": pair["question_id"], + "task_type": pair["task_type"], + "prev_correct": int(pair["prev_correct"]), + "curr_correct": int(pair["curr_correct"]), + "category": pair["category"], + } + for pair in pairs + ] + from app.harness.log import HarnessLog + + with HarnessLog(db_path, run_id) as log: + log.create_table("quadrant_pair", _QUADRANT_COLS) + if records: + log.insert_many("quadrant_pair", records) + + +def read_quadrant_pairs(db_path: str, *, run_id: str) -> list[dict[str, Any]]: + """回读指定 run 的 quadrant_pair 全部行（纯读，不污染运行状态）。""" + return _read_table(db_path, "quadrant_pair", run_id) + + +# --------------------------------------------------------------------------- +# gate_evidence +# --------------------------------------------------------------------------- + + +def write_gate_evidence( + db_path: str, + *, + run_id: str, + epoch: int, + step: int, + rows: list[dict[str, Any]], +) -> None: + """落 gate_evidence 逐题行：CE-Gate 每次决策的可回放审计记录。 + + 参数: + db_path: SQLite 路径。 + run_id: 训练 run ID。 + epoch: 该 gate 所属的轮次（1-based）。 + step: epoch 内 step 序号（0-based）。 + rows: 每题一行，含 question_id/task_type/block_idx/baseline_correct/ + candidate_correct/e_value（该题所在块判定后的累计 e 值）/ + stop_reason（仅最后一题携带最终 stop_reason，其余空串）。 + + 关键实现: + 逐行 insert（非 insert_many），保证每行独立事务。 + """ + from app.harness.log import HarnessLog + + with HarnessLog(db_path, run_id) as log: + log.create_table("gate_evidence", _GATE_EVIDENCE_COLS) + for row in rows: + log.insert("gate_evidence", {"epoch": epoch, "step": step, **row}) + + +def read_gate_evidence(db_path: str, *, run_id: str) -> list[dict[str, Any]]: + """回读指定 run 的 gate_evidence 全部行（纯读，不污染运行状态）。""" + return _read_table(db_path, "gate_evidence", run_id) + + +# --------------------------------------------------------------------------- +# 报告函数（从 TRM4 loop_report.py 迁移） +# --------------------------------------------------------------------------- + + +def write_step_report( + workspace_dir: Path, + epoch: int, + step: int, + global_step: int, + task_type: str, + gate_action: str, + candidate_acc: float, + class_baseline_acc: float, + edit_budget: int, + rank_clip_triggered: bool, + gate_w: int | None, + gate_l: int | None, + gate_e_value: float | None, + gate_n_used: int | None, + gate_stop_reason: str | None, +) -> Path: + """写单个 (step, task_type) 快路径 gate 的最小观测记录 JSON。 + + 文件名按 (epoch, step, task_type) 命名，slug 由 task_type 规范化（小写、空格转 '-'）得到。 + + 参数: + workspace_dir: 实验工作区目录。 + epoch: 当前轮次（1-based）。 + step: epoch 内 step 序号（0-based）。 + global_step: 全局步计数（驱动 edit_budget 退火）。 + task_type: 本条 gate 的任务类型。 + gate_action: 闸门动作（accept_confirmed / accept_provisional / reject / + skipped / cooldown）。 + candidate_acc: 候选在 gate 已观测题上的准确率（观测口径）。 + class_baseline_acc: 基线在 gate 已观测题上的准确率（观测口径）。 + edit_budget: 该 step 按 global_step 退火得到的 per-target 编辑预算上限。 + rank_clip_triggered: 该 skill 进化是否触发了 rank 裁剪。 + gate_w: e-process 累计 W（基线错->候选对翻转数）；skipped/cooldown 路径传 None。 + gate_l: e-process 累计 L（基线对->候选错翻转数）；skipped/cooldown 路径传 None。 + gate_e_value: 停时的 e 值；skipped/cooldown 路径传 None。 + gate_n_used: gate 实际消费的阶梯题数；skipped/cooldown 路径传 None。 + gate_stop_reason: e-process 停止原因；skipped/cooldown 路径传 None。 + + 返回: + 写入的 step_report 文件路径。 + """ + report = { + "epoch": epoch, + "step": step, + "global_step": global_step, + "task_type": task_type, + "gate_action": gate_action, + "candidate_acc": candidate_acc, + "class_baseline_acc": class_baseline_acc, + "edit_budget": edit_budget, + "rank_clip_triggered": rank_clip_triggered, + "gate_w": gate_w, + "gate_l": gate_l, + "gate_e_value": gate_e_value, + "gate_n_used": gate_n_used, + "gate_stop_reason": gate_stop_reason, + } + slug = task_type.lower().replace(" ", "-") + out_dir = workspace_dir / "analyses" + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / f"step_report_e{epoch}_s{step}_{slug}.json" + path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + return path + + +def write_epoch_report( + workspace_dir: Path, + epoch: int, + system_tool_action: str, + momentum_updated_task_types: list[str], + best_val_acc: float, +) -> Path: + """写 epoch 末慢更新汇总 JSON。 + + 慢更新无单一 ValidationOutcome，故本函数只落慢更新可观测的最小集： + system/tool gate 动作、本 epoch 写过 momentum 的题型、慢更新后的全局 best。 + + 参数: + workspace_dir: 实验工作区目录。 + epoch: 当前轮次（1-based）。 + system_tool_action: 慢更新 system/tool 动作（updated / reverted / none）。 + momentum_updated_task_types: 本 epoch 写过 momentum 的题型列表。 + best_val_acc: 慢更新后（含 best argmax）的全局 best 验证准确率。 + + 返回: + 写入的 epoch_report 文件路径。 + """ + report = { + "epoch": epoch, + "system_tool_action": system_tool_action, + "momentum_updated_task_types": momentum_updated_task_types, + "best_val_acc": best_val_acc, + } + out_dir = workspace_dir / "analyses" + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / f"epoch_report_{epoch}.json" + path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + return path diff --git a/tests/unit/test_harness_observation.py b/tests/unit/test_harness_observation.py new file mode 100644 index 0000000..d37cc40 --- /dev/null +++ b/tests/unit/test_harness_observation.py @@ -0,0 +1,354 @@ +"""五张观测表 + step/epoch 报告的单元测试。""" + +from __future__ import annotations + +import json +import sqlite3 +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + +from app.harness.observation import ( + read_dual_metric, + read_gate_evidence, + read_holdout_eval, + read_quadrant_pairs, + read_shadow_gate, + write_dual_metric, + write_epoch_report, + write_gate_evidence, + write_holdout_eval, + write_quadrant_pairs, + write_shadow_gate, + write_step_report, +) + + +@pytest.fixture() +def db_path(tmp_path: Path) -> str: + """返回临时 SQLite 路径。""" + return str(tmp_path / "test_obs.db") + + +@pytest.fixture() +def run_id() -> str: + return "run-obs-001" + + +# --------------------------------------------------------------------------- +# dual_metric_eval +# --------------------------------------------------------------------------- + + +def test_write_read_dual_metric(db_path: str, run_id: str) -> None: + """写入 dual_metric_eval 后回读应与输入一致。""" + write_dual_metric( + db_path, + run_id=run_id, + epoch=1, + version_kind="baseline", + skills_version="v0", + prompts_version="v0", + pool="val", + hard_acc=0.75, + soft_score=0.80, + mixed_score=0.775, + ) + rows = read_dual_metric(db_path, run_id=run_id) + assert len(rows) == 1 + row = rows[0] + assert row["epoch"] == 1 + assert row["version_kind"] == "baseline" + assert row["skills_version"] == "v0" + assert row["prompts_version"] == "v0" + assert row["pool"] == "val" + assert row["hard_acc"] == pytest.approx(0.75) + assert row["soft_score"] == pytest.approx(0.80) + assert row["mixed_score"] == pytest.approx(0.775) + assert row["run_id"] == run_id + + +# --------------------------------------------------------------------------- +# shadow_gate +# --------------------------------------------------------------------------- + + +def test_write_read_shadow_gate(db_path: str, run_id: str) -> None: + """写入 shadow_gate 后回读应与输入一致，is_mixed_best 布尔转 int。""" + write_shadow_gate( + db_path, + run_id=run_id, + epoch=2, + candidate_version="skills/v1+prompts/v1", + hard_acc=0.82, + soft_score=0.78, + mixed_score=0.80, + is_mixed_best=True, + ) + rows = read_shadow_gate(db_path, run_id=run_id) + assert len(rows) == 1 + row = rows[0] + assert row["epoch"] == 2 + assert row["candidate_version"] == "skills/v1+prompts/v1" + assert row["hard_acc"] == pytest.approx(0.82) + assert row["is_mixed_best"] == 1 + + +# --------------------------------------------------------------------------- +# holdout_eval +# --------------------------------------------------------------------------- + + +def test_write_read_holdout_eval(db_path: str, run_id: str) -> None: + """写入 holdout_eval 后回读应与输入一致。""" + per_task = json.dumps({"temporal": {"accuracy": 0.9, "total": 10, "correct": 9}}) + write_holdout_eval( + db_path, + run_id=run_id, + epoch=1, + version_kind="best_hard", + hard_acc=0.85, + soft_score=0.70, + mixed_score=0.775, + per_task_type_json=per_task, + ) + rows = read_holdout_eval(db_path, run_id=run_id) + assert len(rows) == 1 + row = rows[0] + assert row["version_kind"] == "best_hard" + assert row["hard_acc"] == pytest.approx(0.85) + parsed = json.loads(row["per_task_type_json"]) + assert parsed["temporal"]["correct"] == 9 + + +# --------------------------------------------------------------------------- +# gate_evidence +# --------------------------------------------------------------------------- + + +def test_write_read_gate_evidence(db_path: str, run_id: str) -> None: + """写入 gate_evidence 后回读应与输入一致，逐行插入。""" + evidence_rows = [ + { + "task_type": "temporal", + "question_id": "q1", + "block_idx": 0, + "baseline_correct": 1, + "candidate_correct": 1, + "e_value": 1.0, + "stop_reason": "", + }, + { + "task_type": "temporal", + "question_id": "q2", + "block_idx": 0, + "baseline_correct": 0, + "candidate_correct": 1, + "e_value": 2.0, + "stop_reason": "confirmed", + }, + ] + write_gate_evidence(db_path, run_id=run_id, epoch=1, step=0, rows=evidence_rows) + rows = read_gate_evidence(db_path, run_id=run_id) + assert len(rows) == 2 + assert rows[0]["question_id"] == "q1" + assert rows[1]["stop_reason"] == "confirmed" + assert rows[1]["e_value"] == pytest.approx(2.0) + + +# --------------------------------------------------------------------------- +# quadrant_pair +# --------------------------------------------------------------------------- + + +def test_write_read_quadrant_pairs(db_path: str, run_id: str) -> None: + """写入 quadrant_pair 后回读，bool -> int 转换正确。""" + pairs = [ + { + "question_id": "q1", + "task_type": "causal", + "prev_correct": True, + "curr_correct": False, + "category": "regression", + }, + { + "question_id": "q2", + "task_type": "causal", + "prev_correct": False, + "curr_correct": True, + "category": "improvement", + }, + ] + write_quadrant_pairs(db_path, run_id=run_id, epoch=1, step=0, pairs=pairs) + rows = read_quadrant_pairs(db_path, run_id=run_id) + assert len(rows) == 2 + # bool -> int 转换 + assert rows[0]["prev_correct"] == 1 + assert rows[0]["curr_correct"] == 0 + assert rows[1]["prev_correct"] == 0 + assert rows[1]["curr_correct"] == 1 + + +# --------------------------------------------------------------------------- +# NULL vs 0 语义 +# --------------------------------------------------------------------------- + + +def test_null_not_zero_for_soft(db_path: str, run_id: str) -> None: + """soft_score/mixed_score 为 None 时存 NULL（非 0），回读也是 None。""" + write_dual_metric( + db_path, + run_id=run_id, + epoch=1, + version_kind="baseline", + skills_version="v0", + prompts_version="v0", + pool="val", + hard_acc=0.75, + soft_score=None, + mixed_score=None, + ) + rows = read_dual_metric(db_path, run_id=run_id) + assert len(rows) == 1 + row = rows[0] + assert row["soft_score"] is None + assert row["mixed_score"] is None + + # 用原生 SQL 确认存的是 NULL 而非 0 + conn = sqlite3.connect(db_path) + cursor = conn.execute( + "SELECT soft_score, mixed_score FROM dual_metric_eval WHERE run_id=?", + (run_id,), + ) + raw = cursor.fetchone() + conn.close() + assert raw[0] is None + assert raw[1] is None + + +# --------------------------------------------------------------------------- +# 只读连接隔离 +# --------------------------------------------------------------------------- + + +def test_read_only_connection(db_path: str, run_id: str) -> None: + """read_* 使用独立只读连接，不向 _runs 表插入新行。""" + write_dual_metric( + db_path, + run_id=run_id, + epoch=1, + version_kind="baseline", + skills_version="v0", + prompts_version="v0", + pool="val", + hard_acc=0.5, + soft_score=None, + mixed_score=None, + ) + + # 用另一个 run_id 回读——不应在 _runs 表中创建新行 + other_run = "run-obs-ghost" + rows = read_dual_metric(db_path, run_id=other_run) + assert rows == [] + + conn = sqlite3.connect(db_path) + cursor = conn.execute("SELECT run_id FROM _runs") + run_ids = [r[0] for r in cursor.fetchall()] + conn.close() + assert other_run not in run_ids + + +# --------------------------------------------------------------------------- +# step_report +# --------------------------------------------------------------------------- + + +def test_write_step_report(tmp_path: Path) -> None: + """step_report 写入 JSON 文件，内容字段完整。""" + workspace = tmp_path / "ws" + workspace.mkdir() + path = write_step_report( + workspace_dir=workspace, + epoch=1, + step=2, + global_step=12, + task_type="Temporal Order", + gate_action="accept_confirmed", + candidate_acc=0.85, + class_baseline_acc=0.70, + edit_budget=5, + rank_clip_triggered=False, + gate_w=3, + gate_l=1, + gate_e_value=4.2, + gate_n_used=8, + gate_stop_reason="confirmed", + ) + assert path.exists() + assert path.name == "step_report_e1_s2_temporal-order.json" + data = json.loads(path.read_text(encoding="utf-8")) + assert data["epoch"] == 1 + assert data["step"] == 2 + assert data["global_step"] == 12 + assert data["task_type"] == "Temporal Order" + assert data["gate_action"] == "accept_confirmed" + assert data["gate_w"] == 3 + assert data["gate_stop_reason"] == "confirmed" + assert data["rank_clip_triggered"] is False + + +def test_write_step_report_skipped_null_fields(tmp_path: Path) -> None: + """skipped/cooldown 路径的 gate 字段应为 null。""" + workspace = tmp_path / "ws" + workspace.mkdir() + path = write_step_report( + workspace_dir=workspace, + epoch=1, + step=0, + global_step=0, + task_type="causal", + gate_action="skipped", + candidate_acc=0.0, + class_baseline_acc=0.0, + edit_budget=10, + rank_clip_triggered=False, + gate_w=None, + gate_l=None, + gate_e_value=None, + gate_n_used=None, + gate_stop_reason=None, + ) + data = json.loads(path.read_text(encoding="utf-8")) + assert data["gate_w"] is None + assert data["gate_l"] is None + assert data["gate_e_value"] is None + assert data["gate_n_used"] is None + assert data["gate_stop_reason"] is None + + +# --------------------------------------------------------------------------- +# epoch_report +# --------------------------------------------------------------------------- + + +def test_write_epoch_report(tmp_path: Path) -> None: + """epoch_report 写入 JSON 文件，内容字段完整。""" + workspace = tmp_path / "ws" + workspace.mkdir() + path = write_epoch_report( + workspace_dir=workspace, + epoch=3, + system_tool_action="updated", + momentum_updated_task_types=["temporal", "causal"], + best_val_acc=0.88, + ) + assert path.exists() + assert path.name == "epoch_report_3.json" + data = json.loads(path.read_text(encoding="utf-8")) + assert data["epoch"] == 3 + assert data["system_tool_action"] == "updated" + assert data["momentum_updated_task_types"] == ["temporal", "causal"] + assert data["best_val_acc"] == pytest.approx(0.88) From 886a444d1d5d49ada3a352cd769184cdcd9bc9e2 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 13:01:04 -0400 Subject: [PATCH 13/19] =?UTF-8?q?feat(harness):=20momentum.py=20=E2=80=94?= =?UTF-8?q?=20async=20=E6=85=A2=E6=9B=B4=E6=96=B0=E5=8A=A8=E9=87=8F?= =?UTF-8?q?=E7=94=9F=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/harness/momentum.py | 178 +++++++++++++++++++ tests/unit/test_harness_momentum.py | 261 ++++++++++++++++++++++++++++ 2 files changed, 439 insertions(+) create mode 100644 app/harness/momentum.py create mode 100644 tests/unit/test_harness_momentum.py diff --git a/app/harness/momentum.py b/app/harness/momentum.py new file mode 100644 index 0000000..bbd26ae --- /dev/null +++ b/app/harness/momentum.py @@ -0,0 +1,178 @@ +"""慢更新动量生成 — epoch 末为单个 skill 产出新的动量指导。 + +对标 SkillOpt 的 slow_update 机制：拿上一 epoch 末与当前 epoch 末两版 skill， +在固定样本上各跑一遍得到纵向对比（comparison_pairs），反思上一轮动量指导是否奏效、 +本轮正文改动是改善还是漂移，据此重写动量指导。新指导经 patch 引擎的 replace_momentum +写回 skill 的 momentum 受保护区，作为下一轮进化的方向锚。 + +从 TRM4 core/harness/momentum.py（156 行）迁移 + async 化。 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from loguru import logger + +from core.evolution.diagnose import extract_json_from_response + +if TYPE_CHECKING: + from pathlib import Path + + from core.protocols import LLMProvider + + +# ========================================================================= +# 四类纵向对比类别名（单一真源） +# ========================================================================= + +IMPROVED = "improved" # 错→对 +REGRESSED = "regressed" # 对→错 +PERSISTENT_FAIL = "persistent_fail" # 错→错 +STABLE_SUCCESS = "stable_success" # 对→对 + +# 类别名 → 展示标题，列表顺序即展示顺序。 +# 回退（REGRESSED）刻意排在改善（IMPROVED）之前——它是最该警惕的伤害信号。 +_CATEGORY_LABELS: tuple[tuple[str, str], ...] = ( + (REGRESSED, "从对变错（回退，最高优先级）"), + (PERSISTENT_FAIL, "始终答错（持续失败）"), + (IMPROVED, "从错变对（改善）"), + (STABLE_SUCCESS, "始终答对（稳定成功）"), +) + + +# ========================================================================= +# 辅助函数 +# ========================================================================= + + +def _categorize_pair(pair: dict[str, Any]) -> str: + """按两版正误派生纵向对比类别。 + + 用键值的真值（bool(...)）表示该题在两版上各自的正误：缺 correct_prev/ + correct_curr 键时直接抛 KeyError 向上传播——这是上游数据损坏（不是裁判语义 + 歧义），静默当 False 会伪造 persistent_fail 证据、污染动量指导，故不掩盖。 + + 参数: + pair: 单个纵向对比对，须含 correct_prev/correct_curr 两键。 + + 返回: + 四个类别命名常量之一：IMPROVED（错→对）/REGRESSED（对→错）/ + PERSISTENT_FAIL（错→错）/STABLE_SUCCESS（对→对）。 + + 异常: + KeyError: 缺 correct_prev 或 correct_curr 键时。 + """ + correct_prev = bool(pair["correct_prev"]) + correct_curr = bool(pair["correct_curr"]) + if not correct_prev and correct_curr: + return IMPROVED + if correct_prev and not correct_curr: + return REGRESSED + if not correct_prev and not correct_curr: + return PERSISTENT_FAIL + return STABLE_SUCCESS + + +def _format_comparison_pairs(comparison_pairs: list[dict[str, Any]]) -> str: + """将纵向对比对格式化为裁判可读文本，按 _CATEGORY_LABELS 分组与排序。 + + 参数: + comparison_pairs: 每个 dict 含 question/prev_prediction/curr_prediction/ + correct_prev/correct_curr 字段，描述一道固定样本上两版的成对结果。 + + 返回: + 可读的纵向对比文本；空列表返回占位说明。 + + 异常: + KeyError: 任一 pair 缺 correct_prev/correct_curr 键时；不掩盖的理由见 + _categorize_pair docstring。 + """ + if not comparison_pairs: + return "（本轮无可用纵向对比样本）" + + grouped: dict[str, list[dict[str, Any]]] = {key: [] for key, _ in _CATEGORY_LABELS} + for pair in comparison_pairs: + grouped[_categorize_pair(pair)].append(pair) + + lines: list[str] = [f"固定样本总数：{len(comparison_pairs)}"] + for key, label in _CATEGORY_LABELS: + entries = grouped[key] + lines.append(f"\n### {label}（{len(entries)} 题）") + if not entries: + lines.append("（无）") + continue + for pair in entries: + lines.append( + f"- 题目：{pair.get('question', '')}\n" + f" 上版预测：{pair.get('prev_prediction', '')} | " + f"当前版预测：{pair.get('curr_prediction', '')}" + ) + return "\n".join(lines) + + +# ========================================================================= +# 入口 +# ========================================================================= + + +async def run_slow_momentum( + llm: LLMProvider, + diagnose_prompts_dir: Path, + skill_content: str, + prev_skill: str, + prev_guidance: str, + comparison_pairs: list[dict[str, Any]], +) -> str: + """为单个 skill 生成新的慢更新动量指导。 + + 参数: + llm: LLM 端口（async chat）。 + diagnose_prompts_dir: 诊断 prompt 目录（根 prompts/，slow_momentum.md 在此）。 + skill_content: 当前版 skill 正文。 + prev_skill: 上一版 skill 正文。 + prev_guidance: 上一轮写下的动量指导。 + comparison_pairs: 固定样本上两版 rollout 的成对结果（含 question/ + prev_prediction/curr_prediction/correct_prev/correct_curr）。 + + 返回: + 新的动量指导文本；解析失败时保留 prev_guidance。 + + 关键实现细节: + - _format_comparison_pairs 刻意置于 try 块之外（prompt 构造阶段）：它对每个 + pair 取 correct_prev/correct_curr，缺键抛 KeyError 直接向上传播，不被下方 + 针对裁判语义歧义的 except ValueError 吞掉。 + - 解析失败保留上轮指导：extract_json_from_response 抛 ValueError、缺 + slow_update_content 字段、或该字段非 str，均视为语义解析失败，返回 + prev_guidance（判不准时保守保留上轮指导，对标 diagnose 的保护性 fallback）。 + - P5 边界：仅捕 ValueError 这一语义歧义；llm.chat 的基础设施失败 + （网络/API 异常）刻意不捕，向上传播，绝不用默认值掩盖。 + """ + system_prompt = (diagnose_prompts_dir / "slow_momentum.md").read_text(encoding="utf-8") + # _format_comparison_pairs 刻意置于下方 try 块之外（prompt 构造阶段）：它对每个 + # pair 取 correct_prev/correct_curr，缺键抛 KeyError 直接向上传播，不被下方针对 + # 裁判语义歧义的 except ValueError 吞掉。异常类型选 KeyError（非 ValueError）， + # 即便位置疏忽落入 try 也不会被误吞。 + user_prompt = ( + f"## 上一版 skill 正文\n{prev_skill}\n\n" + f"## 当前版 skill 正文\n{skill_content}\n\n" + f"## 上一轮的动量指导\n{prev_guidance}\n\n" + f"## 固定样本纵向对比（上版 vs 当前版）\n" + f"{_format_comparison_pairs(comparison_pairs)}" + ) + response = await llm.chat( + [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ] + ) + raw = response.content + try: + parsed = extract_json_from_response(raw) + new_guidance = parsed.get("slow_update_content") + if not isinstance(new_guidance, str): + raise ValueError("slow_update_content 字段缺失或非字符串") + except ValueError: + logger.warning("慢更新动量解析失败，保留上轮动量指导") + return prev_guidance + return new_guidance diff --git a/tests/unit/test_harness_momentum.py b/tests/unit/test_harness_momentum.py new file mode 100644 index 0000000..0869b52 --- /dev/null +++ b/tests/unit/test_harness_momentum.py @@ -0,0 +1,261 @@ +"""慢更新动量生成（app/harness/momentum.py）单元测试。 + +覆盖: +- _categorize_pair 四分类 + 缺键 KeyError +- _format_comparison_pairs 分组排序 + 空列表 +- run_slow_momentum 正常解析 + 解析失败保留 prev_guidance +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any +from unittest.mock import AsyncMock + +import pytest + +from app.harness.momentum import ( + IMPROVED, + PERSISTENT_FAIL, + REGRESSED, + STABLE_SUCCESS, + _categorize_pair, + _format_comparison_pairs, + run_slow_momentum, +) + +if TYPE_CHECKING: + from pathlib import Path + + +# ========================================================================= +# _categorize_pair +# ========================================================================= + + +class TestCategorizePair: + """_categorize_pair 四分类测试。""" + + def test_categorize_pair_all_four(self) -> None: + """四种 correct_prev/correct_curr 组合应返回对应类别常量。""" + assert _categorize_pair({"correct_prev": False, "correct_curr": True}) == IMPROVED + assert _categorize_pair({"correct_prev": True, "correct_curr": False}) == REGRESSED + assert _categorize_pair({"correct_prev": False, "correct_curr": False}) == PERSISTENT_FAIL + assert _categorize_pair({"correct_prev": True, "correct_curr": True}) == STABLE_SUCCESS + + def test_categorize_pair_truthy_values(self) -> None: + """非布尔真值（int / str）也能正确分类。""" + assert _categorize_pair({"correct_prev": 0, "correct_curr": 1}) == IMPROVED + assert _categorize_pair({"correct_prev": "yes", "correct_curr": ""}) == REGRESSED + + def test_categorize_pair_missing_key(self) -> None: + """缺 correct_prev 或 correct_curr 键时抛 KeyError。""" + with pytest.raises(KeyError): + _categorize_pair({"correct_prev": True}) + with pytest.raises(KeyError): + _categorize_pair({"correct_curr": False}) + with pytest.raises(KeyError): + _categorize_pair({}) + + +# ========================================================================= +# _format_comparison_pairs +# ========================================================================= + + +class TestFormatComparisonPairs: + """_format_comparison_pairs 分组排序测试。""" + + def test_format_comparison_pairs_empty(self) -> None: + """空列表返回占位说明。""" + result = _format_comparison_pairs([]) + assert "无可用" in result + + def test_format_comparison_pairs_order(self) -> None: + """REGRESSED 标题应出现在 IMPROVED 标题之前（伤害信号优先）。""" + pairs = [ + { + "question": "Q1", + "prev_prediction": "A", + "curr_prediction": "B", + "correct_prev": False, + "correct_curr": True, + }, + { + "question": "Q2", + "prev_prediction": "C", + "curr_prediction": "D", + "correct_prev": True, + "correct_curr": False, + }, + ] + result = _format_comparison_pairs(pairs) + regressed_pos = result.index("回退") + improved_pos = result.index("改善") + assert regressed_pos < improved_pos, "REGRESSED 应排在 IMPROVED 之前" + + def test_format_comparison_pairs_all_categories(self) -> None: + """四种类别的 pair 都能被正确分组。""" + pairs = [ + {"question": "Q1", "correct_prev": False, "correct_curr": True}, + {"question": "Q2", "correct_prev": True, "correct_curr": False}, + {"question": "Q3", "correct_prev": False, "correct_curr": False}, + {"question": "Q4", "correct_prev": True, "correct_curr": True}, + ] + result = _format_comparison_pairs(pairs) + assert "固定样本总数：4" in result + # 每个类别都应标注 1 题 + assert "1 题" in result + + def test_format_comparison_pairs_missing_key(self) -> None: + """pair 缺键时 KeyError 不被吞。""" + with pytest.raises(KeyError): + _format_comparison_pairs([{"question": "Q1"}]) + + +# ========================================================================= +# run_slow_momentum +# ========================================================================= + + +def _make_mock_llm(response_content: str) -> Any: + """构造一个返回指定 content 的 mock LLMProvider。""" + + @dataclass(frozen=True) + class _FakeResponse: + content: str + thinking: str = "" + model: str = "mock" + provider: str = "mock" + prompt_tokens: int = 0 + completion_tokens: int = 0 + latency_ms: int = 0 + ttft_ms: float | None = None + max_inter_token_ms: float | None = None + cache_hit: bool = False + call_id: str = "test-call-id" + + mock_llm = AsyncMock() + mock_llm.chat.return_value = _FakeResponse(content=response_content) + return mock_llm + + +@pytest.mark.asyncio +async def test_run_slow_momentum_basic(tmp_path: Path) -> None: + """正常解析时返回新的动量指导文本。""" + # 准备 prompt 文件 + prompt_file = tmp_path / "slow_momentum.md" + prompt_file.write_text("你是一个慢更新动量裁判。", encoding="utf-8") + + new_guidance_text = "新一轮的动量指导内容" + llm_response = json.dumps({"slow_update_content": new_guidance_text}, ensure_ascii=False) + mock_llm = _make_mock_llm(llm_response) + + result = await run_slow_momentum( + llm=mock_llm, + diagnose_prompts_dir=tmp_path, + skill_content="当前 skill 正文", + prev_skill="上一版 skill 正文", + prev_guidance="旧的动量指导", + comparison_pairs=[ + { + "question": "Q1", + "prev_prediction": "A", + "curr_prediction": "B", + "correct_prev": False, + "correct_curr": True, + } + ], + ) + assert result == new_guidance_text + mock_llm.chat.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_run_slow_momentum_parse_failure(tmp_path: Path) -> None: + """LLM 返回无法解析的内容时保留 prev_guidance。""" + prompt_file = tmp_path / "slow_momentum.md" + prompt_file.write_text("你是一个慢更新动量裁判。", encoding="utf-8") + + mock_llm = _make_mock_llm("这不是 JSON，无法解析") + prev_guidance = "应该被保留的旧动量指导" + + result = await run_slow_momentum( + llm=mock_llm, + diagnose_prompts_dir=tmp_path, + skill_content="当前 skill", + prev_skill="上一版 skill", + prev_guidance=prev_guidance, + comparison_pairs=[ + { + "question": "Q1", + "prev_prediction": "A", + "curr_prediction": "B", + "correct_prev": True, + "correct_curr": True, + } + ], + ) + assert result == prev_guidance + + +@pytest.mark.asyncio +async def test_run_slow_momentum_missing_field(tmp_path: Path) -> None: + """LLM 返回合法 JSON 但缺少 slow_update_content 字段时保留 prev_guidance。""" + prompt_file = tmp_path / "slow_momentum.md" + prompt_file.write_text("你是一个慢更新动量裁判。", encoding="utf-8") + + llm_response = json.dumps({"other_field": "无关内容"}, ensure_ascii=False) + mock_llm = _make_mock_llm(llm_response) + prev_guidance = "应该被保留的旧动量指导" + + result = await run_slow_momentum( + llm=mock_llm, + diagnose_prompts_dir=tmp_path, + skill_content="当前 skill", + prev_skill="上一版 skill", + prev_guidance=prev_guidance, + comparison_pairs=[], + ) + assert result == prev_guidance + + +@pytest.mark.asyncio +async def test_run_slow_momentum_keyerror_not_swallowed(tmp_path: Path) -> None: + """comparison_pairs 缺键时 KeyError 不被 ValueError 吞掉。""" + prompt_file = tmp_path / "slow_momentum.md" + prompt_file.write_text("你是一个慢更新动量裁判。", encoding="utf-8") + + mock_llm = _make_mock_llm('{"slow_update_content": "ok"}') + + with pytest.raises(KeyError): + await run_slow_momentum( + llm=mock_llm, + diagnose_prompts_dir=tmp_path, + skill_content="当前 skill", + prev_skill="上一版 skill", + prev_guidance="旧指导", + comparison_pairs=[{"question": "Q1"}], # 缺 correct_prev/correct_curr + ) + + +@pytest.mark.asyncio +async def test_run_slow_momentum_fenced_json(tmp_path: Path) -> None: + """LLM 返回 fenced code block 中的 JSON 也能正确解析。""" + prompt_file = tmp_path / "slow_momentum.md" + prompt_file.write_text("你是一个慢更新动量裁判。", encoding="utf-8") + + new_guidance = "从 fenced block 中提取的指导" + llm_response = f'```json\n{{"slow_update_content": "{new_guidance}"}}\n```' + mock_llm = _make_mock_llm(llm_response) + + result = await run_slow_momentum( + llm=mock_llm, + diagnose_prompts_dir=tmp_path, + skill_content="当前 skill", + prev_skill="上一版 skill", + prev_guidance="旧指导", + comparison_pairs=[], + ) + assert result == new_guidance From d7f1bdeea6fe7556c43f4d8883e2d07e95a8ca17 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 13:04:26 -0400 Subject: [PATCH 14/19] =?UTF-8?q?feat(harness):=20inference.py=20=E2=80=94?= =?UTF-8?q?=20async=20run=5Finference=20+=20DI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/harness/inference.py | 441 ++++++++++++++++++ tests/unit/test_harness_inference.py | 659 +++++++++++++++++++++++++++ 2 files changed, 1100 insertions(+) create mode 100644 app/harness/inference.py create mode 100644 tests/unit/test_harness_inference.py diff --git a/app/harness/inference.py b/app/harness/inference.py new file mode 100644 index 0000000..3924892 --- /dev/null +++ b/app/harness/inference.py @@ -0,0 +1,441 @@ +"""async 推理编排 — 训练循环的 forward()。 + +从 TRM4 core/harness/inference.py (~560 行) 迁移，重大重构: +- 同步 ThreadPoolExecutor → asyncio.Semaphore + asyncio.gather +- LLMClient.from_env() 每题构造 → llm: LLMProvider 注入共享 +- SentenceTransformer/OCR 内部构造 → 调用方通过 tool_dispatch_fn 注入 +- run_id 必传，空串 → ValueError +- _aggregate_results 从内存 results 聚合（非 DB 回读） +- record_run 由调用方（Runner）负责 +- prompt 构建由调用方注入 prompt_builder +""" + +from __future__ import annotations + +import asyncio +import json +from collections import defaultdict +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from loguru import logger + +from core.agent.loop import AgentLoop + +if TYPE_CHECKING: + from collections.abc import Callable + + from app.harness.log import HarnessLog + from core.agent.types import LoopResult + from core.protocols import LLMProvider + from core.types import GeneratedQuestion + + +@dataclass(frozen=True) +class InferenceResult: + """推理聚合结果。 + + 属性: + run_id: 运行标识。 + accuracy: 总正确率。 + total: 总题数。 + correct: 正确题数。 + per_task_type: 按题型分组的指标 {task_type: {accuracy, total, correct}}。 + steps_mean: 平均步数。 + token_usage: token 总用量 {prompt_tokens, completion_tokens}。 + stop_reason_counts: 终止原因计数 {reason: count}。 + """ + + run_id: str + accuracy: float + total: int + correct: int + per_task_type: dict[str, dict] + steps_mean: float + token_usage: dict[str, int] + stop_reason_counts: dict[str, int] + + +# --------------------------------------------------------------------------- +# 表 Schema 定义（5 张表，保留 TRM4 全部 schema） +# --------------------------------------------------------------------------- + +PREDICTIONS_SCHEMA: dict[str, str] = { + "video_id": "TEXT", + "question_id": "TEXT", + "task_type": "TEXT", + "prediction": "TEXT", + "answer": "TEXT", + "evidence": "TEXT", + "reasoning": "TEXT", + "steps_used": "INTEGER", + "prompt_tokens": "INTEGER", + "completion_tokens": "INTEGER", + "stop_reason": "TEXT", + "steps_json": "JSON", +} + +TRACES_SCHEMA: dict[str, str] = { + "video_id": "TEXT", + "question_id": "TEXT", + "step": "INTEGER", + "tool_name": "TEXT", + "tool_args": "JSON", + "tool_output": "TEXT", + "thought": "TEXT", +} + +VALIDATION_FLAGS_SCHEMA: dict[str, str] = { + "video_id": "TEXT", + "question_id": "TEXT", + "has_l3_visit": "INTEGER", + "l1_count": "INTEGER", + "l2_count": "INTEGER", + "l3_count": "INTEGER", +} + +ANCHOR_CHECK_SCHEMA: dict[str, str] = { + "video_id": "TEXT", + "question_id": "TEXT", + "step": "INTEGER", + "n_assertions": "INTEGER", + "n_anchored": "INTEGER", + "n_illegal": "INTEGER", + "n_expanded": "INTEGER", + "n_trunc": "INTEGER", + "output_chars": "INTEGER", +} + +OF_HEALTH_SCHEMA: dict[str, str] = { + "video_id": "TEXT", + "question_id": "TEXT", + "step": "INTEGER", + "ocr_injected": "INTEGER", + "ocr_chars": "INTEGER", + "ocr_failed": "INTEGER", + "discrepancy": "INTEGER", + "abstain": "INTEGER", +} + + +# --------------------------------------------------------------------------- +# 内部工具 +# --------------------------------------------------------------------------- + + +class _DispatcherAdapter: + """将裸 async callable 包装为 ToolDispatcher Protocol 实例。 + + AgentLoop 要求 ToolDispatcher（有 dispatch 方法），而 run_inference + 接收的 tool_dispatch_fn 是裸 async callable。此适配器桥接两者。 + + 参数: + fn: async def (tool_name, args, *, context) -> str。 + """ + + def __init__(self, fn: Callable[..., Any]) -> None: + self._fn = fn + + async def dispatch( + self, tool_name: str, args: dict[str, Any], *, context: dict[str, Any] + ) -> str: + """转发工具调用给被包装的 callable。""" + return await self._fn(tool_name, args, context=context) + + +def _to_text_field(value: Any) -> str: + """把 prediction 的 evidence/reasoning 归一为可入库的文本。 + + LLM 有时把这些字段返回成 list 或 dict（而非字符串）。sqlite 无法绑定 + 非标量类型，直接入库会抛 ProgrammingError 致该题丢失预测行、进而触发 + rollout 完整性护栏中止整轮。凡非 str 一律 JSON 序列化为文本。 + + 参数: + value: evidence/reasoning 原始值（可能是 str/list/dict）。 + + 返回: + 可直接入库的字符串。 + """ + if isinstance(value, str): + return value + return json.dumps(value, ensure_ascii=False) + + +def _zero_result(run_id: str) -> InferenceResult: + """空记录时的零值 InferenceResult。 + + 参数: + run_id: 运行标识。 + + 返回: + 全零的 InferenceResult。 + """ + return InferenceResult( + run_id=run_id, + accuracy=0.0, + total=0, + correct=0, + per_task_type={}, + steps_mean=0.0, + token_usage={"prompt_tokens": 0, "completion_tokens": 0}, + stop_reason_counts={}, + ) + + +def _group_by_task_type(records: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """按 task_type 分组聚合正确率指标。 + + 参数: + records: 预测记录列表。 + + 返回: + {task_type: {accuracy, total, correct}} 映射。 + """ + task_groups: dict[str, list[dict[str, Any]]] = defaultdict(list) + for r in records: + task_groups[r["task_type"]].append(r) + + per_task_type: dict[str, dict[str, Any]] = {} + for task_type, group in task_groups.items(): + t_total = len(group) + t_correct = sum(1 for r in group if r["prediction"] == r["answer"]) + per_task_type[task_type] = { + "accuracy": t_correct / t_total, + "total": t_total, + "correct": t_correct, + } + return per_task_type + + +def _aggregate_results(records: list[dict[str, Any]], run_id: str) -> InferenceResult: + """从内存 records 聚合推理指标。 + + TRM4 从 DB 回读 predictions 表聚合；TRM5 改为从内存直接聚合， + 避免 DB 回读的同步开销和额外依赖。 + + 参数: + records: _run_single_question 返回的 record 列表。 + run_id: 当前运行标识。 + + 返回: + InferenceResult 冻结实例。 + """ + total = len(records) + if total == 0: + return _zero_result(run_id) + + correct = sum(1 for r in records if r["prediction"] == r["answer"]) + stop_counts: dict[str, int] = defaultdict(int) + for r in records: + stop_counts[r["stop_reason"]] += 1 + + return InferenceResult( + run_id=run_id, + accuracy=correct / total, + total=total, + correct=correct, + per_task_type=_group_by_task_type(records), + steps_mean=sum(r["steps_used"] for r in records) / total, + token_usage={ + "prompt_tokens": sum(r["prompt_tokens"] for r in records), + "completion_tokens": sum(r["completion_tokens"] for r in records), + }, + stop_reason_counts=dict(stop_counts), + ) + + +# --------------------------------------------------------------------------- +# 单题推理 +# --------------------------------------------------------------------------- + + +async def _run_single_question( + qa: GeneratedQuestion, + *, + llm: LLMProvider, + tool_dispatch_fn: Callable[..., Any], + prompt_builder: Callable[[GeneratedQuestion], tuple[str, str]], + log: HarnessLog, + max_steps: int, + plugins: list[object], +) -> dict[str, Any]: + """执行单道题目的 Agent 推理。 + + 悲观默认值：record 初始 stop_reason="error"，成功后覆盖。 + prediction 必落库：log.insert 在 try/except 之后（无论成败）。 + + 参数: + qa: 待推理的题目。 + llm: LLMProvider 共享实例。 + tool_dispatch_fn: async 工具调度函数 (tool_name, args, *, context) -> str。 + prompt_builder: (GeneratedQuestion) -> (system_prompt, user_prompt)。 + log: HarnessLog 实例（线程安全）。 + max_steps: AgentLoop 最大步数。 + plugins: pluggy 插件列表。 + + 返回: + 预测结果字典（含 video_id, question_id, prediction, answer 等）。 + """ + record: dict[str, Any] = { + "video_id": qa.video_id, + "question_id": qa.question_id, + "task_type": qa.task_type, + "prediction": None, + "answer": qa.answer, + "evidence": "", + "reasoning": "", + "steps_used": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + "stop_reason": "error", # 悲观默认 + "steps_json": "[]", + } + + try: + system_prompt, user_prompt = prompt_builder(qa) + dispatcher = _DispatcherAdapter(tool_dispatch_fn) + loop = AgentLoop(llm, max_steps=max_steps) + loop_result: LoopResult = await loop.run( + system_prompt, + user_prompt, + dispatcher, + plugins=plugins, + session_id=qa.question_id, + ) + + result_dict = loop_result.result if isinstance(loop_result.result, dict) else {} + evidence = _to_text_field(result_dict.get("evidence", "")) + reasoning = _to_text_field(result_dict.get("reasoning", "")) + record.update( + { + "prediction": result_dict.get("answer"), + "evidence": evidence, + "reasoning": reasoning, + "steps_used": loop_result.steps_used, + "prompt_tokens": loop_result.token_usage["prompt_tokens"], + "completion_tokens": loop_result.token_usage["completion_tokens"], + "stop_reason": loop_result.stop_reason, + "steps_json": json.dumps( + [ + { + "thought": s.thought, + "tool_call": s.tool_call, + "tool_output": s.tool_output, + } + for s in loop_result.steps + ], + ensure_ascii=False, + ), + } + ) + except Exception: + logger.exception("[{}] QA {} 执行异常", qa.video_id, qa.question_id) + + # prediction 必落库（try 外，无论成败） + await asyncio.to_thread(log.insert, "predictions", record) + return record + + +# --------------------------------------------------------------------------- +# 建表 +# --------------------------------------------------------------------------- + + +def _ensure_tables(log: HarnessLog) -> None: + """创建推理所需的 5 张表。 + + 参数: + log: HarnessLog 实例。 + """ + log.create_table("predictions", PREDICTIONS_SCHEMA) + log.create_table("traces", TRACES_SCHEMA) + log.create_table("validation_flags", VALIDATION_FLAGS_SCHEMA) + log.create_table("anchor_check", ANCHOR_CHECK_SCHEMA) + log.create_table("observe_frame_health", OF_HEALTH_SCHEMA) + + +# --------------------------------------------------------------------------- +# 公共入口 +# --------------------------------------------------------------------------- + + +async def run_inference( + questions: list[GeneratedQuestion], + *, + llm: LLMProvider, + tool_dispatch_fn: Callable[..., Any], + prompt_builder: Callable[[GeneratedQuestion], tuple[str, str]], + log: HarnessLog, + run_id: str, + concurrency: int, + max_steps: int, + skill_mode: str, + plugins_factory: Callable[[str, str], list[object]] | None = None, +) -> InferenceResult: + """在视频树上执行 Agent 推理，对应训练循环的 forward()。 + + 参数: + questions: 待推理的题目列表。 + llm: LLMProvider 共享实例（依赖注入）。 + tool_dispatch_fn: async 工具调度函数 (tool_name, args, *, context) -> str。 + prompt_builder: prompt 构建函数 (GeneratedQuestion) -> (system_prompt, user_prompt)。 + log: HarnessLog 实例（由调用方管理生命周期）。 + run_id: 运行标识（必传，空串 → ValueError）。 + concurrency: 最大并发数（asyncio.Semaphore 控制）。 + max_steps: AgentLoop 单题最大步数。 + skill_mode: "auto" / "manual" / "none"（传递给调用方的 prompt/plugin 构建逻辑）。 + plugins_factory: 可选的插件工厂 (video_id, question_id) -> plugins 列表。 + + 返回: + InferenceResult（含 accuracy、per_task_type 等聚合指标）。 + + 异常: + ValueError: run_id 为空串或纯空白。 + """ + if not run_id or not run_id.strip(): + raise ValueError("run_id 不得为空串或纯空白") + + _ensure_tables(log) + + if not questions: + logger.info("题目列表为空，返回零值 InferenceResult") + return _aggregate_results([], run_id) + + sem = asyncio.Semaphore(concurrency) + total_count = len(questions) + + async def _bounded(index: int, qa: GeneratedQuestion) -> dict[str, Any]: + """信号量限流的单题推理包装。""" + async with sem: + plugins = ( + plugins_factory(qa.video_id, qa.question_id) if plugins_factory is not None else [] + ) + result = await _run_single_question( + qa, + llm=llm, + tool_dispatch_fn=tool_dispatch_fn, + prompt_builder=prompt_builder, + log=log, + max_steps=max_steps, + plugins=plugins, + ) + logger.info( + "[{}/{}] {} QA {} 完成 (stop={})", + index + 1, + total_count, + qa.video_id, + qa.question_id, + result["stop_reason"], + ) + return result + + results = await asyncio.gather(*[_bounded(i, qa) for i, qa in enumerate(questions)]) + + inference_result = _aggregate_results(list(results), run_id) + logger.info( + "推理完成: accuracy={:.2%} ({}/{})", + inference_result.accuracy, + inference_result.correct, + inference_result.total, + ) + return inference_result diff --git a/tests/unit/test_harness_inference.py b/tests/unit/test_harness_inference.py new file mode 100644 index 0000000..2c6e5d6 --- /dev/null +++ b/tests/unit/test_harness_inference.py @@ -0,0 +1,659 @@ +"""app/harness/inference.py 单元测试。 + +测试覆盖: +- run_inference 基本流程（mock LLM + tool_dispatch） +- 异常时 prediction 仍落库（stop_reason=error） +- _to_text_field 归一化 +- run_id 空串 → ValueError +- _aggregate_results 内存聚合 +- 空 questions 列表零值返回 +- 并发控制 Semaphore +- plugins_factory 调用 +""" + +from __future__ import annotations + +import json +from typing import Any +from unittest.mock import AsyncMock + +import pytest + +from app.harness.inference import ( + InferenceResult, + _aggregate_results, + _to_text_field, + run_inference, +) +from app.harness.log import HarnessLog +from core.types import GeneratedQuestion, LLMResponse + +# ── 测试基础设施 ────────────────────────────────────────────────── + + +def _make_question( + question_id: str = "q1", + video_id: str = "v1", + task_type: str = "Action Reasoning", + answer: str = "B", +) -> GeneratedQuestion: + """构造测试用题目。""" + return GeneratedQuestion( + question_id=question_id, + video_id=video_id, + task_type=task_type, + question="测试问题", + options=("A. 选项A", "B. 选项B", "C. 选项C", "D. 选项D"), + answer=answer, + source_nodes=("L1_001",), + difficulty="medium", + ) + + +def _make_llm_response(answer: str = "B") -> LLMResponse: + """构造测试用 LLMResponse（submit_answer 场景）。""" + content = json.dumps( + { + "reflect": {"observation": "找到答案"}, + "plan": {"next_step": "提交"}, + "action": { + "tool": "submit_answer", + "args": { + "answer": answer, + "evidence": "证据文本", + "reasoning": "推理过程", + }, + }, + } + ) + return LLMResponse( + content=content, + thinking="思考过程", + model="test-model", + provider="test", + prompt_tokens=100, + completion_tokens=50, + latency_ms=200, + ttft_ms=30.0, + max_inter_token_ms=5.0, + cache_hit=False, + call_id="test-call-001", + ) + + +def _make_error_llm_response() -> LLMResponse: + """构造触发解析失败的 LLMResponse。""" + return LLMResponse( + content="这不是JSON", + thinking="", + model="test-model", + provider="test", + prompt_tokens=10, + completion_tokens=5, + latency_ms=50, + ttft_ms=10.0, + max_inter_token_ms=2.0, + cache_hit=False, + call_id="test-call-err", + ) + + +async def _stub_tool_dispatch( + tool_name: str, args: dict[str, Any], *, context: dict[str, Any] +) -> str: + """测试用工具调度函数。""" + if tool_name == "submit_answer": + return "答案已提交" + raise ValueError(f"未知工具: {tool_name}") + + +def _stub_prompt_builder(qa: GeneratedQuestion) -> tuple[str, str]: + """测试用 prompt 构建函数。""" + return "系统提示词", f"用户问题: {qa.question}" + + +@pytest.fixture +def harness_log(tmp_path: Any, request: Any) -> HarnessLog: + """创建临时 HarnessLog 实例。 + + 使用 test 节点名称的 hash 作为 db 文件名，避免冲突。 + run_id 固定为 "test-run"，实际 run_inference 中传入的 run_id + 由 HarnessLog.insert 自动覆盖为 HarnessLog 构造时的值。 + """ + db_name = f"harness_{id(request)}.db" + db_path = str(tmp_path / db_name) + log = HarnessLog(db_path, "test-run") + yield log + log.close() + + +# ── 测试用例 ────────────────────────────────────────────────── + + +class TestToTextField: + """_to_text_field 归一化测试。""" + + @pytest.mark.asyncio + async def test_string_passthrough(self) -> None: + """字符串原样返回。""" + assert _to_text_field("hello") == "hello" + + @pytest.mark.asyncio + async def test_empty_string(self) -> None: + """空字符串原样返回。""" + assert _to_text_field("") == "" + + @pytest.mark.asyncio + async def test_list_serialized(self) -> None: + """list 被 JSON 序列化。""" + result = _to_text_field(["a", "b"]) + assert result == '["a", "b"]' + + @pytest.mark.asyncio + async def test_dict_serialized(self) -> None: + """dict 被 JSON 序列化。""" + result = _to_text_field({"key": "值"}) + assert '"key"' in result + assert '"值"' in result + + @pytest.mark.asyncio + async def test_int_serialized(self) -> None: + """int 被 JSON 序列化。""" + assert _to_text_field(42) == "42" + + @pytest.mark.asyncio + async def test_none_serialized(self) -> None: + """None 被 JSON 序列化。""" + assert _to_text_field(None) == "null" + + @pytest.mark.asyncio + async def test_unicode_preserved(self) -> None: + """ensure_ascii=False 保留中文。""" + result = _to_text_field(["中文"]) + assert "中文" in result + assert "\\u" not in result + + +class TestAggregateResults: + """_aggregate_results 内存聚合测试。""" + + @pytest.mark.asyncio + async def test_empty_records(self) -> None: + """空列表返回零值 InferenceResult。""" + result = _aggregate_results([], "run-empty") + assert result.run_id == "run-empty" + assert result.accuracy == 0.0 + assert result.total == 0 + assert result.correct == 0 + assert result.per_task_type == {} + assert result.steps_mean == 0.0 + assert result.token_usage == {"prompt_tokens": 0, "completion_tokens": 0} + assert result.stop_reason_counts == {} + + @pytest.mark.asyncio + async def test_single_correct(self) -> None: + """单条正确记录 → accuracy=1.0。""" + records = [ + { + "prediction": "B", + "answer": "B", + "task_type": "AR", + "steps_used": 3, + "prompt_tokens": 100, + "completion_tokens": 50, + "stop_reason": "finished", + } + ] + result = _aggregate_results(records, "run-1") + assert result.accuracy == 1.0 + assert result.total == 1 + assert result.correct == 1 + assert result.steps_mean == 3.0 + + @pytest.mark.asyncio + async def test_mixed_correct_wrong(self) -> None: + """混合正确/错误 → 准确率与步数均正确聚合。""" + records = [ + { + "prediction": "B", + "answer": "B", + "task_type": "AR", + "steps_used": 2, + "prompt_tokens": 100, + "completion_tokens": 50, + "stop_reason": "finished", + }, + { + "prediction": "C", + "answer": "A", + "task_type": "AR", + "steps_used": 4, + "prompt_tokens": 200, + "completion_tokens": 100, + "stop_reason": "budget_exceeded", + }, + { + "prediction": "D", + "answer": "D", + "task_type": "SP", + "steps_used": 1, + "prompt_tokens": 50, + "completion_tokens": 25, + "stop_reason": "finished", + }, + ] + result = _aggregate_results(records, "run-mix") + assert result.total == 3 + assert result.correct == 2 + assert abs(result.accuracy - 2 / 3) < 1e-9 + assert abs(result.steps_mean - 7 / 3) < 1e-9 + assert result.token_usage == {"prompt_tokens": 350, "completion_tokens": 175} + assert result.stop_reason_counts == {"finished": 2, "budget_exceeded": 1} + + @pytest.mark.asyncio + async def test_per_task_type_grouping(self) -> None: + """按 task_type 分组聚合。""" + records = [ + { + "prediction": "B", + "answer": "B", + "task_type": "AR", + "steps_used": 1, + "prompt_tokens": 10, + "completion_tokens": 5, + "stop_reason": "finished", + }, + { + "prediction": "A", + "answer": "C", + "task_type": "AR", + "steps_used": 2, + "prompt_tokens": 20, + "completion_tokens": 10, + "stop_reason": "finished", + }, + { + "prediction": "D", + "answer": "D", + "task_type": "SP", + "steps_used": 3, + "prompt_tokens": 30, + "completion_tokens": 15, + "stop_reason": "finished", + }, + ] + result = _aggregate_results(records, "run-task") + assert "AR" in result.per_task_type + assert "SP" in result.per_task_type + assert result.per_task_type["AR"]["total"] == 2 + assert result.per_task_type["AR"]["correct"] == 1 + assert result.per_task_type["AR"]["accuracy"] == 0.5 + assert result.per_task_type["SP"]["total"] == 1 + assert result.per_task_type["SP"]["correct"] == 1 + assert result.per_task_type["SP"]["accuracy"] == 1.0 + + +class TestRunIdValidation: + """run_id 校验测试。""" + + @pytest.mark.asyncio + async def test_empty_string_raises(self, harness_log: HarnessLog) -> None: + """空串 run_id → ValueError。""" + llm = AsyncMock() + with pytest.raises(ValueError, match="run_id 不得为空"): + await run_inference( + [], + llm=llm, + tool_dispatch_fn=_stub_tool_dispatch, + prompt_builder=_stub_prompt_builder, + log=harness_log, + run_id="", + concurrency=1, + max_steps=10, + skill_mode="auto", + ) + + @pytest.mark.asyncio + async def test_whitespace_only_raises(self, harness_log: HarnessLog) -> None: + """纯空白 run_id → ValueError。""" + llm = AsyncMock() + with pytest.raises(ValueError, match="run_id 不得为空"): + await run_inference( + [], + llm=llm, + tool_dispatch_fn=_stub_tool_dispatch, + prompt_builder=_stub_prompt_builder, + log=harness_log, + run_id=" ", + concurrency=1, + max_steps=10, + skill_mode="auto", + ) + + +class TestEmptyQuestions: + """空题目列表测试。""" + + @pytest.mark.asyncio + async def test_empty_questions_returns_zero(self, harness_log: HarnessLog) -> None: + """空 questions 列表直接返回零值 InferenceResult。""" + llm = AsyncMock() + result = await run_inference( + [], + llm=llm, + tool_dispatch_fn=_stub_tool_dispatch, + prompt_builder=_stub_prompt_builder, + log=harness_log, + run_id="run-empty", + concurrency=1, + max_steps=10, + skill_mode="auto", + ) + assert isinstance(result, InferenceResult) + assert result.run_id == "run-empty" + assert result.accuracy == 0.0 + assert result.total == 0 + assert result.correct == 0 + # LLM 未被调用 + llm.chat.assert_not_called() + + +class TestRunInferenceBasic: + """run_inference 基本流程测试。""" + + @pytest.mark.asyncio + async def test_single_question_correct(self, harness_log: HarnessLog) -> None: + """单题正确推理 → accuracy=1.0, stop_reason=finished。""" + llm = AsyncMock() + llm.chat.return_value = _make_llm_response(answer="B") + + result = await run_inference( + [_make_question(answer="B")], + llm=llm, + tool_dispatch_fn=_stub_tool_dispatch, + prompt_builder=_stub_prompt_builder, + log=harness_log, + run_id="run-basic", + concurrency=1, + max_steps=10, + skill_mode="auto", + ) + + assert result.accuracy == 1.0 + assert result.total == 1 + assert result.correct == 1 + assert result.stop_reason_counts.get("finished") == 1 + + @pytest.mark.asyncio + async def test_single_question_wrong(self, harness_log: HarnessLog) -> None: + """单题错误推理 → accuracy=0.0。""" + llm = AsyncMock() + llm.chat.return_value = _make_llm_response(answer="C") + + result = await run_inference( + [_make_question(answer="B")], + llm=llm, + tool_dispatch_fn=_stub_tool_dispatch, + prompt_builder=_stub_prompt_builder, + log=harness_log, + run_id="run-wrong", + concurrency=1, + max_steps=10, + skill_mode="auto", + ) + + assert result.accuracy == 0.0 + assert result.total == 1 + assert result.correct == 0 + + @pytest.mark.asyncio + async def test_multiple_questions_concurrent(self, harness_log: HarnessLog) -> None: + """3 题并发推理 → 结果正确聚合。""" + llm = AsyncMock() + llm.chat.return_value = _make_llm_response(answer="B") + + questions = [ + _make_question(question_id="q1", answer="B"), + _make_question(question_id="q2", answer="B"), + _make_question(question_id="q3", answer="A"), + ] + + result = await run_inference( + questions, + llm=llm, + tool_dispatch_fn=_stub_tool_dispatch, + prompt_builder=_stub_prompt_builder, + log=harness_log, + run_id="run-multi", + concurrency=3, + max_steps=10, + skill_mode="auto", + ) + + assert result.total == 3 + assert result.correct == 2 + assert abs(result.accuracy - 2 / 3) < 1e-9 + + @pytest.mark.asyncio + async def test_token_usage_accumulated(self, harness_log: HarnessLog) -> None: + """多题 token 累加验证。""" + llm = AsyncMock() + llm.chat.return_value = _make_llm_response(answer="B") + + questions = [ + _make_question(question_id="q1"), + _make_question(question_id="q2"), + ] + + result = await run_inference( + questions, + llm=llm, + tool_dispatch_fn=_stub_tool_dispatch, + prompt_builder=_stub_prompt_builder, + log=harness_log, + run_id="run-token", + concurrency=2, + max_steps=10, + skill_mode="auto", + ) + + assert result.token_usage["prompt_tokens"] == 200 + assert result.token_usage["completion_tokens"] == 100 + + +class TestPredictionAlwaysWritten: + """异常时 prediction 仍落库测试。""" + + @pytest.mark.asyncio + async def test_error_still_persisted(self, harness_log: HarnessLog) -> None: + """LLM 调用异常时，prediction 仍以 stop_reason=error 落库。""" + llm = AsyncMock() + llm.chat.side_effect = RuntimeError("LLM API 不可用") + + result = await run_inference( + [_make_question()], + llm=llm, + tool_dispatch_fn=_stub_tool_dispatch, + prompt_builder=_stub_prompt_builder, + log=harness_log, + run_id="run-error", + concurrency=1, + max_steps=10, + skill_mode="auto", + ) + + assert result.total == 1 + assert result.correct == 0 + assert result.stop_reason_counts.get("error") == 1 + + # 验证 DB 中的记录（HarnessLog.insert 使用构造时的 run_id） + rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("test-run",)) + assert len(rows) == 1 + assert rows[0]["stop_reason"] == "error" + assert rows[0]["prediction"] is None + + @pytest.mark.asyncio + async def test_parse_error_still_persisted(self, harness_log: HarnessLog) -> None: + """LLM 返回非 JSON 内容，parse_error 后 prediction 仍落库。""" + llm = AsyncMock() + llm.chat.return_value = _make_error_llm_response() + + result = await run_inference( + [_make_question()], + llm=llm, + tool_dispatch_fn=_stub_tool_dispatch, + prompt_builder=_stub_prompt_builder, + log=harness_log, + run_id="run-parse-err", + concurrency=1, + max_steps=10, + skill_mode="auto", + ) + + assert result.total == 1 + # HarnessLog.insert 使用构造时的 run_id + rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("test-run",)) + assert len(rows) == 1 + assert rows[0]["prediction"] is None + + +class TestPluginsFactory: + """plugins_factory 调用测试。""" + + @pytest.mark.asyncio + async def test_factory_called_per_question(self, harness_log: HarnessLog) -> None: + """每题调用 plugins_factory，传入 (video_id, question_id)。""" + llm = AsyncMock() + llm.chat.return_value = _make_llm_response(answer="B") + + factory_calls: list[tuple[str, str]] = [] + + def _factory(video_id: str, question_id: str) -> list[object]: + factory_calls.append((video_id, question_id)) + return [] + + questions = [ + _make_question(question_id="q1", video_id="v1"), + _make_question(question_id="q2", video_id="v2"), + ] + + await run_inference( + questions, + llm=llm, + tool_dispatch_fn=_stub_tool_dispatch, + prompt_builder=_stub_prompt_builder, + log=harness_log, + run_id="run-factory", + concurrency=2, + max_steps=10, + skill_mode="auto", + plugins_factory=_factory, + ) + + assert len(factory_calls) == 2 + call_set = set(factory_calls) + assert ("v1", "q1") in call_set + assert ("v2", "q2") in call_set + + @pytest.mark.asyncio + async def test_no_factory_uses_empty_plugins(self, harness_log: HarnessLog) -> None: + """plugins_factory=None 时使用空 plugins 列表。""" + llm = AsyncMock() + llm.chat.return_value = _make_llm_response(answer="B") + + result = await run_inference( + [_make_question()], + llm=llm, + tool_dispatch_fn=_stub_tool_dispatch, + prompt_builder=_stub_prompt_builder, + log=harness_log, + run_id="run-no-factory", + concurrency=1, + max_steps=10, + skill_mode="auto", + plugins_factory=None, + ) + + assert result.total == 1 + assert result.stop_reason_counts.get("finished") == 1 + + +class TestConcurrencyControl: + """并发控制 Semaphore 测试。""" + + @pytest.mark.asyncio + async def test_concurrency_semaphore_limits(self, harness_log: HarnessLog) -> None: + """Semaphore(1) 限制并发为 1 — 通过最大并发计数器验证。""" + import asyncio + + llm = AsyncMock() + current_concurrent = 0 + max_concurrent = 0 + + original_response = _make_llm_response(answer="B") + + async def _slow_chat( + messages: Any, + *, + session_id: str | None = None, + parent_call_id: str | None = None, + ) -> LLMResponse: + nonlocal current_concurrent, max_concurrent + current_concurrent += 1 + max_concurrent = max(max_concurrent, current_concurrent) + await asyncio.sleep(0.01) + current_concurrent -= 1 + return original_response + + llm.chat.side_effect = _slow_chat + + questions = [_make_question(question_id=f"q{i}") for i in range(5)] + + await run_inference( + questions, + llm=llm, + tool_dispatch_fn=_stub_tool_dispatch, + prompt_builder=_stub_prompt_builder, + log=harness_log, + run_id="run-sem", + concurrency=1, + max_steps=10, + skill_mode="auto", + ) + + assert max_concurrent == 1 + + +class TestTablesCreated: + """表创建测试。""" + + @pytest.mark.asyncio + async def test_five_tables_created(self, harness_log: HarnessLog) -> None: + """run_inference 启动时创建 5 张推理表。""" + llm = AsyncMock() + + await run_inference( + [], + llm=llm, + tool_dispatch_fn=_stub_tool_dispatch, + prompt_builder=_stub_prompt_builder, + log=harness_log, + run_id="run-tables", + concurrency=1, + max_steps=10, + skill_mode="auto", + ) + + expected_tables = [ + "predictions", + "traces", + "validation_flags", + "anchor_check", + "observe_frame_health", + ] + for table_name in expected_tables: + rows = harness_log.query( + "SELECT name FROM sqlite_master WHERE type='table' AND name=?", + (table_name,), + ) + assert len(rows) == 1, f"表 {table_name} 未创建" From a6b816db94269a1bcc7bda4d3335a9d976039f4e Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 13:13:37 -0400 Subject: [PATCH 15/19] =?UTF-8?q?feat(harness):=20checkpoint.py=20?= =?UTF-8?q?=E2=80=94=20TrainState=20=E5=BA=8F=E5=88=97=E5=8C=96=20+=20?= =?UTF-8?q?=E5=8E=9F=E5=AD=90=E5=86=99=20+=20=E6=8C=87=E7=BA=B9=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/harness/checkpoint.py | 301 +++++++++++++++++++ tests/unit/test_harness_checkpoint.py | 413 ++++++++++++++++++++++++++ 2 files changed, 714 insertions(+) create mode 100644 app/harness/checkpoint.py create mode 100644 tests/unit/test_harness_checkpoint.py diff --git a/app/harness/checkpoint.py b/app/harness/checkpoint.py new file mode 100644 index 0000000..04199e1 --- /dev/null +++ b/app/harness/checkpoint.py @@ -0,0 +1,301 @@ +"""step 级续训 checkpoint：_TrainState 可持久化字段的序列化 / 反序列化。 + +_TrainState 的累加包均为扁平纯数据 dataclass，经 dataclasses.asdict 序列化为 +纯 JSON dict；反序列化时用 Cls(**d) 还原，其中 SystemCasePack 含嵌套 CaseSample +列表、Probation 含嵌套 RejectedEdit 列表，需逐个重建。 + +不持久化的字段：gate_pools / baseline_cache（各自文件级自持久化，resume 时按 +指纹重载）、best_*（从 manifest best 指针读）、global_step（存 progress 块， +由 train 单独赋值）。gate_epoch_observed 持久化：warm p-hat 在 gate_pools.json +幸存，观测开关须随行，否则 resume 后阶梯排序回退冷启动序。 +""" + +from __future__ import annotations + +import json +import os +from dataclasses import asdict, dataclass, field +from typing import TYPE_CHECKING, Any + +from core.evolution.types import ( + CaseSample, + RejectedEdit, + SystemCasePack, + ToolCasePack, +) + +if TYPE_CHECKING: + from pathlib import Path + +CHECKPOINT_SCHEMA_VERSION = 1 + + +# --------------------------------------------------------------------------- +# Probation 类型（Task 10 validate.py 尚未就绪，暂定义在此供 checkpoint 使用） +# Task 10 完成后迁移至 app/harness/validate.py 并改为 re-export。 +# --------------------------------------------------------------------------- + + +@dataclass +class Probation: + """一个题型的在途试用账本（每题型至多一个）。 + + 属性: + task_type: 题型。 + anchor_skills_version: 锚版本名（最近一个 CONFIRMED 的 skills 版本）。 + target_file: 该题型解析后的 skill 文件名。 + correctness_snapshot: 开账时该题型 val 题的对错快照（回滚时恢复）。 + opened_step: 开账时的 global_step（观测用）。 + pending_edits: 试用链上全部候选 edit 的黑名单素材（回滚时整链入黑名单）。 + """ + + task_type: str + anchor_skills_version: str + target_file: str + correctness_snapshot: dict[str, bool] + opened_step: int + pending_edits: list[RejectedEdit] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# 结构性 / 决策性指纹键 +# --------------------------------------------------------------------------- + +_STRUCTURAL_KEYS = ( + "batch_size", + "min_class_per_batch", + "epochs", + "diag_size", + "val_size", + "batch_correct_ratio", +) + +_DECISION_KEYS = ( + "edit_budget_start", + "edit_budget_end", + "early_stop_patience", + "use_slow_momentum", + "skill_update_mode", + "appendix_consolidate_threshold", + "momentum_samples", + "gate_e_confirm", + "gate_e_provisional", + "gate_w_net_min", + "gate_delta_min", + "gate_lambda_dir", + "gate_e_rollback", + "gate_block", + "gate_n_max", + "gate_p_low", + "gate_p_high", + "gate_probe_quota", + "gate_gamma_decay", + "gate_cooldown_steps", + "gate_guard_err", +) + + +# --------------------------------------------------------------------------- +# 序列化 / 反序列化 +# --------------------------------------------------------------------------- + + +def serialize_state(state: Any) -> dict[str, Any]: + """把 _TrainState 的可持久化字段转为纯 JSON dict。 + + 参数: + state: _TrainState 实例（duck-typed，仅需含可持久化字段）。 + + 返回: + 纯 JSON 可序列化的 dict，不含 gate_pools / baseline_cache / + best_* / global_step。 + + 关键实现细节: + - changed_task_types_this_epoch 是 set，JSON 无 set，故 sorted 成有序列表。 + - dataclass 均经 asdict 递归转 dict（含 SystemCasePack 嵌套 CaseSample、 + Probation 嵌套 RejectedEdit）。 + """ + return { + "correctness": state.correctness, + "eval_prev_acc": state.eval_prev_acc, + "eval_prev_run_id": state.eval_prev_run_id, + "baseline_skills_version": state.baseline_skills_version, + "baseline_prompts_version": state.baseline_prompts_version, + "steps_since_best_improved": state.steps_since_best_improved, + "epoch_start_skills": state.epoch_start_skills, + "changed_task_types_this_epoch": sorted(state.changed_task_types_this_epoch), + "rejected_buffer": {k: [asdict(x) for x in v] for k, v in state.rejected_buffer.items()}, + "system_packs": [asdict(x) for x in state.system_packs], + "tool_packs": [asdict(x) for x in state.tool_packs], + "probations": {t: asdict(p) for t, p in state.probations.items()}, + "gate_cooldown": state.gate_cooldown, + "gate_epoch_observed": state.gate_epoch_observed, + } + + +def _restore_system_pack(d: dict[str, Any]) -> SystemCasePack: + """还原 SystemCasePack，含嵌套 CaseSample 列表。 + + 参数: + d: asdict(SystemCasePack) 产出的 dict。 + + 返回: + 复活的 SystemCasePack；failure_cases / success_cases 重建为 CaseSample 实例。 + """ + return SystemCasePack( + stats=d["stats"], + failure_cases=[CaseSample(**c) for c in d["failure_cases"]], + success_cases=[CaseSample(**c) for c in d["success_cases"]], + ) + + +def deserialize_state_fields(d: dict[str, Any]) -> dict[str, Any]: + """把序列化 dict 还原为可填入 _TrainState 的字段字典（dataclass 复活）。 + + 参数: + d: serialize_state 产出并经 JSON 往返的 dict。 + + 返回: + 字段名 -> 值的 dict，可直接铺到 _TrainState；其中各 dataclass 已复活、 + changed_task_types_this_epoch 还原为 set。 + + 关键实现细节: + - RejectedEdit / ToolCasePack 字段均为标量/dict/list[dict]，Cls(**d) 直接构造。 + - SystemCasePack 含嵌套 CaseSample，交由 _restore_system_pack 重建。 + - Probation 含嵌套 RejectedEdit 列表（pending_edits），先重建内层再构造外层。 + - 直接取 d[...] 不用 .get 兜底：serialize 后的 checkpoint 必带全部键， + 缺键即 checkpoint 损坏，应硬失败（P5 不掩盖）。 + """ + return { + "correctness": d["correctness"], + "eval_prev_acc": d["eval_prev_acc"], + "eval_prev_run_id": d["eval_prev_run_id"], + "baseline_skills_version": d["baseline_skills_version"], + "baseline_prompts_version": d["baseline_prompts_version"], + "steps_since_best_improved": d["steps_since_best_improved"], + "epoch_start_skills": d["epoch_start_skills"], + "changed_task_types_this_epoch": set(d["changed_task_types_this_epoch"]), + "rejected_buffer": { + k: [RejectedEdit(**x) for x in v] for k, v in d["rejected_buffer"].items() + }, + "system_packs": [_restore_system_pack(x) for x in d["system_packs"]], + "tool_packs": [ToolCasePack(**x) for x in d["tool_packs"]], + "probations": { + t: Probation( + **{ + **d_p, + "pending_edits": [RejectedEdit(**x) for x in d_p["pending_edits"]], + } + ) + for t, d_p in d["probations"].items() + }, + "gate_cooldown": d["gate_cooldown"], + "gate_epoch_observed": d["gate_epoch_observed"], + } + + +# --------------------------------------------------------------------------- +# 配置指纹 +# --------------------------------------------------------------------------- + + +def compute_fingerprint(config: Any) -> dict[str, Any]: + """采集影响训练轨迹的配置项（结构性 + 决策性）。 + + 参数: + config: 训练配置对象（duck-typed，需含 _STRUCTURAL_KEYS + _DECISION_KEYS 属性）。 + + 返回: + 指纹 dict，键为配置项名，值为对应配置值。 + """ + return {k: getattr(config, k) for k in _STRUCTURAL_KEYS + _DECISION_KEYS} + + +def check_fingerprint(saved: dict[str, Any], config: Any) -> tuple[list[str], list[str]]: + """比对保存的指纹与当前配置。返回 (结构性不一致项, 决策性不一致项)。 + + 参数: + saved: checkpoint 中保存的 config_fingerprint。 + config: 当前训练配置对象。 + + 返回: + (structural, decision) 两个不一致项名列表。 + + 关键实现细节: + 结构性不一致（batch_size/min_class_per_batch/epochs/diag_size/val_size/ + batch_correct_ratio）→ 调用方应拒绝 resume；决策性不一致 → 仅告警放行。 + """ + cur = compute_fingerprint(config) + structural = [k for k in _STRUCTURAL_KEYS if saved.get(k) != cur[k]] + decision = [k for k in _DECISION_KEYS if saved.get(k) != cur[k]] + return structural, decision + + +# --------------------------------------------------------------------------- +# 读写 checkpoint +# --------------------------------------------------------------------------- + + +def write_checkpoint( + workspace_dir: Path, + *, + state: Any, + epoch: int, + step_completed: int, + phase: str, + global_step: int, + total_steps: int, + version_snapshot: dict[str, str], + epoch_batches: list[list[str]], + config: Any, +) -> None: + """原子写 checkpoint.json（.tmp 再 os.replace）。 + + 参数: + workspace_dir: workspace 目录，checkpoint.json 写入其下。 + state: _TrainState 实例，交由 serialize_state 序列化。 + epoch: 当前 epoch 序号。 + step_completed: 本 epoch 内已完成的 step 数。 + phase: 续训阶段标识（如 "in_epoch"）。 + global_step: 全局 step 序号。 + total_steps: 全局总 step 数。 + version_snapshot: skills/prompts 版本快照。 + epoch_batches: 本 epoch 的 batch 划分（question_id 列表的列表）。 + config: 训练配置对象，用于计算 config_fingerprint。 + + 关键实现细节: + 先写 checkpoint.json.tmp 再 os.replace，保证 checkpoint 不被写一半的中断破坏。 + """ + payload = { + "schema_version": CHECKPOINT_SCHEMA_VERSION, + "progress": { + "epoch": epoch, + "step_completed": step_completed, + "phase": phase, + "global_step": global_step, + "total_steps": total_steps, + }, + "version_snapshot": version_snapshot, + "epoch_batches": epoch_batches, + "config_fingerprint": compute_fingerprint(config), + "state": serialize_state(state), + } + path = workspace_dir / "checkpoint.json" + tmp = path.with_name("checkpoint.json.tmp") + tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2)) + os.replace(tmp, path) + + +def load_checkpoint(workspace_dir: Path) -> dict[str, Any] | None: + """读 checkpoint.json，不存在返回 None。 + + 参数: + workspace_dir: workspace 目录。 + + 返回: + checkpoint payload dict；checkpoint.json 不存在时返回 None。 + """ + path = workspace_dir / "checkpoint.json" + if not path.exists(): + return None + return json.loads(path.read_text()) diff --git a/tests/unit/test_harness_checkpoint.py b/tests/unit/test_harness_checkpoint.py new file mode 100644 index 0000000..e5f0b95 --- /dev/null +++ b/tests/unit/test_harness_checkpoint.py @@ -0,0 +1,413 @@ +"""app/harness/checkpoint.py 单元测试。 + +覆盖序列化/反序列化往返、嵌套 dataclass 复活、缺键硬失败、 +配置指纹结构性 vs 决策性判定、原子写与 load 缺失场景。 +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + +from app.harness.checkpoint import ( + CHECKPOINT_SCHEMA_VERSION, + Probation, + check_fingerprint, + compute_fingerprint, + deserialize_state_fields, + load_checkpoint, + serialize_state, + write_checkpoint, +) +from core.evolution.types import ( + CaseSample, + RejectedEdit, + SystemCasePack, + ToolCasePack, +) + +# --------------------------------------------------------------------------- +# fixtures: 模拟 _TrainState 与 RunConfig +# --------------------------------------------------------------------------- + + +def _make_case_sample(**overrides: Any) -> CaseSample: + """构造一个最小可用 CaseSample。""" + defaults: dict[str, Any] = { + "question_id": "q001", + "video_id": "v001", + "task_type": "temporal", + "question": "What happened?", + "options": ["A", "B", "C"], + "answer": "A", + "prediction": "B", + "correct": False, + "error_type": "reasoning", + "selection_reason": "worst", + "metrics": {"acc": 0.5}, + "trace": [{"step": 1, "action": "search"}], + } + defaults.update(overrides) + return CaseSample(**defaults) + + +def _make_rejected_edit(**overrides: Any) -> RejectedEdit: + """构造一个最小可用 RejectedEdit。""" + defaults: dict[str, Any] = { + "target_file": "temporal-reasoning.md", + "target_type": "skill", + "change_summary": "added step", + "delta": -0.05, + "source_version": "v2", + "epoch": 1, + "gate_w": 3, + "gate_l": 5, + "gate_e_value": 0.8, + "gate_delta_shrunk": -0.02, + } + defaults.update(overrides) + return RejectedEdit(**defaults) + + +def _make_system_pack() -> SystemCasePack: + """构造包含嵌套 CaseSample 的 SystemCasePack。""" + return SystemCasePack( + stats={"pattern": "repeat_visit", "count": 3}, + failure_cases=[_make_case_sample(question_id="q010")], + success_cases=[_make_case_sample(question_id="q011", correct=True, error_type=None)], + ) + + +def _make_tool_pack() -> ToolCasePack: + """构造 ToolCasePack。""" + return ToolCasePack( + tool_name="search_subtree", + target_files=["search_subtree_extract.md"], + stats={"completeness": 0.8}, + failure_spans=[{"step": 2, "issue": "missing"}], + success_spans=[{"step": 3, "quality": "good"}], + ) + + +def _make_probation() -> Probation: + """构造包含嵌套 RejectedEdit 的 Probation。""" + return Probation( + task_type="temporal", + anchor_skills_version="v1", + target_file="temporal-reasoning.md", + correctness_snapshot={"q001": True, "q002": False}, + opened_step=5, + pending_edits=[_make_rejected_edit()], + ) + + +@dataclass +class _FakeState: + """模拟 _TrainState 全部可持久化字段。""" + + correctness: dict[str, bool] + eval_prev_acc: float + eval_prev_run_id: str + baseline_skills_version: str + baseline_prompts_version: str + steps_since_best_improved: int + epoch_start_skills: str + changed_task_types_this_epoch: set[str] + rejected_buffer: dict[str, list[RejectedEdit]] + system_packs: list[SystemCasePack] + tool_packs: list[ToolCasePack] + probations: dict[str, Probation] + gate_cooldown: dict[str, int] + gate_epoch_observed: dict[str, bool] + + +def _make_state() -> _FakeState: + """构造一个填满全部字段的 _FakeState。""" + return _FakeState( + correctness={"q001": True, "q002": False}, + eval_prev_acc=0.65, + eval_prev_run_id="run-abc", + baseline_skills_version="v1", + baseline_prompts_version="v1", + steps_since_best_improved=2, + epoch_start_skills="v1", + changed_task_types_this_epoch={"temporal", "causal"}, + rejected_buffer={"temporal": [_make_rejected_edit()]}, + system_packs=[_make_system_pack()], + tool_packs=[_make_tool_pack()], + probations={"temporal": _make_probation()}, + gate_cooldown={"temporal": 3}, + gate_epoch_observed={"temporal": True}, + ) + + +@dataclass(frozen=True) +class _FakeConfig: + """模拟 RunConfig 的指纹相关字段。""" + + batch_size: int = 8 + min_class_per_batch: int = 2 + epochs: int = 5 + diag_size: int = 30 + val_size: int = 50 + batch_correct_ratio: float = 0.5 + edit_budget_start: int = 6 + edit_budget_end: int = 3 + early_stop_patience: int = 3 + use_slow_momentum: bool = True + skill_update_mode: str = "patch" + appendix_consolidate_threshold: int = 10 + momentum_samples: int = 20 + gate_e_confirm: float = 20.0 + gate_e_provisional: float = 6.0 + gate_w_net_min: int = 2 + gate_delta_min: float = 0.02 + gate_lambda_dir: float = -3.0 + gate_e_rollback: float = 10.0 + gate_block: int = 4 + gate_n_max: int = 40 + gate_p_low: float = 0.1 + gate_p_high: float = 0.9 + gate_probe_quota: float = 0.2 + gate_gamma_decay: float = 0.9 + gate_cooldown_steps: int = 2 + gate_guard_err: float = 0.3 + + +# ========================================================================= +# 测试用例 +# ========================================================================= + + +class TestSerializeDeserializeRoundtrip: + """序列化 → JSON 往返 → 反序列化应完全复原。""" + + def test_serialize_deserialize_roundtrip(self) -> None: + state = _make_state() + serialized = serialize_state(state) + # JSON 往返（模拟实际落盘-读回） + json_str = json.dumps(serialized, ensure_ascii=False) + loaded = json.loads(json_str) + restored = deserialize_state_fields(loaded) + + assert restored["correctness"] == state.correctness + assert restored["eval_prev_acc"] == state.eval_prev_acc + assert restored["eval_prev_run_id"] == state.eval_prev_run_id + assert restored["baseline_skills_version"] == state.baseline_skills_version + assert restored["baseline_prompts_version"] == state.baseline_prompts_version + assert restored["steps_since_best_improved"] == state.steps_since_best_improved + assert restored["epoch_start_skills"] == state.epoch_start_skills + assert restored["changed_task_types_this_epoch"] == state.changed_task_types_this_epoch + assert restored["gate_cooldown"] == state.gate_cooldown + assert restored["gate_epoch_observed"] == state.gate_epoch_observed + + +class TestSerializeSetToSortedList: + """set 字段序列化为排序列表。""" + + def test_serialize_set_to_sorted_list(self) -> None: + state = _make_state() + state.changed_task_types_this_epoch = {"z_type", "a_type", "m_type"} + serialized = serialize_state(state) + assert serialized["changed_task_types_this_epoch"] == ["a_type", "m_type", "z_type"] + + +class TestDeserializeNestedSystemPack: + """SystemCasePack 内嵌套的 CaseSample 正确复活。""" + + def test_deserialize_nested_system_pack(self) -> None: + state = _make_state() + serialized = serialize_state(state) + json_str = json.dumps(serialized, ensure_ascii=False) + loaded = json.loads(json_str) + restored = deserialize_state_fields(loaded) + + packs = restored["system_packs"] + assert len(packs) == 1 + pack = packs[0] + assert isinstance(pack, SystemCasePack) + assert len(pack.failure_cases) == 1 + assert isinstance(pack.failure_cases[0], CaseSample) + assert pack.failure_cases[0].question_id == "q010" + assert len(pack.success_cases) == 1 + assert isinstance(pack.success_cases[0], CaseSample) + assert pack.success_cases[0].question_id == "q011" + + +class TestDeserializeNestedProbation: + """Probation 内嵌套的 RejectedEdit 正确复活。""" + + def test_deserialize_nested_probation(self) -> None: + state = _make_state() + serialized = serialize_state(state) + json_str = json.dumps(serialized, ensure_ascii=False) + loaded = json.loads(json_str) + restored = deserialize_state_fields(loaded) + + probations = restored["probations"] + assert "temporal" in probations + prob = probations["temporal"] + assert isinstance(prob, Probation) + assert prob.task_type == "temporal" + assert prob.anchor_skills_version == "v1" + assert prob.correctness_snapshot == {"q001": True, "q002": False} + assert len(prob.pending_edits) == 1 + edit = prob.pending_edits[0] + assert isinstance(edit, RejectedEdit) + assert edit.target_file == "temporal-reasoning.md" + assert edit.delta == -0.05 + + +class TestDeserializeMissingKeyRaises: + """缺键即 checkpoint 损坏，应硬失败。""" + + def test_deserialize_missing_key_raises(self) -> None: + state = _make_state() + serialized = serialize_state(state) + del serialized["gate_epoch_observed"] + with pytest.raises(KeyError): + deserialize_state_fields(serialized) + + +class TestFingerprintStructuralVsDecision: + """compute_fingerprint 包含全部结构性 + 决策性键。""" + + def test_fingerprint_structural_vs_decision(self) -> None: + config = _FakeConfig() + fp = compute_fingerprint(config) + + structural = { + "batch_size", + "min_class_per_batch", + "epochs", + "diag_size", + "val_size", + "batch_correct_ratio", + } + decision = { + "edit_budget_start", + "edit_budget_end", + "early_stop_patience", + "use_slow_momentum", + "skill_update_mode", + "appendix_consolidate_threshold", + "momentum_samples", + "gate_e_confirm", + "gate_e_provisional", + "gate_w_net_min", + "gate_delta_min", + "gate_lambda_dir", + "gate_e_rollback", + "gate_block", + "gate_n_max", + "gate_p_low", + "gate_p_high", + "gate_probe_quota", + "gate_gamma_decay", + "gate_cooldown_steps", + "gate_guard_err", + } + assert structural | decision == set(fp.keys()) + assert fp["batch_size"] == 8 + assert fp["gate_e_confirm"] == 20.0 + + +class TestCheckFingerprintStructuralReject: + """结构性键变化应出现在 structural 列表中。""" + + def test_check_fingerprint_structural_reject(self) -> None: + config_old = _FakeConfig() + saved = compute_fingerprint(config_old) + # 修改结构性参数 + config_new = _FakeConfig(batch_size=16, epochs=10) + structural, decision = check_fingerprint(saved, config_new) + assert "batch_size" in structural + assert "epochs" in structural + assert len(decision) == 0 + + +class TestCheckFingerprintDecisionWarn: + """决策性键变化应出现在 decision 列表中，structural 为空。""" + + def test_check_fingerprint_decision_warn(self) -> None: + config_old = _FakeConfig() + saved = compute_fingerprint(config_old) + config_new = _FakeConfig(early_stop_patience=10, gate_e_confirm=50.0) + structural, decision = check_fingerprint(saved, config_new) + assert len(structural) == 0 + assert "early_stop_patience" in decision + assert "gate_e_confirm" in decision + + +class TestWriteCheckpointAtomic: + """原子写：先 .tmp 再 os.replace。""" + + def test_write_checkpoint_atomic(self, tmp_path: Path) -> None: + state = _make_state() + config = _FakeConfig() + write_checkpoint( + tmp_path, + state=state, + epoch=2, + step_completed=5, + phase="in_epoch", + global_step=15, + total_steps=40, + version_snapshot={"skills": "v3", "prompts": "v2"}, + epoch_batches=[["q001", "q002"], ["q003"]], + config=config, + ) + ckpt_path = tmp_path / "checkpoint.json" + assert ckpt_path.exists() + # .tmp 应已被 os.replace 移除 + assert not (tmp_path / "checkpoint.json.tmp").exists() + + payload = json.loads(ckpt_path.read_text()) + assert payload["schema_version"] == CHECKPOINT_SCHEMA_VERSION + assert payload["progress"]["epoch"] == 2 + assert payload["progress"]["step_completed"] == 5 + assert payload["progress"]["phase"] == "in_epoch" + assert payload["progress"]["global_step"] == 15 + assert payload["progress"]["total_steps"] == 40 + assert payload["version_snapshot"] == {"skills": "v3", "prompts": "v2"} + assert payload["epoch_batches"] == [["q001", "q002"], ["q003"]] + assert "config_fingerprint" in payload + assert "state" in payload + + +class TestLoadCheckpointMissing: + """checkpoint.json 不存在时返回 None。""" + + def test_load_checkpoint_missing(self, tmp_path: Path) -> None: + result = load_checkpoint(tmp_path) + assert result is None + + def test_load_checkpoint_exists(self, tmp_path: Path) -> None: + """checkpoint.json 存在时正确读回。""" + state = _make_state() + config = _FakeConfig() + write_checkpoint( + tmp_path, + state=state, + epoch=1, + step_completed=3, + phase="post_evolve", + global_step=8, + total_steps=20, + version_snapshot={"skills": "v2", "prompts": "v1"}, + epoch_batches=[["q001"]], + config=config, + ) + loaded = load_checkpoint(tmp_path) + assert loaded is not None + assert loaded["schema_version"] == CHECKPOINT_SCHEMA_VERSION + assert loaded["progress"]["epoch"] == 1 + # 完整往返测试：state 可 deserialize + restored = deserialize_state_fields(loaded["state"]) + assert restored["eval_prev_acc"] == 0.65 From 7bc6fc752cdd1a642df1213ae371ef40fe17f63c Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 13:20:43 -0400 Subject: [PATCH 16/19] =?UTF-8?q?feat(harness):=20validate.py=20=E2=80=94?= =?UTF-8?q?=20async=20=E5=9D=97=E5=BA=8F=E8=B4=AF=E9=AA=8C=E8=AF=81?= =?UTF-8?q?=E7=BC=96=E6=8E=92=20+=20Probation=20=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E5=AE=9A=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/harness/checkpoint.py | 26 +- app/harness/validate.py | 665 ++++++++++++++++++++++++++++ tests/unit/test_harness_validate.py | 554 +++++++++++++++++++++++ 3 files changed, 1220 insertions(+), 25 deletions(-) create mode 100644 app/harness/validate.py create mode 100644 tests/unit/test_harness_validate.py diff --git a/app/harness/checkpoint.py b/app/harness/checkpoint.py index 04199e1..e389546 100644 --- a/app/harness/checkpoint.py +++ b/app/harness/checkpoint.py @@ -30,31 +30,7 @@ if TYPE_CHECKING: CHECKPOINT_SCHEMA_VERSION = 1 -# --------------------------------------------------------------------------- -# Probation 类型（Task 10 validate.py 尚未就绪，暂定义在此供 checkpoint 使用） -# Task 10 完成后迁移至 app/harness/validate.py 并改为 re-export。 -# --------------------------------------------------------------------------- - - -@dataclass -class Probation: - """一个题型的在途试用账本（每题型至多一个）。 - - 属性: - task_type: 题型。 - anchor_skills_version: 锚版本名（最近一个 CONFIRMED 的 skills 版本）。 - target_file: 该题型解析后的 skill 文件名。 - correctness_snapshot: 开账时该题型 val 题的对错快照（回滚时恢复）。 - opened_step: 开账时的 global_step（观测用）。 - pending_edits: 试用链上全部候选 edit 的黑名单素材（回滚时整链入黑名单）。 - """ - - task_type: str - anchor_skills_version: str - target_file: str - correctness_snapshot: dict[str, bool] - opened_step: int - pending_edits: list[RejectedEdit] = field(default_factory=list) +from app.harness.validate import Probation # noqa: E402 # --------------------------------------------------------------------------- diff --git a/app/harness/validate.py b/app/harness/validate.py new file mode 100644 index 0000000..d4878b2 --- /dev/null +++ b/app/harness/validate.py @@ -0,0 +1,665 @@ +"""async 块序贯验证编排 — CE-Gate 局部验证的唯一独立子编排器。 + +从 TRM4 core/harness/validate.py (626 行) 迁移，重大重构： +- 同步 → async（run_inference 注入为 async callable） +- _classify_quadrants → core.evolution.classify_quadrants 纯函数 +- 配对逻辑 → 复用 core.evolution.pair_block + 本地证据行组装 +- _load_run_rows / _candidate_correctness_from_db → 共享 log.query() +- materialize_candidate_skill 保持同步（纯文件操作） + +基线与候选在同一阶梯前缀上逐块配对，只数翻转（基线错→候选对 = W， +基线对→候选错 = L），每块结束调 gate_decision 做四出口判定。 +基线侧逐题对错走 BaselineCache 内容寻址缓存，miss 才新鲜跑。 +判定逻辑全部在 core/evolution/gate，本模块只负责推理编排与证据收集。 +""" + +from __future__ import annotations + +import json +import shutil +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable + +from loguru import logger + +from app.harness.gate_ladder import BaselineCache, skill_hash +from core.evolution import ( + GateParams, + GateVerdict, + RejectedEdit, + classify_quadrants, + gate_decision, + pair_block, +) + +if TYPE_CHECKING: + from app.harness.inference import InferenceResult + from app.harness.log import HarnessLog + from core.types import GeneratedQuestion + + +# gate_decision 的 decision → ValidationOutcome.stop_reason 映射 +_STOP_REASON_BY_DECISION: dict[str, str] = { + "accept_confirmed": "confirmed", + "reject_directional": "directional", + "reject_futility": "futility", + "accept_provisional": "provisional", + "reject_inertia": "inertia", +} + + +# --------------------------------------------------------------------------- +# 注入协议 +# --------------------------------------------------------------------------- + + +@runtime_checkable +class RunInferenceFn(Protocol): + """注入的推理函数协议。 + + 调用方（runner）负责绑定 llm、tool_dispatch_fn、prompt_builder、 + log、concurrency、max_steps、skill_mode 等共享依赖。 + validate 侧只传 questions、run_id、skills_dir 三个逐块变化的参数。 + """ + + async def __call__( + self, + questions: list[GeneratedQuestion], + *, + run_id: str, + skills_dir: Path, + ) -> InferenceResult: ... + + +# --------------------------------------------------------------------------- +# 数据类型 +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class InferenceRunConfig: + """一次推理运行的配置三元组，把"如何跑推理"内聚成一组。 + + 字段: + concurrency: 推理并发度。 + max_steps: 单题最大推理步数。 + skill_mode: 推理 skill 模式（"auto" / "manual" / "none"）。 + """ + + concurrency: int + max_steps: int + skill_mode: str + + +@dataclass +class ValidationOutcome: + """CE-Gate 局部验证结果：三态动作 + e-process 证据 + 已观测题逐题对错。 + + correctness 二轨语义：candidate_correctness 只含已观测题（早停后是 + 阶梯前缀子集）；accept 时由 runner 按题粒度增量合并进 state.correctness。 + """ + + action: str # accept_confirmed | accept_provisional | reject + accepted: bool + stop_reason: str # confirmed | directional | futility | provisional | inertia + e_value: float + w: int + l: int # noqa: E741 + n_used: int + delta_hat: float + delta_shrunk: float + baseline_acc: float # 已观测题上的基线准确率（观测口径） + candidate_acc: float # 已观测题上的候选准确率（观测口径） + improvements: list[str] = field(default_factory=list) + regressions: list[str] = field(default_factory=list) + persistent_fails: list[str] = field(default_factory=list) + stable_successes: list[str] = field(default_factory=list) + candidate_correctness: dict[str, bool] = field(default_factory=dict) + evidence_rows: list[dict] = field(default_factory=list) # gate_evidence 逐题行，runner 落库 + + +@dataclass +class Probation: + """一个题型的在途试用账本（每题型至多一个）。 + + 字段: + task_type: 题型。 + anchor_skills_version: 锚版本名（最近一个 CONFIRMED 的 skills 版本）—— + 回滚时恢复该版本中本题型 skill 文件的内容。 + target_file: 该题型解析后的 skill 文件名。 + correctness_snapshot: 开账时该题型 val 题的对错快照（回滚时恢复）。 + opened_step: 开账时的 global_step（观测用）。 + pending_edits: 试用链上全部候选 edit 的黑名单素材（回滚时整链入黑名单）。 + """ + + task_type: str + anchor_skills_version: str + target_file: str + correctness_snapshot: dict[str, bool] + opened_step: int + pending_edits: list[RejectedEdit] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# 同步辅助函数 +# --------------------------------------------------------------------------- + + +def materialize_candidate_skill( + workspace_dir: Path, + base_skills_version: str, + target_file: str, + content: str, +) -> Path: + """将候选 skill 正文物化为 workspace 专用临时目录下唯一命名的候选 skills 目录。 + + 复制基线 skills 目录到 .cand_tmp/ 下的唯一命名临时目录，然后覆写 target_file。 + 构建失败时尽力清理已建临时目录再重抛原始异常。 + + 参数: + workspace_dir: Workspace 根目录。基线 skills 从 workspace_dir/skills/ + 复制，临时候选落 workspace_dir/.cand_tmp/。 + base_skills_version: 基线 skills 版本名。 + target_file: 被替换的 skill 文件名。 + content: 候选 skill 文件全文。 + + 返回: + 新建的临时候选目录绝对路径。 + + 契约: + 构建失败（OSError）时尽力清理已建临时目录再重抛原始异常； + 清理本身失败记 warning。 + """ + cand_tmp_root = workspace_dir / ".cand_tmp" + cand_tmp_root.mkdir(parents=True, exist_ok=True) + cand_dir = Path(tempfile.mkdtemp(prefix=f"{base_skills_version}_cand_", dir=cand_tmp_root)) + try: + base_dir = workspace_dir / "skills" / base_skills_version + shutil.copytree(base_dir, cand_dir, dirs_exist_ok=True) + (cand_dir / target_file).write_text(content, encoding="utf-8") + except OSError: + try: + shutil.rmtree(cand_dir) + except OSError as cleanup_err: + logger.warning("候选物化失败后清理临时目录也失败 {}: {}", cand_dir, cleanup_err) + raise + return cand_dir + + +def _load_run_rows( + log: HarnessLog, + run_id: str, +) -> dict[str, dict[str, Any]]: + """读取单个 run 的逐题预测行并规范化轨迹字段。 + + 从 predictions 表读取指定 run 的题目级记录，补充 _correct + 与规范化后的 steps 字段。保持同步（log.query）——仅在推理完成后调用。 + + 参数: + log: HarnessLog 共享实例（用 query 方法做只读 SELECT）。 + run_id: 待读取的预测 run_id。 + + 返回: + 以 question_id 为键的行字典。每行至少包含 prediction、answer、 + _correct、steps 等字段。 + """ + rows = log.query( + "SELECT question_id, prediction, answer, steps_json FROM predictions WHERE run_id=?", + (run_id,), + ) + normalized: dict[str, dict[str, Any]] = {} + for row in rows: + raw_steps = row.get("steps_json") + parsed_steps: Any = raw_steps + if isinstance(raw_steps, str): + try: + parsed_steps = json.loads(raw_steps) + except json.JSONDecodeError: + parsed_steps = [] + steps = parsed_steps if isinstance(parsed_steps, list) else [] + normalized[row["question_id"]] = { + **row, + "_correct": row.get("prediction") == row.get("answer"), + "steps": steps, + } + return normalized + + +def _candidate_correctness_from_db( + log: HarnessLog, + run_id: str, + chunk: list[GeneratedQuestion], +) -> dict[str, bool]: + """从 db 读取候选/基线 run 在指定题目上的逐题对错。 + + 参数: + log: HarnessLog 共享实例。 + run_id: 推理 run_id。 + chunk: 题目列表。 + + 返回: + question_id -> 是否答对的映射。缺行的题目记为 False。 + """ + rows = _load_run_rows(log, run_id) + return {q.question_id: rows.get(q.question_id, {}).get("_correct", False) for q in chunk} + + +# --------------------------------------------------------------------------- +# 块级 async 函数 +# --------------------------------------------------------------------------- + + +async def _resolve_baseline_block( + chunk: list[GeneratedQuestion], + task_type: str, + s_hash: str, + prompts_version: str, + baseline_cache: BaselineCache, + base_skills_dir: Path, + run_inference: RunInferenceFn, + log: HarnessLog, + run_id: str, +) -> tuple[dict[str, bool], int, int]: + """基线侧处理一个块：缓存优先，miss 的题新鲜跑基线版本并回写缓存。 + + 参数: + chunk: 当前块的题目列表。 + task_type: 当前验证题型（缓存键成分）。 + s_hash: 基线侧生效 skill 的内容哈希（缓存键成分）。 + prompts_version: 当前 prompts 版本（缓存键成分）。 + baseline_cache: 基线侧逐题对错缓存。 + base_skills_dir: 基线 skills 版本目录。 + run_inference: 注入的 async 推理函数。 + log: HarnessLog 共享实例（推理后读预测）。 + run_id: 本块基线 run_id。 + + 返回: + (b_map, errors_inc, denom_inc)：块内 question_id -> 基线对错、 + 本块新增的 INFRA error 计数与推理题次分母增量（全命中时为 0, 0）。 + """ + misses = [ + q + for q in chunk + if baseline_cache.get(task_type, s_hash, prompts_version, q.question_id) is None + ] + errors_inc = 0 + denom_inc = 0 + if misses: + r_b = await run_inference(misses, run_id=run_id, skills_dir=base_skills_dir) + errors_inc = r_b.stop_reason_counts.get("error", 0) + denom_inc = r_b.total + fresh = _candidate_correctness_from_db(log, r_b.run_id, misses) + for qid, correct in fresh.items(): + baseline_cache.put(task_type, s_hash, prompts_version, qid, correct) + + b_map: dict[str, bool] = {} + for q in chunk: + val = baseline_cache.get(task_type, s_hash, prompts_version, q.question_id) + assert val is not None, f"基线缓存补齐后仍有 miss: {q.question_id} run_id={run_id}" + b_map[q.question_id] = val + return b_map, errors_inc, denom_inc + + +async def _run_candidate_block( + chunk: list[GeneratedQuestion], + cand_dir: Path, + run_inference: RunInferenceFn, + log: HarnessLog, + run_id: str, +) -> tuple[dict[str, bool], int, int]: + """候选侧处理一个块：全块新鲜跑候选版本并从 db 读逐题对错。 + + 参数: + chunk: 当前块的题目列表。 + cand_dir: 已物化的候选 skills 目录。 + run_inference: 注入的 async 推理函数。 + log: HarnessLog 共享实例（推理后读预测）。 + run_id: 本块候选 run_id。 + + 返回: + (c_map, errors_inc, denom_inc)。 + """ + r_c = await run_inference(chunk, run_id=run_id, skills_dir=cand_dir) + c_map = _candidate_correctness_from_db(log, r_c.run_id, chunk) + return c_map, r_c.stop_reason_counts.get("error", 0), r_c.total + + +def _build_evidence_rows( + chunk: list[GeneratedQuestion], + b_map: dict[str, bool], + c_map: dict[str, bool], + task_type: str, + block_idx: int, +) -> list[dict]: + """组装一个块的 gate_evidence 逐题证据行。 + + e_value 留 None 待块判定后回填，stop_reason 留空串待终态回填。 + + 参数: + chunk: 当前块的题目列表。 + b_map: 块内 question_id -> 基线对错。 + c_map: 块内 question_id -> 候选对错。 + task_type: 当前验证题型。 + block_idx: 当前块序号。 + + 返回: + 逐题证据行列表。 + """ + return [ + { + "question_id": q.question_id, + "task_type": task_type, + "block_idx": block_idx, + "baseline_correct": b_map[q.question_id], + "candidate_correct": c_map[q.question_id], + "e_value": None, + "stop_reason": "", + } + for q in chunk + ] + + +# --------------------------------------------------------------------------- +# INFRA 护栏 +# --------------------------------------------------------------------------- + + +def _check_infra_guard(errors: int, infra_denom: int, gate_guard_err: float) -> None: + """跨块累计 INFRA 错误率护栏：分母 >=10 且超阈值时 raise。 + + 参数: + errors: 两侧累计 error 计数。 + infra_denom: 两侧累计推理题次分母。 + gate_guard_err: 错误率阈值。 + + 异常: + RuntimeError: 错误率超阈值。 + """ + if infra_denom >= 10 and errors / infra_denom > gate_guard_err: + raise RuntimeError(f"gate 推理累计错误率过高 {errors / infra_denom:.0%}，中止本轮") + + +# --------------------------------------------------------------------------- +# 终态组装 +# --------------------------------------------------------------------------- + + +def _finalize_outcome( + verdict: GateVerdict, + w: int, + l: int, # noqa: E741 + n_used: int, + n_plan: int, + base_obs: dict[str, bool], + cand_obs: dict[str, bool], + evidence_rows: list[dict], + task_type: str, +) -> ValidationOutcome: + """将块循环终态判定组装为 ValidationOutcome。 + + 参数: + verdict: 最后一块的 gate 判定结果。 + w: 累计 W（基线错→候选对翻转）。 + l: 累计 L（基线对→候选错翻转）。 + n_used: 已消费的阶梯题数。 + n_plan: 阶梯总题数。 + base_obs: 累计基线已观测对错。 + cand_obs: 累计候选已观测对错。 + evidence_rows: 逐题证据行。 + task_type: 验证题型（日志用）。 + + 返回: + ValidationOutcome。 + """ + action = { + "accept_confirmed": "accept_confirmed", + "accept_provisional": "accept_provisional", + }.get(verdict.decision, "reject") + stop_reason = _STOP_REASON_BY_DECISION[verdict.decision] + # 只有终态题的证据行才携带 stop_reason + evidence_rows[-1]["stop_reason"] = stop_reason + + quadrants = classify_quadrants({qid: (base_obs[qid], cand_obs[qid]) for qid in base_obs}) + baseline_acc = sum(base_obs.values()) / len(base_obs) + candidate_acc = sum(cand_obs.values()) / len(cand_obs) + accepted = action != "reject" + + logger.info( + "gate 局部验证[{}]: 基线{:.1%} → 候选{:.1%} (W={} L={} E={:.2f} n={}/{}) {}", + task_type, + baseline_acc, + candidate_acc, + w, + l, + verdict.e_value, + n_used, + n_plan, + "接受" if accepted else "回滚", + ) + + return ValidationOutcome( + action=action, + accepted=accepted, + stop_reason=stop_reason, + e_value=verdict.e_value, + w=w, + l=l, + n_used=n_used, + delta_hat=verdict.delta_hat, + delta_shrunk=verdict.delta_shrunk, + baseline_acc=baseline_acc, + candidate_acc=candidate_acc, + improvements=quadrants.improvements, + regressions=quadrants.regressions, + persistent_fails=quadrants.persistent_fails, + stable_successes=quadrants.stable_successes, + candidate_correctness=cand_obs, + evidence_rows=evidence_rows, + ) + + +# --------------------------------------------------------------------------- +# 主编排 +# --------------------------------------------------------------------------- + + +async def _run_local_validation( + workspace_dir: Path, + cand_dir: Path, + base_skills_version: str, + task_type: str, + base_skill_content: str, + plan: list[GeneratedQuestion], + gate_params: GateParams, + gate_block: int, + gate_guard_err: float, + baseline_cache: BaselineCache, + prompts_version: str, + run_inference: RunInferenceFn, + log: HarnessLog, + gate_run_prefix: str, +) -> ValidationOutcome: + """块序贯循环主体：逐块基线（缓存优先）/候选配对推理，块间 e-process 判定。 + + 按 gate_block 切阶梯前缀，每块先补齐基线侧缓存 miss（新鲜跑基线版本 + 并逐题写 BaselineCache），再全块跑候选，配对累计 W/L 后调 gate_decision； + 非 continue 即早停。题尽时最后一块的判定即终态（n_remaining=0 走 + provisional/inertia 分支），无循环外补判。 + + 参数: + workspace_dir: Workspace 根目录。 + cand_dir: 已物化的候选 skills 目录。 + base_skills_version: 基线 skills 版本名。 + task_type: 当前验证题型。 + base_skill_content: 基线侧生效 skill 全文（skill_hash 作缓存键成分）。 + plan: 已截断到 gate_n_max 的阶梯出题序。 + gate_params: e-process 判据阈值组。 + gate_block: 块大小。 + gate_guard_err: 跨块累计 INFRA 错误率护栏（分母 >=10 才触发）。 + baseline_cache: 基线侧逐题对错缓存。 + prompts_version: 当前 prompts 版本（缓存键成分）。 + run_inference: 注入的 async 推理函数。 + log: HarnessLog 共享实例。 + gate_run_prefix: 块 run_id 前缀（含 "_gate_" 标记）。 + + 返回: + ValidationOutcome。 + + 关键实现: + INFRA 护栏跨块累计基线+候选两侧的 error 计数，分母（总推理题次）>=10 + 且错误率超 gate_guard_err 时直接 raise，避免坏批次污染判定。 + """ + w = 0 + l = 0 # noqa: E741 + n_used = 0 + errors = 0 + infra_denom = 0 + evidence_rows: list[dict] = [] + base_obs: dict[str, bool] = {} + cand_obs: dict[str, bool] = {} + s_hash = skill_hash(base_skill_content) + base_skills_dir = workspace_dir / "skills" / base_skills_version + chunks = [plan[i : i + gate_block] for i in range(0, len(plan), gate_block)] + verdict: GateVerdict | None = None + + for block_idx, chunk in enumerate(chunks): + # Phase 1: 基线侧（缓存优先，miss 新鲜跑）+ 候选侧（全块新鲜跑） + b_map, err_b, den_b = await _resolve_baseline_block( + chunk=chunk, + task_type=task_type, + s_hash=s_hash, + prompts_version=prompts_version, + baseline_cache=baseline_cache, + base_skills_dir=base_skills_dir, + run_inference=run_inference, + log=log, + run_id=f"{gate_run_prefix}_b{block_idx}_base", + ) + c_map, err_c, den_c = await _run_candidate_block( + chunk=chunk, + cand_dir=cand_dir, + run_inference=run_inference, + log=log, + run_id=f"{gate_run_prefix}_b{block_idx}_cand", + ) + + # Phase 2: INFRA 护栏（跨块累计，分母 >=10 才触发） + errors += err_b + err_c + infra_denom += den_b + den_c + _check_infra_guard(errors, infra_denom, gate_guard_err) + + # Phase 3: 配对 + 证据行 + 块间判定 + qids = [q.question_id for q in chunk] + pair_result = pair_block(b_map, c_map, qids) + for qid, (b, c) in pair_result.observed.items(): + base_obs[qid] = b + cand_obs[qid] = c + + block_rows = _build_evidence_rows(chunk, b_map, c_map, task_type, block_idx) + + w += pair_result.w + l += pair_result.l # noqa: E741 + n_used += len(chunk) + verdict = gate_decision(w, l, n_used, len(plan) - n_used, params=gate_params) + + for row in block_rows: + row["e_value"] = verdict.e_value + evidence_rows.extend(block_rows) + + if verdict.decision != "continue": + break + + # 最后一块判定即终态（n_remaining=0 → provisional/inertia） + assert verdict is not None, "空阶梯应已在 validate_skill_local 入口拒绝" + return _finalize_outcome( + verdict=verdict, + w=w, + l=l, + n_used=n_used, + n_plan=len(plan), + base_obs=base_obs, + cand_obs=cand_obs, + evidence_rows=evidence_rows, + task_type=task_type, + ) + + +async def validate_skill_local( + workspace_dir: Path, + base_skills_version: str, + task_type: str, + target_file: str, + candidate_content: str, + base_skill_content: str, + ladder_items: list[GeneratedQuestion], + gate_params: GateParams, + gate_block: int, + gate_n_max: int, + gate_guard_err: float, + baseline_cache: BaselineCache, + prompts_version: str, + run_inference: RunInferenceFn, + log: HarnessLog, + gate_run_prefix: str, +) -> ValidationOutcome: + """块序贯配对验证：阶梯出题，基线/候选逐块配对，e-process 四出口早停。 + + 参数: + workspace_dir: workspace 根目录。 + base_skills_version: 基线 skills 版本名（候选物化复制源）。 + task_type: 待验证题型。 + target_file: fallback 解析后该题型的真实生效 skill 文件名 + （record.target_file，可能是共享 default-strategy.md）； + 候选物化写此文件，与 accept 路径同源。 + candidate_content: 候选 skill 全文。 + base_skill_content: 基线侧该题型解析后生效 skill 文件全文 + （skill_hash(base_skill_content) 作 BaselineCache 键成分）。 + ladder_items: 阶梯序题目列表（已排除本 step 案例包题）。 + gate_params: e-process 判据阈值组。 + gate_block: 块大小。 + gate_n_max: 单 gate 题数上限。 + gate_guard_err: 跨块累计 INFRA 错误率护栏（分母 >=10 才触发）。 + baseline_cache: 基线侧逐题对错缓存。 + prompts_version: 当前 prompts 版本（缓存键成分）。 + run_inference: 注入的 async 推理函数（RunInferenceFn 协议）。 + log: HarnessLog 共享实例（供 DB 回读逐题对错）。 + gate_run_prefix: gate 内推理 run_id 前缀，必须含 "_gate_" + （防泄露过滤靠它识别）。块 run_id = f"{prefix}_b{block_idx}_{arm}"。 + + 返回: + ValidationOutcome。逐题证据记入 outcome.evidence_rows 随结果返回， + gate_evidence 落库由调用方（runner）负责。 + """ + if "_gate_" not in gate_run_prefix: + raise ValueError(f"gate_run_prefix 必须含 '_gate_'（防泄露过滤依赖）: {gate_run_prefix!r}") + if not ladder_items: + raise ValueError(f"task_type={task_type} 阶梯为空，无法验证") + + plan = ladder_items[:gate_n_max] + cand_dir = materialize_candidate_skill( + workspace_dir, base_skills_version, target_file, candidate_content + ) + try: + return await _run_local_validation( + workspace_dir=workspace_dir, + cand_dir=cand_dir, + base_skills_version=base_skills_version, + task_type=task_type, + base_skill_content=base_skill_content, + plan=plan, + gate_params=gate_params, + gate_block=gate_block, + gate_guard_err=gate_guard_err, + baseline_cache=baseline_cache, + prompts_version=prompts_version, + run_inference=run_inference, + log=log, + gate_run_prefix=gate_run_prefix, + ) + finally: + try: + shutil.rmtree(cand_dir) + except OSError as e: + logger.warning("候选临时目录清理失败 {}: {}", cand_dir, e) diff --git a/tests/unit/test_harness_validate.py b/tests/unit/test_harness_validate.py new file mode 100644 index 0000000..2734408 --- /dev/null +++ b/tests/unit/test_harness_validate.py @@ -0,0 +1,554 @@ +"""tests/unit/test_harness_validate.py — app/harness/validate.py 的单元测试。 + +覆盖：数据类型字段、materialize 物化与清理、async validate_skill_local +（accept/reject/prefix 校验/INFRA 护栏/缓存命中/最后一块终态）。 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest + +from app.harness.gate_ladder import BaselineCache, skill_hash +from app.harness.inference import PREDICTIONS_SCHEMA, InferenceResult +from app.harness.log import HarnessLog +from app.harness.validate import ( + Probation, + ValidationOutcome, + materialize_candidate_skill, + validate_skill_local, +) +from core.evolution import GateParams, RejectedEdit +from core.types import GeneratedQuestion + +if TYPE_CHECKING: + from pathlib import Path + +# --------------------------------------------------------------------------- +# 辅助工具 +# --------------------------------------------------------------------------- + +_DEFAULT_GATE_PARAMS = GateParams( + e_confirm=20.0, + e_provisional=3.0, + w_net_min=2, + delta_min=0.05, + lambda_dir=-2.0, + e_rollback=10.0, +) + + +def _make_questions( + n: int, + task_type: str = "temporal", + prefix: str = "q", +) -> list[GeneratedQuestion]: + """生成 n 个测试用 GeneratedQuestion。""" + return [ + GeneratedQuestion( + question_id=f"{prefix}{i}", + video_id=f"v{i}", + task_type=task_type, + question=f"Question {i}?", + options=("A", "B", "C", "D"), + answer="A", + source_nodes=(), + difficulty="easy", + ) + for i in range(n) + ] + + +def _setup_workspace(tmp_path: Path) -> Path: + """在 tmp_path 下构建最小 workspace 结构。""" + skills_dir = tmp_path / "skills" / "v1" + skills_dir.mkdir(parents=True) + (skills_dir / "temporal.md").write_text("baseline skill content", encoding="utf-8") + return tmp_path + + +def _make_log(workspace: Path, run_id: str = "test_master") -> HarnessLog: + """创建 HarnessLog 并初始化 predictions 表。""" + db_path = workspace / "harness.db" + log = HarnessLog(str(db_path), run_id) + log.create_table("predictions", PREDICTIONS_SCHEMA) + return log + + +def _insert_predictions( + log: HarnessLog, + run_id: str, + correctness: dict[str, bool], + answer: str = "A", +) -> None: + """向 predictions 表插入指定 run_id 的逐题预测记录。 + + 通过在 record 中显式传入 run_id 覆盖 log 的默认 run_id。 + """ + for qid, correct in correctness.items(): + prediction = answer if correct else "Z" + log.insert( + "predictions", + { + "run_id": run_id, + "video_id": "v0", + "question_id": qid, + "task_type": "temporal", + "prediction": prediction, + "answer": answer, + "evidence": "", + "reasoning": "", + "steps_used": 1, + "prompt_tokens": 10, + "completion_tokens": 10, + "stop_reason": "completed", + "steps_json": "[]", + }, + ) + + +def _make_mock_run_inference( + log: HarnessLog, + baseline_correctness: dict[str, bool], + candidate_correctness: dict[str, bool], + error_count: int = 0, +): + """构建 mock RunInferenceFn。 + + 根据 run_id 中的 arm 标记（_base / _cand）决定使用基线或候选对错映射， + 将预测写入 log 的同一 DB，返回 InferenceResult。 + """ + call_log: list[dict[str, Any]] = [] + + async def mock_fn( + questions: list[GeneratedQuestion], + *, + run_id: str, + skills_dir: Path, + ) -> InferenceResult: + is_baseline = run_id.endswith("_base") + correctness = baseline_correctness if is_baseline else candidate_correctness + + call_log.append({"run_id": run_id, "skills_dir": skills_dir, "n": len(questions)}) + per_q = {q.question_id: correctness.get(q.question_id, False) for q in questions} + _insert_predictions(log, run_id, per_q) + + correct = sum(per_q.values()) + total = len(questions) + stop_counts: dict[str, int] = {"completed": total - error_count} + if error_count > 0: + stop_counts["error"] = error_count + return InferenceResult( + run_id=run_id, + accuracy=correct / total if total else 0.0, + total=total, + correct=correct, + per_task_type={}, + steps_mean=1.0, + token_usage={"prompt_tokens": 10, "completion_tokens": 10}, + stop_reason_counts=stop_counts, + ) + + return mock_fn, call_log + + +# =========================================================================== +# 数据类型测试 +# =========================================================================== + + +class TestValidationOutcomeFields: + """ValidationOutcome 数据类型字段完整性测试。""" + + def test_validation_outcome_fields(self) -> None: + """所有字段可构造、默认值合理。""" + outcome = ValidationOutcome( + action="accept_confirmed", + accepted=True, + stop_reason="confirmed", + e_value=25.0, + w=5, + l=1, + n_used=10, + delta_hat=0.4, + delta_shrunk=0.3, + baseline_acc=0.6, + candidate_acc=0.9, + ) + assert outcome.action == "accept_confirmed" + assert outcome.accepted is True + assert outcome.stop_reason == "confirmed" + assert outcome.e_value == 25.0 + assert outcome.w == 5 + assert outcome.l == 1 + assert outcome.n_used == 10 + assert outcome.delta_hat == 0.4 + assert outcome.delta_shrunk == 0.3 + assert outcome.baseline_acc == 0.6 + assert outcome.candidate_acc == 0.9 + assert outcome.improvements == [] + assert outcome.regressions == [] + assert outcome.persistent_fails == [] + assert outcome.stable_successes == [] + assert outcome.candidate_correctness == {} + assert outcome.evidence_rows == [] + + +class TestProbationFields: + """Probation 数据类型字段完整性测试。""" + + def test_probation_fields(self) -> None: + """所有字段可构造、pending_edits 默认空列表。""" + prob = Probation( + task_type="temporal", + anchor_skills_version="v1", + target_file="temporal.md", + correctness_snapshot={"q0": True, "q1": False}, + opened_step=5, + ) + assert prob.task_type == "temporal" + assert prob.anchor_skills_version == "v1" + assert prob.target_file == "temporal.md" + assert prob.correctness_snapshot == {"q0": True, "q1": False} + assert prob.opened_step == 5 + assert prob.pending_edits == [] + + def test_probation_with_pending_edits(self) -> None: + """pending_edits 可附加 RejectedEdit。""" + edit = RejectedEdit( + target_file="temporal.md", + target_type="skill", + change_summary="bad change", + delta=-0.1, + source_version="v2", + epoch=1, + ) + prob = Probation( + task_type="temporal", + anchor_skills_version="v1", + target_file="temporal.md", + correctness_snapshot={}, + opened_step=3, + pending_edits=[edit], + ) + assert len(prob.pending_edits) == 1 + assert prob.pending_edits[0].change_summary == "bad change" + + +# =========================================================================== +# materialize 测试 +# =========================================================================== + + +class TestMaterializeCandidateSkill: + """materialize_candidate_skill 物化与清理测试。""" + + def test_materialize_candidate_skill(self, tmp_path: Path) -> None: + """正常物化：基线目录被复制，target_file 被覆写为候选内容。""" + workspace = _setup_workspace(tmp_path) + cand_dir = materialize_candidate_skill( + workspace, "v1", "temporal.md", "candidate skill content" + ) + try: + assert cand_dir.exists() + assert cand_dir.parent == workspace / ".cand_tmp" + assert (cand_dir / "temporal.md").read_text(encoding="utf-8") == ( + "candidate skill content" + ) + finally: + import shutil + + shutil.rmtree(cand_dir) + + def test_materialize_cleanup_on_failure(self, tmp_path: Path) -> None: + """基线目录不存在时 OSError，临时目录被清理。""" + workspace = tmp_path / "ws" + workspace.mkdir() + # 不创建 skills/v1，copytree 应失败 + with pytest.raises(OSError): + materialize_candidate_skill(workspace, "v1", "temporal.md", "content") + # .cand_tmp 可能存在但内部应被清理 + cand_tmp = workspace / ".cand_tmp" + if cand_tmp.exists(): + remaining = list(cand_tmp.iterdir()) + assert remaining == [], f"临时目录未被清理: {remaining}" + + +# =========================================================================== +# async 验证测试 +# =========================================================================== + + +@pytest.mark.asyncio +async def test_validate_skill_local_accept(tmp_path: Path) -> None: + """候选全对、基线全错 → 高 e 值 → accept_confirmed。""" + workspace = _setup_workspace(tmp_path) + log = _make_log(workspace) + questions = _make_questions(6) + cache = BaselineCache(workspace / "baseline_cache.json") + + # 基线全错，候选全对 → W=6, L=0 → E=18.14 → CONFIRMED（e_confirm=15） + baseline_correct = {f"q{i}": False for i in range(6)} + candidate_correct = {f"q{i}": True for i in range(6)} + mock_fn, call_log = _make_mock_run_inference(log, baseline_correct, candidate_correct) + + # e_confirm=15 使 E=18.14 超过阈值触发 CONFIRMED + accept_params = GateParams( + e_confirm=15.0, + e_provisional=3.0, + w_net_min=2, + delta_min=0.05, + lambda_dir=-2.0, + e_rollback=10.0, + ) + + try: + outcome = await validate_skill_local( + workspace_dir=workspace, + base_skills_version="v1", + task_type="temporal", + target_file="temporal.md", + candidate_content="improved skill", + base_skill_content="baseline skill content", + ladder_items=questions, + gate_params=accept_params, + gate_block=6, + gate_n_max=20, + gate_guard_err=0.5, + baseline_cache=cache, + prompts_version="p1", + run_inference=mock_fn, + log=log, + gate_run_prefix="step1_gate_test", + ) + + assert outcome.accepted is True + assert outcome.action == "accept_confirmed" + assert outcome.stop_reason == "confirmed" + assert outcome.w == 6 + assert outcome.l == 0 + assert outcome.n_used == 6 + assert outcome.candidate_acc == 1.0 + assert outcome.baseline_acc == 0.0 + assert len(outcome.evidence_rows) == 6 + # 终态证据行携带 stop_reason + assert outcome.evidence_rows[-1]["stop_reason"] == "confirmed" + # 候选临时目录应被清理 + cand_tmp = workspace / ".cand_tmp" + if cand_tmp.exists(): + assert list(cand_tmp.iterdir()) == [] + finally: + log.close() + + +@pytest.mark.asyncio +async def test_validate_skill_local_reject(tmp_path: Path) -> None: + """候选全错、基线全对 → L 高 → 方向拒绝。""" + workspace = _setup_workspace(tmp_path) + log = _make_log(workspace) + questions = _make_questions(6) + cache = BaselineCache(workspace / "baseline_cache.json") + + # 基线全对，候选全错 → W=0, L=6 → 方向拒绝 + baseline_correct = {f"q{i}": True for i in range(6)} + candidate_correct = {f"q{i}": False for i in range(6)} + mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct) + + try: + outcome = await validate_skill_local( + workspace_dir=workspace, + base_skills_version="v1", + task_type="temporal", + target_file="temporal.md", + candidate_content="bad skill", + base_skill_content="baseline skill content", + ladder_items=questions, + gate_params=_DEFAULT_GATE_PARAMS, + gate_block=6, + gate_n_max=20, + gate_guard_err=0.5, + baseline_cache=cache, + prompts_version="p1", + run_inference=mock_fn, + log=log, + gate_run_prefix="step1_gate_test", + ) + + assert outcome.accepted is False + assert outcome.action == "reject" + assert outcome.stop_reason == "directional" + assert outcome.w == 0 + assert outcome.l == 6 + finally: + log.close() + + +@pytest.mark.asyncio +async def test_gate_prefix_must_contain_gate(tmp_path: Path) -> None: + """gate_run_prefix 不含 '_gate_' 时抛 ValueError。""" + workspace = _setup_workspace(tmp_path) + log = _make_log(workspace) + questions = _make_questions(4) + cache = BaselineCache(workspace / "baseline_cache.json") + + async def noop_fn(questions, *, run_id, skills_dir): + raise AssertionError("不应被调用") + + try: + with pytest.raises(ValueError, match="_gate_"): + await validate_skill_local( + workspace_dir=workspace, + base_skills_version="v1", + task_type="temporal", + target_file="temporal.md", + candidate_content="content", + base_skill_content="baseline", + ladder_items=questions, + gate_params=_DEFAULT_GATE_PARAMS, + gate_block=4, + gate_n_max=20, + gate_guard_err=0.5, + baseline_cache=cache, + prompts_version="p1", + run_inference=noop_fn, + log=log, + gate_run_prefix="step1_no_marker", + ) + finally: + log.close() + + +@pytest.mark.asyncio +async def test_infra_guard_threshold(tmp_path: Path) -> None: + """推理错误率超阈值时抛 RuntimeError。""" + workspace = _setup_workspace(tmp_path) + log = _make_log(workspace) + # 需要 >=10 题次才触发 INFRA 护栏 + questions = _make_questions(6) + cache = BaselineCache(workspace / "baseline_cache.json") + + baseline_correct = {f"q{i}": False for i in range(6)} + candidate_correct = {f"q{i}": False for i in range(6)} + # 每次 run_inference 报 error_count=5，两侧各 5 → 10/12 > 0.5 + mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct, error_count=5) + + try: + with pytest.raises(RuntimeError, match="错误率过高"): + await validate_skill_local( + workspace_dir=workspace, + base_skills_version="v1", + task_type="temporal", + target_file="temporal.md", + candidate_content="content", + base_skill_content="baseline skill content", + ladder_items=questions, + gate_params=_DEFAULT_GATE_PARAMS, + gate_block=6, + gate_n_max=20, + gate_guard_err=0.5, + baseline_cache=cache, + prompts_version="p1", + run_inference=mock_fn, + log=log, + gate_run_prefix="step1_gate_test", + ) + finally: + log.close() + + +@pytest.mark.asyncio +async def test_baseline_cache_hit(tmp_path: Path) -> None: + """基线缓存全命中时不发起基线侧推理。""" + workspace = _setup_workspace(tmp_path) + log = _make_log(workspace) + questions = _make_questions(4) + cache = BaselineCache(workspace / "baseline_cache.json") + + s_hash = skill_hash("baseline skill content") + # 预填充缓存：全部题目基线全错 + for q in questions: + cache.put("temporal", s_hash, "p1", q.question_id, False) + + # 候选全对 → accept + candidate_correct = {f"q{i}": True for i in range(4)} + baseline_correct = {f"q{i}": False for i in range(4)} + mock_fn, call_log = _make_mock_run_inference(log, baseline_correct, candidate_correct) + + try: + outcome = await validate_skill_local( + workspace_dir=workspace, + base_skills_version="v1", + task_type="temporal", + target_file="temporal.md", + candidate_content="improved skill", + base_skill_content="baseline skill content", + ladder_items=questions, + gate_params=_DEFAULT_GATE_PARAMS, + gate_block=4, + gate_n_max=20, + gate_guard_err=0.5, + baseline_cache=cache, + prompts_version="p1", + run_inference=mock_fn, + log=log, + gate_run_prefix="step1_gate_test", + ) + + # 只有候选侧调用了 run_inference（_cand），基线侧全命中不调用 + base_calls = [c for c in call_log if c["run_id"].endswith("_base")] + cand_calls = [c for c in call_log if c["run_id"].endswith("_cand")] + assert len(base_calls) == 0, "基线缓存全命中不应发起推理" + assert len(cand_calls) == 1 + assert outcome.accepted is True + finally: + log.close() + + +@pytest.mark.asyncio +async def test_last_block_terminal(tmp_path: Path) -> None: + """单块 + n_remaining=0 → 终态判定（provisional 或 inertia），非 continue。""" + workspace = _setup_workspace(tmp_path) + log = _make_log(workspace) + # 4 题，gate_block=4 → 一块走完，n_remaining=0 + questions = _make_questions(4) + cache = BaselineCache(workspace / "baseline_cache.json") + + # 两题翻转（W=2, L=0），但 e_confirm=20 难以达到 → provisional 或 inertia + baseline_correct = {"q0": False, "q1": False, "q2": True, "q3": True} + candidate_correct = {"q0": True, "q1": True, "q2": True, "q3": True} + mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct) + + try: + outcome = await validate_skill_local( + workspace_dir=workspace, + base_skills_version="v1", + task_type="temporal", + target_file="temporal.md", + candidate_content="candidate skill", + base_skill_content="baseline skill content", + ladder_items=questions, + gate_params=_DEFAULT_GATE_PARAMS, + gate_block=4, + gate_n_max=4, + gate_guard_err=0.5, + baseline_cache=cache, + prompts_version="p1", + run_inference=mock_fn, + log=log, + gate_run_prefix="step1_gate_test", + ) + + # n_remaining=0 → 不可能是 continue + assert outcome.stop_reason in ( + "confirmed", + "provisional", + "inertia", + "directional", + "futility", + ) + assert outcome.n_used == 4 + # 终态行标记 stop_reason + assert outcome.evidence_rows[-1]["stop_reason"] != "" + finally: + log.close() From 2296134f733c3728443a277488e51060494892d9 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 13:43:20 -0400 Subject: [PATCH 17/19] =?UTF-8?q?feat(harness):=20runner.py=20=E2=80=94=20?= =?UTF-8?q?train=20loop=20orchestrator=20(#13=20algorithm=20fidelity)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three-level nesting (epoch -> step -> per-skill), slow update 10-step sequence, checkpoint/resume, early stop, probation accept/reject/rollback. Key TRM4->TRM5 changes: - sync -> async (all inference/diagnosis/evolve/validate awaited) - LLMClient.from_env -> injected LLMProvider (DI via constructor) - Direct DB/file access -> module functions (workspace/store/log) - _TrainState as train() local, explicit param passing to helpers Module-level pure functions extracted for testability: resume_plan, _guard_infra_failures, _apply_batch_correctness, _compute_total_steps, _should_early_stop, _format_applied_edits, _fallback_summary, _write_skip_report, _outcome_to_quadrant_pairs, _build_comparison_pairs, _batch_from_ids, _snapshot_current_skills. Tests: 34 unit tests covering 13a-13e sub-tasks. Radon: all functions Grade B or better. --- app/harness/runner.py | 2146 +++++++++++++++++++++++++++++ tests/unit/test_harness_runner.py | 682 +++++++++ 2 files changed, 2828 insertions(+) create mode 100644 app/harness/runner.py create mode 100644 tests/unit/test_harness_runner.py diff --git a/app/harness/runner.py b/app/harness/runner.py new file mode 100644 index 0000000..2cc4e28 --- /dev/null +++ b/app/harness/runner.py @@ -0,0 +1,2146 @@ +"""实验运行器（瘦编排器），对标 PyTorch Trainer。 + +三级嵌套（epoch → step → per-skill）训练循环 + 慢更新十步序 + 断点续训。 +算法保真 #13：训练循环编排从 TRM4 runner.py（2273 行）迁移，逻辑不可简化。 + +关键重构（TRM4 → TRM5）： +- sync → async（await run_inference / run_diagnosis / evolve_* / validate_*） +- LLMClient.from_env → 注入 LLMProvider（self._llm / self._evolve_llm） +- 直接 DB/文件操作 → 通过模块函数（workspace / store / log / observation） +- 瘦身 2273 → ~500 行（推理/诊断/进化/验证全委托模块函数） +""" + +from __future__ import annotations + +import json +import math +import random +import shutil +import sqlite3 +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from loguru import logger + +from app.harness.batching import build_batches +from app.harness.checkpoint import ( + check_fingerprint, + deserialize_state_fields, + load_checkpoint, + write_checkpoint, +) +from app.harness.config import RunConfig # noqa: TC001 — 运行时 _compute_total_steps 使用 +from app.harness.gate_ladder import BaselineCache, GatePools, build_or_load_gate_pools +from app.harness.observation import ( + write_dual_metric, + write_epoch_report, + write_gate_evidence, + write_holdout_eval, + write_quadrant_pairs, + write_shadow_gate, + write_step_report, +) +from app.harness.store import advance_version +from app.harness.validate import Probation, ValidationOutcome +from app.harness.workspace import ( + ResolvedPaths, + archive_workspace, + init_workspace, + init_workspace_from_seed, + load_manifest, + read_best, + resolve_paths, + update_best, + update_manifest, +) +from core.evolution import ( + DiagnosisResult, + GateParams, + RejectedEdit, + edit_budget_at, + momentum_inner, + probation_verdict, + replace_momentum, + resolve_skill_file, +) +from core.evolution.diagnose import merge_system_packs, merge_tool_packs + +if TYPE_CHECKING: + from app.harness.inference import InferenceResult + from app.harness.pools import Pools + from core.evolution.types import ( + EvolutionRecord, + SystemCasePack, + ToolCasePack, + ) + from core.protocols import LLMProvider, TelemetryRecorder, VLMProvider + from core.types import GeneratedQuestion + + +class _InterruptError(RuntimeError): + """测试用中断注入信号：_run_step 末尾可选抛出以模拟进程中断。""" + + +# --------------------------------------------------------------------------- +# _TrainState: 19 个可变字段 +# --------------------------------------------------------------------------- + + +@dataclass +class _TrainState: + """一次 train() 的跨 step 可变状态（训练循环的"权重/缓冲"）。 + + 字段说明见 TRM4 同名 dataclass（完整保留 19 字段语义）。 + TRM5 移除 evolve_client（改走构造注入），其余 18 字段 + gate_epoch_observed 不变。 + """ + + correctness: dict[str, bool] + gate_pools: GatePools + baseline_cache: BaselineCache + eval_prev_acc: float + eval_prev_run_id: str + best_val_acc: float + best_skills_version: str + best_prompts_version: str + baseline_skills_version: str = "" + baseline_prompts_version: str = "" + rejected_buffer: dict[str, list[RejectedEdit]] = field(default_factory=dict) + system_packs: list[SystemCasePack] = field(default_factory=list) + tool_packs: list[ToolCasePack] = field(default_factory=list) + global_step: int = 0 + changed_task_types_this_epoch: set[str] = field(default_factory=set) + epoch_start_skills: dict[str, str] = field(default_factory=dict) + steps_since_best_improved: int = 0 + gate_epoch_observed: bool = False + probations: dict[str, Probation] = field(default_factory=dict) + gate_cooldown: dict[str, int] = field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# 纯函数辅助（不依赖 self） +# --------------------------------------------------------------------------- + + +def resume_plan(epoch: int, phase: str, step_completed: int) -> dict: + """据 checkpoint 进度算续跑计划（纯函数，便于单测）。 + + 参数: + epoch: checkpoint 落库时的 epoch 序号。 + phase: "in_epoch" 或 "epoch_done"。 + step_completed: 该 epoch 内最后完整完成的 step 序号。 + + 返回: + {"first_epoch": int, "resume_epoch": int | None, "resume_step_from": int}。 + """ + if phase == "epoch_done": + return {"first_epoch": epoch + 1, "resume_epoch": None, "resume_step_from": 0} + return { + "first_epoch": epoch, + "resume_epoch": epoch, + "resume_step_from": step_completed + 1, + } + + +def _guard_infra_failures(result: InferenceResult, context: str) -> None: + """基础设施失败护栏：stop_reason="error" 占比 > 10% 即硬终止。 + + 参数: + result: 推理聚合结果。 + context: 出错时报错的推理路径名（仅诊断用）。 + + 异常: + RuntimeError: error 占比 > 10%。 + """ + error_rate = result.stop_reason_counts.get("error", 0) / max(result.total, 1) + if error_rate > 0.1: + raise RuntimeError( + f"{context} 推理基础设施失败率过高 {error_rate:.0%}（stop_reason=error），中止本轮" + ) + + +def _apply_batch_correctness( + correctness: dict[str, bool], + log: Any, + run_id: str, + batch: list[GeneratedQuestion], +) -> None: + """从该 run 的 predictions 读 batch 各题新对错，就地增量更新 correctness。 + + 参数: + correctness: question_id -> 是否答对，就地更新。 + log: HarnessLog 实例。 + run_id: rollout 的 run_id。 + batch: 本 step 的题目列表。 + + 异常: + RuntimeError: rollout 不完整（缺预测行）。 + """ + from app.harness.validate import _load_run_rows + + rows = _load_run_rows(log, run_id) + missing = [q.question_id for q in batch if q.question_id not in rows] + if missing: + raise RuntimeError( + f"rollout 不完整：run_id={run_id} 缺 {len(missing)} 道题预测行 {missing}，中止本步" + ) + for q in batch: + correctness[q.question_id] = rows[q.question_id]["_correct"] + + +def _accumulate_slow_packs(diagnosis: DiagnosisResult, state: _TrainState) -> None: + """把本 step 诊断的 system/tool 案例包只累加不更新，留给 epoch 末慢更新消费。""" + if diagnosis.system_case_pack is not None: + state.system_packs.append(diagnosis.system_case_pack) + state.tool_packs.extend(diagnosis.tool_case_packs.values()) + + +def _batch_from_ids(pools: Pools, ids: list[str]) -> list[GeneratedQuestion]: + """按 question_id 从诊断池重建一个 batch（保持原 epoch 划分）。 + + 参数: + pools: 三池容器。 + ids: 一个 batch 的 question_id 列表。 + + 返回: + 按 ids 顺序取出的 GeneratedQuestion 列表。 + """ + by_id = {q.question_id: q for q in pools.diagnosis} + return [by_id[i] for i in ids] + + +def _snapshot_current_skills(skills_dir: Path) -> dict[str, str]: + """快照当前 skills 版本目录下各 skill 文件的正文（文件名 -> 全文）。 + + 参数: + skills_dir: 当前 skills 版本目录。 + + 返回: + {文件名: 全文}，作 momentum 的上一版基准。 + """ + snapshot: dict[str, str] = {} + for path in sorted(skills_dir.glob("*.md")): + snapshot[path.name] = path.read_text(encoding="utf-8") + return snapshot + + +def _should_early_stop( + workspace_dir: Path, + epoch: int, + steps_this_epoch: int, + state: _TrainState, + patience: int, +) -> bool: + """步粒度 early stop：本 epoch best 未刷新则累加本 epoch 步数。 + + 参数: + workspace_dir: workspace 目录（读 manifest best）。 + epoch: 当前 epoch。 + steps_this_epoch: 本 epoch 的 step 总数。 + state: 训练状态（steps_since_best_improved 就地更新）。 + patience: early_stop_patience。 + + 返回: + 是否触发 early stop。 + """ + best = read_best(workspace_dir) + improved_this_epoch = best is not None and best.get("epoch") == epoch + if improved_this_epoch: + state.steps_since_best_improved = 0 + return False + state.steps_since_best_improved += steps_this_epoch + return state.steps_since_best_improved >= patience + + +def _compute_total_steps(pools: Pools, correctness: dict[str, bool], config: RunConfig) -> int: + """退火地平线：用 build_batches 试切一轮拿 selected_count，再乘 epochs。""" + _, selected_count = build_batches( + pools.diagnosis, + correctness, + config.batch_size, + config.min_class_per_batch, + seed=1, + correct_ratio=config.batch_correct_ratio, + ) + steps_per_epoch = max(1, math.ceil(selected_count / config.batch_size)) + return config.epochs * steps_per_epoch + + +def _outcome_to_quadrant_pairs(task_type: str, outcome: ValidationOutcome) -> list[dict]: + """把 ValidationOutcome 的四象限拍平为逐题 pair（供 quadrant_pair 表落库观测）。 + + 参数: + task_type: 该批 gate 的任务类型。 + outcome: 局部验证决策结果。 + + 返回: + 每条含 question_id/task_type/prev_correct/curr_correct/category 的 dict 列表。 + """ + from app.harness.momentum import ( + IMPROVED, + PERSISTENT_FAIL, + REGRESSED, + STABLE_SUCCESS, + ) + + spec = [ + (outcome.improvements, IMPROVED, False, True), + (outcome.regressions, REGRESSED, True, False), + (outcome.persistent_fails, PERSISTENT_FAIL, False, False), + (outcome.stable_successes, STABLE_SUCCESS, True, True), + ] + pairs: list[dict] = [] + for qids, category, prev_ok, curr_ok in spec: + for qid in qids: + pairs.append( + { + "question_id": qid, + "task_type": task_type, + "prev_correct": prev_ok, + "curr_correct": curr_ok, + "category": category, + } + ) + return pairs + + +def _build_comparison_pairs( + sampled: list[GeneratedQuestion], + prev_rows: dict[str, dict], + curr_rows: dict[str, dict], +) -> list[dict]: + """为采样好的诊断池题目构造 momentum 纵向对比对。""" + pairs: list[dict] = [] + for q in sampled: + prev = prev_rows.get(q.question_id, {}) + curr = curr_rows.get(q.question_id, {}) + pairs.append( + { + "question": q.question, + "prev_prediction": prev.get("prediction", ""), + "curr_prediction": curr.get("prediction", ""), + "correct_prev": prev.get("_correct", False), + "correct_curr": curr.get("_correct", False), + } + ) + return pairs + + +def _filter_applied_edits(edits: list[dict], reports: list[dict]) -> list[dict] | str: + """按 apply_report 过滤出真正 applied 的 edit。 + + 参数: + edits: EvolutionRecord.edits 列表。 + reports: EvolutionRecord.apply_report 列表（与 edits 同序对齐）。 + + 返回: + 过滤后的 edit 列表；0 applied 时返回信息性消息字符串。 + reports 为空时返回原 edits 不过滤。 + """ + if not reports: + return edits + applied = [ + edit + for edit, report in zip(edits, reports, strict=True) + if str(report.get("status", "")).startswith("applied") + ] + if not applied: + return "上轮改法全部未成功应用(0 applied)，本条无已验证信息" + return applied + + +def _format_applied_edits(record: Any) -> str | None: + """从 EvolutionRecord 中提取真正 applied 的 edit 并格式化为摘要。 + + 参数: + record: EvolutionRecord（duck-typed，需含 edits / apply_report）。 + + 返回: + 已 applied edit 的格式化摘要；无 edit 或无 applied 时返回 None， + 0 applied 时返回信息性消息（非 None）。 + """ + rec_edits = getattr(record, "edits", []) or [] + if not rec_edits: + return None + filtered = _filter_applied_edits(rec_edits, getattr(record, "apply_report", []) or []) + if isinstance(filtered, str): + return filtered + summary = "; ".join( + f"[{edit.get('op')}]{(edit.get('target') or edit.get('content', ''))[:40]}" + for edit in filtered + if isinstance(edit, dict) + ) + return summary or None + + +def _fallback_summary(record: Any, outcome: Any) -> str: + """从 suggestions 或跌幅信息构造兜底黑名单摘要。 + + 参数: + record: EvolutionRecord（duck-typed，需含 suggestions）。 + outcome: ValidationOutcome（duck-typed，需含 delta_hat）。 + + 返回: + 兜底摘要字符串。 + """ + return "; ".join(s.get("change", "") for s in record.suggestions) or ( + f"上轮对本文件改写被拒(delta {outcome.delta_hat:+.2f})，换方向" + ) + + +def _write_skip_report( + workspace_dir: Path, + epoch: int, + step: int, + global_step: int, + task_type: str, + action: str, + baseline_acc: float, + budget: int, + rank_clip_triggered: bool = False, +) -> None: + """为 cooldown / skipped 路径写 step_report（无 gate 证据）。 + + 参数: + workspace_dir: 工作区目录。 + epoch: 轮次。 + step: epoch 内 step。 + global_step: 全局步计数。 + task_type: 任务类型。 + action: "cooldown" 或 "skipped"。 + baseline_acc: 当前类基线准确率。 + budget: 编辑预算。 + rank_clip_triggered: 是否触发 rank 裁剪（skipped 路径需要）。 + """ + write_step_report( + workspace_dir, + epoch=epoch, + step=step, + global_step=global_step, + task_type=task_type, + gate_action=action, + candidate_acc=baseline_acc, + class_baseline_acc=baseline_acc, + edit_budget=budget, + rank_clip_triggered=rank_clip_triggered, + gate_w=None, + gate_l=None, + gate_e_value=None, + gate_n_used=None, + gate_stop_reason=None, + ) + + +# --------------------------------------------------------------------------- +# Runner 主类 +# --------------------------------------------------------------------------- + + +class Runner: + """实验运行器（瘦编排器），通过 RunConfig 驱动训练/推理/诊断/评估等模式。 + + DI 纪律：self 只持注入依赖 + _paths。_TrainState 是 train() 内局部变量， + 显式传参给模块函数。 + + 参数: + config: 运行配置。 + llm: 推理用 LLMProvider。 + evolve_llm: 进化用 LLMProvider（thinking=True）。 + vlm: VLMProvider。 + telemetry: 遥测记录端口。 + """ + + def __init__( + self, + config: RunConfig, + *, + llm: LLMProvider, + evolve_llm: LLMProvider, + vlm: VLMProvider, + telemetry: TelemetryRecorder, + ) -> None: + self._config = config + self._llm = llm + self._evolve_llm = evolve_llm + self._vlm = vlm + self._telemetry = telemetry + self._ensure_workspace() + self._paths: ResolvedPaths = resolve_paths(config.workspace_dir) + + # ----------------------------------------------------------------------- + # workspace 三态逻辑 + # ----------------------------------------------------------------------- + + def _ensure_workspace(self) -> None: + """train 模式按 --resume/--fresh 分三态；其余模式仅要求 ws 已存在并复用。 + + 三态逻辑: + resume+fresh → ValueError + fresh+已有 → archive + init_from_seed + resume+无进度 → RuntimeError + 无flag+已有 → SystemExit + """ + manifest = self._config.workspace_dir / "manifest.json" + has_progress = manifest.exists() + if self._config.mode != "train": + if not has_progress: + raise RuntimeError( + f"{self._config.mode} 模式要求 workspace 已存在: {self._config.workspace_dir}" + ) + return + if self._config.resume and self._config.fresh: + raise ValueError("--resume 与 --fresh 互斥") + if self._config.fresh: + if has_progress: + logger.info( + "旧 workspace 已归档: {}", + archive_workspace(self._config.workspace_dir), + ) + init_workspace_from_seed( + self._config.workspace_dir, + self._config.store_dir, + self._config.seed, + self._config.questions, + ) + return + if self._config.resume: + if not has_progress: + raise RuntimeError("--resume 但 workspace 无已有进度") + return + if has_progress: + raise SystemExit("workspace 已有进度；用 --resume 续训 或 --fresh 归档重开") + init_workspace( + self._config.workspace_dir, + self._config.store_dir, + self._config.questions, + self._config.skills_version, + self._config.prompts_version, + ) + + # ----------------------------------------------------------------------- + # 公共入口：infer / eval / diagnose / promote + # ----------------------------------------------------------------------- + + async def infer(self, task_types: list[str] | None = None) -> InferenceResult: + """执行单次推理（forward-only）。 + + 参数: + task_types: 若非 None，只保留指定题型。 + + 返回: + InferenceResult 冻结实例。 + """ + from app.harness.inference import run_inference + from app.harness.log import HarnessLog + from app.question_gen import load_benchmark + + questions = load_benchmark(self._paths.questions_dir) + if task_types: + allowed = set(task_types) + questions = [q for q in questions if q.task_type in allowed] + if self._config.n_samples > 0: + questions = questions[: self._config.n_samples] + + run_id = f"infer_{self._config.run_id}" if self._config.run_id else "infer_adhoc" + record_run_dir = self._record_run(run_id) # noqa: F841 + + logger.info( + "启动推理: {} 道题, concurrency={}, max_steps={}, skill_mode={}", + len(questions), + self._config.concurrency, + self._config.max_steps, + self._config.skill_mode, + ) + + with HarnessLog(str(self._paths.db_path), run_id) as log: + return await run_inference( + questions=questions, + llm=self._llm, + tool_dispatch_fn=self._make_tool_dispatch_fn(), + prompt_builder=self._make_prompt_builder(), + log=log, + run_id=run_id, + concurrency=self._config.concurrency, + max_steps=self._config.max_steps, + skill_mode=self._config.skill_mode, + ) + + async def eval(self, version: str) -> InferenceResult: + """用指定 skills 版本跑完整题库，全量记录落 db + 版本回填。 + + 参数: + version: skills 版本号。 + + 返回: + InferenceResult。 + """ + from datetime import UTC, datetime + + from app.harness.inference import run_inference + from app.harness.log import HarnessLog + from app.question_gen import load_benchmark + + cur = load_manifest(self._config.workspace_dir)["current"] + prompts_v = cur["prompts"].split("/")[-1] + skills_dir = self._paths.workspace_dir / "skills" / version + prompts_dir = self._paths.workspace_dir / "prompts" / prompts_v + if not skills_dir.is_dir(): + raise FileNotFoundError(f"skills 版本目录不存在: {skills_dir}") + if not prompts_dir.is_dir(): + raise FileNotFoundError(f"prompts 版本目录不存在: {prompts_dir}") + + questions = load_benchmark(self._paths.questions_dir) + if self._config.n_samples > 0: + questions = questions[: self._config.n_samples] + + ts = datetime.now(UTC).strftime("%Y%m%d_%H%M%S") + run_id = f"eval_{version}-{prompts_v}_{ts}" + self._record_run(run_id) + logger.info( + "eval: 版本 skills/{}+prompts/{} 跑 {} 题 (run_id={})", + version, + prompts_v, + len(questions), + run_id, + ) + + with HarnessLog(str(self._paths.db_path), run_id) as log: + result = await run_inference( + questions=questions, + llm=self._llm, + tool_dispatch_fn=self._make_tool_dispatch_fn(skills_dir=skills_dir), + prompt_builder=self._make_prompt_builder( + skills_dir=skills_dir, prompts_dir=prompts_dir + ), + log=log, + run_id=run_id, + concurrency=self._config.concurrency, + max_steps=self._config.max_steps, + skill_mode=self._config.skill_mode, + ) + + # C4 回填 + self._backfill_run_versions(run_id, version, prompts_v) + self._write_eval_report(run_id, version, prompts_v, result) + return result + + async def diagnose(self, run_id: str) -> DiagnosisResult: + """执行指定 run 的两阶段诊断。 + + 参数: + run_id: 待诊断的 run_id。 + + 返回: + DiagnosisResult。 + """ + + return await self._run_diagnosis(run_id) + + def promote(self, version: str, eval_run_id: str, name: str) -> None: + """把当前 ws 的指定版本提升为 Store 新种子。 + + 参数: + version: 要提升的 skills 版本号。 + eval_run_id: canonical eval run。 + name: 新种子名。 + """ + from app.harness.store import promote_to_seed + + seed_dir = promote_to_seed( + self._config.workspace_dir, + self._config.store_dir, + version, + eval_run_id, + name, + description=f"promote from {self._config.workspace_dir.name} {version}", + ) + logger.info("已提升为种子: {}", seed_dir) + + # ----------------------------------------------------------------------- + # train() 三级嵌套（核心） + # ----------------------------------------------------------------------- + + async def train(self, pools: Pools) -> None: + """mini-batch 快慢双速闭环：epoch 内多 step、每 step 按类 per-skill gate。 + + 三级嵌套：epoch → batch(step) → per-skill。 + epoch 末 _slow_update_cycle 十步序。 + 训练收尾 _deliver_best + _final_test_eval。 + """ + state, total_steps, plan, saved_batches = await self._setup_train_run(pools) + for epoch in range(plan["first_epoch"], self._config.epochs + 1): + if epoch == plan["resume_epoch"]: + batches = [_batch_from_ids(pools, ids) for ids in saved_batches] + step_from = plan["resume_step_from"] + else: + logger.info("=== Epoch {} ===", epoch) + state.system_packs = [] + state.tool_packs = [] + state.changed_task_types_this_epoch = set() + state.epoch_start_skills = _snapshot_current_skills(self._paths.skills_dir) + batches, _ = build_batches( + pools.diagnosis, + state.correctness, + self._config.batch_size, + self._config.min_class_per_batch, + seed=epoch, + correct_ratio=self._config.batch_correct_ratio, + ) + step_from = 0 + batch_ids = [[q.question_id for q in b] for b in batches] + for step in range(step_from, len(batches)): + await self._run_step(epoch, step, total_steps, batches[step], pools, state) + state.global_step += 1 + write_checkpoint( + self._config.workspace_dir, + state=state, + epoch=epoch, + step_completed=step, + phase="in_epoch", + global_step=state.global_step, + total_steps=total_steps, + version_snapshot=self._current_version_snapshot(), + epoch_batches=batch_ids, + config=self._config, + ) + await self._slow_update_cycle(epoch, pools, state) + state.system_packs = [] + state.tool_packs = [] + state.changed_task_types_this_epoch = set() + write_checkpoint( + self._config.workspace_dir, + state=state, + epoch=epoch, + step_completed=len(batches) - 1, + phase="epoch_done", + global_step=state.global_step, + total_steps=total_steps, + version_snapshot=self._current_version_snapshot(), + epoch_batches=batch_ids, + config=self._config, + ) + if _should_early_stop( + self._config.workspace_dir, + epoch, + len(batches), + state, + self._config.early_stop_patience, + ): + logger.info("Epoch {} 触发 early stop（best 连续无新高）", epoch) + break + self._deliver_best(state.best_skills_version, state.best_prompts_version) + await self._final_test_eval(pools) + + # ----------------------------------------------------------------------- + # 训练初始化 + # ----------------------------------------------------------------------- + + async def _setup_train_run(self, pools: Pools) -> tuple[_TrainState, int, dict, list | None]: + """据是否 --resume 准备训练起点。 + + 返回: + (state, total_steps, plan, saved_batches)。 + """ + ckpt = load_checkpoint(self._config.workspace_dir) if self._config.resume else None + if self._config.resume and ckpt is None: + raise RuntimeError("--resume 但 checkpoint.json 不存在，拒绝静默从头重训") + gate_pools, baseline_cache = self._init_gate_pools(pools) + if not ckpt: + state = self._init_train_state(pools, gate_pools, baseline_cache) + total_steps = _compute_total_steps(pools, state.correctness, self._config) + plan = {"first_epoch": 1, "resume_epoch": None, "resume_step_from": 0} + return state, total_steps, plan, None + struct, decision = check_fingerprint(ckpt["config_fingerprint"], self._config) + if struct: + raise RuntimeError(f"结构性配置变化，拒绝 resume: {struct}") + if decision: + logger.warning("决策性配置变化，继续 resume: {}", decision) + state = self._restore_train_state(ckpt, pools, gate_pools, baseline_cache) + state.global_step = ckpt["progress"]["global_step"] + update_manifest( + self._config.workspace_dir, + skills=ckpt["version_snapshot"]["skills"], + prompts=ckpt["version_snapshot"]["prompts"], + ) + self._paths = resolve_paths(self._config.workspace_dir) + plan = resume_plan( + ckpt["progress"]["epoch"], + ckpt["progress"]["phase"], + ckpt["progress"]["step_completed"], + ) + return state, ckpt["progress"]["total_steps"], plan, ckpt["epoch_batches"] + + def _init_gate_pools(self, pools: Pools) -> tuple[GatePools, BaselineCache]: + """构建/加载 CE-Gate 信息量阶梯与基线缓存。 + + 副作用：设置 self._gate_questions_by_id（不进 checkpoint）。 + + 参数: + pools: 冻结三池。 + + 返回: + (GatePools, BaselineCache)。 + """ + from app.harness.log import HarnessLog + from app.question_gen import load_benchmark + + questions = load_benchmark(self._paths.questions_dir) + self._gate_questions_by_id: dict[str, GeneratedQuestion] = { + q.question_id: q for q in questions + } + with HarnessLog(str(self._paths.db_path), pools.baseline_run_id) as log: + rows = log.query( + "SELECT question_id, prediction, answer FROM predictions WHERE run_id=?", + (pools.baseline_run_id,), + ) + if not rows: + raise RuntimeError( + f"基线 run {pools.baseline_run_id} 在 predictions 表无任何行，" + "无法构建 gate 阶梯（检查种子基线是否完整落库）" + ) + baseline_correctness = {r["question_id"]: r["prediction"] == r["answer"] for r in rows} + logger.info("gate 阶梯基线对错覆盖 {} 题", len(baseline_correctness)) + gate_task_types = sorted({q.task_type for q in pools.diagnosis}) + gate_pools = build_or_load_gate_pools( + workspace_dir=self._config.workspace_dir, + questions=questions, + test_qids={q.question_id for q in pools.test}, + baseline_correctness=baseline_correctness, + task_types=gate_task_types, + probe_quota=self._config.gate_probe_quota, + seed=1, + baseline_run_id=pools.baseline_run_id, + ) + baseline_cache = BaselineCache(self._config.workspace_dir / "baseline_cache.json") + return gate_pools, baseline_cache + + def _init_train_state( + self, pools: Pools, gate_pools: GatePools, baseline_cache: BaselineCache + ) -> _TrainState: + """初始化跨 step 训练状态。""" + skills_v = self._current_version("skills") + prompts_v = self._current_version("prompts") + update_best( + self._config.workspace_dir, + skills=f"skills/{skills_v}", + prompts=f"prompts/{prompts_v}", + val_acc=pools.baseline_val_accuracy, + run_id=pools.baseline_run_id, + epoch=0, + ) + if read_best(self._config.workspace_dir) is None: + raise RuntimeError("best 指针初始化失败") + return _TrainState( + correctness=dict(pools.correctness), + gate_pools=gate_pools, + baseline_cache=baseline_cache, + eval_prev_acc=pools.baseline_val_accuracy, + eval_prev_run_id=pools.baseline_run_id, + best_val_acc=pools.baseline_val_accuracy, + best_skills_version=skills_v, + best_prompts_version=prompts_v, + baseline_skills_version=skills_v, + baseline_prompts_version=prompts_v, + ) + + def _restore_train_state( + self, + ckpt: dict, + pools: Pools, + gate_pools: GatePools, + baseline_cache: BaselineCache, + ) -> _TrainState: + """从 checkpoint 重建 _TrainState。""" + fields = deserialize_state_fields(ckpt["state"]) + best = read_best(self._config.workspace_dir) or {} + return _TrainState( + gate_pools=gate_pools, + baseline_cache=baseline_cache, + best_val_acc=best.get("val_acc", pools.baseline_val_accuracy), + best_skills_version=best.get("skills", "skills/v1").split("/")[-1], + best_prompts_version=best.get("prompts", "prompts/v1").split("/")[-1], + **fields, + ) + + # ----------------------------------------------------------------------- + # _run_step：rollout → correctness → diagnose → accumulate → gate → cooldown + # ----------------------------------------------------------------------- + + async def _run_step( + self, + epoch: int, + step: int, + total_steps: int, + batch: list[GeneratedQuestion], + pools: Pools, + state: _TrainState, + ) -> None: + """单 step：rollout → correctness 增量 → 诊断 → 累加 system/tool → 按类 gate。""" + run_id = f"{pools.baseline_run_id}_e{epoch}_s{step}" + await self._rollout_batch(batch, run_id) + + from app.harness.log import HarnessLog + + with HarnessLog(str(self._paths.db_path), run_id) as log: + _apply_batch_correctness(state.correctness, log, run_id, batch) + + diagnosis = await self._run_diagnosis(run_id, question_ids=[q.question_id for q in batch]) + _accumulate_slow_packs(diagnosis, state) + await self._gate_batch_skills(epoch, step, diagnosis, total_steps, pools, state) + # 冷却计数每 step 递减、归零剔除 + state.gate_cooldown = {t: n - 1 for t, n in state.gate_cooldown.items() if n - 1 > 0} + # 测试中断注入点 + if getattr(self, "_interrupt_after_step", None) == step: + raise _InterruptError(f"模拟中断于 epoch{epoch} step{step}") + + async def _rollout_batch(self, batch: list[GeneratedQuestion], run_id: str) -> None: + """用当前 skill 版本重推该 batch。""" + result = await self._run_inference_on_pool( + batch, run_id, self._paths.skills_dir, self._paths.prompts_dir + ) + _guard_infra_failures(result, context="rollout") + + # ----------------------------------------------------------------------- + # _gate_batch_skills：per task_type gate + # ----------------------------------------------------------------------- + + async def _gate_batch_skills( + self, + epoch: int, + step: int, + diagnosis: DiagnosisResult, + total_steps: int, + pools: Pools, + state: _TrainState, + ) -> None: + """按 task_type 独立 evolve → 局部验证 → accept/reject。""" + from app.harness.workspace import VersionedSkillStore + from core.evolution import evolve_single_skill + + budget = edit_budget_at( + global_step=state.global_step, + total_steps=total_steps, + start=self._config.edit_budget_start, + end=self._config.edit_budget_end, + ) + for task_type in sorted(diagnosis.skill_case_packs): + # 冷却 admission control + if state.gate_cooldown.get(task_type, 0) > 0: + _write_skip_report( + self._config.workspace_dir, + epoch, + step, + state.global_step, + task_type, + action="cooldown", + baseline_acc=self._class_baseline_acc( + task_type, pools.validation, state.correctness + ), + budget=budget, + ) + continue + + pack = diagnosis.skill_case_packs[task_type] + skill_store = VersionedSkillStore(self._paths.skills_dir) + evolve_prompts = self._load_evolve_prompts() + record = await evolve_single_skill( + self._evolve_llm, + pack, + skill_store, + evolve_prompts, + self._current_version("skills"), + budget, + self._config.appendix_consolidate_threshold, + skill_update_mode=self._config.skill_update_mode, + rejected=state.rejected_buffer.get(task_type, []), + ) + # 进化未产出真实改动 + if record.status in ("rejected", "skipped") or ( + record.evolved_content == record.original_content + ): + _write_skip_report( + self._config.workspace_dir, + epoch, + step, + state.global_step, + task_type, + action="skipped", + baseline_acc=self._class_baseline_acc( + task_type, pools.validation, state.correctness + ), + budget=budget, + rank_clip_triggered=bool(record.clip_info.get("triggered", False)), + ) + continue + + outcome = await self._run_gate_validation( + epoch, step, task_type, pack, record, pools, state + ) + # 观测落库 + write_gate_evidence( + str(self._paths.db_path), + run_id=pools.baseline_run_id, + epoch=epoch, + step=step, + rows=outcome.evidence_rows, + ) + write_step_report( + self._config.workspace_dir, + epoch=epoch, + step=step, + global_step=state.global_step, + task_type=task_type, + gate_action=outcome.action, + candidate_acc=outcome.candidate_acc, + class_baseline_acc=outcome.baseline_acc, + edit_budget=budget, + rank_clip_triggered=bool(record.clip_info.get("triggered", False)), + gate_w=outcome.w, + gate_l=outcome.l, + gate_e_value=outcome.e_value, + gate_n_used=outcome.n_used, + gate_stop_reason=outcome.stop_reason, + ) + write_quadrant_pairs( + str(self._paths.db_path), + run_id=pools.baseline_run_id, + epoch=epoch, + step=step, + pairs=_outcome_to_quadrant_pairs(task_type, outcome), + ) + if outcome.accepted: + self._accept_skill(task_type, record, outcome, state, pools) + else: + self._record_rejected_skill( + state.rejected_buffer, task_type, record, outcome, state.global_step + ) + + async def _run_gate_validation( + self, + epoch: int, + step: int, + task_type: str, + pack: Any, + record: EvolutionRecord, + pools: Pools, + state: _TrainState, + ) -> ValidationOutcome: + """CE-Gate 块序贯配对验证：阶梯出题 → 基线/候选逐块配对 → e-process 四出口。 + + 参数: + epoch: 轮次。 + step: epoch 内 step。 + task_type: 待验证题型。 + pack: SkillCasePack。 + record: 进化产物。 + pools: 冻结三池。 + state: 训练状态。 + + 返回: + ValidationOutcome。 + """ + from app.harness.log import HarnessLog + from app.harness.validate import validate_skill_local + + exclude_qids = {c.question_id for c in pack.failure_cases + pack.success_cases} + ladder_qids = state.gate_pools.ladder_for( + task_type, + exclude_qids, + p_low=self._config.gate_p_low, + p_high=self._config.gate_p_high, + cold=not state.gate_epoch_observed, + ) + missing = [qid for qid in ladder_qids if qid not in self._gate_questions_by_id] + if missing: + raise ValueError( + f"gate 阶梯[{task_type}] 含 benchmark 中不存在的题: " + f"{missing[:5]}（gate_pools.json 与题库失配）" + ) + ladder_items = [self._gate_questions_by_id[qid] for qid in ladder_qids] + base_skill_content = (self._paths.skills_dir / record.target_file).read_text( + encoding="utf-8" + ) + slug = task_type.lower().replace(" ", "-") + run_inference_fn = self._make_validate_run_inference_fn() + with HarnessLog(str(self._paths.db_path), f"gate_{slug}") as gate_log: + return await validate_skill_local( + workspace_dir=self._config.workspace_dir, + base_skills_version=self._current_version("skills"), + task_type=task_type, + target_file=record.target_file, + candidate_content=record.evolved_content, + base_skill_content=base_skill_content, + ladder_items=ladder_items, + gate_params=GateParams( + e_confirm=self._config.gate_e_confirm, + e_provisional=self._config.gate_e_provisional, + w_net_min=self._config.gate_w_net_min, + delta_min=self._config.gate_delta_min, + lambda_dir=self._config.gate_lambda_dir, + e_rollback=self._config.gate_e_rollback, + ), + gate_block=self._config.gate_block, + gate_n_max=self._config.gate_n_max, + gate_guard_err=self._config.gate_guard_err, + baseline_cache=state.baseline_cache, + prompts_version=self._current_version("prompts"), + run_inference=run_inference_fn, + log=gate_log, + gate_run_prefix=(f"{pools.baseline_run_id}_e{epoch}_s{step}_gate_{slug}"), + ) + + # ----------------------------------------------------------------------- + # accept / reject / probation + # ----------------------------------------------------------------------- + + def _accept_skill( + self, + task_type: str, + record: EvolutionRecord, + outcome: ValidationOutcome, + state: _TrainState, + pools: Pools, + ) -> None: + """accept：写候选为新 skills 版本 → manifest → 路径 → 前移 correctness。 + + probation 分岔：provisional + 无现有试用 + 非 default-strategy.md → 开账。 + 试用中追加 pending_edits。 + """ + pre_accept_version = self._current_version("skills") + # 开账快照在合并前拍取 + pre_merge_snapshot = { + q.question_id: state.correctness.get(q.question_id, False) + for q in pools.validation + if q.task_type == task_type + } + new_version = self._promote_skill_version(record.evolved_content, record.target_file) + update_manifest(self._config.workspace_dir, skills=f"skills/{new_version}") + self._paths = resolve_paths(self._config.workspace_dir) + # correctness 二轨合并（只合并已观测题） + state.correctness.update(outcome.candidate_correctness) + # 清该类黑名单 + state.rejected_buffer.pop(task_type, None) + state.changed_task_types_this_epoch.add(task_type) + # probation 分岔 + if ( + outcome.action == "accept_provisional" + and task_type not in state.probations + and record.target_file != "default-strategy.md" + ): + state.probations[task_type] = Probation( + task_type=task_type, + anchor_skills_version=pre_accept_version, + target_file=record.target_file, + correctness_snapshot=pre_merge_snapshot, + opened_step=state.global_step, + ) + elif outcome.action == "accept_provisional" and record.target_file == "default-strategy.md": + logger.warning( + "按类 gate[{}] provisional 落在共享 default-strategy.md，跳过试用直接转正", + task_type, + ) + if task_type in state.probations: + state.probations[task_type].pending_edits.append( + RejectedEdit( + target_file=record.target_file, + target_type=record.target_type, + change_summary=self._rejected_summary(record, outcome), + delta=outcome.delta_hat, + source_version=record.source_version, + epoch=state.global_step, + gate_w=outcome.w, + gate_l=outcome.l, + gate_e_value=outcome.e_value, + gate_delta_shrunk=outcome.delta_shrunk, + ) + ) + logger.info( + "按类 gate[{}] accept: 候选{:.1%} (观测基线{:.1%}) → skills/{}", + task_type, + outcome.candidate_acc, + outcome.baseline_acc, + new_version, + ) + + def _rollback_probation(self, probation: Probation, state: _TrainState) -> None: + """试用期回滚：文件级 revert 到锚版本 + 恢复快照 + 冷却 + 证据入黑名单。""" + anchor_content = ( + self._config.workspace_dir + / "skills" + / probation.anchor_skills_version + / probation.target_file + ).read_text(encoding="utf-8") + new_version = self._promote_skill_version(anchor_content, probation.target_file) + update_manifest(self._config.workspace_dir, skills=f"skills/{new_version}") + self._paths = resolve_paths(self._config.workspace_dir) + state.correctness.update(probation.correctness_snapshot) + state.gate_cooldown[probation.task_type] = self._config.gate_cooldown_steps + state.rejected_buffer.setdefault(probation.task_type, []).extend(probation.pending_edits) + logger.info( + "probation 回滚[{}]: skills 文件 {} 恢复至锚版本 {} → 新版本 {}，冷却 {} step", + probation.task_type, + probation.target_file, + probation.anchor_skills_version, + new_version, + self._config.gate_cooldown_steps, + ) + + @staticmethod + def _record_rejected_skill( + rejected_buffer: dict[str, list], + task_type: str, + record: EvolutionRecord, + outcome: ValidationOutcome, + global_step: int, + ) -> None: + """reject：按 task_type 累加 A5 黑名单。""" + rejected_buffer.setdefault(task_type, []).append( + RejectedEdit( + target_file=record.target_file, + target_type=record.target_type, + change_summary=Runner._rejected_summary_static(record, outcome), + delta=outcome.delta_hat, + source_version=record.source_version, + epoch=global_step, + gate_w=outcome.w, + gate_l=outcome.l, + gate_e_value=outcome.e_value, + gate_delta_shrunk=outcome.delta_shrunk, + ) + ) + logger.info( + "按类 gate[{}] reject: 候选{:.1%} (观测基线{:.1%}) 回退该 skill", + task_type, + outcome.candidate_acc, + outcome.baseline_acc, + ) + + def _rejected_summary(self, record: EvolutionRecord, outcome: ValidationOutcome) -> str: + """为被拒进化记录生成黑名单摘要：只拼真正 applied 的 edit。""" + return Runner._rejected_summary_static(record, outcome) + + @staticmethod + def _rejected_summary_static(record: EvolutionRecord, outcome: ValidationOutcome) -> str: + """黑名单摘要实现（静态方法，供 accept / reject 两侧复用）。 + + 为何只记 applied：未 applied 的 edit 从未写进候选正文、从未被 gate + 验证过，进黑名单会污染「已验证无效」语义。 + """ + applied_summary = _format_applied_edits(record) + if applied_summary is not None: + return applied_summary + return _fallback_summary(record, outcome) + + # ----------------------------------------------------------------------- + # _slow_update_cycle 十步序 + # ----------------------------------------------------------------------- + + async def _slow_update_cycle(self, epoch: int, pools: Pools, state: _TrainState) -> None: + """epoch 末慢更新十步序。 + + 1. 捕获版本快照 → 全 val 重跑 R + 2. soft score + dual_metric 落库 + 3. R 逐题对错无条件回写 + 4. probation 结算（回滚者覆盖 step 3） + 5. best argmax（严格大于） + 6. momentum（不可变新版本，按 skill 文件分组） + 7. system/tool 慢更新（edit_budget_end） + 8. R2 闭环 + 9. 三态标签 + epoch_report + 四向 held-out + 10. gate 阶梯刷新 + """ + # Phase 1 + eval_skills_version = self._current_version("skills") + eval_prompts_version = self._current_version("prompts") + eval_r = await self._eval_full_val(epoch, pools) + + # Phase 2: soft + dual_metric + eval_soft = await self._try_soft_score(eval_r.run_id, pools.validation) + write_dual_metric( + str(self._paths.db_path), + run_id=self._config.run_id, + epoch=epoch, + version_kind="final", + skills_version=eval_skills_version, + prompts_version=eval_prompts_version, + pool="val", + hard_acc=eval_r.accuracy, + soft_score=eval_soft, + mixed_score=(None if eval_soft is None else 0.5 * eval_r.accuracy + 0.5 * eval_soft), + ) + + # Phase 3: 无条件回写 + self._writeback_val_correctness(eval_r.run_id, pools, state) + + # Phase 4: probation 结算 + self._settle_probations(eval_r.run_id, state) + + # Phase 5: best argmax + self._maybe_promote_best( + eval_skills_version, + eval_prompts_version, + eval_r.accuracy, + eval_r.run_id, + epoch, + state, + ) + + # Phase 6: momentum + momentum_task_types = await self._write_momentum_for_changed_skills( + state, pools, epoch, eval_skills_version + ) + + # Phase 7: system/tool 慢更新 + pre_prompts_version = self._current_version("prompts") + system_tool_updated = await self._update_system_tool(epoch, state) + system_tool_reverted = False + + # Phase 8: R2 闭环 + r2_kept_run_ids: list[str] | None = None + if system_tool_updated: + r2_skills_version = self._current_version("skills") + new_prompts_version = self._current_version("prompts") + eval_r2 = await self._eval_full_val(epoch, pools, run_suffix="_p2") + write_dual_metric( + str(self._paths.db_path), + run_id=self._config.run_id, + epoch=epoch, + version_kind="final", + skills_version=r2_skills_version, + prompts_version=new_prompts_version, + pool="val", + hard_acc=eval_r2.accuracy, + soft_score=None, + mixed_score=None, + ) + system_tool_reverted = eval_r2.accuracy < eval_r.accuracy + if system_tool_reverted: + self._revert_system_tool(pre_prompts_version) + else: + self._writeback_val_correctness(eval_r2.run_id, pools, state) + self._maybe_promote_best( + r2_skills_version, + new_prompts_version, + eval_r2.accuracy, + eval_r2.run_id, + epoch, + state, + ) + state.eval_prev_acc = eval_r2.accuracy + state.eval_prev_run_id = eval_r2.run_id + r2_kept_run_ids = [eval_r2.run_id] + if (not system_tool_updated) or system_tool_reverted: + state.eval_prev_acc = eval_r.accuracy + state.eval_prev_run_id = eval_r.run_id + + # Phase 9: 三态标签 + epoch_report + held-out + if system_tool_reverted: + system_tool_action = "reverted" + elif system_tool_updated: + system_tool_action = "updated" + else: + system_tool_action = "none" + write_epoch_report( + self._config.workspace_dir, + epoch=epoch, + system_tool_action=system_tool_action, + momentum_updated_task_types=momentum_task_types, + best_val_acc=state.best_val_acc, + ) + await self._holdout_four_way(epoch, pools, state, eval_skills_version, eval_prompts_version) + + # Phase 10: gate 阶梯刷新 + self._refresh_gate_ladder( + epoch, pools.baseline_run_id, state, extra_run_ids=r2_kept_run_ids + ) + + # ----------------------------------------------------------------------- + # 慢更新内部方法 + # ----------------------------------------------------------------------- + + def _settle_probations(self, eval_run_id: str, state: _TrainState) -> None: + """epoch 末试用期一次性结算：全 val 重跑逐题结果与锚快照配对。 + + 参数: + eval_run_id: 本 epoch 全 val 重跑（R）的 run_id。 + state: 训练状态（probations 结算后清空）。 + + 异常: + RuntimeError: 重跑缺某快照题的预测行。 + """ + if not state.probations: + return + from app.harness.log import HarnessLog + from app.harness.validate import _load_run_rows + + with HarnessLog(str(self._paths.db_path), eval_run_id) as log: + rows = _load_run_rows(log, eval_run_id) + + params = GateParams( + e_confirm=self._config.gate_e_confirm, + e_provisional=self._config.gate_e_provisional, + w_net_min=self._config.gate_w_net_min, + delta_min=self._config.gate_delta_min, + lambda_dir=self._config.gate_lambda_dir, + e_rollback=self._config.gate_e_rollback, + ) + for task_type in sorted(state.probations): + probation = state.probations[task_type] + w = l = 0 # noqa: E741 + for qid, snap_correct in probation.correctness_snapshot.items(): + row = rows.get(qid) + if row is None: + raise RuntimeError( + f"probation 结算缺预测行: {task_type}/{qid}（run={eval_run_id}）" + ) + cur = row["_correct"] + if not snap_correct and cur: + w += 1 + elif snap_correct and not cur: + l += 1 # noqa: E741 + verdict = probation_verdict(w, l, params=params) + logger.info("probation 结算[{}]: W={} L={} → {}", task_type, w, l, verdict) + if verdict == "rollback": + self._rollback_probation(probation, state) + state.probations.clear() + + async def _eval_full_val( + self, epoch: int, pools: Pools, run_suffix: str = "" + ) -> InferenceResult: + """全验证池重跑一次并护栏。""" + run_id = f"{self._config.run_id}_slow_e{epoch}{run_suffix}" + result = await self._run_inference_on_pool( + pools.validation, run_id, self._paths.skills_dir, self._paths.prompts_dir + ) + _guard_infra_failures(result, context="全 val 重跑") + return result + + def _writeback_val_correctness( + self, eval_run_id: str, pools: Pools, state: _TrainState + ) -> None: + """把全 val 重跑逐题对错回写进 state.correctness。""" + from app.harness.log import HarnessLog + from app.harness.validate import _load_run_rows + + with HarnessLog(str(self._paths.db_path), eval_run_id) as log: + rows = _load_run_rows(log, eval_run_id) + for q in pools.validation: + row = rows.get(q.question_id) + if row is not None: + state.correctness[q.question_id] = row["_correct"] + + def _maybe_promote_best( + self, + skills_v: str, + prompts_v: str, + eval_acc: float, + run_id: str, + epoch: int, + state: _TrainState, + ) -> None: + """全局 best argmax（严格大于才推进）。""" + if eval_acc <= state.best_val_acc: + return + state.best_val_acc = eval_acc + state.best_skills_version = skills_v + state.best_prompts_version = prompts_v + state.steps_since_best_improved = 0 + update_best( + self._config.workspace_dir, + skills=f"skills/{skills_v}", + prompts=f"prompts/{prompts_v}", + val_acc=eval_acc, + run_id=run_id, + epoch=epoch, + ) + logger.info( + "全局 best argmax 刷新: {:.1%} → skills/{} prompts/{}", + eval_acc, + skills_v, + prompts_v, + ) + + async def _write_momentum_for_changed_skills( + self, + state: _TrainState, + pools: Pools, + epoch: int, + eval_skills_version: str, + ) -> list[str]: + """为本 epoch 改过的题型写 momentum：推进不可变新版本。 + + 返回: + 实际写过 momentum 的题型列表。 + """ + + if not self._config.use_slow_momentum: + return [] + if not state.changed_task_types_this_epoch: + return [] + + file_to_task_types = self._group_changed_task_types_by_file( + state.changed_task_types_this_epoch + ) + with tempfile.TemporaryDirectory() as tmp: + staged_skills = Path(tmp) / "skills" + shutil.copytree(self._paths.skills_dir, staged_skills, dirs_exist_ok=True) + for target_file in sorted(file_to_task_types): + await self._stage_momentum_for_file( + target_file, + file_to_task_types[target_file], + state, + pools, + staged_skills, + epoch, + ) + new_version = advance_version( + self._paths.workspace_dir, + "skills", + staged_skills, + { + "source": "slow_momentum", + "parent": eval_skills_version, + "description": "epoch 末 momentum（不可变新版本）", + }, + ) + update_manifest(self._config.workspace_dir, skills=f"skills/{new_version}") + self._paths = resolve_paths(self._config.workspace_dir) + logger.info( + "Epoch 末 momentum → skills/{}（不改 eval 版本 {}）", + new_version, + eval_skills_version, + ) + return sorted(state.changed_task_types_this_epoch) + + async def _stage_momentum_for_file( + self, + target_file: str, + task_types: list[str], + state: _TrainState, + pools: Pools, + staged_skills: Path, + epoch: int, + ) -> None: + """单个 skill 文件的 momentum 生成：诊断池采样 → 两版 rollout → 纵向对比。""" + from app.harness.log import HarnessLog + from app.harness.momentum import run_slow_momentum + from app.harness.validate import _load_run_rows + + skill_path = staged_skills / target_file + skill_content = skill_path.read_text(encoding="utf-8") + prev_skill = state.epoch_start_skills.get(target_file, skill_content) + prev_guidance = momentum_inner(skill_content) + + # 采样 + allowed = set(task_types) + candidates = [q for q in pools.diagnosis if q.task_type in allowed] + rng = random.Random(epoch) + n = min(self._config.momentum_samples, len(candidates)) + sampled = rng.sample(candidates, n) if n > 0 else [] + + if not sampled: + skill_path.write_text( + replace_momentum(skill_content, prev_guidance or ""), + encoding="utf-8", + ) + logger.debug( + "Epoch {} momentum 跳过 {}：诊断池无匹配题型 {} 的样本", + epoch, + target_file, + sorted(task_types), + ) + return + + # 两版 rollout + prev_run_id = f"momentum_prev_e{epoch}_{target_file.replace('.md', '')}" + curr_run_id = f"momentum_curr_e{epoch}_{target_file.replace('.md', '')}" + + with tempfile.TemporaryDirectory() as prev_tmp: + prev_skills_dir = Path(prev_tmp) / "skills" + shutil.copytree(self._paths.skills_dir, prev_skills_dir, dirs_exist_ok=True) + (prev_skills_dir / target_file).write_text(prev_skill, encoding="utf-8") + await self._run_inference_on_pool( + sampled, prev_run_id, prev_skills_dir, self._paths.prompts_dir + ) + + await self._run_inference_on_pool( + sampled, curr_run_id, self._paths.skills_dir, self._paths.prompts_dir + ) + + with HarnessLog(str(self._paths.db_path), prev_run_id) as log: + prev_rows = _load_run_rows(log, prev_run_id) + with HarnessLog(str(self._paths.db_path), curr_run_id) as log: + curr_rows = _load_run_rows(log, curr_run_id) + + comparison_pairs = _build_comparison_pairs(sampled, prev_rows, curr_rows) + guidance = await run_slow_momentum( + llm=self._evolve_llm, + diagnose_prompts_dir=Path("prompts"), + skill_content=skill_content, + prev_skill=prev_skill, + prev_guidance=prev_guidance, + comparison_pairs=comparison_pairs, + ) + new_content = replace_momentum(skill_content, guidance) + skill_path.write_text(new_content, encoding="utf-8") + logger.info( + "Epoch {} momentum 写入 skill 文件 {}（题型 {}，采样 {} 题）", + epoch, + target_file, + sorted(task_types), + len(sampled), + ) + + async def _update_system_tool(self, epoch: int, state: _TrainState) -> bool: + """merge 本 epoch 累加的 system/tool 案例包 → 进化 → accept 写新 prompts 版本。 + + 返回: + 是否实际写了新 prompts 版本。 + """ + from app.harness.workspace import VersionedPromptStore + from core.evolution import evolve_single_tool, evolve_system_prompt + + merged_system = merge_system_packs(state.system_packs) + merged_tools = merge_tool_packs(state.tool_packs) + source_version = self._current_version("prompts") + max_edits = self._config.edit_budget_end + evolve_prompts = self._load_evolve_prompts() + prompt_store = VersionedPromptStore(self._paths.prompts_dir) + + records: list[EvolutionRecord] = [] + if merged_system is not None: + records.append( + await evolve_system_prompt( + self._evolve_llm, + merged_system, + prompt_store, + evolve_prompts, + source_version, + max_edits, + ) + ) + for tool_name in sorted(merged_tools): + records.append( + await evolve_single_tool( + self._evolve_llm, + merged_tools[tool_name], + prompt_store, + evolve_prompts, + source_version, + max_edits, + ) + ) + accepted = [r for r in records if r.status == "accepted"] + if not accepted: + logger.debug("Epoch {} 慢更新：无 system/tool 改动被接受", epoch) + return False + + new_version = self._write_accepted_prompts_version(accepted, source_version) + if new_version is None: + return False + update_manifest(self._config.workspace_dir, prompts=f"prompts/{new_version}") + self._paths = resolve_paths(self._config.workspace_dir) + logger.info("Epoch {} 慢更新：system/tool → prompts/{}", epoch, new_version) + return True + + def _write_accepted_prompts_version( + self, accepted: list[EvolutionRecord], source_version: str + ) -> str | None: + """将 accepted system/tool records 写成新 prompts 版本。 + + 参数: + accepted: 状态为 accepted 的 EvolutionRecord 列表。 + source_version: 改写前 prompts 版本。 + + 返回: + 新版本号，或 None（无实际变化时）。 + """ + with tempfile.TemporaryDirectory() as tmp: + staged = Path(tmp) / "prompts" + shutil.copytree(self._paths.prompts_dir, staged, dirs_exist_ok=True) + any_changed = False + for rec in accepted: + if rec.target_type == "tool": + # tool: evolved_content = json.dumps({"extract": ..., "verify": ...}) + combined = json.loads(rec.evolved_content) + for key in ("extract", "verify"): + fname = rec.target_file.replace("_extract.md", f"_{key}.md") + (staged / fname).write_text(combined[key], encoding="utf-8") + any_changed = True + else: + (staged / rec.target_file).write_text(rec.evolved_content, encoding="utf-8") + any_changed = True + if not any_changed: + return None + return advance_version( + self._paths.workspace_dir, + "prompts", + staged, + { + "source": "evolution", + "parent": source_version, + "description": "epoch 末 system/tool 慢更新", + }, + ) + + def _revert_system_tool(self, pre_prompts_version: str) -> None: + """prompts-only delta 退步时回退到更新前版本。""" + update_manifest(self._config.workspace_dir, prompts=f"prompts/{pre_prompts_version}") + self._paths = resolve_paths(self._config.workspace_dir) + logger.info( + "慢更新 prompts-only delta 退步：system/tool 回退到 prompts/{}", + pre_prompts_version, + ) + + def _refresh_gate_ladder( + self, + epoch: int, + base_run_id: str, + state: _TrainState, + extra_run_ids: list[str] | None = None, + ) -> None: + """用本 epoch 非 gate run 的逐题观测 γ-EMA 更新阶梯 p-hat 并落盘。 + + 精确三源：step rollout GLOB 排除 _gate_ + slow R + kept R2。 + """ + db_path = resolve_paths(self._config.workspace_dir).db_path + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + try: + rows = conn.execute( + "SELECT question_id, prediction, answer FROM predictions " + "WHERE run_id GLOB ? AND run_id NOT GLOB '*_gate_*' " + "ORDER BY rowid", + (f"{base_run_id}_e{epoch}_s*",), + ).fetchall() + slow_rows = conn.execute( + "SELECT question_id, prediction, answer FROM predictions " + "WHERE run_id=? ORDER BY rowid", + (f"{self._config.run_id}_slow_e{epoch}",), + ).fetchall() + extra_rows_lists = [ + conn.execute( + "SELECT question_id, prediction, answer FROM predictions " + "WHERE run_id=? ORDER BY rowid", + (rid,), + ).fetchall() + for rid in (extra_run_ids or []) + ] + finally: + conn.close() + + obs = {r["question_id"]: r["prediction"] == r["answer"] for r in rows} + obs.update({r["question_id"]: r["prediction"] == r["answer"] for r in slow_rows}) + for extra_rows in extra_rows_lists: + obs.update({r["question_id"]: r["prediction"] == r["answer"] for r in extra_rows}) + state.gate_pools.update_probs(obs, gamma=self._config.gate_gamma_decay) + state.gate_pools.save(self._config.workspace_dir / "gate_pools.json") + state.gate_epoch_observed = True + + async def _holdout_four_way( + self, + epoch: int, + pools: Pools, + state: _TrainState, + eval_skills_version: str, + eval_prompts_version: str, + ) -> None: + """四向 held-out：baseline/best_hard/final/best_mixed 各在 test 池评估。 + + test 池仅观测落库，绝不进 gate/best/early-stop/调参。 + """ + best_mixed = await self._pick_mixed_best( + epoch, pools, state, eval_skills_version, eval_prompts_version + ) + versions: dict[str, tuple[str, str] | None] = { + "baseline": (state.baseline_skills_version, state.baseline_prompts_version), + "best_hard": (state.best_skills_version, state.best_prompts_version), + "final": (eval_skills_version, eval_prompts_version), + "best_mixed": best_mixed, + } + for version_kind, version in versions.items(): + if version is None: + continue + sv, pv = version + run_id = f"{self._config.run_id}_holdout_{version_kind}_e{epoch}" + res = await self._eval_version_on_pool( + sv, pv, pools.test, run_id, context=f"held-out {version_kind}" + ) + soft = await self._try_soft_score(run_id, pools.test) + mixed = None if soft is None else 0.5 * res.accuracy + 0.5 * soft + write_holdout_eval( + str(self._paths.db_path), + run_id=self._config.run_id, + epoch=epoch, + version_kind=version_kind, + hard_acc=res.accuracy, + soft_score=soft, + mixed_score=mixed, + per_task_type_json=json.dumps(res.per_task_type, ensure_ascii=False), + ) + + async def _pick_mixed_best( + self, + epoch: int, + pools: Pools, + state: _TrainState, + eval_skills_version: str, + eval_prompts_version: str, + ) -> tuple[str, str] | None: + """在 val 池对候选集算 mixed，落 shadow_gate，返回 argmax mixed 版本。 + + 只观测落库，绝不改 manifest/best/early-stop。 + """ + candidates = { + "best_hard": (state.best_skills_version, state.best_prompts_version), + "final": (eval_skills_version, eval_prompts_version), + } + best_kind: str | None = None + best_mixed: float | None = None + for kind, (sv, pv) in candidates.items(): + run_id = f"{self._config.run_id}_shadow_{kind}_e{epoch}" + res = await self._eval_version_on_pool( + sv, pv, pools.validation, run_id, context=f"mixed 影子 {kind}" + ) + soft = await self._try_soft_score(run_id, pools.validation) + mixed = None if soft is None else 0.5 * res.accuracy + 0.5 * soft + write_shadow_gate( + str(self._paths.db_path), + run_id=self._config.run_id, + epoch=epoch, + candidate_version=f"skills/{sv}+prompts/{pv}", + hard_acc=res.accuracy, + soft_score=soft, + mixed_score=mixed, + is_mixed_best=False, + ) + if mixed is not None and (best_mixed is None or mixed > best_mixed): + best_mixed, best_kind = mixed, kind + if best_kind is None: + return None + self._mark_shadow_best(epoch, candidates[best_kind]) + return candidates[best_kind] + + def _mark_shadow_best(self, epoch: int, best_version: tuple[str, str]) -> None: + """回标 shadow_gate 中 argmax mixed 选中的版本 is_mixed_best=1。""" + sv, pv = best_version + candidate_version = f"skills/{sv}+prompts/{pv}" + conn = sqlite3.connect(str(self._paths.db_path)) + try: + conn.execute( + "UPDATE shadow_gate SET is_mixed_best=1 WHERE rowid = (" + " SELECT rowid FROM shadow_gate " + " WHERE run_id=? AND epoch=? AND candidate_version=? LIMIT 1" + ")", + (self._config.run_id, epoch, candidate_version), + ) + conn.commit() + finally: + conn.close() + + # ----------------------------------------------------------------------- + # 收尾 + # ----------------------------------------------------------------------- + + def _deliver_best(self, best_skills_version: str, best_prompts_version: str) -> None: + """若当前 current 不是历史最优，回滚 manifest 到 best 并刷新路径。""" + cur = load_manifest(self._config.workspace_dir)["current"] + if ( + cur["skills"] != f"skills/{best_skills_version}" + or cur["prompts"] != f"prompts/{best_prompts_version}" + ): + update_manifest( + self._config.workspace_dir, + skills=f"skills/{best_skills_version}", + prompts=f"prompts/{best_prompts_version}", + ) + self._paths = resolve_paths(self._config.workspace_dir) + logger.info( + "收尾交付 best：current → skills/{} prompts/{}", + best_skills_version, + best_prompts_version, + ) + + async def _final_test_eval(self, pools: Pools) -> None: + """收尾在 held-out test 池跑一次评估。""" + run_id = f"{self._config.run_id}_final_test" + result = await self._run_inference_on_pool( + pools.test, run_id, self._paths.skills_dir, self._paths.prompts_dir + ) + _guard_infra_failures(result, context="held-out test 评估") + report = { + "run_id": result.run_id, + "accuracy": result.accuracy, + "total": result.total, + "correct": result.correct, + "per_task_type": result.per_task_type, + } + path = self._config.workspace_dir / "analyses" / "final_test_eval.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + logger.info("held-out test 评估写入: {} (acc={:.1%})", path, result.accuracy) + + # ----------------------------------------------------------------------- + # 私有辅助 + # ----------------------------------------------------------------------- + + def _current_version(self, kind: str) -> str: + """读取 manifest current 指针中某类资源的当前版本名。""" + return load_manifest(self._config.workspace_dir)["current"][kind].split("/")[-1] + + def _current_version_snapshot(self) -> dict[str, str]: + """读 manifest.current 的 skills/prompts 指针。""" + cur = load_manifest(self._config.workspace_dir)["current"] + return {"skills": cur["skills"], "prompts": cur["prompts"]} + + def _class_baseline_acc( + self, + task_type: str, + validation: list[GeneratedQuestion], + correctness: dict[str, bool], + ) -> float: + """该 task_type 验证子集在当前 correctness 下的准确率。""" + class_items = [q for q in validation if q.task_type == task_type] + assert class_items, f"task_type={task_type} 在验证池中无对应题目" + correct = sum(1 for q in class_items if correctness.get(q.question_id, False)) + return correct / len(class_items) + + def _promote_skill_version(self, content: str, target_file: str) -> str: + """把候选 skill 内容写成新正式 skills 版本。""" + with tempfile.TemporaryDirectory() as tmp: + src = Path(tmp) / "skills" + shutil.copytree(self._paths.skills_dir, src, dirs_exist_ok=True) + (src / target_file).write_text(content, encoding="utf-8") + return advance_version( + self._paths.workspace_dir, + "skills", + src, + { + "source": "evolution", + "parent": self._current_version("skills"), + "description": f"按类 gate accept {target_file}", + }, + ) + + def _group_changed_task_types_by_file( + self, changed_task_types: set[str] + ) -> dict[str, list[str]]: + """把改过的 task_type 集合经 fallback 解析映射到 skill 文件，按文件分组。""" + from app.harness.workspace import VersionedSkillStore + + skill_store = VersionedSkillStore(self._paths.skills_dir) + grouped: dict[str, list[str]] = {} + for task_type in changed_task_types: + skill_file = resolve_skill_file(skill_store, task_type) + grouped.setdefault(skill_file, []).append(task_type) + return grouped + + def _record_run(self, run_id: str) -> Path: + """将 current 版本快照追加到 manifest history，创建 run 目录。""" + from app.harness.workspace import record_run + + return record_run(self._config.workspace_dir, run_id) + + def _backfill_run_versions( + self, run_id: str, skills_version: str, prompts_version: str + ) -> None: + """eval run 的 skills/prompts 版本对 + questions_ref 回填进 _runs。""" + from app.harness.log import HarnessLog + + with HarnessLog(str(self._paths.db_path), run_id) as log: + log.execute( + "UPDATE _runs SET skills_version = ?, prompts_version = ?, " + "questions_ref = ? WHERE run_id = ?", + (skills_version, prompts_version, self._config.questions, run_id), + ) + + def _write_eval_report( + self, + run_id: str, + skills_version: str, + prompts_version: str, + result: InferenceResult, + ) -> None: + """写 eval 评测报告 analyses/eval_{run_id}.json。""" + report = { + "run_id": run_id, + "skills_version": skills_version, + "prompts_version": prompts_version, + "accuracy": result.accuracy, + "total": result.total, + "correct": result.correct, + "stop_reason_counts": result.stop_reason_counts, + } + analyses_dir = self._config.workspace_dir / "analyses" + analyses_dir.mkdir(parents=True, exist_ok=True) + path = analyses_dir / f"eval_{run_id}.json" + path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + logger.info("eval 报告写入: {} (acc={:.1%})", path, result.accuracy) + + async def _run_inference_on_pool( + self, + questions: list[GeneratedQuestion], + run_id: str, + skills_dir: Path, + prompts_dir: Path, + ) -> InferenceResult: + """用指定版本在给定题池跑一次 run_inference。""" + from app.harness.inference import run_inference + from app.harness.log import HarnessLog + + self._record_run(run_id) + with HarnessLog(str(self._paths.db_path), run_id) as log: + return await run_inference( + questions=questions, + llm=self._llm, + tool_dispatch_fn=self._make_tool_dispatch_fn(skills_dir=skills_dir), + prompt_builder=self._make_prompt_builder( + skills_dir=skills_dir, prompts_dir=prompts_dir + ), + log=log, + run_id=run_id, + concurrency=self._config.concurrency, + max_steps=self._config.max_steps, + skill_mode=self._config.skill_mode, + ) + + async def _eval_version_on_pool( + self, + skills_version: str, + prompts_version: str, + questions: list[GeneratedQuestion], + run_id: str, + context: str, + ) -> InferenceResult: + """用指定版本在给定池跑一次推理并护栏。""" + skills_dir = self._paths.workspace_dir / "skills" / skills_version + prompts_dir = self._paths.workspace_dir / "prompts" / prompts_version + result = await self._run_inference_on_pool(questions, run_id, skills_dir, prompts_dir) + _guard_infra_failures(result, context=context) + return result + + async def _run_diagnosis( + self, run_id: str, *, question_ids: list[str] | None = None + ) -> DiagnosisResult: + """执行两阶段诊断。""" + from app.harness.log import RunLogImpl + from app.harness.workspace import VersionedSkillStore + from app.question_gen import load_benchmark + from core.evolution.diagnose import run_diagnosis + + questions = load_benchmark(self._paths.questions_dir) + run_log = RunLogImpl(str(self._paths.db_path)) + skill_store = VersionedSkillStore(self._paths.skills_dir) + diagnose_prompts = self._load_diagnose_prompts() + + return await run_diagnosis( + run_id=run_id, + questions=questions, + tree_data={}, # tree_data 由诊断管线内部按需加载 + llm=self._llm, + run_log=run_log, + skill_store=skill_store, + prompts=diagnose_prompts, + concurrency=self._config.concurrency, + question_ids=question_ids, + ) + + async def _try_soft_score( + self, run_id: str, questions: list[GeneratedQuestion] + ) -> float | None: + """尝试计算 soft score，失败降级为 None。""" + try: + # soft score 暂不实现（需诊断 span_evaluations 表），降级为 None + return None + except Exception: + logger.warning("soft score 计算失败（run={}），降级为 None", run_id) + return None + + # ----------------------------------------------------------------------- + # 注入工厂（暂用占位，由 main.py 绑定实际实现） + # ----------------------------------------------------------------------- + + def _make_tool_dispatch_fn(self, *, skills_dir: Path | None = None): + """构造工具调度函数（由子类或 main.py 覆盖）。""" + + async def _noop_dispatch(tool_name: str, args: dict, *, context: dict) -> str: + raise NotImplementedError( + f"工具 {tool_name} 调度未配置（需由 main.py 注入 tool_dispatch_fn）" + ) + + return _noop_dispatch + + def _make_prompt_builder( + self, *, skills_dir: Path | None = None, prompts_dir: Path | None = None + ): + """构造 prompt 构建函数（由子类或 main.py 覆盖）。""" + + def _noop_builder(qa: GeneratedQuestion) -> tuple[str, str]: + raise NotImplementedError("prompt_builder 未配置（需由 main.py 注入）") + + return _noop_builder + + def _make_validate_run_inference_fn(self): + """构造 validate 用的 RunInferenceFn（绑定共享依赖）。""" + from app.harness.inference import run_inference + from app.harness.log import HarnessLog + + async def _run( + questions: list[GeneratedQuestion], + *, + run_id: str, + skills_dir: Path, + ) -> InferenceResult: + self._record_run(run_id) + with HarnessLog(str(self._paths.db_path), run_id) as log: + return await run_inference( + questions=questions, + llm=self._llm, + tool_dispatch_fn=self._make_tool_dispatch_fn(skills_dir=skills_dir), + prompt_builder=self._make_prompt_builder( + skills_dir=skills_dir, prompts_dir=self._paths.prompts_dir + ), + log=log, + run_id=run_id, + concurrency=self._config.concurrency, + max_steps=self._config.max_steps, + skill_mode=self._config.skill_mode, + ) + + return _run + + def _load_evolve_prompts(self): + """加载进化模板束（从项目根 prompts/ 读取诊断标尺模板）。""" + from core.evolution.types import EvolvePrompts + + def _read(name: str) -> str: + p = Path("prompts") / name + return p.read_text(encoding="utf-8") if p.exists() else "" + + return EvolvePrompts( + evolve_skill=_read("evolve_skill.md"), + evolve_system=_read("evolve_system.md"), + evolve_tool=_read("evolve_tool.md"), + evolve_rank=_read("evolve_rank.md"), + consolidate_system=_read("consolidate_system.md"), + ) + + def _load_diagnose_prompts(self): + """加载诊断模板束（从项目根 prompts/ 读取）。""" + from core.evolution.types import DiagnosePrompts + + def _read(name: str) -> str: + p = Path("prompts") / name + return p.read_text(encoding="utf-8") if p.exists() else "" + + return DiagnosePrompts( + defect_vs_lapse=_read("defect_vs_lapse.md"), + reasoning_sub=_read("reasoning_sub.md"), + span_eval_system=_read("span_eval_system.md"), + span_eval_user=_read("span_eval_user.md"), + missed_nodes=_read("missed_nodes.md"), + skill_adherence=_read("skill_adherence.md"), + confirmation_bias=_read("confirmation_bias.md"), + evidence_sufficiency=_read("evidence_sufficiency.md"), + ) diff --git a/tests/unit/test_harness_runner.py b/tests/unit/test_harness_runner.py new file mode 100644 index 0000000..76e0211 --- /dev/null +++ b/tests/unit/test_harness_runner.py @@ -0,0 +1,682 @@ +"""runner.py 单元测试（算法保真 #13）。 + +覆盖 13a-13e 五个子任务，测试 Runner 骨架、三级嵌套、gate/accept/reject/probation、 +慢更新十步序、deliver_best + early stop。大部分测试用纯函数或 mock 构造避免真实推理。 +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path # noqa: TC003 — 运行时 tmp_path 标注使用 +from unittest.mock import MagicMock, patch + +import pytest + +from app.harness.runner import ( + _apply_batch_correctness, + _batch_from_ids, + _build_comparison_pairs, + _compute_total_steps, + _fallback_summary, + _format_applied_edits, + _guard_infra_failures, + _outcome_to_quadrant_pairs, + _should_early_stop, + _snapshot_current_skills, + _TrainState, + _write_skip_report, + resume_plan, +) +from app.harness.validate import Probation, ValidationOutcome +from core.evolution import RejectedEdit + +# --------------------------------------------------------------------------- +# 测试辅助 +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _FakeInferenceResult: + """InferenceResult 替身。""" + + run_id: str = "test_run" + accuracy: float = 0.5 + total: int = 10 + correct: int = 5 + per_task_type: dict = field(default_factory=dict) + steps_mean: float = 3.0 + token_usage: dict = field(default_factory=lambda: {"prompt_tokens": 0, "completion_tokens": 0}) + stop_reason_counts: dict = field(default_factory=lambda: {"finished": 10}) + + +@dataclass(frozen=True) +class _FakeQuestion: + """GeneratedQuestion 替身。""" + + question_id: str + video_id: str = "v1" + task_type: str = "Action Reasoning" + question: str = "问题" + options: tuple = ("A", "B", "C", "D") + answer: str = "A" + source_nodes: tuple = () + difficulty: str = "medium" + + +@dataclass +class _FakePools: + """Pools 替身。""" + + diagnosis: list = field(default_factory=list) + validation: list = field(default_factory=list) + test: list = field(default_factory=list) + baseline_run_id: str = "baseline_run" + baseline_val_accuracy: float = 0.5 + correctness: dict = field(default_factory=dict) + + +# ========================================================================= +# 13a: Runner 骨架 + 纯函数 +# ========================================================================= + + +class TestResumePlan: + """resume_plan 纯函数。""" + + def test_epoch_done_advances_epoch(self) -> None: + """epoch_done 阶段：下一 epoch 从头开始。""" + plan = resume_plan(epoch=3, phase="epoch_done", step_completed=5) + assert plan["first_epoch"] == 4 + assert plan["resume_epoch"] is None + assert plan["resume_step_from"] == 0 + + def test_in_epoch_resumes_same_epoch(self) -> None: + """in_epoch 阶段：从同 epoch 的下一个 step 续跑。""" + plan = resume_plan(epoch=2, phase="in_epoch", step_completed=3) + assert plan["first_epoch"] == 2 + assert plan["resume_epoch"] == 2 + assert plan["resume_step_from"] == 4 + + def test_in_epoch_step_zero(self) -> None: + """in_epoch step_completed=0：从 step 1 续跑。""" + plan = resume_plan(epoch=1, phase="in_epoch", step_completed=0) + assert plan["resume_step_from"] == 1 + + +class TestGuardInfraFailures: + """_guard_infra_failures 基础设施护栏。""" + + def test_low_error_rate_passes(self) -> None: + """error 率 <= 10% 不抛异常。""" + result = _FakeInferenceResult( + total=100, + stop_reason_counts={"finished": 95, "error": 5}, + ) + _guard_infra_failures(result, context="test") # 不应抛异常 + + def test_high_error_rate_raises(self) -> None: + """error 率 > 10% 抛 RuntimeError。""" + result = _FakeInferenceResult( + total=10, + stop_reason_counts={"finished": 8, "error": 2}, + ) + with pytest.raises(RuntimeError, match="基础设施失败率过高"): + _guard_infra_failures(result, context="test") + + def test_zero_total_does_not_crash(self) -> None: + """total=0 时不除零崩溃。""" + result = _FakeInferenceResult(total=0, stop_reason_counts={}) + _guard_infra_failures(result, context="test") # 不应抛异常 + + def test_no_error_key_passes(self) -> None: + """stop_reason_counts 无 error 键时正常通过。""" + result = _FakeInferenceResult( + total=10, + stop_reason_counts={"finished": 10}, + ) + _guard_infra_failures(result, context="test") + + +# ========================================================================= +# 13b: _apply_batch_correctness + _compute_total_steps +# ========================================================================= + + +class TestApplyBatchCorrectness: + """_apply_batch_correctness rollout 完整性护栏。""" + + def test_complete_batch_updates_correctness(self) -> None: + """完整 rollout 正常更新 correctness。""" + batch = [_FakeQuestion(question_id="q1"), _FakeQuestion(question_id="q2")] + correctness: dict[str, bool] = {} + + # mock HarnessLog + mock_log = MagicMock() + mock_log.query.return_value = [ + {"question_id": "q1", "prediction": "A", "answer": "A", "steps_json": "[]"}, + {"question_id": "q2", "prediction": "B", "answer": "A", "steps_json": "[]"}, + ] + + with patch("app.harness.validate._load_run_rows") as mock_load: + mock_load.return_value = { + "q1": {"prediction": "A", "answer": "A", "_correct": True, "steps": []}, + "q2": {"prediction": "B", "answer": "A", "_correct": False, "steps": []}, + } + _apply_batch_correctness(correctness, mock_log, "run_1", batch) + + assert correctness["q1"] is True + assert correctness["q2"] is False + + def test_missing_prediction_raises(self) -> None: + """rollout 缺预测行时抛 RuntimeError。""" + batch = [_FakeQuestion(question_id="q1"), _FakeQuestion(question_id="q2")] + correctness: dict[str, bool] = {} + + mock_log = MagicMock() + with patch("app.harness.validate._load_run_rows") as mock_load: + mock_load.return_value = { + "q1": {"prediction": "A", "answer": "A", "_correct": True, "steps": []}, + # q2 缺失 + } + with pytest.raises(RuntimeError, match="rollout 不完整"): + _apply_batch_correctness(correctness, mock_log, "run_1", batch) + + +class TestComputeTotalSteps: + """_compute_total_steps 退火地平线。""" + + def test_basic_calculation(self) -> None: + """基本退火地平线计算。""" + questions = [ + _FakeQuestion(question_id=f"q{i}", task_type="Action Reasoning") for i in range(20) + ] + # 全错题 + correctness = {q.question_id: False for q in questions} + + config = MagicMock() + config.batch_size = 5 + config.min_class_per_batch = 1 + config.batch_correct_ratio = 0.0 + config.epochs = 3 + + pools = _FakePools(diagnosis=questions, correctness=correctness) + total = _compute_total_steps(pools, correctness, config) + # 20 题 / batch_size 5 = 4 步/epoch * 3 epochs = 12 + assert total == 12 + + +# ========================================================================= +# 13b: _batch_from_ids +# ========================================================================= + + +class TestBatchFromIds: + """_batch_from_ids 按 ID 重建 batch。""" + + def test_preserves_order(self) -> None: + """按 ids 顺序取出，保持原 batch 划分。""" + q1 = _FakeQuestion(question_id="q1") + q2 = _FakeQuestion(question_id="q2") + q3 = _FakeQuestion(question_id="q3") + pools = _FakePools(diagnosis=[q1, q2, q3]) + + batch = _batch_from_ids(pools, ["q3", "q1"]) + assert [q.question_id for q in batch] == ["q3", "q1"] + + +# ========================================================================= +# 13b: _snapshot_current_skills +# ========================================================================= + + +class TestSnapshotCurrentSkills: + """_snapshot_current_skills 快照 skill 文件。""" + + def test_snapshots_md_files(self, tmp_path: Path) -> None: + """只快照 .md 文件。""" + (tmp_path / "action-reasoning.md").write_text("skill content 1") + (tmp_path / "temporal.md").write_text("skill content 2") + (tmp_path / "meta.json").write_text("{}") + + snapshot = _snapshot_current_skills(tmp_path) + assert "action-reasoning.md" in snapshot + assert "temporal.md" in snapshot + assert "meta.json" not in snapshot + assert snapshot["action-reasoning.md"] == "skill content 1" + + +# ========================================================================= +# 13c: _outcome_to_quadrant_pairs +# ========================================================================= + + +class TestOutcomeToQuadrantPairs: + """_outcome_to_quadrant_pairs 四象限拍平。""" + + def test_all_quadrants(self) -> None: + """四象限各有一个 qid 时生成 4 条 pair。""" + outcome = ValidationOutcome( + action="accept_confirmed", + accepted=True, + stop_reason="confirmed", + e_value=10.0, + w=3, + l=0, + n_used=10, + delta_hat=0.3, + delta_shrunk=0.2, + baseline_acc=0.7, + candidate_acc=0.9, + improvements=["q1"], + regressions=["q2"], + persistent_fails=["q3"], + stable_successes=["q4"], + ) + pairs = _outcome_to_quadrant_pairs("Action Reasoning", outcome) + assert len(pairs) == 4 + by_qid = {p["question_id"]: p for p in pairs} + assert by_qid["q1"]["category"] == "improved" + assert by_qid["q1"]["prev_correct"] is False + assert by_qid["q1"]["curr_correct"] is True + assert by_qid["q2"]["category"] == "regressed" + assert by_qid["q3"]["category"] == "persistent_fail" + assert by_qid["q4"]["category"] == "stable_success" + + def test_empty_outcome(self) -> None: + """四象限全空时返回空列表。""" + outcome = ValidationOutcome( + action="reject", + accepted=False, + stop_reason="directional", + e_value=0.5, + w=0, + l=2, + n_used=5, + delta_hat=-0.1, + delta_shrunk=-0.05, + baseline_acc=0.8, + candidate_acc=0.6, + ) + assert _outcome_to_quadrant_pairs("Any", outcome) == [] + + +# ========================================================================= +# 13c: _build_comparison_pairs +# ========================================================================= + + +class TestBuildComparisonPairs: + """_build_comparison_pairs momentum 纵向对比对。""" + + def test_builds_pairs(self) -> None: + """正确构造对比对。""" + sampled = [_FakeQuestion(question_id="q1", question="问题1")] + prev_rows = { + "q1": {"prediction": "A", "_correct": True}, + } + curr_rows = { + "q1": {"prediction": "B", "_correct": False}, + } + pairs = _build_comparison_pairs(sampled, prev_rows, curr_rows) + assert len(pairs) == 1 + assert pairs[0]["question"] == "问题1" + assert pairs[0]["prev_prediction"] == "A" + assert pairs[0]["curr_prediction"] == "B" + assert pairs[0]["correct_prev"] is True + assert pairs[0]["correct_curr"] is False + + def test_missing_rows_use_defaults(self) -> None: + """缺失行时使用默认值。""" + sampled = [_FakeQuestion(question_id="q1")] + pairs = _build_comparison_pairs(sampled, {}, {}) + assert pairs[0]["prev_prediction"] == "" + assert pairs[0]["correct_prev"] is False + + +# ========================================================================= +# 13e: _should_early_stop +# ========================================================================= + + +class TestShouldEarlyStop: + """_should_early_stop 步粒度 early stop。""" + + def test_improved_this_epoch_resets(self, tmp_path: Path) -> None: + """本 epoch best 刷新时重置计数器。""" + # 写 manifest + best + manifest = { + "name": "test", + "store": ".", + "current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"}, + "best": {"epoch": 2, "val_acc": 0.9}, + "history": [], + } + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + + state = MagicMock() + state.steps_since_best_improved = 10 + + result = _should_early_stop(tmp_path, epoch=2, steps_this_epoch=5, state=state, patience=20) + assert result is False + assert state.steps_since_best_improved == 0 + + def test_no_improvement_accumulates(self, tmp_path: Path) -> None: + """未刷新时累加步数。""" + manifest = { + "name": "test", + "store": ".", + "current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"}, + "best": {"epoch": 1, "val_acc": 0.5}, + "history": [], + } + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + + state = MagicMock() + state.steps_since_best_improved = 15 + + result = _should_early_stop(tmp_path, epoch=3, steps_this_epoch=5, state=state, patience=20) + assert result is True # 15 + 5 = 20 >= 20 + assert state.steps_since_best_improved == 20 + + def test_below_patience_continues(self, tmp_path: Path) -> None: + """累计步数未达阈值时继续。""" + manifest = { + "name": "test", + "store": ".", + "current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"}, + "best": {"epoch": 1, "val_acc": 0.5}, + "history": [], + } + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + + state = MagicMock() + state.steps_since_best_improved = 10 + + result = _should_early_stop(tmp_path, epoch=3, steps_this_epoch=5, state=state, patience=20) + assert result is False + assert state.steps_since_best_improved == 15 + + def test_step_granularity(self, tmp_path: Path) -> None: + """步粒度而非 epoch 粒度。""" + manifest = { + "name": "test", + "store": ".", + "current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"}, + "best": {"epoch": 1, "val_acc": 0.5}, + "history": [], + } + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + + state = MagicMock() + # 连续 3 个 epoch，每个 3 步 + state.steps_since_best_improved = 0 + for ep in range(2, 5): + stopped = _should_early_stop( + tmp_path, epoch=ep, steps_this_epoch=3, state=state, patience=10 + ) + if ep < 4: + assert stopped is False + else: + # 3+3+3=9 < 10 但第三轮后 9+3=12>=10 在 ep=5 触发 + # 实际：ep=2 → 3, ep=3 → 6, ep=4 → 9 + assert stopped is False + stopped = _should_early_stop( + tmp_path, epoch=5, steps_this_epoch=3, state=state, patience=10 + ) + assert stopped is True # 9+3=12>=10 + + +# ========================================================================= +# 13c: Probation 数据结构测试 +# ========================================================================= + + +class TestProbation: + """Probation 数据结构。""" + + def test_probation_fields(self) -> None: + """Probation 必须具备全部字段。""" + p = Probation( + task_type="Action Reasoning", + anchor_skills_version="v1", + target_file="action-reasoning.md", + correctness_snapshot={"q1": True}, + opened_step=5, + ) + assert p.task_type == "Action Reasoning" + assert p.pending_edits == [] + + def test_pending_edits_append(self) -> None: + """pending_edits 可追加 RejectedEdit。""" + p = Probation( + task_type="Action Reasoning", + anchor_skills_version="v1", + target_file="action-reasoning.md", + correctness_snapshot={}, + opened_step=0, + ) + edit = RejectedEdit( + target_file="action-reasoning.md", + target_type="skill", + change_summary="test", + delta=0.1, + source_version="v1", + epoch=0, + gate_w=3, + gate_l=1, + gate_e_value=2.5, + gate_delta_shrunk=0.05, + ) + p.pending_edits.append(edit) + assert len(p.pending_edits) == 1 + + +# ========================================================================= +# 13c: RejectedSummary 黑名单防污染 +# ========================================================================= + + +class TestFormatAppliedEdits: + """_format_applied_edits 只拼 applied 的 edit。""" + + def test_only_applied_edits_in_summary(self) -> None: + """只有 applied 状态的 edit 进入摘要。""" + record = MagicMock() + record.edits = [ + {"op": "replace", "target": "section1"}, + {"op": "insert", "content": "new_content"}, + {"op": "delete", "target": "old_stuff"}, + ] + record.apply_report = [ + {"status": "applied_exact"}, + {"status": "skipped_not_found"}, + {"status": "applied_fuzzy"}, + ] + + summary = _format_applied_edits(record) + assert summary is not None + assert "section1" in summary + assert "old_stuff" in summary + assert "new_content" not in summary + + def test_zero_applied_returns_info_message(self) -> None: + """0 applied 返回信息性消息（非 None）。""" + record = MagicMock() + record.edits = [{"op": "replace", "target": "sec"}] + record.apply_report = [{"status": "skipped_not_found"}] + + summary = _format_applied_edits(record) + assert summary is not None + assert "0 applied" in summary + + def test_no_edits_returns_none(self) -> None: + """无 edit 时返回 None。""" + record = MagicMock() + record.edits = [] + + assert _format_applied_edits(record) is None + + def test_no_report_includes_all_edits(self) -> None: + """无 apply_report 时包含所有 edit。""" + record = MagicMock() + record.edits = [{"op": "replace", "target": "foo"}] + record.apply_report = [] + + summary = _format_applied_edits(record) + assert summary is not None + assert "foo" in summary + + +class TestFallbackSummary: + """_fallback_summary 兜底黑名单摘要。""" + + def test_from_suggestions(self) -> None: + """有 suggestions 时拼接 change 字段。""" + record = MagicMock() + record.suggestions = [{"change": "改 A"}, {"change": "改 B"}] + outcome = MagicMock() + outcome.delta_hat = -0.1 + + summary = _fallback_summary(record, outcome) + assert "改 A" in summary + assert "改 B" in summary + + def test_no_suggestions_uses_delta(self) -> None: + """无 suggestions 时使用 delta 信息。""" + record = MagicMock() + record.suggestions = [] + outcome = MagicMock() + outcome.delta_hat = -0.15 + + summary = _fallback_summary(record, outcome) + assert "delta" in summary + assert "-0.15" in summary + + +class TestRejectedSummaryIntegration: + """_rejected_summary_static 集成：两个子函数组合。""" + + def test_static_delegates_to_format_applied(self) -> None: + """有 applied edit 时 static 方法返回 _format_applied_edits 结果。""" + from app.harness.runner import Runner + + record = MagicMock() + record.edits = [{"op": "replace", "target": "section1"}] + record.apply_report = [{"status": "applied_exact"}] + record.suggestions = [] + outcome = MagicMock() + outcome.delta_hat = 0.1 + + summary = Runner._rejected_summary_static(record, outcome) + assert "section1" in summary + + def test_static_falls_back_to_suggestions(self) -> None: + """无 edit 时 static 方法使用 _fallback_summary。""" + from app.harness.runner import Runner + + record = MagicMock() + record.edits = [] + record.suggestions = [{"change": "尝试 X"}] + outcome = MagicMock() + outcome.delta_hat = -0.2 + + summary = Runner._rejected_summary_static(record, outcome) + assert "尝试 X" in summary + + +class TestWriteSkipReport: + """_write_skip_report 辅助函数。""" + + def test_writes_cooldown_report(self, tmp_path: Path) -> None: + """cooldown 路径写 step_report JSON 文件。""" + (tmp_path / "analyses").mkdir() + _write_skip_report( + tmp_path, + epoch=1, + step=0, + global_step=5, + task_type="Action Reasoning", + action="cooldown", + baseline_acc=0.75, + budget=3, + ) + report_path = tmp_path / "analyses" / "step_report_e1_s0_action-reasoning.json" + assert report_path.exists() + data = json.loads(report_path.read_text()) + assert data["gate_action"] == "cooldown" + assert data["candidate_acc"] == 0.75 + assert data["gate_w"] is None + + def test_writes_skipped_report(self, tmp_path: Path) -> None: + """skipped 路径写 step_report 并传递 rank_clip_triggered。""" + (tmp_path / "analyses").mkdir() + _write_skip_report( + tmp_path, + epoch=2, + step=1, + global_step=10, + task_type="Temporal Reasoning", + action="skipped", + baseline_acc=0.6, + budget=2, + rank_clip_triggered=True, + ) + report_path = tmp_path / "analyses" / "step_report_e2_s1_temporal-reasoning.json" + assert report_path.exists() + data = json.loads(report_path.read_text()) + assert data["gate_action"] == "skipped" + assert data["rank_clip_triggered"] is True + + +# ========================================================================= +# 13a: _TrainState 基本构造 +# ========================================================================= + + +class TestTrainState: + """_TrainState dataclass 基本构造与字段默认值。""" + + def test_default_fields(self) -> None: + """默认字段值正确。""" + state = _TrainState( + correctness={"q1": True}, + gate_pools=MagicMock(), + baseline_cache=MagicMock(), + eval_prev_acc=0.5, + eval_prev_run_id="run1", + best_val_acc=0.5, + best_skills_version="v1", + best_prompts_version="v1", + ) + assert state.global_step == 0 + assert state.gate_epoch_observed is False + assert state.probations == {} + assert state.gate_cooldown == {} + assert state.rejected_buffer == {} + assert state.system_packs == [] + assert state.tool_packs == [] + assert state.changed_task_types_this_epoch == set() + assert state.steps_since_best_improved == 0 + + +# ========================================================================= +# 13c: cooldown 递减测试 +# ========================================================================= + + +class TestCooldownDecrement: + """gate_cooldown 每 step 递减、归零剔除。""" + + def test_decrement_and_remove(self) -> None: + """冷却值递减，归零剔除。""" + cooldown = {"type_a": 3, "type_b": 1, "type_c": 2} + # 模拟 _run_step 末尾的冷却递减 + cooldown = {t: n - 1 for t, n in cooldown.items() if n - 1 > 0} + assert cooldown == {"type_a": 2, "type_c": 1} + + cooldown = {t: n - 1 for t, n in cooldown.items() if n - 1 > 0} + assert cooldown == {"type_a": 1} + + cooldown = {t: n - 1 for t, n in cooldown.items() if n - 1 > 0} + assert cooldown == {} From be0e89401ecf3dfb99ec1632ab2cb35ce7b4da4d Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 13:45:48 -0400 Subject: [PATCH 18/19] feat(harness): __init__.py public API + lint fixes --- app/harness/__init__.py | 36 ++++++++++++++++++++++++++++++++++++ app/harness/checkpoint.py | 7 ++----- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/app/harness/__init__.py b/app/harness/__init__.py index e69de29..5ec5abc 100644 --- a/app/harness/__init__.py +++ b/app/harness/__init__.py @@ -0,0 +1,36 @@ +"""app/harness/ — 训练循环编排层。 + +组合 core/evolution/（决策内核）+ core/agent/（AgentLoop）+ adapters/（LLM/VLM/telemetry）， +实现自进化闭环的训练循环三级嵌套、块序贯验证、快慢双速进化、checkpoint/resume。 +""" + +from app.harness.config import RunConfig, load_config +from app.harness.inference import InferenceResult, run_inference +from app.harness.log import HarnessLog, RunLogImpl +from app.harness.pools import Pools, build_or_load_pools, build_pools, load_pools, save_pools +from app.harness.runner import Runner +from app.harness.workspace import ( + ResolvedPaths, + VersionedPromptStore, + VersionedSkillStore, + resolve_paths, +) + +__all__ = [ + "HarnessLog", + "InferenceResult", + "Pools", + "ResolvedPaths", + "RunConfig", + "RunLogImpl", + "Runner", + "VersionedPromptStore", + "VersionedSkillStore", + "build_or_load_pools", + "build_pools", + "load_config", + "load_pools", + "resolve_paths", + "run_inference", + "save_pools", +] diff --git a/app/harness/checkpoint.py b/app/harness/checkpoint.py index e389546..d08b0a5 100644 --- a/app/harness/checkpoint.py +++ b/app/harness/checkpoint.py @@ -14,9 +14,10 @@ from __future__ import annotations import json import os -from dataclasses import asdict, dataclass, field +from dataclasses import asdict from typing import TYPE_CHECKING, Any +from app.harness.validate import Probation from core.evolution.types import ( CaseSample, RejectedEdit, @@ -29,10 +30,6 @@ if TYPE_CHECKING: CHECKPOINT_SCHEMA_VERSION = 1 - -from app.harness.validate import Probation # noqa: E402 - - # --------------------------------------------------------------------------- # 结构性 / 决策性指纹键 # --------------------------------------------------------------------------- From 7a00bc1a28665a1ce157b72e4568999c19fe6f08 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 7 Jul 2026 20:34:48 -0400 Subject: [PATCH 19/19] style(harness): ruff format batching.py + log.py --- app/harness/batching.py | 4 +--- app/harness/log.py | 28 +++++++--------------------- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/app/harness/batching.py b/app/harness/batching.py index 5e24d3e..6385839 100644 --- a/app/harness/batching.py +++ b/app/harness/batching.py @@ -139,9 +139,7 @@ def _select_mixed_by_task_type( n_correct = round(len(errs) * correct_ratio / (1 - correct_ratio)) available = correct_by_type.get(task_type, []) sampled = ( - list(available) - if len(available) <= n_correct - else rng.sample(available, n_correct) + list(available) if len(available) <= n_correct else rng.sample(available, n_correct) ) grouped[task_type] = errs + sampled diff --git a/app/harness/log.py b/app/harness/log.py index 8d9f970..d2f54fd 100644 --- a/app/harness/log.py +++ b/app/harness/log.py @@ -68,9 +68,7 @@ class HarnessLog: self._conn.execute("PRAGMA journal_mode=WAL") self._init_fixed_tables() resolved_sha = git_sha or _get_git_sha() - config_json = ( - json.dumps(config_snapshot, ensure_ascii=False) if config_snapshot else None - ) + config_json = json.dumps(config_snapshot, ensure_ascii=False) if config_snapshot else None self._conn.execute( "INSERT OR IGNORE INTO _runs" " (run_id, git_sha, started_at, config, status)" @@ -143,18 +141,14 @@ class HarnessLog: col_names = ", ".join(cols) values = [enriched[c] for c in cols] if mode == "upsert": - sql = ( - f"INSERT OR REPLACE INTO {table} ({col_names}) VALUES ({placeholders})" - ) + sql = f"INSERT OR REPLACE INTO {table} ({col_names}) VALUES ({placeholders})" else: sql = f"INSERT INTO {table} ({col_names}) VALUES ({placeholders})" with self._lock: self._conn.execute(sql, values) self._conn.commit() - def insert_many( - self, table: str, records: list[dict[str, Any]], mode: str = "append" - ) -> None: + def insert_many(self, table: str, records: list[dict[str, Any]], mode: str = "append") -> None: """批量插入多条记录。 参数: @@ -192,9 +186,7 @@ class HarnessLog: with self._lock: cursor = self._conn.execute(sql, params) columns = [desc[0] for desc in cursor.description] - return [ - dict(zip(columns, row, strict=True)) for row in cursor.fetchall() - ] + return [dict(zip(columns, row, strict=True)) for row in cursor.fetchall()] def log_event(self, event_type: str, payload: dict[str, Any]) -> None: """向 _events 表写入一条事件。 @@ -205,8 +197,7 @@ class HarnessLog: """ with self._lock: self._conn.execute( - "INSERT INTO _events (run_id, timestamp, event_type, payload)" - " VALUES (?, ?, ?, ?)", + "INSERT INTO _events (run_id, timestamp, event_type, payload) VALUES (?, ?, ?, ?)", ( self._run_id, _now_iso(), @@ -280,15 +271,10 @@ def _read_table( if question_ids is not None: placeholders = ", ".join(["?"] * len(question_ids)) - sql = ( - f"SELECT * FROM {table}" - f" WHERE run_id = ? AND question_id IN ({placeholders})" - ) + sql = f"SELECT * FROM {table} WHERE run_id = ? AND question_id IN ({placeholders})" rows = conn.execute(sql, (run_id, *question_ids)).fetchall() else: - rows = conn.execute( - f"SELECT * FROM {table} WHERE run_id = ?", (run_id,) - ).fetchall() + rows = conn.execute(f"SELECT * FROM {table} WHERE run_id = ?", (run_id,)).fetchall() return [dict(r) for r in rows] finally: