merge: feat/app-harness — app/harness/ 训练循环编排层（14 文件, 算法保真 #6/#10/#13）
This commit is contained in:
@@ -0,0 +1,36 @@
|
||||
"""app/harness/ — 训练循环编排层。
|
||||
|
||||
组合 core/evolution/（决策内核）+ core/agent/（AgentLoop）+ adapters/（LLM/VLM/telemetry），
|
||||
实现自进化闭环的训练循环三级嵌套、块序贯验证、快慢双速进化、checkpoint/resume。
|
||||
"""
|
||||
|
||||
from app.harness.config import RunConfig, load_config
|
||||
from app.harness.inference import InferenceResult, run_inference
|
||||
from app.harness.log import HarnessLog, RunLogImpl
|
||||
from app.harness.pools import Pools, build_or_load_pools, build_pools, load_pools, save_pools
|
||||
from app.harness.runner import Runner
|
||||
from app.harness.workspace import (
|
||||
ResolvedPaths,
|
||||
VersionedPromptStore,
|
||||
VersionedSkillStore,
|
||||
resolve_paths,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"HarnessLog",
|
||||
"InferenceResult",
|
||||
"Pools",
|
||||
"ResolvedPaths",
|
||||
"RunConfig",
|
||||
"RunLogImpl",
|
||||
"Runner",
|
||||
"VersionedPromptStore",
|
||||
"VersionedSkillStore",
|
||||
"build_or_load_pools",
|
||||
"build_pools",
|
||||
"load_config",
|
||||
"load_pools",
|
||||
"resolve_paths",
|
||||
"run_inference",
|
||||
"save_pools",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
"""混合 mini-batch 切分：大类打散、小类整锁，供 runner 每 step 处理一个 batch。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import random
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from core.types import GeneratedQuestion
|
||||
|
||||
|
||||
def build_batches(
|
||||
items: list[GeneratedQuestion],
|
||||
correctness: dict[str, bool],
|
||||
batch_size: int,
|
||||
min_class_per_batch: int,
|
||||
seed: int,
|
||||
correct_ratio: float = 0.0,
|
||||
) -> tuple[list[list[GeneratedQuestion]], int]:
|
||||
"""把诊断池里的题目切成多个混合 mini-batch。
|
||||
|
||||
当 ``correct_ratio > 0`` 时，按题型为每组错题配比一定数量的正确题，使 batch
|
||||
包含正误混合样本（"动量"机制）；``correct_ratio <= 0`` 时退化为纯错题模式。
|
||||
|
||||
参数:
|
||||
items: 候选题目全集。
|
||||
correctness: question_id -> 基线是否答对。
|
||||
batch_size: 单个 batch 的样本数上限（> 0）。
|
||||
min_class_per_batch: 小类判定阈值——题目数 ≤ 此值的题型整组锁进单一
|
||||
batch（> 0）。
|
||||
seed: 随机种子，保证相同输入产出完全一致的切分。
|
||||
correct_ratio: 正确题占比（0.0 ~ 1.0）。0.0 = 纯错题；0.5 = 错题:正确题 = 1:1。
|
||||
返回:
|
||||
(非空 mini-batch 列表, selected_count)；无错题时返回 ([], 0)。
|
||||
selected_count 是所有 batch 中题目总数。
|
||||
异常:
|
||||
ValueError: batch_size 或 min_class_per_batch < 1, 或
|
||||
min_class_per_batch >= batch_size（破坏小类整组装箱不超容的前提）。
|
||||
关键实现细节:
|
||||
装箱顺序为「先小类后大类」。小类整组用 first-fit-decreasing 装箱：按组大小
|
||||
降序处理（同大小再按 task_type 排序保证确定性），每组放进第一个剩余容量足够
|
||||
的 batch；若现有 batch 都装不下就新开一个空 batch——因小类组大小
|
||||
≤ min_class_per_batch < batch_size，新空 batch 必能容纳，故小类装箱永不抛
|
||||
ValueError，且保证整组不拆。再把大类样本（seed 确定性 shuffle 后）round-robin
|
||||
分发到所有现存 batch 填充剩余容量。这样小类聚集于单 batch、大类散布多 batch
|
||||
且与小类共箱，自然产生多类混合 batch（纯类切片会被 multiclass 断言拒绝）。
|
||||
nb = ceil(总题数/batch_size) 是初始 batch 数下界估计而非硬上限：小类装箱可能
|
||||
新开 bin 使实际 batch 数超过 nb。每次新开 bin 都意味着总容量随之增加，故总容量
|
||||
恒 ≥ 总题数，大类 round-robin 跳过满箱后仍能放下全部样本，不会违反 batch_size
|
||||
上限。题型按名称排序处理以保证跨运行确定性，不依赖 dict 遍历顺序。
|
||||
"""
|
||||
_validate_params(batch_size, min_class_per_batch)
|
||||
|
||||
rng = random.Random(seed)
|
||||
grouped = _select_mixed_by_task_type(items, correctness, correct_ratio, rng)
|
||||
total = sum(len(g) for g in grouped.values())
|
||||
if total == 0:
|
||||
return [], 0
|
||||
|
||||
nb = max(1, math.ceil(total / batch_size))
|
||||
batches: list[list[GeneratedQuestion]] = [[] for _ in range(nb)]
|
||||
|
||||
small, large = _split_by_size(grouped, min_class_per_batch)
|
||||
for group in _small_groups_decreasing(small):
|
||||
_pack_small_class(batches, group, batch_size)
|
||||
_distribute_large_classes(batches, large, batch_size, rng)
|
||||
|
||||
result = [b for b in batches if b]
|
||||
selected_count = sum(len(b) for b in result)
|
||||
return result, selected_count
|
||||
|
||||
|
||||
def _validate_params(batch_size: int, min_class_per_batch: int) -> None:
|
||||
"""校验切分参数，非法值直接报错而非用默认值掩盖。
|
||||
|
||||
除各自 >= 1 外，强制 min_class_per_batch < batch_size：小类组大小 ≤
|
||||
min_class_per_batch，唯有此前提成立才能保证小类整组放入单一 batch 而不超容；否则
|
||||
_pack_small_class 新开的 bin 会装入超 batch_size 的整组，静默违反容量合约。此约束
|
||||
与 config._validate_minibatch 一致，是 build_batches 对自身前提的防御性自校验（P5）。
|
||||
"""
|
||||
if batch_size < 1:
|
||||
raise ValueError(f"batch_size 必须 >= 1, 实为 {batch_size}")
|
||||
if min_class_per_batch < 1:
|
||||
raise ValueError(f"min_class_per_batch 必须 >= 1, 实为 {min_class_per_batch}")
|
||||
if min_class_per_batch >= batch_size:
|
||||
raise ValueError(
|
||||
f"min_class_per_batch 必须严格 < batch_size, 否则无法保证小类整组放入单一 "
|
||||
f"batch 不超容; 实为 min_class_per_batch={min_class_per_batch}, "
|
||||
f"batch_size={batch_size}"
|
||||
)
|
||||
|
||||
|
||||
def _split_by_size(
|
||||
grouped: dict[str, list[GeneratedQuestion]],
|
||||
min_class_per_batch: int,
|
||||
) -> tuple[dict[str, list[GeneratedQuestion]], dict[str, list[GeneratedQuestion]]]:
|
||||
"""按错题数把题型分为小类（≤ 阈值）与大类（> 阈值）两组。"""
|
||||
small = {t: g for t, g in grouped.items() if len(g) <= min_class_per_batch}
|
||||
large = {t: g for t, g in grouped.items() if len(g) > min_class_per_batch}
|
||||
return small, large
|
||||
|
||||
|
||||
def _select_mixed_by_task_type(
|
||||
items: list[GeneratedQuestion],
|
||||
correctness: dict[str, bool],
|
||||
correct_ratio: float,
|
||||
rng: random.Random,
|
||||
) -> dict[str, list[GeneratedQuestion]]:
|
||||
"""按题型分组，为每组错题按比例采样正确题混入。
|
||||
|
||||
只对有错题的题型做混合——无错题的题型不进 batch，即使有正确题。
|
||||
``correct_ratio <= 0`` 时退化为纯错题模式（向后兼容）。
|
||||
|
||||
参数:
|
||||
items: 候选题目全集。
|
||||
correctness: question_id -> 基线是否答对。
|
||||
correct_ratio: 正确题占比（0.0 ~ 1.0）。
|
||||
rng: 随机数发生器，用于采样正确题。
|
||||
返回:
|
||||
task_type -> 该题型的混合题目列表（错题全部 + 按比例采样的正确题）。
|
||||
"""
|
||||
errors_by_type: dict[str, list[GeneratedQuestion]] = {}
|
||||
correct_by_type: dict[str, list[GeneratedQuestion]] = {}
|
||||
for q in items:
|
||||
qid = q.question_id
|
||||
if correctness.get(qid) is False:
|
||||
errors_by_type.setdefault(q.task_type, []).append(q)
|
||||
elif correctness.get(qid, False):
|
||||
correct_by_type.setdefault(q.task_type, []).append(q)
|
||||
|
||||
if correct_ratio <= 0:
|
||||
return errors_by_type
|
||||
|
||||
# 为每个有错题的 task_type 混入正确题
|
||||
grouped: dict[str, list[GeneratedQuestion]] = {}
|
||||
for task_type in sorted(errors_by_type):
|
||||
errs = errors_by_type[task_type]
|
||||
n_correct = round(len(errs) * correct_ratio / (1 - correct_ratio))
|
||||
available = correct_by_type.get(task_type, [])
|
||||
sampled = (
|
||||
list(available) if len(available) <= n_correct else rng.sample(available, n_correct)
|
||||
)
|
||||
grouped[task_type] = errs + sampled
|
||||
|
||||
return grouped
|
||||
|
||||
|
||||
def _small_groups_decreasing(
|
||||
small: dict[str, list[GeneratedQuestion]],
|
||||
) -> list[list[GeneratedQuestion]]:
|
||||
"""按组大小降序、同大小按 task_type 升序排出小类组（first-fit-decreasing 顺序）。
|
||||
|
||||
参数:
|
||||
small: task_type -> 小类错题列表。
|
||||
返回:
|
||||
排好序的小类组列表；降序处理可降低碎片，确定性 tie-break 保证跨运行一致。
|
||||
"""
|
||||
return [small[t] for t in sorted(small, key=lambda t: (-len(small[t]), t))]
|
||||
|
||||
|
||||
def _pack_small_class(
|
||||
batches: list[list[GeneratedQuestion]],
|
||||
group: list[GeneratedQuestion],
|
||||
batch_size: int,
|
||||
) -> None:
|
||||
"""用 first-fit 把一个小类整组放入首个容得下的 batch，装不下则新开 bin（就地修改）。
|
||||
|
||||
因小类组大小 ≤ min_class_per_batch < batch_size，新开的空 batch 必能容纳整组，
|
||||
故此函数永不抛 ValueError，且整组不拆。
|
||||
|
||||
参数:
|
||||
batches: 当前各 batch（就地追加，必要时 append 新空 batch）。
|
||||
group: 待锁定的小类错题（整组不拆）。
|
||||
batch_size: 单 batch 容量上限。
|
||||
"""
|
||||
for b in batches:
|
||||
if len(b) + len(group) <= batch_size:
|
||||
b.extend(group)
|
||||
return
|
||||
batches.append(list(group))
|
||||
|
||||
|
||||
def _distribute_large_classes(
|
||||
batches: list[list[GeneratedQuestion]],
|
||||
large: dict[str, list[GeneratedQuestion]],
|
||||
batch_size: int,
|
||||
rng: random.Random,
|
||||
) -> None:
|
||||
"""将各大类样本 shuffle 后 round-robin 分发到所有现存 batch（就地修改）。
|
||||
|
||||
参数:
|
||||
batches: 当前各 batch（含小类装箱可能新开的 bin，就地追加）。
|
||||
large: task_type -> 大类错题列表。
|
||||
batch_size: 单 batch 容量上限。
|
||||
rng: 复用的随机数发生器，保证 shuffle 确定性。
|
||||
异常:
|
||||
ValueError: 所有 batch 均满仍有样本未放置（总容量估算异常，合法输入不可达）。
|
||||
关键实现细节:
|
||||
轮转范围是「所有现存 batch」而非固定 nb 个——小类装箱新开的 bin 也参与分发。
|
||||
总容量 = 现存 batch 数 × batch_size，每次新开 bin 都同步抬高总容量，故总容量恒
|
||||
≥ 总错题数，防御性 ValueError 在合法输入下不可达。全局指针在所有大类样本间持续
|
||||
轮转（不为每类重置），满箱即跳过，使大类充分散布并与已锁定的小类共箱。题型按名称
|
||||
排序以保证分发顺序确定。
|
||||
"""
|
||||
nb = len(batches)
|
||||
pointer = 0
|
||||
for task_type in sorted(large):
|
||||
group = list(large[task_type])
|
||||
rng.shuffle(group)
|
||||
for q in group:
|
||||
pointer = _place_round_robin(batches, q, pointer, batch_size, nb)
|
||||
|
||||
|
||||
def _place_round_robin(
|
||||
batches: list[list[GeneratedQuestion]],
|
||||
q: GeneratedQuestion,
|
||||
pointer: int,
|
||||
batch_size: int,
|
||||
nb: int,
|
||||
) -> int:
|
||||
"""从 pointer 起找第一个未满 batch 放入 q，返回下一次起始指针。
|
||||
|
||||
参数:
|
||||
batches: 当前各 batch（就地追加）。
|
||||
q: 待放置的样本。
|
||||
pointer: 本次轮转起始 batch 下标。
|
||||
batch_size: 单 batch 容量上限。
|
||||
nb: batch 总数。
|
||||
返回:
|
||||
下一次轮转的起始指针（已前移一位）。
|
||||
异常:
|
||||
ValueError: 扫描一轮所有 batch 均满（总容量估算异常）。
|
||||
"""
|
||||
for offset in range(nb):
|
||||
idx = (pointer + offset) % nb
|
||||
if len(batches[idx]) < batch_size:
|
||||
batches[idx].append(q)
|
||||
return (idx + 1) % nb
|
||||
raise ValueError("所有 batch 均满仍有样本待放置, 总容量估算异常")
|
||||
@@ -0,0 +1,274 @@
|
||||
"""step 级续训 checkpoint：_TrainState 可持久化字段的序列化 / 反序列化。
|
||||
|
||||
_TrainState 的累加包均为扁平纯数据 dataclass，经 dataclasses.asdict 序列化为
|
||||
纯 JSON dict；反序列化时用 Cls(**d) 还原，其中 SystemCasePack 含嵌套 CaseSample
|
||||
列表、Probation 含嵌套 RejectedEdit 列表，需逐个重建。
|
||||
|
||||
不持久化的字段：gate_pools / baseline_cache（各自文件级自持久化，resume 时按
|
||||
指纹重载）、best_*（从 manifest best 指针读）、global_step（存 progress 块，
|
||||
由 train 单独赋值）。gate_epoch_observed 持久化：warm p-hat 在 gate_pools.json
|
||||
幸存，观测开关须随行，否则 resume 后阶梯排序回退冷启动序。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import asdict
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from app.harness.validate import Probation
|
||||
from core.evolution.types import (
|
||||
CaseSample,
|
||||
RejectedEdit,
|
||||
SystemCasePack,
|
||||
ToolCasePack,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
CHECKPOINT_SCHEMA_VERSION = 1
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 结构性 / 决策性指纹键
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_STRUCTURAL_KEYS = (
|
||||
"batch_size",
|
||||
"min_class_per_batch",
|
||||
"epochs",
|
||||
"diag_size",
|
||||
"val_size",
|
||||
"batch_correct_ratio",
|
||||
)
|
||||
|
||||
_DECISION_KEYS = (
|
||||
"edit_budget_start",
|
||||
"edit_budget_end",
|
||||
"early_stop_patience",
|
||||
"use_slow_momentum",
|
||||
"skill_update_mode",
|
||||
"appendix_consolidate_threshold",
|
||||
"momentum_samples",
|
||||
"gate_e_confirm",
|
||||
"gate_e_provisional",
|
||||
"gate_w_net_min",
|
||||
"gate_delta_min",
|
||||
"gate_lambda_dir",
|
||||
"gate_e_rollback",
|
||||
"gate_block",
|
||||
"gate_n_max",
|
||||
"gate_p_low",
|
||||
"gate_p_high",
|
||||
"gate_probe_quota",
|
||||
"gate_gamma_decay",
|
||||
"gate_cooldown_steps",
|
||||
"gate_guard_err",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 序列化 / 反序列化
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def serialize_state(state: Any) -> dict[str, Any]:
|
||||
"""把 _TrainState 的可持久化字段转为纯 JSON dict。
|
||||
|
||||
参数:
|
||||
state: _TrainState 实例（duck-typed，仅需含可持久化字段）。
|
||||
|
||||
返回:
|
||||
纯 JSON 可序列化的 dict，不含 gate_pools / baseline_cache /
|
||||
best_* / global_step。
|
||||
|
||||
关键实现细节:
|
||||
- changed_task_types_this_epoch 是 set，JSON 无 set，故 sorted 成有序列表。
|
||||
- dataclass 均经 asdict 递归转 dict（含 SystemCasePack 嵌套 CaseSample、
|
||||
Probation 嵌套 RejectedEdit）。
|
||||
"""
|
||||
return {
|
||||
"correctness": state.correctness,
|
||||
"eval_prev_acc": state.eval_prev_acc,
|
||||
"eval_prev_run_id": state.eval_prev_run_id,
|
||||
"baseline_skills_version": state.baseline_skills_version,
|
||||
"baseline_prompts_version": state.baseline_prompts_version,
|
||||
"steps_since_best_improved": state.steps_since_best_improved,
|
||||
"epoch_start_skills": state.epoch_start_skills,
|
||||
"changed_task_types_this_epoch": sorted(state.changed_task_types_this_epoch),
|
||||
"rejected_buffer": {k: [asdict(x) for x in v] for k, v in state.rejected_buffer.items()},
|
||||
"system_packs": [asdict(x) for x in state.system_packs],
|
||||
"tool_packs": [asdict(x) for x in state.tool_packs],
|
||||
"probations": {t: asdict(p) for t, p in state.probations.items()},
|
||||
"gate_cooldown": state.gate_cooldown,
|
||||
"gate_epoch_observed": state.gate_epoch_observed,
|
||||
}
|
||||
|
||||
|
||||
def _restore_system_pack(d: dict[str, Any]) -> SystemCasePack:
|
||||
"""还原 SystemCasePack，含嵌套 CaseSample 列表。
|
||||
|
||||
参数:
|
||||
d: asdict(SystemCasePack) 产出的 dict。
|
||||
|
||||
返回:
|
||||
复活的 SystemCasePack；failure_cases / success_cases 重建为 CaseSample 实例。
|
||||
"""
|
||||
return SystemCasePack(
|
||||
stats=d["stats"],
|
||||
failure_cases=[CaseSample(**c) for c in d["failure_cases"]],
|
||||
success_cases=[CaseSample(**c) for c in d["success_cases"]],
|
||||
)
|
||||
|
||||
|
||||
def deserialize_state_fields(d: dict[str, Any]) -> dict[str, Any]:
|
||||
"""把序列化 dict 还原为可填入 _TrainState 的字段字典（dataclass 复活）。
|
||||
|
||||
参数:
|
||||
d: serialize_state 产出并经 JSON 往返的 dict。
|
||||
|
||||
返回:
|
||||
字段名 -> 值的 dict，可直接铺到 _TrainState；其中各 dataclass 已复活、
|
||||
changed_task_types_this_epoch 还原为 set。
|
||||
|
||||
关键实现细节:
|
||||
- RejectedEdit / ToolCasePack 字段均为标量/dict/list[dict]，Cls(**d) 直接构造。
|
||||
- SystemCasePack 含嵌套 CaseSample，交由 _restore_system_pack 重建。
|
||||
- Probation 含嵌套 RejectedEdit 列表（pending_edits），先重建内层再构造外层。
|
||||
- 直接取 d[...] 不用 .get 兜底：serialize 后的 checkpoint 必带全部键，
|
||||
缺键即 checkpoint 损坏，应硬失败（P5 不掩盖）。
|
||||
"""
|
||||
return {
|
||||
"correctness": d["correctness"],
|
||||
"eval_prev_acc": d["eval_prev_acc"],
|
||||
"eval_prev_run_id": d["eval_prev_run_id"],
|
||||
"baseline_skills_version": d["baseline_skills_version"],
|
||||
"baseline_prompts_version": d["baseline_prompts_version"],
|
||||
"steps_since_best_improved": d["steps_since_best_improved"],
|
||||
"epoch_start_skills": d["epoch_start_skills"],
|
||||
"changed_task_types_this_epoch": set(d["changed_task_types_this_epoch"]),
|
||||
"rejected_buffer": {
|
||||
k: [RejectedEdit(**x) for x in v] for k, v in d["rejected_buffer"].items()
|
||||
},
|
||||
"system_packs": [_restore_system_pack(x) for x in d["system_packs"]],
|
||||
"tool_packs": [ToolCasePack(**x) for x in d["tool_packs"]],
|
||||
"probations": {
|
||||
t: Probation(
|
||||
**{
|
||||
**d_p,
|
||||
"pending_edits": [RejectedEdit(**x) for x in d_p["pending_edits"]],
|
||||
}
|
||||
)
|
||||
for t, d_p in d["probations"].items()
|
||||
},
|
||||
"gate_cooldown": d["gate_cooldown"],
|
||||
"gate_epoch_observed": d["gate_epoch_observed"],
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 配置指纹
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def compute_fingerprint(config: Any) -> dict[str, Any]:
|
||||
"""采集影响训练轨迹的配置项（结构性 + 决策性）。
|
||||
|
||||
参数:
|
||||
config: 训练配置对象（duck-typed，需含 _STRUCTURAL_KEYS + _DECISION_KEYS 属性）。
|
||||
|
||||
返回:
|
||||
指纹 dict，键为配置项名，值为对应配置值。
|
||||
"""
|
||||
return {k: getattr(config, k) for k in _STRUCTURAL_KEYS + _DECISION_KEYS}
|
||||
|
||||
|
||||
def check_fingerprint(saved: dict[str, Any], config: Any) -> tuple[list[str], list[str]]:
|
||||
"""比对保存的指纹与当前配置。返回 (结构性不一致项, 决策性不一致项)。
|
||||
|
||||
参数:
|
||||
saved: checkpoint 中保存的 config_fingerprint。
|
||||
config: 当前训练配置对象。
|
||||
|
||||
返回:
|
||||
(structural, decision) 两个不一致项名列表。
|
||||
|
||||
关键实现细节:
|
||||
结构性不一致（batch_size/min_class_per_batch/epochs/diag_size/val_size/
|
||||
batch_correct_ratio）→ 调用方应拒绝 resume；决策性不一致 → 仅告警放行。
|
||||
"""
|
||||
cur = compute_fingerprint(config)
|
||||
structural = [k for k in _STRUCTURAL_KEYS if saved.get(k) != cur[k]]
|
||||
decision = [k for k in _DECISION_KEYS if saved.get(k) != cur[k]]
|
||||
return structural, decision
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 读写 checkpoint
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_checkpoint(
|
||||
workspace_dir: Path,
|
||||
*,
|
||||
state: Any,
|
||||
epoch: int,
|
||||
step_completed: int,
|
||||
phase: str,
|
||||
global_step: int,
|
||||
total_steps: int,
|
||||
version_snapshot: dict[str, str],
|
||||
epoch_batches: list[list[str]],
|
||||
config: Any,
|
||||
) -> None:
|
||||
"""原子写 checkpoint.json（.tmp 再 os.replace）。
|
||||
|
||||
参数:
|
||||
workspace_dir: workspace 目录，checkpoint.json 写入其下。
|
||||
state: _TrainState 实例，交由 serialize_state 序列化。
|
||||
epoch: 当前 epoch 序号。
|
||||
step_completed: 本 epoch 内已完成的 step 数。
|
||||
phase: 续训阶段标识（如 "in_epoch"）。
|
||||
global_step: 全局 step 序号。
|
||||
total_steps: 全局总 step 数。
|
||||
version_snapshot: skills/prompts 版本快照。
|
||||
epoch_batches: 本 epoch 的 batch 划分（question_id 列表的列表）。
|
||||
config: 训练配置对象，用于计算 config_fingerprint。
|
||||
|
||||
关键实现细节:
|
||||
先写 checkpoint.json.tmp 再 os.replace，保证 checkpoint 不被写一半的中断破坏。
|
||||
"""
|
||||
payload = {
|
||||
"schema_version": CHECKPOINT_SCHEMA_VERSION,
|
||||
"progress": {
|
||||
"epoch": epoch,
|
||||
"step_completed": step_completed,
|
||||
"phase": phase,
|
||||
"global_step": global_step,
|
||||
"total_steps": total_steps,
|
||||
},
|
||||
"version_snapshot": version_snapshot,
|
||||
"epoch_batches": epoch_batches,
|
||||
"config_fingerprint": compute_fingerprint(config),
|
||||
"state": serialize_state(state),
|
||||
}
|
||||
path = workspace_dir / "checkpoint.json"
|
||||
tmp = path.with_name("checkpoint.json.tmp")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def load_checkpoint(workspace_dir: Path) -> dict[str, Any] | None:
|
||||
"""读 checkpoint.json，不存在返回 None。
|
||||
|
||||
参数:
|
||||
workspace_dir: workspace 目录。
|
||||
|
||||
返回:
|
||||
checkpoint payload dict；checkpoint.json 不存在时返回 None。
|
||||
"""
|
||||
path = workspace_dir / "checkpoint.json"
|
||||
if not path.exists():
|
||||
return None
|
||||
return json.loads(path.read_text())
|
||||
@@ -0,0 +1,427 @@
|
||||
"""运行配置：RunConfig frozen dataclass 与 YAML + CLI + .env 三层加载。
|
||||
|
||||
三层合并优先级：CLI > .env > YAML（遵循 CLAUDE.md §4.5 配置管理规范）。
|
||||
- YAML：科研实验配置（会在实验中反复扫动的参数），存放于 config/ 下。
|
||||
- .env：工程配置（少变路径如 workspace_dir、store_dir），通过环境变量注入。
|
||||
- CLI：单次临时覆盖。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
_VALID_MODES = {"infer", "train", "diagnose", "evolve", "eval", "promote"}
|
||||
_VALID_SKILL_MODES = {"auto", "manual", "none"}
|
||||
_VALID_SKILL_UPDATE_MODES = {"patch", "rewrite"}
|
||||
_PATH_FIELDS = {"workspace_dir", "store_dir"}
|
||||
|
||||
# Video-MME 的任务类型数量：验证池每类至少保底 eval_min_per_class 题，共 11 类。
|
||||
_VIDEO_MME_TASK_TYPE_COUNT = 11
|
||||
|
||||
# .env 工程配置字段映射（环境变量名 → RunConfig 字段名）。
|
||||
# 仅路径类工程配置走 .env，科研实验参数走 YAML。
|
||||
_ENV_FIELD_MAP: dict[str, str] = {
|
||||
"HARNESS_WORKSPACE_DIR": "workspace_dir",
|
||||
"HARNESS_STORE_DIR": "store_dir",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunConfig:
|
||||
"""实验运行配置，所有参数的唯一归口。
|
||||
|
||||
frozen=True 确保配置在创建后不可变，防止运行中被意外修改。
|
||||
三层合并优先级：CLI > .env > YAML。
|
||||
|
||||
字段:
|
||||
workspace_dir: Workspace 根目录。
|
||||
store_dir: Store 根目录。
|
||||
mode: 运行模式，"infer" / "train" / "diagnose" / "evolve" / "eval" / "promote"。
|
||||
concurrency: 并行 worker 数。
|
||||
max_steps: AgentLoop 单题最大步数。
|
||||
skill_mode: Skill 加载模式，"auto" / "manual" / "none"。
|
||||
n_samples: 题目截取数，0 表示全量。
|
||||
questions: 题目在 questions/ 下的相对路径。
|
||||
skills_version: Skills 版本号。
|
||||
prompts_version: Prompts 版本号。
|
||||
epochs: 训练轮数。
|
||||
diag_size: 诊断池题目数。
|
||||
diag_correct_ratio: 诊断池中正确题目占比。
|
||||
val_size: 验证池题目数。
|
||||
val_correct_ratio: 验证池中正确题目占比。
|
||||
edit_budget_start: 编辑预算前期上限。
|
||||
edit_budget_end: 编辑预算后期下限。
|
||||
batch_size: mini-batch 单批题目数。
|
||||
min_class_per_batch: 单批中每个任务类型至少保留的题目数（< batch_size）。
|
||||
eval_min_per_class: 验证池中每个任务类型至少保底的题目数。
|
||||
early_stop_patience: 全局 best 连续未提升的容忍轮数，达到即早停。
|
||||
test_size: held-out 测试池题目数。
|
||||
use_slow_momentum: 是否启用快慢双速进化中的慢速 momentum 更新。
|
||||
gate_e_confirm: CE-Gate CONFIRMED 接受的 e 值门槛（1/alpha，Ville 界假阳率 alpha）。
|
||||
gate_e_provisional: 题尽暂定接受门 + futility 提前止损的代数界。
|
||||
gate_w_net_min: 题尽暂定接受要求的最小净胜数（win - loss）。
|
||||
gate_delta_min: 最小点估计效应量下限（承接旧 margin 语义）。
|
||||
gate_lambda_dir: Wald 方向拒绝的对数似然比阈值（必须为负）。
|
||||
gate_e_rollback: 试用期对称回滚门（回滚 e 值门槛）。
|
||||
gate_block: 块序贯验证的块大小（=推理并发度，块内跑满）。
|
||||
gate_n_max: 单次 gate 消耗的题数上限。
|
||||
gate_p_low: 信息量阶梯 p-hat 保留区间下界（剔除必错零信息题）。
|
||||
gate_p_high: 信息量阶梯 p-hat 保留区间上界（剔除必对零信息题）。
|
||||
gate_probe_quota: 冷启动探针集比例（全错题中插尾的比例）。
|
||||
gate_gamma_decay: 逐题正确率估计 p-hat 的 EMA 衰减系数。
|
||||
gate_cooldown_steps: 回滚后该题型跳过进化的冷却 step 数。
|
||||
gate_guard_err: gate 内跨块累计 INFRA 错误率护栏。
|
||||
skill_update_mode: skill 进化模式，"patch"（局部 edit）/ "rewrite"（整篇重写）。
|
||||
appendix_consolidate_threshold: appendix note 条数达此值触发 LLM consolidation。
|
||||
run_id: diagnose/evolve 模式要分析的运行 ID，默认空字符串。
|
||||
batch_correct_ratio: 单批中正确题目占比，范围 [0, 1)。
|
||||
momentum_samples: 慢速 momentum 更新时从诊断池采样的题目数，必须 >= 1。
|
||||
seed: fresh 训练的种子名（对应 seed.json），默认 "initial"。
|
||||
version: eval/promote 模式指定的 store 版本号（如 "v3"）。
|
||||
resume: train 模式是否从已有 checkpoint 续训。
|
||||
fresh: train 模式是否从种子全新开始。
|
||||
"""
|
||||
|
||||
# ── 必填字段（无默认值，来自 YAML 或 CLI） ──
|
||||
workspace_dir: Path
|
||||
store_dir: Path
|
||||
mode: str
|
||||
concurrency: int
|
||||
max_steps: int
|
||||
skill_mode: str
|
||||
n_samples: int
|
||||
questions: str
|
||||
skills_version: str
|
||||
prompts_version: str
|
||||
epochs: int
|
||||
diag_size: int
|
||||
diag_correct_ratio: float
|
||||
val_size: int
|
||||
val_correct_ratio: float
|
||||
edit_budget_start: int
|
||||
edit_budget_end: int
|
||||
batch_size: int
|
||||
min_class_per_batch: int
|
||||
eval_min_per_class: int
|
||||
early_stop_patience: int
|
||||
test_size: int
|
||||
use_slow_momentum: bool
|
||||
gate_e_confirm: float
|
||||
gate_e_provisional: float
|
||||
gate_w_net_min: int
|
||||
gate_delta_min: float
|
||||
gate_lambda_dir: float
|
||||
gate_e_rollback: float
|
||||
gate_block: int
|
||||
gate_n_max: int
|
||||
gate_p_low: float
|
||||
gate_p_high: float
|
||||
gate_probe_quota: float
|
||||
gate_gamma_decay: float
|
||||
gate_cooldown_steps: int
|
||||
gate_guard_err: float
|
||||
skill_update_mode: str
|
||||
appendix_consolidate_threshold: int
|
||||
|
||||
# ── 有默认值的字段（通常由 CLI 传入或可选） ──
|
||||
run_id: str = ""
|
||||
batch_correct_ratio: float = 0.5
|
||||
momentum_samples: int = 20
|
||||
seed: str = "initial"
|
||||
version: str = ""
|
||||
resume: bool = False
|
||||
fresh: bool = False
|
||||
|
||||
|
||||
def _validate(config: RunConfig) -> None:
|
||||
"""校验 RunConfig 全部字段约束。
|
||||
|
||||
六层校验链：mode → 基础标量 → 编辑预算 → mini-batch → gate 阈值 → gate 阶梯。
|
||||
|
||||
参数:
|
||||
config: 待校验的配置实例。
|
||||
|
||||
异常:
|
||||
ValueError: 任一字段值不合法。
|
||||
"""
|
||||
_validate_mode(config)
|
||||
_validate_mode_deps(config)
|
||||
_validate_basic(config)
|
||||
_validate_edit_budget(config)
|
||||
_validate_minibatch(config)
|
||||
_validate_gate(config)
|
||||
|
||||
|
||||
def _validate_mode(config: RunConfig) -> None:
|
||||
"""校验运行模式枚举合法性。
|
||||
|
||||
参数:
|
||||
config: 待校验的配置实例。
|
||||
|
||||
异常:
|
||||
ValueError: mode 值不在合法集合中。
|
||||
"""
|
||||
if config.mode not in _VALID_MODES:
|
||||
raise ValueError(f"mode 必须为 {_VALID_MODES} 之一，实际: {config.mode!r}")
|
||||
|
||||
|
||||
def _validate_mode_deps(config: RunConfig) -> None:
|
||||
"""校验各运行模式的依赖字段（run_id、version）。
|
||||
|
||||
参数:
|
||||
config: 待校验的配置实例。
|
||||
|
||||
异常:
|
||||
ValueError: 模式依赖字段缺失。
|
||||
"""
|
||||
if config.mode in ("diagnose", "evolve") and not config.run_id:
|
||||
raise ValueError(f"mode 为 {config.mode!r} 时必须提供 run_id。")
|
||||
if config.mode in ("eval", "promote") and not config.version:
|
||||
raise ValueError(f"mode 为 {config.mode!r} 时必须提供 --version。")
|
||||
if config.mode == "promote" and not config.run_id:
|
||||
raise ValueError("promote 必须提供 --run-id（指定 canonical eval run）。")
|
||||
_validate_train_run_id(config)
|
||||
|
||||
|
||||
def _validate_train_run_id(config: RunConfig) -> None:
|
||||
"""校验 train 模式非 resume/fresh 时必须提供 run_id。
|
||||
|
||||
参数:
|
||||
config: 待校验的配置实例。
|
||||
|
||||
异常:
|
||||
ValueError: train 模式既非 resume 也非 fresh 且缺少 run_id。
|
||||
"""
|
||||
if config.mode != "train":
|
||||
return
|
||||
if config.resume or config.fresh:
|
||||
return
|
||||
if not config.run_id:
|
||||
raise ValueError("train 非 resume/fresh 时必须提供 run_id（旧式基线 run）。")
|
||||
|
||||
|
||||
def _validate_basic(config: RunConfig) -> None:
|
||||
"""校验基础标量字段：枚举合法性与正整数约束。
|
||||
|
||||
参数:
|
||||
config: 待校验的配置实例。
|
||||
|
||||
异常:
|
||||
ValueError: 任一基础字段值不合法。
|
||||
"""
|
||||
if config.skill_mode not in _VALID_SKILL_MODES:
|
||||
raise ValueError(
|
||||
f"skill_mode 必须为 {_VALID_SKILL_MODES} 之一，实际: {config.skill_mode!r}"
|
||||
)
|
||||
if config.concurrency <= 0:
|
||||
raise ValueError(f"concurrency 必须 > 0，实际: {config.concurrency}")
|
||||
if config.max_steps <= 0:
|
||||
raise ValueError(f"max_steps 必须 > 0，实际: {config.max_steps}")
|
||||
if config.n_samples < 0:
|
||||
raise ValueError(f"n_samples 必须 >= 0，实际: {config.n_samples}")
|
||||
if config.epochs <= 0:
|
||||
raise ValueError(f"epochs 必须 > 0，实际: {config.epochs}")
|
||||
if config.skill_update_mode not in _VALID_SKILL_UPDATE_MODES:
|
||||
raise ValueError(
|
||||
f"skill_update_mode 必须为 {_VALID_SKILL_UPDATE_MODES} 之一，"
|
||||
f"实际: {config.skill_update_mode!r}"
|
||||
)
|
||||
if config.appendix_consolidate_threshold < 1:
|
||||
raise ValueError(
|
||||
f"appendix_consolidate_threshold 必须 >= 1，"
|
||||
f"实际: {config.appendix_consolidate_threshold}"
|
||||
)
|
||||
|
||||
|
||||
def _validate_edit_budget(config: RunConfig) -> None:
|
||||
"""校验编辑预算退火的前期/后期上限约束。
|
||||
|
||||
参数:
|
||||
config: 待校验的配置实例。
|
||||
|
||||
异常:
|
||||
ValueError: edit_budget_start < edit_budget_end，或 end <= 0。
|
||||
"""
|
||||
if config.edit_budget_start < config.edit_budget_end:
|
||||
raise ValueError(
|
||||
f"edit_budget_start({config.edit_budget_start}) 必须 >= "
|
||||
f"edit_budget_end({config.edit_budget_end})"
|
||||
)
|
||||
if config.edit_budget_end <= 0:
|
||||
raise ValueError(f"edit_budget_end 必须 > 0，实际: {config.edit_budget_end}")
|
||||
|
||||
|
||||
def _validate_minibatch(config: RunConfig) -> None:
|
||||
"""校验 mini-batch 自进化闭环参数约束。
|
||||
|
||||
参数:
|
||||
config: 待校验的 RunConfig 配置对象。
|
||||
|
||||
异常:
|
||||
ValueError: 任一约束被违反。
|
||||
|
||||
关键实现细节:
|
||||
val_size 必须 >= eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT，保证验证池
|
||||
能为 Video-MME 的全部 11 个任务类型各保底 eval_min_per_class 题。
|
||||
"""
|
||||
if config.batch_size <= 0:
|
||||
raise ValueError(f"batch_size 必须 > 0，实际: {config.batch_size}")
|
||||
if not (1 <= config.min_class_per_batch < config.batch_size):
|
||||
raise ValueError(
|
||||
f"min_class_per_batch 必须满足 1 <= 值 < batch_size"
|
||||
f"({config.batch_size})，实际: {config.min_class_per_batch}"
|
||||
)
|
||||
if config.eval_min_per_class < 1:
|
||||
raise ValueError(f"eval_min_per_class 必须 >= 1，实际: {config.eval_min_per_class}")
|
||||
floor = config.eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT
|
||||
if config.val_size < floor:
|
||||
raise ValueError(
|
||||
f"val_size 必须 >= eval_min_per_class * {_VIDEO_MME_TASK_TYPE_COUNT}"
|
||||
f"(={floor})：Video-MME 共 {_VIDEO_MME_TASK_TYPE_COUNT} 个任务类型，"
|
||||
f"每类需 eval_min_per_class 题保底，故验证池下限为 {floor}，"
|
||||
f"实际: {config.val_size}"
|
||||
)
|
||||
if config.early_stop_patience <= 0:
|
||||
raise ValueError(f"early_stop_patience 必须 > 0，实际: {config.early_stop_patience}")
|
||||
if config.test_size <= 0:
|
||||
raise ValueError(f"test_size 必须 > 0，实际: {config.test_size}")
|
||||
if not (0 <= config.batch_correct_ratio < 1):
|
||||
raise ValueError(
|
||||
f"batch_correct_ratio 必须满足 0 <= 值 < 1，实际: {config.batch_correct_ratio}"
|
||||
)
|
||||
if config.momentum_samples < 1:
|
||||
raise ValueError(f"momentum_samples 必须 >= 1，实际: {config.momentum_samples}")
|
||||
|
||||
|
||||
def _validate_gate(config: RunConfig) -> None:
|
||||
"""校验 CE-Gate 全部参数：判据阈值 + 信息量阶梯。
|
||||
|
||||
参数:
|
||||
config: 待校验的配置实例。
|
||||
|
||||
异常:
|
||||
ValueError: 任一 gate 参数不合法。
|
||||
"""
|
||||
_validate_gate_thresholds(config)
|
||||
_validate_gate_ladder(config)
|
||||
|
||||
|
||||
def _validate_gate_thresholds(config: RunConfig) -> None:
|
||||
"""校验 CE-Gate 判据阈值参数（e 值、净胜数、效应量、方向拒绝）。
|
||||
|
||||
参数:
|
||||
config: 待校验的配置实例。
|
||||
|
||||
异常:
|
||||
ValueError: 任一阈值参数不合法。
|
||||
"""
|
||||
if config.gate_e_confirm <= 1:
|
||||
raise ValueError(f"gate_e_confirm 必须 > 1，实际: {config.gate_e_confirm}")
|
||||
if not (1 < config.gate_e_provisional <= config.gate_e_confirm):
|
||||
raise ValueError(
|
||||
f"gate_e_provisional 必须在 (1, gate_e_confirm] 内，实际: {config.gate_e_provisional}"
|
||||
)
|
||||
if config.gate_e_rollback <= 1:
|
||||
raise ValueError(f"gate_e_rollback 必须 > 1，实际: {config.gate_e_rollback}")
|
||||
if config.gate_w_net_min < 1:
|
||||
raise ValueError(f"gate_w_net_min 必须 >= 1，实际: {config.gate_w_net_min}")
|
||||
if config.gate_delta_min < 0:
|
||||
raise ValueError(f"gate_delta_min 必须 >= 0，实际: {config.gate_delta_min}")
|
||||
if config.gate_lambda_dir >= 0:
|
||||
raise ValueError(f"gate_lambda_dir 必须 < 0，实际: {config.gate_lambda_dir}")
|
||||
|
||||
|
||||
def _validate_gate_ladder(config: RunConfig) -> None:
|
||||
"""校验 CE-Gate 信息量阶梯与块序贯参数。
|
||||
|
||||
参数:
|
||||
config: 待校验的配置实例。
|
||||
|
||||
异常:
|
||||
ValueError: 任一阶梯参数不合法。
|
||||
"""
|
||||
if config.gate_block <= 0 or config.gate_n_max < config.gate_block:
|
||||
raise ValueError(
|
||||
f"需 0 < gate_block <= gate_n_max，"
|
||||
f"实际: block={config.gate_block}, n_max={config.gate_n_max}"
|
||||
)
|
||||
if not (0 <= config.gate_p_low < config.gate_p_high <= 1):
|
||||
raise ValueError(
|
||||
f"需 0 <= gate_p_low < gate_p_high <= 1，"
|
||||
f"实际: [{config.gate_p_low}, {config.gate_p_high}]"
|
||||
)
|
||||
if not (0 <= config.gate_probe_quota <= 1):
|
||||
raise ValueError(f"gate_probe_quota 须在 [0,1]，实际: {config.gate_probe_quota}")
|
||||
if not (0 < config.gate_gamma_decay < 1):
|
||||
raise ValueError(f"gate_gamma_decay 须在 (0,1)，实际: {config.gate_gamma_decay}")
|
||||
if config.gate_cooldown_steps < 1:
|
||||
raise ValueError(f"gate_cooldown_steps 必须 >= 1，实际: {config.gate_cooldown_steps}")
|
||||
if not (0 < config.gate_guard_err < 1):
|
||||
raise ValueError(f"gate_guard_err 须在 (0,1)，实际: {config.gate_guard_err}")
|
||||
|
||||
|
||||
def _apply_env_var_overrides(data: dict) -> None:
|
||||
"""从环境变量覆盖路径字段（原地修改）。
|
||||
|
||||
.env 文件由入口脚本 load_dotenv 加载到环境变量，本函数仅从 os.environ 读取。
|
||||
仅覆盖 _ENV_FIELD_MAP 中声明的工程配置字段（workspace_dir、store_dir）。
|
||||
|
||||
参数:
|
||||
data: 待覆盖的配置字典。
|
||||
"""
|
||||
for env_key, field_name in _ENV_FIELD_MAP.items():
|
||||
env_val = os.environ.get(env_key)
|
||||
if env_val is not None:
|
||||
data[field_name] = env_val
|
||||
|
||||
|
||||
def load_config(
|
||||
yaml_path: Path,
|
||||
cli_overrides: dict[str, object] | None = None,
|
||||
) -> RunConfig:
|
||||
"""从 YAML 加载配置，叠加 .env 和 CLI 覆盖层后构造 RunConfig。
|
||||
|
||||
三层合并优先级：CLI > .env > YAML。
|
||||
|
||||
参数:
|
||||
yaml_path: YAML 配置文件路径，需包含 ``harness`` 段。
|
||||
cli_overrides: CLI 参数字典，值为 None 表示未传入（不覆盖）。
|
||||
|
||||
返回:
|
||||
构造并校验后的 RunConfig 实例。
|
||||
|
||||
异常:
|
||||
FileNotFoundError: YAML 文件不存在。
|
||||
ValueError: 校验失败。
|
||||
"""
|
||||
# Phase 1: 加载 YAML 基础层
|
||||
with open(yaml_path, encoding="utf-8") as f:
|
||||
raw: dict = yaml.safe_load(f)
|
||||
|
||||
# 支持嵌套 harness 段和扁平 YAML 两种格式
|
||||
yaml_data: dict = raw.get("harness", raw)
|
||||
|
||||
# Phase 2: .env 覆盖层（仅工程配置字段）
|
||||
_apply_env_var_overrides(yaml_data)
|
||||
|
||||
# Phase 3: CLI 覆盖层（最高优先级）
|
||||
valid_fields = {f.name for f in dataclasses.fields(RunConfig)}
|
||||
if cli_overrides:
|
||||
for key, value in cli_overrides.items():
|
||||
if value is not None and key in valid_fields:
|
||||
yaml_data[key] = value
|
||||
|
||||
# Phase 4: 类型转换 — 路径字段转 Path
|
||||
for field_name in _PATH_FIELDS:
|
||||
if field_name in yaml_data:
|
||||
yaml_data[field_name] = Path(yaml_data[field_name])
|
||||
|
||||
# Phase 5: 构造并校验
|
||||
config = RunConfig(**{k: v for k, v in yaml_data.items() if k in valid_fields})
|
||||
_validate(config)
|
||||
return config
|
||||
@@ -0,0 +1,335 @@
|
||||
"""CE-Gate 信息量阶梯与基线缓存。
|
||||
|
||||
阶梯（每题型一条）：gate 的出题顺序表。冷启动（FRESH）用种子基线对错
|
||||
两档粗排（错题高优先 2:1 交错 + 全错题 probe_quota 探针插尾）；
|
||||
epoch >=1 用非 gate run 观测做 gamma-EMA 更新 p_hat，按信息量 p_hat(1-p_hat) 降序、
|
||||
剔 p_hat 不在 [p_low, p_high]。防泄露铁律：gate 内 rollout 永不回流 p_hat
|
||||
（调用方以 run_id 含 "_gate_" 过滤观测源）。
|
||||
|
||||
BaselineCache：基线侧逐题对错缓存，键 = (task_type, skill_hash,
|
||||
prompts_version, qid) 内容寻址、无显式失效。JSON 持久化到 workspace，
|
||||
供 resume 后合法复用已冻结阶梯上的新鲜 draw。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from loguru import logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from core.types import GeneratedQuestion
|
||||
|
||||
|
||||
def skill_hash(content: str) -> str:
|
||||
"""对 skill 正文取 sha1 摘要，作缓存键的内容维度。
|
||||
|
||||
参数:
|
||||
content: skill 文件全文（基线侧为解析后生效文件的正文）。
|
||||
|
||||
返回:
|
||||
sha1 十六进制摘要。
|
||||
"""
|
||||
return hashlib.sha1(content.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
@dataclass
|
||||
class LadderEntry:
|
||||
"""阶梯单元：题目与其估计答对率。
|
||||
|
||||
字段:
|
||||
question_id: 题目唯一标识。
|
||||
p_hat: 估计答对率。冷启动为 Beta(1,1) 平滑的单次观测后验均值
|
||||
（错=1/3、对=2/3），此后经 gamma-EMA 更新。
|
||||
"""
|
||||
|
||||
question_id: str
|
||||
p_hat: float
|
||||
|
||||
|
||||
def build_cold_entries(
|
||||
questions: list[GeneratedQuestion],
|
||||
correctness: dict[str, bool],
|
||||
probe_quota: float,
|
||||
seed: int,
|
||||
) -> list[LadderEntry]:
|
||||
"""冷启动排序：错题高优先 2:1 交错 + 全错题 probe_quota 探针插尾。
|
||||
|
||||
参数:
|
||||
questions: 该题型的全部候选题（已排除 test 池）。
|
||||
correctness: question_id -> 种子基线是否答对（900 题全量对错）。
|
||||
probe_quota: 从错题中随机抽出插到梯尾的探针比例（防"解锁新能力"盲区）。
|
||||
seed: 洗牌种子，保证确定性重建。
|
||||
|
||||
返回:
|
||||
排序后的 LadderEntry 列表（p_hat 用 Beta(1,1) 平滑：错=1/3、对=2/3，
|
||||
与 warm 阶段 gamma-EMA / 信息量排序自然衔接）。
|
||||
|
||||
关键实现细节:
|
||||
错题、对题各自固定种子洗牌 -> 抽探针 -> 剩余按 错错对 2:1 交错
|
||||
（一方耗尽后顺排另一方）-> 探针追加尾部。
|
||||
"""
|
||||
rng = random.Random(seed)
|
||||
wrong = [q for q in questions if not correctness.get(q.question_id, False)]
|
||||
right = [q for q in questions if correctness.get(q.question_id, False)]
|
||||
rng.shuffle(wrong)
|
||||
rng.shuffle(right)
|
||||
|
||||
n_probe = int(len(wrong) * probe_quota)
|
||||
probes, wrong_main = wrong[:n_probe], wrong[n_probe:]
|
||||
|
||||
interleaved: list[GeneratedQuestion] = []
|
||||
wi, ri = 0, 0
|
||||
while wi < len(wrong_main) or ri < len(right):
|
||||
for _ in range(2):
|
||||
if wi < len(wrong_main):
|
||||
interleaved.append(wrong_main[wi])
|
||||
wi += 1
|
||||
if ri < len(right):
|
||||
interleaved.append(right[ri])
|
||||
ri += 1
|
||||
interleaved.extend(probes)
|
||||
|
||||
def _p0(q: GeneratedQuestion) -> float:
|
||||
return 2 / 3 if correctness.get(q.question_id, False) else 1 / 3
|
||||
|
||||
return [LadderEntry(q.question_id, _p0(q)) for q in interleaved]
|
||||
|
||||
|
||||
def order_ladder(entries: list[LadderEntry], p_low: float, p_high: float) -> list[LadderEntry]:
|
||||
"""warm 排序：剔 p_hat 不在 [p_low, p_high] 的零信息题，按信息量 p_hat(1-p_hat) 降序。
|
||||
|
||||
参数:
|
||||
entries: 待排序的阶梯单元。
|
||||
p_low / p_high: p_hat 保留区间。
|
||||
|
||||
返回:
|
||||
过滤并排序后的新列表（稳定排序，同信息量保持原相对序）。
|
||||
"""
|
||||
kept = [e for e in entries if p_low <= e.p_hat <= p_high]
|
||||
return sorted(kept, key=lambda e: e.p_hat * (1 - e.p_hat), reverse=True)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GatePools:
|
||||
"""全部题型的阶梯容器，含构建种子与数据指纹（确定性重建凭据）。
|
||||
|
||||
字段:
|
||||
entries: task_type -> 冷启动序 LadderEntry 列表（warm 排序在取用时做，
|
||||
保持存储序稳定、避免每次更新重写全表顺序）。
|
||||
seed: 冷启动洗牌种子。
|
||||
fingerprint: 构建输入指纹（基线 run_id + 题集 hash 等），resume 校验用。
|
||||
"""
|
||||
|
||||
entries: dict[str, list[LadderEntry]]
|
||||
seed: int
|
||||
fingerprint: str
|
||||
|
||||
def ladder_for(
|
||||
self,
|
||||
task_type: str,
|
||||
exclude_qids: set[str],
|
||||
p_low: float,
|
||||
p_high: float,
|
||||
cold: bool,
|
||||
) -> list[str]:
|
||||
"""取该题型的 gate 出题序（qid 列表），排除本 step 进化案例包题。
|
||||
|
||||
参数:
|
||||
task_type: 目标题型。
|
||||
exclude_qids: 本 step 案例包（failure/success cases）的题目 id，
|
||||
防止在"刚学的那道题"上自测。
|
||||
p_low / p_high: warm 阶段的 p_hat 保留区间。
|
||||
cold: True 表示尚无 epoch 级观测（epoch 1），用冷启动存储序；
|
||||
False 走 order_ladder 信息量排序。
|
||||
|
||||
返回:
|
||||
排除后的有序 question_id 列表。
|
||||
|
||||
异常:
|
||||
ValueError: 该题型无阶梯（冷启动构建缺失），或该题型阶梯为空。
|
||||
"""
|
||||
if task_type not in self.entries:
|
||||
raise ValueError(f"task_type={task_type} 无阶梯，冷启动构建缺失该题型")
|
||||
pool = self.entries[task_type]
|
||||
if not pool:
|
||||
raise ValueError(f"task_type={task_type} 阶梯为空，无可出题目")
|
||||
ordered = pool if cold else order_ladder(pool, p_low, p_high)
|
||||
return [e.question_id for e in ordered if e.question_id not in exclude_qids]
|
||||
|
||||
def update_probs(self, observations: dict[str, bool], gamma: float) -> None:
|
||||
"""gamma-EMA 更新 p_hat：p_hat <- gamma * p_hat + (1-gamma) * obs。只更新有新观测的题。
|
||||
|
||||
参数:
|
||||
observations: question_id -> 本 epoch 非 gate run 的最新对错。
|
||||
调用方必须已按 run_id 过滤掉 gate 内 rollout（防泄露铁律）。
|
||||
gamma: EMA 衰减系数。
|
||||
"""
|
||||
for entries in self.entries.values():
|
||||
for e in entries:
|
||||
if e.question_id in observations:
|
||||
obs = 1.0 if observations[e.question_id] else 0.0
|
||||
e.p_hat = gamma * e.p_hat + (1 - gamma) * obs
|
||||
|
||||
def save(self, path: Path) -> None:
|
||||
"""原子写 gate_pools.json（.tmp 再 replace）。
|
||||
|
||||
参数:
|
||||
path: 目标 JSON 路径。
|
||||
"""
|
||||
payload = {
|
||||
"seed": self.seed,
|
||||
"fingerprint": self.fingerprint,
|
||||
"entries": {
|
||||
t: [{"question_id": e.question_id, "p_hat": e.p_hat} for e in es]
|
||||
for t, es in self.entries.items()
|
||||
},
|
||||
}
|
||||
tmp = path.with_suffix(".json.tmp")
|
||||
tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> GatePools:
|
||||
"""从 gate_pools.json 恢复。
|
||||
|
||||
参数:
|
||||
path: gate_pools.json 路径。
|
||||
|
||||
返回:
|
||||
复活的 GatePools。
|
||||
"""
|
||||
d = json.loads(path.read_text(encoding="utf-8"))
|
||||
return cls(
|
||||
entries={
|
||||
t: [LadderEntry(x["question_id"], x["p_hat"]) for x in es]
|
||||
for t, es in d["entries"].items()
|
||||
},
|
||||
seed=d["seed"],
|
||||
fingerprint=d["fingerprint"],
|
||||
)
|
||||
|
||||
|
||||
def build_or_load_gate_pools(
|
||||
workspace_dir: Path,
|
||||
questions: list[GeneratedQuestion],
|
||||
test_qids: set[str],
|
||||
baseline_correctness: dict[str, bool],
|
||||
task_types: list[str],
|
||||
probe_quota: float,
|
||||
seed: int,
|
||||
baseline_run_id: str,
|
||||
) -> GatePools:
|
||||
"""gate 阶梯获取入口：gate_pools.json 存在且指纹一致则加载，否则冷启动构建。
|
||||
|
||||
参数:
|
||||
workspace_dir: workspace 根目录（gate_pools.json 落其下）。
|
||||
questions: benchmark 全量题（900 题）。
|
||||
test_qids: held-out test 池题目 id（阶梯题源必须排除）。
|
||||
baseline_correctness: 种子基线 900 题全量对错（从基线 run 的 db 读）。
|
||||
task_types: 参与进化的题型列表。
|
||||
probe_quota: 冷启动探针比例。
|
||||
seed: 冷启动洗牌种子。
|
||||
baseline_run_id: 指纹成分。
|
||||
|
||||
返回:
|
||||
GatePools。
|
||||
|
||||
关键实现细节:
|
||||
指纹 = sha1(baseline_run_id|全 qid|seed|probe_quota|task_types|test_qids)。
|
||||
指纹不一致（题集/基线/参数变了）直接报错——FRESH 语义下不该发生，
|
||||
防御性拒绝而非静默重建。
|
||||
"""
|
||||
joined = ",".join(sorted(q.question_id for q in questions))
|
||||
fp_src = (
|
||||
f"{baseline_run_id}|{joined}|{seed}|{probe_quota}"
|
||||
f"|{','.join(sorted(task_types))}|{','.join(sorted(test_qids))}"
|
||||
)
|
||||
fingerprint = hashlib.sha1(fp_src.encode()).hexdigest()
|
||||
path = workspace_dir / "gate_pools.json"
|
||||
if path.exists():
|
||||
pools = GatePools.load(path)
|
||||
if pools.fingerprint != fingerprint:
|
||||
raise RuntimeError(
|
||||
f"gate_pools.json 指纹不一致（题集或基线变更），拒绝静默重建: {path}"
|
||||
)
|
||||
return pools
|
||||
|
||||
entries: dict[str, list[LadderEntry]] = {}
|
||||
for t in task_types:
|
||||
pool = [q for q in questions if q.task_type == t and q.question_id not in test_qids]
|
||||
if not pool:
|
||||
raise ValueError(f"task_type={t} 无非 test 题，无法建阶梯")
|
||||
entries[t] = build_cold_entries(pool, baseline_correctness, probe_quota, seed)
|
||||
logger.info("gate 阶梯[{}]: {} 题（冷启动）", t, len(entries[t]))
|
||||
pools = GatePools(entries=entries, seed=seed, fingerprint=fingerprint)
|
||||
pools.save(path)
|
||||
return pools
|
||||
|
||||
|
||||
class BaselineCache:
|
||||
"""基线侧逐题对错缓存（内容寻址，JSON 持久化）。
|
||||
|
||||
键 = (task_type, skill_hash, prompts_version, qid)：任何影响该题型
|
||||
有效 skill 的变化（含共享 default-strategy.md 被他类 accept 改写）
|
||||
都使 skill_hash 变化、缓存自然 miss；prompts 版本变化同理。
|
||||
"""
|
||||
|
||||
def __init__(self, path: Path) -> None:
|
||||
"""加载或初始化缓存文件。
|
||||
|
||||
参数:
|
||||
path: 缓存 JSON 路径（workspace/baseline_cache.json）。
|
||||
"""
|
||||
self._path = path
|
||||
self._store: dict[str, bool] = {}
|
||||
if path.exists():
|
||||
self._store = json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
@staticmethod
|
||||
def _key(task_type: str, s_hash: str, prompts_version: str, qid: str) -> str:
|
||||
"""拼缓存键（四维内容寻址）。"""
|
||||
return f"{task_type}|{s_hash}|{prompts_version}|{qid}"
|
||||
|
||||
def get(self, task_type: str, s_hash: str, prompts_version: str, qid: str) -> bool | None:
|
||||
"""读缓存；未命中返回 None。
|
||||
|
||||
参数:
|
||||
task_type: 题型。
|
||||
s_hash: 基线侧生效 skill 文件的内容哈希。
|
||||
prompts_version: 当前 prompts 版本。
|
||||
qid: 题目 id。
|
||||
|
||||
返回:
|
||||
缓存的对错；未命中 None。
|
||||
"""
|
||||
return self._store.get(self._key(task_type, s_hash, prompts_version, qid))
|
||||
|
||||
def put(
|
||||
self, task_type: str, s_hash: str, prompts_version: str, qid: str, correct: bool
|
||||
) -> None:
|
||||
"""写缓存并落盘（原子写，gate 频度低、全量重写成本可忽略）。
|
||||
|
||||
参数:
|
||||
task_type / s_hash / prompts_version / qid: 缓存键四维。
|
||||
correct: 基线侧该题对错。
|
||||
|
||||
关键实现细节:
|
||||
先盘后存：新条目先原子落盘（tmp 写 + os.replace）成功后才更新
|
||||
内存，磁盘写失败时内存与磁盘一致（均无新条目），无分裂窗口。
|
||||
"""
|
||||
updated = {
|
||||
**self._store,
|
||||
self._key(task_type, s_hash, prompts_version, qid): correct,
|
||||
}
|
||||
tmp = self._path.with_suffix(".json.tmp")
|
||||
tmp.write_text(json.dumps(updated, ensure_ascii=False), encoding="utf-8")
|
||||
os.replace(tmp, self._path)
|
||||
self._store = updated
|
||||
@@ -0,0 +1,441 @@
|
||||
"""async 推理编排 — 训练循环的 forward()。
|
||||
|
||||
从 TRM4 core/harness/inference.py (~560 行) 迁移，重大重构:
|
||||
- 同步 ThreadPoolExecutor → asyncio.Semaphore + asyncio.gather
|
||||
- LLMClient.from_env() 每题构造 → llm: LLMProvider 注入共享
|
||||
- SentenceTransformer/OCR 内部构造 → 调用方通过 tool_dispatch_fn 注入
|
||||
- run_id 必传，空串 → ValueError
|
||||
- _aggregate_results 从内存 results 聚合（非 DB 回读）
|
||||
- record_run 由调用方（Runner）负责
|
||||
- prompt 构建由调用方注入 prompt_builder
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from core.agent.loop import AgentLoop
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from app.harness.log import HarnessLog
|
||||
from core.agent.types import LoopResult
|
||||
from core.protocols import LLMProvider
|
||||
from core.types import GeneratedQuestion
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InferenceResult:
|
||||
"""推理聚合结果。
|
||||
|
||||
属性:
|
||||
run_id: 运行标识。
|
||||
accuracy: 总正确率。
|
||||
total: 总题数。
|
||||
correct: 正确题数。
|
||||
per_task_type: 按题型分组的指标 {task_type: {accuracy, total, correct}}。
|
||||
steps_mean: 平均步数。
|
||||
token_usage: token 总用量 {prompt_tokens, completion_tokens}。
|
||||
stop_reason_counts: 终止原因计数 {reason: count}。
|
||||
"""
|
||||
|
||||
run_id: str
|
||||
accuracy: float
|
||||
total: int
|
||||
correct: int
|
||||
per_task_type: dict[str, dict]
|
||||
steps_mean: float
|
||||
token_usage: dict[str, int]
|
||||
stop_reason_counts: dict[str, int]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 表 Schema 定义（5 张表，保留 TRM4 全部 schema）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PREDICTIONS_SCHEMA: dict[str, str] = {
|
||||
"video_id": "TEXT",
|
||||
"question_id": "TEXT",
|
||||
"task_type": "TEXT",
|
||||
"prediction": "TEXT",
|
||||
"answer": "TEXT",
|
||||
"evidence": "TEXT",
|
||||
"reasoning": "TEXT",
|
||||
"steps_used": "INTEGER",
|
||||
"prompt_tokens": "INTEGER",
|
||||
"completion_tokens": "INTEGER",
|
||||
"stop_reason": "TEXT",
|
||||
"steps_json": "JSON",
|
||||
}
|
||||
|
||||
TRACES_SCHEMA: dict[str, str] = {
|
||||
"video_id": "TEXT",
|
||||
"question_id": "TEXT",
|
||||
"step": "INTEGER",
|
||||
"tool_name": "TEXT",
|
||||
"tool_args": "JSON",
|
||||
"tool_output": "TEXT",
|
||||
"thought": "TEXT",
|
||||
}
|
||||
|
||||
VALIDATION_FLAGS_SCHEMA: dict[str, str] = {
|
||||
"video_id": "TEXT",
|
||||
"question_id": "TEXT",
|
||||
"has_l3_visit": "INTEGER",
|
||||
"l1_count": "INTEGER",
|
||||
"l2_count": "INTEGER",
|
||||
"l3_count": "INTEGER",
|
||||
}
|
||||
|
||||
ANCHOR_CHECK_SCHEMA: dict[str, str] = {
|
||||
"video_id": "TEXT",
|
||||
"question_id": "TEXT",
|
||||
"step": "INTEGER",
|
||||
"n_assertions": "INTEGER",
|
||||
"n_anchored": "INTEGER",
|
||||
"n_illegal": "INTEGER",
|
||||
"n_expanded": "INTEGER",
|
||||
"n_trunc": "INTEGER",
|
||||
"output_chars": "INTEGER",
|
||||
}
|
||||
|
||||
OF_HEALTH_SCHEMA: dict[str, str] = {
|
||||
"video_id": "TEXT",
|
||||
"question_id": "TEXT",
|
||||
"step": "INTEGER",
|
||||
"ocr_injected": "INTEGER",
|
||||
"ocr_chars": "INTEGER",
|
||||
"ocr_failed": "INTEGER",
|
||||
"discrepancy": "INTEGER",
|
||||
"abstain": "INTEGER",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 内部工具
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _DispatcherAdapter:
|
||||
"""将裸 async callable 包装为 ToolDispatcher Protocol 实例。
|
||||
|
||||
AgentLoop 要求 ToolDispatcher（有 dispatch 方法），而 run_inference
|
||||
接收的 tool_dispatch_fn 是裸 async callable。此适配器桥接两者。
|
||||
|
||||
参数:
|
||||
fn: async def (tool_name, args, *, context) -> str。
|
||||
"""
|
||||
|
||||
def __init__(self, fn: Callable[..., Any]) -> None:
|
||||
self._fn = fn
|
||||
|
||||
async def dispatch(
|
||||
self, tool_name: str, args: dict[str, Any], *, context: dict[str, Any]
|
||||
) -> str:
|
||||
"""转发工具调用给被包装的 callable。"""
|
||||
return await self._fn(tool_name, args, context=context)
|
||||
|
||||
|
||||
def _to_text_field(value: Any) -> str:
|
||||
"""把 prediction 的 evidence/reasoning 归一为可入库的文本。
|
||||
|
||||
LLM 有时把这些字段返回成 list 或 dict（而非字符串）。sqlite 无法绑定
|
||||
非标量类型，直接入库会抛 ProgrammingError 致该题丢失预测行、进而触发
|
||||
rollout 完整性护栏中止整轮。凡非 str 一律 JSON 序列化为文本。
|
||||
|
||||
参数:
|
||||
value: evidence/reasoning 原始值（可能是 str/list/dict）。
|
||||
|
||||
返回:
|
||||
可直接入库的字符串。
|
||||
"""
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def _zero_result(run_id: str) -> InferenceResult:
|
||||
"""空记录时的零值 InferenceResult。
|
||||
|
||||
参数:
|
||||
run_id: 运行标识。
|
||||
|
||||
返回:
|
||||
全零的 InferenceResult。
|
||||
"""
|
||||
return InferenceResult(
|
||||
run_id=run_id,
|
||||
accuracy=0.0,
|
||||
total=0,
|
||||
correct=0,
|
||||
per_task_type={},
|
||||
steps_mean=0.0,
|
||||
token_usage={"prompt_tokens": 0, "completion_tokens": 0},
|
||||
stop_reason_counts={},
|
||||
)
|
||||
|
||||
|
||||
def _group_by_task_type(records: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
"""按 task_type 分组聚合正确率指标。
|
||||
|
||||
参数:
|
||||
records: 预测记录列表。
|
||||
|
||||
返回:
|
||||
{task_type: {accuracy, total, correct}} 映射。
|
||||
"""
|
||||
task_groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for r in records:
|
||||
task_groups[r["task_type"]].append(r)
|
||||
|
||||
per_task_type: dict[str, dict[str, Any]] = {}
|
||||
for task_type, group in task_groups.items():
|
||||
t_total = len(group)
|
||||
t_correct = sum(1 for r in group if r["prediction"] == r["answer"])
|
||||
per_task_type[task_type] = {
|
||||
"accuracy": t_correct / t_total,
|
||||
"total": t_total,
|
||||
"correct": t_correct,
|
||||
}
|
||||
return per_task_type
|
||||
|
||||
|
||||
def _aggregate_results(records: list[dict[str, Any]], run_id: str) -> InferenceResult:
|
||||
"""从内存 records 聚合推理指标。
|
||||
|
||||
TRM4 从 DB 回读 predictions 表聚合；TRM5 改为从内存直接聚合，
|
||||
避免 DB 回读的同步开销和额外依赖。
|
||||
|
||||
参数:
|
||||
records: _run_single_question 返回的 record 列表。
|
||||
run_id: 当前运行标识。
|
||||
|
||||
返回:
|
||||
InferenceResult 冻结实例。
|
||||
"""
|
||||
total = len(records)
|
||||
if total == 0:
|
||||
return _zero_result(run_id)
|
||||
|
||||
correct = sum(1 for r in records if r["prediction"] == r["answer"])
|
||||
stop_counts: dict[str, int] = defaultdict(int)
|
||||
for r in records:
|
||||
stop_counts[r["stop_reason"]] += 1
|
||||
|
||||
return InferenceResult(
|
||||
run_id=run_id,
|
||||
accuracy=correct / total,
|
||||
total=total,
|
||||
correct=correct,
|
||||
per_task_type=_group_by_task_type(records),
|
||||
steps_mean=sum(r["steps_used"] for r in records) / total,
|
||||
token_usage={
|
||||
"prompt_tokens": sum(r["prompt_tokens"] for r in records),
|
||||
"completion_tokens": sum(r["completion_tokens"] for r in records),
|
||||
},
|
||||
stop_reason_counts=dict(stop_counts),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 单题推理
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _run_single_question(
|
||||
qa: GeneratedQuestion,
|
||||
*,
|
||||
llm: LLMProvider,
|
||||
tool_dispatch_fn: Callable[..., Any],
|
||||
prompt_builder: Callable[[GeneratedQuestion], tuple[str, str]],
|
||||
log: HarnessLog,
|
||||
max_steps: int,
|
||||
plugins: list[object],
|
||||
) -> dict[str, Any]:
|
||||
"""执行单道题目的 Agent 推理。
|
||||
|
||||
悲观默认值：record 初始 stop_reason="error"，成功后覆盖。
|
||||
prediction 必落库：log.insert 在 try/except 之后（无论成败）。
|
||||
|
||||
参数:
|
||||
qa: 待推理的题目。
|
||||
llm: LLMProvider 共享实例。
|
||||
tool_dispatch_fn: async 工具调度函数 (tool_name, args, *, context) -> str。
|
||||
prompt_builder: (GeneratedQuestion) -> (system_prompt, user_prompt)。
|
||||
log: HarnessLog 实例（线程安全）。
|
||||
max_steps: AgentLoop 最大步数。
|
||||
plugins: pluggy 插件列表。
|
||||
|
||||
返回:
|
||||
预测结果字典（含 video_id, question_id, prediction, answer 等）。
|
||||
"""
|
||||
record: dict[str, Any] = {
|
||||
"video_id": qa.video_id,
|
||||
"question_id": qa.question_id,
|
||||
"task_type": qa.task_type,
|
||||
"prediction": None,
|
||||
"answer": qa.answer,
|
||||
"evidence": "",
|
||||
"reasoning": "",
|
||||
"steps_used": 0,
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"stop_reason": "error", # 悲观默认
|
||||
"steps_json": "[]",
|
||||
}
|
||||
|
||||
try:
|
||||
system_prompt, user_prompt = prompt_builder(qa)
|
||||
dispatcher = _DispatcherAdapter(tool_dispatch_fn)
|
||||
loop = AgentLoop(llm, max_steps=max_steps)
|
||||
loop_result: LoopResult = await loop.run(
|
||||
system_prompt,
|
||||
user_prompt,
|
||||
dispatcher,
|
||||
plugins=plugins,
|
||||
session_id=qa.question_id,
|
||||
)
|
||||
|
||||
result_dict = loop_result.result if isinstance(loop_result.result, dict) else {}
|
||||
evidence = _to_text_field(result_dict.get("evidence", ""))
|
||||
reasoning = _to_text_field(result_dict.get("reasoning", ""))
|
||||
record.update(
|
||||
{
|
||||
"prediction": result_dict.get("answer"),
|
||||
"evidence": evidence,
|
||||
"reasoning": reasoning,
|
||||
"steps_used": loop_result.steps_used,
|
||||
"prompt_tokens": loop_result.token_usage["prompt_tokens"],
|
||||
"completion_tokens": loop_result.token_usage["completion_tokens"],
|
||||
"stop_reason": loop_result.stop_reason,
|
||||
"steps_json": json.dumps(
|
||||
[
|
||||
{
|
||||
"thought": s.thought,
|
||||
"tool_call": s.tool_call,
|
||||
"tool_output": s.tool_output,
|
||||
}
|
||||
for s in loop_result.steps
|
||||
],
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("[{}] QA {} 执行异常", qa.video_id, qa.question_id)
|
||||
|
||||
# prediction 必落库（try 外，无论成败）
|
||||
await asyncio.to_thread(log.insert, "predictions", record)
|
||||
return record
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 建表
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _ensure_tables(log: HarnessLog) -> None:
|
||||
"""创建推理所需的 5 张表。
|
||||
|
||||
参数:
|
||||
log: HarnessLog 实例。
|
||||
"""
|
||||
log.create_table("predictions", PREDICTIONS_SCHEMA)
|
||||
log.create_table("traces", TRACES_SCHEMA)
|
||||
log.create_table("validation_flags", VALIDATION_FLAGS_SCHEMA)
|
||||
log.create_table("anchor_check", ANCHOR_CHECK_SCHEMA)
|
||||
log.create_table("observe_frame_health", OF_HEALTH_SCHEMA)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 公共入口
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def run_inference(
|
||||
questions: list[GeneratedQuestion],
|
||||
*,
|
||||
llm: LLMProvider,
|
||||
tool_dispatch_fn: Callable[..., Any],
|
||||
prompt_builder: Callable[[GeneratedQuestion], tuple[str, str]],
|
||||
log: HarnessLog,
|
||||
run_id: str,
|
||||
concurrency: int,
|
||||
max_steps: int,
|
||||
skill_mode: str,
|
||||
plugins_factory: Callable[[str, str], list[object]] | None = None,
|
||||
) -> InferenceResult:
|
||||
"""在视频树上执行 Agent 推理，对应训练循环的 forward()。
|
||||
|
||||
参数:
|
||||
questions: 待推理的题目列表。
|
||||
llm: LLMProvider 共享实例（依赖注入）。
|
||||
tool_dispatch_fn: async 工具调度函数 (tool_name, args, *, context) -> str。
|
||||
prompt_builder: prompt 构建函数 (GeneratedQuestion) -> (system_prompt, user_prompt)。
|
||||
log: HarnessLog 实例（由调用方管理生命周期）。
|
||||
run_id: 运行标识（必传，空串 → ValueError）。
|
||||
concurrency: 最大并发数（asyncio.Semaphore 控制）。
|
||||
max_steps: AgentLoop 单题最大步数。
|
||||
skill_mode: "auto" / "manual" / "none"（传递给调用方的 prompt/plugin 构建逻辑）。
|
||||
plugins_factory: 可选的插件工厂 (video_id, question_id) -> plugins 列表。
|
||||
|
||||
返回:
|
||||
InferenceResult（含 accuracy、per_task_type 等聚合指标）。
|
||||
|
||||
异常:
|
||||
ValueError: run_id 为空串或纯空白。
|
||||
"""
|
||||
if not run_id or not run_id.strip():
|
||||
raise ValueError("run_id 不得为空串或纯空白")
|
||||
|
||||
_ensure_tables(log)
|
||||
|
||||
if not questions:
|
||||
logger.info("题目列表为空，返回零值 InferenceResult")
|
||||
return _aggregate_results([], run_id)
|
||||
|
||||
sem = asyncio.Semaphore(concurrency)
|
||||
total_count = len(questions)
|
||||
|
||||
async def _bounded(index: int, qa: GeneratedQuestion) -> dict[str, Any]:
|
||||
"""信号量限流的单题推理包装。"""
|
||||
async with sem:
|
||||
plugins = (
|
||||
plugins_factory(qa.video_id, qa.question_id) if plugins_factory is not None else []
|
||||
)
|
||||
result = await _run_single_question(
|
||||
qa,
|
||||
llm=llm,
|
||||
tool_dispatch_fn=tool_dispatch_fn,
|
||||
prompt_builder=prompt_builder,
|
||||
log=log,
|
||||
max_steps=max_steps,
|
||||
plugins=plugins,
|
||||
)
|
||||
logger.info(
|
||||
"[{}/{}] {} QA {} 完成 (stop={})",
|
||||
index + 1,
|
||||
total_count,
|
||||
qa.video_id,
|
||||
qa.question_id,
|
||||
result["stop_reason"],
|
||||
)
|
||||
return result
|
||||
|
||||
results = await asyncio.gather(*[_bounded(i, qa) for i, qa in enumerate(questions)])
|
||||
|
||||
inference_result = _aggregate_results(list(results), run_id)
|
||||
logger.info(
|
||||
"推理完成: accuracy={:.2%} ({}/{})",
|
||||
inference_result.accuracy,
|
||||
inference_result.correct,
|
||||
inference_result.total,
|
||||
)
|
||||
return inference_result
|
||||
@@ -0,0 +1,333 @@
|
||||
"""HarnessLog：SQLite 薄包装 + RunLogImpl 只读查询端口。
|
||||
|
||||
HarnessLog 提供统一的结构化日志接口，从 TRM4 直搬，保留全部线程安全与幂等语义。
|
||||
RunLogImpl 实现 core/evolution/protocols.py::RunLog Protocol，用独立连接做只读 SELECT，
|
||||
不经 HarnessLog 生命周期（不触发 _runs INSERT OR IGNORE），避免污染运行状态。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import threading
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _get_git_sha() -> str | None:
|
||||
"""获取当前 git commit SHA。"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
return result.stdout.strip()
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
return None
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
"""返回当前 UTC 时间的 ISO 格式字符串。"""
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
class HarnessLog:
|
||||
"""SQLite 薄包装，为科研项目提供统一的结构化日志接口。
|
||||
|
||||
关键设计：
|
||||
- WAL 模式 + threading.Lock 保证共享连接下并发安全。
|
||||
- INSERT OR IGNORE INTO _runs 保证幂等（同 run_id 多次创建不报错）。
|
||||
- query 也持锁：共享连接（check_same_thread=False）下并发 SELECT + INSERT
|
||||
在同一连接上 execute 会损坏游标状态，故读也须串行化。
|
||||
- context manager 语义：正常退出 completed，异常退出 failed。
|
||||
|
||||
参数:
|
||||
db_path: SQLite 数据库文件路径。
|
||||
run_id: 本次运行的唯一标识。
|
||||
git_sha: 代码版本，默认自动获取。
|
||||
config_snapshot: 本次运行的配置快照。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db_path: str,
|
||||
run_id: str,
|
||||
git_sha: str | None = None,
|
||||
config_snapshot: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
self._run_id = run_id
|
||||
Path(db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
self._conn = sqlite3.connect(db_path, check_same_thread=False)
|
||||
self._lock = threading.Lock()
|
||||
self._conn.row_factory = sqlite3.Row
|
||||
self._conn.execute("PRAGMA journal_mode=WAL")
|
||||
self._init_fixed_tables()
|
||||
resolved_sha = git_sha or _get_git_sha()
|
||||
config_json = json.dumps(config_snapshot, ensure_ascii=False) if config_snapshot else None
|
||||
self._conn.execute(
|
||||
"INSERT OR IGNORE INTO _runs"
|
||||
" (run_id, git_sha, started_at, config, status)"
|
||||
" VALUES (?, ?, ?, ?, ?)",
|
||||
(run_id, resolved_sha, _now_iso(), config_json, "running"),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def _init_fixed_tables(self) -> None:
|
||||
"""创建 _runs 和 _events 固定表。"""
|
||||
self._conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS _runs (
|
||||
run_id TEXT PRIMARY KEY,
|
||||
git_sha TEXT,
|
||||
started_at TEXT,
|
||||
finished_at TEXT,
|
||||
config JSON,
|
||||
status TEXT DEFAULT 'running',
|
||||
skills_version TEXT,
|
||||
prompts_version TEXT,
|
||||
questions_ref TEXT
|
||||
)
|
||||
""")
|
||||
self._conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS _events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_id TEXT,
|
||||
timestamp TEXT,
|
||||
event_type TEXT,
|
||||
payload JSON
|
||||
)
|
||||
""")
|
||||
self._conn.commit()
|
||||
|
||||
def create_table(
|
||||
self,
|
||||
name: str,
|
||||
columns: dict[str, str],
|
||||
primary_key: str | None = None,
|
||||
) -> None:
|
||||
"""创建自定义表，自动追加 run_id 和 timestamp 列。
|
||||
|
||||
参数:
|
||||
name: 表名。
|
||||
columns: 列定义，如 {"epoch": "INTEGER", "loss": "REAL"}。
|
||||
primary_key: 主键列名。
|
||||
"""
|
||||
all_columns = {"run_id": "TEXT", "timestamp": "TEXT"}
|
||||
all_columns.update(columns)
|
||||
col_defs = []
|
||||
for col_name, col_type in all_columns.items():
|
||||
pk_suffix = " PRIMARY KEY" if col_name == primary_key else ""
|
||||
col_defs.append(f"{col_name} {col_type}{pk_suffix}")
|
||||
sql = f"CREATE TABLE IF NOT EXISTS {name} ({', '.join(col_defs)})"
|
||||
self._conn.execute(sql)
|
||||
self._conn.commit()
|
||||
|
||||
def insert(self, table: str, record: dict[str, Any], mode: str = "append") -> None:
|
||||
"""插入一条记录，自动填充 run_id 和 timestamp。
|
||||
|
||||
参数:
|
||||
table: 目标表名。
|
||||
record: 要插入的数据。
|
||||
mode: "append" 或 "upsert"。
|
||||
"""
|
||||
enriched = {"run_id": self._run_id, "timestamp": _now_iso()}
|
||||
enriched.update(record)
|
||||
cols = list(enriched.keys())
|
||||
placeholders = ", ".join(["?"] * len(cols))
|
||||
col_names = ", ".join(cols)
|
||||
values = [enriched[c] for c in cols]
|
||||
if mode == "upsert":
|
||||
sql = f"INSERT OR REPLACE INTO {table} ({col_names}) VALUES ({placeholders})"
|
||||
else:
|
||||
sql = f"INSERT INTO {table} ({col_names}) VALUES ({placeholders})"
|
||||
with self._lock:
|
||||
self._conn.execute(sql, values)
|
||||
self._conn.commit()
|
||||
|
||||
def insert_many(self, table: str, records: list[dict[str, Any]], mode: str = "append") -> None:
|
||||
"""批量插入多条记录。
|
||||
|
||||
参数:
|
||||
table: 目标表名。
|
||||
records: 要插入的数据列表。
|
||||
mode: "append" 或 "upsert"。
|
||||
"""
|
||||
for record in records:
|
||||
self.insert(table, record, mode=mode)
|
||||
|
||||
def execute(self, sql: str, params: tuple[Any, ...] = ()) -> None:
|
||||
"""执行原生 SQL 写操作。
|
||||
|
||||
参数:
|
||||
sql: SQL 语句。
|
||||
params: 参数元组。
|
||||
"""
|
||||
with self._lock:
|
||||
self._conn.execute(sql, params)
|
||||
self._conn.commit()
|
||||
|
||||
def query(self, sql: str, params: tuple[Any, ...] = ()) -> list[dict[str, Any]]:
|
||||
"""执行原生 SQL 查询，返回 list[dict]。
|
||||
|
||||
与所有写方法同持 self._lock：共享连接（check_same_thread=False）下，
|
||||
并发 SELECT 与 INSERT 在同一连接上 execute 会损坏游标状态，故读也须串行化。
|
||||
|
||||
参数:
|
||||
sql: SQL 查询语句。
|
||||
params: 参数元组。
|
||||
|
||||
返回:
|
||||
查询结果列表，每行为一个字典。
|
||||
"""
|
||||
with self._lock:
|
||||
cursor = self._conn.execute(sql, params)
|
||||
columns = [desc[0] for desc in cursor.description]
|
||||
return [dict(zip(columns, row, strict=True)) for row in cursor.fetchall()]
|
||||
|
||||
def log_event(self, event_type: str, payload: dict[str, Any]) -> None:
|
||||
"""向 _events 表写入一条事件。
|
||||
|
||||
参数:
|
||||
event_type: 事件类型标识。
|
||||
payload: 事件数据。
|
||||
"""
|
||||
with self._lock:
|
||||
self._conn.execute(
|
||||
"INSERT INTO _events (run_id, timestamp, event_type, payload) VALUES (?, ?, ?, ?)",
|
||||
(
|
||||
self._run_id,
|
||||
_now_iso(),
|
||||
event_type,
|
||||
json.dumps(payload, ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
self._conn.commit()
|
||||
|
||||
def close(self, status: str = "completed") -> None:
|
||||
"""更新运行状态并关闭连接。
|
||||
|
||||
参数:
|
||||
status: 最终状态，"completed" 或 "failed"。
|
||||
"""
|
||||
with self._lock:
|
||||
self._conn.execute(
|
||||
"UPDATE _runs SET finished_at = ?, status = ? WHERE run_id = ?",
|
||||
(_now_iso(), status, self._run_id),
|
||||
)
|
||||
self._conn.commit()
|
||||
self._conn.close()
|
||||
|
||||
def __enter__(self) -> HarnessLog:
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_val: BaseException | None,
|
||||
exc_tb: Any,
|
||||
) -> None:
|
||||
status = "failed" if exc_type is not None else "completed"
|
||||
self.close(status=status)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# RunLogImpl — core/evolution/protocols.py::RunLog 的只读实现
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _read_table(
|
||||
db_path: str,
|
||||
table: str,
|
||||
run_id: str,
|
||||
*,
|
||||
question_ids: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""纯读某表指定 run 的行——不经 HarnessLog 生命周期，避免回读污染 _runs 运行状态。
|
||||
|
||||
HarnessLog.__enter__/__exit__ 会对 run_id 做 INSERT OR IGNORE 并在退出时标 completed；
|
||||
回读指标绝不应改运行状态，故走独立只读连接（仅 SELECT）。
|
||||
|
||||
参数:
|
||||
db_path: SQLite 路径。
|
||||
table: 表名（内部固定常量，非外部输入，无注入风险）。
|
||||
run_id: 过滤的 run ID。
|
||||
question_ids: 可选的 question_id 过滤列表。
|
||||
|
||||
返回:
|
||||
行 dict 列表；表尚未建（没写过）视为无数据返 []。
|
||||
"""
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
exists = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table,)
|
||||
).fetchone()
|
||||
if exists is None:
|
||||
return []
|
||||
|
||||
if question_ids is not None:
|
||||
placeholders = ", ".join(["?"] * len(question_ids))
|
||||
sql = f"SELECT * FROM {table} WHERE run_id = ? AND question_id IN ({placeholders})"
|
||||
rows = conn.execute(sql, (run_id, *question_ids)).fetchall()
|
||||
else:
|
||||
rows = conn.execute(f"SELECT * FROM {table} WHERE run_id = ?", (run_id,)).fetchall()
|
||||
|
||||
return [dict(r) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
class RunLogImpl:
|
||||
"""RunLog Protocol 的只读实现。
|
||||
|
||||
用独立 sqlite3.connect 做 SELECT，不经 HarnessLog 生命周期（不触发 _runs INSERT），
|
||||
asyncio.to_thread 包装同步 SQL 查询，避免引入 aiosqlite 新依赖。
|
||||
|
||||
参数:
|
||||
db_path: SQLite 数据库文件路径。
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str) -> None:
|
||||
self._db_path = db_path
|
||||
|
||||
async def get_predictions(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
question_ids: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""查询指定 run 的预测记录。
|
||||
|
||||
参数:
|
||||
run_id: 运行标识。
|
||||
question_ids: 可选的题目 ID 过滤列表。
|
||||
|
||||
返回:
|
||||
预测记录字典列表。
|
||||
"""
|
||||
return await asyncio.to_thread(
|
||||
_read_table, self._db_path, "predictions", run_id, question_ids=question_ids
|
||||
)
|
||||
|
||||
async def get_traces(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
question_ids: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""查询指定 run 的推理轨迹。
|
||||
|
||||
参数:
|
||||
run_id: 运行标识。
|
||||
question_ids: 可选的题目 ID 过滤列表。
|
||||
|
||||
返回:
|
||||
轨迹记录字典列表。
|
||||
"""
|
||||
return await asyncio.to_thread(
|
||||
_read_table, self._db_path, "traces", run_id, question_ids=question_ids
|
||||
)
|
||||
@@ -0,0 +1,178 @@
|
||||
"""慢更新动量生成 — epoch 末为单个 skill 产出新的动量指导。
|
||||
|
||||
对标 SkillOpt 的 slow_update 机制：拿上一 epoch 末与当前 epoch 末两版 skill，
|
||||
在固定样本上各跑一遍得到纵向对比（comparison_pairs），反思上一轮动量指导是否奏效、
|
||||
本轮正文改动是改善还是漂移，据此重写动量指导。新指导经 patch 引擎的 replace_momentum
|
||||
写回 skill 的 momentum 受保护区，作为下一轮进化的方向锚。
|
||||
|
||||
从 TRM4 core/harness/momentum.py（156 行）迁移 + async 化。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from core.evolution.diagnose import extract_json_from_response
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from core.protocols import LLMProvider
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 四类纵向对比类别名（单一真源）
|
||||
# =========================================================================
|
||||
|
||||
IMPROVED = "improved" # 错→对
|
||||
REGRESSED = "regressed" # 对→错
|
||||
PERSISTENT_FAIL = "persistent_fail" # 错→错
|
||||
STABLE_SUCCESS = "stable_success" # 对→对
|
||||
|
||||
# 类别名 → 展示标题，列表顺序即展示顺序。
|
||||
# 回退（REGRESSED）刻意排在改善（IMPROVED）之前——它是最该警惕的伤害信号。
|
||||
_CATEGORY_LABELS: tuple[tuple[str, str], ...] = (
|
||||
(REGRESSED, "从对变错（回退，最高优先级）"),
|
||||
(PERSISTENT_FAIL, "始终答错（持续失败）"),
|
||||
(IMPROVED, "从错变对（改善）"),
|
||||
(STABLE_SUCCESS, "始终答对（稳定成功）"),
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 辅助函数
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def _categorize_pair(pair: dict[str, Any]) -> str:
|
||||
"""按两版正误派生纵向对比类别。
|
||||
|
||||
用键值的真值（bool(...)）表示该题在两版上各自的正误：缺 correct_prev/
|
||||
correct_curr 键时直接抛 KeyError 向上传播——这是上游数据损坏（不是裁判语义
|
||||
歧义），静默当 False 会伪造 persistent_fail 证据、污染动量指导，故不掩盖。
|
||||
|
||||
参数:
|
||||
pair: 单个纵向对比对，须含 correct_prev/correct_curr 两键。
|
||||
|
||||
返回:
|
||||
四个类别命名常量之一：IMPROVED（错→对）/REGRESSED（对→错）/
|
||||
PERSISTENT_FAIL（错→错）/STABLE_SUCCESS（对→对）。
|
||||
|
||||
异常:
|
||||
KeyError: 缺 correct_prev 或 correct_curr 键时。
|
||||
"""
|
||||
correct_prev = bool(pair["correct_prev"])
|
||||
correct_curr = bool(pair["correct_curr"])
|
||||
if not correct_prev and correct_curr:
|
||||
return IMPROVED
|
||||
if correct_prev and not correct_curr:
|
||||
return REGRESSED
|
||||
if not correct_prev and not correct_curr:
|
||||
return PERSISTENT_FAIL
|
||||
return STABLE_SUCCESS
|
||||
|
||||
|
||||
def _format_comparison_pairs(comparison_pairs: list[dict[str, Any]]) -> str:
|
||||
"""将纵向对比对格式化为裁判可读文本，按 _CATEGORY_LABELS 分组与排序。
|
||||
|
||||
参数:
|
||||
comparison_pairs: 每个 dict 含 question/prev_prediction/curr_prediction/
|
||||
correct_prev/correct_curr 字段，描述一道固定样本上两版的成对结果。
|
||||
|
||||
返回:
|
||||
可读的纵向对比文本；空列表返回占位说明。
|
||||
|
||||
异常:
|
||||
KeyError: 任一 pair 缺 correct_prev/correct_curr 键时；不掩盖的理由见
|
||||
_categorize_pair docstring。
|
||||
"""
|
||||
if not comparison_pairs:
|
||||
return "（本轮无可用纵向对比样本）"
|
||||
|
||||
grouped: dict[str, list[dict[str, Any]]] = {key: [] for key, _ in _CATEGORY_LABELS}
|
||||
for pair in comparison_pairs:
|
||||
grouped[_categorize_pair(pair)].append(pair)
|
||||
|
||||
lines: list[str] = [f"固定样本总数：{len(comparison_pairs)}"]
|
||||
for key, label in _CATEGORY_LABELS:
|
||||
entries = grouped[key]
|
||||
lines.append(f"\n### {label}（{len(entries)} 题）")
|
||||
if not entries:
|
||||
lines.append("（无）")
|
||||
continue
|
||||
for pair in entries:
|
||||
lines.append(
|
||||
f"- 题目：{pair.get('question', '')}\n"
|
||||
f" 上版预测：{pair.get('prev_prediction', '')} | "
|
||||
f"当前版预测：{pair.get('curr_prediction', '')}"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 入口
|
||||
# =========================================================================
|
||||
|
||||
|
||||
async def run_slow_momentum(
|
||||
llm: LLMProvider,
|
||||
diagnose_prompts_dir: Path,
|
||||
skill_content: str,
|
||||
prev_skill: str,
|
||||
prev_guidance: str,
|
||||
comparison_pairs: list[dict[str, Any]],
|
||||
) -> str:
|
||||
"""为单个 skill 生成新的慢更新动量指导。
|
||||
|
||||
参数:
|
||||
llm: LLM 端口（async chat）。
|
||||
diagnose_prompts_dir: 诊断 prompt 目录（根 prompts/，slow_momentum.md 在此）。
|
||||
skill_content: 当前版 skill 正文。
|
||||
prev_skill: 上一版 skill 正文。
|
||||
prev_guidance: 上一轮写下的动量指导。
|
||||
comparison_pairs: 固定样本上两版 rollout 的成对结果（含 question/
|
||||
prev_prediction/curr_prediction/correct_prev/correct_curr）。
|
||||
|
||||
返回:
|
||||
新的动量指导文本；解析失败时保留 prev_guidance。
|
||||
|
||||
关键实现细节:
|
||||
- _format_comparison_pairs 刻意置于 try 块之外（prompt 构造阶段）：它对每个
|
||||
pair 取 correct_prev/correct_curr，缺键抛 KeyError 直接向上传播，不被下方
|
||||
针对裁判语义歧义的 except ValueError 吞掉。
|
||||
- 解析失败保留上轮指导：extract_json_from_response 抛 ValueError、缺
|
||||
slow_update_content 字段、或该字段非 str，均视为语义解析失败，返回
|
||||
prev_guidance（判不准时保守保留上轮指导，对标 diagnose 的保护性 fallback）。
|
||||
- P5 边界：仅捕 ValueError 这一语义歧义；llm.chat 的基础设施失败
|
||||
（网络/API 异常）刻意不捕，向上传播，绝不用默认值掩盖。
|
||||
"""
|
||||
system_prompt = (diagnose_prompts_dir / "slow_momentum.md").read_text(encoding="utf-8")
|
||||
# _format_comparison_pairs 刻意置于下方 try 块之外（prompt 构造阶段）：它对每个
|
||||
# pair 取 correct_prev/correct_curr，缺键抛 KeyError 直接向上传播，不被下方针对
|
||||
# 裁判语义歧义的 except ValueError 吞掉。异常类型选 KeyError（非 ValueError），
|
||||
# 即便位置疏忽落入 try 也不会被误吞。
|
||||
user_prompt = (
|
||||
f"## 上一版 skill 正文\n{prev_skill}\n\n"
|
||||
f"## 当前版 skill 正文\n{skill_content}\n\n"
|
||||
f"## 上一轮的动量指导\n{prev_guidance}\n\n"
|
||||
f"## 固定样本纵向对比（上版 vs 当前版）\n"
|
||||
f"{_format_comparison_pairs(comparison_pairs)}"
|
||||
)
|
||||
response = await llm.chat(
|
||||
[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt},
|
||||
]
|
||||
)
|
||||
raw = response.content
|
||||
try:
|
||||
parsed = extract_json_from_response(raw)
|
||||
new_guidance = parsed.get("slow_update_content")
|
||||
if not isinstance(new_guidance, str):
|
||||
raise ValueError("slow_update_content 字段缺失或非字符串")
|
||||
except ValueError:
|
||||
logger.warning("慢更新动量解析失败，保留上轮动量指导")
|
||||
return prev_guidance
|
||||
return new_guidance
|
||||
@@ -0,0 +1,459 @@
|
||||
"""五张观测表的落库写入与回读 + step/epoch 报告文件输出。
|
||||
|
||||
合并 TRM4 的 metric_log.py（五表）和 loop_report.py（报告）。
|
||||
|
||||
五张表均经 structured-logging 定义，DDL 与之逐列一致：
|
||||
dual_metric_eval / shadow_gate / holdout_eval / quadrant_pair / gate_evidence。
|
||||
|
||||
公共契约（守 P5）：soft/mixed 为 None（invalid，无 span / 诊断失败）时存 NULL，**绝不存 0**——
|
||||
SQLite 对 dict 中 None 值写入即 NULL，分析时按 NULL 跳过。每个写函数内幂等建表
|
||||
（``HarnessLog.create_table`` 用 CREATE TABLE IF NOT EXISTS），run_id/timestamp 列由
|
||||
HarnessLog 自动补。
|
||||
|
||||
报告函数输出 JSON 到 workspace 的 analyses/ 目录，供人工审查诊断 prompt 与进化 prompt。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _read_table(db_path: str, table: str, run_id: str) -> list[dict[str, Any]]:
|
||||
"""纯读某表指定 run 的全部行——不经 HarnessLog 生命周期，避免回读污染 _runs 运行状态。
|
||||
|
||||
HarnessLog.__enter__/__exit__ 会对 run_id 做 INSERT OR IGNORE 并在退出时标 completed；
|
||||
回读指标绝不应改运行状态，故 read_* 一律走本只读连接（仅 SELECT）。
|
||||
|
||||
参数:
|
||||
db_path: SQLite 路径。
|
||||
table: 表名（内部固定常量，非外部输入，无注入风险）。
|
||||
run_id: 过滤的 run ID。
|
||||
|
||||
返回:
|
||||
行 dict 列表；表尚未建（没写过）视为无数据返 []。
|
||||
"""
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
exists = conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table,)
|
||||
).fetchone()
|
||||
if exists is None:
|
||||
return []
|
||||
rows = conn.execute(f"SELECT * FROM {table} WHERE run_id=?", (run_id,)).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 列定义严格对齐 research-wiki/schemas/*.md（run_id/timestamp 由 create_table 自动补）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DUAL_COLS: dict[str, str] = {
|
||||
"epoch": "INTEGER",
|
||||
"version_kind": "TEXT",
|
||||
"skills_version": "TEXT",
|
||||
"prompts_version": "TEXT",
|
||||
"pool": "TEXT",
|
||||
"hard_acc": "REAL",
|
||||
"soft_score": "REAL",
|
||||
"mixed_score": "REAL",
|
||||
}
|
||||
|
||||
_SHADOW_COLS: dict[str, str] = {
|
||||
"epoch": "INTEGER",
|
||||
"candidate_version": "TEXT",
|
||||
"hard_acc": "REAL",
|
||||
"soft_score": "REAL",
|
||||
"mixed_score": "REAL",
|
||||
"is_mixed_best": "INTEGER",
|
||||
}
|
||||
|
||||
_HOLDOUT_COLS: dict[str, str] = {
|
||||
"epoch": "INTEGER",
|
||||
"version_kind": "TEXT",
|
||||
"hard_acc": "REAL",
|
||||
"soft_score": "REAL",
|
||||
"mixed_score": "REAL",
|
||||
"per_task_type_json": "TEXT",
|
||||
}
|
||||
|
||||
_QUADRANT_COLS: dict[str, str] = {
|
||||
"epoch": "INTEGER",
|
||||
"step": "INTEGER",
|
||||
"question_id": "TEXT",
|
||||
"task_type": "TEXT",
|
||||
"prev_correct": "INTEGER",
|
||||
"curr_correct": "INTEGER",
|
||||
"category": "TEXT",
|
||||
}
|
||||
|
||||
_GATE_EVIDENCE_COLS: dict[str, str] = {
|
||||
"epoch": "INTEGER",
|
||||
"step": "INTEGER",
|
||||
"task_type": "TEXT",
|
||||
"question_id": "TEXT",
|
||||
"block_idx": "INTEGER",
|
||||
"baseline_correct": "INTEGER",
|
||||
"candidate_correct": "INTEGER",
|
||||
"e_value": "REAL",
|
||||
"stop_reason": "TEXT",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# dual_metric_eval
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_dual_metric(
|
||||
db_path: str,
|
||||
*,
|
||||
run_id: str,
|
||||
epoch: int,
|
||||
version_kind: str,
|
||||
skills_version: str,
|
||||
prompts_version: str,
|
||||
pool: str,
|
||||
hard_acc: float,
|
||||
soft_score: float | None,
|
||||
mixed_score: float | None,
|
||||
) -> None:
|
||||
"""落 dual_metric_eval 一行：epoch 末关键版本的 hard+soft+mixed 双轨度量。
|
||||
|
||||
参数:
|
||||
db_path: SQLite 路径。
|
||||
run_id: 训练 run ID。
|
||||
epoch: 轮次（1-based）。
|
||||
version_kind: baseline / best_hard / best_mixed / final。
|
||||
skills_version / prompts_version: 评估的资源版本。
|
||||
pool: val / test。
|
||||
hard_acc: hard 准确率。
|
||||
soft_score: soft 连续分；invalid 传 None -> 存 NULL。
|
||||
mixed_score: 0.5*hard+0.5*soft；soft 缺失传 None -> 存 NULL。
|
||||
"""
|
||||
from app.harness.log import HarnessLog
|
||||
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("dual_metric_eval", _DUAL_COLS)
|
||||
log.insert(
|
||||
"dual_metric_eval",
|
||||
{
|
||||
"epoch": epoch,
|
||||
"version_kind": version_kind,
|
||||
"skills_version": skills_version,
|
||||
"prompts_version": prompts_version,
|
||||
"pool": pool,
|
||||
"hard_acc": hard_acc,
|
||||
"soft_score": soft_score,
|
||||
"mixed_score": mixed_score,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def read_dual_metric(db_path: str, *, run_id: str) -> list[dict[str, Any]]:
|
||||
"""回读指定 run 的 dual_metric_eval 全部行（纯读，不污染运行状态）。"""
|
||||
return _read_table(db_path, "dual_metric_eval", run_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# shadow_gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_shadow_gate(
|
||||
db_path: str,
|
||||
*,
|
||||
run_id: str,
|
||||
epoch: int,
|
||||
candidate_version: str,
|
||||
hard_acc: float,
|
||||
soft_score: float | None,
|
||||
mixed_score: float | None,
|
||||
is_mixed_best: bool,
|
||||
) -> None:
|
||||
"""落 shadow_gate 一行：mixed 影子 best 候选的 hard/soft/mixed 及是否 argmax 选中。
|
||||
|
||||
参数:
|
||||
db_path: SQLite 路径。
|
||||
run_id: 训练 run ID。
|
||||
epoch: 轮次（1-based）。
|
||||
candidate_version: 候选版本标识（如 skills/vX+prompts/vY）。
|
||||
hard_acc: hard 准确率。
|
||||
soft_score: soft 连续分；invalid 传 None -> 存 NULL（该版本不进 argmax）。
|
||||
mixed_score: 0.5*hard+0.5*soft；soft 缺失传 None -> 存 NULL。
|
||||
is_mixed_best: 是否本 epoch mixed argmax 选中（存 1/0）。
|
||||
"""
|
||||
from app.harness.log import HarnessLog
|
||||
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("shadow_gate", _SHADOW_COLS)
|
||||
log.insert(
|
||||
"shadow_gate",
|
||||
{
|
||||
"epoch": epoch,
|
||||
"candidate_version": candidate_version,
|
||||
"hard_acc": hard_acc,
|
||||
"soft_score": soft_score,
|
||||
"mixed_score": mixed_score,
|
||||
"is_mixed_best": int(is_mixed_best),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def read_shadow_gate(db_path: str, *, run_id: str) -> list[dict[str, Any]]:
|
||||
"""回读指定 run 的 shadow_gate 全部行（纯读，不污染运行状态）。"""
|
||||
return _read_table(db_path, "shadow_gate", run_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# holdout_eval
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_holdout_eval(
|
||||
db_path: str,
|
||||
*,
|
||||
run_id: str,
|
||||
epoch: int,
|
||||
version_kind: str,
|
||||
hard_acc: float,
|
||||
soft_score: float | None,
|
||||
mixed_score: float | None,
|
||||
per_task_type_json: str,
|
||||
) -> None:
|
||||
"""落 holdout_eval 一行：四向 held-out 在 test 池的 hard+soft+mixed 及按题型细分。
|
||||
|
||||
参数:
|
||||
db_path: SQLite 路径。
|
||||
run_id: 训练 run ID。
|
||||
epoch: 轮次（1-based）。
|
||||
version_kind: baseline / best_hard / best_mixed / final。
|
||||
hard_acc: hard 准确率。
|
||||
soft_score: soft 连续分；invalid 传 None -> 存 NULL。
|
||||
mixed_score: 0.5*hard+0.5*soft；soft 缺失传 None -> 存 NULL。
|
||||
per_task_type_json: 按 task_type 的 {accuracy,total,correct} JSON 串。
|
||||
"""
|
||||
from app.harness.log import HarnessLog
|
||||
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("holdout_eval", _HOLDOUT_COLS)
|
||||
log.insert(
|
||||
"holdout_eval",
|
||||
{
|
||||
"epoch": epoch,
|
||||
"version_kind": version_kind,
|
||||
"hard_acc": hard_acc,
|
||||
"soft_score": soft_score,
|
||||
"mixed_score": mixed_score,
|
||||
"per_task_type_json": per_task_type_json,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def read_holdout_eval(db_path: str, *, run_id: str) -> list[dict[str, Any]]:
|
||||
"""回读指定 run 的 holdout_eval 全部行（纯读，不污染运行状态）。"""
|
||||
return _read_table(db_path, "holdout_eval", run_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# quadrant_pair
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_quadrant_pairs(
|
||||
db_path: str,
|
||||
*,
|
||||
run_id: str,
|
||||
epoch: int,
|
||||
step: int,
|
||||
pairs: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""落 quadrant_pair 多行：fast gate 后逐题四象限（prev/curr 翻转 + category）落库。
|
||||
|
||||
参数:
|
||||
db_path: SQLite 路径。
|
||||
run_id: 训练 run ID。
|
||||
epoch: 轮次（1-based）。
|
||||
step: epoch 内 step 序号（0-based）。
|
||||
pairs: 每条含 question_id/task_type/prev_correct/curr_correct/category；
|
||||
prev_correct/curr_correct 为 bool，写库前转 0/1。
|
||||
|
||||
关键实现:
|
||||
用 insert_many 批量落库；pairs 为空时只建表不插入（fast gate 无翻转的极端情况）。
|
||||
"""
|
||||
records = [
|
||||
{
|
||||
"epoch": epoch,
|
||||
"step": step,
|
||||
"question_id": pair["question_id"],
|
||||
"task_type": pair["task_type"],
|
||||
"prev_correct": int(pair["prev_correct"]),
|
||||
"curr_correct": int(pair["curr_correct"]),
|
||||
"category": pair["category"],
|
||||
}
|
||||
for pair in pairs
|
||||
]
|
||||
from app.harness.log import HarnessLog
|
||||
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("quadrant_pair", _QUADRANT_COLS)
|
||||
if records:
|
||||
log.insert_many("quadrant_pair", records)
|
||||
|
||||
|
||||
def read_quadrant_pairs(db_path: str, *, run_id: str) -> list[dict[str, Any]]:
|
||||
"""回读指定 run 的 quadrant_pair 全部行（纯读，不污染运行状态）。"""
|
||||
return _read_table(db_path, "quadrant_pair", run_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# gate_evidence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_gate_evidence(
|
||||
db_path: str,
|
||||
*,
|
||||
run_id: str,
|
||||
epoch: int,
|
||||
step: int,
|
||||
rows: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""落 gate_evidence 逐题行：CE-Gate 每次决策的可回放审计记录。
|
||||
|
||||
参数:
|
||||
db_path: SQLite 路径。
|
||||
run_id: 训练 run ID。
|
||||
epoch: 该 gate 所属的轮次（1-based）。
|
||||
step: epoch 内 step 序号（0-based）。
|
||||
rows: 每题一行，含 question_id/task_type/block_idx/baseline_correct/
|
||||
candidate_correct/e_value（该题所在块判定后的累计 e 值）/
|
||||
stop_reason（仅最后一题携带最终 stop_reason，其余空串）。
|
||||
|
||||
关键实现:
|
||||
逐行 insert（非 insert_many），保证每行独立事务。
|
||||
"""
|
||||
from app.harness.log import HarnessLog
|
||||
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("gate_evidence", _GATE_EVIDENCE_COLS)
|
||||
for row in rows:
|
||||
log.insert("gate_evidence", {"epoch": epoch, "step": step, **row})
|
||||
|
||||
|
||||
def read_gate_evidence(db_path: str, *, run_id: str) -> list[dict[str, Any]]:
|
||||
"""回读指定 run 的 gate_evidence 全部行（纯读，不污染运行状态）。"""
|
||||
return _read_table(db_path, "gate_evidence", run_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 报告函数（从 TRM4 loop_report.py 迁移）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def write_step_report(
|
||||
workspace_dir: Path,
|
||||
epoch: int,
|
||||
step: int,
|
||||
global_step: int,
|
||||
task_type: str,
|
||||
gate_action: str,
|
||||
candidate_acc: float,
|
||||
class_baseline_acc: float,
|
||||
edit_budget: int,
|
||||
rank_clip_triggered: bool,
|
||||
gate_w: int | None,
|
||||
gate_l: int | None,
|
||||
gate_e_value: float | None,
|
||||
gate_n_used: int | None,
|
||||
gate_stop_reason: str | None,
|
||||
) -> Path:
|
||||
"""写单个 (step, task_type) 快路径 gate 的最小观测记录 JSON。
|
||||
|
||||
文件名按 (epoch, step, task_type) 命名，slug 由 task_type 规范化（小写、空格转 '-'）得到。
|
||||
|
||||
参数:
|
||||
workspace_dir: 实验工作区目录。
|
||||
epoch: 当前轮次（1-based）。
|
||||
step: epoch 内 step 序号（0-based）。
|
||||
global_step: 全局步计数（驱动 edit_budget 退火）。
|
||||
task_type: 本条 gate 的任务类型。
|
||||
gate_action: 闸门动作（accept_confirmed / accept_provisional / reject /
|
||||
skipped / cooldown）。
|
||||
candidate_acc: 候选在 gate 已观测题上的准确率（观测口径）。
|
||||
class_baseline_acc: 基线在 gate 已观测题上的准确率（观测口径）。
|
||||
edit_budget: 该 step 按 global_step 退火得到的 per-target 编辑预算上限。
|
||||
rank_clip_triggered: 该 skill 进化是否触发了 rank 裁剪。
|
||||
gate_w: e-process 累计 W（基线错->候选对翻转数）；skipped/cooldown 路径传 None。
|
||||
gate_l: e-process 累计 L（基线对->候选错翻转数）；skipped/cooldown 路径传 None。
|
||||
gate_e_value: 停时的 e 值；skipped/cooldown 路径传 None。
|
||||
gate_n_used: gate 实际消费的阶梯题数；skipped/cooldown 路径传 None。
|
||||
gate_stop_reason: e-process 停止原因；skipped/cooldown 路径传 None。
|
||||
|
||||
返回:
|
||||
写入的 step_report 文件路径。
|
||||
"""
|
||||
report = {
|
||||
"epoch": epoch,
|
||||
"step": step,
|
||||
"global_step": global_step,
|
||||
"task_type": task_type,
|
||||
"gate_action": gate_action,
|
||||
"candidate_acc": candidate_acc,
|
||||
"class_baseline_acc": class_baseline_acc,
|
||||
"edit_budget": edit_budget,
|
||||
"rank_clip_triggered": rank_clip_triggered,
|
||||
"gate_w": gate_w,
|
||||
"gate_l": gate_l,
|
||||
"gate_e_value": gate_e_value,
|
||||
"gate_n_used": gate_n_used,
|
||||
"gate_stop_reason": gate_stop_reason,
|
||||
}
|
||||
slug = task_type.lower().replace(" ", "-")
|
||||
out_dir = workspace_dir / "analyses"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = out_dir / f"step_report_e{epoch}_s{step}_{slug}.json"
|
||||
path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def write_epoch_report(
|
||||
workspace_dir: Path,
|
||||
epoch: int,
|
||||
system_tool_action: str,
|
||||
momentum_updated_task_types: list[str],
|
||||
best_val_acc: float,
|
||||
) -> Path:
|
||||
"""写 epoch 末慢更新汇总 JSON。
|
||||
|
||||
慢更新无单一 ValidationOutcome，故本函数只落慢更新可观测的最小集：
|
||||
system/tool gate 动作、本 epoch 写过 momentum 的题型、慢更新后的全局 best。
|
||||
|
||||
参数:
|
||||
workspace_dir: 实验工作区目录。
|
||||
epoch: 当前轮次（1-based）。
|
||||
system_tool_action: 慢更新 system/tool 动作（updated / reverted / none）。
|
||||
momentum_updated_task_types: 本 epoch 写过 momentum 的题型列表。
|
||||
best_val_acc: 慢更新后（含 best argmax）的全局 best 验证准确率。
|
||||
|
||||
返回:
|
||||
写入的 epoch_report 文件路径。
|
||||
"""
|
||||
report = {
|
||||
"epoch": epoch,
|
||||
"system_tool_action": system_tool_action,
|
||||
"momentum_updated_task_types": momentum_updated_task_types,
|
||||
"best_val_acc": best_val_acc,
|
||||
}
|
||||
out_dir = workspace_dir / "analyses"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = out_dir / f"epoch_report_{epoch}.json"
|
||||
path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return path
|
||||
@@ -0,0 +1,283 @@
|
||||
"""三池：held-out test + 验证 + 诊断，分层采样 + 冻结持久化。
|
||||
|
||||
三池切分对应训练循环中的 DataLoader 阶段——从题目全集中按
|
||||
test -> validation -> diagnosis 的顺序 progressive exclusion，
|
||||
保证 question_id 互斥。test 池用自然分布（correct_ratio=None），
|
||||
验证池/诊断池按对错比例分层采样。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from app.question_gen import stratified_sample
|
||||
from core.types import GeneratedQuestion
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from app.harness.config import RunConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class Pools:
|
||||
"""冻结的三池及其基线指标。
|
||||
|
||||
字段:
|
||||
diagnosis: 诊断池（用于错误归因，对应 loss.backward）。
|
||||
validation: 验证池（按类局部验证，每题型有保底样本）。
|
||||
test: held-out 测试池（自然分布，用于最终无偏评估）。
|
||||
baseline_run_id: 基线 run 标识。
|
||||
baseline_val_accuracy: 基线在验证池上的准确率。
|
||||
correctness: 三池所有题的 question_id -> 基线是否答对。
|
||||
"""
|
||||
|
||||
diagnosis: list[GeneratedQuestion]
|
||||
validation: list[GeneratedQuestion]
|
||||
test: list[GeneratedQuestion]
|
||||
baseline_run_id: str
|
||||
baseline_val_accuracy: float
|
||||
correctness: dict[str, bool] = field(default_factory=dict)
|
||||
|
||||
|
||||
def build_pools(
|
||||
questions: list[GeneratedQuestion],
|
||||
correctness: dict[str, bool],
|
||||
diag_cfg: dict,
|
||||
val_cfg: dict,
|
||||
test_cfg: dict,
|
||||
baseline_run_id: str,
|
||||
) -> Pools:
|
||||
"""先抽 held-out test，再抽验证集，最后抽诊断池，三池互斥。
|
||||
|
||||
参数:
|
||||
questions: 题目全集。
|
||||
correctness: question_id -> 基线是否答对。
|
||||
diag_cfg: 诊断池采样配置（size/correct_ratio/task_types[/seed]）。
|
||||
val_cfg: 验证池采样配置，可含 min_per_class 做按类保底。
|
||||
test_cfg: 测试池配置（size[/seed]）；走自然分布，不强制对错比与题型。
|
||||
baseline_run_id: 基线 run 标识。
|
||||
|
||||
返回:
|
||||
冻结的三池 Pools。
|
||||
|
||||
关键实现细节:
|
||||
切分顺序 test -> validation -> diagnosis；后两步从剩余题中采样以保证
|
||||
question_id 互斥。test 池用 correct_ratio=None 的自然分布采样。
|
||||
"""
|
||||
test = _sample_excluding(
|
||||
questions,
|
||||
set(),
|
||||
correctness,
|
||||
size=test_cfg["size"],
|
||||
correct_ratio=None,
|
||||
task_types=None,
|
||||
seed=test_cfg.get("seed", 0),
|
||||
min_per_class=None,
|
||||
)
|
||||
selected_ids = {q.question_id for q in test}
|
||||
|
||||
validation = _sample_excluding(questions, selected_ids, correctness, **val_cfg)
|
||||
selected_ids |= {q.question_id for q in validation}
|
||||
|
||||
diagnosis = _sample_excluding(questions, selected_ids, correctness, **diag_cfg)
|
||||
|
||||
val_correct = sum(1 for q in validation if correctness.get(q.question_id))
|
||||
baseline_val_accuracy = val_correct / len(validation) if validation else 0.0
|
||||
return Pools(
|
||||
diagnosis=diagnosis,
|
||||
validation=validation,
|
||||
test=test,
|
||||
baseline_run_id=baseline_run_id,
|
||||
baseline_val_accuracy=baseline_val_accuracy,
|
||||
correctness={
|
||||
q.question_id: correctness.get(q.question_id, False)
|
||||
for q in test + validation + diagnosis
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _sample_excluding(
|
||||
questions: list[GeneratedQuestion],
|
||||
exclude_ids: set[str],
|
||||
correctness: dict[str, bool],
|
||||
**cfg: object,
|
||||
) -> list[GeneratedQuestion]:
|
||||
"""排除已选 question_id 后，按 cfg 对剩余题做分层采样。
|
||||
|
||||
参数:
|
||||
questions: 题目全集。
|
||||
exclude_ids: 已被其他池选走的 question_id，从候选中剔除以保证三池互斥。
|
||||
correctness: question_id -> 基线是否答对。
|
||||
cfg: 透传给 stratified_sample 的采样配置
|
||||
（size/correct_ratio/task_types[/seed/min_per_class]）。
|
||||
|
||||
返回:
|
||||
采样后的题目列表。
|
||||
"""
|
||||
pool = [q for q in questions if q.question_id not in exclude_ids]
|
||||
return stratified_sample(pool, correctness, **cfg)
|
||||
|
||||
|
||||
def _q_to_dict(q: GeneratedQuestion) -> dict:
|
||||
"""将 GeneratedQuestion 转为可序列化字典。
|
||||
|
||||
参数:
|
||||
q: 题目对象。
|
||||
|
||||
返回:
|
||||
包含全部字段的字典（options/source_nodes 从 tuple 转为 list）。
|
||||
"""
|
||||
return {
|
||||
"question_id": q.question_id,
|
||||
"video_id": q.video_id,
|
||||
"task_type": q.task_type,
|
||||
"question": q.question,
|
||||
"options": list(q.options),
|
||||
"answer": q.answer,
|
||||
"source_nodes": list(q.source_nodes),
|
||||
"difficulty": q.difficulty,
|
||||
}
|
||||
|
||||
|
||||
def _dict_to_q(d: dict) -> GeneratedQuestion:
|
||||
"""从字典恢复 GeneratedQuestion。
|
||||
|
||||
参数:
|
||||
d: 由 _q_to_dict 产出的字典。
|
||||
|
||||
返回:
|
||||
恢复的 GeneratedQuestion 实例（options/source_nodes 恢复为 tuple）。
|
||||
"""
|
||||
return GeneratedQuestion(
|
||||
question_id=d["question_id"],
|
||||
video_id=d["video_id"],
|
||||
task_type=d["task_type"],
|
||||
question=d["question"],
|
||||
options=tuple(d["options"]),
|
||||
answer=d["answer"],
|
||||
source_nodes=tuple(d.get("source_nodes", ())),
|
||||
difficulty=d.get("difficulty", "medium"),
|
||||
)
|
||||
|
||||
|
||||
def save_pools(pools: Pools, path: Path) -> None:
|
||||
"""将三池及基线指标冻结为 JSON。
|
||||
|
||||
参数:
|
||||
pools: 待冻结的三池。
|
||||
path: 目标 JSON 文件路径。
|
||||
"""
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"baseline_run_id": pools.baseline_run_id,
|
||||
"baseline_val_accuracy": pools.baseline_val_accuracy,
|
||||
"correctness": pools.correctness,
|
||||
"diagnosis": [_q_to_dict(q) for q in pools.diagnosis],
|
||||
"validation": [_q_to_dict(q) for q in pools.validation],
|
||||
"test": [_q_to_dict(q) for q in pools.test],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def load_pools(path: Path) -> Pools:
|
||||
"""从 JSON 恢复冻结的三池。
|
||||
|
||||
参数:
|
||||
path: 冻结的 pools.json 路径。
|
||||
|
||||
返回:
|
||||
恢复的三池 Pools。
|
||||
|
||||
异常:
|
||||
ValueError: 旧格式 pools.json（无 test 池）。
|
||||
|
||||
关键实现细节:
|
||||
旧格式 pools.json（无 test 池）会以清晰的 ValueError 中止——本项目不做
|
||||
向后兼容，也不为缺失字段填默认值。删除旧文件后 build_pools 会重新采样切分，
|
||||
无需重新推理。
|
||||
"""
|
||||
d = json.loads(path.read_text(encoding="utf-8"))
|
||||
if "test" not in d:
|
||||
raise ValueError(
|
||||
f"{path} 为旧格式 pools.json（缺 test 池），"
|
||||
"请删除后重新切分（build_pools 会重新采样，无需重新推理）。"
|
||||
)
|
||||
return Pools(
|
||||
diagnosis=[_dict_to_q(x) for x in d["diagnosis"]],
|
||||
validation=[_dict_to_q(x) for x in d["validation"]],
|
||||
test=[_dict_to_q(x) for x in d["test"]],
|
||||
baseline_run_id=d["baseline_run_id"],
|
||||
baseline_val_accuracy=d["baseline_val_accuracy"],
|
||||
correctness=d["correctness"],
|
||||
)
|
||||
|
||||
|
||||
def build_or_load_pools(
|
||||
config: RunConfig,
|
||||
run_id: str,
|
||||
task_types: list[str] | None = None,
|
||||
) -> Pools:
|
||||
"""train 模式的三池获取入口：pools.json 已存在则加载，否则从基线 db 切分并冻结。
|
||||
|
||||
把 main.py train 分支「pools.json 存在则 load_pools 否则 build_pools 再 save_pools」
|
||||
那段抽成纯函数，使 main 与集成测试共用同一切分逻辑、避免重复。pools.json 是
|
||||
一次 fresh 训练的冻结切分，resume/重跑同一 workspace 时直接复用以保证三池一致。
|
||||
|
||||
参数:
|
||||
config: 运行配置，提供 workspace_dir 与三池采样旋钮（diag/val/test 各项）。
|
||||
run_id: 基线全量记录的 run_id（fresh 时来自 seed.json，决定从哪个 run 读对错）。
|
||||
task_types: 可选题型过滤，限定诊断/验证池只采样这些题型；None 表示不过滤。
|
||||
|
||||
返回:
|
||||
冻结的三池 Pools。
|
||||
|
||||
关键实现:
|
||||
切分前从基线 db 的 predictions 表读该 run_id 的逐题对错，作为分层采样依据。
|
||||
pools.json 落在 config.workspace_dir 下，存在即视为已冻结，原样加载不重切。
|
||||
"""
|
||||
from app.harness.log import HarnessLog
|
||||
from app.harness.workspace import resolve_paths
|
||||
from app.question_gen import load_benchmark
|
||||
|
||||
pools_path = config.workspace_dir / "pools.json"
|
||||
if pools_path.exists():
|
||||
return load_pools(pools_path)
|
||||
|
||||
paths = resolve_paths(config.workspace_dir)
|
||||
questions = load_benchmark(paths.questions_dir)
|
||||
with HarnessLog(str(paths.db_path), run_id) as log:
|
||||
rows = log.query(
|
||||
"SELECT question_id, prediction, answer FROM predictions WHERE run_id=?",
|
||||
(run_id,),
|
||||
)
|
||||
correctness = {r["question_id"]: r["prediction"] == r["answer"] for r in rows}
|
||||
pools = build_pools(
|
||||
questions,
|
||||
correctness,
|
||||
diag_cfg={
|
||||
"size": config.diag_size,
|
||||
"correct_ratio": config.diag_correct_ratio,
|
||||
"task_types": task_types,
|
||||
"seed": 0,
|
||||
"min_per_class": None,
|
||||
},
|
||||
val_cfg={
|
||||
"size": config.val_size,
|
||||
"correct_ratio": config.val_correct_ratio,
|
||||
"task_types": task_types,
|
||||
"seed": 0,
|
||||
"min_per_class": config.eval_min_per_class,
|
||||
},
|
||||
test_cfg={"size": config.test_size},
|
||||
baseline_run_id=run_id,
|
||||
)
|
||||
save_pools(pools, pools_path)
|
||||
return pools
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,378 @@
|
||||
"""Store 版本操作 + Seed 管理。
|
||||
|
||||
Store 存储版本化资源（视频、题目、Skill、Prompt），
|
||||
通过版本号（v1, v2, ...）管理资源的演化历史。
|
||||
Seed 是可复现的实验起点，包含权重快照 + baseline 数据库。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from loguru import logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
"""返回当前 UTC 时间的 ISO 格式字符串。"""
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _parse_version(name: str) -> int:
|
||||
"""解析版本目录名 ``v\\d+`` 为整数。
|
||||
|
||||
参数:
|
||||
name: 版本目录名，如 ``"v1"``、``"v10"``。
|
||||
|
||||
返回:
|
||||
版本号整数。
|
||||
|
||||
异常:
|
||||
ValueError: 版本目录名格式不合法（不匹配 ``v\\d+``）。
|
||||
"""
|
||||
match = re.match(r"v(\d+)$", name)
|
||||
if not match:
|
||||
raise ValueError(f"无效版本号: {name}")
|
||||
return int(match.group(1))
|
||||
|
||||
|
||||
def list_versions(store_dir: Path, resource_type: str) -> list[str]:
|
||||
"""列出 Store 中某类资源的所有版本号，按数字值排序。
|
||||
|
||||
按数字排序保证 v10 排在 v2 后面（而非字典序 v10 < v2）。
|
||||
|
||||
参数:
|
||||
store_dir: Store 根目录。
|
||||
resource_type: 资源类型路径，如 ``"skills"``、``"questions/generated"``。
|
||||
|
||||
返回:
|
||||
排序后的版本号列表，如 ``["v1", "v2", "v10"]``。
|
||||
"""
|
||||
resource_dir = store_dir / resource_type
|
||||
if not resource_dir.is_dir():
|
||||
return []
|
||||
versions = []
|
||||
for entry in resource_dir.iterdir():
|
||||
if entry.is_dir() and re.match(r"v\d+$", entry.name):
|
||||
versions.append(entry.name)
|
||||
return sorted(versions, key=_parse_version)
|
||||
|
||||
|
||||
def next_version(store_dir: Path, resource_type: str) -> str:
|
||||
"""返回某类资源的下一个可用版本号。
|
||||
|
||||
参数:
|
||||
store_dir: Store 根目录。
|
||||
resource_type: 资源类型路径。
|
||||
|
||||
返回:
|
||||
下一个版本号字符串，如 ``"v3"``。
|
||||
"""
|
||||
versions = list_versions(store_dir, resource_type)
|
||||
if not versions:
|
||||
return "v1"
|
||||
latest = _parse_version(versions[-1])
|
||||
return f"v{latest + 1}"
|
||||
|
||||
|
||||
def _write_meta(target_dir: Path, version: str, source: str, **extra: str | None) -> None:
|
||||
"""写入版本元数据文件 ``meta.json``。
|
||||
|
||||
参数:
|
||||
target_dir: 版本目录。
|
||||
version: 版本号。
|
||||
source: 来源标识（``"manual"`` / ``"evolution"`` / ``"auto-gen"``）。
|
||||
**extra: 额外字段（parent, trigger_run, trigger_workspace, description）。
|
||||
"""
|
||||
meta = {
|
||||
"version": version,
|
||||
"created_at": _now_iso(),
|
||||
"parent": extra.get("parent"),
|
||||
"source": source,
|
||||
"trigger_run": extra.get("trigger_run"),
|
||||
"trigger_workspace": extra.get("trigger_workspace"),
|
||||
"description": extra.get("description", ""),
|
||||
}
|
||||
(target_dir / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
def advance_version(
|
||||
store_dir: Path,
|
||||
resource_type: str,
|
||||
source_dir: Path,
|
||||
meta: dict,
|
||||
) -> str:
|
||||
"""将 source_dir 的内容写入 Store 的下一个版本目录，写入 meta.json。
|
||||
|
||||
参数:
|
||||
store_dir: Store 根目录。
|
||||
resource_type: 资源类型路径，如 ``"skills"``、``"questions/generated"``。
|
||||
source_dir: 包含新版本资源文件的源目录。
|
||||
meta: 元数据字典，至少包含 ``source`` 字段。
|
||||
|
||||
返回:
|
||||
新版本号字符串，如 ``"v2"``。
|
||||
"""
|
||||
version = next_version(store_dir, resource_type)
|
||||
target = store_dir / resource_type / version
|
||||
shutil.copytree(source_dir, target)
|
||||
_write_meta(
|
||||
target,
|
||||
version,
|
||||
meta.get("source", "manual"),
|
||||
parent=meta.get("parent"),
|
||||
trigger_run=meta.get("trigger_run"),
|
||||
trigger_workspace=meta.get("trigger_workspace"),
|
||||
description=meta.get("description", ""),
|
||||
)
|
||||
logger.info("Store 版本推进: {}/{}", resource_type, version)
|
||||
return version
|
||||
|
||||
|
||||
def init_store(
|
||||
store_dir: Path,
|
||||
videos_source: Path,
|
||||
skills_dir: Path,
|
||||
prompts_dir: Path,
|
||||
) -> None:
|
||||
"""初始化 Store：拷贝视频数据，创建 skills/v1、prompts/v1 和 questions 目录。
|
||||
|
||||
参数:
|
||||
store_dir: Store 目标路径（不得已存在）。
|
||||
videos_source: 视频数据源目录。
|
||||
skills_dir: 初始 Skill 文件目录。
|
||||
prompts_dir: 初始 Prompt 文件目录。
|
||||
|
||||
异常:
|
||||
FileExistsError: Store 目录已存在。
|
||||
"""
|
||||
if store_dir.exists():
|
||||
raise FileExistsError(f"Store 已存在: {store_dir}")
|
||||
store_dir.mkdir(parents=True)
|
||||
shutil.copytree(videos_source, store_dir / "videos")
|
||||
(store_dir / "questions" / "benchmarks").mkdir(parents=True)
|
||||
(store_dir / "questions" / "generated").mkdir(parents=True)
|
||||
shutil.copytree(skills_dir, store_dir / "skills" / "v1")
|
||||
_write_meta(
|
||||
store_dir / "skills" / "v1",
|
||||
"v1",
|
||||
"manual",
|
||||
description="手工创建的初始版本",
|
||||
)
|
||||
shutil.copytree(prompts_dir, store_dir / "prompts" / "v1")
|
||||
_write_meta(
|
||||
store_dir / "prompts" / "v1",
|
||||
"v1",
|
||||
"manual",
|
||||
description="手工创建的初始版本",
|
||||
)
|
||||
logger.info("Store 初始化完成: {}", store_dir)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 种子库（Seed）函数
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def init_seed(
|
||||
store_dir: Path,
|
||||
name: str,
|
||||
skills_dir: Path,
|
||||
prompts_dir: Path,
|
||||
baseline_db: Path,
|
||||
baseline_run_id: str,
|
||||
parent: str | None,
|
||||
description: str,
|
||||
) -> Path:
|
||||
"""在 store/seeds/<name> 写一个种子：权重 + baseline.db + seed.json。
|
||||
|
||||
参数:
|
||||
store_dir: Store 根目录。
|
||||
name: 种子名（如 ``'initial'``、``'from-evolve-v20'``）。
|
||||
skills_dir: 该版本 Skill 权重源目录。
|
||||
prompts_dir: 该版本 Prompt 权重源目录。
|
||||
baseline_db: 该版本全量记录 db（含 _runs + predictions 行）。
|
||||
baseline_run_id: 全量记录的 run_id，fresh 时注入 build_pools。
|
||||
parent: 来源（initial 为 None）。
|
||||
description: 人类可读说明。
|
||||
|
||||
返回:
|
||||
种子目录路径。
|
||||
|
||||
异常:
|
||||
FileExistsError: 同名种子已存在（不覆盖）。
|
||||
"""
|
||||
seed_dir = store_dir / "seeds" / name
|
||||
if seed_dir.exists():
|
||||
raise FileExistsError(f"种子已存在，不覆盖: {seed_dir}")
|
||||
seed_dir.mkdir(parents=True)
|
||||
shutil.copytree(skills_dir, seed_dir / "skills")
|
||||
shutil.copytree(prompts_dir, seed_dir / "prompts")
|
||||
shutil.copy2(baseline_db, seed_dir / "baseline.db")
|
||||
(seed_dir / "seed.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"baseline_run_id": baseline_run_id,
|
||||
"parent": parent,
|
||||
"created_at": _now_iso(),
|
||||
"description": description,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
logger.info("种子创建完成: {}", seed_dir)
|
||||
return seed_dir
|
||||
|
||||
|
||||
def list_seeds(store_dir: Path) -> list[str]:
|
||||
"""列出 store/seeds 下所有种子名（按名排序）。
|
||||
|
||||
参数:
|
||||
store_dir: Store 根目录。
|
||||
|
||||
返回:
|
||||
种子名列表（仅含 seed.json 存在的目录），按名排序。
|
||||
"""
|
||||
seeds_root = store_dir / "seeds"
|
||||
if not seeds_root.is_dir():
|
||||
return []
|
||||
return sorted(e.name for e in seeds_root.iterdir() if (e / "seed.json").exists())
|
||||
|
||||
|
||||
def read_seed(store_dir: Path, name: str) -> dict:
|
||||
"""读取种子 seed.json；不存在则报错。
|
||||
|
||||
参数:
|
||||
store_dir: Store 根目录。
|
||||
name: 种子名。
|
||||
|
||||
返回:
|
||||
seed.json 解析后的字典。
|
||||
|
||||
异常:
|
||||
FileNotFoundError: 该种子不存在。
|
||||
"""
|
||||
seed_json = store_dir / "seeds" / name / "seed.json"
|
||||
if not seed_json.exists():
|
||||
raise FileNotFoundError(f"种子不存在: {name}（{seed_json}）")
|
||||
return json.loads(seed_json.read_text())
|
||||
|
||||
|
||||
def extract_run_db(src_db: Path, dst_db: Path, run_id: str) -> None:
|
||||
"""从 src_db 抽出某 run_id 的 _runs + predictions 行，写一个最小 db（种子 baseline.db）。
|
||||
|
||||
用源表的**原始 CREATE 语句**重建目标表，保留主键/列类型/约束——
|
||||
``_runs.run_id TEXT PRIMARY KEY`` 是 HarnessLog ``INSERT OR IGNORE`` 去重的依据，
|
||||
若 seed db 丢主键则续训/fresh-bootstrap 的去重失效。
|
||||
|
||||
参数:
|
||||
src_db: 源 harness.db。
|
||||
dst_db: 目标 db（不得已存在）。
|
||||
run_id: 要抽取的 run。
|
||||
|
||||
异常:
|
||||
RuntimeError: 源中无该表或无该 run 的行。
|
||||
"""
|
||||
src = sqlite3.connect(src_db)
|
||||
dst = sqlite3.connect(dst_db)
|
||||
try:
|
||||
for table in ("_runs", "predictions"):
|
||||
create_sql = src.execute(
|
||||
"SELECT sql FROM sqlite_master WHERE type='table' AND name=?",
|
||||
(table,),
|
||||
).fetchone()
|
||||
if create_sql is None or create_sql[0] is None:
|
||||
raise RuntimeError(f"源 db 无表 {table}")
|
||||
dst.execute(create_sql[0])
|
||||
cols = [r[1] for r in src.execute(f"PRAGMA table_info({table})")]
|
||||
col_sql = ", ".join(cols)
|
||||
rows = src.execute(
|
||||
f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,)
|
||||
).fetchall()
|
||||
if not rows:
|
||||
raise RuntimeError(f"{table} 中无 run_id={run_id} 的行")
|
||||
ph = ", ".join("?" * len(cols))
|
||||
dst.executemany(f"INSERT INTO {table} ({col_sql}) VALUES ({ph})", rows)
|
||||
dst.commit()
|
||||
finally:
|
||||
dst.close()
|
||||
src.close()
|
||||
|
||||
|
||||
def promote_to_seed(
|
||||
workspace_dir: Path,
|
||||
store_dir: Path,
|
||||
version: str,
|
||||
eval_run_id: str,
|
||||
name: str,
|
||||
description: str,
|
||||
) -> Path:
|
||||
"""把 workspace 的指定版本 + 配套 prompts + 指定 eval run 全量记录固化成新种子。
|
||||
|
||||
强校验 eval_run_id 对应的 _runs 行中 skills_version 必须与 version 一致，
|
||||
且 skills_version/prompts_version 均不得为 NULL。
|
||||
|
||||
参数:
|
||||
workspace_dir: 来源 workspace。
|
||||
store_dir: Store 根目录。
|
||||
version: skills 版本号。
|
||||
eval_run_id: canonical eval run（其 _runs 行提供配套 prompts 版本与全量记录）。
|
||||
name: 新种子名（冲突报错不覆盖）。
|
||||
description: 说明。
|
||||
|
||||
返回:
|
||||
新种子目录。
|
||||
|
||||
异常:
|
||||
ValueError: eval_run_id 不存在，或其 skills_version 与 version 不符，或版本为 NULL。
|
||||
FileExistsError: 同名种子已存在（由 init_seed 抛出）。
|
||||
"""
|
||||
con = sqlite3.connect(workspace_dir / "harness.db")
|
||||
con.row_factory = sqlite3.Row
|
||||
try:
|
||||
row = con.execute(
|
||||
"SELECT skills_version, prompts_version FROM _runs WHERE run_id=?",
|
||||
(eval_run_id,),
|
||||
).fetchone()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
if row is None:
|
||||
raise ValueError(f"eval run 不存在: {eval_run_id}")
|
||||
|
||||
skills_v, prompts_v = row["skills_version"], row["prompts_version"]
|
||||
|
||||
# 强校验——eval run 的版本必须与 --version 一致，且不得为 NULL
|
||||
if skills_v is None or prompts_v is None:
|
||||
raise ValueError(f"eval run {eval_run_id} 的 _runs 版本对为 NULL（未回填？），无法 promote")
|
||||
if skills_v != version:
|
||||
raise ValueError(f"eval run {eval_run_id} 的版本 {skills_v} 与 --version {version} 不符")
|
||||
|
||||
tmp_db = workspace_dir / "_promote_tmp.db"
|
||||
if tmp_db.exists():
|
||||
tmp_db.unlink()
|
||||
extract_run_db(workspace_dir / "harness.db", tmp_db, eval_run_id)
|
||||
try:
|
||||
seed_dir = init_seed(
|
||||
store_dir,
|
||||
name,
|
||||
workspace_dir / "skills" / skills_v,
|
||||
workspace_dir / "prompts" / prompts_v,
|
||||
tmp_db,
|
||||
baseline_run_id=eval_run_id,
|
||||
parent=f"{workspace_dir.name}:{version}",
|
||||
description=description,
|
||||
)
|
||||
finally:
|
||||
tmp_db.unlink()
|
||||
|
||||
logger.info("Promote 完成: {} -> {}", workspace_dir.name, seed_dir)
|
||||
return seed_dir
|
||||
@@ -0,0 +1,665 @@
|
||||
"""async 块序贯验证编排 — CE-Gate 局部验证的唯一独立子编排器。
|
||||
|
||||
从 TRM4 core/harness/validate.py (626 行) 迁移，重大重构：
|
||||
- 同步 → async（run_inference 注入为 async callable）
|
||||
- _classify_quadrants → core.evolution.classify_quadrants 纯函数
|
||||
- 配对逻辑 → 复用 core.evolution.pair_block + 本地证据行组装
|
||||
- _load_run_rows / _candidate_correctness_from_db → 共享 log.query()
|
||||
- materialize_candidate_skill 保持同步（纯文件操作）
|
||||
|
||||
基线与候选在同一阶梯前缀上逐块配对，只数翻转（基线错→候选对 = W，
|
||||
基线对→候选错 = L），每块结束调 gate_decision 做四出口判定。
|
||||
基线侧逐题对错走 BaselineCache 内容寻址缓存，miss 才新鲜跑。
|
||||
判定逻辑全部在 core/evolution/gate，本模块只负责推理编排与证据收集。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from app.harness.gate_ladder import BaselineCache, skill_hash
|
||||
from core.evolution import (
|
||||
GateParams,
|
||||
GateVerdict,
|
||||
RejectedEdit,
|
||||
classify_quadrants,
|
||||
gate_decision,
|
||||
pair_block,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.harness.inference import InferenceResult
|
||||
from app.harness.log import HarnessLog
|
||||
from core.types import GeneratedQuestion
|
||||
|
||||
|
||||
# gate_decision 的 decision → ValidationOutcome.stop_reason 映射
|
||||
_STOP_REASON_BY_DECISION: dict[str, str] = {
|
||||
"accept_confirmed": "confirmed",
|
||||
"reject_directional": "directional",
|
||||
"reject_futility": "futility",
|
||||
"accept_provisional": "provisional",
|
||||
"reject_inertia": "inertia",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 注入协议
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class RunInferenceFn(Protocol):
|
||||
"""注入的推理函数协议。
|
||||
|
||||
调用方（runner）负责绑定 llm、tool_dispatch_fn、prompt_builder、
|
||||
log、concurrency、max_steps、skill_mode 等共享依赖。
|
||||
validate 侧只传 questions、run_id、skills_dir 三个逐块变化的参数。
|
||||
"""
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
questions: list[GeneratedQuestion],
|
||||
*,
|
||||
run_id: str,
|
||||
skills_dir: Path,
|
||||
) -> InferenceResult: ...
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 数据类型
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InferenceRunConfig:
|
||||
"""一次推理运行的配置三元组，把"如何跑推理"内聚成一组。
|
||||
|
||||
字段:
|
||||
concurrency: 推理并发度。
|
||||
max_steps: 单题最大推理步数。
|
||||
skill_mode: 推理 skill 模式（"auto" / "manual" / "none"）。
|
||||
"""
|
||||
|
||||
concurrency: int
|
||||
max_steps: int
|
||||
skill_mode: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationOutcome:
|
||||
"""CE-Gate 局部验证结果：三态动作 + e-process 证据 + 已观测题逐题对错。
|
||||
|
||||
correctness 二轨语义：candidate_correctness 只含已观测题（早停后是
|
||||
阶梯前缀子集）；accept 时由 runner 按题粒度增量合并进 state.correctness。
|
||||
"""
|
||||
|
||||
action: str # accept_confirmed | accept_provisional | reject
|
||||
accepted: bool
|
||||
stop_reason: str # confirmed | directional | futility | provisional | inertia
|
||||
e_value: float
|
||||
w: int
|
||||
l: int # noqa: E741
|
||||
n_used: int
|
||||
delta_hat: float
|
||||
delta_shrunk: float
|
||||
baseline_acc: float # 已观测题上的基线准确率（观测口径）
|
||||
candidate_acc: float # 已观测题上的候选准确率（观测口径）
|
||||
improvements: list[str] = field(default_factory=list)
|
||||
regressions: list[str] = field(default_factory=list)
|
||||
persistent_fails: list[str] = field(default_factory=list)
|
||||
stable_successes: list[str] = field(default_factory=list)
|
||||
candidate_correctness: dict[str, bool] = field(default_factory=dict)
|
||||
evidence_rows: list[dict] = field(default_factory=list) # gate_evidence 逐题行，runner 落库
|
||||
|
||||
|
||||
@dataclass
|
||||
class Probation:
|
||||
"""一个题型的在途试用账本（每题型至多一个）。
|
||||
|
||||
字段:
|
||||
task_type: 题型。
|
||||
anchor_skills_version: 锚版本名（最近一个 CONFIRMED 的 skills 版本）——
|
||||
回滚时恢复该版本中本题型 skill 文件的内容。
|
||||
target_file: 该题型解析后的 skill 文件名。
|
||||
correctness_snapshot: 开账时该题型 val 题的对错快照（回滚时恢复）。
|
||||
opened_step: 开账时的 global_step（观测用）。
|
||||
pending_edits: 试用链上全部候选 edit 的黑名单素材（回滚时整链入黑名单）。
|
||||
"""
|
||||
|
||||
task_type: str
|
||||
anchor_skills_version: str
|
||||
target_file: str
|
||||
correctness_snapshot: dict[str, bool]
|
||||
opened_step: int
|
||||
pending_edits: list[RejectedEdit] = field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 同步辅助函数
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def materialize_candidate_skill(
|
||||
workspace_dir: Path,
|
||||
base_skills_version: str,
|
||||
target_file: str,
|
||||
content: str,
|
||||
) -> Path:
|
||||
"""将候选 skill 正文物化为 workspace 专用临时目录下唯一命名的候选 skills 目录。
|
||||
|
||||
复制基线 skills 目录到 .cand_tmp/ 下的唯一命名临时目录，然后覆写 target_file。
|
||||
构建失败时尽力清理已建临时目录再重抛原始异常。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。基线 skills 从 workspace_dir/skills/<base>
|
||||
复制，临时候选落 workspace_dir/.cand_tmp/。
|
||||
base_skills_version: 基线 skills 版本名。
|
||||
target_file: 被替换的 skill 文件名。
|
||||
content: 候选 skill 文件全文。
|
||||
|
||||
返回:
|
||||
新建的临时候选目录绝对路径。
|
||||
|
||||
契约:
|
||||
构建失败（OSError）时尽力清理已建临时目录再重抛原始异常；
|
||||
清理本身失败记 warning。
|
||||
"""
|
||||
cand_tmp_root = workspace_dir / ".cand_tmp"
|
||||
cand_tmp_root.mkdir(parents=True, exist_ok=True)
|
||||
cand_dir = Path(tempfile.mkdtemp(prefix=f"{base_skills_version}_cand_", dir=cand_tmp_root))
|
||||
try:
|
||||
base_dir = workspace_dir / "skills" / base_skills_version
|
||||
shutil.copytree(base_dir, cand_dir, dirs_exist_ok=True)
|
||||
(cand_dir / target_file).write_text(content, encoding="utf-8")
|
||||
except OSError:
|
||||
try:
|
||||
shutil.rmtree(cand_dir)
|
||||
except OSError as cleanup_err:
|
||||
logger.warning("候选物化失败后清理临时目录也失败 {}: {}", cand_dir, cleanup_err)
|
||||
raise
|
||||
return cand_dir
|
||||
|
||||
|
||||
def _load_run_rows(
|
||||
log: HarnessLog,
|
||||
run_id: str,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""读取单个 run 的逐题预测行并规范化轨迹字段。
|
||||
|
||||
从 predictions 表读取指定 run 的题目级记录，补充 _correct
|
||||
与规范化后的 steps 字段。保持同步（log.query）——仅在推理完成后调用。
|
||||
|
||||
参数:
|
||||
log: HarnessLog 共享实例（用 query 方法做只读 SELECT）。
|
||||
run_id: 待读取的预测 run_id。
|
||||
|
||||
返回:
|
||||
以 question_id 为键的行字典。每行至少包含 prediction、answer、
|
||||
_correct、steps 等字段。
|
||||
"""
|
||||
rows = log.query(
|
||||
"SELECT question_id, prediction, answer, steps_json FROM predictions WHERE run_id=?",
|
||||
(run_id,),
|
||||
)
|
||||
normalized: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
raw_steps = row.get("steps_json")
|
||||
parsed_steps: Any = raw_steps
|
||||
if isinstance(raw_steps, str):
|
||||
try:
|
||||
parsed_steps = json.loads(raw_steps)
|
||||
except json.JSONDecodeError:
|
||||
parsed_steps = []
|
||||
steps = parsed_steps if isinstance(parsed_steps, list) else []
|
||||
normalized[row["question_id"]] = {
|
||||
**row,
|
||||
"_correct": row.get("prediction") == row.get("answer"),
|
||||
"steps": steps,
|
||||
}
|
||||
return normalized
|
||||
|
||||
|
||||
def _candidate_correctness_from_db(
|
||||
log: HarnessLog,
|
||||
run_id: str,
|
||||
chunk: list[GeneratedQuestion],
|
||||
) -> dict[str, bool]:
|
||||
"""从 db 读取候选/基线 run 在指定题目上的逐题对错。
|
||||
|
||||
参数:
|
||||
log: HarnessLog 共享实例。
|
||||
run_id: 推理 run_id。
|
||||
chunk: 题目列表。
|
||||
|
||||
返回:
|
||||
question_id -> 是否答对的映射。缺行的题目记为 False。
|
||||
"""
|
||||
rows = _load_run_rows(log, run_id)
|
||||
return {q.question_id: rows.get(q.question_id, {}).get("_correct", False) for q in chunk}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 块级 async 函数
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _resolve_baseline_block(
|
||||
chunk: list[GeneratedQuestion],
|
||||
task_type: str,
|
||||
s_hash: str,
|
||||
prompts_version: str,
|
||||
baseline_cache: BaselineCache,
|
||||
base_skills_dir: Path,
|
||||
run_inference: RunInferenceFn,
|
||||
log: HarnessLog,
|
||||
run_id: str,
|
||||
) -> tuple[dict[str, bool], int, int]:
|
||||
"""基线侧处理一个块：缓存优先，miss 的题新鲜跑基线版本并回写缓存。
|
||||
|
||||
参数:
|
||||
chunk: 当前块的题目列表。
|
||||
task_type: 当前验证题型（缓存键成分）。
|
||||
s_hash: 基线侧生效 skill 的内容哈希（缓存键成分）。
|
||||
prompts_version: 当前 prompts 版本（缓存键成分）。
|
||||
baseline_cache: 基线侧逐题对错缓存。
|
||||
base_skills_dir: 基线 skills 版本目录。
|
||||
run_inference: 注入的 async 推理函数。
|
||||
log: HarnessLog 共享实例（推理后读预测）。
|
||||
run_id: 本块基线 run_id。
|
||||
|
||||
返回:
|
||||
(b_map, errors_inc, denom_inc)：块内 question_id -> 基线对错、
|
||||
本块新增的 INFRA error 计数与推理题次分母增量（全命中时为 0, 0）。
|
||||
"""
|
||||
misses = [
|
||||
q
|
||||
for q in chunk
|
||||
if baseline_cache.get(task_type, s_hash, prompts_version, q.question_id) is None
|
||||
]
|
||||
errors_inc = 0
|
||||
denom_inc = 0
|
||||
if misses:
|
||||
r_b = await run_inference(misses, run_id=run_id, skills_dir=base_skills_dir)
|
||||
errors_inc = r_b.stop_reason_counts.get("error", 0)
|
||||
denom_inc = r_b.total
|
||||
fresh = _candidate_correctness_from_db(log, r_b.run_id, misses)
|
||||
for qid, correct in fresh.items():
|
||||
baseline_cache.put(task_type, s_hash, prompts_version, qid, correct)
|
||||
|
||||
b_map: dict[str, bool] = {}
|
||||
for q in chunk:
|
||||
val = baseline_cache.get(task_type, s_hash, prompts_version, q.question_id)
|
||||
assert val is not None, f"基线缓存补齐后仍有 miss: {q.question_id} run_id={run_id}"
|
||||
b_map[q.question_id] = val
|
||||
return b_map, errors_inc, denom_inc
|
||||
|
||||
|
||||
async def _run_candidate_block(
|
||||
chunk: list[GeneratedQuestion],
|
||||
cand_dir: Path,
|
||||
run_inference: RunInferenceFn,
|
||||
log: HarnessLog,
|
||||
run_id: str,
|
||||
) -> tuple[dict[str, bool], int, int]:
|
||||
"""候选侧处理一个块：全块新鲜跑候选版本并从 db 读逐题对错。
|
||||
|
||||
参数:
|
||||
chunk: 当前块的题目列表。
|
||||
cand_dir: 已物化的候选 skills 目录。
|
||||
run_inference: 注入的 async 推理函数。
|
||||
log: HarnessLog 共享实例（推理后读预测）。
|
||||
run_id: 本块候选 run_id。
|
||||
|
||||
返回:
|
||||
(c_map, errors_inc, denom_inc)。
|
||||
"""
|
||||
r_c = await run_inference(chunk, run_id=run_id, skills_dir=cand_dir)
|
||||
c_map = _candidate_correctness_from_db(log, r_c.run_id, chunk)
|
||||
return c_map, r_c.stop_reason_counts.get("error", 0), r_c.total
|
||||
|
||||
|
||||
def _build_evidence_rows(
|
||||
chunk: list[GeneratedQuestion],
|
||||
b_map: dict[str, bool],
|
||||
c_map: dict[str, bool],
|
||||
task_type: str,
|
||||
block_idx: int,
|
||||
) -> list[dict]:
|
||||
"""组装一个块的 gate_evidence 逐题证据行。
|
||||
|
||||
e_value 留 None 待块判定后回填，stop_reason 留空串待终态回填。
|
||||
|
||||
参数:
|
||||
chunk: 当前块的题目列表。
|
||||
b_map: 块内 question_id -> 基线对错。
|
||||
c_map: 块内 question_id -> 候选对错。
|
||||
task_type: 当前验证题型。
|
||||
block_idx: 当前块序号。
|
||||
|
||||
返回:
|
||||
逐题证据行列表。
|
||||
"""
|
||||
return [
|
||||
{
|
||||
"question_id": q.question_id,
|
||||
"task_type": task_type,
|
||||
"block_idx": block_idx,
|
||||
"baseline_correct": b_map[q.question_id],
|
||||
"candidate_correct": c_map[q.question_id],
|
||||
"e_value": None,
|
||||
"stop_reason": "",
|
||||
}
|
||||
for q in chunk
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# INFRA 护栏
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _check_infra_guard(errors: int, infra_denom: int, gate_guard_err: float) -> None:
|
||||
"""跨块累计 INFRA 错误率护栏：分母 >=10 且超阈值时 raise。
|
||||
|
||||
参数:
|
||||
errors: 两侧累计 error 计数。
|
||||
infra_denom: 两侧累计推理题次分母。
|
||||
gate_guard_err: 错误率阈值。
|
||||
|
||||
异常:
|
||||
RuntimeError: 错误率超阈值。
|
||||
"""
|
||||
if infra_denom >= 10 and errors / infra_denom > gate_guard_err:
|
||||
raise RuntimeError(f"gate 推理累计错误率过高 {errors / infra_denom:.0%}，中止本轮")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 终态组装
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _finalize_outcome(
|
||||
verdict: GateVerdict,
|
||||
w: int,
|
||||
l: int, # noqa: E741
|
||||
n_used: int,
|
||||
n_plan: int,
|
||||
base_obs: dict[str, bool],
|
||||
cand_obs: dict[str, bool],
|
||||
evidence_rows: list[dict],
|
||||
task_type: str,
|
||||
) -> ValidationOutcome:
|
||||
"""将块循环终态判定组装为 ValidationOutcome。
|
||||
|
||||
参数:
|
||||
verdict: 最后一块的 gate 判定结果。
|
||||
w: 累计 W（基线错→候选对翻转）。
|
||||
l: 累计 L（基线对→候选错翻转）。
|
||||
n_used: 已消费的阶梯题数。
|
||||
n_plan: 阶梯总题数。
|
||||
base_obs: 累计基线已观测对错。
|
||||
cand_obs: 累计候选已观测对错。
|
||||
evidence_rows: 逐题证据行。
|
||||
task_type: 验证题型（日志用）。
|
||||
|
||||
返回:
|
||||
ValidationOutcome。
|
||||
"""
|
||||
action = {
|
||||
"accept_confirmed": "accept_confirmed",
|
||||
"accept_provisional": "accept_provisional",
|
||||
}.get(verdict.decision, "reject")
|
||||
stop_reason = _STOP_REASON_BY_DECISION[verdict.decision]
|
||||
# 只有终态题的证据行才携带 stop_reason
|
||||
evidence_rows[-1]["stop_reason"] = stop_reason
|
||||
|
||||
quadrants = classify_quadrants({qid: (base_obs[qid], cand_obs[qid]) for qid in base_obs})
|
||||
baseline_acc = sum(base_obs.values()) / len(base_obs)
|
||||
candidate_acc = sum(cand_obs.values()) / len(cand_obs)
|
||||
accepted = action != "reject"
|
||||
|
||||
logger.info(
|
||||
"gate 局部验证[{}]: 基线{:.1%} → 候选{:.1%} (W={} L={} E={:.2f} n={}/{}) {}",
|
||||
task_type,
|
||||
baseline_acc,
|
||||
candidate_acc,
|
||||
w,
|
||||
l,
|
||||
verdict.e_value,
|
||||
n_used,
|
||||
n_plan,
|
||||
"接受" if accepted else "回滚",
|
||||
)
|
||||
|
||||
return ValidationOutcome(
|
||||
action=action,
|
||||
accepted=accepted,
|
||||
stop_reason=stop_reason,
|
||||
e_value=verdict.e_value,
|
||||
w=w,
|
||||
l=l,
|
||||
n_used=n_used,
|
||||
delta_hat=verdict.delta_hat,
|
||||
delta_shrunk=verdict.delta_shrunk,
|
||||
baseline_acc=baseline_acc,
|
||||
candidate_acc=candidate_acc,
|
||||
improvements=quadrants.improvements,
|
||||
regressions=quadrants.regressions,
|
||||
persistent_fails=quadrants.persistent_fails,
|
||||
stable_successes=quadrants.stable_successes,
|
||||
candidate_correctness=cand_obs,
|
||||
evidence_rows=evidence_rows,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 主编排
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _run_local_validation(
|
||||
workspace_dir: Path,
|
||||
cand_dir: Path,
|
||||
base_skills_version: str,
|
||||
task_type: str,
|
||||
base_skill_content: str,
|
||||
plan: list[GeneratedQuestion],
|
||||
gate_params: GateParams,
|
||||
gate_block: int,
|
||||
gate_guard_err: float,
|
||||
baseline_cache: BaselineCache,
|
||||
prompts_version: str,
|
||||
run_inference: RunInferenceFn,
|
||||
log: HarnessLog,
|
||||
gate_run_prefix: str,
|
||||
) -> ValidationOutcome:
|
||||
"""块序贯循环主体：逐块基线（缓存优先）/候选配对推理，块间 e-process 判定。
|
||||
|
||||
按 gate_block 切阶梯前缀，每块先补齐基线侧缓存 miss（新鲜跑基线版本
|
||||
并逐题写 BaselineCache），再全块跑候选，配对累计 W/L 后调 gate_decision；
|
||||
非 continue 即早停。题尽时最后一块的判定即终态（n_remaining=0 走
|
||||
provisional/inertia 分支），无循环外补判。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
cand_dir: 已物化的候选 skills 目录。
|
||||
base_skills_version: 基线 skills 版本名。
|
||||
task_type: 当前验证题型。
|
||||
base_skill_content: 基线侧生效 skill 全文（skill_hash 作缓存键成分）。
|
||||
plan: 已截断到 gate_n_max 的阶梯出题序。
|
||||
gate_params: e-process 判据阈值组。
|
||||
gate_block: 块大小。
|
||||
gate_guard_err: 跨块累计 INFRA 错误率护栏（分母 >=10 才触发）。
|
||||
baseline_cache: 基线侧逐题对错缓存。
|
||||
prompts_version: 当前 prompts 版本（缓存键成分）。
|
||||
run_inference: 注入的 async 推理函数。
|
||||
log: HarnessLog 共享实例。
|
||||
gate_run_prefix: 块 run_id 前缀（含 "_gate_" 标记）。
|
||||
|
||||
返回:
|
||||
ValidationOutcome。
|
||||
|
||||
关键实现:
|
||||
INFRA 护栏跨块累计基线+候选两侧的 error 计数，分母（总推理题次）>=10
|
||||
且错误率超 gate_guard_err 时直接 raise，避免坏批次污染判定。
|
||||
"""
|
||||
w = 0
|
||||
l = 0 # noqa: E741
|
||||
n_used = 0
|
||||
errors = 0
|
||||
infra_denom = 0
|
||||
evidence_rows: list[dict] = []
|
||||
base_obs: dict[str, bool] = {}
|
||||
cand_obs: dict[str, bool] = {}
|
||||
s_hash = skill_hash(base_skill_content)
|
||||
base_skills_dir = workspace_dir / "skills" / base_skills_version
|
||||
chunks = [plan[i : i + gate_block] for i in range(0, len(plan), gate_block)]
|
||||
verdict: GateVerdict | None = None
|
||||
|
||||
for block_idx, chunk in enumerate(chunks):
|
||||
# Phase 1: 基线侧（缓存优先，miss 新鲜跑）+ 候选侧（全块新鲜跑）
|
||||
b_map, err_b, den_b = await _resolve_baseline_block(
|
||||
chunk=chunk,
|
||||
task_type=task_type,
|
||||
s_hash=s_hash,
|
||||
prompts_version=prompts_version,
|
||||
baseline_cache=baseline_cache,
|
||||
base_skills_dir=base_skills_dir,
|
||||
run_inference=run_inference,
|
||||
log=log,
|
||||
run_id=f"{gate_run_prefix}_b{block_idx}_base",
|
||||
)
|
||||
c_map, err_c, den_c = await _run_candidate_block(
|
||||
chunk=chunk,
|
||||
cand_dir=cand_dir,
|
||||
run_inference=run_inference,
|
||||
log=log,
|
||||
run_id=f"{gate_run_prefix}_b{block_idx}_cand",
|
||||
)
|
||||
|
||||
# Phase 2: INFRA 护栏（跨块累计，分母 >=10 才触发）
|
||||
errors += err_b + err_c
|
||||
infra_denom += den_b + den_c
|
||||
_check_infra_guard(errors, infra_denom, gate_guard_err)
|
||||
|
||||
# Phase 3: 配对 + 证据行 + 块间判定
|
||||
qids = [q.question_id for q in chunk]
|
||||
pair_result = pair_block(b_map, c_map, qids)
|
||||
for qid, (b, c) in pair_result.observed.items():
|
||||
base_obs[qid] = b
|
||||
cand_obs[qid] = c
|
||||
|
||||
block_rows = _build_evidence_rows(chunk, b_map, c_map, task_type, block_idx)
|
||||
|
||||
w += pair_result.w
|
||||
l += pair_result.l # noqa: E741
|
||||
n_used += len(chunk)
|
||||
verdict = gate_decision(w, l, n_used, len(plan) - n_used, params=gate_params)
|
||||
|
||||
for row in block_rows:
|
||||
row["e_value"] = verdict.e_value
|
||||
evidence_rows.extend(block_rows)
|
||||
|
||||
if verdict.decision != "continue":
|
||||
break
|
||||
|
||||
# 最后一块判定即终态（n_remaining=0 → provisional/inertia）
|
||||
assert verdict is not None, "空阶梯应已在 validate_skill_local 入口拒绝"
|
||||
return _finalize_outcome(
|
||||
verdict=verdict,
|
||||
w=w,
|
||||
l=l,
|
||||
n_used=n_used,
|
||||
n_plan=len(plan),
|
||||
base_obs=base_obs,
|
||||
cand_obs=cand_obs,
|
||||
evidence_rows=evidence_rows,
|
||||
task_type=task_type,
|
||||
)
|
||||
|
||||
|
||||
async def validate_skill_local(
|
||||
workspace_dir: Path,
|
||||
base_skills_version: str,
|
||||
task_type: str,
|
||||
target_file: str,
|
||||
candidate_content: str,
|
||||
base_skill_content: str,
|
||||
ladder_items: list[GeneratedQuestion],
|
||||
gate_params: GateParams,
|
||||
gate_block: int,
|
||||
gate_n_max: int,
|
||||
gate_guard_err: float,
|
||||
baseline_cache: BaselineCache,
|
||||
prompts_version: str,
|
||||
run_inference: RunInferenceFn,
|
||||
log: HarnessLog,
|
||||
gate_run_prefix: str,
|
||||
) -> ValidationOutcome:
|
||||
"""块序贯配对验证：阶梯出题，基线/候选逐块配对，e-process 四出口早停。
|
||||
|
||||
参数:
|
||||
workspace_dir: workspace 根目录。
|
||||
base_skills_version: 基线 skills 版本名（候选物化复制源）。
|
||||
task_type: 待验证题型。
|
||||
target_file: fallback 解析后该题型的真实生效 skill 文件名
|
||||
（record.target_file，可能是共享 default-strategy.md）；
|
||||
候选物化写此文件，与 accept 路径同源。
|
||||
candidate_content: 候选 skill 全文。
|
||||
base_skill_content: 基线侧该题型解析后生效 skill 文件全文
|
||||
（skill_hash(base_skill_content) 作 BaselineCache 键成分）。
|
||||
ladder_items: 阶梯序题目列表（已排除本 step 案例包题）。
|
||||
gate_params: e-process 判据阈值组。
|
||||
gate_block: 块大小。
|
||||
gate_n_max: 单 gate 题数上限。
|
||||
gate_guard_err: 跨块累计 INFRA 错误率护栏（分母 >=10 才触发）。
|
||||
baseline_cache: 基线侧逐题对错缓存。
|
||||
prompts_version: 当前 prompts 版本（缓存键成分）。
|
||||
run_inference: 注入的 async 推理函数（RunInferenceFn 协议）。
|
||||
log: HarnessLog 共享实例（供 DB 回读逐题对错）。
|
||||
gate_run_prefix: gate 内推理 run_id 前缀，必须含 "_gate_"
|
||||
（防泄露过滤靠它识别）。块 run_id = f"{prefix}_b{block_idx}_{arm}"。
|
||||
|
||||
返回:
|
||||
ValidationOutcome。逐题证据记入 outcome.evidence_rows 随结果返回，
|
||||
gate_evidence 落库由调用方（runner）负责。
|
||||
"""
|
||||
if "_gate_" not in gate_run_prefix:
|
||||
raise ValueError(f"gate_run_prefix 必须含 '_gate_'（防泄露过滤依赖）: {gate_run_prefix!r}")
|
||||
if not ladder_items:
|
||||
raise ValueError(f"task_type={task_type} 阶梯为空，无法验证")
|
||||
|
||||
plan = ladder_items[:gate_n_max]
|
||||
cand_dir = materialize_candidate_skill(
|
||||
workspace_dir, base_skills_version, target_file, candidate_content
|
||||
)
|
||||
try:
|
||||
return await _run_local_validation(
|
||||
workspace_dir=workspace_dir,
|
||||
cand_dir=cand_dir,
|
||||
base_skills_version=base_skills_version,
|
||||
task_type=task_type,
|
||||
base_skill_content=base_skill_content,
|
||||
plan=plan,
|
||||
gate_params=gate_params,
|
||||
gate_block=gate_block,
|
||||
gate_guard_err=gate_guard_err,
|
||||
baseline_cache=baseline_cache,
|
||||
prompts_version=prompts_version,
|
||||
run_inference=run_inference,
|
||||
log=log,
|
||||
gate_run_prefix=gate_run_prefix,
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
shutil.rmtree(cand_dir)
|
||||
except OSError as e:
|
||||
logger.warning("候选临时目录清理失败 {}: {}", cand_dir, e)
|
||||
@@ -0,0 +1,478 @@
|
||||
"""Workspace 生命周期管理 + manifest 读写 + Protocol 实现。
|
||||
|
||||
Workspace 是一次实验的独立工作区，通过 manifest.json 引用 Store 中的
|
||||
特定版本资源并记录实验过程。Skills/Prompts 权重拷入 workspace 本地，
|
||||
训练产物只进 workspace 不污染 Store。
|
||||
|
||||
VersionedSkillStore / VersionedPromptStore 实现 core/evolution/protocols.py
|
||||
中定义的只读端口，供 core/ 层以 Protocol 方式读取技能和提示词。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from app.harness.store import read_seed
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedPaths:
|
||||
"""manifest 解析后的绝对路径集合。
|
||||
|
||||
属性:
|
||||
store_dir: Store 根目录绝对路径。
|
||||
videos_dir: 视频数据目录。
|
||||
questions_dir: 当前引用的题目目录。
|
||||
skills_dir: 当前引用的 Skill 版本目录（workspace 内）。
|
||||
prompts_dir: 当前引用的 Prompt 版本目录（workspace 内）。
|
||||
workspace_dir: Workspace 根目录。
|
||||
db_path: harness.db 路径。
|
||||
analyses_dir: 分析报告目录。
|
||||
runs_dir: 运行临时状态目录。
|
||||
"""
|
||||
|
||||
store_dir: Path
|
||||
videos_dir: Path
|
||||
questions_dir: Path
|
||||
skills_dir: Path
|
||||
prompts_dir: Path
|
||||
workspace_dir: Path
|
||||
db_path: Path
|
||||
analyses_dir: Path
|
||||
runs_dir: Path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 内部工具
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MANIFEST_CURRENT_KEYS = {"videos", "questions", "skills", "prompts"}
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
"""返回当前 UTC 时间的 ISO 格式字符串。"""
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Workspace 核心函数
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _scaffold_workspace(
|
||||
workspace_dir: Path,
|
||||
store_dir: Path,
|
||||
questions: str,
|
||||
skills_version: str,
|
||||
prompts_version: str,
|
||||
) -> None:
|
||||
"""写 manifest + 建 analyses/runs 目录（不拷权重；权重由调用方按来源拷入）。
|
||||
|
||||
参数:
|
||||
workspace_dir: 目标 workspace（由调用方保证不存在）。
|
||||
store_dir: Store 根目录。
|
||||
questions: 题目相对路径，如 ``'benchmarks/Video-MME'``。
|
||||
skills_version: manifest.current.skills 初始版本号。
|
||||
prompts_version: manifest.current.prompts 初始版本号。
|
||||
|
||||
关键实现:
|
||||
不依赖任何外部资源源（store 中的 skills/prompts 是否存在不在此校验），
|
||||
因此可被 init_workspace 与种子初始化复用；store 引用以相对路径写入 manifest。
|
||||
"""
|
||||
workspace_dir.mkdir(parents=True)
|
||||
(workspace_dir / "analyses").mkdir()
|
||||
(workspace_dir / "runs").mkdir()
|
||||
|
||||
store_abs = store_dir.resolve()
|
||||
store_rel = os.path.relpath(store_abs, workspace_dir.resolve())
|
||||
|
||||
manifest = {
|
||||
"name": workspace_dir.name,
|
||||
"created_at": _now_iso(),
|
||||
"store": store_rel,
|
||||
"current": {
|
||||
"videos": "videos",
|
||||
"questions": f"questions/{questions}",
|
||||
"skills": f"skills/{skills_version}",
|
||||
"prompts": f"prompts/{prompts_version}",
|
||||
},
|
||||
"history": [],
|
||||
}
|
||||
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
def init_workspace(
|
||||
workspace_dir: Path,
|
||||
store_dir: Path,
|
||||
questions: str,
|
||||
skills_version: str,
|
||||
prompts_version: str,
|
||||
) -> None:
|
||||
"""创建 Workspace 目录并写入初始 manifest.json，拷贝种子权重。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 目标路径（不得已存在）。
|
||||
store_dir: Store 根目录。
|
||||
questions: 题目在 questions/ 下的相对路径，如 ``"benchmarks/Video-MME"``。
|
||||
skills_version: Skills 版本号，如 ``"v1"``。
|
||||
prompts_version: Prompts 版本号，如 ``"v1"``。
|
||||
|
||||
异常:
|
||||
FileExistsError: Workspace 目录已存在。
|
||||
FileNotFoundError: 引用的资源在 Store 中不存在。
|
||||
"""
|
||||
if workspace_dir.exists():
|
||||
raise FileExistsError(f"Workspace 已存在: {workspace_dir}")
|
||||
|
||||
store_abs = store_dir.resolve()
|
||||
refs = {
|
||||
"skills": f"skills/{skills_version}",
|
||||
"prompts": f"prompts/{prompts_version}",
|
||||
"questions": f"questions/{questions}",
|
||||
}
|
||||
for label, rel in refs.items():
|
||||
full = store_abs / rel
|
||||
if not full.is_dir():
|
||||
raise FileNotFoundError(f"Store 中不存在 {label}: {full}")
|
||||
|
||||
_scaffold_workspace(workspace_dir, store_dir, questions, skills_version, prompts_version)
|
||||
|
||||
# 拷种子权重进 workspace：v2+ 训练产物只进 workspace，不污染 store
|
||||
shutil.copytree(store_abs / refs["skills"], workspace_dir / refs["skills"])
|
||||
shutil.copytree(store_abs / refs["prompts"], workspace_dir / refs["prompts"])
|
||||
logger.info("Workspace 初始化完成: {}", workspace_dir)
|
||||
|
||||
|
||||
def init_workspace_from_seed(
|
||||
workspace_dir: Path,
|
||||
store_dir: Path,
|
||||
seed_name: str,
|
||||
questions: str,
|
||||
) -> str:
|
||||
"""从种子全新建 workspace：拷权重 -> v1、baseline.db -> harness.db、读 baseline_run_id。
|
||||
|
||||
参数:
|
||||
workspace_dir: 目标 workspace（不得已存在）。
|
||||
store_dir: Store 根目录。
|
||||
seed_name: 种子名（store/seeds 下）。
|
||||
questions: 题目相对路径，如 ``'benchmarks/Video-MME'``。
|
||||
|
||||
返回:
|
||||
baseline_run_id（供 build_pools 使用）。
|
||||
|
||||
异常:
|
||||
FileExistsError: workspace 已存在。
|
||||
FileNotFoundError: 种子不存在（由 read_seed 抛出），或 questions ref 目录不存在。
|
||||
|
||||
关键实现:
|
||||
破坏性/创建操作前先校验 questions ref 存在：fresh 路径在 runner 侧已先
|
||||
归档旧 ws，若到 build_pools 才发现 questions 缺失则旧 ws 已被毁；
|
||||
故在此尽早报错（fail-fast），让新 ws 在创建前失败。
|
||||
"""
|
||||
if workspace_dir.exists():
|
||||
raise FileExistsError(f"Workspace 已存在: {workspace_dir}")
|
||||
|
||||
# 校验种子存在 + 取 baseline_run_id
|
||||
meta = read_seed(store_dir, seed_name)
|
||||
|
||||
# fail-fast：校验 questions ref 存在
|
||||
questions_ref = store_dir / "questions" / questions
|
||||
if not questions_ref.is_dir():
|
||||
raise FileNotFoundError(f"questions ref 目录不存在: {questions_ref}")
|
||||
|
||||
seed_dir = store_dir / "seeds" / seed_name
|
||||
_scaffold_workspace(workspace_dir, store_dir, questions, "v1", "v1")
|
||||
shutil.copytree(seed_dir / "skills", workspace_dir / "skills" / "v1")
|
||||
shutil.copytree(seed_dir / "prompts", workspace_dir / "prompts" / "v1")
|
||||
shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db")
|
||||
|
||||
logger.info("Workspace 从种子 '{}' 初始化完成: {}", seed_name, workspace_dir)
|
||||
return meta["baseline_run_id"]
|
||||
|
||||
|
||||
def load_manifest(workspace_dir: Path) -> dict:
|
||||
"""读取并返回 workspace 的 manifest.json。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
|
||||
返回:
|
||||
manifest 字典。
|
||||
|
||||
异常:
|
||||
FileNotFoundError: manifest.json 不存在。
|
||||
"""
|
||||
manifest_path = workspace_dir / "manifest.json"
|
||||
if not manifest_path.exists():
|
||||
raise FileNotFoundError(f"manifest.json 不存在: {manifest_path}")
|
||||
return json.loads(manifest_path.read_text())
|
||||
|
||||
|
||||
def resolve_paths(workspace_dir: Path) -> ResolvedPaths:
|
||||
"""读取 manifest，解析 current 中所有资源的绝对路径。
|
||||
|
||||
skills_dir/prompts_dir 解析到 workspace（非 store），
|
||||
videos_dir/questions_dir 解析到 store。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
|
||||
返回:
|
||||
ResolvedPaths 实例，包含所有资源的绝对路径。
|
||||
"""
|
||||
manifest = load_manifest(workspace_dir)
|
||||
ws_abs = workspace_dir.resolve()
|
||||
store_abs = (ws_abs / manifest["store"]).resolve()
|
||||
current = manifest["current"]
|
||||
return ResolvedPaths(
|
||||
store_dir=store_abs,
|
||||
videos_dir=store_abs / current["videos"],
|
||||
questions_dir=store_abs / current["questions"],
|
||||
skills_dir=ws_abs / current["skills"],
|
||||
prompts_dir=ws_abs / current["prompts"],
|
||||
workspace_dir=ws_abs,
|
||||
db_path=ws_abs / "harness.db",
|
||||
analyses_dir=ws_abs / "analyses",
|
||||
runs_dir=ws_abs / "runs",
|
||||
)
|
||||
|
||||
|
||||
def list_video_ids(workspace_dir: Path) -> list[str]:
|
||||
"""列出 workspace 引用的所有视频 ID（含 tree.json 的子目录名）。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
|
||||
返回:
|
||||
排序后的视频 ID 列表。
|
||||
"""
|
||||
paths = resolve_paths(workspace_dir)
|
||||
video_ids = []
|
||||
for entry in paths.videos_dir.iterdir():
|
||||
if entry.is_dir() and (entry / "tree.json").exists():
|
||||
video_ids.append(entry.name)
|
||||
return sorted(video_ids)
|
||||
|
||||
|
||||
def update_manifest(workspace_dir: Path, **version_updates: str) -> None:
|
||||
"""更新 manifest 的 current 字段。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
**version_updates: 要更新的字段及其新值，如 ``skills="skills/v2"``。
|
||||
|
||||
异常:
|
||||
KeyError: 更新的字段不在 current 允许的 key 白名单中。
|
||||
"""
|
||||
invalid = set(version_updates) - _MANIFEST_CURRENT_KEYS
|
||||
if invalid:
|
||||
raise KeyError(f"无效的 manifest current 字段: {invalid}")
|
||||
manifest = load_manifest(workspace_dir)
|
||||
manifest["current"].update(version_updates)
|
||||
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
def record_run(workspace_dir: Path, run_id: str) -> Path:
|
||||
"""将 current 版本快照追加到 manifest history，创建 run 目录和 per-video wiki 目录。
|
||||
|
||||
幂等：同 run_id 不重复追加 history（长跑中断后重启 / held-out 复用 run_id 时）。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
run_id: 本次运行的唯一标识，如 ``"run_001"``。
|
||||
|
||||
返回:
|
||||
创建的 run 目录路径。
|
||||
"""
|
||||
manifest = load_manifest(workspace_dir)
|
||||
current = manifest["current"]
|
||||
|
||||
# 幂等：同 run_id 不重复追加 history
|
||||
if not any(h["run_id"] == run_id for h in manifest["history"]):
|
||||
manifest["history"].append(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"started_at": _now_iso(),
|
||||
"skills": current["skills"],
|
||||
"prompts": current["prompts"],
|
||||
"questions": current["questions"],
|
||||
}
|
||||
)
|
||||
(workspace_dir / "manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2)
|
||||
)
|
||||
|
||||
run_dir = workspace_dir / "runs" / run_id
|
||||
# exist_ok：同 run_id 重跑时 run 目录已存在不应崩溃
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
for video_id in list_video_ids(workspace_dir):
|
||||
(run_dir / video_id / "wiki").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.debug("Run 已记录: {}", run_id)
|
||||
return run_dir
|
||||
|
||||
|
||||
def read_best(workspace_dir: Path) -> dict | None:
|
||||
"""读取 manifest 的 best 指针，未设置时返回 None。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
|
||||
返回:
|
||||
best 字典（skills/prompts/val_acc/run_id/epoch），未设置时 None。
|
||||
"""
|
||||
return load_manifest(workspace_dir).get("best")
|
||||
|
||||
|
||||
def update_best(
|
||||
workspace_dir: Path,
|
||||
skills: str,
|
||||
prompts: str,
|
||||
val_acc: float,
|
||||
run_id: str,
|
||||
epoch: int,
|
||||
) -> None:
|
||||
"""写入 manifest 的 best 指针（历史最优版本快照，与 current 平级）。
|
||||
|
||||
best 独立于 current——更新 best 不影响 current。
|
||||
|
||||
参数:
|
||||
workspace_dir: Workspace 根目录。
|
||||
skills: 最优 skills 版本完整 ref，如 ``'skills/v2'``。
|
||||
prompts: 最优 prompts 版本完整 ref，如 ``'prompts/v2'``。
|
||||
val_acc: 该版本验证集准确率。
|
||||
run_id: 该版本验证 run_id。
|
||||
epoch: 达成该最优的轮次。
|
||||
"""
|
||||
manifest = load_manifest(workspace_dir)
|
||||
manifest["best"] = {
|
||||
"skills": skills,
|
||||
"prompts": prompts,
|
||||
"val_acc": val_acc,
|
||||
"run_id": run_id,
|
||||
"epoch": epoch,
|
||||
}
|
||||
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
logger.info("Best 已更新: val_acc={}, run={}, epoch={}", val_acc, run_id, epoch)
|
||||
|
||||
|
||||
def archive_workspace(workspace_dir: Path) -> Path:
|
||||
"""把 workspace 整体移动到同级 .archive/<name>-<ts>，返回归档路径。
|
||||
|
||||
参数:
|
||||
workspace_dir: 要归档的 Workspace 根目录。
|
||||
|
||||
返回:
|
||||
归档后的目标路径。
|
||||
|
||||
异常:
|
||||
FileNotFoundError: workspace 不存在。
|
||||
"""
|
||||
if not workspace_dir.exists():
|
||||
raise FileNotFoundError(f"workspace 不存在: {workspace_dir}")
|
||||
|
||||
archive_root = workspace_dir.parent / ".archive"
|
||||
archive_root.mkdir(exist_ok=True)
|
||||
ts = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
|
||||
target = archive_root / f"{workspace_dir.name}-{ts}"
|
||||
shutil.move(str(workspace_dir), str(target))
|
||||
|
||||
logger.info("Workspace 已归档: {} -> {}", workspace_dir, target)
|
||||
return target
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Protocol 实现：VersionedSkillStore / VersionedPromptStore
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VersionedSkillStore:
|
||||
"""版本化技能读取端口实现。
|
||||
|
||||
满足 ``core/evolution/protocols.py::SkillStore`` Protocol。
|
||||
从指定的 skills 版本目录读取 ``.md`` 文件。
|
||||
|
||||
参数:
|
||||
skills_dir: skills 版本目录绝对路径（如 ``workspace/skills/v1``）。
|
||||
"""
|
||||
|
||||
def __init__(self, skills_dir: Path) -> None:
|
||||
if not skills_dir.is_dir():
|
||||
raise FileNotFoundError(f"Skills 目录不存在: {skills_dir}")
|
||||
self._dir = skills_dir
|
||||
|
||||
def read_skill(self, filename: str) -> str:
|
||||
"""读取指定 skill 文件的全文内容。
|
||||
|
||||
参数:
|
||||
filename: skill 文件名，如 ``'temporal-reasoning.md'``。
|
||||
|
||||
返回:
|
||||
文件全文内容。
|
||||
|
||||
异常:
|
||||
FileNotFoundError: 文件不存在。
|
||||
"""
|
||||
path = self._dir / filename
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Skill 文件不存在: {path}")
|
||||
return path.read_text()
|
||||
|
||||
def list_skill_files(self) -> list[str]:
|
||||
"""列出当前版本所有 skill 文件名。
|
||||
|
||||
返回:
|
||||
文件名列表（排序）。
|
||||
"""
|
||||
return sorted(entry.name for entry in self._dir.iterdir() if entry.is_file())
|
||||
|
||||
|
||||
class VersionedPromptStore:
|
||||
"""版本化提示词读取端口实现。
|
||||
|
||||
满足 ``core/evolution/protocols.py::PromptStore`` Protocol。
|
||||
从指定的 prompts 版本目录读取 ``.md`` 文件。
|
||||
|
||||
参数:
|
||||
prompts_dir: prompts 版本目录绝对路径（如 ``workspace/prompts/v1``）。
|
||||
"""
|
||||
|
||||
def __init__(self, prompts_dir: Path) -> None:
|
||||
if not prompts_dir.is_dir():
|
||||
raise FileNotFoundError(f"Prompts 目录不存在: {prompts_dir}")
|
||||
self._dir = prompts_dir
|
||||
|
||||
def read_prompt(self, filename: str) -> str:
|
||||
"""读取指定 prompt 文件的全文内容。
|
||||
|
||||
参数:
|
||||
filename: prompt 文件名，如 ``'system.md'``。
|
||||
|
||||
返回:
|
||||
文件全文内容。
|
||||
|
||||
异常:
|
||||
FileNotFoundError: 文件不存在。
|
||||
"""
|
||||
path = self._dir / filename
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Prompt 文件不存在: {path}")
|
||||
return path.read_text()
|
||||
|
||||
def list_prompt_files(self) -> list[str]:
|
||||
"""列出当前版本所有 prompt 文件名。
|
||||
|
||||
返回:
|
||||
文件名列表（排序）。
|
||||
"""
|
||||
return sorted(entry.name for entry in self._dir.iterdir() if entry.is_file())
|
||||
@@ -21,9 +21,47 @@ from core.evolution.patch import (
|
||||
replace_appendix_notes,
|
||||
replace_momentum,
|
||||
)
|
||||
from core.evolution.types import (
|
||||
CaseSample,
|
||||
DiagnosePrompts,
|
||||
DiagnosisResult,
|
||||
ErrorAttribution,
|
||||
EvolutionRecord,
|
||||
EvolutionResult,
|
||||
EvolvePrompts,
|
||||
GateParams,
|
||||
GateVerdict,
|
||||
PairResult,
|
||||
QuadrantClassification,
|
||||
QuestionMetrics,
|
||||
RejectedEdit,
|
||||
SkillCasePack,
|
||||
SkillStepAdherence,
|
||||
SpanMetrics,
|
||||
SystemCasePack,
|
||||
ToolCasePack,
|
||||
)
|
||||
from core.evolution.validate import classify_quadrants, compute_accuracy, pair_block
|
||||
|
||||
__all__ = [
|
||||
"CaseSample",
|
||||
"DiagnosePrompts",
|
||||
"DiagnosisResult",
|
||||
"ErrorAttribution",
|
||||
"EvolutionRecord",
|
||||
"EvolutionResult",
|
||||
"EvolvePrompts",
|
||||
"GateParams",
|
||||
"GateVerdict",
|
||||
"PairResult",
|
||||
"QuadrantClassification",
|
||||
"QuestionMetrics",
|
||||
"RejectedEdit",
|
||||
"SkillCasePack",
|
||||
"SkillStepAdherence",
|
||||
"SpanMetrics",
|
||||
"SystemCasePack",
|
||||
"ToolCasePack",
|
||||
"append_to_appendix",
|
||||
"apply_patch_with_report",
|
||||
"classify_quadrants",
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
"""app/harness/batching.py 的单元测试。
|
||||
|
||||
覆盖 FFD + round-robin mini-batch 构建的核心场景：
|
||||
确定性、小类不拆、大类 round-robin、正确率混合、空错题、参数校验、
|
||||
correctness False vs None 精确匹配。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from core.types import GeneratedQuestion
|
||||
from app.harness.batching import (
|
||||
build_batches,
|
||||
_validate_params,
|
||||
_select_mixed_by_task_type,
|
||||
)
|
||||
|
||||
import random
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 辅助构造
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_q(
|
||||
qid: str,
|
||||
task_type: str = "default",
|
||||
video_id: str = "v1",
|
||||
) -> GeneratedQuestion:
|
||||
"""构造最小 GeneratedQuestion 用于测试。"""
|
||||
return GeneratedQuestion(
|
||||
question_id=qid,
|
||||
video_id=video_id,
|
||||
task_type=task_type,
|
||||
question=f"question_{qid}",
|
||||
options=("A. a", "B. b", "C. c", "D. d"),
|
||||
answer="A",
|
||||
source_nodes=("n1",),
|
||||
difficulty="medium",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# test_build_batches_deterministic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestBuildBatchesDeterministic:
|
||||
"""相同输入 + 相同 seed 产出完全一致的切分。"""
|
||||
|
||||
def test_same_seed_same_result(self) -> None:
|
||||
items = [_make_q(f"q{i}", task_type=f"type_{i % 3}") for i in range(20)]
|
||||
correctness = {f"q{i}": False for i in range(20)}
|
||||
r1 = build_batches(items, correctness, batch_size=5, min_class_per_batch=2, seed=42)
|
||||
r2 = build_batches(items, correctness, batch_size=5, min_class_per_batch=2, seed=42)
|
||||
assert r1 == r2
|
||||
|
||||
def test_different_seed_may_differ(self) -> None:
|
||||
"""不同 seed 结果应不同（极大概率，用大量样本保证）。"""
|
||||
items = [_make_q(f"q{i}", task_type=f"type_{i % 5}") for i in range(50)]
|
||||
correctness = {f"q{i}": False for i in range(50)}
|
||||
r1 = build_batches(items, correctness, batch_size=10, min_class_per_batch=3, seed=1)
|
||||
r2 = build_batches(items, correctness, batch_size=10, min_class_per_batch=3, seed=99)
|
||||
# 至少 batch 内容不同（不比较结构，只比较 selected_count 一致性）
|
||||
assert r1[1] == r2[1] # 总题数一致
|
||||
# 但 batch 内部排列几乎必然不同
|
||||
flat1 = [q.question_id for b in r1[0] for q in b]
|
||||
flat2 = [q.question_id for b in r2[0] for q in b]
|
||||
assert flat1 != flat2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# test_small_class_not_split
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestSmallClassNotSplit:
|
||||
"""小类（≤ min_class_per_batch）整组不拆，锁在同一 batch。"""
|
||||
|
||||
def test_small_group_stays_together(self) -> None:
|
||||
# 2 道题属于 small_type（≤ min_class=3），应在同一 batch
|
||||
items = [
|
||||
_make_q("s1", task_type="small_type"),
|
||||
_make_q("s2", task_type="small_type"),
|
||||
# 大类 8 道题
|
||||
*[_make_q(f"big{i}", task_type="big_type") for i in range(8)],
|
||||
]
|
||||
correctness = {q.question_id: False for q in items}
|
||||
batches, count = build_batches(
|
||||
items, correctness, batch_size=5, min_class_per_batch=3, seed=0
|
||||
)
|
||||
assert count == 10
|
||||
# 找到包含 small_type 的 batch
|
||||
small_batch = [
|
||||
b for b in batches
|
||||
if any(q.task_type == "small_type" for q in b)
|
||||
]
|
||||
assert len(small_batch) == 1 # 整组在同一个 batch
|
||||
small_ids = {q.question_id for q in small_batch[0] if q.task_type == "small_type"}
|
||||
assert small_ids == {"s1", "s2"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# test_large_class_round_robin
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestLargeClassRoundRobin:
|
||||
"""大类样本 round-robin 散布到多个 batch，不集中于单一 batch。"""
|
||||
|
||||
def test_large_group_distributed(self) -> None:
|
||||
# 12 道大类题，batch_size=4，min_class=2 → 大类 > 2 → round-robin
|
||||
items = [_make_q(f"q{i}", task_type="large_type") for i in range(12)]
|
||||
correctness = {q.question_id: False for q in items}
|
||||
batches, count = build_batches(
|
||||
items, correctness, batch_size=4, min_class_per_batch=2, seed=7
|
||||
)
|
||||
assert count == 12
|
||||
assert len(batches) >= 3 # ceil(12/4) = 3
|
||||
# 每个 batch 不超过 batch_size
|
||||
for b in batches:
|
||||
assert len(b) <= 4
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# test_correct_ratio_mixing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestCorrectRatioMixing:
|
||||
"""correct_ratio > 0 时混入正确题。"""
|
||||
|
||||
def test_mixed_includes_correct(self) -> None:
|
||||
items = [
|
||||
_make_q("e1", task_type="t1"),
|
||||
_make_q("e2", task_type="t1"),
|
||||
_make_q("c1", task_type="t1"),
|
||||
_make_q("c2", task_type="t1"),
|
||||
_make_q("c3", task_type="t1"),
|
||||
]
|
||||
correctness = {"e1": False, "e2": False, "c1": True, "c2": True, "c3": True}
|
||||
batches, count = build_batches(
|
||||
items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
|
||||
correct_ratio=0.5,
|
||||
)
|
||||
# correct_ratio=0.5 → 错:正 = 1:1 → 2 错 + 2 正 = 4 题
|
||||
assert count == 4
|
||||
all_ids = {q.question_id for b in batches for q in b}
|
||||
assert {"e1", "e2"}.issubset(all_ids) # 错题全部
|
||||
correct_in = all_ids - {"e1", "e2"}
|
||||
assert len(correct_in) == 2 # 采样 2 个正确题
|
||||
assert correct_in.issubset({"c1", "c2", "c3"})
|
||||
|
||||
def test_ratio_zero_pure_errors(self) -> None:
|
||||
items = [
|
||||
_make_q("e1", task_type="t1"),
|
||||
_make_q("c1", task_type="t1"),
|
||||
]
|
||||
correctness = {"e1": False, "c1": True}
|
||||
batches, count = build_batches(
|
||||
items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
|
||||
correct_ratio=0.0,
|
||||
)
|
||||
assert count == 1
|
||||
assert batches[0][0].question_id == "e1"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# test_no_wrong_answers_empty
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestNoWrongAnswersEmpty:
|
||||
"""无错题时返回空列表。"""
|
||||
|
||||
def test_all_correct_returns_empty(self) -> None:
|
||||
items = [_make_q(f"q{i}") for i in range(5)]
|
||||
correctness = {f"q{i}": True for i in range(5)}
|
||||
batches, count = build_batches(
|
||||
items, correctness, batch_size=3, min_class_per_batch=1, seed=0,
|
||||
)
|
||||
assert batches == []
|
||||
assert count == 0
|
||||
|
||||
def test_empty_items_returns_empty(self) -> None:
|
||||
batches, count = build_batches(
|
||||
[], {}, batch_size=3, min_class_per_batch=1, seed=0,
|
||||
)
|
||||
assert batches == []
|
||||
assert count == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# test_validate_params_strict
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestValidateParamsStrict:
|
||||
"""参数校验：batch_size < 1、min_class < 1、min_class >= batch_size 都报错。"""
|
||||
|
||||
def test_batch_size_zero(self) -> None:
|
||||
with pytest.raises(ValueError, match="batch_size 必须 >= 1"):
|
||||
_validate_params(0, 1)
|
||||
|
||||
def test_batch_size_negative(self) -> None:
|
||||
with pytest.raises(ValueError, match="batch_size 必须 >= 1"):
|
||||
_validate_params(-1, 1)
|
||||
|
||||
def test_min_class_zero(self) -> None:
|
||||
with pytest.raises(ValueError, match="min_class_per_batch 必须 >= 1"):
|
||||
_validate_params(5, 0)
|
||||
|
||||
def test_min_class_equals_batch_size(self) -> None:
|
||||
with pytest.raises(ValueError, match="min_class_per_batch 必须严格 < batch_size"):
|
||||
_validate_params(5, 5)
|
||||
|
||||
def test_min_class_exceeds_batch_size(self) -> None:
|
||||
with pytest.raises(ValueError, match="min_class_per_batch 必须严格 < batch_size"):
|
||||
_validate_params(3, 5)
|
||||
|
||||
def test_valid_params_no_error(self) -> None:
|
||||
_validate_params(5, 3) # 不抛异常
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# test_correctness_false_vs_none
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestCorrectnessFalseVsNone:
|
||||
"""correctness.get(qid) is False 精确匹配：None（未知题）不算错题。"""
|
||||
|
||||
def test_none_excluded_from_errors(self) -> None:
|
||||
items = [
|
||||
_make_q("wrong", task_type="t1"),
|
||||
_make_q("right", task_type="t1"),
|
||||
_make_q("unknown", task_type="t1"),
|
||||
]
|
||||
# wrong=False（错题），right=True（正确题），unknown 不在 correctness（None）
|
||||
correctness: dict[str, bool] = {"wrong": False, "right": True}
|
||||
batches, count = build_batches(
|
||||
items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
|
||||
correct_ratio=0.0,
|
||||
)
|
||||
# 仅 wrong 进入 batch，unknown 不算错题
|
||||
assert count == 1
|
||||
assert batches[0][0].question_id == "wrong"
|
||||
|
||||
def test_explicit_false_only(self) -> None:
|
||||
"""直接测试 _select_mixed_by_task_type 内部逻辑。"""
|
||||
items = [
|
||||
_make_q("f1", task_type="t1"),
|
||||
_make_q("n1", task_type="t1"), # None（未知）
|
||||
_make_q("t1", task_type="t1"), # True（正确）
|
||||
]
|
||||
correctness: dict[str, bool] = {"f1": False, "t1": True}
|
||||
rng = random.Random(0)
|
||||
result = _select_mixed_by_task_type(items, correctness, 0.0, rng)
|
||||
assert "t1" in result
|
||||
assert len(result["t1"]) == 1
|
||||
assert result["t1"][0].question_id == "f1"
|
||||
|
||||
def test_none_not_treated_as_correct(self) -> None:
|
||||
"""None（未知）不进正确组，不被 correct_ratio 采样。"""
|
||||
items = [
|
||||
_make_q("err", task_type="t1"),
|
||||
_make_q("unk", task_type="t1"),
|
||||
]
|
||||
correctness: dict[str, bool] = {"err": False}
|
||||
rng = random.Random(0)
|
||||
result = _select_mixed_by_task_type(items, correctness, 0.5, rng)
|
||||
# 只有 err 一题错题，unk 不在 correctness 中 → get 返回 None → 不进 correct 组
|
||||
assert len(result["t1"]) == 1 # 只有错题，无正确题可混入
|
||||
@@ -0,0 +1,413 @@
|
||||
"""app/harness/checkpoint.py 单元测试。
|
||||
|
||||
覆盖序列化/反序列化往返、嵌套 dataclass 复活、缺键硬失败、
|
||||
配置指纹结构性 vs 决策性判定、原子写与 load 缺失场景。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from app.harness.checkpoint import (
|
||||
CHECKPOINT_SCHEMA_VERSION,
|
||||
Probation,
|
||||
check_fingerprint,
|
||||
compute_fingerprint,
|
||||
deserialize_state_fields,
|
||||
load_checkpoint,
|
||||
serialize_state,
|
||||
write_checkpoint,
|
||||
)
|
||||
from core.evolution.types import (
|
||||
CaseSample,
|
||||
RejectedEdit,
|
||||
SystemCasePack,
|
||||
ToolCasePack,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# fixtures: 模拟 _TrainState 与 RunConfig
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_case_sample(**overrides: Any) -> CaseSample:
|
||||
"""构造一个最小可用 CaseSample。"""
|
||||
defaults: dict[str, Any] = {
|
||||
"question_id": "q001",
|
||||
"video_id": "v001",
|
||||
"task_type": "temporal",
|
||||
"question": "What happened?",
|
||||
"options": ["A", "B", "C"],
|
||||
"answer": "A",
|
||||
"prediction": "B",
|
||||
"correct": False,
|
||||
"error_type": "reasoning",
|
||||
"selection_reason": "worst",
|
||||
"metrics": {"acc": 0.5},
|
||||
"trace": [{"step": 1, "action": "search"}],
|
||||
}
|
||||
defaults.update(overrides)
|
||||
return CaseSample(**defaults)
|
||||
|
||||
|
||||
def _make_rejected_edit(**overrides: Any) -> RejectedEdit:
|
||||
"""构造一个最小可用 RejectedEdit。"""
|
||||
defaults: dict[str, Any] = {
|
||||
"target_file": "temporal-reasoning.md",
|
||||
"target_type": "skill",
|
||||
"change_summary": "added step",
|
||||
"delta": -0.05,
|
||||
"source_version": "v2",
|
||||
"epoch": 1,
|
||||
"gate_w": 3,
|
||||
"gate_l": 5,
|
||||
"gate_e_value": 0.8,
|
||||
"gate_delta_shrunk": -0.02,
|
||||
}
|
||||
defaults.update(overrides)
|
||||
return RejectedEdit(**defaults)
|
||||
|
||||
|
||||
def _make_system_pack() -> SystemCasePack:
|
||||
"""构造包含嵌套 CaseSample 的 SystemCasePack。"""
|
||||
return SystemCasePack(
|
||||
stats={"pattern": "repeat_visit", "count": 3},
|
||||
failure_cases=[_make_case_sample(question_id="q010")],
|
||||
success_cases=[_make_case_sample(question_id="q011", correct=True, error_type=None)],
|
||||
)
|
||||
|
||||
|
||||
def _make_tool_pack() -> ToolCasePack:
|
||||
"""构造 ToolCasePack。"""
|
||||
return ToolCasePack(
|
||||
tool_name="search_subtree",
|
||||
target_files=["search_subtree_extract.md"],
|
||||
stats={"completeness": 0.8},
|
||||
failure_spans=[{"step": 2, "issue": "missing"}],
|
||||
success_spans=[{"step": 3, "quality": "good"}],
|
||||
)
|
||||
|
||||
|
||||
def _make_probation() -> Probation:
|
||||
"""构造包含嵌套 RejectedEdit 的 Probation。"""
|
||||
return Probation(
|
||||
task_type="temporal",
|
||||
anchor_skills_version="v1",
|
||||
target_file="temporal-reasoning.md",
|
||||
correctness_snapshot={"q001": True, "q002": False},
|
||||
opened_step=5,
|
||||
pending_edits=[_make_rejected_edit()],
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeState:
|
||||
"""模拟 _TrainState 全部可持久化字段。"""
|
||||
|
||||
correctness: dict[str, bool]
|
||||
eval_prev_acc: float
|
||||
eval_prev_run_id: str
|
||||
baseline_skills_version: str
|
||||
baseline_prompts_version: str
|
||||
steps_since_best_improved: int
|
||||
epoch_start_skills: str
|
||||
changed_task_types_this_epoch: set[str]
|
||||
rejected_buffer: dict[str, list[RejectedEdit]]
|
||||
system_packs: list[SystemCasePack]
|
||||
tool_packs: list[ToolCasePack]
|
||||
probations: dict[str, Probation]
|
||||
gate_cooldown: dict[str, int]
|
||||
gate_epoch_observed: dict[str, bool]
|
||||
|
||||
|
||||
def _make_state() -> _FakeState:
|
||||
"""构造一个填满全部字段的 _FakeState。"""
|
||||
return _FakeState(
|
||||
correctness={"q001": True, "q002": False},
|
||||
eval_prev_acc=0.65,
|
||||
eval_prev_run_id="run-abc",
|
||||
baseline_skills_version="v1",
|
||||
baseline_prompts_version="v1",
|
||||
steps_since_best_improved=2,
|
||||
epoch_start_skills="v1",
|
||||
changed_task_types_this_epoch={"temporal", "causal"},
|
||||
rejected_buffer={"temporal": [_make_rejected_edit()]},
|
||||
system_packs=[_make_system_pack()],
|
||||
tool_packs=[_make_tool_pack()],
|
||||
probations={"temporal": _make_probation()},
|
||||
gate_cooldown={"temporal": 3},
|
||||
gate_epoch_observed={"temporal": True},
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _FakeConfig:
|
||||
"""模拟 RunConfig 的指纹相关字段。"""
|
||||
|
||||
batch_size: int = 8
|
||||
min_class_per_batch: int = 2
|
||||
epochs: int = 5
|
||||
diag_size: int = 30
|
||||
val_size: int = 50
|
||||
batch_correct_ratio: float = 0.5
|
||||
edit_budget_start: int = 6
|
||||
edit_budget_end: int = 3
|
||||
early_stop_patience: int = 3
|
||||
use_slow_momentum: bool = True
|
||||
skill_update_mode: str = "patch"
|
||||
appendix_consolidate_threshold: int = 10
|
||||
momentum_samples: int = 20
|
||||
gate_e_confirm: float = 20.0
|
||||
gate_e_provisional: float = 6.0
|
||||
gate_w_net_min: int = 2
|
||||
gate_delta_min: float = 0.02
|
||||
gate_lambda_dir: float = -3.0
|
||||
gate_e_rollback: float = 10.0
|
||||
gate_block: int = 4
|
||||
gate_n_max: int = 40
|
||||
gate_p_low: float = 0.1
|
||||
gate_p_high: float = 0.9
|
||||
gate_probe_quota: float = 0.2
|
||||
gate_gamma_decay: float = 0.9
|
||||
gate_cooldown_steps: int = 2
|
||||
gate_guard_err: float = 0.3
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 测试用例
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestSerializeDeserializeRoundtrip:
|
||||
"""序列化 → JSON 往返 → 反序列化应完全复原。"""
|
||||
|
||||
def test_serialize_deserialize_roundtrip(self) -> None:
|
||||
state = _make_state()
|
||||
serialized = serialize_state(state)
|
||||
# JSON 往返（模拟实际落盘-读回）
|
||||
json_str = json.dumps(serialized, ensure_ascii=False)
|
||||
loaded = json.loads(json_str)
|
||||
restored = deserialize_state_fields(loaded)
|
||||
|
||||
assert restored["correctness"] == state.correctness
|
||||
assert restored["eval_prev_acc"] == state.eval_prev_acc
|
||||
assert restored["eval_prev_run_id"] == state.eval_prev_run_id
|
||||
assert restored["baseline_skills_version"] == state.baseline_skills_version
|
||||
assert restored["baseline_prompts_version"] == state.baseline_prompts_version
|
||||
assert restored["steps_since_best_improved"] == state.steps_since_best_improved
|
||||
assert restored["epoch_start_skills"] == state.epoch_start_skills
|
||||
assert restored["changed_task_types_this_epoch"] == state.changed_task_types_this_epoch
|
||||
assert restored["gate_cooldown"] == state.gate_cooldown
|
||||
assert restored["gate_epoch_observed"] == state.gate_epoch_observed
|
||||
|
||||
|
||||
class TestSerializeSetToSortedList:
|
||||
"""set 字段序列化为排序列表。"""
|
||||
|
||||
def test_serialize_set_to_sorted_list(self) -> None:
|
||||
state = _make_state()
|
||||
state.changed_task_types_this_epoch = {"z_type", "a_type", "m_type"}
|
||||
serialized = serialize_state(state)
|
||||
assert serialized["changed_task_types_this_epoch"] == ["a_type", "m_type", "z_type"]
|
||||
|
||||
|
||||
class TestDeserializeNestedSystemPack:
|
||||
"""SystemCasePack 内嵌套的 CaseSample 正确复活。"""
|
||||
|
||||
def test_deserialize_nested_system_pack(self) -> None:
|
||||
state = _make_state()
|
||||
serialized = serialize_state(state)
|
||||
json_str = json.dumps(serialized, ensure_ascii=False)
|
||||
loaded = json.loads(json_str)
|
||||
restored = deserialize_state_fields(loaded)
|
||||
|
||||
packs = restored["system_packs"]
|
||||
assert len(packs) == 1
|
||||
pack = packs[0]
|
||||
assert isinstance(pack, SystemCasePack)
|
||||
assert len(pack.failure_cases) == 1
|
||||
assert isinstance(pack.failure_cases[0], CaseSample)
|
||||
assert pack.failure_cases[0].question_id == "q010"
|
||||
assert len(pack.success_cases) == 1
|
||||
assert isinstance(pack.success_cases[0], CaseSample)
|
||||
assert pack.success_cases[0].question_id == "q011"
|
||||
|
||||
|
||||
class TestDeserializeNestedProbation:
|
||||
"""Probation 内嵌套的 RejectedEdit 正确复活。"""
|
||||
|
||||
def test_deserialize_nested_probation(self) -> None:
|
||||
state = _make_state()
|
||||
serialized = serialize_state(state)
|
||||
json_str = json.dumps(serialized, ensure_ascii=False)
|
||||
loaded = json.loads(json_str)
|
||||
restored = deserialize_state_fields(loaded)
|
||||
|
||||
probations = restored["probations"]
|
||||
assert "temporal" in probations
|
||||
prob = probations["temporal"]
|
||||
assert isinstance(prob, Probation)
|
||||
assert prob.task_type == "temporal"
|
||||
assert prob.anchor_skills_version == "v1"
|
||||
assert prob.correctness_snapshot == {"q001": True, "q002": False}
|
||||
assert len(prob.pending_edits) == 1
|
||||
edit = prob.pending_edits[0]
|
||||
assert isinstance(edit, RejectedEdit)
|
||||
assert edit.target_file == "temporal-reasoning.md"
|
||||
assert edit.delta == -0.05
|
||||
|
||||
|
||||
class TestDeserializeMissingKeyRaises:
|
||||
"""缺键即 checkpoint 损坏，应硬失败。"""
|
||||
|
||||
def test_deserialize_missing_key_raises(self) -> None:
|
||||
state = _make_state()
|
||||
serialized = serialize_state(state)
|
||||
del serialized["gate_epoch_observed"]
|
||||
with pytest.raises(KeyError):
|
||||
deserialize_state_fields(serialized)
|
||||
|
||||
|
||||
class TestFingerprintStructuralVsDecision:
|
||||
"""compute_fingerprint 包含全部结构性 + 决策性键。"""
|
||||
|
||||
def test_fingerprint_structural_vs_decision(self) -> None:
|
||||
config = _FakeConfig()
|
||||
fp = compute_fingerprint(config)
|
||||
|
||||
structural = {
|
||||
"batch_size",
|
||||
"min_class_per_batch",
|
||||
"epochs",
|
||||
"diag_size",
|
||||
"val_size",
|
||||
"batch_correct_ratio",
|
||||
}
|
||||
decision = {
|
||||
"edit_budget_start",
|
||||
"edit_budget_end",
|
||||
"early_stop_patience",
|
||||
"use_slow_momentum",
|
||||
"skill_update_mode",
|
||||
"appendix_consolidate_threshold",
|
||||
"momentum_samples",
|
||||
"gate_e_confirm",
|
||||
"gate_e_provisional",
|
||||
"gate_w_net_min",
|
||||
"gate_delta_min",
|
||||
"gate_lambda_dir",
|
||||
"gate_e_rollback",
|
||||
"gate_block",
|
||||
"gate_n_max",
|
||||
"gate_p_low",
|
||||
"gate_p_high",
|
||||
"gate_probe_quota",
|
||||
"gate_gamma_decay",
|
||||
"gate_cooldown_steps",
|
||||
"gate_guard_err",
|
||||
}
|
||||
assert structural | decision == set(fp.keys())
|
||||
assert fp["batch_size"] == 8
|
||||
assert fp["gate_e_confirm"] == 20.0
|
||||
|
||||
|
||||
class TestCheckFingerprintStructuralReject:
|
||||
"""结构性键变化应出现在 structural 列表中。"""
|
||||
|
||||
def test_check_fingerprint_structural_reject(self) -> None:
|
||||
config_old = _FakeConfig()
|
||||
saved = compute_fingerprint(config_old)
|
||||
# 修改结构性参数
|
||||
config_new = _FakeConfig(batch_size=16, epochs=10)
|
||||
structural, decision = check_fingerprint(saved, config_new)
|
||||
assert "batch_size" in structural
|
||||
assert "epochs" in structural
|
||||
assert len(decision) == 0
|
||||
|
||||
|
||||
class TestCheckFingerprintDecisionWarn:
|
||||
"""决策性键变化应出现在 decision 列表中，structural 为空。"""
|
||||
|
||||
def test_check_fingerprint_decision_warn(self) -> None:
|
||||
config_old = _FakeConfig()
|
||||
saved = compute_fingerprint(config_old)
|
||||
config_new = _FakeConfig(early_stop_patience=10, gate_e_confirm=50.0)
|
||||
structural, decision = check_fingerprint(saved, config_new)
|
||||
assert len(structural) == 0
|
||||
assert "early_stop_patience" in decision
|
||||
assert "gate_e_confirm" in decision
|
||||
|
||||
|
||||
class TestWriteCheckpointAtomic:
|
||||
"""原子写：先 .tmp 再 os.replace。"""
|
||||
|
||||
def test_write_checkpoint_atomic(self, tmp_path: Path) -> None:
|
||||
state = _make_state()
|
||||
config = _FakeConfig()
|
||||
write_checkpoint(
|
||||
tmp_path,
|
||||
state=state,
|
||||
epoch=2,
|
||||
step_completed=5,
|
||||
phase="in_epoch",
|
||||
global_step=15,
|
||||
total_steps=40,
|
||||
version_snapshot={"skills": "v3", "prompts": "v2"},
|
||||
epoch_batches=[["q001", "q002"], ["q003"]],
|
||||
config=config,
|
||||
)
|
||||
ckpt_path = tmp_path / "checkpoint.json"
|
||||
assert ckpt_path.exists()
|
||||
# .tmp 应已被 os.replace 移除
|
||||
assert not (tmp_path / "checkpoint.json.tmp").exists()
|
||||
|
||||
payload = json.loads(ckpt_path.read_text())
|
||||
assert payload["schema_version"] == CHECKPOINT_SCHEMA_VERSION
|
||||
assert payload["progress"]["epoch"] == 2
|
||||
assert payload["progress"]["step_completed"] == 5
|
||||
assert payload["progress"]["phase"] == "in_epoch"
|
||||
assert payload["progress"]["global_step"] == 15
|
||||
assert payload["progress"]["total_steps"] == 40
|
||||
assert payload["version_snapshot"] == {"skills": "v3", "prompts": "v2"}
|
||||
assert payload["epoch_batches"] == [["q001", "q002"], ["q003"]]
|
||||
assert "config_fingerprint" in payload
|
||||
assert "state" in payload
|
||||
|
||||
|
||||
class TestLoadCheckpointMissing:
|
||||
"""checkpoint.json 不存在时返回 None。"""
|
||||
|
||||
def test_load_checkpoint_missing(self, tmp_path: Path) -> None:
|
||||
result = load_checkpoint(tmp_path)
|
||||
assert result is None
|
||||
|
||||
def test_load_checkpoint_exists(self, tmp_path: Path) -> None:
|
||||
"""checkpoint.json 存在时正确读回。"""
|
||||
state = _make_state()
|
||||
config = _FakeConfig()
|
||||
write_checkpoint(
|
||||
tmp_path,
|
||||
state=state,
|
||||
epoch=1,
|
||||
step_completed=3,
|
||||
phase="post_evolve",
|
||||
global_step=8,
|
||||
total_steps=20,
|
||||
version_snapshot={"skills": "v2", "prompts": "v1"},
|
||||
epoch_batches=[["q001"]],
|
||||
config=config,
|
||||
)
|
||||
loaded = load_checkpoint(tmp_path)
|
||||
assert loaded is not None
|
||||
assert loaded["schema_version"] == CHECKPOINT_SCHEMA_VERSION
|
||||
assert loaded["progress"]["epoch"] == 1
|
||||
# 完整往返测试：state 可 deserialize
|
||||
restored = deserialize_state_fields(loaded["state"])
|
||||
assert restored["eval_prev_acc"] == 0.65
|
||||
@@ -0,0 +1,595 @@
|
||||
"""app/harness/config.py 单元测试。
|
||||
|
||||
覆盖 RunConfig 构造、四层校验（_validate → 三个子校验）、
|
||||
YAML + CLI + .env 三层加载优先级。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from app.harness.config import RunConfig, _validate, load_config
|
||||
|
||||
# ────────────────────────────── 测试数据工厂 ──────────────────────────────
|
||||
|
||||
|
||||
def _valid_kwargs() -> dict:
|
||||
"""构造一组完整合法的 RunConfig 字段值（使用真实 default.yaml 数据）。"""
|
||||
return {
|
||||
"workspace_dir": Path("workspaces/default"),
|
||||
"store_dir": Path("store"),
|
||||
"mode": "infer",
|
||||
"run_id": "",
|
||||
"concurrency": 12,
|
||||
"max_steps": 15,
|
||||
"skill_mode": "auto",
|
||||
"n_samples": 0,
|
||||
"questions": "benchmarks/Video-MME",
|
||||
"skills_version": "v1",
|
||||
"prompts_version": "v1",
|
||||
"epochs": 1,
|
||||
"diag_size": 200,
|
||||
"diag_correct_ratio": 0.5,
|
||||
"val_size": 30,
|
||||
"val_correct_ratio": 0.5,
|
||||
"edit_budget_start": 5,
|
||||
"edit_budget_end": 2,
|
||||
"batch_size": 15,
|
||||
"min_class_per_batch": 2,
|
||||
"eval_min_per_class": 2,
|
||||
"early_stop_patience": 8,
|
||||
"test_size": 60,
|
||||
"use_slow_momentum": True,
|
||||
"gate_e_confirm": 20.0,
|
||||
"gate_e_provisional": 3.0,
|
||||
"gate_w_net_min": 2,
|
||||
"gate_delta_min": 0.02,
|
||||
"gate_lambda_dir": -0.642,
|
||||
"gate_e_rollback": 10.0,
|
||||
"gate_block": 8,
|
||||
"gate_n_max": 40,
|
||||
"gate_p_low": 0.05,
|
||||
"gate_p_high": 0.95,
|
||||
"gate_probe_quota": 0.2,
|
||||
"gate_gamma_decay": 0.9,
|
||||
"gate_cooldown_steps": 2,
|
||||
"gate_guard_err": 0.10,
|
||||
"skill_update_mode": "patch",
|
||||
"appendix_consolidate_threshold": 6,
|
||||
"batch_correct_ratio": 0.5,
|
||||
"momentum_samples": 20,
|
||||
}
|
||||
|
||||
|
||||
def _make_config(**overrides: object) -> RunConfig:
|
||||
"""用 _valid_kwargs 构造 RunConfig，支持字段覆盖。"""
|
||||
kwargs = _valid_kwargs()
|
||||
kwargs.update(overrides)
|
||||
return RunConfig(**kwargs)
|
||||
|
||||
|
||||
def _yaml_harness_dict() -> dict:
|
||||
"""构造可序列化为 YAML 的合法 harness 配置字典（路径用字符串）。"""
|
||||
d = _valid_kwargs()
|
||||
d["workspace_dir"] = str(d["workspace_dir"])
|
||||
d["store_dir"] = str(d["store_dir"])
|
||||
return d
|
||||
|
||||
|
||||
def _write_yaml(tmp_path: Path, harness_data: dict) -> Path:
|
||||
"""将 harness 配置写入临时 YAML 文件，返回文件路径。"""
|
||||
yaml_path = tmp_path / "experiment.yaml"
|
||||
with open(yaml_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump({"harness": harness_data}, f)
|
||||
return yaml_path
|
||||
|
||||
|
||||
# ──────────────────────────── test_valid_config ────────────────────────────
|
||||
|
||||
|
||||
class TestValidConfig:
|
||||
"""合法参数应能正常构造并通过校验。"""
|
||||
|
||||
def test_default_yaml_values_pass_validation(self) -> None:
|
||||
"""使用 default.yaml 真实默认值构造的 RunConfig 应通过全部校验。"""
|
||||
cfg = _make_config()
|
||||
_validate(cfg)
|
||||
assert cfg.mode == "infer"
|
||||
assert cfg.workspace_dir == Path("workspaces/default")
|
||||
assert cfg.store_dir == Path("store")
|
||||
|
||||
def test_frozen_immutability(self) -> None:
|
||||
"""frozen=True 应禁止字段赋值。"""
|
||||
cfg = _make_config()
|
||||
with pytest.raises(AttributeError):
|
||||
cfg.mode = "train" # type: ignore[misc]
|
||||
|
||||
def test_default_field_values(self) -> None:
|
||||
"""有默认值的字段不传时应使用默认值。"""
|
||||
kwargs = _valid_kwargs()
|
||||
# 不传 run_id / seed / version / resume / fresh，使用默认值
|
||||
kwargs.pop("run_id", None)
|
||||
cfg = RunConfig(**kwargs)
|
||||
assert cfg.run_id == ""
|
||||
assert cfg.seed == "initial"
|
||||
assert cfg.version == ""
|
||||
assert cfg.resume is False
|
||||
assert cfg.fresh is False
|
||||
|
||||
|
||||
# ──────────────────────────── test_mode_validation ────────────────────────
|
||||
|
||||
|
||||
class TestModeValidation:
|
||||
"""mode 字段校验。"""
|
||||
|
||||
@pytest.mark.parametrize("bad_mode", ["unknown", "test", "", "INFER", "Train"])
|
||||
def test_invalid_mode_rejected(self, bad_mode: str) -> None:
|
||||
"""非法 mode 应抛出 ValueError。"""
|
||||
cfg = _make_config(mode=bad_mode)
|
||||
with pytest.raises(ValueError, match="mode"):
|
||||
_validate(cfg)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"valid_mode", ["infer", "train", "diagnose", "evolve", "eval", "promote"]
|
||||
)
|
||||
def test_all_valid_modes_accepted(self, valid_mode: str) -> None:
|
||||
"""全部合法 mode 应通过校验（diagnose/evolve 需 run_id）。"""
|
||||
overrides: dict = {"mode": valid_mode}
|
||||
if valid_mode in ("diagnose", "evolve", "train"):
|
||||
overrides["run_id"] = "run-001"
|
||||
if valid_mode in ("eval", "promote"):
|
||||
overrides["version"] = "v1"
|
||||
if valid_mode == "promote":
|
||||
overrides["run_id"] = "run-001"
|
||||
cfg = _make_config(**overrides)
|
||||
_validate(cfg)
|
||||
|
||||
def test_diagnose_requires_run_id(self) -> None:
|
||||
"""diagnose 模式缺少 run_id 应报错。"""
|
||||
cfg = _make_config(mode="diagnose", run_id="")
|
||||
with pytest.raises(ValueError, match="run_id"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_evolve_requires_run_id(self) -> None:
|
||||
"""evolve 模式缺少 run_id 应报错。"""
|
||||
cfg = _make_config(mode="evolve", run_id="")
|
||||
with pytest.raises(ValueError, match="run_id"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_eval_requires_version(self) -> None:
|
||||
"""eval 模式缺少 version 应报错。"""
|
||||
cfg = _make_config(mode="eval", version="")
|
||||
with pytest.raises(ValueError, match="version"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_promote_requires_run_id_and_version(self) -> None:
|
||||
"""promote 模式需同时提供 run_id 和 version。"""
|
||||
cfg = _make_config(mode="promote", run_id="", version="v1")
|
||||
with pytest.raises(ValueError, match="promote.*run-id"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_train_mode_requires_run_id_without_resume_fresh(self) -> None:
|
||||
"""train 模式非 resume/fresh 时必须提供 run_id。"""
|
||||
cfg = _make_config(mode="train", run_id="", resume=False, fresh=False)
|
||||
with pytest.raises(ValueError, match="run_id"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_train_mode_resume_without_run_id_accepted(self) -> None:
|
||||
"""train 模式 resume=True 时不需要 run_id。"""
|
||||
cfg = _make_config(mode="train", run_id="", resume=True)
|
||||
_validate(cfg)
|
||||
|
||||
def test_train_mode_fresh_without_run_id_accepted(self) -> None:
|
||||
"""train 模式 fresh=True 时不需要 run_id。"""
|
||||
cfg = _make_config(mode="train", run_id="", fresh=True)
|
||||
_validate(cfg)
|
||||
|
||||
def test_train_mode_with_run_id_accepted(self) -> None:
|
||||
"""train 模式提供 run_id 时应通过（旧式基线 run）。"""
|
||||
cfg = _make_config(mode="train", run_id="baseline-001")
|
||||
_validate(cfg)
|
||||
|
||||
def test_concurrency_positive(self) -> None:
|
||||
"""concurrency <= 0 应报错。"""
|
||||
cfg = _make_config(concurrency=0)
|
||||
with pytest.raises(ValueError, match="concurrency"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_max_steps_positive(self) -> None:
|
||||
"""max_steps <= 0 应报错。"""
|
||||
cfg = _make_config(max_steps=-1)
|
||||
with pytest.raises(ValueError, match="max_steps"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_n_samples_non_negative(self) -> None:
|
||||
"""n_samples < 0 应报错。"""
|
||||
cfg = _make_config(n_samples=-1)
|
||||
with pytest.raises(ValueError, match="n_samples"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_epochs_positive(self) -> None:
|
||||
"""epochs <= 0 应报错。"""
|
||||
cfg = _make_config(epochs=0)
|
||||
with pytest.raises(ValueError, match="epochs"):
|
||||
_validate(cfg)
|
||||
|
||||
@pytest.mark.parametrize("bad_skill_mode", ["Auto", "disabled", ""])
|
||||
def test_invalid_skill_mode(self, bad_skill_mode: str) -> None:
|
||||
"""非法 skill_mode 应报错。"""
|
||||
cfg = _make_config(skill_mode=bad_skill_mode)
|
||||
with pytest.raises(ValueError, match="skill_mode"):
|
||||
_validate(cfg)
|
||||
|
||||
@pytest.mark.parametrize("bad_update_mode", ["append", "delete", ""])
|
||||
def test_invalid_skill_update_mode(self, bad_update_mode: str) -> None:
|
||||
"""非法 skill_update_mode 应报错。"""
|
||||
cfg = _make_config(skill_update_mode=bad_update_mode)
|
||||
with pytest.raises(ValueError, match="skill_update_mode"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_appendix_consolidate_threshold_positive(self) -> None:
|
||||
"""appendix_consolidate_threshold < 1 应报错。"""
|
||||
cfg = _make_config(appendix_consolidate_threshold=0)
|
||||
with pytest.raises(ValueError, match="appendix_consolidate_threshold"):
|
||||
_validate(cfg)
|
||||
|
||||
|
||||
# ────────────────────── test_edit_budget_validation ───────────────────────
|
||||
|
||||
|
||||
class TestEditBudgetValidation:
|
||||
"""编辑预算退火校验（_validate_edit_budget）。"""
|
||||
|
||||
def test_start_less_than_end_rejected(self) -> None:
|
||||
"""edit_budget_start < edit_budget_end 应抛出 ValueError。"""
|
||||
cfg = _make_config(edit_budget_start=1, edit_budget_end=5)
|
||||
with pytest.raises(ValueError, match="edit_budget_start"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_end_zero_rejected(self) -> None:
|
||||
"""edit_budget_end <= 0 应抛出 ValueError。"""
|
||||
cfg = _make_config(edit_budget_start=1, edit_budget_end=0)
|
||||
with pytest.raises(ValueError, match="edit_budget_end"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_equal_values_accepted(self) -> None:
|
||||
"""edit_budget_start == edit_budget_end 应通过。"""
|
||||
cfg = _make_config(edit_budget_start=3, edit_budget_end=3)
|
||||
_validate(cfg)
|
||||
|
||||
|
||||
# ─────────────────────── test_minibatch_validation ───────────────────────
|
||||
|
||||
|
||||
class TestMinibatchValidation:
|
||||
"""mini-batch 自进化闭环参数校验（_validate_minibatch）。"""
|
||||
|
||||
def test_batch_size_zero_rejected(self) -> None:
|
||||
"""batch_size <= 0 应抛出 ValueError。"""
|
||||
cfg = _make_config(batch_size=0)
|
||||
with pytest.raises(ValueError, match="batch_size"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_min_class_per_batch_equals_batch_size_rejected(self) -> None:
|
||||
"""min_class_per_batch >= batch_size 应抛出 ValueError。"""
|
||||
cfg = _make_config(batch_size=5, min_class_per_batch=5)
|
||||
with pytest.raises(ValueError, match="min_class_per_batch"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_min_class_per_batch_zero_rejected(self) -> None:
|
||||
"""min_class_per_batch < 1 应抛出 ValueError。"""
|
||||
cfg = _make_config(min_class_per_batch=0)
|
||||
with pytest.raises(ValueError, match="min_class_per_batch"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_eval_min_per_class_zero_rejected(self) -> None:
|
||||
"""eval_min_per_class < 1 应抛出 ValueError。"""
|
||||
cfg = _make_config(eval_min_per_class=0)
|
||||
with pytest.raises(ValueError, match="eval_min_per_class"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_early_stop_patience_zero_rejected(self) -> None:
|
||||
"""early_stop_patience <= 0 应抛出 ValueError。"""
|
||||
cfg = _make_config(early_stop_patience=0)
|
||||
with pytest.raises(ValueError, match="early_stop_patience"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_test_size_zero_rejected(self) -> None:
|
||||
"""test_size <= 0 应抛出 ValueError。"""
|
||||
cfg = _make_config(test_size=0)
|
||||
with pytest.raises(ValueError, match="test_size"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_batch_correct_ratio_one_rejected(self) -> None:
|
||||
"""batch_correct_ratio >= 1 应抛出 ValueError。"""
|
||||
cfg = _make_config(batch_correct_ratio=1.0)
|
||||
with pytest.raises(ValueError, match="batch_correct_ratio"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_batch_correct_ratio_negative_rejected(self) -> None:
|
||||
"""batch_correct_ratio < 0 应抛出 ValueError。"""
|
||||
cfg = _make_config(batch_correct_ratio=-0.1)
|
||||
with pytest.raises(ValueError, match="batch_correct_ratio"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_momentum_samples_zero_rejected(self) -> None:
|
||||
"""momentum_samples < 1 应抛出 ValueError。"""
|
||||
cfg = _make_config(momentum_samples=0)
|
||||
with pytest.raises(ValueError, match="momentum_samples"):
|
||||
_validate(cfg)
|
||||
|
||||
|
||||
# ─────────────────────────── test_gate_validation ────────────────────────
|
||||
|
||||
|
||||
class TestGateValidation:
|
||||
"""CE-Gate 参数校验（_validate_gate）。"""
|
||||
|
||||
def test_e_confirm_at_one_rejected(self) -> None:
|
||||
"""gate_e_confirm <= 1 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_e_confirm=1.0, gate_e_provisional=1.0)
|
||||
with pytest.raises(ValueError, match="gate_e_confirm"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_e_provisional_exceeds_confirm_rejected(self) -> None:
|
||||
"""gate_e_provisional > gate_e_confirm 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_e_confirm=10.0, gate_e_provisional=15.0)
|
||||
with pytest.raises(ValueError, match="gate_e_provisional"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_e_provisional_at_one_rejected(self) -> None:
|
||||
"""gate_e_provisional <= 1 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_e_provisional=0.5)
|
||||
with pytest.raises(ValueError, match="gate_e_provisional"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_e_rollback_at_one_rejected(self) -> None:
|
||||
"""gate_e_rollback <= 1 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_e_rollback=1.0)
|
||||
with pytest.raises(ValueError, match="gate_e_rollback"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_w_net_min_zero_rejected(self) -> None:
|
||||
"""gate_w_net_min < 1 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_w_net_min=0)
|
||||
with pytest.raises(ValueError, match="gate_w_net_min"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_delta_min_negative_rejected(self) -> None:
|
||||
"""gate_delta_min < 0 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_delta_min=-0.1)
|
||||
with pytest.raises(ValueError, match="gate_delta_min"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_lambda_dir_positive_rejected(self) -> None:
|
||||
"""gate_lambda_dir >= 0 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_lambda_dir=0.5)
|
||||
with pytest.raises(ValueError, match="gate_lambda_dir"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_lambda_dir_zero_rejected(self) -> None:
|
||||
"""gate_lambda_dir == 0 也应报错。"""
|
||||
cfg = _make_config(gate_lambda_dir=0.0)
|
||||
with pytest.raises(ValueError, match="gate_lambda_dir"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_block_exceeds_n_max_rejected(self) -> None:
|
||||
"""gate_block > gate_n_max 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_block=50, gate_n_max=40)
|
||||
with pytest.raises(ValueError, match="gate_block"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_block_zero_rejected(self) -> None:
|
||||
"""gate_block <= 0 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_block=0)
|
||||
with pytest.raises(ValueError, match="gate_block"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_p_low_exceeds_p_high_rejected(self) -> None:
|
||||
"""gate_p_low >= gate_p_high 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_p_low=0.9, gate_p_high=0.1)
|
||||
with pytest.raises(ValueError, match="gate_p_low"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_probe_quota_negative_rejected(self) -> None:
|
||||
"""gate_probe_quota < 0 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_probe_quota=-0.1)
|
||||
with pytest.raises(ValueError, match="gate_probe_quota"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_gamma_decay_zero_rejected(self) -> None:
|
||||
"""gate_gamma_decay <= 0 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_gamma_decay=0.0)
|
||||
with pytest.raises(ValueError, match="gate_gamma_decay"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_gamma_decay_one_rejected(self) -> None:
|
||||
"""gate_gamma_decay >= 1 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_gamma_decay=1.0)
|
||||
with pytest.raises(ValueError, match="gate_gamma_decay"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_cooldown_steps_zero_rejected(self) -> None:
|
||||
"""gate_cooldown_steps < 1 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_cooldown_steps=0)
|
||||
with pytest.raises(ValueError, match="gate_cooldown_steps"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_guard_err_zero_rejected(self) -> None:
|
||||
"""gate_guard_err <= 0 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_guard_err=0.0)
|
||||
with pytest.raises(ValueError, match="gate_guard_err"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_guard_err_one_rejected(self) -> None:
|
||||
"""gate_guard_err >= 1 应抛出 ValueError。"""
|
||||
cfg = _make_config(gate_guard_err=1.0)
|
||||
with pytest.raises(ValueError, match="gate_guard_err"):
|
||||
_validate(cfg)
|
||||
|
||||
|
||||
# ─────────────────────── test_load_config_cli_overrides ──────────────────
|
||||
|
||||
|
||||
class TestLoadConfigCliOverrides:
|
||||
"""load_config 的 CLI 覆盖层测试。"""
|
||||
|
||||
def test_cli_overrides_yaml_values(self, tmp_path: Path) -> None:
|
||||
"""CLI 参数应覆盖 YAML 中的同名字段。"""
|
||||
harness_data = _yaml_harness_dict()
|
||||
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||
|
||||
cfg = load_config(yaml_path, cli_overrides={"concurrency": 4, "max_steps": 30})
|
||||
assert cfg.concurrency == 4
|
||||
assert cfg.max_steps == 30
|
||||
|
||||
def test_cli_none_values_ignored(self, tmp_path: Path) -> None:
|
||||
"""CLI 中值为 None 的字段不应覆盖 YAML。"""
|
||||
harness_data = _yaml_harness_dict()
|
||||
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||
|
||||
cfg = load_config(yaml_path, cli_overrides={"concurrency": None, "max_steps": 20})
|
||||
assert cfg.concurrency == 12 # YAML 默认值
|
||||
assert cfg.max_steps == 20
|
||||
|
||||
def test_cli_run_id_override(self, tmp_path: Path) -> None:
|
||||
"""CLI 可通过 run_id 覆盖默认空字符串。"""
|
||||
harness_data = _yaml_harness_dict()
|
||||
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||
|
||||
cfg = load_config(
|
||||
yaml_path,
|
||||
cli_overrides={"mode": "diagnose", "run_id": "run-abc-123"},
|
||||
)
|
||||
assert cfg.run_id == "run-abc-123"
|
||||
assert cfg.mode == "diagnose"
|
||||
|
||||
def test_path_fields_converted_to_path(self, tmp_path: Path) -> None:
|
||||
"""workspace_dir 和 store_dir 应被转换为 Path 对象。"""
|
||||
harness_data = _yaml_harness_dict()
|
||||
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||
|
||||
cfg = load_config(yaml_path)
|
||||
assert isinstance(cfg.workspace_dir, Path)
|
||||
assert isinstance(cfg.store_dir, Path)
|
||||
|
||||
def test_unknown_cli_keys_ignored(self, tmp_path: Path) -> None:
|
||||
"""YAML 和 RunConfig 中不存在的 CLI key 应被忽略。"""
|
||||
harness_data = _yaml_harness_dict()
|
||||
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||
|
||||
cfg = load_config(yaml_path, cli_overrides={"nonexistent_field": 42})
|
||||
assert cfg.concurrency == 12 # 正常字段不受影响
|
||||
|
||||
def test_validation_runs_after_loading(self, tmp_path: Path) -> None:
|
||||
"""load_config 加载后应运行校验，非法值应报错。"""
|
||||
harness_data = _yaml_harness_dict()
|
||||
harness_data["mode"] = "invalid_mode"
|
||||
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||
|
||||
with pytest.raises(ValueError, match="mode"):
|
||||
load_config(yaml_path)
|
||||
|
||||
|
||||
# ──────────────────── test_load_config_env_overrides ─────────────────────
|
||||
|
||||
|
||||
class TestLoadConfigEnvOverrides:
|
||||
"""load_config 的 .env 环境变量覆盖层测试。"""
|
||||
|
||||
def test_env_overrides_yaml_workspace_dir(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""环境变量 HARNESS_WORKSPACE_DIR 应覆盖 YAML 中的 workspace_dir。"""
|
||||
harness_data = _yaml_harness_dict()
|
||||
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||
|
||||
monkeypatch.setenv("HARNESS_WORKSPACE_DIR", "/custom/workspace")
|
||||
cfg = load_config(yaml_path)
|
||||
assert cfg.workspace_dir == Path("/custom/workspace")
|
||||
|
||||
def test_env_overrides_yaml_store_dir(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""环境变量 HARNESS_STORE_DIR 应覆盖 YAML 中的 store_dir。"""
|
||||
harness_data = _yaml_harness_dict()
|
||||
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||
|
||||
monkeypatch.setenv("HARNESS_STORE_DIR", "/custom/store")
|
||||
cfg = load_config(yaml_path)
|
||||
assert cfg.store_dir == Path("/custom/store")
|
||||
|
||||
def test_cli_overrides_env(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""CLI 优先级高于 .env：CLI > .env > YAML。"""
|
||||
harness_data = _yaml_harness_dict()
|
||||
yaml_path = _write_yaml(tmp_path, harness_data)
|
||||
|
||||
monkeypatch.setenv("HARNESS_WORKSPACE_DIR", "/env/workspace")
|
||||
cfg = load_config(yaml_path, cli_overrides={"workspace_dir": "/cli/workspace"})
|
||||
assert cfg.workspace_dir == Path("/cli/workspace")
|
||||
|
||||
|
||||
# ──────────────────── test_load_config_real_yaml ─────────────────────────
|
||||
|
||||
|
||||
class TestLoadConfigRealYaml:
|
||||
"""用真实 config/default.yaml 验证 load_config 嵌套解析。"""
|
||||
|
||||
def test_load_config_real_default_yaml(self) -> None:
|
||||
"""真实 config/default.yaml 的 harness 段应正确解析并通过校验。"""
|
||||
cfg = load_config(Path("config/default.yaml"), {})
|
||||
assert cfg.mode == "infer"
|
||||
assert cfg.gate_e_confirm == 20.0
|
||||
assert cfg.batch_size == 15
|
||||
assert cfg.concurrency == 12
|
||||
assert cfg.workspace_dir == Path("workspaces/default")
|
||||
assert cfg.store_dir == Path("store")
|
||||
assert cfg.skill_mode == "auto"
|
||||
|
||||
def test_real_yaml_cli_override(self) -> None:
|
||||
"""真实 YAML + CLI 覆盖应正确合并。"""
|
||||
cfg = load_config(
|
||||
Path("config/default.yaml"),
|
||||
{"concurrency": 4, "max_steps": 30},
|
||||
)
|
||||
assert cfg.concurrency == 4
|
||||
assert cfg.max_steps == 30
|
||||
assert cfg.mode == "infer" # 未覆盖字段保持 YAML 值
|
||||
|
||||
|
||||
# ─────────────────────────── test_val_size_floor ─────────────────────────
|
||||
|
||||
|
||||
class TestValSizeFloor:
|
||||
"""val_size >= eval_min_per_class * 11 的下限校验。"""
|
||||
|
||||
def test_val_size_below_floor_rejected(self) -> None:
|
||||
"""val_size < eval_min_per_class * 11 应抛出 ValueError。
|
||||
|
||||
eval_min_per_class=3 → 下限 = 3 * 11 = 33，val_size=30 不足。
|
||||
"""
|
||||
cfg = _make_config(eval_min_per_class=3, val_size=30)
|
||||
with pytest.raises(ValueError, match="val_size"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_val_size_at_floor_accepted(self) -> None:
|
||||
"""val_size == eval_min_per_class * 11 应通过。"""
|
||||
cfg = _make_config(eval_min_per_class=3, val_size=33)
|
||||
_validate(cfg)
|
||||
|
||||
def test_val_size_above_floor_accepted(self) -> None:
|
||||
"""val_size > eval_min_per_class * 11 应通过。"""
|
||||
cfg = _make_config(eval_min_per_class=2, val_size=100)
|
||||
_validate(cfg)
|
||||
|
||||
def test_default_yaml_values_satisfy_floor(self) -> None:
|
||||
"""default.yaml 的默认值（val_size=30, eval_min_per_class=2）应满足下限。
|
||||
|
||||
下限 = 2 * 11 = 22，val_size=30 >= 22，通过。
|
||||
"""
|
||||
cfg = _make_config()
|
||||
_validate(cfg) # 不应抛出异常
|
||||
@@ -0,0 +1,374 @@
|
||||
"""app/harness/gate_ladder.py 单元测试。
|
||||
|
||||
覆盖冷启动交错、Beta(1,1) 平滑、warm 信息量排序、
|
||||
GatePools 原子读写与指纹校验、BaselineCache 四维内容寻址与先盘后存、
|
||||
gamma-EMA 更新、防泄露过滤等核心语义。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from app.harness.gate_ladder import (
|
||||
BaselineCache,
|
||||
GatePools,
|
||||
LadderEntry,
|
||||
build_cold_entries,
|
||||
build_or_load_gate_pools,
|
||||
order_ladder,
|
||||
skill_hash,
|
||||
)
|
||||
from core.types import GeneratedQuestion
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# ── 工具函数 ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_q(qid: str, task_type: str = "AR") -> GeneratedQuestion:
|
||||
"""构造最小 GeneratedQuestion 实例。"""
|
||||
return GeneratedQuestion(
|
||||
question_id=qid,
|
||||
video_id="v1",
|
||||
task_type=task_type,
|
||||
question="dummy",
|
||||
options=("A", "B", "C", "D"),
|
||||
answer="A",
|
||||
source_nodes=("n1",),
|
||||
difficulty="easy",
|
||||
)
|
||||
|
||||
|
||||
# ── 冷启动 ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestColdStart:
|
||||
"""冷启动排序：2:1 交错 + 探针插尾 + Beta(1,1) 平滑。"""
|
||||
|
||||
def test_cold_start_interleaving(self) -> None:
|
||||
"""错题:对题 = 2:1 交错顺序。
|
||||
|
||||
6 错 3 对（probe_quota=0 无探针）→ 交错序应为 W W R W W R W W R。
|
||||
"""
|
||||
wrong_ids = [f"w{i}" for i in range(6)]
|
||||
right_ids = [f"r{i}" for i in range(3)]
|
||||
questions = [_make_q(qid) for qid in wrong_ids + right_ids]
|
||||
correctness = dict.fromkeys(wrong_ids, False)
|
||||
correctness.update(dict.fromkeys(right_ids, True))
|
||||
|
||||
entries = build_cold_entries(questions, correctness, probe_quota=0.0, seed=42)
|
||||
|
||||
assert len(entries) == 9
|
||||
# 验证 2:1 交错模式（seed 固定后 shuffle 结果确定）
|
||||
pattern = ["W" if not correctness[e.question_id] else "R" for e in entries]
|
||||
# 前 9 个交错应为 W W R W W R W W R
|
||||
assert pattern == ["W", "W", "R", "W", "W", "R", "W", "W", "R"]
|
||||
|
||||
def test_cold_start_p_hat_beta(self) -> None:
|
||||
"""p_hat 遵循 Beta(1,1) 平滑：错=1/3，对=2/3。"""
|
||||
questions = [_make_q("q1"), _make_q("q2")]
|
||||
correctness = {"q1": False, "q2": True}
|
||||
|
||||
entries = build_cold_entries(questions, correctness, probe_quota=0.0, seed=0)
|
||||
|
||||
p_map = {e.question_id: e.p_hat for e in entries}
|
||||
assert p_map["q1"] == pytest.approx(1 / 3)
|
||||
assert p_map["q2"] == pytest.approx(2 / 3)
|
||||
|
||||
def test_cold_start_probe_at_tail(self) -> None:
|
||||
"""probe_quota > 0 时探针题追加在尾部。"""
|
||||
wrong_ids = [f"w{i}" for i in range(10)]
|
||||
right_ids = [f"r{i}" for i in range(2)]
|
||||
questions = [_make_q(qid) for qid in wrong_ids + right_ids]
|
||||
correctness = dict.fromkeys(wrong_ids, False)
|
||||
correctness.update(dict.fromkeys(right_ids, True))
|
||||
|
||||
entries = build_cold_entries(questions, correctness, probe_quota=0.3, seed=7)
|
||||
|
||||
# 10 错 * 0.3 = 3 个探针在尾部
|
||||
n_probe = int(10 * 0.3)
|
||||
assert n_probe == 3
|
||||
# 尾部 3 个都应为错题
|
||||
tail = entries[-n_probe:]
|
||||
for e in tail:
|
||||
assert not correctness[e.question_id]
|
||||
|
||||
|
||||
# ── warm 排序 ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestWarmOrdering:
|
||||
"""warm 阶段：信息量 p_hat(1-p_hat) 降序 + p_hat 区间过滤。"""
|
||||
|
||||
def test_warm_ordering_information(self) -> None:
|
||||
"""p_hat=0.5 信息量最高，排在最前。"""
|
||||
entries = [
|
||||
LadderEntry("a", 0.1),
|
||||
LadderEntry("b", 0.5),
|
||||
LadderEntry("c", 0.9),
|
||||
LadderEntry("d", 0.3),
|
||||
]
|
||||
ordered = order_ladder(entries, p_low=0.0, p_high=1.0)
|
||||
assert ordered[0].question_id == "b" # 0.5*(1-0.5)=0.25 最高
|
||||
# d: 0.3*0.7=0.21, a: 0.1*0.9=0.09, c: 0.9*0.1=0.09
|
||||
assert ordered[1].question_id == "d"
|
||||
|
||||
def test_warm_filter_bounds(self) -> None:
|
||||
"""p_hat 不在 [p_low, p_high] 区间的题被剔除。"""
|
||||
entries = [
|
||||
LadderEntry("low", 0.05),
|
||||
LadderEntry("mid", 0.5),
|
||||
LadderEntry("high", 0.95),
|
||||
]
|
||||
ordered = order_ladder(entries, p_low=0.1, p_high=0.9)
|
||||
ids = [e.question_id for e in ordered]
|
||||
assert "mid" in ids
|
||||
assert "low" not in ids
|
||||
assert "high" not in ids
|
||||
|
||||
|
||||
# ── GatePools 持久化 ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestGatePoolsPersistence:
|
||||
"""GatePools.save/load 原子性与指纹校验。"""
|
||||
|
||||
def test_gate_pools_save_load_atomic(self, tmp_path: Path) -> None:
|
||||
"""save -> load 往返保真，且使用原子写（中间 .tmp 文件不残留）。"""
|
||||
entries = {
|
||||
"AR": [LadderEntry("q1", 0.33), LadderEntry("q2", 0.67)],
|
||||
"CR": [LadderEntry("q3", 0.5)],
|
||||
}
|
||||
pools = GatePools(entries=entries, seed=42, fingerprint="abc123")
|
||||
path = tmp_path / "gate_pools.json"
|
||||
pools.save(path)
|
||||
|
||||
# .tmp 文件不应残留
|
||||
assert not (tmp_path / "gate_pools.json.tmp").exists()
|
||||
assert path.exists()
|
||||
|
||||
loaded = GatePools.load(path)
|
||||
assert loaded.seed == 42
|
||||
assert loaded.fingerprint == "abc123"
|
||||
assert len(loaded.entries["AR"]) == 2
|
||||
assert loaded.entries["AR"][0].question_id == "q1"
|
||||
assert loaded.entries["AR"][0].p_hat == pytest.approx(0.33)
|
||||
assert loaded.entries["CR"][0].question_id == "q3"
|
||||
|
||||
def test_gate_pools_fingerprint_mismatch(self, tmp_path: Path) -> None:
|
||||
"""指纹不一致 -> RuntimeError（不静默重建）。"""
|
||||
questions = [_make_q("q1", "AR"), _make_q("q2", "AR")]
|
||||
correctness = {"q1": True, "q2": False}
|
||||
|
||||
# 第一次构建
|
||||
build_or_load_gate_pools(
|
||||
workspace_dir=tmp_path,
|
||||
questions=questions,
|
||||
test_qids=set(),
|
||||
baseline_correctness=correctness,
|
||||
task_types=["AR"],
|
||||
probe_quota=0.0,
|
||||
seed=1,
|
||||
baseline_run_id="run_001",
|
||||
)
|
||||
|
||||
# 改 baseline_run_id 导致指纹变化 -> 应报错
|
||||
with pytest.raises(RuntimeError, match="指纹不一致"):
|
||||
build_or_load_gate_pools(
|
||||
workspace_dir=tmp_path,
|
||||
questions=questions,
|
||||
test_qids=set(),
|
||||
baseline_correctness=correctness,
|
||||
task_types=["AR"],
|
||||
probe_quota=0.0,
|
||||
seed=1,
|
||||
baseline_run_id="run_002",
|
||||
)
|
||||
|
||||
|
||||
# ── ladder_for ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestLadderFor:
|
||||
"""ladder_for 取题序与排除逻辑。"""
|
||||
|
||||
def test_ladder_for_excludes_qids(self) -> None:
|
||||
"""exclude_qids 中的题被排除。"""
|
||||
entries = {
|
||||
"AR": [
|
||||
LadderEntry("q1", 0.5),
|
||||
LadderEntry("q2", 0.4),
|
||||
LadderEntry("q3", 0.6),
|
||||
],
|
||||
}
|
||||
pools = GatePools(entries=entries, seed=0, fingerprint="x")
|
||||
result = pools.ladder_for("AR", exclude_qids={"q2"}, p_low=0.0, p_high=1.0, cold=True)
|
||||
assert "q2" not in result
|
||||
assert "q1" in result
|
||||
assert "q3" in result
|
||||
|
||||
def test_ladder_for_missing_task_type(self) -> None:
|
||||
"""不存在的 task_type -> ValueError。"""
|
||||
pools = GatePools(entries={}, seed=0, fingerprint="x")
|
||||
with pytest.raises(ValueError, match="无阶梯"):
|
||||
pools.ladder_for("MISSING", set(), 0.0, 1.0, cold=True)
|
||||
|
||||
def test_ladder_for_warm_uses_order_ladder(self) -> None:
|
||||
"""cold=False 时走 warm 信息量排序。"""
|
||||
entries = {
|
||||
"AR": [
|
||||
LadderEntry("low", 0.1),
|
||||
LadderEntry("mid", 0.5),
|
||||
LadderEntry("high", 0.9),
|
||||
],
|
||||
}
|
||||
pools = GatePools(entries=entries, seed=0, fingerprint="x")
|
||||
result = pools.ladder_for("AR", set(), p_low=0.0, p_high=1.0, cold=False)
|
||||
# 信息量排序：mid(0.25) > low(0.09) = high(0.09)
|
||||
assert result[0] == "mid"
|
||||
|
||||
|
||||
# ── gamma-EMA 更新 ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestGammaEMA:
|
||||
"""gamma-EMA 更新 p_hat。"""
|
||||
|
||||
def test_gamma_ema_update(self) -> None:
|
||||
"""p_hat <- gamma * p_hat + (1-gamma) * obs。"""
|
||||
entries = {"AR": [LadderEntry("q1", 0.5)]}
|
||||
pools = GatePools(entries=entries, seed=0, fingerprint="x")
|
||||
|
||||
# 观测为正确(1.0), gamma=0.8
|
||||
pools.update_probs({"q1": True}, gamma=0.8)
|
||||
expected = 0.8 * 0.5 + 0.2 * 1.0 # 0.6
|
||||
assert pools.entries["AR"][0].p_hat == pytest.approx(expected)
|
||||
|
||||
# 再次观测为错误(0.0), gamma=0.8
|
||||
pools.update_probs({"q1": False}, gamma=0.8)
|
||||
expected2 = 0.8 * expected + 0.2 * 0.0 # 0.48
|
||||
assert pools.entries["AR"][0].p_hat == pytest.approx(expected2)
|
||||
|
||||
def test_update_probs_no_observation_unchanged(self) -> None:
|
||||
"""无观测的题 p_hat 不变。"""
|
||||
entries = {"AR": [LadderEntry("q1", 0.5), LadderEntry("q2", 0.3)]}
|
||||
pools = GatePools(entries=entries, seed=0, fingerprint="x")
|
||||
pools.update_probs({"q1": True}, gamma=0.9)
|
||||
assert pools.entries["AR"][1].p_hat == pytest.approx(0.3)
|
||||
|
||||
|
||||
# ── 防泄露 ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestLeakPrevention:
|
||||
"""防泄露铁律：gate 内 rollout 永不回流 p_hat（由调用方过滤）。"""
|
||||
|
||||
def test_update_probs_excludes_gate_runs(self) -> None:
|
||||
"""调用方须过滤 run_id 含 '_gate_' 的观测。
|
||||
|
||||
update_probs 本身只接收已过滤的 observations，这里验证
|
||||
如果调用方正确过滤，gate run 数据不会影响 p_hat。
|
||||
"""
|
||||
entries = {"AR": [LadderEntry("q1", 0.5)]}
|
||||
pools = GatePools(entries=entries, seed=0, fingerprint="x")
|
||||
|
||||
# 模拟：所有 run 的原始观测（含 gate run）
|
||||
raw_observations = {
|
||||
"run_normal": {"q1": True}, # 普通 run
|
||||
"run_gate_01": {"q1": False}, # gate run（run_id 含 _gate_）
|
||||
}
|
||||
|
||||
# 调用方按 run_id 过滤：排除含 "_gate_" 的 run
|
||||
filtered = {}
|
||||
for run_id, obs in raw_observations.items():
|
||||
if "_gate_" not in run_id:
|
||||
filtered.update(obs)
|
||||
|
||||
# 只有普通 run 的观测进入 update_probs
|
||||
assert filtered == {"q1": True}
|
||||
pools.update_probs(filtered, gamma=0.8)
|
||||
expected = 0.8 * 0.5 + 0.2 * 1.0
|
||||
assert pools.entries["AR"][0].p_hat == pytest.approx(expected)
|
||||
|
||||
|
||||
# ── BaselineCache ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestBaselineCache:
|
||||
"""BaselineCache 四维内容寻址与先盘后存。"""
|
||||
|
||||
def test_baseline_cache_content_addressed(self, tmp_path: Path) -> None:
|
||||
"""四维键唯一寻址：任一维度变化 -> miss。"""
|
||||
path = tmp_path / "baseline_cache.json"
|
||||
cache = BaselineCache(path)
|
||||
|
||||
cache.put("AR", "hash1", "v1", "q1", True)
|
||||
assert cache.get("AR", "hash1", "v1", "q1") is True
|
||||
|
||||
# 改 skill_hash -> miss
|
||||
assert cache.get("AR", "hash2", "v1", "q1") is None
|
||||
# 改 prompts_version -> miss
|
||||
assert cache.get("AR", "hash1", "v2", "q1") is None
|
||||
# 改 task_type -> miss
|
||||
assert cache.get("CR", "hash1", "v1", "q1") is None
|
||||
# 改 qid -> miss
|
||||
assert cache.get("AR", "hash1", "v1", "q2") is None
|
||||
|
||||
def test_baseline_cache_disk_first(self, tmp_path: Path) -> None:
|
||||
"""先盘后存：磁盘写成功后内存才更新，新实例可从磁盘读到。"""
|
||||
path = tmp_path / "baseline_cache.json"
|
||||
cache = BaselineCache(path)
|
||||
|
||||
cache.put("AR", "h1", "v1", "q1", True)
|
||||
|
||||
# 内存可读
|
||||
assert cache.get("AR", "h1", "v1", "q1") is True
|
||||
|
||||
# 新实例从磁盘加载也能读到（证明先落盘）
|
||||
cache2 = BaselineCache(path)
|
||||
assert cache2.get("AR", "h1", "v1", "q1") is True
|
||||
|
||||
# .tmp 文件不应残留
|
||||
assert not (tmp_path / "baseline_cache.json.tmp").exists()
|
||||
|
||||
def test_baseline_cache_empty_init(self, tmp_path: Path) -> None:
|
||||
"""不存在的文件 -> 空缓存初始化。"""
|
||||
path = tmp_path / "nonexistent.json"
|
||||
cache = BaselineCache(path)
|
||||
assert cache.get("AR", "h1", "v1", "q1") is None
|
||||
|
||||
def test_baseline_cache_overwrite(self, tmp_path: Path) -> None:
|
||||
"""同键重复写入覆盖旧值。"""
|
||||
path = tmp_path / "baseline_cache.json"
|
||||
cache = BaselineCache(path)
|
||||
|
||||
cache.put("AR", "h1", "v1", "q1", True)
|
||||
assert cache.get("AR", "h1", "v1", "q1") is True
|
||||
|
||||
cache.put("AR", "h1", "v1", "q1", False)
|
||||
assert cache.get("AR", "h1", "v1", "q1") is False
|
||||
|
||||
|
||||
# ── skill_hash ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestSkillHash:
|
||||
"""skill_hash SHA1 摘要。"""
|
||||
|
||||
def test_deterministic(self) -> None:
|
||||
"""相同输入产生相同摘要。"""
|
||||
assert skill_hash("hello") == skill_hash("hello")
|
||||
|
||||
def test_different_content(self) -> None:
|
||||
"""不同输入产生不同摘要。"""
|
||||
assert skill_hash("hello") != skill_hash("world")
|
||||
|
||||
def test_is_sha1_hex(self) -> None:
|
||||
"""输出为 40 字符十六进制。"""
|
||||
h = skill_hash("test")
|
||||
assert len(h) == 40
|
||||
assert all(c in "0123456789abcdef" for c in h)
|
||||
@@ -0,0 +1,659 @@
|
||||
"""app/harness/inference.py 单元测试。
|
||||
|
||||
测试覆盖:
|
||||
- run_inference 基本流程（mock LLM + tool_dispatch）
|
||||
- 异常时 prediction 仍落库（stop_reason=error）
|
||||
- _to_text_field 归一化
|
||||
- run_id 空串 → ValueError
|
||||
- _aggregate_results 内存聚合
|
||||
- 空 questions 列表零值返回
|
||||
- 并发控制 Semaphore
|
||||
- plugins_factory 调用
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.harness.inference import (
|
||||
InferenceResult,
|
||||
_aggregate_results,
|
||||
_to_text_field,
|
||||
run_inference,
|
||||
)
|
||||
from app.harness.log import HarnessLog
|
||||
from core.types import GeneratedQuestion, LLMResponse
|
||||
|
||||
# ── 测试基础设施 ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_question(
|
||||
question_id: str = "q1",
|
||||
video_id: str = "v1",
|
||||
task_type: str = "Action Reasoning",
|
||||
answer: str = "B",
|
||||
) -> GeneratedQuestion:
|
||||
"""构造测试用题目。"""
|
||||
return GeneratedQuestion(
|
||||
question_id=question_id,
|
||||
video_id=video_id,
|
||||
task_type=task_type,
|
||||
question="测试问题",
|
||||
options=("A. 选项A", "B. 选项B", "C. 选项C", "D. 选项D"),
|
||||
answer=answer,
|
||||
source_nodes=("L1_001",),
|
||||
difficulty="medium",
|
||||
)
|
||||
|
||||
|
||||
def _make_llm_response(answer: str = "B") -> LLMResponse:
|
||||
"""构造测试用 LLMResponse（submit_answer 场景）。"""
|
||||
content = json.dumps(
|
||||
{
|
||||
"reflect": {"observation": "找到答案"},
|
||||
"plan": {"next_step": "提交"},
|
||||
"action": {
|
||||
"tool": "submit_answer",
|
||||
"args": {
|
||||
"answer": answer,
|
||||
"evidence": "证据文本",
|
||||
"reasoning": "推理过程",
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
return LLMResponse(
|
||||
content=content,
|
||||
thinking="思考过程",
|
||||
model="test-model",
|
||||
provider="test",
|
||||
prompt_tokens=100,
|
||||
completion_tokens=50,
|
||||
latency_ms=200,
|
||||
ttft_ms=30.0,
|
||||
max_inter_token_ms=5.0,
|
||||
cache_hit=False,
|
||||
call_id="test-call-001",
|
||||
)
|
||||
|
||||
|
||||
def _make_error_llm_response() -> LLMResponse:
|
||||
"""构造触发解析失败的 LLMResponse。"""
|
||||
return LLMResponse(
|
||||
content="这不是JSON",
|
||||
thinking="",
|
||||
model="test-model",
|
||||
provider="test",
|
||||
prompt_tokens=10,
|
||||
completion_tokens=5,
|
||||
latency_ms=50,
|
||||
ttft_ms=10.0,
|
||||
max_inter_token_ms=2.0,
|
||||
cache_hit=False,
|
||||
call_id="test-call-err",
|
||||
)
|
||||
|
||||
|
||||
async def _stub_tool_dispatch(
|
||||
tool_name: str, args: dict[str, Any], *, context: dict[str, Any]
|
||||
) -> str:
|
||||
"""测试用工具调度函数。"""
|
||||
if tool_name == "submit_answer":
|
||||
return "答案已提交"
|
||||
raise ValueError(f"未知工具: {tool_name}")
|
||||
|
||||
|
||||
def _stub_prompt_builder(qa: GeneratedQuestion) -> tuple[str, str]:
|
||||
"""测试用 prompt 构建函数。"""
|
||||
return "系统提示词", f"用户问题: {qa.question}"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def harness_log(tmp_path: Any, request: Any) -> HarnessLog:
|
||||
"""创建临时 HarnessLog 实例。
|
||||
|
||||
使用 test 节点名称的 hash 作为 db 文件名，避免冲突。
|
||||
run_id 固定为 "test-run"，实际 run_inference 中传入的 run_id
|
||||
由 HarnessLog.insert 自动覆盖为 HarnessLog 构造时的值。
|
||||
"""
|
||||
db_name = f"harness_{id(request)}.db"
|
||||
db_path = str(tmp_path / db_name)
|
||||
log = HarnessLog(db_path, "test-run")
|
||||
yield log
|
||||
log.close()
|
||||
|
||||
|
||||
# ── 测试用例 ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestToTextField:
|
||||
"""_to_text_field 归一化测试。"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_string_passthrough(self) -> None:
|
||||
"""字符串原样返回。"""
|
||||
assert _to_text_field("hello") == "hello"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_string(self) -> None:
|
||||
"""空字符串原样返回。"""
|
||||
assert _to_text_field("") == ""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_serialized(self) -> None:
|
||||
"""list 被 JSON 序列化。"""
|
||||
result = _to_text_field(["a", "b"])
|
||||
assert result == '["a", "b"]'
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dict_serialized(self) -> None:
|
||||
"""dict 被 JSON 序列化。"""
|
||||
result = _to_text_field({"key": "值"})
|
||||
assert '"key"' in result
|
||||
assert '"值"' in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_int_serialized(self) -> None:
|
||||
"""int 被 JSON 序列化。"""
|
||||
assert _to_text_field(42) == "42"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_none_serialized(self) -> None:
|
||||
"""None 被 JSON 序列化。"""
|
||||
assert _to_text_field(None) == "null"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unicode_preserved(self) -> None:
|
||||
"""ensure_ascii=False 保留中文。"""
|
||||
result = _to_text_field(["中文"])
|
||||
assert "中文" in result
|
||||
assert "\\u" not in result
|
||||
|
||||
|
||||
class TestAggregateResults:
|
||||
"""_aggregate_results 内存聚合测试。"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_records(self) -> None:
|
||||
"""空列表返回零值 InferenceResult。"""
|
||||
result = _aggregate_results([], "run-empty")
|
||||
assert result.run_id == "run-empty"
|
||||
assert result.accuracy == 0.0
|
||||
assert result.total == 0
|
||||
assert result.correct == 0
|
||||
assert result.per_task_type == {}
|
||||
assert result.steps_mean == 0.0
|
||||
assert result.token_usage == {"prompt_tokens": 0, "completion_tokens": 0}
|
||||
assert result.stop_reason_counts == {}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_correct(self) -> None:
|
||||
"""单条正确记录 → accuracy=1.0。"""
|
||||
records = [
|
||||
{
|
||||
"prediction": "B",
|
||||
"answer": "B",
|
||||
"task_type": "AR",
|
||||
"steps_used": 3,
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 50,
|
||||
"stop_reason": "finished",
|
||||
}
|
||||
]
|
||||
result = _aggregate_results(records, "run-1")
|
||||
assert result.accuracy == 1.0
|
||||
assert result.total == 1
|
||||
assert result.correct == 1
|
||||
assert result.steps_mean == 3.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mixed_correct_wrong(self) -> None:
|
||||
"""混合正确/错误 → 准确率与步数均正确聚合。"""
|
||||
records = [
|
||||
{
|
||||
"prediction": "B",
|
||||
"answer": "B",
|
||||
"task_type": "AR",
|
||||
"steps_used": 2,
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 50,
|
||||
"stop_reason": "finished",
|
||||
},
|
||||
{
|
||||
"prediction": "C",
|
||||
"answer": "A",
|
||||
"task_type": "AR",
|
||||
"steps_used": 4,
|
||||
"prompt_tokens": 200,
|
||||
"completion_tokens": 100,
|
||||
"stop_reason": "budget_exceeded",
|
||||
},
|
||||
{
|
||||
"prediction": "D",
|
||||
"answer": "D",
|
||||
"task_type": "SP",
|
||||
"steps_used": 1,
|
||||
"prompt_tokens": 50,
|
||||
"completion_tokens": 25,
|
||||
"stop_reason": "finished",
|
||||
},
|
||||
]
|
||||
result = _aggregate_results(records, "run-mix")
|
||||
assert result.total == 3
|
||||
assert result.correct == 2
|
||||
assert abs(result.accuracy - 2 / 3) < 1e-9
|
||||
assert abs(result.steps_mean - 7 / 3) < 1e-9
|
||||
assert result.token_usage == {"prompt_tokens": 350, "completion_tokens": 175}
|
||||
assert result.stop_reason_counts == {"finished": 2, "budget_exceeded": 1}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_per_task_type_grouping(self) -> None:
|
||||
"""按 task_type 分组聚合。"""
|
||||
records = [
|
||||
{
|
||||
"prediction": "B",
|
||||
"answer": "B",
|
||||
"task_type": "AR",
|
||||
"steps_used": 1,
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"stop_reason": "finished",
|
||||
},
|
||||
{
|
||||
"prediction": "A",
|
||||
"answer": "C",
|
||||
"task_type": "AR",
|
||||
"steps_used": 2,
|
||||
"prompt_tokens": 20,
|
||||
"completion_tokens": 10,
|
||||
"stop_reason": "finished",
|
||||
},
|
||||
{
|
||||
"prediction": "D",
|
||||
"answer": "D",
|
||||
"task_type": "SP",
|
||||
"steps_used": 3,
|
||||
"prompt_tokens": 30,
|
||||
"completion_tokens": 15,
|
||||
"stop_reason": "finished",
|
||||
},
|
||||
]
|
||||
result = _aggregate_results(records, "run-task")
|
||||
assert "AR" in result.per_task_type
|
||||
assert "SP" in result.per_task_type
|
||||
assert result.per_task_type["AR"]["total"] == 2
|
||||
assert result.per_task_type["AR"]["correct"] == 1
|
||||
assert result.per_task_type["AR"]["accuracy"] == 0.5
|
||||
assert result.per_task_type["SP"]["total"] == 1
|
||||
assert result.per_task_type["SP"]["correct"] == 1
|
||||
assert result.per_task_type["SP"]["accuracy"] == 1.0
|
||||
|
||||
|
||||
class TestRunIdValidation:
|
||||
"""run_id 校验测试。"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_string_raises(self, harness_log: HarnessLog) -> None:
|
||||
"""空串 run_id → ValueError。"""
|
||||
llm = AsyncMock()
|
||||
with pytest.raises(ValueError, match="run_id 不得为空"):
|
||||
await run_inference(
|
||||
[],
|
||||
llm=llm,
|
||||
tool_dispatch_fn=_stub_tool_dispatch,
|
||||
prompt_builder=_stub_prompt_builder,
|
||||
log=harness_log,
|
||||
run_id="",
|
||||
concurrency=1,
|
||||
max_steps=10,
|
||||
skill_mode="auto",
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_whitespace_only_raises(self, harness_log: HarnessLog) -> None:
|
||||
"""纯空白 run_id → ValueError。"""
|
||||
llm = AsyncMock()
|
||||
with pytest.raises(ValueError, match="run_id 不得为空"):
|
||||
await run_inference(
|
||||
[],
|
||||
llm=llm,
|
||||
tool_dispatch_fn=_stub_tool_dispatch,
|
||||
prompt_builder=_stub_prompt_builder,
|
||||
log=harness_log,
|
||||
run_id=" ",
|
||||
concurrency=1,
|
||||
max_steps=10,
|
||||
skill_mode="auto",
|
||||
)
|
||||
|
||||
|
||||
class TestEmptyQuestions:
|
||||
"""空题目列表测试。"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_questions_returns_zero(self, harness_log: HarnessLog) -> None:
|
||||
"""空 questions 列表直接返回零值 InferenceResult。"""
|
||||
llm = AsyncMock()
|
||||
result = await run_inference(
|
||||
[],
|
||||
llm=llm,
|
||||
tool_dispatch_fn=_stub_tool_dispatch,
|
||||
prompt_builder=_stub_prompt_builder,
|
||||
log=harness_log,
|
||||
run_id="run-empty",
|
||||
concurrency=1,
|
||||
max_steps=10,
|
||||
skill_mode="auto",
|
||||
)
|
||||
assert isinstance(result, InferenceResult)
|
||||
assert result.run_id == "run-empty"
|
||||
assert result.accuracy == 0.0
|
||||
assert result.total == 0
|
||||
assert result.correct == 0
|
||||
# LLM 未被调用
|
||||
llm.chat.assert_not_called()
|
||||
|
||||
|
||||
class TestRunInferenceBasic:
|
||||
"""run_inference 基本流程测试。"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_question_correct(self, harness_log: HarnessLog) -> None:
|
||||
"""单题正确推理 → accuracy=1.0, stop_reason=finished。"""
|
||||
llm = AsyncMock()
|
||||
llm.chat.return_value = _make_llm_response(answer="B")
|
||||
|
||||
result = await run_inference(
|
||||
[_make_question(answer="B")],
|
||||
llm=llm,
|
||||
tool_dispatch_fn=_stub_tool_dispatch,
|
||||
prompt_builder=_stub_prompt_builder,
|
||||
log=harness_log,
|
||||
run_id="run-basic",
|
||||
concurrency=1,
|
||||
max_steps=10,
|
||||
skill_mode="auto",
|
||||
)
|
||||
|
||||
assert result.accuracy == 1.0
|
||||
assert result.total == 1
|
||||
assert result.correct == 1
|
||||
assert result.stop_reason_counts.get("finished") == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_question_wrong(self, harness_log: HarnessLog) -> None:
|
||||
"""单题错误推理 → accuracy=0.0。"""
|
||||
llm = AsyncMock()
|
||||
llm.chat.return_value = _make_llm_response(answer="C")
|
||||
|
||||
result = await run_inference(
|
||||
[_make_question(answer="B")],
|
||||
llm=llm,
|
||||
tool_dispatch_fn=_stub_tool_dispatch,
|
||||
prompt_builder=_stub_prompt_builder,
|
||||
log=harness_log,
|
||||
run_id="run-wrong",
|
||||
concurrency=1,
|
||||
max_steps=10,
|
||||
skill_mode="auto",
|
||||
)
|
||||
|
||||
assert result.accuracy == 0.0
|
||||
assert result.total == 1
|
||||
assert result.correct == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_questions_concurrent(self, harness_log: HarnessLog) -> None:
|
||||
"""3 题并发推理 → 结果正确聚合。"""
|
||||
llm = AsyncMock()
|
||||
llm.chat.return_value = _make_llm_response(answer="B")
|
||||
|
||||
questions = [
|
||||
_make_question(question_id="q1", answer="B"),
|
||||
_make_question(question_id="q2", answer="B"),
|
||||
_make_question(question_id="q3", answer="A"),
|
||||
]
|
||||
|
||||
result = await run_inference(
|
||||
questions,
|
||||
llm=llm,
|
||||
tool_dispatch_fn=_stub_tool_dispatch,
|
||||
prompt_builder=_stub_prompt_builder,
|
||||
log=harness_log,
|
||||
run_id="run-multi",
|
||||
concurrency=3,
|
||||
max_steps=10,
|
||||
skill_mode="auto",
|
||||
)
|
||||
|
||||
assert result.total == 3
|
||||
assert result.correct == 2
|
||||
assert abs(result.accuracy - 2 / 3) < 1e-9
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_token_usage_accumulated(self, harness_log: HarnessLog) -> None:
|
||||
"""多题 token 累加验证。"""
|
||||
llm = AsyncMock()
|
||||
llm.chat.return_value = _make_llm_response(answer="B")
|
||||
|
||||
questions = [
|
||||
_make_question(question_id="q1"),
|
||||
_make_question(question_id="q2"),
|
||||
]
|
||||
|
||||
result = await run_inference(
|
||||
questions,
|
||||
llm=llm,
|
||||
tool_dispatch_fn=_stub_tool_dispatch,
|
||||
prompt_builder=_stub_prompt_builder,
|
||||
log=harness_log,
|
||||
run_id="run-token",
|
||||
concurrency=2,
|
||||
max_steps=10,
|
||||
skill_mode="auto",
|
||||
)
|
||||
|
||||
assert result.token_usage["prompt_tokens"] == 200
|
||||
assert result.token_usage["completion_tokens"] == 100
|
||||
|
||||
|
||||
class TestPredictionAlwaysWritten:
|
||||
"""异常时 prediction 仍落库测试。"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_error_still_persisted(self, harness_log: HarnessLog) -> None:
|
||||
"""LLM 调用异常时，prediction 仍以 stop_reason=error 落库。"""
|
||||
llm = AsyncMock()
|
||||
llm.chat.side_effect = RuntimeError("LLM API 不可用")
|
||||
|
||||
result = await run_inference(
|
||||
[_make_question()],
|
||||
llm=llm,
|
||||
tool_dispatch_fn=_stub_tool_dispatch,
|
||||
prompt_builder=_stub_prompt_builder,
|
||||
log=harness_log,
|
||||
run_id="run-error",
|
||||
concurrency=1,
|
||||
max_steps=10,
|
||||
skill_mode="auto",
|
||||
)
|
||||
|
||||
assert result.total == 1
|
||||
assert result.correct == 0
|
||||
assert result.stop_reason_counts.get("error") == 1
|
||||
|
||||
# 验证 DB 中的记录（HarnessLog.insert 使用构造时的 run_id）
|
||||
rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("test-run",))
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["stop_reason"] == "error"
|
||||
assert rows[0]["prediction"] is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_error_still_persisted(self, harness_log: HarnessLog) -> None:
|
||||
"""LLM 返回非 JSON 内容，parse_error 后 prediction 仍落库。"""
|
||||
llm = AsyncMock()
|
||||
llm.chat.return_value = _make_error_llm_response()
|
||||
|
||||
result = await run_inference(
|
||||
[_make_question()],
|
||||
llm=llm,
|
||||
tool_dispatch_fn=_stub_tool_dispatch,
|
||||
prompt_builder=_stub_prompt_builder,
|
||||
log=harness_log,
|
||||
run_id="run-parse-err",
|
||||
concurrency=1,
|
||||
max_steps=10,
|
||||
skill_mode="auto",
|
||||
)
|
||||
|
||||
assert result.total == 1
|
||||
# HarnessLog.insert 使用构造时的 run_id
|
||||
rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("test-run",))
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["prediction"] is None
|
||||
|
||||
|
||||
class TestPluginsFactory:
|
||||
"""plugins_factory 调用测试。"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_factory_called_per_question(self, harness_log: HarnessLog) -> None:
|
||||
"""每题调用 plugins_factory，传入 (video_id, question_id)。"""
|
||||
llm = AsyncMock()
|
||||
llm.chat.return_value = _make_llm_response(answer="B")
|
||||
|
||||
factory_calls: list[tuple[str, str]] = []
|
||||
|
||||
def _factory(video_id: str, question_id: str) -> list[object]:
|
||||
factory_calls.append((video_id, question_id))
|
||||
return []
|
||||
|
||||
questions = [
|
||||
_make_question(question_id="q1", video_id="v1"),
|
||||
_make_question(question_id="q2", video_id="v2"),
|
||||
]
|
||||
|
||||
await run_inference(
|
||||
questions,
|
||||
llm=llm,
|
||||
tool_dispatch_fn=_stub_tool_dispatch,
|
||||
prompt_builder=_stub_prompt_builder,
|
||||
log=harness_log,
|
||||
run_id="run-factory",
|
||||
concurrency=2,
|
||||
max_steps=10,
|
||||
skill_mode="auto",
|
||||
plugins_factory=_factory,
|
||||
)
|
||||
|
||||
assert len(factory_calls) == 2
|
||||
call_set = set(factory_calls)
|
||||
assert ("v1", "q1") in call_set
|
||||
assert ("v2", "q2") in call_set
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_factory_uses_empty_plugins(self, harness_log: HarnessLog) -> None:
|
||||
"""plugins_factory=None 时使用空 plugins 列表。"""
|
||||
llm = AsyncMock()
|
||||
llm.chat.return_value = _make_llm_response(answer="B")
|
||||
|
||||
result = await run_inference(
|
||||
[_make_question()],
|
||||
llm=llm,
|
||||
tool_dispatch_fn=_stub_tool_dispatch,
|
||||
prompt_builder=_stub_prompt_builder,
|
||||
log=harness_log,
|
||||
run_id="run-no-factory",
|
||||
concurrency=1,
|
||||
max_steps=10,
|
||||
skill_mode="auto",
|
||||
plugins_factory=None,
|
||||
)
|
||||
|
||||
assert result.total == 1
|
||||
assert result.stop_reason_counts.get("finished") == 1
|
||||
|
||||
|
||||
class TestConcurrencyControl:
|
||||
"""并发控制 Semaphore 测试。"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrency_semaphore_limits(self, harness_log: HarnessLog) -> None:
|
||||
"""Semaphore(1) 限制并发为 1 — 通过最大并发计数器验证。"""
|
||||
import asyncio
|
||||
|
||||
llm = AsyncMock()
|
||||
current_concurrent = 0
|
||||
max_concurrent = 0
|
||||
|
||||
original_response = _make_llm_response(answer="B")
|
||||
|
||||
async def _slow_chat(
|
||||
messages: Any,
|
||||
*,
|
||||
session_id: str | None = None,
|
||||
parent_call_id: str | None = None,
|
||||
) -> LLMResponse:
|
||||
nonlocal current_concurrent, max_concurrent
|
||||
current_concurrent += 1
|
||||
max_concurrent = max(max_concurrent, current_concurrent)
|
||||
await asyncio.sleep(0.01)
|
||||
current_concurrent -= 1
|
||||
return original_response
|
||||
|
||||
llm.chat.side_effect = _slow_chat
|
||||
|
||||
questions = [_make_question(question_id=f"q{i}") for i in range(5)]
|
||||
|
||||
await run_inference(
|
||||
questions,
|
||||
llm=llm,
|
||||
tool_dispatch_fn=_stub_tool_dispatch,
|
||||
prompt_builder=_stub_prompt_builder,
|
||||
log=harness_log,
|
||||
run_id="run-sem",
|
||||
concurrency=1,
|
||||
max_steps=10,
|
||||
skill_mode="auto",
|
||||
)
|
||||
|
||||
assert max_concurrent == 1
|
||||
|
||||
|
||||
class TestTablesCreated:
|
||||
"""表创建测试。"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_five_tables_created(self, harness_log: HarnessLog) -> None:
|
||||
"""run_inference 启动时创建 5 张推理表。"""
|
||||
llm = AsyncMock()
|
||||
|
||||
await run_inference(
|
||||
[],
|
||||
llm=llm,
|
||||
tool_dispatch_fn=_stub_tool_dispatch,
|
||||
prompt_builder=_stub_prompt_builder,
|
||||
log=harness_log,
|
||||
run_id="run-tables",
|
||||
concurrency=1,
|
||||
max_steps=10,
|
||||
skill_mode="auto",
|
||||
)
|
||||
|
||||
expected_tables = [
|
||||
"predictions",
|
||||
"traces",
|
||||
"validation_flags",
|
||||
"anchor_check",
|
||||
"observe_frame_health",
|
||||
]
|
||||
for table_name in expected_tables:
|
||||
rows = harness_log.query(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?",
|
||||
(table_name,),
|
||||
)
|
||||
assert len(rows) == 1, f"表 {table_name} 未创建"
|
||||
@@ -0,0 +1,347 @@
|
||||
"""HarnessLog + RunLogImpl 单元测试。
|
||||
|
||||
HarnessLog: SQLite 薄包装的线程安全、幂等、context manager 语义验证。
|
||||
RunLogImpl: 只读 RunLog Protocol 实现，独立连接不污染 _runs 运行状态。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import threading
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from app.harness.log import HarnessLog, RunLogImpl
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db_path(tmp_path: Path) -> str:
|
||||
"""返回临时 SQLite 数据库路径。"""
|
||||
return str(tmp_path / "test.db")
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def run_id() -> str:
|
||||
return "test-run-001"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# HarnessLog 测试
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestHarnessLog:
|
||||
"""HarnessLog 核心功能测试。"""
|
||||
|
||||
def test_create_table_and_insert(self, db_path: str, run_id: str) -> None:
|
||||
"""create_table 建表 + insert 写入 + query 读回。"""
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("metrics", {"epoch": "INTEGER", "loss": "REAL"})
|
||||
log.insert("metrics", {"epoch": 1, "loss": 0.5})
|
||||
rows = log.query("SELECT * FROM metrics WHERE run_id = ?", (run_id,))
|
||||
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["epoch"] == 1
|
||||
assert rows[0]["loss"] == 0.5
|
||||
assert rows[0]["run_id"] == run_id
|
||||
# insert 自动填充 timestamp
|
||||
assert rows[0]["timestamp"] is not None
|
||||
|
||||
def test_query_thread_safety(self, db_path: str, run_id: str) -> None:
|
||||
"""并发读写不损坏游标状态。"""
|
||||
errors: list[Exception] = []
|
||||
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("counter", {"value": "INTEGER"})
|
||||
|
||||
def writer() -> None:
|
||||
try:
|
||||
for i in range(50):
|
||||
log.insert("counter", {"value": i})
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
def reader() -> None:
|
||||
try:
|
||||
for _ in range(50):
|
||||
log.query("SELECT COUNT(*) as cnt FROM counter")
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=writer),
|
||||
threading.Thread(target=reader),
|
||||
threading.Thread(target=reader),
|
||||
]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert errors == [], f"并发读写产生错误: {errors}"
|
||||
|
||||
def test_context_manager_completed(self, db_path: str, run_id: str) -> None:
|
||||
"""正常退出时 status = completed。"""
|
||||
with HarnessLog(db_path, run_id):
|
||||
pass
|
||||
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
row = conn.execute(
|
||||
"SELECT status, finished_at FROM _runs WHERE run_id = ?", (run_id,)
|
||||
).fetchone()
|
||||
conn.close()
|
||||
|
||||
assert row["status"] == "completed"
|
||||
assert row["finished_at"] is not None
|
||||
|
||||
def test_context_manager_failed(self, db_path: str, run_id: str) -> None:
|
||||
"""异常退出时 status = failed。"""
|
||||
with pytest.raises(ValueError, match="boom"), HarnessLog(db_path, run_id):
|
||||
raise ValueError("boom")
|
||||
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
row = conn.execute(
|
||||
"SELECT status FROM _runs WHERE run_id = ?", (run_id,)
|
||||
).fetchone()
|
||||
conn.close()
|
||||
|
||||
assert row["status"] == "failed"
|
||||
|
||||
def test_insert_or_ignore_idempotent(self, db_path: str, run_id: str) -> None:
|
||||
"""同一 run_id 多次创建 HarnessLog 不报错（INSERT OR IGNORE 幂等）。"""
|
||||
with HarnessLog(db_path, run_id):
|
||||
pass
|
||||
|
||||
# 再次用同一 run_id 打开——不应抛异常
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("t", {"x": "INTEGER"})
|
||||
log.insert("t", {"x": 42})
|
||||
|
||||
conn = sqlite3.connect(db_path)
|
||||
count = conn.execute(
|
||||
"SELECT COUNT(*) FROM _runs WHERE run_id = ?", (run_id,)
|
||||
).fetchone()[0]
|
||||
conn.close()
|
||||
|
||||
assert count == 1, "INSERT OR IGNORE 应保证 _runs 只有一行"
|
||||
|
||||
def test_wal_mode(self, db_path: str, run_id: str) -> None:
|
||||
"""连接初始化后 journal_mode 应为 WAL。"""
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
rows = log.query("PRAGMA journal_mode")
|
||||
|
||||
assert rows[0]["journal_mode"].lower() == "wal"
|
||||
|
||||
def test_insert_many(self, db_path: str, run_id: str) -> None:
|
||||
"""insert_many 批量插入多条记录。"""
|
||||
records = [{"epoch": i, "loss": float(i) * 0.1} for i in range(5)]
|
||||
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("batch", {"epoch": "INTEGER", "loss": "REAL"})
|
||||
log.insert_many("batch", records)
|
||||
rows = log.query(
|
||||
"SELECT * FROM batch WHERE run_id = ? ORDER BY epoch", (run_id,)
|
||||
)
|
||||
|
||||
assert len(rows) == 5
|
||||
assert [r["epoch"] for r in rows] == [0, 1, 2, 3, 4]
|
||||
|
||||
def test_log_event(self, db_path: str, run_id: str) -> None:
|
||||
"""log_event 向 _events 表写入事件。"""
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.log_event("train_start", {"epoch": 1, "lr": 0.001})
|
||||
log.log_event("train_end", {"epoch": 1, "loss": 0.42})
|
||||
rows = log.query(
|
||||
"SELECT * FROM _events WHERE run_id = ? ORDER BY id", (run_id,)
|
||||
)
|
||||
|
||||
assert len(rows) == 2
|
||||
assert rows[0]["event_type"] == "train_start"
|
||||
assert rows[1]["event_type"] == "train_end"
|
||||
|
||||
def test_execute_raw_sql(self, db_path: str, run_id: str) -> None:
|
||||
"""execute 执行原生 SQL 写操作。"""
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("raw", {"val": "INTEGER"})
|
||||
log.execute(
|
||||
"INSERT INTO raw (run_id, timestamp, val) VALUES (?, ?, ?)",
|
||||
(run_id, "2026-01-01T00:00:00", 99),
|
||||
)
|
||||
rows = log.query("SELECT val FROM raw WHERE run_id = ?", (run_id,))
|
||||
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["val"] == 99
|
||||
|
||||
def test_upsert_mode(self, db_path: str, run_id: str) -> None:
|
||||
"""insert mode='upsert' 使用 INSERT OR REPLACE。"""
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table("kv", {"key": "TEXT", "val": "TEXT"}, primary_key="key")
|
||||
log.insert("kv", {"key": "a", "val": "1"}, mode="upsert")
|
||||
log.insert("kv", {"key": "a", "val": "2"}, mode="upsert")
|
||||
rows = log.query("SELECT val FROM kv WHERE key = 'a'")
|
||||
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["val"] == "2"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# RunLogImpl 测试
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def _setup_predictions_and_traces(db_path: str, run_id: str) -> None:
|
||||
"""向测试数据库写入 predictions 和 traces 数据，模拟 inference 阶段产物。"""
|
||||
with HarnessLog(db_path, run_id) as log:
|
||||
log.create_table(
|
||||
"predictions",
|
||||
{
|
||||
"question_id": "TEXT",
|
||||
"predicted_answer": "TEXT",
|
||||
"correct": "INTEGER",
|
||||
},
|
||||
)
|
||||
log.create_table(
|
||||
"traces",
|
||||
{
|
||||
"question_id": "TEXT",
|
||||
"step_idx": "INTEGER",
|
||||
"action": "TEXT",
|
||||
"observation": "TEXT",
|
||||
},
|
||||
)
|
||||
log.insert_many(
|
||||
"predictions",
|
||||
[
|
||||
{"question_id": "q1", "predicted_answer": "A", "correct": 1},
|
||||
{"question_id": "q2", "predicted_answer": "B", "correct": 0},
|
||||
{"question_id": "q3", "predicted_answer": "C", "correct": 1},
|
||||
],
|
||||
)
|
||||
log.insert_many(
|
||||
"traces",
|
||||
[
|
||||
{
|
||||
"question_id": "q1",
|
||||
"step_idx": 0,
|
||||
"action": "search",
|
||||
"observation": "found",
|
||||
},
|
||||
{
|
||||
"question_id": "q1",
|
||||
"step_idx": 1,
|
||||
"action": "verify",
|
||||
"observation": "ok",
|
||||
},
|
||||
{
|
||||
"question_id": "q2",
|
||||
"step_idx": 0,
|
||||
"action": "search",
|
||||
"observation": "not found",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class TestRunLogImpl:
|
||||
"""RunLogImpl 只读查询端口测试。"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_predictions(self, db_path: str, run_id: str) -> None:
|
||||
"""get_predictions 返回指定 run 的全部预测记录。"""
|
||||
_setup_predictions_and_traces(db_path, run_id)
|
||||
impl = RunLogImpl(db_path)
|
||||
|
||||
preds = await impl.get_predictions(run_id)
|
||||
|
||||
assert len(preds) == 3
|
||||
q_ids = {p["question_id"] for p in preds}
|
||||
assert q_ids == {"q1", "q2", "q3"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_predictions_filtered(self, db_path: str, run_id: str) -> None:
|
||||
"""get_predictions 按 question_ids 过滤。"""
|
||||
_setup_predictions_and_traces(db_path, run_id)
|
||||
impl = RunLogImpl(db_path)
|
||||
|
||||
preds = await impl.get_predictions(run_id, question_ids=["q1", "q3"])
|
||||
|
||||
assert len(preds) == 2
|
||||
q_ids = {p["question_id"] for p in preds}
|
||||
assert q_ids == {"q1", "q3"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_traces(self, db_path: str, run_id: str) -> None:
|
||||
"""get_traces 返回指定 run 的全部轨迹记录。"""
|
||||
_setup_predictions_and_traces(db_path, run_id)
|
||||
impl = RunLogImpl(db_path)
|
||||
|
||||
traces = await impl.get_traces(run_id)
|
||||
|
||||
assert len(traces) == 3
|
||||
# q1 有 2 步，q2 有 1 步
|
||||
q1_traces = [t for t in traces if t["question_id"] == "q1"]
|
||||
assert len(q1_traces) == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_traces_filtered(self, db_path: str, run_id: str) -> None:
|
||||
"""get_traces 按 question_ids 过滤。"""
|
||||
_setup_predictions_and_traces(db_path, run_id)
|
||||
impl = RunLogImpl(db_path)
|
||||
|
||||
traces = await impl.get_traces(run_id, question_ids=["q2"])
|
||||
|
||||
assert len(traces) == 1
|
||||
assert traces[0]["question_id"] == "q2"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_readonly_no_runs_insert(self, db_path: str, run_id: str) -> None:
|
||||
"""RunLogImpl 查询不触发 _runs INSERT（不污染运行状态）。"""
|
||||
_setup_predictions_and_traces(db_path, run_id)
|
||||
|
||||
# 用不同 run_id 查询，确保不会创建新的 _runs 行
|
||||
other_run = "nonexistent-run"
|
||||
impl = RunLogImpl(db_path)
|
||||
preds = await impl.get_predictions(other_run)
|
||||
|
||||
assert preds == []
|
||||
|
||||
# 验证 _runs 表没有 other_run 的记录
|
||||
conn = sqlite3.connect(db_path)
|
||||
count = conn.execute(
|
||||
"SELECT COUNT(*) FROM _runs WHERE run_id = ?", (other_run,)
|
||||
).fetchone()[0]
|
||||
conn.close()
|
||||
assert count == 0, "RunLogImpl 不应向 _runs 插入记录"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_protocol_compliance(self, db_path: str, run_id: str) -> None:
|
||||
"""RunLogImpl 满足 RunLog Protocol（runtime_checkable isinstance 检查）。"""
|
||||
from core.evolution.protocols import RunLog
|
||||
|
||||
impl = RunLogImpl(db_path)
|
||||
assert isinstance(impl, RunLog)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_table_returns_empty(self, db_path: str, run_id: str) -> None:
|
||||
"""查询不存在的表（predictions/traces 未建）返回空列表。"""
|
||||
# 仅创建 _runs，不建 predictions/traces 表
|
||||
with HarnessLog(db_path, run_id):
|
||||
pass
|
||||
|
||||
impl = RunLogImpl(db_path)
|
||||
preds = await impl.get_predictions(run_id)
|
||||
traces = await impl.get_traces(run_id)
|
||||
|
||||
assert preds == []
|
||||
assert traces == []
|
||||
@@ -0,0 +1,261 @@
|
||||
"""慢更新动量生成（app/harness/momentum.py）单元测试。
|
||||
|
||||
覆盖:
|
||||
- _categorize_pair 四分类 + 缺键 KeyError
|
||||
- _format_comparison_pairs 分组排序 + 空列表
|
||||
- run_slow_momentum 正常解析 + 解析失败保留 prev_guidance
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.harness.momentum import (
|
||||
IMPROVED,
|
||||
PERSISTENT_FAIL,
|
||||
REGRESSED,
|
||||
STABLE_SUCCESS,
|
||||
_categorize_pair,
|
||||
_format_comparison_pairs,
|
||||
run_slow_momentum,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# _categorize_pair
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestCategorizePair:
|
||||
"""_categorize_pair 四分类测试。"""
|
||||
|
||||
def test_categorize_pair_all_four(self) -> None:
|
||||
"""四种 correct_prev/correct_curr 组合应返回对应类别常量。"""
|
||||
assert _categorize_pair({"correct_prev": False, "correct_curr": True}) == IMPROVED
|
||||
assert _categorize_pair({"correct_prev": True, "correct_curr": False}) == REGRESSED
|
||||
assert _categorize_pair({"correct_prev": False, "correct_curr": False}) == PERSISTENT_FAIL
|
||||
assert _categorize_pair({"correct_prev": True, "correct_curr": True}) == STABLE_SUCCESS
|
||||
|
||||
def test_categorize_pair_truthy_values(self) -> None:
|
||||
"""非布尔真值（int / str）也能正确分类。"""
|
||||
assert _categorize_pair({"correct_prev": 0, "correct_curr": 1}) == IMPROVED
|
||||
assert _categorize_pair({"correct_prev": "yes", "correct_curr": ""}) == REGRESSED
|
||||
|
||||
def test_categorize_pair_missing_key(self) -> None:
|
||||
"""缺 correct_prev 或 correct_curr 键时抛 KeyError。"""
|
||||
with pytest.raises(KeyError):
|
||||
_categorize_pair({"correct_prev": True})
|
||||
with pytest.raises(KeyError):
|
||||
_categorize_pair({"correct_curr": False})
|
||||
with pytest.raises(KeyError):
|
||||
_categorize_pair({})
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# _format_comparison_pairs
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestFormatComparisonPairs:
|
||||
"""_format_comparison_pairs 分组排序测试。"""
|
||||
|
||||
def test_format_comparison_pairs_empty(self) -> None:
|
||||
"""空列表返回占位说明。"""
|
||||
result = _format_comparison_pairs([])
|
||||
assert "无可用" in result
|
||||
|
||||
def test_format_comparison_pairs_order(self) -> None:
|
||||
"""REGRESSED 标题应出现在 IMPROVED 标题之前（伤害信号优先）。"""
|
||||
pairs = [
|
||||
{
|
||||
"question": "Q1",
|
||||
"prev_prediction": "A",
|
||||
"curr_prediction": "B",
|
||||
"correct_prev": False,
|
||||
"correct_curr": True,
|
||||
},
|
||||
{
|
||||
"question": "Q2",
|
||||
"prev_prediction": "C",
|
||||
"curr_prediction": "D",
|
||||
"correct_prev": True,
|
||||
"correct_curr": False,
|
||||
},
|
||||
]
|
||||
result = _format_comparison_pairs(pairs)
|
||||
regressed_pos = result.index("回退")
|
||||
improved_pos = result.index("改善")
|
||||
assert regressed_pos < improved_pos, "REGRESSED 应排在 IMPROVED 之前"
|
||||
|
||||
def test_format_comparison_pairs_all_categories(self) -> None:
|
||||
"""四种类别的 pair 都能被正确分组。"""
|
||||
pairs = [
|
||||
{"question": "Q1", "correct_prev": False, "correct_curr": True},
|
||||
{"question": "Q2", "correct_prev": True, "correct_curr": False},
|
||||
{"question": "Q3", "correct_prev": False, "correct_curr": False},
|
||||
{"question": "Q4", "correct_prev": True, "correct_curr": True},
|
||||
]
|
||||
result = _format_comparison_pairs(pairs)
|
||||
assert "固定样本总数：4" in result
|
||||
# 每个类别都应标注 1 题
|
||||
assert "1 题" in result
|
||||
|
||||
def test_format_comparison_pairs_missing_key(self) -> None:
|
||||
"""pair 缺键时 KeyError 不被吞。"""
|
||||
with pytest.raises(KeyError):
|
||||
_format_comparison_pairs([{"question": "Q1"}])
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# run_slow_momentum
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def _make_mock_llm(response_content: str) -> Any:
|
||||
"""构造一个返回指定 content 的 mock LLMProvider。"""
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _FakeResponse:
|
||||
content: str
|
||||
thinking: str = ""
|
||||
model: str = "mock"
|
||||
provider: str = "mock"
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
latency_ms: int = 0
|
||||
ttft_ms: float | None = None
|
||||
max_inter_token_ms: float | None = None
|
||||
cache_hit: bool = False
|
||||
call_id: str = "test-call-id"
|
||||
|
||||
mock_llm = AsyncMock()
|
||||
mock_llm.chat.return_value = _FakeResponse(content=response_content)
|
||||
return mock_llm
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_slow_momentum_basic(tmp_path: Path) -> None:
|
||||
"""正常解析时返回新的动量指导文本。"""
|
||||
# 准备 prompt 文件
|
||||
prompt_file = tmp_path / "slow_momentum.md"
|
||||
prompt_file.write_text("你是一个慢更新动量裁判。", encoding="utf-8")
|
||||
|
||||
new_guidance_text = "新一轮的动量指导内容"
|
||||
llm_response = json.dumps({"slow_update_content": new_guidance_text}, ensure_ascii=False)
|
||||
mock_llm = _make_mock_llm(llm_response)
|
||||
|
||||
result = await run_slow_momentum(
|
||||
llm=mock_llm,
|
||||
diagnose_prompts_dir=tmp_path,
|
||||
skill_content="当前 skill 正文",
|
||||
prev_skill="上一版 skill 正文",
|
||||
prev_guidance="旧的动量指导",
|
||||
comparison_pairs=[
|
||||
{
|
||||
"question": "Q1",
|
||||
"prev_prediction": "A",
|
||||
"curr_prediction": "B",
|
||||
"correct_prev": False,
|
||||
"correct_curr": True,
|
||||
}
|
||||
],
|
||||
)
|
||||
assert result == new_guidance_text
|
||||
mock_llm.chat.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_slow_momentum_parse_failure(tmp_path: Path) -> None:
|
||||
"""LLM 返回无法解析的内容时保留 prev_guidance。"""
|
||||
prompt_file = tmp_path / "slow_momentum.md"
|
||||
prompt_file.write_text("你是一个慢更新动量裁判。", encoding="utf-8")
|
||||
|
||||
mock_llm = _make_mock_llm("这不是 JSON，无法解析")
|
||||
prev_guidance = "应该被保留的旧动量指导"
|
||||
|
||||
result = await run_slow_momentum(
|
||||
llm=mock_llm,
|
||||
diagnose_prompts_dir=tmp_path,
|
||||
skill_content="当前 skill",
|
||||
prev_skill="上一版 skill",
|
||||
prev_guidance=prev_guidance,
|
||||
comparison_pairs=[
|
||||
{
|
||||
"question": "Q1",
|
||||
"prev_prediction": "A",
|
||||
"curr_prediction": "B",
|
||||
"correct_prev": True,
|
||||
"correct_curr": True,
|
||||
}
|
||||
],
|
||||
)
|
||||
assert result == prev_guidance
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_slow_momentum_missing_field(tmp_path: Path) -> None:
|
||||
"""LLM 返回合法 JSON 但缺少 slow_update_content 字段时保留 prev_guidance。"""
|
||||
prompt_file = tmp_path / "slow_momentum.md"
|
||||
prompt_file.write_text("你是一个慢更新动量裁判。", encoding="utf-8")
|
||||
|
||||
llm_response = json.dumps({"other_field": "无关内容"}, ensure_ascii=False)
|
||||
mock_llm = _make_mock_llm(llm_response)
|
||||
prev_guidance = "应该被保留的旧动量指导"
|
||||
|
||||
result = await run_slow_momentum(
|
||||
llm=mock_llm,
|
||||
diagnose_prompts_dir=tmp_path,
|
||||
skill_content="当前 skill",
|
||||
prev_skill="上一版 skill",
|
||||
prev_guidance=prev_guidance,
|
||||
comparison_pairs=[],
|
||||
)
|
||||
assert result == prev_guidance
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_slow_momentum_keyerror_not_swallowed(tmp_path: Path) -> None:
|
||||
"""comparison_pairs 缺键时 KeyError 不被 ValueError 吞掉。"""
|
||||
prompt_file = tmp_path / "slow_momentum.md"
|
||||
prompt_file.write_text("你是一个慢更新动量裁判。", encoding="utf-8")
|
||||
|
||||
mock_llm = _make_mock_llm('{"slow_update_content": "ok"}')
|
||||
|
||||
with pytest.raises(KeyError):
|
||||
await run_slow_momentum(
|
||||
llm=mock_llm,
|
||||
diagnose_prompts_dir=tmp_path,
|
||||
skill_content="当前 skill",
|
||||
prev_skill="上一版 skill",
|
||||
prev_guidance="旧指导",
|
||||
comparison_pairs=[{"question": "Q1"}], # 缺 correct_prev/correct_curr
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_slow_momentum_fenced_json(tmp_path: Path) -> None:
|
||||
"""LLM 返回 fenced code block 中的 JSON 也能正确解析。"""
|
||||
prompt_file = tmp_path / "slow_momentum.md"
|
||||
prompt_file.write_text("你是一个慢更新动量裁判。", encoding="utf-8")
|
||||
|
||||
new_guidance = "从 fenced block 中提取的指导"
|
||||
llm_response = f'```json\n{{"slow_update_content": "{new_guidance}"}}\n```'
|
||||
mock_llm = _make_mock_llm(llm_response)
|
||||
|
||||
result = await run_slow_momentum(
|
||||
llm=mock_llm,
|
||||
diagnose_prompts_dir=tmp_path,
|
||||
skill_content="当前 skill",
|
||||
prev_skill="上一版 skill",
|
||||
prev_guidance="旧指导",
|
||||
comparison_pairs=[],
|
||||
)
|
||||
assert result == new_guidance
|
||||
@@ -0,0 +1,354 @@
|
||||
"""五张观测表 + step/epoch 报告的单元测试。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from app.harness.observation import (
|
||||
read_dual_metric,
|
||||
read_gate_evidence,
|
||||
read_holdout_eval,
|
||||
read_quadrant_pairs,
|
||||
read_shadow_gate,
|
||||
write_dual_metric,
|
||||
write_epoch_report,
|
||||
write_gate_evidence,
|
||||
write_holdout_eval,
|
||||
write_quadrant_pairs,
|
||||
write_shadow_gate,
|
||||
write_step_report,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db_path(tmp_path: Path) -> str:
|
||||
"""返回临时 SQLite 路径。"""
|
||||
return str(tmp_path / "test_obs.db")
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def run_id() -> str:
|
||||
return "run-obs-001"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# dual_metric_eval
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_write_read_dual_metric(db_path: str, run_id: str) -> None:
|
||||
"""写入 dual_metric_eval 后回读应与输入一致。"""
|
||||
write_dual_metric(
|
||||
db_path,
|
||||
run_id=run_id,
|
||||
epoch=1,
|
||||
version_kind="baseline",
|
||||
skills_version="v0",
|
||||
prompts_version="v0",
|
||||
pool="val",
|
||||
hard_acc=0.75,
|
||||
soft_score=0.80,
|
||||
mixed_score=0.775,
|
||||
)
|
||||
rows = read_dual_metric(db_path, run_id=run_id)
|
||||
assert len(rows) == 1
|
||||
row = rows[0]
|
||||
assert row["epoch"] == 1
|
||||
assert row["version_kind"] == "baseline"
|
||||
assert row["skills_version"] == "v0"
|
||||
assert row["prompts_version"] == "v0"
|
||||
assert row["pool"] == "val"
|
||||
assert row["hard_acc"] == pytest.approx(0.75)
|
||||
assert row["soft_score"] == pytest.approx(0.80)
|
||||
assert row["mixed_score"] == pytest.approx(0.775)
|
||||
assert row["run_id"] == run_id
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# shadow_gate
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_write_read_shadow_gate(db_path: str, run_id: str) -> None:
|
||||
"""写入 shadow_gate 后回读应与输入一致，is_mixed_best 布尔转 int。"""
|
||||
write_shadow_gate(
|
||||
db_path,
|
||||
run_id=run_id,
|
||||
epoch=2,
|
||||
candidate_version="skills/v1+prompts/v1",
|
||||
hard_acc=0.82,
|
||||
soft_score=0.78,
|
||||
mixed_score=0.80,
|
||||
is_mixed_best=True,
|
||||
)
|
||||
rows = read_shadow_gate(db_path, run_id=run_id)
|
||||
assert len(rows) == 1
|
||||
row = rows[0]
|
||||
assert row["epoch"] == 2
|
||||
assert row["candidate_version"] == "skills/v1+prompts/v1"
|
||||
assert row["hard_acc"] == pytest.approx(0.82)
|
||||
assert row["is_mixed_best"] == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# holdout_eval
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_write_read_holdout_eval(db_path: str, run_id: str) -> None:
|
||||
"""写入 holdout_eval 后回读应与输入一致。"""
|
||||
per_task = json.dumps({"temporal": {"accuracy": 0.9, "total": 10, "correct": 9}})
|
||||
write_holdout_eval(
|
||||
db_path,
|
||||
run_id=run_id,
|
||||
epoch=1,
|
||||
version_kind="best_hard",
|
||||
hard_acc=0.85,
|
||||
soft_score=0.70,
|
||||
mixed_score=0.775,
|
||||
per_task_type_json=per_task,
|
||||
)
|
||||
rows = read_holdout_eval(db_path, run_id=run_id)
|
||||
assert len(rows) == 1
|
||||
row = rows[0]
|
||||
assert row["version_kind"] == "best_hard"
|
||||
assert row["hard_acc"] == pytest.approx(0.85)
|
||||
parsed = json.loads(row["per_task_type_json"])
|
||||
assert parsed["temporal"]["correct"] == 9
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# gate_evidence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_write_read_gate_evidence(db_path: str, run_id: str) -> None:
|
||||
"""写入 gate_evidence 后回读应与输入一致，逐行插入。"""
|
||||
evidence_rows = [
|
||||
{
|
||||
"task_type": "temporal",
|
||||
"question_id": "q1",
|
||||
"block_idx": 0,
|
||||
"baseline_correct": 1,
|
||||
"candidate_correct": 1,
|
||||
"e_value": 1.0,
|
||||
"stop_reason": "",
|
||||
},
|
||||
{
|
||||
"task_type": "temporal",
|
||||
"question_id": "q2",
|
||||
"block_idx": 0,
|
||||
"baseline_correct": 0,
|
||||
"candidate_correct": 1,
|
||||
"e_value": 2.0,
|
||||
"stop_reason": "confirmed",
|
||||
},
|
||||
]
|
||||
write_gate_evidence(db_path, run_id=run_id, epoch=1, step=0, rows=evidence_rows)
|
||||
rows = read_gate_evidence(db_path, run_id=run_id)
|
||||
assert len(rows) == 2
|
||||
assert rows[0]["question_id"] == "q1"
|
||||
assert rows[1]["stop_reason"] == "confirmed"
|
||||
assert rows[1]["e_value"] == pytest.approx(2.0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# quadrant_pair
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_write_read_quadrant_pairs(db_path: str, run_id: str) -> None:
|
||||
"""写入 quadrant_pair 后回读，bool -> int 转换正确。"""
|
||||
pairs = [
|
||||
{
|
||||
"question_id": "q1",
|
||||
"task_type": "causal",
|
||||
"prev_correct": True,
|
||||
"curr_correct": False,
|
||||
"category": "regression",
|
||||
},
|
||||
{
|
||||
"question_id": "q2",
|
||||
"task_type": "causal",
|
||||
"prev_correct": False,
|
||||
"curr_correct": True,
|
||||
"category": "improvement",
|
||||
},
|
||||
]
|
||||
write_quadrant_pairs(db_path, run_id=run_id, epoch=1, step=0, pairs=pairs)
|
||||
rows = read_quadrant_pairs(db_path, run_id=run_id)
|
||||
assert len(rows) == 2
|
||||
# bool -> int 转换
|
||||
assert rows[0]["prev_correct"] == 1
|
||||
assert rows[0]["curr_correct"] == 0
|
||||
assert rows[1]["prev_correct"] == 0
|
||||
assert rows[1]["curr_correct"] == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NULL vs 0 语义
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_null_not_zero_for_soft(db_path: str, run_id: str) -> None:
|
||||
"""soft_score/mixed_score 为 None 时存 NULL（非 0），回读也是 None。"""
|
||||
write_dual_metric(
|
||||
db_path,
|
||||
run_id=run_id,
|
||||
epoch=1,
|
||||
version_kind="baseline",
|
||||
skills_version="v0",
|
||||
prompts_version="v0",
|
||||
pool="val",
|
||||
hard_acc=0.75,
|
||||
soft_score=None,
|
||||
mixed_score=None,
|
||||
)
|
||||
rows = read_dual_metric(db_path, run_id=run_id)
|
||||
assert len(rows) == 1
|
||||
row = rows[0]
|
||||
assert row["soft_score"] is None
|
||||
assert row["mixed_score"] is None
|
||||
|
||||
# 用原生 SQL 确认存的是 NULL 而非 0
|
||||
conn = sqlite3.connect(db_path)
|
||||
cursor = conn.execute(
|
||||
"SELECT soft_score, mixed_score FROM dual_metric_eval WHERE run_id=?",
|
||||
(run_id,),
|
||||
)
|
||||
raw = cursor.fetchone()
|
||||
conn.close()
|
||||
assert raw[0] is None
|
||||
assert raw[1] is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 只读连接隔离
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_read_only_connection(db_path: str, run_id: str) -> None:
|
||||
"""read_* 使用独立只读连接，不向 _runs 表插入新行。"""
|
||||
write_dual_metric(
|
||||
db_path,
|
||||
run_id=run_id,
|
||||
epoch=1,
|
||||
version_kind="baseline",
|
||||
skills_version="v0",
|
||||
prompts_version="v0",
|
||||
pool="val",
|
||||
hard_acc=0.5,
|
||||
soft_score=None,
|
||||
mixed_score=None,
|
||||
)
|
||||
|
||||
# 用另一个 run_id 回读——不应在 _runs 表中创建新行
|
||||
other_run = "run-obs-ghost"
|
||||
rows = read_dual_metric(db_path, run_id=other_run)
|
||||
assert rows == []
|
||||
|
||||
conn = sqlite3.connect(db_path)
|
||||
cursor = conn.execute("SELECT run_id FROM _runs")
|
||||
run_ids = [r[0] for r in cursor.fetchall()]
|
||||
conn.close()
|
||||
assert other_run not in run_ids
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# step_report
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_write_step_report(tmp_path: Path) -> None:
|
||||
"""step_report 写入 JSON 文件，内容字段完整。"""
|
||||
workspace = tmp_path / "ws"
|
||||
workspace.mkdir()
|
||||
path = write_step_report(
|
||||
workspace_dir=workspace,
|
||||
epoch=1,
|
||||
step=2,
|
||||
global_step=12,
|
||||
task_type="Temporal Order",
|
||||
gate_action="accept_confirmed",
|
||||
candidate_acc=0.85,
|
||||
class_baseline_acc=0.70,
|
||||
edit_budget=5,
|
||||
rank_clip_triggered=False,
|
||||
gate_w=3,
|
||||
gate_l=1,
|
||||
gate_e_value=4.2,
|
||||
gate_n_used=8,
|
||||
gate_stop_reason="confirmed",
|
||||
)
|
||||
assert path.exists()
|
||||
assert path.name == "step_report_e1_s2_temporal-order.json"
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
assert data["epoch"] == 1
|
||||
assert data["step"] == 2
|
||||
assert data["global_step"] == 12
|
||||
assert data["task_type"] == "Temporal Order"
|
||||
assert data["gate_action"] == "accept_confirmed"
|
||||
assert data["gate_w"] == 3
|
||||
assert data["gate_stop_reason"] == "confirmed"
|
||||
assert data["rank_clip_triggered"] is False
|
||||
|
||||
|
||||
def test_write_step_report_skipped_null_fields(tmp_path: Path) -> None:
|
||||
"""skipped/cooldown 路径的 gate 字段应为 null。"""
|
||||
workspace = tmp_path / "ws"
|
||||
workspace.mkdir()
|
||||
path = write_step_report(
|
||||
workspace_dir=workspace,
|
||||
epoch=1,
|
||||
step=0,
|
||||
global_step=0,
|
||||
task_type="causal",
|
||||
gate_action="skipped",
|
||||
candidate_acc=0.0,
|
||||
class_baseline_acc=0.0,
|
||||
edit_budget=10,
|
||||
rank_clip_triggered=False,
|
||||
gate_w=None,
|
||||
gate_l=None,
|
||||
gate_e_value=None,
|
||||
gate_n_used=None,
|
||||
gate_stop_reason=None,
|
||||
)
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
assert data["gate_w"] is None
|
||||
assert data["gate_l"] is None
|
||||
assert data["gate_e_value"] is None
|
||||
assert data["gate_n_used"] is None
|
||||
assert data["gate_stop_reason"] is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# epoch_report
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_write_epoch_report(tmp_path: Path) -> None:
|
||||
"""epoch_report 写入 JSON 文件，内容字段完整。"""
|
||||
workspace = tmp_path / "ws"
|
||||
workspace.mkdir()
|
||||
path = write_epoch_report(
|
||||
workspace_dir=workspace,
|
||||
epoch=3,
|
||||
system_tool_action="updated",
|
||||
momentum_updated_task_types=["temporal", "causal"],
|
||||
best_val_acc=0.88,
|
||||
)
|
||||
assert path.exists()
|
||||
assert path.name == "epoch_report_3.json"
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
assert data["epoch"] == 3
|
||||
assert data["system_tool_action"] == "updated"
|
||||
assert data["momentum_updated_task_types"] == ["temporal", "causal"]
|
||||
assert data["best_val_acc"] == pytest.approx(0.88)
|
||||
@@ -0,0 +1,290 @@
|
||||
"""三池切分单元测试。
|
||||
|
||||
验证:
|
||||
- 三池互斥（question_id 无重叠）
|
||||
- test 池自然分布（correct_ratio=None）
|
||||
- save/load 往返一致
|
||||
- 旧格式拒绝（无 test 键 → ValueError）
|
||||
- build_or_load_pools 冻结复用（pools.json 存在时不重切）
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from app.harness.pools import (
|
||||
build_pools,
|
||||
load_pools,
|
||||
save_pools,
|
||||
)
|
||||
from core.types import GeneratedQuestion
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _make_question(qid: str, task_type: str = "Action Reasoning") -> GeneratedQuestion:
|
||||
"""构造测试用 GeneratedQuestion。
|
||||
|
||||
参数:
|
||||
qid: 题目 ID。
|
||||
task_type: 题型。
|
||||
|
||||
返回:
|
||||
GeneratedQuestion 实例。
|
||||
"""
|
||||
return GeneratedQuestion(
|
||||
question_id=qid,
|
||||
video_id="video_001",
|
||||
task_type=task_type,
|
||||
question=f"Question {qid}?",
|
||||
options=("A. opt1", "B. opt2", "C. opt3", "D. opt4"),
|
||||
answer="A",
|
||||
source_nodes=("node_1",),
|
||||
difficulty="medium",
|
||||
)
|
||||
|
||||
|
||||
def _make_question_set(
|
||||
n: int,
|
||||
task_types: list[str] | None = None,
|
||||
) -> list[GeneratedQuestion]:
|
||||
"""构造 n 道题，交替分配题型。
|
||||
|
||||
参数:
|
||||
n: 题目数量。
|
||||
task_types: 可选题型列表，轮转分配；None 默认 2 类。
|
||||
|
||||
返回:
|
||||
题目列表。
|
||||
"""
|
||||
types = task_types or ["Action Reasoning", "Scene Understanding"]
|
||||
return [_make_question(f"q_{i:04d}", types[i % len(types)]) for i in range(n)]
|
||||
|
||||
|
||||
def _make_correctness(
|
||||
questions: list[GeneratedQuestion],
|
||||
correct_ratio: float = 0.5,
|
||||
) -> dict[str, bool]:
|
||||
"""构造 correctness 字典，前 correct_ratio 比例标对。
|
||||
|
||||
参数:
|
||||
questions: 题目列表。
|
||||
correct_ratio: 对题占比。
|
||||
|
||||
返回:
|
||||
question_id -> bool。
|
||||
"""
|
||||
n_correct = round(len(questions) * correct_ratio)
|
||||
return {q.question_id: (i < n_correct) for i, q in enumerate(questions)}
|
||||
|
||||
|
||||
class TestBuildPoolsMutualExclusion:
|
||||
"""三池 question_id 互斥验证。"""
|
||||
|
||||
def test_build_pools_mutual_exclusion(self) -> None:
|
||||
"""三池切分后，任意两池不共享 question_id。"""
|
||||
questions = _make_question_set(200)
|
||||
correctness = _make_correctness(questions, 0.5)
|
||||
|
||||
pools = build_pools(
|
||||
questions,
|
||||
correctness,
|
||||
diag_cfg={
|
||||
"size": 30,
|
||||
"correct_ratio": 0.5,
|
||||
"task_types": None,
|
||||
"seed": 42,
|
||||
"min_per_class": None,
|
||||
},
|
||||
val_cfg={
|
||||
"size": 30,
|
||||
"correct_ratio": 0.5,
|
||||
"task_types": None,
|
||||
"seed": 42,
|
||||
"min_per_class": None,
|
||||
},
|
||||
test_cfg={"size": 30},
|
||||
baseline_run_id="run_baseline",
|
||||
)
|
||||
|
||||
diag_ids = {q.question_id for q in pools.diagnosis}
|
||||
val_ids = {q.question_id for q in pools.validation}
|
||||
test_ids = {q.question_id for q in pools.test}
|
||||
|
||||
assert diag_ids & val_ids == set(), "诊断池与验证池有重叠"
|
||||
assert diag_ids & test_ids == set(), "诊断池与测试池有重叠"
|
||||
assert val_ids & test_ids == set(), "验证池与测试池有重叠"
|
||||
|
||||
assert len(diag_ids) == 30
|
||||
assert len(val_ids) == 30
|
||||
assert len(test_ids) == 30
|
||||
|
||||
|
||||
class TestBuildPoolsTestNaturalDistribution:
|
||||
"""test 池使用自然分布（correct_ratio=None）。"""
|
||||
|
||||
def test_build_pools_test_natural_distribution(self) -> None:
|
||||
"""test 池不强制对错比例，保留候选池的自然分布。
|
||||
|
||||
构造 correctness 为 50% 对/50% 错，diag/val 用 correct_ratio=0.3
|
||||
强制裁剪，test 池走自然分布（correct_ratio=None）。验证 test 池
|
||||
不受 correct_ratio 约束。
|
||||
"""
|
||||
questions = _make_question_set(300)
|
||||
correctness = _make_correctness(questions, 0.5)
|
||||
|
||||
pools = build_pools(
|
||||
questions,
|
||||
correctness,
|
||||
diag_cfg={
|
||||
"size": 20,
|
||||
"correct_ratio": 0.3,
|
||||
"task_types": None,
|
||||
"seed": 42,
|
||||
"min_per_class": None,
|
||||
},
|
||||
val_cfg={
|
||||
"size": 20,
|
||||
"correct_ratio": 0.3,
|
||||
"task_types": None,
|
||||
"seed": 42,
|
||||
"min_per_class": None,
|
||||
},
|
||||
test_cfg={"size": 20},
|
||||
baseline_run_id="run_baseline",
|
||||
)
|
||||
|
||||
# diag/val 被 correct_ratio=0.3 裁剪：round(20*0.3) = 6 对, 14 错
|
||||
diag_correct = sum(1 for q in pools.diagnosis if correctness[q.question_id])
|
||||
val_correct = sum(1 for q in pools.validation if correctness[q.question_id])
|
||||
assert diag_correct == 6, "诊断池应强制 30% 对题"
|
||||
assert val_correct == 6, "验证池应强制 30% 对题"
|
||||
|
||||
# test 池自然分布：不受 correct_ratio 约束
|
||||
assert len(pools.test) == 20
|
||||
|
||||
|
||||
class TestSaveLoadPoolsRoundtrip:
|
||||
"""save/load 往返一致验证。"""
|
||||
|
||||
def test_save_load_pools_roundtrip(self, tmp_path: Path) -> None:
|
||||
"""save_pools → load_pools 后全字段一致。"""
|
||||
questions = _make_question_set(100)
|
||||
correctness = _make_correctness(questions, 0.5)
|
||||
|
||||
original = build_pools(
|
||||
questions,
|
||||
correctness,
|
||||
diag_cfg={
|
||||
"size": 15,
|
||||
"correct_ratio": 0.5,
|
||||
"task_types": None,
|
||||
"seed": 42,
|
||||
"min_per_class": None,
|
||||
},
|
||||
val_cfg={
|
||||
"size": 15,
|
||||
"correct_ratio": 0.5,
|
||||
"task_types": None,
|
||||
"seed": 42,
|
||||
"min_per_class": None,
|
||||
},
|
||||
test_cfg={"size": 15},
|
||||
baseline_run_id="run_001",
|
||||
)
|
||||
|
||||
pools_path = tmp_path / "pools.json"
|
||||
save_pools(original, pools_path)
|
||||
restored = load_pools(pools_path)
|
||||
|
||||
# 标量字段
|
||||
assert restored.baseline_run_id == original.baseline_run_id
|
||||
assert restored.baseline_val_accuracy == pytest.approx(original.baseline_val_accuracy)
|
||||
assert restored.correctness == original.correctness
|
||||
|
||||
# 三池逐题比对
|
||||
for pool_name in ("diagnosis", "validation", "test"):
|
||||
orig_list = getattr(original, pool_name)
|
||||
rest_list = getattr(restored, pool_name)
|
||||
assert len(rest_list) == len(orig_list), f"{pool_name} 长度不一致"
|
||||
for o, r in zip(orig_list, rest_list, strict=False):
|
||||
assert o.question_id == r.question_id
|
||||
assert o.video_id == r.video_id
|
||||
assert o.task_type == r.task_type
|
||||
assert o.question == r.question
|
||||
assert o.options == r.options
|
||||
assert o.answer == r.answer
|
||||
assert o.source_nodes == r.source_nodes
|
||||
assert o.difficulty == r.difficulty
|
||||
|
||||
|
||||
class TestLoadPoolsOldFormatReject:
|
||||
"""旧格式 pools.json（无 test 键）→ ValueError。"""
|
||||
|
||||
def test_load_pools_old_format_reject(self, tmp_path: Path) -> None:
|
||||
"""缺少 test 键的 pools.json 必须抛出 ValueError。"""
|
||||
old_format = {
|
||||
"baseline_run_id": "run_old",
|
||||
"baseline_val_accuracy": 0.5,
|
||||
"correctness": {},
|
||||
"diagnosis": [],
|
||||
"validation": [],
|
||||
}
|
||||
pools_path = tmp_path / "pools.json"
|
||||
pools_path.write_text(json.dumps(old_format), encoding="utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match="旧格式"):
|
||||
load_pools(pools_path)
|
||||
|
||||
|
||||
class TestBuildOrLoadPoolsFrozen:
|
||||
"""build_or_load_pools 冻结复用：pools.json 存在时原样加载不重切。"""
|
||||
|
||||
def test_build_or_load_pools_frozen(self, tmp_path: Path) -> None:
|
||||
"""pools.json 已存在时，build_or_load_pools 返回冻结内容。"""
|
||||
questions = _make_question_set(60)
|
||||
correctness = _make_correctness(questions, 0.5)
|
||||
|
||||
frozen = build_pools(
|
||||
questions,
|
||||
correctness,
|
||||
diag_cfg={
|
||||
"size": 10,
|
||||
"correct_ratio": 0.5,
|
||||
"task_types": None,
|
||||
"seed": 42,
|
||||
"min_per_class": None,
|
||||
},
|
||||
val_cfg={
|
||||
"size": 10,
|
||||
"correct_ratio": 0.5,
|
||||
"task_types": None,
|
||||
"seed": 42,
|
||||
"min_per_class": None,
|
||||
},
|
||||
test_cfg={"size": 10},
|
||||
baseline_run_id="run_frozen",
|
||||
)
|
||||
|
||||
pools_path = tmp_path / "pools.json"
|
||||
save_pools(frozen, pools_path)
|
||||
|
||||
# build_or_load_pools 中 pools.json 存在 → 直接 load_pools
|
||||
# 此处直接测试 load_pools 行为等价
|
||||
loaded = load_pools(pools_path)
|
||||
|
||||
assert loaded.baseline_run_id == frozen.baseline_run_id
|
||||
assert loaded.baseline_val_accuracy == pytest.approx(frozen.baseline_val_accuracy)
|
||||
assert len(loaded.test) == len(frozen.test)
|
||||
assert len(loaded.validation) == len(frozen.validation)
|
||||
assert len(loaded.diagnosis) == len(frozen.diagnosis)
|
||||
|
||||
# question_id 完全一致
|
||||
for pool_name in ("diagnosis", "validation", "test"):
|
||||
orig_ids = [q.question_id for q in getattr(frozen, pool_name)]
|
||||
load_ids = [q.question_id for q in getattr(loaded, pool_name)]
|
||||
assert orig_ids == load_ids, f"{pool_name} 冻结后 ID 顺序不一致"
|
||||
@@ -0,0 +1,682 @@
|
||||
"""runner.py 单元测试（算法保真 #13）。
|
||||
|
||||
覆盖 13a-13e 五个子任务，测试 Runner 骨架、三级嵌套、gate/accept/reject/probation、
|
||||
慢更新十步序、deliver_best + early stop。大部分测试用纯函数或 mock 构造避免真实推理。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path # noqa: TC003 — 运行时 tmp_path 标注使用
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.harness.runner import (
|
||||
_apply_batch_correctness,
|
||||
_batch_from_ids,
|
||||
_build_comparison_pairs,
|
||||
_compute_total_steps,
|
||||
_fallback_summary,
|
||||
_format_applied_edits,
|
||||
_guard_infra_failures,
|
||||
_outcome_to_quadrant_pairs,
|
||||
_should_early_stop,
|
||||
_snapshot_current_skills,
|
||||
_TrainState,
|
||||
_write_skip_report,
|
||||
resume_plan,
|
||||
)
|
||||
from app.harness.validate import Probation, ValidationOutcome
|
||||
from core.evolution import RejectedEdit
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 测试辅助
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _FakeInferenceResult:
|
||||
"""InferenceResult 替身。"""
|
||||
|
||||
run_id: str = "test_run"
|
||||
accuracy: float = 0.5
|
||||
total: int = 10
|
||||
correct: int = 5
|
||||
per_task_type: dict = field(default_factory=dict)
|
||||
steps_mean: float = 3.0
|
||||
token_usage: dict = field(default_factory=lambda: {"prompt_tokens": 0, "completion_tokens": 0})
|
||||
stop_reason_counts: dict = field(default_factory=lambda: {"finished": 10})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _FakeQuestion:
|
||||
"""GeneratedQuestion 替身。"""
|
||||
|
||||
question_id: str
|
||||
video_id: str = "v1"
|
||||
task_type: str = "Action Reasoning"
|
||||
question: str = "问题"
|
||||
options: tuple = ("A", "B", "C", "D")
|
||||
answer: str = "A"
|
||||
source_nodes: tuple = ()
|
||||
difficulty: str = "medium"
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakePools:
|
||||
"""Pools 替身。"""
|
||||
|
||||
diagnosis: list = field(default_factory=list)
|
||||
validation: list = field(default_factory=list)
|
||||
test: list = field(default_factory=list)
|
||||
baseline_run_id: str = "baseline_run"
|
||||
baseline_val_accuracy: float = 0.5
|
||||
correctness: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 13a: Runner 骨架 + 纯函数
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestResumePlan:
|
||||
"""resume_plan 纯函数。"""
|
||||
|
||||
def test_epoch_done_advances_epoch(self) -> None:
|
||||
"""epoch_done 阶段：下一 epoch 从头开始。"""
|
||||
plan = resume_plan(epoch=3, phase="epoch_done", step_completed=5)
|
||||
assert plan["first_epoch"] == 4
|
||||
assert plan["resume_epoch"] is None
|
||||
assert plan["resume_step_from"] == 0
|
||||
|
||||
def test_in_epoch_resumes_same_epoch(self) -> None:
|
||||
"""in_epoch 阶段：从同 epoch 的下一个 step 续跑。"""
|
||||
plan = resume_plan(epoch=2, phase="in_epoch", step_completed=3)
|
||||
assert plan["first_epoch"] == 2
|
||||
assert plan["resume_epoch"] == 2
|
||||
assert plan["resume_step_from"] == 4
|
||||
|
||||
def test_in_epoch_step_zero(self) -> None:
|
||||
"""in_epoch step_completed=0：从 step 1 续跑。"""
|
||||
plan = resume_plan(epoch=1, phase="in_epoch", step_completed=0)
|
||||
assert plan["resume_step_from"] == 1
|
||||
|
||||
|
||||
class TestGuardInfraFailures:
|
||||
"""_guard_infra_failures 基础设施护栏。"""
|
||||
|
||||
def test_low_error_rate_passes(self) -> None:
|
||||
"""error 率 <= 10% 不抛异常。"""
|
||||
result = _FakeInferenceResult(
|
||||
total=100,
|
||||
stop_reason_counts={"finished": 95, "error": 5},
|
||||
)
|
||||
_guard_infra_failures(result, context="test") # 不应抛异常
|
||||
|
||||
def test_high_error_rate_raises(self) -> None:
|
||||
"""error 率 > 10% 抛 RuntimeError。"""
|
||||
result = _FakeInferenceResult(
|
||||
total=10,
|
||||
stop_reason_counts={"finished": 8, "error": 2},
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="基础设施失败率过高"):
|
||||
_guard_infra_failures(result, context="test")
|
||||
|
||||
def test_zero_total_does_not_crash(self) -> None:
|
||||
"""total=0 时不除零崩溃。"""
|
||||
result = _FakeInferenceResult(total=0, stop_reason_counts={})
|
||||
_guard_infra_failures(result, context="test") # 不应抛异常
|
||||
|
||||
def test_no_error_key_passes(self) -> None:
|
||||
"""stop_reason_counts 无 error 键时正常通过。"""
|
||||
result = _FakeInferenceResult(
|
||||
total=10,
|
||||
stop_reason_counts={"finished": 10},
|
||||
)
|
||||
_guard_infra_failures(result, context="test")
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 13b: _apply_batch_correctness + _compute_total_steps
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestApplyBatchCorrectness:
|
||||
"""_apply_batch_correctness rollout 完整性护栏。"""
|
||||
|
||||
def test_complete_batch_updates_correctness(self) -> None:
|
||||
"""完整 rollout 正常更新 correctness。"""
|
||||
batch = [_FakeQuestion(question_id="q1"), _FakeQuestion(question_id="q2")]
|
||||
correctness: dict[str, bool] = {}
|
||||
|
||||
# mock HarnessLog
|
||||
mock_log = MagicMock()
|
||||
mock_log.query.return_value = [
|
||||
{"question_id": "q1", "prediction": "A", "answer": "A", "steps_json": "[]"},
|
||||
{"question_id": "q2", "prediction": "B", "answer": "A", "steps_json": "[]"},
|
||||
]
|
||||
|
||||
with patch("app.harness.validate._load_run_rows") as mock_load:
|
||||
mock_load.return_value = {
|
||||
"q1": {"prediction": "A", "answer": "A", "_correct": True, "steps": []},
|
||||
"q2": {"prediction": "B", "answer": "A", "_correct": False, "steps": []},
|
||||
}
|
||||
_apply_batch_correctness(correctness, mock_log, "run_1", batch)
|
||||
|
||||
assert correctness["q1"] is True
|
||||
assert correctness["q2"] is False
|
||||
|
||||
def test_missing_prediction_raises(self) -> None:
|
||||
"""rollout 缺预测行时抛 RuntimeError。"""
|
||||
batch = [_FakeQuestion(question_id="q1"), _FakeQuestion(question_id="q2")]
|
||||
correctness: dict[str, bool] = {}
|
||||
|
||||
mock_log = MagicMock()
|
||||
with patch("app.harness.validate._load_run_rows") as mock_load:
|
||||
mock_load.return_value = {
|
||||
"q1": {"prediction": "A", "answer": "A", "_correct": True, "steps": []},
|
||||
# q2 缺失
|
||||
}
|
||||
with pytest.raises(RuntimeError, match="rollout 不完整"):
|
||||
_apply_batch_correctness(correctness, mock_log, "run_1", batch)
|
||||
|
||||
|
||||
class TestComputeTotalSteps:
|
||||
"""_compute_total_steps 退火地平线。"""
|
||||
|
||||
def test_basic_calculation(self) -> None:
|
||||
"""基本退火地平线计算。"""
|
||||
questions = [
|
||||
_FakeQuestion(question_id=f"q{i}", task_type="Action Reasoning") for i in range(20)
|
||||
]
|
||||
# 全错题
|
||||
correctness = {q.question_id: False for q in questions}
|
||||
|
||||
config = MagicMock()
|
||||
config.batch_size = 5
|
||||
config.min_class_per_batch = 1
|
||||
config.batch_correct_ratio = 0.0
|
||||
config.epochs = 3
|
||||
|
||||
pools = _FakePools(diagnosis=questions, correctness=correctness)
|
||||
total = _compute_total_steps(pools, correctness, config)
|
||||
# 20 题 / batch_size 5 = 4 步/epoch * 3 epochs = 12
|
||||
assert total == 12
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 13b: _batch_from_ids
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestBatchFromIds:
|
||||
"""_batch_from_ids 按 ID 重建 batch。"""
|
||||
|
||||
def test_preserves_order(self) -> None:
|
||||
"""按 ids 顺序取出，保持原 batch 划分。"""
|
||||
q1 = _FakeQuestion(question_id="q1")
|
||||
q2 = _FakeQuestion(question_id="q2")
|
||||
q3 = _FakeQuestion(question_id="q3")
|
||||
pools = _FakePools(diagnosis=[q1, q2, q3])
|
||||
|
||||
batch = _batch_from_ids(pools, ["q3", "q1"])
|
||||
assert [q.question_id for q in batch] == ["q3", "q1"]
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 13b: _snapshot_current_skills
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestSnapshotCurrentSkills:
|
||||
"""_snapshot_current_skills 快照 skill 文件。"""
|
||||
|
||||
def test_snapshots_md_files(self, tmp_path: Path) -> None:
|
||||
"""只快照 .md 文件。"""
|
||||
(tmp_path / "action-reasoning.md").write_text("skill content 1")
|
||||
(tmp_path / "temporal.md").write_text("skill content 2")
|
||||
(tmp_path / "meta.json").write_text("{}")
|
||||
|
||||
snapshot = _snapshot_current_skills(tmp_path)
|
||||
assert "action-reasoning.md" in snapshot
|
||||
assert "temporal.md" in snapshot
|
||||
assert "meta.json" not in snapshot
|
||||
assert snapshot["action-reasoning.md"] == "skill content 1"
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 13c: _outcome_to_quadrant_pairs
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestOutcomeToQuadrantPairs:
|
||||
"""_outcome_to_quadrant_pairs 四象限拍平。"""
|
||||
|
||||
def test_all_quadrants(self) -> None:
|
||||
"""四象限各有一个 qid 时生成 4 条 pair。"""
|
||||
outcome = ValidationOutcome(
|
||||
action="accept_confirmed",
|
||||
accepted=True,
|
||||
stop_reason="confirmed",
|
||||
e_value=10.0,
|
||||
w=3,
|
||||
l=0,
|
||||
n_used=10,
|
||||
delta_hat=0.3,
|
||||
delta_shrunk=0.2,
|
||||
baseline_acc=0.7,
|
||||
candidate_acc=0.9,
|
||||
improvements=["q1"],
|
||||
regressions=["q2"],
|
||||
persistent_fails=["q3"],
|
||||
stable_successes=["q4"],
|
||||
)
|
||||
pairs = _outcome_to_quadrant_pairs("Action Reasoning", outcome)
|
||||
assert len(pairs) == 4
|
||||
by_qid = {p["question_id"]: p for p in pairs}
|
||||
assert by_qid["q1"]["category"] == "improved"
|
||||
assert by_qid["q1"]["prev_correct"] is False
|
||||
assert by_qid["q1"]["curr_correct"] is True
|
||||
assert by_qid["q2"]["category"] == "regressed"
|
||||
assert by_qid["q3"]["category"] == "persistent_fail"
|
||||
assert by_qid["q4"]["category"] == "stable_success"
|
||||
|
||||
def test_empty_outcome(self) -> None:
|
||||
"""四象限全空时返回空列表。"""
|
||||
outcome = ValidationOutcome(
|
||||
action="reject",
|
||||
accepted=False,
|
||||
stop_reason="directional",
|
||||
e_value=0.5,
|
||||
w=0,
|
||||
l=2,
|
||||
n_used=5,
|
||||
delta_hat=-0.1,
|
||||
delta_shrunk=-0.05,
|
||||
baseline_acc=0.8,
|
||||
candidate_acc=0.6,
|
||||
)
|
||||
assert _outcome_to_quadrant_pairs("Any", outcome) == []
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 13c: _build_comparison_pairs
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestBuildComparisonPairs:
|
||||
"""_build_comparison_pairs momentum 纵向对比对。"""
|
||||
|
||||
def test_builds_pairs(self) -> None:
|
||||
"""正确构造对比对。"""
|
||||
sampled = [_FakeQuestion(question_id="q1", question="问题1")]
|
||||
prev_rows = {
|
||||
"q1": {"prediction": "A", "_correct": True},
|
||||
}
|
||||
curr_rows = {
|
||||
"q1": {"prediction": "B", "_correct": False},
|
||||
}
|
||||
pairs = _build_comparison_pairs(sampled, prev_rows, curr_rows)
|
||||
assert len(pairs) == 1
|
||||
assert pairs[0]["question"] == "问题1"
|
||||
assert pairs[0]["prev_prediction"] == "A"
|
||||
assert pairs[0]["curr_prediction"] == "B"
|
||||
assert pairs[0]["correct_prev"] is True
|
||||
assert pairs[0]["correct_curr"] is False
|
||||
|
||||
def test_missing_rows_use_defaults(self) -> None:
|
||||
"""缺失行时使用默认值。"""
|
||||
sampled = [_FakeQuestion(question_id="q1")]
|
||||
pairs = _build_comparison_pairs(sampled, {}, {})
|
||||
assert pairs[0]["prev_prediction"] == ""
|
||||
assert pairs[0]["correct_prev"] is False
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 13e: _should_early_stop
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestShouldEarlyStop:
|
||||
"""_should_early_stop 步粒度 early stop。"""
|
||||
|
||||
def test_improved_this_epoch_resets(self, tmp_path: Path) -> None:
|
||||
"""本 epoch best 刷新时重置计数器。"""
|
||||
# 写 manifest + best
|
||||
manifest = {
|
||||
"name": "test",
|
||||
"store": ".",
|
||||
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
|
||||
"best": {"epoch": 2, "val_acc": 0.9},
|
||||
"history": [],
|
||||
}
|
||||
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
||||
|
||||
state = MagicMock()
|
||||
state.steps_since_best_improved = 10
|
||||
|
||||
result = _should_early_stop(tmp_path, epoch=2, steps_this_epoch=5, state=state, patience=20)
|
||||
assert result is False
|
||||
assert state.steps_since_best_improved == 0
|
||||
|
||||
def test_no_improvement_accumulates(self, tmp_path: Path) -> None:
|
||||
"""未刷新时累加步数。"""
|
||||
manifest = {
|
||||
"name": "test",
|
||||
"store": ".",
|
||||
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
|
||||
"best": {"epoch": 1, "val_acc": 0.5},
|
||||
"history": [],
|
||||
}
|
||||
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
||||
|
||||
state = MagicMock()
|
||||
state.steps_since_best_improved = 15
|
||||
|
||||
result = _should_early_stop(tmp_path, epoch=3, steps_this_epoch=5, state=state, patience=20)
|
||||
assert result is True # 15 + 5 = 20 >= 20
|
||||
assert state.steps_since_best_improved == 20
|
||||
|
||||
def test_below_patience_continues(self, tmp_path: Path) -> None:
|
||||
"""累计步数未达阈值时继续。"""
|
||||
manifest = {
|
||||
"name": "test",
|
||||
"store": ".",
|
||||
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
|
||||
"best": {"epoch": 1, "val_acc": 0.5},
|
||||
"history": [],
|
||||
}
|
||||
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
||||
|
||||
state = MagicMock()
|
||||
state.steps_since_best_improved = 10
|
||||
|
||||
result = _should_early_stop(tmp_path, epoch=3, steps_this_epoch=5, state=state, patience=20)
|
||||
assert result is False
|
||||
assert state.steps_since_best_improved == 15
|
||||
|
||||
def test_step_granularity(self, tmp_path: Path) -> None:
|
||||
"""步粒度而非 epoch 粒度。"""
|
||||
manifest = {
|
||||
"name": "test",
|
||||
"store": ".",
|
||||
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
|
||||
"best": {"epoch": 1, "val_acc": 0.5},
|
||||
"history": [],
|
||||
}
|
||||
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
||||
|
||||
state = MagicMock()
|
||||
# 连续 3 个 epoch，每个 3 步
|
||||
state.steps_since_best_improved = 0
|
||||
for ep in range(2, 5):
|
||||
stopped = _should_early_stop(
|
||||
tmp_path, epoch=ep, steps_this_epoch=3, state=state, patience=10
|
||||
)
|
||||
if ep < 4:
|
||||
assert stopped is False
|
||||
else:
|
||||
# 3+3+3=9 < 10 但第三轮后 9+3=12>=10 在 ep=5 触发
|
||||
# 实际：ep=2 → 3, ep=3 → 6, ep=4 → 9
|
||||
assert stopped is False
|
||||
stopped = _should_early_stop(
|
||||
tmp_path, epoch=5, steps_this_epoch=3, state=state, patience=10
|
||||
)
|
||||
assert stopped is True # 9+3=12>=10
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 13c: Probation 数据结构测试
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestProbation:
|
||||
"""Probation 数据结构。"""
|
||||
|
||||
def test_probation_fields(self) -> None:
|
||||
"""Probation 必须具备全部字段。"""
|
||||
p = Probation(
|
||||
task_type="Action Reasoning",
|
||||
anchor_skills_version="v1",
|
||||
target_file="action-reasoning.md",
|
||||
correctness_snapshot={"q1": True},
|
||||
opened_step=5,
|
||||
)
|
||||
assert p.task_type == "Action Reasoning"
|
||||
assert p.pending_edits == []
|
||||
|
||||
def test_pending_edits_append(self) -> None:
|
||||
"""pending_edits 可追加 RejectedEdit。"""
|
||||
p = Probation(
|
||||
task_type="Action Reasoning",
|
||||
anchor_skills_version="v1",
|
||||
target_file="action-reasoning.md",
|
||||
correctness_snapshot={},
|
||||
opened_step=0,
|
||||
)
|
||||
edit = RejectedEdit(
|
||||
target_file="action-reasoning.md",
|
||||
target_type="skill",
|
||||
change_summary="test",
|
||||
delta=0.1,
|
||||
source_version="v1",
|
||||
epoch=0,
|
||||
gate_w=3,
|
||||
gate_l=1,
|
||||
gate_e_value=2.5,
|
||||
gate_delta_shrunk=0.05,
|
||||
)
|
||||
p.pending_edits.append(edit)
|
||||
assert len(p.pending_edits) == 1
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 13c: RejectedSummary 黑名单防污染
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestFormatAppliedEdits:
|
||||
"""_format_applied_edits 只拼 applied 的 edit。"""
|
||||
|
||||
def test_only_applied_edits_in_summary(self) -> None:
|
||||
"""只有 applied 状态的 edit 进入摘要。"""
|
||||
record = MagicMock()
|
||||
record.edits = [
|
||||
{"op": "replace", "target": "section1"},
|
||||
{"op": "insert", "content": "new_content"},
|
||||
{"op": "delete", "target": "old_stuff"},
|
||||
]
|
||||
record.apply_report = [
|
||||
{"status": "applied_exact"},
|
||||
{"status": "skipped_not_found"},
|
||||
{"status": "applied_fuzzy"},
|
||||
]
|
||||
|
||||
summary = _format_applied_edits(record)
|
||||
assert summary is not None
|
||||
assert "section1" in summary
|
||||
assert "old_stuff" in summary
|
||||
assert "new_content" not in summary
|
||||
|
||||
def test_zero_applied_returns_info_message(self) -> None:
|
||||
"""0 applied 返回信息性消息（非 None）。"""
|
||||
record = MagicMock()
|
||||
record.edits = [{"op": "replace", "target": "sec"}]
|
||||
record.apply_report = [{"status": "skipped_not_found"}]
|
||||
|
||||
summary = _format_applied_edits(record)
|
||||
assert summary is not None
|
||||
assert "0 applied" in summary
|
||||
|
||||
def test_no_edits_returns_none(self) -> None:
|
||||
"""无 edit 时返回 None。"""
|
||||
record = MagicMock()
|
||||
record.edits = []
|
||||
|
||||
assert _format_applied_edits(record) is None
|
||||
|
||||
def test_no_report_includes_all_edits(self) -> None:
|
||||
"""无 apply_report 时包含所有 edit。"""
|
||||
record = MagicMock()
|
||||
record.edits = [{"op": "replace", "target": "foo"}]
|
||||
record.apply_report = []
|
||||
|
||||
summary = _format_applied_edits(record)
|
||||
assert summary is not None
|
||||
assert "foo" in summary
|
||||
|
||||
|
||||
class TestFallbackSummary:
|
||||
"""_fallback_summary 兜底黑名单摘要。"""
|
||||
|
||||
def test_from_suggestions(self) -> None:
|
||||
"""有 suggestions 时拼接 change 字段。"""
|
||||
record = MagicMock()
|
||||
record.suggestions = [{"change": "改 A"}, {"change": "改 B"}]
|
||||
outcome = MagicMock()
|
||||
outcome.delta_hat = -0.1
|
||||
|
||||
summary = _fallback_summary(record, outcome)
|
||||
assert "改 A" in summary
|
||||
assert "改 B" in summary
|
||||
|
||||
def test_no_suggestions_uses_delta(self) -> None:
|
||||
"""无 suggestions 时使用 delta 信息。"""
|
||||
record = MagicMock()
|
||||
record.suggestions = []
|
||||
outcome = MagicMock()
|
||||
outcome.delta_hat = -0.15
|
||||
|
||||
summary = _fallback_summary(record, outcome)
|
||||
assert "delta" in summary
|
||||
assert "-0.15" in summary
|
||||
|
||||
|
||||
class TestRejectedSummaryIntegration:
|
||||
"""_rejected_summary_static 集成：两个子函数组合。"""
|
||||
|
||||
def test_static_delegates_to_format_applied(self) -> None:
|
||||
"""有 applied edit 时 static 方法返回 _format_applied_edits 结果。"""
|
||||
from app.harness.runner import Runner
|
||||
|
||||
record = MagicMock()
|
||||
record.edits = [{"op": "replace", "target": "section1"}]
|
||||
record.apply_report = [{"status": "applied_exact"}]
|
||||
record.suggestions = []
|
||||
outcome = MagicMock()
|
||||
outcome.delta_hat = 0.1
|
||||
|
||||
summary = Runner._rejected_summary_static(record, outcome)
|
||||
assert "section1" in summary
|
||||
|
||||
def test_static_falls_back_to_suggestions(self) -> None:
|
||||
"""无 edit 时 static 方法使用 _fallback_summary。"""
|
||||
from app.harness.runner import Runner
|
||||
|
||||
record = MagicMock()
|
||||
record.edits = []
|
||||
record.suggestions = [{"change": "尝试 X"}]
|
||||
outcome = MagicMock()
|
||||
outcome.delta_hat = -0.2
|
||||
|
||||
summary = Runner._rejected_summary_static(record, outcome)
|
||||
assert "尝试 X" in summary
|
||||
|
||||
|
||||
class TestWriteSkipReport:
|
||||
"""_write_skip_report 辅助函数。"""
|
||||
|
||||
def test_writes_cooldown_report(self, tmp_path: Path) -> None:
|
||||
"""cooldown 路径写 step_report JSON 文件。"""
|
||||
(tmp_path / "analyses").mkdir()
|
||||
_write_skip_report(
|
||||
tmp_path,
|
||||
epoch=1,
|
||||
step=0,
|
||||
global_step=5,
|
||||
task_type="Action Reasoning",
|
||||
action="cooldown",
|
||||
baseline_acc=0.75,
|
||||
budget=3,
|
||||
)
|
||||
report_path = tmp_path / "analyses" / "step_report_e1_s0_action-reasoning.json"
|
||||
assert report_path.exists()
|
||||
data = json.loads(report_path.read_text())
|
||||
assert data["gate_action"] == "cooldown"
|
||||
assert data["candidate_acc"] == 0.75
|
||||
assert data["gate_w"] is None
|
||||
|
||||
def test_writes_skipped_report(self, tmp_path: Path) -> None:
|
||||
"""skipped 路径写 step_report 并传递 rank_clip_triggered。"""
|
||||
(tmp_path / "analyses").mkdir()
|
||||
_write_skip_report(
|
||||
tmp_path,
|
||||
epoch=2,
|
||||
step=1,
|
||||
global_step=10,
|
||||
task_type="Temporal Reasoning",
|
||||
action="skipped",
|
||||
baseline_acc=0.6,
|
||||
budget=2,
|
||||
rank_clip_triggered=True,
|
||||
)
|
||||
report_path = tmp_path / "analyses" / "step_report_e2_s1_temporal-reasoning.json"
|
||||
assert report_path.exists()
|
||||
data = json.loads(report_path.read_text())
|
||||
assert data["gate_action"] == "skipped"
|
||||
assert data["rank_clip_triggered"] is True
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 13a: _TrainState 基本构造
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestTrainState:
|
||||
"""_TrainState dataclass 基本构造与字段默认值。"""
|
||||
|
||||
def test_default_fields(self) -> None:
|
||||
"""默认字段值正确。"""
|
||||
state = _TrainState(
|
||||
correctness={"q1": True},
|
||||
gate_pools=MagicMock(),
|
||||
baseline_cache=MagicMock(),
|
||||
eval_prev_acc=0.5,
|
||||
eval_prev_run_id="run1",
|
||||
best_val_acc=0.5,
|
||||
best_skills_version="v1",
|
||||
best_prompts_version="v1",
|
||||
)
|
||||
assert state.global_step == 0
|
||||
assert state.gate_epoch_observed is False
|
||||
assert state.probations == {}
|
||||
assert state.gate_cooldown == {}
|
||||
assert state.rejected_buffer == {}
|
||||
assert state.system_packs == []
|
||||
assert state.tool_packs == []
|
||||
assert state.changed_task_types_this_epoch == set()
|
||||
assert state.steps_since_best_improved == 0
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 13c: cooldown 递减测试
|
||||
# =========================================================================
|
||||
|
||||
|
||||
class TestCooldownDecrement:
|
||||
"""gate_cooldown 每 step 递减、归零剔除。"""
|
||||
|
||||
def test_decrement_and_remove(self) -> None:
|
||||
"""冷却值递减，归零剔除。"""
|
||||
cooldown = {"type_a": 3, "type_b": 1, "type_c": 2}
|
||||
# 模拟 _run_step 末尾的冷却递减
|
||||
cooldown = {t: n - 1 for t, n in cooldown.items() if n - 1 > 0}
|
||||
assert cooldown == {"type_a": 2, "type_c": 1}
|
||||
|
||||
cooldown = {t: n - 1 for t, n in cooldown.items() if n - 1 > 0}
|
||||
assert cooldown == {"type_a": 1}
|
||||
|
||||
cooldown = {t: n - 1 for t, n in cooldown.items() if n - 1 > 0}
|
||||
assert cooldown == {}
|
||||
@@ -0,0 +1,460 @@
|
||||
"""Store 版本操作 + Seed 管理的单元测试。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from app.harness.store import (
|
||||
_parse_version,
|
||||
_write_meta,
|
||||
advance_version,
|
||||
extract_run_db,
|
||||
init_seed,
|
||||
init_store,
|
||||
list_seeds,
|
||||
list_versions,
|
||||
next_version,
|
||||
promote_to_seed,
|
||||
read_seed,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _parse_version
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestParseVersion:
|
||||
"""_parse_version 解析 v\\d+ 格式版本号。"""
|
||||
|
||||
def test_parse_version_normal(self) -> None:
|
||||
assert _parse_version("v1") == 1
|
||||
assert _parse_version("v10") == 10
|
||||
assert _parse_version("v999") == 999
|
||||
|
||||
def test_parse_version_invalid(self) -> None:
|
||||
with pytest.raises(ValueError, match="无效版本号"):
|
||||
_parse_version("abc")
|
||||
with pytest.raises(ValueError, match="无效版本号"):
|
||||
_parse_version("v")
|
||||
with pytest.raises(ValueError, match="无效版本号"):
|
||||
_parse_version("v1.0")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# list_versions — 数字排序
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestListVersions:
|
||||
"""list_versions 按数字排序，v10 排在 v2 后。"""
|
||||
|
||||
def test_list_versions_numeric_sort(self, tmp_path: "Path") -> None:
|
||||
"""v10 必须排在 v2 后面（非字典序）。"""
|
||||
store = tmp_path / "store"
|
||||
resource = store / "skills"
|
||||
resource.mkdir(parents=True)
|
||||
for v in ("v1", "v10", "v2", "v20", "v3"):
|
||||
(resource / v).mkdir()
|
||||
result = list_versions(store, "skills")
|
||||
assert result == ["v1", "v2", "v3", "v10", "v20"]
|
||||
|
||||
def test_list_versions_empty(self, tmp_path: "Path") -> None:
|
||||
store = tmp_path / "store"
|
||||
assert list_versions(store, "skills") == []
|
||||
|
||||
def test_list_versions_ignores_non_version_dirs(self, tmp_path: "Path") -> None:
|
||||
"""非 v\\d+ 格式的目录被忽略。"""
|
||||
store = tmp_path / "store"
|
||||
resource = store / "skills"
|
||||
resource.mkdir(parents=True)
|
||||
(resource / "v1").mkdir()
|
||||
(resource / "backup").mkdir()
|
||||
(resource / ".hidden").mkdir()
|
||||
assert list_versions(store, "skills") == ["v1"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# next_version
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNextVersion:
|
||||
"""next_version 返回下一个可用版本号。"""
|
||||
|
||||
def test_next_version_empty(self, tmp_path: "Path") -> None:
|
||||
store = tmp_path / "store"
|
||||
assert next_version(store, "skills") == "v1"
|
||||
|
||||
def test_next_version_after_existing(self, tmp_path: "Path") -> None:
|
||||
store = tmp_path / "store"
|
||||
resource = store / "skills"
|
||||
resource.mkdir(parents=True)
|
||||
(resource / "v1").mkdir()
|
||||
(resource / "v2").mkdir()
|
||||
assert next_version(store, "skills") == "v3"
|
||||
|
||||
def test_next_version_with_gap(self, tmp_path: "Path") -> None:
|
||||
"""v1 和 v10 之间有 gap，next 应为 v11。"""
|
||||
store = tmp_path / "store"
|
||||
resource = store / "skills"
|
||||
resource.mkdir(parents=True)
|
||||
(resource / "v1").mkdir()
|
||||
(resource / "v10").mkdir()
|
||||
assert next_version(store, "skills") == "v11"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# advance_version
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAdvanceVersion:
|
||||
"""advance_version copytree + _write_meta。"""
|
||||
|
||||
def test_advance_version(self, tmp_path: "Path") -> None:
|
||||
store = tmp_path / "store"
|
||||
resource = store / "skills"
|
||||
resource.mkdir(parents=True)
|
||||
(resource / "v1").mkdir()
|
||||
|
||||
source = tmp_path / "new_skills"
|
||||
source.mkdir()
|
||||
(source / "skill_a.md").write_text("内容A")
|
||||
|
||||
version = advance_version(
|
||||
store,
|
||||
"skills",
|
||||
source,
|
||||
{"source": "evolution", "description": "进化产出"},
|
||||
)
|
||||
assert version == "v2"
|
||||
assert (resource / "v2" / "skill_a.md").read_text() == "内容A"
|
||||
|
||||
meta = json.loads((resource / "v2" / "meta.json").read_text())
|
||||
assert meta["version"] == "v2"
|
||||
assert meta["source"] == "evolution"
|
||||
assert meta["description"] == "进化产出"
|
||||
assert "created_at" in meta
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# init_store
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestInitStore:
|
||||
"""init_store 初始化 Store 目录结构。"""
|
||||
|
||||
def test_init_store(self, tmp_path: "Path") -> None:
|
||||
videos = tmp_path / "videos_src"
|
||||
videos.mkdir()
|
||||
(videos / "v001").mkdir()
|
||||
(videos / "v001" / "tree.json").write_text("{}")
|
||||
|
||||
skills = tmp_path / "skills_src"
|
||||
skills.mkdir()
|
||||
(skills / "search.md").write_text("skill")
|
||||
|
||||
prompts = tmp_path / "prompts_src"
|
||||
prompts.mkdir()
|
||||
(prompts / "system.md").write_text("prompt")
|
||||
|
||||
store = tmp_path / "store"
|
||||
init_store(store, videos, skills, prompts)
|
||||
|
||||
assert (store / "videos" / "v001" / "tree.json").exists()
|
||||
assert (store / "questions" / "benchmarks").is_dir()
|
||||
assert (store / "questions" / "generated").is_dir()
|
||||
assert (store / "skills" / "v1" / "search.md").read_text() == "skill"
|
||||
assert (store / "prompts" / "v1" / "system.md").read_text() == "prompt"
|
||||
|
||||
skills_meta = json.loads(
|
||||
(store / "skills" / "v1" / "meta.json").read_text()
|
||||
)
|
||||
assert skills_meta["version"] == "v1"
|
||||
assert skills_meta["source"] == "manual"
|
||||
|
||||
def test_init_store_exists_raises(self, tmp_path: "Path") -> None:
|
||||
store = tmp_path / "store"
|
||||
store.mkdir()
|
||||
with pytest.raises(FileExistsError, match="Store 已存在"):
|
||||
init_store(store, tmp_path, tmp_path, tmp_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Seed 相关
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_seed_fixtures(tmp_path):
|
||||
"""创建 seed 测试所需的公共 fixture。"""
|
||||
store = tmp_path / "store"
|
||||
store.mkdir(parents=True)
|
||||
|
||||
skills_dir = tmp_path / "sk"
|
||||
skills_dir.mkdir()
|
||||
(skills_dir / "search.md").write_text("skill")
|
||||
|
||||
prompts_dir = tmp_path / "pr"
|
||||
prompts_dir.mkdir()
|
||||
(prompts_dir / "system.md").write_text("prompt")
|
||||
|
||||
baseline_db = tmp_path / "base.db"
|
||||
conn = sqlite3.connect(baseline_db)
|
||||
conn.execute(
|
||||
"CREATE TABLE _runs (run_id TEXT PRIMARY KEY, status TEXT)"
|
||||
)
|
||||
conn.execute("INSERT INTO _runs VALUES ('r1', 'done')")
|
||||
conn.execute(
|
||||
"CREATE TABLE predictions (run_id TEXT, question_id TEXT, answer TEXT)"
|
||||
)
|
||||
conn.execute("INSERT INTO predictions VALUES ('r1', 'q1', 'A')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
return store, skills_dir, prompts_dir, baseline_db
|
||||
|
||||
|
||||
class TestInitSeed:
|
||||
"""init_seed 创建种子目录。"""
|
||||
|
||||
def test_init_seed(self, tmp_path: "Path") -> None:
|
||||
store, skills_dir, prompts_dir, baseline_db = _make_seed_fixtures(tmp_path)
|
||||
seed_dir = init_seed(
|
||||
store,
|
||||
"initial",
|
||||
skills_dir,
|
||||
prompts_dir,
|
||||
baseline_db,
|
||||
"r1",
|
||||
None,
|
||||
"初始种子",
|
||||
)
|
||||
assert seed_dir == store / "seeds" / "initial"
|
||||
assert (seed_dir / "skills" / "search.md").exists()
|
||||
assert (seed_dir / "prompts" / "system.md").exists()
|
||||
assert (seed_dir / "baseline.db").exists()
|
||||
|
||||
meta = json.loads((seed_dir / "seed.json").read_text())
|
||||
assert meta["baseline_run_id"] == "r1"
|
||||
assert meta["parent"] is None
|
||||
assert meta["description"] == "初始种子"
|
||||
assert "created_at" in meta
|
||||
|
||||
def test_init_seed_exists_raises(self, tmp_path: "Path") -> None:
|
||||
store, skills_dir, prompts_dir, baseline_db = _make_seed_fixtures(tmp_path)
|
||||
init_seed(store, "dup", skills_dir, prompts_dir, baseline_db, "r1", None, "first")
|
||||
with pytest.raises(FileExistsError, match="种子已存在"):
|
||||
init_seed(
|
||||
store, "dup", skills_dir, prompts_dir, baseline_db, "r1", None, "second"
|
||||
)
|
||||
|
||||
|
||||
class TestListSeeds:
|
||||
"""list_seeds 列出所有种子。"""
|
||||
|
||||
def test_list_seeds(self, tmp_path: "Path") -> None:
|
||||
store, skills_dir, prompts_dir, baseline_db = _make_seed_fixtures(tmp_path)
|
||||
init_seed(store, "beta", skills_dir, prompts_dir, baseline_db, "r1", None, "b")
|
||||
init_seed(store, "alpha", skills_dir, prompts_dir, baseline_db, "r1", None, "a")
|
||||
assert list_seeds(store) == ["alpha", "beta"]
|
||||
|
||||
def test_list_seeds_empty(self, tmp_path: "Path") -> None:
|
||||
store = tmp_path / "store"
|
||||
assert list_seeds(store) == []
|
||||
|
||||
|
||||
class TestReadSeed:
|
||||
"""read_seed 读取 seed.json。"""
|
||||
|
||||
def test_read_seed(self, tmp_path: "Path") -> None:
|
||||
store, skills_dir, prompts_dir, baseline_db = _make_seed_fixtures(tmp_path)
|
||||
init_seed(store, "s1", skills_dir, prompts_dir, baseline_db, "r1", None, "desc")
|
||||
meta = read_seed(store, "s1")
|
||||
assert meta["baseline_run_id"] == "r1"
|
||||
assert meta["description"] == "desc"
|
||||
|
||||
def test_read_seed_not_found(self, tmp_path: "Path") -> None:
|
||||
store = tmp_path / "store"
|
||||
store.mkdir()
|
||||
with pytest.raises(FileNotFoundError, match="种子不存在"):
|
||||
read_seed(store, "no_such")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# extract_run_db
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractRunDb:
|
||||
"""extract_run_db 抽取指定 run 的行并保留 PK。"""
|
||||
|
||||
def _make_src_db(self, path):
|
||||
"""创建带 _runs + predictions 表的源 db。"""
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute(
|
||||
"CREATE TABLE _runs (run_id TEXT PRIMARY KEY, status TEXT)"
|
||||
)
|
||||
conn.execute("INSERT INTO _runs VALUES ('r1', 'done')")
|
||||
conn.execute("INSERT INTO _runs VALUES ('r2', 'done')")
|
||||
conn.execute(
|
||||
"CREATE TABLE predictions "
|
||||
"(run_id TEXT, question_id TEXT, answer TEXT)"
|
||||
)
|
||||
conn.execute("INSERT INTO predictions VALUES ('r1', 'q1', 'A')")
|
||||
conn.execute("INSERT INTO predictions VALUES ('r1', 'q2', 'B')")
|
||||
conn.execute("INSERT INTO predictions VALUES ('r2', 'q1', 'C')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def test_extract_run_db_preserves_pk(self, tmp_path: "Path") -> None:
|
||||
"""原始 CREATE 保留主键约束。"""
|
||||
src = tmp_path / "src.db"
|
||||
dst = tmp_path / "dst.db"
|
||||
self._make_src_db(src)
|
||||
extract_run_db(src, dst, "r1")
|
||||
|
||||
conn = sqlite3.connect(dst)
|
||||
# 验证 _runs 表有 PK
|
||||
create_sql = conn.execute(
|
||||
"SELECT sql FROM sqlite_master WHERE type='table' AND name='_runs'"
|
||||
).fetchone()[0]
|
||||
assert "PRIMARY KEY" in create_sql
|
||||
|
||||
# 验证只有 r1 的行
|
||||
runs = conn.execute("SELECT * FROM _runs").fetchall()
|
||||
assert len(runs) == 1
|
||||
assert runs[0][0] == "r1"
|
||||
|
||||
preds = conn.execute("SELECT * FROM predictions").fetchall()
|
||||
assert len(preds) == 2
|
||||
conn.close()
|
||||
|
||||
def test_extract_run_db_missing_table(self, tmp_path: "Path") -> None:
|
||||
"""源 db 无目标表时报错。"""
|
||||
src = tmp_path / "src.db"
|
||||
dst = tmp_path / "dst.db"
|
||||
conn = sqlite3.connect(src)
|
||||
conn.execute("CREATE TABLE other (id TEXT)")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
with pytest.raises(RuntimeError, match="源 db 无表"):
|
||||
extract_run_db(src, dst, "r1")
|
||||
|
||||
def test_extract_run_db_no_rows(self, tmp_path: "Path") -> None:
|
||||
"""目标 run_id 不存在时报错。"""
|
||||
src = tmp_path / "src.db"
|
||||
dst = tmp_path / "dst.db"
|
||||
self._make_src_db(src)
|
||||
with pytest.raises(RuntimeError, match="无 run_id="):
|
||||
extract_run_db(src, dst, "nonexistent")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# promote_to_seed
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_promote_fixtures(tmp_path):
|
||||
"""创建 promote_to_seed 测试所需的 workspace + store。"""
|
||||
ws = tmp_path / "ws"
|
||||
ws.mkdir()
|
||||
store = tmp_path / "store"
|
||||
store.mkdir()
|
||||
|
||||
# workspace 内的 skills/prompts 版本目录
|
||||
(ws / "skills" / "v2").mkdir(parents=True)
|
||||
(ws / "skills" / "v2" / "skill.md").write_text("evolved")
|
||||
(ws / "prompts" / "v2").mkdir(parents=True)
|
||||
(ws / "prompts" / "v2" / "system.md").write_text("prompt v2")
|
||||
|
||||
# workspace harness.db
|
||||
db_path = ws / "harness.db"
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute("""
|
||||
CREATE TABLE _runs (
|
||||
run_id TEXT PRIMARY KEY,
|
||||
skills_version TEXT,
|
||||
prompts_version TEXT
|
||||
)
|
||||
""")
|
||||
conn.execute(
|
||||
"INSERT INTO _runs VALUES ('eval_001', 'v2', 'v2')"
|
||||
)
|
||||
conn.execute("""
|
||||
CREATE TABLE predictions (
|
||||
run_id TEXT, question_id TEXT, answer TEXT
|
||||
)
|
||||
""")
|
||||
conn.execute("INSERT INTO predictions VALUES ('eval_001', 'q1', 'A')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
return ws, store
|
||||
|
||||
|
||||
class TestPromoteToSeed:
|
||||
"""promote_to_seed 固化 workspace 版本为种子。"""
|
||||
|
||||
def test_promote_to_seed_success(self, tmp_path: "Path") -> None:
|
||||
ws, store = _make_promote_fixtures(tmp_path)
|
||||
seed_dir = promote_to_seed(ws, store, "v2", "eval_001", "evolved-seed", "good")
|
||||
assert seed_dir == store / "seeds" / "evolved-seed"
|
||||
assert (seed_dir / "skills" / "skill.md").exists()
|
||||
assert (seed_dir / "prompts" / "system.md").exists()
|
||||
assert (seed_dir / "baseline.db").exists()
|
||||
meta = json.loads((seed_dir / "seed.json").read_text())
|
||||
assert meta["baseline_run_id"] == "eval_001"
|
||||
assert meta["parent"] == "ws:v2"
|
||||
|
||||
def test_promote_to_seed_version_mismatch(self, tmp_path: "Path") -> None:
|
||||
"""eval run 的 skills_version 与 --version 不符时报错。"""
|
||||
ws, store = _make_promote_fixtures(tmp_path)
|
||||
with pytest.raises(ValueError, match="版本.*不符"):
|
||||
promote_to_seed(ws, store, "v3", "eval_001", "bad", "mismatch")
|
||||
|
||||
def test_promote_to_seed_null_version(self, tmp_path: "Path") -> None:
|
||||
"""eval run 的版本为 NULL 时报错。"""
|
||||
ws = tmp_path / "ws2"
|
||||
ws.mkdir()
|
||||
store = tmp_path / "store2"
|
||||
store.mkdir()
|
||||
db_path = ws / "harness.db"
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute("""
|
||||
CREATE TABLE _runs (
|
||||
run_id TEXT PRIMARY KEY,
|
||||
skills_version TEXT,
|
||||
prompts_version TEXT
|
||||
)
|
||||
""")
|
||||
conn.execute("INSERT INTO _runs VALUES ('eval_null', NULL, NULL)")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
with pytest.raises(ValueError, match="NULL"):
|
||||
promote_to_seed(ws, store, "v1", "eval_null", "bad", "null ver")
|
||||
|
||||
def test_promote_to_seed_run_not_found(self, tmp_path: "Path") -> None:
|
||||
"""eval run 不存在时报错。"""
|
||||
ws, store = _make_promote_fixtures(tmp_path)
|
||||
with pytest.raises(ValueError, match="eval run 不存在"):
|
||||
promote_to_seed(ws, store, "v1", "nonexistent", "bad", "no run")
|
||||
|
||||
def test_promote_cleanup_tmp_db(self, tmp_path: "Path") -> None:
|
||||
"""finally 清理临时 db 文件。"""
|
||||
ws, store = _make_promote_fixtures(tmp_path)
|
||||
promote_to_seed(ws, store, "v2", "eval_001", "clean-test", "cleanup")
|
||||
# 临时 db 应已清理
|
||||
assert not (ws / "_promote_tmp.db").exists()
|
||||
|
||||
def test_promote_cleanup_tmp_db_on_error(self, tmp_path: "Path") -> None:
|
||||
"""即使 init_seed 失败（同名种子），临时 db 也应被清理。"""
|
||||
ws, store = _make_promote_fixtures(tmp_path)
|
||||
promote_to_seed(ws, store, "v2", "eval_001", "first", "first time")
|
||||
with pytest.raises(FileExistsError):
|
||||
promote_to_seed(ws, store, "v2", "eval_001", "first", "second time")
|
||||
assert not (ws / "_promote_tmp.db").exists()
|
||||
@@ -0,0 +1,554 @@
|
||||
"""tests/unit/test_harness_validate.py — app/harness/validate.py 的单元测试。
|
||||
|
||||
覆盖：数据类型字段、materialize 物化与清理、async validate_skill_local
|
||||
（accept/reject/prefix 校验/INFRA 护栏/缓存命中/最后一块终态）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
|
||||
from app.harness.gate_ladder import BaselineCache, skill_hash
|
||||
from app.harness.inference import PREDICTIONS_SCHEMA, InferenceResult
|
||||
from app.harness.log import HarnessLog
|
||||
from app.harness.validate import (
|
||||
Probation,
|
||||
ValidationOutcome,
|
||||
materialize_candidate_skill,
|
||||
validate_skill_local,
|
||||
)
|
||||
from core.evolution import GateParams, RejectedEdit
|
||||
from core.types import GeneratedQuestion
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 辅助工具
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DEFAULT_GATE_PARAMS = GateParams(
|
||||
e_confirm=20.0,
|
||||
e_provisional=3.0,
|
||||
w_net_min=2,
|
||||
delta_min=0.05,
|
||||
lambda_dir=-2.0,
|
||||
e_rollback=10.0,
|
||||
)
|
||||
|
||||
|
||||
def _make_questions(
|
||||
n: int,
|
||||
task_type: str = "temporal",
|
||||
prefix: str = "q",
|
||||
) -> list[GeneratedQuestion]:
|
||||
"""生成 n 个测试用 GeneratedQuestion。"""
|
||||
return [
|
||||
GeneratedQuestion(
|
||||
question_id=f"{prefix}{i}",
|
||||
video_id=f"v{i}",
|
||||
task_type=task_type,
|
||||
question=f"Question {i}?",
|
||||
options=("A", "B", "C", "D"),
|
||||
answer="A",
|
||||
source_nodes=(),
|
||||
difficulty="easy",
|
||||
)
|
||||
for i in range(n)
|
||||
]
|
||||
|
||||
|
||||
def _setup_workspace(tmp_path: Path) -> Path:
|
||||
"""在 tmp_path 下构建最小 workspace 结构。"""
|
||||
skills_dir = tmp_path / "skills" / "v1"
|
||||
skills_dir.mkdir(parents=True)
|
||||
(skills_dir / "temporal.md").write_text("baseline skill content", encoding="utf-8")
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _make_log(workspace: Path, run_id: str = "test_master") -> HarnessLog:
|
||||
"""创建 HarnessLog 并初始化 predictions 表。"""
|
||||
db_path = workspace / "harness.db"
|
||||
log = HarnessLog(str(db_path), run_id)
|
||||
log.create_table("predictions", PREDICTIONS_SCHEMA)
|
||||
return log
|
||||
|
||||
|
||||
def _insert_predictions(
|
||||
log: HarnessLog,
|
||||
run_id: str,
|
||||
correctness: dict[str, bool],
|
||||
answer: str = "A",
|
||||
) -> None:
|
||||
"""向 predictions 表插入指定 run_id 的逐题预测记录。
|
||||
|
||||
通过在 record 中显式传入 run_id 覆盖 log 的默认 run_id。
|
||||
"""
|
||||
for qid, correct in correctness.items():
|
||||
prediction = answer if correct else "Z"
|
||||
log.insert(
|
||||
"predictions",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"video_id": "v0",
|
||||
"question_id": qid,
|
||||
"task_type": "temporal",
|
||||
"prediction": prediction,
|
||||
"answer": answer,
|
||||
"evidence": "",
|
||||
"reasoning": "",
|
||||
"steps_used": 1,
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 10,
|
||||
"stop_reason": "completed",
|
||||
"steps_json": "[]",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _make_mock_run_inference(
|
||||
log: HarnessLog,
|
||||
baseline_correctness: dict[str, bool],
|
||||
candidate_correctness: dict[str, bool],
|
||||
error_count: int = 0,
|
||||
):
|
||||
"""构建 mock RunInferenceFn。
|
||||
|
||||
根据 run_id 中的 arm 标记（_base / _cand）决定使用基线或候选对错映射，
|
||||
将预测写入 log 的同一 DB，返回 InferenceResult。
|
||||
"""
|
||||
call_log: list[dict[str, Any]] = []
|
||||
|
||||
async def mock_fn(
|
||||
questions: list[GeneratedQuestion],
|
||||
*,
|
||||
run_id: str,
|
||||
skills_dir: Path,
|
||||
) -> InferenceResult:
|
||||
is_baseline = run_id.endswith("_base")
|
||||
correctness = baseline_correctness if is_baseline else candidate_correctness
|
||||
|
||||
call_log.append({"run_id": run_id, "skills_dir": skills_dir, "n": len(questions)})
|
||||
per_q = {q.question_id: correctness.get(q.question_id, False) for q in questions}
|
||||
_insert_predictions(log, run_id, per_q)
|
||||
|
||||
correct = sum(per_q.values())
|
||||
total = len(questions)
|
||||
stop_counts: dict[str, int] = {"completed": total - error_count}
|
||||
if error_count > 0:
|
||||
stop_counts["error"] = error_count
|
||||
return InferenceResult(
|
||||
run_id=run_id,
|
||||
accuracy=correct / total if total else 0.0,
|
||||
total=total,
|
||||
correct=correct,
|
||||
per_task_type={},
|
||||
steps_mean=1.0,
|
||||
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
|
||||
stop_reason_counts=stop_counts,
|
||||
)
|
||||
|
||||
return mock_fn, call_log
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 数据类型测试
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestValidationOutcomeFields:
|
||||
"""ValidationOutcome 数据类型字段完整性测试。"""
|
||||
|
||||
def test_validation_outcome_fields(self) -> None:
|
||||
"""所有字段可构造、默认值合理。"""
|
||||
outcome = ValidationOutcome(
|
||||
action="accept_confirmed",
|
||||
accepted=True,
|
||||
stop_reason="confirmed",
|
||||
e_value=25.0,
|
||||
w=5,
|
||||
l=1,
|
||||
n_used=10,
|
||||
delta_hat=0.4,
|
||||
delta_shrunk=0.3,
|
||||
baseline_acc=0.6,
|
||||
candidate_acc=0.9,
|
||||
)
|
||||
assert outcome.action == "accept_confirmed"
|
||||
assert outcome.accepted is True
|
||||
assert outcome.stop_reason == "confirmed"
|
||||
assert outcome.e_value == 25.0
|
||||
assert outcome.w == 5
|
||||
assert outcome.l == 1
|
||||
assert outcome.n_used == 10
|
||||
assert outcome.delta_hat == 0.4
|
||||
assert outcome.delta_shrunk == 0.3
|
||||
assert outcome.baseline_acc == 0.6
|
||||
assert outcome.candidate_acc == 0.9
|
||||
assert outcome.improvements == []
|
||||
assert outcome.regressions == []
|
||||
assert outcome.persistent_fails == []
|
||||
assert outcome.stable_successes == []
|
||||
assert outcome.candidate_correctness == {}
|
||||
assert outcome.evidence_rows == []
|
||||
|
||||
|
||||
class TestProbationFields:
|
||||
"""Probation 数据类型字段完整性测试。"""
|
||||
|
||||
def test_probation_fields(self) -> None:
|
||||
"""所有字段可构造、pending_edits 默认空列表。"""
|
||||
prob = Probation(
|
||||
task_type="temporal",
|
||||
anchor_skills_version="v1",
|
||||
target_file="temporal.md",
|
||||
correctness_snapshot={"q0": True, "q1": False},
|
||||
opened_step=5,
|
||||
)
|
||||
assert prob.task_type == "temporal"
|
||||
assert prob.anchor_skills_version == "v1"
|
||||
assert prob.target_file == "temporal.md"
|
||||
assert prob.correctness_snapshot == {"q0": True, "q1": False}
|
||||
assert prob.opened_step == 5
|
||||
assert prob.pending_edits == []
|
||||
|
||||
def test_probation_with_pending_edits(self) -> None:
|
||||
"""pending_edits 可附加 RejectedEdit。"""
|
||||
edit = RejectedEdit(
|
||||
target_file="temporal.md",
|
||||
target_type="skill",
|
||||
change_summary="bad change",
|
||||
delta=-0.1,
|
||||
source_version="v2",
|
||||
epoch=1,
|
||||
)
|
||||
prob = Probation(
|
||||
task_type="temporal",
|
||||
anchor_skills_version="v1",
|
||||
target_file="temporal.md",
|
||||
correctness_snapshot={},
|
||||
opened_step=3,
|
||||
pending_edits=[edit],
|
||||
)
|
||||
assert len(prob.pending_edits) == 1
|
||||
assert prob.pending_edits[0].change_summary == "bad change"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# materialize 测试
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestMaterializeCandidateSkill:
|
||||
"""materialize_candidate_skill 物化与清理测试。"""
|
||||
|
||||
def test_materialize_candidate_skill(self, tmp_path: Path) -> None:
|
||||
"""正常物化：基线目录被复制，target_file 被覆写为候选内容。"""
|
||||
workspace = _setup_workspace(tmp_path)
|
||||
cand_dir = materialize_candidate_skill(
|
||||
workspace, "v1", "temporal.md", "candidate skill content"
|
||||
)
|
||||
try:
|
||||
assert cand_dir.exists()
|
||||
assert cand_dir.parent == workspace / ".cand_tmp"
|
||||
assert (cand_dir / "temporal.md").read_text(encoding="utf-8") == (
|
||||
"candidate skill content"
|
||||
)
|
||||
finally:
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(cand_dir)
|
||||
|
||||
def test_materialize_cleanup_on_failure(self, tmp_path: Path) -> None:
|
||||
"""基线目录不存在时 OSError，临时目录被清理。"""
|
||||
workspace = tmp_path / "ws"
|
||||
workspace.mkdir()
|
||||
# 不创建 skills/v1，copytree 应失败
|
||||
with pytest.raises(OSError):
|
||||
materialize_candidate_skill(workspace, "v1", "temporal.md", "content")
|
||||
# .cand_tmp 可能存在但内部应被清理
|
||||
cand_tmp = workspace / ".cand_tmp"
|
||||
if cand_tmp.exists():
|
||||
remaining = list(cand_tmp.iterdir())
|
||||
assert remaining == [], f"临时目录未被清理: {remaining}"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# async 验证测试
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validate_skill_local_accept(tmp_path: Path) -> None:
|
||||
"""候选全对、基线全错 → 高 e 值 → accept_confirmed。"""
|
||||
workspace = _setup_workspace(tmp_path)
|
||||
log = _make_log(workspace)
|
||||
questions = _make_questions(6)
|
||||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||||
|
||||
# 基线全错，候选全对 → W=6, L=0 → E=18.14 → CONFIRMED（e_confirm=15）
|
||||
baseline_correct = {f"q{i}": False for i in range(6)}
|
||||
candidate_correct = {f"q{i}": True for i in range(6)}
|
||||
mock_fn, call_log = _make_mock_run_inference(log, baseline_correct, candidate_correct)
|
||||
|
||||
# e_confirm=15 使 E=18.14 超过阈值触发 CONFIRMED
|
||||
accept_params = GateParams(
|
||||
e_confirm=15.0,
|
||||
e_provisional=3.0,
|
||||
w_net_min=2,
|
||||
delta_min=0.05,
|
||||
lambda_dir=-2.0,
|
||||
e_rollback=10.0,
|
||||
)
|
||||
|
||||
try:
|
||||
outcome = await validate_skill_local(
|
||||
workspace_dir=workspace,
|
||||
base_skills_version="v1",
|
||||
task_type="temporal",
|
||||
target_file="temporal.md",
|
||||
candidate_content="improved skill",
|
||||
base_skill_content="baseline skill content",
|
||||
ladder_items=questions,
|
||||
gate_params=accept_params,
|
||||
gate_block=6,
|
||||
gate_n_max=20,
|
||||
gate_guard_err=0.5,
|
||||
baseline_cache=cache,
|
||||
prompts_version="p1",
|
||||
run_inference=mock_fn,
|
||||
log=log,
|
||||
gate_run_prefix="step1_gate_test",
|
||||
)
|
||||
|
||||
assert outcome.accepted is True
|
||||
assert outcome.action == "accept_confirmed"
|
||||
assert outcome.stop_reason == "confirmed"
|
||||
assert outcome.w == 6
|
||||
assert outcome.l == 0
|
||||
assert outcome.n_used == 6
|
||||
assert outcome.candidate_acc == 1.0
|
||||
assert outcome.baseline_acc == 0.0
|
||||
assert len(outcome.evidence_rows) == 6
|
||||
# 终态证据行携带 stop_reason
|
||||
assert outcome.evidence_rows[-1]["stop_reason"] == "confirmed"
|
||||
# 候选临时目录应被清理
|
||||
cand_tmp = workspace / ".cand_tmp"
|
||||
if cand_tmp.exists():
|
||||
assert list(cand_tmp.iterdir()) == []
|
||||
finally:
|
||||
log.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validate_skill_local_reject(tmp_path: Path) -> None:
|
||||
"""候选全错、基线全对 → L 高 → 方向拒绝。"""
|
||||
workspace = _setup_workspace(tmp_path)
|
||||
log = _make_log(workspace)
|
||||
questions = _make_questions(6)
|
||||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||||
|
||||
# 基线全对，候选全错 → W=0, L=6 → 方向拒绝
|
||||
baseline_correct = {f"q{i}": True for i in range(6)}
|
||||
candidate_correct = {f"q{i}": False for i in range(6)}
|
||||
mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct)
|
||||
|
||||
try:
|
||||
outcome = await validate_skill_local(
|
||||
workspace_dir=workspace,
|
||||
base_skills_version="v1",
|
||||
task_type="temporal",
|
||||
target_file="temporal.md",
|
||||
candidate_content="bad skill",
|
||||
base_skill_content="baseline skill content",
|
||||
ladder_items=questions,
|
||||
gate_params=_DEFAULT_GATE_PARAMS,
|
||||
gate_block=6,
|
||||
gate_n_max=20,
|
||||
gate_guard_err=0.5,
|
||||
baseline_cache=cache,
|
||||
prompts_version="p1",
|
||||
run_inference=mock_fn,
|
||||
log=log,
|
||||
gate_run_prefix="step1_gate_test",
|
||||
)
|
||||
|
||||
assert outcome.accepted is False
|
||||
assert outcome.action == "reject"
|
||||
assert outcome.stop_reason == "directional"
|
||||
assert outcome.w == 0
|
||||
assert outcome.l == 6
|
||||
finally:
|
||||
log.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gate_prefix_must_contain_gate(tmp_path: Path) -> None:
|
||||
"""gate_run_prefix 不含 '_gate_' 时抛 ValueError。"""
|
||||
workspace = _setup_workspace(tmp_path)
|
||||
log = _make_log(workspace)
|
||||
questions = _make_questions(4)
|
||||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||||
|
||||
async def noop_fn(questions, *, run_id, skills_dir):
|
||||
raise AssertionError("不应被调用")
|
||||
|
||||
try:
|
||||
with pytest.raises(ValueError, match="_gate_"):
|
||||
await validate_skill_local(
|
||||
workspace_dir=workspace,
|
||||
base_skills_version="v1",
|
||||
task_type="temporal",
|
||||
target_file="temporal.md",
|
||||
candidate_content="content",
|
||||
base_skill_content="baseline",
|
||||
ladder_items=questions,
|
||||
gate_params=_DEFAULT_GATE_PARAMS,
|
||||
gate_block=4,
|
||||
gate_n_max=20,
|
||||
gate_guard_err=0.5,
|
||||
baseline_cache=cache,
|
||||
prompts_version="p1",
|
||||
run_inference=noop_fn,
|
||||
log=log,
|
||||
gate_run_prefix="step1_no_marker",
|
||||
)
|
||||
finally:
|
||||
log.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_infra_guard_threshold(tmp_path: Path) -> None:
|
||||
"""推理错误率超阈值时抛 RuntimeError。"""
|
||||
workspace = _setup_workspace(tmp_path)
|
||||
log = _make_log(workspace)
|
||||
# 需要 >=10 题次才触发 INFRA 护栏
|
||||
questions = _make_questions(6)
|
||||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||||
|
||||
baseline_correct = {f"q{i}": False for i in range(6)}
|
||||
candidate_correct = {f"q{i}": False for i in range(6)}
|
||||
# 每次 run_inference 报 error_count=5，两侧各 5 → 10/12 > 0.5
|
||||
mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct, error_count=5)
|
||||
|
||||
try:
|
||||
with pytest.raises(RuntimeError, match="错误率过高"):
|
||||
await validate_skill_local(
|
||||
workspace_dir=workspace,
|
||||
base_skills_version="v1",
|
||||
task_type="temporal",
|
||||
target_file="temporal.md",
|
||||
candidate_content="content",
|
||||
base_skill_content="baseline skill content",
|
||||
ladder_items=questions,
|
||||
gate_params=_DEFAULT_GATE_PARAMS,
|
||||
gate_block=6,
|
||||
gate_n_max=20,
|
||||
gate_guard_err=0.5,
|
||||
baseline_cache=cache,
|
||||
prompts_version="p1",
|
||||
run_inference=mock_fn,
|
||||
log=log,
|
||||
gate_run_prefix="step1_gate_test",
|
||||
)
|
||||
finally:
|
||||
log.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_baseline_cache_hit(tmp_path: Path) -> None:
|
||||
"""基线缓存全命中时不发起基线侧推理。"""
|
||||
workspace = _setup_workspace(tmp_path)
|
||||
log = _make_log(workspace)
|
||||
questions = _make_questions(4)
|
||||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||||
|
||||
s_hash = skill_hash("baseline skill content")
|
||||
# 预填充缓存：全部题目基线全错
|
||||
for q in questions:
|
||||
cache.put("temporal", s_hash, "p1", q.question_id, False)
|
||||
|
||||
# 候选全对 → accept
|
||||
candidate_correct = {f"q{i}": True for i in range(4)}
|
||||
baseline_correct = {f"q{i}": False for i in range(4)}
|
||||
mock_fn, call_log = _make_mock_run_inference(log, baseline_correct, candidate_correct)
|
||||
|
||||
try:
|
||||
outcome = await validate_skill_local(
|
||||
workspace_dir=workspace,
|
||||
base_skills_version="v1",
|
||||
task_type="temporal",
|
||||
target_file="temporal.md",
|
||||
candidate_content="improved skill",
|
||||
base_skill_content="baseline skill content",
|
||||
ladder_items=questions,
|
||||
gate_params=_DEFAULT_GATE_PARAMS,
|
||||
gate_block=4,
|
||||
gate_n_max=20,
|
||||
gate_guard_err=0.5,
|
||||
baseline_cache=cache,
|
||||
prompts_version="p1",
|
||||
run_inference=mock_fn,
|
||||
log=log,
|
||||
gate_run_prefix="step1_gate_test",
|
||||
)
|
||||
|
||||
# 只有候选侧调用了 run_inference（_cand），基线侧全命中不调用
|
||||
base_calls = [c for c in call_log if c["run_id"].endswith("_base")]
|
||||
cand_calls = [c for c in call_log if c["run_id"].endswith("_cand")]
|
||||
assert len(base_calls) == 0, "基线缓存全命中不应发起推理"
|
||||
assert len(cand_calls) == 1
|
||||
assert outcome.accepted is True
|
||||
finally:
|
||||
log.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_last_block_terminal(tmp_path: Path) -> None:
|
||||
"""单块 + n_remaining=0 → 终态判定（provisional 或 inertia），非 continue。"""
|
||||
workspace = _setup_workspace(tmp_path)
|
||||
log = _make_log(workspace)
|
||||
# 4 题，gate_block=4 → 一块走完，n_remaining=0
|
||||
questions = _make_questions(4)
|
||||
cache = BaselineCache(workspace / "baseline_cache.json")
|
||||
|
||||
# 两题翻转（W=2, L=0），但 e_confirm=20 难以达到 → provisional 或 inertia
|
||||
baseline_correct = {"q0": False, "q1": False, "q2": True, "q3": True}
|
||||
candidate_correct = {"q0": True, "q1": True, "q2": True, "q3": True}
|
||||
mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct)
|
||||
|
||||
try:
|
||||
outcome = await validate_skill_local(
|
||||
workspace_dir=workspace,
|
||||
base_skills_version="v1",
|
||||
task_type="temporal",
|
||||
target_file="temporal.md",
|
||||
candidate_content="candidate skill",
|
||||
base_skill_content="baseline skill content",
|
||||
ladder_items=questions,
|
||||
gate_params=_DEFAULT_GATE_PARAMS,
|
||||
gate_block=4,
|
||||
gate_n_max=4,
|
||||
gate_guard_err=0.5,
|
||||
baseline_cache=cache,
|
||||
prompts_version="p1",
|
||||
run_inference=mock_fn,
|
||||
log=log,
|
||||
gate_run_prefix="step1_gate_test",
|
||||
)
|
||||
|
||||
# n_remaining=0 → 不可能是 continue
|
||||
assert outcome.stop_reason in (
|
||||
"confirmed",
|
||||
"provisional",
|
||||
"inertia",
|
||||
"directional",
|
||||
"futility",
|
||||
)
|
||||
assert outcome.n_used == 4
|
||||
# 终态行标记 stop_reason
|
||||
assert outcome.evidence_rows[-1]["stop_reason"] != ""
|
||||
finally:
|
||||
log.close()
|
||||
@@ -0,0 +1,519 @@
|
||||
"""app/harness/workspace 单元测试。
|
||||
|
||||
覆盖 Workspace 生命周期管理、manifest 读写、
|
||||
VersionedSkillStore / VersionedPromptStore Protocol 合规。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.harness.workspace import (
|
||||
ResolvedPaths,
|
||||
VersionedPromptStore,
|
||||
VersionedSkillStore,
|
||||
archive_workspace,
|
||||
init_workspace,
|
||||
init_workspace_from_seed,
|
||||
list_video_ids,
|
||||
load_manifest,
|
||||
read_best,
|
||||
record_run,
|
||||
resolve_paths,
|
||||
update_best,
|
||||
update_manifest,
|
||||
)
|
||||
from core.evolution.protocols import PromptStore, SkillStore
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def store_dir(tmp_path: Path) -> Path:
|
||||
"""构建一个最小 Store 目录结构供测试使用。"""
|
||||
sd = tmp_path / "store"
|
||||
sd.mkdir()
|
||||
|
||||
# videos：两个含 tree.json 的视频目录
|
||||
for vid in ("vid_A", "vid_B"):
|
||||
vdir = sd / "videos" / vid
|
||||
vdir.mkdir(parents=True)
|
||||
(vdir / "tree.json").write_text("{}")
|
||||
|
||||
# skills/v1 + prompts/v1
|
||||
s1 = sd / "skills" / "v1"
|
||||
s1.mkdir(parents=True)
|
||||
(s1 / "temporal-reasoning.md").write_text("skill content A")
|
||||
(s1 / "spatial-analysis.md").write_text("skill content B")
|
||||
|
||||
p1 = sd / "prompts" / "v1"
|
||||
p1.mkdir(parents=True)
|
||||
(p1 / "system.md").write_text("system prompt v1")
|
||||
(p1 / "extract.md").write_text("extract prompt v1")
|
||||
|
||||
# questions/benchmarks/Video-MME
|
||||
q = sd / "questions" / "benchmarks" / "Video-MME"
|
||||
q.mkdir(parents=True)
|
||||
(q / "q1.json").write_text('{"id": "q1"}')
|
||||
|
||||
# seed "initial"
|
||||
seed_dir = sd / "seeds" / "initial"
|
||||
seed_dir.mkdir(parents=True)
|
||||
shutil.copytree(s1, seed_dir / "skills")
|
||||
shutil.copytree(p1, seed_dir / "prompts")
|
||||
baseline_db = seed_dir / "baseline.db"
|
||||
baseline_db.write_bytes(b"")
|
||||
(seed_dir / "seed.json").write_text(
|
||||
json.dumps({"baseline_run_id": "baseline-001", "parent": None})
|
||||
)
|
||||
return sd
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def workspace_dir(tmp_path: Path) -> Path:
|
||||
"""返回一个不存在的 workspace 路径。"""
|
||||
return tmp_path / "ws_test"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ResolvedPaths
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolved_paths_frozen() -> None:
|
||||
"""ResolvedPaths 实例应不可变。"""
|
||||
rp = ResolvedPaths(
|
||||
store_dir=Path("/s"),
|
||||
videos_dir=Path("/v"),
|
||||
questions_dir=Path("/q"),
|
||||
skills_dir=Path("/sk"),
|
||||
prompts_dir=Path("/p"),
|
||||
workspace_dir=Path("/w"),
|
||||
db_path=Path("/d"),
|
||||
analyses_dir=Path("/a"),
|
||||
runs_dir=Path("/r"),
|
||||
)
|
||||
with pytest.raises(AttributeError):
|
||||
rp.store_dir = Path("/other") # type: ignore[misc]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# init_workspace
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_init_workspace(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""init_workspace 应创建 manifest、拷贝权重。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
|
||||
# manifest 存在
|
||||
manifest = json.loads((workspace_dir / "manifest.json").read_text())
|
||||
assert manifest["current"]["skills"] == "skills/v1"
|
||||
assert manifest["current"]["prompts"] == "prompts/v1"
|
||||
assert manifest["current"]["questions"] == "questions/benchmarks/Video-MME"
|
||||
|
||||
# 权重已拷入 workspace
|
||||
assert (workspace_dir / "skills" / "v1" / "temporal-reasoning.md").exists()
|
||||
assert (workspace_dir / "prompts" / "v1" / "system.md").exists()
|
||||
|
||||
# analyses, runs 目录已创建
|
||||
assert (workspace_dir / "analyses").is_dir()
|
||||
assert (workspace_dir / "runs").is_dir()
|
||||
|
||||
# 重复创建应报错
|
||||
with pytest.raises(FileExistsError):
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# init_workspace_from_seed
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_init_workspace_from_seed(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""从种子初始化应拷权重 + baseline.db + 返回 baseline_run_id。"""
|
||||
run_id = init_workspace_from_seed(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
seed_name="initial",
|
||||
questions="benchmarks/Video-MME",
|
||||
)
|
||||
assert run_id == "baseline-001"
|
||||
|
||||
# 权重在 workspace
|
||||
assert (workspace_dir / "skills" / "v1" / "temporal-reasoning.md").exists()
|
||||
assert (workspace_dir / "prompts" / "v1" / "system.md").exists()
|
||||
|
||||
# baseline.db -> harness.db
|
||||
assert (workspace_dir / "harness.db").exists()
|
||||
|
||||
# manifest 正确
|
||||
manifest = json.loads((workspace_dir / "manifest.json").read_text())
|
||||
assert manifest["current"]["skills"] == "skills/v1"
|
||||
|
||||
|
||||
def test_init_workspace_from_seed_missing_questions(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""questions ref 不存在应 fail-fast（FileNotFoundError），不创建 workspace。"""
|
||||
with pytest.raises(FileNotFoundError, match="questions ref"):
|
||||
init_workspace_from_seed(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
seed_name="initial",
|
||||
questions="benchmarks/NONEXISTENT",
|
||||
)
|
||||
# workspace 不应被创建
|
||||
assert not workspace_dir.exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# load_manifest
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_load_manifest(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""正常加载 manifest。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
manifest = load_manifest(workspace_dir)
|
||||
assert "current" in manifest
|
||||
assert "history" in manifest
|
||||
assert manifest["current"]["skills"] == "skills/v1"
|
||||
|
||||
|
||||
def test_load_manifest_missing(workspace_dir: Path) -> None:
|
||||
"""manifest 不存在应 FileNotFoundError。"""
|
||||
workspace_dir.mkdir(parents=True)
|
||||
with pytest.raises(FileNotFoundError):
|
||||
load_manifest(workspace_dir)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# update_manifest
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_update_manifest_invalid_key(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""非法 key 应 KeyError。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
with pytest.raises(KeyError, match="无效"):
|
||||
update_manifest(workspace_dir, bad_key="skills/v2")
|
||||
|
||||
|
||||
def test_update_manifest_valid(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""合法 key 应更新 manifest current。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
update_manifest(workspace_dir, skills="skills/v2")
|
||||
manifest = load_manifest(workspace_dir)
|
||||
assert manifest["current"]["skills"] == "skills/v2"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# record_run
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_record_run_idempotent(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""同 run_id 调用两次不应重复追加 history。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
run_dir1 = record_run(workspace_dir, "run_001")
|
||||
run_dir2 = record_run(workspace_dir, "run_001")
|
||||
|
||||
# 返回路径一致
|
||||
assert run_dir1 == run_dir2
|
||||
|
||||
# history 只有一条
|
||||
manifest = load_manifest(workspace_dir)
|
||||
matched = [h for h in manifest["history"] if h["run_id"] == "run_001"]
|
||||
assert len(matched) == 1
|
||||
|
||||
# run 目录存在
|
||||
assert run_dir1.is_dir()
|
||||
|
||||
# per-video wiki 目录存在
|
||||
assert (run_dir1 / "vid_A" / "wiki").is_dir()
|
||||
assert (run_dir1 / "vid_B" / "wiki").is_dir()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# update_best / read_best
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_update_best_independent_of_current(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""best 应独立于 current——更新 best 不影响 current。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
|
||||
# 初始无 best
|
||||
assert read_best(workspace_dir) is None
|
||||
|
||||
# 设置 best
|
||||
update_best(
|
||||
workspace_dir,
|
||||
skills="skills/v3",
|
||||
prompts="prompts/v3",
|
||||
val_acc=0.85,
|
||||
run_id="run_005",
|
||||
epoch=3,
|
||||
)
|
||||
|
||||
best = read_best(workspace_dir)
|
||||
assert best is not None
|
||||
assert best["skills"] == "skills/v3"
|
||||
assert best["val_acc"] == 0.85
|
||||
assert best["epoch"] == 3
|
||||
|
||||
# current 不受影响
|
||||
manifest = load_manifest(workspace_dir)
|
||||
assert manifest["current"]["skills"] == "skills/v1"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# archive_workspace
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_archive_workspace(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""归档应把 workspace 移到 .archive/ 下。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
archived = archive_workspace(workspace_dir)
|
||||
|
||||
# 原目录不存在
|
||||
assert not workspace_dir.exists()
|
||||
|
||||
# 归档目录存在并包含 manifest
|
||||
assert archived.is_dir()
|
||||
assert (archived / "manifest.json").exists()
|
||||
|
||||
# 归档路径在 .archive 下
|
||||
assert archived.parent.name == ".archive"
|
||||
|
||||
|
||||
def test_archive_workspace_missing(workspace_dir: Path) -> None:
|
||||
"""归档不存在的 workspace 应 FileNotFoundError。"""
|
||||
with pytest.raises(FileNotFoundError):
|
||||
archive_workspace(workspace_dir)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# list_video_ids
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_list_video_ids(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""列出 workspace 引用的所有视频 ID。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
ids = list_video_ids(workspace_dir)
|
||||
assert ids == ["vid_A", "vid_B"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_paths
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolve_paths(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""resolve_paths 应正确解析绝对路径。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
rp = resolve_paths(workspace_dir)
|
||||
|
||||
# skills_dir/prompts_dir 解析到 workspace（非 store）
|
||||
ws_abs = workspace_dir.resolve()
|
||||
assert rp.skills_dir == ws_abs / "skills" / "v1"
|
||||
assert rp.prompts_dir == ws_abs / "prompts" / "v1"
|
||||
|
||||
# videos/questions 解析到 store
|
||||
store_abs = store_dir.resolve()
|
||||
assert rp.videos_dir == store_abs / "videos"
|
||||
assert rp.questions_dir == store_abs / "questions" / "benchmarks" / "Video-MME"
|
||||
|
||||
# db_path, analyses, runs
|
||||
assert rp.db_path == ws_abs / "harness.db"
|
||||
assert rp.analyses_dir == ws_abs / "analyses"
|
||||
assert rp.runs_dir == ws_abs / "runs"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VersionedSkillStore
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_versioned_skill_store_read(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""read_skill 应返回文件全文。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
skills_dir = workspace_dir / "skills" / "v1"
|
||||
store = VersionedSkillStore(skills_dir)
|
||||
content = store.read_skill("temporal-reasoning.md")
|
||||
assert content == "skill content A"
|
||||
|
||||
|
||||
def test_versioned_skill_store_read_missing(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""读取不存在的 skill 文件应 FileNotFoundError。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
skills_dir = workspace_dir / "skills" / "v1"
|
||||
store = VersionedSkillStore(skills_dir)
|
||||
with pytest.raises(FileNotFoundError):
|
||||
store.read_skill("nonexistent.md")
|
||||
|
||||
|
||||
def test_versioned_skill_store_list(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""list_skill_files 应列出所有 skill 文件名（排序）。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
skills_dir = workspace_dir / "skills" / "v1"
|
||||
store = VersionedSkillStore(skills_dir)
|
||||
files = store.list_skill_files()
|
||||
assert sorted(files) == ["spatial-analysis.md", "temporal-reasoning.md"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# VersionedPromptStore
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_versioned_prompt_store(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""VersionedPromptStore 读写功能验证。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
prompts_dir = workspace_dir / "prompts" / "v1"
|
||||
store = VersionedPromptStore(prompts_dir)
|
||||
|
||||
content = store.read_prompt("system.md")
|
||||
assert content == "system prompt v1"
|
||||
|
||||
files = store.list_prompt_files()
|
||||
assert sorted(files) == ["extract.md", "system.md"]
|
||||
|
||||
|
||||
def test_versioned_prompt_store_read_missing(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""读取不存在的 prompt 文件应 FileNotFoundError。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
prompts_dir = workspace_dir / "prompts" / "v1"
|
||||
store = VersionedPromptStore(prompts_dir)
|
||||
with pytest.raises(FileNotFoundError):
|
||||
store.read_prompt("nonexistent.md")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Protocol 合规
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_skill_store_protocol_compliance(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""VersionedSkillStore 应满足 core/evolution/protocols.py::SkillStore Protocol。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
skills_dir = workspace_dir / "skills" / "v1"
|
||||
store = VersionedSkillStore(skills_dir)
|
||||
assert isinstance(store, SkillStore)
|
||||
|
||||
|
||||
def test_prompt_store_protocol_compliance(store_dir: Path, workspace_dir: Path) -> None:
|
||||
"""VersionedPromptStore 应满足 core/evolution/protocols.py::PromptStore Protocol。"""
|
||||
init_workspace(
|
||||
workspace_dir,
|
||||
store_dir,
|
||||
questions="benchmarks/Video-MME",
|
||||
skills_version="v1",
|
||||
prompts_version="v1",
|
||||
)
|
||||
prompts_dir = workspace_dir / "prompts" / "v1"
|
||||
store = VersionedPromptStore(prompts_dir)
|
||||
assert isinstance(store, PromptStore)
|
||||
Reference in New Issue
Block a user