Merge branch 'feat/preflight-train-fixes': preflight training fixes (4 WPs, 21+ defects)

This commit is contained in:
2026-07-16 07:25:50 -04:00
77 changed files with 4813 additions and 277 deletions
+1
View File
@@ -46,6 +46,7 @@ LLM_CIRCUIT_BREAKER_COOLDOWN=60
LLM_TTFT_TIMEOUT=30
LLM_INTER_TOKEN_TIMEOUT=15
LLM_RETRY_MAX_DELAY=30.0
# 正整数秒,禁止 0(0 会被拒绝启动);训练场景建议 >= 单次训练时长
REDIS_CACHE_TTL=86400
# 建树批量并行:全局 VLM/LLM 在途调用上限(Spec-2 工程配置)
+14 -4
View File
@@ -106,21 +106,31 @@ class SqliteDiagnosisSignalStore:
)
self._conn.commit()
def done_question_ids(self, baseline_run_id: str, diag_fingerprint: str) -> set[str]:
def done_question_ids(
self,
baseline_run_id: str,
diag_fingerprint: str,
*,
retry_uncertain: bool = False,
) -> set[str]:
"""查询指定 run 与诊断指纹下已完成的 question_id 集合。
参数:
baseline_run_id: baseline run 标识。
diag_fingerprint: 诊断口径指纹。
retry_uncertain: True 时追加 `AND tier != 'uncertain'`,把 uncertain
(信号不可信降级)题排除出已完成集,令其被重新诊断;默认 False。
返回:
已落盘信号的 question_id 集合;无匹配时为空集,供断点续跑跳过。
"""
cursor = self._conn.execute(
sql = (
"SELECT DISTINCT question_id FROM baseline_diagnosis"
" WHERE baseline_run_id = ? AND diag_fingerprint = ?",
(baseline_run_id, diag_fingerprint),
" WHERE baseline_run_id = ? AND diag_fingerprint = ?"
)
if retry_uncertain:
sql += " AND tier != 'uncertain'"
cursor = self._conn.execute(sql, (baseline_run_id, diag_fingerprint))
return {r["question_id"] for r in cursor.fetchall()}
def load(self, baseline_run_id: str, diag_fingerprint: str) -> list[DiagnosisSignalRow]:
+18 -3
View File
@@ -20,21 +20,32 @@ class CircuitBreaker:
self._cooldown_s = cooldown_s
self._fails: dict[str, int] = {}
self._open_until: dict[str, float] = {}
self._half_open_inflight: dict[str, bool] = {}
def is_open(self, source_name: str, now: float) -> bool:
"""判断指定源是否处于开路状态。
冷却截止时刻之前为开路;到期返回 False(放行一个试探,即半开)。
冷却截止时刻之前为开路;到期进入半开,**只放行一个探针**(其余仍被挡),
避免冷却到期瞬间惊群重连再次压垮上游。"检查+标记探针"在 asyncio 单线程内
同步执行,天然原子无竞态。
Args:
source_name: 被熔断的源标识。
now: 当前时刻(秒级时间戳),由调用方注入。
Returns:
True 表示开路(拒绝请求),False 表示关闭或半开放行
True 表示开路(拒绝请求),False 表示关闭或半开放行探针
"""
until = self._open_until.get(source_name)
return until is not None and now < until
if until is None:
return False
if now < until:
return True # 冷却中,全挡
# 冷却到期:half-open,只放行一个探针
if self._half_open_inflight.get(source_name):
return True # 已有探针在途,继续挡
self._half_open_inflight[source_name] = True
return False
def record_failure(self, source_name: str, now: float) -> None:
"""记录一次失败;累计达阈值则开路至 now + cooldown。
@@ -47,6 +58,8 @@ class CircuitBreaker:
self._fails[source_name] = count
if count >= self._fail_threshold:
self._open_until[source_name] = now + self._cooldown_s
# 探针失败清在途标记,使下一轮 cooldown 到期后可再放行探针
self._half_open_inflight.pop(source_name, None)
def force_open(self, source_name: str, now: float) -> None:
"""强制开路(用于 401/403 等不可恢复错误),一次即熔断。
@@ -59,6 +72,7 @@ class CircuitBreaker:
"""
self._fails[source_name] = self._fail_threshold
self._open_until[source_name] = now + self._cooldown_s
self._half_open_inflight.pop(source_name, None)
def record_success(self, source_name: str) -> None:
"""记录一次成功;清零失败计数与开路状态(关闭熔断器)。
@@ -68,3 +82,4 @@ class CircuitBreaker:
"""
self._fails.pop(source_name, None)
self._open_until.pop(source_name, None)
self._half_open_inflight.pop(source_name, None)
+16 -3
View File
@@ -179,7 +179,10 @@ def _is_transient_error(exc: Exception) -> bool:
返回:
True 表示可重试,False 表示不可重试。
"""
if isinstance(exc, (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout)):
# 两族基类覆盖断连族:TimeoutExceptionConnectTimeout/ReadTimeout/WriteTimeout/PoolTimeout
# 与 TransportErrorConnectError/ReadError/RemoteProtocolError 等)。
# 注意 HTTPStatusError 非 TransportError 子类,401/403 致命分支不受影响。
if isinstance(exc, (httpx.TimeoutException, httpx.TransportError)):
return True
if isinstance(exc, httpx.HTTPStatusError):
return exc.response.status_code in _TRANSIENT_STATUS_CODES
@@ -274,6 +277,7 @@ class GovernedLLMClient:
*,
session_id: str | None = None,
parent_call_id: str | None = None,
cache_salt: str | None = None,
) -> LLMResponse:
"""发起 LLM 调用,经四层治理栈:熔断 → 缓存 → 重试+流式 → 遥测。
@@ -281,6 +285,7 @@ class GovernedLLMClient:
messages: OpenAI 格式消息列表。
session_id: 会话 ID(传递到遥测)。
parent_call_id: 父调用 ID(传递到遥测)。
cache_salt: 可选缓存盐,透传到 Redis 缓存键(如跨 epoch 重采样)。
返回:
LLMResponse 统一响应。
@@ -296,7 +301,11 @@ class GovernedLLMClient:
raise CircuitOpenError(f"熔断器已开启,拒绝调用 provider={self._provider}")
# ② 缓存查询(cache 为 None 时跳过)— call_id 在缓存路径独立生成
cached = await self._cache.get(self._model, messages) if self._cache is not None else None
cached = (
await self._cache.get(self._model, messages, cache_salt)
if self._cache is not None
else None
)
if cached is not None:
cache_call_id = str(uuid4())
response = LLMResponse(
@@ -370,7 +379,7 @@ class GovernedLLMClient:
# ④ 写缓存(cache 为 None 时跳过)
if self._cache is not None:
await self._cache.set(self._model, messages, response)
await self._cache.set(self._model, messages, response, cache_salt)
# ⑤ 遥测
await self._telemetry.record_llm_call(
@@ -572,6 +581,10 @@ class GovernedLLMClient:
else:
thinking_parts.append(text)
# 流耗尽但未收 [DONE] → 服务端截断,视为可重试的 SSE 异常(不写缓存/不当成功)
if not usage_sink.get("done"):
raise _SseAnomaly("truncated_no_done")
content = "".join(content_parts)
thinking = "".join(thinking_parts)
usage = usage_sink.get("usage", {})
+43 -9
View File
@@ -12,6 +12,26 @@ from loguru import logger
from core.types import LLMResponse
def _resolve_cache_ttl(ttl: int) -> int:
"""校验 Redis 缓存 TTL:必须为正整数(消灭 0=永不过期 的隐式语义)。
Args:
ttl: 待校验的 TTL 秒数。
Returns:
校验通过的正整数 TTL。
Raises:
ValueError: ttl <= 0。
"""
if ttl <= 0:
raise ValueError(
f"REDIS_CACHE_TTL 必须为正整数秒,实际 {ttl}"
"训练场景建议 >= 单次训练时长(如 86400)。"
)
return ttl
class RedisResponseCache:
"""基于 Redis 的 LLM 响应缓存。
@@ -29,36 +49,48 @@ class RedisResponseCache:
self._redis = redis
self._ttl_s = ttl_s
def _build_key(self, model: str, messages: list[dict[str, str]]) -> str:
def _build_key(
self,
model: str,
messages: list[dict[str, str]],
cache_salt: str | None = None,
) -> str:
"""构造 content-addressed 缓存键。
Args:
model: 模型名称。
messages: 消息列表。
cache_salt: 可选缓存盐(如跨 epoch 强制重采样)。仅当非 None 时才加入
键 payload,保证默认 None 时键结构与旧缓存一字节不差、旧键不失效。
Returns:
sha256 哈希字符串作为 Redis 键。
"""
payload = json.dumps(
{"model": model, "messages": messages},
sort_keys=True,
ensure_ascii=False,
)
key_obj: dict[str, Any] = {"model": model, "messages": messages}
if cache_salt is not None:
key_obj["salt"] = cache_salt
payload = json.dumps(key_obj, sort_keys=True, ensure_ascii=False)
digest = hashlib.sha256(payload.encode("utf-8")).hexdigest()
return f"llm_cache:{digest}"
async def get(self, model: str, messages: list[dict[str, str]]) -> LLMResponse | None:
async def get(
self,
model: str,
messages: list[dict[str, str]],
cache_salt: str | None = None,
) -> LLMResponse | None:
"""从缓存读取 LLM 响应。
Args:
model: 模型名称。
messages: 消息列表。
cache_salt: 可选缓存盐,透传到键构造。
Returns:
缓存命中时返回 LLMResponse,未命中或 Redis 异常时返回 None。
"""
try:
key = self._build_key(model, messages)
key = self._build_key(model, messages, cache_salt)
raw = await self._redis.get(key)
except Exception:
logger.warning("Redis 缓存读取失败,降级为未命中")
@@ -75,6 +107,7 @@ class RedisResponseCache:
model: str,
messages: list[dict[str, str]],
response: LLMResponse,
cache_salt: str | None = None,
) -> None:
"""将 LLM 响应写入缓存。
@@ -82,9 +115,10 @@ class RedisResponseCache:
model: 模型名称。
messages: 消息列表。
response: 待缓存的 LLMResponse。
cache_salt: 可选缓存盐,透传到键构造。
"""
try:
key = self._build_key(model, messages)
key = self._build_key(model, messages, cache_salt)
value = json.dumps(dataclasses.asdict(response), ensure_ascii=False)
if self._ttl_s:
await self._redis.set(key, value, ex=self._ttl_s)
+3
View File
@@ -36,6 +36,7 @@ class GovernedVLMClient:
*,
session_id: str | None = None,
parent_call_id: str | None = None,
cache_salt: str | None = None,
) -> LLMResponse:
"""图文调用:将图片编码为 base64 嵌入 messages,委托给 LLM 客户端。
@@ -45,6 +46,7 @@ class GovernedVLMClient:
images: 图片文件路径列表。
session_id: 会话 ID(遥测用)。
parent_call_id: 父调用 ID(遥测用)。
cache_salt: 可选缓存盐,透传到底层 LLM 缓存键。
返回:
LLMResponse。
@@ -54,6 +56,7 @@ class GovernedVLMClient:
vision_messages,
session_id=session_id,
parent_call_id=parent_call_id,
cache_salt=cache_salt,
)
@staticmethod
+13 -5
View File
@@ -6,7 +6,9 @@
包装内层 RunLog,兼容 traces 未落表的历史 run);
3. 把 error_attributions / infra / degraded 三类产物确定性投影为
DiagnosisSignalRowtier 由 split_selection.score_signal 判定);
4. 逐行 store.upsert 落盘,单行单事务 → 崩溃最多丢正在写的一行。
4. run 末(Phase 3逐行 store.upsert 落库——诊断在 Phase 2 全部跑完后才落库,
故崩溃丢本次 run 未落库的全部结果(不是"仅一行");靠 GovernedLLMClient 的
Redis 缓存缓解重跑时的 LLM 重烧,下次调用命中缓存直接续。
错误处理诚实标注(不谎称全传播):
- run_diagnosis 的 C1/C2 阶段(指标计算、错误归因)网络/API 失败经
@@ -70,8 +72,9 @@ async def run_baseline_diagnosis(
questions: dict[str, GeneratedQuestion],
store: DiagnosisSignalStore,
deps: DiagnosisDeps,
retry_uncertain: bool = False,
) -> None:
"""对 baseline run 的错题跑离线诊断并把信号逐行落库(断点续跑幂等)。
"""对 baseline run 的错题跑离线诊断并把信号落库(断点续跑幂等)。
参数:
baseline_run_id: baseline run 标识(如 "infer_adhoc"),信号行主键之一。
@@ -79,8 +82,10 @@ async def run_baseline_diagnosis(
wrong_ids: 本次待诊断的可诊断错题 question_id 列表(保序)。
questions: question_id → GeneratedQuestion 映射,需覆盖 wrong_ids 全部题
及 run_diagnosis 返回的所有 infra/degraded 题(用于取 video_id/task_type)。
store: 诊断信号存储端口,逐行 upsert 落盘并提供 done_question_ids 续跑查询。
store: 诊断信号存储端口,upsert 落盘并提供 done_question_ids 续跑查询。
deps: 外部依赖束(见 DiagnosisDeps)。
retry_uncertain: True 时把已落 tier='uncertain'(信号不可信降级)的题也纳入
remaining 重新诊断,透传给 store.done_question_ids;默认 False。
返回:
None。副作用为把逐题 DiagnosisSignalRow 写入 store。
@@ -93,7 +98,9 @@ async def run_baseline_diagnosis(
T0)、degraded_question_idsuncertain)。
"""
# Phase 1: 算 remaining(续跑幂等)
done = store.done_question_ids(baseline_run_id, diag_fingerprint)
done = store.done_question_ids(
baseline_run_id, diag_fingerprint, retry_uncertain=retry_uncertain
)
remaining = [qid for qid in wrong_ids if qid not in done]
if not remaining:
logger.info(
@@ -169,7 +176,8 @@ def _project_and_persist(
{tier: 行数} 计数字典(T2/T1/T0/uncertain),供上层日志与 manifest。
关键实现:
逐行 upsert(单行单事务),中途崩溃最多丢正在写的一行。三桶**非互斥**:
本函数在 run 末(Phase 3)逐行 upsert(单行单事务);诊断已在 Phase 2 全部
跑完,故本阶段中途崩溃丢本次 run 未落库的余下行。三桶**非互斥**:
同一 degraded 错题可能同时出现在 error_attributionsjudge 解析失败仍建
attribution)里,故按 **degraded > infra > attribution** 优先级去重——先落
degraded/infra,再在 attribution 循环跳过已落题,保证**每题恰写一行、
+87 -5
View File
@@ -12,8 +12,9 @@ test,再以视频组为原子切出诊断 / 验证池,原子冻结 pools.jso
from __future__ import annotations
import hashlib
import json
import sqlite3
from collections import Counter
from collections import Counter, defaultdict
from dataclasses import asdict, dataclass, field
from typing import TYPE_CHECKING
@@ -59,6 +60,8 @@ class SplitBuildConfig:
select_seed: 贪心选择器预洗牌种子(打破等增益平局)。
val_ratio: validation 占 trainval 视频组总数的比例。
split_seed: 视频组题级切分的洗牌种子。
val_wrong_min: validation 池最少错题数,切分时保证功效(不足则从 diag 换入
低 T2 错题组补足,耗尽 fail-loud)。
"""
n_trainval: int
@@ -68,6 +71,7 @@ class SplitBuildConfig:
select_seed: int
val_ratio: float
split_seed: int
val_wrong_min: int
@dataclass(frozen=True)
@@ -101,6 +105,72 @@ class SplitBuildResult:
return getattr(self, key)
def _unique_backup_path(path: Path, suffix: str) -> Path:
"""求 path 的唯一 .bak.<suffix> 备份路径,已存在则追加递增序号避免覆盖。
首选 ``<name>.bak.<suffix>``;若已存在,退化为 ``<name>.bak.<suffix>.2``、
``.3`` … 直到找到不存在的名字。保证连续 forced freeze(同 suffix 或都缺
manifest 用 'prev')不会静默覆盖此前保留的备份。
参数:
path: 待备份的原文件路径。
suffix: 备份后缀(旧 pools_sha256 前 8 位或 'prev')。
返回:
目录内唯一、尚不存在的备份路径。
"""
candidate = path.with_name(f"{path.name}.bak.{suffix}")
counter = 2
while candidate.exists():
candidate = path.with_name(f"{path.name}.bak.{suffix}.{counter}")
counter += 1
return candidate
def _guard_frozen_products(out_path: Path, manifest_path: Path, *, force: bool) -> None:
"""冻结前的覆盖保护:产物已存在时按 force 决定报错或备份。
参数:
out_path: 目标 pools.json 路径。
manifest_path: 目标 split_manifest.json 路径。
force: False 时已存在即 FileExistsErrorTrue 时把旧产物重命名为唯一的
.bak.<旧 pools_sha256 前 8 位或 'prev'>(同名已存在则追加递增序号)再放行。
异常:
FileExistsError: force=False 且产物已存在(防静默覆盖冻结锚点)。
OSError: 备份 rename 失败;已备份的文件先 rollback 回原名再抛出,保证
要么两文件都备份、要么都不动(原子性,不留半备份的不一致目录)。
"""
if not out_path.exists() and not manifest_path.exists():
return
if not force:
raise FileExistsError(
f"已存在冻结产物 {out_path}(或其 manifest)。重跑切分会覆盖训练依赖的"
"冻结锚点——确认要替换请加 --force(旧产物将备份为 .bak.*)。"
)
# 备份后缀取旧 manifest 的 pools_sha256 前 8 位,无则用 'prev'
suffix = "prev"
if manifest_path.exists():
try:
old = json.loads(manifest_path.read_text(encoding="utf-8"))
suffix = str(old.get("pools_sha256", "prev"))[:8] or "prev"
except (json.JSONDecodeError, OSError):
suffix = "prev"
# 先为每个存在的文件求唯一备份路径(互不冲突),再逐个 rename;
# 中途失败则把已备份的 rollback 回原名,保证原子性。
to_backup = [p for p in (out_path, manifest_path) if p.exists()]
done: list[tuple[Path, Path]] = [] # (备份路径, 原路径),供 rollback
try:
for p in to_backup:
dst = _unique_backup_path(p, suffix)
p.rename(dst)
done.append((dst, p))
except OSError:
for backup_path, original in reversed(done):
backup_path.rename(original)
raise
def build_split(
*,
db_path: Path,
@@ -112,6 +182,7 @@ def build_split(
out_path: Path,
manifest_path: Path,
generated_at: str,
force: bool = False,
) -> SplitBuildResult:
"""顶层编排结果驱动视频级切分,冻结 pools.json + manifest 并跑防御断言。
@@ -119,10 +190,10 @@ def build_split(
test → 加载题库并以视频归属切三池 → 原子冻结 pools.json → 写溯源 manifest →
六条防御断言 fail-fast 校验。
契约(Task 11,非疏漏):build_split 有意保持 val_wrong_min-agnostic——内部调
split_by_video_assignment 时不传 val_wrong_min(默认 0,不校验 validation 错题
数)。McNemar 功效护栏是切分**冻结后**的独立校验,由 CLI 的 check_mcnemar_power
在 build_split 返回后执行;切分构造本身不因功效阈失败,二者关注点分离
契约(Task 11):val_wrong_min 前置到切分内保证功效——build_split 计算
wrong_tier_by_video 并连同 config.val_wrong_min 传入 split_by_video_assignment
切分时若 val 错题不足即从 diag 换入低 T2 错题组补足(耗尽 fail-loud)。CLI 的
check_mcnemar_power 作切分冻结后的冗余最终确认
参数:
db_path: harness.db 路径(只读读取 predictions,不改动)。
@@ -134,6 +205,9 @@ def build_split(
out_path: 冻结 pools.json 目标路径(原子写)。
manifest_path: 溯源 manifest 目标路径(原子写)。
generated_at: 生成时间戳(ISO 字符串),由调用方传入以保证可复现。
force: 覆盖保护开关。False(默认)时若 out_path/manifest_path 已存在即
FileExistsError(防静默覆盖训练依赖的冻结锚点);True 时先把旧产物备份为
.bak.* 再放行覆盖。
返回:
SplitBuildResult,含 pools / manifest / assignment,支持字典式访问。
@@ -181,6 +255,11 @@ def build_split(
# Phase 3: 加载题库 + 视频归属切三池 + 原子冻结。
questions = load_benchmark(questions_dir)
correctness = {pred["question_id"]: pred["correct"] for pred in preds}
tier_by_q = {row["question_id"]: row["tier"] for row in signal_rows}
wrong_tier_by_video: dict[str, int] = defaultdict(int)
for pred in preds:
if not pred["correct"] and tier_by_q.get(pred["question_id"]) == "T2":
wrong_tier_by_video[pred["video_id"]] += 1
pools = split_by_video_assignment(
questions,
assignment,
@@ -188,7 +267,10 @@ def build_split(
config.val_ratio,
config.split_seed,
baseline_run_id=baseline_run_id,
val_wrong_min=config.val_wrong_min,
wrong_tier_by_video=dict(wrong_tier_by_video),
)
_guard_frozen_products(out_path, manifest_path, force=force)
save_pools(pools, out_path)
# Phase 4: 溯源 manifestpools_sha256 锚定冻结内容)。
+3 -2
View File
@@ -41,6 +41,7 @@ _STRUCTURAL_KEYS = (
"diag_size",
"val_size",
"batch_correct_ratio",
"trainable_min_units",
)
_DECISION_KEYS = (
@@ -94,7 +95,7 @@ def serialize_state(state: Any) -> dict[str, Any]:
"eval_prev_run_id": state.eval_prev_run_id,
"baseline_skills_version": state.baseline_skills_version,
"baseline_prompts_version": state.baseline_prompts_version,
"steps_since_best_improved": state.steps_since_best_improved,
"epochs_since_best_improved": state.epochs_since_best_improved,
"epoch_start_skills": state.epoch_start_skills,
"changed_task_types_this_epoch": sorted(state.changed_task_types_this_epoch),
"rejected_buffer": {k: [asdict(x) for x in v] for k, v in state.rejected_buffer.items()},
@@ -145,7 +146,7 @@ def deserialize_state_fields(d: dict[str, Any]) -> dict[str, Any]:
"eval_prev_run_id": d["eval_prev_run_id"],
"baseline_skills_version": d["baseline_skills_version"],
"baseline_prompts_version": d["baseline_prompts_version"],
"steps_since_best_improved": d["steps_since_best_improved"],
"epochs_since_best_improved": d["epochs_since_best_improved"],
"epoch_start_skills": d["epoch_start_skills"],
"changed_task_types_this_epoch": set(d["changed_task_types_this_epoch"]),
"rejected_buffer": {
+4
View File
@@ -60,6 +60,7 @@ class RunConfig:
batch_size: mini-batch 单批题目数。
min_class_per_batch: 单批中每个任务类型至少保留的题目数(< batch_size)。
eval_min_per_class: 验证池中每个任务类型至少保底的题目数。
trainable_min_units: 可训练性预检:每题型 diag+val 单元数下限,低于则剔除该题型。
early_stop_patience: 全局 best 连续未提升的容忍轮数,达到即早停。
test_size: held-out 测试池题目数。
use_slow_momentum: 是否启用快慢双速进化中的慢速 momentum 更新。
@@ -114,6 +115,7 @@ class RunConfig:
batch_size: int
min_class_per_batch: int
eval_min_per_class: int
trainable_min_units: int
early_stop_patience: int
test_size: int
use_slow_momentum: bool
@@ -297,6 +299,8 @@ def _validate_minibatch(config: RunConfig) -> None:
)
if config.eval_min_per_class < 1:
raise ValueError(f"eval_min_per_class 必须 >= 1,实际: {config.eval_min_per_class}")
if config.trainable_min_units < 1:
raise ValueError(f"trainable_min_units 必须 >= 1,实际: {config.trainable_min_units}")
if config.pool_split_mode != "per_category":
floor = config.eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT
if config.val_size < floor:
+37 -3
View File
@@ -14,6 +14,7 @@ from __future__ import annotations
import asyncio
import json
import sqlite3
from collections import defaultdict
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
@@ -162,6 +163,24 @@ def _to_text_field(value: Any) -> str:
return json.dumps(value, ensure_ascii=False)
def _normalize_prediction(answer: object) -> str | None:
"""归一化 prediction 落库值。
LLM 提交的 answer 有时是 list/dict(如 {'answer': ['B']}),sqlite 无法绑定
非标量类型直接入库会抛 ProgrammingError 击穿整轮 gather。None 保留(INFRA 空
预测语义,供正确率判定天然计错);str 原样;其余 JSON 序列化为文本。
参数:
answer: LoopResult.result 中的 answer 原始值(可能是 None/str/list/dict)。
返回:
None(保留空预测语义)或可直接入库的字符串。
"""
if answer is None or isinstance(answer, str):
return answer
return _to_text_field(answer)
def _zero_result(run_id: str) -> InferenceResult:
"""空记录时的零值 InferenceResult。
@@ -369,6 +388,7 @@ async def _run_single_question(
log: HarnessLog,
max_steps: int,
plugins: list[object],
run_id: str,
) -> dict[str, Any]:
"""执行单道题目的 Agent 推理。
@@ -383,6 +403,8 @@ async def _run_single_question(
log: HarnessLog 实例(线程安全)。
max_steps: AgentLoop 最大步数。
plugins: pluggy 插件列表。
run_id: 运行标识,用作 cache_salt——run_id 含 _e{epoch} 天然跨 epoch 重采样、
同 epoch 续跑命中缓存(算法 #10 透传)。
返回:
预测结果字典(含 video_id, question_id, prediction, answer 等)。
@@ -412,6 +434,7 @@ async def _run_single_question(
dispatcher,
plugins=plugins,
session_id=qa.question_id,
cache_salt=run_id,
)
result_dict = loop_result.result if isinstance(loop_result.result, dict) else {}
@@ -419,7 +442,7 @@ async def _run_single_question(
reasoning = _to_text_field(result_dict.get("reasoning", ""))
record.update(
{
"prediction": result_dict.get("answer"),
"prediction": _normalize_prediction(result_dict.get("answer")),
"evidence": evidence,
"reasoning": reasoning,
"steps_used": loop_result.steps_used,
@@ -442,8 +465,18 @@ async def _run_single_question(
except Exception:
logger.exception("[{}] QA {} 执行异常", qa.video_id, qa.question_id)
# prediction 必落库(try 外,无论成败)
await asyncio.to_thread(log.insert, "predictions", record)
# prediction 必落库(try 外,无论成败);绑定异常降级为最小 error 行,不击穿 gather
try:
await asyncio.to_thread(log.insert, "predictions", record)
except (sqlite3.InterfaceError, sqlite3.ProgrammingError):
logger.exception("[{}] QA {} 落库绑定异常,降级为 error 行", qa.video_id, qa.question_id)
record["prediction"] = None
record["stop_reason"] = "error"
await asyncio.to_thread(
log.insert,
"predictions",
{k: v for k, v in record.items() if isinstance(v, (str, int, float, type(None)))},
)
return record
@@ -529,6 +562,7 @@ async def run_inference(
log=log,
max_steps=max_steps,
plugins=plugins,
run_id=run_id,
)
logger.info(
"[{}/{}] {} QA {} 完成 (stop={})",
+32 -18
View File
@@ -52,6 +52,9 @@ class HarnessLog:
run_id: 本次运行的唯一标识。
git_sha: 代码版本,默认自动获取。
config_snapshot: 本次运行的配置快照。
register_run: 是否注册运行(upsert _runs + 退出时同步 status)。默认 True
只读查询已有 run(如基线预测回读)时传 False,避免把该 run 的
started_at/config/status 改写、把基线元数据污染成本次进程的运行状态。
"""
def __init__(
@@ -60,27 +63,33 @@ class HarnessLog:
run_id: str,
git_sha: str | None = None,
config_snapshot: dict[str, Any] | None = None,
*,
register_run: bool = True,
) -> None:
self._run_id = run_id
self._register_run = register_run
Path(db_path).parent.mkdir(parents=True, exist_ok=True)
self._conn = sqlite3.connect(db_path, check_same_thread=False)
self._lock = threading.Lock()
self._conn.row_factory = sqlite3.Row
self._conn.execute("PRAGMA journal_mode=WAL")
self._init_fixed_tables()
resolved_sha = git_sha or _get_git_sha()
config_json = json.dumps(config_snapshot, ensure_ascii=False) if config_snapshot else None
self._conn.execute(
"INSERT INTO _runs"
" (run_id, git_sha, started_at, config, status)"
" VALUES (?, ?, ?, ?, ?)"
" ON CONFLICT(run_id) DO UPDATE SET"
" started_at=excluded.started_at,"
" config=excluded.config,"
" status=excluded.status",
(run_id, resolved_sha, _now_iso(), config_json, "running"),
)
self._conn.commit()
if register_run:
resolved_sha = git_sha or _get_git_sha()
config_json = (
json.dumps(config_snapshot, ensure_ascii=False) if config_snapshot else None
)
self._conn.execute(
"INSERT INTO _runs"
" (run_id, git_sha, started_at, config, status)"
" VALUES (?, ?, ?, ?, ?)"
" ON CONFLICT(run_id) DO UPDATE SET"
" started_at=excluded.started_at,"
" config=excluded.config,"
" status=excluded.status",
(run_id, resolved_sha, _now_iso(), config_json, "running"),
)
self._conn.commit()
def _init_fixed_tables(self) -> None:
"""创建 _runs 和 _events 固定表。"""
@@ -217,13 +226,18 @@ class HarnessLog:
参数:
status: 最终状态,"completed""failed"
关键实现:
register_run=False(只读打开)时跳过 status 更新,仅关闭连接,
避免只读回读把已有 run 的 finished_at/status 改写。
"""
with self._lock:
self._conn.execute(
"UPDATE _runs SET finished_at = ?, status = ? WHERE run_id = ?",
(_now_iso(), status, self._run_id),
)
self._conn.commit()
if self._register_run:
self._conn.execute(
"UPDATE _runs SET finished_at = ?, status = ? WHERE run_id = ?",
(_now_iso(), status, self._run_id),
)
self._conn.commit()
self._conn.close()
def __enter__(self) -> HarnessLog:
+2 -1
View File
@@ -135,7 +135,8 @@ def write_dual_metric(
db_path: SQLite 路径。
run_id: 训练 run ID。
epoch: 轮次(1-based)。
version_kind: baseline / best_hard / best_mixed / final
version_kind: baseline / best_hard / best_mixed / final / slow_candidate
slow_candidate = 慢更新 R2 可能被 revert 的候选,不占 epoch 终值 final 口径)。
skills_version / prompts_version: 评估的资源版本。
pool: val / test。
hard_acc: hard 准确率。
+75 -16
View File
@@ -131,6 +131,7 @@ def split_by_video_assignment(
seed: int,
baseline_run_id: str = "",
val_wrong_min: int = 0,
wrong_tier_by_video: dict[str, int] | None = None,
) -> Pools:
"""按视频归属做原子切分:同一视频所有题绝不跨 trainval/test 池。
@@ -146,7 +147,11 @@ def split_by_video_assignment(
seed: 随机种子,保证视频组 shuffle 可复现。
baseline_run_id: 基线 run 标识;离线切分阶段可留空,由调用方回填。
val_wrong_min: validation 池最少错题数(默认 0 = 不检查,保持既有调用契约)。
> 0 时切出 val 后统计其错题数,不足即 fail loud(见 InsufficientValSignal)。
> 0 时切分时保证(不足则从 diag 换入低 T2 错题组补足,耗尽 fail-loud
见 InsufficientValSignal)。
wrong_tier_by_video: video_id -> 该视频错题中 T2(defect) 数量;透传给
_split_trainval_by_video_group 做 tier 感知 diag/val 分配,None 时退化为
原随机 shuffle。
返回:
冻结的三池 Poolsdiagnosis/validation 仍是逐题 GeneratedQuestion 列表
@@ -173,18 +178,11 @@ def split_by_video_assignment(
trainval_qs, test_qs = _partition_by_video_assignment(questions, assignment, correctness)
diagnosis, validation = _split_trainval_by_video_group(
trainval_qs, correctness, val_ratio, random.Random(seed)
trainval_qs, correctness, val_ratio, random.Random(seed),
wrong_tier_by_video=wrong_tier_by_video,
val_wrong_min=val_wrong_min,
)
if val_wrong_min > 0:
val_wrong = sum(1 for q in validation if not correctness[q.question_id])
if val_wrong < val_wrong_min:
raise InsufficientValSignal(
f"validation 池错题数 {val_wrong} < val_wrong_min={val_wrong_min}"
"验证信号不足以支撑可靠比较(如 McNemar 检验功效),"
"请放大 val_ratio / 调整 trainval 归属或调低阈值。"
)
val_correct = sum(1 for q in validation if correctness.get(q.question_id))
baseline_val_accuracy = val_correct / len(validation) if validation else 0.0
return Pools(
@@ -291,6 +289,8 @@ def _split_trainval_by_video_group(
correctness: dict[str, bool],
val_ratio: float,
rng: random.Random,
wrong_tier_by_video: dict[str, int] | None = None,
val_wrong_min: int = 0,
) -> tuple[list[GeneratedQuestion], list[GeneratedQuestion]]:
"""以视频组为原子对 trainval 题集做 correctness 分层,切出 (diagnosis, validation)。
@@ -299,6 +299,12 @@ def _split_trainval_by_video_group(
correctness: question_id -> 基线是否答对;视频组正确性取组内全部题 AND。
val_ratio: validation 占视频组总数的比例。
rng: 随机数生成器,保证视频组 shuffle 可复现。
wrong_tier_by_video: video_id -> 该视频错题中 T2(defect) 的数量。提供时错题
视频组按 T2 含量升序进 val(T2 高的组保留在 diagnosis,把高价值缺陷信号
留给诊断),确定性排序取代随机 shuffle;None 时退化为原随机 shuffle。
val_wrong_min: validation 池最少错题数(切分时保证功效)。> 0 且初分 val 错题
不足时,从 diag 侧的错题组按 T2 升序换入 val 直到满足(每组至多移动一次),
耗尽仍不足则抛 InsufficientValSignalfail loudP5)。
返回:
(diagnosis, validation) 逐题列表元组;同一 video 的全部题整组落在同一侧,
@@ -306,7 +312,8 @@ def _split_trainval_by_video_group(
关键实现细节:
与 _split_one_category 同构:先按视频组 correctness 分正确组/错误组,按比例
把 n_val 个组分层落入 validation(全正确或全错误时退化为非分层随机划分),
把 n_val 个组分层落入 validation(全正确退化为非分层随机划分;全错误时若有
wrong_tier_by_video 仍按 T2 升序分配,否则随机划分),
再把选中组内所有题展开。视频组按 video_id 排序后再 shuffle,保证确定性。
"""
groups: dict[str, list[GeneratedQuestion]] = defaultdict(list)
@@ -320,7 +327,11 @@ def _split_trainval_by_video_group(
correct_vids, wrong_vids = _partition_video_groups_by_correctness(groups, correctness)
n_correct = len(correct_vids)
if n_correct == 0 or n_correct == n_total:
if n_correct == 0 and wrong_tier_by_video is not None:
# 全部错误 + 有 tier 信号:按 T2 升序,低 T2 组优先进 val(保留高 T2 在 diag
wrong_vids.sort(key=lambda v: (wrong_tier_by_video.get(v, 0), v))
val_vids = set(wrong_vids[:n_val])
elif n_correct == 0 or n_correct == n_total:
label = "全部正确" if n_correct == n_total else "全部错误"
logger.warning("trainval 视频组 {} ({} 组),退化为非分层随机划分", label, n_total)
shuffled = list(video_ids)
@@ -330,9 +341,33 @@ def _split_trainval_by_video_group(
val_correct = math.floor(n_correct * n_val / n_total)
val_wrong = n_val - val_correct
rng.shuffle(correct_vids)
rng.shuffle(wrong_vids)
if wrong_tier_by_video is None:
rng.shuffle(wrong_vids)
else:
# T2 少的错题组优先进 val(保留 T2 高的组在 diag),确定性排序
wrong_vids.sort(key=lambda v: (wrong_tier_by_video.get(v, 0), v))
val_vids = set(correct_vids[:val_correct] + wrong_vids[:val_wrong])
if val_wrong_min > 0:
val_wrong_now = sum(
1 for v in val_vids for q in groups[v] if not correctness[q.question_id]
)
# diag 侧仍在的错题组,按 T2 升序(低价值优先移交 val)
diag_wrong_pool = sorted(
(v for v in wrong_vids if v not in val_vids),
key=lambda v: ((wrong_tier_by_video or {}).get(v, 0), v),
)
for v in diag_wrong_pool:
if val_wrong_now >= val_wrong_min:
break
val_vids.add(v)
val_wrong_now += sum(1 for q in groups[v] if not correctness[q.question_id])
if val_wrong_now < val_wrong_min:
raise InsufficientValSignal(
f"trainval 错题不足以让 val 达到 val_wrong_min={val_wrong_min}"
f"(修复后仅 {val_wrong_now}),请放大 val_ratio 或调整 trainval 归属。"
)
diagnosis = [q for q in trainval_qs if q.video_id not in val_vids]
validation = [q for q in trainval_qs if q.video_id in val_vids]
return diagnosis, validation
@@ -762,7 +797,9 @@ def build_or_load_pools(
# 增量构建新类别
paths = resolve_paths(config.workspace_dir)
questions = load_benchmark(paths.questions_dir)
with HarnessLog(str(db_path), baseline_run_id) as hlog:
with HarnessLog(
str(db_path), baseline_run_id, register_run=False
) as hlog:
rows = hlog.query(
"SELECT question_id, prediction, answer "
"FROM predictions WHERE run_id=?",
@@ -809,13 +846,35 @@ def build_or_load_pools(
len(new_types),
sorted(new_types),
)
else:
# global:校验 baseline_run_id 与(若有)manifest 内容指纹,
# 拒绝静默加载与 seed 错配 / 被篡改的冻结切分(P5 fail loud)。
frozen_baseline = raw.get("baseline_run_id")
if frozen_baseline != baseline_run_id:
raise ValueError(
f"冻结 pools.json 的 baseline_run_id={frozen_baseline!r} 与 seed "
f"{baseline_run_id!r} 不一致,拒绝静默加载错配切分。"
)
manifest_path = config.workspace_dir / "split_manifest.json"
if manifest_path.exists():
import hashlib
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
actual_sha = hashlib.sha256(
pools_path.read_text(encoding="utf-8").encode("utf-8")
).hexdigest()
if manifest.get("pools_sha256") != actual_sha:
raise ValueError(
"pools.json 内容指纹与 split_manifest.pools_sha256 不符,"
"冻结产物疑被篡改,拒绝加载。"
)
return load_pools(pools_path)
# ── 全新构建 ──
paths = resolve_paths(config.workspace_dir)
questions = load_benchmark(paths.questions_dir)
with HarnessLog(str(db_path), baseline_run_id) as hlog:
with HarnessLog(str(db_path), baseline_run_id, register_run=False) as hlog:
rows = hlog.query(
"SELECT question_id, prediction, answer FROM predictions WHERE run_id=?",
(baseline_run_id,),
+225 -44
View File
@@ -18,7 +18,8 @@ import random
import shutil
import sqlite3
import tempfile
from dataclasses import dataclass, field
from collections import Counter
from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import TYPE_CHECKING, Any
@@ -115,10 +116,13 @@ class _TrainState:
global_step: int = 0
changed_task_types_this_epoch: set[str] = field(default_factory=set)
epoch_start_skills: dict[str, str] = field(default_factory=dict)
steps_since_best_improved: int = 0
epochs_since_best_improved: int = 0
gate_epoch_observed: bool = False
probations: dict[str, Probation] = field(default_factory=dict)
gate_cooldown: dict[str, int] = field(default_factory=dict)
# 进程内 holdout 去重备忘录 (skills_v, prompts_v) -> test 评估结果;不进 checkpoint
# resume 后清空(重评一次是可接受代价,换取零 schema 变更)。
holdout_memo: dict[tuple[str, str], InferenceResult] = field(default_factory=dict)
# ---------------------------------------------------------------------------
@@ -288,21 +292,82 @@ def _snapshot_current_skills(skills_dir: Path) -> dict[str, str]:
return snapshot
def _filter_untrainable_types(
pools: Pools,
task_types: list[str] | None,
eval_min_per_class: int,
trainable_min_units: int,
) -> tuple[Pools, list[str] | None]:
"""剔除不可训练题型(val 单元<eval_min_per_class 或 diag+val 单元<trainable_min_units)。
计数以**单元(unit)**为原子:AR pair 孪生对折叠计 1 个单元(等于 gate 阶梯该类
候选数),非按题目计数——否则 pair 题型会以 2 倍题目数误通过阈值。test 池不过滤
(继续报告全题型准确率)。在 gate 建立前调用,避免样本不足的微型题型进入信息量
阶梯导致门控崩溃。
参数:
pools: 冻结三池。
task_types: 显式题型子集(None 表示全部),过滤后按 keep 收窄。
eval_min_per_class: 验证池每类保底单元数下限。
trainable_min_units: 每类可训练所需最小 diag+val 单元数。
返回:
过滤后的 (pools, task_types)pools.diagnosis/validation 仅保留 keep 题型,
test 原样;task_types 收窄为 keep(原 None 时返回 sorted(keep))。
异常:
RuntimeError: 过滤后无任何可训练题型(切分/阈值需调整,fail-fast 不空转训练)。
"""
diag_by_type = Counter(u.task_type for u in build_units(pools.diagnosis))
val_by_type = Counter(u.task_type for u in build_units(pools.validation))
# 先按调用方显式 task_types 收窄候选集:未请求的题型(哪怕可训练)不得进入
# keep,否则冻结全局 pools 后 batch/diagnosis 会训练非请求题型,而 gate 只覆盖
# 请求题型 → 静默语义偏差(I-4)。task_types=None 表示全部题型皆为候选。
candidates = set(diag_by_type) | set(val_by_type)
if task_types is not None:
candidates &= set(task_types)
keep: set[str] = set()
dropped: list[tuple[str, str]] = []
for tt in candidates:
n_val = val_by_type.get(tt, 0)
n_units = diag_by_type.get(tt, 0) + n_val
if n_val < eval_min_per_class:
dropped.append((tt, f"val_units={n_val}<{eval_min_per_class}"))
elif n_units < trainable_min_units:
dropped.append((tt, f"units={n_units}<{trainable_min_units}"))
else:
keep.add(tt)
for tt, why in sorted(dropped):
logger.warning("可训练性预检剔除题型 {}{}", tt, why)
if not keep:
detail = "".join(f"{tt}{why}" for tt, why in sorted(dropped))
raise RuntimeError(
"可训练性预检剔除了全部题型,无题型满足 "
f"val_units>={eval_min_per_class} 且 units>={trainable_min_units}"
f"{detail}。请调整池切分或降低阈值。"
)
new_pools = replace(
pools,
diagnosis=[q for q in pools.diagnosis if q.task_type in keep],
validation=[q for q in pools.validation if q.task_type in keep],
)
new_types = [t for t in task_types if t in keep] if task_types is not None else sorted(keep)
return new_pools, new_types
def _should_early_stop(
workspace_dir: Path,
epoch: int,
steps_this_epoch: int,
state: _TrainState,
patience: int,
) -> bool:
"""粒度 early stop:本 epoch best 未刷新则累加本 epoch 步数
"""epoch 粒度 early stop:本 epoch best 未刷新则计数 +1
参数:
workspace_dir: workspace 目录(读 manifest best)。
epoch: 当前 epoch。
steps_this_epoch: 本 epoch 的 step 总数
state: 训练状态(steps_since_best_improved 就地更新)。
patience: early_stop_patience。
state: 训练状态(epochs_since_best_improved 就地更新)
patience: early_stop_patience(连续无刷新的 epoch 数上限)。
返回:
是否触发 early stop。
@@ -310,10 +375,10 @@ def _should_early_stop(
best = read_best(workspace_dir)
improved_this_epoch = best is not None and best.get("epoch") == epoch
if improved_this_epoch:
state.steps_since_best_improved = 0
state.epochs_since_best_improved = 0
return False
state.steps_since_best_improved += steps_this_epoch
return state.steps_since_best_improved >= patience
state.epochs_since_best_improved += 1
return state.epochs_since_best_improved >= patience
def _compute_total_steps(pools: Pools, correctness: dict[str, bool], config: RunConfig) -> int:
@@ -792,8 +857,20 @@ class Runner:
三级嵌套:epoch → batch(step) → per-skill。
epoch 末 _slow_update_cycle 十步序。
训练收尾 _deliver_best + _final_test_eval。
入口第一步做可训练性预检:剔除样本不足的微型题型(防 gate 阶梯崩溃),
过滤后的 pools 贯穿 batch/step/slow-update/final-eval 全部消费;filtered_task_types
透传到 gate 建立(不改 frozen RunConfig)。
"""
state, total_steps, plan, saved_batches = await self._setup_train_run(pools)
pools, filtered_task_types = _filter_untrainable_types(
pools,
list(self._config.task_types) if self._config.task_types is not None else None,
self._config.eval_min_per_class,
self._config.trainable_min_units,
)
state, total_steps, plan, saved_batches = await self._setup_train_run(
pools, filtered_task_types
)
for epoch in range(plan["first_epoch"], self._config.epochs + 1):
if epoch == plan["resume_epoch"]:
batches = [_batch_from_ids(pools, ids) for ids in saved_batches]
@@ -830,26 +907,19 @@ class Runner:
epoch_batches=batch_unit_ids,
config=self._config,
)
await self._slow_update_cycle(epoch, pools, state)
state.system_packs = []
state.tool_packs = []
state.changed_task_types_this_epoch = set()
write_checkpoint(
self._config.workspace_dir,
state=state,
epoch=epoch,
step_completed=len(batches) - 1,
phase="epoch_done",
global_step=state.global_step,
# checkpoint 落盘移入 _slow_update_cycle 末尾(gate_pools.save 之后立即写),
# 消除 gate_epoch_observed 在 gate_pools.json 与 checkpoint 间的双计窗口。
await self._slow_update_cycle(
epoch,
pools,
state,
total_steps=total_steps,
version_snapshot=self._current_version_snapshot(),
step_completed=len(batches) - 1,
epoch_batches=batch_unit_ids,
config=self._config,
)
if _should_early_stop(
self._config.workspace_dir,
epoch,
len(batches),
state,
self._config.early_stop_patience,
):
@@ -862,16 +932,22 @@ class Runner:
# 训练初始化
# -----------------------------------------------------------------------
async def _setup_train_run(self, pools: Pools) -> tuple[_TrainState, int, dict, list | None]:
async def _setup_train_run(
self, pools: Pools, filtered_task_types: list[str] | None
) -> tuple[_TrainState, int, dict, list | None]:
"""据是否 --resume 准备训练起点。
参数:
pools: 已过可训练性预检的三池。
filtered_task_types: 预检后保留的题型(None 表示不限,由 gate 从 diag 推导)。
返回:
(state, total_steps, plan, saved_batches)。
"""
ckpt = load_checkpoint(self._config.workspace_dir) if self._config.resume else None
if self._config.resume and ckpt is None:
raise RuntimeError("--resume 但 checkpoint.json 不存在,拒绝静默从头重训")
gate_pools, baseline_cache = self._init_gate_pools(pools)
gate_pools, baseline_cache = self._init_gate_pools(pools, filtered_task_types)
if not ckpt:
state = self._init_train_state(pools, gate_pools, baseline_cache)
total_steps = _compute_total_steps(pools, state.correctness, self._config)
@@ -897,13 +973,16 @@ class Runner:
)
return state, ckpt["progress"]["total_steps"], plan, ckpt["epoch_batches"]
def _init_gate_pools(self, pools: Pools) -> tuple[GatePools, BaselineCache]:
def _init_gate_pools(
self, pools: Pools, filtered_task_types: list[str] | None
) -> tuple[GatePools, BaselineCache]:
"""构建/加载 CE-Gate 信息量阶梯与基线缓存。
副作用:设置 self._gate_questions_by_id(不进 checkpoint)。
参数:
pools: 冻结三池。
pools: 冻结三池(已过可训练性预检)
filtered_task_types: 预检保留的题型;None 时从 pools.diagnosis 推导。
返回:
(GatePools, BaselineCache)。
@@ -920,7 +999,9 @@ class Runner:
self._gate_units_by_id: dict[str, QuestionUnit] = {
u.unit_id: u for u in build_units(questions)
}
with HarnessLog(str(self._paths.db_path), pools.baseline_run_id) as log:
with HarnessLog(
str(self._paths.db_path), pools.baseline_run_id, register_run=False
) as log:
rows = log.query(
"SELECT question_id, prediction, answer FROM predictions WHERE run_id=?",
(pools.baseline_run_id,),
@@ -932,7 +1013,12 @@ class Runner:
)
baseline_correctness = {r["question_id"]: r["prediction"] == r["answer"] for r in rows}
logger.info("gate 阶梯基线对错覆盖 {}", len(baseline_correctness))
gate_task_types = sorted({q.task_type for q in pools.diagnosis})
# 预检保留的题型优先;None 时从(已过滤的)诊断池推导,二者一致
gate_task_types = (
sorted(filtered_task_types)
if filtered_task_types is not None
else sorted({q.task_type for q in pools.diagnosis})
)
gate_pools = build_or_load_gate_pools(
workspace_dir=self._config.workspace_dir,
questions=questions,
@@ -1009,14 +1095,32 @@ class Runner:
) -> None:
"""单 steprollout → correctness 增量 → 诊断 → 累加 system/tool → 按类 gate。"""
run_id = f"{pools.baseline_run_id}_e{epoch}_s{step}"
await self._rollout_batch(batch, run_id)
from app.harness.inference import PREDICTIONS_SCHEMA, TRACES_SCHEMA
from app.harness.log import HarnessLog
# 幂等:重跑同一 step 前先清旧行,避免断点续跑重复累计双计。
# 先 CREATE TABLE IF NOT EXISTSfresh workspace 首跑时表尚未由 run_inference 建),
# register_run=False 避免只读清理污染 _runs 运行状态。
with HarnessLog(str(self._paths.db_path), run_id, register_run=False) as log:
log.create_table("predictions", PREDICTIONS_SCHEMA)
log.create_table("traces", TRACES_SCHEMA)
log.execute("DELETE FROM predictions WHERE run_id=?", (run_id,))
log.execute("DELETE FROM traces WHERE run_id=?", (run_id,))
await self._rollout_batch(batch, run_id)
with HarnessLog(str(self._paths.db_path), run_id) as log:
_apply_batch_correctness(state.correctness, log, run_id, batch)
diagnosis = await self._run_diagnosis(run_id, question_ids=[q.question_id for q in batch])
# 降级占比过高疑似 judge 基础设施故障:不以降级信号驱动进化,直接中止
n_wrong = sum(1 for q in batch if not state.correctness.get(q.question_id, True))
if n_wrong > 0 and diagnosis.degraded_count / n_wrong > 0.5:
raise RuntimeError(
f"本 step 诊断降级占比 {diagnosis.degraded_count}/{n_wrong} > 50%"
"疑似 judge 基础设施故障,中止训练(不以降级信号驱动进化)。"
)
_accumulate_slow_packs(diagnosis, state)
await self._gate_batch_skills(epoch, step, diagnosis, total_steps, pools, state)
# 冷却计数每 step 递减、归零剔除
@@ -1375,7 +1479,16 @@ class Runner:
# _slow_update_cycle 十步序
# -----------------------------------------------------------------------
async def _slow_update_cycle(self, epoch: int, pools: Pools, state: _TrainState) -> None:
async def _slow_update_cycle(
self,
epoch: int,
pools: Pools,
state: _TrainState,
*,
total_steps: int,
step_completed: int,
epoch_batches: list[list[str]],
) -> None:
"""epoch 末慢更新十步序。
1. 捕获版本快照 → 全 val 重跑 R
@@ -1387,7 +1500,12 @@ class Runner:
7. system/tool 慢更新(edit_budget_end
8. R2 闭环
9. 三态标签 + epoch_report + 四向 held-out
10. gate 阶梯刷新
10. gate 阶梯刷新 → 重置 epoch 累加器 → 立即落 epoch_done checkpoint
参数:
total_steps: 全局总 step 数(checkpoint 用)。
step_completed: 本 epoch 已完成 step 数(checkpoint 用)。
epoch_batches: 本 epoch batch 的 unit_id 划分(checkpoint 用)。
"""
# Phase 1
eval_skills_version = self._current_version("skills")
@@ -1441,11 +1559,13 @@ class Runner:
r2_skills_version = self._current_version("skills")
new_prompts_version = self._current_version("prompts")
eval_r2 = await self._eval_full_val(epoch, pools, run_suffix="_p2")
# R2 是可能被 revert 的慢更新候选,用 slow_candidate 口径,不占 final
# epoch 终值唯一由 Phase 2 的 final 承载)
write_dual_metric(
str(self._paths.db_path),
run_id=self._config.run_id,
epoch=epoch,
version_kind="final",
version_kind="slow_candidate",
skills_version=r2_skills_version,
prompts_version=new_prompts_version,
pool="val",
@@ -1497,6 +1617,24 @@ class Runner:
epoch, pools.baseline_run_id, state, extra_run_ids=r2_kept_run_ids
)
# 重置 epoch 累加器 + 立即落 epoch_done checkpoint:与 _refresh_gate_ladder 内的
# gate_pools.save + gate_epoch_observed=True 同刻一致,消除断点续跑的双计窗口。
state.system_packs = []
state.tool_packs = []
state.changed_task_types_this_epoch = set()
write_checkpoint(
self._config.workspace_dir,
state=state,
epoch=epoch,
step_completed=step_completed,
phase="epoch_done",
global_step=state.global_step,
total_steps=total_steps,
version_snapshot=self._current_version_snapshot(),
epoch_batches=epoch_batches,
config=self._config,
)
# -----------------------------------------------------------------------
# 慢更新内部方法
# -----------------------------------------------------------------------
@@ -1584,7 +1722,7 @@ class Runner:
state.best_val_acc = eval_acc
state.best_skills_version = skills_v
state.best_prompts_version = prompts_v
state.steps_since_best_improved = 0
state.epochs_since_best_improved = 0
update_best(
self._config.workspace_dir,
skills=f"skills/{skills_v}",
@@ -1888,11 +2026,18 @@ class Runner:
) -> None:
"""四向 held-outbaseline/best_hard/final/best_mixed 各在 test 池评估。
去重(进程内备忘录 state.holdout_memo,不改 schema):
- baseline:不跑推理,从基线 predictions 推导 test 结果(0 推理),存 memo 跨 epoch 复用。
- final:真评 test,存 memo[(final_sv,final_pv)]。
- best_hard:其版本已在 memo== final 或往轮已评)则引用,否则真评并存 memo。
- best_mixed:赢家必是 best_hard 或 final 之一,其结果已在 memo,直接引用(0 推理)。
test 池仅观测落库,绝不进 gate/best/early-stop/调参。
"""
best_mixed = await self._pick_mixed_best(
epoch, pools, state, eval_skills_version, eval_prompts_version
)
memo = state.holdout_memo
versions: dict[str, tuple[str, str] | None] = {
"baseline": (state.baseline_skills_version, state.baseline_prompts_version),
"best_hard": (state.best_skills_version, state.best_prompts_version),
@@ -1904,9 +2049,14 @@ class Runner:
continue
sv, pv = version
run_id = f"{self._config.run_id}_holdout_{version_kind}_e{epoch}"
res = await self._eval_version_on_pool(
sv, pv, pools.test, run_id, context=f"held-out {version_kind}"
)
if version not in memo:
if version_kind == "baseline":
memo[version] = self._derive_baseline_test_result(pools, run_id)
else:
memo[version] = await self._eval_version_on_pool(
sv, pv, pools.test, run_id, context=f"held-out {version_kind}"
)
res = memo[version]
soft = await self._try_soft_score(run_id, pools.test)
mixed = None if soft is None else 0.5 * res.accuracy + 0.5 * soft
write_holdout_eval(
@@ -1920,6 +2070,33 @@ class Runner:
per_task_type_json=json.dumps(res.per_task_type, ensure_ascii=False),
)
def _derive_baseline_test_result(self, pools: Pools, run_id: str) -> InferenceResult:
"""从基线 run 的 predictions 推导 test 池评估结果(0 推理)。
基线 runpools.baseline_run_id)已对全题库推理并落库,test 题在其中;此处
按 test 题回读基线预测、经 _aggregate_results 折叠为 unit 级 InferenceResult
避免重复推理基线版本(同版本不重采样)。
参数:
pools: 冻结三池(提供 test 与 baseline_run_id)。
run_id: 本次 holdout baseline 向的 run_id(仅用作结果标识)。
返回:
unit 级 InferenceResultaccuracy / per_task_type 与真评同口径)。
"""
from app.harness.inference import _aggregate_results
from app.harness.log import HarnessLog
qids = [q.question_id for q in pools.test]
with HarnessLog(
str(self._paths.db_path), pools.baseline_run_id, register_run=False
) as log:
placeholders = ", ".join(["?"] * len(qids))
rows = log.query(
f"SELECT * FROM predictions WHERE run_id=? AND question_id IN ({placeholders})",
(pools.baseline_run_id, *qids),
)
return _aggregate_results(rows, pools.test, run_id)
async def _pick_mixed_best(
self,
epoch: int,
@@ -2164,13 +2341,15 @@ class Runner:
self, run_id: str, *, question_ids: list[str] | None = None
) -> DiagnosisResult:
"""执行两阶段诊断。"""
from app.harness.baseline_run_log import StepsJsonRunLog
from app.harness.log import RunLogImpl
from app.harness.workspace import VersionedSkillStore
from app.question_gen import load_benchmark
from core.evolution.diagnose import run_diagnosis
questions = load_benchmark(self._paths.questions_dir)
run_log = RunLogImpl(str(self._paths.db_path))
# traces 表空时(如训练 rollout 只落 steps_json)从 steps_json 重建轨迹,恢复算法 #7
run_log = StepsJsonRunLog(RunLogImpl(str(self._paths.db_path)))
skill_store = VersionedSkillStore(self._paths.skills_dir)
diagnose_prompts = self._load_diagnose_prompts()
@@ -2269,14 +2448,15 @@ class Runner:
def _read(name: str) -> str:
p = Path("prompts") / name
return p.read_text(encoding="utf-8") if p.exists() else ""
if not p.exists():
raise FileNotFoundError(f"缺进化/诊断模板: {p}(请从 TRM4 迁移或检查 prompts/")
return p.read_text(encoding="utf-8")
return EvolvePrompts(
evolve_skill=_read("evolve_skill.md"),
evolve_system=_read("evolve_system.md"),
evolve_tool=_read("evolve_tool.md"),
evolve_rank=_read("evolve_rank.md"),
consolidate_system=_read("consolidate_system.md"),
)
def _load_diagnose_prompts(self):
@@ -2285,13 +2465,14 @@ class Runner:
def _read(name: str) -> str:
p = Path("prompts") / name
return p.read_text(encoding="utf-8") if p.exists() else ""
if not p.exists():
raise FileNotFoundError(f"缺进化/诊断模板: {p}(请从 TRM4 迁移或检查 prompts/")
return p.read_text(encoding="utf-8")
return DiagnosePrompts(
defect_vs_lapse=_read("defect_vs_lapse.md"),
reasoning_sub=_read("reasoning_sub.md"),
span_eval_system=_read("span_eval_system.md"),
span_eval_user=_read("span_eval_user.md"),
missed_nodes=_read("missed_nodes.md"),
skill_adherence=_read("skill_adherence.md"),
confirmation_bias=_read("confirmation_bias.md"),
+28 -4
View File
@@ -190,6 +190,9 @@ def init_seed(
baseline_run_id: str,
parent: str | None,
description: str,
*,
pools_json: Path | None = None,
split_manifest: Path | None = None,
) -> Path:
"""在 store/seeds/<name> 写一个种子:权重 + baseline.db + seed.json。
@@ -202,6 +205,10 @@ def init_seed(
baseline_run_id: 全量记录的 run_idfresh 时注入 build_pools
parent: 来源initial None
description: 人类可读说明
pools_json: 可选冻结切分 pools.json 源路径提供时拷入 seed 目录
fresh 训练时携带冻结切分进 workspace init_workspace_from_seed
split_manifest: 可选冻结切分 split_manifest.json 源路径提供时拷入 seed
目录供加载时校验 pools.json 内容指纹pools_sha256
返回:
种子目录路径
@@ -216,6 +223,10 @@ def init_seed(
shutil.copytree(skills_dir, seed_dir / "skills")
shutil.copytree(prompts_dir, seed_dir / "prompts")
shutil.copy2(baseline_db, seed_dir / "baseline.db")
if pools_json is not None:
shutil.copy2(pools_json, seed_dir / "pools.json")
if split_manifest is not None:
shutil.copy2(split_manifest, seed_dir / "split_manifest.json")
(seed_dir / "seed.json").write_text(
json.dumps(
{
@@ -266,7 +277,9 @@ def read_seed(store_dir: Path, name: str) -> dict:
return json.loads(seed_json.read_text())
def extract_run_db(src_db: Path, dst_db: Path, run_id: str) -> None:
def extract_run_db(
src_db: Path, dst_db: Path, run_id: str, *, dedupe_per_question: bool = False
) -> None:
"""从 src_db 抽出某 run_id 的 _runs + predictions 行,写一个最小 db(种子 baseline.db)。
用源表的**原始 CREATE 语句**重建目标表保留主键/列类型/约束
@@ -277,6 +290,9 @@ def extract_run_db(src_db: Path, dst_db: Path, run_id: str) -> None:
src_db: harness.db
dst_db: 目标 db不得已存在
run_id: 要抽取的 run
dedupe_per_question: True predictions 表每 question_id 仅保留 rowid 最小
的首行对齐 canonical question_id 取第一行 ORDER BY rowid口径
902900_runs 表不受影响
异常:
RuntimeError: 源中无该表或无该 run 的行
@@ -294,9 +310,17 @@ def extract_run_db(src_db: Path, dst_db: Path, run_id: str) -> None:
dst.execute(create_sql[0])
cols = [r[1] for r in src.execute(f"PRAGMA table_info({table})")]
col_sql = ", ".join(cols)
rows = src.execute(
f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,)
).fetchall()
if table == "predictions" and dedupe_per_question:
rows = src.execute(
f"SELECT {col_sql} FROM {table} WHERE run_id=? "
"AND rowid IN (SELECT MIN(rowid) FROM predictions "
"WHERE run_id=? GROUP BY question_id)",
(run_id, run_id),
).fetchall()
else:
rows = src.execute(
f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,)
).fetchall()
if not rows:
raise RuntimeError(f"{table} 中无 run_id={run_id} 的行")
ph = ", ".join("?" * len(cols))
+105 -21
View File
@@ -27,6 +27,7 @@ from loguru import logger
from app.harness.gate_ladder import BaselineCache, skill_hash
from app.harness.question_units import build_units, flatten_units, unit_correctness_view
from core.evolution import (
INFRA_STOP_REASONS,
GateParams,
GateVerdict,
RejectedEdit,
@@ -35,6 +36,10 @@ from core.evolution import (
pair_block,
)
# INFRA_STOP_REASONS 复用 core.evolution.diagnose 的单一定义(M-2):INFRA 故障
# stop_reason(推理侧基础设施错误,非模型答错)在诊断与 gate 两处必须同口径,
# 避免各自维护副本致未来漂移。
if TYPE_CHECKING:
from app.harness.inference import InferenceResult
from app.harness.log import HarnessLog
@@ -209,7 +214,8 @@ def _load_run_rows(
_correctsteps 等字段
"""
rows = log.query(
"SELECT question_id, prediction, answer, steps_json FROM predictions WHERE run_id=?",
"SELECT question_id, prediction, answer, stop_reason, steps_json "
"FROM predictions WHERE run_id=?",
(run_id,),
)
normalized: dict[str, dict[str, Any]] = {}
@@ -230,6 +236,46 @@ def _load_run_rows(
return normalized
def _infra_question_ids_from_db(
log: HarnessLog,
run_id: str,
chunk: list[GeneratedQuestion],
) -> set[str]:
"""从 db 读取一个 run 中 stop_reason 属 INFRA 故障族的 question_id 集合。
参数:
log: HarnessLog 共享实例
run_id: 推理 run_id
chunk: 题目列表
返回:
stop_reason {"error", "parse_error"} question_id 集合
"""
rows = _load_run_rows(log, run_id)
return {
q.question_id
for q in chunk
if rows.get(q.question_id, {}).get("stop_reason") in INFRA_STOP_REASONS
}
def _count_infra_units(units: list[QuestionUnit], infra_qids: set[str]) -> int:
"""统计含 INFRA record 的 unit 数(一个 unit 任一题 INFRA 即计 1)。
使护栏分子与分母r.totalunit 粒度同口径AR pair unit 含两 record
record 计数会放大分子致 gate_guard_err 误触发破坏 unit 粒度一致性
核心算法保真 #5/#6)。
参数:
units: 当前块的单元列表single AR pair
infra_qids: run stop_reason INFRA 故障族的 question_id 集合
返回:
含至少一题 INFRA unit
"""
return sum(1 for u in units if any(q.question_id in infra_qids for q in u.questions))
def _candidate_correctness_from_db(
log: HarnessLog,
run_id: str,
@@ -264,13 +310,18 @@ async def _resolve_baseline_block(
run_inference: RunInferenceFn,
log: HarnessLog,
run_id: str,
) -> tuple[dict[str, bool], int, int]:
) -> tuple[dict[str, bool], list[QuestionUnit], int, int]:
"""基线侧处理一个块:缓存优先(unit 键),miss 的单元新鲜跑基线版本并回写缓存。
缓存以 unit_id 为键存单元级对错AR pair 双向 AND 折叠后一个布尔
miss 的单元展开为逐题送推理读回逐题预测后经 unit_correctness_view 折叠成
单元级对错再写缓存核心算法保真 #5)。逐题 predictions 仍逐题落库溯源。
INFRA 隔离算法 #6):miss 单元内**任一题** stop_reason ∈ {error, parse_error}
即判定该单元为 INFRA 故障**不写 BaselineCache**否则瞬时故障永久污染基线
快照**不入 b_units**并从返回的有效单元集中剔除避免污染 W/L 翻转与配对
命中缓存的单元恒为有效此前已成功验证过
参数:
units: 当前块的单元列表single AR pair
task_type: 当前验证题型缓存键成分
@@ -283,8 +334,9 @@ async def _resolve_baseline_block(
run_id: 本块基线 run_id
返回:
(b_units, errors_inc, denom_inc)块内 unit_id -> 基线单元对错
本块新增的 INFRA error 计数与推理题次分母增量全命中时为 0, 0
(b_units, valid_units, errors_inc, denom_inc)块内有效 unit_id -> 基线单元
对错剔除 INFRA 后的有效单元列表本块新增的 INFRA error 计数与推理题次
分母增量全命中时为 0, 0
"""
miss_units = [
u
@@ -293,22 +345,34 @@ async def _resolve_baseline_block(
]
errors_inc = 0
denom_inc = 0
infra_qids: set[str] = set()
if miss_units:
miss_questions = flatten_units(miss_units)
r_b = await run_inference(miss_questions, run_id=run_id, skills_dir=base_skills_dir)
errors_inc = r_b.stop_reason_counts.get("error", 0)
infra_qids = _infra_question_ids_from_db(log, r_b.run_id, miss_questions)
# 护栏分子与分母(r.total,unit 粒度)同口径:含 INFRA record 的 unit 计 1
# 避免 AR pair(一 unit 两 record)逐 record 计数放大分子致误触发;仍涵盖
# error + parse_error_infra_question_ids_from_db 口径),parse_error 风暴不被绕过。
errors_inc = _count_infra_units(miss_units, infra_qids)
denom_inc = r_b.total
fresh_per_q = _candidate_correctness_from_db(log, r_b.run_id, miss_questions)
fresh_units = unit_correctness_view(miss_units, fresh_per_q)
for uid, correct in fresh_units.items():
baseline_cache.put(task_type, s_hash, prompts_version, uid, correct)
# 只回写非 INFRA 单元;INFRA 单元不入缓存(不永久污染基线快照)
for u in miss_units:
if any(q.question_id in infra_qids for q in u.questions):
continue
baseline_cache.put(task_type, s_hash, prompts_version, u.unit_id, fresh_units[u.unit_id])
valid_units = [
u for u in units if not any(q.question_id in infra_qids for q in u.questions)
]
b_units: dict[str, bool] = {}
for u in units:
for u in valid_units:
val = baseline_cache.get(task_type, s_hash, prompts_version, u.unit_id)
assert val is not None, f"基线缓存补齐后仍有 miss: unit={u.unit_id} run_id={run_id}"
b_units[u.unit_id] = val
return b_units, errors_inc, denom_inc
return b_units, valid_units, errors_inc, denom_inc
async def _run_candidate_block(
@@ -336,7 +400,11 @@ async def _run_candidate_block(
questions = flatten_units(units)
r_c = await run_inference(questions, run_id=run_id, skills_dir=cand_dir)
c_per_q = _candidate_correctness_from_db(log, r_c.run_id, questions)
return c_per_q, r_c.stop_reason_counts.get("error", 0), r_c.total
infra_qids = _infra_question_ids_from_db(log, r_c.run_id, questions)
# 护栏分子与分母(r.total,unit 粒度)同口径:含 INFRA record 的 unit 计 1
# (见 _count_infra_units),涵盖 error + parse_error。
errors_inc = _count_infra_units(units, infra_qids)
return c_per_q, errors_inc, r_c.total
def _build_evidence_rows(
@@ -554,6 +622,7 @@ async def _run_local_validation(
w = 0
l = 0 # noqa: E741
n_used = 0
n_excluded = 0 # 累计被 INFRA 隔离剔除的单元数(从阶梯分母扣除)
errors = 0
infra_denom = 0
evidence_rows: list[dict] = []
@@ -567,8 +636,8 @@ async def _run_local_validation(
verdict: GateVerdict | None = None
for block_idx, unit_chunk in enumerate(unit_chunks):
# Phase 1: 基线侧(缓存优先,miss 新鲜跑+ 候选侧(全块新鲜跑
b_units, err_b, den_b = await _resolve_baseline_block(
# Phase 1: 基线侧(缓存优先,miss 新鲜跑INFRA 单元剔除
b_units, valid_chunk, err_b, den_b = await _resolve_baseline_block(
units=unit_chunk,
task_type=task_type,
s_hash=s_hash,
@@ -579,34 +648,46 @@ async def _run_local_validation(
log=log,
run_id=f"{gate_run_prefix}_b{block_idx}_base",
)
# 本块全 INFRA:无有效单元可配对——候选无需空跑,仅把基线侧错误计入护栏后
# 累计剔除数进入下一块(护栏仍能在整轮 INFRA 错误率超阈值时熔断)。
n_excluded += len(unit_chunk) - len(valid_chunk)
if not valid_chunk:
errors += err_b
infra_denom += den_b
_check_infra_guard(errors, infra_denom, gate_guard_err)
continue
# 候选侧只跑基线侧判定有效(非 INFRA)的单元,保证配对 unit_ids 两侧一致
c_per_q, err_c, den_c = await _run_candidate_block(
units=unit_chunk,
units=valid_chunk,
cand_dir=cand_dir,
run_inference=run_inference,
log=log,
run_id=f"{gate_run_prefix}_b{block_idx}_cand",
)
# Phase 2: INFRA 护栏(跨块累计,分母 >=10 才触发)
# Phase 2: INFRA 护栏(跨块累计,分母 >=10 才触发)——写缓存前置于此已由
# _resolve_baseline_block 保证 INFRA 单元不落缓存,此处仅做整轮错误率熔断。
errors += err_b + err_c
infra_denom += den_b + den_c
_check_infra_guard(errors, infra_denom, gate_guard_err)
# Phase 3: 折叠成单元视图 + 配对 + 证据行 + 块间判定
c_units = unit_correctness_view(unit_chunk, c_per_q)
# Phase 3: 折叠成单元视图 + 配对 + 证据行 + 块间判定(均用有效单元)
c_units = unit_correctness_view(valid_chunk, c_per_q)
candidate_per_q.update(c_per_q)
unit_ids = [u.unit_id for u in unit_chunk]
unit_ids = [u.unit_id for u in valid_chunk]
pair_result = pair_block(b_units, c_units, unit_ids)
for uid, (b, c) in pair_result.observed.items():
base_obs[uid] = b
cand_obs[uid] = c
block_rows = _build_evidence_rows(unit_chunk, b_units, c_units, task_type, block_idx)
block_rows = _build_evidence_rows(valid_chunk, b_units, c_units, task_type, block_idx)
w += pair_result.w
l += pair_result.l # noqa: E741
n_used += len(unit_chunk)
verdict = gate_decision(w, l, n_used, n_plan - n_used, params=gate_params)
n_used += len(valid_chunk)
# 阶梯剩余按扣除 INFRA 后的有效分母计:n_remaining = (n_plan - n_excluded) - n_used
verdict = gate_decision(w, l, n_used, (n_plan - n_excluded) - n_used, params=gate_params)
for row in block_rows:
row["e_value"] = verdict.e_value
@@ -615,8 +696,11 @@ async def _run_local_validation(
if verdict.decision != "continue":
break
# verdict 仍为 None ⟺ 全部单元被 INFRA 排除(空 ladder 已在入口拒绝)。
# 明确失败,避免落到误导性的"空阶梯"断言而无法定位为 INFRA 原因。
if verdict is None:
raise RuntimeError("gate 阶梯所有 unit 被判为 INFRA 排除,无法验证(检查推理基础设施)")
# 最后一块判定即终态(n_remaining=0 → provisional/inertia
assert verdict is not None, "空阶梯应已在 validate_skill_local 入口拒绝"
return _finalize_outcome(
verdict=verdict,
w=w,
+31 -7
View File
@@ -248,13 +248,15 @@ def _build_redis_cache(settings: Any) -> Any | None:
"""按 .env redis_url 构建响应缓存(不可用则降级 None,与 main 一致)。"""
if not settings.redis_url:
return None
from adapters.redis_cache import RedisResponseCache, _resolve_cache_ttl
# 配置校验 fail-loud(不属于 Redis 连接故障,不得被下方降级 except 吞掉)
ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl)
try:
import redis.asyncio as aioredis
from adapters.redis_cache import RedisResponseCache
redis_client = aioredis.from_url(settings.redis_url, decode_responses=True)
ttl_s = settings.redis_cache_ttl if settings.redis_cache_ttl > 0 else None
return RedisResponseCache(redis=redis_client, ttl_s=ttl_s)
except Exception:
logger.warning("Redis 缓存不可用,诊断降级为无缓存模式")
@@ -365,13 +367,14 @@ def _load_diagnose_prompts() -> Any:
def _read(name: str) -> str:
p = Path("prompts") / name
return p.read_text(encoding="utf-8") if p.exists() else ""
if not p.exists():
raise FileNotFoundError(f"缺进化/诊断模板: {p}(请从 TRM4 迁移或检查 prompts/")
return p.read_text(encoding="utf-8")
return DiagnosePrompts(
defect_vs_lapse=_read("defect_vs_lapse.md"),
reasoning_sub=_read("reasoning_sub.md"),
span_eval_system=_read("span_eval_system.md"),
span_eval_user=_read("span_eval_user.md"),
missed_nodes=_read("missed_nodes.md"),
skill_adherence=_read("skill_adherence.md"),
confirmation_bias=_read("confirmation_bias.md"),
@@ -476,8 +479,9 @@ def load_questions_by_id(questions_dir: Path) -> dict[str, GeneratedQuestion]:
def check_mcnemar_power(pools: Pools, val_wrong_min: int) -> int:
"""校验 validation 池错题数达 McNemar 功效阈,不足即 fail loud。
build_split 按契约用 split_by_video_assignment(val_wrong_min=0)不破契约
故功效护栏在 capstone 层单独核验val 错题数 < 验证信号不足以支撑可靠比较
val_wrong_min 已前置到 build_split 内的切分保证功效不足即从 diag 换入低 T2
错题组补足耗尽 fail-loud本函数作切分冻结后的冗余最终确认val 错题数 <
验证信号不足以支撑可靠比较
参数:
pools: 冻结三池 validation correctness
@@ -517,6 +521,8 @@ async def run_pipeline(
questions_dir: Path,
out_dir: Path,
generated_at: str,
force: bool = False,
retry_uncertain: bool = False,
) -> SplitBuildResult:
"""内联三阶段:Phase 0 INFRA T0 补录 → Phase 1 诊断 → Phase 2 冻结切分 → McNemar 护栏。
@@ -532,6 +538,8 @@ async def run_pipeline(
questions_dir: benchmark 题库目录Phase 2 加载题库切池
out_dir: 冻结产物目录pools.json + split_manifest.json
generated_at: 生成时间戳ISO 字符串由调用方传入见模块 C-2 复现锚点约定
force: 覆盖已存在冻结产物开关透传给 build_splitFalse 时已存在即报错
retry_uncertain: 透传给 run_baseline_diagnosis令已落 uncertain 题被重新诊断
返回:
SplitBuildResult冻结三池 + manifest + assignment
@@ -551,6 +559,7 @@ async def run_pipeline(
questions=questions,
store=signal_store,
deps=diagnosis_deps,
retry_uncertain=retry_uncertain,
)
# Phase 2: 冻结切分(读诊断信号 → 贪心选择 → 视频组原子切三池 → 冻结 + 六条断言)。
@@ -570,10 +579,12 @@ async def run_pipeline(
select_seed=config.seed,
val_ratio=config.val_ratio,
split_seed=config.seed,
val_wrong_min=config.val_wrong_min,
),
out_path=out_dir / "pools.json",
manifest_path=out_dir / "split_manifest.json",
generated_at=generated_at,
force=force,
)
# McNemar 功效护栏(build_split 契约外的 capstone 层校验)。
@@ -647,6 +658,8 @@ def _execute_real(config: VideoSplitConfig, fingerprint: str, args: argparse.Nam
questions_dir=questions_dir,
out_dir=out_dir,
generated_at=generated_at,
force=args.force,
retry_uncertain=args.retry_uncertain,
)
)
finally:
@@ -770,6 +783,17 @@ def build_arg_parser() -> argparse.ArgumentParser:
"对 manifest 做字节级复现比对。"
),
)
parser.add_argument(
"--force",
action="store_true",
help="覆盖已存在的冻结 pools.json/manifest(旧产物备份为 .bak.*",
)
parser.add_argument(
"--retry-uncertain",
action="store_true",
dest="retry_uncertain",
help="把已落 tier='uncertain'(信号不可信降级)的题重新诊断,而非当作已完成跳过",
)
return parser
+26 -6
View File
@@ -64,6 +64,21 @@ def _now_iso() -> str:
return datetime.now(UTC).isoformat()
def _atomic_write_json(path: Path, data: dict) -> None:
"""原子写 JSONtmp + os.replace(对齐 checkpoint.py 范式,防半截损坏)。
先写同目录临时文件 os.replace 原子替换目标替换阶段崩溃不会留下半截
JSON原文件保持完好
参数:
path: 目标 JSON 文件路径
data: 待序列化的字典
"""
tmp = path.with_name(path.name + ".tmp")
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
os.replace(tmp, path)
# ---------------------------------------------------------------------------
# Workspace 核心函数
# ---------------------------------------------------------------------------
@@ -108,7 +123,7 @@ def _scaffold_workspace(
},
"history": [],
}
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
_atomic_write_json(workspace_dir / "manifest.json", manifest)
def init_workspace(
@@ -196,6 +211,13 @@ def init_workspace_from_seed(
shutil.copytree(seed_dir / "prompts", workspace_dir / "prompts" / "v1")
shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db")
seed_pools = seed_dir / "pools.json"
if seed_pools.exists():
shutil.copy2(seed_pools, workspace_dir / "pools.json")
seed_manifest = seed_dir / "split_manifest.json"
if seed_manifest.exists():
shutil.copy2(seed_manifest, workspace_dir / "split_manifest.json")
logger.info("Workspace 从种子 '{}' 初始化完成: {}", seed_name, workspace_dir)
return meta["baseline_run_id"]
@@ -279,7 +301,7 @@ def update_manifest(workspace_dir: Path, **version_updates: str) -> None:
raise KeyError(f"无效的 manifest current 字段: {invalid}")
manifest = load_manifest(workspace_dir)
manifest["current"].update(version_updates)
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
_atomic_write_json(workspace_dir / "manifest.json", manifest)
def record_run(workspace_dir: Path, run_id: str) -> Path:
@@ -308,9 +330,7 @@ def record_run(workspace_dir: Path, run_id: str) -> Path:
"questions": current["questions"],
}
)
(workspace_dir / "manifest.json").write_text(
json.dumps(manifest, ensure_ascii=False, indent=2)
)
_atomic_write_json(workspace_dir / "manifest.json", manifest)
run_dir = workspace_dir / "runs" / run_id
# exist_ok:同 run_id 重跑时 run 目录已存在不应崩溃
@@ -362,7 +382,7 @@ def update_best(
"run_id": run_id,
"epoch": epoch,
}
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
_atomic_write_json(workspace_dir / "manifest.json", manifest)
logger.info("Best 已更新: val_acc={}, run={}, epoch={}", val_acc, run_id, epoch)
+1
View File
@@ -67,6 +67,7 @@ harness:
batch_correct_ratio: 0.5
momentum_samples: 20
eval_min_per_class: 2
trainable_min_units: 8
early_stop_patience: 8
use_slow_momentum: true
# 池构建策略
+1
View File
@@ -48,6 +48,7 @@ harness:
batch_correct_ratio: 0.5
momentum_samples: 20
eval_min_per_class: 2
trainable_min_units: 8
early_stop_patience: 4
test_size: 63
diag_size: 20
+1
View File
@@ -49,6 +49,7 @@ harness:
batch_correct_ratio: 0.5
momentum_samples: 20
eval_min_per_class: 2
trainable_min_units: 8
early_stop_patience: 4
test_size: 63
diag_size: 20
+1 -1
View File
@@ -12,7 +12,7 @@ video_split:
epsilon: 0.1 # test 相对全局最大允许分布偏差(题型占比 / 难度画像两维,逐桶)
report_floor: 27 # per-type 报告门限:题数 ≥ 此值的 task_type 才入 ε 代表性约束
val_wrong_min: 20 # validation 池最少错题数(McNemar 检验功效阈 ≈ 20,低于则信号不足)
val_ratio: 0.3 # validation 占 trainval 视频组总数的比例
val_ratio: 0.4 # validation 占 trainval 视频组总数的比例(0.3→0.4 提升整包终审功效,WP2)
seed: 7 # 贪心选择器预洗牌 + 视频组题级切分种子(打破等增益 / 等槽平局)
floor_k: # 各 task_type 的 T2 defect 下限(硬约束)—— 均衡覆盖全 11 类,标定于 1cb1c203 真实 T2 分布
Object Reasoning: 5 # T2=25
+10 -3
View File
@@ -108,6 +108,7 @@ class AgentLoop:
plugins: list[object] | None = None,
*,
session_id: str | None = None,
cache_salt: str | None = None,
) -> LoopResult:
"""执行 Thinking+JSON 推理循环。
@@ -117,6 +118,7 @@ class AgentLoop:
tool_dispatcher: 工具调度器ToolDispatcher Protocol 实例
plugins: pluggy 插件列表
session_id: 会话 ID透传给 LLMProvider
cache_salt: 缓存盐透传给 LLMProvider如训练用 run_id epoch 重采样
返回:
LoopResult 实例包含推理步骤token 用量终止原因
@@ -138,7 +140,7 @@ class AgentLoop:
# Phase 1: LLM 调用(步级重试:防穿透 GovernedLLMClient 的瞬时异常)
try:
response = await self._call_llm_with_step_retry(
messages, token_usage, session_id=session_id
messages, token_usage, session_id=session_id, cache_salt=cache_salt
)
except Exception as e:
logger.error("LLM API 调用失败({}: {}", type(e).__name__, e)
@@ -269,6 +271,7 @@ class AgentLoop:
token_usage: dict[str, int],
*,
session_id: str | None = None,
cache_salt: str | None = None,
) -> LLMResponse:
"""带步级重试的 LLM 调用,兜底穿透治理层重试栈的瞬时异常。
@@ -292,7 +295,9 @@ class AgentLoop:
step_attempt = 0
while True:
try:
return await self._call_llm(messages, token_usage, session_id=session_id)
return await self._call_llm(
messages, token_usage, session_id=session_id, cache_salt=cache_salt
)
except self._retryable_exceptions as e:
step_attempt += 1
if step_attempt > self._step_retries:
@@ -315,6 +320,7 @@ class AgentLoop:
token_usage: dict[str, int],
*,
session_id: str | None = None,
cache_salt: str | None = None,
) -> LLMResponse:
"""调用 LLMProvider 并累加 token 使用量。
@@ -322,11 +328,12 @@ class AgentLoop:
messages: 消息历史
token_usage: 可变字典就地累加
session_id: 会话 ID透传给 LLMProvider
cache_salt: 缓存盐透传给 LLMProvider epoch 重采样
返回:
LLMResponse 实例
"""
response = await self._llm.chat(messages, session_id=session_id)
response = await self._llm.chat(messages, session_id=session_id, cache_salt=cache_salt)
token_usage["prompt_tokens"] += response.prompt_tokens
token_usage["completion_tokens"] += response.completion_tokens
return response
+2 -1
View File
@@ -4,7 +4,7 @@
只依赖 Protocol 接口和标准库可搬到无 adapters 的环境用假实现原样运行
"""
from core.evolution.diagnose import run_diagnosis
from core.evolution.diagnose import INFRA_STOP_REASONS, run_diagnosis
from core.evolution.evolve import (
edit_budget_at,
evolve_single_skill,
@@ -44,6 +44,7 @@ from core.evolution.types import (
from core.evolution.validate import classify_quadrants, compute_accuracy, pair_block
__all__ = [
"INFRA_STOP_REASONS",
"CaseSample",
"DiagnosePrompts",
"DiagnosisResult",
+10 -7
View File
@@ -53,7 +53,7 @@ if TYPE_CHECKING:
_SPAN_EVAL_TOOLS: frozenset[str] = frozenset({"view_node", "search_similar", "observe_frame"})
"""span 级评估涵盖的工具集合。"""
_INFRA_STOP_REASONS: frozenset[str] = frozenset({"error", "parse_error"})
INFRA_STOP_REASONS: frozenset[str] = frozenset({"error", "parse_error"})
"""执行/解析层失败导致排除的 stop_reason 集合。"""
@@ -1489,12 +1489,15 @@ def _build_skill_case_packs(
if qm.correct:
continue
attr = attribution_map.get(qm.question_id)
if attr is not None and attr.cause_category == "lapse":
if attr.lapse_note and attr.lapse_note.strip():
# 仅明确 defect 且非 degraded 才进正文进化路径;
# lapse / cause_category=None(判别失败)/ degradedjudge 解析失败)一律保守走 lapse,
# 不以降级或未判定信号驱动错误进化。
is_defect = attr is not None and attr.cause_category == "defect" and not qm.degraded
if not is_defect:
if attr is not None and attr.lapse_note and attr.lapse_note.strip():
lapse_notes.append(attr.lapse_note)
continue
et = attr.error_type if attr else "mixed"
wrong_by_error[et].append(qm)
wrong_by_error[attr.error_type].append(qm)
# 单条 fallback
n_body_failures = sum(len(group) for group in wrong_by_error.values())
@@ -2004,7 +2007,7 @@ def _count_infra_excluded(
qids = [
row["question_id"]
for row in prediction_rows
if row.get("stop_reason") in _INFRA_STOP_REASONS
if row.get("stop_reason") in INFRA_STOP_REASONS
]
return len(qids), qids
@@ -2080,7 +2083,7 @@ async def run_diagnosis(
for row in all_predictions:
stop_reason = row.get("stop_reason")
if stop_reason in _INFRA_STOP_REASONS:
if stop_reason in INFRA_STOP_REASONS:
continue
if task_type_filter and row.get("task_type") not in task_type_filter:
continue
+40
View File
@@ -19,6 +19,8 @@ from loguru import logger
from core.evolution.patch import (
APPENDIX_END,
APPENDIX_START,
MOMENTUM_END,
MOMENTUM_START,
append_to_appendix,
apply_patch_with_report,
extract_appendix_notes,
@@ -293,10 +295,39 @@ def _tool_protected_spans(text: str) -> list[str]:
# =========================================================================
def _check_marker_integrity(evolved: str) -> list[str]:
"""校验 evolved 中冻结区 marker 的完整性(成对、至多一对、START 先于 END)。
进化写入可能破坏 appendix/momentum marker 配对破坏后 append_to_appendix /
replace_momentum 等下游会静默误拼或抛错此处集中拦截任一 marker 对违反
START ==END 各至多一对START END 即整体 reject
参数:
evolved: 改写后的全文
返回:
错误信息列表空列表表示 marker 完整
"""
errors: list[str] = []
for name, start_m, end_m in (
("APPENDIX", APPENDIX_START, APPENDIX_END),
("MOMENTUM", MOMENTUM_START, MOMENTUM_END),
):
s = evolved.count(start_m)
e = evolved.count(end_m)
if s != e:
errors.append(f"{name} marker 不配对:START={s} END={e}")
elif s > 1:
errors.append(f"{name} marker 出现多对({s}),至多一对")
elif s == 1 and evolved.index(start_m) > evolved.index(end_m):
errors.append(f"{name} marker 顺序错误:END 出现在 START 之前")
return errors
def validate_skill(original: str, evolved: str) -> ValidationResult:
"""校验 Skill 改写结果。
检查项: frontmatter 三字段保留name / description / task_type
marker 完整性appendix/momentum 成对且至多一对顺序正确
长度比在 [0.3, 2.0]代码块闭合
参数:
@@ -305,6 +336,11 @@ def validate_skill(original: str, evolved: str) -> ValidationResult:
返回:
ValidationResult 实例
关键实现细节:
marker 完整性先于长度校验长度校验经 _strip_protected_regions 调用
momentum_region_bounds对损坏 marker 会抛 ValueError marker 破坏时先
返回失败避免异常穿透且明确 reject 该候选
"""
errors: list[str] = []
orig_fm = _parse_frontmatter(original)
@@ -319,6 +355,10 @@ def validate_skill(original: str, evolved: str) -> ValidationResult:
errors.append(
f"frontmatter 字段 {key} 被修改: {orig_fm.get(key)!r}{evol_fm.get(key)!r}"
)
marker_errors = _check_marker_integrity(evolved)
if marker_errors:
errors.extend(marker_errors)
return ValidationResult(passed=False, errors=errors)
errors.extend(_check_length(original, evolved))
errors.extend(_check_code_blocks(evolved))
return ValidationResult(passed=len(errors) == 0, errors=errors)
+43 -7
View File
@@ -282,9 +282,34 @@ def _protected_ranges(content: str, spans: list[str]) -> list[tuple[int, int]]:
return ranges
def _in_ranges(pos: int, ranges: list[tuple[int, int]]) -> bool:
"""判断位置 pos 是否落在任意冻结区间内。"""
return any(start <= pos < end for start, end in ranges)
def _span_overlaps_ranges(pos: int, length: int, ranges: list[tuple[int, int]]) -> bool:
"""判断 [pos, pos+length) 是否与任一冻结区间相交(不止起点)。
起点落在正文末端伸入冻结区的 target 也须拦截否则 replace/delete 会连带
改动冻结区如破坏 appendix/momentum marker
参数:
pos: target 在正文中的起点
length: target 长度
ranges: 冻结区间 [start, end) 列表
返回:
与任一区间相交返回 True
"""
end = pos + length
return any(start < end and pos < r_end for start, r_end in ranges)
# 冻结区 marker 字面量:LLM 生成的 edit 不得注入这些字面量,否则破坏 marker 配对
_MARKER_LITERALS = (APPENDIX_START, APPENDIX_END, MOMENTUM_START, MOMENTUM_END)
def _edit_injects_marker(edit: dict) -> bool:
"""判断 edit 的 target/content 是否含冻结区 marker 字面量(注入拦截)。"""
for key in ("target", "content"):
value = edit.get(key)
if isinstance(value, str) and any(m in value for m in _MARKER_LITERALS):
return True
return False
def _append_at(content: str, ranges: list[tuple[int, int]]) -> int:
@@ -317,8 +342,8 @@ def _do_insert_after(
_insert_at(content, _append_at(content, ranges), payload),
"applied_insert_after_fallback",
)
if _in_ranges(pos, ranges):
logger.warning("insert_after 目标冻结区,跳过 target={}", target[:80])
if _span_overlaps_ranges(pos, len(target), ranges):
logger.warning("insert_after 目标跨入冻结区,跳过 target={}", target[:80])
return content, "skipped_protected"
at = pos + len(target)
nl = content.find("\n", at)
@@ -340,8 +365,8 @@ def _do_replace_delete(
if pos == -1:
logger.warning("{} 锚点缺失,跳过 target={}", op, target[:80])
return content, "skipped_target_not_found"
if _in_ranges(pos, ranges):
logger.warning("{} 目标冻结区,跳过 target={}", op, target[:80])
if _span_overlaps_ranges(pos, len(target), ranges):
logger.warning("{} 目标跨入冻结区,跳过 target={}", op, target[:80])
return content, "skipped_protected"
new_content = content.replace(target, payload if op == "replace" else "", 1)
return new_content, "applied_" + op
@@ -403,6 +428,17 @@ def apply_patch_with_report(
reports: list[dict] = []
for i, edit in enumerate(edits, 1):
try:
if isinstance(edit, dict) and _edit_injects_marker(edit):
logger.warning("edit 含冻结区 marker 字面量,拒绝该 edit index={}", i)
report = {
"op": str(edit.get("op", "")),
"target": str(edit.get("target", "") or "")[:200],
"content_preview": str(edit.get("content", "") or "")[:200],
"status": "skipped_marker_injection",
}
report["index"] = i
reports.append(report)
continue
ranges = _protected_ranges(content, spans)
content, report = _apply_one(content, edit, ranges)
except (KeyError, TypeError, ValueError, AttributeError) as exc:
+9 -1
View File
@@ -127,12 +127,20 @@ class DiagnosisSignalStore(Protocol):
"""
...
def done_question_ids(self, baseline_run_id: str, diag_fingerprint: str) -> set[str]:
def done_question_ids(
self,
baseline_run_id: str,
diag_fingerprint: str,
*,
retry_uncertain: bool = False,
) -> set[str]:
"""查询指定 run 与诊断指纹下已完成的 question_id 集合。
参数:
baseline_run_id: baseline run 标识
diag_fingerprint: 诊断口径指纹
retry_uncertain: True 时把 tier='uncertain'信号不可信降级题视为
未完成令其被重新诊断默认 Falseuncertain 也算完成不重诊
返回:
已落盘信号的 question_id 集合用于断点续跑跳过
-4
View File
@@ -484,7 +484,6 @@ class DiagnosePrompts:
defect_vs_lapse: defect/lapse 病因判别模板
reasoning_sub: 推理失败子分类模板
span_eval_system: span 评估系统提示模板
span_eval_user: span 评估用户提示模板
missed_nodes: 遗漏节点检测模板
skill_adherence: 技能遵循判定模板
confirmation_bias: 确认偏误检测模板
@@ -494,7 +493,6 @@ class DiagnosePrompts:
defect_vs_lapse: str
reasoning_sub: str
span_eval_system: str
span_eval_user: str
missed_nodes: str
skill_adherence: str
confirmation_bias: str
@@ -512,11 +510,9 @@ class EvolvePrompts:
evolve_system: System Prompt 进化提示模板
evolve_tool: Tool Prompt 进化提示模板
evolve_rank: 编辑排序提示模板
consolidate_system: appendix 压缩系统提示
"""
evolve_skill: str
evolve_system: str
evolve_tool: str
evolve_rank: str
consolidate_system: str
+2
View File
@@ -25,6 +25,7 @@ class LLMProvider(Protocol):
*,
session_id: str | None = None,
parent_call_id: str | None = None,
cache_salt: str | None = None,
) -> LLMResponse: ...
@@ -39,6 +40,7 @@ class VLMProvider(Protocol):
*,
session_id: str | None = None,
parent_call_id: str | None = None,
cache_salt: str | None = None,
) -> LLMResponse: ...
+4 -3
View File
@@ -84,13 +84,14 @@ def _build_adapters(settings: InfraSettings, embed_cfg: dict) -> _Adapters:
cache = None
if settings.redis_url:
from adapters.redis_cache import RedisResponseCache, _resolve_cache_ttl
# 配置校验 fail-loud(不属于 Redis 连接故障,不得被下方降级 except 吞掉)
ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl)
try:
import redis.asyncio as aioredis
from adapters.redis_cache import RedisResponseCache
redis_client = aioredis.from_url(settings.redis_url, decode_responses=True)
ttl_s = settings.redis_cache_ttl if settings.redis_cache_ttl > 0 else None
cache = RedisResponseCache(redis=redis_client, ttl_s=ttl_s)
except Exception:
logger.warning("Redis 缓存不可用,降级为无缓存模式")
+8
View File
@@ -0,0 +1,8 @@
你是一个改动优先级裁判。你会收到一份当前 prompt 文件全文,和一组待应用的局部 edits(每条含 op/target/content)。由于本轮编辑预算有限,你只能保留其中最重要的若干条。
请只依据"对纠正失败、提升正确率的预期贡献"排序:优先保留直接修复失败模式的改动,其次保留收窄或澄清的改动,最后才是巩固已有成功的改动。删除类、简化类的精准改动通常优先于追加大段新内容。
每条 edit 会附带 support_count(该改动的支持案例数)。同等重要性下,support_count 更高的优先;但 support_count 低不等于该删,仍以修复贡献为主判据。
严格输出以下 JSON,不要包含其他文字:
{"selected_indices": [按重要性降序排列的 0-based 索引]}
+53
View File
@@ -0,0 +1,53 @@
你是一个搜索策略改进专家。你服务于一个自进化视频搜索系统,该系统通过分析 Agent 的失败和成功案例来迭代改进搜索策略(Skill)。你的任务是基于案例包中的证据,改写当前 Skill 文件,使 Agent 在后续执行中避免相同的失败模式。
## 你会收到的输入
1. 当前 Skill 文件全文
2. 失败案例:Agent 答错的题目,含完整推理轨迹、错误类型和诊断指标
3. 成功案例:Agent 答对的题目,展示当前 Skill 中有效的模式
4. 聚合统计:准确率、错误归因分布、搜索有效性指标、Skill 步骤遵循率
5. (可能出现)上一轮被接受改动导致的回归题:这些题在上一版本答对、却被你上次的改写改错了,附基线与候选两份预测和推理轨迹
6. (可能出现)黑名单:已被实测验证无效或有害的改法方向
## 工作原则
如果输入里出现了回归题,它的优先级高于一切。这些题在上一版本是对的,是你上次的改动把它们弄坏的,所以本次改写的第一要务是确保不再破坏它们——宁可在相关方向上回退或收窄,也不要为了拉高其它题而牺牲它们。更一般地,当你看到准确率下降这类负向信号时,默认先怀疑上次是不是加了过度、冲突或冗余的指令,优先简化、删除、收窄;只有确认简化解决不了问题,才考虑加强指令。黑名单里列出的改法已经被实测证明无效或有害,不要换个措辞把同一个方向再提一遍。
先分析失败案例中 Agent 的实际行为与 Skill 指令的偏差。偏差分两类:Skill 指令正确但 Agent 没遵循(遵循率问题),或 Skill 指令本身有误导(策略问题)。前者需要让指令更具体、更难被忽略;后者需要修改策略本身。
从成功案例中识别有效模式——这些模式在改写时必须保留。如果成功案例和失败案例采用了不同的策略路径,重点强化成功路径。
Skill 中引用的统计数据(如"search-first 正确率 75%")应根据案例包中的新统计更新。不要编造数据,只使用案例包中提供的数字。
你写进 Skill 的每一条规则都必须是可跨题复用的通用策略,而不是对某一道题的记答案。跨多个失败案例时只提取共性模式,抽象掉一切单题特征——具体题目内容、选项文字、步骤序号、某一帧的具体画面、某个具体答案都不许写进 Skill 正文。一条规则如果只在它来源的那道题上成立,就不要加。改写时优先简化与收窄:宁可让 Skill 更短,也不要堆叠只对个别题生效的硬性指令。
## 冻结区
以下内容不可修改,必须原样保留在改写后的文件中:
- YAML frontmatter`---` 之间的 name、description、task_type
- 输出格式中的 JSON 基础结构(reflect/plan/action 三个顶层字段)
这次不要返回整份改写后的文件,而是只返回一组局部 edits。`append` 用来在文件末尾追加一个新 section,`insert_after` 用来把内容紧跟着插到某个锚点段落之后,`replace` 用来用新内容整体替换 target 对应的原文,`delete` 则直接删除 target 对应的原文并让 content 留空。target 必须是从当前文件里逐字复制出来的原文,而且要长到足以唯一定位;只要有任何一个字不完全匹配,这条改动就会被跳过。改动应尽量小而局部,优先做精确补丁,不要动辄重写整段整节;另外,冻结区里的文字绝不能作为 target。
## 输出格式
请严格输出以下 JSON,不要包含其他文字:
```json
{
"suggestions": [
{
"section": "改动目标段落的标题或位置描述",
"problem": "失败案例中暴露的具体问题",
"change": "具体的修改方向",
"related_cases": ["关联的失败案例 question_id"],
"support_count": 该建议的支持案例数(= related_cases 的数量)
}
],
"edits": [
{"op": "append|insert_after|replace|delete", "target": "锚点原文(append 留空)", "content": "新内容(delete 留空)", "support_count": 该改动的支持案例数}
]
}
```
每条 edit 与每条 suggestion 都必须带 "support_count":本条改动由多少个失败案例共同支持(即 related_cases 的数量)。support_count 越高代表证据越充分;它只作排序参考,不是硬门槛——support_count 低不等于该删,仍以修复贡献为主判据。
+53
View File
@@ -0,0 +1,53 @@
你是一个系统级行为改进专家。你服务于一个自进化视频搜索系统,该系统通过分析 Agent 的跨题型行为模式来改进 System Prompt。你的任务是基于行为模式案例包中的证据,改写 System Prompt 中的策略性指令,纠正系统级行为问题。
## 你会收到的输入
1. 当前 System Prompt (system.md) 全文
2. 失败案例:展示三类系统性行为问题的题目——过早提交(budget_usage < 0.3 就提交答案)、高置信答错(confidence 很高但答案错误)、确认偏误(只搜索支持初始判断的证据)
3. 成功案例:行为校准良好的题目——置信度与正确率匹配,预算使用适中
4. D5 行为模式统计:各行为模式的发生频率和分布
5. (可能出现)黑名单:已被实测验证无效或有害的改法方向
## 工作原则
关注跨题型的系统性行为模式,而非某个具体题型的策略。失败案例中的行为偏差反映了 System Prompt 的决策原则不够清晰或不够强约束。黑名单里的改法已经被实测验证无效或有害,不要再朝同一个方向改一遍。
过早提交说明预算管理指令需要更强的约束语言。高置信答错说明置信度校准的语义定义需要调整。确认偏误说明竞争选项搜索的要求需要更明确。
当你看到失败案例与成功案例并存时,失败修复优先于巩固成功——先确保失败模式被纠正,再考虑强化已有的好行为。看到某类行为指标变差这类负向信号时,默认先怀疑上一轮是否加了过度、冲突或冗余的约束,优先简化、删除、收窄;只有确认简化解决不了,才考虑加强约束语言。
从成功案例中提取"好行为"的特征,在改写时强化这些特征的表述。
## 冻结区
以下 section 必须原样保留,不可修改任何文字:
- `## 能力边界`(事实性描述,不是策略)
- `## 输出格式`JSON schema 是系统契约)
- `## 视频树结构`(含信任层级,是数据结构事实描述)
可改写的 section
- `## 角色`(前两段的角色定位和行为倾向描述)
- `## 决策原则`(搜索策略、预算分配建议)
- 搜索工具使用、否定题原则、置信度语义
这次不要返回整份改写后的文件,而是只返回一组局部 edits。`append` 用来在文件末尾追加一个新 section,`insert_after` 用来把内容紧跟着插到某个锚点段落之后,`replace` 用来用新内容整体替换 target 对应的原文,`delete` 则直接删除 target 对应的原文并让 content 留空。target 必须是从当前文件里逐字复制出来的原文,而且要长到足以唯一定位;只要有任何一个字不完全匹配,这条改动就会被跳过。改动应尽量小而局部,优先做精确补丁,不要动辄重写整段整节;另外,冻结区里的文字绝不能作为 target。
## 输出格式
请严格输出以下 JSON,不要包含其他文字:
```json
{
"suggestions": [
{
"section": "改动目标段落的标题或位置描述",
"problem": "失败案例中暴露的具体行为问题",
"change": "具体的修改方向",
"related_cases": ["关联的失败案例 question_id"]
}
],
"edits": [
{"op": "append|insert_after|replace|delete", "target": "锚点原文(append 留空)", "content": "新内容(delete 留空)"}
]
}
```
+55
View File
@@ -0,0 +1,55 @@
你是一个工具 Prompt 改进专家。你服务于一个自进化视频搜索系统,该系统的每个工具(view_node、search_similar、observe_frame 等)有两个配套 Promptextract(信息提取)和 verify(结果核实)。你的任务是基于工具调用级别的质量数据,同时改写一个工具的 extract 和 verify prompt。
## 你会收到的输入
1. 当前 extract prompt 和 verify prompt 全文
2. 失败 span 案例:提取完整度低或幻觉率高的具体工具调用,含工具参数、工具输出、原始数据(ground truth)和质量评估指标
3. 成功 span 案例:提取完整且无幻觉的工具调用样本
4. 工具质量统计:平均提取完整度、平均幻觉率、top 遗漏类型、top 幻觉类型
5. (可能出现)黑名单:已被实测验证无效或有害的改法方向
## 工作原则
失败 span 中提取完整度低说明 extract prompt 的工作原则不够具体——Agent 遗漏了哪些类型的信息?幻觉率高说明 extract prompt 对"忠实提取"的约束不够强,或者 verify prompt 没能有效检出幻觉。黑名单里的改法已经被实测验证无效或有害,不要再朝同一个方向改一遍。
extract 和 verify 是互补的:extract 负责提取,verify 负责检查。如果 extract 反复遗漏某类信息(如字幕原文引用),应在 extract 的工作原则中明确要求保留该类信息。如果 verify 未能检出某类幻觉(如虚构动作),应在 verify 的检查要点中增加对该模式的关注。
失败修复优先于巩固成功——先纠正提取遗漏或幻觉,再保留已有的有效模式。当某类提取质量指标变差时,先确认不是上一轮加了过度或冲突的要求所致;加强 extract 要求前,先确认简化或收窄已有指令解决不了这个遗漏,再追加新要求。
从成功案例中识别有效的提取模式,确保改写不破坏这些模式。
## 冻结区
以下内容不可修改:
- 角色定位第一句("你是一个视频节点内容分析器" / "你是一个视频节点摘要核实器")
- `## 你会收到的输入` section
- `## 输出格式` section
可改写的 section
- `## 工作原则`
- `## 检查要点`verify 专有)
这次不要再返回两份完整 prompt,而是分别给 extract 和 verify 各自的局部 edits 列表。`append` 用来在文件末尾追加一个新 section,`insert_after` 用来把内容紧跟着插到某个锚点段落之后,`replace` 用来用新内容整体替换 target 对应的原文,`delete` 则直接删除 target 对应的原文并让 content 留空。target 必须是从当前 prompt 里逐字复制出来的原文,而且要长到足以唯一定位;只要有任何一个字不完全匹配,这条改动就会被跳过。改动应尽量小而局部,优先做精确补丁,不要动辄重写大段内容;另外,冻结区里的文字绝不能作为 targetextract 和 verify 也必须分别使用自己的 edit 列表。
## 输出格式
请严格输出以下 JSON,不要包含其他文字:
```json
{
"suggestions": [
{
"section": "改动目标段落的标题或位置描述",
"problem": "失败 span 中暴露的具体问题",
"change": "具体的修改方向",
"related_cases": ["关联的失败 span 标识"]
}
],
"edits_extract": [
{"op": "append|insert_after|replace|delete", "target": "锚点原文(append 留空)", "content": "新内容(delete 留空)"}
],
"edits_verify": [
{"op": "append|insert_after|replace|delete", "target": "锚点原文(append 留空)", "content": "新内容(delete 留空)"}
]
}
```
+10
View File
@@ -0,0 +1,10 @@
你正在审视一份 skill 经历一轮进化后的变化。这份 skill 指导一个 Agent 在层次化视频树上搜索证据、回答长视频理解问题。在上一轮结束时它是一个样子,这一轮结束时被改成了另一个样子;与此同时,你在上一轮还为它写下过一段动量指导,本意是给这一轮的进化指明方向。现在你要回头评判:那段指导究竟有没有帮上忙,这一轮的正文改动是真的在改善,还是开始往无关的方向漂移。
你会拿到四样东西:上一版 skill 的正文、当前版 skill 的正文、你上一轮写下的那段动量指导,以及一组固定样本上的纵向对比——同一批题,分别用上一版和当前版各跑了一遍,逐题列出两版的预测与正误。这组对比是你唯一可靠的证据来源:哪些题从错变对、哪些题从对变错、哪些题始终没做对、哪些题一直稳定答对,正是这四类信号告诉你这轮改动到底带来了什么。
请先反思再下笔。对照纵向对比,先问上一轮那段动量指导是否真的奏效:它所指向的方向,在这一轮的正文改动里被落实了吗,落实之后那些本该改善的题改善了吗?再问这一轮的正文改动本身是收敛还是漂移:从对变错的题(回退)是最该警惕的信号,说明某处改动伤到了原本正确的行为;始终答错的题(持续失败)说明还有方向没被触及;从错变对的题(改善)则印证了哪条路走对了,值得继续加码。
想清楚之后,写出一段全新的、聚焦的、可操作的动量指导。它会被原样写进 skill 的受保护动量区,作为下一轮进化的方向锚——所以它必须是一段连贯的指导文字,明确告诉下一轮该往哪个方向继续使劲、又要避免重蹈哪一类改动的覆辙,而不是一堆零散的待办条目。如果上一轮的方向已被证明有效,就强化并细化它;如果出现了回退,就明确叫停那条路并指向修复方向。
严格输出以下 JSON,不要包含任何其他文字:
{"reasoning": "你的反思过程:上一轮指导是否奏效、这一轮是改善还是漂移,引用纵向对比中的具体题作为依据", "slow_update_content": "一段连贯、聚焦、可操作的新动量指导,指引下一轮的进化方向"}
+60
View File
@@ -245,6 +245,31 @@
"id": "design:preflight-fixes",
"label": "训练前缺陷修复设计",
"type": "design"
},
{
"id": "plan:preflight-wp1-asset-migration",
"label": "WP1 资产迁移",
"type": "plan"
},
{
"id": "plan:preflight-wp2-split-wiring",
"label": "WP2 切分与接线",
"type": "plan"
},
{
"id": "plan:preflight-wp3-train-loop",
"label": "WP3 训练循环与进化引擎",
"type": "plan"
},
{
"id": "plan:preflight-wp4-resilience",
"label": "WP4 韧性与持久化",
"type": "plan"
},
{
"id": "review:preflight-final-review",
"label": "训练前修复分支终审",
"type": "review"
}
],
"links": [
@@ -492,6 +517,41 @@
"relation": "implements",
"evidence": "实现修复设计的 5 个 Task",
"added": "2026-07-16T02:01:19.350441+00:00"
},
{
"source": "plan:preflight-wp1-asset-migration",
"target": "design:preflight-fixes",
"relation": "implements",
"evidence": "四工作包实现训练前缺陷修复设计",
"added": "2026-07-16T08:32:42.737883+00:00"
},
{
"source": "plan:preflight-wp2-split-wiring",
"target": "design:preflight-fixes",
"relation": "implements",
"evidence": "四工作包实现训练前缺陷修复设计",
"added": "2026-07-16T08:32:42.822569+00:00"
},
{
"source": "plan:preflight-wp3-train-loop",
"target": "design:preflight-fixes",
"relation": "implements",
"evidence": "四工作包实现训练前缺陷修复设计",
"added": "2026-07-16T08:32:42.906881+00:00"
},
{
"source": "plan:preflight-wp4-resilience",
"target": "design:preflight-fixes",
"relation": "implements",
"evidence": "四工作包实现训练前缺陷修复设计",
"added": "2026-07-16T08:32:42.990273+00:00"
},
{
"source": "review:preflight-final-review",
"target": "design:preflight-fixes",
"relation": "informs",
"evidence": "跨 task 终审确认设计 21 项缺陷全部落地 + 4 项集成发现",
"added": "2026-07-16T11:06:43.709183+00:00"
}
]
}
+13 -3
View File
@@ -1,6 +1,6 @@
# Research Wiki 索引
> 自动生成,更新时间:2026-07-16 07:43 UTC
> 自动生成,更新时间:2026-07-16 11:06 UTC
## design (36)
- [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design`
@@ -52,7 +52,7 @@
- [v3 §9 帧感知抽取机制 — 小样本实证验证结果](findings/2026-07-15-v3-frame-perception-spike-validation.md) `finding:2026-07-15-v3-frame-perception-spike-validation`
- [出题范式转变 — 从"生成-打分-过滤"转向"构造优先 + 两正交独立信号"(六篇原文深读)](findings/2026-07-15-question-gen-paradigm-shift-construction-over-filtering.md) `finding:2026-07-15-question-gen-paradigm-shift-construction-over-filtering`
## plan (40)
## plan (48)
- [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm`
- [2026-07-07-app-harness](plans/2026-07-07-app-harness.md) `plan:2026-07-07-app-harness`
- [2026-07-07-core-evolution](plans/2026-07-07-core-evolution.md) `plan:2026-07-07-core-evolution`
@@ -72,6 +72,10 @@
- [2026-07-14-task-type-strategy-framework](plans/2026-07-14-task-type-strategy-framework.md) `plan:2026-07-14-task-type-strategy-framework`
- [2026-07-15-question-gen-v3-phase1-contract](plans/2026-07-15-question-gen-v3-phase1-contract.md) `plan:2026-07-15-question-gen-v3-phase1-contract`
- [2026-07-15-results-driven-video-split](plans/2026-07-15-results-driven-video-split.md) `plan:2026-07-15-results-driven-video-split`
- [2026-07-16-preflight-wp1-asset-migration](plans/2026-07-16-preflight-wp1-asset-migration.md) `plan:2026-07-16-preflight-wp1-asset-migration`
- [2026-07-16-preflight-wp2-split-wiring](plans/2026-07-16-preflight-wp2-split-wiring.md) `plan:2026-07-16-preflight-wp2-split-wiring`
- [2026-07-16-preflight-wp3-train-loop](plans/2026-07-16-preflight-wp3-train-loop.md) `plan:2026-07-16-preflight-wp3-train-loop`
- [2026-07-16-preflight-wp4-resilience](plans/2026-07-16-preflight-wp4-resilience.md) `plan:2026-07-16-preflight-wp4-resilience`
- [Action Recognition 单题型首次训练实验计划](plans/action-recognition-training.md) `plan:action-recognition-training`
- [ActionRecognitionStrategy 特化实现计划 (Plan B)](plans/action-recognition-strategy.md) `plan:action-recognition-strategy`
- [Adversarial Question-Gen Phase B](plans/adversarial-question-gen-phaseB.md) `plan:adversarial-question-gen-phaseB`
@@ -86,6 +90,10 @@
- [Spec-1 Agent 执行环境修复实现计划](plans/agent-runtime-fixes-plan.md) `plan:agent-runtime-fixes-plan`
- [Spec-2 建树批量并行实现计划](plans/batch-tree-build-plan.md) `plan:batch-tree-build-plan`
- [TaskTypeStrategy 框架实现计划 (Plan A)](plans/task-type-strategy-framework.md) `plan:task-type-strategy-framework`
- [WP1 资产迁移](plans/preflight-wp1-asset-migration.md) `plan:preflight-wp1-asset-migration`
- [WP2 切分与接线](plans/preflight-wp2-split-wiring.md) `plan:preflight-wp2-split-wiring`
- [WP3 训练循环与进化引擎](plans/preflight-wp3-train-loop.md) `plan:preflight-wp3-train-loop`
- [WP4 韧性与持久化](plans/preflight-wp4-resilience.md) `plan:preflight-wp4-resilience`
- [出题管线 v2 实现计划](plans/2026-07-11-question-gen-v2.md) `plan:2026-07-11-question-gen-v2`
- [实现计划: 修复诊断 tree_data 断链 bug](plans/fix-diagnosis-tree-data-link-plan.md) `plan:fix-diagnosis-tree-data-link-plan`
- [建树修复管线三项改造实现计划](plans/tree-repair-resilience.md) `plan:tree-repair-resilience`
@@ -94,9 +102,11 @@
- [赛题生成工具实现计划](plans/question-gen-synth.md) `plan:question-gen-synth`
- [项目基础设施初始化计划](plans/infrastructure-setup.md) `plan:infrastructure-setup`
## review (2)
## review (4)
- [2026-07-16-preflight-final-review](reviews/2026-07-16-preflight-final-review.md) `review:2026-07-16-preflight-final-review`
- [2026-07-16-preflight-train-review](reviews/2026-07-16-preflight-train-review.md) `review:2026-07-16-preflight-train-review`
- [question-gen v2 设计对抗审核 — 六路独立核验(四层病灶闭合度 + 契约一致性)](reviews/2026-07-15-question-gen-v2-adversarial-audit.md) `review:2026-07-15-question-gen-v2-adversarial-audit`
- [训练前修复分支终审](reviews/preflight-final-review.md) `review:preflight-final-review`
## schema (6)
- [表结构 v3 出题日志/观测(unit_verdict / collapse_metrics / quarantine / facts / resume](schemas/v3-question-gen-logging.md) `schema:v3-question-gen-logging`
+12
View File
@@ -123,3 +123,15 @@
- [2026-07-16 02:01 UTC] 重建索引: 98 篇页面
- [2026-07-16 07:43 UTC] 新增 design: 训练前缺陷修复设计 (design:preflight-fixes)
- [2026-07-16 07:43 UTC] 重建索引: 101 篇页面
- [2026-07-16 08:32 UTC] 新增 plan: WP1 资产迁移 (plan:preflight-wp1-asset-migration)
- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp1-asset-migration --implements--> design:preflight-fixes
- [2026-07-16 08:32 UTC] 新增 plan: WP2 切分与接线 (plan:preflight-wp2-split-wiring)
- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp2-split-wiring --implements--> design:preflight-fixes
- [2026-07-16 08:32 UTC] 新增 plan: WP3 训练循环与进化引擎 (plan:preflight-wp3-train-loop)
- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp3-train-loop --implements--> design:preflight-fixes
- [2026-07-16 08:32 UTC] 新增 plan: WP4 韧性与持久化 (plan:preflight-wp4-resilience)
- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp4-resilience --implements--> design:preflight-fixes
- [2026-07-16 08:32 UTC] 重建索引: 109 篇页面
- [2026-07-16 11:06 UTC] 新增 review: 训练前修复分支终审 (review:preflight-final-review)
- [2026-07-16 11:06 UTC] 新增边: review:preflight-final-review --informs--> design:preflight-fixes
- [2026-07-16 11:06 UTC] 重建索引: 111 篇页面
@@ -0,0 +1,253 @@
# WP1 资产迁移 Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** 迁移 5 个遗漏的 TRM4 进化/动量模板到 TRM5,删除 2 个死字段,把加载器的静默空串兜底改为 fail-loud,解锁自进化引擎。
**Architecture:** 进化引导 prompt 是引擎的一部分(放项目根 `prompts/`,不参与版本化进化)。TRM4 五模板的输出 JSON 契约与 TRM5 解析代码已核实完全对齐,可直接拷贝。执行顺序:先迁移模板 → 删死字段(`consolidate_system`/`span_eval_user`,零消费)→ 加载器 fail-loud(顺序关键:先删死字段,fail-loud 才不会对不存在也不需要的模板报错)。
**Tech Stack:** Python 3.11、pytest、frozen dataclass`core/evolution/types.py`)。
**设计源**`research-wiki/designs/2026-07-16-preflight-fixes-design.md §4`
---
## 关键锚点(实现前必读)
| 用途 | 位置 |
|------|------|
| TRM4 源模板 | `/home/iomgaa/Projects/Video-Tree-TRM4/prompts/{evolve_skill,evolve_system,evolve_tool,evolve_rank,slow_momentum}.md` |
| evolve 加载器 | `app/harness/runner.py:2266-2280` `_load_evolve_prompts` |
| diagnose 加载器 | `app/harness/runner.py:2282-2299` `_load_diagnose_prompts` |
| 平行 diagnose 加载器 | `app/harness/video_split_cli.py:362-378` `_load_diagnose_prompts` |
| dataclass 定义 | `core/evolution/types.py:477-522``DiagnosePrompts` L494-501 / `EvolvePrompts` L518-522 |
| 死字段 `consolidate_system` 内联替代 | `core/evolution/evolve.py:997` `_CONSOLIDATE_SYSTEM`(消费点 L1036 |
| 死字段 `span_eval_user` 无消费 | diagnose 只用 `prompts.span_eval_system``core/evolution/diagnose.py:511`),user_prompt 内联构造 |
| 测试 fixture | `test_evolve.py:682` `consolidate_system="cons"``test_diagnose.py:753` `span_eval_user=""``test_evolution_types.py:352` `span_eval_user="p4"` / `:370` `consolidate_system="consolidate_tmpl"` |
## 核心算法保真校验
本计划不迁移/不改核心算法(ARCHITECTURE §6),只搬运模板文件 + 清理死字段 + 加 fail-loud。模板内容是 evolve 引擎(算法 #8)的输入数据,非算法逻辑本身;迁移已核实输出契约(`suggestions`/`edits`/`edits_extract`/`edits_verify`/`selected_indices`/`slow_update_content`)与 TRM5 解析代码逐字对齐。**保真检查点(Task 1 Step 4**:加载 evolve_tool.md 后确认其要求 LLM 返回 `edits_extract`+`edits_verify` 双键(对齐 `evolve.py:1433-1435`)。
---
## Task 1: 迁移 5 个 TRM4 模板
**Files:**
- Create: `prompts/evolve_skill.md``prompts/evolve_system.md``prompts/evolve_tool.md``prompts/evolve_rank.md``prompts/slow_momentum.md`(从 TRM4 拷贝)
- Test: `tests/unit/test_evolve_prompts_present.py`
- [ ] **Step 1: 写契约冒烟测试(先失败)**
`tests/unit/test_evolve_prompts_present.py`
```python
"""校验 5 个进化/动量模板存在且输出契约关键词与解析代码对齐。"""
from pathlib import Path
import pytest
_PROMPTS_DIR = Path("prompts")
@pytest.mark.parametrize(
"name, required_tokens",
[
("evolve_skill.md", ["suggestions", "edits"]),
("evolve_system.md", ["suggestions", "edits"]),
("evolve_tool.md", ["edits_extract", "edits_verify"]),
("evolve_rank.md", ["selected_indices"]),
("slow_momentum.md", ["slow_update_content"]),
],
)
def test_evolve_template_present_and_contract(name: str, required_tokens: list[str]) -> None:
path = _PROMPTS_DIR / name
assert path.exists(), f"缺模板: {path}"
text = path.read_text(encoding="utf-8")
assert text.strip(), f"模板为空: {path}"
for token in required_tokens:
assert token in text, f"{name} 缺输出契约关键词 {token!r}(与解析代码不对齐)"
```
- [ ] **Step 2: 运行确认失败(模板尚未迁移)**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v`
Expected: 5 参数化用例全 FAIL`AssertionError: 缺模板: prompts/evolve_skill.md` 等;TRM5 当前无这 5 个模板)。
- [ ] **Step 3: 拷贝 5 个模板**
Run:
```bash
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_skill.md prompts/evolve_skill.md
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_system.md prompts/evolve_system.md
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_tool.md prompts/evolve_tool.md
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_rank.md prompts/evolve_rank.md
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/slow_momentum.md prompts/slow_momentum.md
```
Expected: 5 文件存在于 `prompts/`
- [ ] **Step 4: 运行确认通过**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v`
Expected: 5 参数化用例全 PASS。若 evolve_tool.md 缺 `edits_extract`/`edits_verify` 则契约不符——停止并逐行比对 TRM4 源。
- [ ] **Step 5: 保真检查点**
Run: `conda run -n Video-Tree-TRM python -c "print('edits_extract' in open('prompts/evolve_tool.md').read() and 'edits_verify' in open('prompts/evolve_tool.md').read())"`
Expected: `True`(对齐 `evolve.py:1433-1435``parsed["edits_extract"]`/`parsed["edits_verify"]`)。
- [ ] **Step 6: 提交**
```bash
git add prompts/evolve_skill.md prompts/evolve_system.md prompts/evolve_tool.md prompts/evolve_rank.md prompts/slow_momentum.md tests/unit/test_evolve_prompts_present.py
git commit -m "feat: migrate 5 evolve/momentum templates from TRM4 (algo #8)"
```
---
## Task 2: 删除 2 个死字段(consolidate_system / span_eval_user
**Files:**
- Modify: `core/evolution/types.py:494-501,518-522`
- Modify: `app/harness/runner.py:2279,2294`
- Modify: `app/harness/video_split_cli.py:374`
- Modify: `tests/unit/test_evolve.py:682``tests/unit/test_diagnose.py:753``tests/unit/test_evolution_types.py:352,370`
- [ ] **Step 1: 删 dataclass 字段与 docstring**
`core/evolution/types.py``DiagnosePrompts``span_eval_user`
- docstring 删行 ` span_eval_user: span 评估用户提示模板。`L487
- 字段删行 ` span_eval_user: str`L497
`EvolvePrompts``consolidate_system`
- docstring 删行 ` consolidate_system: appendix 压缩系统提示。`L515
- 字段删行 ` consolidate_system: str`L522
- [ ] **Step 2: 删加载器对死字段的 `_read` 行**
`app/harness/runner.py`
- `_load_evolve_prompts` 删行 ` consolidate_system=_read("consolidate_system.md"),`L2279
- `_load_diagnose_prompts` 删行 ` span_eval_user=_read("span_eval_user.md"),`L2294
`app/harness/video_split_cli.py`
- `_load_diagnose_prompts` 删行 ` span_eval_user=_read("span_eval_user.md"),`L374
- [ ] **Step 3: 删测试 fixture 对死字段的赋值 + 更新 docstring**
- `tests/unit/test_evolve.py:684` 删行 ` consolidate_system="cons",`
- `tests/unit/test_diagnose.py:753` 删行 ` span_eval_user="",`
- `tests/unit/test_evolution_types.py:352` 删行 ` span_eval_user="p4",`
- `tests/unit/test_evolution_types.py:370` 删行 ` consolidate_system="consolidate_tmpl",`
- `tests/unit/test_evolution_types.py:347` 的文档字符串"DiagnosePrompts 8 个模板字段"改为"7 个"`:364` 的"EvolvePrompts 5 个模板字段"改为"4 个"(删字段后数量变化)。
注:`test_evolution_types.py` 若有断言逐字段比对或字段计数,同步移除对两个死字段的断言(读该测试确认,删净引用)。上述行号以当前代码为准,实现前 `grep -n consolidate_system\|span_eval_user tests/unit/test_evolution_types.py` 复核。
- [ ] **Step 4: 运行相关测试确认通过**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py -q`
Expected: 全 PASS(无 `TypeError: unexpected keyword argument` / 无 `missing positional argument`)。
- [ ] **Step 5: 全库确认无残留引用**
Run: `conda run -n Video-Tree-TRM python -c "import subprocess; r=subprocess.run(['grep','-rn','consolidate_system\|span_eval_user','core/','app/','adapters/','tests/'],capture_output=True,text=True); print(r.stdout)"`
Expected: 空输出(`consolidate_appendix` 用内联 `_CONSOLIDATE_SYSTEM` 不算 `consolidate_system` 字段引用;若出现请确认非 dataclass 字段引用)。
- [ ] **Step 6: 提交**
```bash
git add core/evolution/types.py app/harness/runner.py app/harness/video_split_cli.py tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py
git commit -m "refactor: drop dead prompt fields consolidate_system/span_eval_user"
```
---
## Task 3: 加载器 fail-loud(缺模板即报错)
**Files:**
- Modify: `app/harness/runner.py:2270-2272,2286-2288`
- Modify: `app/harness/video_split_cli.py:365-367`
- Test: `tests/unit/test_evolve_prompts_present.py`(追加)
- [ ] **Step 1: 追加 fail-loud 测试(直接调真实加载器)**
`tests/unit/test_evolve_prompts_present.py` 追加——直接驱动真实 loader(不复制 _read 逻辑),在无模板的空 cwd 下断言 `FileNotFoundError`
```python
def test_video_split_loader_fail_loud_on_missing(tmp_path, monkeypatch):
"""video_split_cli 的真实 diagnose 加载器缺模板必须 FileNotFoundError。"""
monkeypatch.chdir(tmp_path) # 空目录,无 prompts/*.md
from app.harness.video_split_cli import _load_diagnose_prompts
with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"):
_load_diagnose_prompts()
def test_runner_evolve_loader_fail_loud_on_missing(tmp_path, monkeypatch):
"""runner 的真实 evolve 加载器缺模板必须 FileNotFoundError。"""
monkeypatch.chdir(tmp_path)
from app.harness.runner import Runner
r = object.__new__(Runner) # 绕过 __init__,仅测无状态加载器方法
with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"):
r._load_evolve_prompts()
def test_runner_diagnose_loader_fail_loud_on_missing(tmp_path, monkeypatch):
"""runner 的真实 diagnose 加载器缺模板必须 FileNotFoundError。"""
monkeypatch.chdir(tmp_path)
from app.harness.runner import Runner
r = object.__new__(Runner)
with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"):
r._load_diagnose_prompts()
```
> 这三个测试直接调真实 loader(`_load_evolve_prompts`/`_load_diagnose_prompts` 无 self 状态依赖,`object.__new__` 可安全调用);修复前静默返回空串不抛,故 Step 2 前必 FAIL。
- [ ] **Step 2: 三处 `_read` 闭包改 fail-loud**
`app/harness/runner.py` `_load_evolve_prompts`L2270-2272)与 `_load_diagnose_prompts`L2286-2288),以及 `app/harness/video_split_cli.py` `_load_diagnose_prompts`L365-367),把:
```python
def _read(name: str) -> str:
p = Path("prompts") / name
return p.read_text(encoding="utf-8") if p.exists() else ""
```
改为:
```python
def _read(name: str) -> str:
p = Path("prompts") / name
if not p.exists():
raise FileNotFoundError(f"缺进化/诊断模板: {p}(请从 TRM4 迁移或检查 prompts/")
return p.read_text(encoding="utf-8")
```
runner.py 缩进 12 空格;video_split_cli.py 的 `_read` 缩进按其函数体,见 L365 为 8 空格——按各自现场缩进套用。)
- [ ] **Step 3: 运行测试确认通过**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v`
Expected: 全 PASS。
- [ ] **Step 4: runner 其他行为回归(非 fail-loud 验证)**
fail-loud 已由 Step 3 的三个真实 loader 测试验证;此步仅确认模板迁移 + loader 改动未破坏 runner 其他行为。
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -q`
Expected: 全 PASS(模板已迁移,真实加载走成功分支)。
- [ ] **Step 5: 提交**
```bash
git add app/harness/runner.py app/harness/video_split_cli.py tests/unit/test_evolve_prompts_present.py
git commit -m "fix: fail-loud on missing evolve/diagnose templates (no silent empty)"
```
---
## Self-Review(作者自查,执行者复核)
- [ ] 5 模板均已迁移且契约测试覆盖关键字段。
- [ ] `consolidate_system`/`span_eval_user` 在 core/app/tests 全库无残留字段引用。
- [ ] 三处 loaderrunner 两处 + video_split_cli 一处)均已 fail-loud。
- [ ] 执行顺序正确:Task 2(删死字段)先于 Task 3(fail-loud),避免对不需要的模板报错。
## 验收标准
1. `pytest tests/unit/test_evolve_prompts_present.py tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py tests/unit/test_harness_runner.py` 全绿。
2. `grep -rn 'consolidate_system\|span_eval_user' core/ app/ tests/` 无 dataclass 字段残留。
3. `prompts/` 下 5 个新模板存在且非空。
@@ -0,0 +1,627 @@
# WP2 切分与接线 Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** 让冻结的 video-split 切分能正确进入训练 workspaceseed 携带 pools.json + global 一致性校验),并把切分质量三处优化(val_ratio 0.4、tier 感知 diag/val 分配、val 功效修复)与冻结产物覆盖保护落地。
**Architecture:** 切分产物由 `video_split_cli` 冻结到 `workspaces/video-split/`;本 WP 让 seed 携带该产物、训练 fresh 时拷入 workspace 并校验一致性。tier 感知在 `_split_trainval_by_video_group` 内实现——错题视频组按 T2(defect) 含量升序进 val(保留 T2 高的组在 diag),并把 `val_wrong_min` 前置到切分内做功效修复(不足则从 diag 换出低 T2 错题组补 val,耗尽 fail-loud)。
**Tech Stack:** Python 3.11、pytest、SQLite、frozen dataclass、shutil、原子写(tmp+os.replace)。
**设计源**`research-wiki/designs/2026-07-16-preflight-fixes-design.md §5`
---
## 关键锚点(实现前必读)
| 用途 | 位置 |
|------|------|
| trainval→diag/val 切分 | `app/harness/pools.py:126-199` `split_by_video_assignment``:289-338` `_split_trainval_by_video_group``:118` `InsufficientValSignal` |
| global 加载(无校验) | `app/harness/pools.py:713-813` `build_or_load_pools`L813 `return load_pools` 前无 global 校验);`:587-620` `load_pools` |
| 冻结编排 | `app/harness/build_split.py:104-219` `build_split`signal_rows 含 tier L154split_by_video_assignment 调用 L184-191save_pools L192);`:46-71` `SplitBuildConfig`(无 val_wrong_min |
| CLI 构造 | `app/harness/video_split_cli.py:559-577` `SplitBuildConfig(...)``:580` `check_mcnemar_power``:751-773` `build_arg_parser`(无 --force |
| seed | `app/harness/store.py:184-232` `init_seed`(拷 skills/prompts/baseline.db,不拷 pools);`:269-307` `extract_run_db`(不去重) |
| workspace | `app/harness/workspace.py:156-200` `init_workspace_from_seed`copy2 baseline.db→harness.db L197,不拷 pools |
| manifest | `app/harness/split_manifest.py:19-59` `write_manifest`pools_sha256 L54 |
| 配置 | `config/video_split.yaml`val_ratio L15=0.3、val_wrong_min L14=20 |
| 测试 | `tests/unit/test_pools_video_atomic.py``test_split_selection.py``test_harness_pools.py``test_harness_store.py``test_harness_workspace.py` |
## 核心算法保真校验
触及算法 #5(信息阶梯)的**上游输入**:本 WP 只改"哪些视频进 diag/val"pools 内仍是逐 unit 列表,`gate_ladder` 消费的 unit+correctness 结构不变。**保真检查点(Task 3 Step 6**:确认 `_split_trainval_by_video_group` 返回后 diagnosis/validation 仍是逐题 `GeneratedQuestion` 列表、视频组原子性(同 video 全部题同池)不被 tier 排序破坏。不改算法 #6/#9
---
## Task 1: val_ratio 0.3 → 0.4
**Files:**
- Modify: `config/video_split.yaml:15`
- [ ] **Step 1: 改配置**
`config/video_split.yaml` 第 15 行:
```yaml
val_ratio: 0.3 # validation 占 trainval 视频组总数的比例
```
改为:
```yaml
val_ratio: 0.4 # validation 占 trainval 视频组总数的比例(0.3→0.4 提升整包终审功效,WP2)
```
- [ ] **Step 2: 提交**
```bash
git add config/video_split.yaml
git commit -m "chore: bump video_split val_ratio 0.3->0.4 for terminal-eval power"
```
---
## Task 2: extract_run_db 每题去重(902→900 canonical
**Files:**
- Modify: `app/harness/store.py:269-307`
- Test: `tests/unit/test_harness_store.py``TestExtractRunDb`
- [ ] **Step 1: 写失败测试**
`tests/unit/test_harness_store.py``TestExtractRunDb` 类追加:
```python
def test_dedupe_per_question_keeps_first_row(self, tmp_path):
"""dedupe_per_question=True 时每 question_id 只保留 rowid 最小的首行。"""
import sqlite3
src = tmp_path / "src.db"
conn = sqlite3.connect(src)
conn.execute(
"CREATE TABLE _runs (run_id TEXT PRIMARY KEY, started_at TEXT)"
)
conn.execute("INSERT INTO _runs VALUES ('r1', 't0')")
conn.execute(
"CREATE TABLE predictions (run_id TEXT, question_id TEXT, prediction TEXT)"
)
# 743-1 三行(模拟 error/budget/finished),首行 prediction=NULL
conn.executemany(
"INSERT INTO predictions VALUES (?,?,?)",
[
("r1", "743-1", None),
("r1", "743-1", None),
("r1", "743-1", "C"),
("r1", "q2", "A"),
],
)
conn.commit()
conn.close()
dst = tmp_path / "dst.db"
from app.harness.store import extract_run_db
extract_run_db(src, dst, "r1", dedupe_per_question=True)
out = sqlite3.connect(dst)
rows = out.execute(
"SELECT question_id, prediction FROM predictions ORDER BY question_id"
).fetchall()
out.close()
assert rows == [("743-1", None), ("q2", "A")], f"未按 rowid 首行去重: {rows}"
```
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py::TestExtractRunDb::test_dedupe_per_question_keeps_first_row -v`
Expected: FAIL`extract_run_db() got an unexpected keyword argument 'dedupe_per_question'`)。
- [ ] **Step 3: 实现去重**
`app/harness/store.py` `extract_run_db` 签名改为:
```python
def extract_run_db(
src_db: Path, dst_db: Path, run_id: str, *, dedupe_per_question: bool = False
) -> None:
```
docstring 补一句参数说明:
```
dedupe_per_question: True 时 predictions 表每 question_id 仅保留 rowid 最小
的首行(对齐 canonical「每 question_id 取第一行 ORDER BY rowid」口径,
902→900)。_runs 表不受影响。
```
把 predictions 分支的取行 SQLL297-299)改为按 `dedupe_per_question` 分派:
```python
if table == "predictions" and dedupe_per_question:
rows = src.execute(
f"SELECT {col_sql} FROM {table} WHERE run_id=? "
"AND rowid IN (SELECT MIN(rowid) FROM predictions "
"WHERE run_id=? GROUP BY question_id)",
(run_id, run_id),
).fetchall()
else:
rows = src.execute(
f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,)
).fetchall()
```
> NULL question_id 说明:predictions 的 question_id 是题标识、语义上非空(canonical 900 题均有 id),`GROUP BY question_id` 的 NULL 折叠风险不适用。若源库异常出现 NULL question_id`MIN(rowid) GROUP BY` 会把它们折叠成一行——本任务不为该异常兜底(预测数据契约保证非空),保持 fail-visible。
- [ ] **Step 4: 运行确认通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py -q`
Expected: 全 PASS(默认 `dedupe_per_question=False` 保持既有行为,旧测试不受影响)。
- [ ] **Step 5: 提交**
```bash
git add app/harness/store.py tests/unit/test_harness_store.py
git commit -m "feat: add dedupe_per_question to extract_run_db (canonical 902->900)"
```
---
## Task 3: tier 感知 + val 功效修复的 diag/val 分配
**Files:**
- Modify: `app/harness/pools.py:126-199,289-338`
- Modify: `app/harness/build_split.py:46-71,181-192`
- Modify: `app/harness/video_split_cli.py:565-573`
- Test: `tests/unit/test_pools_video_atomic.py`
- [ ] **Step 1: 写失败测试(tier 优先 + 功效修复)**
`tests/unit/test_pools_video_atomic.py` 追加:
```python
def test_tier_aware_keeps_high_t2_in_diag():
"""错题视频组按 T2 含量升序进 val:T2 高的组保留在 diagnosis。"""
from app.harness.pools import split_by_video_assignment
from core.types import GeneratedQuestion
def _q(qid, vid):
return GeneratedQuestion(
question_id=qid, video_id=vid, task_type="X", question="q",
options=["A", "B"], answer="A", source_nodes=[], difficulty="easy",
)
# 4 个错题视频(每视频 1 题),T2 数分别 2/1/0/0
questions = [_q(f"{v}-1", v) for v in ("vA", "vB", "vC", "vD")]
assignment = {v: "trainval" for v in ("vA", "vB", "vC", "vD")}
correctness = {f"{v}-1": False for v in ("vA", "vB", "vC", "vD")}
wrong_tier = {"vA": 2, "vB": 1, "vC": 0, "vD": 0}
pools = split_by_video_assignment(
questions, assignment, correctness, val_ratio=0.5, seed=7,
wrong_tier_by_video=wrong_tier,
)
diag_vids = {q.video_id for q in pools.diagnosis}
# T2 最高的 vA 必留 diagT2=0 的组优先进 val
assert "vA" in diag_vids
assert "vB" in diag_vids
def test_val_wrong_min_repair_pulls_from_diag():
"""val 错题不足 val_wrong_min 时从 diag 换入低 T2 错题组补足。"""
from app.harness.pools import split_by_video_assignment
from core.types import GeneratedQuestion
def _q(qid, vid, correct):
return GeneratedQuestion(
question_id=qid, video_id=vid, task_type="X", question="q",
options=["A", "B"], answer="A", source_nodes=[], difficulty="easy",
)
# 8 错题视频 + 2 正确视频;val_ratio 小使初分 val 错题不足,触发修复
vids_wrong = [f"w{i}" for i in range(8)]
vids_correct = ["c0", "c1"]
questions = [_q(f"{v}-1", v, False) for v in vids_wrong] + [
_q(f"{v}-1", v, True) for v in vids_correct
]
assignment = {v: "trainval" for v in vids_wrong + vids_correct}
correctness = {f"{v}-1": False for v in vids_wrong}
correctness.update({f"{v}-1": True for v in vids_correct})
wrong_tier = {v: i for i, v in enumerate(vids_wrong)} # 递增 T2
pools = split_by_video_assignment(
questions, assignment, correctness, val_ratio=0.1, seed=7,
wrong_tier_by_video=wrong_tier, val_wrong_min=4,
)
val_wrong = sum(1 for q in pools.validation if not correctness[q.question_id])
assert val_wrong >= 4, f"功效修复后 val 错题 {val_wrong} < 4"
```
> 注:`GeneratedQuestion` 的真实字段以 `app/question_gen/types.py` 为准;若构造签名不符,读该文件对齐必填字段(勿臆造)。
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_pools_video_atomic.py -k "tier_aware or val_wrong_min_repair" -v`
Expected: FAIL`unexpected keyword argument 'wrong_tier_by_video'`)。
- [ ] **Step 3: 改 `_split_trainval_by_video_group` 加 tier 感知 + 功效修复**
`app/harness/pools.py` 函数签名改为:
```python
def _split_trainval_by_video_group(
trainval_qs: list[GeneratedQuestion],
correctness: dict[str, bool],
val_ratio: float,
rng: random.Random,
wrong_tier_by_video: dict[str, int] | None = None,
val_wrong_min: int = 0,
) -> tuple[list[GeneratedQuestion], list[GeneratedQuestion]]:
```
把分层块(L329-334 的 else 分支)改为 tier 感知:`wrong_vids` 按 T2 含量升序(T2 少的优先进 val),保留 T2 高的组在 diag;`wrong_tier_by_video=None` 时退化为原 shuffle
```python
else:
val_correct = math.floor(n_correct * n_val / n_total)
val_wrong = n_val - val_correct
rng.shuffle(correct_vids)
if wrong_tier_by_video is None:
rng.shuffle(wrong_vids)
else:
# T2 少的错题组优先进 val(保留 T2 高的组在 diag),确定性排序
wrong_vids.sort(key=lambda v: (wrong_tier_by_video.get(v, 0), v))
val_vids = set(correct_vids[:val_correct] + wrong_vids[:val_wrong])
```
`val_vids` 确定后、返回前,加**功效修复**(从 diag 的错题组按 T2 升序补入 val 直到满足 val_wrong_min):
```python
if val_wrong_min > 0:
val_wrong_now = sum(
1 for v in val_vids for q in groups[v] if not correctness[q.question_id]
)
# diag 侧仍在的错题组,按 T2 升序(低价值优先移交 val)
diag_wrong_pool = sorted(
(v for v in wrong_vids if v not in val_vids),
key=lambda v: ((wrong_tier_by_video or {}).get(v, 0), v),
)
for v in diag_wrong_pool:
if val_wrong_now >= val_wrong_min:
break
val_vids.add(v)
val_wrong_now += sum(1 for q in groups[v] if not correctness[q.question_id])
if val_wrong_now < val_wrong_min:
raise InsufficientValSignal(
f"trainval 错题不足以让 val 达到 val_wrong_min={val_wrong_min}"
f"(修复后仅 {val_wrong_now}),请放大 val_ratio 或调整 trainval 归属。"
)
```
`InsufficientValSignal` 已在 pools.py:118 定义,无需新增;需确认函数内可见 `math`/`defaultdict`,文件顶部已 import。)
- [ ] **Step 4: `split_by_video_assignment` 透传新参数**
`app/harness/pools.py` `split_by_video_assignment` 签名加 `wrong_tier_by_video: dict[str, int] | None = None`(放在 `val_wrong_min` 之后),并把 `_split_trainval_by_video_group` 调用(L175-177)改为:
```python
diagnosis, validation = _split_trainval_by_video_group(
trainval_qs, correctness, val_ratio, random.Random(seed),
wrong_tier_by_video=wrong_tier_by_video,
val_wrong_min=val_wrong_min,
)
```
删除原 L179-186 的独立 `val_wrong_min` 事后校验块(功效已在 `_split_trainval_by_video_group` 内保证,避免重复校验语义)。docstring 的 `val_wrong_min` 说明改为"切分时保证(不足则从 diag 换入低 T2 错题组补足,耗尽 fail-loud"。
- [ ] **Step 5: build_split 计算并传入 tier + val_wrong_min**
`app/harness/build_split.py``SplitBuildConfig` 加字段 `val_wrong_min: int`(放 `split_seed` 之后,docstring 补"validation 池最少错题数,切分时保证功效")。build_split Phase 3L182-191)改为:
```python
questions = load_benchmark(questions_dir)
correctness = {pred["question_id"]: pred["correct"] for pred in preds}
tier_by_q = {row["question_id"]: row["tier"] for row in signal_rows}
wrong_tier_by_video: dict[str, int] = defaultdict(int)
for pred in preds:
if not pred["correct"] and tier_by_q.get(pred["question_id"]) == "T2":
wrong_tier_by_video[pred["video_id"]] += 1
pools = split_by_video_assignment(
questions,
assignment,
correctness,
config.val_ratio,
config.split_seed,
baseline_run_id=baseline_run_id,
val_wrong_min=config.val_wrong_min,
wrong_tier_by_video=dict(wrong_tier_by_video),
)
```
更新 build_split docstring 的"契约(Task 11..."段:删除"有意保持 val_wrong_min-agnostic"表述,改为"val_wrong_min 前置到切分内保证功效;CLI 的 check_mcnemar_power 作冗余最终确认"。确认 `defaultdict` 已 import`from collections import Counter, defaultdict`)。
- [ ] **Step 6: CLI 传 val_wrong_min + 保真检查**
`app/harness/video_split_cli.py``SplitBuildConfig(...)`L565-573)加一行 `val_wrong_min=config.val_wrong_min,`
保真检查点:确认 `pools.diagnosis`/`pools.validation` 仍是逐题 `GeneratedQuestion` 列表、同 video 全部题同池(`test_pools_video_atomic.py::test_video_group_atomic_in_trainval_split` 覆盖)。
- [ ] **Step 7: 运行测试确认通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_pools_video_atomic.py tests/unit/test_harness_pools.py tests/unit/test_split_selection.py -q`
Expected: 全 PASS。
- [ ] **Step 8: 提交**
```bash
git add app/harness/pools.py app/harness/build_split.py app/harness/video_split_cli.py tests/unit/test_pools_video_atomic.py
git commit -m "feat: tier-aware diag/val split with val-power repair (design 5.1)"
```
---
## Task 4: 冻结产物覆盖保护 + --force
**Files:**
- Modify: `app/harness/build_split.py:104-192`
- Modify: `app/harness/video_split_cli.py:751-773`build_arg_parser+ run_pipeline 传参
- Test: `tests/unit/test_video_split_cli.py`
- [ ] **Step 1: 写失败测试**
`tests/unit/test_video_split_cli.py` 追加(用最小 build_split 覆盖场景,或直接测保护函数):
```python
def test_build_split_refuses_overwrite_without_force(tmp_path):
"""已存在指纹不同的 pools.json 时,force=False 必须报错不覆盖。"""
from app.harness.build_split import _guard_frozen_products
out_path = tmp_path / "pools.json"
out_path.write_text('{"split_mode":"global"}', encoding="utf-8")
manifest_path = tmp_path / "split_manifest.json"
with pytest.raises(FileExistsError, match="已存在冻结产物"):
_guard_frozen_products(out_path, manifest_path, force=False)
def test_build_split_force_backs_up_old(tmp_path):
"""force=True 时旧产物被备份为 .bak.* 再允许覆盖。"""
from app.harness.build_split import _guard_frozen_products
out_path = tmp_path / "pools.json"
out_path.write_text('{"old":1}', encoding="utf-8")
manifest_path = tmp_path / "split_manifest.json"
manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8")
_guard_frozen_products(out_path, manifest_path, force=True)
baks = list(tmp_path.glob("pools.json.bak.*"))
assert len(baks) == 1, f"未备份旧产物: {list(tmp_path.iterdir())}"
```
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -k "refuses_overwrite or force_backs_up" -v`
Expected: FAIL`cannot import name '_guard_frozen_products'`)。
- [ ] **Step 3: 实现覆盖保护函数**
`app/harness/build_split.py` 顶部 import 区确认有 `import shutil`(无则加)。新增函数(放 build_split 之前):
```python
def _guard_frozen_products(out_path: Path, manifest_path: Path, *, force: bool) -> None:
"""冻结前的覆盖保护:产物已存在时按 force 决定报错或备份。
参数:
out_path: 目标 pools.json 路径。
manifest_path: 目标 split_manifest.json 路径。
force: False 时已存在即 FileExistsErrorTrue 时把旧产物重命名为
.bak.<旧 pools_sha256 前 8 位或 timestamp-less 序号> 再放行。
异常:
FileExistsError: force=False 且产物已存在(防静默覆盖冻结锚点)。
"""
if not out_path.exists() and not manifest_path.exists():
return
if not force:
raise FileExistsError(
f"已存在冻结产物 {out_path}(或其 manifest)。重跑切分会覆盖训练依赖的"
"冻结锚点——确认要替换请加 --force(旧产物将备份为 .bak.*)。"
)
# 备份后缀取旧 manifest 的 pools_sha256 前 8 位,无则用 'prev'
suffix = "prev"
if manifest_path.exists():
try:
old = json.loads(manifest_path.read_text(encoding="utf-8"))
suffix = str(old.get("pools_sha256", "prev"))[:8] or "prev"
except (json.JSONDecodeError, OSError):
suffix = "prev"
for p in (out_path, manifest_path):
if p.exists():
p.rename(p.with_name(f"{p.name}.bak.{suffix}"))
```
确认 build_split.py 已 import `json`(无则加 `import json`)。在 `build_split` 签名加参数 `force: bool = False`(放 `generated_at` 之后),并在 Phase 3 `save_pools` 之前(L192 前)调用 `_guard_frozen_products(out_path, manifest_path, force=force)`
- [ ] **Step 4: CLI 暴露 --force 并透传**
`app/harness/video_split_cli.py` `build_arg_parser`L751-773)追加:
```python
parser.add_argument(
"--force",
action="store_true",
help="覆盖已存在的冻结 pools.json/manifest(旧产物备份为 .bak.*",
)
```
`run_pipeline` 签名加 `force: bool = False` 参数,build_split 调用(L559-577)加 `force=force,``main()` 里把 `args.force` 透传给 `run_pipeline`
- [ ] **Step 5: 运行测试确认通过 + CLI 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -q`
Expected: 全 PASS。
- [ ] **Step 6: 提交**
```bash
git add app/harness/build_split.py app/harness/video_split_cli.py tests/unit/test_video_split_cli.py
git commit -m "feat: guard frozen split products against silent overwrite (--force)"
```
---
## Task 5: seed 携带 pools.json + 训练拷入
**Files:**
- Modify: `app/harness/store.py:184-232`init_seed
- Modify: `app/harness/workspace.py:156-200`init_workspace_from_seed
- Test: `tests/unit/test_harness_store.py``tests/unit/test_harness_workspace.py`
- [ ] **Step 1: 写失败测试(seed 携带)**
`tests/unit/test_harness_store.py::TestInitSeed` 追加:
```python
def test_init_seed_carries_pools(self, tmp_path):
"""提供 pools_json/split_manifest 时拷入 seed 目录。"""
from app.harness.store import init_seed
store = tmp_path / "store"
skills = tmp_path / "sk"; skills.mkdir(); (skills / "s.md").write_text("x")
prompts = tmp_path / "pr"; prompts.mkdir(); (prompts / "p.md").write_text("y")
db = tmp_path / "b.db"; db.write_text("db")
pools = tmp_path / "pools.json"; pools.write_text('{"split_mode":"global"}')
manifest = tmp_path / "split_manifest.json"; manifest.write_text('{"pools_sha256":"a"}')
seed_dir = init_seed(
store, "s1", skills, prompts, db, "infer_adhoc", None, "d",
pools_json=pools, split_manifest=manifest,
)
assert (seed_dir / "pools.json").exists()
assert (seed_dir / "split_manifest.json").exists()
```
`tests/unit/test_harness_workspace.py` 追加:
```python
def test_init_workspace_from_seed_carries_pools(store_dir, workspace_dir):
"""seed 目录含 pools.json 时拷入 workspace。"""
import shutil
from app.harness.store import init_seed
from app.harness.workspace import init_workspace_from_seed
# 复用现有 fixture 构造 seed 的方式;此处补 pools.json 到 seed 后初始化 workspace
# (具体 fixture 依 test_harness_workspace.py 现有 helper,读文件对齐)
...
```
> 该 workspace 测试需依 `test_harness_workspace.py` 现有 fixture`store_dir`/`workspace_dir` 及既有 seed 构造 helper)填充;实现前读该文件 `test_init_workspace_from_seed`(L151)复用其 seed 搭建,再在 seed 目录写 `pools.json` 后断言 workspace 内出现 `pools.json`
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py::TestInitSeed::test_init_seed_carries_pools -v`
Expected: FAIL`unexpected keyword argument 'pools_json'`)。
- [ ] **Step 3: init_seed 加可选携带参数**
`app/harness/store.py` `init_seed` 签名加:
```python
def init_seed(
store_dir: Path,
name: str,
skills_dir: Path,
prompts_dir: Path,
baseline_db: Path,
baseline_run_id: str,
parent: str | None,
description: str,
*,
pools_json: Path | None = None,
split_manifest: Path | None = None,
) -> Path:
```
`copy2(baseline_db, ...)`L218)之后加:
```python
if pools_json is not None:
shutil.copy2(pools_json, seed_dir / "pools.json")
if split_manifest is not None:
shutil.copy2(split_manifest, seed_dir / "split_manifest.json")
```
docstring 补两参说明。
- [ ] **Step 4: init_workspace_from_seed 拷入 pools**
`app/harness/workspace.py` `init_workspace_from_seed``shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db")`L197)之后加:
```python
seed_pools = seed_dir / "pools.json"
if seed_pools.exists():
shutil.copy2(seed_pools, workspace_dir / "pools.json")
seed_manifest = seed_dir / "split_manifest.json"
if seed_manifest.exists():
shutil.copy2(seed_manifest, workspace_dir / "split_manifest.json")
```
- [ ] **Step 5: 运行测试确认通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py -q`
Expected: 全 PASS。
- [ ] **Step 6: 提交**
```bash
git add app/harness/store.py app/harness/workspace.py tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py
git commit -m "feat: seed carries frozen pools.json into training workspace"
```
---
## Task 6: build_or_load_pools global 一致性校验
**Files:**
- Modify: `app/harness/pools.py:746-813`
- Test: `tests/unit/test_harness_pools.py`
- [ ] **Step 1: 写失败测试**
`tests/unit/test_harness_pools.py::TestBuildOrLoadPoolsFrozen` 追加:
```python
def test_global_frozen_rejects_baseline_mismatch(self, tmp_path, ...):
"""global 冻结 pools 的 baseline_run_id 与 seed 不符时 fail-loud。"""
# 依现有 fixture 造 workspace + 冻结 pools.jsonsplit_mode=global,
# baseline_run_id="other"),seed.json baseline_run_id="infer_adhoc"
# 调 build_or_load_pools 应 raise ValueError(match="baseline_run_id")
...
```
> 依 `TestBuildOrLoadPoolsFrozen`L246)现有 fixture 复用其 workspace/seed 搭建;实现前读该类对齐 RunConfig/strategy 构造,勿臆造。
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_pools.py::TestBuildOrLoadPoolsFrozen -v`
Expected: 新用例 FAIL(当前 global 分支无校验,误加载不报错)。
- [ ] **Step 3: 加 global 一致性校验**
`app/harness/pools.py` `build_or_load_pools`,在 global 加载分支(`if pools_path.exists():` 块内、`per_category` 校验的 `else` 侧,即 L813 `return load_pools(pools_path)` 之前)加:
```python
else: # global:校验 baseline_run_id 与(若有)manifest 内容指纹
frozen_baseline = raw.get("baseline_run_id")
if frozen_baseline != baseline_run_id:
raise ValueError(
f"冻结 pools.json 的 baseline_run_id={frozen_baseline!r} 与 seed "
f"的 {baseline_run_id!r} 不一致,拒绝静默加载错配切分。"
)
manifest_path = config.workspace_dir / "split_manifest.json"
if manifest_path.exists():
import hashlib
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
actual_sha = hashlib.sha256(
pools_path.read_text(encoding="utf-8").encode("utf-8")
).hexdigest()
if manifest.get("pools_sha256") != actual_sha:
raise ValueError(
"pools.json 内容指纹与 split_manifest.pools_sha256 不符,"
"冻结产物疑被篡改,拒绝加载。"
)
```
(确认该 `else` 与 L751 `if frozen_split_mode == "per_category":` 配对;若现有结构非 if/else 而是 if 后直接 return,则把校验插在 `return load_pools(pools_path)` 前并用 `if frozen_split_mode != "per_category":` 守卫。实现前读 L746-813 对齐控制流。)
- [ ] **Step 4: 运行测试确认通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_pools.py -q`
Expected: 全 PASS。
- [ ] **Step 5: 提交**
```bash
git add app/harness/pools.py tests/unit/test_harness_pools.py
git commit -m "fix: validate global frozen pools baseline_run_id + sha256 on load"
```
---
## Self-Review(作者自查,执行者复核)
- [ ] val_ratio=0.4 已改;tier 感知 + 功效修复在同一函数、退化路径(`wrong_tier_by_video=None`)保持旧行为。
- [ ] val_wrong_min 从 CLI→SplitBuildConfig→build_split→split_by_video_assignment→_split_trainval_by_video_group 全链路贯通;旧的 pools.py 事后校验块已删(不重复)。
- [ ] extract_run_db 去重默认关闭,不破坏既有调用。
- [ ] seed 携带 + workspace 拷入 + global 一致性校验三者闭环:冻结产物有唯一路径进训练且被校验。
- [ ] 覆盖保护默认 force=False,离线 CLI 重跑需显式 --force。
## 核心算法保真校验结论
本计划触及算法 #5 的上游输入(哪些视频进 diag/val),**不改** gate_ladder 的 unit+correctness 消费结构;Task 3 Step 6 已设保真检查点确认逐 unit 列表与视频组原子性。不涉及算法 #4/#6/#8/#9 逻辑。
## 验收标准
1. `pytest tests/unit/test_pools_video_atomic.py tests/unit/test_harness_pools.py tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py tests/unit/test_split_selection.py tests/unit/test_video_split_cli.py` 全绿。
2. tier 感知:T2 高的错题视频组留 diag,T2 低的优先进 val。
3. seed 携带 pools.json → init_workspace_from_seed 拷入 → build_or_load_pools 校验 baseline_run_id + sha256。
4. 冻结产物 force=False 时拒绝覆盖。
@@ -0,0 +1,609 @@
# WP3 训练循环与进化引擎 Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
> **前置依赖:WP1(模板已迁移,进化引擎可运行)+ WP4(cache_salt 能力已就绪)必须先完成。**
**Goal:** 修复训练循环与进化引擎的 9 处缺陷,使诊断拿到真实轨迹、早停按 epoch 语义、微型题型不崩、崩溃可幂等续跑、进化 patch 不破坏冻结区、降级信号不驱动错误进化、跨 epoch 评估真实重采样。
**Architecture:** 诊断经 `StepsJsonRunLog` 从 steps_json 重建轨迹(算法 #7 恢复);早停计数单位 step→epoch;可训练性预检在 gate 建立前剔除微型题型;`_run_step` 幂等(先 DELETE 再写);patch 冻结区检查整个 target 跨度(算法 #8 加固);降级/未判定题按 lapse 保守分流;训练推理用 run_id 作 cache_saltrun_id 已含 epoch,天然跨 epoch 重采样)。
**Tech Stack:** Python 3.11、asyncio、SQLite、pytest。
**设计源**`research-wiki/designs/2026-07-16-preflight-fixes-design.md §6-7`
---
## 关键锚点(实现前必读)
| 用途 | 位置 |
|------|------|
| 训练主流程 | `app/harness/runner.py:789-859` `train``_setup_train_run:865-879`(预检插入点 L874 前);`_run_step:1001-1026`run_id L1011、诊断 L1019、无 DELETE |
| 早停 | `app/harness/runner.py:291-316` `_should_early_stop`L315 `+= steps_this_epoch`);`_TrainState:95-121``steps_since_best_improved` L118);`_maybe_promote_best:1587`(置 0 |
| 诊断调用 | `app/harness/runner.py:2163-2196` `_run_diagnosis``RunLogImpl` L2173 未包 StepsJsonRunLog);DiagnosisResult `degraded_count` 未被引用 |
| gate 刷新 | `app/harness/runner.py:1833-1879` `_refresh_gate_ladder`save L1878 → set observed L1879);checkpoint 落盘晚在 train L837 |
| holdout | `app/harness/runner.py:1881-1921` `_holdout_four_way``_pick_mixed_best:1923-1963``_eval_version_on_pool:2148-2161` |
| 推理落库 | `app/harness/inference.py:363-447` `_run_single_question`prediction L422 未归一、insert L446 try 外);`_to_text_field:147-162``run_inference:473` |
| Agent Loop | `core/agent/loop.py:103` `run`session_id L110);`_call_llm` chat 调用 `:329``self._llm.chat(messages, session_id=session_id)` |
| batching | `app/harness/batching.py:186-202` `_classify_unit`L200 缺 correctness→None |
| checkpoint | `app/harness/checkpoint.py:37-44` `_STRUCTURAL_KEYS``serialize_state:76-106``write_checkpoint:212-260`(原子写) |
| 诊断分流 | `core/evolution/diagnose.py:1485-1519` `_build_skill_case_packs`lapse 分流 L1492);`_process_question:2139-2206`except L2194 cause_category 留 None |
| traces 适配 | `app/harness/baseline_run_log.py:13` `StepsJsonRunLog``steps_json_traces.py:13` |
| patch | `core/evolution/patch.py:285-287` `_in_ranges``_do_insert_after:309-326`L320);`_do_replace_delete:329-347`L343);markers L11-18`validate_skill` in `evolve.py:296` |
## 核心算法保真校验
触及算法 #7(诊断瀑布,Task 1 恢复轨迹)、#8patch 引擎,Task 5 冻结区加固)、#10Agent LoopTask 9 透传 cache_salt)、#5/#12(信息阶梯/训练编排,Task 7 checkpoint 时序)。**均为恢复/加固/透传,不改算法逻辑**:Task 1 让诊断拿到本就该有的轨迹;Task 5 把"只查起点"补成"查整跨度"(保护方向不变);Task 9 只加透传参数;Task 7 只调 checkpoint 落盘时机。每个相关 Task 设保真检查点。
---
## Task 1: traces 适配(诊断拿到真实轨迹,算法 #7
**Files:**
- Modify: `app/harness/runner.py:2163-2196`_run_diagnosis
- Test: `tests/unit/test_runner_diag_tree_inject.py``test_harness_runner.py`
- [ ] **Step 1: 写失败测试**
`tests/unit/test_runner_diag_tree_inject.py` 追加(构造只写 steps_json 不写 traces 表的 run,断言诊断能拿到轨迹):
```python
@pytest.mark.asyncio
async def test_diagnosis_reads_traces_from_steps_json(...):
"""traces 表为空但 predictions.steps_json 有轨迹时,诊断仍拿到非空 traces。"""
# 依现有 runner 测试 fixture 造一个 runpredictions 有 steps_jsontraces 表空;
# 调 _run_diagnosis 后断言 diagnose 收到的 traces 非空(可 patch run_diagnosis 捕获入参)
...
```
> 依 `test_runner_diag_tree_inject.py` 现有 fixture;实现前读对齐 runner 构造与 patch 点。
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_runner_diag_tree_inject.py -k reads_traces_from_steps_json -v`
Expected: FAIL(当前 RunLogImpl 直读空 traces 表)。
- [ ] **Step 3: 包 StepsJsonRunLog**
`app/harness/runner.py` `_run_diagnosis`,把传给 `run_diagnosis``run_log`(当前 `RunLogImpl(...)`L2173 附近)包一层:
```python
from app.harness.baseline_run_log import StepsJsonRunLog
from app.harness.log import RunLogImpl
run_log = StepsJsonRunLog(RunLogImpl(str(self._paths.db_path)))
```
`StepsJsonRunLog.get_traces` 在底层 traces 空时从 predictions.steps_json 经 `steps_json_to_trace_rows` 重建;`get_predictions` 透传。)
- [ ] **Step 4: 保真检查点 + 测试通过**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_runner_diag_tree_inject.py tests/unit/test_baseline_run_log.py -q`
Expected: 全 PASS。保真:确认诊断瀑布拿到的是逐 step `{tool_name,tool_args,tool_output,thought}` 行(对齐 TRM4 诊断输入)。
- [ ] **Step 5: 提交**
```bash
git add app/harness/runner.py tests/unit/test_runner_diag_tree_inject.py
git commit -m "fix: wrap diagnosis run_log with StepsJsonRunLog (restore algo #7 traces)"
```
---
## Task 2: prediction 归一化 + 落库加固
**Files:**
- Modify: `app/harness/inference.py:417-447`
- Test: `tests/unit/test_harness_inference.py`
- [ ] **Step 1: 写失败测试**
`tests/unit/test_harness_inference.py` 追加(LLM 提交非标量 answer 不崩 gather):
```python
@pytest.mark.asyncio
async def test_nonscalar_prediction_does_not_crash(...):
"""submit_answer 返回 {'answer': ['B']} 等非标量时归一化落库,不抛 sqlite 绑定异常。"""
# 依现有 inference 测试 fixture,让 AgentLoop 返回 result={'answer': ['B']}
# run_inference 应正常完成、predictions 行 prediction 为字符串(如 '["B"]'),不崩
...
```
> 依 `test_harness_inference.py` 现有 fake loop/dispatch fixture;实现前读对齐。
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py -k nonscalar_prediction -v`
Expected: FAILsqlite `InterfaceError: Error binding parameter`)。
- [ ] **Step 3: 归一化 prediction + insert 加固**
`app/harness/inference.py`:新增归一化 helper(None 保留、str 原样、其余 `_to_text_field`):
```python
def _normalize_prediction(answer: object) -> str | None:
"""归一化 predictionNone 保留(INFRA 空预测语义),str 原样,其余 JSON 序列化。"""
if answer is None or isinstance(answer, str):
return answer
return _to_text_field(answer)
```
L422 `"prediction": result_dict.get("answer"),` 改为 `"prediction": _normalize_prediction(result_dict.get("answer")),`
L446 的 `await asyncio.to_thread(log.insert, "predictions", record)` 包 try,绑定异常降级为最小 error 行不击穿 gather
```python
try:
await asyncio.to_thread(log.insert, "predictions", record)
except (sqlite3.InterfaceError, sqlite3.ProgrammingError):
logger.exception("[{}] QA {} 落库绑定异常,降级为 error 行", qa.video_id, qa.question_id)
record["prediction"] = None
record["stop_reason"] = "error"
await asyncio.to_thread(
log.insert, "predictions",
{k: v for k, v in record.items() if isinstance(v, (str, int, float, type(None)))},
)
```
确认 `import sqlite3` 在文件顶部(无则加)。
- [ ] **Step 4: 测试通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py -q`
Expected: 全 PASS。
- [ ] **Step 5: 提交**
```bash
git add app/harness/inference.py tests/unit/test_harness_inference.py
git commit -m "fix: normalize non-scalar prediction; harden predictions insert"
```
---
## Task 3: early_stop 改 epoch 计数
**Files:**
- Modify: `app/harness/runner.py:291-316,118,1587,849-857`
- Modify: `app/harness/checkpoint.py`(若字段入 state
- Test: `tests/unit/test_harness_runner.py`
- [ ] **Step 1: 写失败测试**
`tests/unit/test_harness_runner.py` 追加:
```python
def test_early_stop_counts_epochs_not_steps(tmp_path):
"""patience=2 表示连续 2 个 epoch 无 best 刷新才停(不是步数)。"""
from app.harness.runner import _should_early_stop, _TrainState
# 造 state + workspacebest 停在 epoch 1
# epoch 2 无刷新 → epochs_since_best_improved=1 → 不停;
# epoch 3 无刷新 → =2 → 停
...
```
> 依现有 `_TrainState`/`read_best` fixture;实现前读对齐 workspace best 写入。
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k early_stop_counts_epochs -v`
Expected: FAIL(当前累加 steps_this_epoch)。
- [ ] **Step 3: 字段改名 + 计数改 epoch**
全局把 `steps_since_best_improved` 改名 `epochs_since_best_improved``grep -rn steps_since_best_improved app/``_TrainState:118``_should_early_stop:313,315``_maybe_promote_best:1587`,以及 checkpoint serialize/deserialize 若含此字段)。
`_should_early_stop`L315`state.steps_since_best_improved += steps_this_epoch` 改为 `state.epochs_since_best_improved += 1`;签名删除 `steps_this_epoch` 参数(改为 `_should_early_stop(workspace_dir, epoch, state, patience)`),train 调用点(L849-857)同步去掉 `len(batches)` 实参。docstring 改为"epoch 粒度"。
- [ ] **Step 4: checkpoint 兼容**
`epochs_since_best_improved` 入 checkpoint state`grep -n steps_since_best_improved app/harness/checkpoint.py`),同步改名。本轮为 fresh 训练无旧 checkpoint,无迁移负担。
- [ ] **Step 5: 测试通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py tests/unit/test_harness_checkpoint.py -q`
Expected: 全 PASS。
- [ ] **Step 6: 提交**
```bash
git add app/harness/runner.py app/harness/checkpoint.py tests/unit/test_harness_runner.py
git commit -m "fix: early_stop patience counts epochs not steps"
```
---
## Task 4: 可训练性预检
**Files:**
- Modify: `app/harness/runner.py``train` 入口过滤 + `_setup_train_run` 接收 filtered task_types
- Modify: `app/harness/config.py`RunConfig 加 `trainable_min_units` + 正整数校验)
- Modify: `app/harness/checkpoint.py:37``trainable_min_units``_STRUCTURAL_KEYS` 指纹)
- Test: `tests/unit/test_harness_runner.py``tests/unit/test_harness_checkpoint.py`
- [ ] **Step 1: 写失败测试**
```python
def test_untrainable_types_filtered_before_gate():
"""val<eval_min_per_class 或 非test单元<trainable_min_units 的题型从 diag/val/task_types 剔除。"""
from app.harness.runner import _filter_untrainable_types
# 构造 pools:题型 Aval=5, units=40)可训;Bval=0, units=2)不可训
# 调 _filter_untrainable_types(pools, task_types=[A,B], eval_min_per_class=2, trainable_min_units=8)
# 断言返回 pools 不含 B、task_types 不含 B
...
```
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k untrainable_types_filtered -v`
Expected: FAIL`_filter_untrainable_types` 不存在)。
- [ ] **Step 3: 实现纯函数预检**
`app/harness/runner.py` 新增模块级纯函数:
```python
def _filter_untrainable_types(
pools: Pools,
task_types: list[str] | None,
eval_min_per_class: int,
trainable_min_units: int,
) -> tuple[Pools, list[str] | None]:
"""剔除不可训练题型(val<eval_min_per_class 或 非test单元<trainable_min_units)。
非test单元数 = 该题型 diag+val 题数(single 题 unit==题;等于 gate 阶梯该类候选数)。
test 池不过滤(继续报告全题型准确率)。返回过滤后 (pools, task_types)。
"""
from collections import Counter
diag_by_type = Counter(q.task_type for q in pools.diagnosis)
val_by_type = Counter(q.task_type for q in pools.validation)
keep: set[str] = set()
dropped: list[tuple[str, str]] = []
for tt in set(diag_by_type) | set(val_by_type):
n_val = val_by_type.get(tt, 0)
n_units = diag_by_type.get(tt, 0) + n_val
if n_val < eval_min_per_class:
dropped.append((tt, f"val={n_val}<{eval_min_per_class}"))
elif n_units < trainable_min_units:
dropped.append((tt, f"units={n_units}<{trainable_min_units}"))
else:
keep.add(tt)
for tt, why in sorted(dropped):
logger.warning("可训练性预检剔除题型 {}{}", tt, why)
new_pools = replace(
pools,
diagnosis=[q for q in pools.diagnosis if q.task_type in keep],
validation=[q for q in pools.validation if q.task_type in keep],
)
new_types = [t for t in task_types if t in keep] if task_types is not None else sorted(keep)
return new_pools, new_types
```
(确认 `from dataclasses import replace` 已 import`Pools` 是否 frozen dataclass 支持 `replace`——若非,按其构造方式重建。)
**过滤结果必须回传主循环(Codex Critical**`RunConfig``@dataclass(frozen=True)`**不能** `self._config.task_types = ...`(会 FrozenInstanceError),且过滤后的 pools 必须被 `train()` 后续的 batch/step/slow-update/final-eval 全部使用。实现方式:
- 在 `train(pools)` **入口第一步**`_setup_train_run` 调用之前)过滤:
```python
pools, filtered_task_types = _filter_untrainable_types(
pools, self._config.task_types,
self._config.eval_min_per_class, self._config.trainable_min_units,
)
```
- 把 filtered `pools` 传给 `_setup_train_run(pools)` 与 train() 后续所有消费点(build_batches / slow_update / final_eval 均用这个 filtered pools,不再触碰原始 pools)。
- 把 `filtered_task_types` 传给 gate 建立(`_setup_train_run`/`_init_gate_pools` 用它而非 `self._config.task_types`)——新增参数透传,不改 frozen config。
`app/harness/config.py``RunConfig` 加字段 `trainable_min_units: int`(无默认,显式配置;train yaml 提供);在配置校验函数(如 `validate_config`config.py:277 附近)加 `trainable_min_units >= 1` 断言(<1 报错)。
`app/harness/checkpoint.py:37` `_STRUCTURAL_KEYS` 加入 `"trainable_min_units"`——该值改变会改变 pools 过滤结果与训练轨迹,必须纳入 checkpoint 结构指纹,resume 时变化即拒绝复用旧 checkpoint。
- [ ] **Step 4: 测试通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -q`
Expected: 全 PASS。
- [ ] **Step 5: 提交**
```bash
git add app/harness/runner.py app/harness/config.py tests/unit/test_harness_runner.py
git commit -m "feat: pre-flight filter of untrainable task types before gate"
```
---
## Task 5: patch 冻结区跨度 + 注入 + marker 校验(算法 #8
**Files:**
- Modify: `core/evolution/patch.py:285-347`
- Modify: `core/evolution/evolve.py:296`validate_skill
- Test: `tests/unit/test_patch.py`
- [ ] **Step 1: 写失败测试**
`tests/unit/test_patch.py` 追加:
```python
def test_replace_spanning_into_protected_is_skipped():
"""target 起点在正文、末端伸入冻结区的 replace 被跳过(不破坏 marker)。"""
from core.evolution.patch import apply_patch_with_report, APPENDIX_START, APPENDIX_END
body = "正文最后一段。"
appendix = f"{APPENDIX_START}\n## 执行提醒\n- 规则A\n{APPENDIX_END}"
content = body + "\n\n" + appendix
# target 从正文末尾跨入 APPENDIX_START
target = "正文最后一段。\n\n" + APPENDIX_START
edits = [{"op": "delete", "target": target, "content": ""}]
new_content, report = apply_patch_with_report(content, edits, protected_spans=[appendix])
assert APPENDIX_START in new_content and APPENDIX_END in new_content # marker 未被破坏
def test_edit_payload_with_marker_literal_rejected():
"""edit payload/target 含 marker 字面量 → 拒绝该 edit。"""
from core.evolution.patch import apply_patch_with_report, APPENDIX_START
edits = [{"op": "append", "target": "", "content": f"注入 {APPENDIX_START} 破坏"}]
_, report = apply_patch_with_report("正文", edits, protected_spans=[])
assert any("marker" in str(s).lower() or "reject" in str(s).lower() for s in report)
```
> marker 常量名以 `core/evolution/patch.py:11-18` 为准(`APPENDIX_START` 等),实现前读对齐 import 名。
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_patch.py -k "spanning_into_protected or marker_literal" -v`
Expected: FAIL(当前只查起点 pos,跨入未拦;无注入检查)。
- [ ] **Step 3: 跨度检查 + 注入检查**
`core/evolution/patch.py` 新增跨度 helper
```python
def _span_overlaps_ranges(pos: int, length: int, ranges: list[tuple[int, int]]) -> bool:
"""判断 [pos, pos+length) 是否与任一冻结区间相交(不止起点)。"""
end = pos + length
return any(start < end and pos < r_end for start, r_end in ranges)
```
`_do_insert_after` L320 `if _in_ranges(pos, ranges):` 改为 `if _span_overlaps_ranges(pos, len(target), ranges):`
`_do_replace_delete` L343 `if _in_ranges(pos, ranges):` 改为 `if _span_overlaps_ranges(pos, len(target), ranges):`
`apply_patch_with_report`(L387)应用每个 edit 前加注入检查:payload/target 含任一 marker 字面量(`APPENDIX_START/END``MOMENTUM_START/END`)→ 跳过该 edit 并记 `skipped_marker_injection`
- [ ] **Step 4: validate_skill 加 marker 完整性校验**
`core/evolution/evolve.py:296` `validate_skill`:在现有 frontmatter/长度/代码块校验后加——统计 evolved 中 `APPENDIX_START/END``MOMENTUM_START/END` 出现次数,要求成对(START 数==END 数)、各至多一对、START 在 END 前;违反则返回校验失败(该候选整体 reject)。
- [ ] **Step 5: 保真检查点 + 测试通过**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_patch.py tests/unit/test_evolve.py -q`
Expected: 全 PASS。保真:确认"保护跨度"方向未变(仍是保护 appendix/momentum 不被误改),只是从"查起点"补成"查整跨度"+ 注入/完整性双防线。
- [ ] **Step 6: 提交**
```bash
git add core/evolution/patch.py core/evolution/evolve.py tests/unit/test_patch.py
git commit -m "fix: patch checks full target span + marker injection/integrity (algo #8)"
```
---
## Task 6: 诊断降级分流 + 占比中止
**Files:**
- Modify: `core/evolution/diagnose.py:1485-1497`
- Modify: `app/harness/runner.py:1019`(诊断后 degraded 占比检查)
- Test: `tests/unit/test_diagnose.py``tests/unit/test_harness_runner.py`
- [ ] **Step 1: 写失败测试(分流)**
`tests/unit/test_diagnose.py` 追加:
```python
def test_none_cause_and_degraded_route_to_lapse():
"""cause_category=None(判别失败)与 degraded 题按 lapse 处置,不进 defect 正文路径。"""
from core.evolution.diagnose import _build_skill_case_packs
# 构造 metrics_group:一题 attr.cause_category=None(非 degraded)、一题 qm.degraded=True
# 断言二者都不出现在 failure_caseswrong_by_error),只可能进 lapse_notes
...
```
> 依 `test_diagnose.py` 现有 QuestionMetrics/ErrorAttribution 构造(`:753` 附近);实现前读对齐字段。
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_diagnose.py -k none_cause_and_degraded_route -v`
Expected: FAIL(当前 None → wrong_by_error 走 defect 正文)。
- [ ] **Step 3: 改分流逻辑**
`core/evolution/diagnose.py` `_build_skill_case_packs` 的分流循环(L1488-1497)改为"仅明确 defect 且非 degraded 才进正文路径"
```python
for qm in metrics_group:
if qm.correct:
continue
attr = attribution_map.get(qm.question_id)
is_defect = (
attr is not None
and attr.cause_category == "defect"
and not qm.degraded
)
if not is_defect:
# lapse / None(判别失败)/ degraded → 保守,不驱动正文进化
if attr is not None and attr.lapse_note and attr.lapse_note.strip():
lapse_notes.append(attr.lapse_note)
continue
wrong_by_error[attr.error_type].append(qm)
```
- [ ] **Step 4: 写失败测试(占比中止)+ 实现**
`tests/unit/test_harness_runner.py` 追加:诊断结果 degraded_count/总题数 > 0.5 时 `_run_step` 后应 raise(疑似基础设施故障)。
`app/harness/runner.py` `_run_step`L1019 拿到 `diagnosis` 后)加:
```python
n_wrong = sum(1 for q in batch if not state.correctness.get(q.question_id, True))
if n_wrong > 0 and diagnosis.degraded_count / n_wrong > 0.5:
raise RuntimeError(
f"本 step 诊断降级占比 {diagnosis.degraded_count}/{n_wrong} > 50%"
"疑似 judge 基础设施故障,中止训练(不以降级信号驱动进化)。"
)
```
(确认 `DiagnosisResult.degraded_count` 字段可用,定义在 `core/evolution/types.py:299`。)
- [ ] **Step 5: 测试通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_diagnose.py tests/unit/test_harness_runner.py -q`
Expected: 全 PASS。
- [ ] **Step 6: 提交**
```bash
git add core/evolution/diagnose.py app/harness/runner.py tests/unit/test_diagnose.py tests/unit/test_harness_runner.py
git commit -m "fix: route None/degraded diagnoses to lapse; abort on high degrade rate"
```
---
## Task 7: step 幂等(DELETE+ gate_epoch_observed 立即落盘
**Files:**
- Modify: `app/harness/runner.py:1001-1026`_run_step 开头 DELETE
- Modify: `app/harness/runner.py:1378-1498,1833-1879`checkpoint 提到 gate save 之后)
- Test: `tests/unit/test_harness_runner.py`
- [ ] **Step 1: 写失败测试(幂等)**
```python
@pytest.mark.asyncio
async def test_run_step_deletes_stale_rows_before_rerun(...):
"""同 run_id 重跑前先清 predictions/traces,避免重复行双计。"""
# 预置该 step run_id 的旧 predictions 行;调 _run_step;断言旧行被清、只剩本次
...
```
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k deletes_stale_rows -v`
Expected: FAIL(当前 append,无 DELETE)。
- [ ] **Step 3: _run_step 开头 DELETE**
`app/harness/runner.py` `_run_step`,在 rolloutL1012)之前加(**用 `IF EXISTS` 避免 fresh workspace 首跑时 predictions/traces 表尚未由 `run_inference._ensure_tables` 创建导致 `OperationalError: no such table`**):
```python
with HarnessLog(str(self._paths.db_path), run_id, register_run=False) as log:
log.execute("DELETE FROM predictions WHERE run_id=?", (run_id,))
log.execute("DELETE FROM traces WHERE run_id=?", (run_id,))
```
> SQLite `DELETE FROM <t>` 对不存在的表会抛 `no such table`。两种消解方式择一(实现前读 log.py 确认):① rollout 由 `run_inference` 先建表——把 DELETE 移到**首次 rollout 之后、诊断之前**并只在 resume 重跑(step 已有旧行)时执行;② 或 DELETE 前先 `CREATE TABLE IF NOT EXISTS`(复用 inference 的 PREDICTIONS_SCHEMA/TRACES_SCHEMA),保证幂等无害。推荐 ②(无害且简单)。`register_run=False` 来自 WP4;若 HarnessLog 无 `execute` 便捷方法,用其现有连接接口。
- [ ] **Step 4: checkpoint 提到 gate save 之后(消除双计窗口)**
目标:`_refresh_gate_ladder``gate_pools.save`L1878+ `gate_epoch_observed=True`L1879)之后,**立即** `write_checkpoint`phase="epoch_done"),不等到 `train` L837。实现:把 `_slow_update_cycle` Phase 10(调 `_refresh_gate_ladder` L1496-1498)之后的 checkpoint 落盘从 `train`L837)移入 `_slow_update_cycle` 末尾,或让 `_refresh_gate_ladder` 接收 checkpoint 所需上下文(epoch/progress/batches)并在 save 后落盘。实现前读 `write_checkpoint` 签名(checkpoint.py:212)与 `train` L833-848 对齐参数,确保 gate_pools.json 与 checkpoint 的 `gate_epoch_observed` 同一时刻一致。
- [ ] **Step 5: 保真检查点 + 测试通过**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py tests/unit/test_harness_checkpoint.py -q`
Expected: 全 PASS。保真(算法 #5/#12):确认 γ-EMA 更新(`update_probs`)仍每 epoch 一次、checkpoint 落盘不改变慢更新十步序的语义顺序。
- [ ] **Step 6: 提交**
```bash
git add app/harness/runner.py tests/unit/test_harness_runner.py
git commit -m "fix: idempotent _run_step (DELETE stale) + checkpoint after gate save"
```
---
## Task 8: holdout 四向去重
**Files:**
- Modify: `app/harness/runner.py:1881-1963`_holdout_four_way / _pick_mixed_best
- Test: `tests/unit/test_harness_runner.py`
> **说明**:目标是每 epoch 的四向 test 评估从"4×600 全跑"降为"仅 final 必跑 + best_hard 未评过才跑 + baseline 从基线预测推导 + best_mixed 引用赢家"。去重做在 harness 逻辑层(配合 WP4 epoch 盐,同版本不重采样)。
- [ ] **Step 1: 写失败测试**
```python
@pytest.mark.asyncio
async def test_holdout_dedup_skips_reevaluated_versions(...):
"""baseline 不跑推理(从基线预测推导);best_hard==final 时不重复评估。"""
# 统计 _eval_version_on_pool 被调次数:baseline=0best_hard==final 时该向复用不重跑
...
```
> 依现有 runner holdout fixture;实现前读 `_holdout_four_way`/`write_holdout_eval` 对齐。
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k holdout_dedup -v`
Expected: FAIL(当前四向各跑一次)。
- [ ] **Step 3: 实现去重(进程内备忘录,不改 holdout_eval schema**
> **schema 约束(Codex Critical**:当前 `holdout_eval` 表(observation.py:78)不存 skills_version/prompts_version/pointer,无法按版本反查做跨-run hydrate。本 task **不扩展该 schema**(避免结构性风险),改用 **train() 进程内备忘录** `dict[(skills_v,prompts_v), float]` 去重。代价:resume 后备忘录清空、已评版本会重评一次——resume 是异常路径、重评 600 题成本可接受,换取零 schema 变更风险。完整跨-run hydrate 记 future work。
`app/harness/runner.py` `_holdout_four_way`(在 `_TrainState` 加一个 `holdout_memo: dict[tuple[str,str], float] = field(default_factory=dict)` 字段):
- **baseline 向**:不调 `_eval_version_on_pool`,改从基线 predictions`baseline_run_id`)读 test 题对错算 acctest 题在 infer_adhoc 已全推理过);结果存 memoepoch>1 直接复用(0 推理)。
- **final 向**:真评 600,算完存 `memo[(final_sv,final_pv)]`
- **best_hard 向**:若 `(best_sv,best_pv)` 已在 `memo`(== final 或往轮已评)则引用,否则真评并存 memo。
- **best_mixed 向**`_pick_mixed_best` 选出的赢家必是 best_hard 或 final 之一,其 test acc 已在 memo,直接引用写 holdout_eval0 推理。
实现前完整读 `_holdout_four_way`~1885)、`write_holdout_eval``_eval_version_on_pool` 对齐;`write_holdout_eval` 调用保持不变(仍逐向写观测行,只是 acc 来源改为 memo 复用/基线推导)。
- [ ] **Step 4: 测试通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -q`
Expected: 全 PASS。
- [ ] **Step 5: 提交**
```bash
git add app/harness/runner.py tests/unit/test_harness_runner.py
git commit -m "perf: dedup holdout four-way eval (baseline derive, best_hard memo)"
```
---
## Task 9: 训练推理注入 epoch 盐(cache_salt=run_id,算法 #10 透传)
**Files:**
- Modify: `core/agent/loop.py:103-141,271-329`
- Modify: `app/harness/inference.py:408-415,473`
- Test: `tests/unit/test_agent_loop`(或现有 loop 测试)、`tests/unit/test_harness_inference.py`
> **原理**:训练/val/test/holdout 推理的 run_id 已含 `_e{epoch}`(如 `{base}_e{epoch}_s{step}``{run_id}_holdout_{kind}_e{epoch}`),用 run_id 作 cache_salt 即天然跨 epoch 重采样、同 epoch 续跑仍命中。judge/evolve 不经此路径(默认 salt=None)。gate 基线臂走 BaselineCache 不受影响;候选臂 messages 含 skill 版本天然区分。
- [ ] **Step 1: 写失败测试**
```python
@pytest.mark.asyncio
async def test_agent_loop_forwards_cache_salt():
"""AgentLoop.run(cache_salt=...) 透传到 llm.chat。"""
from core.agent.loop import AgentLoop
# fake llm 记录 chat 收到的 cache_salt kwargloop.run(..., cache_salt='run:e2')
# 断言 fake_llm.chat 收到 cache_salt='run:e2'
...
```
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest -k agent_loop_forwards_cache_salt -v`
Expected: FAIL`run()` 无 cache_salt 参数)。
- [ ] **Step 3: AgentLoop 透传 cache_salt**
`core/agent/loop.py``run`L103)、`_step`/`_call_llm`L271,317)签名加 `cache_salt: str | None = None`keyword,随 session_id 透传);L329 `self._llm.chat(messages, session_id=session_id)` 改为 `self._llm.chat(messages, session_id=session_id, cache_salt=cache_salt)`
- [ ] **Step 4: inference 用 run_id 作 salt**
`app/harness/inference.py` `_run_single_question``loop.run(...)`L409)加 `cache_salt=run_id`run_id 从 run_inference 透传到每题;`_run_single_question` 已有 run_id 上下文——若无则从 run_inference 参数透传)。确认 run_inference→_run_single_question 的 run_id 传递链完整。
- [ ] **Step 5: 保真检查点 + 测试通过**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py tests/integration/test_agent_governed_e2e.py -q`
Expected: 全 PASS。保真(算法 #10):确认只加透传参数,Thinking+JSON/json_repair/pluggy hook 逻辑不变。
- [ ] **Step 6: 提交**
```bash
git add core/agent/loop.py app/harness/inference.py tests/unit/
git commit -m "feat: inject run_id as cache_salt for per-epoch resampling (algo #10)"
```
---
## Self-Review(作者自查,执行者复核)
- [ ] traces 适配后诊断拿到真实轨迹(算法 #7 恢复)。
- [ ] prediction 归一化 None 保留、非标量序列化;insert 绑定异常不击穿 gather。
- [ ] early_stop 字段全局改名一致、计数改 epoch。
- [ ] 预检剔除不可训练题型(test 池不动)。
- [ ] patch 查整跨度 + 注入 + marker 完整性三防线(算法 #8 保护方向不变)。
- [ ] None/degraded 按 lapse 保守分流;降级占比>50% 中止。
- [ ] _run_step 幂等;gate_pools 与 checkpoint 的 gate_epoch_observed 同刻一致。
- [ ] holdout 去重后 baseline 0 推理、best_hard 备忘录 resume 可 hydrate。
- [ ] cache_salt=run_id 贯穿 AgentLooprun_id 含 epoch 保证跨 epoch 重采样。
## 核心算法保真校验结论
触及算法 #5/#7/#8/#10/#12,均为恢复(#7 轨迹)/加固(#8 跨度)/透传(#10 salt/时序(#5/#12 checkpoint),各 Task 已设保真检查点,不改算法核心逻辑。Task 5/7/9 需在实现时对照 TRM4 参考确认无行为漂移。
## 验收标准
1. `pytest tests/unit/test_harness_runner.py tests/unit/test_harness_inference.py tests/unit/test_diagnose.py tests/unit/test_patch.py tests/unit/test_harness_checkpoint.py tests/unit/test_baseline_run_log.py tests/unit/test_runner_diag_tree_inject.py` 全绿。
2. 诊断拿到非空轨迹;非标量 prediction 不崩;early_stop 按 epoch。
3. 微型题型被预检剔除;patch 不破坏 marker;降级题不驱动进化。
4. _run_step 幂等;cache_salt=run_id 贯穿。
@@ -0,0 +1,624 @@
# WP4 韧性与持久化 Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** 修复 LLM 治理栈与持久化层的 10 处韧性缺陷,使训练信号不被缓存重放/截断响应/断连/INFRA 故障污染,且 workspace 元数据崩溃可恢复。
**Architecture:** 缓存加 salt 维度让跨 epoch 评估真实重采样(同 epoch 续跑仍命中);SSE 未收 `[DONE]` 视为截断进重试且不写缓存;扩展瞬时错误清单覆盖断连族;熔断半开只放一个探针;gate 基线臂 INFRA 故障不污染 BaselineCache(算法 #6 保真区,逐行比对);manifest 补原子写;只读查询不改基线元数据。
**Tech Stack:** Python 3.11、httpx、Redis、asyncio、SQLite、pytest、pydantic-settings。
**设计源**`research-wiki/designs/2026-07-16-preflight-fixes-design.md §8`
---
## 关键锚点(实现前必读)
| 用途 | 位置 |
|------|------|
| chat 缓存/重试 | `adapters/llm.py:271` chatget L299 / set L373 / 重试 L336);`_call_streaming:506``_consume_stream:533`(不校验 done);`_iter_sse_deltas:94`done 标志 L116);`_is_transient_error:173-186``_SseAnomaly:40` |
| 缓存键 | `adapters/redis_cache.py:32` `_build_key`(无 salt);`get:50``set:73`ttl 分支 L89-92 |
| Protocol | `core/protocols.py:22-28` `LLMProvider.chat` |
| TTL | `main.py:41` `redis_cache_ttl=86400``:93` `ttl_s = ... if >0 else None` |
| 熔断 | `adapters/breaker.py:24` `is_open`(L36-37 到期即放行,无探针锁);`record_failure:39``record_success:63` |
| gate INFRA | `app/harness/validate.py:257` `_resolve_baseline_block`put L303-304);`_check_infra_guard:384`(累计检查 L593);`BaselineCache` in `gate_ladder.py:344`put L384 原子 L401-403 |
| 持久化 | `app/harness/workspace.py` 4 处非原子 `manifest.json` write_textWP2 后行号):`_scaffold:111`/`update_manifest:289`/`record_run:318`/`update_best:372`;原子范式 `checkpoint.py:257-260`。**以 `grep -n 'manifest.json").write_text' app/harness/workspace.py` 现场定位为准** |
| 基线元数据 | `app/harness/log.py:57` `HarnessLog.__init__`upsert `_runs` L73-82);`RunLogImpl._read_table:247`(只读) |
| dual_metric | `app/harness/observation.py:119` `write_dual_metric`version_kind final L138);runner 调用 `runner.py:1444-1448` |
| 配置 | `.env:65` `REDIS_CACHE_TTL=0``.env.example:49` `=86400` |
## 核心算法保真校验
触及算法 #6(块顺序验证:基线缓存/INFRA 护栏/配对翻转)——Task 6gate 基线臂 INFRA 隔离)改 `_resolve_baseline_block` 与护栏时序,**必须逐行比对** TRM4 `/home/iomgaa/Projects/Video-Tree-TRM4/core/harness/validate.py` 的 INFRA 护栏语义,确保只改"INFRA unit 不写缓存/不计 W/L + 护栏前置",不动配对翻转与基线快照复用逻辑。其余 task 不触碰核心算法。
---
## Task 1: 缓存 cache_salt 贯穿
**Files:**
- Modify: `core/protocols.py:22-28`
- Modify: `adapters/llm.py:271-373`
- Modify: `adapters/redis_cache.py:32-92`
- Modify: VLM 适配转发(`adapters/` 内实现 `LLMProvider`/转发 chat 的类,grep 定位)
- Test: `tests/unit/test_redis_cache.py``tests/unit/test_governed_llm.py`
- [ ] **Step 1: 写失败测试(salt 入键)**
`tests/unit/test_redis_cache.py` 追加:
```python
@pytest.mark.asyncio
async def test_salt_changes_key(fake_redis):
from adapters.redis_cache import RedisResponseCache
cache = RedisResponseCache(redis=fake_redis, ttl_s=None)
k_none = cache._build_key("m", [{"role": "user", "content": "x"}], None)
k_e1 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e1")
k_e2 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e2")
assert k_none != k_e1 != k_e2 and k_e1 != k_e2
```
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_redis_cache.py::test_salt_changes_key -v`
Expected: FAIL`_build_key() takes 3 positional arguments but 4 were given`)。
- [ ] **Step 3: redis_cache 加 salt 维度**
`adapters/redis_cache.py`
- `_build_key(self, model, messages, cache_salt: str | None = None)`**仅当 `cache_salt is not None` 才加入 `salt` 字段**(默认 None 时 payload 结构与现状一字节不差,旧缓存键不失效):
```python
key_obj: dict = {"model": model, "messages": messages}
if cache_salt is not None:
key_obj["salt"] = cache_salt
payload = json.dumps(key_obj, sort_keys=True, ensure_ascii=False)
```
- `get(self, model, messages, cache_salt: str | None = None)``key = self._build_key(model, messages, cache_salt)`
- `set(self, model, messages, response, cache_salt: str | None = None)`:同样传 `cache_salt`
- [ ] **Step 4: Protocol + chat 透传 salt**
`core/protocols.py` `LLMProvider.chat`L22)签名加 `cache_salt: str | None = None`keyword-only,放 `parent_call_id` 后)。
`adapters/llm.py` `chat`L271)签名加同参;L299 改 `await self._cache.get(self._model, messages, cache_salt)`L373 改 `await self._cache.set(self._model, messages, response, cache_salt)`
`adapters/redis_cache.py` `get`L50/`set`L73)签名各加 `cache_salt: str | None = None`,内部 `_build_key(model, messages, cache_salt)`
**VLM 转发**`core/protocols.py` `VLMProvider.chat_with_images`L35)与 `adapters/vlm.py` `GovernedVLMClient.chat_with_images`L32)签名加 keyword-only `cache_salt: str | None = None`L53 转发 `self._llm.chat(..., cache_salt=cache_salt)`
- [ ] **Step 5: 运行测试确认通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_redis_cache.py tests/unit/test_governed_llm.py -q`
Expected: 全 PASS(默认 `cache_salt=None` 保持旧键,既有缓存测试不受影响)。
- [ ] **Step 6: 提交**
```bash
git add core/protocols.py adapters/llm.py adapters/redis_cache.py tests/unit/test_redis_cache.py
git commit -m "feat: add cache_salt dimension to LLM response cache"
```
> 注:训练链路注入 epoch 盐(`f"{run_id}:e{epoch}"`)在 WP3 的推理调用点完成(本 WP 只提供 salt 能力)。
---
## Task 2: TTL 语义修正(≤0 报错)
**Files:**
- Modify: `adapters/redis_cache.py`(新增 `_resolve_cache_ttl`
- Modify: `main.py:93``app/harness/video_split_cli.py:257`(两处 TTL 入口共用)
- Modify: `.env:65``.env.example:49`
- Test: 新增 `tests/unit/test_infra_settings.py`
- [ ] **Step 1: 写失败测试**
新增 `tests/unit/test_infra_settings.py`
```python
import pytest
def test_redis_cache_ttl_zero_rejected():
"""REDIS_CACHE_TTL<=0 必须启动即报错,消灭'0=永不过期'隐式语义。"""
from adapters.redis_cache import _resolve_cache_ttl # Step 3 抽出的纯函数
with pytest.raises(ValueError, match="REDIS_CACHE_TTL"):
_resolve_cache_ttl(0)
with pytest.raises(ValueError, match="REDIS_CACHE_TTL"):
_resolve_cache_ttl(-1)
assert _resolve_cache_ttl(86400) == 86400
```
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_infra_settings.py -v`
Expected: FAIL`cannot import name '_resolve_cache_ttl'`)。
- [ ] **Step 3: 抽出 fail-loud 并替换两处 TTL 入口**
`adapters/redis_cache.py` 新增模块级纯函数(配置校验贴近 cache 实现、避免 app→main 依赖):
```python
def _resolve_cache_ttl(ttl: int) -> int:
"""校验 Redis 缓存 TTL:必须为正整数(消灭 0=永不过期 的隐式语义)。"""
if ttl <= 0:
raise ValueError(
f"REDIS_CACHE_TTL 必须为正整数秒,实际 {ttl}。"
"训练场景建议 >= 单次训练时长(如 86400)。"
)
return ttl
```
`main.py:93` 改为 `ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl)`import 上述函数,删 `if >0 else None`)。
`app/harness/video_split_cli.py:257``_build_redis_cache` 内)同样从 `ttl_s = ... if > 0 else None` 改为 `ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl)`——消灭第二条 `0=永不过期` 入口。
- [ ] **Step 4: 改 .env / .env.example**
`.env:65` `REDIS_CACHE_TTL=0``REDIS_CACHE_TTL=86400`
`.env.example:49` 确认为正整数(当前 86400,OK);补注释 `# 正整数秒,禁止 0(0 会被拒绝启动)`
- [ ] **Step 5: 运行测试确认通过**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_infra_settings.py -v`
Expected: PASS。
- [ ] **Step 6: 提交**
```bash
git add main.py .env.example tests/unit/test_infra_settings.py
git commit -m "fix: reject REDIS_CACHE_TTL<=0 (kill implicit never-expire)"
```
> 注:`.env` 不提交(gitignore);改动需手动同步到运行环境,runbook 会提示。
---
## Task 3: SSE 截断检测(未收 [DONE] 即重试,不写缓存)
**Files:**
- Modify: `adapters/llm.py:533-578`_consume_stream
- Test: `tests/unit/test_governed_llm.py`
- [ ] **Step 1: 写失败测试**
`tests/unit/test_governed_llm.py` 追加:
```python
@pytest.mark.asyncio
async def test_truncated_stream_without_done_raises(_build_client):
"""SSE 流耗尽但未收 [DONE] → _SseAnomaly(进重试,不当成功)。"""
from adapters.llm import _SseAnomaly, GovernedLLMClient
async def _lines():
yield 'data: {"choices":[{"delta":{"content":"半"}}]}'
# 无 data: [DONE] —— 模拟服务端截断
client = _build_client() # 依现有 fixture
with pytest.raises(_SseAnomaly):
await client._consume_stream(_lines())
```
> 依 `test_governed_llm.py` 现有 `_build_client`L50/`_FakeRedisCache` 构造;实现前读该文件对齐 client 构造签名。
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -k truncated_stream -v`
Expected: FAIL(当前耗尽即正常返回,不抛)。
- [ ] **Step 3: _consume_stream 校验 done**
`adapters/llm.py` `_consume_stream``return content, thinking, ttft_ms, max_inter_token_ms, usage`L578)之前加:
```python
if not usage_sink.get("done"):
raise _SseAnomaly("truncated_no_done")
```
`_iter_sse_deltas` 收到 `[DONE]` 时置 `usage_sink["done"]=True`L116;未置说明流被截断。`_SseAnomaly` 已在 `_is_transient_error` L186 归为可重试,故自动进重试梯且不走成功路径、不写缓存。)
- [ ] **Step 4: 运行测试确认通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py tests/unit/test_streaming.py -q`
Expected: 全 PASS(正常流带 `[DONE]` 的既有测试仍通过;若既有 streaming 测试的假流未含 `[DONE]`,同步补 `data: [DONE]` 终帧)。
- [ ] **Step 5: 提交**
```bash
git add adapters/llm.py tests/unit/test_governed_llm.py
git commit -m "fix: treat SSE stream without [DONE] as truncated (retry, no cache)"
```
---
## Task 4: 瞬时错误清单扩展(断连族)
**Files:**
- Modify: `adapters/llm.py:182-186`
- Test: `tests/unit/test_governed_llm.py`
- [ ] **Step 1: 写失败测试**
`tests/unit/test_governed_llm.py` 追加:
```python
def test_transient_covers_disconnect_family():
import httpx
from adapters.llm import _is_transient_error
assert _is_transient_error(httpx.RemoteProtocolError("peer reset"))
assert _is_transient_error(httpx.ReadError("read"))
assert _is_transient_error(httpx.ConnectTimeout("ct"))
assert _is_transient_error(httpx.PoolTimeout("pt"))
```
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -k disconnect_family -v`
Expected: FAILRemoteProtocolError/PoolTimeout 不在当前清单)。
- [ ] **Step 3: 扩展 _is_transient_error**
`adapters/llm.py` L182-183 的:
```python
if isinstance(exc, (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout)):
return True
```
改为(两族基类覆盖 RemoteProtocolError/ReadError/ConnectTimeout/PoolTimeout 等):
```python
if isinstance(exc, (httpx.TimeoutException, httpx.TransportError)):
return True
```
- [ ] **Step 4: 运行测试确认通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -q`
Expected: 全 PASS`httpx.HTTPStatusError` 非 TransportError 子类,401/403 致命分支不受影响)。
- [ ] **Step 5: 提交**
```bash
git add adapters/llm.py tests/unit/test_governed_llm.py
git commit -m "fix: cover httpx disconnect family in transient error set"
```
---
## Task 5: 熔断半开单探针锁
**Files:**
- Modify: `adapters/breaker.py:18-70`
- Test: `tests/unit/test_breaker.py`
- [ ] **Step 1: 写失败测试**
`tests/unit/test_breaker.py::TestCircuitBreaker` 追加:
```python
def test_half_open_admits_single_probe(self):
from adapters.breaker import CircuitBreaker
b = CircuitBreaker(fail_threshold=2, cooldown_s=10.0)
b.record_failure("p", now=0.0)
b.record_failure("p", now=0.0) # 开路至 t=10
assert b.is_open("p", now=5.0) is True # 冷却中
# 冷却到期:只放行第一个探针
assert b.is_open("p", now=11.0) is False # 探针 1 放行
assert b.is_open("p", now=11.0) is True # 探针 2 被挡(探针在途)
b.record_success("p") # 探针成功 → 闭合
assert b.is_open("p", now=12.0) is False
```
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_breaker.py -k half_open_admits_single -v`
Expected: FAIL(当前到期后所有调用都返回 False)。
- [ ] **Step 3: 实现半开单探针锁**
`adapters/breaker.py`
- `__init__``self._half_open_inflight: dict[str, bool] = {}`
- `is_open` 改为:
```python
def is_open(self, source_name: str, now: float) -> bool:
until = self._open_until.get(source_name)
if until is None:
return False
if now < until:
return True # 冷却中,全挡
# 冷却到期:half-open,只放行一个探针
if self._half_open_inflight.get(source_name):
return True # 已有探针在途,继续挡
self._half_open_inflight[source_name] = True
return False
```
- `record_success``self._half_open_inflight.pop(source_name, None)`(探针成功 → 清在途 + 已有的清 fails/open_until
- `record_failure`:探针失败会累计并可能重开路;末尾加 `self._half_open_inflight.pop(source_name, None)`(让下一轮 cooldown 后可再探)
- `force_open`:加 `self._half_open_inflight.pop(source_name, None)`
is_open 在 asyncio 单线程内同步执行,"检查+标记探针"原子,无竞态。)
- [ ] **Step 4: 运行测试确认通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_breaker.py tests/unit/test_governed_llm.py -q`
Expected: 全 PASS(既有 circuit_open 测试若假设"到期即多次放行"需同步更新为单探针语义)。
- [ ] **Step 5: 提交**
```bash
git add adapters/breaker.py tests/unit/test_breaker.py
git commit -m "fix: half-open circuit admits single probe (no thundering herd)"
```
---
## Task 6: gate 基线臂 INFRA 隔离(算法 #6 保真区)
**Files:**
- Modify: `app/harness/validate.py:257-311`_resolve_baseline_block+ 护栏时序
- Test: `tests/unit/test_harness_validate.py`
> **保真前置**:实现前完整读 `app/harness/validate.py:257-311` `_resolve_baseline_block``_candidate_correctness_from_db`、块循环(575-600),并逐行比对 TRM4 `core/harness/validate.py` 的 INFRA 护栏语义。目标仅为:① 基线臂 INFRA 故障(stop_reason∈{error,parse_error})的 unit **不写 BaselineCache**、**不计入 W/L 翻转**;② 护栏检查移到"写缓存之前"。不得改动配对翻转、基线快照复用、unit 折叠逻辑。
- [ ] **Step 1: 写失败测试**
`tests/unit/test_harness_validate.py` 追加(用 fake run_inference 让某基线 unit 返回 stop_reason=error):
```python
@pytest.mark.asyncio
async def test_baseline_infra_error_not_cached(tmp_path):
"""基线臂 INFRA error 的 unit 不写入 BaselineCache(不永久污染)。"""
# 依现有 test_harness_validate.py 的 fake run_inference / BaselineCache fixture 构造;
# 让 miss unit u1 的推理返回 prediction=None, stop_reason='error'
# 调 _resolve_baseline_block 后断言 baseline_cache.get(..., u1) is None
...
```
> 依 `test_harness_validate.py` 现有 fixturefake `run_inference``BaselineCache``HarnessLog`);实现前读该文件对齐构造,勿臆造签名。
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_validate.py -k baseline_infra_error_not_cached -v`
Expected: FAIL(当前 INFRA unit 的对错被 put 进缓存)。
- [ ] **Step 3: 实现 INFRA 隔离(精确数据路径)**
**数据路径**(Codex 审指出:当前 `_load_run_rows`validate.py:212)只查 `question_id,prediction,answer,steps_json`,拿不到 per-unit stop_reason,只有汇总 `stop_reason_counts` 无法定位哪个 unit 是 INFRA):
1. `_load_run_rows`validate.py:194-212)的 SELECT 增加 `stop_reason` 列;`_candidate_correctness_from_db` 相应可返回每题 stop_reason(或新增 `_infra_question_ids_from_db(log, run_id, questions)` 返回 stop_reason∈{"error","parse_error"} 的 qid 集)。
2. `_resolve_baseline_block`:miss 跑完后,计算本块 INFRA unit 集(unit 内**任一题** stop_reason∈{"error","parse_error"})。对这些 unit**不 `baseline_cache.put`**、**不计入 `b_units`**、并从返回给调用方的有效 unit 集中排除。护栏所需 `errors_inc` 在 put 之前累计并调 `_check_infra_guard`(护栏前置于缓存写入)。
3. **贯穿调用方**validate.py:575-600):`_resolve_baseline_block` 返回"有效 unit 子集 `valid_unit_chunk`",后续 `_run_candidate_block``unit_correctness_view``pair_block``_build_evidence_rows` 全部用 `valid_unit_chunk`(而非原始 `unit_chunk`),确保配对 `unit_ids` 与基线侧一致、不含 INFRA unit。
4. 保真:非 INFRA unit 的配对翻转、基线快照复用、unit 折叠逻辑一字不改;仅"把 INFRA unit 从本块整体剔除"。按 Step 前"保真前置"逐行比对 TRM4。
- [ ] **Step 4: 运行测试确认通过 + 保真回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_validate.py tests/unit/test_gate_block_unit.py tests/unit/test_gates.py -q`
Expected: 全 PASS(配对翻转/e-process 既有测试不受影响 = 保真达成)。
- [ ] **Step 5: 提交**
```bash
git add app/harness/validate.py tests/unit/test_harness_validate.py
git commit -m "fix: isolate gate baseline-arm INFRA errors from BaselineCache (algo #6)"
```
---
## Task 7: manifest 原子写
**Files:**
- Modify: `app/harness/workspace.py``_scaffold:111`/`update_manifest:282`/`record_run:311`/`update_best:365`
- Test: `tests/unit/test_harness_workspace.py`
- [ ] **Step 1: 写失败测试**
`tests/unit/test_harness_workspace.py` 追加:
```python
def test_update_manifest_is_atomic(workspace_dir, monkeypatch):
"""写 manifest 途中崩溃不产生半截 JSON(原子写:tmp 存在即失败也不损原文件)。"""
import json
from app.harness import workspace as ws
# 先建合法 manifest
... # 依现有 fixture 初始化 workspace
original = (workspace_dir / "manifest.json").read_text()
# monkeypatch os.replace 抛异常,模拟替换阶段崩溃
def _boom(src, dst):
raise OSError("crash during replace")
monkeypatch.setattr(ws.os, "replace", _boom)
with pytest.raises(OSError):
ws.update_manifest(workspace_dir, skills="skills/v2")
# 原 manifest 未被破坏
assert (workspace_dir / "manifest.json").read_text() == original
assert json.loads((workspace_dir / "manifest.json").read_text())
```
> 依 `test_harness_workspace.py` 现有 workspace 初始化 fixture;实现前读对齐。
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_workspace.py -k update_manifest_is_atomic -v`
Expected: FAIL(当前 write_text 非原子,崩溃留半截)。
- [ ] **Step 3: 加原子写 helper 并替换 4 处**
`app/harness/workspace.py` 顶部确认 `import os`。新增模块级 helper
```python
def _atomic_write_json(path: Path, data: dict) -> None:
"""原子写 JSONtmp + os.replace(对齐 checkpoint.py 范式,防半截损坏)。"""
tmp = path.with_name(path.name + ".tmp")
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
os.replace(tmp, path)
```
`_scaffold_workspace`~L111)、`update_manifest`~L289)、`record_run`~L318)、`update_best`~L372)四处的 `(workspace_dir / "manifest.json").write_text(json.dumps(...))` 替换为 `_atomic_write_json(workspace_dir / "manifest.json", <data>)`(用 `grep -n 'manifest.json").write_text' app/harness/workspace.py` 定位全部 4 处,行号随 WP2 改动漂移)。
- [ ] **Step 4: 运行测试确认通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_workspace.py -q`
Expected: 全 PASS。
- [ ] **Step 5: 提交**
```bash
git add app/harness/workspace.py tests/unit/test_harness_workspace.py
git commit -m "fix: atomic writes for manifest/record_run/update_best (tmp+replace)"
```
---
## Task 8: 只读查询不改基线元数据
**Files:**
- Modify: `app/harness/log.py:57-83`HarnessLog.__init__ 加 register_run 开关)
- Modify: 只读查询基线的调用点(`app/harness/runner.py:923` `_init_gate``app/harness/pools.py:765,818`
- Test: `tests/unit/test_harness_log.py`
- [ ] **Step 1: 写失败测试**
`tests/unit/test_harness_log.py::TestHarnessLogUpsert` 追加:
```python
def test_register_run_false_skips_upsert(self, tmp_path):
"""register_run=False 时只读打开不改写已有 _runs 行(started_at/status 不变)。"""
db = str(tmp_path / "h.db")
from app.harness.log import HarnessLog
with HarnessLog(db, "r1") as log:
log # 初次注册
row0 = _read_run_row(db, "r1") # 依现有 helper 读 started_at/status
with HarnessLog(db, "r1", register_run=False) as log:
log.query("SELECT 1") # 只读
row1 = _read_run_row(db, "r1")
assert row1["started_at"] == row0["started_at"]
assert row1["status"] == row0["status"]
```
> 依 `test_harness_log.py` 现有读行 helper;实现前读 `TestHarnessLogUpsert`L196)对齐。
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_log.py -k register_run_false -v`
Expected: FAIL`unexpected keyword argument 'register_run'`)。
- [ ] **Step 3: HarnessLog 加 register_run 开关**
`app/harness/log.py` `__init__` 签名加 `register_run: bool = True`keyword)。把 L73-82 的 `_runs` upsert 包进 `if register_run:``register_run=False` 时跳过 upsert(仅 `_init_fixed_tables` 建表 + 连接,供只读查询)。`__exit__`/`close` 的 status 更新同样在 `register_run` 为 True 时才执行(避免只读关闭把 status 改 completed)。
- [ ] **Step 4: 只读调用点传 register_run=False**
`app/harness/runner.py:923`_init_gate 用 baseline_run_id 只读查 predictions)、`app/harness/pools.py:765,818`build_or_load_pools 只读查 predictions)三处 `HarnessLog(...)` 调用加 `register_run=False`
- [ ] **Step 5: 运行测试确认通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_log.py tests/unit/test_harness_pools.py -q`
Expected: 全 PASS。
- [ ] **Step 6: 提交**
```bash
git add app/harness/log.py app/harness/runner.py app/harness/pools.py tests/unit/test_harness_log.py
git commit -m "fix: read-only baseline queries skip _runs upsert (register_run flag)"
```
---
## Task 9: dual_metric version_kind 口径修正
**Files:**
- Modify: `app/harness/runner.py:1444-1448`(第二次 write_dual_metric
- Test: `tests/unit/test_harness_observation.py``test_harness_runner.py`
- [ ] **Step 1: 写失败测试**
`read_dual_metric` 真实签名为 `(db_path, *, run_id)`(无 epoch 参数,observation.py:164),故 observation 层 write/read 对任意 version_kind 都已支持——真正要验证的是 **runner 慢更新第二次写出的是 `slow_candidate` 而非 `final`**。在 `tests/unit/test_harness_observation.py` 追加 observation 层可区分性基线测试:
```python
def test_dual_metric_kinds_distinguishable(tmp_path):
from app.harness.observation import write_dual_metric, read_dual_metric
db = str(tmp_path / "h.db")
write_dual_metric(db, run_id="r", epoch=1, version_kind="final",
skills_version="v1", prompts_version="v1", pool="val",
hard_acc=0.7, soft_score=None, mixed_score=None)
write_dual_metric(db, run_id="r", epoch=1, version_kind="slow_candidate",
skills_version="v2", prompts_version="v2", pool="val",
hard_acc=0.6, soft_score=None, mixed_score=None)
rows = read_dual_metric(db, run_id="r") # 真实签名无 epoch,返回该 run 全部行
kinds = {row["version_kind"] for row in rows if row["epoch"] == 1}
assert kinds == {"final", "slow_candidate"}
```
> 实现前读 `test_harness_observation.py``write_dual_metric`/`read_dual_metric`observation.py:119/164)确认参数名(epoch/pool/hard_acc 等)与返回行字段。真正的 `slow_candidate` 语义由 Step 3 改 runner 调用点落地。
- [ ] **Step 2: 运行基线(observation 层已支持任意 kind**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_observation.py -k dual_metric_kinds -v`
Expected: PASSobservation 层本就接受任意 version_kind);本 task 的实质改动在 Step 3 的 runner 调用点。
- [ ] **Step 3: 改 runner 慢更新第二次写为 slow_candidate**
`app/harness/runner.py` L1444-1448 的 `write_dual_metric(..., version_kind="final", ...)``_slow_update_cycle` Phase 8 的 R2 行)改为 `version_kind="slow_candidate"`,使被 revert 的慢更新候选不再占用 `final` 语义。确认 Phase 2 的第一次(L1399-1403)仍为 `final`epoch 终值唯一)。
- [ ] **Step 4: 运行测试确认通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_observation.py tests/unit/test_harness_runner.py -q`
Expected: 全 PASS。
- [ ] **Step 5: 提交**
```bash
git add app/harness/runner.py tests/unit/test_harness_observation.py
git commit -m "fix: slow-update R2 dual_metric uses slow_candidate kind (not final)"
```
---
## Task 10: 离线 --retry-uncertain + docstring 修正
**Files:**
- Modify: `core/evolution/protocols.py:130``DiagnosisSignalStore.done_question_ids` 加参数)
- Modify: `adapters/baseline_diagnosis_store.py:109`SqliteDiagnosisSignalStore 实现 + SQL
- Modify: `app/harness/baseline_diagnosis.py:96`(透传 + docstring
- Modify: `app/harness/video_split_cli.py`build_arg_parser + run_pipeline 透传)
- Test: `tests/unit/`(对 SqliteDiagnosisSignalStore.done_question_ids
- [ ] **Step 1: 写失败测试**
`done_question_ids` 的真实定义在 Protocol `core/evolution/protocols.py:130` 与实现 `adapters/baseline_diagnosis_store.py:109`(不在 baseline_diagnosis.py)。对 SqliteDiagnosisSignalStore 追加测试:
```python
def test_done_question_ids_retry_uncertain_excludes(tmp_path):
# 构造 baseline_diagnosis 表含 tier=T2 行与 tier=uncertain 行(同 run+fingerprint);
from adapters.baseline_diagnosis_store import SqliteDiagnosisSignalStore
store = SqliteDiagnosisSignalStore(str(tmp_path / "h.db"))
# ... upsert 一条 T2、一条 uncertain ...
assert "<uncertain_qid>" in store.done_question_ids("run", "fp") # 默认含
assert "<uncertain_qid>" not in store.done_question_ids("run", "fp", retry_uncertain=True) # 排除
assert "<t2_qid>" in store.done_question_ids("run", "fp", retry_uncertain=True) # T2 仍算完成
```
> 实现前读 `adapters/baseline_diagnosis_store.py` 的 upsert/表结构与 `done_question_ids` SQL 对齐构造。
- [ ] **Step 2: 运行确认失败**
Run: `conda run -n Video-Tree-TRM python -m pytest -k done_question_ids_retry_uncertain -v`
Expected: FAIL`unexpected keyword argument 'retry_uncertain'`)。
- [ ] **Step 3: 实现 retry_uncertain(贯穿 Protocol→Adapter→编排→CLI**
- `core/evolution/protocols.py:130` `DiagnosisSignalStore.done_question_ids` 签名加 `retry_uncertain: bool = False`keyword)。
- `adapters/baseline_diagnosis_store.py:109` 实现同签名;SQL 在 `retry_uncertain=True` 时追加 `AND tier != 'uncertain'`(uncertain 题不算完成,会被重诊)。
- `app/harness/baseline_diagnosis.py:96` `done = store.done_question_ids(baseline_run_id, diag_fingerprint)` 透传 `retry_uncertain`;修正模块 docstring 把"崩溃最多丢正在写的一行/逐行落库"改为实际的"run 末批量落库(Phase 3),崩溃丢本次 run 未落库结果,靠 Redis 缓存缓解重烧"。
- `app/harness/video_split_cli.py``build_arg_parser``--retry-uncertain`store_true);`run_pipeline` 透传到 `run_baseline_diagnosis`
- [ ] **Step 4: 运行测试确认通过 + 回归**
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -q`
Expected: 全 PASS。
- [ ] **Step 5: 提交**
```bash
git add app/harness/video_split_cli.py app/harness/baseline_diagnosis.py tests/unit/
git commit -m "feat: --retry-uncertain re-diagnoses uncertain rows; fix docstring"
```
---
## Self-Review(作者自查,执行者复核)
- [ ] cache_salt 四处(protocols/llm/redis_cache/VLM 转发)贯通,默认 None 保持旧键。
- [ ] TTL≤0 fail-loud + .env/.env.example 同步(.env 手动,runbook 提示)。
- [ ] SSE 未收 [DONE] → _SseAnomaly(已在瞬时清单 → 重试 + 不写缓存)。
- [ ] 瞬时清单用 TimeoutException+TransportError 两族基类覆盖断连族,不误纳 HTTPStatusError。
- [ ] 熔断半开单探针:is_open 检查+标记原子(asyncio 单线程)。
- [ ] Task 6 触及算法 #6,已设保真前置(逐行比对 TRM4,只改 INFRA 不写缓存 + 护栏前置)。
- [ ] manifest 4 处原子写;只读查询 register_run=False 不改基线元数据。
## 核心算法保真校验结论
Task 6 触及算法 #6(块顺序验证),已在该 Task 设"保真前置"要求逐行比对 TRM4 `validate.py`,仅改 INFRA 隔离与护栏时序,不动配对翻转/基线快照/unit 折叠。其余 Task 均为治理栈/持久化层,不涉及核心算法。
## 验收标准
1. `pytest tests/unit/test_redis_cache.py tests/unit/test_governed_llm.py tests/unit/test_breaker.py tests/unit/test_streaming.py tests/unit/test_harness_validate.py tests/unit/test_harness_workspace.py tests/unit/test_harness_log.py tests/unit/test_harness_observation.py tests/unit/test_infra_settings.py` 全绿。
2. 缓存加 salt 后跨 epoch 键不同、同 epoch 键相同。
3. 熔断半开只放一个探针;SSE 截断进重试不写缓存。
4. manifest 原子写;基线 _runs 元数据只读查询不被改写。
@@ -0,0 +1,9 @@
---
type: plan
node_id: plan:preflight-wp1-asset-migration
title: "WP1 资产迁移"
date: 2026-07-16
---
# WP1 资产迁移
@@ -0,0 +1,9 @@
---
type: plan
node_id: plan:preflight-wp2-split-wiring
title: "WP2 切分与接线"
date: 2026-07-16
---
# WP2 切分与接线
@@ -0,0 +1,9 @@
---
type: plan
node_id: plan:preflight-wp3-train-loop
title: "WP3 训练循环与进化引擎"
date: 2026-07-16
---
# WP3 训练循环与进化引擎
@@ -0,0 +1,9 @@
---
type: plan
node_id: plan:preflight-wp4-resilience
title: "WP4 韧性与持久化"
date: 2026-07-16
---
# WP4 韧性与持久化
@@ -0,0 +1,49 @@
# 训练前修复分支 final whole-implementation review
> 2026-07-16。分支 `feat/preflight-train-fixes`4 工作包 33 个实现 commit。SDD 每 task 三审 + 本次 Codex 跨 task 整分支终审(gpt-5.4 xhigh)。
> 覆盖矩阵:21 项确认缺陷 + 接线 2 项 + WP1 死字段清理**全部实现落地**Codex 逐项核对 file:line 证据)。核心算法保真(#4/#5/#6/#7/#8/#9/#10/#12)、依赖方向(core 不依赖 app/adapters)、loguru、原子写、死字段无残留——全部 verified OK。
## 执行结果汇总
| WP | 内容 | commit 数 | Codex 三审结果 |
|----|------|----------|---------------|
| WP1 | 模板迁移 + 死字段 + fail-loud | 3 | 无任何问题(模板 cmp TRM4 逐字节一致) |
| WP2 | 切分与接线(tier/功效/seed/覆盖保护) | 7 | 无 Critical2 Important--force 备份健壮性)已修 |
| WP4 | 韧性与持久化(10 项) | 11 | 无 Critical2 Important(全 INFRA/parse_error 护栏)已修 |
| WP3 | 训练循环与进化(9 项,5 算法保真区) | 12 | 无 Critical2 Important(单元计数/全过滤 fail-fast)已修 |
全量 1523 tests passedruff 全绿。
## Final review 发现与处置
### C-1(必须,runbook)——.env 未同步导致 train 启动崩溃
`.env:65``REDIS_CACHE_TTL=0`WP4 Task 2 的 fail-loud 会在 `main.py` / `video_split_cli` 构建 Redis 缓存时抛 ValueError,训练进不到 `runner.train``.env.example` 已改 86400 但 `.env`gitignore)需**手动**改。→ 训练前 runbook 第一步:`REDIS_CACHE_TTL` 改为正整数(如 86400)。
### I-3 / I-4(真实缺陷,✅ 已修)
| # | 缺陷 | 首跑是否触发 | 处置(commit) |
|---|------|:---:|------|
| I-4 | 显式 task_types 子集不过滤冻结全局 pools → 训练非请求题型 | 全 12 类首跑**不触发** | ✅ `ee69721``_filter_untrainable_types` 候选集先与 task_types 取交集,非请求题型剔除,fail-fast 保留 |
| I-3 | gate INFRA 护栏分子按 record、分母按 unitAR pair 误触发 gate_guard_err | 全 single 首跑**不触发** | ✅ `b3ba11c`INFRA 分子改按 unit 数;顺带修正 2 个既有护栏测试的 mock 不真实性(per-record 与 summary 不一致,真实推理不会发生),保真套件全绿 |
> M-2`6911c83`):`INFRA_STOP_REASONS` 提为 core 公共常量、app import。**残留 future work**`app/harness/video_split_cli.py` 仍有第三份独立副本(本次 scope 外),待后续 dedup。
### I-1 / I-2(技术论证:实际影响可控,记录不强修)
| # | Codex 关切 | 论证 | 首跑建议 |
|---|-----------|------|---------|
| I-1 | cache_salt 未贯穿工具内 LLM/VLMobserve_frame VLM、summarizer | 工具内是**确定性子程序**——同帧→同 VLM 描述、同轨迹→同 summary,重放语义正确甚至期望;需跨 epoch 重采样的 agent **决策** LLM(主 loop chat)已正确注入 run_id 盐 | 改 .env 后可跑;若谨慎可首跑关 Redis 缓存 |
| I-2 | 基线臂 miss 新鲜推理注入 epoch salt,不符固定快照 | BaselineCache`baseline_cache.json`)**跨 run 持久**:快照一旦建立即稳定,epoch salt 只作用于"首次建立快照的那一次采样"(本就要采一次),中断重跑命中持久缓存 | 影响限于首次采样,可接受 |
> 若后续要彻底贯彻 P1-1(把 cache_salt 显式化、基线臂传 None、工具内也隔离),记 future work——需把 Task 9 的"inference 内部用 run_id"重构为"调用方显式传 cache_salt",风险中等。
### Minorfuture work / 顺手)
- **M-2**INFRA stop-reason 集合在 core/app 各一份 → 已派 app import core 常量消除漂移。
- **M-1**`_atomic_write_json` 在 pools.py/workspace.py 两份等价实现(行为一致,可抽共享 helper)。
- **M-3**:部分 analyses JSON(非 checkpoint/manifest/pools)仍直接 write_text(不破坏续跑主状态)。
## 训练前 runbook(结合本分支)
1. **改 `.env``REDIS_CACHE_TTL=0` → `86400`**C-1,不改则启动崩)。
2. 备份 `workspaces/video-split/` 冻结产物。改 `config/video_split.yaml` 已含 `val_ratio: 0.4`WP2),重跑 `build_video_split.sh`(诊断命中缓存秒级重切)→ 核对 val 错题≥20 / T2 入 diag / tier 感知生效。
3. 建 seed`extract_run_db(infer_adhoc, dedupe_per_question=True)` + `init_seed('adhoc-baseline', pools_json=…, split_manifest=…)`WP2 接线)。
4. 新建 `config/train_videomme.yaml` + `scripts/train_videomme.sh``epochs=3``early_stop_patience=2`epoch 语义 WP3)、`run_holdout_eval=true`(去重版 WP3)、`trainable_min_units=8`(WP3 新字段,必填)、全 12 类(避免 I-4)、其余 gate/batch 沿用 default.yaml。
5. tmux 启动 `CUDA_VISIBLE_DEVICES=0 bash scripts/train_videomme.sh`。预检会自动剔除微型题型(OCR/Spatial/Temporal Perception 等 5 类),打印剔除清单。
@@ -0,0 +1,9 @@
---
type: review
node_id: review:preflight-final-review
title: 训练前修复分支终审
date: 2026-07-16
---
# 训练前修复分支终审
@@ -171,6 +171,7 @@ class _MockLLM:
*,
session_id: str | None = None,
parent_call_id: str | None = None,
cache_salt: str | None = None,
) -> LLMResponse:
content = json.dumps(
{"action": {"tool": "submit_answer", "args": {"answer": self._answer}}}
@@ -101,6 +101,7 @@ def test_end_to_end_freezes_valid_pools(tmp_path: Path) -> None:
select_seed=7,
val_ratio=0.3,
split_seed=7,
val_wrong_min=0,
)
result = build_split(
+2 -1
View File
@@ -90,7 +90,7 @@ class _FakeState:
eval_prev_run_id: str = "run-0"
baseline_skills_version: str = "v1"
baseline_prompts_version: str = "v1"
steps_since_best_improved: int = 0
epochs_since_best_improved: int = 0
epoch_start_skills: str = "v1"
changed_task_types_this_epoch: set[str] = field(default_factory=set)
rejected_buffer: dict = field(default_factory=dict)
@@ -111,6 +111,7 @@ class _FakeConfig:
diag_size: int = 30
val_size: int = 50
batch_correct_ratio: float = 0.0
trainable_min_units: int = 8
edit_budget_start: int = 6
edit_budget_end: int = 3
early_stop_patience: int = 3
+11
View File
@@ -94,6 +94,17 @@ def _invalid_tool_json() -> str:
class TestAgentLoop:
"""AgentLoop 推理循环引擎测试。"""
@pytest.mark.asyncio
async def test_forwards_cache_salt(self) -> None:
"""AgentLoop.run(cache_salt=...) 透传到 llm.chat(算法 #10 跨 epoch 重采样)。"""
llm = AsyncMock()
llm.chat.return_value = _make_response(_submit_json())
loop = AgentLoop(llm=llm, max_steps=10)
await loop.run("system", "user", _StubDispatcher(), cache_salt="run:e2")
assert llm.chat.call_args.kwargs["cache_salt"] == "run:e2"
@pytest.mark.asyncio
async def test_submit_answer_terminates_loop(self) -> None:
"""submit_answer 终止循环 → finished, result=args, steps_used=1。"""
@@ -85,6 +85,33 @@ def test_null_fields_for_infra_row(tmp_path):
assert rows[0].error_type is None and rows[0].evolution_target is None and rows[0].infra is True
def test_done_question_ids_retry_uncertain_excludes(tmp_path):
"""retry_uncertain=True 时 uncertain 题不算完成(会被重诊),非 uncertain 仍算完成。"""
s = _store(tmp_path)
# T2defect)题:已完成
s.upsert(
DiagnosisSignalRow(
"t2_qid", "v1", "run", "fp", "Counting",
"search_failure", "defect", "T2", "skill",
degraded=False, infra=False, session_id="s",
)
)
# uncertaindegraded)题:信号不可信
s.upsert(
DiagnosisSignalRow(
"uncertain_qid", "v2", "run", "fp", "Counting",
None, None, "uncertain", None,
degraded=True, infra=False, session_id=None,
)
)
# 默认:两者都算完成
assert s.done_question_ids("run", "fp") == {"t2_qid", "uncertain_qid"}
# retry_uncertainuncertain 被排除,T2 仍算完成
done = s.done_question_ids("run", "fp", retry_uncertain=True)
assert "uncertain_qid" not in done
assert "t2_qid" in done
def test_context_manager_closes_connection(tmp_path):
# with 块退出后连接关闭,再操作应报 ProgrammingError
with SqliteDiagnosisSignalStore(str(tmp_path / "h.db")) as s:
+12
View File
@@ -58,6 +58,18 @@ class TestCircuitBreaker:
breaker.force_open("llm", now=5.0)
assert breaker.is_open("llm", now=5.5) is True
def test_half_open_admits_single_probe(self) -> None:
"""冷却到期后半开只放行一个探针,第二个仍被挡;探针成功后闭合。"""
b = CircuitBreaker(fail_threshold=2, cooldown_s=10.0)
b.record_failure("p", now=0.0)
b.record_failure("p", now=0.0) # 开路至 t=10
assert b.is_open("p", now=5.0) is True # 冷却中
# 冷却到期:只放行第一个探针
assert b.is_open("p", now=11.0) is False # 探针 1 放行
assert b.is_open("p", now=11.0) is True # 探针 2 被挡(探针在途)
b.record_success("p") # 探针成功 → 闭合
assert b.is_open("p", now=12.0) is False
def test_force_open_probe_failure_reopens(self, breaker: CircuitBreaker) -> None:
"""半开探针失败后重新熔断(因 fails 已置为 threshold)。"""
breaker.force_open("llm", now=0.0)
+51 -1
View File
@@ -17,6 +17,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from core.evolution.diagnose import (
_build_skill_case_packs,
_percentile,
_trigrams,
aggregate_d2,
@@ -43,6 +44,7 @@ from core.evolution.types import (
CaseSample,
DiagnosePrompts,
DiagnosisResult,
ErrorAttribution,
QuestionMetrics,
SkillStepAdherence,
SpanMetrics,
@@ -750,7 +752,6 @@ class TestRunDiagnosis:
defect_vs_lapse="",
reasoning_sub="",
span_eval_system="",
span_eval_user="",
missed_nodes="",
skill_adherence="",
confirmation_bias="",
@@ -772,3 +773,52 @@ class TestRunDiagnosis:
assert result.run_id == "run1"
assert result.error_attributions == []
assert result.degraded_count == 0
class TestBuildSkillCasePacksDegradeRouting:
"""_build_skill_case_packs 降级/未判定分流:仅 defect 且非 degraded 进正文路径。"""
def test_none_cause_and_degraded_route_to_lapse(self) -> None:
"""cause_category=None(判别失败)与 degraded 题按 lapse 处置,不进 defect 正文路径。"""
tt = "Action Reasoning"
qm_none = _make_qm(question_id="q-none", task_type=tt, correct=False, degraded=False)
qm_degraded = _make_qm(question_id="q-deg", task_type=tt, correct=False, degraded=True)
# 补两道正确题,避免退化
qm_ok1 = _make_qm(question_id="q-ok1", task_type=tt, correct=True)
qm_ok2 = _make_qm(question_id="q-ok2", task_type=tt, correct=True)
metrics = [qm_none, qm_degraded, qm_ok1, qm_ok2]
attributions = [
# 判别失败:cause_category=None
ErrorAttribution(
question_id="q-none",
error_type="reasoning",
reasoning_failure_type=None,
cause_category=None,
lapse_note="复核该类推理规则",
),
# degraded 题即便被判 defect,也须走 lapse(不驱动正文进化)
ErrorAttribution(
question_id="q-deg",
error_type="reasoning",
reasoning_failure_type=None,
cause_category="defect",
lapse_note=None,
),
]
packs = _build_skill_case_packs(
all_metrics=metrics,
error_attributions=attributions,
traces_by_question={},
predictions=[],
d3_stats={},
d4_stats={},
)
pack = packs[tt]
failure_ids = {c.question_id for c in pack.failure_cases}
assert "q-none" not in failure_ids
assert "q-deg" not in failure_ids
# None-cause 的 lapse_note 应被收进 lapse_notes
assert any("复核该类推理规则" in n for n in pack.lapse_notes)
+2 -4
View File
@@ -344,12 +344,11 @@ def test_quadrant_classification_frozen():
def test_diagnose_prompts_frozen():
"""DiagnosePrompts 是 frozen dataclass8 个模板字段。"""
"""DiagnosePrompts 是 frozen dataclass7 个模板字段。"""
dp = DiagnosePrompts(
defect_vs_lapse="p1",
reasoning_sub="p2",
span_eval_system="p3",
span_eval_user="p4",
missed_nodes="p5",
skill_adherence="p6",
confirmation_bias="p7",
@@ -361,13 +360,12 @@ def test_diagnose_prompts_frozen():
def test_evolve_prompts_frozen():
"""EvolvePrompts 是 frozen dataclass5 个模板字段。"""
"""EvolvePrompts 是 frozen dataclass4 个模板字段。"""
ep = EvolvePrompts(
evolve_skill="skill_tmpl",
evolve_system="system_tmpl",
evolve_tool="tool_tmpl",
evolve_rank="rank_tmpl",
consolidate_system="consolidate_tmpl",
)
assert ep.evolve_rank == "rank_tmpl"
with pytest.raises(AttributeError):
+30 -1
View File
@@ -309,6 +309,36 @@ class TestValidateSkill:
result = validate_skill(orig, "no frontmatter body")
assert not result.passed
def test_unpaired_appendix_marker_fails(self) -> None:
"""evolved 出现孤立 APPENDIX_START(无配对 END)→ marker 完整性校验失败。"""
orig = "---\nname: a\ndescription: d\ntask_type: t\n---\nbody body body"
evol = f"---\nname: a\ndescription: d\ntask_type: t\n---\nbody {APPENDIX_START} body"
result = validate_skill(orig, evol)
assert not result.passed
assert any("marker" in e.lower() or "配对" in e for e in result.errors)
def test_duplicate_momentum_marker_fails(self) -> None:
"""evolved 出现两对 MOMENTUM marker → 完整性校验失败(各至多一对)。"""
orig = "---\nname: a\ndescription: d\ntask_type: t\n---\nbody body body"
evol = (
"---\nname: a\ndescription: d\ntask_type: t\n---\n"
f"{MOMENTUM_START}x{MOMENTUM_END} mid {MOMENTUM_START}y{MOMENTUM_END}"
)
result = validate_skill(orig, evol)
assert not result.passed
assert any("marker" in e.lower() or "配对" in e for e in result.errors)
def test_paired_markers_pass(self) -> None:
"""evolved 含成对 appendix+momentum marker(各一对,顺序正确)→ 校验通过。"""
body = "body " * 20
orig = f"---\nname: a\ndescription: d\ntask_type: t\n---\n{body}"
evol = (
f"---\nname: a\ndescription: d\ntask_type: t\n---\n{body}"
f"{APPENDIX_START}\n- n\n{APPENDIX_END}\n{MOMENTUM_START}\nm\n{MOMENTUM_END}"
)
result = validate_skill(orig, evol)
assert result.passed, result.errors
class TestValidateSystem:
"""validate_system 测试。"""
@@ -681,7 +711,6 @@ _PROMPTS = EvolvePrompts(
evolve_system="sys",
evolve_tool="tool",
evolve_rank="rank",
consolidate_system="cons",
)
+54
View File
@@ -0,0 +1,54 @@
"""校验 5 个进化/动量模板存在且输出契约关键词与解析代码对齐。"""
from pathlib import Path
import pytest
_PROMPTS_DIR = Path("prompts")
@pytest.mark.parametrize(
"name, required_tokens",
[
("evolve_skill.md", ["suggestions", "edits"]),
("evolve_system.md", ["suggestions", "edits"]),
("evolve_tool.md", ["edits_extract", "edits_verify"]),
("evolve_rank.md", ["selected_indices"]),
("slow_momentum.md", ["slow_update_content"]),
],
)
def test_evolve_template_present_and_contract(name: str, required_tokens: list[str]) -> None:
path = _PROMPTS_DIR / name
assert path.exists(), f"缺模板: {path}"
text = path.read_text(encoding="utf-8")
assert text.strip(), f"模板为空: {path}"
for token in required_tokens:
assert token in text, f"{name} 缺输出契约关键词 {token!r}(与解析代码不对齐)"
def test_video_split_loader_fail_loud_on_missing(tmp_path, monkeypatch):
"""video_split_cli 的真实 diagnose 加载器缺模板必须 FileNotFoundError。"""
monkeypatch.chdir(tmp_path) # 空目录,无 prompts/*.md
from app.harness.video_split_cli import _load_diagnose_prompts
with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"):
_load_diagnose_prompts()
def test_runner_evolve_loader_fail_loud_on_missing(tmp_path, monkeypatch):
"""runner 的真实 evolve 加载器缺模板必须 FileNotFoundError。"""
monkeypatch.chdir(tmp_path)
from app.harness.runner import Runner
r = object.__new__(Runner) # 绕过 __init__,仅测无状态加载器方法
with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"):
r._load_evolve_prompts()
def test_runner_diagnose_loader_fail_loud_on_missing(tmp_path, monkeypatch):
"""runner 的真实 diagnose 加载器缺模板必须 FileNotFoundError。"""
monkeypatch.chdir(tmp_path)
from app.harness.runner import Runner
r = object.__new__(Runner)
with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"):
r._load_diagnose_prompts()
+46 -3
View File
@@ -23,8 +23,13 @@ class _FakeRedisCache:
def __init__(self) -> None:
self._store: dict[str, LLMResponse] = {}
async def get(self, model: str, messages: list[dict[str, str]]) -> LLMResponse | None:
key = f"{model}:{json.dumps(messages, sort_keys=True)}"
async def get(
self,
model: str,
messages: list[dict[str, str]],
cache_salt: str | None = None,
) -> LLMResponse | None:
key = f"{model}:{cache_salt}:{json.dumps(messages, sort_keys=True)}"
return self._store.get(key)
async def set(
@@ -32,8 +37,9 @@ class _FakeRedisCache:
model: str,
messages: list[dict[str, str]],
response: LLMResponse,
cache_salt: str | None = None,
) -> None:
key = f"{model}:{json.dumps(messages, sort_keys=True)}"
key = f"{model}:{cache_salt}:{json.dumps(messages, sort_keys=True)}"
self._store[key] = response
@@ -253,6 +259,43 @@ async def test_qwen_thinking_stripped():
assert thinking2 == ""
def test_transient_covers_disconnect_family():
"""瞬时错误清单覆盖断连族(RemoteProtocolError/ReadError/ConnectTimeout/PoolTimeout)。"""
from adapters.llm import _is_transient_error
assert _is_transient_error(httpx.RemoteProtocolError("peer reset"))
assert _is_transient_error(httpx.ReadError("read"))
assert _is_transient_error(httpx.ConnectTimeout("ct"))
assert _is_transient_error(httpx.PoolTimeout("pt"))
@pytest.mark.asyncio
async def test_truncated_stream_without_done_raises():
"""SSE 流耗尽但未收 [DONE] → _SseAnomaly(进重试,不当成功)。"""
from adapters.llm import _SseAnomaly
async def _lines():
yield 'data: {"choices":[{"delta":{"content":""}}]}'
# 无 data: [DONE] —— 模拟服务端截断
client = _build_client()
with pytest.raises(_SseAnomaly):
await client._consume_stream(_lines())
@pytest.mark.asyncio
async def test_complete_stream_with_done_ok():
"""正常带 [DONE] 的流不抛异常,正确累积 content。"""
async def _lines():
yield 'data: {"choices":[{"delta":{"content":"完整"}}]}'
yield "data: [DONE]"
client = _build_client()
content, _thinking, _ttft, _gap, _usage = await client._consume_stream(_lines())
assert content == "完整"
@pytest.mark.asyncio
async def test_parent_call_id_forwarded_to_telemetry():
"""parent_call_id 和 session_id 正确传递到遥测记录。"""
+5 -3
View File
@@ -116,7 +116,7 @@ class _FakeState:
eval_prev_run_id: str
baseline_skills_version: str
baseline_prompts_version: str
steps_since_best_improved: int
epochs_since_best_improved: int
epoch_start_skills: str
changed_task_types_this_epoch: set[str]
rejected_buffer: dict[str, list[RejectedEdit]]
@@ -135,7 +135,7 @@ def _make_state() -> _FakeState:
eval_prev_run_id="run-abc",
baseline_skills_version="v1",
baseline_prompts_version="v1",
steps_since_best_improved=2,
epochs_since_best_improved=2,
epoch_start_skills="v1",
changed_task_types_this_epoch={"temporal", "causal"},
rejected_buffer={"temporal": [_make_rejected_edit()]},
@@ -157,6 +157,7 @@ class _FakeConfig:
diag_size: int = 30
val_size: int = 50
batch_correct_ratio: float = 0.5
trainable_min_units: int = 8
edit_budget_start: int = 6
edit_budget_end: int = 3
early_stop_patience: int = 3
@@ -201,7 +202,7 @@ class TestSerializeDeserializeRoundtrip:
assert restored["eval_prev_run_id"] == state.eval_prev_run_id
assert restored["baseline_skills_version"] == state.baseline_skills_version
assert restored["baseline_prompts_version"] == state.baseline_prompts_version
assert restored["steps_since_best_improved"] == state.steps_since_best_improved
assert restored["epochs_since_best_improved"] == state.epochs_since_best_improved
assert restored["epoch_start_skills"] == state.epoch_start_skills
assert restored["changed_task_types_this_epoch"] == state.changed_task_types_this_epoch
assert restored["gate_cooldown"] == state.gate_cooldown
@@ -289,6 +290,7 @@ class TestFingerprintStructuralVsDecision:
"diag_size",
"val_size",
"batch_correct_ratio",
"trainable_min_units",
}
decision = {
"edit_budget_start",
+1
View File
@@ -40,6 +40,7 @@ def _valid_kwargs() -> dict:
"batch_size": 15,
"min_class_per_batch": 2,
"eval_min_per_class": 2,
"trainable_min_units": 8,
"early_stop_patience": 8,
"test_size": 60,
"use_slow_momentum": True,
+60
View File
@@ -559,6 +559,65 @@ class TestPredictionAlwaysWritten:
assert rows[0]["prediction"] is None
def _make_nonscalar_llm_response() -> LLMResponse:
"""构造 submit_answer 提交非标量 answerlist)的 LLMResponse。"""
content = json.dumps(
{
"reflect": {"observation": "找到答案"},
"plan": {"next_step": "提交"},
"action": {
"tool": "submit_answer",
"args": {
"answer": ["B"],
"evidence": "证据文本",
"reasoning": "推理过程",
},
},
}
)
return LLMResponse(
content=content,
thinking="思考过程",
model="test-model",
provider="test",
prompt_tokens=100,
completion_tokens=50,
latency_ms=200,
ttft_ms=30.0,
max_inter_token_ms=5.0,
cache_hit=False,
call_id="test-call-nonscalar",
)
class TestNonScalarPrediction:
"""非标量 prediction 归一化 + 落库加固测试。"""
@pytest.mark.asyncio
async def test_nonscalar_prediction_does_not_crash(self, harness_log: HarnessLog) -> None:
"""submit_answer 返回 {'answer': ['B']} 时归一化落库,不抛 sqlite 绑定异常。"""
llm = AsyncMock()
llm.chat.return_value = _make_nonscalar_llm_response()
result = await run_inference(
[_make_question(answer="B")],
llm=llm,
tool_dispatch_fn=_stub_tool_dispatch,
prompt_builder=_stub_prompt_builder,
log=harness_log,
run_id="run-nonscalar",
concurrency=1,
max_steps=10,
skill_mode="auto",
)
assert result.total == 1
rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("test-run",))
assert len(rows) == 1
# prediction 被 JSON 序列化为字符串,不再是 Python list
assert rows[0]["prediction"] == '["B"]'
class TestPluginsFactory:
"""plugins_factory 调用测试。"""
@@ -639,6 +698,7 @@ class TestConcurrencyControl:
*,
session_id: str | None = None,
parent_call_id: str | None = None,
cache_salt: str | None = None,
) -> LLMResponse:
nonlocal current_concurrent, max_concurrent
current_concurrent += 1
+24
View File
@@ -215,6 +215,30 @@ class TestHarnessLogUpsert:
rows = log3.query("SELECT COUNT(*) as cnt FROM _runs WHERE run_id='run_1'")
assert rows[0]["cnt"] == 1
def test_register_run_false_skips_upsert(self, tmp_path: Path) -> None:
"""register_run=False 时只读打开不改写已有 _runs 行(started_at/status 不变)。"""
db = str(tmp_path / "h.db")
def _read_run_row(run_id: str) -> dict:
# 用 register_run=False 只读,避免读取本身污染 _runs
with HarnessLog(db, run_id, register_run=False) as log:
rows = log.query(
"SELECT started_at, status FROM _runs WHERE run_id=?", (run_id,)
)
return rows[0]
with HarnessLog(db, "r1"):
pass # 初次注册 + 正常退出置 completed
row0 = _read_run_row("r1")
time.sleep(0.02)
with HarnessLog(db, "r1", register_run=False) as log:
log.query("SELECT 1") # 只读
row1 = _read_run_row("r1")
assert row1["started_at"] == row0["started_at"]
assert row1["status"] == row0["status"]
# ===========================================================================
# RunLogImpl 测试
+31
View File
@@ -71,6 +71,37 @@ def test_write_read_dual_metric(db_path: str, run_id: str) -> None:
assert row["run_id"] == run_id
def test_dual_metric_kinds_distinguishable(db_path: str, run_id: str) -> None:
"""observation 层接受任意 version_kindfinal 与 slow_candidate 可区分共存。"""
write_dual_metric(
db_path,
run_id=run_id,
epoch=1,
version_kind="final",
skills_version="v1",
prompts_version="v1",
pool="val",
hard_acc=0.7,
soft_score=None,
mixed_score=None,
)
write_dual_metric(
db_path,
run_id=run_id,
epoch=1,
version_kind="slow_candidate",
skills_version="v2",
prompts_version="v2",
pool="val",
hard_acc=0.6,
soft_score=None,
mixed_score=None,
)
rows = read_dual_metric(db_path, run_id=run_id)
kinds = {row["version_kind"] for row in rows if row["epoch"] == 1}
assert kinds == {"final", "slow_candidate"}
# ---------------------------------------------------------------------------
# shadow_gate
# ---------------------------------------------------------------------------
+74
View File
@@ -18,6 +18,7 @@ import pytest
from app.harness.pools import (
GlobalPoolStrategy,
PerCategoryPoolStrategy,
build_or_load_pools,
build_pools,
load_pools,
save_pools,
@@ -291,6 +292,77 @@ class TestBuildOrLoadPoolsFrozen:
load_ids = [q.question_id for q in getattr(loaded, pool_name)]
assert orig_ids == load_ids, f"{pool_name} 冻结后 ID 顺序不一致"
def _run_config_for_frozen(self, tmp_path: Path, seed_name: str) -> object:
"""构造指向 tmp workspace/store + 指定种子名的最小 train RunConfig。"""
from app.harness.config import RunConfig
return RunConfig(
workspace_dir=tmp_path / "ws",
store_dir=tmp_path / "store",
mode="train",
concurrency=4,
max_steps=10,
skill_mode="auto",
n_samples=0,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
epochs=1,
diag_size=10,
diag_correct_ratio=0.5,
val_size=10,
val_correct_ratio=0.5,
edit_budget_start=5,
edit_budget_end=2,
batch_size=15,
min_class_per_batch=2,
eval_min_per_class=2,
trainable_min_units=8,
early_stop_patience=4,
test_size=10,
use_slow_momentum=True,
gate_e_confirm=20.0,
gate_e_provisional=3.0,
gate_w_net_min=2,
gate_delta_min=0.02,
gate_lambda_dir=-0.642,
gate_e_rollback=10.0,
gate_block=8,
gate_n_max=40,
gate_p_low=0.05,
gate_p_high=0.95,
gate_probe_quota=0.2,
gate_gamma_decay=0.9,
gate_cooldown_steps=2,
gate_guard_err=0.10,
skill_update_mode="patch",
appendix_consolidate_threshold=6,
fresh=True,
seed=seed_name,
test_questions="", # 绕过 _to_pool_config 的 resolve_paths(manifest 依赖)
)
def test_global_frozen_rejects_baseline_mismatch(self, tmp_path: Path) -> None:
"""global 冻结 pools 的 baseline_run_id 与 seed 不符时 fail-loud。"""
seed_name = "myseed"
seed_dir = tmp_path / "store" / "seeds" / seed_name
seed_dir.mkdir(parents=True)
(seed_dir / "seed.json").write_text(
json.dumps({"baseline_run_id": "infer_adhoc", "parent": None})
)
ws = tmp_path / "ws"
ws.mkdir()
(ws / "pools.json").write_text(
json.dumps({"split_mode": "global", "baseline_run_id": "other"}),
encoding="utf-8",
)
config = self._run_config_for_frozen(tmp_path, seed_name)
strategy = GlobalPoolStrategy()
with pytest.raises(ValueError, match="baseline_run_id"):
build_or_load_pools(config, strategy, tmp_path / "nonexistent.db")
class TestGlobalPoolStrategy:
"""GlobalPoolStrategy 封装现有全局三分逻辑。"""
@@ -799,6 +871,7 @@ class TestRunHoldoutEvalConfig:
batch_size=15,
min_class_per_batch=2,
eval_min_per_class=2,
trainable_min_units=8,
early_stop_patience=4,
test_size=30,
use_slow_momentum=True,
@@ -849,6 +922,7 @@ class TestRunHoldoutEvalConfig:
batch_size=15,
min_class_per_batch=2,
eval_min_per_class=2,
trainable_min_units=8,
early_stop_patience=4,
test_size=30,
use_slow_momentum=True,
+174 -69
View File
@@ -20,6 +20,7 @@ from app.harness.runner import (
_build_comparison_pairs,
_compute_total_steps,
_fallback_summary,
_filter_untrainable_types,
_format_applied_edits,
_guard_infra_failures,
_outcome_to_quadrant_pairs,
@@ -74,6 +75,26 @@ class _FakeQuestion:
object.__setattr__(self, "unit_id", self.pair_id or self.question_id)
def _fake_pair(pair_id: str, task_type: str, video_id: str = "v1") -> list[_FakeQuestion]:
"""构造合法孪生对(original + mirror),共享 pair_id/video_id/task_type/flip_axis。"""
return [
_FakeQuestion(
question_id=f"{pair_id}-o",
video_id=video_id,
task_type=task_type,
pair_id=pair_id,
question_role="pair_original",
),
_FakeQuestion(
question_id=f"{pair_id}-m",
video_id=video_id,
task_type=task_type,
pair_id=pair_id,
question_role="pair_mirror",
),
]
@dataclass
class _FakePools:
"""Pools 替身。"""
@@ -349,92 +370,175 @@ class TestBuildComparisonPairs:
# =========================================================================
def _write_manifest_with_best(tmp_path: Path, best_epoch: int) -> None:
"""写含 best.epoch 的 manifest,供 _should_early_stop 读 read_best。"""
manifest = {
"name": "test",
"store": ".",
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
"best": {"epoch": best_epoch, "val_acc": 0.5},
"history": [],
}
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
class TestShouldEarlyStop:
"""_should_early_stop 粒度 early stop。"""
"""_should_early_stop epoch 粒度 early stoppatience 以 epoch 计)"""
def test_improved_this_epoch_resets(self, tmp_path: Path) -> None:
"""本 epoch best 刷新时重置计数器。"""
# 写 manifest + best
manifest = {
"name": "test",
"store": ".",
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
"best": {"epoch": 2, "val_acc": 0.9},
"history": [],
}
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
_write_manifest_with_best(tmp_path, best_epoch=2)
state = MagicMock()
state.steps_since_best_improved = 10
state.epochs_since_best_improved = 3
result = _should_early_stop(tmp_path, epoch=2, steps_this_epoch=5, state=state, patience=20)
result = _should_early_stop(tmp_path, epoch=2, state=state, patience=2)
assert result is False
assert state.steps_since_best_improved == 0
def test_no_improvement_accumulates(self, tmp_path: Path) -> None:
"""未刷新时累加步数。"""
manifest = {
"name": "test",
"store": ".",
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
"best": {"epoch": 1, "val_acc": 0.5},
"history": [],
}
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
state = MagicMock()
state.steps_since_best_improved = 15
result = _should_early_stop(tmp_path, epoch=3, steps_this_epoch=5, state=state, patience=20)
assert result is True # 15 + 5 = 20 >= 20
assert state.steps_since_best_improved == 20
assert state.epochs_since_best_improved == 0
def test_below_patience_continues(self, tmp_path: Path) -> None:
"""累计步数未达阈值时继续。"""
manifest = {
"name": "test",
"store": ".",
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
"best": {"epoch": 1, "val_acc": 0.5},
"history": [],
}
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
"""未达 patience 个 epoch 无刷新时继续。"""
_write_manifest_with_best(tmp_path, best_epoch=1)
state = MagicMock()
state.steps_since_best_improved = 10
state.epochs_since_best_improved = 0
result = _should_early_stop(tmp_path, epoch=3, steps_this_epoch=5, state=state, patience=20)
result = _should_early_stop(tmp_path, epoch=2, state=state, patience=3)
assert result is False
assert state.steps_since_best_improved == 15
assert state.epochs_since_best_improved == 1
def test_step_granularity(self, tmp_path: Path) -> None:
"""步粒度而非 epoch 粒度"""
manifest = {
"name": "test",
"store": ".",
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
"best": {"epoch": 1, "val_acc": 0.5},
"history": [],
}
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
def test_early_stop_counts_epochs_not_steps(self, tmp_path: Path) -> None:
"""patience=2 表示连续 2 个 epoch 无 best 刷新才停(不是步数)"""
_write_manifest_with_best(tmp_path, best_epoch=1)
state = MagicMock()
# 连续 3 个 epoch,每个 3 步
state.steps_since_best_improved = 0
for ep in range(2, 5):
stopped = _should_early_stop(
tmp_path, epoch=ep, steps_this_epoch=3, state=state, patience=10
)
if ep < 4:
assert stopped is False
else:
# 3+3+3=9 < 10 但第三轮后 9+3=12>=10 在 ep=5 触发
# 实际:ep=2 → 3, ep=3 → 6, ep=4 → 9
assert stopped is False
stopped = _should_early_stop(
tmp_path, epoch=5, steps_this_epoch=3, state=state, patience=10
state.epochs_since_best_improved = 0
# epoch 2 无刷新 → 1 → 不停
assert _should_early_stop(tmp_path, epoch=2, state=state, patience=2) is False
assert state.epochs_since_best_improved == 1
# epoch 3 无刷新 → 2 → 停
assert _should_early_stop(tmp_path, epoch=3, state=state, patience=2) is True
assert state.epochs_since_best_improved == 2
class TestFilterUntrainableTypes:
"""_filter_untrainable_types 可训练性预检纯函数。"""
def test_untrainable_types_filtered_before_gate(self) -> None:
"""val<eval_min_per_class 或 非test单元<trainable_min_units 的题型被剔除。"""
# 题型 Adiag=5 + val=5 → units=10、val=5,可训
diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(5)]
val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(5)]
# 题型 Bval=0<eval_min_per_class)不可训
diag += [_FakeQuestion(question_id=f"B-d{i}", task_type="B") for i in range(2)]
pools = _FakePools(diagnosis=diag, validation=val, test=[])
new_pools, new_types = _filter_untrainable_types(
pools,
task_types=["A", "B"],
eval_min_per_class=2,
trainable_min_units=8,
)
assert stopped is True # 9+3=12>=10
assert {q.task_type for q in new_pools.diagnosis} == {"A"}
assert {q.task_type for q in new_pools.validation} == {"A"}
assert new_types == ["A"]
def test_units_below_threshold_filtered(self) -> None:
"""val 达标但 diag+val 单元数 < trainable_min_units 的题型被剔除(保留另一可训题型)。"""
# 可训题型 Adiag=8 + val=2 → units=10、val=2,保留
keep_diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)]
keep_val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(2)]
# 题型 Cval=2>=2)但 units=2+1=3 < 8 → 剔除
val = keep_val + [_FakeQuestion(question_id=f"C-v{i}", task_type="C") for i in range(2)]
diag = keep_diag + [_FakeQuestion(question_id="C-d0", task_type="C")]
pools = _FakePools(diagnosis=diag, validation=val, test=[])
new_pools, new_types = _filter_untrainable_types(
pools, task_types=None, eval_min_per_class=2, trainable_min_units=8
)
assert {q.task_type for q in new_pools.diagnosis} == {"A"}
assert {q.task_type for q in new_pools.validation} == {"A"}
assert new_types == ["A"]
def test_ar_pair_counted_as_units_not_questions(self) -> None:
"""AR pair 按单元折叠计数:题目数达标但单元数不足的题型仍被剔除。"""
# 可训题型 Adiag=8 + val=2 single → units=10,保留
keep_diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)]
keep_val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(2)]
# 题型 Pdiag 3 对(6 题=3 单元)+ val 2 对(4 题=2 单元)→ 单元数=5<8,
# 但题目数=10>=8。按单元计数须剔除(按题目计数会误通过)。
pair_diag: list[_FakeQuestion] = []
for i in range(3):
pair_diag.extend(_fake_pair(f"P-d{i}", "P"))
pair_val: list[_FakeQuestion] = []
for i in range(2):
pair_val.extend(_fake_pair(f"P-v{i}", "P"))
pools = _FakePools(
diagnosis=keep_diag + pair_diag,
validation=keep_val + pair_val,
test=[],
)
new_pools, new_types = _filter_untrainable_types(
pools, task_types=None, eval_min_per_class=2, trainable_min_units=8
)
assert "P" not in {q.task_type for q in new_pools.diagnosis}
assert "P" not in new_types
assert "A" in new_types
def test_task_types_subset_drops_non_requested_trainable(self) -> None:
"""指定 task_types 子集时:非请求但可训练的题型也被剔除出 diag/val。
回归 I-4pools 过滤此前只按可训练性 keep不按 task_types 收窄导致
冻结全局 pools batch/diagnosis 会训练非请求题型gate 只覆盖请求题型
静默语义偏差此处 AB 均可训仅请求 AB 必须被剔除
"""
# 题型 Adiag=8 + val=2 → units=10、val=2,可训
diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)]
val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(2)]
# 题型 Bdiag=8 + val=2 → units=10、val=2,同样可训(但未被请求)
diag += [_FakeQuestion(question_id=f"B-d{i}", task_type="B") for i in range(8)]
val += [_FakeQuestion(question_id=f"B-v{i}", task_type="B") for i in range(2)]
pools = _FakePools(diagnosis=diag, validation=val, test=[])
new_pools, new_types = _filter_untrainable_types(
pools,
task_types=["A"],
eval_min_per_class=2,
trainable_min_units=8,
)
# 非请求题型 B(可训)被剔除出 diag/val
assert {q.task_type for q in new_pools.diagnosis} == {"A"}
assert {q.task_type for q in new_pools.validation} == {"A"}
assert new_types == ["A"]
def test_all_filtered_raises(self) -> None:
"""所有题型都被剔除时 fail-fastraise RuntimeError 并列出剔除原因。"""
diag = [_FakeQuestion(question_id="B-d0", task_type="B")] # val=0 < 2
pools = _FakePools(diagnosis=diag, validation=[], test=[])
with pytest.raises(RuntimeError, match="可训练"):
_filter_untrainable_types(
pools, task_types=None, eval_min_per_class=2, trainable_min_units=8
)
def test_test_pool_untouched(self) -> None:
"""test 池不参与过滤(继续报告全题型准确率)。"""
val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(8)]
diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)]
test = [_FakeQuestion(question_id="B-t0", task_type="B")]
pools = _FakePools(diagnosis=diag, validation=val, test=test)
new_pools, _ = _filter_untrainable_types(
pools, task_types=None, eval_min_per_class=2, trainable_min_units=8
)
assert new_pools.test == test
# =========================================================================
@@ -667,7 +771,7 @@ class TestTrainState:
assert state.system_packs == []
assert state.tool_packs == []
assert state.changed_task_types_this_epoch == set()
assert state.steps_since_best_improved == 0
assert state.epochs_since_best_improved == 0
# =========================================================================
@@ -726,6 +830,7 @@ class TestRunnerFactoryInjection:
"batch_size": 5,
"min_class_per_batch": 2,
"eval_min_per_class": 2,
"trainable_min_units": 8,
"early_stop_patience": 3,
"test_size": 10,
"use_slow_momentum": False,
+56
View File
@@ -243,6 +243,24 @@ class TestInitSeed:
with pytest.raises(FileExistsError, match="种子已存在"):
init_seed(store, "dup", skills_dir, prompts_dir, baseline_db, "r1", None, "second")
def test_init_seed_carries_pools(self, tmp_path):
"""提供 pools_json/split_manifest 时拷入 seed 目录。"""
from app.harness.store import init_seed
store = tmp_path / "store"
skills = tmp_path / "sk"; skills.mkdir(); (skills / "s.md").write_text("x")
prompts = tmp_path / "pr"; prompts.mkdir(); (prompts / "p.md").write_text("y")
db = tmp_path / "b.db"; db.write_text("db")
pools = tmp_path / "pools.json"; pools.write_text('{"split_mode":"global"}')
manifest = tmp_path / "split_manifest.json"; manifest.write_text('{"pools_sha256":"a"}')
seed_dir = init_seed(
store, "s1", skills, prompts, db, "infer_adhoc", None, "d",
pools_json=pools, split_manifest=manifest,
)
assert (seed_dir / "pools.json").exists()
assert (seed_dir / "split_manifest.json").exists()
class TestListSeeds:
"""list_seeds 列出所有种子。"""
@@ -338,6 +356,44 @@ class TestExtractRunDb:
with pytest.raises(RuntimeError, match="无 run_id="):
extract_run_db(src, dst, "nonexistent")
def test_dedupe_per_question_keeps_first_row(self, tmp_path):
"""dedupe_per_question=True 时每 question_id 只保留 rowid 最小的首行。"""
import sqlite3
src = tmp_path / "src.db"
conn = sqlite3.connect(src)
conn.execute(
"CREATE TABLE _runs (run_id TEXT PRIMARY KEY, started_at TEXT)"
)
conn.execute("INSERT INTO _runs VALUES ('r1', 't0')")
conn.execute(
"CREATE TABLE predictions (run_id TEXT, question_id TEXT, prediction TEXT)"
)
# 743-1 三行(模拟 error/budget/finished),首行 prediction=NULL
conn.executemany(
"INSERT INTO predictions VALUES (?,?,?)",
[
("r1", "743-1", None),
("r1", "743-1", None),
("r1", "743-1", "C"),
("r1", "q2", "A"),
],
)
conn.commit()
conn.close()
dst = tmp_path / "dst.db"
from app.harness.store import extract_run_db
extract_run_db(src, dst, "r1", dedupe_per_question=True)
out = sqlite3.connect(dst)
rows = out.execute(
"SELECT question_id, prediction FROM predictions ORDER BY question_id"
).fetchall()
out.close()
assert rows == [("743-1", None), ("q2", "A")], f"未按 rowid 首行去重: {rows}"
# ---------------------------------------------------------------------------
# promote_to_seed
+331 -14
View File
@@ -112,7 +112,6 @@ def _make_mock_run_inference(
log: HarnessLog,
baseline_correctness: dict[str, bool],
candidate_correctness: dict[str, bool],
error_count: int = 0,
):
"""构建 mock RunInferenceFn。
@@ -136,9 +135,6 @@ def _make_mock_run_inference(
correct = sum(per_q.values())
total = len(questions)
stop_counts: dict[str, int] = {"completed": total - error_count}
if error_count > 0:
stop_counts["error"] = error_count
return InferenceResult(
run_id=run_id,
accuracy=correct / total if total else 0.0,
@@ -147,12 +143,71 @@ def _make_mock_run_inference(
per_task_type={},
steps_mean=1.0,
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
stop_reason_counts=stop_counts,
stop_reason_counts={"completed": total},
)
return mock_fn, call_log
def _make_all_infra_mock(log: HarnessLog, stop_reason: str):
"""构建基线全 INFRA 的 mock:每 record 写指定 INFRA stop_reasonerror/parse_error)。
与真实推理一致per-record DB stop_reason 与汇总 stop_reason_counts 同源护栏
分子按 unit DB _infra_question_ids_from_db故须真实落 DBtotal 返回
unit 粒度single == 题数使护栏分子/分母同粒度
"""
call_log: list[dict[str, Any]] = []
async def mock_fn(
questions: list[GeneratedQuestion],
*,
run_id: str,
skills_dir: Path,
) -> InferenceResult:
call_log.append({"run_id": run_id, "n": len(questions)})
for q in questions:
log.insert(
"predictions",
{
"run_id": run_id,
"video_id": "v0",
"question_id": q.question_id,
"task_type": "temporal",
"prediction": "",
"answer": "A",
"evidence": "",
"reasoning": "",
"steps_used": 1,
"prompt_tokens": 10,
"completion_tokens": 10,
"stop_reason": stop_reason,
"steps_json": "[]",
},
)
total = len(questions) # 全 single → unit 数 == 题数
return InferenceResult(
run_id=run_id,
accuracy=0.0,
total=total,
correct=0,
per_task_type={},
steps_mean=1.0,
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
stop_reason_counts={stop_reason: total},
)
return mock_fn, call_log
def test_infra_stop_reasons_single_source() -> None:
"""app 侧 INFRA_STOP_REASONS 复用 core 常量(同一对象),杜绝未来漂移(M-2)。"""
from app.harness import validate
from core.evolution import diagnose
assert validate.INFRA_STOP_REASONS is diagnose.INFRA_STOP_REASONS
assert frozenset({"error", "parse_error"}) == diagnose.INFRA_STOP_REASONS
# ===========================================================================
# 数据类型测试
# ===========================================================================
@@ -421,17 +476,14 @@ async def test_gate_prefix_must_contain_gate(tmp_path: Path) -> None:
@pytest.mark.asyncio
async def test_infra_guard_threshold(tmp_path: Path) -> None:
"""推理错误率超阈值时抛 RuntimeError。"""
"""推理错误率超阈值时抛 RuntimeError(护栏分子/分母 unit 同粒度)"""
workspace = _setup_workspace(tmp_path)
log = _make_log(workspace)
# 需要 >=10 题次才触发 INFRA 护栏
questions = _make_questions(6)
# 需要 >=10 unit 分母才触发护栏:12 个 single,基线全 INFRA error。
# 首块全 INFRA → valid_chunk 空 → errors=12/denom=12=1.0>0.5 触发护栏。
questions = _make_questions(12)
cache = BaselineCache(workspace / "baseline_cache.json")
baseline_correct = {f"q{i}": False for i in range(6)}
candidate_correct = {f"q{i}": False for i in range(6)}
# 每次 run_inference 报 error_count=5,两侧各 5 → 10/12 > 0.5
mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct, error_count=5)
mock_fn, _ = _make_all_infra_mock(log, "error")
try:
with pytest.raises(RuntimeError, match="错误率过高"):
@@ -444,7 +496,7 @@ async def test_infra_guard_threshold(tmp_path: Path) -> None:
base_skill_content="baseline skill content",
ladder_items=questions,
gate_params=_DEFAULT_GATE_PARAMS,
gate_block=6,
gate_block=12,
gate_n_max=20,
gate_guard_err=0.5,
baseline_cache=cache,
@@ -505,6 +557,271 @@ async def test_baseline_cache_hit(tmp_path: Path) -> None:
log.close()
@pytest.mark.asyncio
async def test_baseline_infra_error_not_cached(tmp_path: Path) -> None:
"""基线臂 INFRA error 的 unit 不写入 BaselineCache(不永久污染),且从有效单元排除。"""
from app.harness.gate_ladder import skill_hash
from app.harness.question_units import build_units
from app.harness.validate import _resolve_baseline_block
workspace = _setup_workspace(tmp_path)
log = _make_log(workspace)
questions = _make_questions(2) # q0 干净, q1 INFRA error
units = build_units(questions)
cache = BaselineCache(workspace / "baseline_cache.json")
s_hash = skill_hash("baseline skill content")
async def mock_fn(qs, *, run_id, skills_dir):
for q in qs:
is_err = q.question_id == "q1"
log.insert(
"predictions",
{
"run_id": run_id,
"video_id": "v0",
"question_id": q.question_id,
"task_type": "temporal",
"prediction": "" if is_err else "A",
"answer": "A",
"evidence": "",
"reasoning": "",
"steps_used": 1,
"prompt_tokens": 10,
"completion_tokens": 10,
"stop_reason": "error" if is_err else "completed",
"steps_json": "[]",
},
)
return InferenceResult(
run_id=run_id,
accuracy=0.5,
total=2,
correct=1,
per_task_type={},
steps_mean=1.0,
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
stop_reason_counts={"completed": 1, "error": 1},
)
try:
b_units, valid_units, _errors_inc, _denom_inc = await _resolve_baseline_block(
units=units,
task_type="temporal",
s_hash=s_hash,
prompts_version="p1",
baseline_cache=cache,
base_skills_dir=workspace / "skills" / "v1",
run_inference=mock_fn,
log=log,
run_id="step1_gate_b0_base",
)
# q1 是 INFRA:不写缓存、不入 b_units、不在有效单元里
assert cache.get("temporal", s_hash, "p1", "q1") is None
assert "q1" not in b_units
assert all(u.unit_id != "q1" for u in valid_units)
# q0 干净:正常缓存并入 b_units/valid_units
assert cache.get("temporal", s_hash, "p1", "q0") is True
assert b_units["q0"] is True
assert any(u.unit_id == "q0" for u in valid_units)
finally:
log.close()
@pytest.mark.asyncio
async def test_infra_guard_counts_units_not_records(tmp_path: Path) -> None:
"""护栏分子按 unit 计:AR pair 两 record 全 INFRA 只计 1 个 INFRA unit(而非 2)。
回归 I-3分子此前用 stop_reason_counts record 计数分母 denom_inc=r.total
unit 粒度AR pair unit record致分子被放大误触发 gate_guard_err
分子改为"含 INFRA record 的 unit 数"后与分母同粒度核心算法保真 #5/#6)。
"""
from app.harness.gate_ladder import skill_hash
from app.harness.question_units import build_units
from app.harness.validate import _resolve_baseline_block
workspace = _setup_workspace(tmp_path)
log = _make_log(workspace)
# 一个 AR pair(两成员共享 pair_id)→ build_units 折叠为 1 个 pair unit
common = {
"video_id": "vp",
"task_type": "temporal",
"question": "Q?",
"options": ("A", "B", "C", "D"),
"answer": "A",
"source_nodes": (),
"difficulty": "easy",
"pair_id": "p1",
"flip_axis": "before_after",
}
pair = [
GeneratedQuestion(question_id="p1_o", question_role="pair_original", **common),
GeneratedQuestion(question_id="p1_m", question_role="pair_mirror", **common),
]
units = build_units(pair)
assert len(units) == 1 # 前置:pair 折叠为 1 个 unit
cache = BaselineCache(workspace / "baseline_cache.json")
s_hash = skill_hash("baseline skill content")
async def mock_fn(qs, *, run_id, skills_dir):
# 两 record 皆 INFRA error
for q in qs:
log.insert(
"predictions",
{
"run_id": run_id,
"video_id": "vp",
"question_id": q.question_id,
"task_type": "temporal",
"prediction": "",
"answer": "A",
"evidence": "",
"reasoning": "",
"steps_used": 1,
"prompt_tokens": 10,
"completion_tokens": 10,
"stop_reason": "error",
"steps_json": "[]",
},
)
# total 为 unit 粒度(1 个 pair unit);stop_reason_counts 为 record 粒度(2
return InferenceResult(
run_id=run_id,
accuracy=0.0,
total=1,
correct=0,
per_task_type={},
steps_mean=1.0,
token_usage={"prompt_tokens": 20, "completion_tokens": 20},
stop_reason_counts={"error": 2},
)
try:
_b_units, valid_units, errors_inc, denom_inc = await _resolve_baseline_block(
units=units,
task_type="temporal",
s_hash=s_hash,
prompts_version="p1",
baseline_cache=cache,
base_skills_dir=workspace / "skills" / "v1",
run_inference=mock_fn,
log=log,
run_id="step1_gate_b0_base",
)
# 分子按 unit 计:1 个 INFRA unit(不是 2 条 record);分母同粒度 = r.total = 1
assert errors_inc == 1
assert denom_inc == 1
# 整对 INFRA → 从有效单元剔除
assert valid_units == []
finally:
log.close()
@pytest.mark.asyncio
async def test_all_infra_ladder_raises_clear_error(tmp_path: Path) -> None:
"""整个阶梯所有 unit 都被判为 INFRA 排除 → 明确 RuntimeError(非误导性空阶梯断言)。"""
workspace = _setup_workspace(tmp_path)
log = _make_log(workspace)
questions = _make_questions(4)
cache = BaselineCache(workspace / "baseline_cache.json")
candidate_calls: list[str] = []
async def mock_fn(qs, *, run_id, skills_dir):
if run_id.endswith("_cand"):
candidate_calls.append(run_id)
# 基线臂逐题全部 INFRA error(候选臂在修复后不应被空跑)
for q in qs:
log.insert(
"predictions",
{
"run_id": run_id,
"video_id": "v0",
"question_id": q.question_id,
"task_type": "temporal",
"prediction": "",
"answer": "A",
"evidence": "",
"reasoning": "",
"steps_used": 1,
"prompt_tokens": 10,
"completion_tokens": 10,
"stop_reason": "error",
"steps_json": "[]",
},
)
total = len(qs)
return InferenceResult(
run_id=run_id,
accuracy=0.0,
total=total,
correct=0,
per_task_type={},
steps_mean=1.0,
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
stop_reason_counts={"error": total},
)
try:
with pytest.raises(RuntimeError, match="INFRA"):
await validate_skill_local(
workspace_dir=workspace,
base_skills_version="v1",
task_type="temporal",
target_file="temporal.md",
candidate_content="content",
base_skill_content="baseline skill content",
ladder_items=questions,
gate_params=_DEFAULT_GATE_PARAMS,
gate_block=4,
gate_n_max=20,
gate_guard_err=0.9, # 高阈值:4 题 <10 分母不触发错误率护栏
baseline_cache=cache,
prompts_version="p1",
run_inference=mock_fn,
log=log,
gate_run_prefix="step1_gate_test",
)
# 全 INFRA 块不应触发候选空跑
assert candidate_calls == []
finally:
log.close()
@pytest.mark.asyncio
async def test_parse_error_counts_toward_guard(tmp_path: Path) -> None:
"""stop_reason=parse_error 也计入护栏错误率(与 INFRA 判定口径一致)→ 超阈值熔断。"""
workspace = _setup_workspace(tmp_path)
log = _make_log(workspace)
# 12 个 single,基线全 parse_errorper-record 落 DB,护栏按 unit 从 DB 读)。
# 首块全 INFRA → errors=12/denom=12=1.0>0.5 → parse_error 亦触发护栏。
questions = _make_questions(12)
cache = BaselineCache(workspace / "baseline_cache.json")
mock_fn, _ = _make_all_infra_mock(log, "parse_error")
try:
with pytest.raises(RuntimeError, match="错误率过高"):
await validate_skill_local(
workspace_dir=workspace,
base_skills_version="v1",
task_type="temporal",
target_file="temporal.md",
candidate_content="content",
base_skill_content="baseline skill content",
ladder_items=questions,
gate_params=_DEFAULT_GATE_PARAMS,
gate_block=12,
gate_n_max=20,
gate_guard_err=0.5,
baseline_cache=cache,
prompts_version="p1",
run_inference=mock_fn,
log=log,
gate_run_prefix="step1_gate_test",
)
finally:
log.close()
@pytest.mark.asyncio
async def test_last_block_terminal(tmp_path: Path) -> None:
"""单块 + n_remaining=0 → 终态判定(provisional 或 inertia),非 continue。"""
+44
View File
@@ -183,6 +183,23 @@ def test_init_workspace_from_seed_missing_questions(store_dir: Path, workspace_d
assert not workspace_dir.exists()
def test_init_workspace_from_seed_carries_pools(store_dir: Path, workspace_dir: Path) -> None:
"""seed 目录含 pools.json/split_manifest.json 时拷入 workspace。"""
# store_dir fixture 已构造 seed "initial";补冻结产物到 seed 目录再初始化 workspace。
seed_dir = store_dir / "seeds" / "initial"
(seed_dir / "pools.json").write_text('{"split_mode":"global"}')
(seed_dir / "split_manifest.json").write_text('{"pools_sha256":"a"}')
init_workspace_from_seed(
workspace_dir,
store_dir,
seed_name="initial",
questions="benchmarks/Video-MME",
)
assert (workspace_dir / "pools.json").exists()
assert (workspace_dir / "split_manifest.json").exists()
# ---------------------------------------------------------------------------
# load_manifest
# ---------------------------------------------------------------------------
@@ -242,6 +259,33 @@ def test_update_manifest_valid(store_dir: Path, workspace_dir: Path) -> None:
assert manifest["current"]["skills"] == "skills/v2"
def test_update_manifest_is_atomic(
store_dir: Path, workspace_dir: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""写 manifest 途中崩溃不产生半截 JSON(原子写:os.replace 失败也不损原文件)。"""
from app.harness import workspace as ws
init_workspace(
workspace_dir,
store_dir,
questions="benchmarks/Video-MME",
skills_version="v1",
prompts_version="v1",
)
original = (workspace_dir / "manifest.json").read_text()
def _boom(_src: object, _dst: object) -> None:
raise OSError("crash during replace")
monkeypatch.setattr(ws.os, "replace", _boom)
with pytest.raises(OSError, match="crash during replace"):
update_manifest(workspace_dir, skills="skills/v2")
# 原 manifest 未被破坏(内容不变且仍是合法 JSON)
assert (workspace_dir / "manifest.json").read_text() == original
assert json.loads((workspace_dir / "manifest.json").read_text())
# ---------------------------------------------------------------------------
# record_run
# ---------------------------------------------------------------------------
+16
View File
@@ -0,0 +1,16 @@
"""工程配置校验单元测试(Redis 缓存 TTL fail-loud)。"""
from __future__ import annotations
import pytest
def test_redis_cache_ttl_zero_rejected() -> None:
"""REDIS_CACHE_TTL<=0 必须启动即报错,消灭'0=永不过期'隐式语义。"""
from adapters.redis_cache import _resolve_cache_ttl
with pytest.raises(ValueError, match="REDIS_CACHE_TTL"):
_resolve_cache_ttl(0)
with pytest.raises(ValueError, match="REDIS_CACHE_TTL"):
_resolve_cache_ttl(-1)
assert _resolve_cache_ttl(86400) == 86400
+65
View File
@@ -383,3 +383,68 @@ class TestApplyPatch:
assert "line2" not in out
assert "TAIL" in out
assert all(r["status"].startswith("applied") for r in reports)
class TestSpanOverlapProtection:
"""冻结区跨度保护 + marker 注入拦截(算法 #8 加固)。"""
def test_replace_spanning_into_protected_marker_preserved(self) -> None:
"""target 末端跨入含 marker 的 appendix 冻结区时被拦截,marker 不被破坏。"""
body = "正文最后一段。"
appendix = f"{APPENDIX_START}\n## 执行提醒\n- 规则A\n{APPENDIX_END}"
content = body + "\n\n" + appendix
# target 从正文末尾跨入 APPENDIX_START(含 marker 字面量)
target = "正文最后一段。\n\n" + APPENDIX_START
edits = [{"op": "delete", "target": target, "content": ""}]
new_content, report = apply_patch_with_report(
content, edits, protected_spans=[appendix]
)
# 无论经 marker 注入拦截还是跨度拦截,marker 都必须完整保留
assert APPENDIX_START in new_content and APPENDIX_END in new_content
assert report[0]["status"] in ("skipped_protected", "skipped_marker_injection")
def test_replace_spanning_into_frozen_section_is_skipped(self) -> None:
"""target 起点在正文、末端伸入无 marker 的冻结区段 → 跨度拦截跳过。"""
body = "可改正文段落。"
frozen = "## 输出格式\n必须输出 JSON。"
content = body + "\n" + frozen
# target 从可改正文跨入冻结区段(不含任何 marker 字面量)
target = "可改正文段落。\n## 输出格式"
edits = [{"op": "replace", "target": target, "content": "破坏内容"}]
new_content, report = apply_patch_with_report(
content, edits, protected_spans=[frozen]
)
assert "破坏内容" not in new_content
assert "## 输出格式" in new_content
assert report[0]["status"] == "skipped_protected"
def test_insert_after_spanning_into_frozen_section_is_skipped(self) -> None:
"""insert_after 的 target 末端伸入无 marker 冻结区段时跨度拦截跳过。"""
body = "可改正文。"
frozen = "## 输出格式\n固定内容。"
content = body + "\n" + frozen
target = "可改正文。\n## 输出格式"
edits = [{"op": "insert_after", "target": target, "content": "注入内容"}]
new_content, report = apply_patch_with_report(
content, edits, protected_spans=[frozen]
)
assert "注入内容" not in new_content
assert report[0]["status"] == "skipped_protected"
def test_edit_payload_with_marker_literal_rejected(self) -> None:
"""edit payload 含 marker 字面量 → 拒绝该 edit。"""
edits = [
{"op": "append", "target": "", "content": f"注入 {APPENDIX_START} 破坏"}
]
_, report = apply_patch_with_report("正文", edits, protected_spans=[])
assert any(
"marker" in str(s).lower() or "reject" in str(s).lower() for s in report[0].values()
)
def test_edit_target_with_marker_literal_rejected(self) -> None:
"""edit target 含 momentum marker 字面量 → 拒绝该 edit。"""
edits = [
{"op": "replace", "target": f"{MOMENTUM_START}x", "content": "y"}
]
_, report = apply_patch_with_report("正文", edits, protected_spans=[])
assert report[0]["status"] == "skipped_marker_injection"
+57
View File
@@ -135,3 +135,60 @@ def test_baseline_val_accuracy_reflects_validation():
assert len(pools.validation) == 2
assert pools.baseline_val_accuracy == pytest.approx(0.5)
assert pools.diagnosis == []
def test_tier_aware_keeps_high_t2_in_diag():
"""错题视频组按 T2 含量升序进 val:T2 高的组保留在 diagnosis。"""
from app.harness.pools import split_by_video_assignment
from core.types import GeneratedQuestion
def _q(qid, vid):
return GeneratedQuestion(
question_id=qid, video_id=vid, task_type="X", question="q",
options=["A", "B"], answer="A", source_nodes=[], difficulty="easy",
)
# 4 个错题视频(每视频 1 题),T2 数分别 2/1/0/0
questions = [_q(f"{v}-1", v) for v in ("vA", "vB", "vC", "vD")]
assignment = {v: "trainval" for v in ("vA", "vB", "vC", "vD")}
correctness = {f"{v}-1": False for v in ("vA", "vB", "vC", "vD")}
wrong_tier = {"vA": 2, "vB": 1, "vC": 0, "vD": 0}
pools = split_by_video_assignment(
questions, assignment, correctness, val_ratio=0.5, seed=7,
wrong_tier_by_video=wrong_tier,
)
diag_vids = {q.video_id for q in pools.diagnosis}
# T2 最高的 vA 必留 diagT2=0 的组优先进 val
assert "vA" in diag_vids
assert "vB" in diag_vids
def test_val_wrong_min_repair_pulls_from_diag():
"""val 错题不足 val_wrong_min 时从 diag 换入低 T2 错题组补足。"""
from app.harness.pools import split_by_video_assignment
from core.types import GeneratedQuestion
def _q(qid, vid, correct):
return GeneratedQuestion(
question_id=qid, video_id=vid, task_type="X", question="q",
options=["A", "B"], answer="A", source_nodes=[], difficulty="easy",
)
# 8 错题视频 + 2 正确视频;val_ratio 小使初分 val 错题不足,触发修复
vids_wrong = [f"w{i}" for i in range(8)]
vids_correct = ["c0", "c1"]
questions = [_q(f"{v}-1", v, False) for v in vids_wrong] + [
_q(f"{v}-1", v, True) for v in vids_correct
]
assignment = {v: "trainval" for v in vids_wrong + vids_correct}
correctness = {f"{v}-1": False for v in vids_wrong}
correctness.update({f"{v}-1": True for v in vids_correct})
wrong_tier = {v: i for i, v in enumerate(vids_wrong)} # 递增 T2
pools = split_by_video_assignment(
questions, assignment, correctness, val_ratio=0.1, seed=7,
wrong_tier_by_video=wrong_tier, val_wrong_min=4,
)
val_wrong = sum(1 for q in pools.validation if not correctness[q.question_id])
assert val_wrong >= 4, f"功效修复后 val 错题 {val_wrong} < 4"
+12
View File
@@ -109,6 +109,18 @@ async def test_different_models_different_keys(
assert await cache.get("claude-3-opus", MESSAGES) is None
@pytest.mark.asyncio
async def test_salt_changes_key(fake_redis: object) -> None:
"""cache_salt 进入键:不同 salt 产生不同键,None 保持旧键结构。"""
cache = RedisResponseCache(redis=fake_redis, ttl_s=None)
k_none = cache._build_key("m", [{"role": "user", "content": "x"}], None)
k_e1 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e1")
k_e2 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e2")
assert k_none != k_e1
assert k_none != k_e2
assert k_e1 != k_e2
@pytest.mark.asyncio
async def test_graceful_degradation_on_error() -> None:
"""Redis 不可用时静默降级:get→None,set→不报错。"""
+189
View File
@@ -54,6 +54,7 @@ def _base_config(workspace_dir: Path, store_dir: Path) -> RunConfig:
batch_size=5,
min_class_per_batch=2,
eval_min_per_class=2,
trainable_min_units=8,
early_stop_patience=3,
test_size=10,
use_slow_momentum=False,
@@ -147,6 +148,194 @@ def _fake_question(question_id: str, video_id: str) -> object:
)
@pytest.mark.asyncio
async def test_diagnosis_reads_traces_from_steps_json(
runner_with_real_store: Runner,
) -> None:
"""traces 表为空但 predictions.steps_json 有轨迹时,诊断仍拿到非空 traces。
构造一条只写 steps_json不写 traces 表的 predictions patch run_diagnosis
捕获传入的 run_log直接 await get_traces 断言经 StepsJsonRunLog
steps_json 重建出非空轨迹算法 #7 恢复)。
"""
from app.harness.inference import PREDICTIONS_SCHEMA
from app.harness.log import HarnessLog
steps_json = json.dumps(
[
{
"thought": "先看整体",
"tool_call": {"tool": "search_tree", "args": {"query": "开场"}},
"tool_output": "命中 L2 节点 A",
}
],
ensure_ascii=False,
)
with HarnessLog(str(runner_with_real_store._paths.db_path), "infer_adhoc") as log:
log.create_table("predictions", PREDICTIONS_SCHEMA)
log.create_table("traces", {"video_id": "TEXT", "question_id": "TEXT", "step": "INTEGER"})
log.insert(
"predictions",
{
"video_id": _REAL_VIDEO_ID,
"question_id": _REAL_QUESTION_ID,
"task_type": "Action Reasoning",
"prediction": "A",
"answer": "B",
"evidence": "",
"reasoning": "",
"steps_used": 1,
"prompt_tokens": 0,
"completion_tokens": 0,
"stop_reason": "finished",
"steps_json": steps_json,
},
)
captured: dict[str, object] = {}
async def _fake_run_diagnosis(**kwargs: object) -> DiagnosisResult:
captured["run_log"] = kwargs["run_log"]
return _empty_diagnosis_result()
with patch(
"core.evolution.diagnose.run_diagnosis",
new=AsyncMock(side_effect=_fake_run_diagnosis),
):
await runner_with_real_store._run_diagnosis(
"infer_adhoc", question_ids=[_REAL_QUESTION_ID]
)
run_log = captured["run_log"]
traces = await run_log.get_traces("infer_adhoc", question_ids=[_REAL_QUESTION_ID])
assert traces, "traces 表空时应从 steps_json 重建出非空轨迹"
assert traces[0]["tool_name"] == "search_tree"
def _fake_inference_result(accuracy: float) -> object:
"""构造 InferenceResult 供 holdout 去重测试(per_task_type 空、token 归零)。"""
from app.harness.inference import InferenceResult
return InferenceResult(
run_id="x",
accuracy=accuracy,
total=10,
correct=int(accuracy * 10),
per_task_type={},
steps_mean=1.0,
token_usage={"prompt_tokens": 0, "completion_tokens": 0},
stop_reason_counts={},
)
@pytest.mark.asyncio
async def test_holdout_dedup_skips_reevaluated_versions(
runner_with_real_store: Runner,
) -> None:
"""baseline 不跑推理(从基线预测推导);best_hard==final 时不重复评估。"""
from unittest.mock import AsyncMock
runner = runner_with_real_store
eval_calls: list[str] = []
async def _fake_eval(sv, pv, questions, run_id, context): # noqa: ANN001
eval_calls.append(run_id)
return _fake_inference_result(0.5)
runner._eval_version_on_pool = AsyncMock(side_effect=_fake_eval)
# best_mixed 赢家取 final 版本(必落在 memo,0 推理)
runner._pick_mixed_best = AsyncMock(return_value=("skills_final", "prompts_final"))
# baseline 推导:patch 为 0 推理的假结果(不经 _eval_version_on_pool
runner._derive_baseline_test_result = MagicMock(return_value=_fake_inference_result(0.4))
state = MagicMock()
state.holdout_memo = {}
state.baseline_skills_version = "skills_base"
state.baseline_prompts_version = "prompts_base"
# best_hard 版本 == final 版本 → 去重
state.best_skills_version = "skills_final"
state.best_prompts_version = "prompts_final"
pools = MagicMock()
pools.baseline_run_id = "infer_adhoc"
pools.test = [_fake_question("q1", "vA")]
await runner._holdout_four_way(
1, pools, state, eval_skills_version="skills_final", eval_prompts_version="prompts_final"
)
# baseline=0(推导)、best_hard=1(真评并存 memo)、final/best_mixed 引用 memo0
assert len(eval_calls) == 1
runner._derive_baseline_test_result.assert_called_once()
@pytest.mark.asyncio
async def test_run_step_deletes_stale_rows_before_rerun(
runner_with_real_store: Runner, monkeypatch: pytest.MonkeyPatch
) -> None:
"""同 run_id 重跑前先清 predictions/traces,避免重复行双计(幂等)。"""
from unittest.mock import AsyncMock
from app.harness.inference import PREDICTIONS_SCHEMA, TRACES_SCHEMA
from app.harness.log import HarnessLog
run_id = "infer_adhoc_e1_s0"
# 预置该 step run_id 的旧 predictions/traces 行
with HarnessLog(str(runner_with_real_store._paths.db_path), run_id) as log:
log.create_table("predictions", PREDICTIONS_SCHEMA)
log.create_table("traces", TRACES_SCHEMA)
log.insert("predictions", {"video_id": "vA", "question_id": "q1", "prediction": "A"})
log.insert("traces", {"video_id": "vA", "question_id": "q1", "step": 0})
batch = [_fake_question("q1", "vA")]
runner_with_real_store._rollout_batch = AsyncMock()
monkeypatch.setattr("app.harness.runner._apply_batch_correctness", lambda *a, **k: None)
runner_with_real_store._run_diagnosis = AsyncMock(return_value=DiagnosisResult(run_id=run_id))
runner_with_real_store._gate_batch_skills = AsyncMock()
state = MagicMock()
state.correctness = {"q1": True}
state.gate_cooldown = {}
pools = MagicMock()
pools.baseline_run_id = "infer_adhoc"
await runner_with_real_store._run_step(1, 0, 10, batch, pools, state)
with HarnessLog(
str(runner_with_real_store._paths.db_path), run_id, register_run=False
) as log:
preds = log.query("SELECT * FROM predictions WHERE run_id=?", (run_id,))
traces = log.query("SELECT * FROM traces WHERE run_id=?", (run_id,))
assert preds == []
assert traces == []
@pytest.mark.asyncio
async def test_run_step_aborts_on_high_degrade_rate(
runner_with_real_store: Runner, monkeypatch: pytest.MonkeyPatch
) -> None:
"""诊断降级占比 > 50% 时 _run_step 中止(疑似 judge 基础设施故障)。"""
from unittest.mock import AsyncMock
batch = [_fake_question("q1", "vA"), _fake_question("q2", "vA")]
runner_with_real_store._rollout_batch = AsyncMock()
monkeypatch.setattr("app.harness.runner._apply_batch_correctness", lambda *a, **k: None)
runner_with_real_store._run_diagnosis = AsyncMock(
return_value=DiagnosisResult(run_id="r", degraded_count=2)
)
runner_with_real_store._gate_batch_skills = AsyncMock()
state = MagicMock()
state.correctness = {"q1": False, "q2": False}
state.gate_cooldown = {}
pools = MagicMock()
pools.baseline_run_id = "infer_adhoc"
with pytest.raises(RuntimeError, match="降级占比"):
await runner_with_real_store._run_step(1, 0, 10, batch, pools, state)
@pytest.mark.asyncio
async def test_run_diagnosis_full_scan_loads_all_video_trees(
runner_with_real_store: Runner,
+79
View File
@@ -303,3 +303,82 @@ def test_git_short_sha_nonempty():
sha = cli.git_short_sha()
assert sha
assert len(sha) >= 4
def test_build_split_refuses_overwrite_without_force(tmp_path):
"""已存在指纹不同的 pools.json 时,force=False 必须报错不覆盖。"""
from app.harness.build_split import _guard_frozen_products
out_path = tmp_path / "pools.json"
out_path.write_text('{"split_mode":"global"}', encoding="utf-8")
manifest_path = tmp_path / "split_manifest.json"
with pytest.raises(FileExistsError, match="已存在冻结产物"):
_guard_frozen_products(out_path, manifest_path, force=False)
def test_build_split_force_backs_up_old(tmp_path):
"""force=True 时旧产物被备份为 .bak.* 再允许覆盖。"""
from app.harness.build_split import _guard_frozen_products
out_path = tmp_path / "pools.json"
out_path.write_text('{"old":1}', encoding="utf-8")
manifest_path = tmp_path / "split_manifest.json"
manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8")
_guard_frozen_products(out_path, manifest_path, force=True)
baks = list(tmp_path.glob("pools.json.bak.*"))
assert len(baks) == 1, f"未备份旧产物: {list(tmp_path.iterdir())}"
def test_build_split_force_twice_same_suffix_keeps_both(tmp_path):
"""连续两次 force 命中同后缀时,第二次备份不覆盖第一次(追加递增序号)。"""
from app.harness.build_split import _guard_frozen_products
out_path = tmp_path / "pools.json"
manifest_path = tmp_path / "split_manifest.json"
# 第一次 force:两文件都无 manifest 里的 pools_sha256 之外的差异,后缀恒为同值。
out_path.write_text('{"gen":1}', encoding="utf-8")
manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8")
_guard_frozen_products(out_path, manifest_path, force=True)
# 第二次 force:写入相同 sha 的新产物,触发同后缀备份。
out_path.write_text('{"gen":2}', encoding="utf-8")
manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8")
_guard_frozen_products(out_path, manifest_path, force=True)
pools_baks = sorted(p.name for p in tmp_path.glob("pools.json.bak.*"))
assert len(pools_baks) == 2, f"同后缀第二次备份覆盖了第一次: {pools_baks}"
def test_build_split_force_rollback_on_partial_failure(tmp_path, monkeypatch):
"""备份第二个文件失败时,第一个已备份文件被 rollback 回原名(原子性)。"""
from pathlib import Path
from app.harness import build_split as bs
out_path = tmp_path / "pools.json"
manifest_path = tmp_path / "split_manifest.json"
out_path.write_text('{"gen":1}', encoding="utf-8")
manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8")
real_rename = Path.rename
calls = {"n": 0}
def flaky_rename(self, target):
# 第一次 rename(备份 pools.json)成功,第二次(备份 manifest)抛错。
calls["n"] += 1
if calls["n"] == 2:
raise OSError("模拟第二个备份 rename 失败")
return real_rename(self, target)
monkeypatch.setattr(Path, "rename", flaky_rename)
with pytest.raises(OSError, match="模拟第二个备份"):
bs._guard_frozen_products(out_path, manifest_path, force=True)
# rollback 后:两原文件仍在原位,无残留 .bak.*
assert out_path.exists(), "第一个文件未被 rollback 回原名"
assert manifest_path.exists(), "第二个文件不应被移动"
assert not list(tmp_path.glob("*.bak.*")), f"残留半备份: {list(tmp_path.iterdir())}"