Merge branch 'feat/preflight-train-fixes': preflight training fixes (4 WPs, 21+ defects)
This commit is contained in:
@@ -46,6 +46,7 @@ LLM_CIRCUIT_BREAKER_COOLDOWN=60
|
|||||||
LLM_TTFT_TIMEOUT=30
|
LLM_TTFT_TIMEOUT=30
|
||||||
LLM_INTER_TOKEN_TIMEOUT=15
|
LLM_INTER_TOKEN_TIMEOUT=15
|
||||||
LLM_RETRY_MAX_DELAY=30.0
|
LLM_RETRY_MAX_DELAY=30.0
|
||||||
|
# 正整数秒,禁止 0(0 会被拒绝启动);训练场景建议 >= 单次训练时长
|
||||||
REDIS_CACHE_TTL=86400
|
REDIS_CACHE_TTL=86400
|
||||||
|
|
||||||
# 建树批量并行:全局 VLM/LLM 在途调用上限(Spec-2 工程配置)
|
# 建树批量并行:全局 VLM/LLM 在途调用上限(Spec-2 工程配置)
|
||||||
|
|||||||
@@ -106,21 +106,31 @@ class SqliteDiagnosisSignalStore:
|
|||||||
)
|
)
|
||||||
self._conn.commit()
|
self._conn.commit()
|
||||||
|
|
||||||
def done_question_ids(self, baseline_run_id: str, diag_fingerprint: str) -> set[str]:
|
def done_question_ids(
|
||||||
|
self,
|
||||||
|
baseline_run_id: str,
|
||||||
|
diag_fingerprint: str,
|
||||||
|
*,
|
||||||
|
retry_uncertain: bool = False,
|
||||||
|
) -> set[str]:
|
||||||
"""查询指定 run 与诊断指纹下已完成的 question_id 集合。
|
"""查询指定 run 与诊断指纹下已完成的 question_id 集合。
|
||||||
|
|
||||||
参数:
|
参数:
|
||||||
baseline_run_id: baseline run 标识。
|
baseline_run_id: baseline run 标识。
|
||||||
diag_fingerprint: 诊断口径指纹。
|
diag_fingerprint: 诊断口径指纹。
|
||||||
|
retry_uncertain: True 时追加 `AND tier != 'uncertain'`,把 uncertain
|
||||||
|
(信号不可信降级)题排除出已完成集,令其被重新诊断;默认 False。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
已落盘信号的 question_id 集合;无匹配时为空集,供断点续跑跳过。
|
已落盘信号的 question_id 集合;无匹配时为空集,供断点续跑跳过。
|
||||||
"""
|
"""
|
||||||
cursor = self._conn.execute(
|
sql = (
|
||||||
"SELECT DISTINCT question_id FROM baseline_diagnosis"
|
"SELECT DISTINCT question_id FROM baseline_diagnosis"
|
||||||
" WHERE baseline_run_id = ? AND diag_fingerprint = ?",
|
" WHERE baseline_run_id = ? AND diag_fingerprint = ?"
|
||||||
(baseline_run_id, diag_fingerprint),
|
|
||||||
)
|
)
|
||||||
|
if retry_uncertain:
|
||||||
|
sql += " AND tier != 'uncertain'"
|
||||||
|
cursor = self._conn.execute(sql, (baseline_run_id, diag_fingerprint))
|
||||||
return {r["question_id"] for r in cursor.fetchall()}
|
return {r["question_id"] for r in cursor.fetchall()}
|
||||||
|
|
||||||
def load(self, baseline_run_id: str, diag_fingerprint: str) -> list[DiagnosisSignalRow]:
|
def load(self, baseline_run_id: str, diag_fingerprint: str) -> list[DiagnosisSignalRow]:
|
||||||
|
|||||||
+18
-3
@@ -20,21 +20,32 @@ class CircuitBreaker:
|
|||||||
self._cooldown_s = cooldown_s
|
self._cooldown_s = cooldown_s
|
||||||
self._fails: dict[str, int] = {}
|
self._fails: dict[str, int] = {}
|
||||||
self._open_until: dict[str, float] = {}
|
self._open_until: dict[str, float] = {}
|
||||||
|
self._half_open_inflight: dict[str, bool] = {}
|
||||||
|
|
||||||
def is_open(self, source_name: str, now: float) -> bool:
|
def is_open(self, source_name: str, now: float) -> bool:
|
||||||
"""判断指定源是否处于开路状态。
|
"""判断指定源是否处于开路状态。
|
||||||
|
|
||||||
冷却截止时刻之前为开路;到期返回 False(放行一个试探,即半开)。
|
冷却截止时刻之前为开路;到期进入半开,**只放行一个探针**(其余仍被挡),
|
||||||
|
避免冷却到期瞬间惊群重连再次压垮上游。"检查+标记探针"在 asyncio 单线程内
|
||||||
|
同步执行,天然原子无竞态。
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
source_name: 被熔断的源标识。
|
source_name: 被熔断的源标识。
|
||||||
now: 当前时刻(秒级时间戳),由调用方注入。
|
now: 当前时刻(秒级时间戳),由调用方注入。
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True 表示开路(拒绝请求),False 表示关闭或半开(放行)。
|
True 表示开路(拒绝请求),False 表示关闭或半开放行探针。
|
||||||
"""
|
"""
|
||||||
until = self._open_until.get(source_name)
|
until = self._open_until.get(source_name)
|
||||||
return until is not None and now < until
|
if until is None:
|
||||||
|
return False
|
||||||
|
if now < until:
|
||||||
|
return True # 冷却中,全挡
|
||||||
|
# 冷却到期:half-open,只放行一个探针
|
||||||
|
if self._half_open_inflight.get(source_name):
|
||||||
|
return True # 已有探针在途,继续挡
|
||||||
|
self._half_open_inflight[source_name] = True
|
||||||
|
return False
|
||||||
|
|
||||||
def record_failure(self, source_name: str, now: float) -> None:
|
def record_failure(self, source_name: str, now: float) -> None:
|
||||||
"""记录一次失败;累计达阈值则开路至 now + cooldown。
|
"""记录一次失败;累计达阈值则开路至 now + cooldown。
|
||||||
@@ -47,6 +58,8 @@ class CircuitBreaker:
|
|||||||
self._fails[source_name] = count
|
self._fails[source_name] = count
|
||||||
if count >= self._fail_threshold:
|
if count >= self._fail_threshold:
|
||||||
self._open_until[source_name] = now + self._cooldown_s
|
self._open_until[source_name] = now + self._cooldown_s
|
||||||
|
# 探针失败清在途标记,使下一轮 cooldown 到期后可再放行探针
|
||||||
|
self._half_open_inflight.pop(source_name, None)
|
||||||
|
|
||||||
def force_open(self, source_name: str, now: float) -> None:
|
def force_open(self, source_name: str, now: float) -> None:
|
||||||
"""强制开路(用于 401/403 等不可恢复错误),一次即熔断。
|
"""强制开路(用于 401/403 等不可恢复错误),一次即熔断。
|
||||||
@@ -59,6 +72,7 @@ class CircuitBreaker:
|
|||||||
"""
|
"""
|
||||||
self._fails[source_name] = self._fail_threshold
|
self._fails[source_name] = self._fail_threshold
|
||||||
self._open_until[source_name] = now + self._cooldown_s
|
self._open_until[source_name] = now + self._cooldown_s
|
||||||
|
self._half_open_inflight.pop(source_name, None)
|
||||||
|
|
||||||
def record_success(self, source_name: str) -> None:
|
def record_success(self, source_name: str) -> None:
|
||||||
"""记录一次成功;清零失败计数与开路状态(关闭熔断器)。
|
"""记录一次成功;清零失败计数与开路状态(关闭熔断器)。
|
||||||
@@ -68,3 +82,4 @@ class CircuitBreaker:
|
|||||||
"""
|
"""
|
||||||
self._fails.pop(source_name, None)
|
self._fails.pop(source_name, None)
|
||||||
self._open_until.pop(source_name, None)
|
self._open_until.pop(source_name, None)
|
||||||
|
self._half_open_inflight.pop(source_name, None)
|
||||||
|
|||||||
+16
-3
@@ -179,7 +179,10 @@ def _is_transient_error(exc: Exception) -> bool:
|
|||||||
返回:
|
返回:
|
||||||
True 表示可重试,False 表示不可重试。
|
True 表示可重试,False 表示不可重试。
|
||||||
"""
|
"""
|
||||||
if isinstance(exc, (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout)):
|
# 两族基类覆盖断连族:TimeoutException(ConnectTimeout/ReadTimeout/WriteTimeout/PoolTimeout)
|
||||||
|
# 与 TransportError(ConnectError/ReadError/RemoteProtocolError 等)。
|
||||||
|
# 注意 HTTPStatusError 非 TransportError 子类,401/403 致命分支不受影响。
|
||||||
|
if isinstance(exc, (httpx.TimeoutException, httpx.TransportError)):
|
||||||
return True
|
return True
|
||||||
if isinstance(exc, httpx.HTTPStatusError):
|
if isinstance(exc, httpx.HTTPStatusError):
|
||||||
return exc.response.status_code in _TRANSIENT_STATUS_CODES
|
return exc.response.status_code in _TRANSIENT_STATUS_CODES
|
||||||
@@ -274,6 +277,7 @@ class GovernedLLMClient:
|
|||||||
*,
|
*,
|
||||||
session_id: str | None = None,
|
session_id: str | None = None,
|
||||||
parent_call_id: str | None = None,
|
parent_call_id: str | None = None,
|
||||||
|
cache_salt: str | None = None,
|
||||||
) -> LLMResponse:
|
) -> LLMResponse:
|
||||||
"""发起 LLM 调用,经四层治理栈:熔断 → 缓存 → 重试+流式 → 遥测。
|
"""发起 LLM 调用,经四层治理栈:熔断 → 缓存 → 重试+流式 → 遥测。
|
||||||
|
|
||||||
@@ -281,6 +285,7 @@ class GovernedLLMClient:
|
|||||||
messages: OpenAI 格式消息列表。
|
messages: OpenAI 格式消息列表。
|
||||||
session_id: 会话 ID(传递到遥测)。
|
session_id: 会话 ID(传递到遥测)。
|
||||||
parent_call_id: 父调用 ID(传递到遥测)。
|
parent_call_id: 父调用 ID(传递到遥测)。
|
||||||
|
cache_salt: 可选缓存盐,透传到 Redis 缓存键(如跨 epoch 重采样)。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
LLMResponse 统一响应。
|
LLMResponse 统一响应。
|
||||||
@@ -296,7 +301,11 @@ class GovernedLLMClient:
|
|||||||
raise CircuitOpenError(f"熔断器已开启,拒绝调用 provider={self._provider}")
|
raise CircuitOpenError(f"熔断器已开启,拒绝调用 provider={self._provider}")
|
||||||
|
|
||||||
# ② 缓存查询(cache 为 None 时跳过)— call_id 在缓存路径独立生成
|
# ② 缓存查询(cache 为 None 时跳过)— call_id 在缓存路径独立生成
|
||||||
cached = await self._cache.get(self._model, messages) if self._cache is not None else None
|
cached = (
|
||||||
|
await self._cache.get(self._model, messages, cache_salt)
|
||||||
|
if self._cache is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
if cached is not None:
|
if cached is not None:
|
||||||
cache_call_id = str(uuid4())
|
cache_call_id = str(uuid4())
|
||||||
response = LLMResponse(
|
response = LLMResponse(
|
||||||
@@ -370,7 +379,7 @@ class GovernedLLMClient:
|
|||||||
|
|
||||||
# ④ 写缓存(cache 为 None 时跳过)
|
# ④ 写缓存(cache 为 None 时跳过)
|
||||||
if self._cache is not None:
|
if self._cache is not None:
|
||||||
await self._cache.set(self._model, messages, response)
|
await self._cache.set(self._model, messages, response, cache_salt)
|
||||||
|
|
||||||
# ⑤ 遥测
|
# ⑤ 遥测
|
||||||
await self._telemetry.record_llm_call(
|
await self._telemetry.record_llm_call(
|
||||||
@@ -572,6 +581,10 @@ class GovernedLLMClient:
|
|||||||
else:
|
else:
|
||||||
thinking_parts.append(text)
|
thinking_parts.append(text)
|
||||||
|
|
||||||
|
# 流耗尽但未收 [DONE] → 服务端截断,视为可重试的 SSE 异常(不写缓存/不当成功)
|
||||||
|
if not usage_sink.get("done"):
|
||||||
|
raise _SseAnomaly("truncated_no_done")
|
||||||
|
|
||||||
content = "".join(content_parts)
|
content = "".join(content_parts)
|
||||||
thinking = "".join(thinking_parts)
|
thinking = "".join(thinking_parts)
|
||||||
usage = usage_sink.get("usage", {})
|
usage = usage_sink.get("usage", {})
|
||||||
|
|||||||
+43
-9
@@ -12,6 +12,26 @@ from loguru import logger
|
|||||||
from core.types import LLMResponse
|
from core.types import LLMResponse
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_cache_ttl(ttl: int) -> int:
|
||||||
|
"""校验 Redis 缓存 TTL:必须为正整数(消灭 0=永不过期 的隐式语义)。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
ttl: 待校验的 TTL 秒数。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
校验通过的正整数 TTL。
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: ttl <= 0。
|
||||||
|
"""
|
||||||
|
if ttl <= 0:
|
||||||
|
raise ValueError(
|
||||||
|
f"REDIS_CACHE_TTL 必须为正整数秒,实际 {ttl}。"
|
||||||
|
"训练场景建议 >= 单次训练时长(如 86400)。"
|
||||||
|
)
|
||||||
|
return ttl
|
||||||
|
|
||||||
|
|
||||||
class RedisResponseCache:
|
class RedisResponseCache:
|
||||||
"""基于 Redis 的 LLM 响应缓存。
|
"""基于 Redis 的 LLM 响应缓存。
|
||||||
|
|
||||||
@@ -29,36 +49,48 @@ class RedisResponseCache:
|
|||||||
self._redis = redis
|
self._redis = redis
|
||||||
self._ttl_s = ttl_s
|
self._ttl_s = ttl_s
|
||||||
|
|
||||||
def _build_key(self, model: str, messages: list[dict[str, str]]) -> str:
|
def _build_key(
|
||||||
|
self,
|
||||||
|
model: str,
|
||||||
|
messages: list[dict[str, str]],
|
||||||
|
cache_salt: str | None = None,
|
||||||
|
) -> str:
|
||||||
"""构造 content-addressed 缓存键。
|
"""构造 content-addressed 缓存键。
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
model: 模型名称。
|
model: 模型名称。
|
||||||
messages: 消息列表。
|
messages: 消息列表。
|
||||||
|
cache_salt: 可选缓存盐(如跨 epoch 强制重采样)。仅当非 None 时才加入
|
||||||
|
键 payload,保证默认 None 时键结构与旧缓存一字节不差、旧键不失效。
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
sha256 哈希字符串作为 Redis 键。
|
sha256 哈希字符串作为 Redis 键。
|
||||||
"""
|
"""
|
||||||
payload = json.dumps(
|
key_obj: dict[str, Any] = {"model": model, "messages": messages}
|
||||||
{"model": model, "messages": messages},
|
if cache_salt is not None:
|
||||||
sort_keys=True,
|
key_obj["salt"] = cache_salt
|
||||||
ensure_ascii=False,
|
payload = json.dumps(key_obj, sort_keys=True, ensure_ascii=False)
|
||||||
)
|
|
||||||
digest = hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
digest = hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||||
return f"llm_cache:{digest}"
|
return f"llm_cache:{digest}"
|
||||||
|
|
||||||
async def get(self, model: str, messages: list[dict[str, str]]) -> LLMResponse | None:
|
async def get(
|
||||||
|
self,
|
||||||
|
model: str,
|
||||||
|
messages: list[dict[str, str]],
|
||||||
|
cache_salt: str | None = None,
|
||||||
|
) -> LLMResponse | None:
|
||||||
"""从缓存读取 LLM 响应。
|
"""从缓存读取 LLM 响应。
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
model: 模型名称。
|
model: 模型名称。
|
||||||
messages: 消息列表。
|
messages: 消息列表。
|
||||||
|
cache_salt: 可选缓存盐,透传到键构造。
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
缓存命中时返回 LLMResponse,未命中或 Redis 异常时返回 None。
|
缓存命中时返回 LLMResponse,未命中或 Redis 异常时返回 None。
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
key = self._build_key(model, messages)
|
key = self._build_key(model, messages, cache_salt)
|
||||||
raw = await self._redis.get(key)
|
raw = await self._redis.get(key)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.warning("Redis 缓存读取失败,降级为未命中")
|
logger.warning("Redis 缓存读取失败,降级为未命中")
|
||||||
@@ -75,6 +107,7 @@ class RedisResponseCache:
|
|||||||
model: str,
|
model: str,
|
||||||
messages: list[dict[str, str]],
|
messages: list[dict[str, str]],
|
||||||
response: LLMResponse,
|
response: LLMResponse,
|
||||||
|
cache_salt: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""将 LLM 响应写入缓存。
|
"""将 LLM 响应写入缓存。
|
||||||
|
|
||||||
@@ -82,9 +115,10 @@ class RedisResponseCache:
|
|||||||
model: 模型名称。
|
model: 模型名称。
|
||||||
messages: 消息列表。
|
messages: 消息列表。
|
||||||
response: 待缓存的 LLMResponse。
|
response: 待缓存的 LLMResponse。
|
||||||
|
cache_salt: 可选缓存盐,透传到键构造。
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
key = self._build_key(model, messages)
|
key = self._build_key(model, messages, cache_salt)
|
||||||
value = json.dumps(dataclasses.asdict(response), ensure_ascii=False)
|
value = json.dumps(dataclasses.asdict(response), ensure_ascii=False)
|
||||||
if self._ttl_s:
|
if self._ttl_s:
|
||||||
await self._redis.set(key, value, ex=self._ttl_s)
|
await self._redis.set(key, value, ex=self._ttl_s)
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ class GovernedVLMClient:
|
|||||||
*,
|
*,
|
||||||
session_id: str | None = None,
|
session_id: str | None = None,
|
||||||
parent_call_id: str | None = None,
|
parent_call_id: str | None = None,
|
||||||
|
cache_salt: str | None = None,
|
||||||
) -> LLMResponse:
|
) -> LLMResponse:
|
||||||
"""图文调用:将图片编码为 base64 嵌入 messages,委托给 LLM 客户端。
|
"""图文调用:将图片编码为 base64 嵌入 messages,委托给 LLM 客户端。
|
||||||
|
|
||||||
@@ -45,6 +46,7 @@ class GovernedVLMClient:
|
|||||||
images: 图片文件路径列表。
|
images: 图片文件路径列表。
|
||||||
session_id: 会话 ID(遥测用)。
|
session_id: 会话 ID(遥测用)。
|
||||||
parent_call_id: 父调用 ID(遥测用)。
|
parent_call_id: 父调用 ID(遥测用)。
|
||||||
|
cache_salt: 可选缓存盐,透传到底层 LLM 缓存键。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
LLMResponse。
|
LLMResponse。
|
||||||
@@ -54,6 +56,7 @@ class GovernedVLMClient:
|
|||||||
vision_messages,
|
vision_messages,
|
||||||
session_id=session_id,
|
session_id=session_id,
|
||||||
parent_call_id=parent_call_id,
|
parent_call_id=parent_call_id,
|
||||||
|
cache_salt=cache_salt,
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@@ -6,7 +6,9 @@
|
|||||||
包装内层 RunLog,兼容 traces 未落表的历史 run);
|
包装内层 RunLog,兼容 traces 未落表的历史 run);
|
||||||
3. 把 error_attributions / infra / degraded 三类产物确定性投影为
|
3. 把 error_attributions / infra / degraded 三类产物确定性投影为
|
||||||
DiagnosisSignalRow(tier 由 split_selection.score_signal 判定);
|
DiagnosisSignalRow(tier 由 split_selection.score_signal 判定);
|
||||||
4. 逐行 store.upsert 落盘,单行单事务 → 崩溃最多丢正在写的一行。
|
4. run 末(Phase 3)逐行 store.upsert 落库——诊断在 Phase 2 全部跑完后才落库,
|
||||||
|
故崩溃丢本次 run 未落库的全部结果(不是"仅一行");靠 GovernedLLMClient 的
|
||||||
|
Redis 缓存缓解重跑时的 LLM 重烧,下次调用命中缓存直接续。
|
||||||
|
|
||||||
错误处理诚实标注(不谎称全传播):
|
错误处理诚实标注(不谎称全传播):
|
||||||
- run_diagnosis 的 C1/C2 阶段(指标计算、错误归因)网络/API 失败经
|
- run_diagnosis 的 C1/C2 阶段(指标计算、错误归因)网络/API 失败经
|
||||||
@@ -70,8 +72,9 @@ async def run_baseline_diagnosis(
|
|||||||
questions: dict[str, GeneratedQuestion],
|
questions: dict[str, GeneratedQuestion],
|
||||||
store: DiagnosisSignalStore,
|
store: DiagnosisSignalStore,
|
||||||
deps: DiagnosisDeps,
|
deps: DiagnosisDeps,
|
||||||
|
retry_uncertain: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""对 baseline run 的错题跑离线诊断并把信号逐行落库(断点续跑幂等)。
|
"""对 baseline run 的错题跑离线诊断并把信号落库(断点续跑幂等)。
|
||||||
|
|
||||||
参数:
|
参数:
|
||||||
baseline_run_id: baseline run 标识(如 "infer_adhoc"),信号行主键之一。
|
baseline_run_id: baseline run 标识(如 "infer_adhoc"),信号行主键之一。
|
||||||
@@ -79,8 +82,10 @@ async def run_baseline_diagnosis(
|
|||||||
wrong_ids: 本次待诊断的可诊断错题 question_id 列表(保序)。
|
wrong_ids: 本次待诊断的可诊断错题 question_id 列表(保序)。
|
||||||
questions: question_id → GeneratedQuestion 映射,需覆盖 wrong_ids 全部题
|
questions: question_id → GeneratedQuestion 映射,需覆盖 wrong_ids 全部题
|
||||||
及 run_diagnosis 返回的所有 infra/degraded 题(用于取 video_id/task_type)。
|
及 run_diagnosis 返回的所有 infra/degraded 题(用于取 video_id/task_type)。
|
||||||
store: 诊断信号存储端口,逐行 upsert 落盘并提供 done_question_ids 续跑查询。
|
store: 诊断信号存储端口,upsert 落盘并提供 done_question_ids 续跑查询。
|
||||||
deps: 外部依赖束(见 DiagnosisDeps)。
|
deps: 外部依赖束(见 DiagnosisDeps)。
|
||||||
|
retry_uncertain: True 时把已落 tier='uncertain'(信号不可信降级)的题也纳入
|
||||||
|
remaining 重新诊断,透传给 store.done_question_ids;默认 False。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
None。副作用为把逐题 DiagnosisSignalRow 写入 store。
|
None。副作用为把逐题 DiagnosisSignalRow 写入 store。
|
||||||
@@ -93,7 +98,9 @@ async def run_baseline_diagnosis(
|
|||||||
(T0)、degraded_question_ids(uncertain)。
|
(T0)、degraded_question_ids(uncertain)。
|
||||||
"""
|
"""
|
||||||
# Phase 1: 算 remaining(续跑幂等)
|
# Phase 1: 算 remaining(续跑幂等)
|
||||||
done = store.done_question_ids(baseline_run_id, diag_fingerprint)
|
done = store.done_question_ids(
|
||||||
|
baseline_run_id, diag_fingerprint, retry_uncertain=retry_uncertain
|
||||||
|
)
|
||||||
remaining = [qid for qid in wrong_ids if qid not in done]
|
remaining = [qid for qid in wrong_ids if qid not in done]
|
||||||
if not remaining:
|
if not remaining:
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -169,7 +176,8 @@ def _project_and_persist(
|
|||||||
{tier: 行数} 计数字典(T2/T1/T0/uncertain),供上层日志与 manifest。
|
{tier: 行数} 计数字典(T2/T1/T0/uncertain),供上层日志与 manifest。
|
||||||
|
|
||||||
关键实现:
|
关键实现:
|
||||||
逐行 upsert(单行单事务),中途崩溃最多丢正在写的一行。三桶**非互斥**:
|
本函数在 run 末(Phase 3)逐行 upsert(单行单事务);诊断已在 Phase 2 全部
|
||||||
|
跑完,故本阶段中途崩溃丢本次 run 未落库的余下行。三桶**非互斥**:
|
||||||
同一 degraded 错题可能同时出现在 error_attributions(judge 解析失败仍建
|
同一 degraded 错题可能同时出现在 error_attributions(judge 解析失败仍建
|
||||||
attribution)里,故按 **degraded > infra > attribution** 优先级去重——先落
|
attribution)里,故按 **degraded > infra > attribution** 优先级去重——先落
|
||||||
degraded/infra,再在 attribution 循环跳过已落题,保证**每题恰写一行、
|
degraded/infra,再在 attribution 循环跳过已落题,保证**每题恰写一行、
|
||||||
|
|||||||
@@ -12,8 +12,9 @@ test,再以视频组为原子切出诊断 / 验证池,原子冻结 pools.jso
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import json
|
||||||
import sqlite3
|
import sqlite3
|
||||||
from collections import Counter
|
from collections import Counter, defaultdict
|
||||||
from dataclasses import asdict, dataclass, field
|
from dataclasses import asdict, dataclass, field
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
@@ -59,6 +60,8 @@ class SplitBuildConfig:
|
|||||||
select_seed: 贪心选择器预洗牌种子(打破等增益平局)。
|
select_seed: 贪心选择器预洗牌种子(打破等增益平局)。
|
||||||
val_ratio: validation 占 trainval 视频组总数的比例。
|
val_ratio: validation 占 trainval 视频组总数的比例。
|
||||||
split_seed: 视频组题级切分的洗牌种子。
|
split_seed: 视频组题级切分的洗牌种子。
|
||||||
|
val_wrong_min: validation 池最少错题数,切分时保证功效(不足则从 diag 换入
|
||||||
|
低 T2 错题组补足,耗尽 fail-loud)。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
n_trainval: int
|
n_trainval: int
|
||||||
@@ -68,6 +71,7 @@ class SplitBuildConfig:
|
|||||||
select_seed: int
|
select_seed: int
|
||||||
val_ratio: float
|
val_ratio: float
|
||||||
split_seed: int
|
split_seed: int
|
||||||
|
val_wrong_min: int
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -101,6 +105,72 @@ class SplitBuildResult:
|
|||||||
return getattr(self, key)
|
return getattr(self, key)
|
||||||
|
|
||||||
|
|
||||||
|
def _unique_backup_path(path: Path, suffix: str) -> Path:
|
||||||
|
"""求 path 的唯一 .bak.<suffix> 备份路径,已存在则追加递增序号避免覆盖。
|
||||||
|
|
||||||
|
首选 ``<name>.bak.<suffix>``;若已存在,退化为 ``<name>.bak.<suffix>.2``、
|
||||||
|
``.3`` … 直到找到不存在的名字。保证连续 forced freeze(同 suffix 或都缺
|
||||||
|
manifest 用 'prev')不会静默覆盖此前保留的备份。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
path: 待备份的原文件路径。
|
||||||
|
suffix: 备份后缀(旧 pools_sha256 前 8 位或 'prev')。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
目录内唯一、尚不存在的备份路径。
|
||||||
|
"""
|
||||||
|
candidate = path.with_name(f"{path.name}.bak.{suffix}")
|
||||||
|
counter = 2
|
||||||
|
while candidate.exists():
|
||||||
|
candidate = path.with_name(f"{path.name}.bak.{suffix}.{counter}")
|
||||||
|
counter += 1
|
||||||
|
return candidate
|
||||||
|
|
||||||
|
|
||||||
|
def _guard_frozen_products(out_path: Path, manifest_path: Path, *, force: bool) -> None:
|
||||||
|
"""冻结前的覆盖保护:产物已存在时按 force 决定报错或备份。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
out_path: 目标 pools.json 路径。
|
||||||
|
manifest_path: 目标 split_manifest.json 路径。
|
||||||
|
force: False 时已存在即 FileExistsError;True 时把旧产物重命名为唯一的
|
||||||
|
.bak.<旧 pools_sha256 前 8 位或 'prev'>(同名已存在则追加递增序号)再放行。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileExistsError: force=False 且产物已存在(防静默覆盖冻结锚点)。
|
||||||
|
OSError: 备份 rename 失败;已备份的文件先 rollback 回原名再抛出,保证
|
||||||
|
要么两文件都备份、要么都不动(原子性,不留半备份的不一致目录)。
|
||||||
|
"""
|
||||||
|
if not out_path.exists() and not manifest_path.exists():
|
||||||
|
return
|
||||||
|
if not force:
|
||||||
|
raise FileExistsError(
|
||||||
|
f"已存在冻结产物 {out_path}(或其 manifest)。重跑切分会覆盖训练依赖的"
|
||||||
|
"冻结锚点——确认要替换请加 --force(旧产物将备份为 .bak.*)。"
|
||||||
|
)
|
||||||
|
# 备份后缀取旧 manifest 的 pools_sha256 前 8 位,无则用 'prev'
|
||||||
|
suffix = "prev"
|
||||||
|
if manifest_path.exists():
|
||||||
|
try:
|
||||||
|
old = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
suffix = str(old.get("pools_sha256", "prev"))[:8] or "prev"
|
||||||
|
except (json.JSONDecodeError, OSError):
|
||||||
|
suffix = "prev"
|
||||||
|
# 先为每个存在的文件求唯一备份路径(互不冲突),再逐个 rename;
|
||||||
|
# 中途失败则把已备份的 rollback 回原名,保证原子性。
|
||||||
|
to_backup = [p for p in (out_path, manifest_path) if p.exists()]
|
||||||
|
done: list[tuple[Path, Path]] = [] # (备份路径, 原路径),供 rollback
|
||||||
|
try:
|
||||||
|
for p in to_backup:
|
||||||
|
dst = _unique_backup_path(p, suffix)
|
||||||
|
p.rename(dst)
|
||||||
|
done.append((dst, p))
|
||||||
|
except OSError:
|
||||||
|
for backup_path, original in reversed(done):
|
||||||
|
backup_path.rename(original)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
def build_split(
|
def build_split(
|
||||||
*,
|
*,
|
||||||
db_path: Path,
|
db_path: Path,
|
||||||
@@ -112,6 +182,7 @@ def build_split(
|
|||||||
out_path: Path,
|
out_path: Path,
|
||||||
manifest_path: Path,
|
manifest_path: Path,
|
||||||
generated_at: str,
|
generated_at: str,
|
||||||
|
force: bool = False,
|
||||||
) -> SplitBuildResult:
|
) -> SplitBuildResult:
|
||||||
"""顶层编排结果驱动视频级切分,冻结 pools.json + manifest 并跑防御断言。
|
"""顶层编排结果驱动视频级切分,冻结 pools.json + manifest 并跑防御断言。
|
||||||
|
|
||||||
@@ -119,10 +190,10 @@ def build_split(
|
|||||||
test → 加载题库并以视频归属切三池 → 原子冻结 pools.json → 写溯源 manifest →
|
test → 加载题库并以视频归属切三池 → 原子冻结 pools.json → 写溯源 manifest →
|
||||||
六条防御断言 fail-fast 校验。
|
六条防御断言 fail-fast 校验。
|
||||||
|
|
||||||
契约(Task 11,非疏漏):build_split 有意保持 val_wrong_min-agnostic——内部调
|
契约(Task 11):val_wrong_min 前置到切分内保证功效——build_split 计算
|
||||||
split_by_video_assignment 时不传 val_wrong_min(默认 0,不校验 validation 错题
|
wrong_tier_by_video 并连同 config.val_wrong_min 传入 split_by_video_assignment,
|
||||||
数)。McNemar 功效护栏是切分**冻结后**的独立校验,由 CLI 的 check_mcnemar_power
|
切分时若 val 错题不足即从 diag 换入低 T2 错题组补足(耗尽 fail-loud)。CLI 的
|
||||||
在 build_split 返回后执行;切分构造本身不因功效阈失败,二者关注点分离。
|
check_mcnemar_power 作切分冻结后的冗余最终确认。
|
||||||
|
|
||||||
参数:
|
参数:
|
||||||
db_path: harness.db 路径(只读读取 predictions,不改动)。
|
db_path: harness.db 路径(只读读取 predictions,不改动)。
|
||||||
@@ -134,6 +205,9 @@ def build_split(
|
|||||||
out_path: 冻结 pools.json 目标路径(原子写)。
|
out_path: 冻结 pools.json 目标路径(原子写)。
|
||||||
manifest_path: 溯源 manifest 目标路径(原子写)。
|
manifest_path: 溯源 manifest 目标路径(原子写)。
|
||||||
generated_at: 生成时间戳(ISO 字符串),由调用方传入以保证可复现。
|
generated_at: 生成时间戳(ISO 字符串),由调用方传入以保证可复现。
|
||||||
|
force: 覆盖保护开关。False(默认)时若 out_path/manifest_path 已存在即
|
||||||
|
FileExistsError(防静默覆盖训练依赖的冻结锚点);True 时先把旧产物备份为
|
||||||
|
.bak.* 再放行覆盖。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
SplitBuildResult,含 pools / manifest / assignment,支持字典式访问。
|
SplitBuildResult,含 pools / manifest / assignment,支持字典式访问。
|
||||||
@@ -181,6 +255,11 @@ def build_split(
|
|||||||
# Phase 3: 加载题库 + 视频归属切三池 + 原子冻结。
|
# Phase 3: 加载题库 + 视频归属切三池 + 原子冻结。
|
||||||
questions = load_benchmark(questions_dir)
|
questions = load_benchmark(questions_dir)
|
||||||
correctness = {pred["question_id"]: pred["correct"] for pred in preds}
|
correctness = {pred["question_id"]: pred["correct"] for pred in preds}
|
||||||
|
tier_by_q = {row["question_id"]: row["tier"] for row in signal_rows}
|
||||||
|
wrong_tier_by_video: dict[str, int] = defaultdict(int)
|
||||||
|
for pred in preds:
|
||||||
|
if not pred["correct"] and tier_by_q.get(pred["question_id"]) == "T2":
|
||||||
|
wrong_tier_by_video[pred["video_id"]] += 1
|
||||||
pools = split_by_video_assignment(
|
pools = split_by_video_assignment(
|
||||||
questions,
|
questions,
|
||||||
assignment,
|
assignment,
|
||||||
@@ -188,7 +267,10 @@ def build_split(
|
|||||||
config.val_ratio,
|
config.val_ratio,
|
||||||
config.split_seed,
|
config.split_seed,
|
||||||
baseline_run_id=baseline_run_id,
|
baseline_run_id=baseline_run_id,
|
||||||
|
val_wrong_min=config.val_wrong_min,
|
||||||
|
wrong_tier_by_video=dict(wrong_tier_by_video),
|
||||||
)
|
)
|
||||||
|
_guard_frozen_products(out_path, manifest_path, force=force)
|
||||||
save_pools(pools, out_path)
|
save_pools(pools, out_path)
|
||||||
|
|
||||||
# Phase 4: 溯源 manifest(pools_sha256 锚定冻结内容)。
|
# Phase 4: 溯源 manifest(pools_sha256 锚定冻结内容)。
|
||||||
|
|||||||
@@ -41,6 +41,7 @@ _STRUCTURAL_KEYS = (
|
|||||||
"diag_size",
|
"diag_size",
|
||||||
"val_size",
|
"val_size",
|
||||||
"batch_correct_ratio",
|
"batch_correct_ratio",
|
||||||
|
"trainable_min_units",
|
||||||
)
|
)
|
||||||
|
|
||||||
_DECISION_KEYS = (
|
_DECISION_KEYS = (
|
||||||
@@ -94,7 +95,7 @@ def serialize_state(state: Any) -> dict[str, Any]:
|
|||||||
"eval_prev_run_id": state.eval_prev_run_id,
|
"eval_prev_run_id": state.eval_prev_run_id,
|
||||||
"baseline_skills_version": state.baseline_skills_version,
|
"baseline_skills_version": state.baseline_skills_version,
|
||||||
"baseline_prompts_version": state.baseline_prompts_version,
|
"baseline_prompts_version": state.baseline_prompts_version,
|
||||||
"steps_since_best_improved": state.steps_since_best_improved,
|
"epochs_since_best_improved": state.epochs_since_best_improved,
|
||||||
"epoch_start_skills": state.epoch_start_skills,
|
"epoch_start_skills": state.epoch_start_skills,
|
||||||
"changed_task_types_this_epoch": sorted(state.changed_task_types_this_epoch),
|
"changed_task_types_this_epoch": sorted(state.changed_task_types_this_epoch),
|
||||||
"rejected_buffer": {k: [asdict(x) for x in v] for k, v in state.rejected_buffer.items()},
|
"rejected_buffer": {k: [asdict(x) for x in v] for k, v in state.rejected_buffer.items()},
|
||||||
@@ -145,7 +146,7 @@ def deserialize_state_fields(d: dict[str, Any]) -> dict[str, Any]:
|
|||||||
"eval_prev_run_id": d["eval_prev_run_id"],
|
"eval_prev_run_id": d["eval_prev_run_id"],
|
||||||
"baseline_skills_version": d["baseline_skills_version"],
|
"baseline_skills_version": d["baseline_skills_version"],
|
||||||
"baseline_prompts_version": d["baseline_prompts_version"],
|
"baseline_prompts_version": d["baseline_prompts_version"],
|
||||||
"steps_since_best_improved": d["steps_since_best_improved"],
|
"epochs_since_best_improved": d["epochs_since_best_improved"],
|
||||||
"epoch_start_skills": d["epoch_start_skills"],
|
"epoch_start_skills": d["epoch_start_skills"],
|
||||||
"changed_task_types_this_epoch": set(d["changed_task_types_this_epoch"]),
|
"changed_task_types_this_epoch": set(d["changed_task_types_this_epoch"]),
|
||||||
"rejected_buffer": {
|
"rejected_buffer": {
|
||||||
|
|||||||
@@ -60,6 +60,7 @@ class RunConfig:
|
|||||||
batch_size: mini-batch 单批题目数。
|
batch_size: mini-batch 单批题目数。
|
||||||
min_class_per_batch: 单批中每个任务类型至少保留的题目数(< batch_size)。
|
min_class_per_batch: 单批中每个任务类型至少保留的题目数(< batch_size)。
|
||||||
eval_min_per_class: 验证池中每个任务类型至少保底的题目数。
|
eval_min_per_class: 验证池中每个任务类型至少保底的题目数。
|
||||||
|
trainable_min_units: 可训练性预检:每题型 diag+val 单元数下限,低于则剔除该题型。
|
||||||
early_stop_patience: 全局 best 连续未提升的容忍轮数,达到即早停。
|
early_stop_patience: 全局 best 连续未提升的容忍轮数,达到即早停。
|
||||||
test_size: held-out 测试池题目数。
|
test_size: held-out 测试池题目数。
|
||||||
use_slow_momentum: 是否启用快慢双速进化中的慢速 momentum 更新。
|
use_slow_momentum: 是否启用快慢双速进化中的慢速 momentum 更新。
|
||||||
@@ -114,6 +115,7 @@ class RunConfig:
|
|||||||
batch_size: int
|
batch_size: int
|
||||||
min_class_per_batch: int
|
min_class_per_batch: int
|
||||||
eval_min_per_class: int
|
eval_min_per_class: int
|
||||||
|
trainable_min_units: int
|
||||||
early_stop_patience: int
|
early_stop_patience: int
|
||||||
test_size: int
|
test_size: int
|
||||||
use_slow_momentum: bool
|
use_slow_momentum: bool
|
||||||
@@ -297,6 +299,8 @@ def _validate_minibatch(config: RunConfig) -> None:
|
|||||||
)
|
)
|
||||||
if config.eval_min_per_class < 1:
|
if config.eval_min_per_class < 1:
|
||||||
raise ValueError(f"eval_min_per_class 必须 >= 1,实际: {config.eval_min_per_class}")
|
raise ValueError(f"eval_min_per_class 必须 >= 1,实际: {config.eval_min_per_class}")
|
||||||
|
if config.trainable_min_units < 1:
|
||||||
|
raise ValueError(f"trainable_min_units 必须 >= 1,实际: {config.trainable_min_units}")
|
||||||
if config.pool_split_mode != "per_category":
|
if config.pool_split_mode != "per_category":
|
||||||
floor = config.eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT
|
floor = config.eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT
|
||||||
if config.val_size < floor:
|
if config.val_size < floor:
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
|
import sqlite3
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
@@ -162,6 +163,24 @@ def _to_text_field(value: Any) -> str:
|
|||||||
return json.dumps(value, ensure_ascii=False)
|
return json.dumps(value, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_prediction(answer: object) -> str | None:
|
||||||
|
"""归一化 prediction 落库值。
|
||||||
|
|
||||||
|
LLM 提交的 answer 有时是 list/dict(如 {'answer': ['B']}),sqlite 无法绑定
|
||||||
|
非标量类型直接入库会抛 ProgrammingError 击穿整轮 gather。None 保留(INFRA 空
|
||||||
|
预测语义,供正确率判定天然计错);str 原样;其余 JSON 序列化为文本。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
answer: LoopResult.result 中的 answer 原始值(可能是 None/str/list/dict)。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
None(保留空预测语义)或可直接入库的字符串。
|
||||||
|
"""
|
||||||
|
if answer is None or isinstance(answer, str):
|
||||||
|
return answer
|
||||||
|
return _to_text_field(answer)
|
||||||
|
|
||||||
|
|
||||||
def _zero_result(run_id: str) -> InferenceResult:
|
def _zero_result(run_id: str) -> InferenceResult:
|
||||||
"""空记录时的零值 InferenceResult。
|
"""空记录时的零值 InferenceResult。
|
||||||
|
|
||||||
@@ -369,6 +388,7 @@ async def _run_single_question(
|
|||||||
log: HarnessLog,
|
log: HarnessLog,
|
||||||
max_steps: int,
|
max_steps: int,
|
||||||
plugins: list[object],
|
plugins: list[object],
|
||||||
|
run_id: str,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""执行单道题目的 Agent 推理。
|
"""执行单道题目的 Agent 推理。
|
||||||
|
|
||||||
@@ -383,6 +403,8 @@ async def _run_single_question(
|
|||||||
log: HarnessLog 实例(线程安全)。
|
log: HarnessLog 实例(线程安全)。
|
||||||
max_steps: AgentLoop 最大步数。
|
max_steps: AgentLoop 最大步数。
|
||||||
plugins: pluggy 插件列表。
|
plugins: pluggy 插件列表。
|
||||||
|
run_id: 运行标识,用作 cache_salt——run_id 含 _e{epoch} 天然跨 epoch 重采样、
|
||||||
|
同 epoch 续跑命中缓存(算法 #10 透传)。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
预测结果字典(含 video_id, question_id, prediction, answer 等)。
|
预测结果字典(含 video_id, question_id, prediction, answer 等)。
|
||||||
@@ -412,6 +434,7 @@ async def _run_single_question(
|
|||||||
dispatcher,
|
dispatcher,
|
||||||
plugins=plugins,
|
plugins=plugins,
|
||||||
session_id=qa.question_id,
|
session_id=qa.question_id,
|
||||||
|
cache_salt=run_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
result_dict = loop_result.result if isinstance(loop_result.result, dict) else {}
|
result_dict = loop_result.result if isinstance(loop_result.result, dict) else {}
|
||||||
@@ -419,7 +442,7 @@ async def _run_single_question(
|
|||||||
reasoning = _to_text_field(result_dict.get("reasoning", ""))
|
reasoning = _to_text_field(result_dict.get("reasoning", ""))
|
||||||
record.update(
|
record.update(
|
||||||
{
|
{
|
||||||
"prediction": result_dict.get("answer"),
|
"prediction": _normalize_prediction(result_dict.get("answer")),
|
||||||
"evidence": evidence,
|
"evidence": evidence,
|
||||||
"reasoning": reasoning,
|
"reasoning": reasoning,
|
||||||
"steps_used": loop_result.steps_used,
|
"steps_used": loop_result.steps_used,
|
||||||
@@ -442,8 +465,18 @@ async def _run_single_question(
|
|||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("[{}] QA {} 执行异常", qa.video_id, qa.question_id)
|
logger.exception("[{}] QA {} 执行异常", qa.video_id, qa.question_id)
|
||||||
|
|
||||||
# prediction 必落库(try 外,无论成败)
|
# prediction 必落库(try 外,无论成败);绑定异常降级为最小 error 行,不击穿 gather
|
||||||
await asyncio.to_thread(log.insert, "predictions", record)
|
try:
|
||||||
|
await asyncio.to_thread(log.insert, "predictions", record)
|
||||||
|
except (sqlite3.InterfaceError, sqlite3.ProgrammingError):
|
||||||
|
logger.exception("[{}] QA {} 落库绑定异常,降级为 error 行", qa.video_id, qa.question_id)
|
||||||
|
record["prediction"] = None
|
||||||
|
record["stop_reason"] = "error"
|
||||||
|
await asyncio.to_thread(
|
||||||
|
log.insert,
|
||||||
|
"predictions",
|
||||||
|
{k: v for k, v in record.items() if isinstance(v, (str, int, float, type(None)))},
|
||||||
|
)
|
||||||
return record
|
return record
|
||||||
|
|
||||||
|
|
||||||
@@ -529,6 +562,7 @@ async def run_inference(
|
|||||||
log=log,
|
log=log,
|
||||||
max_steps=max_steps,
|
max_steps=max_steps,
|
||||||
plugins=plugins,
|
plugins=plugins,
|
||||||
|
run_id=run_id,
|
||||||
)
|
)
|
||||||
logger.info(
|
logger.info(
|
||||||
"[{}/{}] {} QA {} 完成 (stop={})",
|
"[{}/{}] {} QA {} 完成 (stop={})",
|
||||||
|
|||||||
+32
-18
@@ -52,6 +52,9 @@ class HarnessLog:
|
|||||||
run_id: 本次运行的唯一标识。
|
run_id: 本次运行的唯一标识。
|
||||||
git_sha: 代码版本,默认自动获取。
|
git_sha: 代码版本,默认自动获取。
|
||||||
config_snapshot: 本次运行的配置快照。
|
config_snapshot: 本次运行的配置快照。
|
||||||
|
register_run: 是否注册运行(upsert _runs + 退出时同步 status)。默认 True;
|
||||||
|
只读查询已有 run(如基线预测回读)时传 False,避免把该 run 的
|
||||||
|
started_at/config/status 改写、把基线元数据污染成本次进程的运行状态。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -60,27 +63,33 @@ class HarnessLog:
|
|||||||
run_id: str,
|
run_id: str,
|
||||||
git_sha: str | None = None,
|
git_sha: str | None = None,
|
||||||
config_snapshot: dict[str, Any] | None = None,
|
config_snapshot: dict[str, Any] | None = None,
|
||||||
|
*,
|
||||||
|
register_run: bool = True,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._run_id = run_id
|
self._run_id = run_id
|
||||||
|
self._register_run = register_run
|
||||||
Path(db_path).parent.mkdir(parents=True, exist_ok=True)
|
Path(db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
self._conn = sqlite3.connect(db_path, check_same_thread=False)
|
self._conn = sqlite3.connect(db_path, check_same_thread=False)
|
||||||
self._lock = threading.Lock()
|
self._lock = threading.Lock()
|
||||||
self._conn.row_factory = sqlite3.Row
|
self._conn.row_factory = sqlite3.Row
|
||||||
self._conn.execute("PRAGMA journal_mode=WAL")
|
self._conn.execute("PRAGMA journal_mode=WAL")
|
||||||
self._init_fixed_tables()
|
self._init_fixed_tables()
|
||||||
resolved_sha = git_sha or _get_git_sha()
|
if register_run:
|
||||||
config_json = json.dumps(config_snapshot, ensure_ascii=False) if config_snapshot else None
|
resolved_sha = git_sha or _get_git_sha()
|
||||||
self._conn.execute(
|
config_json = (
|
||||||
"INSERT INTO _runs"
|
json.dumps(config_snapshot, ensure_ascii=False) if config_snapshot else None
|
||||||
" (run_id, git_sha, started_at, config, status)"
|
)
|
||||||
" VALUES (?, ?, ?, ?, ?)"
|
self._conn.execute(
|
||||||
" ON CONFLICT(run_id) DO UPDATE SET"
|
"INSERT INTO _runs"
|
||||||
" started_at=excluded.started_at,"
|
" (run_id, git_sha, started_at, config, status)"
|
||||||
" config=excluded.config,"
|
" VALUES (?, ?, ?, ?, ?)"
|
||||||
" status=excluded.status",
|
" ON CONFLICT(run_id) DO UPDATE SET"
|
||||||
(run_id, resolved_sha, _now_iso(), config_json, "running"),
|
" started_at=excluded.started_at,"
|
||||||
)
|
" config=excluded.config,"
|
||||||
self._conn.commit()
|
" status=excluded.status",
|
||||||
|
(run_id, resolved_sha, _now_iso(), config_json, "running"),
|
||||||
|
)
|
||||||
|
self._conn.commit()
|
||||||
|
|
||||||
def _init_fixed_tables(self) -> None:
|
def _init_fixed_tables(self) -> None:
|
||||||
"""创建 _runs 和 _events 固定表。"""
|
"""创建 _runs 和 _events 固定表。"""
|
||||||
@@ -217,13 +226,18 @@ class HarnessLog:
|
|||||||
|
|
||||||
参数:
|
参数:
|
||||||
status: 最终状态,"completed" 或 "failed"。
|
status: 最终状态,"completed" 或 "failed"。
|
||||||
|
|
||||||
|
关键实现:
|
||||||
|
register_run=False(只读打开)时跳过 status 更新,仅关闭连接,
|
||||||
|
避免只读回读把已有 run 的 finished_at/status 改写。
|
||||||
"""
|
"""
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self._conn.execute(
|
if self._register_run:
|
||||||
"UPDATE _runs SET finished_at = ?, status = ? WHERE run_id = ?",
|
self._conn.execute(
|
||||||
(_now_iso(), status, self._run_id),
|
"UPDATE _runs SET finished_at = ?, status = ? WHERE run_id = ?",
|
||||||
)
|
(_now_iso(), status, self._run_id),
|
||||||
self._conn.commit()
|
)
|
||||||
|
self._conn.commit()
|
||||||
self._conn.close()
|
self._conn.close()
|
||||||
|
|
||||||
def __enter__(self) -> HarnessLog:
|
def __enter__(self) -> HarnessLog:
|
||||||
|
|||||||
@@ -135,7 +135,8 @@ def write_dual_metric(
|
|||||||
db_path: SQLite 路径。
|
db_path: SQLite 路径。
|
||||||
run_id: 训练 run ID。
|
run_id: 训练 run ID。
|
||||||
epoch: 轮次(1-based)。
|
epoch: 轮次(1-based)。
|
||||||
version_kind: baseline / best_hard / best_mixed / final。
|
version_kind: baseline / best_hard / best_mixed / final / slow_candidate
|
||||||
|
(slow_candidate = 慢更新 R2 可能被 revert 的候选,不占 epoch 终值 final 口径)。
|
||||||
skills_version / prompts_version: 评估的资源版本。
|
skills_version / prompts_version: 评估的资源版本。
|
||||||
pool: val / test。
|
pool: val / test。
|
||||||
hard_acc: hard 准确率。
|
hard_acc: hard 准确率。
|
||||||
|
|||||||
+75
-16
@@ -131,6 +131,7 @@ def split_by_video_assignment(
|
|||||||
seed: int,
|
seed: int,
|
||||||
baseline_run_id: str = "",
|
baseline_run_id: str = "",
|
||||||
val_wrong_min: int = 0,
|
val_wrong_min: int = 0,
|
||||||
|
wrong_tier_by_video: dict[str, int] | None = None,
|
||||||
) -> Pools:
|
) -> Pools:
|
||||||
"""按视频归属做原子切分:同一视频所有题绝不跨 trainval/test 池。
|
"""按视频归属做原子切分:同一视频所有题绝不跨 trainval/test 池。
|
||||||
|
|
||||||
@@ -146,7 +147,11 @@ def split_by_video_assignment(
|
|||||||
seed: 随机种子,保证视频组 shuffle 可复现。
|
seed: 随机种子,保证视频组 shuffle 可复现。
|
||||||
baseline_run_id: 基线 run 标识;离线切分阶段可留空,由调用方回填。
|
baseline_run_id: 基线 run 标识;离线切分阶段可留空,由调用方回填。
|
||||||
val_wrong_min: validation 池最少错题数(默认 0 = 不检查,保持既有调用契约)。
|
val_wrong_min: validation 池最少错题数(默认 0 = 不检查,保持既有调用契约)。
|
||||||
> 0 时切出 val 后统计其错题数,不足即 fail loud(见 InsufficientValSignal)。
|
> 0 时切分时保证(不足则从 diag 换入低 T2 错题组补足,耗尽 fail-loud,
|
||||||
|
见 InsufficientValSignal)。
|
||||||
|
wrong_tier_by_video: video_id -> 该视频错题中 T2(defect) 数量;透传给
|
||||||
|
_split_trainval_by_video_group 做 tier 感知 diag/val 分配,None 时退化为
|
||||||
|
原随机 shuffle。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
冻结的三池 Pools:diagnosis/validation 仍是逐题 GeneratedQuestion 列表
|
冻结的三池 Pools:diagnosis/validation 仍是逐题 GeneratedQuestion 列表
|
||||||
@@ -173,18 +178,11 @@ def split_by_video_assignment(
|
|||||||
trainval_qs, test_qs = _partition_by_video_assignment(questions, assignment, correctness)
|
trainval_qs, test_qs = _partition_by_video_assignment(questions, assignment, correctness)
|
||||||
|
|
||||||
diagnosis, validation = _split_trainval_by_video_group(
|
diagnosis, validation = _split_trainval_by_video_group(
|
||||||
trainval_qs, correctness, val_ratio, random.Random(seed)
|
trainval_qs, correctness, val_ratio, random.Random(seed),
|
||||||
|
wrong_tier_by_video=wrong_tier_by_video,
|
||||||
|
val_wrong_min=val_wrong_min,
|
||||||
)
|
)
|
||||||
|
|
||||||
if val_wrong_min > 0:
|
|
||||||
val_wrong = sum(1 for q in validation if not correctness[q.question_id])
|
|
||||||
if val_wrong < val_wrong_min:
|
|
||||||
raise InsufficientValSignal(
|
|
||||||
f"validation 池错题数 {val_wrong} < val_wrong_min={val_wrong_min},"
|
|
||||||
"验证信号不足以支撑可靠比较(如 McNemar 检验功效),"
|
|
||||||
"请放大 val_ratio / 调整 trainval 归属或调低阈值。"
|
|
||||||
)
|
|
||||||
|
|
||||||
val_correct = sum(1 for q in validation if correctness.get(q.question_id))
|
val_correct = sum(1 for q in validation if correctness.get(q.question_id))
|
||||||
baseline_val_accuracy = val_correct / len(validation) if validation else 0.0
|
baseline_val_accuracy = val_correct / len(validation) if validation else 0.0
|
||||||
return Pools(
|
return Pools(
|
||||||
@@ -291,6 +289,8 @@ def _split_trainval_by_video_group(
|
|||||||
correctness: dict[str, bool],
|
correctness: dict[str, bool],
|
||||||
val_ratio: float,
|
val_ratio: float,
|
||||||
rng: random.Random,
|
rng: random.Random,
|
||||||
|
wrong_tier_by_video: dict[str, int] | None = None,
|
||||||
|
val_wrong_min: int = 0,
|
||||||
) -> tuple[list[GeneratedQuestion], list[GeneratedQuestion]]:
|
) -> tuple[list[GeneratedQuestion], list[GeneratedQuestion]]:
|
||||||
"""以视频组为原子对 trainval 题集做 correctness 分层,切出 (diagnosis, validation)。
|
"""以视频组为原子对 trainval 题集做 correctness 分层,切出 (diagnosis, validation)。
|
||||||
|
|
||||||
@@ -299,6 +299,12 @@ def _split_trainval_by_video_group(
|
|||||||
correctness: question_id -> 基线是否答对;视频组正确性取组内全部题 AND。
|
correctness: question_id -> 基线是否答对;视频组正确性取组内全部题 AND。
|
||||||
val_ratio: validation 占视频组总数的比例。
|
val_ratio: validation 占视频组总数的比例。
|
||||||
rng: 随机数生成器,保证视频组 shuffle 可复现。
|
rng: 随机数生成器,保证视频组 shuffle 可复现。
|
||||||
|
wrong_tier_by_video: video_id -> 该视频错题中 T2(defect) 的数量。提供时错题
|
||||||
|
视频组按 T2 含量升序进 val(T2 高的组保留在 diagnosis,把高价值缺陷信号
|
||||||
|
留给诊断),确定性排序取代随机 shuffle;None 时退化为原随机 shuffle。
|
||||||
|
val_wrong_min: validation 池最少错题数(切分时保证功效)。> 0 且初分 val 错题
|
||||||
|
不足时,从 diag 侧的错题组按 T2 升序换入 val 直到满足(每组至多移动一次),
|
||||||
|
耗尽仍不足则抛 InsufficientValSignal(fail loud,P5)。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
(diagnosis, validation) 逐题列表元组;同一 video 的全部题整组落在同一侧,
|
(diagnosis, validation) 逐题列表元组;同一 video 的全部题整组落在同一侧,
|
||||||
@@ -306,7 +312,8 @@ def _split_trainval_by_video_group(
|
|||||||
|
|
||||||
关键实现细节:
|
关键实现细节:
|
||||||
与 _split_one_category 同构:先按视频组 correctness 分正确组/错误组,按比例
|
与 _split_one_category 同构:先按视频组 correctness 分正确组/错误组,按比例
|
||||||
把 n_val 个组分层落入 validation(全正确或全错误时退化为非分层随机划分),
|
把 n_val 个组分层落入 validation(全正确退化为非分层随机划分;全错误时若有
|
||||||
|
wrong_tier_by_video 仍按 T2 升序分配,否则随机划分),
|
||||||
再把选中组内所有题展开。视频组按 video_id 排序后再 shuffle,保证确定性。
|
再把选中组内所有题展开。视频组按 video_id 排序后再 shuffle,保证确定性。
|
||||||
"""
|
"""
|
||||||
groups: dict[str, list[GeneratedQuestion]] = defaultdict(list)
|
groups: dict[str, list[GeneratedQuestion]] = defaultdict(list)
|
||||||
@@ -320,7 +327,11 @@ def _split_trainval_by_video_group(
|
|||||||
correct_vids, wrong_vids = _partition_video_groups_by_correctness(groups, correctness)
|
correct_vids, wrong_vids = _partition_video_groups_by_correctness(groups, correctness)
|
||||||
n_correct = len(correct_vids)
|
n_correct = len(correct_vids)
|
||||||
|
|
||||||
if n_correct == 0 or n_correct == n_total:
|
if n_correct == 0 and wrong_tier_by_video is not None:
|
||||||
|
# 全部错误 + 有 tier 信号:按 T2 升序,低 T2 组优先进 val(保留高 T2 在 diag)
|
||||||
|
wrong_vids.sort(key=lambda v: (wrong_tier_by_video.get(v, 0), v))
|
||||||
|
val_vids = set(wrong_vids[:n_val])
|
||||||
|
elif n_correct == 0 or n_correct == n_total:
|
||||||
label = "全部正确" if n_correct == n_total else "全部错误"
|
label = "全部正确" if n_correct == n_total else "全部错误"
|
||||||
logger.warning("trainval 视频组 {} ({} 组),退化为非分层随机划分", label, n_total)
|
logger.warning("trainval 视频组 {} ({} 组),退化为非分层随机划分", label, n_total)
|
||||||
shuffled = list(video_ids)
|
shuffled = list(video_ids)
|
||||||
@@ -330,9 +341,33 @@ def _split_trainval_by_video_group(
|
|||||||
val_correct = math.floor(n_correct * n_val / n_total)
|
val_correct = math.floor(n_correct * n_val / n_total)
|
||||||
val_wrong = n_val - val_correct
|
val_wrong = n_val - val_correct
|
||||||
rng.shuffle(correct_vids)
|
rng.shuffle(correct_vids)
|
||||||
rng.shuffle(wrong_vids)
|
if wrong_tier_by_video is None:
|
||||||
|
rng.shuffle(wrong_vids)
|
||||||
|
else:
|
||||||
|
# T2 少的错题组优先进 val(保留 T2 高的组在 diag),确定性排序
|
||||||
|
wrong_vids.sort(key=lambda v: (wrong_tier_by_video.get(v, 0), v))
|
||||||
val_vids = set(correct_vids[:val_correct] + wrong_vids[:val_wrong])
|
val_vids = set(correct_vids[:val_correct] + wrong_vids[:val_wrong])
|
||||||
|
|
||||||
|
if val_wrong_min > 0:
|
||||||
|
val_wrong_now = sum(
|
||||||
|
1 for v in val_vids for q in groups[v] if not correctness[q.question_id]
|
||||||
|
)
|
||||||
|
# diag 侧仍在的错题组,按 T2 升序(低价值优先移交 val)
|
||||||
|
diag_wrong_pool = sorted(
|
||||||
|
(v for v in wrong_vids if v not in val_vids),
|
||||||
|
key=lambda v: ((wrong_tier_by_video or {}).get(v, 0), v),
|
||||||
|
)
|
||||||
|
for v in diag_wrong_pool:
|
||||||
|
if val_wrong_now >= val_wrong_min:
|
||||||
|
break
|
||||||
|
val_vids.add(v)
|
||||||
|
val_wrong_now += sum(1 for q in groups[v] if not correctness[q.question_id])
|
||||||
|
if val_wrong_now < val_wrong_min:
|
||||||
|
raise InsufficientValSignal(
|
||||||
|
f"trainval 错题不足以让 val 达到 val_wrong_min={val_wrong_min}"
|
||||||
|
f"(修复后仅 {val_wrong_now}),请放大 val_ratio 或调整 trainval 归属。"
|
||||||
|
)
|
||||||
|
|
||||||
diagnosis = [q for q in trainval_qs if q.video_id not in val_vids]
|
diagnosis = [q for q in trainval_qs if q.video_id not in val_vids]
|
||||||
validation = [q for q in trainval_qs if q.video_id in val_vids]
|
validation = [q for q in trainval_qs if q.video_id in val_vids]
|
||||||
return diagnosis, validation
|
return diagnosis, validation
|
||||||
@@ -762,7 +797,9 @@ def build_or_load_pools(
|
|||||||
# 增量构建新类别
|
# 增量构建新类别
|
||||||
paths = resolve_paths(config.workspace_dir)
|
paths = resolve_paths(config.workspace_dir)
|
||||||
questions = load_benchmark(paths.questions_dir)
|
questions = load_benchmark(paths.questions_dir)
|
||||||
with HarnessLog(str(db_path), baseline_run_id) as hlog:
|
with HarnessLog(
|
||||||
|
str(db_path), baseline_run_id, register_run=False
|
||||||
|
) as hlog:
|
||||||
rows = hlog.query(
|
rows = hlog.query(
|
||||||
"SELECT question_id, prediction, answer "
|
"SELECT question_id, prediction, answer "
|
||||||
"FROM predictions WHERE run_id=?",
|
"FROM predictions WHERE run_id=?",
|
||||||
@@ -809,13 +846,35 @@ def build_or_load_pools(
|
|||||||
len(new_types),
|
len(new_types),
|
||||||
sorted(new_types),
|
sorted(new_types),
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
# global:校验 baseline_run_id 与(若有)manifest 内容指纹,
|
||||||
|
# 拒绝静默加载与 seed 错配 / 被篡改的冻结切分(P5 fail loud)。
|
||||||
|
frozen_baseline = raw.get("baseline_run_id")
|
||||||
|
if frozen_baseline != baseline_run_id:
|
||||||
|
raise ValueError(
|
||||||
|
f"冻结 pools.json 的 baseline_run_id={frozen_baseline!r} 与 seed "
|
||||||
|
f"的 {baseline_run_id!r} 不一致,拒绝静默加载错配切分。"
|
||||||
|
)
|
||||||
|
manifest_path = config.workspace_dir / "split_manifest.json"
|
||||||
|
if manifest_path.exists():
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
actual_sha = hashlib.sha256(
|
||||||
|
pools_path.read_text(encoding="utf-8").encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
if manifest.get("pools_sha256") != actual_sha:
|
||||||
|
raise ValueError(
|
||||||
|
"pools.json 内容指纹与 split_manifest.pools_sha256 不符,"
|
||||||
|
"冻结产物疑被篡改,拒绝加载。"
|
||||||
|
)
|
||||||
|
|
||||||
return load_pools(pools_path)
|
return load_pools(pools_path)
|
||||||
|
|
||||||
# ── 全新构建 ──
|
# ── 全新构建 ──
|
||||||
paths = resolve_paths(config.workspace_dir)
|
paths = resolve_paths(config.workspace_dir)
|
||||||
questions = load_benchmark(paths.questions_dir)
|
questions = load_benchmark(paths.questions_dir)
|
||||||
with HarnessLog(str(db_path), baseline_run_id) as hlog:
|
with HarnessLog(str(db_path), baseline_run_id, register_run=False) as hlog:
|
||||||
rows = hlog.query(
|
rows = hlog.query(
|
||||||
"SELECT question_id, prediction, answer FROM predictions WHERE run_id=?",
|
"SELECT question_id, prediction, answer FROM predictions WHERE run_id=?",
|
||||||
(baseline_run_id,),
|
(baseline_run_id,),
|
||||||
|
|||||||
+225
-44
@@ -18,7 +18,8 @@ import random
|
|||||||
import shutil
|
import shutil
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import tempfile
|
import tempfile
|
||||||
from dataclasses import dataclass, field
|
from collections import Counter
|
||||||
|
from dataclasses import dataclass, field, replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
@@ -115,10 +116,13 @@ class _TrainState:
|
|||||||
global_step: int = 0
|
global_step: int = 0
|
||||||
changed_task_types_this_epoch: set[str] = field(default_factory=set)
|
changed_task_types_this_epoch: set[str] = field(default_factory=set)
|
||||||
epoch_start_skills: dict[str, str] = field(default_factory=dict)
|
epoch_start_skills: dict[str, str] = field(default_factory=dict)
|
||||||
steps_since_best_improved: int = 0
|
epochs_since_best_improved: int = 0
|
||||||
gate_epoch_observed: bool = False
|
gate_epoch_observed: bool = False
|
||||||
probations: dict[str, Probation] = field(default_factory=dict)
|
probations: dict[str, Probation] = field(default_factory=dict)
|
||||||
gate_cooldown: dict[str, int] = field(default_factory=dict)
|
gate_cooldown: dict[str, int] = field(default_factory=dict)
|
||||||
|
# 进程内 holdout 去重备忘录 (skills_v, prompts_v) -> test 评估结果;不进 checkpoint,
|
||||||
|
# resume 后清空(重评一次是可接受代价,换取零 schema 变更)。
|
||||||
|
holdout_memo: dict[tuple[str, str], InferenceResult] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -288,21 +292,82 @@ def _snapshot_current_skills(skills_dir: Path) -> dict[str, str]:
|
|||||||
return snapshot
|
return snapshot
|
||||||
|
|
||||||
|
|
||||||
|
def _filter_untrainable_types(
|
||||||
|
pools: Pools,
|
||||||
|
task_types: list[str] | None,
|
||||||
|
eval_min_per_class: int,
|
||||||
|
trainable_min_units: int,
|
||||||
|
) -> tuple[Pools, list[str] | None]:
|
||||||
|
"""剔除不可训练题型(val 单元<eval_min_per_class 或 diag+val 单元<trainable_min_units)。
|
||||||
|
|
||||||
|
计数以**单元(unit)**为原子:AR pair 孪生对折叠计 1 个单元(等于 gate 阶梯该类
|
||||||
|
候选数),非按题目计数——否则 pair 题型会以 2 倍题目数误通过阈值。test 池不过滤
|
||||||
|
(继续报告全题型准确率)。在 gate 建立前调用,避免样本不足的微型题型进入信息量
|
||||||
|
阶梯导致门控崩溃。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
pools: 冻结三池。
|
||||||
|
task_types: 显式题型子集(None 表示全部),过滤后按 keep 收窄。
|
||||||
|
eval_min_per_class: 验证池每类保底单元数下限。
|
||||||
|
trainable_min_units: 每类可训练所需最小 diag+val 单元数。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
过滤后的 (pools, task_types):pools.diagnosis/validation 仅保留 keep 题型,
|
||||||
|
test 原样;task_types 收窄为 keep(原 None 时返回 sorted(keep))。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
RuntimeError: 过滤后无任何可训练题型(切分/阈值需调整,fail-fast 不空转训练)。
|
||||||
|
"""
|
||||||
|
diag_by_type = Counter(u.task_type for u in build_units(pools.diagnosis))
|
||||||
|
val_by_type = Counter(u.task_type for u in build_units(pools.validation))
|
||||||
|
# 先按调用方显式 task_types 收窄候选集:未请求的题型(哪怕可训练)不得进入
|
||||||
|
# keep,否则冻结全局 pools 后 batch/diagnosis 会训练非请求题型,而 gate 只覆盖
|
||||||
|
# 请求题型 → 静默语义偏差(I-4)。task_types=None 表示全部题型皆为候选。
|
||||||
|
candidates = set(diag_by_type) | set(val_by_type)
|
||||||
|
if task_types is not None:
|
||||||
|
candidates &= set(task_types)
|
||||||
|
keep: set[str] = set()
|
||||||
|
dropped: list[tuple[str, str]] = []
|
||||||
|
for tt in candidates:
|
||||||
|
n_val = val_by_type.get(tt, 0)
|
||||||
|
n_units = diag_by_type.get(tt, 0) + n_val
|
||||||
|
if n_val < eval_min_per_class:
|
||||||
|
dropped.append((tt, f"val_units={n_val}<{eval_min_per_class}"))
|
||||||
|
elif n_units < trainable_min_units:
|
||||||
|
dropped.append((tt, f"units={n_units}<{trainable_min_units}"))
|
||||||
|
else:
|
||||||
|
keep.add(tt)
|
||||||
|
for tt, why in sorted(dropped):
|
||||||
|
logger.warning("可训练性预检剔除题型 {}({})", tt, why)
|
||||||
|
if not keep:
|
||||||
|
detail = ";".join(f"{tt}({why})" for tt, why in sorted(dropped))
|
||||||
|
raise RuntimeError(
|
||||||
|
"可训练性预检剔除了全部题型,无题型满足 "
|
||||||
|
f"val_units>={eval_min_per_class} 且 units>={trainable_min_units}:"
|
||||||
|
f"{detail}。请调整池切分或降低阈值。"
|
||||||
|
)
|
||||||
|
new_pools = replace(
|
||||||
|
pools,
|
||||||
|
diagnosis=[q for q in pools.diagnosis if q.task_type in keep],
|
||||||
|
validation=[q for q in pools.validation if q.task_type in keep],
|
||||||
|
)
|
||||||
|
new_types = [t for t in task_types if t in keep] if task_types is not None else sorted(keep)
|
||||||
|
return new_pools, new_types
|
||||||
|
|
||||||
|
|
||||||
def _should_early_stop(
|
def _should_early_stop(
|
||||||
workspace_dir: Path,
|
workspace_dir: Path,
|
||||||
epoch: int,
|
epoch: int,
|
||||||
steps_this_epoch: int,
|
|
||||||
state: _TrainState,
|
state: _TrainState,
|
||||||
patience: int,
|
patience: int,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""步粒度 early stop:本 epoch best 未刷新则累加本 epoch 步数。
|
"""epoch 粒度 early stop:本 epoch best 未刷新则计数 +1。
|
||||||
|
|
||||||
参数:
|
参数:
|
||||||
workspace_dir: workspace 目录(读 manifest best)。
|
workspace_dir: workspace 目录(读 manifest best)。
|
||||||
epoch: 当前 epoch。
|
epoch: 当前 epoch。
|
||||||
steps_this_epoch: 本 epoch 的 step 总数。
|
state: 训练状态(epochs_since_best_improved 就地更新)。
|
||||||
state: 训练状态(steps_since_best_improved 就地更新)。
|
patience: early_stop_patience(连续无刷新的 epoch 数上限)。
|
||||||
patience: early_stop_patience。
|
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
是否触发 early stop。
|
是否触发 early stop。
|
||||||
@@ -310,10 +375,10 @@ def _should_early_stop(
|
|||||||
best = read_best(workspace_dir)
|
best = read_best(workspace_dir)
|
||||||
improved_this_epoch = best is not None and best.get("epoch") == epoch
|
improved_this_epoch = best is not None and best.get("epoch") == epoch
|
||||||
if improved_this_epoch:
|
if improved_this_epoch:
|
||||||
state.steps_since_best_improved = 0
|
state.epochs_since_best_improved = 0
|
||||||
return False
|
return False
|
||||||
state.steps_since_best_improved += steps_this_epoch
|
state.epochs_since_best_improved += 1
|
||||||
return state.steps_since_best_improved >= patience
|
return state.epochs_since_best_improved >= patience
|
||||||
|
|
||||||
|
|
||||||
def _compute_total_steps(pools: Pools, correctness: dict[str, bool], config: RunConfig) -> int:
|
def _compute_total_steps(pools: Pools, correctness: dict[str, bool], config: RunConfig) -> int:
|
||||||
@@ -792,8 +857,20 @@ class Runner:
|
|||||||
三级嵌套:epoch → batch(step) → per-skill。
|
三级嵌套:epoch → batch(step) → per-skill。
|
||||||
epoch 末 _slow_update_cycle 十步序。
|
epoch 末 _slow_update_cycle 十步序。
|
||||||
训练收尾 _deliver_best + _final_test_eval。
|
训练收尾 _deliver_best + _final_test_eval。
|
||||||
|
|
||||||
|
入口第一步做可训练性预检:剔除样本不足的微型题型(防 gate 阶梯崩溃),
|
||||||
|
过滤后的 pools 贯穿 batch/step/slow-update/final-eval 全部消费;filtered_task_types
|
||||||
|
透传到 gate 建立(不改 frozen RunConfig)。
|
||||||
"""
|
"""
|
||||||
state, total_steps, plan, saved_batches = await self._setup_train_run(pools)
|
pools, filtered_task_types = _filter_untrainable_types(
|
||||||
|
pools,
|
||||||
|
list(self._config.task_types) if self._config.task_types is not None else None,
|
||||||
|
self._config.eval_min_per_class,
|
||||||
|
self._config.trainable_min_units,
|
||||||
|
)
|
||||||
|
state, total_steps, plan, saved_batches = await self._setup_train_run(
|
||||||
|
pools, filtered_task_types
|
||||||
|
)
|
||||||
for epoch in range(plan["first_epoch"], self._config.epochs + 1):
|
for epoch in range(plan["first_epoch"], self._config.epochs + 1):
|
||||||
if epoch == plan["resume_epoch"]:
|
if epoch == plan["resume_epoch"]:
|
||||||
batches = [_batch_from_ids(pools, ids) for ids in saved_batches]
|
batches = [_batch_from_ids(pools, ids) for ids in saved_batches]
|
||||||
@@ -830,26 +907,19 @@ class Runner:
|
|||||||
epoch_batches=batch_unit_ids,
|
epoch_batches=batch_unit_ids,
|
||||||
config=self._config,
|
config=self._config,
|
||||||
)
|
)
|
||||||
await self._slow_update_cycle(epoch, pools, state)
|
# checkpoint 落盘移入 _slow_update_cycle 末尾(gate_pools.save 之后立即写),
|
||||||
state.system_packs = []
|
# 消除 gate_epoch_observed 在 gate_pools.json 与 checkpoint 间的双计窗口。
|
||||||
state.tool_packs = []
|
await self._slow_update_cycle(
|
||||||
state.changed_task_types_this_epoch = set()
|
epoch,
|
||||||
write_checkpoint(
|
pools,
|
||||||
self._config.workspace_dir,
|
state,
|
||||||
state=state,
|
|
||||||
epoch=epoch,
|
|
||||||
step_completed=len(batches) - 1,
|
|
||||||
phase="epoch_done",
|
|
||||||
global_step=state.global_step,
|
|
||||||
total_steps=total_steps,
|
total_steps=total_steps,
|
||||||
version_snapshot=self._current_version_snapshot(),
|
step_completed=len(batches) - 1,
|
||||||
epoch_batches=batch_unit_ids,
|
epoch_batches=batch_unit_ids,
|
||||||
config=self._config,
|
|
||||||
)
|
)
|
||||||
if _should_early_stop(
|
if _should_early_stop(
|
||||||
self._config.workspace_dir,
|
self._config.workspace_dir,
|
||||||
epoch,
|
epoch,
|
||||||
len(batches),
|
|
||||||
state,
|
state,
|
||||||
self._config.early_stop_patience,
|
self._config.early_stop_patience,
|
||||||
):
|
):
|
||||||
@@ -862,16 +932,22 @@ class Runner:
|
|||||||
# 训练初始化
|
# 训练初始化
|
||||||
# -----------------------------------------------------------------------
|
# -----------------------------------------------------------------------
|
||||||
|
|
||||||
async def _setup_train_run(self, pools: Pools) -> tuple[_TrainState, int, dict, list | None]:
|
async def _setup_train_run(
|
||||||
|
self, pools: Pools, filtered_task_types: list[str] | None
|
||||||
|
) -> tuple[_TrainState, int, dict, list | None]:
|
||||||
"""据是否 --resume 准备训练起点。
|
"""据是否 --resume 准备训练起点。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
pools: 已过可训练性预检的三池。
|
||||||
|
filtered_task_types: 预检后保留的题型(None 表示不限,由 gate 从 diag 推导)。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
(state, total_steps, plan, saved_batches)。
|
(state, total_steps, plan, saved_batches)。
|
||||||
"""
|
"""
|
||||||
ckpt = load_checkpoint(self._config.workspace_dir) if self._config.resume else None
|
ckpt = load_checkpoint(self._config.workspace_dir) if self._config.resume else None
|
||||||
if self._config.resume and ckpt is None:
|
if self._config.resume and ckpt is None:
|
||||||
raise RuntimeError("--resume 但 checkpoint.json 不存在,拒绝静默从头重训")
|
raise RuntimeError("--resume 但 checkpoint.json 不存在,拒绝静默从头重训")
|
||||||
gate_pools, baseline_cache = self._init_gate_pools(pools)
|
gate_pools, baseline_cache = self._init_gate_pools(pools, filtered_task_types)
|
||||||
if not ckpt:
|
if not ckpt:
|
||||||
state = self._init_train_state(pools, gate_pools, baseline_cache)
|
state = self._init_train_state(pools, gate_pools, baseline_cache)
|
||||||
total_steps = _compute_total_steps(pools, state.correctness, self._config)
|
total_steps = _compute_total_steps(pools, state.correctness, self._config)
|
||||||
@@ -897,13 +973,16 @@ class Runner:
|
|||||||
)
|
)
|
||||||
return state, ckpt["progress"]["total_steps"], plan, ckpt["epoch_batches"]
|
return state, ckpt["progress"]["total_steps"], plan, ckpt["epoch_batches"]
|
||||||
|
|
||||||
def _init_gate_pools(self, pools: Pools) -> tuple[GatePools, BaselineCache]:
|
def _init_gate_pools(
|
||||||
|
self, pools: Pools, filtered_task_types: list[str] | None
|
||||||
|
) -> tuple[GatePools, BaselineCache]:
|
||||||
"""构建/加载 CE-Gate 信息量阶梯与基线缓存。
|
"""构建/加载 CE-Gate 信息量阶梯与基线缓存。
|
||||||
|
|
||||||
副作用:设置 self._gate_questions_by_id(不进 checkpoint)。
|
副作用:设置 self._gate_questions_by_id(不进 checkpoint)。
|
||||||
|
|
||||||
参数:
|
参数:
|
||||||
pools: 冻结三池。
|
pools: 冻结三池(已过可训练性预检)。
|
||||||
|
filtered_task_types: 预检保留的题型;None 时从 pools.diagnosis 推导。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
(GatePools, BaselineCache)。
|
(GatePools, BaselineCache)。
|
||||||
@@ -920,7 +999,9 @@ class Runner:
|
|||||||
self._gate_units_by_id: dict[str, QuestionUnit] = {
|
self._gate_units_by_id: dict[str, QuestionUnit] = {
|
||||||
u.unit_id: u for u in build_units(questions)
|
u.unit_id: u for u in build_units(questions)
|
||||||
}
|
}
|
||||||
with HarnessLog(str(self._paths.db_path), pools.baseline_run_id) as log:
|
with HarnessLog(
|
||||||
|
str(self._paths.db_path), pools.baseline_run_id, register_run=False
|
||||||
|
) as log:
|
||||||
rows = log.query(
|
rows = log.query(
|
||||||
"SELECT question_id, prediction, answer FROM predictions WHERE run_id=?",
|
"SELECT question_id, prediction, answer FROM predictions WHERE run_id=?",
|
||||||
(pools.baseline_run_id,),
|
(pools.baseline_run_id,),
|
||||||
@@ -932,7 +1013,12 @@ class Runner:
|
|||||||
)
|
)
|
||||||
baseline_correctness = {r["question_id"]: r["prediction"] == r["answer"] for r in rows}
|
baseline_correctness = {r["question_id"]: r["prediction"] == r["answer"] for r in rows}
|
||||||
logger.info("gate 阶梯基线对错覆盖 {} 题", len(baseline_correctness))
|
logger.info("gate 阶梯基线对错覆盖 {} 题", len(baseline_correctness))
|
||||||
gate_task_types = sorted({q.task_type for q in pools.diagnosis})
|
# 预检保留的题型优先;None 时从(已过滤的)诊断池推导,二者一致
|
||||||
|
gate_task_types = (
|
||||||
|
sorted(filtered_task_types)
|
||||||
|
if filtered_task_types is not None
|
||||||
|
else sorted({q.task_type for q in pools.diagnosis})
|
||||||
|
)
|
||||||
gate_pools = build_or_load_gate_pools(
|
gate_pools = build_or_load_gate_pools(
|
||||||
workspace_dir=self._config.workspace_dir,
|
workspace_dir=self._config.workspace_dir,
|
||||||
questions=questions,
|
questions=questions,
|
||||||
@@ -1009,14 +1095,32 @@ class Runner:
|
|||||||
) -> None:
|
) -> None:
|
||||||
"""单 step:rollout → correctness 增量 → 诊断 → 累加 system/tool → 按类 gate。"""
|
"""单 step:rollout → correctness 增量 → 诊断 → 累加 system/tool → 按类 gate。"""
|
||||||
run_id = f"{pools.baseline_run_id}_e{epoch}_s{step}"
|
run_id = f"{pools.baseline_run_id}_e{epoch}_s{step}"
|
||||||
await self._rollout_batch(batch, run_id)
|
|
||||||
|
|
||||||
|
from app.harness.inference import PREDICTIONS_SCHEMA, TRACES_SCHEMA
|
||||||
from app.harness.log import HarnessLog
|
from app.harness.log import HarnessLog
|
||||||
|
|
||||||
|
# 幂等:重跑同一 step 前先清旧行,避免断点续跑重复累计双计。
|
||||||
|
# 先 CREATE TABLE IF NOT EXISTS(fresh workspace 首跑时表尚未由 run_inference 建),
|
||||||
|
# register_run=False 避免只读清理污染 _runs 运行状态。
|
||||||
|
with HarnessLog(str(self._paths.db_path), run_id, register_run=False) as log:
|
||||||
|
log.create_table("predictions", PREDICTIONS_SCHEMA)
|
||||||
|
log.create_table("traces", TRACES_SCHEMA)
|
||||||
|
log.execute("DELETE FROM predictions WHERE run_id=?", (run_id,))
|
||||||
|
log.execute("DELETE FROM traces WHERE run_id=?", (run_id,))
|
||||||
|
|
||||||
|
await self._rollout_batch(batch, run_id)
|
||||||
|
|
||||||
with HarnessLog(str(self._paths.db_path), run_id) as log:
|
with HarnessLog(str(self._paths.db_path), run_id) as log:
|
||||||
_apply_batch_correctness(state.correctness, log, run_id, batch)
|
_apply_batch_correctness(state.correctness, log, run_id, batch)
|
||||||
|
|
||||||
diagnosis = await self._run_diagnosis(run_id, question_ids=[q.question_id for q in batch])
|
diagnosis = await self._run_diagnosis(run_id, question_ids=[q.question_id for q in batch])
|
||||||
|
# 降级占比过高疑似 judge 基础设施故障:不以降级信号驱动进化,直接中止
|
||||||
|
n_wrong = sum(1 for q in batch if not state.correctness.get(q.question_id, True))
|
||||||
|
if n_wrong > 0 and diagnosis.degraded_count / n_wrong > 0.5:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"本 step 诊断降级占比 {diagnosis.degraded_count}/{n_wrong} > 50%,"
|
||||||
|
"疑似 judge 基础设施故障,中止训练(不以降级信号驱动进化)。"
|
||||||
|
)
|
||||||
_accumulate_slow_packs(diagnosis, state)
|
_accumulate_slow_packs(diagnosis, state)
|
||||||
await self._gate_batch_skills(epoch, step, diagnosis, total_steps, pools, state)
|
await self._gate_batch_skills(epoch, step, diagnosis, total_steps, pools, state)
|
||||||
# 冷却计数每 step 递减、归零剔除
|
# 冷却计数每 step 递减、归零剔除
|
||||||
@@ -1375,7 +1479,16 @@ class Runner:
|
|||||||
# _slow_update_cycle 十步序
|
# _slow_update_cycle 十步序
|
||||||
# -----------------------------------------------------------------------
|
# -----------------------------------------------------------------------
|
||||||
|
|
||||||
async def _slow_update_cycle(self, epoch: int, pools: Pools, state: _TrainState) -> None:
|
async def _slow_update_cycle(
|
||||||
|
self,
|
||||||
|
epoch: int,
|
||||||
|
pools: Pools,
|
||||||
|
state: _TrainState,
|
||||||
|
*,
|
||||||
|
total_steps: int,
|
||||||
|
step_completed: int,
|
||||||
|
epoch_batches: list[list[str]],
|
||||||
|
) -> None:
|
||||||
"""epoch 末慢更新十步序。
|
"""epoch 末慢更新十步序。
|
||||||
|
|
||||||
1. 捕获版本快照 → 全 val 重跑 R
|
1. 捕获版本快照 → 全 val 重跑 R
|
||||||
@@ -1387,7 +1500,12 @@ class Runner:
|
|||||||
7. system/tool 慢更新(edit_budget_end)
|
7. system/tool 慢更新(edit_budget_end)
|
||||||
8. R2 闭环
|
8. R2 闭环
|
||||||
9. 三态标签 + epoch_report + 四向 held-out
|
9. 三态标签 + epoch_report + 四向 held-out
|
||||||
10. gate 阶梯刷新
|
10. gate 阶梯刷新 → 重置 epoch 累加器 → 立即落 epoch_done checkpoint
|
||||||
|
|
||||||
|
参数:
|
||||||
|
total_steps: 全局总 step 数(checkpoint 用)。
|
||||||
|
step_completed: 本 epoch 已完成 step 数(checkpoint 用)。
|
||||||
|
epoch_batches: 本 epoch batch 的 unit_id 划分(checkpoint 用)。
|
||||||
"""
|
"""
|
||||||
# Phase 1
|
# Phase 1
|
||||||
eval_skills_version = self._current_version("skills")
|
eval_skills_version = self._current_version("skills")
|
||||||
@@ -1441,11 +1559,13 @@ class Runner:
|
|||||||
r2_skills_version = self._current_version("skills")
|
r2_skills_version = self._current_version("skills")
|
||||||
new_prompts_version = self._current_version("prompts")
|
new_prompts_version = self._current_version("prompts")
|
||||||
eval_r2 = await self._eval_full_val(epoch, pools, run_suffix="_p2")
|
eval_r2 = await self._eval_full_val(epoch, pools, run_suffix="_p2")
|
||||||
|
# R2 是可能被 revert 的慢更新候选,用 slow_candidate 口径,不占 final
|
||||||
|
# (epoch 终值唯一由 Phase 2 的 final 承载)
|
||||||
write_dual_metric(
|
write_dual_metric(
|
||||||
str(self._paths.db_path),
|
str(self._paths.db_path),
|
||||||
run_id=self._config.run_id,
|
run_id=self._config.run_id,
|
||||||
epoch=epoch,
|
epoch=epoch,
|
||||||
version_kind="final",
|
version_kind="slow_candidate",
|
||||||
skills_version=r2_skills_version,
|
skills_version=r2_skills_version,
|
||||||
prompts_version=new_prompts_version,
|
prompts_version=new_prompts_version,
|
||||||
pool="val",
|
pool="val",
|
||||||
@@ -1497,6 +1617,24 @@ class Runner:
|
|||||||
epoch, pools.baseline_run_id, state, extra_run_ids=r2_kept_run_ids
|
epoch, pools.baseline_run_id, state, extra_run_ids=r2_kept_run_ids
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# 重置 epoch 累加器 + 立即落 epoch_done checkpoint:与 _refresh_gate_ladder 内的
|
||||||
|
# gate_pools.save + gate_epoch_observed=True 同刻一致,消除断点续跑的双计窗口。
|
||||||
|
state.system_packs = []
|
||||||
|
state.tool_packs = []
|
||||||
|
state.changed_task_types_this_epoch = set()
|
||||||
|
write_checkpoint(
|
||||||
|
self._config.workspace_dir,
|
||||||
|
state=state,
|
||||||
|
epoch=epoch,
|
||||||
|
step_completed=step_completed,
|
||||||
|
phase="epoch_done",
|
||||||
|
global_step=state.global_step,
|
||||||
|
total_steps=total_steps,
|
||||||
|
version_snapshot=self._current_version_snapshot(),
|
||||||
|
epoch_batches=epoch_batches,
|
||||||
|
config=self._config,
|
||||||
|
)
|
||||||
|
|
||||||
# -----------------------------------------------------------------------
|
# -----------------------------------------------------------------------
|
||||||
# 慢更新内部方法
|
# 慢更新内部方法
|
||||||
# -----------------------------------------------------------------------
|
# -----------------------------------------------------------------------
|
||||||
@@ -1584,7 +1722,7 @@ class Runner:
|
|||||||
state.best_val_acc = eval_acc
|
state.best_val_acc = eval_acc
|
||||||
state.best_skills_version = skills_v
|
state.best_skills_version = skills_v
|
||||||
state.best_prompts_version = prompts_v
|
state.best_prompts_version = prompts_v
|
||||||
state.steps_since_best_improved = 0
|
state.epochs_since_best_improved = 0
|
||||||
update_best(
|
update_best(
|
||||||
self._config.workspace_dir,
|
self._config.workspace_dir,
|
||||||
skills=f"skills/{skills_v}",
|
skills=f"skills/{skills_v}",
|
||||||
@@ -1888,11 +2026,18 @@ class Runner:
|
|||||||
) -> None:
|
) -> None:
|
||||||
"""四向 held-out:baseline/best_hard/final/best_mixed 各在 test 池评估。
|
"""四向 held-out:baseline/best_hard/final/best_mixed 各在 test 池评估。
|
||||||
|
|
||||||
|
去重(进程内备忘录 state.holdout_memo,不改 schema):
|
||||||
|
- baseline:不跑推理,从基线 predictions 推导 test 结果(0 推理),存 memo 跨 epoch 复用。
|
||||||
|
- final:真评 test,存 memo[(final_sv,final_pv)]。
|
||||||
|
- best_hard:其版本已在 memo(== final 或往轮已评)则引用,否则真评并存 memo。
|
||||||
|
- best_mixed:赢家必是 best_hard 或 final 之一,其结果已在 memo,直接引用(0 推理)。
|
||||||
|
|
||||||
test 池仅观测落库,绝不进 gate/best/early-stop/调参。
|
test 池仅观测落库,绝不进 gate/best/early-stop/调参。
|
||||||
"""
|
"""
|
||||||
best_mixed = await self._pick_mixed_best(
|
best_mixed = await self._pick_mixed_best(
|
||||||
epoch, pools, state, eval_skills_version, eval_prompts_version
|
epoch, pools, state, eval_skills_version, eval_prompts_version
|
||||||
)
|
)
|
||||||
|
memo = state.holdout_memo
|
||||||
versions: dict[str, tuple[str, str] | None] = {
|
versions: dict[str, tuple[str, str] | None] = {
|
||||||
"baseline": (state.baseline_skills_version, state.baseline_prompts_version),
|
"baseline": (state.baseline_skills_version, state.baseline_prompts_version),
|
||||||
"best_hard": (state.best_skills_version, state.best_prompts_version),
|
"best_hard": (state.best_skills_version, state.best_prompts_version),
|
||||||
@@ -1904,9 +2049,14 @@ class Runner:
|
|||||||
continue
|
continue
|
||||||
sv, pv = version
|
sv, pv = version
|
||||||
run_id = f"{self._config.run_id}_holdout_{version_kind}_e{epoch}"
|
run_id = f"{self._config.run_id}_holdout_{version_kind}_e{epoch}"
|
||||||
res = await self._eval_version_on_pool(
|
if version not in memo:
|
||||||
sv, pv, pools.test, run_id, context=f"held-out {version_kind}"
|
if version_kind == "baseline":
|
||||||
)
|
memo[version] = self._derive_baseline_test_result(pools, run_id)
|
||||||
|
else:
|
||||||
|
memo[version] = await self._eval_version_on_pool(
|
||||||
|
sv, pv, pools.test, run_id, context=f"held-out {version_kind}"
|
||||||
|
)
|
||||||
|
res = memo[version]
|
||||||
soft = await self._try_soft_score(run_id, pools.test)
|
soft = await self._try_soft_score(run_id, pools.test)
|
||||||
mixed = None if soft is None else 0.5 * res.accuracy + 0.5 * soft
|
mixed = None if soft is None else 0.5 * res.accuracy + 0.5 * soft
|
||||||
write_holdout_eval(
|
write_holdout_eval(
|
||||||
@@ -1920,6 +2070,33 @@ class Runner:
|
|||||||
per_task_type_json=json.dumps(res.per_task_type, ensure_ascii=False),
|
per_task_type_json=json.dumps(res.per_task_type, ensure_ascii=False),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _derive_baseline_test_result(self, pools: Pools, run_id: str) -> InferenceResult:
|
||||||
|
"""从基线 run 的 predictions 推导 test 池评估结果(0 推理)。
|
||||||
|
|
||||||
|
基线 run(pools.baseline_run_id)已对全题库推理并落库,test 题在其中;此处
|
||||||
|
按 test 题回读基线预测、经 _aggregate_results 折叠为 unit 级 InferenceResult,
|
||||||
|
避免重复推理基线版本(同版本不重采样)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
pools: 冻结三池(提供 test 与 baseline_run_id)。
|
||||||
|
run_id: 本次 holdout baseline 向的 run_id(仅用作结果标识)。
|
||||||
|
返回:
|
||||||
|
unit 级 InferenceResult(accuracy / per_task_type 与真评同口径)。
|
||||||
|
"""
|
||||||
|
from app.harness.inference import _aggregate_results
|
||||||
|
from app.harness.log import HarnessLog
|
||||||
|
|
||||||
|
qids = [q.question_id for q in pools.test]
|
||||||
|
with HarnessLog(
|
||||||
|
str(self._paths.db_path), pools.baseline_run_id, register_run=False
|
||||||
|
) as log:
|
||||||
|
placeholders = ", ".join(["?"] * len(qids))
|
||||||
|
rows = log.query(
|
||||||
|
f"SELECT * FROM predictions WHERE run_id=? AND question_id IN ({placeholders})",
|
||||||
|
(pools.baseline_run_id, *qids),
|
||||||
|
)
|
||||||
|
return _aggregate_results(rows, pools.test, run_id)
|
||||||
|
|
||||||
async def _pick_mixed_best(
|
async def _pick_mixed_best(
|
||||||
self,
|
self,
|
||||||
epoch: int,
|
epoch: int,
|
||||||
@@ -2164,13 +2341,15 @@ class Runner:
|
|||||||
self, run_id: str, *, question_ids: list[str] | None = None
|
self, run_id: str, *, question_ids: list[str] | None = None
|
||||||
) -> DiagnosisResult:
|
) -> DiagnosisResult:
|
||||||
"""执行两阶段诊断。"""
|
"""执行两阶段诊断。"""
|
||||||
|
from app.harness.baseline_run_log import StepsJsonRunLog
|
||||||
from app.harness.log import RunLogImpl
|
from app.harness.log import RunLogImpl
|
||||||
from app.harness.workspace import VersionedSkillStore
|
from app.harness.workspace import VersionedSkillStore
|
||||||
from app.question_gen import load_benchmark
|
from app.question_gen import load_benchmark
|
||||||
from core.evolution.diagnose import run_diagnosis
|
from core.evolution.diagnose import run_diagnosis
|
||||||
|
|
||||||
questions = load_benchmark(self._paths.questions_dir)
|
questions = load_benchmark(self._paths.questions_dir)
|
||||||
run_log = RunLogImpl(str(self._paths.db_path))
|
# traces 表空时(如训练 rollout 只落 steps_json)从 steps_json 重建轨迹,恢复算法 #7
|
||||||
|
run_log = StepsJsonRunLog(RunLogImpl(str(self._paths.db_path)))
|
||||||
skill_store = VersionedSkillStore(self._paths.skills_dir)
|
skill_store = VersionedSkillStore(self._paths.skills_dir)
|
||||||
diagnose_prompts = self._load_diagnose_prompts()
|
diagnose_prompts = self._load_diagnose_prompts()
|
||||||
|
|
||||||
@@ -2269,14 +2448,15 @@ class Runner:
|
|||||||
|
|
||||||
def _read(name: str) -> str:
|
def _read(name: str) -> str:
|
||||||
p = Path("prompts") / name
|
p = Path("prompts") / name
|
||||||
return p.read_text(encoding="utf-8") if p.exists() else ""
|
if not p.exists():
|
||||||
|
raise FileNotFoundError(f"缺进化/诊断模板: {p}(请从 TRM4 迁移或检查 prompts/)")
|
||||||
|
return p.read_text(encoding="utf-8")
|
||||||
|
|
||||||
return EvolvePrompts(
|
return EvolvePrompts(
|
||||||
evolve_skill=_read("evolve_skill.md"),
|
evolve_skill=_read("evolve_skill.md"),
|
||||||
evolve_system=_read("evolve_system.md"),
|
evolve_system=_read("evolve_system.md"),
|
||||||
evolve_tool=_read("evolve_tool.md"),
|
evolve_tool=_read("evolve_tool.md"),
|
||||||
evolve_rank=_read("evolve_rank.md"),
|
evolve_rank=_read("evolve_rank.md"),
|
||||||
consolidate_system=_read("consolidate_system.md"),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def _load_diagnose_prompts(self):
|
def _load_diagnose_prompts(self):
|
||||||
@@ -2285,13 +2465,14 @@ class Runner:
|
|||||||
|
|
||||||
def _read(name: str) -> str:
|
def _read(name: str) -> str:
|
||||||
p = Path("prompts") / name
|
p = Path("prompts") / name
|
||||||
return p.read_text(encoding="utf-8") if p.exists() else ""
|
if not p.exists():
|
||||||
|
raise FileNotFoundError(f"缺进化/诊断模板: {p}(请从 TRM4 迁移或检查 prompts/)")
|
||||||
|
return p.read_text(encoding="utf-8")
|
||||||
|
|
||||||
return DiagnosePrompts(
|
return DiagnosePrompts(
|
||||||
defect_vs_lapse=_read("defect_vs_lapse.md"),
|
defect_vs_lapse=_read("defect_vs_lapse.md"),
|
||||||
reasoning_sub=_read("reasoning_sub.md"),
|
reasoning_sub=_read("reasoning_sub.md"),
|
||||||
span_eval_system=_read("span_eval_system.md"),
|
span_eval_system=_read("span_eval_system.md"),
|
||||||
span_eval_user=_read("span_eval_user.md"),
|
|
||||||
missed_nodes=_read("missed_nodes.md"),
|
missed_nodes=_read("missed_nodes.md"),
|
||||||
skill_adherence=_read("skill_adherence.md"),
|
skill_adherence=_read("skill_adherence.md"),
|
||||||
confirmation_bias=_read("confirmation_bias.md"),
|
confirmation_bias=_read("confirmation_bias.md"),
|
||||||
|
|||||||
+28
-4
@@ -190,6 +190,9 @@ def init_seed(
|
|||||||
baseline_run_id: str,
|
baseline_run_id: str,
|
||||||
parent: str | None,
|
parent: str | None,
|
||||||
description: str,
|
description: str,
|
||||||
|
*,
|
||||||
|
pools_json: Path | None = None,
|
||||||
|
split_manifest: Path | None = None,
|
||||||
) -> Path:
|
) -> Path:
|
||||||
"""在 store/seeds/<name> 写一个种子:权重 + baseline.db + seed.json。
|
"""在 store/seeds/<name> 写一个种子:权重 + baseline.db + seed.json。
|
||||||
|
|
||||||
@@ -202,6 +205,10 @@ def init_seed(
|
|||||||
baseline_run_id: 全量记录的 run_id,fresh 时注入 build_pools。
|
baseline_run_id: 全量记录的 run_id,fresh 时注入 build_pools。
|
||||||
parent: 来源(initial 为 None)。
|
parent: 来源(initial 为 None)。
|
||||||
description: 人类可读说明。
|
description: 人类可读说明。
|
||||||
|
pools_json: 可选,冻结切分 pools.json 源路径;提供时拷入 seed 目录,
|
||||||
|
供 fresh 训练时携带冻结切分进 workspace(见 init_workspace_from_seed)。
|
||||||
|
split_manifest: 可选,冻结切分 split_manifest.json 源路径;提供时拷入 seed
|
||||||
|
目录,供加载时校验 pools.json 内容指纹(pools_sha256)。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
种子目录路径。
|
种子目录路径。
|
||||||
@@ -216,6 +223,10 @@ def init_seed(
|
|||||||
shutil.copytree(skills_dir, seed_dir / "skills")
|
shutil.copytree(skills_dir, seed_dir / "skills")
|
||||||
shutil.copytree(prompts_dir, seed_dir / "prompts")
|
shutil.copytree(prompts_dir, seed_dir / "prompts")
|
||||||
shutil.copy2(baseline_db, seed_dir / "baseline.db")
|
shutil.copy2(baseline_db, seed_dir / "baseline.db")
|
||||||
|
if pools_json is not None:
|
||||||
|
shutil.copy2(pools_json, seed_dir / "pools.json")
|
||||||
|
if split_manifest is not None:
|
||||||
|
shutil.copy2(split_manifest, seed_dir / "split_manifest.json")
|
||||||
(seed_dir / "seed.json").write_text(
|
(seed_dir / "seed.json").write_text(
|
||||||
json.dumps(
|
json.dumps(
|
||||||
{
|
{
|
||||||
@@ -266,7 +277,9 @@ def read_seed(store_dir: Path, name: str) -> dict:
|
|||||||
return json.loads(seed_json.read_text())
|
return json.loads(seed_json.read_text())
|
||||||
|
|
||||||
|
|
||||||
def extract_run_db(src_db: Path, dst_db: Path, run_id: str) -> None:
|
def extract_run_db(
|
||||||
|
src_db: Path, dst_db: Path, run_id: str, *, dedupe_per_question: bool = False
|
||||||
|
) -> None:
|
||||||
"""从 src_db 抽出某 run_id 的 _runs + predictions 行,写一个最小 db(种子 baseline.db)。
|
"""从 src_db 抽出某 run_id 的 _runs + predictions 行,写一个最小 db(种子 baseline.db)。
|
||||||
|
|
||||||
用源表的**原始 CREATE 语句**重建目标表,保留主键/列类型/约束——
|
用源表的**原始 CREATE 语句**重建目标表,保留主键/列类型/约束——
|
||||||
@@ -277,6 +290,9 @@ def extract_run_db(src_db: Path, dst_db: Path, run_id: str) -> None:
|
|||||||
src_db: 源 harness.db。
|
src_db: 源 harness.db。
|
||||||
dst_db: 目标 db(不得已存在)。
|
dst_db: 目标 db(不得已存在)。
|
||||||
run_id: 要抽取的 run。
|
run_id: 要抽取的 run。
|
||||||
|
dedupe_per_question: True 时 predictions 表每 question_id 仅保留 rowid 最小
|
||||||
|
的首行(对齐 canonical「每 question_id 取第一行 ORDER BY rowid」口径,
|
||||||
|
902→900)。_runs 表不受影响。
|
||||||
|
|
||||||
异常:
|
异常:
|
||||||
RuntimeError: 源中无该表或无该 run 的行。
|
RuntimeError: 源中无该表或无该 run 的行。
|
||||||
@@ -294,9 +310,17 @@ def extract_run_db(src_db: Path, dst_db: Path, run_id: str) -> None:
|
|||||||
dst.execute(create_sql[0])
|
dst.execute(create_sql[0])
|
||||||
cols = [r[1] for r in src.execute(f"PRAGMA table_info({table})")]
|
cols = [r[1] for r in src.execute(f"PRAGMA table_info({table})")]
|
||||||
col_sql = ", ".join(cols)
|
col_sql = ", ".join(cols)
|
||||||
rows = src.execute(
|
if table == "predictions" and dedupe_per_question:
|
||||||
f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,)
|
rows = src.execute(
|
||||||
).fetchall()
|
f"SELECT {col_sql} FROM {table} WHERE run_id=? "
|
||||||
|
"AND rowid IN (SELECT MIN(rowid) FROM predictions "
|
||||||
|
"WHERE run_id=? GROUP BY question_id)",
|
||||||
|
(run_id, run_id),
|
||||||
|
).fetchall()
|
||||||
|
else:
|
||||||
|
rows = src.execute(
|
||||||
|
f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,)
|
||||||
|
).fetchall()
|
||||||
if not rows:
|
if not rows:
|
||||||
raise RuntimeError(f"{table} 中无 run_id={run_id} 的行")
|
raise RuntimeError(f"{table} 中无 run_id={run_id} 的行")
|
||||||
ph = ", ".join("?" * len(cols))
|
ph = ", ".join("?" * len(cols))
|
||||||
|
|||||||
+105
-21
@@ -27,6 +27,7 @@ from loguru import logger
|
|||||||
from app.harness.gate_ladder import BaselineCache, skill_hash
|
from app.harness.gate_ladder import BaselineCache, skill_hash
|
||||||
from app.harness.question_units import build_units, flatten_units, unit_correctness_view
|
from app.harness.question_units import build_units, flatten_units, unit_correctness_view
|
||||||
from core.evolution import (
|
from core.evolution import (
|
||||||
|
INFRA_STOP_REASONS,
|
||||||
GateParams,
|
GateParams,
|
||||||
GateVerdict,
|
GateVerdict,
|
||||||
RejectedEdit,
|
RejectedEdit,
|
||||||
@@ -35,6 +36,10 @@ from core.evolution import (
|
|||||||
pair_block,
|
pair_block,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# INFRA_STOP_REASONS 复用 core.evolution.diagnose 的单一定义(M-2):INFRA 故障
|
||||||
|
# stop_reason(推理侧基础设施错误,非模型答错)在诊断与 gate 两处必须同口径,
|
||||||
|
# 避免各自维护副本致未来漂移。
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from app.harness.inference import InferenceResult
|
from app.harness.inference import InferenceResult
|
||||||
from app.harness.log import HarnessLog
|
from app.harness.log import HarnessLog
|
||||||
@@ -209,7 +214,8 @@ def _load_run_rows(
|
|||||||
_correct、steps 等字段。
|
_correct、steps 等字段。
|
||||||
"""
|
"""
|
||||||
rows = log.query(
|
rows = log.query(
|
||||||
"SELECT question_id, prediction, answer, steps_json FROM predictions WHERE run_id=?",
|
"SELECT question_id, prediction, answer, stop_reason, steps_json "
|
||||||
|
"FROM predictions WHERE run_id=?",
|
||||||
(run_id,),
|
(run_id,),
|
||||||
)
|
)
|
||||||
normalized: dict[str, dict[str, Any]] = {}
|
normalized: dict[str, dict[str, Any]] = {}
|
||||||
@@ -230,6 +236,46 @@ def _load_run_rows(
|
|||||||
return normalized
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
def _infra_question_ids_from_db(
|
||||||
|
log: HarnessLog,
|
||||||
|
run_id: str,
|
||||||
|
chunk: list[GeneratedQuestion],
|
||||||
|
) -> set[str]:
|
||||||
|
"""从 db 读取一个 run 中 stop_reason 属 INFRA 故障族的 question_id 集合。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
log: HarnessLog 共享实例。
|
||||||
|
run_id: 推理 run_id。
|
||||||
|
chunk: 题目列表。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
stop_reason ∈ {"error", "parse_error"} 的 question_id 集合。
|
||||||
|
"""
|
||||||
|
rows = _load_run_rows(log, run_id)
|
||||||
|
return {
|
||||||
|
q.question_id
|
||||||
|
for q in chunk
|
||||||
|
if rows.get(q.question_id, {}).get("stop_reason") in INFRA_STOP_REASONS
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _count_infra_units(units: list[QuestionUnit], infra_qids: set[str]) -> int:
|
||||||
|
"""统计含 INFRA record 的 unit 数(一个 unit 任一题 INFRA 即计 1)。
|
||||||
|
|
||||||
|
使护栏分子与分母(r.total,unit 粒度)同口径:AR pair 一 unit 含两 record,
|
||||||
|
逐 record 计数会放大分子致 gate_guard_err 误触发,破坏 unit 粒度一致性
|
||||||
|
(核心算法保真 #5/#6)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
units: 当前块的单元列表(single 或 AR pair)。
|
||||||
|
infra_qids: 本 run 中 stop_reason 属 INFRA 故障族的 question_id 集合。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
含至少一题 INFRA 的 unit 数。
|
||||||
|
"""
|
||||||
|
return sum(1 for u in units if any(q.question_id in infra_qids for q in u.questions))
|
||||||
|
|
||||||
|
|
||||||
def _candidate_correctness_from_db(
|
def _candidate_correctness_from_db(
|
||||||
log: HarnessLog,
|
log: HarnessLog,
|
||||||
run_id: str,
|
run_id: str,
|
||||||
@@ -264,13 +310,18 @@ async def _resolve_baseline_block(
|
|||||||
run_inference: RunInferenceFn,
|
run_inference: RunInferenceFn,
|
||||||
log: HarnessLog,
|
log: HarnessLog,
|
||||||
run_id: str,
|
run_id: str,
|
||||||
) -> tuple[dict[str, bool], int, int]:
|
) -> tuple[dict[str, bool], list[QuestionUnit], int, int]:
|
||||||
"""基线侧处理一个块:缓存优先(unit 键),miss 的单元新鲜跑基线版本并回写缓存。
|
"""基线侧处理一个块:缓存优先(unit 键),miss 的单元新鲜跑基线版本并回写缓存。
|
||||||
|
|
||||||
缓存以 unit_id 为键、存单元级对错(AR pair 双向 AND 折叠后一个布尔)。
|
缓存以 unit_id 为键、存单元级对错(AR pair 双向 AND 折叠后一个布尔)。
|
||||||
miss 的单元展开为逐题送推理,读回逐题预测后经 unit_correctness_view 折叠成
|
miss 的单元展开为逐题送推理,读回逐题预测后经 unit_correctness_view 折叠成
|
||||||
单元级对错再写缓存(核心算法保真 #5)。逐题 predictions 仍逐题落库溯源。
|
单元级对错再写缓存(核心算法保真 #5)。逐题 predictions 仍逐题落库溯源。
|
||||||
|
|
||||||
|
INFRA 隔离(算法 #6):miss 单元内**任一题** stop_reason ∈ {error, parse_error}
|
||||||
|
即判定该单元为 INFRA 故障——**不写 BaselineCache**(否则瞬时故障永久污染基线
|
||||||
|
快照)、**不入 b_units**、并从返回的有效单元集中剔除,避免污染 W/L 翻转与配对。
|
||||||
|
命中缓存的单元恒为有效(此前已成功验证过)。
|
||||||
|
|
||||||
参数:
|
参数:
|
||||||
units: 当前块的单元列表(single 或 AR pair)。
|
units: 当前块的单元列表(single 或 AR pair)。
|
||||||
task_type: 当前验证题型(缓存键成分)。
|
task_type: 当前验证题型(缓存键成分)。
|
||||||
@@ -283,8 +334,9 @@ async def _resolve_baseline_block(
|
|||||||
run_id: 本块基线 run_id。
|
run_id: 本块基线 run_id。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
(b_units, errors_inc, denom_inc):块内 unit_id -> 基线单元对错、
|
(b_units, valid_units, errors_inc, denom_inc):块内有效 unit_id -> 基线单元
|
||||||
本块新增的 INFRA error 计数与推理题次分母增量(全命中时为 0, 0)。
|
对错、剔除 INFRA 后的有效单元列表、本块新增的 INFRA error 计数与推理题次
|
||||||
|
分母增量(全命中时为 0, 0)。
|
||||||
"""
|
"""
|
||||||
miss_units = [
|
miss_units = [
|
||||||
u
|
u
|
||||||
@@ -293,22 +345,34 @@ async def _resolve_baseline_block(
|
|||||||
]
|
]
|
||||||
errors_inc = 0
|
errors_inc = 0
|
||||||
denom_inc = 0
|
denom_inc = 0
|
||||||
|
infra_qids: set[str] = set()
|
||||||
if miss_units:
|
if miss_units:
|
||||||
miss_questions = flatten_units(miss_units)
|
miss_questions = flatten_units(miss_units)
|
||||||
r_b = await run_inference(miss_questions, run_id=run_id, skills_dir=base_skills_dir)
|
r_b = await run_inference(miss_questions, run_id=run_id, skills_dir=base_skills_dir)
|
||||||
errors_inc = r_b.stop_reason_counts.get("error", 0)
|
infra_qids = _infra_question_ids_from_db(log, r_b.run_id, miss_questions)
|
||||||
|
# 护栏分子与分母(r.total,unit 粒度)同口径:含 INFRA record 的 unit 计 1,
|
||||||
|
# 避免 AR pair(一 unit 两 record)逐 record 计数放大分子致误触发;仍涵盖
|
||||||
|
# error + parse_error(_infra_question_ids_from_db 口径),parse_error 风暴不被绕过。
|
||||||
|
errors_inc = _count_infra_units(miss_units, infra_qids)
|
||||||
denom_inc = r_b.total
|
denom_inc = r_b.total
|
||||||
fresh_per_q = _candidate_correctness_from_db(log, r_b.run_id, miss_questions)
|
fresh_per_q = _candidate_correctness_from_db(log, r_b.run_id, miss_questions)
|
||||||
fresh_units = unit_correctness_view(miss_units, fresh_per_q)
|
fresh_units = unit_correctness_view(miss_units, fresh_per_q)
|
||||||
for uid, correct in fresh_units.items():
|
# 只回写非 INFRA 单元;INFRA 单元不入缓存(不永久污染基线快照)
|
||||||
baseline_cache.put(task_type, s_hash, prompts_version, uid, correct)
|
for u in miss_units:
|
||||||
|
if any(q.question_id in infra_qids for q in u.questions):
|
||||||
|
continue
|
||||||
|
baseline_cache.put(task_type, s_hash, prompts_version, u.unit_id, fresh_units[u.unit_id])
|
||||||
|
|
||||||
|
valid_units = [
|
||||||
|
u for u in units if not any(q.question_id in infra_qids for q in u.questions)
|
||||||
|
]
|
||||||
|
|
||||||
b_units: dict[str, bool] = {}
|
b_units: dict[str, bool] = {}
|
||||||
for u in units:
|
for u in valid_units:
|
||||||
val = baseline_cache.get(task_type, s_hash, prompts_version, u.unit_id)
|
val = baseline_cache.get(task_type, s_hash, prompts_version, u.unit_id)
|
||||||
assert val is not None, f"基线缓存补齐后仍有 miss: unit={u.unit_id} run_id={run_id}"
|
assert val is not None, f"基线缓存补齐后仍有 miss: unit={u.unit_id} run_id={run_id}"
|
||||||
b_units[u.unit_id] = val
|
b_units[u.unit_id] = val
|
||||||
return b_units, errors_inc, denom_inc
|
return b_units, valid_units, errors_inc, denom_inc
|
||||||
|
|
||||||
|
|
||||||
async def _run_candidate_block(
|
async def _run_candidate_block(
|
||||||
@@ -336,7 +400,11 @@ async def _run_candidate_block(
|
|||||||
questions = flatten_units(units)
|
questions = flatten_units(units)
|
||||||
r_c = await run_inference(questions, run_id=run_id, skills_dir=cand_dir)
|
r_c = await run_inference(questions, run_id=run_id, skills_dir=cand_dir)
|
||||||
c_per_q = _candidate_correctness_from_db(log, r_c.run_id, questions)
|
c_per_q = _candidate_correctness_from_db(log, r_c.run_id, questions)
|
||||||
return c_per_q, r_c.stop_reason_counts.get("error", 0), r_c.total
|
infra_qids = _infra_question_ids_from_db(log, r_c.run_id, questions)
|
||||||
|
# 护栏分子与分母(r.total,unit 粒度)同口径:含 INFRA record 的 unit 计 1
|
||||||
|
# (见 _count_infra_units),涵盖 error + parse_error。
|
||||||
|
errors_inc = _count_infra_units(units, infra_qids)
|
||||||
|
return c_per_q, errors_inc, r_c.total
|
||||||
|
|
||||||
|
|
||||||
def _build_evidence_rows(
|
def _build_evidence_rows(
|
||||||
@@ -554,6 +622,7 @@ async def _run_local_validation(
|
|||||||
w = 0
|
w = 0
|
||||||
l = 0 # noqa: E741
|
l = 0 # noqa: E741
|
||||||
n_used = 0
|
n_used = 0
|
||||||
|
n_excluded = 0 # 累计被 INFRA 隔离剔除的单元数(从阶梯分母扣除)
|
||||||
errors = 0
|
errors = 0
|
||||||
infra_denom = 0
|
infra_denom = 0
|
||||||
evidence_rows: list[dict] = []
|
evidence_rows: list[dict] = []
|
||||||
@@ -567,8 +636,8 @@ async def _run_local_validation(
|
|||||||
verdict: GateVerdict | None = None
|
verdict: GateVerdict | None = None
|
||||||
|
|
||||||
for block_idx, unit_chunk in enumerate(unit_chunks):
|
for block_idx, unit_chunk in enumerate(unit_chunks):
|
||||||
# Phase 1: 基线侧(缓存优先,miss 新鲜跑)+ 候选侧(全块新鲜跑)
|
# Phase 1: 基线侧(缓存优先,miss 新鲜跑,INFRA 单元剔除)
|
||||||
b_units, err_b, den_b = await _resolve_baseline_block(
|
b_units, valid_chunk, err_b, den_b = await _resolve_baseline_block(
|
||||||
units=unit_chunk,
|
units=unit_chunk,
|
||||||
task_type=task_type,
|
task_type=task_type,
|
||||||
s_hash=s_hash,
|
s_hash=s_hash,
|
||||||
@@ -579,34 +648,46 @@ async def _run_local_validation(
|
|||||||
log=log,
|
log=log,
|
||||||
run_id=f"{gate_run_prefix}_b{block_idx}_base",
|
run_id=f"{gate_run_prefix}_b{block_idx}_base",
|
||||||
)
|
)
|
||||||
|
# 本块全 INFRA:无有效单元可配对——候选无需空跑,仅把基线侧错误计入护栏后
|
||||||
|
# 累计剔除数进入下一块(护栏仍能在整轮 INFRA 错误率超阈值时熔断)。
|
||||||
|
n_excluded += len(unit_chunk) - len(valid_chunk)
|
||||||
|
if not valid_chunk:
|
||||||
|
errors += err_b
|
||||||
|
infra_denom += den_b
|
||||||
|
_check_infra_guard(errors, infra_denom, gate_guard_err)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 候选侧只跑基线侧判定有效(非 INFRA)的单元,保证配对 unit_ids 两侧一致
|
||||||
c_per_q, err_c, den_c = await _run_candidate_block(
|
c_per_q, err_c, den_c = await _run_candidate_block(
|
||||||
units=unit_chunk,
|
units=valid_chunk,
|
||||||
cand_dir=cand_dir,
|
cand_dir=cand_dir,
|
||||||
run_inference=run_inference,
|
run_inference=run_inference,
|
||||||
log=log,
|
log=log,
|
||||||
run_id=f"{gate_run_prefix}_b{block_idx}_cand",
|
run_id=f"{gate_run_prefix}_b{block_idx}_cand",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Phase 2: INFRA 护栏(跨块累计,分母 >=10 才触发)
|
# Phase 2: INFRA 护栏(跨块累计,分母 >=10 才触发)——写缓存前置于此已由
|
||||||
|
# _resolve_baseline_block 保证 INFRA 单元不落缓存,此处仅做整轮错误率熔断。
|
||||||
errors += err_b + err_c
|
errors += err_b + err_c
|
||||||
infra_denom += den_b + den_c
|
infra_denom += den_b + den_c
|
||||||
_check_infra_guard(errors, infra_denom, gate_guard_err)
|
_check_infra_guard(errors, infra_denom, gate_guard_err)
|
||||||
|
|
||||||
# Phase 3: 折叠成单元视图 + 配对 + 证据行 + 块间判定
|
# Phase 3: 折叠成单元视图 + 配对 + 证据行 + 块间判定(均用有效单元)
|
||||||
c_units = unit_correctness_view(unit_chunk, c_per_q)
|
c_units = unit_correctness_view(valid_chunk, c_per_q)
|
||||||
candidate_per_q.update(c_per_q)
|
candidate_per_q.update(c_per_q)
|
||||||
unit_ids = [u.unit_id for u in unit_chunk]
|
unit_ids = [u.unit_id for u in valid_chunk]
|
||||||
pair_result = pair_block(b_units, c_units, unit_ids)
|
pair_result = pair_block(b_units, c_units, unit_ids)
|
||||||
for uid, (b, c) in pair_result.observed.items():
|
for uid, (b, c) in pair_result.observed.items():
|
||||||
base_obs[uid] = b
|
base_obs[uid] = b
|
||||||
cand_obs[uid] = c
|
cand_obs[uid] = c
|
||||||
|
|
||||||
block_rows = _build_evidence_rows(unit_chunk, b_units, c_units, task_type, block_idx)
|
block_rows = _build_evidence_rows(valid_chunk, b_units, c_units, task_type, block_idx)
|
||||||
|
|
||||||
w += pair_result.w
|
w += pair_result.w
|
||||||
l += pair_result.l # noqa: E741
|
l += pair_result.l # noqa: E741
|
||||||
n_used += len(unit_chunk)
|
n_used += len(valid_chunk)
|
||||||
verdict = gate_decision(w, l, n_used, n_plan - n_used, params=gate_params)
|
# 阶梯剩余按扣除 INFRA 后的有效分母计:n_remaining = (n_plan - n_excluded) - n_used
|
||||||
|
verdict = gate_decision(w, l, n_used, (n_plan - n_excluded) - n_used, params=gate_params)
|
||||||
|
|
||||||
for row in block_rows:
|
for row in block_rows:
|
||||||
row["e_value"] = verdict.e_value
|
row["e_value"] = verdict.e_value
|
||||||
@@ -615,8 +696,11 @@ async def _run_local_validation(
|
|||||||
if verdict.decision != "continue":
|
if verdict.decision != "continue":
|
||||||
break
|
break
|
||||||
|
|
||||||
|
# verdict 仍为 None ⟺ 全部单元被 INFRA 排除(空 ladder 已在入口拒绝)。
|
||||||
|
# 明确失败,避免落到误导性的"空阶梯"断言而无法定位为 INFRA 原因。
|
||||||
|
if verdict is None:
|
||||||
|
raise RuntimeError("gate 阶梯所有 unit 被判为 INFRA 排除,无法验证(检查推理基础设施)")
|
||||||
# 最后一块判定即终态(n_remaining=0 → provisional/inertia)
|
# 最后一块判定即终态(n_remaining=0 → provisional/inertia)
|
||||||
assert verdict is not None, "空阶梯应已在 validate_skill_local 入口拒绝"
|
|
||||||
return _finalize_outcome(
|
return _finalize_outcome(
|
||||||
verdict=verdict,
|
verdict=verdict,
|
||||||
w=w,
|
w=w,
|
||||||
|
|||||||
@@ -248,13 +248,15 @@ def _build_redis_cache(settings: Any) -> Any | None:
|
|||||||
"""按 .env redis_url 构建响应缓存(不可用则降级 None,与 main 一致)。"""
|
"""按 .env redis_url 构建响应缓存(不可用则降级 None,与 main 一致)。"""
|
||||||
if not settings.redis_url:
|
if not settings.redis_url:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
from adapters.redis_cache import RedisResponseCache, _resolve_cache_ttl
|
||||||
|
|
||||||
|
# 配置校验 fail-loud(不属于 Redis 连接故障,不得被下方降级 except 吞掉)
|
||||||
|
ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl)
|
||||||
try:
|
try:
|
||||||
import redis.asyncio as aioredis
|
import redis.asyncio as aioredis
|
||||||
|
|
||||||
from adapters.redis_cache import RedisResponseCache
|
|
||||||
|
|
||||||
redis_client = aioredis.from_url(settings.redis_url, decode_responses=True)
|
redis_client = aioredis.from_url(settings.redis_url, decode_responses=True)
|
||||||
ttl_s = settings.redis_cache_ttl if settings.redis_cache_ttl > 0 else None
|
|
||||||
return RedisResponseCache(redis=redis_client, ttl_s=ttl_s)
|
return RedisResponseCache(redis=redis_client, ttl_s=ttl_s)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.warning("Redis 缓存不可用,诊断降级为无缓存模式")
|
logger.warning("Redis 缓存不可用,诊断降级为无缓存模式")
|
||||||
@@ -365,13 +367,14 @@ def _load_diagnose_prompts() -> Any:
|
|||||||
|
|
||||||
def _read(name: str) -> str:
|
def _read(name: str) -> str:
|
||||||
p = Path("prompts") / name
|
p = Path("prompts") / name
|
||||||
return p.read_text(encoding="utf-8") if p.exists() else ""
|
if not p.exists():
|
||||||
|
raise FileNotFoundError(f"缺进化/诊断模板: {p}(请从 TRM4 迁移或检查 prompts/)")
|
||||||
|
return p.read_text(encoding="utf-8")
|
||||||
|
|
||||||
return DiagnosePrompts(
|
return DiagnosePrompts(
|
||||||
defect_vs_lapse=_read("defect_vs_lapse.md"),
|
defect_vs_lapse=_read("defect_vs_lapse.md"),
|
||||||
reasoning_sub=_read("reasoning_sub.md"),
|
reasoning_sub=_read("reasoning_sub.md"),
|
||||||
span_eval_system=_read("span_eval_system.md"),
|
span_eval_system=_read("span_eval_system.md"),
|
||||||
span_eval_user=_read("span_eval_user.md"),
|
|
||||||
missed_nodes=_read("missed_nodes.md"),
|
missed_nodes=_read("missed_nodes.md"),
|
||||||
skill_adherence=_read("skill_adherence.md"),
|
skill_adherence=_read("skill_adherence.md"),
|
||||||
confirmation_bias=_read("confirmation_bias.md"),
|
confirmation_bias=_read("confirmation_bias.md"),
|
||||||
@@ -476,8 +479,9 @@ def load_questions_by_id(questions_dir: Path) -> dict[str, GeneratedQuestion]:
|
|||||||
def check_mcnemar_power(pools: Pools, val_wrong_min: int) -> int:
|
def check_mcnemar_power(pools: Pools, val_wrong_min: int) -> int:
|
||||||
"""校验 validation 池错题数达 McNemar 功效阈,不足即 fail loud。
|
"""校验 validation 池错题数达 McNemar 功效阈,不足即 fail loud。
|
||||||
|
|
||||||
build_split 按契约用 split_by_video_assignment(val_wrong_min=0)(不破契约),
|
val_wrong_min 已前置到 build_split 内的切分保证功效(不足即从 diag 换入低 T2
|
||||||
故功效护栏在 capstone 层单独核验:val 错题数 < 阈 → 验证信号不足以支撑可靠比较。
|
错题组补足,耗尽 fail-loud);本函数作切分冻结后的冗余最终确认:val 错题数 < 阈
|
||||||
|
→ 验证信号不足以支撑可靠比较。
|
||||||
|
|
||||||
参数:
|
参数:
|
||||||
pools: 冻结三池(含 validation 与 correctness)。
|
pools: 冻结三池(含 validation 与 correctness)。
|
||||||
@@ -517,6 +521,8 @@ async def run_pipeline(
|
|||||||
questions_dir: Path,
|
questions_dir: Path,
|
||||||
out_dir: Path,
|
out_dir: Path,
|
||||||
generated_at: str,
|
generated_at: str,
|
||||||
|
force: bool = False,
|
||||||
|
retry_uncertain: bool = False,
|
||||||
) -> SplitBuildResult:
|
) -> SplitBuildResult:
|
||||||
"""内联三阶段:Phase 0 INFRA T0 补录 → Phase 1 诊断 → Phase 2 冻结切分 → McNemar 护栏。
|
"""内联三阶段:Phase 0 INFRA T0 补录 → Phase 1 诊断 → Phase 2 冻结切分 → McNemar 护栏。
|
||||||
|
|
||||||
@@ -532,6 +538,8 @@ async def run_pipeline(
|
|||||||
questions_dir: benchmark 题库目录(Phase 2 加载题库切池)。
|
questions_dir: benchmark 题库目录(Phase 2 加载题库切池)。
|
||||||
out_dir: 冻结产物目录(pools.json + split_manifest.json)。
|
out_dir: 冻结产物目录(pools.json + split_manifest.json)。
|
||||||
generated_at: 生成时间戳(ISO 字符串,由调用方传入;见模块 C-2 复现锚点约定)。
|
generated_at: 生成时间戳(ISO 字符串,由调用方传入;见模块 C-2 复现锚点约定)。
|
||||||
|
force: 覆盖已存在冻结产物开关,透传给 build_split(False 时已存在即报错)。
|
||||||
|
retry_uncertain: 透传给 run_baseline_diagnosis,令已落 uncertain 题被重新诊断。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
SplitBuildResult(冻结三池 + manifest + assignment)。
|
SplitBuildResult(冻结三池 + manifest + assignment)。
|
||||||
@@ -551,6 +559,7 @@ async def run_pipeline(
|
|||||||
questions=questions,
|
questions=questions,
|
||||||
store=signal_store,
|
store=signal_store,
|
||||||
deps=diagnosis_deps,
|
deps=diagnosis_deps,
|
||||||
|
retry_uncertain=retry_uncertain,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Phase 2: 冻结切分(读诊断信号 → 贪心选择 → 视频组原子切三池 → 冻结 + 六条断言)。
|
# Phase 2: 冻结切分(读诊断信号 → 贪心选择 → 视频组原子切三池 → 冻结 + 六条断言)。
|
||||||
@@ -570,10 +579,12 @@ async def run_pipeline(
|
|||||||
select_seed=config.seed,
|
select_seed=config.seed,
|
||||||
val_ratio=config.val_ratio,
|
val_ratio=config.val_ratio,
|
||||||
split_seed=config.seed,
|
split_seed=config.seed,
|
||||||
|
val_wrong_min=config.val_wrong_min,
|
||||||
),
|
),
|
||||||
out_path=out_dir / "pools.json",
|
out_path=out_dir / "pools.json",
|
||||||
manifest_path=out_dir / "split_manifest.json",
|
manifest_path=out_dir / "split_manifest.json",
|
||||||
generated_at=generated_at,
|
generated_at=generated_at,
|
||||||
|
force=force,
|
||||||
)
|
)
|
||||||
|
|
||||||
# McNemar 功效护栏(build_split 契约外的 capstone 层校验)。
|
# McNemar 功效护栏(build_split 契约外的 capstone 层校验)。
|
||||||
@@ -647,6 +658,8 @@ def _execute_real(config: VideoSplitConfig, fingerprint: str, args: argparse.Nam
|
|||||||
questions_dir=questions_dir,
|
questions_dir=questions_dir,
|
||||||
out_dir=out_dir,
|
out_dir=out_dir,
|
||||||
generated_at=generated_at,
|
generated_at=generated_at,
|
||||||
|
force=args.force,
|
||||||
|
retry_uncertain=args.retry_uncertain,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
@@ -770,6 +783,17 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
|||||||
"对 manifest 做字节级复现比对。"
|
"对 manifest 做字节级复现比对。"
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--force",
|
||||||
|
action="store_true",
|
||||||
|
help="覆盖已存在的冻结 pools.json/manifest(旧产物备份为 .bak.*)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--retry-uncertain",
|
||||||
|
action="store_true",
|
||||||
|
dest="retry_uncertain",
|
||||||
|
help="把已落 tier='uncertain'(信号不可信降级)的题重新诊断,而非当作已完成跳过",
|
||||||
|
)
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -64,6 +64,21 @@ def _now_iso() -> str:
|
|||||||
return datetime.now(UTC).isoformat()
|
return datetime.now(UTC).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _atomic_write_json(path: Path, data: dict) -> None:
|
||||||
|
"""原子写 JSON:tmp + os.replace(对齐 checkpoint.py 范式,防半截损坏)。
|
||||||
|
|
||||||
|
先写同目录临时文件,再 os.replace 原子替换目标;替换阶段崩溃不会留下半截
|
||||||
|
JSON,原文件保持完好。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
path: 目标 JSON 文件路径。
|
||||||
|
data: 待序列化的字典。
|
||||||
|
"""
|
||||||
|
tmp = path.with_name(path.name + ".tmp")
|
||||||
|
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
|
os.replace(tmp, path)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Workspace 核心函数
|
# Workspace 核心函数
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -108,7 +123,7 @@ def _scaffold_workspace(
|
|||||||
},
|
},
|
||||||
"history": [],
|
"history": [],
|
||||||
}
|
}
|
||||||
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
_atomic_write_json(workspace_dir / "manifest.json", manifest)
|
||||||
|
|
||||||
|
|
||||||
def init_workspace(
|
def init_workspace(
|
||||||
@@ -196,6 +211,13 @@ def init_workspace_from_seed(
|
|||||||
shutil.copytree(seed_dir / "prompts", workspace_dir / "prompts" / "v1")
|
shutil.copytree(seed_dir / "prompts", workspace_dir / "prompts" / "v1")
|
||||||
shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db")
|
shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db")
|
||||||
|
|
||||||
|
seed_pools = seed_dir / "pools.json"
|
||||||
|
if seed_pools.exists():
|
||||||
|
shutil.copy2(seed_pools, workspace_dir / "pools.json")
|
||||||
|
seed_manifest = seed_dir / "split_manifest.json"
|
||||||
|
if seed_manifest.exists():
|
||||||
|
shutil.copy2(seed_manifest, workspace_dir / "split_manifest.json")
|
||||||
|
|
||||||
logger.info("Workspace 从种子 '{}' 初始化完成: {}", seed_name, workspace_dir)
|
logger.info("Workspace 从种子 '{}' 初始化完成: {}", seed_name, workspace_dir)
|
||||||
return meta["baseline_run_id"]
|
return meta["baseline_run_id"]
|
||||||
|
|
||||||
@@ -279,7 +301,7 @@ def update_manifest(workspace_dir: Path, **version_updates: str) -> None:
|
|||||||
raise KeyError(f"无效的 manifest current 字段: {invalid}")
|
raise KeyError(f"无效的 manifest current 字段: {invalid}")
|
||||||
manifest = load_manifest(workspace_dir)
|
manifest = load_manifest(workspace_dir)
|
||||||
manifest["current"].update(version_updates)
|
manifest["current"].update(version_updates)
|
||||||
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
_atomic_write_json(workspace_dir / "manifest.json", manifest)
|
||||||
|
|
||||||
|
|
||||||
def record_run(workspace_dir: Path, run_id: str) -> Path:
|
def record_run(workspace_dir: Path, run_id: str) -> Path:
|
||||||
@@ -308,9 +330,7 @@ def record_run(workspace_dir: Path, run_id: str) -> Path:
|
|||||||
"questions": current["questions"],
|
"questions": current["questions"],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
(workspace_dir / "manifest.json").write_text(
|
_atomic_write_json(workspace_dir / "manifest.json", manifest)
|
||||||
json.dumps(manifest, ensure_ascii=False, indent=2)
|
|
||||||
)
|
|
||||||
|
|
||||||
run_dir = workspace_dir / "runs" / run_id
|
run_dir = workspace_dir / "runs" / run_id
|
||||||
# exist_ok:同 run_id 重跑时 run 目录已存在不应崩溃
|
# exist_ok:同 run_id 重跑时 run 目录已存在不应崩溃
|
||||||
@@ -362,7 +382,7 @@ def update_best(
|
|||||||
"run_id": run_id,
|
"run_id": run_id,
|
||||||
"epoch": epoch,
|
"epoch": epoch,
|
||||||
}
|
}
|
||||||
(workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2))
|
_atomic_write_json(workspace_dir / "manifest.json", manifest)
|
||||||
logger.info("Best 已更新: val_acc={}, run={}, epoch={}", val_acc, run_id, epoch)
|
logger.info("Best 已更新: val_acc={}, run={}, epoch={}", val_acc, run_id, epoch)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -67,6 +67,7 @@ harness:
|
|||||||
batch_correct_ratio: 0.5
|
batch_correct_ratio: 0.5
|
||||||
momentum_samples: 20
|
momentum_samples: 20
|
||||||
eval_min_per_class: 2
|
eval_min_per_class: 2
|
||||||
|
trainable_min_units: 8
|
||||||
early_stop_patience: 8
|
early_stop_patience: 8
|
||||||
use_slow_momentum: true
|
use_slow_momentum: true
|
||||||
# 池构建策略
|
# 池构建策略
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ harness:
|
|||||||
batch_correct_ratio: 0.5
|
batch_correct_ratio: 0.5
|
||||||
momentum_samples: 20
|
momentum_samples: 20
|
||||||
eval_min_per_class: 2
|
eval_min_per_class: 2
|
||||||
|
trainable_min_units: 8
|
||||||
early_stop_patience: 4
|
early_stop_patience: 4
|
||||||
test_size: 63
|
test_size: 63
|
||||||
diag_size: 20
|
diag_size: 20
|
||||||
|
|||||||
@@ -49,6 +49,7 @@ harness:
|
|||||||
batch_correct_ratio: 0.5
|
batch_correct_ratio: 0.5
|
||||||
momentum_samples: 20
|
momentum_samples: 20
|
||||||
eval_min_per_class: 2
|
eval_min_per_class: 2
|
||||||
|
trainable_min_units: 8
|
||||||
early_stop_patience: 4
|
early_stop_patience: 4
|
||||||
test_size: 63
|
test_size: 63
|
||||||
diag_size: 20
|
diag_size: 20
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ video_split:
|
|||||||
epsilon: 0.1 # test 相对全局最大允许分布偏差(题型占比 / 难度画像两维,逐桶)
|
epsilon: 0.1 # test 相对全局最大允许分布偏差(题型占比 / 难度画像两维,逐桶)
|
||||||
report_floor: 27 # per-type 报告门限:题数 ≥ 此值的 task_type 才入 ε 代表性约束
|
report_floor: 27 # per-type 报告门限:题数 ≥ 此值的 task_type 才入 ε 代表性约束
|
||||||
val_wrong_min: 20 # validation 池最少错题数(McNemar 检验功效阈 ≈ 20,低于则信号不足)
|
val_wrong_min: 20 # validation 池最少错题数(McNemar 检验功效阈 ≈ 20,低于则信号不足)
|
||||||
val_ratio: 0.3 # validation 占 trainval 视频组总数的比例
|
val_ratio: 0.4 # validation 占 trainval 视频组总数的比例(0.3→0.4 提升整包终审功效,WP2)
|
||||||
seed: 7 # 贪心选择器预洗牌 + 视频组题级切分种子(打破等增益 / 等槽平局)
|
seed: 7 # 贪心选择器预洗牌 + 视频组题级切分种子(打破等增益 / 等槽平局)
|
||||||
floor_k: # 各 task_type 的 T2 defect 下限(硬约束)—— 均衡覆盖全 11 类,标定于 1cb1c203 真实 T2 分布
|
floor_k: # 各 task_type 的 T2 defect 下限(硬约束)—— 均衡覆盖全 11 类,标定于 1cb1c203 真实 T2 分布
|
||||||
Object Reasoning: 5 # T2=25
|
Object Reasoning: 5 # T2=25
|
||||||
|
|||||||
+10
-3
@@ -108,6 +108,7 @@ class AgentLoop:
|
|||||||
plugins: list[object] | None = None,
|
plugins: list[object] | None = None,
|
||||||
*,
|
*,
|
||||||
session_id: str | None = None,
|
session_id: str | None = None,
|
||||||
|
cache_salt: str | None = None,
|
||||||
) -> LoopResult:
|
) -> LoopResult:
|
||||||
"""执行 Thinking+JSON 推理循环。
|
"""执行 Thinking+JSON 推理循环。
|
||||||
|
|
||||||
@@ -117,6 +118,7 @@ class AgentLoop:
|
|||||||
tool_dispatcher: 工具调度器,ToolDispatcher Protocol 实例。
|
tool_dispatcher: 工具调度器,ToolDispatcher Protocol 实例。
|
||||||
plugins: pluggy 插件列表。
|
plugins: pluggy 插件列表。
|
||||||
session_id: 会话 ID,透传给 LLMProvider。
|
session_id: 会话 ID,透传给 LLMProvider。
|
||||||
|
cache_salt: 缓存盐,透传给 LLMProvider(如训练用 run_id 跨 epoch 重采样)。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
LoopResult 实例,包含推理步骤、token 用量、终止原因。
|
LoopResult 实例,包含推理步骤、token 用量、终止原因。
|
||||||
@@ -138,7 +140,7 @@ class AgentLoop:
|
|||||||
# Phase 1: LLM 调用(步级重试:防穿透 GovernedLLMClient 的瞬时异常)
|
# Phase 1: LLM 调用(步级重试:防穿透 GovernedLLMClient 的瞬时异常)
|
||||||
try:
|
try:
|
||||||
response = await self._call_llm_with_step_retry(
|
response = await self._call_llm_with_step_retry(
|
||||||
messages, token_usage, session_id=session_id
|
messages, token_usage, session_id=session_id, cache_salt=cache_salt
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error("LLM API 调用失败({}): {}", type(e).__name__, e)
|
logger.error("LLM API 调用失败({}): {}", type(e).__name__, e)
|
||||||
@@ -269,6 +271,7 @@ class AgentLoop:
|
|||||||
token_usage: dict[str, int],
|
token_usage: dict[str, int],
|
||||||
*,
|
*,
|
||||||
session_id: str | None = None,
|
session_id: str | None = None,
|
||||||
|
cache_salt: str | None = None,
|
||||||
) -> LLMResponse:
|
) -> LLMResponse:
|
||||||
"""带步级重试的 LLM 调用,兜底穿透治理层重试栈的瞬时异常。
|
"""带步级重试的 LLM 调用,兜底穿透治理层重试栈的瞬时异常。
|
||||||
|
|
||||||
@@ -292,7 +295,9 @@ class AgentLoop:
|
|||||||
step_attempt = 0
|
step_attempt = 0
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
return await self._call_llm(messages, token_usage, session_id=session_id)
|
return await self._call_llm(
|
||||||
|
messages, token_usage, session_id=session_id, cache_salt=cache_salt
|
||||||
|
)
|
||||||
except self._retryable_exceptions as e:
|
except self._retryable_exceptions as e:
|
||||||
step_attempt += 1
|
step_attempt += 1
|
||||||
if step_attempt > self._step_retries:
|
if step_attempt > self._step_retries:
|
||||||
@@ -315,6 +320,7 @@ class AgentLoop:
|
|||||||
token_usage: dict[str, int],
|
token_usage: dict[str, int],
|
||||||
*,
|
*,
|
||||||
session_id: str | None = None,
|
session_id: str | None = None,
|
||||||
|
cache_salt: str | None = None,
|
||||||
) -> LLMResponse:
|
) -> LLMResponse:
|
||||||
"""调用 LLMProvider 并累加 token 使用量。
|
"""调用 LLMProvider 并累加 token 使用量。
|
||||||
|
|
||||||
@@ -322,11 +328,12 @@ class AgentLoop:
|
|||||||
messages: 消息历史。
|
messages: 消息历史。
|
||||||
token_usage: 可变字典,就地累加。
|
token_usage: 可变字典,就地累加。
|
||||||
session_id: 会话 ID,透传给 LLMProvider。
|
session_id: 会话 ID,透传给 LLMProvider。
|
||||||
|
cache_salt: 缓存盐,透传给 LLMProvider(跨 epoch 重采样)。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
LLMResponse 实例。
|
LLMResponse 实例。
|
||||||
"""
|
"""
|
||||||
response = await self._llm.chat(messages, session_id=session_id)
|
response = await self._llm.chat(messages, session_id=session_id, cache_salt=cache_salt)
|
||||||
token_usage["prompt_tokens"] += response.prompt_tokens
|
token_usage["prompt_tokens"] += response.prompt_tokens
|
||||||
token_usage["completion_tokens"] += response.completion_tokens
|
token_usage["completion_tokens"] += response.completion_tokens
|
||||||
return response
|
return response
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
只依赖 Protocol 接口和标准库,可搬到无 adapters 的环境用假实现原样运行。
|
只依赖 Protocol 接口和标准库,可搬到无 adapters 的环境用假实现原样运行。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from core.evolution.diagnose import run_diagnosis
|
from core.evolution.diagnose import INFRA_STOP_REASONS, run_diagnosis
|
||||||
from core.evolution.evolve import (
|
from core.evolution.evolve import (
|
||||||
edit_budget_at,
|
edit_budget_at,
|
||||||
evolve_single_skill,
|
evolve_single_skill,
|
||||||
@@ -44,6 +44,7 @@ from core.evolution.types import (
|
|||||||
from core.evolution.validate import classify_quadrants, compute_accuracy, pair_block
|
from core.evolution.validate import classify_quadrants, compute_accuracy, pair_block
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
"INFRA_STOP_REASONS",
|
||||||
"CaseSample",
|
"CaseSample",
|
||||||
"DiagnosePrompts",
|
"DiagnosePrompts",
|
||||||
"DiagnosisResult",
|
"DiagnosisResult",
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ if TYPE_CHECKING:
|
|||||||
_SPAN_EVAL_TOOLS: frozenset[str] = frozenset({"view_node", "search_similar", "observe_frame"})
|
_SPAN_EVAL_TOOLS: frozenset[str] = frozenset({"view_node", "search_similar", "observe_frame"})
|
||||||
"""span 级评估涵盖的工具集合。"""
|
"""span 级评估涵盖的工具集合。"""
|
||||||
|
|
||||||
_INFRA_STOP_REASONS: frozenset[str] = frozenset({"error", "parse_error"})
|
INFRA_STOP_REASONS: frozenset[str] = frozenset({"error", "parse_error"})
|
||||||
"""执行/解析层失败导致排除的 stop_reason 集合。"""
|
"""执行/解析层失败导致排除的 stop_reason 集合。"""
|
||||||
|
|
||||||
|
|
||||||
@@ -1489,12 +1489,15 @@ def _build_skill_case_packs(
|
|||||||
if qm.correct:
|
if qm.correct:
|
||||||
continue
|
continue
|
||||||
attr = attribution_map.get(qm.question_id)
|
attr = attribution_map.get(qm.question_id)
|
||||||
if attr is not None and attr.cause_category == "lapse":
|
# 仅明确 defect 且非 degraded 才进正文进化路径;
|
||||||
if attr.lapse_note and attr.lapse_note.strip():
|
# lapse / cause_category=None(判别失败)/ degraded(judge 解析失败)一律保守走 lapse,
|
||||||
|
# 不以降级或未判定信号驱动错误进化。
|
||||||
|
is_defect = attr is not None and attr.cause_category == "defect" and not qm.degraded
|
||||||
|
if not is_defect:
|
||||||
|
if attr is not None and attr.lapse_note and attr.lapse_note.strip():
|
||||||
lapse_notes.append(attr.lapse_note)
|
lapse_notes.append(attr.lapse_note)
|
||||||
continue
|
continue
|
||||||
et = attr.error_type if attr else "mixed"
|
wrong_by_error[attr.error_type].append(qm)
|
||||||
wrong_by_error[et].append(qm)
|
|
||||||
|
|
||||||
# 单条 fallback
|
# 单条 fallback
|
||||||
n_body_failures = sum(len(group) for group in wrong_by_error.values())
|
n_body_failures = sum(len(group) for group in wrong_by_error.values())
|
||||||
@@ -2004,7 +2007,7 @@ def _count_infra_excluded(
|
|||||||
qids = [
|
qids = [
|
||||||
row["question_id"]
|
row["question_id"]
|
||||||
for row in prediction_rows
|
for row in prediction_rows
|
||||||
if row.get("stop_reason") in _INFRA_STOP_REASONS
|
if row.get("stop_reason") in INFRA_STOP_REASONS
|
||||||
]
|
]
|
||||||
return len(qids), qids
|
return len(qids), qids
|
||||||
|
|
||||||
@@ -2080,7 +2083,7 @@ async def run_diagnosis(
|
|||||||
|
|
||||||
for row in all_predictions:
|
for row in all_predictions:
|
||||||
stop_reason = row.get("stop_reason")
|
stop_reason = row.get("stop_reason")
|
||||||
if stop_reason in _INFRA_STOP_REASONS:
|
if stop_reason in INFRA_STOP_REASONS:
|
||||||
continue
|
continue
|
||||||
if task_type_filter and row.get("task_type") not in task_type_filter:
|
if task_type_filter and row.get("task_type") not in task_type_filter:
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ from loguru import logger
|
|||||||
from core.evolution.patch import (
|
from core.evolution.patch import (
|
||||||
APPENDIX_END,
|
APPENDIX_END,
|
||||||
APPENDIX_START,
|
APPENDIX_START,
|
||||||
|
MOMENTUM_END,
|
||||||
|
MOMENTUM_START,
|
||||||
append_to_appendix,
|
append_to_appendix,
|
||||||
apply_patch_with_report,
|
apply_patch_with_report,
|
||||||
extract_appendix_notes,
|
extract_appendix_notes,
|
||||||
@@ -293,10 +295,39 @@ def _tool_protected_spans(text: str) -> list[str]:
|
|||||||
# =========================================================================
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
def _check_marker_integrity(evolved: str) -> list[str]:
|
||||||
|
"""校验 evolved 中冻结区 marker 的完整性(成对、至多一对、START 先于 END)。
|
||||||
|
|
||||||
|
进化写入可能破坏 appendix/momentum marker 配对,破坏后 append_to_appendix /
|
||||||
|
replace_momentum 等下游会静默误拼或抛错。此处集中拦截:任一 marker 对违反
|
||||||
|
「START 数==END 数、各至多一对、START 在 END 前」即整体 reject。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
evolved: 改写后的全文。
|
||||||
|
返回:
|
||||||
|
错误信息列表(空列表表示 marker 完整)。
|
||||||
|
"""
|
||||||
|
errors: list[str] = []
|
||||||
|
for name, start_m, end_m in (
|
||||||
|
("APPENDIX", APPENDIX_START, APPENDIX_END),
|
||||||
|
("MOMENTUM", MOMENTUM_START, MOMENTUM_END),
|
||||||
|
):
|
||||||
|
s = evolved.count(start_m)
|
||||||
|
e = evolved.count(end_m)
|
||||||
|
if s != e:
|
||||||
|
errors.append(f"{name} marker 不配对:START={s} END={e}")
|
||||||
|
elif s > 1:
|
||||||
|
errors.append(f"{name} marker 出现多对({s}),至多一对")
|
||||||
|
elif s == 1 and evolved.index(start_m) > evolved.index(end_m):
|
||||||
|
errors.append(f"{name} marker 顺序错误:END 出现在 START 之前")
|
||||||
|
return errors
|
||||||
|
|
||||||
|
|
||||||
def validate_skill(original: str, evolved: str) -> ValidationResult:
|
def validate_skill(original: str, evolved: str) -> ValidationResult:
|
||||||
"""校验 Skill 改写结果。
|
"""校验 Skill 改写结果。
|
||||||
|
|
||||||
检查项: frontmatter 三字段保留(name / description / task_type)、
|
检查项: frontmatter 三字段保留(name / description / task_type)、
|
||||||
|
marker 完整性(appendix/momentum 成对且至多一对、顺序正确)、
|
||||||
长度比在 [0.3, 2.0]、代码块闭合。
|
长度比在 [0.3, 2.0]、代码块闭合。
|
||||||
|
|
||||||
参数:
|
参数:
|
||||||
@@ -305,6 +336,11 @@ def validate_skill(original: str, evolved: str) -> ValidationResult:
|
|||||||
|
|
||||||
返回:
|
返回:
|
||||||
ValidationResult 实例。
|
ValidationResult 实例。
|
||||||
|
|
||||||
|
关键实现细节:
|
||||||
|
marker 完整性先于长度校验——长度校验经 _strip_protected_regions 调用
|
||||||
|
momentum_region_bounds,对损坏 marker 会抛 ValueError;故 marker 破坏时先
|
||||||
|
返回失败,避免异常穿透且明确 reject 该候选。
|
||||||
"""
|
"""
|
||||||
errors: list[str] = []
|
errors: list[str] = []
|
||||||
orig_fm = _parse_frontmatter(original)
|
orig_fm = _parse_frontmatter(original)
|
||||||
@@ -319,6 +355,10 @@ def validate_skill(original: str, evolved: str) -> ValidationResult:
|
|||||||
errors.append(
|
errors.append(
|
||||||
f"frontmatter 字段 {key} 被修改: {orig_fm.get(key)!r} → {evol_fm.get(key)!r}"
|
f"frontmatter 字段 {key} 被修改: {orig_fm.get(key)!r} → {evol_fm.get(key)!r}"
|
||||||
)
|
)
|
||||||
|
marker_errors = _check_marker_integrity(evolved)
|
||||||
|
if marker_errors:
|
||||||
|
errors.extend(marker_errors)
|
||||||
|
return ValidationResult(passed=False, errors=errors)
|
||||||
errors.extend(_check_length(original, evolved))
|
errors.extend(_check_length(original, evolved))
|
||||||
errors.extend(_check_code_blocks(evolved))
|
errors.extend(_check_code_blocks(evolved))
|
||||||
return ValidationResult(passed=len(errors) == 0, errors=errors)
|
return ValidationResult(passed=len(errors) == 0, errors=errors)
|
||||||
|
|||||||
+43
-7
@@ -282,9 +282,34 @@ def _protected_ranges(content: str, spans: list[str]) -> list[tuple[int, int]]:
|
|||||||
return ranges
|
return ranges
|
||||||
|
|
||||||
|
|
||||||
def _in_ranges(pos: int, ranges: list[tuple[int, int]]) -> bool:
|
def _span_overlaps_ranges(pos: int, length: int, ranges: list[tuple[int, int]]) -> bool:
|
||||||
"""判断位置 pos 是否落在任意冻结区间内。"""
|
"""判断 [pos, pos+length) 是否与任一冻结区间相交(不止起点)。
|
||||||
return any(start <= pos < end for start, end in ranges)
|
|
||||||
|
起点落在正文、末端伸入冻结区的 target 也须拦截,否则 replace/delete 会连带
|
||||||
|
改动冻结区(如破坏 appendix/momentum marker)。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
pos: target 在正文中的起点。
|
||||||
|
length: target 长度。
|
||||||
|
ranges: 冻结区间 [start, end) 列表。
|
||||||
|
返回:
|
||||||
|
与任一区间相交返回 True。
|
||||||
|
"""
|
||||||
|
end = pos + length
|
||||||
|
return any(start < end and pos < r_end for start, r_end in ranges)
|
||||||
|
|
||||||
|
|
||||||
|
# 冻结区 marker 字面量:LLM 生成的 edit 不得注入这些字面量,否则破坏 marker 配对
|
||||||
|
_MARKER_LITERALS = (APPENDIX_START, APPENDIX_END, MOMENTUM_START, MOMENTUM_END)
|
||||||
|
|
||||||
|
|
||||||
|
def _edit_injects_marker(edit: dict) -> bool:
|
||||||
|
"""判断 edit 的 target/content 是否含冻结区 marker 字面量(注入拦截)。"""
|
||||||
|
for key in ("target", "content"):
|
||||||
|
value = edit.get(key)
|
||||||
|
if isinstance(value, str) and any(m in value for m in _MARKER_LITERALS):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def _append_at(content: str, ranges: list[tuple[int, int]]) -> int:
|
def _append_at(content: str, ranges: list[tuple[int, int]]) -> int:
|
||||||
@@ -317,8 +342,8 @@ def _do_insert_after(
|
|||||||
_insert_at(content, _append_at(content, ranges), payload),
|
_insert_at(content, _append_at(content, ranges), payload),
|
||||||
"applied_insert_after_fallback",
|
"applied_insert_after_fallback",
|
||||||
)
|
)
|
||||||
if _in_ranges(pos, ranges):
|
if _span_overlaps_ranges(pos, len(target), ranges):
|
||||||
logger.warning("insert_after 目标在冻结区,跳过 target={}", target[:80])
|
logger.warning("insert_after 目标跨入冻结区,跳过 target={}", target[:80])
|
||||||
return content, "skipped_protected"
|
return content, "skipped_protected"
|
||||||
at = pos + len(target)
|
at = pos + len(target)
|
||||||
nl = content.find("\n", at)
|
nl = content.find("\n", at)
|
||||||
@@ -340,8 +365,8 @@ def _do_replace_delete(
|
|||||||
if pos == -1:
|
if pos == -1:
|
||||||
logger.warning("{} 锚点缺失,跳过 target={}", op, target[:80])
|
logger.warning("{} 锚点缺失,跳过 target={}", op, target[:80])
|
||||||
return content, "skipped_target_not_found"
|
return content, "skipped_target_not_found"
|
||||||
if _in_ranges(pos, ranges):
|
if _span_overlaps_ranges(pos, len(target), ranges):
|
||||||
logger.warning("{} 目标在冻结区,跳过 target={}", op, target[:80])
|
logger.warning("{} 目标跨入冻结区,跳过 target={}", op, target[:80])
|
||||||
return content, "skipped_protected"
|
return content, "skipped_protected"
|
||||||
new_content = content.replace(target, payload if op == "replace" else "", 1)
|
new_content = content.replace(target, payload if op == "replace" else "", 1)
|
||||||
return new_content, "applied_" + op
|
return new_content, "applied_" + op
|
||||||
@@ -403,6 +428,17 @@ def apply_patch_with_report(
|
|||||||
reports: list[dict] = []
|
reports: list[dict] = []
|
||||||
for i, edit in enumerate(edits, 1):
|
for i, edit in enumerate(edits, 1):
|
||||||
try:
|
try:
|
||||||
|
if isinstance(edit, dict) and _edit_injects_marker(edit):
|
||||||
|
logger.warning("edit 含冻结区 marker 字面量,拒绝该 edit index={}", i)
|
||||||
|
report = {
|
||||||
|
"op": str(edit.get("op", "")),
|
||||||
|
"target": str(edit.get("target", "") or "")[:200],
|
||||||
|
"content_preview": str(edit.get("content", "") or "")[:200],
|
||||||
|
"status": "skipped_marker_injection",
|
||||||
|
}
|
||||||
|
report["index"] = i
|
||||||
|
reports.append(report)
|
||||||
|
continue
|
||||||
ranges = _protected_ranges(content, spans)
|
ranges = _protected_ranges(content, spans)
|
||||||
content, report = _apply_one(content, edit, ranges)
|
content, report = _apply_one(content, edit, ranges)
|
||||||
except (KeyError, TypeError, ValueError, AttributeError) as exc:
|
except (KeyError, TypeError, ValueError, AttributeError) as exc:
|
||||||
|
|||||||
@@ -127,12 +127,20 @@ class DiagnosisSignalStore(Protocol):
|
|||||||
"""
|
"""
|
||||||
...
|
...
|
||||||
|
|
||||||
def done_question_ids(self, baseline_run_id: str, diag_fingerprint: str) -> set[str]:
|
def done_question_ids(
|
||||||
|
self,
|
||||||
|
baseline_run_id: str,
|
||||||
|
diag_fingerprint: str,
|
||||||
|
*,
|
||||||
|
retry_uncertain: bool = False,
|
||||||
|
) -> set[str]:
|
||||||
"""查询指定 run 与诊断指纹下已完成的 question_id 集合。
|
"""查询指定 run 与诊断指纹下已完成的 question_id 集合。
|
||||||
|
|
||||||
参数:
|
参数:
|
||||||
baseline_run_id: baseline run 标识。
|
baseline_run_id: baseline run 标识。
|
||||||
diag_fingerprint: 诊断口径指纹。
|
diag_fingerprint: 诊断口径指纹。
|
||||||
|
retry_uncertain: True 时把 tier='uncertain'(信号不可信降级)题视为
|
||||||
|
未完成,令其被重新诊断;默认 False(uncertain 也算完成,不重诊)。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
已落盘信号的 question_id 集合,用于断点续跑跳过。
|
已落盘信号的 question_id 集合,用于断点续跑跳过。
|
||||||
|
|||||||
@@ -484,7 +484,6 @@ class DiagnosePrompts:
|
|||||||
defect_vs_lapse: defect/lapse 病因判别模板。
|
defect_vs_lapse: defect/lapse 病因判别模板。
|
||||||
reasoning_sub: 推理失败子分类模板。
|
reasoning_sub: 推理失败子分类模板。
|
||||||
span_eval_system: span 评估系统提示模板。
|
span_eval_system: span 评估系统提示模板。
|
||||||
span_eval_user: span 评估用户提示模板。
|
|
||||||
missed_nodes: 遗漏节点检测模板。
|
missed_nodes: 遗漏节点检测模板。
|
||||||
skill_adherence: 技能遵循判定模板。
|
skill_adherence: 技能遵循判定模板。
|
||||||
confirmation_bias: 确认偏误检测模板。
|
confirmation_bias: 确认偏误检测模板。
|
||||||
@@ -494,7 +493,6 @@ class DiagnosePrompts:
|
|||||||
defect_vs_lapse: str
|
defect_vs_lapse: str
|
||||||
reasoning_sub: str
|
reasoning_sub: str
|
||||||
span_eval_system: str
|
span_eval_system: str
|
||||||
span_eval_user: str
|
|
||||||
missed_nodes: str
|
missed_nodes: str
|
||||||
skill_adherence: str
|
skill_adherence: str
|
||||||
confirmation_bias: str
|
confirmation_bias: str
|
||||||
@@ -512,11 +510,9 @@ class EvolvePrompts:
|
|||||||
evolve_system: System Prompt 进化提示模板。
|
evolve_system: System Prompt 进化提示模板。
|
||||||
evolve_tool: Tool Prompt 进化提示模板。
|
evolve_tool: Tool Prompt 进化提示模板。
|
||||||
evolve_rank: 编辑排序提示模板。
|
evolve_rank: 编辑排序提示模板。
|
||||||
consolidate_system: appendix 压缩系统提示。
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
evolve_skill: str
|
evolve_skill: str
|
||||||
evolve_system: str
|
evolve_system: str
|
||||||
evolve_tool: str
|
evolve_tool: str
|
||||||
evolve_rank: str
|
evolve_rank: str
|
||||||
consolidate_system: str
|
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ class LLMProvider(Protocol):
|
|||||||
*,
|
*,
|
||||||
session_id: str | None = None,
|
session_id: str | None = None,
|
||||||
parent_call_id: str | None = None,
|
parent_call_id: str | None = None,
|
||||||
|
cache_salt: str | None = None,
|
||||||
) -> LLMResponse: ...
|
) -> LLMResponse: ...
|
||||||
|
|
||||||
|
|
||||||
@@ -39,6 +40,7 @@ class VLMProvider(Protocol):
|
|||||||
*,
|
*,
|
||||||
session_id: str | None = None,
|
session_id: str | None = None,
|
||||||
parent_call_id: str | None = None,
|
parent_call_id: str | None = None,
|
||||||
|
cache_salt: str | None = None,
|
||||||
) -> LLMResponse: ...
|
) -> LLMResponse: ...
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -84,13 +84,14 @@ def _build_adapters(settings: InfraSettings, embed_cfg: dict) -> _Adapters:
|
|||||||
|
|
||||||
cache = None
|
cache = None
|
||||||
if settings.redis_url:
|
if settings.redis_url:
|
||||||
|
from adapters.redis_cache import RedisResponseCache, _resolve_cache_ttl
|
||||||
|
|
||||||
|
# 配置校验 fail-loud(不属于 Redis 连接故障,不得被下方降级 except 吞掉)
|
||||||
|
ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl)
|
||||||
try:
|
try:
|
||||||
import redis.asyncio as aioredis
|
import redis.asyncio as aioredis
|
||||||
|
|
||||||
from adapters.redis_cache import RedisResponseCache
|
|
||||||
|
|
||||||
redis_client = aioredis.from_url(settings.redis_url, decode_responses=True)
|
redis_client = aioredis.from_url(settings.redis_url, decode_responses=True)
|
||||||
ttl_s = settings.redis_cache_ttl if settings.redis_cache_ttl > 0 else None
|
|
||||||
cache = RedisResponseCache(redis=redis_client, ttl_s=ttl_s)
|
cache = RedisResponseCache(redis=redis_client, ttl_s=ttl_s)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.warning("Redis 缓存不可用,降级为无缓存模式")
|
logger.warning("Redis 缓存不可用,降级为无缓存模式")
|
||||||
|
|||||||
@@ -0,0 +1,8 @@
|
|||||||
|
你是一个改动优先级裁判。你会收到一份当前 prompt 文件全文,和一组待应用的局部 edits(每条含 op/target/content)。由于本轮编辑预算有限,你只能保留其中最重要的若干条。
|
||||||
|
|
||||||
|
请只依据"对纠正失败、提升正确率的预期贡献"排序:优先保留直接修复失败模式的改动,其次保留收窄或澄清的改动,最后才是巩固已有成功的改动。删除类、简化类的精准改动通常优先于追加大段新内容。
|
||||||
|
|
||||||
|
每条 edit 会附带 support_count(该改动的支持案例数)。同等重要性下,support_count 更高的优先;但 support_count 低不等于该删,仍以修复贡献为主判据。
|
||||||
|
|
||||||
|
严格输出以下 JSON,不要包含其他文字:
|
||||||
|
{"selected_indices": [按重要性降序排列的 0-based 索引]}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
你是一个搜索策略改进专家。你服务于一个自进化视频搜索系统,该系统通过分析 Agent 的失败和成功案例来迭代改进搜索策略(Skill)。你的任务是基于案例包中的证据,改写当前 Skill 文件,使 Agent 在后续执行中避免相同的失败模式。
|
||||||
|
|
||||||
|
## 你会收到的输入
|
||||||
|
|
||||||
|
1. 当前 Skill 文件全文
|
||||||
|
2. 失败案例:Agent 答错的题目,含完整推理轨迹、错误类型和诊断指标
|
||||||
|
3. 成功案例:Agent 答对的题目,展示当前 Skill 中有效的模式
|
||||||
|
4. 聚合统计:准确率、错误归因分布、搜索有效性指标、Skill 步骤遵循率
|
||||||
|
5. (可能出现)上一轮被接受改动导致的回归题:这些题在上一版本答对、却被你上次的改写改错了,附基线与候选两份预测和推理轨迹
|
||||||
|
6. (可能出现)黑名单:已被实测验证无效或有害的改法方向
|
||||||
|
|
||||||
|
## 工作原则
|
||||||
|
|
||||||
|
如果输入里出现了回归题,它的优先级高于一切。这些题在上一版本是对的,是你上次的改动把它们弄坏的,所以本次改写的第一要务是确保不再破坏它们——宁可在相关方向上回退或收窄,也不要为了拉高其它题而牺牲它们。更一般地,当你看到准确率下降这类负向信号时,默认先怀疑上次是不是加了过度、冲突或冗余的指令,优先简化、删除、收窄;只有确认简化解决不了问题,才考虑加强指令。黑名单里列出的改法已经被实测证明无效或有害,不要换个措辞把同一个方向再提一遍。
|
||||||
|
|
||||||
|
先分析失败案例中 Agent 的实际行为与 Skill 指令的偏差。偏差分两类:Skill 指令正确但 Agent 没遵循(遵循率问题),或 Skill 指令本身有误导(策略问题)。前者需要让指令更具体、更难被忽略;后者需要修改策略本身。
|
||||||
|
|
||||||
|
从成功案例中识别有效模式——这些模式在改写时必须保留。如果成功案例和失败案例采用了不同的策略路径,重点强化成功路径。
|
||||||
|
|
||||||
|
Skill 中引用的统计数据(如"search-first 正确率 75%")应根据案例包中的新统计更新。不要编造数据,只使用案例包中提供的数字。
|
||||||
|
|
||||||
|
你写进 Skill 的每一条规则都必须是可跨题复用的通用策略,而不是对某一道题的记答案。跨多个失败案例时只提取共性模式,抽象掉一切单题特征——具体题目内容、选项文字、步骤序号、某一帧的具体画面、某个具体答案都不许写进 Skill 正文。一条规则如果只在它来源的那道题上成立,就不要加。改写时优先简化与收窄:宁可让 Skill 更短,也不要堆叠只对个别题生效的硬性指令。
|
||||||
|
|
||||||
|
## 冻结区
|
||||||
|
|
||||||
|
以下内容不可修改,必须原样保留在改写后的文件中:
|
||||||
|
- YAML frontmatter(`---` 之间的 name、description、task_type)
|
||||||
|
- 输出格式中的 JSON 基础结构(reflect/plan/action 三个顶层字段)
|
||||||
|
|
||||||
|
这次不要返回整份改写后的文件,而是只返回一组局部 edits。`append` 用来在文件末尾追加一个新 section,`insert_after` 用来把内容紧跟着插到某个锚点段落之后,`replace` 用来用新内容整体替换 target 对应的原文,`delete` 则直接删除 target 对应的原文并让 content 留空。target 必须是从当前文件里逐字复制出来的原文,而且要长到足以唯一定位;只要有任何一个字不完全匹配,这条改动就会被跳过。改动应尽量小而局部,优先做精确补丁,不要动辄重写整段整节;另外,冻结区里的文字绝不能作为 target。
|
||||||
|
|
||||||
|
## 输出格式
|
||||||
|
|
||||||
|
请严格输出以下 JSON,不要包含其他文字:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"suggestions": [
|
||||||
|
{
|
||||||
|
"section": "改动目标段落的标题或位置描述",
|
||||||
|
"problem": "失败案例中暴露的具体问题",
|
||||||
|
"change": "具体的修改方向",
|
||||||
|
"related_cases": ["关联的失败案例 question_id"],
|
||||||
|
"support_count": 该建议的支持案例数(= related_cases 的数量)
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"edits": [
|
||||||
|
{"op": "append|insert_after|replace|delete", "target": "锚点原文(append 留空)", "content": "新内容(delete 留空)", "support_count": 该改动的支持案例数}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
每条 edit 与每条 suggestion 都必须带 "support_count":本条改动由多少个失败案例共同支持(即 related_cases 的数量)。support_count 越高代表证据越充分;它只作排序参考,不是硬门槛——support_count 低不等于该删,仍以修复贡献为主判据。
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
你是一个系统级行为改进专家。你服务于一个自进化视频搜索系统,该系统通过分析 Agent 的跨题型行为模式来改进 System Prompt。你的任务是基于行为模式案例包中的证据,改写 System Prompt 中的策略性指令,纠正系统级行为问题。
|
||||||
|
|
||||||
|
## 你会收到的输入
|
||||||
|
|
||||||
|
1. 当前 System Prompt (system.md) 全文
|
||||||
|
2. 失败案例:展示三类系统性行为问题的题目——过早提交(budget_usage < 0.3 就提交答案)、高置信答错(confidence 很高但答案错误)、确认偏误(只搜索支持初始判断的证据)
|
||||||
|
3. 成功案例:行为校准良好的题目——置信度与正确率匹配,预算使用适中
|
||||||
|
4. D5 行为模式统计:各行为模式的发生频率和分布
|
||||||
|
5. (可能出现)黑名单:已被实测验证无效或有害的改法方向
|
||||||
|
|
||||||
|
## 工作原则
|
||||||
|
|
||||||
|
关注跨题型的系统性行为模式,而非某个具体题型的策略。失败案例中的行为偏差反映了 System Prompt 的决策原则不够清晰或不够强约束。黑名单里的改法已经被实测验证无效或有害,不要再朝同一个方向改一遍。
|
||||||
|
|
||||||
|
过早提交说明预算管理指令需要更强的约束语言。高置信答错说明置信度校准的语义定义需要调整。确认偏误说明竞争选项搜索的要求需要更明确。
|
||||||
|
|
||||||
|
当你看到失败案例与成功案例并存时,失败修复优先于巩固成功——先确保失败模式被纠正,再考虑强化已有的好行为。看到某类行为指标变差这类负向信号时,默认先怀疑上一轮是否加了过度、冲突或冗余的约束,优先简化、删除、收窄;只有确认简化解决不了,才考虑加强约束语言。
|
||||||
|
|
||||||
|
从成功案例中提取"好行为"的特征,在改写时强化这些特征的表述。
|
||||||
|
|
||||||
|
## 冻结区
|
||||||
|
|
||||||
|
以下 section 必须原样保留,不可修改任何文字:
|
||||||
|
- `## 能力边界`(事实性描述,不是策略)
|
||||||
|
- `## 输出格式`(JSON schema 是系统契约)
|
||||||
|
- `## 视频树结构`(含信任层级,是数据结构事实描述)
|
||||||
|
|
||||||
|
可改写的 section:
|
||||||
|
- `## 角色`(前两段的角色定位和行为倾向描述)
|
||||||
|
- `## 决策原则`(搜索策略、预算分配建议)
|
||||||
|
- 搜索工具使用、否定题原则、置信度语义
|
||||||
|
|
||||||
|
这次不要返回整份改写后的文件,而是只返回一组局部 edits。`append` 用来在文件末尾追加一个新 section,`insert_after` 用来把内容紧跟着插到某个锚点段落之后,`replace` 用来用新内容整体替换 target 对应的原文,`delete` 则直接删除 target 对应的原文并让 content 留空。target 必须是从当前文件里逐字复制出来的原文,而且要长到足以唯一定位;只要有任何一个字不完全匹配,这条改动就会被跳过。改动应尽量小而局部,优先做精确补丁,不要动辄重写整段整节;另外,冻结区里的文字绝不能作为 target。
|
||||||
|
|
||||||
|
## 输出格式
|
||||||
|
|
||||||
|
请严格输出以下 JSON,不要包含其他文字:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"suggestions": [
|
||||||
|
{
|
||||||
|
"section": "改动目标段落的标题或位置描述",
|
||||||
|
"problem": "失败案例中暴露的具体行为问题",
|
||||||
|
"change": "具体的修改方向",
|
||||||
|
"related_cases": ["关联的失败案例 question_id"]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"edits": [
|
||||||
|
{"op": "append|insert_after|replace|delete", "target": "锚点原文(append 留空)", "content": "新内容(delete 留空)"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
你是一个工具 Prompt 改进专家。你服务于一个自进化视频搜索系统,该系统的每个工具(view_node、search_similar、observe_frame 等)有两个配套 Prompt:extract(信息提取)和 verify(结果核实)。你的任务是基于工具调用级别的质量数据,同时改写一个工具的 extract 和 verify prompt。
|
||||||
|
|
||||||
|
## 你会收到的输入
|
||||||
|
|
||||||
|
1. 当前 extract prompt 和 verify prompt 全文
|
||||||
|
2. 失败 span 案例:提取完整度低或幻觉率高的具体工具调用,含工具参数、工具输出、原始数据(ground truth)和质量评估指标
|
||||||
|
3. 成功 span 案例:提取完整且无幻觉的工具调用样本
|
||||||
|
4. 工具质量统计:平均提取完整度、平均幻觉率、top 遗漏类型、top 幻觉类型
|
||||||
|
5. (可能出现)黑名单:已被实测验证无效或有害的改法方向
|
||||||
|
|
||||||
|
## 工作原则
|
||||||
|
|
||||||
|
失败 span 中提取完整度低说明 extract prompt 的工作原则不够具体——Agent 遗漏了哪些类型的信息?幻觉率高说明 extract prompt 对"忠实提取"的约束不够强,或者 verify prompt 没能有效检出幻觉。黑名单里的改法已经被实测验证无效或有害,不要再朝同一个方向改一遍。
|
||||||
|
|
||||||
|
extract 和 verify 是互补的:extract 负责提取,verify 负责检查。如果 extract 反复遗漏某类信息(如字幕原文引用),应在 extract 的工作原则中明确要求保留该类信息。如果 verify 未能检出某类幻觉(如虚构动作),应在 verify 的检查要点中增加对该模式的关注。
|
||||||
|
|
||||||
|
失败修复优先于巩固成功——先纠正提取遗漏或幻觉,再保留已有的有效模式。当某类提取质量指标变差时,先确认不是上一轮加了过度或冲突的要求所致;加强 extract 要求前,先确认简化或收窄已有指令解决不了这个遗漏,再追加新要求。
|
||||||
|
|
||||||
|
从成功案例中识别有效的提取模式,确保改写不破坏这些模式。
|
||||||
|
|
||||||
|
## 冻结区
|
||||||
|
|
||||||
|
以下内容不可修改:
|
||||||
|
- 角色定位第一句("你是一个视频节点内容分析器" / "你是一个视频节点摘要核实器")
|
||||||
|
- `## 你会收到的输入` section
|
||||||
|
- `## 输出格式` section
|
||||||
|
|
||||||
|
可改写的 section:
|
||||||
|
- `## 工作原则`
|
||||||
|
- `## 检查要点`(verify 专有)
|
||||||
|
|
||||||
|
这次不要再返回两份完整 prompt,而是分别给 extract 和 verify 各自的局部 edits 列表。`append` 用来在文件末尾追加一个新 section,`insert_after` 用来把内容紧跟着插到某个锚点段落之后,`replace` 用来用新内容整体替换 target 对应的原文,`delete` 则直接删除 target 对应的原文并让 content 留空。target 必须是从当前 prompt 里逐字复制出来的原文,而且要长到足以唯一定位;只要有任何一个字不完全匹配,这条改动就会被跳过。改动应尽量小而局部,优先做精确补丁,不要动辄重写大段内容;另外,冻结区里的文字绝不能作为 target,extract 和 verify 也必须分别使用自己的 edit 列表。
|
||||||
|
|
||||||
|
## 输出格式
|
||||||
|
|
||||||
|
请严格输出以下 JSON,不要包含其他文字:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"suggestions": [
|
||||||
|
{
|
||||||
|
"section": "改动目标段落的标题或位置描述",
|
||||||
|
"problem": "失败 span 中暴露的具体问题",
|
||||||
|
"change": "具体的修改方向",
|
||||||
|
"related_cases": ["关联的失败 span 标识"]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"edits_extract": [
|
||||||
|
{"op": "append|insert_after|replace|delete", "target": "锚点原文(append 留空)", "content": "新内容(delete 留空)"}
|
||||||
|
],
|
||||||
|
"edits_verify": [
|
||||||
|
{"op": "append|insert_after|replace|delete", "target": "锚点原文(append 留空)", "content": "新内容(delete 留空)"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
你正在审视一份 skill 经历一轮进化后的变化。这份 skill 指导一个 Agent 在层次化视频树上搜索证据、回答长视频理解问题。在上一轮结束时它是一个样子,这一轮结束时被改成了另一个样子;与此同时,你在上一轮还为它写下过一段动量指导,本意是给这一轮的进化指明方向。现在你要回头评判:那段指导究竟有没有帮上忙,这一轮的正文改动是真的在改善,还是开始往无关的方向漂移。
|
||||||
|
|
||||||
|
你会拿到四样东西:上一版 skill 的正文、当前版 skill 的正文、你上一轮写下的那段动量指导,以及一组固定样本上的纵向对比——同一批题,分别用上一版和当前版各跑了一遍,逐题列出两版的预测与正误。这组对比是你唯一可靠的证据来源:哪些题从错变对、哪些题从对变错、哪些题始终没做对、哪些题一直稳定答对,正是这四类信号告诉你这轮改动到底带来了什么。
|
||||||
|
|
||||||
|
请先反思再下笔。对照纵向对比,先问上一轮那段动量指导是否真的奏效:它所指向的方向,在这一轮的正文改动里被落实了吗,落实之后那些本该改善的题改善了吗?再问这一轮的正文改动本身是收敛还是漂移:从对变错的题(回退)是最该警惕的信号,说明某处改动伤到了原本正确的行为;始终答错的题(持续失败)说明还有方向没被触及;从错变对的题(改善)则印证了哪条路走对了,值得继续加码。
|
||||||
|
|
||||||
|
想清楚之后,写出一段全新的、聚焦的、可操作的动量指导。它会被原样写进 skill 的受保护动量区,作为下一轮进化的方向锚——所以它必须是一段连贯的指导文字,明确告诉下一轮该往哪个方向继续使劲、又要避免重蹈哪一类改动的覆辙,而不是一堆零散的待办条目。如果上一轮的方向已被证明有效,就强化并细化它;如果出现了回退,就明确叫停那条路并指向修复方向。
|
||||||
|
|
||||||
|
严格输出以下 JSON,不要包含任何其他文字:
|
||||||
|
{"reasoning": "你的反思过程:上一轮指导是否奏效、这一轮是改善还是漂移,引用纵向对比中的具体题作为依据", "slow_update_content": "一段连贯、聚焦、可操作的新动量指导,指引下一轮的进化方向"}
|
||||||
@@ -245,6 +245,31 @@
|
|||||||
"id": "design:preflight-fixes",
|
"id": "design:preflight-fixes",
|
||||||
"label": "训练前缺陷修复设计",
|
"label": "训练前缺陷修复设计",
|
||||||
"type": "design"
|
"type": "design"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "plan:preflight-wp1-asset-migration",
|
||||||
|
"label": "WP1 资产迁移",
|
||||||
|
"type": "plan"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "plan:preflight-wp2-split-wiring",
|
||||||
|
"label": "WP2 切分与接线",
|
||||||
|
"type": "plan"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "plan:preflight-wp3-train-loop",
|
||||||
|
"label": "WP3 训练循环与进化引擎",
|
||||||
|
"type": "plan"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "plan:preflight-wp4-resilience",
|
||||||
|
"label": "WP4 韧性与持久化",
|
||||||
|
"type": "plan"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "review:preflight-final-review",
|
||||||
|
"label": "训练前修复分支终审",
|
||||||
|
"type": "review"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"links": [
|
"links": [
|
||||||
@@ -492,6 +517,41 @@
|
|||||||
"relation": "implements",
|
"relation": "implements",
|
||||||
"evidence": "实现修复设计的 5 个 Task",
|
"evidence": "实现修复设计的 5 个 Task",
|
||||||
"added": "2026-07-16T02:01:19.350441+00:00"
|
"added": "2026-07-16T02:01:19.350441+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source": "plan:preflight-wp1-asset-migration",
|
||||||
|
"target": "design:preflight-fixes",
|
||||||
|
"relation": "implements",
|
||||||
|
"evidence": "四工作包实现训练前缺陷修复设计",
|
||||||
|
"added": "2026-07-16T08:32:42.737883+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source": "plan:preflight-wp2-split-wiring",
|
||||||
|
"target": "design:preflight-fixes",
|
||||||
|
"relation": "implements",
|
||||||
|
"evidence": "四工作包实现训练前缺陷修复设计",
|
||||||
|
"added": "2026-07-16T08:32:42.822569+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source": "plan:preflight-wp3-train-loop",
|
||||||
|
"target": "design:preflight-fixes",
|
||||||
|
"relation": "implements",
|
||||||
|
"evidence": "四工作包实现训练前缺陷修复设计",
|
||||||
|
"added": "2026-07-16T08:32:42.906881+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source": "plan:preflight-wp4-resilience",
|
||||||
|
"target": "design:preflight-fixes",
|
||||||
|
"relation": "implements",
|
||||||
|
"evidence": "四工作包实现训练前缺陷修复设计",
|
||||||
|
"added": "2026-07-16T08:32:42.990273+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source": "review:preflight-final-review",
|
||||||
|
"target": "design:preflight-fixes",
|
||||||
|
"relation": "informs",
|
||||||
|
"evidence": "跨 task 终审确认设计 21 项缺陷全部落地 + 4 项集成发现",
|
||||||
|
"added": "2026-07-16T11:06:43.709183+00:00"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
+13
-3
@@ -1,6 +1,6 @@
|
|||||||
# Research Wiki 索引
|
# Research Wiki 索引
|
||||||
|
|
||||||
> 自动生成,更新时间:2026-07-16 07:43 UTC
|
> 自动生成,更新时间:2026-07-16 11:06 UTC
|
||||||
|
|
||||||
## design (36)
|
## design (36)
|
||||||
- [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design`
|
- [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design`
|
||||||
@@ -52,7 +52,7 @@
|
|||||||
- [v3 §9 帧感知抽取机制 — 小样本实证验证结果](findings/2026-07-15-v3-frame-perception-spike-validation.md) `finding:2026-07-15-v3-frame-perception-spike-validation`
|
- [v3 §9 帧感知抽取机制 — 小样本实证验证结果](findings/2026-07-15-v3-frame-perception-spike-validation.md) `finding:2026-07-15-v3-frame-perception-spike-validation`
|
||||||
- [出题范式转变 — 从"生成-打分-过滤"转向"构造优先 + 两正交独立信号"(六篇原文深读)](findings/2026-07-15-question-gen-paradigm-shift-construction-over-filtering.md) `finding:2026-07-15-question-gen-paradigm-shift-construction-over-filtering`
|
- [出题范式转变 — 从"生成-打分-过滤"转向"构造优先 + 两正交独立信号"(六篇原文深读)](findings/2026-07-15-question-gen-paradigm-shift-construction-over-filtering.md) `finding:2026-07-15-question-gen-paradigm-shift-construction-over-filtering`
|
||||||
|
|
||||||
## plan (40)
|
## plan (48)
|
||||||
- [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm`
|
- [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm`
|
||||||
- [2026-07-07-app-harness](plans/2026-07-07-app-harness.md) `plan:2026-07-07-app-harness`
|
- [2026-07-07-app-harness](plans/2026-07-07-app-harness.md) `plan:2026-07-07-app-harness`
|
||||||
- [2026-07-07-core-evolution](plans/2026-07-07-core-evolution.md) `plan:2026-07-07-core-evolution`
|
- [2026-07-07-core-evolution](plans/2026-07-07-core-evolution.md) `plan:2026-07-07-core-evolution`
|
||||||
@@ -72,6 +72,10 @@
|
|||||||
- [2026-07-14-task-type-strategy-framework](plans/2026-07-14-task-type-strategy-framework.md) `plan:2026-07-14-task-type-strategy-framework`
|
- [2026-07-14-task-type-strategy-framework](plans/2026-07-14-task-type-strategy-framework.md) `plan:2026-07-14-task-type-strategy-framework`
|
||||||
- [2026-07-15-question-gen-v3-phase1-contract](plans/2026-07-15-question-gen-v3-phase1-contract.md) `plan:2026-07-15-question-gen-v3-phase1-contract`
|
- [2026-07-15-question-gen-v3-phase1-contract](plans/2026-07-15-question-gen-v3-phase1-contract.md) `plan:2026-07-15-question-gen-v3-phase1-contract`
|
||||||
- [2026-07-15-results-driven-video-split](plans/2026-07-15-results-driven-video-split.md) `plan:2026-07-15-results-driven-video-split`
|
- [2026-07-15-results-driven-video-split](plans/2026-07-15-results-driven-video-split.md) `plan:2026-07-15-results-driven-video-split`
|
||||||
|
- [2026-07-16-preflight-wp1-asset-migration](plans/2026-07-16-preflight-wp1-asset-migration.md) `plan:2026-07-16-preflight-wp1-asset-migration`
|
||||||
|
- [2026-07-16-preflight-wp2-split-wiring](plans/2026-07-16-preflight-wp2-split-wiring.md) `plan:2026-07-16-preflight-wp2-split-wiring`
|
||||||
|
- [2026-07-16-preflight-wp3-train-loop](plans/2026-07-16-preflight-wp3-train-loop.md) `plan:2026-07-16-preflight-wp3-train-loop`
|
||||||
|
- [2026-07-16-preflight-wp4-resilience](plans/2026-07-16-preflight-wp4-resilience.md) `plan:2026-07-16-preflight-wp4-resilience`
|
||||||
- [Action Recognition 单题型首次训练实验计划](plans/action-recognition-training.md) `plan:action-recognition-training`
|
- [Action Recognition 单题型首次训练实验计划](plans/action-recognition-training.md) `plan:action-recognition-training`
|
||||||
- [ActionRecognitionStrategy 特化实现计划 (Plan B)](plans/action-recognition-strategy.md) `plan:action-recognition-strategy`
|
- [ActionRecognitionStrategy 特化实现计划 (Plan B)](plans/action-recognition-strategy.md) `plan:action-recognition-strategy`
|
||||||
- [Adversarial Question-Gen Phase B](plans/adversarial-question-gen-phaseB.md) `plan:adversarial-question-gen-phaseB`
|
- [Adversarial Question-Gen Phase B](plans/adversarial-question-gen-phaseB.md) `plan:adversarial-question-gen-phaseB`
|
||||||
@@ -86,6 +90,10 @@
|
|||||||
- [Spec-1 Agent 执行环境修复实现计划](plans/agent-runtime-fixes-plan.md) `plan:agent-runtime-fixes-plan`
|
- [Spec-1 Agent 执行环境修复实现计划](plans/agent-runtime-fixes-plan.md) `plan:agent-runtime-fixes-plan`
|
||||||
- [Spec-2 建树批量并行实现计划](plans/batch-tree-build-plan.md) `plan:batch-tree-build-plan`
|
- [Spec-2 建树批量并行实现计划](plans/batch-tree-build-plan.md) `plan:batch-tree-build-plan`
|
||||||
- [TaskTypeStrategy 框架实现计划 (Plan A)](plans/task-type-strategy-framework.md) `plan:task-type-strategy-framework`
|
- [TaskTypeStrategy 框架实现计划 (Plan A)](plans/task-type-strategy-framework.md) `plan:task-type-strategy-framework`
|
||||||
|
- [WP1 资产迁移](plans/preflight-wp1-asset-migration.md) `plan:preflight-wp1-asset-migration`
|
||||||
|
- [WP2 切分与接线](plans/preflight-wp2-split-wiring.md) `plan:preflight-wp2-split-wiring`
|
||||||
|
- [WP3 训练循环与进化引擎](plans/preflight-wp3-train-loop.md) `plan:preflight-wp3-train-loop`
|
||||||
|
- [WP4 韧性与持久化](plans/preflight-wp4-resilience.md) `plan:preflight-wp4-resilience`
|
||||||
- [出题管线 v2 实现计划](plans/2026-07-11-question-gen-v2.md) `plan:2026-07-11-question-gen-v2`
|
- [出题管线 v2 实现计划](plans/2026-07-11-question-gen-v2.md) `plan:2026-07-11-question-gen-v2`
|
||||||
- [实现计划: 修复诊断 tree_data 断链 bug](plans/fix-diagnosis-tree-data-link-plan.md) `plan:fix-diagnosis-tree-data-link-plan`
|
- [实现计划: 修复诊断 tree_data 断链 bug](plans/fix-diagnosis-tree-data-link-plan.md) `plan:fix-diagnosis-tree-data-link-plan`
|
||||||
- [建树修复管线三项改造实现计划](plans/tree-repair-resilience.md) `plan:tree-repair-resilience`
|
- [建树修复管线三项改造实现计划](plans/tree-repair-resilience.md) `plan:tree-repair-resilience`
|
||||||
@@ -94,9 +102,11 @@
|
|||||||
- [赛题生成工具实现计划](plans/question-gen-synth.md) `plan:question-gen-synth`
|
- [赛题生成工具实现计划](plans/question-gen-synth.md) `plan:question-gen-synth`
|
||||||
- [项目基础设施初始化计划](plans/infrastructure-setup.md) `plan:infrastructure-setup`
|
- [项目基础设施初始化计划](plans/infrastructure-setup.md) `plan:infrastructure-setup`
|
||||||
|
|
||||||
## review (2)
|
## review (4)
|
||||||
|
- [2026-07-16-preflight-final-review](reviews/2026-07-16-preflight-final-review.md) `review:2026-07-16-preflight-final-review`
|
||||||
- [2026-07-16-preflight-train-review](reviews/2026-07-16-preflight-train-review.md) `review:2026-07-16-preflight-train-review`
|
- [2026-07-16-preflight-train-review](reviews/2026-07-16-preflight-train-review.md) `review:2026-07-16-preflight-train-review`
|
||||||
- [question-gen v2 设计对抗审核 — 六路独立核验(四层病灶闭合度 + 契约一致性)](reviews/2026-07-15-question-gen-v2-adversarial-audit.md) `review:2026-07-15-question-gen-v2-adversarial-audit`
|
- [question-gen v2 设计对抗审核 — 六路独立核验(四层病灶闭合度 + 契约一致性)](reviews/2026-07-15-question-gen-v2-adversarial-audit.md) `review:2026-07-15-question-gen-v2-adversarial-audit`
|
||||||
|
- [训练前修复分支终审](reviews/preflight-final-review.md) `review:preflight-final-review`
|
||||||
|
|
||||||
## schema (6)
|
## schema (6)
|
||||||
- [表结构 v3 出题日志/观测(unit_verdict / collapse_metrics / quarantine / facts / resume)](schemas/v3-question-gen-logging.md) `schema:v3-question-gen-logging`
|
- [表结构 v3 出题日志/观测(unit_verdict / collapse_metrics / quarantine / facts / resume)](schemas/v3-question-gen-logging.md) `schema:v3-question-gen-logging`
|
||||||
|
|||||||
@@ -123,3 +123,15 @@
|
|||||||
- [2026-07-16 02:01 UTC] 重建索引: 98 篇页面
|
- [2026-07-16 02:01 UTC] 重建索引: 98 篇页面
|
||||||
- [2026-07-16 07:43 UTC] 新增 design: 训练前缺陷修复设计 (design:preflight-fixes)
|
- [2026-07-16 07:43 UTC] 新增 design: 训练前缺陷修复设计 (design:preflight-fixes)
|
||||||
- [2026-07-16 07:43 UTC] 重建索引: 101 篇页面
|
- [2026-07-16 07:43 UTC] 重建索引: 101 篇页面
|
||||||
|
- [2026-07-16 08:32 UTC] 新增 plan: WP1 资产迁移 (plan:preflight-wp1-asset-migration)
|
||||||
|
- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp1-asset-migration --implements--> design:preflight-fixes
|
||||||
|
- [2026-07-16 08:32 UTC] 新增 plan: WP2 切分与接线 (plan:preflight-wp2-split-wiring)
|
||||||
|
- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp2-split-wiring --implements--> design:preflight-fixes
|
||||||
|
- [2026-07-16 08:32 UTC] 新增 plan: WP3 训练循环与进化引擎 (plan:preflight-wp3-train-loop)
|
||||||
|
- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp3-train-loop --implements--> design:preflight-fixes
|
||||||
|
- [2026-07-16 08:32 UTC] 新增 plan: WP4 韧性与持久化 (plan:preflight-wp4-resilience)
|
||||||
|
- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp4-resilience --implements--> design:preflight-fixes
|
||||||
|
- [2026-07-16 08:32 UTC] 重建索引: 109 篇页面
|
||||||
|
- [2026-07-16 11:06 UTC] 新增 review: 训练前修复分支终审 (review:preflight-final-review)
|
||||||
|
- [2026-07-16 11:06 UTC] 新增边: review:preflight-final-review --informs--> design:preflight-fixes
|
||||||
|
- [2026-07-16 11:06 UTC] 重建索引: 111 篇页面
|
||||||
|
|||||||
@@ -0,0 +1,253 @@
|
|||||||
|
# WP1 资产迁移 Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** 迁移 5 个遗漏的 TRM4 进化/动量模板到 TRM5,删除 2 个死字段,把加载器的静默空串兜底改为 fail-loud,解锁自进化引擎。
|
||||||
|
|
||||||
|
**Architecture:** 进化引导 prompt 是引擎的一部分(放项目根 `prompts/`,不参与版本化进化)。TRM4 五模板的输出 JSON 契约与 TRM5 解析代码已核实完全对齐,可直接拷贝。执行顺序:先迁移模板 → 删死字段(`consolidate_system`/`span_eval_user`,零消费)→ 加载器 fail-loud(顺序关键:先删死字段,fail-loud 才不会对不存在也不需要的模板报错)。
|
||||||
|
|
||||||
|
**Tech Stack:** Python 3.11、pytest、frozen dataclass(`core/evolution/types.py`)。
|
||||||
|
|
||||||
|
**设计源**:`research-wiki/designs/2026-07-16-preflight-fixes-design.md §4`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 关键锚点(实现前必读)
|
||||||
|
|
||||||
|
| 用途 | 位置 |
|
||||||
|
|------|------|
|
||||||
|
| TRM4 源模板 | `/home/iomgaa/Projects/Video-Tree-TRM4/prompts/{evolve_skill,evolve_system,evolve_tool,evolve_rank,slow_momentum}.md` |
|
||||||
|
| evolve 加载器 | `app/harness/runner.py:2266-2280` `_load_evolve_prompts` |
|
||||||
|
| diagnose 加载器 | `app/harness/runner.py:2282-2299` `_load_diagnose_prompts` |
|
||||||
|
| 平行 diagnose 加载器 | `app/harness/video_split_cli.py:362-378` `_load_diagnose_prompts` |
|
||||||
|
| dataclass 定义 | `core/evolution/types.py:477-522`(`DiagnosePrompts` L494-501 / `EvolvePrompts` L518-522) |
|
||||||
|
| 死字段 `consolidate_system` 内联替代 | `core/evolution/evolve.py:997` `_CONSOLIDATE_SYSTEM`(消费点 L1036) |
|
||||||
|
| 死字段 `span_eval_user` 无消费 | diagnose 只用 `prompts.span_eval_system`(`core/evolution/diagnose.py:511`),user_prompt 内联构造 |
|
||||||
|
| 测试 fixture | `test_evolve.py:682` `consolidate_system="cons"`;`test_diagnose.py:753` `span_eval_user=""`;`test_evolution_types.py:352` `span_eval_user="p4"` / `:370` `consolidate_system="consolidate_tmpl"` |
|
||||||
|
|
||||||
|
## 核心算法保真校验
|
||||||
|
|
||||||
|
本计划不迁移/不改核心算法(ARCHITECTURE §6),只搬运模板文件 + 清理死字段 + 加 fail-loud。模板内容是 evolve 引擎(算法 #8)的输入数据,非算法逻辑本身;迁移已核实输出契约(`suggestions`/`edits`/`edits_extract`/`edits_verify`/`selected_indices`/`slow_update_content`)与 TRM5 解析代码逐字对齐。**保真检查点(Task 1 Step 4)**:加载 evolve_tool.md 后确认其要求 LLM 返回 `edits_extract`+`edits_verify` 双键(对齐 `evolve.py:1433-1435`)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 1: 迁移 5 个 TRM4 模板
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `prompts/evolve_skill.md`、`prompts/evolve_system.md`、`prompts/evolve_tool.md`、`prompts/evolve_rank.md`、`prompts/slow_momentum.md`(从 TRM4 拷贝)
|
||||||
|
- Test: `tests/unit/test_evolve_prompts_present.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写契约冒烟测试(先失败)**
|
||||||
|
|
||||||
|
`tests/unit/test_evolve_prompts_present.py`:
|
||||||
|
```python
|
||||||
|
"""校验 5 个进化/动量模板存在且输出契约关键词与解析代码对齐。"""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
_PROMPTS_DIR = Path("prompts")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"name, required_tokens",
|
||||||
|
[
|
||||||
|
("evolve_skill.md", ["suggestions", "edits"]),
|
||||||
|
("evolve_system.md", ["suggestions", "edits"]),
|
||||||
|
("evolve_tool.md", ["edits_extract", "edits_verify"]),
|
||||||
|
("evolve_rank.md", ["selected_indices"]),
|
||||||
|
("slow_momentum.md", ["slow_update_content"]),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_evolve_template_present_and_contract(name: str, required_tokens: list[str]) -> None:
|
||||||
|
path = _PROMPTS_DIR / name
|
||||||
|
assert path.exists(), f"缺模板: {path}"
|
||||||
|
text = path.read_text(encoding="utf-8")
|
||||||
|
assert text.strip(), f"模板为空: {path}"
|
||||||
|
for token in required_tokens:
|
||||||
|
assert token in text, f"{name} 缺输出契约关键词 {token!r}(与解析代码不对齐)"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败(模板尚未迁移)**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v`
|
||||||
|
Expected: 5 参数化用例全 FAIL(`AssertionError: 缺模板: prompts/evolve_skill.md` 等;TRM5 当前无这 5 个模板)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 拷贝 5 个模板**
|
||||||
|
|
||||||
|
Run:
|
||||||
|
```bash
|
||||||
|
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_skill.md prompts/evolve_skill.md
|
||||||
|
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_system.md prompts/evolve_system.md
|
||||||
|
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_tool.md prompts/evolve_tool.md
|
||||||
|
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_rank.md prompts/evolve_rank.md
|
||||||
|
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/slow_momentum.md prompts/slow_momentum.md
|
||||||
|
```
|
||||||
|
Expected: 5 文件存在于 `prompts/`。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行确认通过**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v`
|
||||||
|
Expected: 5 参数化用例全 PASS。若 evolve_tool.md 缺 `edits_extract`/`edits_verify` 则契约不符——停止并逐行比对 TRM4 源。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 保真检查点**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -c "print('edits_extract' in open('prompts/evolve_tool.md').read() and 'edits_verify' in open('prompts/evolve_tool.md').read())"`
|
||||||
|
Expected: `True`(对齐 `evolve.py:1433-1435` 的 `parsed["edits_extract"]`/`parsed["edits_verify"]`)。
|
||||||
|
|
||||||
|
- [ ] **Step 6: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add prompts/evolve_skill.md prompts/evolve_system.md prompts/evolve_tool.md prompts/evolve_rank.md prompts/slow_momentum.md tests/unit/test_evolve_prompts_present.py
|
||||||
|
git commit -m "feat: migrate 5 evolve/momentum templates from TRM4 (algo #8)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 2: 删除 2 个死字段(consolidate_system / span_eval_user)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `core/evolution/types.py:494-501,518-522`
|
||||||
|
- Modify: `app/harness/runner.py:2279,2294`
|
||||||
|
- Modify: `app/harness/video_split_cli.py:374`
|
||||||
|
- Modify: `tests/unit/test_evolve.py:682`、`tests/unit/test_diagnose.py:753`、`tests/unit/test_evolution_types.py:352,370`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 删 dataclass 字段与 docstring**
|
||||||
|
|
||||||
|
`core/evolution/types.py` — `DiagnosePrompts` 删 `span_eval_user`:
|
||||||
|
- docstring 删行 ` span_eval_user: span 评估用户提示模板。`(L487)
|
||||||
|
- 字段删行 ` span_eval_user: str`(L497)
|
||||||
|
|
||||||
|
`EvolvePrompts` 删 `consolidate_system`:
|
||||||
|
- docstring 删行 ` consolidate_system: appendix 压缩系统提示。`(L515)
|
||||||
|
- 字段删行 ` consolidate_system: str`(L522)
|
||||||
|
|
||||||
|
- [ ] **Step 2: 删加载器对死字段的 `_read` 行**
|
||||||
|
|
||||||
|
`app/harness/runner.py`:
|
||||||
|
- `_load_evolve_prompts` 删行 ` consolidate_system=_read("consolidate_system.md"),`(L2279)
|
||||||
|
- `_load_diagnose_prompts` 删行 ` span_eval_user=_read("span_eval_user.md"),`(L2294)
|
||||||
|
|
||||||
|
`app/harness/video_split_cli.py`:
|
||||||
|
- `_load_diagnose_prompts` 删行 ` span_eval_user=_read("span_eval_user.md"),`(L374)
|
||||||
|
|
||||||
|
- [ ] **Step 3: 删测试 fixture 对死字段的赋值 + 更新 docstring**
|
||||||
|
|
||||||
|
- `tests/unit/test_evolve.py:684` 删行 ` consolidate_system="cons",`
|
||||||
|
- `tests/unit/test_diagnose.py:753` 删行 ` span_eval_user="",`
|
||||||
|
- `tests/unit/test_evolution_types.py:352` 删行 ` span_eval_user="p4",`
|
||||||
|
- `tests/unit/test_evolution_types.py:370` 删行 ` consolidate_system="consolidate_tmpl",`
|
||||||
|
- `tests/unit/test_evolution_types.py:347` 的文档字符串"DiagnosePrompts 8 个模板字段"改为"7 个";`:364` 的"EvolvePrompts 5 个模板字段"改为"4 个"(删字段后数量变化)。
|
||||||
|
|
||||||
|
注:`test_evolution_types.py` 若有断言逐字段比对或字段计数,同步移除对两个死字段的断言(读该测试确认,删净引用)。上述行号以当前代码为准,实现前 `grep -n consolidate_system\|span_eval_user tests/unit/test_evolution_types.py` 复核。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行相关测试确认通过**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py -q`
|
||||||
|
Expected: 全 PASS(无 `TypeError: unexpected keyword argument` / 无 `missing positional argument`)。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 全库确认无残留引用**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -c "import subprocess; r=subprocess.run(['grep','-rn','consolidate_system\|span_eval_user','core/','app/','adapters/','tests/'],capture_output=True,text=True); print(r.stdout)"`
|
||||||
|
Expected: 空输出(`consolidate_appendix` 用内联 `_CONSOLIDATE_SYSTEM` 不算 `consolidate_system` 字段引用;若出现请确认非 dataclass 字段引用)。
|
||||||
|
|
||||||
|
- [ ] **Step 6: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add core/evolution/types.py app/harness/runner.py app/harness/video_split_cli.py tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py
|
||||||
|
git commit -m "refactor: drop dead prompt fields consolidate_system/span_eval_user"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 3: 加载器 fail-loud(缺模板即报错)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/runner.py:2270-2272,2286-2288`
|
||||||
|
- Modify: `app/harness/video_split_cli.py:365-367`
|
||||||
|
- Test: `tests/unit/test_evolve_prompts_present.py`(追加)
|
||||||
|
|
||||||
|
- [ ] **Step 1: 追加 fail-loud 测试(直接调真实加载器)**
|
||||||
|
|
||||||
|
在 `tests/unit/test_evolve_prompts_present.py` 追加——直接驱动真实 loader(不复制 _read 逻辑),在无模板的空 cwd 下断言 `FileNotFoundError`:
|
||||||
|
```python
|
||||||
|
def test_video_split_loader_fail_loud_on_missing(tmp_path, monkeypatch):
|
||||||
|
"""video_split_cli 的真实 diagnose 加载器缺模板必须 FileNotFoundError。"""
|
||||||
|
monkeypatch.chdir(tmp_path) # 空目录,无 prompts/*.md
|
||||||
|
from app.harness.video_split_cli import _load_diagnose_prompts
|
||||||
|
|
||||||
|
with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"):
|
||||||
|
_load_diagnose_prompts()
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_evolve_loader_fail_loud_on_missing(tmp_path, monkeypatch):
|
||||||
|
"""runner 的真实 evolve 加载器缺模板必须 FileNotFoundError。"""
|
||||||
|
monkeypatch.chdir(tmp_path)
|
||||||
|
from app.harness.runner import Runner
|
||||||
|
|
||||||
|
r = object.__new__(Runner) # 绕过 __init__,仅测无状态加载器方法
|
||||||
|
with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"):
|
||||||
|
r._load_evolve_prompts()
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_diagnose_loader_fail_loud_on_missing(tmp_path, monkeypatch):
|
||||||
|
"""runner 的真实 diagnose 加载器缺模板必须 FileNotFoundError。"""
|
||||||
|
monkeypatch.chdir(tmp_path)
|
||||||
|
from app.harness.runner import Runner
|
||||||
|
|
||||||
|
r = object.__new__(Runner)
|
||||||
|
with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"):
|
||||||
|
r._load_diagnose_prompts()
|
||||||
|
```
|
||||||
|
> 这三个测试直接调真实 loader(`_load_evolve_prompts`/`_load_diagnose_prompts` 无 self 状态依赖,`object.__new__` 可安全调用);修复前静默返回空串不抛,故 Step 2 前必 FAIL。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 三处 `_read` 闭包改 fail-loud**
|
||||||
|
|
||||||
|
`app/harness/runner.py` `_load_evolve_prompts`(L2270-2272)与 `_load_diagnose_prompts`(L2286-2288),以及 `app/harness/video_split_cli.py` `_load_diagnose_prompts`(L365-367),把:
|
||||||
|
```python
|
||||||
|
def _read(name: str) -> str:
|
||||||
|
p = Path("prompts") / name
|
||||||
|
return p.read_text(encoding="utf-8") if p.exists() else ""
|
||||||
|
```
|
||||||
|
改为:
|
||||||
|
```python
|
||||||
|
def _read(name: str) -> str:
|
||||||
|
p = Path("prompts") / name
|
||||||
|
if not p.exists():
|
||||||
|
raise FileNotFoundError(f"缺进化/诊断模板: {p}(请从 TRM4 迁移或检查 prompts/)")
|
||||||
|
return p.read_text(encoding="utf-8")
|
||||||
|
```
|
||||||
|
(runner.py 缩进 12 空格;video_split_cli.py 的 `_read` 缩进按其函数体,见 L365 为 8 空格——按各自现场缩进套用。)
|
||||||
|
|
||||||
|
- [ ] **Step 3: 运行测试确认通过**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v`
|
||||||
|
Expected: 全 PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 4: runner 其他行为回归(非 fail-loud 验证)**
|
||||||
|
|
||||||
|
fail-loud 已由 Step 3 的三个真实 loader 测试验证;此步仅确认模板迁移 + loader 改动未破坏 runner 其他行为。
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -q`
|
||||||
|
Expected: 全 PASS(模板已迁移,真实加载走成功分支)。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/runner.py app/harness/video_split_cli.py tests/unit/test_evolve_prompts_present.py
|
||||||
|
git commit -m "fix: fail-loud on missing evolve/diagnose templates (no silent empty)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Self-Review(作者自查,执行者复核)
|
||||||
|
|
||||||
|
- [ ] 5 模板均已迁移且契约测试覆盖关键字段。
|
||||||
|
- [ ] `consolidate_system`/`span_eval_user` 在 core/app/tests 全库无残留字段引用。
|
||||||
|
- [ ] 三处 loader(runner 两处 + video_split_cli 一处)均已 fail-loud。
|
||||||
|
- [ ] 执行顺序正确:Task 2(删死字段)先于 Task 3(fail-loud),避免对不需要的模板报错。
|
||||||
|
|
||||||
|
## 验收标准
|
||||||
|
|
||||||
|
1. `pytest tests/unit/test_evolve_prompts_present.py tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py tests/unit/test_harness_runner.py` 全绿。
|
||||||
|
2. `grep -rn 'consolidate_system\|span_eval_user' core/ app/ tests/` 无 dataclass 字段残留。
|
||||||
|
3. `prompts/` 下 5 个新模板存在且非空。
|
||||||
@@ -0,0 +1,627 @@
|
|||||||
|
# WP2 切分与接线 Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** 让冻结的 video-split 切分能正确进入训练 workspace(seed 携带 pools.json + global 一致性校验),并把切分质量三处优化(val_ratio 0.4、tier 感知 diag/val 分配、val 功效修复)与冻结产物覆盖保护落地。
|
||||||
|
|
||||||
|
**Architecture:** 切分产物由 `video_split_cli` 冻结到 `workspaces/video-split/`;本 WP 让 seed 携带该产物、训练 fresh 时拷入 workspace 并校验一致性。tier 感知在 `_split_trainval_by_video_group` 内实现——错题视频组按 T2(defect) 含量升序进 val(保留 T2 高的组在 diag),并把 `val_wrong_min` 前置到切分内做功效修复(不足则从 diag 换出低 T2 错题组补 val,耗尽 fail-loud)。
|
||||||
|
|
||||||
|
**Tech Stack:** Python 3.11、pytest、SQLite、frozen dataclass、shutil、原子写(tmp+os.replace)。
|
||||||
|
|
||||||
|
**设计源**:`research-wiki/designs/2026-07-16-preflight-fixes-design.md §5`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 关键锚点(实现前必读)
|
||||||
|
|
||||||
|
| 用途 | 位置 |
|
||||||
|
|------|------|
|
||||||
|
| trainval→diag/val 切分 | `app/harness/pools.py:126-199` `split_by_video_assignment`;`:289-338` `_split_trainval_by_video_group`;`:118` `InsufficientValSignal` |
|
||||||
|
| global 加载(无校验) | `app/harness/pools.py:713-813` `build_or_load_pools`(L813 `return load_pools` 前无 global 校验);`:587-620` `load_pools` |
|
||||||
|
| 冻结编排 | `app/harness/build_split.py:104-219` `build_split`(signal_rows 含 tier L154;split_by_video_assignment 调用 L184-191;save_pools L192);`:46-71` `SplitBuildConfig`(无 val_wrong_min) |
|
||||||
|
| CLI 构造 | `app/harness/video_split_cli.py:559-577` `SplitBuildConfig(...)`;`:580` `check_mcnemar_power`;`:751-773` `build_arg_parser`(无 --force) |
|
||||||
|
| seed | `app/harness/store.py:184-232` `init_seed`(拷 skills/prompts/baseline.db,不拷 pools);`:269-307` `extract_run_db`(不去重) |
|
||||||
|
| workspace | `app/harness/workspace.py:156-200` `init_workspace_from_seed`(copy2 baseline.db→harness.db L197,不拷 pools) |
|
||||||
|
| manifest | `app/harness/split_manifest.py:19-59` `write_manifest`(pools_sha256 L54) |
|
||||||
|
| 配置 | `config/video_split.yaml`(val_ratio L15=0.3、val_wrong_min L14=20) |
|
||||||
|
| 测试 | `tests/unit/test_pools_video_atomic.py`、`test_split_selection.py`、`test_harness_pools.py`、`test_harness_store.py`、`test_harness_workspace.py` |
|
||||||
|
|
||||||
|
## 核心算法保真校验
|
||||||
|
|
||||||
|
触及算法 #5(信息阶梯)的**上游输入**:本 WP 只改"哪些视频进 diag/val",pools 内仍是逐 unit 列表,`gate_ladder` 消费的 unit+correctness 结构不变。**保真检查点(Task 3 Step 6)**:确认 `_split_trainval_by_video_group` 返回后 diagnosis/validation 仍是逐题 `GeneratedQuestion` 列表、视频组原子性(同 video 全部题同池)不被 tier 排序破坏。不改算法 #6/#9。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 1: val_ratio 0.3 → 0.4
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `config/video_split.yaml:15`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 改配置**
|
||||||
|
|
||||||
|
`config/video_split.yaml` 第 15 行:
|
||||||
|
```yaml
|
||||||
|
val_ratio: 0.3 # validation 占 trainval 视频组总数的比例
|
||||||
|
```
|
||||||
|
改为:
|
||||||
|
```yaml
|
||||||
|
val_ratio: 0.4 # validation 占 trainval 视频组总数的比例(0.3→0.4 提升整包终审功效,WP2)
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add config/video_split.yaml
|
||||||
|
git commit -m "chore: bump video_split val_ratio 0.3->0.4 for terminal-eval power"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 2: extract_run_db 每题去重(902→900 canonical)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/store.py:269-307`
|
||||||
|
- Test: `tests/unit/test_harness_store.py`(`TestExtractRunDb`)
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
在 `tests/unit/test_harness_store.py` 的 `TestExtractRunDb` 类追加:
|
||||||
|
```python
|
||||||
|
def test_dedupe_per_question_keeps_first_row(self, tmp_path):
|
||||||
|
"""dedupe_per_question=True 时每 question_id 只保留 rowid 最小的首行。"""
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
src = tmp_path / "src.db"
|
||||||
|
conn = sqlite3.connect(src)
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE _runs (run_id TEXT PRIMARY KEY, started_at TEXT)"
|
||||||
|
)
|
||||||
|
conn.execute("INSERT INTO _runs VALUES ('r1', 't0')")
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE predictions (run_id TEXT, question_id TEXT, prediction TEXT)"
|
||||||
|
)
|
||||||
|
# 743-1 三行(模拟 error/budget/finished),首行 prediction=NULL
|
||||||
|
conn.executemany(
|
||||||
|
"INSERT INTO predictions VALUES (?,?,?)",
|
||||||
|
[
|
||||||
|
("r1", "743-1", None),
|
||||||
|
("r1", "743-1", None),
|
||||||
|
("r1", "743-1", "C"),
|
||||||
|
("r1", "q2", "A"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
dst = tmp_path / "dst.db"
|
||||||
|
from app.harness.store import extract_run_db
|
||||||
|
|
||||||
|
extract_run_db(src, dst, "r1", dedupe_per_question=True)
|
||||||
|
|
||||||
|
out = sqlite3.connect(dst)
|
||||||
|
rows = out.execute(
|
||||||
|
"SELECT question_id, prediction FROM predictions ORDER BY question_id"
|
||||||
|
).fetchall()
|
||||||
|
out.close()
|
||||||
|
assert rows == [("743-1", None), ("q2", "A")], f"未按 rowid 首行去重: {rows}"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py::TestExtractRunDb::test_dedupe_per_question_keeps_first_row -v`
|
||||||
|
Expected: FAIL(`extract_run_db() got an unexpected keyword argument 'dedupe_per_question'`)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 实现去重**
|
||||||
|
|
||||||
|
`app/harness/store.py` `extract_run_db` 签名改为:
|
||||||
|
```python
|
||||||
|
def extract_run_db(
|
||||||
|
src_db: Path, dst_db: Path, run_id: str, *, dedupe_per_question: bool = False
|
||||||
|
) -> None:
|
||||||
|
```
|
||||||
|
docstring 补一句参数说明:
|
||||||
|
```
|
||||||
|
dedupe_per_question: True 时 predictions 表每 question_id 仅保留 rowid 最小
|
||||||
|
的首行(对齐 canonical「每 question_id 取第一行 ORDER BY rowid」口径,
|
||||||
|
902→900)。_runs 表不受影响。
|
||||||
|
```
|
||||||
|
把 predictions 分支的取行 SQL(L297-299)改为按 `dedupe_per_question` 分派:
|
||||||
|
```python
|
||||||
|
if table == "predictions" and dedupe_per_question:
|
||||||
|
rows = src.execute(
|
||||||
|
f"SELECT {col_sql} FROM {table} WHERE run_id=? "
|
||||||
|
"AND rowid IN (SELECT MIN(rowid) FROM predictions "
|
||||||
|
"WHERE run_id=? GROUP BY question_id)",
|
||||||
|
(run_id, run_id),
|
||||||
|
).fetchall()
|
||||||
|
else:
|
||||||
|
rows = src.execute(
|
||||||
|
f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,)
|
||||||
|
).fetchall()
|
||||||
|
```
|
||||||
|
> NULL question_id 说明:predictions 的 question_id 是题标识、语义上非空(canonical 900 题均有 id),`GROUP BY question_id` 的 NULL 折叠风险不适用。若源库异常出现 NULL question_id,`MIN(rowid) GROUP BY` 会把它们折叠成一行——本任务不为该异常兜底(预测数据契约保证非空),保持 fail-visible。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行确认通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py -q`
|
||||||
|
Expected: 全 PASS(默认 `dedupe_per_question=False` 保持既有行为,旧测试不受影响)。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/store.py tests/unit/test_harness_store.py
|
||||||
|
git commit -m "feat: add dedupe_per_question to extract_run_db (canonical 902->900)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 3: tier 感知 + val 功效修复的 diag/val 分配
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/pools.py:126-199,289-338`
|
||||||
|
- Modify: `app/harness/build_split.py:46-71,181-192`
|
||||||
|
- Modify: `app/harness/video_split_cli.py:565-573`
|
||||||
|
- Test: `tests/unit/test_pools_video_atomic.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试(tier 优先 + 功效修复)**
|
||||||
|
|
||||||
|
在 `tests/unit/test_pools_video_atomic.py` 追加:
|
||||||
|
```python
|
||||||
|
def test_tier_aware_keeps_high_t2_in_diag():
|
||||||
|
"""错题视频组按 T2 含量升序进 val:T2 高的组保留在 diagnosis。"""
|
||||||
|
from app.harness.pools import split_by_video_assignment
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
|
||||||
|
def _q(qid, vid):
|
||||||
|
return GeneratedQuestion(
|
||||||
|
question_id=qid, video_id=vid, task_type="X", question="q",
|
||||||
|
options=["A", "B"], answer="A", source_nodes=[], difficulty="easy",
|
||||||
|
)
|
||||||
|
|
||||||
|
# 4 个错题视频(每视频 1 题),T2 数分别 2/1/0/0
|
||||||
|
questions = [_q(f"{v}-1", v) for v in ("vA", "vB", "vC", "vD")]
|
||||||
|
assignment = {v: "trainval" for v in ("vA", "vB", "vC", "vD")}
|
||||||
|
correctness = {f"{v}-1": False for v in ("vA", "vB", "vC", "vD")}
|
||||||
|
wrong_tier = {"vA": 2, "vB": 1, "vC": 0, "vD": 0}
|
||||||
|
|
||||||
|
pools = split_by_video_assignment(
|
||||||
|
questions, assignment, correctness, val_ratio=0.5, seed=7,
|
||||||
|
wrong_tier_by_video=wrong_tier,
|
||||||
|
)
|
||||||
|
diag_vids = {q.video_id for q in pools.diagnosis}
|
||||||
|
# T2 最高的 vA 必留 diag;T2=0 的组优先进 val
|
||||||
|
assert "vA" in diag_vids
|
||||||
|
assert "vB" in diag_vids
|
||||||
|
|
||||||
|
|
||||||
|
def test_val_wrong_min_repair_pulls_from_diag():
|
||||||
|
"""val 错题不足 val_wrong_min 时从 diag 换入低 T2 错题组补足。"""
|
||||||
|
from app.harness.pools import split_by_video_assignment
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
|
||||||
|
def _q(qid, vid, correct):
|
||||||
|
return GeneratedQuestion(
|
||||||
|
question_id=qid, video_id=vid, task_type="X", question="q",
|
||||||
|
options=["A", "B"], answer="A", source_nodes=[], difficulty="easy",
|
||||||
|
)
|
||||||
|
|
||||||
|
# 8 错题视频 + 2 正确视频;val_ratio 小使初分 val 错题不足,触发修复
|
||||||
|
vids_wrong = [f"w{i}" for i in range(8)]
|
||||||
|
vids_correct = ["c0", "c1"]
|
||||||
|
questions = [_q(f"{v}-1", v, False) for v in vids_wrong] + [
|
||||||
|
_q(f"{v}-1", v, True) for v in vids_correct
|
||||||
|
]
|
||||||
|
assignment = {v: "trainval" for v in vids_wrong + vids_correct}
|
||||||
|
correctness = {f"{v}-1": False for v in vids_wrong}
|
||||||
|
correctness.update({f"{v}-1": True for v in vids_correct})
|
||||||
|
wrong_tier = {v: i for i, v in enumerate(vids_wrong)} # 递增 T2
|
||||||
|
|
||||||
|
pools = split_by_video_assignment(
|
||||||
|
questions, assignment, correctness, val_ratio=0.1, seed=7,
|
||||||
|
wrong_tier_by_video=wrong_tier, val_wrong_min=4,
|
||||||
|
)
|
||||||
|
val_wrong = sum(1 for q in pools.validation if not correctness[q.question_id])
|
||||||
|
assert val_wrong >= 4, f"功效修复后 val 错题 {val_wrong} < 4"
|
||||||
|
```
|
||||||
|
|
||||||
|
> 注:`GeneratedQuestion` 的真实字段以 `app/question_gen/types.py` 为准;若构造签名不符,读该文件对齐必填字段(勿臆造)。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_pools_video_atomic.py -k "tier_aware or val_wrong_min_repair" -v`
|
||||||
|
Expected: FAIL(`unexpected keyword argument 'wrong_tier_by_video'`)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 改 `_split_trainval_by_video_group` 加 tier 感知 + 功效修复**
|
||||||
|
|
||||||
|
`app/harness/pools.py` 函数签名改为:
|
||||||
|
```python
|
||||||
|
def _split_trainval_by_video_group(
|
||||||
|
trainval_qs: list[GeneratedQuestion],
|
||||||
|
correctness: dict[str, bool],
|
||||||
|
val_ratio: float,
|
||||||
|
rng: random.Random,
|
||||||
|
wrong_tier_by_video: dict[str, int] | None = None,
|
||||||
|
val_wrong_min: int = 0,
|
||||||
|
) -> tuple[list[GeneratedQuestion], list[GeneratedQuestion]]:
|
||||||
|
```
|
||||||
|
把分层块(L329-334 的 else 分支)改为 tier 感知:`wrong_vids` 按 T2 含量升序(T2 少的优先进 val),保留 T2 高的组在 diag;`wrong_tier_by_video=None` 时退化为原 shuffle:
|
||||||
|
```python
|
||||||
|
else:
|
||||||
|
val_correct = math.floor(n_correct * n_val / n_total)
|
||||||
|
val_wrong = n_val - val_correct
|
||||||
|
rng.shuffle(correct_vids)
|
||||||
|
if wrong_tier_by_video is None:
|
||||||
|
rng.shuffle(wrong_vids)
|
||||||
|
else:
|
||||||
|
# T2 少的错题组优先进 val(保留 T2 高的组在 diag),确定性排序
|
||||||
|
wrong_vids.sort(key=lambda v: (wrong_tier_by_video.get(v, 0), v))
|
||||||
|
val_vids = set(correct_vids[:val_correct] + wrong_vids[:val_wrong])
|
||||||
|
```
|
||||||
|
在 `val_vids` 确定后、返回前,加**功效修复**(从 diag 的错题组按 T2 升序补入 val 直到满足 val_wrong_min):
|
||||||
|
```python
|
||||||
|
if val_wrong_min > 0:
|
||||||
|
val_wrong_now = sum(
|
||||||
|
1 for v in val_vids for q in groups[v] if not correctness[q.question_id]
|
||||||
|
)
|
||||||
|
# diag 侧仍在的错题组,按 T2 升序(低价值优先移交 val)
|
||||||
|
diag_wrong_pool = sorted(
|
||||||
|
(v for v in wrong_vids if v not in val_vids),
|
||||||
|
key=lambda v: ((wrong_tier_by_video or {}).get(v, 0), v),
|
||||||
|
)
|
||||||
|
for v in diag_wrong_pool:
|
||||||
|
if val_wrong_now >= val_wrong_min:
|
||||||
|
break
|
||||||
|
val_vids.add(v)
|
||||||
|
val_wrong_now += sum(1 for q in groups[v] if not correctness[q.question_id])
|
||||||
|
if val_wrong_now < val_wrong_min:
|
||||||
|
raise InsufficientValSignal(
|
||||||
|
f"trainval 错题不足以让 val 达到 val_wrong_min={val_wrong_min}"
|
||||||
|
f"(修复后仅 {val_wrong_now}),请放大 val_ratio 或调整 trainval 归属。"
|
||||||
|
)
|
||||||
|
```
|
||||||
|
(`InsufficientValSignal` 已在 pools.py:118 定义,无需新增;需确认函数内可见 `math`/`defaultdict`,文件顶部已 import。)
|
||||||
|
|
||||||
|
- [ ] **Step 4: `split_by_video_assignment` 透传新参数**
|
||||||
|
|
||||||
|
`app/harness/pools.py` `split_by_video_assignment` 签名加 `wrong_tier_by_video: dict[str, int] | None = None`(放在 `val_wrong_min` 之后),并把 `_split_trainval_by_video_group` 调用(L175-177)改为:
|
||||||
|
```python
|
||||||
|
diagnosis, validation = _split_trainval_by_video_group(
|
||||||
|
trainval_qs, correctness, val_ratio, random.Random(seed),
|
||||||
|
wrong_tier_by_video=wrong_tier_by_video,
|
||||||
|
val_wrong_min=val_wrong_min,
|
||||||
|
)
|
||||||
|
```
|
||||||
|
删除原 L179-186 的独立 `val_wrong_min` 事后校验块(功效已在 `_split_trainval_by_video_group` 内保证,避免重复校验语义)。docstring 的 `val_wrong_min` 说明改为"切分时保证(不足则从 diag 换入低 T2 错题组补足,耗尽 fail-loud)"。
|
||||||
|
|
||||||
|
- [ ] **Step 5: build_split 计算并传入 tier + val_wrong_min**
|
||||||
|
|
||||||
|
`app/harness/build_split.py`:`SplitBuildConfig` 加字段 `val_wrong_min: int`(放 `split_seed` 之后,docstring 补"validation 池最少错题数,切分时保证功效")。build_split Phase 3(L182-191)改为:
|
||||||
|
```python
|
||||||
|
questions = load_benchmark(questions_dir)
|
||||||
|
correctness = {pred["question_id"]: pred["correct"] for pred in preds}
|
||||||
|
tier_by_q = {row["question_id"]: row["tier"] for row in signal_rows}
|
||||||
|
wrong_tier_by_video: dict[str, int] = defaultdict(int)
|
||||||
|
for pred in preds:
|
||||||
|
if not pred["correct"] and tier_by_q.get(pred["question_id"]) == "T2":
|
||||||
|
wrong_tier_by_video[pred["video_id"]] += 1
|
||||||
|
pools = split_by_video_assignment(
|
||||||
|
questions,
|
||||||
|
assignment,
|
||||||
|
correctness,
|
||||||
|
config.val_ratio,
|
||||||
|
config.split_seed,
|
||||||
|
baseline_run_id=baseline_run_id,
|
||||||
|
val_wrong_min=config.val_wrong_min,
|
||||||
|
wrong_tier_by_video=dict(wrong_tier_by_video),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
更新 build_split docstring 的"契约(Task 11...)"段:删除"有意保持 val_wrong_min-agnostic"表述,改为"val_wrong_min 前置到切分内保证功效;CLI 的 check_mcnemar_power 作冗余最终确认"。确认 `defaultdict` 已 import(`from collections import Counter, defaultdict`)。
|
||||||
|
|
||||||
|
- [ ] **Step 6: CLI 传 val_wrong_min + 保真检查**
|
||||||
|
|
||||||
|
`app/harness/video_split_cli.py` 的 `SplitBuildConfig(...)`(L565-573)加一行 `val_wrong_min=config.val_wrong_min,`。
|
||||||
|
保真检查点:确认 `pools.diagnosis`/`pools.validation` 仍是逐题 `GeneratedQuestion` 列表、同 video 全部题同池(`test_pools_video_atomic.py::test_video_group_atomic_in_trainval_split` 覆盖)。
|
||||||
|
|
||||||
|
- [ ] **Step 7: 运行测试确认通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_pools_video_atomic.py tests/unit/test_harness_pools.py tests/unit/test_split_selection.py -q`
|
||||||
|
Expected: 全 PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 8: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/pools.py app/harness/build_split.py app/harness/video_split_cli.py tests/unit/test_pools_video_atomic.py
|
||||||
|
git commit -m "feat: tier-aware diag/val split with val-power repair (design 5.1)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 4: 冻结产物覆盖保护 + --force
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/build_split.py:104-192`
|
||||||
|
- Modify: `app/harness/video_split_cli.py:751-773`(build_arg_parser)+ run_pipeline 传参
|
||||||
|
- Test: `tests/unit/test_video_split_cli.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
在 `tests/unit/test_video_split_cli.py` 追加(用最小 build_split 覆盖场景,或直接测保护函数):
|
||||||
|
```python
|
||||||
|
def test_build_split_refuses_overwrite_without_force(tmp_path):
|
||||||
|
"""已存在指纹不同的 pools.json 时,force=False 必须报错不覆盖。"""
|
||||||
|
from app.harness.build_split import _guard_frozen_products
|
||||||
|
|
||||||
|
out_path = tmp_path / "pools.json"
|
||||||
|
out_path.write_text('{"split_mode":"global"}', encoding="utf-8")
|
||||||
|
manifest_path = tmp_path / "split_manifest.json"
|
||||||
|
|
||||||
|
with pytest.raises(FileExistsError, match="已存在冻结产物"):
|
||||||
|
_guard_frozen_products(out_path, manifest_path, force=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_split_force_backs_up_old(tmp_path):
|
||||||
|
"""force=True 时旧产物被备份为 .bak.* 再允许覆盖。"""
|
||||||
|
from app.harness.build_split import _guard_frozen_products
|
||||||
|
|
||||||
|
out_path = tmp_path / "pools.json"
|
||||||
|
out_path.write_text('{"old":1}', encoding="utf-8")
|
||||||
|
manifest_path = tmp_path / "split_manifest.json"
|
||||||
|
manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8")
|
||||||
|
|
||||||
|
_guard_frozen_products(out_path, manifest_path, force=True)
|
||||||
|
baks = list(tmp_path.glob("pools.json.bak.*"))
|
||||||
|
assert len(baks) == 1, f"未备份旧产物: {list(tmp_path.iterdir())}"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -k "refuses_overwrite or force_backs_up" -v`
|
||||||
|
Expected: FAIL(`cannot import name '_guard_frozen_products'`)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 实现覆盖保护函数**
|
||||||
|
|
||||||
|
`app/harness/build_split.py` 顶部 import 区确认有 `import shutil`(无则加)。新增函数(放 build_split 之前):
|
||||||
|
```python
|
||||||
|
def _guard_frozen_products(out_path: Path, manifest_path: Path, *, force: bool) -> None:
|
||||||
|
"""冻结前的覆盖保护:产物已存在时按 force 决定报错或备份。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
out_path: 目标 pools.json 路径。
|
||||||
|
manifest_path: 目标 split_manifest.json 路径。
|
||||||
|
force: False 时已存在即 FileExistsError;True 时把旧产物重命名为
|
||||||
|
.bak.<旧 pools_sha256 前 8 位或 timestamp-less 序号> 再放行。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileExistsError: force=False 且产物已存在(防静默覆盖冻结锚点)。
|
||||||
|
"""
|
||||||
|
if not out_path.exists() and not manifest_path.exists():
|
||||||
|
return
|
||||||
|
if not force:
|
||||||
|
raise FileExistsError(
|
||||||
|
f"已存在冻结产物 {out_path}(或其 manifest)。重跑切分会覆盖训练依赖的"
|
||||||
|
"冻结锚点——确认要替换请加 --force(旧产物将备份为 .bak.*)。"
|
||||||
|
)
|
||||||
|
# 备份后缀取旧 manifest 的 pools_sha256 前 8 位,无则用 'prev'
|
||||||
|
suffix = "prev"
|
||||||
|
if manifest_path.exists():
|
||||||
|
try:
|
||||||
|
old = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
suffix = str(old.get("pools_sha256", "prev"))[:8] or "prev"
|
||||||
|
except (json.JSONDecodeError, OSError):
|
||||||
|
suffix = "prev"
|
||||||
|
for p in (out_path, manifest_path):
|
||||||
|
if p.exists():
|
||||||
|
p.rename(p.with_name(f"{p.name}.bak.{suffix}"))
|
||||||
|
```
|
||||||
|
确认 build_split.py 已 import `json`(无则加 `import json`)。在 `build_split` 签名加参数 `force: bool = False`(放 `generated_at` 之后),并在 Phase 3 `save_pools` 之前(L192 前)调用 `_guard_frozen_products(out_path, manifest_path, force=force)`。
|
||||||
|
|
||||||
|
- [ ] **Step 4: CLI 暴露 --force 并透传**
|
||||||
|
|
||||||
|
`app/harness/video_split_cli.py` `build_arg_parser`(L751-773)追加:
|
||||||
|
```python
|
||||||
|
parser.add_argument(
|
||||||
|
"--force",
|
||||||
|
action="store_true",
|
||||||
|
help="覆盖已存在的冻结 pools.json/manifest(旧产物备份为 .bak.*)",
|
||||||
|
)
|
||||||
|
```
|
||||||
|
`run_pipeline` 签名加 `force: bool = False` 参数,build_split 调用(L559-577)加 `force=force,`;`main()` 里把 `args.force` 透传给 `run_pipeline`。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 运行测试确认通过 + CLI 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -q`
|
||||||
|
Expected: 全 PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 6: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/build_split.py app/harness/video_split_cli.py tests/unit/test_video_split_cli.py
|
||||||
|
git commit -m "feat: guard frozen split products against silent overwrite (--force)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 5: seed 携带 pools.json + 训练拷入
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/store.py:184-232`(init_seed)
|
||||||
|
- Modify: `app/harness/workspace.py:156-200`(init_workspace_from_seed)
|
||||||
|
- Test: `tests/unit/test_harness_store.py`、`tests/unit/test_harness_workspace.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试(seed 携带)**
|
||||||
|
|
||||||
|
在 `tests/unit/test_harness_store.py::TestInitSeed` 追加:
|
||||||
|
```python
|
||||||
|
def test_init_seed_carries_pools(self, tmp_path):
|
||||||
|
"""提供 pools_json/split_manifest 时拷入 seed 目录。"""
|
||||||
|
from app.harness.store import init_seed
|
||||||
|
|
||||||
|
store = tmp_path / "store"
|
||||||
|
skills = tmp_path / "sk"; skills.mkdir(); (skills / "s.md").write_text("x")
|
||||||
|
prompts = tmp_path / "pr"; prompts.mkdir(); (prompts / "p.md").write_text("y")
|
||||||
|
db = tmp_path / "b.db"; db.write_text("db")
|
||||||
|
pools = tmp_path / "pools.json"; pools.write_text('{"split_mode":"global"}')
|
||||||
|
manifest = tmp_path / "split_manifest.json"; manifest.write_text('{"pools_sha256":"a"}')
|
||||||
|
|
||||||
|
seed_dir = init_seed(
|
||||||
|
store, "s1", skills, prompts, db, "infer_adhoc", None, "d",
|
||||||
|
pools_json=pools, split_manifest=manifest,
|
||||||
|
)
|
||||||
|
assert (seed_dir / "pools.json").exists()
|
||||||
|
assert (seed_dir / "split_manifest.json").exists()
|
||||||
|
```
|
||||||
|
|
||||||
|
在 `tests/unit/test_harness_workspace.py` 追加:
|
||||||
|
```python
|
||||||
|
def test_init_workspace_from_seed_carries_pools(store_dir, workspace_dir):
|
||||||
|
"""seed 目录含 pools.json 时拷入 workspace。"""
|
||||||
|
import shutil
|
||||||
|
from app.harness.store import init_seed
|
||||||
|
from app.harness.workspace import init_workspace_from_seed
|
||||||
|
# 复用现有 fixture 构造 seed 的方式;此处补 pools.json 到 seed 后初始化 workspace
|
||||||
|
# (具体 fixture 依 test_harness_workspace.py 现有 helper,读文件对齐)
|
||||||
|
...
|
||||||
|
```
|
||||||
|
> 该 workspace 测试需依 `test_harness_workspace.py` 现有 fixture(`store_dir`/`workspace_dir` 及既有 seed 构造 helper)填充;实现前读该文件 `test_init_workspace_from_seed`(L151)复用其 seed 搭建,再在 seed 目录写 `pools.json` 后断言 workspace 内出现 `pools.json`。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py::TestInitSeed::test_init_seed_carries_pools -v`
|
||||||
|
Expected: FAIL(`unexpected keyword argument 'pools_json'`)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: init_seed 加可选携带参数**
|
||||||
|
|
||||||
|
`app/harness/store.py` `init_seed` 签名加:
|
||||||
|
```python
|
||||||
|
def init_seed(
|
||||||
|
store_dir: Path,
|
||||||
|
name: str,
|
||||||
|
skills_dir: Path,
|
||||||
|
prompts_dir: Path,
|
||||||
|
baseline_db: Path,
|
||||||
|
baseline_run_id: str,
|
||||||
|
parent: str | None,
|
||||||
|
description: str,
|
||||||
|
*,
|
||||||
|
pools_json: Path | None = None,
|
||||||
|
split_manifest: Path | None = None,
|
||||||
|
) -> Path:
|
||||||
|
```
|
||||||
|
在 `copy2(baseline_db, ...)`(L218)之后加:
|
||||||
|
```python
|
||||||
|
if pools_json is not None:
|
||||||
|
shutil.copy2(pools_json, seed_dir / "pools.json")
|
||||||
|
if split_manifest is not None:
|
||||||
|
shutil.copy2(split_manifest, seed_dir / "split_manifest.json")
|
||||||
|
```
|
||||||
|
docstring 补两参说明。
|
||||||
|
|
||||||
|
- [ ] **Step 4: init_workspace_from_seed 拷入 pools**
|
||||||
|
|
||||||
|
`app/harness/workspace.py` `init_workspace_from_seed` 在 `shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db")`(L197)之后加:
|
||||||
|
```python
|
||||||
|
seed_pools = seed_dir / "pools.json"
|
||||||
|
if seed_pools.exists():
|
||||||
|
shutil.copy2(seed_pools, workspace_dir / "pools.json")
|
||||||
|
seed_manifest = seed_dir / "split_manifest.json"
|
||||||
|
if seed_manifest.exists():
|
||||||
|
shutil.copy2(seed_manifest, workspace_dir / "split_manifest.json")
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 5: 运行测试确认通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py -q`
|
||||||
|
Expected: 全 PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 6: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/store.py app/harness/workspace.py tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py
|
||||||
|
git commit -m "feat: seed carries frozen pools.json into training workspace"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 6: build_or_load_pools global 一致性校验
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/pools.py:746-813`
|
||||||
|
- Test: `tests/unit/test_harness_pools.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
在 `tests/unit/test_harness_pools.py::TestBuildOrLoadPoolsFrozen` 追加:
|
||||||
|
```python
|
||||||
|
def test_global_frozen_rejects_baseline_mismatch(self, tmp_path, ...):
|
||||||
|
"""global 冻结 pools 的 baseline_run_id 与 seed 不符时 fail-loud。"""
|
||||||
|
# 依现有 fixture 造 workspace + 冻结 pools.json(split_mode=global,
|
||||||
|
# baseline_run_id="other"),seed.json baseline_run_id="infer_adhoc"
|
||||||
|
# 调 build_or_load_pools 应 raise ValueError(match="baseline_run_id")
|
||||||
|
...
|
||||||
|
```
|
||||||
|
> 依 `TestBuildOrLoadPoolsFrozen`(L246)现有 fixture 复用其 workspace/seed 搭建;实现前读该类对齐 RunConfig/strategy 构造,勿臆造。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_pools.py::TestBuildOrLoadPoolsFrozen -v`
|
||||||
|
Expected: 新用例 FAIL(当前 global 分支无校验,误加载不报错)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 加 global 一致性校验**
|
||||||
|
|
||||||
|
`app/harness/pools.py` `build_or_load_pools`,在 global 加载分支(`if pools_path.exists():` 块内、`per_category` 校验的 `else` 侧,即 L813 `return load_pools(pools_path)` 之前)加:
|
||||||
|
```python
|
||||||
|
else: # global:校验 baseline_run_id 与(若有)manifest 内容指纹
|
||||||
|
frozen_baseline = raw.get("baseline_run_id")
|
||||||
|
if frozen_baseline != baseline_run_id:
|
||||||
|
raise ValueError(
|
||||||
|
f"冻结 pools.json 的 baseline_run_id={frozen_baseline!r} 与 seed "
|
||||||
|
f"的 {baseline_run_id!r} 不一致,拒绝静默加载错配切分。"
|
||||||
|
)
|
||||||
|
manifest_path = config.workspace_dir / "split_manifest.json"
|
||||||
|
if manifest_path.exists():
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
actual_sha = hashlib.sha256(
|
||||||
|
pools_path.read_text(encoding="utf-8").encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
if manifest.get("pools_sha256") != actual_sha:
|
||||||
|
raise ValueError(
|
||||||
|
"pools.json 内容指纹与 split_manifest.pools_sha256 不符,"
|
||||||
|
"冻结产物疑被篡改,拒绝加载。"
|
||||||
|
)
|
||||||
|
```
|
||||||
|
(确认该 `else` 与 L751 `if frozen_split_mode == "per_category":` 配对;若现有结构非 if/else 而是 if 后直接 return,则把校验插在 `return load_pools(pools_path)` 前并用 `if frozen_split_mode != "per_category":` 守卫。实现前读 L746-813 对齐控制流。)
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行测试确认通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_pools.py -q`
|
||||||
|
Expected: 全 PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/pools.py tests/unit/test_harness_pools.py
|
||||||
|
git commit -m "fix: validate global frozen pools baseline_run_id + sha256 on load"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Self-Review(作者自查,执行者复核)
|
||||||
|
|
||||||
|
- [ ] val_ratio=0.4 已改;tier 感知 + 功效修复在同一函数、退化路径(`wrong_tier_by_video=None`)保持旧行为。
|
||||||
|
- [ ] val_wrong_min 从 CLI→SplitBuildConfig→build_split→split_by_video_assignment→_split_trainval_by_video_group 全链路贯通;旧的 pools.py 事后校验块已删(不重复)。
|
||||||
|
- [ ] extract_run_db 去重默认关闭,不破坏既有调用。
|
||||||
|
- [ ] seed 携带 + workspace 拷入 + global 一致性校验三者闭环:冻结产物有唯一路径进训练且被校验。
|
||||||
|
- [ ] 覆盖保护默认 force=False,离线 CLI 重跑需显式 --force。
|
||||||
|
|
||||||
|
## 核心算法保真校验结论
|
||||||
|
|
||||||
|
本计划触及算法 #5 的上游输入(哪些视频进 diag/val),**不改** gate_ladder 的 unit+correctness 消费结构;Task 3 Step 6 已设保真检查点确认逐 unit 列表与视频组原子性。不涉及算法 #4/#6/#8/#9 逻辑。
|
||||||
|
|
||||||
|
## 验收标准
|
||||||
|
|
||||||
|
1. `pytest tests/unit/test_pools_video_atomic.py tests/unit/test_harness_pools.py tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py tests/unit/test_split_selection.py tests/unit/test_video_split_cli.py` 全绿。
|
||||||
|
2. tier 感知:T2 高的错题视频组留 diag,T2 低的优先进 val。
|
||||||
|
3. seed 携带 pools.json → init_workspace_from_seed 拷入 → build_or_load_pools 校验 baseline_run_id + sha256。
|
||||||
|
4. 冻结产物 force=False 时拒绝覆盖。
|
||||||
@@ -0,0 +1,609 @@
|
|||||||
|
# WP3 训练循环与进化引擎 Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
> **前置依赖:WP1(模板已迁移,进化引擎可运行)+ WP4(cache_salt 能力已就绪)必须先完成。**
|
||||||
|
|
||||||
|
**Goal:** 修复训练循环与进化引擎的 9 处缺陷,使诊断拿到真实轨迹、早停按 epoch 语义、微型题型不崩、崩溃可幂等续跑、进化 patch 不破坏冻结区、降级信号不驱动错误进化、跨 epoch 评估真实重采样。
|
||||||
|
|
||||||
|
**Architecture:** 诊断经 `StepsJsonRunLog` 从 steps_json 重建轨迹(算法 #7 恢复);早停计数单位 step→epoch;可训练性预检在 gate 建立前剔除微型题型;`_run_step` 幂等(先 DELETE 再写);patch 冻结区检查整个 target 跨度(算法 #8 加固);降级/未判定题按 lapse 保守分流;训练推理用 run_id 作 cache_salt(run_id 已含 epoch,天然跨 epoch 重采样)。
|
||||||
|
|
||||||
|
**Tech Stack:** Python 3.11、asyncio、SQLite、pytest。
|
||||||
|
|
||||||
|
**设计源**:`research-wiki/designs/2026-07-16-preflight-fixes-design.md §6-7`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 关键锚点(实现前必读)
|
||||||
|
|
||||||
|
| 用途 | 位置 |
|
||||||
|
|------|------|
|
||||||
|
| 训练主流程 | `app/harness/runner.py:789-859` `train`;`_setup_train_run:865-879`(预检插入点 L874 前);`_run_step:1001-1026`(run_id L1011、诊断 L1019、无 DELETE) |
|
||||||
|
| 早停 | `app/harness/runner.py:291-316` `_should_early_stop`(L315 `+= steps_this_epoch`);`_TrainState:95-121`(`steps_since_best_improved` L118);`_maybe_promote_best:1587`(置 0) |
|
||||||
|
| 诊断调用 | `app/harness/runner.py:2163-2196` `_run_diagnosis`(`RunLogImpl` L2173 未包 StepsJsonRunLog);DiagnosisResult `degraded_count` 未被引用 |
|
||||||
|
| gate 刷新 | `app/harness/runner.py:1833-1879` `_refresh_gate_ladder`(save L1878 → set observed L1879);checkpoint 落盘晚在 train L837 |
|
||||||
|
| holdout | `app/harness/runner.py:1881-1921` `_holdout_four_way`;`_pick_mixed_best:1923-1963`;`_eval_version_on_pool:2148-2161` |
|
||||||
|
| 推理落库 | `app/harness/inference.py:363-447` `_run_single_question`(prediction L422 未归一、insert L446 try 外);`_to_text_field:147-162`;`run_inference:473` |
|
||||||
|
| Agent Loop | `core/agent/loop.py:103` `run`(session_id L110);`_call_llm` chat 调用 `:329`(`self._llm.chat(messages, session_id=session_id)`) |
|
||||||
|
| batching | `app/harness/batching.py:186-202` `_classify_unit`(L200 缺 correctness→None) |
|
||||||
|
| checkpoint | `app/harness/checkpoint.py:37-44` `_STRUCTURAL_KEYS`;`serialize_state:76-106`;`write_checkpoint:212-260`(原子写) |
|
||||||
|
| 诊断分流 | `core/evolution/diagnose.py:1485-1519` `_build_skill_case_packs`(lapse 分流 L1492);`_process_question:2139-2206`(except L2194 cause_category 留 None) |
|
||||||
|
| traces 适配 | `app/harness/baseline_run_log.py:13` `StepsJsonRunLog`;`steps_json_traces.py:13` |
|
||||||
|
| patch | `core/evolution/patch.py:285-287` `_in_ranges`;`_do_insert_after:309-326`(L320);`_do_replace_delete:329-347`(L343);markers L11-18;`validate_skill` in `evolve.py:296` |
|
||||||
|
|
||||||
|
## 核心算法保真校验
|
||||||
|
|
||||||
|
触及算法 #7(诊断瀑布,Task 1 恢复轨迹)、#8(patch 引擎,Task 5 冻结区加固)、#10(Agent Loop,Task 9 透传 cache_salt)、#5/#12(信息阶梯/训练编排,Task 7 checkpoint 时序)。**均为恢复/加固/透传,不改算法逻辑**:Task 1 让诊断拿到本就该有的轨迹;Task 5 把"只查起点"补成"查整跨度"(保护方向不变);Task 9 只加透传参数;Task 7 只调 checkpoint 落盘时机。每个相关 Task 设保真检查点。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 1: traces 适配(诊断拿到真实轨迹,算法 #7)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/runner.py:2163-2196`(_run_diagnosis)
|
||||||
|
- Test: `tests/unit/test_runner_diag_tree_inject.py` 或 `test_harness_runner.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
在 `tests/unit/test_runner_diag_tree_inject.py` 追加(构造只写 steps_json 不写 traces 表的 run,断言诊断能拿到轨迹):
|
||||||
|
```python
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_diagnosis_reads_traces_from_steps_json(...):
|
||||||
|
"""traces 表为空但 predictions.steps_json 有轨迹时,诊断仍拿到非空 traces。"""
|
||||||
|
# 依现有 runner 测试 fixture 造一个 run:predictions 有 steps_json,traces 表空;
|
||||||
|
# 调 _run_diagnosis 后断言 diagnose 收到的 traces 非空(可 patch run_diagnosis 捕获入参)
|
||||||
|
...
|
||||||
|
```
|
||||||
|
> 依 `test_runner_diag_tree_inject.py` 现有 fixture;实现前读对齐 runner 构造与 patch 点。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_runner_diag_tree_inject.py -k reads_traces_from_steps_json -v`
|
||||||
|
Expected: FAIL(当前 RunLogImpl 直读空 traces 表)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 包 StepsJsonRunLog**
|
||||||
|
|
||||||
|
`app/harness/runner.py` `_run_diagnosis`,把传给 `run_diagnosis` 的 `run_log`(当前 `RunLogImpl(...)`,L2173 附近)包一层:
|
||||||
|
```python
|
||||||
|
from app.harness.baseline_run_log import StepsJsonRunLog
|
||||||
|
from app.harness.log import RunLogImpl
|
||||||
|
|
||||||
|
run_log = StepsJsonRunLog(RunLogImpl(str(self._paths.db_path)))
|
||||||
|
```
|
||||||
|
(`StepsJsonRunLog.get_traces` 在底层 traces 空时从 predictions.steps_json 经 `steps_json_to_trace_rows` 重建;`get_predictions` 透传。)
|
||||||
|
|
||||||
|
- [ ] **Step 4: 保真检查点 + 测试通过**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_runner_diag_tree_inject.py tests/unit/test_baseline_run_log.py -q`
|
||||||
|
Expected: 全 PASS。保真:确认诊断瀑布拿到的是逐 step `{tool_name,tool_args,tool_output,thought}` 行(对齐 TRM4 诊断输入)。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/runner.py tests/unit/test_runner_diag_tree_inject.py
|
||||||
|
git commit -m "fix: wrap diagnosis run_log with StepsJsonRunLog (restore algo #7 traces)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 2: prediction 归一化 + 落库加固
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/inference.py:417-447`
|
||||||
|
- Test: `tests/unit/test_harness_inference.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
在 `tests/unit/test_harness_inference.py` 追加(LLM 提交非标量 answer 不崩 gather):
|
||||||
|
```python
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_nonscalar_prediction_does_not_crash(...):
|
||||||
|
"""submit_answer 返回 {'answer': ['B']} 等非标量时归一化落库,不抛 sqlite 绑定异常。"""
|
||||||
|
# 依现有 inference 测试 fixture,让 AgentLoop 返回 result={'answer': ['B']};
|
||||||
|
# run_inference 应正常完成、predictions 行 prediction 为字符串(如 '["B"]'),不崩
|
||||||
|
...
|
||||||
|
```
|
||||||
|
> 依 `test_harness_inference.py` 现有 fake loop/dispatch fixture;实现前读对齐。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py -k nonscalar_prediction -v`
|
||||||
|
Expected: FAIL(sqlite `InterfaceError: Error binding parameter`)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 归一化 prediction + insert 加固**
|
||||||
|
|
||||||
|
`app/harness/inference.py`:新增归一化 helper(None 保留、str 原样、其余 `_to_text_field`):
|
||||||
|
```python
|
||||||
|
def _normalize_prediction(answer: object) -> str | None:
|
||||||
|
"""归一化 prediction:None 保留(INFRA 空预测语义),str 原样,其余 JSON 序列化。"""
|
||||||
|
if answer is None or isinstance(answer, str):
|
||||||
|
return answer
|
||||||
|
return _to_text_field(answer)
|
||||||
|
```
|
||||||
|
L422 `"prediction": result_dict.get("answer"),` 改为 `"prediction": _normalize_prediction(result_dict.get("answer")),`。
|
||||||
|
L446 的 `await asyncio.to_thread(log.insert, "predictions", record)` 包 try,绑定异常降级为最小 error 行不击穿 gather:
|
||||||
|
```python
|
||||||
|
try:
|
||||||
|
await asyncio.to_thread(log.insert, "predictions", record)
|
||||||
|
except (sqlite3.InterfaceError, sqlite3.ProgrammingError):
|
||||||
|
logger.exception("[{}] QA {} 落库绑定异常,降级为 error 行", qa.video_id, qa.question_id)
|
||||||
|
record["prediction"] = None
|
||||||
|
record["stop_reason"] = "error"
|
||||||
|
await asyncio.to_thread(
|
||||||
|
log.insert, "predictions",
|
||||||
|
{k: v for k, v in record.items() if isinstance(v, (str, int, float, type(None)))},
|
||||||
|
)
|
||||||
|
```
|
||||||
|
确认 `import sqlite3` 在文件顶部(无则加)。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 测试通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py -q`
|
||||||
|
Expected: 全 PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/inference.py tests/unit/test_harness_inference.py
|
||||||
|
git commit -m "fix: normalize non-scalar prediction; harden predictions insert"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 3: early_stop 改 epoch 计数
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/runner.py:291-316,118,1587,849-857`
|
||||||
|
- Modify: `app/harness/checkpoint.py`(若字段入 state)
|
||||||
|
- Test: `tests/unit/test_harness_runner.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
在 `tests/unit/test_harness_runner.py` 追加:
|
||||||
|
```python
|
||||||
|
def test_early_stop_counts_epochs_not_steps(tmp_path):
|
||||||
|
"""patience=2 表示连续 2 个 epoch 无 best 刷新才停(不是步数)。"""
|
||||||
|
from app.harness.runner import _should_early_stop, _TrainState
|
||||||
|
# 造 state + workspace,best 停在 epoch 1;
|
||||||
|
# epoch 2 无刷新 → epochs_since_best_improved=1 → 不停;
|
||||||
|
# epoch 3 无刷新 → =2 → 停
|
||||||
|
...
|
||||||
|
```
|
||||||
|
> 依现有 `_TrainState`/`read_best` fixture;实现前读对齐 workspace best 写入。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k early_stop_counts_epochs -v`
|
||||||
|
Expected: FAIL(当前累加 steps_this_epoch)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 字段改名 + 计数改 epoch**
|
||||||
|
|
||||||
|
全局把 `steps_since_best_improved` 改名 `epochs_since_best_improved`(`grep -rn steps_since_best_improved app/`:`_TrainState:118`、`_should_early_stop:313,315`、`_maybe_promote_best:1587`,以及 checkpoint serialize/deserialize 若含此字段)。
|
||||||
|
`_should_early_stop`(L315)`state.steps_since_best_improved += steps_this_epoch` 改为 `state.epochs_since_best_improved += 1`;签名删除 `steps_this_epoch` 参数(改为 `_should_early_stop(workspace_dir, epoch, state, patience)`),train 调用点(L849-857)同步去掉 `len(batches)` 实参。docstring 改为"epoch 粒度"。
|
||||||
|
|
||||||
|
- [ ] **Step 4: checkpoint 兼容**
|
||||||
|
|
||||||
|
若 `epochs_since_best_improved` 入 checkpoint state(`grep -n steps_since_best_improved app/harness/checkpoint.py`),同步改名。本轮为 fresh 训练无旧 checkpoint,无迁移负担。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 测试通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py tests/unit/test_harness_checkpoint.py -q`
|
||||||
|
Expected: 全 PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 6: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/runner.py app/harness/checkpoint.py tests/unit/test_harness_runner.py
|
||||||
|
git commit -m "fix: early_stop patience counts epochs not steps"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 4: 可训练性预检
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/runner.py`(`train` 入口过滤 + `_setup_train_run` 接收 filtered task_types)
|
||||||
|
- Modify: `app/harness/config.py`(RunConfig 加 `trainable_min_units` + 正整数校验)
|
||||||
|
- Modify: `app/harness/checkpoint.py:37`(`trainable_min_units` 入 `_STRUCTURAL_KEYS` 指纹)
|
||||||
|
- Test: `tests/unit/test_harness_runner.py`、`tests/unit/test_harness_checkpoint.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
```python
|
||||||
|
def test_untrainable_types_filtered_before_gate():
|
||||||
|
"""val<eval_min_per_class 或 非test单元<trainable_min_units 的题型从 diag/val/task_types 剔除。"""
|
||||||
|
from app.harness.runner import _filter_untrainable_types
|
||||||
|
# 构造 pools:题型 A(val=5, units=40)可训;B(val=0, units=2)不可训
|
||||||
|
# 调 _filter_untrainable_types(pools, task_types=[A,B], eval_min_per_class=2, trainable_min_units=8)
|
||||||
|
# 断言返回 pools 不含 B、task_types 不含 B
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k untrainable_types_filtered -v`
|
||||||
|
Expected: FAIL(`_filter_untrainable_types` 不存在)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 实现纯函数预检**
|
||||||
|
|
||||||
|
`app/harness/runner.py` 新增模块级纯函数:
|
||||||
|
```python
|
||||||
|
def _filter_untrainable_types(
|
||||||
|
pools: Pools,
|
||||||
|
task_types: list[str] | None,
|
||||||
|
eval_min_per_class: int,
|
||||||
|
trainable_min_units: int,
|
||||||
|
) -> tuple[Pools, list[str] | None]:
|
||||||
|
"""剔除不可训练题型(val<eval_min_per_class 或 非test单元<trainable_min_units)。
|
||||||
|
|
||||||
|
非test单元数 = 该题型 diag+val 题数(single 题 unit==题;等于 gate 阶梯该类候选数)。
|
||||||
|
test 池不过滤(继续报告全题型准确率)。返回过滤后 (pools, task_types)。
|
||||||
|
"""
|
||||||
|
from collections import Counter
|
||||||
|
|
||||||
|
diag_by_type = Counter(q.task_type for q in pools.diagnosis)
|
||||||
|
val_by_type = Counter(q.task_type for q in pools.validation)
|
||||||
|
keep: set[str] = set()
|
||||||
|
dropped: list[tuple[str, str]] = []
|
||||||
|
for tt in set(diag_by_type) | set(val_by_type):
|
||||||
|
n_val = val_by_type.get(tt, 0)
|
||||||
|
n_units = diag_by_type.get(tt, 0) + n_val
|
||||||
|
if n_val < eval_min_per_class:
|
||||||
|
dropped.append((tt, f"val={n_val}<{eval_min_per_class}"))
|
||||||
|
elif n_units < trainable_min_units:
|
||||||
|
dropped.append((tt, f"units={n_units}<{trainable_min_units}"))
|
||||||
|
else:
|
||||||
|
keep.add(tt)
|
||||||
|
for tt, why in sorted(dropped):
|
||||||
|
logger.warning("可训练性预检剔除题型 {}({})", tt, why)
|
||||||
|
new_pools = replace(
|
||||||
|
pools,
|
||||||
|
diagnosis=[q for q in pools.diagnosis if q.task_type in keep],
|
||||||
|
validation=[q for q in pools.validation if q.task_type in keep],
|
||||||
|
)
|
||||||
|
new_types = [t for t in task_types if t in keep] if task_types is not None else sorted(keep)
|
||||||
|
return new_pools, new_types
|
||||||
|
```
|
||||||
|
(确认 `from dataclasses import replace` 已 import;`Pools` 是否 frozen dataclass 支持 `replace`——若非,按其构造方式重建。)
|
||||||
|
|
||||||
|
**过滤结果必须回传主循环(Codex Critical)**:`RunConfig` 是 `@dataclass(frozen=True)`,**不能** `self._config.task_types = ...`(会 FrozenInstanceError),且过滤后的 pools 必须被 `train()` 后续的 batch/step/slow-update/final-eval 全部使用。实现方式:
|
||||||
|
- 在 `train(pools)` **入口第一步**(`_setup_train_run` 调用之前)过滤:
|
||||||
|
```python
|
||||||
|
pools, filtered_task_types = _filter_untrainable_types(
|
||||||
|
pools, self._config.task_types,
|
||||||
|
self._config.eval_min_per_class, self._config.trainable_min_units,
|
||||||
|
)
|
||||||
|
```
|
||||||
|
- 把 filtered `pools` 传给 `_setup_train_run(pools)` 与 train() 后续所有消费点(build_batches / slow_update / final_eval 均用这个 filtered pools,不再触碰原始 pools)。
|
||||||
|
- 把 `filtered_task_types` 传给 gate 建立(`_setup_train_run`/`_init_gate_pools` 用它而非 `self._config.task_types`)——新增参数透传,不改 frozen config。
|
||||||
|
|
||||||
|
`app/harness/config.py`:`RunConfig` 加字段 `trainable_min_units: int`(无默认,显式配置;train yaml 提供);在配置校验函数(如 `validate_config`,config.py:277 附近)加 `trainable_min_units >= 1` 断言(<1 报错)。
|
||||||
|
`app/harness/checkpoint.py:37` `_STRUCTURAL_KEYS` 加入 `"trainable_min_units"`——该值改变会改变 pools 过滤结果与训练轨迹,必须纳入 checkpoint 结构指纹,resume 时变化即拒绝复用旧 checkpoint。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 测试通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -q`
|
||||||
|
Expected: 全 PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/runner.py app/harness/config.py tests/unit/test_harness_runner.py
|
||||||
|
git commit -m "feat: pre-flight filter of untrainable task types before gate"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 5: patch 冻结区跨度 + 注入 + marker 校验(算法 #8)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `core/evolution/patch.py:285-347`
|
||||||
|
- Modify: `core/evolution/evolve.py:296`(validate_skill)
|
||||||
|
- Test: `tests/unit/test_patch.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
在 `tests/unit/test_patch.py` 追加:
|
||||||
|
```python
|
||||||
|
def test_replace_spanning_into_protected_is_skipped():
|
||||||
|
"""target 起点在正文、末端伸入冻结区的 replace 被跳过(不破坏 marker)。"""
|
||||||
|
from core.evolution.patch import apply_patch_with_report, APPENDIX_START, APPENDIX_END
|
||||||
|
|
||||||
|
body = "正文最后一段。"
|
||||||
|
appendix = f"{APPENDIX_START}\n## 执行提醒\n- 规则A\n{APPENDIX_END}"
|
||||||
|
content = body + "\n\n" + appendix
|
||||||
|
# target 从正文末尾跨入 APPENDIX_START
|
||||||
|
target = "正文最后一段。\n\n" + APPENDIX_START
|
||||||
|
edits = [{"op": "delete", "target": target, "content": ""}]
|
||||||
|
new_content, report = apply_patch_with_report(content, edits, protected_spans=[appendix])
|
||||||
|
assert APPENDIX_START in new_content and APPENDIX_END in new_content # marker 未被破坏
|
||||||
|
|
||||||
|
|
||||||
|
def test_edit_payload_with_marker_literal_rejected():
|
||||||
|
"""edit payload/target 含 marker 字面量 → 拒绝该 edit。"""
|
||||||
|
from core.evolution.patch import apply_patch_with_report, APPENDIX_START
|
||||||
|
edits = [{"op": "append", "target": "", "content": f"注入 {APPENDIX_START} 破坏"}]
|
||||||
|
_, report = apply_patch_with_report("正文", edits, protected_spans=[])
|
||||||
|
assert any("marker" in str(s).lower() or "reject" in str(s).lower() for s in report)
|
||||||
|
```
|
||||||
|
> marker 常量名以 `core/evolution/patch.py:11-18` 为准(`APPENDIX_START` 等),实现前读对齐 import 名。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_patch.py -k "spanning_into_protected or marker_literal" -v`
|
||||||
|
Expected: FAIL(当前只查起点 pos,跨入未拦;无注入检查)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 跨度检查 + 注入检查**
|
||||||
|
|
||||||
|
`core/evolution/patch.py` 新增跨度 helper:
|
||||||
|
```python
|
||||||
|
def _span_overlaps_ranges(pos: int, length: int, ranges: list[tuple[int, int]]) -> bool:
|
||||||
|
"""判断 [pos, pos+length) 是否与任一冻结区间相交(不止起点)。"""
|
||||||
|
end = pos + length
|
||||||
|
return any(start < end and pos < r_end for start, r_end in ranges)
|
||||||
|
```
|
||||||
|
`_do_insert_after` L320 `if _in_ranges(pos, ranges):` 改为 `if _span_overlaps_ranges(pos, len(target), ranges):`。
|
||||||
|
`_do_replace_delete` L343 `if _in_ranges(pos, ranges):` 改为 `if _span_overlaps_ranges(pos, len(target), ranges):`。
|
||||||
|
在 `apply_patch_with_report`(L387)应用每个 edit 前加注入检查:payload/target 含任一 marker 字面量(`APPENDIX_START/END`、`MOMENTUM_START/END`)→ 跳过该 edit 并记 `skipped_marker_injection`。
|
||||||
|
|
||||||
|
- [ ] **Step 4: validate_skill 加 marker 完整性校验**
|
||||||
|
|
||||||
|
`core/evolution/evolve.py:296` `validate_skill`:在现有 frontmatter/长度/代码块校验后加——统计 evolved 中 `APPENDIX_START/END`、`MOMENTUM_START/END` 出现次数,要求成对(START 数==END 数)、各至多一对、START 在 END 前;违反则返回校验失败(该候选整体 reject)。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 保真检查点 + 测试通过**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_patch.py tests/unit/test_evolve.py -q`
|
||||||
|
Expected: 全 PASS。保真:确认"保护跨度"方向未变(仍是保护 appendix/momentum 不被误改),只是从"查起点"补成"查整跨度"+ 注入/完整性双防线。
|
||||||
|
|
||||||
|
- [ ] **Step 6: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add core/evolution/patch.py core/evolution/evolve.py tests/unit/test_patch.py
|
||||||
|
git commit -m "fix: patch checks full target span + marker injection/integrity (algo #8)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 6: 诊断降级分流 + 占比中止
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `core/evolution/diagnose.py:1485-1497`
|
||||||
|
- Modify: `app/harness/runner.py:1019`(诊断后 degraded 占比检查)
|
||||||
|
- Test: `tests/unit/test_diagnose.py`、`tests/unit/test_harness_runner.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试(分流)**
|
||||||
|
|
||||||
|
在 `tests/unit/test_diagnose.py` 追加:
|
||||||
|
```python
|
||||||
|
def test_none_cause_and_degraded_route_to_lapse():
|
||||||
|
"""cause_category=None(判别失败)与 degraded 题按 lapse 处置,不进 defect 正文路径。"""
|
||||||
|
from core.evolution.diagnose import _build_skill_case_packs
|
||||||
|
# 构造 metrics_group:一题 attr.cause_category=None(非 degraded)、一题 qm.degraded=True;
|
||||||
|
# 断言二者都不出现在 failure_cases(wrong_by_error),只可能进 lapse_notes
|
||||||
|
...
|
||||||
|
```
|
||||||
|
> 依 `test_diagnose.py` 现有 QuestionMetrics/ErrorAttribution 构造(`:753` 附近);实现前读对齐字段。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_diagnose.py -k none_cause_and_degraded_route -v`
|
||||||
|
Expected: FAIL(当前 None → wrong_by_error 走 defect 正文)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 改分流逻辑**
|
||||||
|
|
||||||
|
`core/evolution/diagnose.py` `_build_skill_case_packs` 的分流循环(L1488-1497)改为"仅明确 defect 且非 degraded 才进正文路径":
|
||||||
|
```python
|
||||||
|
for qm in metrics_group:
|
||||||
|
if qm.correct:
|
||||||
|
continue
|
||||||
|
attr = attribution_map.get(qm.question_id)
|
||||||
|
is_defect = (
|
||||||
|
attr is not None
|
||||||
|
and attr.cause_category == "defect"
|
||||||
|
and not qm.degraded
|
||||||
|
)
|
||||||
|
if not is_defect:
|
||||||
|
# lapse / None(判别失败)/ degraded → 保守,不驱动正文进化
|
||||||
|
if attr is not None and attr.lapse_note and attr.lapse_note.strip():
|
||||||
|
lapse_notes.append(attr.lapse_note)
|
||||||
|
continue
|
||||||
|
wrong_by_error[attr.error_type].append(qm)
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: 写失败测试(占比中止)+ 实现**
|
||||||
|
|
||||||
|
`tests/unit/test_harness_runner.py` 追加:诊断结果 degraded_count/总题数 > 0.5 时 `_run_step` 后应 raise(疑似基础设施故障)。
|
||||||
|
`app/harness/runner.py` `_run_step`(L1019 拿到 `diagnosis` 后)加:
|
||||||
|
```python
|
||||||
|
n_wrong = sum(1 for q in batch if not state.correctness.get(q.question_id, True))
|
||||||
|
if n_wrong > 0 and diagnosis.degraded_count / n_wrong > 0.5:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"本 step 诊断降级占比 {diagnosis.degraded_count}/{n_wrong} > 50%,"
|
||||||
|
"疑似 judge 基础设施故障,中止训练(不以降级信号驱动进化)。"
|
||||||
|
)
|
||||||
|
```
|
||||||
|
(确认 `DiagnosisResult.degraded_count` 字段可用,定义在 `core/evolution/types.py:299`。)
|
||||||
|
|
||||||
|
- [ ] **Step 5: 测试通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_diagnose.py tests/unit/test_harness_runner.py -q`
|
||||||
|
Expected: 全 PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 6: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add core/evolution/diagnose.py app/harness/runner.py tests/unit/test_diagnose.py tests/unit/test_harness_runner.py
|
||||||
|
git commit -m "fix: route None/degraded diagnoses to lapse; abort on high degrade rate"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 7: step 幂等(DELETE)+ gate_epoch_observed 立即落盘
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/runner.py:1001-1026`(_run_step 开头 DELETE)
|
||||||
|
- Modify: `app/harness/runner.py:1378-1498,1833-1879`(checkpoint 提到 gate save 之后)
|
||||||
|
- Test: `tests/unit/test_harness_runner.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试(幂等)**
|
||||||
|
|
||||||
|
```python
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_run_step_deletes_stale_rows_before_rerun(...):
|
||||||
|
"""同 run_id 重跑前先清 predictions/traces,避免重复行双计。"""
|
||||||
|
# 预置该 step run_id 的旧 predictions 行;调 _run_step;断言旧行被清、只剩本次
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k deletes_stale_rows -v`
|
||||||
|
Expected: FAIL(当前 append,无 DELETE)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: _run_step 开头 DELETE**
|
||||||
|
|
||||||
|
`app/harness/runner.py` `_run_step`,在 rollout(L1012)之前加(**用 `IF EXISTS` 避免 fresh workspace 首跑时 predictions/traces 表尚未由 `run_inference._ensure_tables` 创建导致 `OperationalError: no such table`**):
|
||||||
|
```python
|
||||||
|
with HarnessLog(str(self._paths.db_path), run_id, register_run=False) as log:
|
||||||
|
log.execute("DELETE FROM predictions WHERE run_id=?", (run_id,))
|
||||||
|
log.execute("DELETE FROM traces WHERE run_id=?", (run_id,))
|
||||||
|
```
|
||||||
|
> SQLite `DELETE FROM <t>` 对不存在的表会抛 `no such table`。两种消解方式择一(实现前读 log.py 确认):① rollout 由 `run_inference` 先建表——把 DELETE 移到**首次 rollout 之后、诊断之前**并只在 resume 重跑(step 已有旧行)时执行;② 或 DELETE 前先 `CREATE TABLE IF NOT EXISTS`(复用 inference 的 PREDICTIONS_SCHEMA/TRACES_SCHEMA),保证幂等无害。推荐 ②(无害且简单)。`register_run=False` 来自 WP4;若 HarnessLog 无 `execute` 便捷方法,用其现有连接接口。
|
||||||
|
|
||||||
|
- [ ] **Step 4: checkpoint 提到 gate save 之后(消除双计窗口)**
|
||||||
|
|
||||||
|
目标:`_refresh_gate_ladder` 内 `gate_pools.save`(L1878)+ `gate_epoch_observed=True`(L1879)之后,**立即** `write_checkpoint`(phase="epoch_done"),不等到 `train` L837。实现:把 `_slow_update_cycle` Phase 10(调 `_refresh_gate_ladder` L1496-1498)之后的 checkpoint 落盘从 `train`(L837)移入 `_slow_update_cycle` 末尾,或让 `_refresh_gate_ladder` 接收 checkpoint 所需上下文(epoch/progress/batches)并在 save 后落盘。实现前读 `write_checkpoint` 签名(checkpoint.py:212)与 `train` L833-848 对齐参数,确保 gate_pools.json 与 checkpoint 的 `gate_epoch_observed` 同一时刻一致。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 保真检查点 + 测试通过**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py tests/unit/test_harness_checkpoint.py -q`
|
||||||
|
Expected: 全 PASS。保真(算法 #5/#12):确认 γ-EMA 更新(`update_probs`)仍每 epoch 一次、checkpoint 落盘不改变慢更新十步序的语义顺序。
|
||||||
|
|
||||||
|
- [ ] **Step 6: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/runner.py tests/unit/test_harness_runner.py
|
||||||
|
git commit -m "fix: idempotent _run_step (DELETE stale) + checkpoint after gate save"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 8: holdout 四向去重
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/runner.py:1881-1963`(_holdout_four_way / _pick_mixed_best)
|
||||||
|
- Test: `tests/unit/test_harness_runner.py`
|
||||||
|
|
||||||
|
> **说明**:目标是每 epoch 的四向 test 评估从"4×600 全跑"降为"仅 final 必跑 + best_hard 未评过才跑 + baseline 从基线预测推导 + best_mixed 引用赢家"。去重做在 harness 逻辑层(配合 WP4 epoch 盐,同版本不重采样)。
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
```python
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_holdout_dedup_skips_reevaluated_versions(...):
|
||||||
|
"""baseline 不跑推理(从基线预测推导);best_hard==final 时不重复评估。"""
|
||||||
|
# 统计 _eval_version_on_pool 被调次数:baseline=0,best_hard==final 时该向复用不重跑
|
||||||
|
...
|
||||||
|
```
|
||||||
|
> 依现有 runner holdout fixture;实现前读 `_holdout_four_way`/`write_holdout_eval` 对齐。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k holdout_dedup -v`
|
||||||
|
Expected: FAIL(当前四向各跑一次)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 实现去重(进程内备忘录,不改 holdout_eval schema)**
|
||||||
|
|
||||||
|
> **schema 约束(Codex Critical)**:当前 `holdout_eval` 表(observation.py:78)不存 skills_version/prompts_version/pointer,无法按版本反查做跨-run hydrate。本 task **不扩展该 schema**(避免结构性风险),改用 **train() 进程内备忘录** `dict[(skills_v,prompts_v), float]` 去重。代价:resume 后备忘录清空、已评版本会重评一次——resume 是异常路径、重评 600 题成本可接受,换取零 schema 变更风险。完整跨-run hydrate 记 future work。
|
||||||
|
|
||||||
|
`app/harness/runner.py` `_holdout_four_way`(在 `_TrainState` 加一个 `holdout_memo: dict[tuple[str,str], float] = field(default_factory=dict)` 字段):
|
||||||
|
- **baseline 向**:不调 `_eval_version_on_pool`,改从基线 predictions(`baseline_run_id`)读 test 题对错算 acc(test 题在 infer_adhoc 已全推理过);结果存 memo,epoch>1 直接复用(0 推理)。
|
||||||
|
- **final 向**:真评 600,算完存 `memo[(final_sv,final_pv)]`。
|
||||||
|
- **best_hard 向**:若 `(best_sv,best_pv)` 已在 `memo`(== final 或往轮已评)则引用,否则真评并存 memo。
|
||||||
|
- **best_mixed 向**:`_pick_mixed_best` 选出的赢家必是 best_hard 或 final 之一,其 test acc 已在 memo,直接引用写 holdout_eval,0 推理。
|
||||||
|
|
||||||
|
实现前完整读 `_holdout_four_way`(~1885)、`write_holdout_eval`、`_eval_version_on_pool` 对齐;`write_holdout_eval` 调用保持不变(仍逐向写观测行,只是 acc 来源改为 memo 复用/基线推导)。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 测试通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -q`
|
||||||
|
Expected: 全 PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/runner.py tests/unit/test_harness_runner.py
|
||||||
|
git commit -m "perf: dedup holdout four-way eval (baseline derive, best_hard memo)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 9: 训练推理注入 epoch 盐(cache_salt=run_id,算法 #10 透传)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `core/agent/loop.py:103-141,271-329`
|
||||||
|
- Modify: `app/harness/inference.py:408-415,473`
|
||||||
|
- Test: `tests/unit/test_agent_loop`(或现有 loop 测试)、`tests/unit/test_harness_inference.py`
|
||||||
|
|
||||||
|
> **原理**:训练/val/test/holdout 推理的 run_id 已含 `_e{epoch}`(如 `{base}_e{epoch}_s{step}`、`{run_id}_holdout_{kind}_e{epoch}`),用 run_id 作 cache_salt 即天然跨 epoch 重采样、同 epoch 续跑仍命中。judge/evolve 不经此路径(默认 salt=None)。gate 基线臂走 BaselineCache 不受影响;候选臂 messages 含 skill 版本天然区分。
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
```python
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_agent_loop_forwards_cache_salt():
|
||||||
|
"""AgentLoop.run(cache_salt=...) 透传到 llm.chat。"""
|
||||||
|
from core.agent.loop import AgentLoop
|
||||||
|
# fake llm 记录 chat 收到的 cache_salt kwarg;loop.run(..., cache_salt='run:e2')
|
||||||
|
# 断言 fake_llm.chat 收到 cache_salt='run:e2'
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest -k agent_loop_forwards_cache_salt -v`
|
||||||
|
Expected: FAIL(`run()` 无 cache_salt 参数)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: AgentLoop 透传 cache_salt**
|
||||||
|
|
||||||
|
`core/agent/loop.py`:`run`(L103)、`_step`/`_call_llm`(L271,317)签名加 `cache_salt: str | None = None`(keyword,随 session_id 透传);L329 `self._llm.chat(messages, session_id=session_id)` 改为 `self._llm.chat(messages, session_id=session_id, cache_salt=cache_salt)`。
|
||||||
|
|
||||||
|
- [ ] **Step 4: inference 用 run_id 作 salt**
|
||||||
|
|
||||||
|
`app/harness/inference.py` `_run_single_question`:`loop.run(...)`(L409)加 `cache_salt=run_id`(run_id 从 run_inference 透传到每题;`_run_single_question` 已有 run_id 上下文——若无则从 run_inference 参数透传)。确认 run_inference→_run_single_question 的 run_id 传递链完整。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 保真检查点 + 测试通过**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py tests/integration/test_agent_governed_e2e.py -q`
|
||||||
|
Expected: 全 PASS。保真(算法 #10):确认只加透传参数,Thinking+JSON/json_repair/pluggy hook 逻辑不变。
|
||||||
|
|
||||||
|
- [ ] **Step 6: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add core/agent/loop.py app/harness/inference.py tests/unit/
|
||||||
|
git commit -m "feat: inject run_id as cache_salt for per-epoch resampling (algo #10)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Self-Review(作者自查,执行者复核)
|
||||||
|
|
||||||
|
- [ ] traces 适配后诊断拿到真实轨迹(算法 #7 恢复)。
|
||||||
|
- [ ] prediction 归一化 None 保留、非标量序列化;insert 绑定异常不击穿 gather。
|
||||||
|
- [ ] early_stop 字段全局改名一致、计数改 epoch。
|
||||||
|
- [ ] 预检剔除不可训练题型(test 池不动)。
|
||||||
|
- [ ] patch 查整跨度 + 注入 + marker 完整性三防线(算法 #8 保护方向不变)。
|
||||||
|
- [ ] None/degraded 按 lapse 保守分流;降级占比>50% 中止。
|
||||||
|
- [ ] _run_step 幂等;gate_pools 与 checkpoint 的 gate_epoch_observed 同刻一致。
|
||||||
|
- [ ] holdout 去重后 baseline 0 推理、best_hard 备忘录 resume 可 hydrate。
|
||||||
|
- [ ] cache_salt=run_id 贯穿 AgentLoop,run_id 含 epoch 保证跨 epoch 重采样。
|
||||||
|
|
||||||
|
## 核心算法保真校验结论
|
||||||
|
|
||||||
|
触及算法 #5/#7/#8/#10/#12,均为恢复(#7 轨迹)/加固(#8 跨度)/透传(#10 salt)/时序(#5/#12 checkpoint),各 Task 已设保真检查点,不改算法核心逻辑。Task 5/7/9 需在实现时对照 TRM4 参考确认无行为漂移。
|
||||||
|
|
||||||
|
## 验收标准
|
||||||
|
|
||||||
|
1. `pytest tests/unit/test_harness_runner.py tests/unit/test_harness_inference.py tests/unit/test_diagnose.py tests/unit/test_patch.py tests/unit/test_harness_checkpoint.py tests/unit/test_baseline_run_log.py tests/unit/test_runner_diag_tree_inject.py` 全绿。
|
||||||
|
2. 诊断拿到非空轨迹;非标量 prediction 不崩;early_stop 按 epoch。
|
||||||
|
3. 微型题型被预检剔除;patch 不破坏 marker;降级题不驱动进化。
|
||||||
|
4. _run_step 幂等;cache_salt=run_id 贯穿。
|
||||||
@@ -0,0 +1,624 @@
|
|||||||
|
# WP4 韧性与持久化 Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** 修复 LLM 治理栈与持久化层的 10 处韧性缺陷,使训练信号不被缓存重放/截断响应/断连/INFRA 故障污染,且 workspace 元数据崩溃可恢复。
|
||||||
|
|
||||||
|
**Architecture:** 缓存加 salt 维度让跨 epoch 评估真实重采样(同 epoch 续跑仍命中);SSE 未收 `[DONE]` 视为截断进重试且不写缓存;扩展瞬时错误清单覆盖断连族;熔断半开只放一个探针;gate 基线臂 INFRA 故障不污染 BaselineCache(算法 #6 保真区,逐行比对);manifest 补原子写;只读查询不改基线元数据。
|
||||||
|
|
||||||
|
**Tech Stack:** Python 3.11、httpx、Redis、asyncio、SQLite、pytest、pydantic-settings。
|
||||||
|
|
||||||
|
**设计源**:`research-wiki/designs/2026-07-16-preflight-fixes-design.md §8`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 关键锚点(实现前必读)
|
||||||
|
|
||||||
|
| 用途 | 位置 |
|
||||||
|
|------|------|
|
||||||
|
| chat 缓存/重试 | `adapters/llm.py:271` chat(get L299 / set L373 / 重试 L336);`_call_streaming:506`;`_consume_stream:533`(不校验 done);`_iter_sse_deltas:94`(done 标志 L116);`_is_transient_error:173-186`;`_SseAnomaly:40` |
|
||||||
|
| 缓存键 | `adapters/redis_cache.py:32` `_build_key`(无 salt);`get:50`;`set:73`(ttl 分支 L89-92) |
|
||||||
|
| Protocol | `core/protocols.py:22-28` `LLMProvider.chat` |
|
||||||
|
| TTL | `main.py:41` `redis_cache_ttl=86400`;`:93` `ttl_s = ... if >0 else None` |
|
||||||
|
| 熔断 | `adapters/breaker.py:24` `is_open`(L36-37 到期即放行,无探针锁);`record_failure:39`;`record_success:63` |
|
||||||
|
| gate INFRA | `app/harness/validate.py:257` `_resolve_baseline_block`(put L303-304);`_check_infra_guard:384`(累计检查 L593);`BaselineCache` in `gate_ladder.py:344`(put L384 原子 L401-403) |
|
||||||
|
| 持久化 | `app/harness/workspace.py` 4 处非原子 `manifest.json` write_text(WP2 后行号):`_scaffold:111`/`update_manifest:289`/`record_run:318`/`update_best:372`;原子范式 `checkpoint.py:257-260`。**以 `grep -n 'manifest.json").write_text' app/harness/workspace.py` 现场定位为准** |
|
||||||
|
| 基线元数据 | `app/harness/log.py:57` `HarnessLog.__init__`(upsert `_runs` L73-82);`RunLogImpl._read_table:247`(只读) |
|
||||||
|
| dual_metric | `app/harness/observation.py:119` `write_dual_metric`(version_kind final L138);runner 调用 `runner.py:1444-1448` |
|
||||||
|
| 配置 | `.env:65` `REDIS_CACHE_TTL=0`;`.env.example:49` `=86400` |
|
||||||
|
|
||||||
|
## 核心算法保真校验
|
||||||
|
|
||||||
|
触及算法 #6(块顺序验证:基线缓存/INFRA 护栏/配对翻转)——Task 6(gate 基线臂 INFRA 隔离)改 `_resolve_baseline_block` 与护栏时序,**必须逐行比对** TRM4 `/home/iomgaa/Projects/Video-Tree-TRM4/core/harness/validate.py` 的 INFRA 护栏语义,确保只改"INFRA unit 不写缓存/不计 W/L + 护栏前置",不动配对翻转与基线快照复用逻辑。其余 task 不触碰核心算法。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 1: 缓存 cache_salt 贯穿
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `core/protocols.py:22-28`
|
||||||
|
- Modify: `adapters/llm.py:271-373`
|
||||||
|
- Modify: `adapters/redis_cache.py:32-92`
|
||||||
|
- Modify: VLM 适配转发(`adapters/` 内实现 `LLMProvider`/转发 chat 的类,grep 定位)
|
||||||
|
- Test: `tests/unit/test_redis_cache.py`、`tests/unit/test_governed_llm.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试(salt 入键)**
|
||||||
|
|
||||||
|
在 `tests/unit/test_redis_cache.py` 追加:
|
||||||
|
```python
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_salt_changes_key(fake_redis):
|
||||||
|
from adapters.redis_cache import RedisResponseCache
|
||||||
|
|
||||||
|
cache = RedisResponseCache(redis=fake_redis, ttl_s=None)
|
||||||
|
k_none = cache._build_key("m", [{"role": "user", "content": "x"}], None)
|
||||||
|
k_e1 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e1")
|
||||||
|
k_e2 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e2")
|
||||||
|
assert k_none != k_e1 != k_e2 and k_e1 != k_e2
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_redis_cache.py::test_salt_changes_key -v`
|
||||||
|
Expected: FAIL(`_build_key() takes 3 positional arguments but 4 were given`)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: redis_cache 加 salt 维度**
|
||||||
|
|
||||||
|
`adapters/redis_cache.py`:
|
||||||
|
- `_build_key(self, model, messages, cache_salt: str | None = None)`:**仅当 `cache_salt is not None` 才加入 `salt` 字段**(默认 None 时 payload 结构与现状一字节不差,旧缓存键不失效):
|
||||||
|
```python
|
||||||
|
key_obj: dict = {"model": model, "messages": messages}
|
||||||
|
if cache_salt is not None:
|
||||||
|
key_obj["salt"] = cache_salt
|
||||||
|
payload = json.dumps(key_obj, sort_keys=True, ensure_ascii=False)
|
||||||
|
```
|
||||||
|
- `get(self, model, messages, cache_salt: str | None = None)`:`key = self._build_key(model, messages, cache_salt)`
|
||||||
|
- `set(self, model, messages, response, cache_salt: str | None = None)`:同样传 `cache_salt`
|
||||||
|
|
||||||
|
- [ ] **Step 4: Protocol + chat 透传 salt**
|
||||||
|
|
||||||
|
`core/protocols.py` `LLMProvider.chat`(L22)签名加 `cache_salt: str | None = None`(keyword-only,放 `parent_call_id` 后)。
|
||||||
|
`adapters/llm.py` `chat`(L271)签名加同参;L299 改 `await self._cache.get(self._model, messages, cache_salt)`;L373 改 `await self._cache.set(self._model, messages, response, cache_salt)`。
|
||||||
|
`adapters/redis_cache.py` `get`(L50)/`set`(L73)签名各加 `cache_salt: str | None = None`,内部 `_build_key(model, messages, cache_salt)`。
|
||||||
|
**VLM 转发**:`core/protocols.py` `VLMProvider.chat_with_images`(L35)与 `adapters/vlm.py` `GovernedVLMClient.chat_with_images`(L32)签名加 keyword-only `cache_salt: str | None = None`,L53 转发 `self._llm.chat(..., cache_salt=cache_salt)`。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 运行测试确认通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_redis_cache.py tests/unit/test_governed_llm.py -q`
|
||||||
|
Expected: 全 PASS(默认 `cache_salt=None` 保持旧键,既有缓存测试不受影响)。
|
||||||
|
|
||||||
|
- [ ] **Step 6: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add core/protocols.py adapters/llm.py adapters/redis_cache.py tests/unit/test_redis_cache.py
|
||||||
|
git commit -m "feat: add cache_salt dimension to LLM response cache"
|
||||||
|
```
|
||||||
|
|
||||||
|
> 注:训练链路注入 epoch 盐(`f"{run_id}:e{epoch}"`)在 WP3 的推理调用点完成(本 WP 只提供 salt 能力)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 2: TTL 语义修正(≤0 报错)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `adapters/redis_cache.py`(新增 `_resolve_cache_ttl`)
|
||||||
|
- Modify: `main.py:93`、`app/harness/video_split_cli.py:257`(两处 TTL 入口共用)
|
||||||
|
- Modify: `.env:65`、`.env.example:49`
|
||||||
|
- Test: 新增 `tests/unit/test_infra_settings.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
新增 `tests/unit/test_infra_settings.py`:
|
||||||
|
```python
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
def test_redis_cache_ttl_zero_rejected():
|
||||||
|
"""REDIS_CACHE_TTL<=0 必须启动即报错,消灭'0=永不过期'隐式语义。"""
|
||||||
|
from adapters.redis_cache import _resolve_cache_ttl # Step 3 抽出的纯函数
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="REDIS_CACHE_TTL"):
|
||||||
|
_resolve_cache_ttl(0)
|
||||||
|
with pytest.raises(ValueError, match="REDIS_CACHE_TTL"):
|
||||||
|
_resolve_cache_ttl(-1)
|
||||||
|
assert _resolve_cache_ttl(86400) == 86400
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_infra_settings.py -v`
|
||||||
|
Expected: FAIL(`cannot import name '_resolve_cache_ttl'`)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 抽出 fail-loud 并替换两处 TTL 入口**
|
||||||
|
|
||||||
|
`adapters/redis_cache.py` 新增模块级纯函数(配置校验贴近 cache 实现、避免 app→main 依赖):
|
||||||
|
```python
|
||||||
|
def _resolve_cache_ttl(ttl: int) -> int:
|
||||||
|
"""校验 Redis 缓存 TTL:必须为正整数(消灭 0=永不过期 的隐式语义)。"""
|
||||||
|
if ttl <= 0:
|
||||||
|
raise ValueError(
|
||||||
|
f"REDIS_CACHE_TTL 必须为正整数秒,实际 {ttl}。"
|
||||||
|
"训练场景建议 >= 单次训练时长(如 86400)。"
|
||||||
|
)
|
||||||
|
return ttl
|
||||||
|
```
|
||||||
|
`main.py:93` 改为 `ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl)`(import 上述函数,删 `if >0 else None`)。
|
||||||
|
`app/harness/video_split_cli.py:257`(`_build_redis_cache` 内)同样从 `ttl_s = ... if > 0 else None` 改为 `ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl)`——消灭第二条 `0=永不过期` 入口。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 改 .env / .env.example**
|
||||||
|
|
||||||
|
`.env:65` `REDIS_CACHE_TTL=0` → `REDIS_CACHE_TTL=86400`。
|
||||||
|
`.env.example:49` 确认为正整数(当前 86400,OK);补注释 `# 正整数秒,禁止 0(0 会被拒绝启动)`。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 运行测试确认通过**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_infra_settings.py -v`
|
||||||
|
Expected: PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 6: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add main.py .env.example tests/unit/test_infra_settings.py
|
||||||
|
git commit -m "fix: reject REDIS_CACHE_TTL<=0 (kill implicit never-expire)"
|
||||||
|
```
|
||||||
|
> 注:`.env` 不提交(gitignore);改动需手动同步到运行环境,runbook 会提示。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 3: SSE 截断检测(未收 [DONE] 即重试,不写缓存)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `adapters/llm.py:533-578`(_consume_stream)
|
||||||
|
- Test: `tests/unit/test_governed_llm.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
在 `tests/unit/test_governed_llm.py` 追加:
|
||||||
|
```python
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_truncated_stream_without_done_raises(_build_client):
|
||||||
|
"""SSE 流耗尽但未收 [DONE] → _SseAnomaly(进重试,不当成功)。"""
|
||||||
|
from adapters.llm import _SseAnomaly, GovernedLLMClient
|
||||||
|
|
||||||
|
async def _lines():
|
||||||
|
yield 'data: {"choices":[{"delta":{"content":"半"}}]}'
|
||||||
|
# 无 data: [DONE] —— 模拟服务端截断
|
||||||
|
|
||||||
|
client = _build_client() # 依现有 fixture
|
||||||
|
with pytest.raises(_SseAnomaly):
|
||||||
|
await client._consume_stream(_lines())
|
||||||
|
```
|
||||||
|
> 依 `test_governed_llm.py` 现有 `_build_client`(L50)/`_FakeRedisCache` 构造;实现前读该文件对齐 client 构造签名。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -k truncated_stream -v`
|
||||||
|
Expected: FAIL(当前耗尽即正常返回,不抛)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: _consume_stream 校验 done**
|
||||||
|
|
||||||
|
`adapters/llm.py` `_consume_stream` 在 `return content, thinking, ttft_ms, max_inter_token_ms, usage`(L578)之前加:
|
||||||
|
```python
|
||||||
|
if not usage_sink.get("done"):
|
||||||
|
raise _SseAnomaly("truncated_no_done")
|
||||||
|
```
|
||||||
|
(`_iter_sse_deltas` 收到 `[DONE]` 时置 `usage_sink["done"]=True`,L116;未置说明流被截断。`_SseAnomaly` 已在 `_is_transient_error` L186 归为可重试,故自动进重试梯且不走成功路径、不写缓存。)
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行测试确认通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py tests/unit/test_streaming.py -q`
|
||||||
|
Expected: 全 PASS(正常流带 `[DONE]` 的既有测试仍通过;若既有 streaming 测试的假流未含 `[DONE]`,同步补 `data: [DONE]` 终帧)。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add adapters/llm.py tests/unit/test_governed_llm.py
|
||||||
|
git commit -m "fix: treat SSE stream without [DONE] as truncated (retry, no cache)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 4: 瞬时错误清单扩展(断连族)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `adapters/llm.py:182-186`
|
||||||
|
- Test: `tests/unit/test_governed_llm.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
在 `tests/unit/test_governed_llm.py` 追加:
|
||||||
|
```python
|
||||||
|
def test_transient_covers_disconnect_family():
|
||||||
|
import httpx
|
||||||
|
from adapters.llm import _is_transient_error
|
||||||
|
|
||||||
|
assert _is_transient_error(httpx.RemoteProtocolError("peer reset"))
|
||||||
|
assert _is_transient_error(httpx.ReadError("read"))
|
||||||
|
assert _is_transient_error(httpx.ConnectTimeout("ct"))
|
||||||
|
assert _is_transient_error(httpx.PoolTimeout("pt"))
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -k disconnect_family -v`
|
||||||
|
Expected: FAIL(RemoteProtocolError/PoolTimeout 不在当前清单)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 扩展 _is_transient_error**
|
||||||
|
|
||||||
|
`adapters/llm.py` L182-183 的:
|
||||||
|
```python
|
||||||
|
if isinstance(exc, (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout)):
|
||||||
|
return True
|
||||||
|
```
|
||||||
|
改为(两族基类覆盖 RemoteProtocolError/ReadError/ConnectTimeout/PoolTimeout 等):
|
||||||
|
```python
|
||||||
|
if isinstance(exc, (httpx.TimeoutException, httpx.TransportError)):
|
||||||
|
return True
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行测试确认通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -q`
|
||||||
|
Expected: 全 PASS(`httpx.HTTPStatusError` 非 TransportError 子类,401/403 致命分支不受影响)。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add adapters/llm.py tests/unit/test_governed_llm.py
|
||||||
|
git commit -m "fix: cover httpx disconnect family in transient error set"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 5: 熔断半开单探针锁
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `adapters/breaker.py:18-70`
|
||||||
|
- Test: `tests/unit/test_breaker.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
在 `tests/unit/test_breaker.py::TestCircuitBreaker` 追加:
|
||||||
|
```python
|
||||||
|
def test_half_open_admits_single_probe(self):
|
||||||
|
from adapters.breaker import CircuitBreaker
|
||||||
|
|
||||||
|
b = CircuitBreaker(fail_threshold=2, cooldown_s=10.0)
|
||||||
|
b.record_failure("p", now=0.0)
|
||||||
|
b.record_failure("p", now=0.0) # 开路至 t=10
|
||||||
|
assert b.is_open("p", now=5.0) is True # 冷却中
|
||||||
|
# 冷却到期:只放行第一个探针
|
||||||
|
assert b.is_open("p", now=11.0) is False # 探针 1 放行
|
||||||
|
assert b.is_open("p", now=11.0) is True # 探针 2 被挡(探针在途)
|
||||||
|
b.record_success("p") # 探针成功 → 闭合
|
||||||
|
assert b.is_open("p", now=12.0) is False
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_breaker.py -k half_open_admits_single -v`
|
||||||
|
Expected: FAIL(当前到期后所有调用都返回 False)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 实现半开单探针锁**
|
||||||
|
|
||||||
|
`adapters/breaker.py`:
|
||||||
|
- `__init__` 加 `self._half_open_inflight: dict[str, bool] = {}`
|
||||||
|
- `is_open` 改为:
|
||||||
|
```python
|
||||||
|
def is_open(self, source_name: str, now: float) -> bool:
|
||||||
|
until = self._open_until.get(source_name)
|
||||||
|
if until is None:
|
||||||
|
return False
|
||||||
|
if now < until:
|
||||||
|
return True # 冷却中,全挡
|
||||||
|
# 冷却到期:half-open,只放行一个探针
|
||||||
|
if self._half_open_inflight.get(source_name):
|
||||||
|
return True # 已有探针在途,继续挡
|
||||||
|
self._half_open_inflight[source_name] = True
|
||||||
|
return False
|
||||||
|
```
|
||||||
|
- `record_success` 加 `self._half_open_inflight.pop(source_name, None)`(探针成功 → 清在途 + 已有的清 fails/open_until)
|
||||||
|
- `record_failure`:探针失败会累计并可能重开路;末尾加 `self._half_open_inflight.pop(source_name, None)`(让下一轮 cooldown 后可再探)
|
||||||
|
- `force_open`:加 `self._half_open_inflight.pop(source_name, None)`
|
||||||
|
|
||||||
|
(is_open 在 asyncio 单线程内同步执行,"检查+标记探针"原子,无竞态。)
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行测试确认通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_breaker.py tests/unit/test_governed_llm.py -q`
|
||||||
|
Expected: 全 PASS(既有 circuit_open 测试若假设"到期即多次放行"需同步更新为单探针语义)。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add adapters/breaker.py tests/unit/test_breaker.py
|
||||||
|
git commit -m "fix: half-open circuit admits single probe (no thundering herd)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 6: gate 基线臂 INFRA 隔离(算法 #6 保真区)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/validate.py:257-311`(_resolve_baseline_block)+ 护栏时序
|
||||||
|
- Test: `tests/unit/test_harness_validate.py`
|
||||||
|
|
||||||
|
> **保真前置**:实现前完整读 `app/harness/validate.py:257-311` `_resolve_baseline_block`、`_candidate_correctness_from_db`、块循环(575-600),并逐行比对 TRM4 `core/harness/validate.py` 的 INFRA 护栏语义。目标仅为:① 基线臂 INFRA 故障(stop_reason∈{error,parse_error})的 unit **不写 BaselineCache**、**不计入 W/L 翻转**;② 护栏检查移到"写缓存之前"。不得改动配对翻转、基线快照复用、unit 折叠逻辑。
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
在 `tests/unit/test_harness_validate.py` 追加(用 fake run_inference 让某基线 unit 返回 stop_reason=error):
|
||||||
|
```python
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_baseline_infra_error_not_cached(tmp_path):
|
||||||
|
"""基线臂 INFRA error 的 unit 不写入 BaselineCache(不永久污染)。"""
|
||||||
|
# 依现有 test_harness_validate.py 的 fake run_inference / BaselineCache fixture 构造;
|
||||||
|
# 让 miss unit u1 的推理返回 prediction=None, stop_reason='error';
|
||||||
|
# 调 _resolve_baseline_block 后断言 baseline_cache.get(..., u1) is None
|
||||||
|
...
|
||||||
|
```
|
||||||
|
> 依 `test_harness_validate.py` 现有 fixture(fake `run_inference`、`BaselineCache`、`HarnessLog`);实现前读该文件对齐构造,勿臆造签名。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_validate.py -k baseline_infra_error_not_cached -v`
|
||||||
|
Expected: FAIL(当前 INFRA unit 的对错被 put 进缓存)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 实现 INFRA 隔离(精确数据路径)**
|
||||||
|
|
||||||
|
**数据路径**(Codex 审指出:当前 `_load_run_rows`(validate.py:212)只查 `question_id,prediction,answer,steps_json`,拿不到 per-unit stop_reason,只有汇总 `stop_reason_counts` 无法定位哪个 unit 是 INFRA):
|
||||||
|
1. `_load_run_rows`(validate.py:194-212)的 SELECT 增加 `stop_reason` 列;`_candidate_correctness_from_db` 相应可返回每题 stop_reason(或新增 `_infra_question_ids_from_db(log, run_id, questions)` 返回 stop_reason∈{"error","parse_error"} 的 qid 集)。
|
||||||
|
2. `_resolve_baseline_block`:miss 跑完后,计算本块 INFRA unit 集(unit 内**任一题** stop_reason∈{"error","parse_error"})。对这些 unit:**不 `baseline_cache.put`**、**不计入 `b_units`**、并从返回给调用方的有效 unit 集中排除。护栏所需 `errors_inc` 在 put 之前累计并调 `_check_infra_guard`(护栏前置于缓存写入)。
|
||||||
|
3. **贯穿调用方**(validate.py:575-600):`_resolve_baseline_block` 返回"有效 unit 子集 `valid_unit_chunk`",后续 `_run_candidate_block`、`unit_correctness_view`、`pair_block`、`_build_evidence_rows` 全部用 `valid_unit_chunk`(而非原始 `unit_chunk`),确保配对 `unit_ids` 与基线侧一致、不含 INFRA unit。
|
||||||
|
4. 保真:非 INFRA unit 的配对翻转、基线快照复用、unit 折叠逻辑一字不改;仅"把 INFRA unit 从本块整体剔除"。按 Step 前"保真前置"逐行比对 TRM4。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行测试确认通过 + 保真回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_validate.py tests/unit/test_gate_block_unit.py tests/unit/test_gates.py -q`
|
||||||
|
Expected: 全 PASS(配对翻转/e-process 既有测试不受影响 = 保真达成)。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/validate.py tests/unit/test_harness_validate.py
|
||||||
|
git commit -m "fix: isolate gate baseline-arm INFRA errors from BaselineCache (algo #6)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 7: manifest 原子写
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/workspace.py`(`_scaffold:111`/`update_manifest:282`/`record_run:311`/`update_best:365`)
|
||||||
|
- Test: `tests/unit/test_harness_workspace.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
在 `tests/unit/test_harness_workspace.py` 追加:
|
||||||
|
```python
|
||||||
|
def test_update_manifest_is_atomic(workspace_dir, monkeypatch):
|
||||||
|
"""写 manifest 途中崩溃不产生半截 JSON(原子写:tmp 存在即失败也不损原文件)。"""
|
||||||
|
import json
|
||||||
|
from app.harness import workspace as ws
|
||||||
|
|
||||||
|
# 先建合法 manifest
|
||||||
|
... # 依现有 fixture 初始化 workspace
|
||||||
|
original = (workspace_dir / "manifest.json").read_text()
|
||||||
|
|
||||||
|
# monkeypatch os.replace 抛异常,模拟替换阶段崩溃
|
||||||
|
def _boom(src, dst):
|
||||||
|
raise OSError("crash during replace")
|
||||||
|
monkeypatch.setattr(ws.os, "replace", _boom)
|
||||||
|
with pytest.raises(OSError):
|
||||||
|
ws.update_manifest(workspace_dir, skills="skills/v2")
|
||||||
|
# 原 manifest 未被破坏
|
||||||
|
assert (workspace_dir / "manifest.json").read_text() == original
|
||||||
|
assert json.loads((workspace_dir / "manifest.json").read_text())
|
||||||
|
```
|
||||||
|
> 依 `test_harness_workspace.py` 现有 workspace 初始化 fixture;实现前读对齐。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_workspace.py -k update_manifest_is_atomic -v`
|
||||||
|
Expected: FAIL(当前 write_text 非原子,崩溃留半截)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 加原子写 helper 并替换 4 处**
|
||||||
|
|
||||||
|
`app/harness/workspace.py` 顶部确认 `import os`。新增模块级 helper:
|
||||||
|
```python
|
||||||
|
def _atomic_write_json(path: Path, data: dict) -> None:
|
||||||
|
"""原子写 JSON:tmp + os.replace(对齐 checkpoint.py 范式,防半截损坏)。"""
|
||||||
|
tmp = path.with_name(path.name + ".tmp")
|
||||||
|
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
|
os.replace(tmp, path)
|
||||||
|
```
|
||||||
|
把 `_scaffold_workspace`(~L111)、`update_manifest`(~L289)、`record_run`(~L318)、`update_best`(~L372)四处的 `(workspace_dir / "manifest.json").write_text(json.dumps(...))` 替换为 `_atomic_write_json(workspace_dir / "manifest.json", <data>)`(用 `grep -n 'manifest.json").write_text' app/harness/workspace.py` 定位全部 4 处,行号随 WP2 改动漂移)。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行测试确认通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_workspace.py -q`
|
||||||
|
Expected: 全 PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/workspace.py tests/unit/test_harness_workspace.py
|
||||||
|
git commit -m "fix: atomic writes for manifest/record_run/update_best (tmp+replace)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 8: 只读查询不改基线元数据
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/log.py:57-83`(HarnessLog.__init__ 加 register_run 开关)
|
||||||
|
- Modify: 只读查询基线的调用点(`app/harness/runner.py:923` `_init_gate`、`app/harness/pools.py:765,818`)
|
||||||
|
- Test: `tests/unit/test_harness_log.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
在 `tests/unit/test_harness_log.py::TestHarnessLogUpsert` 追加:
|
||||||
|
```python
|
||||||
|
def test_register_run_false_skips_upsert(self, tmp_path):
|
||||||
|
"""register_run=False 时只读打开不改写已有 _runs 行(started_at/status 不变)。"""
|
||||||
|
db = str(tmp_path / "h.db")
|
||||||
|
from app.harness.log import HarnessLog
|
||||||
|
|
||||||
|
with HarnessLog(db, "r1") as log:
|
||||||
|
log # 初次注册
|
||||||
|
row0 = _read_run_row(db, "r1") # 依现有 helper 读 started_at/status
|
||||||
|
|
||||||
|
with HarnessLog(db, "r1", register_run=False) as log:
|
||||||
|
log.query("SELECT 1") # 只读
|
||||||
|
row1 = _read_run_row(db, "r1")
|
||||||
|
assert row1["started_at"] == row0["started_at"]
|
||||||
|
assert row1["status"] == row0["status"]
|
||||||
|
```
|
||||||
|
> 依 `test_harness_log.py` 现有读行 helper;实现前读 `TestHarnessLogUpsert`(L196)对齐。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_log.py -k register_run_false -v`
|
||||||
|
Expected: FAIL(`unexpected keyword argument 'register_run'`)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: HarnessLog 加 register_run 开关**
|
||||||
|
|
||||||
|
`app/harness/log.py` `__init__` 签名加 `register_run: bool = True`(keyword)。把 L73-82 的 `_runs` upsert 包进 `if register_run:`;`register_run=False` 时跳过 upsert(仅 `_init_fixed_tables` 建表 + 连接,供只读查询)。`__exit__`/`close` 的 status 更新同样在 `register_run` 为 True 时才执行(避免只读关闭把 status 改 completed)。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 只读调用点传 register_run=False**
|
||||||
|
|
||||||
|
`app/harness/runner.py:923`(_init_gate 用 baseline_run_id 只读查 predictions)、`app/harness/pools.py:765,818`(build_or_load_pools 只读查 predictions)三处 `HarnessLog(...)` 调用加 `register_run=False`。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 运行测试确认通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_log.py tests/unit/test_harness_pools.py -q`
|
||||||
|
Expected: 全 PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 6: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/log.py app/harness/runner.py app/harness/pools.py tests/unit/test_harness_log.py
|
||||||
|
git commit -m "fix: read-only baseline queries skip _runs upsert (register_run flag)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 9: dual_metric version_kind 口径修正
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `app/harness/runner.py:1444-1448`(第二次 write_dual_metric)
|
||||||
|
- Test: `tests/unit/test_harness_observation.py` 或 `test_harness_runner.py`
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
`read_dual_metric` 真实签名为 `(db_path, *, run_id)`(无 epoch 参数,observation.py:164),故 observation 层 write/read 对任意 version_kind 都已支持——真正要验证的是 **runner 慢更新第二次写出的是 `slow_candidate` 而非 `final`**。在 `tests/unit/test_harness_observation.py` 追加 observation 层可区分性基线测试:
|
||||||
|
```python
|
||||||
|
def test_dual_metric_kinds_distinguishable(tmp_path):
|
||||||
|
from app.harness.observation import write_dual_metric, read_dual_metric
|
||||||
|
|
||||||
|
db = str(tmp_path / "h.db")
|
||||||
|
write_dual_metric(db, run_id="r", epoch=1, version_kind="final",
|
||||||
|
skills_version="v1", prompts_version="v1", pool="val",
|
||||||
|
hard_acc=0.7, soft_score=None, mixed_score=None)
|
||||||
|
write_dual_metric(db, run_id="r", epoch=1, version_kind="slow_candidate",
|
||||||
|
skills_version="v2", prompts_version="v2", pool="val",
|
||||||
|
hard_acc=0.6, soft_score=None, mixed_score=None)
|
||||||
|
rows = read_dual_metric(db, run_id="r") # 真实签名无 epoch,返回该 run 全部行
|
||||||
|
kinds = {row["version_kind"] for row in rows if row["epoch"] == 1}
|
||||||
|
assert kinds == {"final", "slow_candidate"}
|
||||||
|
```
|
||||||
|
> 实现前读 `test_harness_observation.py` 与 `write_dual_metric`/`read_dual_metric`(observation.py:119/164)确认参数名(epoch/pool/hard_acc 等)与返回行字段。真正的 `slow_candidate` 语义由 Step 3 改 runner 调用点落地。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行基线(observation 层已支持任意 kind)**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_observation.py -k dual_metric_kinds -v`
|
||||||
|
Expected: PASS(observation 层本就接受任意 version_kind);本 task 的实质改动在 Step 3 的 runner 调用点。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 改 runner 慢更新第二次写为 slow_candidate**
|
||||||
|
|
||||||
|
`app/harness/runner.py` L1444-1448 的 `write_dual_metric(..., version_kind="final", ...)`(`_slow_update_cycle` Phase 8 的 R2 行)改为 `version_kind="slow_candidate"`,使被 revert 的慢更新候选不再占用 `final` 语义。确认 Phase 2 的第一次(L1399-1403)仍为 `final`(epoch 终值唯一)。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行测试确认通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_observation.py tests/unit/test_harness_runner.py -q`
|
||||||
|
Expected: 全 PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/runner.py tests/unit/test_harness_observation.py
|
||||||
|
git commit -m "fix: slow-update R2 dual_metric uses slow_candidate kind (not final)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 10: 离线 --retry-uncertain + docstring 修正
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `core/evolution/protocols.py:130`(`DiagnosisSignalStore.done_question_ids` 加参数)
|
||||||
|
- Modify: `adapters/baseline_diagnosis_store.py:109`(SqliteDiagnosisSignalStore 实现 + SQL)
|
||||||
|
- Modify: `app/harness/baseline_diagnosis.py:96`(透传 + docstring)
|
||||||
|
- Modify: `app/harness/video_split_cli.py`(build_arg_parser + run_pipeline 透传)
|
||||||
|
- Test: `tests/unit/`(对 SqliteDiagnosisSignalStore.done_question_ids)
|
||||||
|
|
||||||
|
- [ ] **Step 1: 写失败测试**
|
||||||
|
|
||||||
|
`done_question_ids` 的真实定义在 Protocol `core/evolution/protocols.py:130` 与实现 `adapters/baseline_diagnosis_store.py:109`(不在 baseline_diagnosis.py)。对 SqliteDiagnosisSignalStore 追加测试:
|
||||||
|
```python
|
||||||
|
def test_done_question_ids_retry_uncertain_excludes(tmp_path):
|
||||||
|
# 构造 baseline_diagnosis 表含 tier=T2 行与 tier=uncertain 行(同 run+fingerprint);
|
||||||
|
from adapters.baseline_diagnosis_store import SqliteDiagnosisSignalStore
|
||||||
|
store = SqliteDiagnosisSignalStore(str(tmp_path / "h.db"))
|
||||||
|
# ... upsert 一条 T2、一条 uncertain ...
|
||||||
|
assert "<uncertain_qid>" in store.done_question_ids("run", "fp") # 默认含
|
||||||
|
assert "<uncertain_qid>" not in store.done_question_ids("run", "fp", retry_uncertain=True) # 排除
|
||||||
|
assert "<t2_qid>" in store.done_question_ids("run", "fp", retry_uncertain=True) # T2 仍算完成
|
||||||
|
```
|
||||||
|
> 实现前读 `adapters/baseline_diagnosis_store.py` 的 upsert/表结构与 `done_question_ids` SQL 对齐构造。
|
||||||
|
|
||||||
|
- [ ] **Step 2: 运行确认失败**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest -k done_question_ids_retry_uncertain -v`
|
||||||
|
Expected: FAIL(`unexpected keyword argument 'retry_uncertain'`)。
|
||||||
|
|
||||||
|
- [ ] **Step 3: 实现 retry_uncertain(贯穿 Protocol→Adapter→编排→CLI)**
|
||||||
|
|
||||||
|
- `core/evolution/protocols.py:130` `DiagnosisSignalStore.done_question_ids` 签名加 `retry_uncertain: bool = False`(keyword)。
|
||||||
|
- `adapters/baseline_diagnosis_store.py:109` 实现同签名;SQL 在 `retry_uncertain=True` 时追加 `AND tier != 'uncertain'`(uncertain 题不算完成,会被重诊)。
|
||||||
|
- `app/harness/baseline_diagnosis.py:96` `done = store.done_question_ids(baseline_run_id, diag_fingerprint)` 透传 `retry_uncertain`;修正模块 docstring 把"崩溃最多丢正在写的一行/逐行落库"改为实际的"run 末批量落库(Phase 3),崩溃丢本次 run 未落库结果,靠 Redis 缓存缓解重烧"。
|
||||||
|
- `app/harness/video_split_cli.py`:`build_arg_parser` 加 `--retry-uncertain`(store_true);`run_pipeline` 透传到 `run_baseline_diagnosis`。
|
||||||
|
|
||||||
|
- [ ] **Step 4: 运行测试确认通过 + 回归**
|
||||||
|
|
||||||
|
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -q`
|
||||||
|
Expected: 全 PASS。
|
||||||
|
|
||||||
|
- [ ] **Step 5: 提交**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add app/harness/video_split_cli.py app/harness/baseline_diagnosis.py tests/unit/
|
||||||
|
git commit -m "feat: --retry-uncertain re-diagnoses uncertain rows; fix docstring"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Self-Review(作者自查,执行者复核)
|
||||||
|
|
||||||
|
- [ ] cache_salt 四处(protocols/llm/redis_cache/VLM 转发)贯通,默认 None 保持旧键。
|
||||||
|
- [ ] TTL≤0 fail-loud + .env/.env.example 同步(.env 手动,runbook 提示)。
|
||||||
|
- [ ] SSE 未收 [DONE] → _SseAnomaly(已在瞬时清单 → 重试 + 不写缓存)。
|
||||||
|
- [ ] 瞬时清单用 TimeoutException+TransportError 两族基类覆盖断连族,不误纳 HTTPStatusError。
|
||||||
|
- [ ] 熔断半开单探针:is_open 检查+标记原子(asyncio 单线程)。
|
||||||
|
- [ ] Task 6 触及算法 #6,已设保真前置(逐行比对 TRM4,只改 INFRA 不写缓存 + 护栏前置)。
|
||||||
|
- [ ] manifest 4 处原子写;只读查询 register_run=False 不改基线元数据。
|
||||||
|
|
||||||
|
## 核心算法保真校验结论
|
||||||
|
|
||||||
|
Task 6 触及算法 #6(块顺序验证),已在该 Task 设"保真前置"要求逐行比对 TRM4 `validate.py`,仅改 INFRA 隔离与护栏时序,不动配对翻转/基线快照/unit 折叠。其余 Task 均为治理栈/持久化层,不涉及核心算法。
|
||||||
|
|
||||||
|
## 验收标准
|
||||||
|
|
||||||
|
1. `pytest tests/unit/test_redis_cache.py tests/unit/test_governed_llm.py tests/unit/test_breaker.py tests/unit/test_streaming.py tests/unit/test_harness_validate.py tests/unit/test_harness_workspace.py tests/unit/test_harness_log.py tests/unit/test_harness_observation.py tests/unit/test_infra_settings.py` 全绿。
|
||||||
|
2. 缓存加 salt 后跨 epoch 键不同、同 epoch 键相同。
|
||||||
|
3. 熔断半开只放一个探针;SSE 截断进重试不写缓存。
|
||||||
|
4. manifest 原子写;基线 _runs 元数据只读查询不被改写。
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
---
|
||||||
|
type: plan
|
||||||
|
node_id: plan:preflight-wp1-asset-migration
|
||||||
|
title: "WP1 资产迁移"
|
||||||
|
date: 2026-07-16
|
||||||
|
---
|
||||||
|
|
||||||
|
# WP1 资产迁移
|
||||||
|
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
---
|
||||||
|
type: plan
|
||||||
|
node_id: plan:preflight-wp2-split-wiring
|
||||||
|
title: "WP2 切分与接线"
|
||||||
|
date: 2026-07-16
|
||||||
|
---
|
||||||
|
|
||||||
|
# WP2 切分与接线
|
||||||
|
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
---
|
||||||
|
type: plan
|
||||||
|
node_id: plan:preflight-wp3-train-loop
|
||||||
|
title: "WP3 训练循环与进化引擎"
|
||||||
|
date: 2026-07-16
|
||||||
|
---
|
||||||
|
|
||||||
|
# WP3 训练循环与进化引擎
|
||||||
|
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
---
|
||||||
|
type: plan
|
||||||
|
node_id: plan:preflight-wp4-resilience
|
||||||
|
title: "WP4 韧性与持久化"
|
||||||
|
date: 2026-07-16
|
||||||
|
---
|
||||||
|
|
||||||
|
# WP4 韧性与持久化
|
||||||
|
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
# 训练前修复分支 final whole-implementation review
|
||||||
|
|
||||||
|
> 2026-07-16。分支 `feat/preflight-train-fixes`,4 工作包 33 个实现 commit。SDD 每 task 三审 + 本次 Codex 跨 task 整分支终审(gpt-5.4 xhigh)。
|
||||||
|
> 覆盖矩阵:21 项确认缺陷 + 接线 2 项 + WP1 死字段清理**全部实现落地**(Codex 逐项核对 file:line 证据)。核心算法保真(#4/#5/#6/#7/#8/#9/#10/#12)、依赖方向(core 不依赖 app/adapters)、loguru、原子写、死字段无残留——全部 verified OK。
|
||||||
|
|
||||||
|
## 执行结果汇总
|
||||||
|
|
||||||
|
| WP | 内容 | commit 数 | Codex 三审结果 |
|
||||||
|
|----|------|----------|---------------|
|
||||||
|
| WP1 | 模板迁移 + 死字段 + fail-loud | 3 | 无任何问题(模板 cmp TRM4 逐字节一致) |
|
||||||
|
| WP2 | 切分与接线(tier/功效/seed/覆盖保护) | 7 | 无 Critical;2 Important(--force 备份健壮性)已修 |
|
||||||
|
| WP4 | 韧性与持久化(10 项) | 11 | 无 Critical;2 Important(全 INFRA/parse_error 护栏)已修 |
|
||||||
|
| WP3 | 训练循环与进化(9 项,5 算法保真区) | 12 | 无 Critical;2 Important(单元计数/全过滤 fail-fast)已修 |
|
||||||
|
|
||||||
|
全量 1523 tests passed,ruff 全绿。
|
||||||
|
|
||||||
|
## Final review 发现与处置
|
||||||
|
|
||||||
|
### C-1(必须,runbook)——.env 未同步导致 train 启动崩溃
|
||||||
|
`.env:65` 仍 `REDIS_CACHE_TTL=0`,WP4 Task 2 的 fail-loud 会在 `main.py` / `video_split_cli` 构建 Redis 缓存时抛 ValueError,训练进不到 `runner.train`。`.env.example` 已改 86400 但 `.env`(gitignore)需**手动**改。→ 训练前 runbook 第一步:`REDIS_CACHE_TTL` 改为正整数(如 86400)。
|
||||||
|
|
||||||
|
### I-3 / I-4(真实缺陷,✅ 已修)
|
||||||
|
| # | 缺陷 | 首跑是否触发 | 处置(commit) |
|
||||||
|
|---|------|:---:|------|
|
||||||
|
| I-4 | 显式 task_types 子集不过滤冻结全局 pools → 训练非请求题型 | 全 12 类首跑**不触发** | ✅ `ee69721`:`_filter_untrainable_types` 候选集先与 task_types 取交集,非请求题型剔除,fail-fast 保留 |
|
||||||
|
| I-3 | gate INFRA 护栏分子按 record、分母按 unit,AR pair 误触发 gate_guard_err | 全 single 首跑**不触发** | ✅ `b3ba11c`:INFRA 分子改按 unit 数;顺带修正 2 个既有护栏测试的 mock 不真实性(per-record 与 summary 不一致,真实推理不会发生),保真套件全绿 |
|
||||||
|
|
||||||
|
> M-2(`6911c83`):`INFRA_STOP_REASONS` 提为 core 公共常量、app import。**残留 future work**:`app/harness/video_split_cli.py` 仍有第三份独立副本(本次 scope 外),待后续 dedup。
|
||||||
|
|
||||||
|
### I-1 / I-2(技术论证:实际影响可控,记录不强修)
|
||||||
|
| # | Codex 关切 | 论证 | 首跑建议 |
|
||||||
|
|---|-----------|------|---------|
|
||||||
|
| I-1 | cache_salt 未贯穿工具内 LLM/VLM(observe_frame VLM、summarizer) | 工具内是**确定性子程序**——同帧→同 VLM 描述、同轨迹→同 summary,重放语义正确甚至期望;需跨 epoch 重采样的 agent **决策** LLM(主 loop chat)已正确注入 run_id 盐 | 改 .env 后可跑;若谨慎可首跑关 Redis 缓存 |
|
||||||
|
| I-2 | 基线臂 miss 新鲜推理注入 epoch salt,不符固定快照 | BaselineCache(`baseline_cache.json`)**跨 run 持久**:快照一旦建立即稳定,epoch salt 只作用于"首次建立快照的那一次采样"(本就要采一次),中断重跑命中持久缓存 | 影响限于首次采样,可接受 |
|
||||||
|
|
||||||
|
> 若后续要彻底贯彻 P1-1(把 cache_salt 显式化、基线臂传 None、工具内也隔离),记 future work——需把 Task 9 的"inference 内部用 run_id"重构为"调用方显式传 cache_salt",风险中等。
|
||||||
|
|
||||||
|
### Minor(future work / 顺手)
|
||||||
|
- **M-2**:INFRA stop-reason 集合在 core/app 各一份 → 已派 app import core 常量消除漂移。
|
||||||
|
- **M-1**:`_atomic_write_json` 在 pools.py/workspace.py 两份等价实现(行为一致,可抽共享 helper)。
|
||||||
|
- **M-3**:部分 analyses JSON(非 checkpoint/manifest/pools)仍直接 write_text(不破坏续跑主状态)。
|
||||||
|
|
||||||
|
## 训练前 runbook(结合本分支)
|
||||||
|
|
||||||
|
1. **改 `.env`:`REDIS_CACHE_TTL=0` → `86400`**(C-1,不改则启动崩)。
|
||||||
|
2. 备份 `workspaces/video-split/` 冻结产物。改 `config/video_split.yaml` 已含 `val_ratio: 0.4`(WP2),重跑 `build_video_split.sh`(诊断命中缓存秒级重切)→ 核对 val 错题≥20 / T2 入 diag / tier 感知生效。
|
||||||
|
3. 建 seed:`extract_run_db(infer_adhoc, dedupe_per_question=True)` + `init_seed('adhoc-baseline', pools_json=…, split_manifest=…)`(WP2 接线)。
|
||||||
|
4. 新建 `config/train_videomme.yaml` + `scripts/train_videomme.sh`:`epochs=3`、`early_stop_patience=2`(epoch 语义 WP3)、`run_holdout_eval=true`(去重版 WP3)、`trainable_min_units=8`(WP3 新字段,必填)、全 12 类(避免 I-4)、其余 gate/batch 沿用 default.yaml。
|
||||||
|
5. tmux 启动 `CUDA_VISIBLE_DEVICES=0 bash scripts/train_videomme.sh`。预检会自动剔除微型题型(OCR/Spatial/Temporal Perception 等 5 类),打印剔除清单。
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
---
|
||||||
|
type: review
|
||||||
|
node_id: review:preflight-final-review
|
||||||
|
title: 训练前修复分支终审
|
||||||
|
date: 2026-07-16
|
||||||
|
---
|
||||||
|
|
||||||
|
# 训练前修复分支终审
|
||||||
|
|
||||||
@@ -171,6 +171,7 @@ class _MockLLM:
|
|||||||
*,
|
*,
|
||||||
session_id: str | None = None,
|
session_id: str | None = None,
|
||||||
parent_call_id: str | None = None,
|
parent_call_id: str | None = None,
|
||||||
|
cache_salt: str | None = None,
|
||||||
) -> LLMResponse:
|
) -> LLMResponse:
|
||||||
content = json.dumps(
|
content = json.dumps(
|
||||||
{"action": {"tool": "submit_answer", "args": {"answer": self._answer}}}
|
{"action": {"tool": "submit_answer", "args": {"answer": self._answer}}}
|
||||||
|
|||||||
@@ -101,6 +101,7 @@ def test_end_to_end_freezes_valid_pools(tmp_path: Path) -> None:
|
|||||||
select_seed=7,
|
select_seed=7,
|
||||||
val_ratio=0.3,
|
val_ratio=0.3,
|
||||||
split_seed=7,
|
split_seed=7,
|
||||||
|
val_wrong_min=0,
|
||||||
)
|
)
|
||||||
|
|
||||||
result = build_split(
|
result = build_split(
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ class _FakeState:
|
|||||||
eval_prev_run_id: str = "run-0"
|
eval_prev_run_id: str = "run-0"
|
||||||
baseline_skills_version: str = "v1"
|
baseline_skills_version: str = "v1"
|
||||||
baseline_prompts_version: str = "v1"
|
baseline_prompts_version: str = "v1"
|
||||||
steps_since_best_improved: int = 0
|
epochs_since_best_improved: int = 0
|
||||||
epoch_start_skills: str = "v1"
|
epoch_start_skills: str = "v1"
|
||||||
changed_task_types_this_epoch: set[str] = field(default_factory=set)
|
changed_task_types_this_epoch: set[str] = field(default_factory=set)
|
||||||
rejected_buffer: dict = field(default_factory=dict)
|
rejected_buffer: dict = field(default_factory=dict)
|
||||||
@@ -111,6 +111,7 @@ class _FakeConfig:
|
|||||||
diag_size: int = 30
|
diag_size: int = 30
|
||||||
val_size: int = 50
|
val_size: int = 50
|
||||||
batch_correct_ratio: float = 0.0
|
batch_correct_ratio: float = 0.0
|
||||||
|
trainable_min_units: int = 8
|
||||||
edit_budget_start: int = 6
|
edit_budget_start: int = 6
|
||||||
edit_budget_end: int = 3
|
edit_budget_end: int = 3
|
||||||
early_stop_patience: int = 3
|
early_stop_patience: int = 3
|
||||||
|
|||||||
@@ -94,6 +94,17 @@ def _invalid_tool_json() -> str:
|
|||||||
class TestAgentLoop:
|
class TestAgentLoop:
|
||||||
"""AgentLoop 推理循环引擎测试。"""
|
"""AgentLoop 推理循环引擎测试。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_forwards_cache_salt(self) -> None:
|
||||||
|
"""AgentLoop.run(cache_salt=...) 透传到 llm.chat(算法 #10 跨 epoch 重采样)。"""
|
||||||
|
llm = AsyncMock()
|
||||||
|
llm.chat.return_value = _make_response(_submit_json())
|
||||||
|
|
||||||
|
loop = AgentLoop(llm=llm, max_steps=10)
|
||||||
|
await loop.run("system", "user", _StubDispatcher(), cache_salt="run:e2")
|
||||||
|
|
||||||
|
assert llm.chat.call_args.kwargs["cache_salt"] == "run:e2"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_submit_answer_terminates_loop(self) -> None:
|
async def test_submit_answer_terminates_loop(self) -> None:
|
||||||
"""submit_answer 终止循环 → finished, result=args, steps_used=1。"""
|
"""submit_answer 终止循环 → finished, result=args, steps_used=1。"""
|
||||||
|
|||||||
@@ -85,6 +85,33 @@ def test_null_fields_for_infra_row(tmp_path):
|
|||||||
assert rows[0].error_type is None and rows[0].evolution_target is None and rows[0].infra is True
|
assert rows[0].error_type is None and rows[0].evolution_target is None and rows[0].infra is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_done_question_ids_retry_uncertain_excludes(tmp_path):
|
||||||
|
"""retry_uncertain=True 时 uncertain 题不算完成(会被重诊),非 uncertain 仍算完成。"""
|
||||||
|
s = _store(tmp_path)
|
||||||
|
# T2(defect)题:已完成
|
||||||
|
s.upsert(
|
||||||
|
DiagnosisSignalRow(
|
||||||
|
"t2_qid", "v1", "run", "fp", "Counting",
|
||||||
|
"search_failure", "defect", "T2", "skill",
|
||||||
|
degraded=False, infra=False, session_id="s",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
# uncertain(degraded)题:信号不可信
|
||||||
|
s.upsert(
|
||||||
|
DiagnosisSignalRow(
|
||||||
|
"uncertain_qid", "v2", "run", "fp", "Counting",
|
||||||
|
None, None, "uncertain", None,
|
||||||
|
degraded=True, infra=False, session_id=None,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
# 默认:两者都算完成
|
||||||
|
assert s.done_question_ids("run", "fp") == {"t2_qid", "uncertain_qid"}
|
||||||
|
# retry_uncertain:uncertain 被排除,T2 仍算完成
|
||||||
|
done = s.done_question_ids("run", "fp", retry_uncertain=True)
|
||||||
|
assert "uncertain_qid" not in done
|
||||||
|
assert "t2_qid" in done
|
||||||
|
|
||||||
|
|
||||||
def test_context_manager_closes_connection(tmp_path):
|
def test_context_manager_closes_connection(tmp_path):
|
||||||
# with 块退出后连接关闭,再操作应报 ProgrammingError
|
# with 块退出后连接关闭,再操作应报 ProgrammingError
|
||||||
with SqliteDiagnosisSignalStore(str(tmp_path / "h.db")) as s:
|
with SqliteDiagnosisSignalStore(str(tmp_path / "h.db")) as s:
|
||||||
|
|||||||
@@ -58,6 +58,18 @@ class TestCircuitBreaker:
|
|||||||
breaker.force_open("llm", now=5.0)
|
breaker.force_open("llm", now=5.0)
|
||||||
assert breaker.is_open("llm", now=5.5) is True
|
assert breaker.is_open("llm", now=5.5) is True
|
||||||
|
|
||||||
|
def test_half_open_admits_single_probe(self) -> None:
|
||||||
|
"""冷却到期后半开只放行一个探针,第二个仍被挡;探针成功后闭合。"""
|
||||||
|
b = CircuitBreaker(fail_threshold=2, cooldown_s=10.0)
|
||||||
|
b.record_failure("p", now=0.0)
|
||||||
|
b.record_failure("p", now=0.0) # 开路至 t=10
|
||||||
|
assert b.is_open("p", now=5.0) is True # 冷却中
|
||||||
|
# 冷却到期:只放行第一个探针
|
||||||
|
assert b.is_open("p", now=11.0) is False # 探针 1 放行
|
||||||
|
assert b.is_open("p", now=11.0) is True # 探针 2 被挡(探针在途)
|
||||||
|
b.record_success("p") # 探针成功 → 闭合
|
||||||
|
assert b.is_open("p", now=12.0) is False
|
||||||
|
|
||||||
def test_force_open_probe_failure_reopens(self, breaker: CircuitBreaker) -> None:
|
def test_force_open_probe_failure_reopens(self, breaker: CircuitBreaker) -> None:
|
||||||
"""半开探针失败后重新熔断(因 fails 已置为 threshold)。"""
|
"""半开探针失败后重新熔断(因 fails 已置为 threshold)。"""
|
||||||
breaker.force_open("llm", now=0.0)
|
breaker.force_open("llm", now=0.0)
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from unittest.mock import AsyncMock, MagicMock
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from core.evolution.diagnose import (
|
from core.evolution.diagnose import (
|
||||||
|
_build_skill_case_packs,
|
||||||
_percentile,
|
_percentile,
|
||||||
_trigrams,
|
_trigrams,
|
||||||
aggregate_d2,
|
aggregate_d2,
|
||||||
@@ -43,6 +44,7 @@ from core.evolution.types import (
|
|||||||
CaseSample,
|
CaseSample,
|
||||||
DiagnosePrompts,
|
DiagnosePrompts,
|
||||||
DiagnosisResult,
|
DiagnosisResult,
|
||||||
|
ErrorAttribution,
|
||||||
QuestionMetrics,
|
QuestionMetrics,
|
||||||
SkillStepAdherence,
|
SkillStepAdherence,
|
||||||
SpanMetrics,
|
SpanMetrics,
|
||||||
@@ -750,7 +752,6 @@ class TestRunDiagnosis:
|
|||||||
defect_vs_lapse="",
|
defect_vs_lapse="",
|
||||||
reasoning_sub="",
|
reasoning_sub="",
|
||||||
span_eval_system="",
|
span_eval_system="",
|
||||||
span_eval_user="",
|
|
||||||
missed_nodes="",
|
missed_nodes="",
|
||||||
skill_adherence="",
|
skill_adherence="",
|
||||||
confirmation_bias="",
|
confirmation_bias="",
|
||||||
@@ -772,3 +773,52 @@ class TestRunDiagnosis:
|
|||||||
assert result.run_id == "run1"
|
assert result.run_id == "run1"
|
||||||
assert result.error_attributions == []
|
assert result.error_attributions == []
|
||||||
assert result.degraded_count == 0
|
assert result.degraded_count == 0
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildSkillCasePacksDegradeRouting:
|
||||||
|
"""_build_skill_case_packs 降级/未判定分流:仅 defect 且非 degraded 进正文路径。"""
|
||||||
|
|
||||||
|
def test_none_cause_and_degraded_route_to_lapse(self) -> None:
|
||||||
|
"""cause_category=None(判别失败)与 degraded 题按 lapse 处置,不进 defect 正文路径。"""
|
||||||
|
tt = "Action Reasoning"
|
||||||
|
qm_none = _make_qm(question_id="q-none", task_type=tt, correct=False, degraded=False)
|
||||||
|
qm_degraded = _make_qm(question_id="q-deg", task_type=tt, correct=False, degraded=True)
|
||||||
|
# 补两道正确题,避免退化
|
||||||
|
qm_ok1 = _make_qm(question_id="q-ok1", task_type=tt, correct=True)
|
||||||
|
qm_ok2 = _make_qm(question_id="q-ok2", task_type=tt, correct=True)
|
||||||
|
metrics = [qm_none, qm_degraded, qm_ok1, qm_ok2]
|
||||||
|
|
||||||
|
attributions = [
|
||||||
|
# 判别失败:cause_category=None
|
||||||
|
ErrorAttribution(
|
||||||
|
question_id="q-none",
|
||||||
|
error_type="reasoning",
|
||||||
|
reasoning_failure_type=None,
|
||||||
|
cause_category=None,
|
||||||
|
lapse_note="复核该类推理规则",
|
||||||
|
),
|
||||||
|
# degraded 题即便被判 defect,也须走 lapse(不驱动正文进化)
|
||||||
|
ErrorAttribution(
|
||||||
|
question_id="q-deg",
|
||||||
|
error_type="reasoning",
|
||||||
|
reasoning_failure_type=None,
|
||||||
|
cause_category="defect",
|
||||||
|
lapse_note=None,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
packs = _build_skill_case_packs(
|
||||||
|
all_metrics=metrics,
|
||||||
|
error_attributions=attributions,
|
||||||
|
traces_by_question={},
|
||||||
|
predictions=[],
|
||||||
|
d3_stats={},
|
||||||
|
d4_stats={},
|
||||||
|
)
|
||||||
|
|
||||||
|
pack = packs[tt]
|
||||||
|
failure_ids = {c.question_id for c in pack.failure_cases}
|
||||||
|
assert "q-none" not in failure_ids
|
||||||
|
assert "q-deg" not in failure_ids
|
||||||
|
# None-cause 的 lapse_note 应被收进 lapse_notes
|
||||||
|
assert any("复核该类推理规则" in n for n in pack.lapse_notes)
|
||||||
|
|||||||
@@ -344,12 +344,11 @@ def test_quadrant_classification_frozen():
|
|||||||
|
|
||||||
|
|
||||||
def test_diagnose_prompts_frozen():
|
def test_diagnose_prompts_frozen():
|
||||||
"""DiagnosePrompts 是 frozen dataclass,8 个模板字段。"""
|
"""DiagnosePrompts 是 frozen dataclass,7 个模板字段。"""
|
||||||
dp = DiagnosePrompts(
|
dp = DiagnosePrompts(
|
||||||
defect_vs_lapse="p1",
|
defect_vs_lapse="p1",
|
||||||
reasoning_sub="p2",
|
reasoning_sub="p2",
|
||||||
span_eval_system="p3",
|
span_eval_system="p3",
|
||||||
span_eval_user="p4",
|
|
||||||
missed_nodes="p5",
|
missed_nodes="p5",
|
||||||
skill_adherence="p6",
|
skill_adherence="p6",
|
||||||
confirmation_bias="p7",
|
confirmation_bias="p7",
|
||||||
@@ -361,13 +360,12 @@ def test_diagnose_prompts_frozen():
|
|||||||
|
|
||||||
|
|
||||||
def test_evolve_prompts_frozen():
|
def test_evolve_prompts_frozen():
|
||||||
"""EvolvePrompts 是 frozen dataclass,5 个模板字段。"""
|
"""EvolvePrompts 是 frozen dataclass,4 个模板字段。"""
|
||||||
ep = EvolvePrompts(
|
ep = EvolvePrompts(
|
||||||
evolve_skill="skill_tmpl",
|
evolve_skill="skill_tmpl",
|
||||||
evolve_system="system_tmpl",
|
evolve_system="system_tmpl",
|
||||||
evolve_tool="tool_tmpl",
|
evolve_tool="tool_tmpl",
|
||||||
evolve_rank="rank_tmpl",
|
evolve_rank="rank_tmpl",
|
||||||
consolidate_system="consolidate_tmpl",
|
|
||||||
)
|
)
|
||||||
assert ep.evolve_rank == "rank_tmpl"
|
assert ep.evolve_rank == "rank_tmpl"
|
||||||
with pytest.raises(AttributeError):
|
with pytest.raises(AttributeError):
|
||||||
|
|||||||
@@ -309,6 +309,36 @@ class TestValidateSkill:
|
|||||||
result = validate_skill(orig, "no frontmatter body")
|
result = validate_skill(orig, "no frontmatter body")
|
||||||
assert not result.passed
|
assert not result.passed
|
||||||
|
|
||||||
|
def test_unpaired_appendix_marker_fails(self) -> None:
|
||||||
|
"""evolved 出现孤立 APPENDIX_START(无配对 END)→ marker 完整性校验失败。"""
|
||||||
|
orig = "---\nname: a\ndescription: d\ntask_type: t\n---\nbody body body"
|
||||||
|
evol = f"---\nname: a\ndescription: d\ntask_type: t\n---\nbody {APPENDIX_START} body"
|
||||||
|
result = validate_skill(orig, evol)
|
||||||
|
assert not result.passed
|
||||||
|
assert any("marker" in e.lower() or "配对" in e for e in result.errors)
|
||||||
|
|
||||||
|
def test_duplicate_momentum_marker_fails(self) -> None:
|
||||||
|
"""evolved 出现两对 MOMENTUM marker → 完整性校验失败(各至多一对)。"""
|
||||||
|
orig = "---\nname: a\ndescription: d\ntask_type: t\n---\nbody body body"
|
||||||
|
evol = (
|
||||||
|
"---\nname: a\ndescription: d\ntask_type: t\n---\n"
|
||||||
|
f"{MOMENTUM_START}x{MOMENTUM_END} mid {MOMENTUM_START}y{MOMENTUM_END}"
|
||||||
|
)
|
||||||
|
result = validate_skill(orig, evol)
|
||||||
|
assert not result.passed
|
||||||
|
assert any("marker" in e.lower() or "配对" in e for e in result.errors)
|
||||||
|
|
||||||
|
def test_paired_markers_pass(self) -> None:
|
||||||
|
"""evolved 含成对 appendix+momentum marker(各一对,顺序正确)→ 校验通过。"""
|
||||||
|
body = "body " * 20
|
||||||
|
orig = f"---\nname: a\ndescription: d\ntask_type: t\n---\n{body}"
|
||||||
|
evol = (
|
||||||
|
f"---\nname: a\ndescription: d\ntask_type: t\n---\n{body}"
|
||||||
|
f"{APPENDIX_START}\n- n\n{APPENDIX_END}\n{MOMENTUM_START}\nm\n{MOMENTUM_END}"
|
||||||
|
)
|
||||||
|
result = validate_skill(orig, evol)
|
||||||
|
assert result.passed, result.errors
|
||||||
|
|
||||||
|
|
||||||
class TestValidateSystem:
|
class TestValidateSystem:
|
||||||
"""validate_system 测试。"""
|
"""validate_system 测试。"""
|
||||||
@@ -681,7 +711,6 @@ _PROMPTS = EvolvePrompts(
|
|||||||
evolve_system="sys",
|
evolve_system="sys",
|
||||||
evolve_tool="tool",
|
evolve_tool="tool",
|
||||||
evolve_rank="rank",
|
evolve_rank="rank",
|
||||||
consolidate_system="cons",
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
"""校验 5 个进化/动量模板存在且输出契约关键词与解析代码对齐。"""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
_PROMPTS_DIR = Path("prompts")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"name, required_tokens",
|
||||||
|
[
|
||||||
|
("evolve_skill.md", ["suggestions", "edits"]),
|
||||||
|
("evolve_system.md", ["suggestions", "edits"]),
|
||||||
|
("evolve_tool.md", ["edits_extract", "edits_verify"]),
|
||||||
|
("evolve_rank.md", ["selected_indices"]),
|
||||||
|
("slow_momentum.md", ["slow_update_content"]),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_evolve_template_present_and_contract(name: str, required_tokens: list[str]) -> None:
|
||||||
|
path = _PROMPTS_DIR / name
|
||||||
|
assert path.exists(), f"缺模板: {path}"
|
||||||
|
text = path.read_text(encoding="utf-8")
|
||||||
|
assert text.strip(), f"模板为空: {path}"
|
||||||
|
for token in required_tokens:
|
||||||
|
assert token in text, f"{name} 缺输出契约关键词 {token!r}(与解析代码不对齐)"
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_split_loader_fail_loud_on_missing(tmp_path, monkeypatch):
|
||||||
|
"""video_split_cli 的真实 diagnose 加载器缺模板必须 FileNotFoundError。"""
|
||||||
|
monkeypatch.chdir(tmp_path) # 空目录,无 prompts/*.md
|
||||||
|
from app.harness.video_split_cli import _load_diagnose_prompts
|
||||||
|
|
||||||
|
with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"):
|
||||||
|
_load_diagnose_prompts()
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_evolve_loader_fail_loud_on_missing(tmp_path, monkeypatch):
|
||||||
|
"""runner 的真实 evolve 加载器缺模板必须 FileNotFoundError。"""
|
||||||
|
monkeypatch.chdir(tmp_path)
|
||||||
|
from app.harness.runner import Runner
|
||||||
|
|
||||||
|
r = object.__new__(Runner) # 绕过 __init__,仅测无状态加载器方法
|
||||||
|
with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"):
|
||||||
|
r._load_evolve_prompts()
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_diagnose_loader_fail_loud_on_missing(tmp_path, monkeypatch):
|
||||||
|
"""runner 的真实 diagnose 加载器缺模板必须 FileNotFoundError。"""
|
||||||
|
monkeypatch.chdir(tmp_path)
|
||||||
|
from app.harness.runner import Runner
|
||||||
|
|
||||||
|
r = object.__new__(Runner)
|
||||||
|
with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"):
|
||||||
|
r._load_diagnose_prompts()
|
||||||
@@ -23,8 +23,13 @@ class _FakeRedisCache:
|
|||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self._store: dict[str, LLMResponse] = {}
|
self._store: dict[str, LLMResponse] = {}
|
||||||
|
|
||||||
async def get(self, model: str, messages: list[dict[str, str]]) -> LLMResponse | None:
|
async def get(
|
||||||
key = f"{model}:{json.dumps(messages, sort_keys=True)}"
|
self,
|
||||||
|
model: str,
|
||||||
|
messages: list[dict[str, str]],
|
||||||
|
cache_salt: str | None = None,
|
||||||
|
) -> LLMResponse | None:
|
||||||
|
key = f"{model}:{cache_salt}:{json.dumps(messages, sort_keys=True)}"
|
||||||
return self._store.get(key)
|
return self._store.get(key)
|
||||||
|
|
||||||
async def set(
|
async def set(
|
||||||
@@ -32,8 +37,9 @@ class _FakeRedisCache:
|
|||||||
model: str,
|
model: str,
|
||||||
messages: list[dict[str, str]],
|
messages: list[dict[str, str]],
|
||||||
response: LLMResponse,
|
response: LLMResponse,
|
||||||
|
cache_salt: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
key = f"{model}:{json.dumps(messages, sort_keys=True)}"
|
key = f"{model}:{cache_salt}:{json.dumps(messages, sort_keys=True)}"
|
||||||
self._store[key] = response
|
self._store[key] = response
|
||||||
|
|
||||||
|
|
||||||
@@ -253,6 +259,43 @@ async def test_qwen_thinking_stripped():
|
|||||||
assert thinking2 == ""
|
assert thinking2 == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_transient_covers_disconnect_family():
|
||||||
|
"""瞬时错误清单覆盖断连族(RemoteProtocolError/ReadError/ConnectTimeout/PoolTimeout)。"""
|
||||||
|
from adapters.llm import _is_transient_error
|
||||||
|
|
||||||
|
assert _is_transient_error(httpx.RemoteProtocolError("peer reset"))
|
||||||
|
assert _is_transient_error(httpx.ReadError("read"))
|
||||||
|
assert _is_transient_error(httpx.ConnectTimeout("ct"))
|
||||||
|
assert _is_transient_error(httpx.PoolTimeout("pt"))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_truncated_stream_without_done_raises():
|
||||||
|
"""SSE 流耗尽但未收 [DONE] → _SseAnomaly(进重试,不当成功)。"""
|
||||||
|
from adapters.llm import _SseAnomaly
|
||||||
|
|
||||||
|
async def _lines():
|
||||||
|
yield 'data: {"choices":[{"delta":{"content":"半"}}]}'
|
||||||
|
# 无 data: [DONE] —— 模拟服务端截断
|
||||||
|
|
||||||
|
client = _build_client()
|
||||||
|
with pytest.raises(_SseAnomaly):
|
||||||
|
await client._consume_stream(_lines())
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_complete_stream_with_done_ok():
|
||||||
|
"""正常带 [DONE] 的流不抛异常,正确累积 content。"""
|
||||||
|
|
||||||
|
async def _lines():
|
||||||
|
yield 'data: {"choices":[{"delta":{"content":"完整"}}]}'
|
||||||
|
yield "data: [DONE]"
|
||||||
|
|
||||||
|
client = _build_client()
|
||||||
|
content, _thinking, _ttft, _gap, _usage = await client._consume_stream(_lines())
|
||||||
|
assert content == "完整"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_parent_call_id_forwarded_to_telemetry():
|
async def test_parent_call_id_forwarded_to_telemetry():
|
||||||
"""parent_call_id 和 session_id 正确传递到遥测记录。"""
|
"""parent_call_id 和 session_id 正确传递到遥测记录。"""
|
||||||
|
|||||||
@@ -116,7 +116,7 @@ class _FakeState:
|
|||||||
eval_prev_run_id: str
|
eval_prev_run_id: str
|
||||||
baseline_skills_version: str
|
baseline_skills_version: str
|
||||||
baseline_prompts_version: str
|
baseline_prompts_version: str
|
||||||
steps_since_best_improved: int
|
epochs_since_best_improved: int
|
||||||
epoch_start_skills: str
|
epoch_start_skills: str
|
||||||
changed_task_types_this_epoch: set[str]
|
changed_task_types_this_epoch: set[str]
|
||||||
rejected_buffer: dict[str, list[RejectedEdit]]
|
rejected_buffer: dict[str, list[RejectedEdit]]
|
||||||
@@ -135,7 +135,7 @@ def _make_state() -> _FakeState:
|
|||||||
eval_prev_run_id="run-abc",
|
eval_prev_run_id="run-abc",
|
||||||
baseline_skills_version="v1",
|
baseline_skills_version="v1",
|
||||||
baseline_prompts_version="v1",
|
baseline_prompts_version="v1",
|
||||||
steps_since_best_improved=2,
|
epochs_since_best_improved=2,
|
||||||
epoch_start_skills="v1",
|
epoch_start_skills="v1",
|
||||||
changed_task_types_this_epoch={"temporal", "causal"},
|
changed_task_types_this_epoch={"temporal", "causal"},
|
||||||
rejected_buffer={"temporal": [_make_rejected_edit()]},
|
rejected_buffer={"temporal": [_make_rejected_edit()]},
|
||||||
@@ -157,6 +157,7 @@ class _FakeConfig:
|
|||||||
diag_size: int = 30
|
diag_size: int = 30
|
||||||
val_size: int = 50
|
val_size: int = 50
|
||||||
batch_correct_ratio: float = 0.5
|
batch_correct_ratio: float = 0.5
|
||||||
|
trainable_min_units: int = 8
|
||||||
edit_budget_start: int = 6
|
edit_budget_start: int = 6
|
||||||
edit_budget_end: int = 3
|
edit_budget_end: int = 3
|
||||||
early_stop_patience: int = 3
|
early_stop_patience: int = 3
|
||||||
@@ -201,7 +202,7 @@ class TestSerializeDeserializeRoundtrip:
|
|||||||
assert restored["eval_prev_run_id"] == state.eval_prev_run_id
|
assert restored["eval_prev_run_id"] == state.eval_prev_run_id
|
||||||
assert restored["baseline_skills_version"] == state.baseline_skills_version
|
assert restored["baseline_skills_version"] == state.baseline_skills_version
|
||||||
assert restored["baseline_prompts_version"] == state.baseline_prompts_version
|
assert restored["baseline_prompts_version"] == state.baseline_prompts_version
|
||||||
assert restored["steps_since_best_improved"] == state.steps_since_best_improved
|
assert restored["epochs_since_best_improved"] == state.epochs_since_best_improved
|
||||||
assert restored["epoch_start_skills"] == state.epoch_start_skills
|
assert restored["epoch_start_skills"] == state.epoch_start_skills
|
||||||
assert restored["changed_task_types_this_epoch"] == state.changed_task_types_this_epoch
|
assert restored["changed_task_types_this_epoch"] == state.changed_task_types_this_epoch
|
||||||
assert restored["gate_cooldown"] == state.gate_cooldown
|
assert restored["gate_cooldown"] == state.gate_cooldown
|
||||||
@@ -289,6 +290,7 @@ class TestFingerprintStructuralVsDecision:
|
|||||||
"diag_size",
|
"diag_size",
|
||||||
"val_size",
|
"val_size",
|
||||||
"batch_correct_ratio",
|
"batch_correct_ratio",
|
||||||
|
"trainable_min_units",
|
||||||
}
|
}
|
||||||
decision = {
|
decision = {
|
||||||
"edit_budget_start",
|
"edit_budget_start",
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ def _valid_kwargs() -> dict:
|
|||||||
"batch_size": 15,
|
"batch_size": 15,
|
||||||
"min_class_per_batch": 2,
|
"min_class_per_batch": 2,
|
||||||
"eval_min_per_class": 2,
|
"eval_min_per_class": 2,
|
||||||
|
"trainable_min_units": 8,
|
||||||
"early_stop_patience": 8,
|
"early_stop_patience": 8,
|
||||||
"test_size": 60,
|
"test_size": 60,
|
||||||
"use_slow_momentum": True,
|
"use_slow_momentum": True,
|
||||||
|
|||||||
@@ -559,6 +559,65 @@ class TestPredictionAlwaysWritten:
|
|||||||
assert rows[0]["prediction"] is None
|
assert rows[0]["prediction"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def _make_nonscalar_llm_response() -> LLMResponse:
|
||||||
|
"""构造 submit_answer 提交非标量 answer(list)的 LLMResponse。"""
|
||||||
|
content = json.dumps(
|
||||||
|
{
|
||||||
|
"reflect": {"observation": "找到答案"},
|
||||||
|
"plan": {"next_step": "提交"},
|
||||||
|
"action": {
|
||||||
|
"tool": "submit_answer",
|
||||||
|
"args": {
|
||||||
|
"answer": ["B"],
|
||||||
|
"evidence": "证据文本",
|
||||||
|
"reasoning": "推理过程",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return LLMResponse(
|
||||||
|
content=content,
|
||||||
|
thinking="思考过程",
|
||||||
|
model="test-model",
|
||||||
|
provider="test",
|
||||||
|
prompt_tokens=100,
|
||||||
|
completion_tokens=50,
|
||||||
|
latency_ms=200,
|
||||||
|
ttft_ms=30.0,
|
||||||
|
max_inter_token_ms=5.0,
|
||||||
|
cache_hit=False,
|
||||||
|
call_id="test-call-nonscalar",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestNonScalarPrediction:
|
||||||
|
"""非标量 prediction 归一化 + 落库加固测试。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_nonscalar_prediction_does_not_crash(self, harness_log: HarnessLog) -> None:
|
||||||
|
"""submit_answer 返回 {'answer': ['B']} 时归一化落库,不抛 sqlite 绑定异常。"""
|
||||||
|
llm = AsyncMock()
|
||||||
|
llm.chat.return_value = _make_nonscalar_llm_response()
|
||||||
|
|
||||||
|
result = await run_inference(
|
||||||
|
[_make_question(answer="B")],
|
||||||
|
llm=llm,
|
||||||
|
tool_dispatch_fn=_stub_tool_dispatch,
|
||||||
|
prompt_builder=_stub_prompt_builder,
|
||||||
|
log=harness_log,
|
||||||
|
run_id="run-nonscalar",
|
||||||
|
concurrency=1,
|
||||||
|
max_steps=10,
|
||||||
|
skill_mode="auto",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.total == 1
|
||||||
|
rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("test-run",))
|
||||||
|
assert len(rows) == 1
|
||||||
|
# prediction 被 JSON 序列化为字符串,不再是 Python list
|
||||||
|
assert rows[0]["prediction"] == '["B"]'
|
||||||
|
|
||||||
|
|
||||||
class TestPluginsFactory:
|
class TestPluginsFactory:
|
||||||
"""plugins_factory 调用测试。"""
|
"""plugins_factory 调用测试。"""
|
||||||
|
|
||||||
@@ -639,6 +698,7 @@ class TestConcurrencyControl:
|
|||||||
*,
|
*,
|
||||||
session_id: str | None = None,
|
session_id: str | None = None,
|
||||||
parent_call_id: str | None = None,
|
parent_call_id: str | None = None,
|
||||||
|
cache_salt: str | None = None,
|
||||||
) -> LLMResponse:
|
) -> LLMResponse:
|
||||||
nonlocal current_concurrent, max_concurrent
|
nonlocal current_concurrent, max_concurrent
|
||||||
current_concurrent += 1
|
current_concurrent += 1
|
||||||
|
|||||||
@@ -215,6 +215,30 @@ class TestHarnessLogUpsert:
|
|||||||
rows = log3.query("SELECT COUNT(*) as cnt FROM _runs WHERE run_id='run_1'")
|
rows = log3.query("SELECT COUNT(*) as cnt FROM _runs WHERE run_id='run_1'")
|
||||||
assert rows[0]["cnt"] == 1
|
assert rows[0]["cnt"] == 1
|
||||||
|
|
||||||
|
def test_register_run_false_skips_upsert(self, tmp_path: Path) -> None:
|
||||||
|
"""register_run=False 时只读打开不改写已有 _runs 行(started_at/status 不变)。"""
|
||||||
|
db = str(tmp_path / "h.db")
|
||||||
|
|
||||||
|
def _read_run_row(run_id: str) -> dict:
|
||||||
|
# 用 register_run=False 只读,避免读取本身污染 _runs
|
||||||
|
with HarnessLog(db, run_id, register_run=False) as log:
|
||||||
|
rows = log.query(
|
||||||
|
"SELECT started_at, status FROM _runs WHERE run_id=?", (run_id,)
|
||||||
|
)
|
||||||
|
return rows[0]
|
||||||
|
|
||||||
|
with HarnessLog(db, "r1"):
|
||||||
|
pass # 初次注册 + 正常退出置 completed
|
||||||
|
row0 = _read_run_row("r1")
|
||||||
|
|
||||||
|
time.sleep(0.02)
|
||||||
|
with HarnessLog(db, "r1", register_run=False) as log:
|
||||||
|
log.query("SELECT 1") # 只读
|
||||||
|
row1 = _read_run_row("r1")
|
||||||
|
|
||||||
|
assert row1["started_at"] == row0["started_at"]
|
||||||
|
assert row1["status"] == row0["status"]
|
||||||
|
|
||||||
|
|
||||||
# ===========================================================================
|
# ===========================================================================
|
||||||
# RunLogImpl 测试
|
# RunLogImpl 测试
|
||||||
|
|||||||
@@ -71,6 +71,37 @@ def test_write_read_dual_metric(db_path: str, run_id: str) -> None:
|
|||||||
assert row["run_id"] == run_id
|
assert row["run_id"] == run_id
|
||||||
|
|
||||||
|
|
||||||
|
def test_dual_metric_kinds_distinguishable(db_path: str, run_id: str) -> None:
|
||||||
|
"""observation 层接受任意 version_kind:final 与 slow_candidate 可区分共存。"""
|
||||||
|
write_dual_metric(
|
||||||
|
db_path,
|
||||||
|
run_id=run_id,
|
||||||
|
epoch=1,
|
||||||
|
version_kind="final",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
pool="val",
|
||||||
|
hard_acc=0.7,
|
||||||
|
soft_score=None,
|
||||||
|
mixed_score=None,
|
||||||
|
)
|
||||||
|
write_dual_metric(
|
||||||
|
db_path,
|
||||||
|
run_id=run_id,
|
||||||
|
epoch=1,
|
||||||
|
version_kind="slow_candidate",
|
||||||
|
skills_version="v2",
|
||||||
|
prompts_version="v2",
|
||||||
|
pool="val",
|
||||||
|
hard_acc=0.6,
|
||||||
|
soft_score=None,
|
||||||
|
mixed_score=None,
|
||||||
|
)
|
||||||
|
rows = read_dual_metric(db_path, run_id=run_id)
|
||||||
|
kinds = {row["version_kind"] for row in rows if row["epoch"] == 1}
|
||||||
|
assert kinds == {"final", "slow_candidate"}
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# shadow_gate
|
# shadow_gate
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import pytest
|
|||||||
from app.harness.pools import (
|
from app.harness.pools import (
|
||||||
GlobalPoolStrategy,
|
GlobalPoolStrategy,
|
||||||
PerCategoryPoolStrategy,
|
PerCategoryPoolStrategy,
|
||||||
|
build_or_load_pools,
|
||||||
build_pools,
|
build_pools,
|
||||||
load_pools,
|
load_pools,
|
||||||
save_pools,
|
save_pools,
|
||||||
@@ -291,6 +292,77 @@ class TestBuildOrLoadPoolsFrozen:
|
|||||||
load_ids = [q.question_id for q in getattr(loaded, pool_name)]
|
load_ids = [q.question_id for q in getattr(loaded, pool_name)]
|
||||||
assert orig_ids == load_ids, f"{pool_name} 冻结后 ID 顺序不一致"
|
assert orig_ids == load_ids, f"{pool_name} 冻结后 ID 顺序不一致"
|
||||||
|
|
||||||
|
def _run_config_for_frozen(self, tmp_path: Path, seed_name: str) -> object:
|
||||||
|
"""构造指向 tmp workspace/store + 指定种子名的最小 train RunConfig。"""
|
||||||
|
from app.harness.config import RunConfig
|
||||||
|
|
||||||
|
return RunConfig(
|
||||||
|
workspace_dir=tmp_path / "ws",
|
||||||
|
store_dir=tmp_path / "store",
|
||||||
|
mode="train",
|
||||||
|
concurrency=4,
|
||||||
|
max_steps=10,
|
||||||
|
skill_mode="auto",
|
||||||
|
n_samples=0,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
epochs=1,
|
||||||
|
diag_size=10,
|
||||||
|
diag_correct_ratio=0.5,
|
||||||
|
val_size=10,
|
||||||
|
val_correct_ratio=0.5,
|
||||||
|
edit_budget_start=5,
|
||||||
|
edit_budget_end=2,
|
||||||
|
batch_size=15,
|
||||||
|
min_class_per_batch=2,
|
||||||
|
eval_min_per_class=2,
|
||||||
|
trainable_min_units=8,
|
||||||
|
early_stop_patience=4,
|
||||||
|
test_size=10,
|
||||||
|
use_slow_momentum=True,
|
||||||
|
gate_e_confirm=20.0,
|
||||||
|
gate_e_provisional=3.0,
|
||||||
|
gate_w_net_min=2,
|
||||||
|
gate_delta_min=0.02,
|
||||||
|
gate_lambda_dir=-0.642,
|
||||||
|
gate_e_rollback=10.0,
|
||||||
|
gate_block=8,
|
||||||
|
gate_n_max=40,
|
||||||
|
gate_p_low=0.05,
|
||||||
|
gate_p_high=0.95,
|
||||||
|
gate_probe_quota=0.2,
|
||||||
|
gate_gamma_decay=0.9,
|
||||||
|
gate_cooldown_steps=2,
|
||||||
|
gate_guard_err=0.10,
|
||||||
|
skill_update_mode="patch",
|
||||||
|
appendix_consolidate_threshold=6,
|
||||||
|
fresh=True,
|
||||||
|
seed=seed_name,
|
||||||
|
test_questions="", # 绕过 _to_pool_config 的 resolve_paths(manifest 依赖)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_global_frozen_rejects_baseline_mismatch(self, tmp_path: Path) -> None:
|
||||||
|
"""global 冻结 pools 的 baseline_run_id 与 seed 不符时 fail-loud。"""
|
||||||
|
seed_name = "myseed"
|
||||||
|
seed_dir = tmp_path / "store" / "seeds" / seed_name
|
||||||
|
seed_dir.mkdir(parents=True)
|
||||||
|
(seed_dir / "seed.json").write_text(
|
||||||
|
json.dumps({"baseline_run_id": "infer_adhoc", "parent": None})
|
||||||
|
)
|
||||||
|
|
||||||
|
ws = tmp_path / "ws"
|
||||||
|
ws.mkdir()
|
||||||
|
(ws / "pools.json").write_text(
|
||||||
|
json.dumps({"split_mode": "global", "baseline_run_id": "other"}),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
config = self._run_config_for_frozen(tmp_path, seed_name)
|
||||||
|
strategy = GlobalPoolStrategy()
|
||||||
|
with pytest.raises(ValueError, match="baseline_run_id"):
|
||||||
|
build_or_load_pools(config, strategy, tmp_path / "nonexistent.db")
|
||||||
|
|
||||||
|
|
||||||
class TestGlobalPoolStrategy:
|
class TestGlobalPoolStrategy:
|
||||||
"""GlobalPoolStrategy 封装现有全局三分逻辑。"""
|
"""GlobalPoolStrategy 封装现有全局三分逻辑。"""
|
||||||
@@ -799,6 +871,7 @@ class TestRunHoldoutEvalConfig:
|
|||||||
batch_size=15,
|
batch_size=15,
|
||||||
min_class_per_batch=2,
|
min_class_per_batch=2,
|
||||||
eval_min_per_class=2,
|
eval_min_per_class=2,
|
||||||
|
trainable_min_units=8,
|
||||||
early_stop_patience=4,
|
early_stop_patience=4,
|
||||||
test_size=30,
|
test_size=30,
|
||||||
use_slow_momentum=True,
|
use_slow_momentum=True,
|
||||||
@@ -849,6 +922,7 @@ class TestRunHoldoutEvalConfig:
|
|||||||
batch_size=15,
|
batch_size=15,
|
||||||
min_class_per_batch=2,
|
min_class_per_batch=2,
|
||||||
eval_min_per_class=2,
|
eval_min_per_class=2,
|
||||||
|
trainable_min_units=8,
|
||||||
early_stop_patience=4,
|
early_stop_patience=4,
|
||||||
test_size=30,
|
test_size=30,
|
||||||
use_slow_momentum=True,
|
use_slow_momentum=True,
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from app.harness.runner import (
|
|||||||
_build_comparison_pairs,
|
_build_comparison_pairs,
|
||||||
_compute_total_steps,
|
_compute_total_steps,
|
||||||
_fallback_summary,
|
_fallback_summary,
|
||||||
|
_filter_untrainable_types,
|
||||||
_format_applied_edits,
|
_format_applied_edits,
|
||||||
_guard_infra_failures,
|
_guard_infra_failures,
|
||||||
_outcome_to_quadrant_pairs,
|
_outcome_to_quadrant_pairs,
|
||||||
@@ -74,6 +75,26 @@ class _FakeQuestion:
|
|||||||
object.__setattr__(self, "unit_id", self.pair_id or self.question_id)
|
object.__setattr__(self, "unit_id", self.pair_id or self.question_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_pair(pair_id: str, task_type: str, video_id: str = "v1") -> list[_FakeQuestion]:
|
||||||
|
"""构造合法孪生对(original + mirror),共享 pair_id/video_id/task_type/flip_axis。"""
|
||||||
|
return [
|
||||||
|
_FakeQuestion(
|
||||||
|
question_id=f"{pair_id}-o",
|
||||||
|
video_id=video_id,
|
||||||
|
task_type=task_type,
|
||||||
|
pair_id=pair_id,
|
||||||
|
question_role="pair_original",
|
||||||
|
),
|
||||||
|
_FakeQuestion(
|
||||||
|
question_id=f"{pair_id}-m",
|
||||||
|
video_id=video_id,
|
||||||
|
task_type=task_type,
|
||||||
|
pair_id=pair_id,
|
||||||
|
question_role="pair_mirror",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class _FakePools:
|
class _FakePools:
|
||||||
"""Pools 替身。"""
|
"""Pools 替身。"""
|
||||||
@@ -349,92 +370,175 @@ class TestBuildComparisonPairs:
|
|||||||
# =========================================================================
|
# =========================================================================
|
||||||
|
|
||||||
|
|
||||||
|
def _write_manifest_with_best(tmp_path: Path, best_epoch: int) -> None:
|
||||||
|
"""写含 best.epoch 的 manifest,供 _should_early_stop 读 read_best。"""
|
||||||
|
manifest = {
|
||||||
|
"name": "test",
|
||||||
|
"store": ".",
|
||||||
|
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
|
||||||
|
"best": {"epoch": best_epoch, "val_acc": 0.5},
|
||||||
|
"history": [],
|
||||||
|
}
|
||||||
|
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
||||||
|
|
||||||
|
|
||||||
class TestShouldEarlyStop:
|
class TestShouldEarlyStop:
|
||||||
"""_should_early_stop 步粒度 early stop。"""
|
"""_should_early_stop epoch 粒度 early stop(patience 以 epoch 计)。"""
|
||||||
|
|
||||||
def test_improved_this_epoch_resets(self, tmp_path: Path) -> None:
|
def test_improved_this_epoch_resets(self, tmp_path: Path) -> None:
|
||||||
"""本 epoch best 刷新时重置计数器。"""
|
"""本 epoch best 刷新时重置计数器。"""
|
||||||
# 写 manifest + best
|
_write_manifest_with_best(tmp_path, best_epoch=2)
|
||||||
manifest = {
|
|
||||||
"name": "test",
|
|
||||||
"store": ".",
|
|
||||||
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
|
|
||||||
"best": {"epoch": 2, "val_acc": 0.9},
|
|
||||||
"history": [],
|
|
||||||
}
|
|
||||||
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
|
||||||
|
|
||||||
state = MagicMock()
|
state = MagicMock()
|
||||||
state.steps_since_best_improved = 10
|
state.epochs_since_best_improved = 3
|
||||||
|
|
||||||
result = _should_early_stop(tmp_path, epoch=2, steps_this_epoch=5, state=state, patience=20)
|
result = _should_early_stop(tmp_path, epoch=2, state=state, patience=2)
|
||||||
assert result is False
|
assert result is False
|
||||||
assert state.steps_since_best_improved == 0
|
assert state.epochs_since_best_improved == 0
|
||||||
|
|
||||||
def test_no_improvement_accumulates(self, tmp_path: Path) -> None:
|
|
||||||
"""未刷新时累加步数。"""
|
|
||||||
manifest = {
|
|
||||||
"name": "test",
|
|
||||||
"store": ".",
|
|
||||||
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
|
|
||||||
"best": {"epoch": 1, "val_acc": 0.5},
|
|
||||||
"history": [],
|
|
||||||
}
|
|
||||||
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
|
||||||
|
|
||||||
state = MagicMock()
|
|
||||||
state.steps_since_best_improved = 15
|
|
||||||
|
|
||||||
result = _should_early_stop(tmp_path, epoch=3, steps_this_epoch=5, state=state, patience=20)
|
|
||||||
assert result is True # 15 + 5 = 20 >= 20
|
|
||||||
assert state.steps_since_best_improved == 20
|
|
||||||
|
|
||||||
def test_below_patience_continues(self, tmp_path: Path) -> None:
|
def test_below_patience_continues(self, tmp_path: Path) -> None:
|
||||||
"""累计步数未达阈值时继续。"""
|
"""未达 patience 个 epoch 无刷新时继续。"""
|
||||||
manifest = {
|
_write_manifest_with_best(tmp_path, best_epoch=1)
|
||||||
"name": "test",
|
|
||||||
"store": ".",
|
|
||||||
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
|
|
||||||
"best": {"epoch": 1, "val_acc": 0.5},
|
|
||||||
"history": [],
|
|
||||||
}
|
|
||||||
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
|
||||||
|
|
||||||
state = MagicMock()
|
state = MagicMock()
|
||||||
state.steps_since_best_improved = 10
|
state.epochs_since_best_improved = 0
|
||||||
|
|
||||||
result = _should_early_stop(tmp_path, epoch=3, steps_this_epoch=5, state=state, patience=20)
|
result = _should_early_stop(tmp_path, epoch=2, state=state, patience=3)
|
||||||
assert result is False
|
assert result is False
|
||||||
assert state.steps_since_best_improved == 15
|
assert state.epochs_since_best_improved == 1
|
||||||
|
|
||||||
def test_step_granularity(self, tmp_path: Path) -> None:
|
def test_early_stop_counts_epochs_not_steps(self, tmp_path: Path) -> None:
|
||||||
"""步粒度而非 epoch 粒度。"""
|
"""patience=2 表示连续 2 个 epoch 无 best 刷新才停(不是步数)。"""
|
||||||
manifest = {
|
_write_manifest_with_best(tmp_path, best_epoch=1)
|
||||||
"name": "test",
|
|
||||||
"store": ".",
|
|
||||||
"current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"},
|
|
||||||
"best": {"epoch": 1, "val_acc": 0.5},
|
|
||||||
"history": [],
|
|
||||||
}
|
|
||||||
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
|
|
||||||
|
|
||||||
state = MagicMock()
|
state = MagicMock()
|
||||||
# 连续 3 个 epoch,每个 3 步
|
state.epochs_since_best_improved = 0
|
||||||
state.steps_since_best_improved = 0
|
|
||||||
for ep in range(2, 5):
|
# epoch 2 无刷新 → 1 → 不停
|
||||||
stopped = _should_early_stop(
|
assert _should_early_stop(tmp_path, epoch=2, state=state, patience=2) is False
|
||||||
tmp_path, epoch=ep, steps_this_epoch=3, state=state, patience=10
|
assert state.epochs_since_best_improved == 1
|
||||||
)
|
# epoch 3 无刷新 → 2 → 停
|
||||||
if ep < 4:
|
assert _should_early_stop(tmp_path, epoch=3, state=state, patience=2) is True
|
||||||
assert stopped is False
|
assert state.epochs_since_best_improved == 2
|
||||||
else:
|
|
||||||
# 3+3+3=9 < 10 但第三轮后 9+3=12>=10 在 ep=5 触发
|
|
||||||
# 实际:ep=2 → 3, ep=3 → 6, ep=4 → 9
|
class TestFilterUntrainableTypes:
|
||||||
assert stopped is False
|
"""_filter_untrainable_types 可训练性预检纯函数。"""
|
||||||
stopped = _should_early_stop(
|
|
||||||
tmp_path, epoch=5, steps_this_epoch=3, state=state, patience=10
|
def test_untrainable_types_filtered_before_gate(self) -> None:
|
||||||
|
"""val<eval_min_per_class 或 非test单元<trainable_min_units 的题型被剔除。"""
|
||||||
|
# 题型 A:diag=5 + val=5 → units=10、val=5,可训
|
||||||
|
diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(5)]
|
||||||
|
val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(5)]
|
||||||
|
# 题型 B:val=0(<eval_min_per_class)不可训
|
||||||
|
diag += [_FakeQuestion(question_id=f"B-d{i}", task_type="B") for i in range(2)]
|
||||||
|
pools = _FakePools(diagnosis=diag, validation=val, test=[])
|
||||||
|
|
||||||
|
new_pools, new_types = _filter_untrainable_types(
|
||||||
|
pools,
|
||||||
|
task_types=["A", "B"],
|
||||||
|
eval_min_per_class=2,
|
||||||
|
trainable_min_units=8,
|
||||||
)
|
)
|
||||||
assert stopped is True # 9+3=12>=10
|
|
||||||
|
assert {q.task_type for q in new_pools.diagnosis} == {"A"}
|
||||||
|
assert {q.task_type for q in new_pools.validation} == {"A"}
|
||||||
|
assert new_types == ["A"]
|
||||||
|
|
||||||
|
def test_units_below_threshold_filtered(self) -> None:
|
||||||
|
"""val 达标但 diag+val 单元数 < trainable_min_units 的题型被剔除(保留另一可训题型)。"""
|
||||||
|
# 可训题型 A:diag=8 + val=2 → units=10、val=2,保留
|
||||||
|
keep_diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)]
|
||||||
|
keep_val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(2)]
|
||||||
|
# 题型 C:val=2(>=2)但 units=2+1=3 < 8 → 剔除
|
||||||
|
val = keep_val + [_FakeQuestion(question_id=f"C-v{i}", task_type="C") for i in range(2)]
|
||||||
|
diag = keep_diag + [_FakeQuestion(question_id="C-d0", task_type="C")]
|
||||||
|
pools = _FakePools(diagnosis=diag, validation=val, test=[])
|
||||||
|
|
||||||
|
new_pools, new_types = _filter_untrainable_types(
|
||||||
|
pools, task_types=None, eval_min_per_class=2, trainable_min_units=8
|
||||||
|
)
|
||||||
|
|
||||||
|
assert {q.task_type for q in new_pools.diagnosis} == {"A"}
|
||||||
|
assert {q.task_type for q in new_pools.validation} == {"A"}
|
||||||
|
assert new_types == ["A"]
|
||||||
|
|
||||||
|
def test_ar_pair_counted_as_units_not_questions(self) -> None:
|
||||||
|
"""AR pair 按单元折叠计数:题目数达标但单元数不足的题型仍被剔除。"""
|
||||||
|
# 可训题型 A:diag=8 + val=2 single → units=10,保留
|
||||||
|
keep_diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)]
|
||||||
|
keep_val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(2)]
|
||||||
|
# 题型 P:diag 3 对(6 题=3 单元)+ val 2 对(4 题=2 单元)→ 单元数=5<8,
|
||||||
|
# 但题目数=10>=8。按单元计数须剔除(按题目计数会误通过)。
|
||||||
|
pair_diag: list[_FakeQuestion] = []
|
||||||
|
for i in range(3):
|
||||||
|
pair_diag.extend(_fake_pair(f"P-d{i}", "P"))
|
||||||
|
pair_val: list[_FakeQuestion] = []
|
||||||
|
for i in range(2):
|
||||||
|
pair_val.extend(_fake_pair(f"P-v{i}", "P"))
|
||||||
|
pools = _FakePools(
|
||||||
|
diagnosis=keep_diag + pair_diag,
|
||||||
|
validation=keep_val + pair_val,
|
||||||
|
test=[],
|
||||||
|
)
|
||||||
|
|
||||||
|
new_pools, new_types = _filter_untrainable_types(
|
||||||
|
pools, task_types=None, eval_min_per_class=2, trainable_min_units=8
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "P" not in {q.task_type for q in new_pools.diagnosis}
|
||||||
|
assert "P" not in new_types
|
||||||
|
assert "A" in new_types
|
||||||
|
|
||||||
|
def test_task_types_subset_drops_non_requested_trainable(self) -> None:
|
||||||
|
"""指定 task_types 子集时:非请求但可训练的题型也被剔除出 diag/val。
|
||||||
|
|
||||||
|
回归 I-4:pools 过滤此前只按可训练性 keep、不按 task_types 收窄,导致
|
||||||
|
冻结全局 pools 后 batch/diagnosis 会训练非请求题型(gate 只覆盖请求题型
|
||||||
|
→ 静默语义偏差)。此处 A、B 均可训,仅请求 A,B 必须被剔除。
|
||||||
|
"""
|
||||||
|
# 题型 A:diag=8 + val=2 → units=10、val=2,可训
|
||||||
|
diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)]
|
||||||
|
val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(2)]
|
||||||
|
# 题型 B:diag=8 + val=2 → units=10、val=2,同样可训(但未被请求)
|
||||||
|
diag += [_FakeQuestion(question_id=f"B-d{i}", task_type="B") for i in range(8)]
|
||||||
|
val += [_FakeQuestion(question_id=f"B-v{i}", task_type="B") for i in range(2)]
|
||||||
|
pools = _FakePools(diagnosis=diag, validation=val, test=[])
|
||||||
|
|
||||||
|
new_pools, new_types = _filter_untrainable_types(
|
||||||
|
pools,
|
||||||
|
task_types=["A"],
|
||||||
|
eval_min_per_class=2,
|
||||||
|
trainable_min_units=8,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 非请求题型 B(可训)被剔除出 diag/val
|
||||||
|
assert {q.task_type for q in new_pools.diagnosis} == {"A"}
|
||||||
|
assert {q.task_type for q in new_pools.validation} == {"A"}
|
||||||
|
assert new_types == ["A"]
|
||||||
|
|
||||||
|
def test_all_filtered_raises(self) -> None:
|
||||||
|
"""所有题型都被剔除时 fail-fast:raise RuntimeError 并列出剔除原因。"""
|
||||||
|
diag = [_FakeQuestion(question_id="B-d0", task_type="B")] # val=0 < 2
|
||||||
|
pools = _FakePools(diagnosis=diag, validation=[], test=[])
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="可训练"):
|
||||||
|
_filter_untrainable_types(
|
||||||
|
pools, task_types=None, eval_min_per_class=2, trainable_min_units=8
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_test_pool_untouched(self) -> None:
|
||||||
|
"""test 池不参与过滤(继续报告全题型准确率)。"""
|
||||||
|
val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(8)]
|
||||||
|
diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)]
|
||||||
|
test = [_FakeQuestion(question_id="B-t0", task_type="B")]
|
||||||
|
pools = _FakePools(diagnosis=diag, validation=val, test=test)
|
||||||
|
|
||||||
|
new_pools, _ = _filter_untrainable_types(
|
||||||
|
pools, task_types=None, eval_min_per_class=2, trainable_min_units=8
|
||||||
|
)
|
||||||
|
|
||||||
|
assert new_pools.test == test
|
||||||
|
|
||||||
|
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
@@ -667,7 +771,7 @@ class TestTrainState:
|
|||||||
assert state.system_packs == []
|
assert state.system_packs == []
|
||||||
assert state.tool_packs == []
|
assert state.tool_packs == []
|
||||||
assert state.changed_task_types_this_epoch == set()
|
assert state.changed_task_types_this_epoch == set()
|
||||||
assert state.steps_since_best_improved == 0
|
assert state.epochs_since_best_improved == 0
|
||||||
|
|
||||||
|
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
@@ -726,6 +830,7 @@ class TestRunnerFactoryInjection:
|
|||||||
"batch_size": 5,
|
"batch_size": 5,
|
||||||
"min_class_per_batch": 2,
|
"min_class_per_batch": 2,
|
||||||
"eval_min_per_class": 2,
|
"eval_min_per_class": 2,
|
||||||
|
"trainable_min_units": 8,
|
||||||
"early_stop_patience": 3,
|
"early_stop_patience": 3,
|
||||||
"test_size": 10,
|
"test_size": 10,
|
||||||
"use_slow_momentum": False,
|
"use_slow_momentum": False,
|
||||||
|
|||||||
@@ -243,6 +243,24 @@ class TestInitSeed:
|
|||||||
with pytest.raises(FileExistsError, match="种子已存在"):
|
with pytest.raises(FileExistsError, match="种子已存在"):
|
||||||
init_seed(store, "dup", skills_dir, prompts_dir, baseline_db, "r1", None, "second")
|
init_seed(store, "dup", skills_dir, prompts_dir, baseline_db, "r1", None, "second")
|
||||||
|
|
||||||
|
def test_init_seed_carries_pools(self, tmp_path):
|
||||||
|
"""提供 pools_json/split_manifest 时拷入 seed 目录。"""
|
||||||
|
from app.harness.store import init_seed
|
||||||
|
|
||||||
|
store = tmp_path / "store"
|
||||||
|
skills = tmp_path / "sk"; skills.mkdir(); (skills / "s.md").write_text("x")
|
||||||
|
prompts = tmp_path / "pr"; prompts.mkdir(); (prompts / "p.md").write_text("y")
|
||||||
|
db = tmp_path / "b.db"; db.write_text("db")
|
||||||
|
pools = tmp_path / "pools.json"; pools.write_text('{"split_mode":"global"}')
|
||||||
|
manifest = tmp_path / "split_manifest.json"; manifest.write_text('{"pools_sha256":"a"}')
|
||||||
|
|
||||||
|
seed_dir = init_seed(
|
||||||
|
store, "s1", skills, prompts, db, "infer_adhoc", None, "d",
|
||||||
|
pools_json=pools, split_manifest=manifest,
|
||||||
|
)
|
||||||
|
assert (seed_dir / "pools.json").exists()
|
||||||
|
assert (seed_dir / "split_manifest.json").exists()
|
||||||
|
|
||||||
|
|
||||||
class TestListSeeds:
|
class TestListSeeds:
|
||||||
"""list_seeds 列出所有种子。"""
|
"""list_seeds 列出所有种子。"""
|
||||||
@@ -338,6 +356,44 @@ class TestExtractRunDb:
|
|||||||
with pytest.raises(RuntimeError, match="无 run_id="):
|
with pytest.raises(RuntimeError, match="无 run_id="):
|
||||||
extract_run_db(src, dst, "nonexistent")
|
extract_run_db(src, dst, "nonexistent")
|
||||||
|
|
||||||
|
def test_dedupe_per_question_keeps_first_row(self, tmp_path):
|
||||||
|
"""dedupe_per_question=True 时每 question_id 只保留 rowid 最小的首行。"""
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
src = tmp_path / "src.db"
|
||||||
|
conn = sqlite3.connect(src)
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE _runs (run_id TEXT PRIMARY KEY, started_at TEXT)"
|
||||||
|
)
|
||||||
|
conn.execute("INSERT INTO _runs VALUES ('r1', 't0')")
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE predictions (run_id TEXT, question_id TEXT, prediction TEXT)"
|
||||||
|
)
|
||||||
|
# 743-1 三行(模拟 error/budget/finished),首行 prediction=NULL
|
||||||
|
conn.executemany(
|
||||||
|
"INSERT INTO predictions VALUES (?,?,?)",
|
||||||
|
[
|
||||||
|
("r1", "743-1", None),
|
||||||
|
("r1", "743-1", None),
|
||||||
|
("r1", "743-1", "C"),
|
||||||
|
("r1", "q2", "A"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
dst = tmp_path / "dst.db"
|
||||||
|
from app.harness.store import extract_run_db
|
||||||
|
|
||||||
|
extract_run_db(src, dst, "r1", dedupe_per_question=True)
|
||||||
|
|
||||||
|
out = sqlite3.connect(dst)
|
||||||
|
rows = out.execute(
|
||||||
|
"SELECT question_id, prediction FROM predictions ORDER BY question_id"
|
||||||
|
).fetchall()
|
||||||
|
out.close()
|
||||||
|
assert rows == [("743-1", None), ("q2", "A")], f"未按 rowid 首行去重: {rows}"
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# promote_to_seed
|
# promote_to_seed
|
||||||
|
|||||||
@@ -112,7 +112,6 @@ def _make_mock_run_inference(
|
|||||||
log: HarnessLog,
|
log: HarnessLog,
|
||||||
baseline_correctness: dict[str, bool],
|
baseline_correctness: dict[str, bool],
|
||||||
candidate_correctness: dict[str, bool],
|
candidate_correctness: dict[str, bool],
|
||||||
error_count: int = 0,
|
|
||||||
):
|
):
|
||||||
"""构建 mock RunInferenceFn。
|
"""构建 mock RunInferenceFn。
|
||||||
|
|
||||||
@@ -136,9 +135,6 @@ def _make_mock_run_inference(
|
|||||||
|
|
||||||
correct = sum(per_q.values())
|
correct = sum(per_q.values())
|
||||||
total = len(questions)
|
total = len(questions)
|
||||||
stop_counts: dict[str, int] = {"completed": total - error_count}
|
|
||||||
if error_count > 0:
|
|
||||||
stop_counts["error"] = error_count
|
|
||||||
return InferenceResult(
|
return InferenceResult(
|
||||||
run_id=run_id,
|
run_id=run_id,
|
||||||
accuracy=correct / total if total else 0.0,
|
accuracy=correct / total if total else 0.0,
|
||||||
@@ -147,12 +143,71 @@ def _make_mock_run_inference(
|
|||||||
per_task_type={},
|
per_task_type={},
|
||||||
steps_mean=1.0,
|
steps_mean=1.0,
|
||||||
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
|
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
|
||||||
stop_reason_counts=stop_counts,
|
stop_reason_counts={"completed": total},
|
||||||
)
|
)
|
||||||
|
|
||||||
return mock_fn, call_log
|
return mock_fn, call_log
|
||||||
|
|
||||||
|
|
||||||
|
def _make_all_infra_mock(log: HarnessLog, stop_reason: str):
|
||||||
|
"""构建基线全 INFRA 的 mock:每 record 写指定 INFRA stop_reason(error/parse_error)。
|
||||||
|
|
||||||
|
与真实推理一致——per-record DB stop_reason 与汇总 stop_reason_counts 同源;护栏
|
||||||
|
分子按 unit 从 DB 读(_infra_question_ids_from_db),故须真实落 DB。total 返回
|
||||||
|
unit 粒度(single 时 == 题数),使护栏分子/分母同粒度。
|
||||||
|
"""
|
||||||
|
call_log: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
async def mock_fn(
|
||||||
|
questions: list[GeneratedQuestion],
|
||||||
|
*,
|
||||||
|
run_id: str,
|
||||||
|
skills_dir: Path,
|
||||||
|
) -> InferenceResult:
|
||||||
|
call_log.append({"run_id": run_id, "n": len(questions)})
|
||||||
|
for q in questions:
|
||||||
|
log.insert(
|
||||||
|
"predictions",
|
||||||
|
{
|
||||||
|
"run_id": run_id,
|
||||||
|
"video_id": "v0",
|
||||||
|
"question_id": q.question_id,
|
||||||
|
"task_type": "temporal",
|
||||||
|
"prediction": "",
|
||||||
|
"answer": "A",
|
||||||
|
"evidence": "",
|
||||||
|
"reasoning": "",
|
||||||
|
"steps_used": 1,
|
||||||
|
"prompt_tokens": 10,
|
||||||
|
"completion_tokens": 10,
|
||||||
|
"stop_reason": stop_reason,
|
||||||
|
"steps_json": "[]",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
total = len(questions) # 全 single → unit 数 == 题数
|
||||||
|
return InferenceResult(
|
||||||
|
run_id=run_id,
|
||||||
|
accuracy=0.0,
|
||||||
|
total=total,
|
||||||
|
correct=0,
|
||||||
|
per_task_type={},
|
||||||
|
steps_mean=1.0,
|
||||||
|
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
|
||||||
|
stop_reason_counts={stop_reason: total},
|
||||||
|
)
|
||||||
|
|
||||||
|
return mock_fn, call_log
|
||||||
|
|
||||||
|
|
||||||
|
def test_infra_stop_reasons_single_source() -> None:
|
||||||
|
"""app 侧 INFRA_STOP_REASONS 复用 core 常量(同一对象),杜绝未来漂移(M-2)。"""
|
||||||
|
from app.harness import validate
|
||||||
|
from core.evolution import diagnose
|
||||||
|
|
||||||
|
assert validate.INFRA_STOP_REASONS is diagnose.INFRA_STOP_REASONS
|
||||||
|
assert frozenset({"error", "parse_error"}) == diagnose.INFRA_STOP_REASONS
|
||||||
|
|
||||||
|
|
||||||
# ===========================================================================
|
# ===========================================================================
|
||||||
# 数据类型测试
|
# 数据类型测试
|
||||||
# ===========================================================================
|
# ===========================================================================
|
||||||
@@ -421,17 +476,14 @@ async def test_gate_prefix_must_contain_gate(tmp_path: Path) -> None:
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_infra_guard_threshold(tmp_path: Path) -> None:
|
async def test_infra_guard_threshold(tmp_path: Path) -> None:
|
||||||
"""推理错误率超阈值时抛 RuntimeError。"""
|
"""推理错误率超阈值时抛 RuntimeError(护栏分子/分母 unit 同粒度)。"""
|
||||||
workspace = _setup_workspace(tmp_path)
|
workspace = _setup_workspace(tmp_path)
|
||||||
log = _make_log(workspace)
|
log = _make_log(workspace)
|
||||||
# 需要 >=10 题次才触发 INFRA 护栏
|
# 需要 >=10 unit 分母才触发护栏:12 个 single,基线全 INFRA error。
|
||||||
questions = _make_questions(6)
|
# 首块全 INFRA → valid_chunk 空 → errors=12/denom=12=1.0>0.5 触发护栏。
|
||||||
|
questions = _make_questions(12)
|
||||||
cache = BaselineCache(workspace / "baseline_cache.json")
|
cache = BaselineCache(workspace / "baseline_cache.json")
|
||||||
|
mock_fn, _ = _make_all_infra_mock(log, "error")
|
||||||
baseline_correct = {f"q{i}": False for i in range(6)}
|
|
||||||
candidate_correct = {f"q{i}": False for i in range(6)}
|
|
||||||
# 每次 run_inference 报 error_count=5,两侧各 5 → 10/12 > 0.5
|
|
||||||
mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct, error_count=5)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with pytest.raises(RuntimeError, match="错误率过高"):
|
with pytest.raises(RuntimeError, match="错误率过高"):
|
||||||
@@ -444,7 +496,7 @@ async def test_infra_guard_threshold(tmp_path: Path) -> None:
|
|||||||
base_skill_content="baseline skill content",
|
base_skill_content="baseline skill content",
|
||||||
ladder_items=questions,
|
ladder_items=questions,
|
||||||
gate_params=_DEFAULT_GATE_PARAMS,
|
gate_params=_DEFAULT_GATE_PARAMS,
|
||||||
gate_block=6,
|
gate_block=12,
|
||||||
gate_n_max=20,
|
gate_n_max=20,
|
||||||
gate_guard_err=0.5,
|
gate_guard_err=0.5,
|
||||||
baseline_cache=cache,
|
baseline_cache=cache,
|
||||||
@@ -505,6 +557,271 @@ async def test_baseline_cache_hit(tmp_path: Path) -> None:
|
|||||||
log.close()
|
log.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_baseline_infra_error_not_cached(tmp_path: Path) -> None:
|
||||||
|
"""基线臂 INFRA error 的 unit 不写入 BaselineCache(不永久污染),且从有效单元排除。"""
|
||||||
|
from app.harness.gate_ladder import skill_hash
|
||||||
|
from app.harness.question_units import build_units
|
||||||
|
from app.harness.validate import _resolve_baseline_block
|
||||||
|
|
||||||
|
workspace = _setup_workspace(tmp_path)
|
||||||
|
log = _make_log(workspace)
|
||||||
|
questions = _make_questions(2) # q0 干净, q1 INFRA error
|
||||||
|
units = build_units(questions)
|
||||||
|
cache = BaselineCache(workspace / "baseline_cache.json")
|
||||||
|
s_hash = skill_hash("baseline skill content")
|
||||||
|
|
||||||
|
async def mock_fn(qs, *, run_id, skills_dir):
|
||||||
|
for q in qs:
|
||||||
|
is_err = q.question_id == "q1"
|
||||||
|
log.insert(
|
||||||
|
"predictions",
|
||||||
|
{
|
||||||
|
"run_id": run_id,
|
||||||
|
"video_id": "v0",
|
||||||
|
"question_id": q.question_id,
|
||||||
|
"task_type": "temporal",
|
||||||
|
"prediction": "" if is_err else "A",
|
||||||
|
"answer": "A",
|
||||||
|
"evidence": "",
|
||||||
|
"reasoning": "",
|
||||||
|
"steps_used": 1,
|
||||||
|
"prompt_tokens": 10,
|
||||||
|
"completion_tokens": 10,
|
||||||
|
"stop_reason": "error" if is_err else "completed",
|
||||||
|
"steps_json": "[]",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return InferenceResult(
|
||||||
|
run_id=run_id,
|
||||||
|
accuracy=0.5,
|
||||||
|
total=2,
|
||||||
|
correct=1,
|
||||||
|
per_task_type={},
|
||||||
|
steps_mean=1.0,
|
||||||
|
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
|
||||||
|
stop_reason_counts={"completed": 1, "error": 1},
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
b_units, valid_units, _errors_inc, _denom_inc = await _resolve_baseline_block(
|
||||||
|
units=units,
|
||||||
|
task_type="temporal",
|
||||||
|
s_hash=s_hash,
|
||||||
|
prompts_version="p1",
|
||||||
|
baseline_cache=cache,
|
||||||
|
base_skills_dir=workspace / "skills" / "v1",
|
||||||
|
run_inference=mock_fn,
|
||||||
|
log=log,
|
||||||
|
run_id="step1_gate_b0_base",
|
||||||
|
)
|
||||||
|
# q1 是 INFRA:不写缓存、不入 b_units、不在有效单元里
|
||||||
|
assert cache.get("temporal", s_hash, "p1", "q1") is None
|
||||||
|
assert "q1" not in b_units
|
||||||
|
assert all(u.unit_id != "q1" for u in valid_units)
|
||||||
|
# q0 干净:正常缓存并入 b_units/valid_units
|
||||||
|
assert cache.get("temporal", s_hash, "p1", "q0") is True
|
||||||
|
assert b_units["q0"] is True
|
||||||
|
assert any(u.unit_id == "q0" for u in valid_units)
|
||||||
|
finally:
|
||||||
|
log.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_infra_guard_counts_units_not_records(tmp_path: Path) -> None:
|
||||||
|
"""护栏分子按 unit 计:AR pair 两 record 全 INFRA 只计 1 个 INFRA unit(而非 2)。
|
||||||
|
|
||||||
|
回归 I-3:分子此前用 stop_reason_counts 逐 record 计数,分母 denom_inc=r.total
|
||||||
|
是 unit 粒度;AR pair(一 unit 两 record)致分子被放大、误触发 gate_guard_err。
|
||||||
|
分子改为"含 INFRA record 的 unit 数"后与分母同粒度(核心算法保真 #5/#6)。
|
||||||
|
"""
|
||||||
|
from app.harness.gate_ladder import skill_hash
|
||||||
|
from app.harness.question_units import build_units
|
||||||
|
from app.harness.validate import _resolve_baseline_block
|
||||||
|
|
||||||
|
workspace = _setup_workspace(tmp_path)
|
||||||
|
log = _make_log(workspace)
|
||||||
|
# 一个 AR pair(两成员共享 pair_id)→ build_units 折叠为 1 个 pair unit
|
||||||
|
common = {
|
||||||
|
"video_id": "vp",
|
||||||
|
"task_type": "temporal",
|
||||||
|
"question": "Q?",
|
||||||
|
"options": ("A", "B", "C", "D"),
|
||||||
|
"answer": "A",
|
||||||
|
"source_nodes": (),
|
||||||
|
"difficulty": "easy",
|
||||||
|
"pair_id": "p1",
|
||||||
|
"flip_axis": "before_after",
|
||||||
|
}
|
||||||
|
pair = [
|
||||||
|
GeneratedQuestion(question_id="p1_o", question_role="pair_original", **common),
|
||||||
|
GeneratedQuestion(question_id="p1_m", question_role="pair_mirror", **common),
|
||||||
|
]
|
||||||
|
units = build_units(pair)
|
||||||
|
assert len(units) == 1 # 前置:pair 折叠为 1 个 unit
|
||||||
|
cache = BaselineCache(workspace / "baseline_cache.json")
|
||||||
|
s_hash = skill_hash("baseline skill content")
|
||||||
|
|
||||||
|
async def mock_fn(qs, *, run_id, skills_dir):
|
||||||
|
# 两 record 皆 INFRA error
|
||||||
|
for q in qs:
|
||||||
|
log.insert(
|
||||||
|
"predictions",
|
||||||
|
{
|
||||||
|
"run_id": run_id,
|
||||||
|
"video_id": "vp",
|
||||||
|
"question_id": q.question_id,
|
||||||
|
"task_type": "temporal",
|
||||||
|
"prediction": "",
|
||||||
|
"answer": "A",
|
||||||
|
"evidence": "",
|
||||||
|
"reasoning": "",
|
||||||
|
"steps_used": 1,
|
||||||
|
"prompt_tokens": 10,
|
||||||
|
"completion_tokens": 10,
|
||||||
|
"stop_reason": "error",
|
||||||
|
"steps_json": "[]",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
# total 为 unit 粒度(1 个 pair unit);stop_reason_counts 为 record 粒度(2)
|
||||||
|
return InferenceResult(
|
||||||
|
run_id=run_id,
|
||||||
|
accuracy=0.0,
|
||||||
|
total=1,
|
||||||
|
correct=0,
|
||||||
|
per_task_type={},
|
||||||
|
steps_mean=1.0,
|
||||||
|
token_usage={"prompt_tokens": 20, "completion_tokens": 20},
|
||||||
|
stop_reason_counts={"error": 2},
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
_b_units, valid_units, errors_inc, denom_inc = await _resolve_baseline_block(
|
||||||
|
units=units,
|
||||||
|
task_type="temporal",
|
||||||
|
s_hash=s_hash,
|
||||||
|
prompts_version="p1",
|
||||||
|
baseline_cache=cache,
|
||||||
|
base_skills_dir=workspace / "skills" / "v1",
|
||||||
|
run_inference=mock_fn,
|
||||||
|
log=log,
|
||||||
|
run_id="step1_gate_b0_base",
|
||||||
|
)
|
||||||
|
# 分子按 unit 计:1 个 INFRA unit(不是 2 条 record);分母同粒度 = r.total = 1
|
||||||
|
assert errors_inc == 1
|
||||||
|
assert denom_inc == 1
|
||||||
|
# 整对 INFRA → 从有效单元剔除
|
||||||
|
assert valid_units == []
|
||||||
|
finally:
|
||||||
|
log.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_all_infra_ladder_raises_clear_error(tmp_path: Path) -> None:
|
||||||
|
"""整个阶梯所有 unit 都被判为 INFRA 排除 → 明确 RuntimeError(非误导性空阶梯断言)。"""
|
||||||
|
workspace = _setup_workspace(tmp_path)
|
||||||
|
log = _make_log(workspace)
|
||||||
|
questions = _make_questions(4)
|
||||||
|
cache = BaselineCache(workspace / "baseline_cache.json")
|
||||||
|
|
||||||
|
candidate_calls: list[str] = []
|
||||||
|
|
||||||
|
async def mock_fn(qs, *, run_id, skills_dir):
|
||||||
|
if run_id.endswith("_cand"):
|
||||||
|
candidate_calls.append(run_id)
|
||||||
|
# 基线臂逐题全部 INFRA error(候选臂在修复后不应被空跑)
|
||||||
|
for q in qs:
|
||||||
|
log.insert(
|
||||||
|
"predictions",
|
||||||
|
{
|
||||||
|
"run_id": run_id,
|
||||||
|
"video_id": "v0",
|
||||||
|
"question_id": q.question_id,
|
||||||
|
"task_type": "temporal",
|
||||||
|
"prediction": "",
|
||||||
|
"answer": "A",
|
||||||
|
"evidence": "",
|
||||||
|
"reasoning": "",
|
||||||
|
"steps_used": 1,
|
||||||
|
"prompt_tokens": 10,
|
||||||
|
"completion_tokens": 10,
|
||||||
|
"stop_reason": "error",
|
||||||
|
"steps_json": "[]",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
total = len(qs)
|
||||||
|
return InferenceResult(
|
||||||
|
run_id=run_id,
|
||||||
|
accuracy=0.0,
|
||||||
|
total=total,
|
||||||
|
correct=0,
|
||||||
|
per_task_type={},
|
||||||
|
steps_mean=1.0,
|
||||||
|
token_usage={"prompt_tokens": 10, "completion_tokens": 10},
|
||||||
|
stop_reason_counts={"error": total},
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with pytest.raises(RuntimeError, match="INFRA"):
|
||||||
|
await validate_skill_local(
|
||||||
|
workspace_dir=workspace,
|
||||||
|
base_skills_version="v1",
|
||||||
|
task_type="temporal",
|
||||||
|
target_file="temporal.md",
|
||||||
|
candidate_content="content",
|
||||||
|
base_skill_content="baseline skill content",
|
||||||
|
ladder_items=questions,
|
||||||
|
gate_params=_DEFAULT_GATE_PARAMS,
|
||||||
|
gate_block=4,
|
||||||
|
gate_n_max=20,
|
||||||
|
gate_guard_err=0.9, # 高阈值:4 题 <10 分母不触发错误率护栏
|
||||||
|
baseline_cache=cache,
|
||||||
|
prompts_version="p1",
|
||||||
|
run_inference=mock_fn,
|
||||||
|
log=log,
|
||||||
|
gate_run_prefix="step1_gate_test",
|
||||||
|
)
|
||||||
|
# 全 INFRA 块不应触发候选空跑
|
||||||
|
assert candidate_calls == []
|
||||||
|
finally:
|
||||||
|
log.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_parse_error_counts_toward_guard(tmp_path: Path) -> None:
|
||||||
|
"""stop_reason=parse_error 也计入护栏错误率(与 INFRA 判定口径一致)→ 超阈值熔断。"""
|
||||||
|
workspace = _setup_workspace(tmp_path)
|
||||||
|
log = _make_log(workspace)
|
||||||
|
# 12 个 single,基线全 parse_error(per-record 落 DB,护栏按 unit 从 DB 读)。
|
||||||
|
# 首块全 INFRA → errors=12/denom=12=1.0>0.5 → parse_error 亦触发护栏。
|
||||||
|
questions = _make_questions(12)
|
||||||
|
cache = BaselineCache(workspace / "baseline_cache.json")
|
||||||
|
mock_fn, _ = _make_all_infra_mock(log, "parse_error")
|
||||||
|
|
||||||
|
try:
|
||||||
|
with pytest.raises(RuntimeError, match="错误率过高"):
|
||||||
|
await validate_skill_local(
|
||||||
|
workspace_dir=workspace,
|
||||||
|
base_skills_version="v1",
|
||||||
|
task_type="temporal",
|
||||||
|
target_file="temporal.md",
|
||||||
|
candidate_content="content",
|
||||||
|
base_skill_content="baseline skill content",
|
||||||
|
ladder_items=questions,
|
||||||
|
gate_params=_DEFAULT_GATE_PARAMS,
|
||||||
|
gate_block=12,
|
||||||
|
gate_n_max=20,
|
||||||
|
gate_guard_err=0.5,
|
||||||
|
baseline_cache=cache,
|
||||||
|
prompts_version="p1",
|
||||||
|
run_inference=mock_fn,
|
||||||
|
log=log,
|
||||||
|
gate_run_prefix="step1_gate_test",
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
log.close()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_last_block_terminal(tmp_path: Path) -> None:
|
async def test_last_block_terminal(tmp_path: Path) -> None:
|
||||||
"""单块 + n_remaining=0 → 终态判定(provisional 或 inertia),非 continue。"""
|
"""单块 + n_remaining=0 → 终态判定(provisional 或 inertia),非 continue。"""
|
||||||
|
|||||||
@@ -183,6 +183,23 @@ def test_init_workspace_from_seed_missing_questions(store_dir: Path, workspace_d
|
|||||||
assert not workspace_dir.exists()
|
assert not workspace_dir.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_init_workspace_from_seed_carries_pools(store_dir: Path, workspace_dir: Path) -> None:
|
||||||
|
"""seed 目录含 pools.json/split_manifest.json 时拷入 workspace。"""
|
||||||
|
# store_dir fixture 已构造 seed "initial";补冻结产物到 seed 目录再初始化 workspace。
|
||||||
|
seed_dir = store_dir / "seeds" / "initial"
|
||||||
|
(seed_dir / "pools.json").write_text('{"split_mode":"global"}')
|
||||||
|
(seed_dir / "split_manifest.json").write_text('{"pools_sha256":"a"}')
|
||||||
|
|
||||||
|
init_workspace_from_seed(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
seed_name="initial",
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
)
|
||||||
|
assert (workspace_dir / "pools.json").exists()
|
||||||
|
assert (workspace_dir / "split_manifest.json").exists()
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# load_manifest
|
# load_manifest
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -242,6 +259,33 @@ def test_update_manifest_valid(store_dir: Path, workspace_dir: Path) -> None:
|
|||||||
assert manifest["current"]["skills"] == "skills/v2"
|
assert manifest["current"]["skills"] == "skills/v2"
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_manifest_is_atomic(
|
||||||
|
store_dir: Path, workspace_dir: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
"""写 manifest 途中崩溃不产生半截 JSON(原子写:os.replace 失败也不损原文件)。"""
|
||||||
|
from app.harness import workspace as ws
|
||||||
|
|
||||||
|
init_workspace(
|
||||||
|
workspace_dir,
|
||||||
|
store_dir,
|
||||||
|
questions="benchmarks/Video-MME",
|
||||||
|
skills_version="v1",
|
||||||
|
prompts_version="v1",
|
||||||
|
)
|
||||||
|
original = (workspace_dir / "manifest.json").read_text()
|
||||||
|
|
||||||
|
def _boom(_src: object, _dst: object) -> None:
|
||||||
|
raise OSError("crash during replace")
|
||||||
|
|
||||||
|
monkeypatch.setattr(ws.os, "replace", _boom)
|
||||||
|
with pytest.raises(OSError, match="crash during replace"):
|
||||||
|
update_manifest(workspace_dir, skills="skills/v2")
|
||||||
|
|
||||||
|
# 原 manifest 未被破坏(内容不变且仍是合法 JSON)
|
||||||
|
assert (workspace_dir / "manifest.json").read_text() == original
|
||||||
|
assert json.loads((workspace_dir / "manifest.json").read_text())
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# record_run
|
# record_run
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
"""工程配置校验单元测试(Redis 缓存 TTL fail-loud)。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
def test_redis_cache_ttl_zero_rejected() -> None:
|
||||||
|
"""REDIS_CACHE_TTL<=0 必须启动即报错,消灭'0=永不过期'隐式语义。"""
|
||||||
|
from adapters.redis_cache import _resolve_cache_ttl
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="REDIS_CACHE_TTL"):
|
||||||
|
_resolve_cache_ttl(0)
|
||||||
|
with pytest.raises(ValueError, match="REDIS_CACHE_TTL"):
|
||||||
|
_resolve_cache_ttl(-1)
|
||||||
|
assert _resolve_cache_ttl(86400) == 86400
|
||||||
@@ -383,3 +383,68 @@ class TestApplyPatch:
|
|||||||
assert "line2" not in out
|
assert "line2" not in out
|
||||||
assert "TAIL" in out
|
assert "TAIL" in out
|
||||||
assert all(r["status"].startswith("applied") for r in reports)
|
assert all(r["status"].startswith("applied") for r in reports)
|
||||||
|
|
||||||
|
|
||||||
|
class TestSpanOverlapProtection:
|
||||||
|
"""冻结区跨度保护 + marker 注入拦截(算法 #8 加固)。"""
|
||||||
|
|
||||||
|
def test_replace_spanning_into_protected_marker_preserved(self) -> None:
|
||||||
|
"""target 末端跨入含 marker 的 appendix 冻结区时被拦截,marker 不被破坏。"""
|
||||||
|
body = "正文最后一段。"
|
||||||
|
appendix = f"{APPENDIX_START}\n## 执行提醒\n- 规则A\n{APPENDIX_END}"
|
||||||
|
content = body + "\n\n" + appendix
|
||||||
|
# target 从正文末尾跨入 APPENDIX_START(含 marker 字面量)
|
||||||
|
target = "正文最后一段。\n\n" + APPENDIX_START
|
||||||
|
edits = [{"op": "delete", "target": target, "content": ""}]
|
||||||
|
new_content, report = apply_patch_with_report(
|
||||||
|
content, edits, protected_spans=[appendix]
|
||||||
|
)
|
||||||
|
# 无论经 marker 注入拦截还是跨度拦截,marker 都必须完整保留
|
||||||
|
assert APPENDIX_START in new_content and APPENDIX_END in new_content
|
||||||
|
assert report[0]["status"] in ("skipped_protected", "skipped_marker_injection")
|
||||||
|
|
||||||
|
def test_replace_spanning_into_frozen_section_is_skipped(self) -> None:
|
||||||
|
"""target 起点在正文、末端伸入无 marker 的冻结区段 → 跨度拦截跳过。"""
|
||||||
|
body = "可改正文段落。"
|
||||||
|
frozen = "## 输出格式\n必须输出 JSON。"
|
||||||
|
content = body + "\n" + frozen
|
||||||
|
# target 从可改正文跨入冻结区段(不含任何 marker 字面量)
|
||||||
|
target = "可改正文段落。\n## 输出格式"
|
||||||
|
edits = [{"op": "replace", "target": target, "content": "破坏内容"}]
|
||||||
|
new_content, report = apply_patch_with_report(
|
||||||
|
content, edits, protected_spans=[frozen]
|
||||||
|
)
|
||||||
|
assert "破坏内容" not in new_content
|
||||||
|
assert "## 输出格式" in new_content
|
||||||
|
assert report[0]["status"] == "skipped_protected"
|
||||||
|
|
||||||
|
def test_insert_after_spanning_into_frozen_section_is_skipped(self) -> None:
|
||||||
|
"""insert_after 的 target 末端伸入无 marker 冻结区段时跨度拦截跳过。"""
|
||||||
|
body = "可改正文。"
|
||||||
|
frozen = "## 输出格式\n固定内容。"
|
||||||
|
content = body + "\n" + frozen
|
||||||
|
target = "可改正文。\n## 输出格式"
|
||||||
|
edits = [{"op": "insert_after", "target": target, "content": "注入内容"}]
|
||||||
|
new_content, report = apply_patch_with_report(
|
||||||
|
content, edits, protected_spans=[frozen]
|
||||||
|
)
|
||||||
|
assert "注入内容" not in new_content
|
||||||
|
assert report[0]["status"] == "skipped_protected"
|
||||||
|
|
||||||
|
def test_edit_payload_with_marker_literal_rejected(self) -> None:
|
||||||
|
"""edit payload 含 marker 字面量 → 拒绝该 edit。"""
|
||||||
|
edits = [
|
||||||
|
{"op": "append", "target": "", "content": f"注入 {APPENDIX_START} 破坏"}
|
||||||
|
]
|
||||||
|
_, report = apply_patch_with_report("正文", edits, protected_spans=[])
|
||||||
|
assert any(
|
||||||
|
"marker" in str(s).lower() or "reject" in str(s).lower() for s in report[0].values()
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_edit_target_with_marker_literal_rejected(self) -> None:
|
||||||
|
"""edit target 含 momentum marker 字面量 → 拒绝该 edit。"""
|
||||||
|
edits = [
|
||||||
|
{"op": "replace", "target": f"{MOMENTUM_START}x", "content": "y"}
|
||||||
|
]
|
||||||
|
_, report = apply_patch_with_report("正文", edits, protected_spans=[])
|
||||||
|
assert report[0]["status"] == "skipped_marker_injection"
|
||||||
|
|||||||
@@ -135,3 +135,60 @@ def test_baseline_val_accuracy_reflects_validation():
|
|||||||
assert len(pools.validation) == 2
|
assert len(pools.validation) == 2
|
||||||
assert pools.baseline_val_accuracy == pytest.approx(0.5)
|
assert pools.baseline_val_accuracy == pytest.approx(0.5)
|
||||||
assert pools.diagnosis == []
|
assert pools.diagnosis == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_tier_aware_keeps_high_t2_in_diag():
|
||||||
|
"""错题视频组按 T2 含量升序进 val:T2 高的组保留在 diagnosis。"""
|
||||||
|
from app.harness.pools import split_by_video_assignment
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
|
||||||
|
def _q(qid, vid):
|
||||||
|
return GeneratedQuestion(
|
||||||
|
question_id=qid, video_id=vid, task_type="X", question="q",
|
||||||
|
options=["A", "B"], answer="A", source_nodes=[], difficulty="easy",
|
||||||
|
)
|
||||||
|
|
||||||
|
# 4 个错题视频(每视频 1 题),T2 数分别 2/1/0/0
|
||||||
|
questions = [_q(f"{v}-1", v) for v in ("vA", "vB", "vC", "vD")]
|
||||||
|
assignment = {v: "trainval" for v in ("vA", "vB", "vC", "vD")}
|
||||||
|
correctness = {f"{v}-1": False for v in ("vA", "vB", "vC", "vD")}
|
||||||
|
wrong_tier = {"vA": 2, "vB": 1, "vC": 0, "vD": 0}
|
||||||
|
|
||||||
|
pools = split_by_video_assignment(
|
||||||
|
questions, assignment, correctness, val_ratio=0.5, seed=7,
|
||||||
|
wrong_tier_by_video=wrong_tier,
|
||||||
|
)
|
||||||
|
diag_vids = {q.video_id for q in pools.diagnosis}
|
||||||
|
# T2 最高的 vA 必留 diag;T2=0 的组优先进 val
|
||||||
|
assert "vA" in diag_vids
|
||||||
|
assert "vB" in diag_vids
|
||||||
|
|
||||||
|
|
||||||
|
def test_val_wrong_min_repair_pulls_from_diag():
|
||||||
|
"""val 错题不足 val_wrong_min 时从 diag 换入低 T2 错题组补足。"""
|
||||||
|
from app.harness.pools import split_by_video_assignment
|
||||||
|
from core.types import GeneratedQuestion
|
||||||
|
|
||||||
|
def _q(qid, vid, correct):
|
||||||
|
return GeneratedQuestion(
|
||||||
|
question_id=qid, video_id=vid, task_type="X", question="q",
|
||||||
|
options=["A", "B"], answer="A", source_nodes=[], difficulty="easy",
|
||||||
|
)
|
||||||
|
|
||||||
|
# 8 错题视频 + 2 正确视频;val_ratio 小使初分 val 错题不足,触发修复
|
||||||
|
vids_wrong = [f"w{i}" for i in range(8)]
|
||||||
|
vids_correct = ["c0", "c1"]
|
||||||
|
questions = [_q(f"{v}-1", v, False) for v in vids_wrong] + [
|
||||||
|
_q(f"{v}-1", v, True) for v in vids_correct
|
||||||
|
]
|
||||||
|
assignment = {v: "trainval" for v in vids_wrong + vids_correct}
|
||||||
|
correctness = {f"{v}-1": False for v in vids_wrong}
|
||||||
|
correctness.update({f"{v}-1": True for v in vids_correct})
|
||||||
|
wrong_tier = {v: i for i, v in enumerate(vids_wrong)} # 递增 T2
|
||||||
|
|
||||||
|
pools = split_by_video_assignment(
|
||||||
|
questions, assignment, correctness, val_ratio=0.1, seed=7,
|
||||||
|
wrong_tier_by_video=wrong_tier, val_wrong_min=4,
|
||||||
|
)
|
||||||
|
val_wrong = sum(1 for q in pools.validation if not correctness[q.question_id])
|
||||||
|
assert val_wrong >= 4, f"功效修复后 val 错题 {val_wrong} < 4"
|
||||||
|
|||||||
@@ -109,6 +109,18 @@ async def test_different_models_different_keys(
|
|||||||
assert await cache.get("claude-3-opus", MESSAGES) is None
|
assert await cache.get("claude-3-opus", MESSAGES) is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_salt_changes_key(fake_redis: object) -> None:
|
||||||
|
"""cache_salt 进入键:不同 salt 产生不同键,None 保持旧键结构。"""
|
||||||
|
cache = RedisResponseCache(redis=fake_redis, ttl_s=None)
|
||||||
|
k_none = cache._build_key("m", [{"role": "user", "content": "x"}], None)
|
||||||
|
k_e1 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e1")
|
||||||
|
k_e2 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e2")
|
||||||
|
assert k_none != k_e1
|
||||||
|
assert k_none != k_e2
|
||||||
|
assert k_e1 != k_e2
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_graceful_degradation_on_error() -> None:
|
async def test_graceful_degradation_on_error() -> None:
|
||||||
"""Redis 不可用时静默降级:get→None,set→不报错。"""
|
"""Redis 不可用时静默降级:get→None,set→不报错。"""
|
||||||
|
|||||||
@@ -54,6 +54,7 @@ def _base_config(workspace_dir: Path, store_dir: Path) -> RunConfig:
|
|||||||
batch_size=5,
|
batch_size=5,
|
||||||
min_class_per_batch=2,
|
min_class_per_batch=2,
|
||||||
eval_min_per_class=2,
|
eval_min_per_class=2,
|
||||||
|
trainable_min_units=8,
|
||||||
early_stop_patience=3,
|
early_stop_patience=3,
|
||||||
test_size=10,
|
test_size=10,
|
||||||
use_slow_momentum=False,
|
use_slow_momentum=False,
|
||||||
@@ -147,6 +148,194 @@ def _fake_question(question_id: str, video_id: str) -> object:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_diagnosis_reads_traces_from_steps_json(
|
||||||
|
runner_with_real_store: Runner,
|
||||||
|
) -> None:
|
||||||
|
"""traces 表为空但 predictions.steps_json 有轨迹时,诊断仍拿到非空 traces。
|
||||||
|
|
||||||
|
构造一条只写 steps_json、不写 traces 表的 predictions 行;patch run_diagnosis
|
||||||
|
捕获传入的 run_log,直接 await 其 get_traces 断言经 StepsJsonRunLog 从
|
||||||
|
steps_json 重建出非空轨迹(算法 #7 恢复)。
|
||||||
|
"""
|
||||||
|
from app.harness.inference import PREDICTIONS_SCHEMA
|
||||||
|
from app.harness.log import HarnessLog
|
||||||
|
|
||||||
|
steps_json = json.dumps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"thought": "先看整体",
|
||||||
|
"tool_call": {"tool": "search_tree", "args": {"query": "开场"}},
|
||||||
|
"tool_output": "命中 L2 节点 A",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
ensure_ascii=False,
|
||||||
|
)
|
||||||
|
with HarnessLog(str(runner_with_real_store._paths.db_path), "infer_adhoc") as log:
|
||||||
|
log.create_table("predictions", PREDICTIONS_SCHEMA)
|
||||||
|
log.create_table("traces", {"video_id": "TEXT", "question_id": "TEXT", "step": "INTEGER"})
|
||||||
|
log.insert(
|
||||||
|
"predictions",
|
||||||
|
{
|
||||||
|
"video_id": _REAL_VIDEO_ID,
|
||||||
|
"question_id": _REAL_QUESTION_ID,
|
||||||
|
"task_type": "Action Reasoning",
|
||||||
|
"prediction": "A",
|
||||||
|
"answer": "B",
|
||||||
|
"evidence": "",
|
||||||
|
"reasoning": "",
|
||||||
|
"steps_used": 1,
|
||||||
|
"prompt_tokens": 0,
|
||||||
|
"completion_tokens": 0,
|
||||||
|
"stop_reason": "finished",
|
||||||
|
"steps_json": steps_json,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
captured: dict[str, object] = {}
|
||||||
|
|
||||||
|
async def _fake_run_diagnosis(**kwargs: object) -> DiagnosisResult:
|
||||||
|
captured["run_log"] = kwargs["run_log"]
|
||||||
|
return _empty_diagnosis_result()
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"core.evolution.diagnose.run_diagnosis",
|
||||||
|
new=AsyncMock(side_effect=_fake_run_diagnosis),
|
||||||
|
):
|
||||||
|
await runner_with_real_store._run_diagnosis(
|
||||||
|
"infer_adhoc", question_ids=[_REAL_QUESTION_ID]
|
||||||
|
)
|
||||||
|
|
||||||
|
run_log = captured["run_log"]
|
||||||
|
traces = await run_log.get_traces("infer_adhoc", question_ids=[_REAL_QUESTION_ID])
|
||||||
|
assert traces, "traces 表空时应从 steps_json 重建出非空轨迹"
|
||||||
|
assert traces[0]["tool_name"] == "search_tree"
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_inference_result(accuracy: float) -> object:
|
||||||
|
"""构造 InferenceResult 供 holdout 去重测试(per_task_type 空、token 归零)。"""
|
||||||
|
from app.harness.inference import InferenceResult
|
||||||
|
|
||||||
|
return InferenceResult(
|
||||||
|
run_id="x",
|
||||||
|
accuracy=accuracy,
|
||||||
|
total=10,
|
||||||
|
correct=int(accuracy * 10),
|
||||||
|
per_task_type={},
|
||||||
|
steps_mean=1.0,
|
||||||
|
token_usage={"prompt_tokens": 0, "completion_tokens": 0},
|
||||||
|
stop_reason_counts={},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_holdout_dedup_skips_reevaluated_versions(
|
||||||
|
runner_with_real_store: Runner,
|
||||||
|
) -> None:
|
||||||
|
"""baseline 不跑推理(从基线预测推导);best_hard==final 时不重复评估。"""
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
|
runner = runner_with_real_store
|
||||||
|
eval_calls: list[str] = []
|
||||||
|
|
||||||
|
async def _fake_eval(sv, pv, questions, run_id, context): # noqa: ANN001
|
||||||
|
eval_calls.append(run_id)
|
||||||
|
return _fake_inference_result(0.5)
|
||||||
|
|
||||||
|
runner._eval_version_on_pool = AsyncMock(side_effect=_fake_eval)
|
||||||
|
# best_mixed 赢家取 final 版本(必落在 memo,0 推理)
|
||||||
|
runner._pick_mixed_best = AsyncMock(return_value=("skills_final", "prompts_final"))
|
||||||
|
# baseline 推导:patch 为 0 推理的假结果(不经 _eval_version_on_pool)
|
||||||
|
runner._derive_baseline_test_result = MagicMock(return_value=_fake_inference_result(0.4))
|
||||||
|
|
||||||
|
state = MagicMock()
|
||||||
|
state.holdout_memo = {}
|
||||||
|
state.baseline_skills_version = "skills_base"
|
||||||
|
state.baseline_prompts_version = "prompts_base"
|
||||||
|
# best_hard 版本 == final 版本 → 去重
|
||||||
|
state.best_skills_version = "skills_final"
|
||||||
|
state.best_prompts_version = "prompts_final"
|
||||||
|
|
||||||
|
pools = MagicMock()
|
||||||
|
pools.baseline_run_id = "infer_adhoc"
|
||||||
|
pools.test = [_fake_question("q1", "vA")]
|
||||||
|
|
||||||
|
await runner._holdout_four_way(
|
||||||
|
1, pools, state, eval_skills_version="skills_final", eval_prompts_version="prompts_final"
|
||||||
|
)
|
||||||
|
|
||||||
|
# baseline=0(推导)、best_hard=1(真评并存 memo)、final/best_mixed 引用 memo(0)
|
||||||
|
assert len(eval_calls) == 1
|
||||||
|
runner._derive_baseline_test_result.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_run_step_deletes_stale_rows_before_rerun(
|
||||||
|
runner_with_real_store: Runner, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
"""同 run_id 重跑前先清 predictions/traces,避免重复行双计(幂等)。"""
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
|
from app.harness.inference import PREDICTIONS_SCHEMA, TRACES_SCHEMA
|
||||||
|
from app.harness.log import HarnessLog
|
||||||
|
|
||||||
|
run_id = "infer_adhoc_e1_s0"
|
||||||
|
# 预置该 step run_id 的旧 predictions/traces 行
|
||||||
|
with HarnessLog(str(runner_with_real_store._paths.db_path), run_id) as log:
|
||||||
|
log.create_table("predictions", PREDICTIONS_SCHEMA)
|
||||||
|
log.create_table("traces", TRACES_SCHEMA)
|
||||||
|
log.insert("predictions", {"video_id": "vA", "question_id": "q1", "prediction": "A"})
|
||||||
|
log.insert("traces", {"video_id": "vA", "question_id": "q1", "step": 0})
|
||||||
|
|
||||||
|
batch = [_fake_question("q1", "vA")]
|
||||||
|
runner_with_real_store._rollout_batch = AsyncMock()
|
||||||
|
monkeypatch.setattr("app.harness.runner._apply_batch_correctness", lambda *a, **k: None)
|
||||||
|
runner_with_real_store._run_diagnosis = AsyncMock(return_value=DiagnosisResult(run_id=run_id))
|
||||||
|
runner_with_real_store._gate_batch_skills = AsyncMock()
|
||||||
|
|
||||||
|
state = MagicMock()
|
||||||
|
state.correctness = {"q1": True}
|
||||||
|
state.gate_cooldown = {}
|
||||||
|
pools = MagicMock()
|
||||||
|
pools.baseline_run_id = "infer_adhoc"
|
||||||
|
|
||||||
|
await runner_with_real_store._run_step(1, 0, 10, batch, pools, state)
|
||||||
|
|
||||||
|
with HarnessLog(
|
||||||
|
str(runner_with_real_store._paths.db_path), run_id, register_run=False
|
||||||
|
) as log:
|
||||||
|
preds = log.query("SELECT * FROM predictions WHERE run_id=?", (run_id,))
|
||||||
|
traces = log.query("SELECT * FROM traces WHERE run_id=?", (run_id,))
|
||||||
|
assert preds == []
|
||||||
|
assert traces == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_run_step_aborts_on_high_degrade_rate(
|
||||||
|
runner_with_real_store: Runner, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
"""诊断降级占比 > 50% 时 _run_step 中止(疑似 judge 基础设施故障)。"""
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
|
batch = [_fake_question("q1", "vA"), _fake_question("q2", "vA")]
|
||||||
|
|
||||||
|
runner_with_real_store._rollout_batch = AsyncMock()
|
||||||
|
monkeypatch.setattr("app.harness.runner._apply_batch_correctness", lambda *a, **k: None)
|
||||||
|
runner_with_real_store._run_diagnosis = AsyncMock(
|
||||||
|
return_value=DiagnosisResult(run_id="r", degraded_count=2)
|
||||||
|
)
|
||||||
|
runner_with_real_store._gate_batch_skills = AsyncMock()
|
||||||
|
|
||||||
|
state = MagicMock()
|
||||||
|
state.correctness = {"q1": False, "q2": False}
|
||||||
|
state.gate_cooldown = {}
|
||||||
|
pools = MagicMock()
|
||||||
|
pools.baseline_run_id = "infer_adhoc"
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="降级占比"):
|
||||||
|
await runner_with_real_store._run_step(1, 0, 10, batch, pools, state)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_run_diagnosis_full_scan_loads_all_video_trees(
|
async def test_run_diagnosis_full_scan_loads_all_video_trees(
|
||||||
runner_with_real_store: Runner,
|
runner_with_real_store: Runner,
|
||||||
|
|||||||
@@ -303,3 +303,82 @@ def test_git_short_sha_nonempty():
|
|||||||
sha = cli.git_short_sha()
|
sha = cli.git_short_sha()
|
||||||
assert sha
|
assert sha
|
||||||
assert len(sha) >= 4
|
assert len(sha) >= 4
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_split_refuses_overwrite_without_force(tmp_path):
|
||||||
|
"""已存在指纹不同的 pools.json 时,force=False 必须报错不覆盖。"""
|
||||||
|
from app.harness.build_split import _guard_frozen_products
|
||||||
|
|
||||||
|
out_path = tmp_path / "pools.json"
|
||||||
|
out_path.write_text('{"split_mode":"global"}', encoding="utf-8")
|
||||||
|
manifest_path = tmp_path / "split_manifest.json"
|
||||||
|
|
||||||
|
with pytest.raises(FileExistsError, match="已存在冻结产物"):
|
||||||
|
_guard_frozen_products(out_path, manifest_path, force=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_split_force_backs_up_old(tmp_path):
|
||||||
|
"""force=True 时旧产物被备份为 .bak.* 再允许覆盖。"""
|
||||||
|
from app.harness.build_split import _guard_frozen_products
|
||||||
|
|
||||||
|
out_path = tmp_path / "pools.json"
|
||||||
|
out_path.write_text('{"old":1}', encoding="utf-8")
|
||||||
|
manifest_path = tmp_path / "split_manifest.json"
|
||||||
|
manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8")
|
||||||
|
|
||||||
|
_guard_frozen_products(out_path, manifest_path, force=True)
|
||||||
|
baks = list(tmp_path.glob("pools.json.bak.*"))
|
||||||
|
assert len(baks) == 1, f"未备份旧产物: {list(tmp_path.iterdir())}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_split_force_twice_same_suffix_keeps_both(tmp_path):
|
||||||
|
"""连续两次 force 命中同后缀时,第二次备份不覆盖第一次(追加递增序号)。"""
|
||||||
|
from app.harness.build_split import _guard_frozen_products
|
||||||
|
|
||||||
|
out_path = tmp_path / "pools.json"
|
||||||
|
manifest_path = tmp_path / "split_manifest.json"
|
||||||
|
|
||||||
|
# 第一次 force:两文件都无 manifest 里的 pools_sha256 之外的差异,后缀恒为同值。
|
||||||
|
out_path.write_text('{"gen":1}', encoding="utf-8")
|
||||||
|
manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8")
|
||||||
|
_guard_frozen_products(out_path, manifest_path, force=True)
|
||||||
|
|
||||||
|
# 第二次 force:写入相同 sha 的新产物,触发同后缀备份。
|
||||||
|
out_path.write_text('{"gen":2}', encoding="utf-8")
|
||||||
|
manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8")
|
||||||
|
_guard_frozen_products(out_path, manifest_path, force=True)
|
||||||
|
|
||||||
|
pools_baks = sorted(p.name for p in tmp_path.glob("pools.json.bak.*"))
|
||||||
|
assert len(pools_baks) == 2, f"同后缀第二次备份覆盖了第一次: {pools_baks}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_split_force_rollback_on_partial_failure(tmp_path, monkeypatch):
|
||||||
|
"""备份第二个文件失败时,第一个已备份文件被 rollback 回原名(原子性)。"""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from app.harness import build_split as bs
|
||||||
|
|
||||||
|
out_path = tmp_path / "pools.json"
|
||||||
|
manifest_path = tmp_path / "split_manifest.json"
|
||||||
|
out_path.write_text('{"gen":1}', encoding="utf-8")
|
||||||
|
manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8")
|
||||||
|
|
||||||
|
real_rename = Path.rename
|
||||||
|
calls = {"n": 0}
|
||||||
|
|
||||||
|
def flaky_rename(self, target):
|
||||||
|
# 第一次 rename(备份 pools.json)成功,第二次(备份 manifest)抛错。
|
||||||
|
calls["n"] += 1
|
||||||
|
if calls["n"] == 2:
|
||||||
|
raise OSError("模拟第二个备份 rename 失败")
|
||||||
|
return real_rename(self, target)
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "rename", flaky_rename)
|
||||||
|
|
||||||
|
with pytest.raises(OSError, match="模拟第二个备份"):
|
||||||
|
bs._guard_frozen_products(out_path, manifest_path, force=True)
|
||||||
|
|
||||||
|
# rollback 后:两原文件仍在原位,无残留 .bak.*
|
||||||
|
assert out_path.exists(), "第一个文件未被 rollback 回原名"
|
||||||
|
assert manifest_path.exists(), "第二个文件不应被移动"
|
||||||
|
assert not list(tmp_path.glob("*.bak.*")), f"残留半备份: {list(tmp_path.iterdir())}"
|
||||||
|
|||||||
Reference in New Issue
Block a user