docs: add 4 work-package implementation plans for preflight fixes
This commit is contained in:
@@ -245,6 +245,26 @@
|
||||
"id": "design:preflight-fixes",
|
||||
"label": "训练前缺陷修复设计",
|
||||
"type": "design"
|
||||
},
|
||||
{
|
||||
"id": "plan:preflight-wp1-asset-migration",
|
||||
"label": "WP1 资产迁移",
|
||||
"type": "plan"
|
||||
},
|
||||
{
|
||||
"id": "plan:preflight-wp2-split-wiring",
|
||||
"label": "WP2 切分与接线",
|
||||
"type": "plan"
|
||||
},
|
||||
{
|
||||
"id": "plan:preflight-wp3-train-loop",
|
||||
"label": "WP3 训练循环与进化引擎",
|
||||
"type": "plan"
|
||||
},
|
||||
{
|
||||
"id": "plan:preflight-wp4-resilience",
|
||||
"label": "WP4 韧性与持久化",
|
||||
"type": "plan"
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
@@ -492,6 +512,34 @@
|
||||
"relation": "implements",
|
||||
"evidence": "实现修复设计的 5 个 Task",
|
||||
"added": "2026-07-16T02:01:19.350441+00:00"
|
||||
},
|
||||
{
|
||||
"source": "plan:preflight-wp1-asset-migration",
|
||||
"target": "design:preflight-fixes",
|
||||
"relation": "implements",
|
||||
"evidence": "四工作包实现训练前缺陷修复设计",
|
||||
"added": "2026-07-16T08:32:42.737883+00:00"
|
||||
},
|
||||
{
|
||||
"source": "plan:preflight-wp2-split-wiring",
|
||||
"target": "design:preflight-fixes",
|
||||
"relation": "implements",
|
||||
"evidence": "四工作包实现训练前缺陷修复设计",
|
||||
"added": "2026-07-16T08:32:42.822569+00:00"
|
||||
},
|
||||
{
|
||||
"source": "plan:preflight-wp3-train-loop",
|
||||
"target": "design:preflight-fixes",
|
||||
"relation": "implements",
|
||||
"evidence": "四工作包实现训练前缺陷修复设计",
|
||||
"added": "2026-07-16T08:32:42.906881+00:00"
|
||||
},
|
||||
{
|
||||
"source": "plan:preflight-wp4-resilience",
|
||||
"target": "design:preflight-fixes",
|
||||
"relation": "implements",
|
||||
"evidence": "四工作包实现训练前缺陷修复设计",
|
||||
"added": "2026-07-16T08:32:42.990273+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
+10
-2
@@ -1,6 +1,6 @@
|
||||
# Research Wiki 索引
|
||||
|
||||
> 自动生成，更新时间：2026-07-16 07:43 UTC
|
||||
> 自动生成，更新时间：2026-07-16 08:32 UTC
|
||||
|
||||
## design (36)
|
||||
- [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design`
|
||||
@@ -52,7 +52,7 @@
|
||||
- [v3 §9 帧感知抽取机制 — 小样本实证验证结果](findings/2026-07-15-v3-frame-perception-spike-validation.md) `finding:2026-07-15-v3-frame-perception-spike-validation`
|
||||
- [出题范式转变 — 从"生成-打分-过滤"转向"构造优先 + 两正交独立信号"（六篇原文深读）](findings/2026-07-15-question-gen-paradigm-shift-construction-over-filtering.md) `finding:2026-07-15-question-gen-paradigm-shift-construction-over-filtering`
|
||||
|
||||
## plan (40)
|
||||
## plan (48)
|
||||
- [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm`
|
||||
- [2026-07-07-app-harness](plans/2026-07-07-app-harness.md) `plan:2026-07-07-app-harness`
|
||||
- [2026-07-07-core-evolution](plans/2026-07-07-core-evolution.md) `plan:2026-07-07-core-evolution`
|
||||
@@ -72,6 +72,10 @@
|
||||
- [2026-07-14-task-type-strategy-framework](plans/2026-07-14-task-type-strategy-framework.md) `plan:2026-07-14-task-type-strategy-framework`
|
||||
- [2026-07-15-question-gen-v3-phase1-contract](plans/2026-07-15-question-gen-v3-phase1-contract.md) `plan:2026-07-15-question-gen-v3-phase1-contract`
|
||||
- [2026-07-15-results-driven-video-split](plans/2026-07-15-results-driven-video-split.md) `plan:2026-07-15-results-driven-video-split`
|
||||
- [2026-07-16-preflight-wp1-asset-migration](plans/2026-07-16-preflight-wp1-asset-migration.md) `plan:2026-07-16-preflight-wp1-asset-migration`
|
||||
- [2026-07-16-preflight-wp2-split-wiring](plans/2026-07-16-preflight-wp2-split-wiring.md) `plan:2026-07-16-preflight-wp2-split-wiring`
|
||||
- [2026-07-16-preflight-wp3-train-loop](plans/2026-07-16-preflight-wp3-train-loop.md) `plan:2026-07-16-preflight-wp3-train-loop`
|
||||
- [2026-07-16-preflight-wp4-resilience](plans/2026-07-16-preflight-wp4-resilience.md) `plan:2026-07-16-preflight-wp4-resilience`
|
||||
- [Action Recognition 单题型首次训练实验计划](plans/action-recognition-training.md) `plan:action-recognition-training`
|
||||
- [ActionRecognitionStrategy 特化实现计划 (Plan B)](plans/action-recognition-strategy.md) `plan:action-recognition-strategy`
|
||||
- [Adversarial Question-Gen Phase B](plans/adversarial-question-gen-phaseB.md) `plan:adversarial-question-gen-phaseB`
|
||||
@@ -86,6 +90,10 @@
|
||||
- [Spec-1 Agent 执行环境修复实现计划](plans/agent-runtime-fixes-plan.md) `plan:agent-runtime-fixes-plan`
|
||||
- [Spec-2 建树批量并行实现计划](plans/batch-tree-build-plan.md) `plan:batch-tree-build-plan`
|
||||
- [TaskTypeStrategy 框架实现计划 (Plan A)](plans/task-type-strategy-framework.md) `plan:task-type-strategy-framework`
|
||||
- [WP1 资产迁移](plans/preflight-wp1-asset-migration.md) `plan:preflight-wp1-asset-migration`
|
||||
- [WP2 切分与接线](plans/preflight-wp2-split-wiring.md) `plan:preflight-wp2-split-wiring`
|
||||
- [WP3 训练循环与进化引擎](plans/preflight-wp3-train-loop.md) `plan:preflight-wp3-train-loop`
|
||||
- [WP4 韧性与持久化](plans/preflight-wp4-resilience.md) `plan:preflight-wp4-resilience`
|
||||
- [出题管线 v2 实现计划](plans/2026-07-11-question-gen-v2.md) `plan:2026-07-11-question-gen-v2`
|
||||
- [实现计划: 修复诊断 tree_data 断链 bug](plans/fix-diagnosis-tree-data-link-plan.md) `plan:fix-diagnosis-tree-data-link-plan`
|
||||
- [建树修复管线三项改造实现计划](plans/tree-repair-resilience.md) `plan:tree-repair-resilience`
|
||||
|
||||
@@ -123,3 +123,12 @@
|
||||
- [2026-07-16 02:01 UTC] 重建索引: 98 篇页面
|
||||
- [2026-07-16 07:43 UTC] 新增 design: 训练前缺陷修复设计 (design:preflight-fixes)
|
||||
- [2026-07-16 07:43 UTC] 重建索引: 101 篇页面
|
||||
- [2026-07-16 08:32 UTC] 新增 plan: WP1 资产迁移 (plan:preflight-wp1-asset-migration)
|
||||
- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp1-asset-migration --implements--> design:preflight-fixes
|
||||
- [2026-07-16 08:32 UTC] 新增 plan: WP2 切分与接线 (plan:preflight-wp2-split-wiring)
|
||||
- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp2-split-wiring --implements--> design:preflight-fixes
|
||||
- [2026-07-16 08:32 UTC] 新增 plan: WP3 训练循环与进化引擎 (plan:preflight-wp3-train-loop)
|
||||
- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp3-train-loop --implements--> design:preflight-fixes
|
||||
- [2026-07-16 08:32 UTC] 新增 plan: WP4 韧性与持久化 (plan:preflight-wp4-resilience)
|
||||
- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp4-resilience --implements--> design:preflight-fixes
|
||||
- [2026-07-16 08:32 UTC] 重建索引: 109 篇页面
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
# WP1 资产迁移 Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** 迁移 5 个遗漏的 TRM4 进化/动量模板到 TRM5，删除 2 个死字段，把加载器的静默空串兜底改为 fail-loud，解锁自进化引擎。
|
||||
|
||||
**Architecture:** 进化引导 prompt 是引擎的一部分（放项目根 `prompts/`，不参与版本化进化）。TRM4 五模板的输出 JSON 契约与 TRM5 解析代码已核实完全对齐，可直接拷贝。执行顺序：先迁移模板 → 删死字段（`consolidate_system`/`span_eval_user`，零消费）→ 加载器 fail-loud（顺序关键：先删死字段，fail-loud 才不会对不存在也不需要的模板报错）。
|
||||
|
||||
**Tech Stack:** Python 3.11、pytest、frozen dataclass（`core/evolution/types.py`）。
|
||||
|
||||
**设计源**：`research-wiki/designs/2026-07-16-preflight-fixes-design.md §4`
|
||||
|
||||
---
|
||||
|
||||
## 关键锚点（实现前必读）
|
||||
|
||||
| 用途 | 位置 |
|
||||
|------|------|
|
||||
| TRM4 源模板 | `/home/iomgaa/Projects/Video-Tree-TRM4/prompts/{evolve_skill,evolve_system,evolve_tool,evolve_rank,slow_momentum}.md` |
|
||||
| evolve 加载器 | `app/harness/runner.py:2266-2280` `_load_evolve_prompts` |
|
||||
| diagnose 加载器 | `app/harness/runner.py:2282-2299` `_load_diagnose_prompts` |
|
||||
| 平行 diagnose 加载器 | `app/harness/video_split_cli.py:362-378` `_load_diagnose_prompts` |
|
||||
| dataclass 定义 | `core/evolution/types.py:477-522`（`DiagnosePrompts` L494-501 / `EvolvePrompts` L518-522） |
|
||||
| 死字段 `consolidate_system` 内联替代 | `core/evolution/evolve.py:997` `_CONSOLIDATE_SYSTEM`（消费点 L1036） |
|
||||
| 死字段 `span_eval_user` 无消费 | diagnose 只用 `prompts.span_eval_system`（`core/evolution/diagnose.py:511`），user_prompt 内联构造 |
|
||||
| 测试 fixture | `test_evolve.py:682` `consolidate_system="cons"`；`test_diagnose.py:753` `span_eval_user=""`；`test_evolution_types.py:352` `span_eval_user="p4"` / `:370` `consolidate_system="consolidate_tmpl"` |
|
||||
|
||||
## 核心算法保真校验
|
||||
|
||||
本计划不迁移/不改核心算法（ARCHITECTURE §6），只搬运模板文件 + 清理死字段 + 加 fail-loud。模板内容是 evolve 引擎（算法 #8）的输入数据，非算法逻辑本身；迁移已核实输出契约（`suggestions`/`edits`/`edits_extract`/`edits_verify`/`selected_indices`/`slow_update_content`）与 TRM5 解析代码逐字对齐。**保真检查点（Task 1 Step 4）**：加载 evolve_tool.md 后确认其要求 LLM 返回 `edits_extract`+`edits_verify` 双键（对齐 `evolve.py:1433-1435`）。
|
||||
|
||||
---
|
||||
|
||||
## Task 1: 迁移 5 个 TRM4 模板
|
||||
|
||||
**Files:**
|
||||
- Create: `prompts/evolve_skill.md`、`prompts/evolve_system.md`、`prompts/evolve_tool.md`、`prompts/evolve_rank.md`、`prompts/slow_momentum.md`（从 TRM4 拷贝）
|
||||
- Test: `tests/unit/test_evolve_prompts_present.py`
|
||||
|
||||
- [ ] **Step 1: 拷贝 5 个模板**
|
||||
|
||||
Run:
|
||||
```bash
|
||||
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_skill.md prompts/evolve_skill.md
|
||||
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_system.md prompts/evolve_system.md
|
||||
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_tool.md prompts/evolve_tool.md
|
||||
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_rank.md prompts/evolve_rank.md
|
||||
cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/slow_momentum.md prompts/slow_momentum.md
|
||||
```
|
||||
Expected: 5 文件存在于 `prompts/`。
|
||||
|
||||
- [ ] **Step 2: 写契约冒烟测试（先失败）**
|
||||
|
||||
`tests/unit/test_evolve_prompts_present.py`：
|
||||
```python
|
||||
"""校验 5 个进化/动量模板存在且输出契约关键词与解析代码对齐。"""
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_PROMPTS_DIR = Path("prompts")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name, required_tokens",
|
||||
[
|
||||
("evolve_skill.md", ["suggestions", "edits"]),
|
||||
("evolve_system.md", ["suggestions", "edits"]),
|
||||
("evolve_tool.md", ["edits_extract", "edits_verify"]),
|
||||
("evolve_rank.md", ["selected_indices"]),
|
||||
("slow_momentum.md", ["slow_update_content"]),
|
||||
],
|
||||
)
|
||||
def test_evolve_template_present_and_contract(name: str, required_tokens: list[str]) -> None:
|
||||
path = _PROMPTS_DIR / name
|
||||
assert path.exists(), f"缺模板: {path}"
|
||||
text = path.read_text(encoding="utf-8")
|
||||
assert text.strip(), f"模板为空: {path}"
|
||||
for token in required_tokens:
|
||||
assert token in text, f"{name} 缺输出契约关键词 {token!r}（与解析代码不对齐）"
|
||||
```
|
||||
|
||||
- [ ] **Step 3: 运行确认通过（模板已在 Step 1 拷入）**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v`
|
||||
Expected: 5 参数化用例全 PASS。若 evolve_tool.md 缺 `edits_extract`/`edits_verify` 则契约不符——停止并逐行比对 TRM4 源。
|
||||
|
||||
- [ ] **Step 4: 保真检查点**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -c "print('edits_extract' in open('prompts/evolve_tool.md').read() and 'edits_verify' in open('prompts/evolve_tool.md').read())"`
|
||||
Expected: `True`（对齐 `evolve.py:1433-1435` 的 `parsed["edits_extract"]`/`parsed["edits_verify"]`）。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add prompts/evolve_skill.md prompts/evolve_system.md prompts/evolve_tool.md prompts/evolve_rank.md prompts/slow_momentum.md tests/unit/test_evolve_prompts_present.py
|
||||
git commit -m "feat: migrate 5 evolve/momentum templates from TRM4 (algo #8)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: 删除 2 个死字段（consolidate_system / span_eval_user）
|
||||
|
||||
**Files:**
|
||||
- Modify: `core/evolution/types.py:494-501,518-522`
|
||||
- Modify: `app/harness/runner.py:2279,2294`
|
||||
- Modify: `app/harness/video_split_cli.py:374`
|
||||
- Modify: `tests/unit/test_evolve.py:682`、`tests/unit/test_diagnose.py:753`、`tests/unit/test_evolution_types.py:352,370`
|
||||
|
||||
- [ ] **Step 1: 删 dataclass 字段与 docstring**
|
||||
|
||||
`core/evolution/types.py` — `DiagnosePrompts` 删 `span_eval_user`：
|
||||
- docstring 删行 ` span_eval_user: span 评估用户提示模板。`（L487）
|
||||
- 字段删行 ` span_eval_user: str`（L497）
|
||||
|
||||
`EvolvePrompts` 删 `consolidate_system`：
|
||||
- docstring 删行 ` consolidate_system: appendix 压缩系统提示。`（L515）
|
||||
- 字段删行 ` consolidate_system: str`（L522）
|
||||
|
||||
- [ ] **Step 2: 删加载器对死字段的 `_read` 行**
|
||||
|
||||
`app/harness/runner.py`：
|
||||
- `_load_evolve_prompts` 删行 ` consolidate_system=_read("consolidate_system.md"),`（L2279）
|
||||
- `_load_diagnose_prompts` 删行 ` span_eval_user=_read("span_eval_user.md"),`（L2294）
|
||||
|
||||
`app/harness/video_split_cli.py`：
|
||||
- `_load_diagnose_prompts` 删行 ` span_eval_user=_read("span_eval_user.md"),`（L374）
|
||||
|
||||
- [ ] **Step 3: 删测试 fixture 对死字段的赋值**
|
||||
|
||||
- `tests/unit/test_evolve.py:682` 删行 ` consolidate_system="cons",`
|
||||
- `tests/unit/test_diagnose.py:753` 删行 ` span_eval_user="",`
|
||||
- `tests/unit/test_evolution_types.py:352` 删行 ` span_eval_user="p4",`
|
||||
- `tests/unit/test_evolution_types.py:370` 删行 ` consolidate_system="consolidate_tmpl",`
|
||||
|
||||
注：`test_evolution_types.py` 可能有断言逐字段比对；若该测试断言字段数量或逐字段相等，同步移除对两个死字段的断言（读该测试确认，删净引用）。
|
||||
|
||||
- [ ] **Step 4: 运行相关测试确认通过**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py -q`
|
||||
Expected: 全 PASS（无 `TypeError: unexpected keyword argument` / 无 `missing positional argument`）。
|
||||
|
||||
- [ ] **Step 5: 全库确认无残留引用**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -c "import subprocess,sys; r=subprocess.run(['grep','-rn','consolidate_system\|span_eval_user','core/','app/','tests/'],capture_output=True,text=True); print(r.stdout)"`
|
||||
Expected: 空输出（`consolidate_appendix` 用内联 `_CONSOLIDATE_SYSTEM` 不算 `consolidate_system` 字段引用；若出现请确认非 dataclass 字段引用）。
|
||||
|
||||
- [ ] **Step 6: 提交**
|
||||
|
||||
```bash
|
||||
git add core/evolution/types.py app/harness/runner.py app/harness/video_split_cli.py tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py
|
||||
git commit -m "refactor: drop dead prompt fields consolidate_system/span_eval_user"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: 加载器 fail-loud（缺模板即报错）
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/runner.py:2270-2272,2286-2288`
|
||||
- Modify: `app/harness/video_split_cli.py:365-367`
|
||||
- Test: `tests/unit/test_evolve_prompts_present.py`（追加）
|
||||
|
||||
- [ ] **Step 1: 追加 fail-loud 测试（先失败）**
|
||||
|
||||
在 `tests/unit/test_evolve_prompts_present.py` 追加：
|
||||
```python
|
||||
def test_loader_fail_loud_on_missing_template(tmp_path, monkeypatch):
|
||||
"""加载器遇缺失模板必须抛 FileNotFoundError，不静默返回空串。"""
|
||||
monkeypatch.chdir(tmp_path) # 空目录下无 prompts/，_read 应 fail-loud
|
||||
(tmp_path / "prompts").mkdir()
|
||||
from app.harness.runner import Runner
|
||||
|
||||
# 直接测私有 _read 语义：借最小构造不便，改测行为——缺文件应 raise
|
||||
def _read(name: str) -> str:
|
||||
p = Path("prompts") / name
|
||||
if not p.exists():
|
||||
raise FileNotFoundError(f"缺进化/诊断模板: {p}（请从 TRM4 迁移或检查 prompts/）")
|
||||
return p.read_text(encoding="utf-8")
|
||||
|
||||
with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"):
|
||||
_read("evolve_skill.md")
|
||||
```
|
||||
|
||||
> 说明：`_read` 是嵌套闭包不易直接导入，此测试固化期望语义（缺文件 raise + 消息含"缺进化/诊断模板"）；Step 2 把三处闭包改成同款实现，集成层由 `test_harness_runner.py` 现有加载路径覆盖。
|
||||
|
||||
- [ ] **Step 2: 三处 `_read` 闭包改 fail-loud**
|
||||
|
||||
`app/harness/runner.py` `_load_evolve_prompts`（L2270-2272）与 `_load_diagnose_prompts`（L2286-2288），以及 `app/harness/video_split_cli.py` `_load_diagnose_prompts`（L365-367），把：
|
||||
```python
|
||||
def _read(name: str) -> str:
|
||||
p = Path("prompts") / name
|
||||
return p.read_text(encoding="utf-8") if p.exists() else ""
|
||||
```
|
||||
改为：
|
||||
```python
|
||||
def _read(name: str) -> str:
|
||||
p = Path("prompts") / name
|
||||
if not p.exists():
|
||||
raise FileNotFoundError(f"缺进化/诊断模板: {p}（请从 TRM4 迁移或检查 prompts/）")
|
||||
return p.read_text(encoding="utf-8")
|
||||
```
|
||||
（runner.py 缩进 12 空格；video_split_cli.py 的 `_read` 缩进按其函数体，见 L365 为 8 空格——按各自现场缩进套用。）
|
||||
|
||||
- [ ] **Step 3: 运行测试确认通过**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v`
|
||||
Expected: 全 PASS。
|
||||
|
||||
- [ ] **Step 4: 运行 runner 加载路径回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -q`
|
||||
Expected: 全 PASS（模板已迁移，真实加载不再命中缺失分支）。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/runner.py app/harness/video_split_cli.py tests/unit/test_evolve_prompts_present.py
|
||||
git commit -m "fix: fail-loud on missing evolve/diagnose templates (no silent empty)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review（作者自查，执行者复核）
|
||||
|
||||
- [ ] 5 模板均已迁移且契约测试覆盖关键字段。
|
||||
- [ ] `consolidate_system`/`span_eval_user` 在 core/app/tests 全库无残留字段引用。
|
||||
- [ ] 三处 loader（runner 两处 + video_split_cli 一处）均已 fail-loud。
|
||||
- [ ] 执行顺序正确：Task 2（删死字段）先于 Task 3（fail-loud），避免对不需要的模板报错。
|
||||
|
||||
## 验收标准
|
||||
|
||||
1. `pytest tests/unit/test_evolve_prompts_present.py tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py tests/unit/test_harness_runner.py` 全绿。
|
||||
2. `grep -rn 'consolidate_system\|span_eval_user' core/ app/ tests/` 无 dataclass 字段残留。
|
||||
3. `prompts/` 下 5 个新模板存在且非空。
|
||||
@@ -0,0 +1,626 @@
|
||||
# WP2 切分与接线 Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** 让冻结的 video-split 切分能正确进入训练 workspace（seed 携带 pools.json + global 一致性校验），并把切分质量三处优化（val_ratio 0.4、tier 感知 diag/val 分配、val 功效修复）与冻结产物覆盖保护落地。
|
||||
|
||||
**Architecture:** 切分产物由 `video_split_cli` 冻结到 `workspaces/video-split/`；本 WP 让 seed 携带该产物、训练 fresh 时拷入 workspace 并校验一致性。tier 感知在 `_split_trainval_by_video_group` 内实现——错题视频组按 T2(defect) 含量升序进 val（保留 T2 高的组在 diag），并把 `val_wrong_min` 前置到切分内做功效修复（不足则从 diag 换出低 T2 错题组补 val，耗尽 fail-loud）。
|
||||
|
||||
**Tech Stack:** Python 3.11、pytest、SQLite、frozen dataclass、shutil、原子写（tmp+os.replace）。
|
||||
|
||||
**设计源**：`research-wiki/designs/2026-07-16-preflight-fixes-design.md §5`
|
||||
|
||||
---
|
||||
|
||||
## 关键锚点（实现前必读）
|
||||
|
||||
| 用途 | 位置 |
|
||||
|------|------|
|
||||
| trainval→diag/val 切分 | `app/harness/pools.py:126-199` `split_by_video_assignment`；`:289-338` `_split_trainval_by_video_group`；`:118` `InsufficientValSignal` |
|
||||
| global 加载（无校验） | `app/harness/pools.py:713-813` `build_or_load_pools`（L813 `return load_pools` 前无 global 校验）；`:587-620` `load_pools` |
|
||||
| 冻结编排 | `app/harness/build_split.py:104-219` `build_split`（signal_rows 含 tier L154；split_by_video_assignment 调用 L184-191；save_pools L192）；`:46-71` `SplitBuildConfig`（无 val_wrong_min） |
|
||||
| CLI 构造 | `app/harness/video_split_cli.py:559-577` `SplitBuildConfig(...)`；`:580` `check_mcnemar_power`；`:751-773` `build_arg_parser`（无 --force） |
|
||||
| seed | `app/harness/store.py:184-232` `init_seed`（拷 skills/prompts/baseline.db，不拷 pools）；`:269-307` `extract_run_db`（不去重） |
|
||||
| workspace | `app/harness/workspace.py:156-200` `init_workspace_from_seed`（copy2 baseline.db→harness.db L197，不拷 pools） |
|
||||
| manifest | `app/harness/split_manifest.py:19-59` `write_manifest`（pools_sha256 L54） |
|
||||
| 配置 | `config/video_split.yaml`（val_ratio L15=0.3、val_wrong_min L14=20） |
|
||||
| 测试 | `tests/unit/test_pools_video_atomic.py`、`test_split_selection.py`、`test_harness_pools.py`、`test_harness_store.py`、`test_harness_workspace.py` |
|
||||
|
||||
## 核心算法保真校验
|
||||
|
||||
触及算法 #5（信息阶梯）的**上游输入**：本 WP 只改"哪些视频进 diag/val"，pools 内仍是逐 unit 列表，`gate_ladder` 消费的 unit+correctness 结构不变。**保真检查点（Task 3 Step 6）**：确认 `_split_trainval_by_video_group` 返回后 diagnosis/validation 仍是逐题 `GeneratedQuestion` 列表、视频组原子性（同 video 全部题同池）不被 tier 排序破坏。不改算法 #6/#9。
|
||||
|
||||
---
|
||||
|
||||
## Task 1: val_ratio 0.3 → 0.4
|
||||
|
||||
**Files:**
|
||||
- Modify: `config/video_split.yaml:15`
|
||||
|
||||
- [ ] **Step 1: 改配置**
|
||||
|
||||
`config/video_split.yaml` 第 15 行：
|
||||
```yaml
|
||||
val_ratio: 0.3 # validation 占 trainval 视频组总数的比例
|
||||
```
|
||||
改为：
|
||||
```yaml
|
||||
val_ratio: 0.4 # validation 占 trainval 视频组总数的比例（0.3→0.4 提升整包终审功效，WP2）
|
||||
```
|
||||
|
||||
- [ ] **Step 2: 提交**
|
||||
|
||||
```bash
|
||||
git add config/video_split.yaml
|
||||
git commit -m "chore: bump video_split val_ratio 0.3->0.4 for terminal-eval power"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: extract_run_db 每题去重（902→900 canonical）
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/store.py:269-307`
|
||||
- Test: `tests/unit/test_harness_store.py`（`TestExtractRunDb`）
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
在 `tests/unit/test_harness_store.py` 的 `TestExtractRunDb` 类追加：
|
||||
```python
|
||||
def test_dedupe_per_question_keeps_first_row(self, tmp_path):
|
||||
"""dedupe_per_question=True 时每 question_id 只保留 rowid 最小的首行。"""
|
||||
import sqlite3
|
||||
|
||||
src = tmp_path / "src.db"
|
||||
conn = sqlite3.connect(src)
|
||||
conn.execute(
|
||||
"CREATE TABLE _runs (run_id TEXT PRIMARY KEY, started_at TEXT)"
|
||||
)
|
||||
conn.execute("INSERT INTO _runs VALUES ('r1', 't0')")
|
||||
conn.execute(
|
||||
"CREATE TABLE predictions (run_id TEXT, question_id TEXT, prediction TEXT)"
|
||||
)
|
||||
# 743-1 三行（模拟 error/budget/finished），首行 prediction=NULL
|
||||
conn.executemany(
|
||||
"INSERT INTO predictions VALUES (?,?,?)",
|
||||
[
|
||||
("r1", "743-1", None),
|
||||
("r1", "743-1", None),
|
||||
("r1", "743-1", "C"),
|
||||
("r1", "q2", "A"),
|
||||
],
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
dst = tmp_path / "dst.db"
|
||||
from app.harness.store import extract_run_db
|
||||
|
||||
extract_run_db(src, dst, "r1", dedupe_per_question=True)
|
||||
|
||||
out = sqlite3.connect(dst)
|
||||
rows = out.execute(
|
||||
"SELECT question_id, prediction FROM predictions ORDER BY question_id"
|
||||
).fetchall()
|
||||
out.close()
|
||||
assert rows == [("743-1", None), ("q2", "A")], f"未按 rowid 首行去重: {rows}"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py::TestExtractRunDb::test_dedupe_per_question_keeps_first_row -v`
|
||||
Expected: FAIL（`extract_run_db() got an unexpected keyword argument 'dedupe_per_question'`）。
|
||||
|
||||
- [ ] **Step 3: 实现去重**
|
||||
|
||||
`app/harness/store.py` `extract_run_db` 签名改为：
|
||||
```python
|
||||
def extract_run_db(
|
||||
src_db: Path, dst_db: Path, run_id: str, *, dedupe_per_question: bool = False
|
||||
) -> None:
|
||||
```
|
||||
docstring 补一句参数说明：
|
||||
```
|
||||
dedupe_per_question: True 时 predictions 表每 question_id 仅保留 rowid 最小
|
||||
的首行（对齐 canonical「每 question_id 取第一行 ORDER BY rowid」口径，
|
||||
902→900）。_runs 表不受影响。
|
||||
```
|
||||
把 predictions 分支的取行 SQL（L297-299）改为按 `dedupe_per_question` 分派：
|
||||
```python
|
||||
if table == "predictions" and dedupe_per_question:
|
||||
rows = src.execute(
|
||||
f"SELECT {col_sql} FROM {table} WHERE run_id=? "
|
||||
"AND rowid IN (SELECT MIN(rowid) FROM predictions "
|
||||
"WHERE run_id=? GROUP BY question_id)",
|
||||
(run_id, run_id),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = src.execute(
|
||||
f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,)
|
||||
).fetchall()
|
||||
```
|
||||
|
||||
- [ ] **Step 4: 运行确认通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py -q`
|
||||
Expected: 全 PASS（默认 `dedupe_per_question=False` 保持既有行为，旧测试不受影响）。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/store.py tests/unit/test_harness_store.py
|
||||
git commit -m "feat: add dedupe_per_question to extract_run_db (canonical 902->900)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: tier 感知 + val 功效修复的 diag/val 分配
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/pools.py:126-199,289-338`
|
||||
- Modify: `app/harness/build_split.py:46-71,181-192`
|
||||
- Modify: `app/harness/video_split_cli.py:565-573`
|
||||
- Test: `tests/unit/test_pools_video_atomic.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试（tier 优先 + 功效修复）**
|
||||
|
||||
在 `tests/unit/test_pools_video_atomic.py` 追加：
|
||||
```python
|
||||
def test_tier_aware_keeps_high_t2_in_diag():
|
||||
"""错题视频组按 T2 含量升序进 val：T2 高的组保留在 diagnosis。"""
|
||||
from app.harness.pools import split_by_video_assignment
|
||||
from app.question_gen.types import GeneratedQuestion
|
||||
|
||||
def _q(qid, vid):
|
||||
return GeneratedQuestion(
|
||||
question_id=qid, video_id=vid, task_type="X", question="q",
|
||||
options=["A", "B"], answer="A", source_nodes=[], difficulty="easy",
|
||||
)
|
||||
|
||||
# 4 个错题视频（每视频 1 题），T2 数分别 2/1/0/0
|
||||
questions = [_q(f"{v}-1", v) for v in ("vA", "vB", "vC", "vD")]
|
||||
assignment = {v: "trainval" for v in ("vA", "vB", "vC", "vD")}
|
||||
correctness = {f"{v}-1": False for v in ("vA", "vB", "vC", "vD")}
|
||||
wrong_tier = {"vA": 2, "vB": 1, "vC": 0, "vD": 0}
|
||||
|
||||
pools = split_by_video_assignment(
|
||||
questions, assignment, correctness, val_ratio=0.5, seed=7,
|
||||
wrong_tier_by_video=wrong_tier,
|
||||
)
|
||||
diag_vids = {q.video_id for q in pools.diagnosis}
|
||||
# T2 最高的 vA 必留 diag；T2=0 的组优先进 val
|
||||
assert "vA" in diag_vids
|
||||
assert "vB" in diag_vids
|
||||
|
||||
|
||||
def test_val_wrong_min_repair_pulls_from_diag():
|
||||
"""val 错题不足 val_wrong_min 时从 diag 换入低 T2 错题组补足。"""
|
||||
from app.harness.pools import split_by_video_assignment
|
||||
from app.question_gen.types import GeneratedQuestion
|
||||
|
||||
def _q(qid, vid, correct):
|
||||
return GeneratedQuestion(
|
||||
question_id=qid, video_id=vid, task_type="X", question="q",
|
||||
options=["A", "B"], answer="A", source_nodes=[], difficulty="easy",
|
||||
)
|
||||
|
||||
# 8 错题视频 + 2 正确视频；val_ratio 小使初分 val 错题不足，触发修复
|
||||
vids_wrong = [f"w{i}" for i in range(8)]
|
||||
vids_correct = ["c0", "c1"]
|
||||
questions = [_q(f"{v}-1", v, False) for v in vids_wrong] + [
|
||||
_q(f"{v}-1", v, True) for v in vids_correct
|
||||
]
|
||||
assignment = {v: "trainval" for v in vids_wrong + vids_correct}
|
||||
correctness = {f"{v}-1": False for v in vids_wrong}
|
||||
correctness.update({f"{v}-1": True for v in vids_correct})
|
||||
wrong_tier = {v: i for i, v in enumerate(vids_wrong)} # 递增 T2
|
||||
|
||||
pools = split_by_video_assignment(
|
||||
questions, assignment, correctness, val_ratio=0.1, seed=7,
|
||||
wrong_tier_by_video=wrong_tier, val_wrong_min=4,
|
||||
)
|
||||
val_wrong = sum(1 for q in pools.validation if not correctness[q.question_id])
|
||||
assert val_wrong >= 4, f"功效修复后 val 错题 {val_wrong} < 4"
|
||||
```
|
||||
|
||||
> 注：`GeneratedQuestion` 的真实字段以 `app/question_gen/types.py` 为准；若构造签名不符，读该文件对齐必填字段（勿臆造）。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_pools_video_atomic.py -k "tier_aware or val_wrong_min_repair" -v`
|
||||
Expected: FAIL（`unexpected keyword argument 'wrong_tier_by_video'`）。
|
||||
|
||||
- [ ] **Step 3: 改 `_split_trainval_by_video_group` 加 tier 感知 + 功效修复**
|
||||
|
||||
`app/harness/pools.py` 函数签名改为：
|
||||
```python
|
||||
def _split_trainval_by_video_group(
|
||||
trainval_qs: list[GeneratedQuestion],
|
||||
correctness: dict[str, bool],
|
||||
val_ratio: float,
|
||||
rng: random.Random,
|
||||
wrong_tier_by_video: dict[str, int] | None = None,
|
||||
val_wrong_min: int = 0,
|
||||
) -> tuple[list[GeneratedQuestion], list[GeneratedQuestion]]:
|
||||
```
|
||||
把分层块（L329-334 的 else 分支）改为 tier 感知：`wrong_vids` 按 T2 含量升序（T2 少的优先进 val），保留 T2 高的组在 diag；`wrong_tier_by_video=None` 时退化为原 shuffle：
|
||||
```python
|
||||
else:
|
||||
val_correct = math.floor(n_correct * n_val / n_total)
|
||||
val_wrong = n_val - val_correct
|
||||
rng.shuffle(correct_vids)
|
||||
if wrong_tier_by_video is None:
|
||||
rng.shuffle(wrong_vids)
|
||||
else:
|
||||
# T2 少的错题组优先进 val（保留 T2 高的组在 diag），确定性排序
|
||||
wrong_vids.sort(key=lambda v: (wrong_tier_by_video.get(v, 0), v))
|
||||
val_vids = set(correct_vids[:val_correct] + wrong_vids[:val_wrong])
|
||||
```
|
||||
在 `val_vids` 确定后、返回前，加**功效修复**（从 diag 的错题组按 T2 升序补入 val 直到满足 val_wrong_min）：
|
||||
```python
|
||||
if val_wrong_min > 0:
|
||||
val_wrong_now = sum(
|
||||
1 for v in val_vids for q in groups[v] if not correctness[q.question_id]
|
||||
)
|
||||
# diag 侧仍在的错题组，按 T2 升序（低价值优先移交 val）
|
||||
diag_wrong_pool = sorted(
|
||||
(v for v in wrong_vids if v not in val_vids),
|
||||
key=lambda v: ((wrong_tier_by_video or {}).get(v, 0), v),
|
||||
)
|
||||
for v in diag_wrong_pool:
|
||||
if val_wrong_now >= val_wrong_min:
|
||||
break
|
||||
val_vids.add(v)
|
||||
val_wrong_now += sum(1 for q in groups[v] if not correctness[q.question_id])
|
||||
if val_wrong_now < val_wrong_min:
|
||||
raise InsufficientValSignal(
|
||||
f"trainval 错题不足以让 val 达到 val_wrong_min={val_wrong_min}"
|
||||
f"（修复后仅 {val_wrong_now}），请放大 val_ratio 或调整 trainval 归属。"
|
||||
)
|
||||
```
|
||||
（`InsufficientValSignal` 已在 pools.py:118 定义，无需新增；需确认函数内可见 `math`/`defaultdict`，文件顶部已 import。）
|
||||
|
||||
- [ ] **Step 4: `split_by_video_assignment` 透传新参数**
|
||||
|
||||
`app/harness/pools.py` `split_by_video_assignment` 签名加 `wrong_tier_by_video: dict[str, int] | None = None`（放在 `val_wrong_min` 之后），并把 `_split_trainval_by_video_group` 调用（L175-177）改为：
|
||||
```python
|
||||
diagnosis, validation = _split_trainval_by_video_group(
|
||||
trainval_qs, correctness, val_ratio, random.Random(seed),
|
||||
wrong_tier_by_video=wrong_tier_by_video,
|
||||
val_wrong_min=val_wrong_min,
|
||||
)
|
||||
```
|
||||
删除原 L179-186 的独立 `val_wrong_min` 事后校验块（功效已在 `_split_trainval_by_video_group` 内保证，避免重复校验语义）。docstring 的 `val_wrong_min` 说明改为"切分时保证（不足则从 diag 换入低 T2 错题组补足，耗尽 fail-loud）"。
|
||||
|
||||
- [ ] **Step 5: build_split 计算并传入 tier + val_wrong_min**
|
||||
|
||||
`app/harness/build_split.py`：`SplitBuildConfig` 加字段 `val_wrong_min: int`（放 `split_seed` 之后，docstring 补"validation 池最少错题数，切分时保证功效"）。build_split Phase 3（L182-191）改为：
|
||||
```python
|
||||
questions = load_benchmark(questions_dir)
|
||||
correctness = {pred["question_id"]: pred["correct"] for pred in preds}
|
||||
tier_by_q = {row["question_id"]: row["tier"] for row in signal_rows}
|
||||
wrong_tier_by_video: dict[str, int] = defaultdict(int)
|
||||
for pred in preds:
|
||||
if not pred["correct"] and tier_by_q.get(pred["question_id"]) == "T2":
|
||||
wrong_tier_by_video[pred["video_id"]] += 1
|
||||
pools = split_by_video_assignment(
|
||||
questions,
|
||||
assignment,
|
||||
correctness,
|
||||
config.val_ratio,
|
||||
config.split_seed,
|
||||
baseline_run_id=baseline_run_id,
|
||||
val_wrong_min=config.val_wrong_min,
|
||||
wrong_tier_by_video=dict(wrong_tier_by_video),
|
||||
)
|
||||
```
|
||||
更新 build_split docstring 的"契约（Task 11...）"段：删除"有意保持 val_wrong_min-agnostic"表述，改为"val_wrong_min 前置到切分内保证功效；CLI 的 check_mcnemar_power 作冗余最终确认"。确认 `defaultdict` 已 import（`from collections import Counter, defaultdict`）。
|
||||
|
||||
- [ ] **Step 6: CLI 传 val_wrong_min + 保真检查**
|
||||
|
||||
`app/harness/video_split_cli.py` 的 `SplitBuildConfig(...)`（L565-573）加一行 `val_wrong_min=config.val_wrong_min,`。
|
||||
保真检查点：确认 `pools.diagnosis`/`pools.validation` 仍是逐题 `GeneratedQuestion` 列表、同 video 全部题同池（`test_pools_video_atomic.py::test_video_group_atomic_in_trainval_split` 覆盖）。
|
||||
|
||||
- [ ] **Step 7: 运行测试确认通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_pools_video_atomic.py tests/unit/test_harness_pools.py tests/unit/test_split_selection.py -q`
|
||||
Expected: 全 PASS。
|
||||
|
||||
- [ ] **Step 8: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/pools.py app/harness/build_split.py app/harness/video_split_cli.py tests/unit/test_pools_video_atomic.py
|
||||
git commit -m "feat: tier-aware diag/val split with val-power repair (design 5.1)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: 冻结产物覆盖保护 + --force
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/build_split.py:104-192`
|
||||
- Modify: `app/harness/video_split_cli.py:751-773`（build_arg_parser）+ run_pipeline 传参
|
||||
- Test: `tests/unit/test_video_split_cli.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
在 `tests/unit/test_video_split_cli.py` 追加（用最小 build_split 覆盖场景，或直接测保护函数）：
|
||||
```python
|
||||
def test_build_split_refuses_overwrite_without_force(tmp_path):
|
||||
"""已存在指纹不同的 pools.json 时，force=False 必须报错不覆盖。"""
|
||||
from app.harness.build_split import _guard_frozen_products
|
||||
|
||||
out_path = tmp_path / "pools.json"
|
||||
out_path.write_text('{"split_mode":"global"}', encoding="utf-8")
|
||||
manifest_path = tmp_path / "split_manifest.json"
|
||||
|
||||
with pytest.raises(FileExistsError, match="已存在冻结产物"):
|
||||
_guard_frozen_products(out_path, manifest_path, force=False)
|
||||
|
||||
|
||||
def test_build_split_force_backs_up_old(tmp_path):
|
||||
"""force=True 时旧产物被备份为 .bak.* 再允许覆盖。"""
|
||||
from app.harness.build_split import _guard_frozen_products
|
||||
|
||||
out_path = tmp_path / "pools.json"
|
||||
out_path.write_text('{"old":1}', encoding="utf-8")
|
||||
manifest_path = tmp_path / "split_manifest.json"
|
||||
manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8")
|
||||
|
||||
_guard_frozen_products(out_path, manifest_path, force=True)
|
||||
baks = list(tmp_path.glob("pools.json.bak.*"))
|
||||
assert len(baks) == 1, f"未备份旧产物: {list(tmp_path.iterdir())}"
|
||||
```
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -k "refuses_overwrite or force_backs_up" -v`
|
||||
Expected: FAIL（`cannot import name '_guard_frozen_products'`）。
|
||||
|
||||
- [ ] **Step 3: 实现覆盖保护函数**
|
||||
|
||||
`app/harness/build_split.py` 顶部 import 区确认有 `import shutil`（无则加）。新增函数（放 build_split 之前）：
|
||||
```python
|
||||
def _guard_frozen_products(out_path: Path, manifest_path: Path, *, force: bool) -> None:
|
||||
"""冻结前的覆盖保护：产物已存在时按 force 决定报错或备份。
|
||||
|
||||
参数:
|
||||
out_path: 目标 pools.json 路径。
|
||||
manifest_path: 目标 split_manifest.json 路径。
|
||||
force: False 时已存在即 FileExistsError；True 时把旧产物重命名为
|
||||
.bak.<旧 pools_sha256 前 8 位或 timestamp-less 序号> 再放行。
|
||||
|
||||
异常:
|
||||
FileExistsError: force=False 且产物已存在（防静默覆盖冻结锚点）。
|
||||
"""
|
||||
if not out_path.exists() and not manifest_path.exists():
|
||||
return
|
||||
if not force:
|
||||
raise FileExistsError(
|
||||
f"已存在冻结产物 {out_path}（或其 manifest）。重跑切分会覆盖训练依赖的"
|
||||
"冻结锚点——确认要替换请加 --force（旧产物将备份为 .bak.*）。"
|
||||
)
|
||||
# 备份后缀取旧 manifest 的 pools_sha256 前 8 位，无则用 'prev'
|
||||
suffix = "prev"
|
||||
if manifest_path.exists():
|
||||
try:
|
||||
old = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
suffix = str(old.get("pools_sha256", "prev"))[:8] or "prev"
|
||||
except (json.JSONDecodeError, OSError):
|
||||
suffix = "prev"
|
||||
for p in (out_path, manifest_path):
|
||||
if p.exists():
|
||||
p.rename(p.with_name(f"{p.name}.bak.{suffix}"))
|
||||
```
|
||||
确认 build_split.py 已 import `json`（无则加 `import json`）。在 `build_split` 签名加参数 `force: bool = False`（放 `generated_at` 之后），并在 Phase 3 `save_pools` 之前（L192 前）调用 `_guard_frozen_products(out_path, manifest_path, force=force)`。
|
||||
|
||||
- [ ] **Step 4: CLI 暴露 --force 并透传**
|
||||
|
||||
`app/harness/video_split_cli.py` `build_arg_parser`（L751-773）追加：
|
||||
```python
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="覆盖已存在的冻结 pools.json/manifest（旧产物备份为 .bak.*）",
|
||||
)
|
||||
```
|
||||
`run_pipeline` 签名加 `force: bool = False` 参数，build_split 调用（L559-577）加 `force=force,`；`main()` 里把 `args.force` 透传给 `run_pipeline`。
|
||||
|
||||
- [ ] **Step 5: 运行测试确认通过 + CLI 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -q`
|
||||
Expected: 全 PASS。
|
||||
|
||||
- [ ] **Step 6: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/build_split.py app/harness/video_split_cli.py tests/unit/test_video_split_cli.py
|
||||
git commit -m "feat: guard frozen split products against silent overwrite (--force)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: seed 携带 pools.json + 训练拷入
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/store.py:184-232`（init_seed）
|
||||
- Modify: `app/harness/workspace.py:156-200`（init_workspace_from_seed）
|
||||
- Test: `tests/unit/test_harness_store.py`、`tests/unit/test_harness_workspace.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试（seed 携带）**
|
||||
|
||||
在 `tests/unit/test_harness_store.py::TestInitSeed` 追加：
|
||||
```python
|
||||
def test_init_seed_carries_pools(self, tmp_path):
|
||||
"""提供 pools_json/split_manifest 时拷入 seed 目录。"""
|
||||
from app.harness.store import init_seed
|
||||
|
||||
store = tmp_path / "store"
|
||||
skills = tmp_path / "sk"; skills.mkdir(); (skills / "s.md").write_text("x")
|
||||
prompts = tmp_path / "pr"; prompts.mkdir(); (prompts / "p.md").write_text("y")
|
||||
db = tmp_path / "b.db"; db.write_text("db")
|
||||
pools = tmp_path / "pools.json"; pools.write_text('{"split_mode":"global"}')
|
||||
manifest = tmp_path / "split_manifest.json"; manifest.write_text('{"pools_sha256":"a"}')
|
||||
|
||||
seed_dir = init_seed(
|
||||
store, "s1", skills, prompts, db, "infer_adhoc", None, "d",
|
||||
pools_json=pools, split_manifest=manifest,
|
||||
)
|
||||
assert (seed_dir / "pools.json").exists()
|
||||
assert (seed_dir / "split_manifest.json").exists()
|
||||
```
|
||||
|
||||
在 `tests/unit/test_harness_workspace.py` 追加：
|
||||
```python
|
||||
def test_init_workspace_from_seed_carries_pools(store_dir, workspace_dir):
|
||||
"""seed 目录含 pools.json 时拷入 workspace。"""
|
||||
import shutil
|
||||
from app.harness.store import init_seed
|
||||
from app.harness.workspace import init_workspace_from_seed
|
||||
# 复用现有 fixture 构造 seed 的方式；此处补 pools.json 到 seed 后初始化 workspace
|
||||
# （具体 fixture 依 test_harness_workspace.py 现有 helper，读文件对齐）
|
||||
...
|
||||
```
|
||||
> 该 workspace 测试需依 `test_harness_workspace.py` 现有 fixture（`store_dir`/`workspace_dir` 及既有 seed 构造 helper）填充；实现前读该文件 `test_init_workspace_from_seed`（L151）复用其 seed 搭建，再在 seed 目录写 `pools.json` 后断言 workspace 内出现 `pools.json`。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py::TestInitSeed::test_init_seed_carries_pools -v`
|
||||
Expected: FAIL（`unexpected keyword argument 'pools_json'`）。
|
||||
|
||||
- [ ] **Step 3: init_seed 加可选携带参数**
|
||||
|
||||
`app/harness/store.py` `init_seed` 签名加：
|
||||
```python
|
||||
def init_seed(
|
||||
store_dir: Path,
|
||||
name: str,
|
||||
skills_dir: Path,
|
||||
prompts_dir: Path,
|
||||
baseline_db: Path,
|
||||
baseline_run_id: str,
|
||||
parent: str | None,
|
||||
description: str,
|
||||
*,
|
||||
pools_json: Path | None = None,
|
||||
split_manifest: Path | None = None,
|
||||
) -> Path:
|
||||
```
|
||||
在 `copy2(baseline_db, ...)`（L218）之后加：
|
||||
```python
|
||||
if pools_json is not None:
|
||||
shutil.copy2(pools_json, seed_dir / "pools.json")
|
||||
if split_manifest is not None:
|
||||
shutil.copy2(split_manifest, seed_dir / "split_manifest.json")
|
||||
```
|
||||
docstring 补两参说明。
|
||||
|
||||
- [ ] **Step 4: init_workspace_from_seed 拷入 pools**
|
||||
|
||||
`app/harness/workspace.py` `init_workspace_from_seed` 在 `shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db")`（L197）之后加：
|
||||
```python
|
||||
seed_pools = seed_dir / "pools.json"
|
||||
if seed_pools.exists():
|
||||
shutil.copy2(seed_pools, workspace_dir / "pools.json")
|
||||
seed_manifest = seed_dir / "split_manifest.json"
|
||||
if seed_manifest.exists():
|
||||
shutil.copy2(seed_manifest, workspace_dir / "split_manifest.json")
|
||||
```
|
||||
|
||||
- [ ] **Step 5: 运行测试确认通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py -q`
|
||||
Expected: 全 PASS。
|
||||
|
||||
- [ ] **Step 6: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/store.py app/harness/workspace.py tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py
|
||||
git commit -m "feat: seed carries frozen pools.json into training workspace"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 6: build_or_load_pools global 一致性校验
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/pools.py:746-813`
|
||||
- Test: `tests/unit/test_harness_pools.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
在 `tests/unit/test_harness_pools.py::TestBuildOrLoadPoolsFrozen` 追加：
|
||||
```python
|
||||
def test_global_frozen_rejects_baseline_mismatch(self, tmp_path, ...):
|
||||
"""global 冻结 pools 的 baseline_run_id 与 seed 不符时 fail-loud。"""
|
||||
# 依现有 fixture 造 workspace + 冻结 pools.json（split_mode=global,
|
||||
# baseline_run_id="other"），seed.json baseline_run_id="infer_adhoc"
|
||||
# 调 build_or_load_pools 应 raise ValueError(match="baseline_run_id")
|
||||
...
|
||||
```
|
||||
> 依 `TestBuildOrLoadPoolsFrozen`（L246）现有 fixture 复用其 workspace/seed 搭建；实现前读该类对齐 RunConfig/strategy 构造，勿臆造。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_pools.py::TestBuildOrLoadPoolsFrozen -v`
|
||||
Expected: 新用例 FAIL（当前 global 分支无校验，误加载不报错）。
|
||||
|
||||
- [ ] **Step 3: 加 global 一致性校验**
|
||||
|
||||
`app/harness/pools.py` `build_or_load_pools`，在 global 加载分支（`if pools_path.exists():` 块内、`per_category` 校验的 `else` 侧，即 L813 `return load_pools(pools_path)` 之前）加：
|
||||
```python
|
||||
else: # global：校验 baseline_run_id 与（若有）manifest 内容指纹
|
||||
frozen_baseline = raw.get("baseline_run_id")
|
||||
if frozen_baseline != baseline_run_id:
|
||||
raise ValueError(
|
||||
f"冻结 pools.json 的 baseline_run_id={frozen_baseline!r} 与 seed "
|
||||
f"的 {baseline_run_id!r} 不一致，拒绝静默加载错配切分。"
|
||||
)
|
||||
manifest_path = config.workspace_dir / "split_manifest.json"
|
||||
if manifest_path.exists():
|
||||
import hashlib
|
||||
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
actual_sha = hashlib.sha256(
|
||||
pools_path.read_text(encoding="utf-8").encode("utf-8")
|
||||
).hexdigest()
|
||||
if manifest.get("pools_sha256") != actual_sha:
|
||||
raise ValueError(
|
||||
"pools.json 内容指纹与 split_manifest.pools_sha256 不符，"
|
||||
"冻结产物疑被篡改，拒绝加载。"
|
||||
)
|
||||
```
|
||||
（确认该 `else` 与 L751 `if frozen_split_mode == "per_category":` 配对；若现有结构非 if/else 而是 if 后直接 return，则把校验插在 `return load_pools(pools_path)` 前并用 `if frozen_split_mode != "per_category":` 守卫。实现前读 L746-813 对齐控制流。）
|
||||
|
||||
- [ ] **Step 4: 运行测试确认通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_pools.py -q`
|
||||
Expected: 全 PASS。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/pools.py tests/unit/test_harness_pools.py
|
||||
git commit -m "fix: validate global frozen pools baseline_run_id + sha256 on load"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review（作者自查，执行者复核）
|
||||
|
||||
- [ ] val_ratio=0.4 已改；tier 感知 + 功效修复在同一函数、退化路径（`wrong_tier_by_video=None`）保持旧行为。
|
||||
- [ ] val_wrong_min 从 CLI→SplitBuildConfig→build_split→split_by_video_assignment→_split_trainval_by_video_group 全链路贯通；旧的 pools.py 事后校验块已删（不重复）。
|
||||
- [ ] extract_run_db 去重默认关闭，不破坏既有调用。
|
||||
- [ ] seed 携带 + workspace 拷入 + global 一致性校验三者闭环：冻结产物有唯一路径进训练且被校验。
|
||||
- [ ] 覆盖保护默认 force=False，离线 CLI 重跑需显式 --force。
|
||||
|
||||
## 核心算法保真校验结论
|
||||
|
||||
本计划触及算法 #5 的上游输入（哪些视频进 diag/val），**不改** gate_ladder 的 unit+correctness 消费结构；Task 3 Step 6 已设保真检查点确认逐 unit 列表与视频组原子性。不涉及算法 #4/#6/#8/#9 逻辑。
|
||||
|
||||
## 验收标准
|
||||
|
||||
1. `pytest tests/unit/test_pools_video_atomic.py tests/unit/test_harness_pools.py tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py tests/unit/test_split_selection.py tests/unit/test_video_split_cli.py` 全绿。
|
||||
2. tier 感知：T2 高的错题视频组留 diag，T2 低的优先进 val。
|
||||
3. seed 携带 pools.json → init_workspace_from_seed 拷入 → build_or_load_pools 校验 baseline_run_id + sha256。
|
||||
4. 冻结产物 force=False 时拒绝覆盖。
|
||||
@@ -0,0 +1,601 @@
|
||||
# WP3 训练循环与进化引擎 Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
> **前置依赖：WP1（模板已迁移，进化引擎可运行）+ WP4（cache_salt 能力已就绪）必须先完成。**
|
||||
|
||||
**Goal:** 修复训练循环与进化引擎的 9 处缺陷，使诊断拿到真实轨迹、早停按 epoch 语义、微型题型不崩、崩溃可幂等续跑、进化 patch 不破坏冻结区、降级信号不驱动错误进化、跨 epoch 评估真实重采样。
|
||||
|
||||
**Architecture:** 诊断经 `StepsJsonRunLog` 从 steps_json 重建轨迹（算法 #7 恢复）；早停计数单位 step→epoch；可训练性预检在 gate 建立前剔除微型题型；`_run_step` 幂等（先 DELETE 再写）；patch 冻结区检查整个 target 跨度（算法 #8 加固）；降级/未判定题按 lapse 保守分流；训练推理用 run_id 作 cache_salt（run_id 已含 epoch，天然跨 epoch 重采样）。
|
||||
|
||||
**Tech Stack:** Python 3.11、asyncio、SQLite、pytest。
|
||||
|
||||
**设计源**：`research-wiki/designs/2026-07-16-preflight-fixes-design.md §6-7`
|
||||
|
||||
---
|
||||
|
||||
## 关键锚点（实现前必读）
|
||||
|
||||
| 用途 | 位置 |
|
||||
|------|------|
|
||||
| 训练主流程 | `app/harness/runner.py:789-859` `train`；`_setup_train_run:865-879`（预检插入点 L874 前）；`_run_step:1001-1026`（run_id L1011、诊断 L1019、无 DELETE） |
|
||||
| 早停 | `app/harness/runner.py:291-316` `_should_early_stop`（L315 `+= steps_this_epoch`）；`_TrainState:95-121`（`steps_since_best_improved` L118）；`_maybe_promote_best:1587`（置 0） |
|
||||
| 诊断调用 | `app/harness/runner.py:2163-2196` `_run_diagnosis`（`RunLogImpl` L2173 未包 StepsJsonRunLog）；DiagnosisResult `degraded_count` 未被引用 |
|
||||
| gate 刷新 | `app/harness/runner.py:1833-1879` `_refresh_gate_ladder`（save L1878 → set observed L1879）；checkpoint 落盘晚在 train L837 |
|
||||
| holdout | `app/harness/runner.py:1881-1921` `_holdout_four_way`；`_pick_mixed_best:1923-1963`；`_eval_version_on_pool:2148-2161` |
|
||||
| 推理落库 | `app/harness/inference.py:363-447` `_run_single_question`（prediction L422 未归一、insert L446 try 外）；`_to_text_field:147-162`；`run_inference:473` |
|
||||
| Agent Loop | `core/agent/loop.py:103` `run`（session_id L110）；`_call_llm` chat 调用 `:329`（`self._llm.chat(messages, session_id=session_id)`） |
|
||||
| batching | `app/harness/batching.py:186-202` `_classify_unit`（L200 缺 correctness→None） |
|
||||
| checkpoint | `app/harness/checkpoint.py:37-44` `_STRUCTURAL_KEYS`；`serialize_state:76-106`；`write_checkpoint:212-260`（原子写） |
|
||||
| 诊断分流 | `core/evolution/diagnose.py:1485-1519` `_build_skill_case_packs`（lapse 分流 L1492）；`_process_question:2139-2206`（except L2194 cause_category 留 None） |
|
||||
| traces 适配 | `app/harness/baseline_run_log.py:13` `StepsJsonRunLog`；`steps_json_traces.py:13` |
|
||||
| patch | `core/evolution/patch.py:285-287` `_in_ranges`；`_do_insert_after:309-326`（L320）；`_do_replace_delete:329-347`（L343）；markers L11-18；`validate_skill` in `evolve.py:296` |
|
||||
|
||||
## 核心算法保真校验
|
||||
|
||||
触及算法 #7（诊断瀑布，Task 1 恢复轨迹）、#8（patch 引擎，Task 5 冻结区加固）、#10（Agent Loop，Task 9 透传 cache_salt）、#5/#12（信息阶梯/训练编排，Task 7 checkpoint 时序）。**均为恢复/加固/透传，不改算法逻辑**：Task 1 让诊断拿到本就该有的轨迹；Task 5 把"只查起点"补成"查整跨度"（保护方向不变）；Task 9 只加透传参数；Task 7 只调 checkpoint 落盘时机。每个相关 Task 设保真检查点。
|
||||
|
||||
---
|
||||
|
||||
## Task 1: traces 适配（诊断拿到真实轨迹，算法 #7）
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/runner.py:2163-2196`（_run_diagnosis）
|
||||
- Test: `tests/unit/test_runner_diag_tree_inject.py` 或 `test_harness_runner.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
在 `tests/unit/test_runner_diag_tree_inject.py` 追加（构造只写 steps_json 不写 traces 表的 run，断言诊断能拿到轨迹）：
|
||||
```python
|
||||
@pytest.mark.asyncio
|
||||
async def test_diagnosis_reads_traces_from_steps_json(...):
|
||||
"""traces 表为空但 predictions.steps_json 有轨迹时，诊断仍拿到非空 traces。"""
|
||||
# 依现有 runner 测试 fixture 造一个 run：predictions 有 steps_json，traces 表空；
|
||||
# 调 _run_diagnosis 后断言 diagnose 收到的 traces 非空（可 patch run_diagnosis 捕获入参）
|
||||
...
|
||||
```
|
||||
> 依 `test_runner_diag_tree_inject.py` 现有 fixture；实现前读对齐 runner 构造与 patch 点。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_runner_diag_tree_inject.py -k reads_traces_from_steps_json -v`
|
||||
Expected: FAIL（当前 RunLogImpl 直读空 traces 表）。
|
||||
|
||||
- [ ] **Step 3: 包 StepsJsonRunLog**
|
||||
|
||||
`app/harness/runner.py` `_run_diagnosis`，把传给 `run_diagnosis` 的 `run_log`（当前 `RunLogImpl(...)`，L2173 附近）包一层：
|
||||
```python
|
||||
from app.harness.baseline_run_log import StepsJsonRunLog
|
||||
from app.harness.log import RunLogImpl
|
||||
|
||||
run_log = StepsJsonRunLog(RunLogImpl(str(self._paths.db_path)))
|
||||
```
|
||||
（`StepsJsonRunLog.get_traces` 在底层 traces 空时从 predictions.steps_json 经 `steps_json_to_trace_rows` 重建；`get_predictions` 透传。）
|
||||
|
||||
- [ ] **Step 4: 保真检查点 + 测试通过**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_runner_diag_tree_inject.py tests/unit/test_baseline_run_log.py -q`
|
||||
Expected: 全 PASS。保真：确认诊断瀑布拿到的是逐 step `{tool_name,tool_args,tool_output,thought}` 行（对齐 TRM4 诊断输入）。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/runner.py tests/unit/test_runner_diag_tree_inject.py
|
||||
git commit -m "fix: wrap diagnosis run_log with StepsJsonRunLog (restore algo #7 traces)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: prediction 归一化 + 落库加固
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/inference.py:417-447`
|
||||
- Test: `tests/unit/test_harness_inference.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
在 `tests/unit/test_harness_inference.py` 追加（LLM 提交非标量 answer 不崩 gather）：
|
||||
```python
|
||||
@pytest.mark.asyncio
|
||||
async def test_nonscalar_prediction_does_not_crash(...):
|
||||
"""submit_answer 返回 {'answer': ['B']} 等非标量时归一化落库，不抛 sqlite 绑定异常。"""
|
||||
# 依现有 inference 测试 fixture，让 AgentLoop 返回 result={'answer': ['B']}；
|
||||
# run_inference 应正常完成、predictions 行 prediction 为字符串（如 '["B"]'），不崩
|
||||
...
|
||||
```
|
||||
> 依 `test_harness_inference.py` 现有 fake loop/dispatch fixture；实现前读对齐。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py -k nonscalar_prediction -v`
|
||||
Expected: FAIL（sqlite `InterfaceError: Error binding parameter`）。
|
||||
|
||||
- [ ] **Step 3: 归一化 prediction + insert 加固**
|
||||
|
||||
`app/harness/inference.py`：新增归一化 helper（None 保留、str 原样、其余 `_to_text_field`）：
|
||||
```python
|
||||
def _normalize_prediction(answer: object) -> str | None:
|
||||
"""归一化 prediction：None 保留（INFRA 空预测语义），str 原样，其余 JSON 序列化。"""
|
||||
if answer is None or isinstance(answer, str):
|
||||
return answer
|
||||
return _to_text_field(answer)
|
||||
```
|
||||
L422 `"prediction": result_dict.get("answer"),` 改为 `"prediction": _normalize_prediction(result_dict.get("answer")),`。
|
||||
L446 的 `await asyncio.to_thread(log.insert, "predictions", record)` 包 try，绑定异常降级为最小 error 行不击穿 gather：
|
||||
```python
|
||||
try:
|
||||
await asyncio.to_thread(log.insert, "predictions", record)
|
||||
except (sqlite3.InterfaceError, sqlite3.ProgrammingError):
|
||||
logger.exception("[{}] QA {} 落库绑定异常，降级为 error 行", qa.video_id, qa.question_id)
|
||||
record["prediction"] = None
|
||||
record["stop_reason"] = "error"
|
||||
await asyncio.to_thread(
|
||||
log.insert, "predictions",
|
||||
{k: v for k, v in record.items() if isinstance(v, (str, int, float, type(None)))},
|
||||
)
|
||||
```
|
||||
确认 `import sqlite3` 在文件顶部（无则加）。
|
||||
|
||||
- [ ] **Step 4: 测试通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py -q`
|
||||
Expected: 全 PASS。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/inference.py tests/unit/test_harness_inference.py
|
||||
git commit -m "fix: normalize non-scalar prediction; harden predictions insert"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: early_stop 改 epoch 计数
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/runner.py:291-316,118,1587,849-857`
|
||||
- Modify: `app/harness/checkpoint.py`（若字段入 state）
|
||||
- Test: `tests/unit/test_harness_runner.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
在 `tests/unit/test_harness_runner.py` 追加：
|
||||
```python
|
||||
def test_early_stop_counts_epochs_not_steps(tmp_path):
|
||||
"""patience=2 表示连续 2 个 epoch 无 best 刷新才停（不是步数）。"""
|
||||
from app.harness.runner import _should_early_stop, _TrainState
|
||||
# 造 state + workspace，best 停在 epoch 1；
|
||||
# epoch 2 无刷新 → epochs_since_best_improved=1 → 不停；
|
||||
# epoch 3 无刷新 → =2 → 停
|
||||
...
|
||||
```
|
||||
> 依现有 `_TrainState`/`read_best` fixture；实现前读对齐 workspace best 写入。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k early_stop_counts_epochs -v`
|
||||
Expected: FAIL（当前累加 steps_this_epoch）。
|
||||
|
||||
- [ ] **Step 3: 字段改名 + 计数改 epoch**
|
||||
|
||||
全局把 `steps_since_best_improved` 改名 `epochs_since_best_improved`（`grep -rn steps_since_best_improved app/`：`_TrainState:118`、`_should_early_stop:313,315`、`_maybe_promote_best:1587`，以及 checkpoint serialize/deserialize 若含此字段）。
|
||||
`_should_early_stop`（L315）`state.steps_since_best_improved += steps_this_epoch` 改为 `state.epochs_since_best_improved += 1`；签名删除 `steps_this_epoch` 参数（改为 `_should_early_stop(workspace_dir, epoch, state, patience)`），train 调用点（L849-857）同步去掉 `len(batches)` 实参。docstring 改为"epoch 粒度"。
|
||||
|
||||
- [ ] **Step 4: checkpoint 兼容**
|
||||
|
||||
若 `epochs_since_best_improved` 入 checkpoint state（`grep -n steps_since_best_improved app/harness/checkpoint.py`），同步改名。本轮为 fresh 训练无旧 checkpoint，无迁移负担。
|
||||
|
||||
- [ ] **Step 5: 测试通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py tests/unit/test_harness_checkpoint.py -q`
|
||||
Expected: 全 PASS。
|
||||
|
||||
- [ ] **Step 6: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/runner.py app/harness/checkpoint.py tests/unit/test_harness_runner.py
|
||||
git commit -m "fix: early_stop patience counts epochs not steps"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: 可训练性预检
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/runner.py:865-879`（_setup_train_run）
|
||||
- Modify: `app/harness/config.py`（RunConfig 加 `trainable_min_units`）+ `config/train_videomme.yaml`（WP 收尾建，先加字段默认）
|
||||
- Test: `tests/unit/test_harness_runner.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
```python
|
||||
def test_untrainable_types_filtered_before_gate():
|
||||
"""val<eval_min_per_class 或 非test单元<trainable_min_units 的题型从 diag/val/task_types 剔除。"""
|
||||
from app.harness.runner import _filter_untrainable_types
|
||||
# 构造 pools：题型 A（val=5, units=40）可训；B（val=0, units=2）不可训
|
||||
# 调 _filter_untrainable_types(pools, task_types=[A,B], eval_min_per_class=2, trainable_min_units=8)
|
||||
# 断言返回 pools 不含 B、task_types 不含 B
|
||||
...
|
||||
```
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k untrainable_types_filtered -v`
|
||||
Expected: FAIL（`_filter_untrainable_types` 不存在）。
|
||||
|
||||
- [ ] **Step 3: 实现纯函数预检**
|
||||
|
||||
`app/harness/runner.py` 新增模块级纯函数：
|
||||
```python
|
||||
def _filter_untrainable_types(
|
||||
pools: Pools,
|
||||
task_types: list[str] | None,
|
||||
eval_min_per_class: int,
|
||||
trainable_min_units: int,
|
||||
) -> tuple[Pools, list[str] | None]:
|
||||
"""剔除不可训练题型（val<eval_min_per_class 或 非test单元<trainable_min_units）。
|
||||
|
||||
非test单元数 = 该题型 diag+val 题数（single 题 unit==题；等于 gate 阶梯该类候选数）。
|
||||
test 池不过滤（继续报告全题型准确率）。返回过滤后 (pools, task_types)。
|
||||
"""
|
||||
from collections import Counter
|
||||
|
||||
diag_by_type = Counter(q.task_type for q in pools.diagnosis)
|
||||
val_by_type = Counter(q.task_type for q in pools.validation)
|
||||
keep: set[str] = set()
|
||||
dropped: list[tuple[str, str]] = []
|
||||
for tt in set(diag_by_type) | set(val_by_type):
|
||||
n_val = val_by_type.get(tt, 0)
|
||||
n_units = diag_by_type.get(tt, 0) + n_val
|
||||
if n_val < eval_min_per_class:
|
||||
dropped.append((tt, f"val={n_val}<{eval_min_per_class}"))
|
||||
elif n_units < trainable_min_units:
|
||||
dropped.append((tt, f"units={n_units}<{trainable_min_units}"))
|
||||
else:
|
||||
keep.add(tt)
|
||||
for tt, why in sorted(dropped):
|
||||
logger.warning("可训练性预检剔除题型 {}（{}）", tt, why)
|
||||
new_pools = replace(
|
||||
pools,
|
||||
diagnosis=[q for q in pools.diagnosis if q.task_type in keep],
|
||||
validation=[q for q in pools.validation if q.task_type in keep],
|
||||
)
|
||||
new_types = [t for t in task_types if t in keep] if task_types is not None else sorted(keep)
|
||||
return new_pools, new_types
|
||||
```
|
||||
（确认 `from dataclasses import replace` 已 import；`Pools` 是否 frozen dataclass 支持 `replace`——若非，按其构造方式重建。）
|
||||
在 `_setup_train_run`（pools 加载后、`_init_gate_pools` L874 前）调用：
|
||||
```python
|
||||
pools, self._config.task_types = _filter_untrainable_types(
|
||||
pools, self._config.task_types,
|
||||
self._config.eval_min_per_class, self._config.trainable_min_units,
|
||||
)
|
||||
```
|
||||
（`RunConfig` 若 frozen，改用局部变量传递而非改 self._config；实现前读 config.py 确认可变性。）
|
||||
`app/harness/config.py` `RunConfig` 加字段 `trainable_min_units: int`（无默认，显式配置；train yaml 提供）。
|
||||
|
||||
- [ ] **Step 4: 测试通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -q`
|
||||
Expected: 全 PASS。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/runner.py app/harness/config.py tests/unit/test_harness_runner.py
|
||||
git commit -m "feat: pre-flight filter of untrainable task types before gate"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: patch 冻结区跨度 + 注入 + marker 校验（算法 #8）
|
||||
|
||||
**Files:**
|
||||
- Modify: `core/evolution/patch.py:285-347`
|
||||
- Modify: `core/evolution/evolve.py:296`（validate_skill）
|
||||
- Test: `tests/unit/test_patch.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
在 `tests/unit/test_patch.py` 追加：
|
||||
```python
|
||||
def test_replace_spanning_into_protected_is_skipped():
|
||||
"""target 起点在正文、末端伸入冻结区的 replace 被跳过（不破坏 marker）。"""
|
||||
from core.evolution.patch import apply_patch_with_report, APPENDIX_START, APPENDIX_END
|
||||
|
||||
body = "正文最后一段。"
|
||||
appendix = f"{APPENDIX_START}\n## 执行提醒\n- 规则A\n{APPENDIX_END}"
|
||||
content = body + "\n\n" + appendix
|
||||
# target 从正文末尾跨入 APPENDIX_START
|
||||
target = "正文最后一段。\n\n" + APPENDIX_START
|
||||
edits = [{"op": "delete", "target": target, "content": ""}]
|
||||
new_content, report = apply_patch_with_report(content, edits, protected_spans=[appendix])
|
||||
assert APPENDIX_START in new_content and APPENDIX_END in new_content # marker 未被破坏
|
||||
|
||||
|
||||
def test_edit_payload_with_marker_literal_rejected():
|
||||
"""edit payload/target 含 marker 字面量 → 拒绝该 edit。"""
|
||||
from core.evolution.patch import apply_patch_with_report, APPENDIX_START
|
||||
edits = [{"op": "append", "target": "", "content": f"注入 {APPENDIX_START} 破坏"}]
|
||||
_, report = apply_patch_with_report("正文", edits, protected_spans=[])
|
||||
assert any("marker" in str(s).lower() or "reject" in str(s).lower() for s in report)
|
||||
```
|
||||
> marker 常量名以 `core/evolution/patch.py:11-18` 为准（`APPENDIX_START` 等），实现前读对齐 import 名。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_patch.py -k "spanning_into_protected or marker_literal" -v`
|
||||
Expected: FAIL（当前只查起点 pos，跨入未拦；无注入检查）。
|
||||
|
||||
- [ ] **Step 3: 跨度检查 + 注入检查**
|
||||
|
||||
`core/evolution/patch.py` 新增跨度 helper：
|
||||
```python
|
||||
def _span_overlaps_ranges(pos: int, length: int, ranges: list[tuple[int, int]]) -> bool:
|
||||
"""判断 [pos, pos+length) 是否与任一冻结区间相交（不止起点）。"""
|
||||
end = pos + length
|
||||
return any(start < end and pos < r_end for start, r_end in ranges)
|
||||
```
|
||||
`_do_insert_after` L320 `if _in_ranges(pos, ranges):` 改为 `if _span_overlaps_ranges(pos, len(target), ranges):`。
|
||||
`_do_replace_delete` L343 `if _in_ranges(pos, ranges):` 改为 `if _span_overlaps_ranges(pos, len(target), ranges):`。
|
||||
在 `apply_patch_with_report`（L387）应用每个 edit 前加注入检查：payload/target 含任一 marker 字面量（`APPENDIX_START/END`、`MOMENTUM_START/END`）→ 跳过该 edit 并记 `skipped_marker_injection`。
|
||||
|
||||
- [ ] **Step 4: validate_skill 加 marker 完整性校验**
|
||||
|
||||
`core/evolution/evolve.py:296` `validate_skill`：在现有 frontmatter/长度/代码块校验后加——统计 evolved 中 `APPENDIX_START/END`、`MOMENTUM_START/END` 出现次数，要求成对（START 数==END 数）、各至多一对、START 在 END 前；违反则返回校验失败（该候选整体 reject）。
|
||||
|
||||
- [ ] **Step 5: 保真检查点 + 测试通过**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_patch.py tests/unit/test_evolve.py -q`
|
||||
Expected: 全 PASS。保真：确认"保护跨度"方向未变（仍是保护 appendix/momentum 不被误改），只是从"查起点"补成"查整跨度"+ 注入/完整性双防线。
|
||||
|
||||
- [ ] **Step 6: 提交**
|
||||
|
||||
```bash
|
||||
git add core/evolution/patch.py core/evolution/evolve.py tests/unit/test_patch.py
|
||||
git commit -m "fix: patch checks full target span + marker injection/integrity (algo #8)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 6: 诊断降级分流 + 占比中止
|
||||
|
||||
**Files:**
|
||||
- Modify: `core/evolution/diagnose.py:1485-1497`
|
||||
- Modify: `app/harness/runner.py:1019`（诊断后 degraded 占比检查）
|
||||
- Test: `tests/unit/test_diagnose.py`、`tests/unit/test_harness_runner.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试（分流）**
|
||||
|
||||
在 `tests/unit/test_diagnose.py` 追加：
|
||||
```python
|
||||
def test_none_cause_and_degraded_route_to_lapse():
|
||||
"""cause_category=None（判别失败）与 degraded 题按 lapse 处置，不进 defect 正文路径。"""
|
||||
from core.evolution.diagnose import _build_skill_case_packs
|
||||
# 构造 metrics_group：一题 attr.cause_category=None（非 degraded）、一题 qm.degraded=True；
|
||||
# 断言二者都不出现在 failure_cases（wrong_by_error），只可能进 lapse_notes
|
||||
...
|
||||
```
|
||||
> 依 `test_diagnose.py` 现有 QuestionMetrics/ErrorAttribution 构造（`:753` 附近）；实现前读对齐字段。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_diagnose.py -k none_cause_and_degraded_route -v`
|
||||
Expected: FAIL（当前 None → wrong_by_error 走 defect 正文）。
|
||||
|
||||
- [ ] **Step 3: 改分流逻辑**
|
||||
|
||||
`core/evolution/diagnose.py` `_build_skill_case_packs` 的分流循环（L1488-1497）改为"仅明确 defect 且非 degraded 才进正文路径"：
|
||||
```python
|
||||
for qm in metrics_group:
|
||||
if qm.correct:
|
||||
continue
|
||||
attr = attribution_map.get(qm.question_id)
|
||||
is_defect = (
|
||||
attr is not None
|
||||
and attr.cause_category == "defect"
|
||||
and not qm.degraded
|
||||
)
|
||||
if not is_defect:
|
||||
# lapse / None（判别失败）/ degraded → 保守，不驱动正文进化
|
||||
if attr is not None and attr.lapse_note and attr.lapse_note.strip():
|
||||
lapse_notes.append(attr.lapse_note)
|
||||
continue
|
||||
wrong_by_error[attr.error_type].append(qm)
|
||||
```
|
||||
|
||||
- [ ] **Step 4: 写失败测试（占比中止）+ 实现**
|
||||
|
||||
`tests/unit/test_harness_runner.py` 追加：诊断结果 degraded_count/总题数 > 0.5 时 `_run_step` 后应 raise（疑似基础设施故障）。
|
||||
`app/harness/runner.py` `_run_step`（L1019 拿到 `diagnosis` 后）加：
|
||||
```python
|
||||
n_wrong = sum(1 for q in batch if not state.correctness.get(q.question_id, True))
|
||||
if n_wrong > 0 and diagnosis.degraded_count / n_wrong > 0.5:
|
||||
raise RuntimeError(
|
||||
f"本 step 诊断降级占比 {diagnosis.degraded_count}/{n_wrong} > 50%，"
|
||||
"疑似 judge 基础设施故障，中止训练（不以降级信号驱动进化）。"
|
||||
)
|
||||
```
|
||||
（确认 `DiagnosisResult.degraded_count` 字段可用，L2311。）
|
||||
|
||||
- [ ] **Step 5: 测试通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_diagnose.py tests/unit/test_harness_runner.py -q`
|
||||
Expected: 全 PASS。
|
||||
|
||||
- [ ] **Step 6: 提交**
|
||||
|
||||
```bash
|
||||
git add core/evolution/diagnose.py app/harness/runner.py tests/unit/test_diagnose.py tests/unit/test_harness_runner.py
|
||||
git commit -m "fix: route None/degraded diagnoses to lapse; abort on high degrade rate"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 7: step 幂等（DELETE）+ gate_epoch_observed 立即落盘
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/runner.py:1001-1026`（_run_step 开头 DELETE）
|
||||
- Modify: `app/harness/runner.py:1378-1498,1833-1879`（checkpoint 提到 gate save 之后）
|
||||
- Test: `tests/unit/test_harness_runner.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试（幂等）**
|
||||
|
||||
```python
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_step_deletes_stale_rows_before_rerun(...):
|
||||
"""同 run_id 重跑前先清 predictions/traces，避免重复行双计。"""
|
||||
# 预置该 step run_id 的旧 predictions 行；调 _run_step；断言旧行被清、只剩本次
|
||||
...
|
||||
```
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k deletes_stale_rows -v`
|
||||
Expected: FAIL（当前 append，无 DELETE）。
|
||||
|
||||
- [ ] **Step 3: _run_step 开头 DELETE**
|
||||
|
||||
`app/harness/runner.py` `_run_step`，在 rollout（L1012）之前加：
|
||||
```python
|
||||
with HarnessLog(str(self._paths.db_path), run_id, register_run=False) as log:
|
||||
log.execute("DELETE FROM predictions WHERE run_id=?", (run_id,))
|
||||
log.execute("DELETE FROM traces WHERE run_id=?", (run_id,))
|
||||
```
|
||||
（`register_run=False` 来自 WP4 Task 8；若 HarnessLog 无 `execute` 便捷方法，用其现有 `query`/连接接口，实现前读 log.py 对齐。）
|
||||
|
||||
- [ ] **Step 4: checkpoint 提到 gate save 之后（消除双计窗口）**
|
||||
|
||||
目标：`_refresh_gate_ladder` 内 `gate_pools.save`（L1878）+ `gate_epoch_observed=True`（L1879）之后，**立即** `write_checkpoint`（phase="epoch_done"），不等到 `train` L837。实现：把 `_slow_update_cycle` Phase 10（调 `_refresh_gate_ladder` L1496-1498）之后的 checkpoint 落盘从 `train`（L837）移入 `_slow_update_cycle` 末尾，或让 `_refresh_gate_ladder` 接收 checkpoint 所需上下文（epoch/progress/batches）并在 save 后落盘。实现前读 `write_checkpoint` 签名（checkpoint.py:212）与 `train` L833-848 对齐参数，确保 gate_pools.json 与 checkpoint 的 `gate_epoch_observed` 同一时刻一致。
|
||||
|
||||
- [ ] **Step 5: 保真检查点 + 测试通过**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py tests/unit/test_harness_checkpoint.py -q`
|
||||
Expected: 全 PASS。保真（算法 #5/#12）：确认 γ-EMA 更新（`update_probs`）仍每 epoch 一次、checkpoint 落盘不改变慢更新十步序的语义顺序。
|
||||
|
||||
- [ ] **Step 6: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/runner.py tests/unit/test_harness_runner.py
|
||||
git commit -m "fix: idempotent _run_step (DELETE stale) + checkpoint after gate save"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 8: holdout 四向去重
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/runner.py:1881-1963`（_holdout_four_way / _pick_mixed_best）
|
||||
- Test: `tests/unit/test_harness_runner.py`
|
||||
|
||||
> **说明**：目标是每 epoch 的四向 test 评估从"4×600 全跑"降为"仅 final 必跑 + best_hard 未评过才跑 + baseline 从基线预测推导 + best_mixed 引用赢家"。去重做在 harness 逻辑层（配合 WP4 epoch 盐，同版本不重采样）。
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
```python
|
||||
@pytest.mark.asyncio
|
||||
async def test_holdout_dedup_skips_reevaluated_versions(...):
|
||||
"""baseline 不跑推理（从基线预测推导）；best_hard==final 时不重复评估。"""
|
||||
# 统计 _eval_version_on_pool 被调次数：baseline=0，best_hard==final 时该向复用不重跑
|
||||
...
|
||||
```
|
||||
> 依现有 runner holdout fixture；实现前读 `_holdout_four_way`/`write_holdout_eval` 对齐。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k holdout_dedup -v`
|
||||
Expected: FAIL（当前四向各跑一次）。
|
||||
|
||||
- [ ] **Step 3: 实现去重**
|
||||
|
||||
`app/harness/runner.py` `_holdout_four_way`：
|
||||
- **baseline 向**：不调 `_eval_version_on_pool`，改从基线 predictions（baseline_run_id）读 test 题对错算 acc（test 题在 infer_adhoc 已推理）；epoch>1 复用首次结果。
|
||||
- **final 向**：真评 600（不变）。
|
||||
- **best_hard 向**：维护"已评版本→test_acc"备忘录，resume 时从 `holdout_eval` 表 hydrate（查已落库 `(skills_v,prompts_v)→acc`）；版本对未评过且 ≠ final 才真评，否则引用。
|
||||
- **best_mixed 向**：引用赢家（best_hard 或 final 之一，`_pick_mixed_best` 已定）的 test 成绩，写 holdout_eval 时标指针，不推理。
|
||||
|
||||
实现前完整读 `_holdout_four_way`（1881-1921）、`write_holdout_eval`、`_eval_version_on_pool`、`read`/hydrate 路径对齐。
|
||||
|
||||
- [ ] **Step 4: 测试通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -q`
|
||||
Expected: 全 PASS。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/runner.py tests/unit/test_harness_runner.py
|
||||
git commit -m "perf: dedup holdout four-way eval (baseline derive, best_hard memo)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 9: 训练推理注入 epoch 盐（cache_salt=run_id，算法 #10 透传）
|
||||
|
||||
**Files:**
|
||||
- Modify: `core/agent/loop.py:103-141,271-329`
|
||||
- Modify: `app/harness/inference.py:408-415,473`
|
||||
- Test: `tests/unit/test_agent_loop`（或现有 loop 测试）、`tests/unit/test_harness_inference.py`
|
||||
|
||||
> **原理**：训练/val/test/holdout 推理的 run_id 已含 `_e{epoch}`（如 `{base}_e{epoch}_s{step}`、`{run_id}_holdout_{kind}_e{epoch}`），用 run_id 作 cache_salt 即天然跨 epoch 重采样、同 epoch 续跑仍命中。judge/evolve 不经此路径（默认 salt=None）。gate 基线臂走 BaselineCache 不受影响；候选臂 messages 含 skill 版本天然区分。
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
```python
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_loop_forwards_cache_salt():
|
||||
"""AgentLoop.run(cache_salt=...) 透传到 llm.chat。"""
|
||||
from core.agent.loop import AgentLoop
|
||||
# fake llm 记录 chat 收到的 cache_salt kwarg；loop.run(..., cache_salt='run:e2')
|
||||
# 断言 fake_llm.chat 收到 cache_salt='run:e2'
|
||||
...
|
||||
```
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest -k agent_loop_forwards_cache_salt -v`
|
||||
Expected: FAIL（`run()` 无 cache_salt 参数）。
|
||||
|
||||
- [ ] **Step 3: AgentLoop 透传 cache_salt**
|
||||
|
||||
`core/agent/loop.py`：`run`（L103）、`_step`/`_call_llm`（L271,317）签名加 `cache_salt: str | None = None`（keyword，随 session_id 透传）；L329 `self._llm.chat(messages, session_id=session_id)` 改为 `self._llm.chat(messages, session_id=session_id, cache_salt=cache_salt)`。
|
||||
|
||||
- [ ] **Step 4: inference 用 run_id 作 salt**
|
||||
|
||||
`app/harness/inference.py` `_run_single_question`：`loop.run(...)`（L409）加 `cache_salt=run_id`（run_id 从 run_inference 透传到每题；`_run_single_question` 已有 run_id 上下文——若无则从 run_inference 参数透传）。确认 run_inference→_run_single_question 的 run_id 传递链完整。
|
||||
|
||||
- [ ] **Step 5: 保真检查点 + 测试通过**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py tests/integration/test_agent_governed_e2e.py -q`
|
||||
Expected: 全 PASS。保真（算法 #10）：确认只加透传参数，Thinking+JSON/json_repair/pluggy hook 逻辑不变。
|
||||
|
||||
- [ ] **Step 6: 提交**
|
||||
|
||||
```bash
|
||||
git add core/agent/loop.py app/harness/inference.py tests/unit/
|
||||
git commit -m "feat: inject run_id as cache_salt for per-epoch resampling (algo #10)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review（作者自查，执行者复核）
|
||||
|
||||
- [ ] traces 适配后诊断拿到真实轨迹（算法 #7 恢复）。
|
||||
- [ ] prediction 归一化 None 保留、非标量序列化；insert 绑定异常不击穿 gather。
|
||||
- [ ] early_stop 字段全局改名一致、计数改 epoch。
|
||||
- [ ] 预检剔除不可训练题型（test 池不动）。
|
||||
- [ ] patch 查整跨度 + 注入 + marker 完整性三防线（算法 #8 保护方向不变）。
|
||||
- [ ] None/degraded 按 lapse 保守分流；降级占比>50% 中止。
|
||||
- [ ] _run_step 幂等；gate_pools 与 checkpoint 的 gate_epoch_observed 同刻一致。
|
||||
- [ ] holdout 去重后 baseline 0 推理、best_hard 备忘录 resume 可 hydrate。
|
||||
- [ ] cache_salt=run_id 贯穿 AgentLoop，run_id 含 epoch 保证跨 epoch 重采样。
|
||||
|
||||
## 核心算法保真校验结论
|
||||
|
||||
触及算法 #5/#7/#8/#10/#12，均为恢复（#7 轨迹）/加固（#8 跨度）/透传（#10 salt）/时序（#5/#12 checkpoint），各 Task 已设保真检查点，不改算法核心逻辑。Task 5/7/9 需在实现时对照 TRM4 参考确认无行为漂移。
|
||||
|
||||
## 验收标准
|
||||
|
||||
1. `pytest tests/unit/test_harness_runner.py tests/unit/test_harness_inference.py tests/unit/test_diagnose.py tests/unit/test_patch.py tests/unit/test_harness_checkpoint.py tests/unit/test_baseline_run_log.py tests/unit/test_runner_diag_tree_inject.py` 全绿。
|
||||
2. 诊断拿到非空轨迹；非标量 prediction 不崩；early_stop 按 epoch。
|
||||
3. 微型题型被预检剔除；patch 不破坏 marker；降级题不驱动进化。
|
||||
4. _run_step 幂等；cache_salt=run_id 贯穿。
|
||||
@@ -0,0 +1,611 @@
|
||||
# WP4 韧性与持久化 Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** 修复 LLM 治理栈与持久化层的 10 处韧性缺陷，使训练信号不被缓存重放/截断响应/断连/INFRA 故障污染，且 workspace 元数据崩溃可恢复。
|
||||
|
||||
**Architecture:** 缓存加 salt 维度让跨 epoch 评估真实重采样（同 epoch 续跑仍命中）；SSE 未收 `[DONE]` 视为截断进重试且不写缓存；扩展瞬时错误清单覆盖断连族；熔断半开只放一个探针；gate 基线臂 INFRA 故障不污染 BaselineCache（算法 #6 保真区，逐行比对）；manifest 补原子写；只读查询不改基线元数据。
|
||||
|
||||
**Tech Stack:** Python 3.11、httpx、Redis、asyncio、SQLite、pytest、pydantic-settings。
|
||||
|
||||
**设计源**：`research-wiki/designs/2026-07-16-preflight-fixes-design.md §8`
|
||||
|
||||
---
|
||||
|
||||
## 关键锚点（实现前必读）
|
||||
|
||||
| 用途 | 位置 |
|
||||
|------|------|
|
||||
| chat 缓存/重试 | `adapters/llm.py:271` chat（get L299 / set L373 / 重试 L336）；`_call_streaming:506`；`_consume_stream:533`（不校验 done）；`_iter_sse_deltas:94`（done 标志 L116）；`_is_transient_error:173-186`；`_SseAnomaly:40` |
|
||||
| 缓存键 | `adapters/redis_cache.py:32` `_build_key`（无 salt）；`get:50`；`set:73`（ttl 分支 L89-92） |
|
||||
| Protocol | `core/protocols.py:22-28` `LLMProvider.chat` |
|
||||
| TTL | `main.py:41` `redis_cache_ttl=86400`；`:93` `ttl_s = ... if >0 else None` |
|
||||
| 熔断 | `adapters/breaker.py:24` `is_open`（L36-37 到期即放行，无探针锁）；`record_failure:39`；`record_success:63` |
|
||||
| gate INFRA | `app/harness/validate.py:257` `_resolve_baseline_block`（put L303-304）；`_check_infra_guard:384`（累计检查 L593）；`BaselineCache` in `gate_ladder.py:344`（put L384 原子 L401-403） |
|
||||
| 持久化 | `app/harness/workspace.py` 4 处非原子 write_text：`_scaffold:111`/`update_manifest:282`/`record_run:311`/`update_best:365`；原子范式 `checkpoint.py:257-260` |
|
||||
| 基线元数据 | `app/harness/log.py:57` `HarnessLog.__init__`（upsert `_runs` L73-82）；`RunLogImpl._read_table:247`（只读） |
|
||||
| dual_metric | `app/harness/observation.py:119` `write_dual_metric`（version_kind final L138）；runner 调用 `runner.py:1444-1448` |
|
||||
| 配置 | `.env:65` `REDIS_CACHE_TTL=0`；`.env.example:49` `=86400` |
|
||||
|
||||
## 核心算法保真校验
|
||||
|
||||
触及算法 #6（块顺序验证：基线缓存/INFRA 护栏/配对翻转）——Task 6（gate 基线臂 INFRA 隔离）改 `_resolve_baseline_block` 与护栏时序，**必须逐行比对** TRM4 `/home/iomgaa/Projects/Video-Tree-TRM4/core/harness/validate.py` 的 INFRA 护栏语义，确保只改"INFRA unit 不写缓存/不计 W/L + 护栏前置"，不动配对翻转与基线快照复用逻辑。其余 task 不触碰核心算法。
|
||||
|
||||
---
|
||||
|
||||
## Task 1: 缓存 cache_salt 贯穿
|
||||
|
||||
**Files:**
|
||||
- Modify: `core/protocols.py:22-28`
|
||||
- Modify: `adapters/llm.py:271-373`
|
||||
- Modify: `adapters/redis_cache.py:32-92`
|
||||
- Modify: VLM 适配转发（`adapters/` 内实现 `LLMProvider`/转发 chat 的类，grep 定位）
|
||||
- Test: `tests/unit/test_redis_cache.py`、`tests/unit/test_governed_llm.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试（salt 入键）**
|
||||
|
||||
在 `tests/unit/test_redis_cache.py` 追加：
|
||||
```python
|
||||
@pytest.mark.asyncio
|
||||
async def test_salt_changes_key(fake_redis):
|
||||
from adapters.redis_cache import RedisResponseCache
|
||||
|
||||
cache = RedisResponseCache(redis=fake_redis, ttl_s=None)
|
||||
k_none = cache._build_key("m", [{"role": "user", "content": "x"}], None)
|
||||
k_e1 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e1")
|
||||
k_e2 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e2")
|
||||
assert k_none != k_e1 != k_e2 and k_e1 != k_e2
|
||||
```
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_redis_cache.py::test_salt_changes_key -v`
|
||||
Expected: FAIL（`_build_key() takes 3 positional arguments but 4 were given`）。
|
||||
|
||||
- [ ] **Step 3: redis_cache 加 salt 维度**
|
||||
|
||||
`adapters/redis_cache.py`：
|
||||
- `_build_key(self, model, messages, cache_salt: str | None = None)`：payload 字典加 `"salt": cache_salt`：
|
||||
```python
|
||||
payload = json.dumps(
|
||||
{"model": model, "messages": messages, "salt": cache_salt},
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
```
|
||||
- `get(self, model, messages, cache_salt: str | None = None)`：`key = self._build_key(model, messages, cache_salt)`
|
||||
- `set(self, model, messages, response, cache_salt: str | None = None)`：同样传 `cache_salt`
|
||||
|
||||
- [ ] **Step 4: Protocol + chat 透传 salt**
|
||||
|
||||
`core/protocols.py` `LLMProvider.chat` 签名加 `cache_salt: str | None = None`（keyword-only，放 `parent_call_id` 后）。
|
||||
`adapters/llm.py` `chat` 签名加同参；L299 改 `await self._cache.get(self._model, messages, cache_salt)`；L373 改 `await self._cache.set(self._model, messages, response, cache_salt)`。
|
||||
grep `adapters/` 找到转发 `chat` 的 VLM/包装类（如有），同步加 `cache_salt` 透传。
|
||||
|
||||
- [ ] **Step 5: 运行测试确认通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_redis_cache.py tests/unit/test_governed_llm.py -q`
|
||||
Expected: 全 PASS（默认 `cache_salt=None` 保持旧键，既有缓存测试不受影响）。
|
||||
|
||||
- [ ] **Step 6: 提交**
|
||||
|
||||
```bash
|
||||
git add core/protocols.py adapters/llm.py adapters/redis_cache.py tests/unit/test_redis_cache.py
|
||||
git commit -m "feat: add cache_salt dimension to LLM response cache"
|
||||
```
|
||||
|
||||
> 注：训练链路注入 epoch 盐（`f"{run_id}:e{epoch}"`）在 WP3 的推理调用点完成（本 WP 只提供 salt 能力）。
|
||||
|
||||
---
|
||||
|
||||
## Task 2: TTL 语义修正（≤0 报错）
|
||||
|
||||
**Files:**
|
||||
- Modify: `main.py:93`
|
||||
- Modify: `.env:65`、`.env.example:49`
|
||||
- Test: `tests/unit/test_governed_llm.py` 或新增 `tests/unit/test_infra_settings.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
新增 `tests/unit/test_infra_settings.py`：
|
||||
```python
|
||||
import pytest
|
||||
|
||||
|
||||
def test_redis_cache_ttl_zero_rejected(monkeypatch):
|
||||
"""REDIS_CACHE_TTL<=0 必须启动即报错，消灭'0=永不过期'隐式语义。"""
|
||||
from main import _resolve_cache_ttl # Step 3 抽出的纯函数
|
||||
|
||||
with pytest.raises(ValueError, match="REDIS_CACHE_TTL"):
|
||||
_resolve_cache_ttl(0)
|
||||
with pytest.raises(ValueError, match="REDIS_CACHE_TTL"):
|
||||
_resolve_cache_ttl(-1)
|
||||
assert _resolve_cache_ttl(86400) == 86400
|
||||
```
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_infra_settings.py -v`
|
||||
Expected: FAIL（`cannot import name '_resolve_cache_ttl'`）。
|
||||
|
||||
- [ ] **Step 3: 抽出并实现 fail-loud**
|
||||
|
||||
`main.py`：新增纯函数并替换 L93 逻辑：
|
||||
```python
|
||||
def _resolve_cache_ttl(ttl: int) -> int:
|
||||
"""校验 Redis 缓存 TTL：必须为正整数（消灭 0=永不过期 的隐式语义）。"""
|
||||
if ttl <= 0:
|
||||
raise ValueError(
|
||||
f"REDIS_CACHE_TTL 必须为正整数秒，实际 {ttl}。"
|
||||
"训练场景建议 >= 单次训练时长（如 86400）。"
|
||||
)
|
||||
return ttl
|
||||
```
|
||||
L93 改为 `ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl)`（删除 `if >0 else None` 分支；`RedisResponseCache` 的 `ttl_s` 从此恒为正）。
|
||||
|
||||
- [ ] **Step 4: 改 .env / .env.example**
|
||||
|
||||
`.env:65` `REDIS_CACHE_TTL=0` → `REDIS_CACHE_TTL=86400`。
|
||||
`.env.example:49` 确认为正整数（当前 86400，OK）；补注释 `# 正整数秒，禁止 0（0 会被拒绝启动）`。
|
||||
|
||||
- [ ] **Step 5: 运行测试确认通过**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_infra_settings.py -v`
|
||||
Expected: PASS。
|
||||
|
||||
- [ ] **Step 6: 提交**
|
||||
|
||||
```bash
|
||||
git add main.py .env.example tests/unit/test_infra_settings.py
|
||||
git commit -m "fix: reject REDIS_CACHE_TTL<=0 (kill implicit never-expire)"
|
||||
```
|
||||
> 注：`.env` 不提交（gitignore）；改动需手动同步到运行环境，runbook 会提示。
|
||||
|
||||
---
|
||||
|
||||
## Task 3: SSE 截断检测（未收 [DONE] 即重试，不写缓存）
|
||||
|
||||
**Files:**
|
||||
- Modify: `adapters/llm.py:533-578`（_consume_stream）
|
||||
- Test: `tests/unit/test_governed_llm.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
在 `tests/unit/test_governed_llm.py` 追加：
|
||||
```python
|
||||
@pytest.mark.asyncio
|
||||
async def test_truncated_stream_without_done_raises(_build_client):
|
||||
"""SSE 流耗尽但未收 [DONE] → _SseAnomaly（进重试，不当成功）。"""
|
||||
from adapters.llm import _SseAnomaly, GovernedLLMClient
|
||||
|
||||
async def _lines():
|
||||
yield 'data: {"choices":[{"delta":{"content":"半"}}]}'
|
||||
# 无 data: [DONE] —— 模拟服务端截断
|
||||
|
||||
client = _build_client() # 依现有 fixture
|
||||
with pytest.raises(_SseAnomaly):
|
||||
await client._consume_stream(_lines())
|
||||
```
|
||||
> 依 `test_governed_llm.py` 现有 `_build_client`（L50）/`_FakeRedisCache` 构造；实现前读该文件对齐 client 构造签名。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -k truncated_stream -v`
|
||||
Expected: FAIL（当前耗尽即正常返回，不抛）。
|
||||
|
||||
- [ ] **Step 3: _consume_stream 校验 done**
|
||||
|
||||
`adapters/llm.py` `_consume_stream` 在 `return content, thinking, ttft_ms, max_inter_token_ms, usage`（L578）之前加：
|
||||
```python
|
||||
if not usage_sink.get("done"):
|
||||
raise _SseAnomaly("truncated_no_done")
|
||||
```
|
||||
（`_iter_sse_deltas` 收到 `[DONE]` 时置 `usage_sink["done"]=True`，L116；未置说明流被截断。`_SseAnomaly` 已在 `_is_transient_error` L186 归为可重试，故自动进重试梯且不走成功路径、不写缓存。）
|
||||
|
||||
- [ ] **Step 4: 运行测试确认通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py tests/unit/test_streaming.py -q`
|
||||
Expected: 全 PASS（正常流带 `[DONE]` 的既有测试仍通过；若既有 streaming 测试的假流未含 `[DONE]`，同步补 `data: [DONE]` 终帧）。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add adapters/llm.py tests/unit/test_governed_llm.py
|
||||
git commit -m "fix: treat SSE stream without [DONE] as truncated (retry, no cache)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: 瞬时错误清单扩展（断连族）
|
||||
|
||||
**Files:**
|
||||
- Modify: `adapters/llm.py:182-186`
|
||||
- Test: `tests/unit/test_governed_llm.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
在 `tests/unit/test_governed_llm.py` 追加：
|
||||
```python
|
||||
def test_transient_covers_disconnect_family():
|
||||
import httpx
|
||||
from adapters.llm import _is_transient_error
|
||||
|
||||
assert _is_transient_error(httpx.RemoteProtocolError("peer reset"))
|
||||
assert _is_transient_error(httpx.ReadError("read"))
|
||||
assert _is_transient_error(httpx.ConnectTimeout("ct"))
|
||||
assert _is_transient_error(httpx.PoolTimeout("pt"))
|
||||
```
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -k disconnect_family -v`
|
||||
Expected: FAIL（RemoteProtocolError/PoolTimeout 不在当前清单）。
|
||||
|
||||
- [ ] **Step 3: 扩展 _is_transient_error**
|
||||
|
||||
`adapters/llm.py` L182-183 的：
|
||||
```python
|
||||
if isinstance(exc, (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout)):
|
||||
return True
|
||||
```
|
||||
改为（两族基类覆盖 RemoteProtocolError/ReadError/ConnectTimeout/PoolTimeout 等）：
|
||||
```python
|
||||
if isinstance(exc, (httpx.TimeoutException, httpx.TransportError)):
|
||||
return True
|
||||
```
|
||||
|
||||
- [ ] **Step 4: 运行测试确认通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -q`
|
||||
Expected: 全 PASS（`httpx.HTTPStatusError` 非 TransportError 子类，401/403 致命分支不受影响）。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add adapters/llm.py tests/unit/test_governed_llm.py
|
||||
git commit -m "fix: cover httpx disconnect family in transient error set"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: 熔断半开单探针锁
|
||||
|
||||
**Files:**
|
||||
- Modify: `adapters/breaker.py:18-70`
|
||||
- Test: `tests/unit/test_breaker.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
在 `tests/unit/test_breaker.py::TestCircuitBreaker` 追加：
|
||||
```python
|
||||
def test_half_open_admits_single_probe(self):
|
||||
from adapters.breaker import CircuitBreaker
|
||||
|
||||
b = CircuitBreaker(fail_threshold=2, cooldown_s=10.0)
|
||||
b.record_failure("p", now=0.0)
|
||||
b.record_failure("p", now=0.0) # 开路至 t=10
|
||||
assert b.is_open("p", now=5.0) is True # 冷却中
|
||||
# 冷却到期：只放行第一个探针
|
||||
assert b.is_open("p", now=11.0) is False # 探针 1 放行
|
||||
assert b.is_open("p", now=11.0) is True # 探针 2 被挡（探针在途）
|
||||
b.record_success("p") # 探针成功 → 闭合
|
||||
assert b.is_open("p", now=12.0) is False
|
||||
```
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_breaker.py -k half_open_admits_single -v`
|
||||
Expected: FAIL（当前到期后所有调用都返回 False）。
|
||||
|
||||
- [ ] **Step 3: 实现半开单探针锁**
|
||||
|
||||
`adapters/breaker.py`：
|
||||
- `__init__` 加 `self._half_open_inflight: dict[str, bool] = {}`
|
||||
- `is_open` 改为：
|
||||
```python
|
||||
def is_open(self, source_name: str, now: float) -> bool:
|
||||
until = self._open_until.get(source_name)
|
||||
if until is None:
|
||||
return False
|
||||
if now < until:
|
||||
return True # 冷却中，全挡
|
||||
# 冷却到期：half-open，只放行一个探针
|
||||
if self._half_open_inflight.get(source_name):
|
||||
return True # 已有探针在途，继续挡
|
||||
self._half_open_inflight[source_name] = True
|
||||
return False
|
||||
```
|
||||
- `record_success` 加 `self._half_open_inflight.pop(source_name, None)`（探针成功 → 清在途 + 已有的清 fails/open_until）
|
||||
- `record_failure`：探针失败会累计并可能重开路；末尾加 `self._half_open_inflight.pop(source_name, None)`（让下一轮 cooldown 后可再探）
|
||||
- `force_open`：加 `self._half_open_inflight.pop(source_name, None)`
|
||||
|
||||
（is_open 在 asyncio 单线程内同步执行，"检查+标记探针"原子，无竞态。）
|
||||
|
||||
- [ ] **Step 4: 运行测试确认通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_breaker.py tests/unit/test_governed_llm.py -q`
|
||||
Expected: 全 PASS（既有 circuit_open 测试若假设"到期即多次放行"需同步更新为单探针语义）。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add adapters/breaker.py tests/unit/test_breaker.py
|
||||
git commit -m "fix: half-open circuit admits single probe (no thundering herd)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 6: gate 基线臂 INFRA 隔离（算法 #6 保真区）
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/validate.py:257-311`（_resolve_baseline_block）+ 护栏时序
|
||||
- Test: `tests/unit/test_harness_validate.py`
|
||||
|
||||
> **保真前置**：实现前完整读 `app/harness/validate.py:257-311` `_resolve_baseline_block`、`_candidate_correctness_from_db`、块循环（575-600），并逐行比对 TRM4 `core/harness/validate.py` 的 INFRA 护栏语义。目标仅为：① 基线臂 INFRA 故障（stop_reason∈{error,parse_error}）的 unit **不写 BaselineCache**、**不计入 W/L 翻转**；② 护栏检查移到"写缓存之前"。不得改动配对翻转、基线快照复用、unit 折叠逻辑。
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
在 `tests/unit/test_harness_validate.py` 追加（用 fake run_inference 让某基线 unit 返回 stop_reason=error）：
|
||||
```python
|
||||
@pytest.mark.asyncio
|
||||
async def test_baseline_infra_error_not_cached(tmp_path):
|
||||
"""基线臂 INFRA error 的 unit 不写入 BaselineCache（不永久污染）。"""
|
||||
# 依现有 test_harness_validate.py 的 fake run_inference / BaselineCache fixture 构造；
|
||||
# 让 miss unit u1 的推理返回 prediction=None, stop_reason='error'；
|
||||
# 调 _resolve_baseline_block 后断言 baseline_cache.get(..., u1) is None
|
||||
...
|
||||
```
|
||||
> 依 `test_harness_validate.py` 现有 fixture（fake `run_inference`、`BaselineCache`、`HarnessLog`）；实现前读该文件对齐构造，勿臆造签名。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_validate.py -k baseline_infra_error_not_cached -v`
|
||||
Expected: FAIL（当前 INFRA unit 的对错被 put 进缓存）。
|
||||
|
||||
- [ ] **Step 3: 实现 INFRA 隔离**
|
||||
|
||||
`_resolve_baseline_block`：miss 跑完后，识别 INFRA 题（从 db 读 stop_reason∈{error,parse_error} 或经 run_inference 结果），其所属 unit：不 `baseline_cache.put`、不计入返回的 `b_units`（从块 unit 集合排除，使配对与 W/L 不含它）。护栏所需的 `errors_inc` 在 put 之前累计并检查（护栏前置）。具体实现按 Step 前"保真前置"要求对齐现场与 TRM4。
|
||||
|
||||
- [ ] **Step 4: 运行测试确认通过 + 保真回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_validate.py tests/unit/test_gate_block_unit.py tests/unit/test_gates.py -q`
|
||||
Expected: 全 PASS（配对翻转/e-process 既有测试不受影响 = 保真达成）。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/validate.py tests/unit/test_harness_validate.py
|
||||
git commit -m "fix: isolate gate baseline-arm INFRA errors from BaselineCache (algo #6)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 7: manifest 原子写
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/workspace.py`（`_scaffold:111`/`update_manifest:282`/`record_run:311`/`update_best:365`）
|
||||
- Test: `tests/unit/test_harness_workspace.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
在 `tests/unit/test_harness_workspace.py` 追加：
|
||||
```python
|
||||
def test_update_manifest_is_atomic(workspace_dir, monkeypatch):
|
||||
"""写 manifest 途中崩溃不产生半截 JSON（原子写：tmp 存在即失败也不损原文件）。"""
|
||||
import json
|
||||
from app.harness import workspace as ws
|
||||
|
||||
# 先建合法 manifest
|
||||
... # 依现有 fixture 初始化 workspace
|
||||
original = (workspace_dir / "manifest.json").read_text()
|
||||
|
||||
# monkeypatch os.replace 抛异常，模拟替换阶段崩溃
|
||||
def _boom(src, dst):
|
||||
raise OSError("crash during replace")
|
||||
monkeypatch.setattr(ws.os, "replace", _boom)
|
||||
with pytest.raises(OSError):
|
||||
ws.update_manifest(workspace_dir, skills="skills/v2")
|
||||
# 原 manifest 未被破坏
|
||||
assert (workspace_dir / "manifest.json").read_text() == original
|
||||
assert json.loads((workspace_dir / "manifest.json").read_text())
|
||||
```
|
||||
> 依 `test_harness_workspace.py` 现有 workspace 初始化 fixture；实现前读对齐。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_workspace.py -k update_manifest_is_atomic -v`
|
||||
Expected: FAIL（当前 write_text 非原子，崩溃留半截）。
|
||||
|
||||
- [ ] **Step 3: 加原子写 helper 并替换 4 处**
|
||||
|
||||
`app/harness/workspace.py` 顶部确认 `import os`。新增模块级 helper：
|
||||
```python
|
||||
def _atomic_write_json(path: Path, data: dict) -> None:
|
||||
"""原子写 JSON：tmp + os.replace（对齐 checkpoint.py 范式，防半截损坏）。"""
|
||||
tmp = path.with_name(path.name + ".tmp")
|
||||
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
```
|
||||
把 `_scaffold_workspace`（L111）、`update_manifest`（L282）、`record_run`（L311）、`update_best`（L365）四处的 `(...).write_text(json.dumps(...))` 替换为 `_atomic_write_json(<path>, <data>)`。
|
||||
|
||||
- [ ] **Step 4: 运行测试确认通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_workspace.py -q`
|
||||
Expected: 全 PASS。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/workspace.py tests/unit/test_harness_workspace.py
|
||||
git commit -m "fix: atomic writes for manifest/record_run/update_best (tmp+replace)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 8: 只读查询不改基线元数据
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/log.py:57-83`（HarnessLog.__init__ 加 register_run 开关）
|
||||
- Modify: 只读查询基线的调用点（`app/harness/runner.py:923` `_init_gate`、`app/harness/pools.py:765,818`）
|
||||
- Test: `tests/unit/test_harness_log.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
在 `tests/unit/test_harness_log.py::TestHarnessLogUpsert` 追加：
|
||||
```python
|
||||
def test_register_run_false_skips_upsert(self, tmp_path):
|
||||
"""register_run=False 时只读打开不改写已有 _runs 行（started_at/status 不变）。"""
|
||||
db = str(tmp_path / "h.db")
|
||||
from app.harness.log import HarnessLog
|
||||
|
||||
with HarnessLog(db, "r1") as log:
|
||||
log # 初次注册
|
||||
row0 = _read_run_row(db, "r1") # 依现有 helper 读 started_at/status
|
||||
|
||||
with HarnessLog(db, "r1", register_run=False) as log:
|
||||
log.query("SELECT 1") # 只读
|
||||
row1 = _read_run_row(db, "r1")
|
||||
assert row1["started_at"] == row0["started_at"]
|
||||
assert row1["status"] == row0["status"]
|
||||
```
|
||||
> 依 `test_harness_log.py` 现有读行 helper；实现前读 `TestHarnessLogUpsert`（L196）对齐。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_log.py -k register_run_false -v`
|
||||
Expected: FAIL（`unexpected keyword argument 'register_run'`）。
|
||||
|
||||
- [ ] **Step 3: HarnessLog 加 register_run 开关**
|
||||
|
||||
`app/harness/log.py` `__init__` 签名加 `register_run: bool = True`（keyword）。把 L73-82 的 `_runs` upsert 包进 `if register_run:`；`register_run=False` 时跳过 upsert（仅 `_init_fixed_tables` 建表 + 连接，供只读查询）。`__exit__`/`close` 的 status 更新同样在 `register_run` 为 True 时才执行（避免只读关闭把 status 改 completed）。
|
||||
|
||||
- [ ] **Step 4: 只读调用点传 register_run=False**
|
||||
|
||||
`app/harness/runner.py:923`（_init_gate 用 baseline_run_id 只读查 predictions）、`app/harness/pools.py:765,818`（build_or_load_pools 只读查 predictions）三处 `HarnessLog(...)` 调用加 `register_run=False`。
|
||||
|
||||
- [ ] **Step 5: 运行测试确认通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_log.py tests/unit/test_harness_pools.py -q`
|
||||
Expected: 全 PASS。
|
||||
|
||||
- [ ] **Step 6: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/log.py app/harness/runner.py app/harness/pools.py tests/unit/test_harness_log.py
|
||||
git commit -m "fix: read-only baseline queries skip _runs upsert (register_run flag)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 9: dual_metric version_kind 口径修正
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/runner.py:1444-1448`（第二次 write_dual_metric）
|
||||
- Test: `tests/unit/test_harness_observation.py` 或 `test_harness_runner.py`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
在 `tests/unit/test_harness_observation.py` 追加（验证 slow_candidate 与 final 可区分）：
|
||||
```python
|
||||
def test_slow_candidate_kind_distinct_from_final(tmp_path):
|
||||
from app.harness.observation import write_dual_metric, read_dual_metric
|
||||
|
||||
db = str(tmp_path / "h.db")
|
||||
write_dual_metric(db, run_id="r", epoch=1, version_kind="final",
|
||||
skills_version="v1", prompts_version="v1", pool="val",
|
||||
hard_acc=0.7, soft_score=None, mixed_score=None)
|
||||
write_dual_metric(db, run_id="r", epoch=1, version_kind="slow_candidate",
|
||||
skills_version="v2", prompts_version="v2", pool="val",
|
||||
hard_acc=0.6, soft_score=None, mixed_score=None)
|
||||
rows = read_dual_metric(db, run_id="r", epoch=1)
|
||||
kinds = {r["version_kind"] for r in rows}
|
||||
assert "final" in kinds and "slow_candidate" in kinds
|
||||
```
|
||||
> 依 `test_harness_observation.py` 现有 read_dual_metric 签名；实现前读对齐。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败/通过基线**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_observation.py -k slow_candidate -v`
|
||||
Expected: 若 read_dual_metric 已支持则 PASS 基线；重点是 Step 3 改 runner 调用点。
|
||||
|
||||
- [ ] **Step 3: 改 runner 慢更新第二次写为 slow_candidate**
|
||||
|
||||
`app/harness/runner.py` L1444-1448 的 `write_dual_metric(..., version_kind="final", ...)`（`_slow_update_cycle` Phase 8 的 R2 行）改为 `version_kind="slow_candidate"`，使被 revert 的慢更新候选不再占用 `final` 语义。确认 Phase 2 的第一次（L1399-1403）仍为 `final`（epoch 终值唯一）。
|
||||
|
||||
- [ ] **Step 4: 运行测试确认通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_observation.py tests/unit/test_harness_runner.py -q`
|
||||
Expected: 全 PASS。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/runner.py tests/unit/test_harness_observation.py
|
||||
git commit -m "fix: slow-update R2 dual_metric uses slow_candidate kind (not final)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 10: 离线 --retry-uncertain + docstring 修正
|
||||
|
||||
**Files:**
|
||||
- Modify: `app/harness/video_split_cli.py`（build_arg_parser + run_pipeline）
|
||||
- Modify: `app/harness/baseline_diagnosis.py`（done 集排除 uncertain + docstring）
|
||||
- Test: `tests/unit/test_video_split_cli.py` 或 `test_baseline_diagnosis`
|
||||
|
||||
- [ ] **Step 1: 写失败测试**
|
||||
|
||||
定位 `baseline_diagnosis` 的 done_question_ids 计算函数（`app/harness/baseline_diagnosis.py`，grep `done_question_ids`），追加测试：`retry_uncertain=True` 时 tier=uncertain 的已落库题不计入 done（会被重诊）。
|
||||
```python
|
||||
def test_retry_uncertain_excludes_uncertain_from_done(...):
|
||||
# 构造 baseline_diagnosis 表含 tier=T2 与 tier=uncertain 行；
|
||||
# done_ids(retry_uncertain=False) 含 uncertain 题；
|
||||
# done_ids(retry_uncertain=True) 不含 uncertain 题
|
||||
...
|
||||
```
|
||||
> 实现前读 `baseline_diagnosis.py` 的 done 集查询函数签名对齐。
|
||||
|
||||
- [ ] **Step 2: 运行确认失败**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest -k retry_uncertain -v`
|
||||
Expected: FAIL（无该参数）。
|
||||
|
||||
- [ ] **Step 3: 实现 retry_uncertain**
|
||||
|
||||
`app/harness/baseline_diagnosis.py`：done 集查询函数加 `retry_uncertain: bool = False`，True 时 `WHERE ... AND tier != 'uncertain'`（uncertain 题不算完成，会被重诊）。修正模块 docstring 把"崩溃最多丢正在写的一行/逐行落库"表述改为实际的"run 末批量落库（Phase 3），崩溃丢本次 run 全部未落库结果，靠 Redis 缓存缓解重烧"。
|
||||
`app/harness/video_split_cli.py`：`build_arg_parser` 加 `--retry-uncertain`（store_true）；`run_pipeline` 透传到 `run_baseline_diagnosis` 的 done 集计算。
|
||||
|
||||
- [ ] **Step 4: 运行测试确认通过 + 回归**
|
||||
|
||||
Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -q`
|
||||
Expected: 全 PASS。
|
||||
|
||||
- [ ] **Step 5: 提交**
|
||||
|
||||
```bash
|
||||
git add app/harness/video_split_cli.py app/harness/baseline_diagnosis.py tests/unit/
|
||||
git commit -m "feat: --retry-uncertain re-diagnoses uncertain rows; fix docstring"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review（作者自查，执行者复核）
|
||||
|
||||
- [ ] cache_salt 四处（protocols/llm/redis_cache/VLM 转发）贯通，默认 None 保持旧键。
|
||||
- [ ] TTL≤0 fail-loud + .env/.env.example 同步（.env 手动，runbook 提示）。
|
||||
- [ ] SSE 未收 [DONE] → _SseAnomaly（已在瞬时清单 → 重试 + 不写缓存）。
|
||||
- [ ] 瞬时清单用 TimeoutException+TransportError 两族基类覆盖断连族，不误纳 HTTPStatusError。
|
||||
- [ ] 熔断半开单探针：is_open 检查+标记原子（asyncio 单线程）。
|
||||
- [ ] Task 6 触及算法 #6，已设保真前置（逐行比对 TRM4，只改 INFRA 不写缓存 + 护栏前置）。
|
||||
- [ ] manifest 4 处原子写；只读查询 register_run=False 不改基线元数据。
|
||||
|
||||
## 核心算法保真校验结论
|
||||
|
||||
Task 6 触及算法 #6（块顺序验证），已在该 Task 设"保真前置"要求逐行比对 TRM4 `validate.py`，仅改 INFRA 隔离与护栏时序，不动配对翻转/基线快照/unit 折叠。其余 Task 均为治理栈/持久化层，不涉及核心算法。
|
||||
|
||||
## 验收标准
|
||||
|
||||
1. `pytest tests/unit/test_redis_cache.py tests/unit/test_governed_llm.py tests/unit/test_breaker.py tests/unit/test_streaming.py tests/unit/test_harness_validate.py tests/unit/test_harness_workspace.py tests/unit/test_harness_log.py tests/unit/test_harness_observation.py tests/unit/test_infra_settings.py` 全绿。
|
||||
2. 缓存加 salt 后跨 epoch 键不同、同 epoch 键相同。
|
||||
3. 熔断半开只放一个探针；SSE 截断进重试不写缓存。
|
||||
4. manifest 原子写；基线 _runs 元数据只读查询不被改写。
|
||||
@@ -0,0 +1,9 @@
|
||||
---
|
||||
type: plan
|
||||
node_id: plan:preflight-wp1-asset-migration
|
||||
title: "WP1 资产迁移"
|
||||
date: 2026-07-16
|
||||
---
|
||||
|
||||
# WP1 资产迁移
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
---
|
||||
type: plan
|
||||
node_id: plan:preflight-wp2-split-wiring
|
||||
title: "WP2 切分与接线"
|
||||
date: 2026-07-16
|
||||
---
|
||||
|
||||
# WP2 切分与接线
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
---
|
||||
type: plan
|
||||
node_id: plan:preflight-wp3-train-loop
|
||||
title: "WP3 训练循环与进化引擎"
|
||||
date: 2026-07-16
|
||||
---
|
||||
|
||||
# WP3 训练循环与进化引擎
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
---
|
||||
type: plan
|
||||
node_id: plan:preflight-wp4-resilience
|
||||
title: "WP4 韧性与持久化"
|
||||
date: 2026-07-16
|
||||
---
|
||||
|
||||
# WP4 韧性与持久化
|
||||
|
||||
Reference in New Issue
Block a user