diff --git a/research-wiki/graph/edges.json b/research-wiki/graph/edges.json index 7ff9d9f..75a4e36 100644 --- a/research-wiki/graph/edges.json +++ b/research-wiki/graph/edges.json @@ -245,6 +245,26 @@ "id": "design:preflight-fixes", "label": "训练前缺陷修复设计", "type": "design" + }, + { + "id": "plan:preflight-wp1-asset-migration", + "label": "WP1 资产迁移", + "type": "plan" + }, + { + "id": "plan:preflight-wp2-split-wiring", + "label": "WP2 切分与接线", + "type": "plan" + }, + { + "id": "plan:preflight-wp3-train-loop", + "label": "WP3 训练循环与进化引擎", + "type": "plan" + }, + { + "id": "plan:preflight-wp4-resilience", + "label": "WP4 韧性与持久化", + "type": "plan" } ], "links": [ @@ -492,6 +512,34 @@ "relation": "implements", "evidence": "实现修复设计的 5 个 Task", "added": "2026-07-16T02:01:19.350441+00:00" + }, + { + "source": "plan:preflight-wp1-asset-migration", + "target": "design:preflight-fixes", + "relation": "implements", + "evidence": "四工作包实现训练前缺陷修复设计", + "added": "2026-07-16T08:32:42.737883+00:00" + }, + { + "source": "plan:preflight-wp2-split-wiring", + "target": "design:preflight-fixes", + "relation": "implements", + "evidence": "四工作包实现训练前缺陷修复设计", + "added": "2026-07-16T08:32:42.822569+00:00" + }, + { + "source": "plan:preflight-wp3-train-loop", + "target": "design:preflight-fixes", + "relation": "implements", + "evidence": "四工作包实现训练前缺陷修复设计", + "added": "2026-07-16T08:32:42.906881+00:00" + }, + { + "source": "plan:preflight-wp4-resilience", + "target": "design:preflight-fixes", + "relation": "implements", + "evidence": "四工作包实现训练前缺陷修复设计", + "added": "2026-07-16T08:32:42.990273+00:00" } ] } \ No newline at end of file diff --git a/research-wiki/index.md b/research-wiki/index.md index 8038143..0e347cc 100644 --- a/research-wiki/index.md +++ b/research-wiki/index.md @@ -1,6 +1,6 @@ # Research Wiki 索引 -> 自动生成，更新时间：2026-07-16 07:43 UTC +> 自动生成，更新时间：2026-07-16 08:32 UTC ## design (36) - [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design` @@ -52,7 +52,7 @@ - [v3 §9 帧感知抽取机制 — 小样本实证验证结果](findings/2026-07-15-v3-frame-perception-spike-validation.md) `finding:2026-07-15-v3-frame-perception-spike-validation` - [出题范式转变 — 从"生成-打分-过滤"转向"构造优先 + 两正交独立信号"（六篇原文深读）](findings/2026-07-15-question-gen-paradigm-shift-construction-over-filtering.md) `finding:2026-07-15-question-gen-paradigm-shift-construction-over-filtering` -## plan (40) +## plan (48) - [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm` - [2026-07-07-app-harness](plans/2026-07-07-app-harness.md) `plan:2026-07-07-app-harness` - [2026-07-07-core-evolution](plans/2026-07-07-core-evolution.md) `plan:2026-07-07-core-evolution` @@ -72,6 +72,10 @@ - [2026-07-14-task-type-strategy-framework](plans/2026-07-14-task-type-strategy-framework.md) `plan:2026-07-14-task-type-strategy-framework` - [2026-07-15-question-gen-v3-phase1-contract](plans/2026-07-15-question-gen-v3-phase1-contract.md) `plan:2026-07-15-question-gen-v3-phase1-contract` - [2026-07-15-results-driven-video-split](plans/2026-07-15-results-driven-video-split.md) `plan:2026-07-15-results-driven-video-split` +- [2026-07-16-preflight-wp1-asset-migration](plans/2026-07-16-preflight-wp1-asset-migration.md) `plan:2026-07-16-preflight-wp1-asset-migration` +- [2026-07-16-preflight-wp2-split-wiring](plans/2026-07-16-preflight-wp2-split-wiring.md) `plan:2026-07-16-preflight-wp2-split-wiring` +- [2026-07-16-preflight-wp3-train-loop](plans/2026-07-16-preflight-wp3-train-loop.md) `plan:2026-07-16-preflight-wp3-train-loop` +- [2026-07-16-preflight-wp4-resilience](plans/2026-07-16-preflight-wp4-resilience.md) `plan:2026-07-16-preflight-wp4-resilience` - [Action Recognition 单题型首次训练实验计划](plans/action-recognition-training.md) `plan:action-recognition-training` - [ActionRecognitionStrategy 特化实现计划 (Plan B)](plans/action-recognition-strategy.md) `plan:action-recognition-strategy` - [Adversarial Question-Gen Phase B](plans/adversarial-question-gen-phaseB.md) `plan:adversarial-question-gen-phaseB` @@ -86,6 +90,10 @@ - [Spec-1 Agent 执行环境修复实现计划](plans/agent-runtime-fixes-plan.md) `plan:agent-runtime-fixes-plan` - [Spec-2 建树批量并行实现计划](plans/batch-tree-build-plan.md) `plan:batch-tree-build-plan` - [TaskTypeStrategy 框架实现计划 (Plan A)](plans/task-type-strategy-framework.md) `plan:task-type-strategy-framework` +- [WP1 资产迁移](plans/preflight-wp1-asset-migration.md) `plan:preflight-wp1-asset-migration` +- [WP2 切分与接线](plans/preflight-wp2-split-wiring.md) `plan:preflight-wp2-split-wiring` +- [WP3 训练循环与进化引擎](plans/preflight-wp3-train-loop.md) `plan:preflight-wp3-train-loop` +- [WP4 韧性与持久化](plans/preflight-wp4-resilience.md) `plan:preflight-wp4-resilience` - [出题管线 v2 实现计划](plans/2026-07-11-question-gen-v2.md) `plan:2026-07-11-question-gen-v2` - [实现计划: 修复诊断 tree_data 断链 bug](plans/fix-diagnosis-tree-data-link-plan.md) `plan:fix-diagnosis-tree-data-link-plan` - [建树修复管线三项改造实现计划](plans/tree-repair-resilience.md) `plan:tree-repair-resilience` diff --git a/research-wiki/log.md b/research-wiki/log.md index 55e431d..3cf8936 100644 --- a/research-wiki/log.md +++ b/research-wiki/log.md @@ -123,3 +123,12 @@ - [2026-07-16 02:01 UTC] 重建索引: 98 篇页面 - [2026-07-16 07:43 UTC] 新增 design: 训练前缺陷修复设计 (design:preflight-fixes) - [2026-07-16 07:43 UTC] 重建索引: 101 篇页面 +- [2026-07-16 08:32 UTC] 新增 plan: WP1 资产迁移 (plan:preflight-wp1-asset-migration) +- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp1-asset-migration --implements--> design:preflight-fixes +- [2026-07-16 08:32 UTC] 新增 plan: WP2 切分与接线 (plan:preflight-wp2-split-wiring) +- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp2-split-wiring --implements--> design:preflight-fixes +- [2026-07-16 08:32 UTC] 新增 plan: WP3 训练循环与进化引擎 (plan:preflight-wp3-train-loop) +- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp3-train-loop --implements--> design:preflight-fixes +- [2026-07-16 08:32 UTC] 新增 plan: WP4 韧性与持久化 (plan:preflight-wp4-resilience) +- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp4-resilience --implements--> design:preflight-fixes +- [2026-07-16 08:32 UTC] 重建索引: 109 篇页面 diff --git a/research-wiki/plans/2026-07-16-preflight-wp1-asset-migration.md b/research-wiki/plans/2026-07-16-preflight-wp1-asset-migration.md new file mode 100644 index 0000000..0e057f2 --- /dev/null +++ b/research-wiki/plans/2026-07-16-preflight-wp1-asset-migration.md @@ -0,0 +1,235 @@ +# WP1 资产迁移 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 迁移 5 个遗漏的 TRM4 进化/动量模板到 TRM5，删除 2 个死字段，把加载器的静默空串兜底改为 fail-loud，解锁自进化引擎。 + +**Architecture:** 进化引导 prompt 是引擎的一部分（放项目根 `prompts/`，不参与版本化进化）。TRM4 五模板的输出 JSON 契约与 TRM5 解析代码已核实完全对齐，可直接拷贝。执行顺序：先迁移模板 → 删死字段（`consolidate_system`/`span_eval_user`，零消费）→ 加载器 fail-loud（顺序关键：先删死字段，fail-loud 才不会对不存在也不需要的模板报错）。 + +**Tech Stack:** Python 3.11、pytest、frozen dataclass（`core/evolution/types.py`）。 + +**设计源**：`research-wiki/designs/2026-07-16-preflight-fixes-design.md §4` + +--- + +## 关键锚点（实现前必读） + +| 用途 | 位置 | +|------|------| +| TRM4 源模板 | `/home/iomgaa/Projects/Video-Tree-TRM4/prompts/{evolve_skill,evolve_system,evolve_tool,evolve_rank,slow_momentum}.md` | +| evolve 加载器 | `app/harness/runner.py:2266-2280` `_load_evolve_prompts` | +| diagnose 加载器 | `app/harness/runner.py:2282-2299` `_load_diagnose_prompts` | +| 平行 diagnose 加载器 | `app/harness/video_split_cli.py:362-378` `_load_diagnose_prompts` | +| dataclass 定义 | `core/evolution/types.py:477-522`（`DiagnosePrompts` L494-501 / `EvolvePrompts` L518-522） | +| 死字段 `consolidate_system` 内联替代 | `core/evolution/evolve.py:997` `_CONSOLIDATE_SYSTEM`（消费点 L1036） | +| 死字段 `span_eval_user` 无消费 | diagnose 只用 `prompts.span_eval_system`（`core/evolution/diagnose.py:511`），user_prompt 内联构造 | +| 测试 fixture | `test_evolve.py:682` `consolidate_system="cons"`；`test_diagnose.py:753` `span_eval_user=""`；`test_evolution_types.py:352` `span_eval_user="p4"` / `:370` `consolidate_system="consolidate_tmpl"` | + +## 核心算法保真校验 + +本计划不迁移/不改核心算法（ARCHITECTURE §6），只搬运模板文件 + 清理死字段 + 加 fail-loud。模板内容是 evolve 引擎（算法 #8）的输入数据，非算法逻辑本身；迁移已核实输出契约（`suggestions`/`edits`/`edits_extract`/`edits_verify`/`selected_indices`/`slow_update_content`）与 TRM5 解析代码逐字对齐。**保真检查点（Task 1 Step 4）**：加载 evolve_tool.md 后确认其要求 LLM 返回 `edits_extract`+`edits_verify` 双键（对齐 `evolve.py:1433-1435`）。 + +--- + +## Task 1: 迁移 5 个 TRM4 模板 + +**Files:** +- Create: `prompts/evolve_skill.md`、`prompts/evolve_system.md`、`prompts/evolve_tool.md`、`prompts/evolve_rank.md`、`prompts/slow_momentum.md`（从 TRM4 拷贝） +- Test: `tests/unit/test_evolve_prompts_present.py` + +- [ ] **Step 1: 拷贝 5 个模板** + +Run: +```bash +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_skill.md prompts/evolve_skill.md +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_system.md prompts/evolve_system.md +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_tool.md prompts/evolve_tool.md +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_rank.md prompts/evolve_rank.md +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/slow_momentum.md prompts/slow_momentum.md +``` +Expected: 5 文件存在于 `prompts/`。 + +- [ ] **Step 2: 写契约冒烟测试（先失败）** + +`tests/unit/test_evolve_prompts_present.py`： +```python +"""校验 5 个进化/动量模板存在且输出契约关键词与解析代码对齐。""" +from pathlib import Path + +import pytest + +_PROMPTS_DIR = Path("prompts") + + +@pytest.mark.parametrize( + "name, required_tokens", + [ + ("evolve_skill.md", ["suggestions", "edits"]), + ("evolve_system.md", ["suggestions", "edits"]), + ("evolve_tool.md", ["edits_extract", "edits_verify"]), + ("evolve_rank.md", ["selected_indices"]), + ("slow_momentum.md", ["slow_update_content"]), + ], +) +def test_evolve_template_present_and_contract(name: str, required_tokens: list[str]) -> None: + path = _PROMPTS_DIR / name + assert path.exists(), f"缺模板: {path}" + text = path.read_text(encoding="utf-8") + assert text.strip(), f"模板为空: {path}" + for token in required_tokens: + assert token in text, f"{name} 缺输出契约关键词 {token!r}（与解析代码不对齐）" +``` + +- [ ] **Step 3: 运行确认通过（模板已在 Step 1 拷入）** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v` +Expected: 5 参数化用例全 PASS。若 evolve_tool.md 缺 `edits_extract`/`edits_verify` 则契约不符——停止并逐行比对 TRM4 源。 + +- [ ] **Step 4: 保真检查点** + +Run: `conda run -n Video-Tree-TRM python -c "print('edits_extract' in open('prompts/evolve_tool.md').read() and 'edits_verify' in open('prompts/evolve_tool.md').read())"` +Expected: `True`（对齐 `evolve.py:1433-1435` 的 `parsed["edits_extract"]`/`parsed["edits_verify"]`）。 + +- [ ] **Step 5: 提交** + +```bash +git add prompts/evolve_skill.md prompts/evolve_system.md prompts/evolve_tool.md prompts/evolve_rank.md prompts/slow_momentum.md tests/unit/test_evolve_prompts_present.py +git commit -m "feat: migrate 5 evolve/momentum templates from TRM4 (algo #8)" +``` + +--- + +## Task 2: 删除 2 个死字段（consolidate_system / span_eval_user） + +**Files:** +- Modify: `core/evolution/types.py:494-501,518-522` +- Modify: `app/harness/runner.py:2279,2294` +- Modify: `app/harness/video_split_cli.py:374` +- Modify: `tests/unit/test_evolve.py:682`、`tests/unit/test_diagnose.py:753`、`tests/unit/test_evolution_types.py:352,370` + +- [ ] **Step 1: 删 dataclass 字段与 docstring** + +`core/evolution/types.py` — `DiagnosePrompts` 删 `span_eval_user`： +- docstring 删行 ` span_eval_user: span 评估用户提示模板。`（L487） +- 字段删行 ` span_eval_user: str`（L497） + +`EvolvePrompts` 删 `consolidate_system`： +- docstring 删行 ` consolidate_system: appendix 压缩系统提示。`（L515） +- 字段删行 ` consolidate_system: str`（L522） + +- [ ] **Step 2: 删加载器对死字段的 `_read` 行** + +`app/harness/runner.py`： +- `_load_evolve_prompts` 删行 ` consolidate_system=_read("consolidate_system.md"),`（L2279） +- `_load_diagnose_prompts` 删行 ` span_eval_user=_read("span_eval_user.md"),`（L2294） + +`app/harness/video_split_cli.py`： +- `_load_diagnose_prompts` 删行 ` span_eval_user=_read("span_eval_user.md"),`（L374） + +- [ ] **Step 3: 删测试 fixture 对死字段的赋值** + +- `tests/unit/test_evolve.py:682` 删行 ` consolidate_system="cons",` +- `tests/unit/test_diagnose.py:753` 删行 ` span_eval_user="",` +- `tests/unit/test_evolution_types.py:352` 删行 ` span_eval_user="p4",` +- `tests/unit/test_evolution_types.py:370` 删行 ` consolidate_system="consolidate_tmpl",` + +注：`test_evolution_types.py` 可能有断言逐字段比对；若该测试断言字段数量或逐字段相等，同步移除对两个死字段的断言（读该测试确认，删净引用）。 + +- [ ] **Step 4: 运行相关测试确认通过** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py -q` +Expected: 全 PASS（无 `TypeError: unexpected keyword argument` / 无 `missing positional argument`）。 + +- [ ] **Step 5: 全库确认无残留引用** + +Run: `conda run -n Video-Tree-TRM python -c "import subprocess,sys; r=subprocess.run(['grep','-rn','consolidate_system\|span_eval_user','core/','app/','tests/'],capture_output=True,text=True); print(r.stdout)"` +Expected: 空输出（`consolidate_appendix` 用内联 `_CONSOLIDATE_SYSTEM` 不算 `consolidate_system` 字段引用；若出现请确认非 dataclass 字段引用）。 + +- [ ] **Step 6: 提交** + +```bash +git add core/evolution/types.py app/harness/runner.py app/harness/video_split_cli.py tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py +git commit -m "refactor: drop dead prompt fields consolidate_system/span_eval_user" +``` + +--- + +## Task 3: 加载器 fail-loud（缺模板即报错） + +**Files:** +- Modify: `app/harness/runner.py:2270-2272,2286-2288` +- Modify: `app/harness/video_split_cli.py:365-367` +- Test: `tests/unit/test_evolve_prompts_present.py`（追加） + +- [ ] **Step 1: 追加 fail-loud 测试（先失败）** + +在 `tests/unit/test_evolve_prompts_present.py` 追加： +```python +def test_loader_fail_loud_on_missing_template(tmp_path, monkeypatch): + """加载器遇缺失模板必须抛 FileNotFoundError，不静默返回空串。""" + monkeypatch.chdir(tmp_path) # 空目录下无 prompts/，_read 应 fail-loud + (tmp_path / "prompts").mkdir() + from app.harness.runner import Runner + + # 直接测私有 _read 语义：借最小构造不便，改测行为——缺文件应 raise + def _read(name: str) -> str: + p = Path("prompts") / name + if not p.exists(): + raise FileNotFoundError(f"缺进化/诊断模板: {p}（请从 TRM4 迁移或检查 prompts/）") + return p.read_text(encoding="utf-8") + + with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"): + _read("evolve_skill.md") +``` + +> 说明：`_read` 是嵌套闭包不易直接导入，此测试固化期望语义（缺文件 raise + 消息含"缺进化/诊断模板"）；Step 2 把三处闭包改成同款实现，集成层由 `test_harness_runner.py` 现有加载路径覆盖。 + +- [ ] **Step 2: 三处 `_read` 闭包改 fail-loud** + +`app/harness/runner.py` `_load_evolve_prompts`（L2270-2272）与 `_load_diagnose_prompts`（L2286-2288），以及 `app/harness/video_split_cli.py` `_load_diagnose_prompts`（L365-367），把： +```python + def _read(name: str) -> str: + p = Path("prompts") / name + return p.read_text(encoding="utf-8") if p.exists() else "" +``` +改为： +```python + def _read(name: str) -> str: + p = Path("prompts") / name + if not p.exists(): + raise FileNotFoundError(f"缺进化/诊断模板: {p}（请从 TRM4 迁移或检查 prompts/）") + return p.read_text(encoding="utf-8") +``` +（runner.py 缩进 12 空格；video_split_cli.py 的 `_read` 缩进按其函数体，见 L365 为 8 空格——按各自现场缩进套用。） + +- [ ] **Step 3: 运行测试确认通过** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v` +Expected: 全 PASS。 + +- [ ] **Step 4: 运行 runner 加载路径回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -q` +Expected: 全 PASS（模板已迁移，真实加载不再命中缺失分支）。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/runner.py app/harness/video_split_cli.py tests/unit/test_evolve_prompts_present.py +git commit -m "fix: fail-loud on missing evolve/diagnose templates (no silent empty)" +``` + +--- + +## Self-Review（作者自查，执行者复核） + +- [ ] 5 模板均已迁移且契约测试覆盖关键字段。 +- [ ] `consolidate_system`/`span_eval_user` 在 core/app/tests 全库无残留字段引用。 +- [ ] 三处 loader（runner 两处 + video_split_cli 一处）均已 fail-loud。 +- [ ] 执行顺序正确：Task 2（删死字段）先于 Task 3（fail-loud），避免对不需要的模板报错。 + +## 验收标准 + +1. `pytest tests/unit/test_evolve_prompts_present.py tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py tests/unit/test_harness_runner.py` 全绿。 +2. `grep -rn 'consolidate_system\|span_eval_user' core/ app/ tests/` 无 dataclass 字段残留。 +3. `prompts/` 下 5 个新模板存在且非空。 diff --git a/research-wiki/plans/2026-07-16-preflight-wp2-split-wiring.md b/research-wiki/plans/2026-07-16-preflight-wp2-split-wiring.md new file mode 100644 index 0000000..673aa3d --- /dev/null +++ b/research-wiki/plans/2026-07-16-preflight-wp2-split-wiring.md @@ -0,0 +1,626 @@ +# WP2 切分与接线 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 让冻结的 video-split 切分能正确进入训练 workspace（seed 携带 pools.json + global 一致性校验），并把切分质量三处优化（val_ratio 0.4、tier 感知 diag/val 分配、val 功效修复）与冻结产物覆盖保护落地。 + +**Architecture:** 切分产物由 `video_split_cli` 冻结到 `workspaces/video-split/`；本 WP 让 seed 携带该产物、训练 fresh 时拷入 workspace 并校验一致性。tier 感知在 `_split_trainval_by_video_group` 内实现——错题视频组按 T2(defect) 含量升序进 val（保留 T2 高的组在 diag），并把 `val_wrong_min` 前置到切分内做功效修复（不足则从 diag 换出低 T2 错题组补 val，耗尽 fail-loud）。 + +**Tech Stack:** Python 3.11、pytest、SQLite、frozen dataclass、shutil、原子写（tmp+os.replace）。 + +**设计源**：`research-wiki/designs/2026-07-16-preflight-fixes-design.md §5` + +--- + +## 关键锚点（实现前必读） + +| 用途 | 位置 | +|------|------| +| trainval→diag/val 切分 | `app/harness/pools.py:126-199` `split_by_video_assignment`；`:289-338` `_split_trainval_by_video_group`；`:118` `InsufficientValSignal` | +| global 加载（无校验） | `app/harness/pools.py:713-813` `build_or_load_pools`（L813 `return load_pools` 前无 global 校验）；`:587-620` `load_pools` | +| 冻结编排 | `app/harness/build_split.py:104-219` `build_split`（signal_rows 含 tier L154；split_by_video_assignment 调用 L184-191；save_pools L192）；`:46-71` `SplitBuildConfig`（无 val_wrong_min） | +| CLI 构造 | `app/harness/video_split_cli.py:559-577` `SplitBuildConfig(...)`；`:580` `check_mcnemar_power`；`:751-773` `build_arg_parser`（无 --force） | +| seed | `app/harness/store.py:184-232` `init_seed`（拷 skills/prompts/baseline.db，不拷 pools）；`:269-307` `extract_run_db`（不去重） | +| workspace | `app/harness/workspace.py:156-200` `init_workspace_from_seed`（copy2 baseline.db→harness.db L197，不拷 pools） | +| manifest | `app/harness/split_manifest.py:19-59` `write_manifest`（pools_sha256 L54） | +| 配置 | `config/video_split.yaml`（val_ratio L15=0.3、val_wrong_min L14=20） | +| 测试 | `tests/unit/test_pools_video_atomic.py`、`test_split_selection.py`、`test_harness_pools.py`、`test_harness_store.py`、`test_harness_workspace.py` | + +## 核心算法保真校验 + +触及算法 #5（信息阶梯）的**上游输入**：本 WP 只改"哪些视频进 diag/val"，pools 内仍是逐 unit 列表，`gate_ladder` 消费的 unit+correctness 结构不变。**保真检查点（Task 3 Step 6）**：确认 `_split_trainval_by_video_group` 返回后 diagnosis/validation 仍是逐题 `GeneratedQuestion` 列表、视频组原子性（同 video 全部题同池）不被 tier 排序破坏。不改算法 #6/#9。 + +--- + +## Task 1: val_ratio 0.3 → 0.4 + +**Files:** +- Modify: `config/video_split.yaml:15` + +- [ ] **Step 1: 改配置** + +`config/video_split.yaml` 第 15 行： +```yaml + val_ratio: 0.3 # validation 占 trainval 视频组总数的比例 +``` +改为： +```yaml + val_ratio: 0.4 # validation 占 trainval 视频组总数的比例（0.3→0.4 提升整包终审功效，WP2） +``` + +- [ ] **Step 2: 提交** + +```bash +git add config/video_split.yaml +git commit -m "chore: bump video_split val_ratio 0.3->0.4 for terminal-eval power" +``` + +--- + +## Task 2: extract_run_db 每题去重（902→900 canonical） + +**Files:** +- Modify: `app/harness/store.py:269-307` +- Test: `tests/unit/test_harness_store.py`（`TestExtractRunDb`） + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_store.py` 的 `TestExtractRunDb` 类追加： +```python + def test_dedupe_per_question_keeps_first_row(self, tmp_path): + """dedupe_per_question=True 时每 question_id 只保留 rowid 最小的首行。""" + import sqlite3 + + src = tmp_path / "src.db" + conn = sqlite3.connect(src) + conn.execute( + "CREATE TABLE _runs (run_id TEXT PRIMARY KEY, started_at TEXT)" + ) + conn.execute("INSERT INTO _runs VALUES ('r1', 't0')") + conn.execute( + "CREATE TABLE predictions (run_id TEXT, question_id TEXT, prediction TEXT)" + ) + # 743-1 三行（模拟 error/budget/finished），首行 prediction=NULL + conn.executemany( + "INSERT INTO predictions VALUES (?,?,?)", + [ + ("r1", "743-1", None), + ("r1", "743-1", None), + ("r1", "743-1", "C"), + ("r1", "q2", "A"), + ], + ) + conn.commit() + conn.close() + + dst = tmp_path / "dst.db" + from app.harness.store import extract_run_db + + extract_run_db(src, dst, "r1", dedupe_per_question=True) + + out = sqlite3.connect(dst) + rows = out.execute( + "SELECT question_id, prediction FROM predictions ORDER BY question_id" + ).fetchall() + out.close() + assert rows == [("743-1", None), ("q2", "A")], f"未按 rowid 首行去重: {rows}" +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py::TestExtractRunDb::test_dedupe_per_question_keeps_first_row -v` +Expected: FAIL（`extract_run_db() got an unexpected keyword argument 'dedupe_per_question'`）。 + +- [ ] **Step 3: 实现去重** + +`app/harness/store.py` `extract_run_db` 签名改为： +```python +def extract_run_db( + src_db: Path, dst_db: Path, run_id: str, *, dedupe_per_question: bool = False +) -> None: +``` +docstring 补一句参数说明： +``` + dedupe_per_question: True 时 predictions 表每 question_id 仅保留 rowid 最小 + 的首行（对齐 canonical「每 question_id 取第一行 ORDER BY rowid」口径， + 902→900）。_runs 表不受影响。 +``` +把 predictions 分支的取行 SQL（L297-299）改为按 `dedupe_per_question` 分派： +```python + if table == "predictions" and dedupe_per_question: + rows = src.execute( + f"SELECT {col_sql} FROM {table} WHERE run_id=? " + "AND rowid IN (SELECT MIN(rowid) FROM predictions " + "WHERE run_id=? GROUP BY question_id)", + (run_id, run_id), + ).fetchall() + else: + rows = src.execute( + f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,) + ).fetchall() +``` + +- [ ] **Step 4: 运行确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py -q` +Expected: 全 PASS（默认 `dedupe_per_question=False` 保持既有行为，旧测试不受影响）。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/store.py tests/unit/test_harness_store.py +git commit -m "feat: add dedupe_per_question to extract_run_db (canonical 902->900)" +``` + +--- + +## Task 3: tier 感知 + val 功效修复的 diag/val 分配 + +**Files:** +- Modify: `app/harness/pools.py:126-199,289-338` +- Modify: `app/harness/build_split.py:46-71,181-192` +- Modify: `app/harness/video_split_cli.py:565-573` +- Test: `tests/unit/test_pools_video_atomic.py` + +- [ ] **Step 1: 写失败测试（tier 优先 + 功效修复）** + +在 `tests/unit/test_pools_video_atomic.py` 追加： +```python +def test_tier_aware_keeps_high_t2_in_diag(): + """错题视频组按 T2 含量升序进 val：T2 高的组保留在 diagnosis。""" + from app.harness.pools import split_by_video_assignment + from app.question_gen.types import GeneratedQuestion + + def _q(qid, vid): + return GeneratedQuestion( + question_id=qid, video_id=vid, task_type="X", question="q", + options=["A", "B"], answer="A", source_nodes=[], difficulty="easy", + ) + + # 4 个错题视频（每视频 1 题），T2 数分别 2/1/0/0 + questions = [_q(f"{v}-1", v) for v in ("vA", "vB", "vC", "vD")] + assignment = {v: "trainval" for v in ("vA", "vB", "vC", "vD")} + correctness = {f"{v}-1": False for v in ("vA", "vB", "vC", "vD")} + wrong_tier = {"vA": 2, "vB": 1, "vC": 0, "vD": 0} + + pools = split_by_video_assignment( + questions, assignment, correctness, val_ratio=0.5, seed=7, + wrong_tier_by_video=wrong_tier, + ) + diag_vids = {q.video_id for q in pools.diagnosis} + # T2 最高的 vA 必留 diag；T2=0 的组优先进 val + assert "vA" in diag_vids + assert "vB" in diag_vids + + +def test_val_wrong_min_repair_pulls_from_diag(): + """val 错题不足 val_wrong_min 时从 diag 换入低 T2 错题组补足。""" + from app.harness.pools import split_by_video_assignment + from app.question_gen.types import GeneratedQuestion + + def _q(qid, vid, correct): + return GeneratedQuestion( + question_id=qid, video_id=vid, task_type="X", question="q", + options=["A", "B"], answer="A", source_nodes=[], difficulty="easy", + ) + + # 8 错题视频 + 2 正确视频；val_ratio 小使初分 val 错题不足，触发修复 + vids_wrong = [f"w{i}" for i in range(8)] + vids_correct = ["c0", "c1"] + questions = [_q(f"{v}-1", v, False) for v in vids_wrong] + [ + _q(f"{v}-1", v, True) for v in vids_correct + ] + assignment = {v: "trainval" for v in vids_wrong + vids_correct} + correctness = {f"{v}-1": False for v in vids_wrong} + correctness.update({f"{v}-1": True for v in vids_correct}) + wrong_tier = {v: i for i, v in enumerate(vids_wrong)} # 递增 T2 + + pools = split_by_video_assignment( + questions, assignment, correctness, val_ratio=0.1, seed=7, + wrong_tier_by_video=wrong_tier, val_wrong_min=4, + ) + val_wrong = sum(1 for q in pools.validation if not correctness[q.question_id]) + assert val_wrong >= 4, f"功效修复后 val 错题 {val_wrong} < 4" +``` + +> 注：`GeneratedQuestion` 的真实字段以 `app/question_gen/types.py` 为准；若构造签名不符，读该文件对齐必填字段（勿臆造）。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_pools_video_atomic.py -k "tier_aware or val_wrong_min_repair" -v` +Expected: FAIL（`unexpected keyword argument 'wrong_tier_by_video'`）。 + +- [ ] **Step 3: 改 `_split_trainval_by_video_group` 加 tier 感知 + 功效修复** + +`app/harness/pools.py` 函数签名改为： +```python +def _split_trainval_by_video_group( + trainval_qs: list[GeneratedQuestion], + correctness: dict[str, bool], + val_ratio: float, + rng: random.Random, + wrong_tier_by_video: dict[str, int] | None = None, + val_wrong_min: int = 0, +) -> tuple[list[GeneratedQuestion], list[GeneratedQuestion]]: +``` +把分层块（L329-334 的 else 分支）改为 tier 感知：`wrong_vids` 按 T2 含量升序（T2 少的优先进 val），保留 T2 高的组在 diag；`wrong_tier_by_video=None` 时退化为原 shuffle： +```python + else: + val_correct = math.floor(n_correct * n_val / n_total) + val_wrong = n_val - val_correct + rng.shuffle(correct_vids) + if wrong_tier_by_video is None: + rng.shuffle(wrong_vids) + else: + # T2 少的错题组优先进 val（保留 T2 高的组在 diag），确定性排序 + wrong_vids.sort(key=lambda v: (wrong_tier_by_video.get(v, 0), v)) + val_vids = set(correct_vids[:val_correct] + wrong_vids[:val_wrong]) +``` +在 `val_vids` 确定后、返回前，加**功效修复**（从 diag 的错题组按 T2 升序补入 val 直到满足 val_wrong_min）： +```python + if val_wrong_min > 0: + val_wrong_now = sum( + 1 for v in val_vids for q in groups[v] if not correctness[q.question_id] + ) + # diag 侧仍在的错题组，按 T2 升序（低价值优先移交 val） + diag_wrong_pool = sorted( + (v for v in wrong_vids if v not in val_vids), + key=lambda v: ((wrong_tier_by_video or {}).get(v, 0), v), + ) + for v in diag_wrong_pool: + if val_wrong_now >= val_wrong_min: + break + val_vids.add(v) + val_wrong_now += sum(1 for q in groups[v] if not correctness[q.question_id]) + if val_wrong_now < val_wrong_min: + raise InsufficientValSignal( + f"trainval 错题不足以让 val 达到 val_wrong_min={val_wrong_min}" + f"（修复后仅 {val_wrong_now}），请放大 val_ratio 或调整 trainval 归属。" + ) +``` +（`InsufficientValSignal` 已在 pools.py:118 定义，无需新增；需确认函数内可见 `math`/`defaultdict`，文件顶部已 import。） + +- [ ] **Step 4: `split_by_video_assignment` 透传新参数** + +`app/harness/pools.py` `split_by_video_assignment` 签名加 `wrong_tier_by_video: dict[str, int] | None = None`（放在 `val_wrong_min` 之后），并把 `_split_trainval_by_video_group` 调用（L175-177）改为： +```python + diagnosis, validation = _split_trainval_by_video_group( + trainval_qs, correctness, val_ratio, random.Random(seed), + wrong_tier_by_video=wrong_tier_by_video, + val_wrong_min=val_wrong_min, + ) +``` +删除原 L179-186 的独立 `val_wrong_min` 事后校验块（功效已在 `_split_trainval_by_video_group` 内保证，避免重复校验语义）。docstring 的 `val_wrong_min` 说明改为"切分时保证（不足则从 diag 换入低 T2 错题组补足，耗尽 fail-loud）"。 + +- [ ] **Step 5: build_split 计算并传入 tier + val_wrong_min** + +`app/harness/build_split.py`：`SplitBuildConfig` 加字段 `val_wrong_min: int`（放 `split_seed` 之后，docstring 补"validation 池最少错题数，切分时保证功效"）。build_split Phase 3（L182-191）改为： +```python + questions = load_benchmark(questions_dir) + correctness = {pred["question_id"]: pred["correct"] for pred in preds} + tier_by_q = {row["question_id"]: row["tier"] for row in signal_rows} + wrong_tier_by_video: dict[str, int] = defaultdict(int) + for pred in preds: + if not pred["correct"] and tier_by_q.get(pred["question_id"]) == "T2": + wrong_tier_by_video[pred["video_id"]] += 1 + pools = split_by_video_assignment( + questions, + assignment, + correctness, + config.val_ratio, + config.split_seed, + baseline_run_id=baseline_run_id, + val_wrong_min=config.val_wrong_min, + wrong_tier_by_video=dict(wrong_tier_by_video), + ) +``` +更新 build_split docstring 的"契约（Task 11...）"段：删除"有意保持 val_wrong_min-agnostic"表述，改为"val_wrong_min 前置到切分内保证功效；CLI 的 check_mcnemar_power 作冗余最终确认"。确认 `defaultdict` 已 import（`from collections import Counter, defaultdict`）。 + +- [ ] **Step 6: CLI 传 val_wrong_min + 保真检查** + +`app/harness/video_split_cli.py` 的 `SplitBuildConfig(...)`（L565-573）加一行 `val_wrong_min=config.val_wrong_min,`。 +保真检查点：确认 `pools.diagnosis`/`pools.validation` 仍是逐题 `GeneratedQuestion` 列表、同 video 全部题同池（`test_pools_video_atomic.py::test_video_group_atomic_in_trainval_split` 覆盖）。 + +- [ ] **Step 7: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_pools_video_atomic.py tests/unit/test_harness_pools.py tests/unit/test_split_selection.py -q` +Expected: 全 PASS。 + +- [ ] **Step 8: 提交** + +```bash +git add app/harness/pools.py app/harness/build_split.py app/harness/video_split_cli.py tests/unit/test_pools_video_atomic.py +git commit -m "feat: tier-aware diag/val split with val-power repair (design 5.1)" +``` + +--- + +## Task 4: 冻结产物覆盖保护 + --force + +**Files:** +- Modify: `app/harness/build_split.py:104-192` +- Modify: `app/harness/video_split_cli.py:751-773`（build_arg_parser）+ run_pipeline 传参 +- Test: `tests/unit/test_video_split_cli.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_video_split_cli.py` 追加（用最小 build_split 覆盖场景，或直接测保护函数）： +```python +def test_build_split_refuses_overwrite_without_force(tmp_path): + """已存在指纹不同的 pools.json 时，force=False 必须报错不覆盖。""" + from app.harness.build_split import _guard_frozen_products + + out_path = tmp_path / "pools.json" + out_path.write_text('{"split_mode":"global"}', encoding="utf-8") + manifest_path = tmp_path / "split_manifest.json" + + with pytest.raises(FileExistsError, match="已存在冻结产物"): + _guard_frozen_products(out_path, manifest_path, force=False) + + +def test_build_split_force_backs_up_old(tmp_path): + """force=True 时旧产物被备份为 .bak.* 再允许覆盖。""" + from app.harness.build_split import _guard_frozen_products + + out_path = tmp_path / "pools.json" + out_path.write_text('{"old":1}', encoding="utf-8") + manifest_path = tmp_path / "split_manifest.json" + manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8") + + _guard_frozen_products(out_path, manifest_path, force=True) + baks = list(tmp_path.glob("pools.json.bak.*")) + assert len(baks) == 1, f"未备份旧产物: {list(tmp_path.iterdir())}" +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -k "refuses_overwrite or force_backs_up" -v` +Expected: FAIL（`cannot import name '_guard_frozen_products'`）。 + +- [ ] **Step 3: 实现覆盖保护函数** + +`app/harness/build_split.py` 顶部 import 区确认有 `import shutil`（无则加）。新增函数（放 build_split 之前）： +```python +def _guard_frozen_products(out_path: Path, manifest_path: Path, *, force: bool) -> None: + """冻结前的覆盖保护：产物已存在时按 force 决定报错或备份。 + + 参数: + out_path: 目标 pools.json 路径。 + manifest_path: 目标 split_manifest.json 路径。 + force: False 时已存在即 FileExistsError；True 时把旧产物重命名为 + .bak.<旧 pools_sha256 前 8 位或 timestamp-less 序号> 再放行。 + + 异常: + FileExistsError: force=False 且产物已存在（防静默覆盖冻结锚点）。 + """ + if not out_path.exists() and not manifest_path.exists(): + return + if not force: + raise FileExistsError( + f"已存在冻结产物 {out_path}（或其 manifest）。重跑切分会覆盖训练依赖的" + "冻结锚点——确认要替换请加 --force（旧产物将备份为 .bak.*）。" + ) + # 备份后缀取旧 manifest 的 pools_sha256 前 8 位，无则用 'prev' + suffix = "prev" + if manifest_path.exists(): + try: + old = json.loads(manifest_path.read_text(encoding="utf-8")) + suffix = str(old.get("pools_sha256", "prev"))[:8] or "prev" + except (json.JSONDecodeError, OSError): + suffix = "prev" + for p in (out_path, manifest_path): + if p.exists(): + p.rename(p.with_name(f"{p.name}.bak.{suffix}")) +``` +确认 build_split.py 已 import `json`（无则加 `import json`）。在 `build_split` 签名加参数 `force: bool = False`（放 `generated_at` 之后），并在 Phase 3 `save_pools` 之前（L192 前）调用 `_guard_frozen_products(out_path, manifest_path, force=force)`。 + +- [ ] **Step 4: CLI 暴露 --force 并透传** + +`app/harness/video_split_cli.py` `build_arg_parser`（L751-773）追加： +```python + parser.add_argument( + "--force", + action="store_true", + help="覆盖已存在的冻结 pools.json/manifest（旧产物备份为 .bak.*）", + ) +``` +`run_pipeline` 签名加 `force: bool = False` 参数，build_split 调用（L559-577）加 `force=force,`；`main()` 里把 `args.force` 透传给 `run_pipeline`。 + +- [ ] **Step 5: 运行测试确认通过 + CLI 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -q` +Expected: 全 PASS。 + +- [ ] **Step 6: 提交** + +```bash +git add app/harness/build_split.py app/harness/video_split_cli.py tests/unit/test_video_split_cli.py +git commit -m "feat: guard frozen split products against silent overwrite (--force)" +``` + +--- + +## Task 5: seed 携带 pools.json + 训练拷入 + +**Files:** +- Modify: `app/harness/store.py:184-232`（init_seed） +- Modify: `app/harness/workspace.py:156-200`（init_workspace_from_seed） +- Test: `tests/unit/test_harness_store.py`、`tests/unit/test_harness_workspace.py` + +- [ ] **Step 1: 写失败测试（seed 携带）** + +在 `tests/unit/test_harness_store.py::TestInitSeed` 追加： +```python + def test_init_seed_carries_pools(self, tmp_path): + """提供 pools_json/split_manifest 时拷入 seed 目录。""" + from app.harness.store import init_seed + + store = tmp_path / "store" + skills = tmp_path / "sk"; skills.mkdir(); (skills / "s.md").write_text("x") + prompts = tmp_path / "pr"; prompts.mkdir(); (prompts / "p.md").write_text("y") + db = tmp_path / "b.db"; db.write_text("db") + pools = tmp_path / "pools.json"; pools.write_text('{"split_mode":"global"}') + manifest = tmp_path / "split_manifest.json"; manifest.write_text('{"pools_sha256":"a"}') + + seed_dir = init_seed( + store, "s1", skills, prompts, db, "infer_adhoc", None, "d", + pools_json=pools, split_manifest=manifest, + ) + assert (seed_dir / "pools.json").exists() + assert (seed_dir / "split_manifest.json").exists() +``` + +在 `tests/unit/test_harness_workspace.py` 追加： +```python +def test_init_workspace_from_seed_carries_pools(store_dir, workspace_dir): + """seed 目录含 pools.json 时拷入 workspace。""" + import shutil + from app.harness.store import init_seed + from app.harness.workspace import init_workspace_from_seed + # 复用现有 fixture 构造 seed 的方式；此处补 pools.json 到 seed 后初始化 workspace + # （具体 fixture 依 test_harness_workspace.py 现有 helper，读文件对齐） + ... +``` +> 该 workspace 测试需依 `test_harness_workspace.py` 现有 fixture（`store_dir`/`workspace_dir` 及既有 seed 构造 helper）填充；实现前读该文件 `test_init_workspace_from_seed`（L151）复用其 seed 搭建，再在 seed 目录写 `pools.json` 后断言 workspace 内出现 `pools.json`。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py::TestInitSeed::test_init_seed_carries_pools -v` +Expected: FAIL（`unexpected keyword argument 'pools_json'`）。 + +- [ ] **Step 3: init_seed 加可选携带参数** + +`app/harness/store.py` `init_seed` 签名加： +```python +def init_seed( + store_dir: Path, + name: str, + skills_dir: Path, + prompts_dir: Path, + baseline_db: Path, + baseline_run_id: str, + parent: str | None, + description: str, + *, + pools_json: Path | None = None, + split_manifest: Path | None = None, +) -> Path: +``` +在 `copy2(baseline_db, ...)`（L218）之后加： +```python + if pools_json is not None: + shutil.copy2(pools_json, seed_dir / "pools.json") + if split_manifest is not None: + shutil.copy2(split_manifest, seed_dir / "split_manifest.json") +``` +docstring 补两参说明。 + +- [ ] **Step 4: init_workspace_from_seed 拷入 pools** + +`app/harness/workspace.py` `init_workspace_from_seed` 在 `shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db")`（L197）之后加： +```python + seed_pools = seed_dir / "pools.json" + if seed_pools.exists(): + shutil.copy2(seed_pools, workspace_dir / "pools.json") + seed_manifest = seed_dir / "split_manifest.json" + if seed_manifest.exists(): + shutil.copy2(seed_manifest, workspace_dir / "split_manifest.json") +``` + +- [ ] **Step 5: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py -q` +Expected: 全 PASS。 + +- [ ] **Step 6: 提交** + +```bash +git add app/harness/store.py app/harness/workspace.py tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py +git commit -m "feat: seed carries frozen pools.json into training workspace" +``` + +--- + +## Task 6: build_or_load_pools global 一致性校验 + +**Files:** +- Modify: `app/harness/pools.py:746-813` +- Test: `tests/unit/test_harness_pools.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_pools.py::TestBuildOrLoadPoolsFrozen` 追加： +```python + def test_global_frozen_rejects_baseline_mismatch(self, tmp_path, ...): + """global 冻结 pools 的 baseline_run_id 与 seed 不符时 fail-loud。""" + # 依现有 fixture 造 workspace + 冻结 pools.json（split_mode=global, + # baseline_run_id="other"），seed.json baseline_run_id="infer_adhoc" + # 调 build_or_load_pools 应 raise ValueError(match="baseline_run_id") + ... +``` +> 依 `TestBuildOrLoadPoolsFrozen`（L246）现有 fixture 复用其 workspace/seed 搭建；实现前读该类对齐 RunConfig/strategy 构造，勿臆造。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_pools.py::TestBuildOrLoadPoolsFrozen -v` +Expected: 新用例 FAIL（当前 global 分支无校验，误加载不报错）。 + +- [ ] **Step 3: 加 global 一致性校验** + +`app/harness/pools.py` `build_or_load_pools`，在 global 加载分支（`if pools_path.exists():` 块内、`per_category` 校验的 `else` 侧，即 L813 `return load_pools(pools_path)` 之前）加： +```python + else: # global：校验 baseline_run_id 与（若有）manifest 内容指纹 + frozen_baseline = raw.get("baseline_run_id") + if frozen_baseline != baseline_run_id: + raise ValueError( + f"冻结 pools.json 的 baseline_run_id={frozen_baseline!r} 与 seed " + f"的 {baseline_run_id!r} 不一致，拒绝静默加载错配切分。" + ) + manifest_path = config.workspace_dir / "split_manifest.json" + if manifest_path.exists(): + import hashlib + + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + actual_sha = hashlib.sha256( + pools_path.read_text(encoding="utf-8").encode("utf-8") + ).hexdigest() + if manifest.get("pools_sha256") != actual_sha: + raise ValueError( + "pools.json 内容指纹与 split_manifest.pools_sha256 不符，" + "冻结产物疑被篡改，拒绝加载。" + ) +``` +（确认该 `else` 与 L751 `if frozen_split_mode == "per_category":` 配对；若现有结构非 if/else 而是 if 后直接 return，则把校验插在 `return load_pools(pools_path)` 前并用 `if frozen_split_mode != "per_category":` 守卫。实现前读 L746-813 对齐控制流。） + +- [ ] **Step 4: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_pools.py -q` +Expected: 全 PASS。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/pools.py tests/unit/test_harness_pools.py +git commit -m "fix: validate global frozen pools baseline_run_id + sha256 on load" +``` + +--- + +## Self-Review（作者自查，执行者复核） + +- [ ] val_ratio=0.4 已改；tier 感知 + 功效修复在同一函数、退化路径（`wrong_tier_by_video=None`）保持旧行为。 +- [ ] val_wrong_min 从 CLI→SplitBuildConfig→build_split→split_by_video_assignment→_split_trainval_by_video_group 全链路贯通；旧的 pools.py 事后校验块已删（不重复）。 +- [ ] extract_run_db 去重默认关闭，不破坏既有调用。 +- [ ] seed 携带 + workspace 拷入 + global 一致性校验三者闭环：冻结产物有唯一路径进训练且被校验。 +- [ ] 覆盖保护默认 force=False，离线 CLI 重跑需显式 --force。 + +## 核心算法保真校验结论 + +本计划触及算法 #5 的上游输入（哪些视频进 diag/val），**不改** gate_ladder 的 unit+correctness 消费结构；Task 3 Step 6 已设保真检查点确认逐 unit 列表与视频组原子性。不涉及算法 #4/#6/#8/#9 逻辑。 + +## 验收标准 + +1. `pytest tests/unit/test_pools_video_atomic.py tests/unit/test_harness_pools.py tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py tests/unit/test_split_selection.py tests/unit/test_video_split_cli.py` 全绿。 +2. tier 感知：T2 高的错题视频组留 diag，T2 低的优先进 val。 +3. seed 携带 pools.json → init_workspace_from_seed 拷入 → build_or_load_pools 校验 baseline_run_id + sha256。 +4. 冻结产物 force=False 时拒绝覆盖。 diff --git a/research-wiki/plans/2026-07-16-preflight-wp3-train-loop.md b/research-wiki/plans/2026-07-16-preflight-wp3-train-loop.md new file mode 100644 index 0000000..7858228 --- /dev/null +++ b/research-wiki/plans/2026-07-16-preflight-wp3-train-loop.md @@ -0,0 +1,601 @@ +# WP3 训练循环与进化引擎 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +> **前置依赖：WP1（模板已迁移，进化引擎可运行）+ WP4（cache_salt 能力已就绪）必须先完成。** + +**Goal:** 修复训练循环与进化引擎的 9 处缺陷，使诊断拿到真实轨迹、早停按 epoch 语义、微型题型不崩、崩溃可幂等续跑、进化 patch 不破坏冻结区、降级信号不驱动错误进化、跨 epoch 评估真实重采样。 + +**Architecture:** 诊断经 `StepsJsonRunLog` 从 steps_json 重建轨迹（算法 #7 恢复）；早停计数单位 step→epoch；可训练性预检在 gate 建立前剔除微型题型；`_run_step` 幂等（先 DELETE 再写）；patch 冻结区检查整个 target 跨度（算法 #8 加固）；降级/未判定题按 lapse 保守分流；训练推理用 run_id 作 cache_salt（run_id 已含 epoch，天然跨 epoch 重采样）。 + +**Tech Stack:** Python 3.11、asyncio、SQLite、pytest。 + +**设计源**：`research-wiki/designs/2026-07-16-preflight-fixes-design.md §6-7` + +--- + +## 关键锚点（实现前必读） + +| 用途 | 位置 | +|------|------| +| 训练主流程 | `app/harness/runner.py:789-859` `train`；`_setup_train_run:865-879`（预检插入点 L874 前）；`_run_step:1001-1026`（run_id L1011、诊断 L1019、无 DELETE） | +| 早停 | `app/harness/runner.py:291-316` `_should_early_stop`（L315 `+= steps_this_epoch`）；`_TrainState:95-121`（`steps_since_best_improved` L118）；`_maybe_promote_best:1587`（置 0） | +| 诊断调用 | `app/harness/runner.py:2163-2196` `_run_diagnosis`（`RunLogImpl` L2173 未包 StepsJsonRunLog）；DiagnosisResult `degraded_count` 未被引用 | +| gate 刷新 | `app/harness/runner.py:1833-1879` `_refresh_gate_ladder`（save L1878 → set observed L1879）；checkpoint 落盘晚在 train L837 | +| holdout | `app/harness/runner.py:1881-1921` `_holdout_four_way`；`_pick_mixed_best:1923-1963`；`_eval_version_on_pool:2148-2161` | +| 推理落库 | `app/harness/inference.py:363-447` `_run_single_question`（prediction L422 未归一、insert L446 try 外）；`_to_text_field:147-162`；`run_inference:473` | +| Agent Loop | `core/agent/loop.py:103` `run`（session_id L110）；`_call_llm` chat 调用 `:329`（`self._llm.chat(messages, session_id=session_id)`） | +| batching | `app/harness/batching.py:186-202` `_classify_unit`（L200 缺 correctness→None） | +| checkpoint | `app/harness/checkpoint.py:37-44` `_STRUCTURAL_KEYS`；`serialize_state:76-106`；`write_checkpoint:212-260`（原子写） | +| 诊断分流 | `core/evolution/diagnose.py:1485-1519` `_build_skill_case_packs`（lapse 分流 L1492）；`_process_question:2139-2206`（except L2194 cause_category 留 None） | +| traces 适配 | `app/harness/baseline_run_log.py:13` `StepsJsonRunLog`；`steps_json_traces.py:13` | +| patch | `core/evolution/patch.py:285-287` `_in_ranges`；`_do_insert_after:309-326`（L320）；`_do_replace_delete:329-347`（L343）；markers L11-18；`validate_skill` in `evolve.py:296` | + +## 核心算法保真校验 + +触及算法 #7（诊断瀑布，Task 1 恢复轨迹）、#8（patch 引擎，Task 5 冻结区加固）、#10（Agent Loop，Task 9 透传 cache_salt）、#5/#12（信息阶梯/训练编排，Task 7 checkpoint 时序）。**均为恢复/加固/透传，不改算法逻辑**：Task 1 让诊断拿到本就该有的轨迹；Task 5 把"只查起点"补成"查整跨度"（保护方向不变）；Task 9 只加透传参数；Task 7 只调 checkpoint 落盘时机。每个相关 Task 设保真检查点。 + +--- + +## Task 1: traces 适配（诊断拿到真实轨迹，算法 #7） + +**Files:** +- Modify: `app/harness/runner.py:2163-2196`（_run_diagnosis） +- Test: `tests/unit/test_runner_diag_tree_inject.py` 或 `test_harness_runner.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_runner_diag_tree_inject.py` 追加（构造只写 steps_json 不写 traces 表的 run，断言诊断能拿到轨迹）： +```python +@pytest.mark.asyncio +async def test_diagnosis_reads_traces_from_steps_json(...): + """traces 表为空但 predictions.steps_json 有轨迹时，诊断仍拿到非空 traces。""" + # 依现有 runner 测试 fixture 造一个 run：predictions 有 steps_json，traces 表空； + # 调 _run_diagnosis 后断言 diagnose 收到的 traces 非空（可 patch run_diagnosis 捕获入参） + ... +``` +> 依 `test_runner_diag_tree_inject.py` 现有 fixture；实现前读对齐 runner 构造与 patch 点。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_runner_diag_tree_inject.py -k reads_traces_from_steps_json -v` +Expected: FAIL（当前 RunLogImpl 直读空 traces 表）。 + +- [ ] **Step 3: 包 StepsJsonRunLog** + +`app/harness/runner.py` `_run_diagnosis`，把传给 `run_diagnosis` 的 `run_log`（当前 `RunLogImpl(...)`，L2173 附近）包一层： +```python + from app.harness.baseline_run_log import StepsJsonRunLog + from app.harness.log import RunLogImpl + + run_log = StepsJsonRunLog(RunLogImpl(str(self._paths.db_path))) +``` +（`StepsJsonRunLog.get_traces` 在底层 traces 空时从 predictions.steps_json 经 `steps_json_to_trace_rows` 重建；`get_predictions` 透传。） + +- [ ] **Step 4: 保真检查点 + 测试通过** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_runner_diag_tree_inject.py tests/unit/test_baseline_run_log.py -q` +Expected: 全 PASS。保真：确认诊断瀑布拿到的是逐 step `{tool_name,tool_args,tool_output,thought}` 行（对齐 TRM4 诊断输入）。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/runner.py tests/unit/test_runner_diag_tree_inject.py +git commit -m "fix: wrap diagnosis run_log with StepsJsonRunLog (restore algo #7 traces)" +``` + +--- + +## Task 2: prediction 归一化 + 落库加固 + +**Files:** +- Modify: `app/harness/inference.py:417-447` +- Test: `tests/unit/test_harness_inference.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_inference.py` 追加（LLM 提交非标量 answer 不崩 gather）： +```python +@pytest.mark.asyncio +async def test_nonscalar_prediction_does_not_crash(...): + """submit_answer 返回 {'answer': ['B']} 等非标量时归一化落库，不抛 sqlite 绑定异常。""" + # 依现有 inference 测试 fixture，让 AgentLoop 返回 result={'answer': ['B']}； + # run_inference 应正常完成、predictions 行 prediction 为字符串（如 '["B"]'），不崩 + ... +``` +> 依 `test_harness_inference.py` 现有 fake loop/dispatch fixture；实现前读对齐。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py -k nonscalar_prediction -v` +Expected: FAIL（sqlite `InterfaceError: Error binding parameter`）。 + +- [ ] **Step 3: 归一化 prediction + insert 加固** + +`app/harness/inference.py`：新增归一化 helper（None 保留、str 原样、其余 `_to_text_field`）： +```python +def _normalize_prediction(answer: object) -> str | None: + """归一化 prediction：None 保留（INFRA 空预测语义），str 原样，其余 JSON 序列化。""" + if answer is None or isinstance(answer, str): + return answer + return _to_text_field(answer) +``` +L422 `"prediction": result_dict.get("answer"),` 改为 `"prediction": _normalize_prediction(result_dict.get("answer")),`。 +L446 的 `await asyncio.to_thread(log.insert, "predictions", record)` 包 try，绑定异常降级为最小 error 行不击穿 gather： +```python + try: + await asyncio.to_thread(log.insert, "predictions", record) + except (sqlite3.InterfaceError, sqlite3.ProgrammingError): + logger.exception("[{}] QA {} 落库绑定异常，降级为 error 行", qa.video_id, qa.question_id) + record["prediction"] = None + record["stop_reason"] = "error" + await asyncio.to_thread( + log.insert, "predictions", + {k: v for k, v in record.items() if isinstance(v, (str, int, float, type(None)))}, + ) +``` +确认 `import sqlite3` 在文件顶部（无则加）。 + +- [ ] **Step 4: 测试通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py -q` +Expected: 全 PASS。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/inference.py tests/unit/test_harness_inference.py +git commit -m "fix: normalize non-scalar prediction; harden predictions insert" +``` + +--- + +## Task 3: early_stop 改 epoch 计数 + +**Files:** +- Modify: `app/harness/runner.py:291-316,118,1587,849-857` +- Modify: `app/harness/checkpoint.py`（若字段入 state） +- Test: `tests/unit/test_harness_runner.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_runner.py` 追加： +```python +def test_early_stop_counts_epochs_not_steps(tmp_path): + """patience=2 表示连续 2 个 epoch 无 best 刷新才停（不是步数）。""" + from app.harness.runner import _should_early_stop, _TrainState + # 造 state + workspace，best 停在 epoch 1； + # epoch 2 无刷新 → epochs_since_best_improved=1 → 不停； + # epoch 3 无刷新 → =2 → 停 + ... +``` +> 依现有 `_TrainState`/`read_best` fixture；实现前读对齐 workspace best 写入。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k early_stop_counts_epochs -v` +Expected: FAIL（当前累加 steps_this_epoch）。 + +- [ ] **Step 3: 字段改名 + 计数改 epoch** + +全局把 `steps_since_best_improved` 改名 `epochs_since_best_improved`（`grep -rn steps_since_best_improved app/`：`_TrainState:118`、`_should_early_stop:313,315`、`_maybe_promote_best:1587`，以及 checkpoint serialize/deserialize 若含此字段）。 +`_should_early_stop`（L315）`state.steps_since_best_improved += steps_this_epoch` 改为 `state.epochs_since_best_improved += 1`；签名删除 `steps_this_epoch` 参数（改为 `_should_early_stop(workspace_dir, epoch, state, patience)`），train 调用点（L849-857）同步去掉 `len(batches)` 实参。docstring 改为"epoch 粒度"。 + +- [ ] **Step 4: checkpoint 兼容** + +若 `epochs_since_best_improved` 入 checkpoint state（`grep -n steps_since_best_improved app/harness/checkpoint.py`），同步改名。本轮为 fresh 训练无旧 checkpoint，无迁移负担。 + +- [ ] **Step 5: 测试通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py tests/unit/test_harness_checkpoint.py -q` +Expected: 全 PASS。 + +- [ ] **Step 6: 提交** + +```bash +git add app/harness/runner.py app/harness/checkpoint.py tests/unit/test_harness_runner.py +git commit -m "fix: early_stop patience counts epochs not steps" +``` + +--- + +## Task 4: 可训练性预检 + +**Files:** +- Modify: `app/harness/runner.py:865-879`（_setup_train_run） +- Modify: `app/harness/config.py`（RunConfig 加 `trainable_min_units`）+ `config/train_videomme.yaml`（WP 收尾建，先加字段默认） +- Test: `tests/unit/test_harness_runner.py` + +- [ ] **Step 1: 写失败测试** + +```python +def test_untrainable_types_filtered_before_gate(): + """val tuple[Pools, list[str] | None]: + """剔除不可训练题型（val marker 常量名以 `core/evolution/patch.py:11-18` 为准（`APPENDIX_START` 等），实现前读对齐 import 名。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_patch.py -k "spanning_into_protected or marker_literal" -v` +Expected: FAIL（当前只查起点 pos，跨入未拦；无注入检查）。 + +- [ ] **Step 3: 跨度检查 + 注入检查** + +`core/evolution/patch.py` 新增跨度 helper： +```python +def _span_overlaps_ranges(pos: int, length: int, ranges: list[tuple[int, int]]) -> bool: + """判断 [pos, pos+length) 是否与任一冻结区间相交（不止起点）。""" + end = pos + length + return any(start < end and pos < r_end for start, r_end in ranges) +``` +`_do_insert_after` L320 `if _in_ranges(pos, ranges):` 改为 `if _span_overlaps_ranges(pos, len(target), ranges):`。 +`_do_replace_delete` L343 `if _in_ranges(pos, ranges):` 改为 `if _span_overlaps_ranges(pos, len(target), ranges):`。 +在 `apply_patch_with_report`（L387）应用每个 edit 前加注入检查：payload/target 含任一 marker 字面量（`APPENDIX_START/END`、`MOMENTUM_START/END`）→ 跳过该 edit 并记 `skipped_marker_injection`。 + +- [ ] **Step 4: validate_skill 加 marker 完整性校验** + +`core/evolution/evolve.py:296` `validate_skill`：在现有 frontmatter/长度/代码块校验后加——统计 evolved 中 `APPENDIX_START/END`、`MOMENTUM_START/END` 出现次数，要求成对（START 数==END 数）、各至多一对、START 在 END 前；违反则返回校验失败（该候选整体 reject）。 + +- [ ] **Step 5: 保真检查点 + 测试通过** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_patch.py tests/unit/test_evolve.py -q` +Expected: 全 PASS。保真：确认"保护跨度"方向未变（仍是保护 appendix/momentum 不被误改），只是从"查起点"补成"查整跨度"+ 注入/完整性双防线。 + +- [ ] **Step 6: 提交** + +```bash +git add core/evolution/patch.py core/evolution/evolve.py tests/unit/test_patch.py +git commit -m "fix: patch checks full target span + marker injection/integrity (algo #8)" +``` + +--- + +## Task 6: 诊断降级分流 + 占比中止 + +**Files:** +- Modify: `core/evolution/diagnose.py:1485-1497` +- Modify: `app/harness/runner.py:1019`（诊断后 degraded 占比检查） +- Test: `tests/unit/test_diagnose.py`、`tests/unit/test_harness_runner.py` + +- [ ] **Step 1: 写失败测试（分流）** + +在 `tests/unit/test_diagnose.py` 追加： +```python +def test_none_cause_and_degraded_route_to_lapse(): + """cause_category=None（判别失败）与 degraded 题按 lapse 处置，不进 defect 正文路径。""" + from core.evolution.diagnose import _build_skill_case_packs + # 构造 metrics_group：一题 attr.cause_category=None（非 degraded）、一题 qm.degraded=True； + # 断言二者都不出现在 failure_cases（wrong_by_error），只可能进 lapse_notes + ... +``` +> 依 `test_diagnose.py` 现有 QuestionMetrics/ErrorAttribution 构造（`:753` 附近）；实现前读对齐字段。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_diagnose.py -k none_cause_and_degraded_route -v` +Expected: FAIL（当前 None → wrong_by_error 走 defect 正文）。 + +- [ ] **Step 3: 改分流逻辑** + +`core/evolution/diagnose.py` `_build_skill_case_packs` 的分流循环（L1488-1497）改为"仅明确 defect 且非 degraded 才进正文路径"： +```python + for qm in metrics_group: + if qm.correct: + continue + attr = attribution_map.get(qm.question_id) + is_defect = ( + attr is not None + and attr.cause_category == "defect" + and not qm.degraded + ) + if not is_defect: + # lapse / None（判别失败）/ degraded → 保守，不驱动正文进化 + if attr is not None and attr.lapse_note and attr.lapse_note.strip(): + lapse_notes.append(attr.lapse_note) + continue + wrong_by_error[attr.error_type].append(qm) +``` + +- [ ] **Step 4: 写失败测试（占比中止）+ 实现** + +`tests/unit/test_harness_runner.py` 追加：诊断结果 degraded_count/总题数 > 0.5 时 `_run_step` 后应 raise（疑似基础设施故障）。 +`app/harness/runner.py` `_run_step`（L1019 拿到 `diagnosis` 后）加： +```python + n_wrong = sum(1 for q in batch if not state.correctness.get(q.question_id, True)) + if n_wrong > 0 and diagnosis.degraded_count / n_wrong > 0.5: + raise RuntimeError( + f"本 step 诊断降级占比 {diagnosis.degraded_count}/{n_wrong} > 50%，" + "疑似 judge 基础设施故障，中止训练（不以降级信号驱动进化）。" + ) +``` +（确认 `DiagnosisResult.degraded_count` 字段可用，L2311。） + +- [ ] **Step 5: 测试通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_diagnose.py tests/unit/test_harness_runner.py -q` +Expected: 全 PASS。 + +- [ ] **Step 6: 提交** + +```bash +git add core/evolution/diagnose.py app/harness/runner.py tests/unit/test_diagnose.py tests/unit/test_harness_runner.py +git commit -m "fix: route None/degraded diagnoses to lapse; abort on high degrade rate" +``` + +--- + +## Task 7: step 幂等（DELETE）+ gate_epoch_observed 立即落盘 + +**Files:** +- Modify: `app/harness/runner.py:1001-1026`（_run_step 开头 DELETE） +- Modify: `app/harness/runner.py:1378-1498,1833-1879`（checkpoint 提到 gate save 之后） +- Test: `tests/unit/test_harness_runner.py` + +- [ ] **Step 1: 写失败测试（幂等）** + +```python +@pytest.mark.asyncio +async def test_run_step_deletes_stale_rows_before_rerun(...): + """同 run_id 重跑前先清 predictions/traces，避免重复行双计。""" + # 预置该 step run_id 的旧 predictions 行；调 _run_step；断言旧行被清、只剩本次 + ... +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k deletes_stale_rows -v` +Expected: FAIL（当前 append，无 DELETE）。 + +- [ ] **Step 3: _run_step 开头 DELETE** + +`app/harness/runner.py` `_run_step`，在 rollout（L1012）之前加： +```python + with HarnessLog(str(self._paths.db_path), run_id, register_run=False) as log: + log.execute("DELETE FROM predictions WHERE run_id=?", (run_id,)) + log.execute("DELETE FROM traces WHERE run_id=?", (run_id,)) +``` +（`register_run=False` 来自 WP4 Task 8；若 HarnessLog 无 `execute` 便捷方法，用其现有 `query`/连接接口，实现前读 log.py 对齐。） + +- [ ] **Step 4: checkpoint 提到 gate save 之后（消除双计窗口）** + +目标：`_refresh_gate_ladder` 内 `gate_pools.save`（L1878）+ `gate_epoch_observed=True`（L1879）之后，**立即** `write_checkpoint`（phase="epoch_done"），不等到 `train` L837。实现：把 `_slow_update_cycle` Phase 10（调 `_refresh_gate_ladder` L1496-1498）之后的 checkpoint 落盘从 `train`（L837）移入 `_slow_update_cycle` 末尾，或让 `_refresh_gate_ladder` 接收 checkpoint 所需上下文（epoch/progress/batches）并在 save 后落盘。实现前读 `write_checkpoint` 签名（checkpoint.py:212）与 `train` L833-848 对齐参数，确保 gate_pools.json 与 checkpoint 的 `gate_epoch_observed` 同一时刻一致。 + +- [ ] **Step 5: 保真检查点 + 测试通过** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py tests/unit/test_harness_checkpoint.py -q` +Expected: 全 PASS。保真（算法 #5/#12）：确认 γ-EMA 更新（`update_probs`）仍每 epoch 一次、checkpoint 落盘不改变慢更新十步序的语义顺序。 + +- [ ] **Step 6: 提交** + +```bash +git add app/harness/runner.py tests/unit/test_harness_runner.py +git commit -m "fix: idempotent _run_step (DELETE stale) + checkpoint after gate save" +``` + +--- + +## Task 8: holdout 四向去重 + +**Files:** +- Modify: `app/harness/runner.py:1881-1963`（_holdout_four_way / _pick_mixed_best） +- Test: `tests/unit/test_harness_runner.py` + +> **说明**：目标是每 epoch 的四向 test 评估从"4×600 全跑"降为"仅 final 必跑 + best_hard 未评过才跑 + baseline 从基线预测推导 + best_mixed 引用赢家"。去重做在 harness 逻辑层（配合 WP4 epoch 盐，同版本不重采样）。 + +- [ ] **Step 1: 写失败测试** + +```python +@pytest.mark.asyncio +async def test_holdout_dedup_skips_reevaluated_versions(...): + """baseline 不跑推理（从基线预测推导）；best_hard==final 时不重复评估。""" + # 统计 _eval_version_on_pool 被调次数：baseline=0，best_hard==final 时该向复用不重跑 + ... +``` +> 依现有 runner holdout fixture；实现前读 `_holdout_four_way`/`write_holdout_eval` 对齐。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k holdout_dedup -v` +Expected: FAIL（当前四向各跑一次）。 + +- [ ] **Step 3: 实现去重** + +`app/harness/runner.py` `_holdout_four_way`： +- **baseline 向**：不调 `_eval_version_on_pool`，改从基线 predictions（baseline_run_id）读 test 题对错算 acc（test 题在 infer_adhoc 已推理）；epoch>1 复用首次结果。 +- **final 向**：真评 600（不变）。 +- **best_hard 向**：维护"已评版本→test_acc"备忘录，resume 时从 `holdout_eval` 表 hydrate（查已落库 `(skills_v,prompts_v)→acc`）；版本对未评过且 ≠ final 才真评，否则引用。 +- **best_mixed 向**：引用赢家（best_hard 或 final 之一，`_pick_mixed_best` 已定）的 test 成绩，写 holdout_eval 时标指针，不推理。 + +实现前完整读 `_holdout_four_way`（1881-1921）、`write_holdout_eval`、`_eval_version_on_pool`、`read`/hydrate 路径对齐。 + +- [ ] **Step 4: 测试通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -q` +Expected: 全 PASS。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/runner.py tests/unit/test_harness_runner.py +git commit -m "perf: dedup holdout four-way eval (baseline derive, best_hard memo)" +``` + +--- + +## Task 9: 训练推理注入 epoch 盐（cache_salt=run_id，算法 #10 透传） + +**Files:** +- Modify: `core/agent/loop.py:103-141,271-329` +- Modify: `app/harness/inference.py:408-415,473` +- Test: `tests/unit/test_agent_loop`（或现有 loop 测试）、`tests/unit/test_harness_inference.py` + +> **原理**：训练/val/test/holdout 推理的 run_id 已含 `_e{epoch}`（如 `{base}_e{epoch}_s{step}`、`{run_id}_holdout_{kind}_e{epoch}`），用 run_id 作 cache_salt 即天然跨 epoch 重采样、同 epoch 续跑仍命中。judge/evolve 不经此路径（默认 salt=None）。gate 基线臂走 BaselineCache 不受影响；候选臂 messages 含 skill 版本天然区分。 + +- [ ] **Step 1: 写失败测试** + +```python +@pytest.mark.asyncio +async def test_agent_loop_forwards_cache_salt(): + """AgentLoop.run(cache_salt=...) 透传到 llm.chat。""" + from core.agent.loop import AgentLoop + # fake llm 记录 chat 收到的 cache_salt kwarg；loop.run(..., cache_salt='run:e2') + # 断言 fake_llm.chat 收到 cache_salt='run:e2' + ... +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest -k agent_loop_forwards_cache_salt -v` +Expected: FAIL（`run()` 无 cache_salt 参数）。 + +- [ ] **Step 3: AgentLoop 透传 cache_salt** + +`core/agent/loop.py`：`run`（L103）、`_step`/`_call_llm`（L271,317）签名加 `cache_salt: str | None = None`（keyword，随 session_id 透传）；L329 `self._llm.chat(messages, session_id=session_id)` 改为 `self._llm.chat(messages, session_id=session_id, cache_salt=cache_salt)`。 + +- [ ] **Step 4: inference 用 run_id 作 salt** + +`app/harness/inference.py` `_run_single_question`：`loop.run(...)`（L409）加 `cache_salt=run_id`（run_id 从 run_inference 透传到每题；`_run_single_question` 已有 run_id 上下文——若无则从 run_inference 参数透传）。确认 run_inference→_run_single_question 的 run_id 传递链完整。 + +- [ ] **Step 5: 保真检查点 + 测试通过** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py tests/integration/test_agent_governed_e2e.py -q` +Expected: 全 PASS。保真（算法 #10）：确认只加透传参数，Thinking+JSON/json_repair/pluggy hook 逻辑不变。 + +- [ ] **Step 6: 提交** + +```bash +git add core/agent/loop.py app/harness/inference.py tests/unit/ +git commit -m "feat: inject run_id as cache_salt for per-epoch resampling (algo #10)" +``` + +--- + +## Self-Review（作者自查，执行者复核） + +- [ ] traces 适配后诊断拿到真实轨迹（算法 #7 恢复）。 +- [ ] prediction 归一化 None 保留、非标量序列化；insert 绑定异常不击穿 gather。 +- [ ] early_stop 字段全局改名一致、计数改 epoch。 +- [ ] 预检剔除不可训练题型（test 池不动）。 +- [ ] patch 查整跨度 + 注入 + marker 完整性三防线（算法 #8 保护方向不变）。 +- [ ] None/degraded 按 lapse 保守分流；降级占比>50% 中止。 +- [ ] _run_step 幂等；gate_pools 与 checkpoint 的 gate_epoch_observed 同刻一致。 +- [ ] holdout 去重后 baseline 0 推理、best_hard 备忘录 resume 可 hydrate。 +- [ ] cache_salt=run_id 贯穿 AgentLoop，run_id 含 epoch 保证跨 epoch 重采样。 + +## 核心算法保真校验结论 + +触及算法 #5/#7/#8/#10/#12，均为恢复（#7 轨迹）/加固（#8 跨度）/透传（#10 salt）/时序（#5/#12 checkpoint），各 Task 已设保真检查点，不改算法核心逻辑。Task 5/7/9 需在实现时对照 TRM4 参考确认无行为漂移。 + +## 验收标准 + +1. `pytest tests/unit/test_harness_runner.py tests/unit/test_harness_inference.py tests/unit/test_diagnose.py tests/unit/test_patch.py tests/unit/test_harness_checkpoint.py tests/unit/test_baseline_run_log.py tests/unit/test_runner_diag_tree_inject.py` 全绿。 +2. 诊断拿到非空轨迹；非标量 prediction 不崩；early_stop 按 epoch。 +3. 微型题型被预检剔除；patch 不破坏 marker；降级题不驱动进化。 +4. _run_step 幂等；cache_salt=run_id 贯穿。 diff --git a/research-wiki/plans/2026-07-16-preflight-wp4-resilience.md b/research-wiki/plans/2026-07-16-preflight-wp4-resilience.md new file mode 100644 index 0000000..9016379 --- /dev/null +++ b/research-wiki/plans/2026-07-16-preflight-wp4-resilience.md @@ -0,0 +1,611 @@ +# WP4 韧性与持久化 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 修复 LLM 治理栈与持久化层的 10 处韧性缺陷，使训练信号不被缓存重放/截断响应/断连/INFRA 故障污染，且 workspace 元数据崩溃可恢复。 + +**Architecture:** 缓存加 salt 维度让跨 epoch 评估真实重采样（同 epoch 续跑仍命中）；SSE 未收 `[DONE]` 视为截断进重试且不写缓存；扩展瞬时错误清单覆盖断连族；熔断半开只放一个探针；gate 基线臂 INFRA 故障不污染 BaselineCache（算法 #6 保真区，逐行比对）；manifest 补原子写；只读查询不改基线元数据。 + +**Tech Stack:** Python 3.11、httpx、Redis、asyncio、SQLite、pytest、pydantic-settings。 + +**设计源**：`research-wiki/designs/2026-07-16-preflight-fixes-design.md §8` + +--- + +## 关键锚点（实现前必读） + +| 用途 | 位置 | +|------|------| +| chat 缓存/重试 | `adapters/llm.py:271` chat（get L299 / set L373 / 重试 L336）；`_call_streaming:506`；`_consume_stream:533`（不校验 done）；`_iter_sse_deltas:94`（done 标志 L116）；`_is_transient_error:173-186`；`_SseAnomaly:40` | +| 缓存键 | `adapters/redis_cache.py:32` `_build_key`（无 salt）；`get:50`；`set:73`（ttl 分支 L89-92） | +| Protocol | `core/protocols.py:22-28` `LLMProvider.chat` | +| TTL | `main.py:41` `redis_cache_ttl=86400`；`:93` `ttl_s = ... if >0 else None` | +| 熔断 | `adapters/breaker.py:24` `is_open`（L36-37 到期即放行，无探针锁）；`record_failure:39`；`record_success:63` | +| gate INFRA | `app/harness/validate.py:257` `_resolve_baseline_block`（put L303-304）；`_check_infra_guard:384`（累计检查 L593）；`BaselineCache` in `gate_ladder.py:344`（put L384 原子 L401-403） | +| 持久化 | `app/harness/workspace.py` 4 处非原子 write_text：`_scaffold:111`/`update_manifest:282`/`record_run:311`/`update_best:365`；原子范式 `checkpoint.py:257-260` | +| 基线元数据 | `app/harness/log.py:57` `HarnessLog.__init__`（upsert `_runs` L73-82）；`RunLogImpl._read_table:247`（只读） | +| dual_metric | `app/harness/observation.py:119` `write_dual_metric`（version_kind final L138）；runner 调用 `runner.py:1444-1448` | +| 配置 | `.env:65` `REDIS_CACHE_TTL=0`；`.env.example:49` `=86400` | + +## 核心算法保真校验 + +触及算法 #6（块顺序验证：基线缓存/INFRA 护栏/配对翻转）——Task 6（gate 基线臂 INFRA 隔离）改 `_resolve_baseline_block` 与护栏时序，**必须逐行比对** TRM4 `/home/iomgaa/Projects/Video-Tree-TRM4/core/harness/validate.py` 的 INFRA 护栏语义，确保只改"INFRA unit 不写缓存/不计 W/L + 护栏前置"，不动配对翻转与基线快照复用逻辑。其余 task 不触碰核心算法。 + +--- + +## Task 1: 缓存 cache_salt 贯穿 + +**Files:** +- Modify: `core/protocols.py:22-28` +- Modify: `adapters/llm.py:271-373` +- Modify: `adapters/redis_cache.py:32-92` +- Modify: VLM 适配转发（`adapters/` 内实现 `LLMProvider`/转发 chat 的类，grep 定位） +- Test: `tests/unit/test_redis_cache.py`、`tests/unit/test_governed_llm.py` + +- [ ] **Step 1: 写失败测试（salt 入键）** + +在 `tests/unit/test_redis_cache.py` 追加： +```python +@pytest.mark.asyncio +async def test_salt_changes_key(fake_redis): + from adapters.redis_cache import RedisResponseCache + + cache = RedisResponseCache(redis=fake_redis, ttl_s=None) + k_none = cache._build_key("m", [{"role": "user", "content": "x"}], None) + k_e1 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e1") + k_e2 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e2") + assert k_none != k_e1 != k_e2 and k_e1 != k_e2 +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_redis_cache.py::test_salt_changes_key -v` +Expected: FAIL（`_build_key() takes 3 positional arguments but 4 were given`）。 + +- [ ] **Step 3: redis_cache 加 salt 维度** + +`adapters/redis_cache.py`： +- `_build_key(self, model, messages, cache_salt: str | None = None)`：payload 字典加 `"salt": cache_salt`： +```python + payload = json.dumps( + {"model": model, "messages": messages, "salt": cache_salt}, + sort_keys=True, + ensure_ascii=False, + ) +``` +- `get(self, model, messages, cache_salt: str | None = None)`：`key = self._build_key(model, messages, cache_salt)` +- `set(self, model, messages, response, cache_salt: str | None = None)`：同样传 `cache_salt` + +- [ ] **Step 4: Protocol + chat 透传 salt** + +`core/protocols.py` `LLMProvider.chat` 签名加 `cache_salt: str | None = None`（keyword-only，放 `parent_call_id` 后）。 +`adapters/llm.py` `chat` 签名加同参；L299 改 `await self._cache.get(self._model, messages, cache_salt)`；L373 改 `await self._cache.set(self._model, messages, response, cache_salt)`。 +grep `adapters/` 找到转发 `chat` 的 VLM/包装类（如有），同步加 `cache_salt` 透传。 + +- [ ] **Step 5: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_redis_cache.py tests/unit/test_governed_llm.py -q` +Expected: 全 PASS（默认 `cache_salt=None` 保持旧键，既有缓存测试不受影响）。 + +- [ ] **Step 6: 提交** + +```bash +git add core/protocols.py adapters/llm.py adapters/redis_cache.py tests/unit/test_redis_cache.py +git commit -m "feat: add cache_salt dimension to LLM response cache" +``` + +> 注：训练链路注入 epoch 盐（`f"{run_id}:e{epoch}"`）在 WP3 的推理调用点完成（本 WP 只提供 salt 能力）。 + +--- + +## Task 2: TTL 语义修正（≤0 报错） + +**Files:** +- Modify: `main.py:93` +- Modify: `.env:65`、`.env.example:49` +- Test: `tests/unit/test_governed_llm.py` 或新增 `tests/unit/test_infra_settings.py` + +- [ ] **Step 1: 写失败测试** + +新增 `tests/unit/test_infra_settings.py`： +```python +import pytest + + +def test_redis_cache_ttl_zero_rejected(monkeypatch): + """REDIS_CACHE_TTL<=0 必须启动即报错，消灭'0=永不过期'隐式语义。""" + from main import _resolve_cache_ttl # Step 3 抽出的纯函数 + + with pytest.raises(ValueError, match="REDIS_CACHE_TTL"): + _resolve_cache_ttl(0) + with pytest.raises(ValueError, match="REDIS_CACHE_TTL"): + _resolve_cache_ttl(-1) + assert _resolve_cache_ttl(86400) == 86400 +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_infra_settings.py -v` +Expected: FAIL（`cannot import name '_resolve_cache_ttl'`）。 + +- [ ] **Step 3: 抽出并实现 fail-loud** + +`main.py`：新增纯函数并替换 L93 逻辑： +```python +def _resolve_cache_ttl(ttl: int) -> int: + """校验 Redis 缓存 TTL：必须为正整数（消灭 0=永不过期 的隐式语义）。""" + if ttl <= 0: + raise ValueError( + f"REDIS_CACHE_TTL 必须为正整数秒，实际 {ttl}。" + "训练场景建议 >= 单次训练时长（如 86400）。" + ) + return ttl +``` +L93 改为 `ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl)`（删除 `if >0 else None` 分支；`RedisResponseCache` 的 `ttl_s` 从此恒为正）。 + +- [ ] **Step 4: 改 .env / .env.example** + +`.env:65` `REDIS_CACHE_TTL=0` → `REDIS_CACHE_TTL=86400`。 +`.env.example:49` 确认为正整数（当前 86400，OK）；补注释 `# 正整数秒，禁止 0（0 会被拒绝启动）`。 + +- [ ] **Step 5: 运行测试确认通过** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_infra_settings.py -v` +Expected: PASS。 + +- [ ] **Step 6: 提交** + +```bash +git add main.py .env.example tests/unit/test_infra_settings.py +git commit -m "fix: reject REDIS_CACHE_TTL<=0 (kill implicit never-expire)" +``` +> 注：`.env` 不提交（gitignore）；改动需手动同步到运行环境，runbook 会提示。 + +--- + +## Task 3: SSE 截断检测（未收 [DONE] 即重试，不写缓存） + +**Files:** +- Modify: `adapters/llm.py:533-578`（_consume_stream） +- Test: `tests/unit/test_governed_llm.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_governed_llm.py` 追加： +```python +@pytest.mark.asyncio +async def test_truncated_stream_without_done_raises(_build_client): + """SSE 流耗尽但未收 [DONE] → _SseAnomaly（进重试，不当成功）。""" + from adapters.llm import _SseAnomaly, GovernedLLMClient + + async def _lines(): + yield 'data: {"choices":[{"delta":{"content":"半"}}]}' + # 无 data: [DONE] —— 模拟服务端截断 + + client = _build_client() # 依现有 fixture + with pytest.raises(_SseAnomaly): + await client._consume_stream(_lines()) +``` +> 依 `test_governed_llm.py` 现有 `_build_client`（L50）/`_FakeRedisCache` 构造；实现前读该文件对齐 client 构造签名。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -k truncated_stream -v` +Expected: FAIL（当前耗尽即正常返回，不抛）。 + +- [ ] **Step 3: _consume_stream 校验 done** + +`adapters/llm.py` `_consume_stream` 在 `return content, thinking, ttft_ms, max_inter_token_ms, usage`（L578）之前加： +```python + if not usage_sink.get("done"): + raise _SseAnomaly("truncated_no_done") +``` +（`_iter_sse_deltas` 收到 `[DONE]` 时置 `usage_sink["done"]=True`，L116；未置说明流被截断。`_SseAnomaly` 已在 `_is_transient_error` L186 归为可重试，故自动进重试梯且不走成功路径、不写缓存。） + +- [ ] **Step 4: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py tests/unit/test_streaming.py -q` +Expected: 全 PASS（正常流带 `[DONE]` 的既有测试仍通过；若既有 streaming 测试的假流未含 `[DONE]`，同步补 `data: [DONE]` 终帧）。 + +- [ ] **Step 5: 提交** + +```bash +git add adapters/llm.py tests/unit/test_governed_llm.py +git commit -m "fix: treat SSE stream without [DONE] as truncated (retry, no cache)" +``` + +--- + +## Task 4: 瞬时错误清单扩展（断连族） + +**Files:** +- Modify: `adapters/llm.py:182-186` +- Test: `tests/unit/test_governed_llm.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_governed_llm.py` 追加： +```python +def test_transient_covers_disconnect_family(): + import httpx + from adapters.llm import _is_transient_error + + assert _is_transient_error(httpx.RemoteProtocolError("peer reset")) + assert _is_transient_error(httpx.ReadError("read")) + assert _is_transient_error(httpx.ConnectTimeout("ct")) + assert _is_transient_error(httpx.PoolTimeout("pt")) +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -k disconnect_family -v` +Expected: FAIL（RemoteProtocolError/PoolTimeout 不在当前清单）。 + +- [ ] **Step 3: 扩展 _is_transient_error** + +`adapters/llm.py` L182-183 的： +```python + if isinstance(exc, (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout)): + return True +``` +改为（两族基类覆盖 RemoteProtocolError/ReadError/ConnectTimeout/PoolTimeout 等）： +```python + if isinstance(exc, (httpx.TimeoutException, httpx.TransportError)): + return True +``` + +- [ ] **Step 4: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -q` +Expected: 全 PASS（`httpx.HTTPStatusError` 非 TransportError 子类，401/403 致命分支不受影响）。 + +- [ ] **Step 5: 提交** + +```bash +git add adapters/llm.py tests/unit/test_governed_llm.py +git commit -m "fix: cover httpx disconnect family in transient error set" +``` + +--- + +## Task 5: 熔断半开单探针锁 + +**Files:** +- Modify: `adapters/breaker.py:18-70` +- Test: `tests/unit/test_breaker.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_breaker.py::TestCircuitBreaker` 追加： +```python + def test_half_open_admits_single_probe(self): + from adapters.breaker import CircuitBreaker + + b = CircuitBreaker(fail_threshold=2, cooldown_s=10.0) + b.record_failure("p", now=0.0) + b.record_failure("p", now=0.0) # 开路至 t=10 + assert b.is_open("p", now=5.0) is True # 冷却中 + # 冷却到期：只放行第一个探针 + assert b.is_open("p", now=11.0) is False # 探针 1 放行 + assert b.is_open("p", now=11.0) is True # 探针 2 被挡（探针在途） + b.record_success("p") # 探针成功 → 闭合 + assert b.is_open("p", now=12.0) is False +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_breaker.py -k half_open_admits_single -v` +Expected: FAIL（当前到期后所有调用都返回 False）。 + +- [ ] **Step 3: 实现半开单探针锁** + +`adapters/breaker.py`： +- `__init__` 加 `self._half_open_inflight: dict[str, bool] = {}` +- `is_open` 改为： +```python + def is_open(self, source_name: str, now: float) -> bool: + until = self._open_until.get(source_name) + if until is None: + return False + if now < until: + return True # 冷却中，全挡 + # 冷却到期：half-open，只放行一个探针 + if self._half_open_inflight.get(source_name): + return True # 已有探针在途，继续挡 + self._half_open_inflight[source_name] = True + return False +``` +- `record_success` 加 `self._half_open_inflight.pop(source_name, None)`（探针成功 → 清在途 + 已有的清 fails/open_until） +- `record_failure`：探针失败会累计并可能重开路；末尾加 `self._half_open_inflight.pop(source_name, None)`（让下一轮 cooldown 后可再探） +- `force_open`：加 `self._half_open_inflight.pop(source_name, None)` + +（is_open 在 asyncio 单线程内同步执行，"检查+标记探针"原子，无竞态。） + +- [ ] **Step 4: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_breaker.py tests/unit/test_governed_llm.py -q` +Expected: 全 PASS（既有 circuit_open 测试若假设"到期即多次放行"需同步更新为单探针语义）。 + +- [ ] **Step 5: 提交** + +```bash +git add adapters/breaker.py tests/unit/test_breaker.py +git commit -m "fix: half-open circuit admits single probe (no thundering herd)" +``` + +--- + +## Task 6: gate 基线臂 INFRA 隔离（算法 #6 保真区） + +**Files:** +- Modify: `app/harness/validate.py:257-311`（_resolve_baseline_block）+ 护栏时序 +- Test: `tests/unit/test_harness_validate.py` + +> **保真前置**：实现前完整读 `app/harness/validate.py:257-311` `_resolve_baseline_block`、`_candidate_correctness_from_db`、块循环（575-600），并逐行比对 TRM4 `core/harness/validate.py` 的 INFRA 护栏语义。目标仅为：① 基线臂 INFRA 故障（stop_reason∈{error,parse_error}）的 unit **不写 BaselineCache**、**不计入 W/L 翻转**；② 护栏检查移到"写缓存之前"。不得改动配对翻转、基线快照复用、unit 折叠逻辑。 + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_validate.py` 追加（用 fake run_inference 让某基线 unit 返回 stop_reason=error）： +```python +@pytest.mark.asyncio +async def test_baseline_infra_error_not_cached(tmp_path): + """基线臂 INFRA error 的 unit 不写入 BaselineCache（不永久污染）。""" + # 依现有 test_harness_validate.py 的 fake run_inference / BaselineCache fixture 构造； + # 让 miss unit u1 的推理返回 prediction=None, stop_reason='error'； + # 调 _resolve_baseline_block 后断言 baseline_cache.get(..., u1) is None + ... +``` +> 依 `test_harness_validate.py` 现有 fixture（fake `run_inference`、`BaselineCache`、`HarnessLog`）；实现前读该文件对齐构造，勿臆造签名。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_validate.py -k baseline_infra_error_not_cached -v` +Expected: FAIL（当前 INFRA unit 的对错被 put 进缓存）。 + +- [ ] **Step 3: 实现 INFRA 隔离** + +`_resolve_baseline_block`：miss 跑完后，识别 INFRA 题（从 db 读 stop_reason∈{error,parse_error} 或经 run_inference 结果），其所属 unit：不 `baseline_cache.put`、不计入返回的 `b_units`（从块 unit 集合排除，使配对与 W/L 不含它）。护栏所需的 `errors_inc` 在 put 之前累计并检查（护栏前置）。具体实现按 Step 前"保真前置"要求对齐现场与 TRM4。 + +- [ ] **Step 4: 运行测试确认通过 + 保真回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_validate.py tests/unit/test_gate_block_unit.py tests/unit/test_gates.py -q` +Expected: 全 PASS（配对翻转/e-process 既有测试不受影响 = 保真达成）。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/validate.py tests/unit/test_harness_validate.py +git commit -m "fix: isolate gate baseline-arm INFRA errors from BaselineCache (algo #6)" +``` + +--- + +## Task 7: manifest 原子写 + +**Files:** +- Modify: `app/harness/workspace.py`（`_scaffold:111`/`update_manifest:282`/`record_run:311`/`update_best:365`） +- Test: `tests/unit/test_harness_workspace.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_workspace.py` 追加： +```python +def test_update_manifest_is_atomic(workspace_dir, monkeypatch): + """写 manifest 途中崩溃不产生半截 JSON（原子写：tmp 存在即失败也不损原文件）。""" + import json + from app.harness import workspace as ws + + # 先建合法 manifest + ... # 依现有 fixture 初始化 workspace + original = (workspace_dir / "manifest.json").read_text() + + # monkeypatch os.replace 抛异常，模拟替换阶段崩溃 + def _boom(src, dst): + raise OSError("crash during replace") + monkeypatch.setattr(ws.os, "replace", _boom) + with pytest.raises(OSError): + ws.update_manifest(workspace_dir, skills="skills/v2") + # 原 manifest 未被破坏 + assert (workspace_dir / "manifest.json").read_text() == original + assert json.loads((workspace_dir / "manifest.json").read_text()) +``` +> 依 `test_harness_workspace.py` 现有 workspace 初始化 fixture；实现前读对齐。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_workspace.py -k update_manifest_is_atomic -v` +Expected: FAIL（当前 write_text 非原子，崩溃留半截）。 + +- [ ] **Step 3: 加原子写 helper 并替换 4 处** + +`app/harness/workspace.py` 顶部确认 `import os`。新增模块级 helper： +```python +def _atomic_write_json(path: Path, data: dict) -> None: + """原子写 JSON：tmp + os.replace（对齐 checkpoint.py 范式，防半截损坏）。""" + tmp = path.with_name(path.name + ".tmp") + tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + os.replace(tmp, path) +``` +把 `_scaffold_workspace`（L111）、`update_manifest`（L282）、`record_run`（L311）、`update_best`（L365）四处的 `(...).write_text(json.dumps(...))` 替换为 `_atomic_write_json(, )`。 + +- [ ] **Step 4: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_workspace.py -q` +Expected: 全 PASS。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/workspace.py tests/unit/test_harness_workspace.py +git commit -m "fix: atomic writes for manifest/record_run/update_best (tmp+replace)" +``` + +--- + +## Task 8: 只读查询不改基线元数据 + +**Files:** +- Modify: `app/harness/log.py:57-83`（HarnessLog.__init__ 加 register_run 开关） +- Modify: 只读查询基线的调用点（`app/harness/runner.py:923` `_init_gate`、`app/harness/pools.py:765,818`） +- Test: `tests/unit/test_harness_log.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_log.py::TestHarnessLogUpsert` 追加： +```python + def test_register_run_false_skips_upsert(self, tmp_path): + """register_run=False 时只读打开不改写已有 _runs 行（started_at/status 不变）。""" + db = str(tmp_path / "h.db") + from app.harness.log import HarnessLog + + with HarnessLog(db, "r1") as log: + log # 初次注册 + row0 = _read_run_row(db, "r1") # 依现有 helper 读 started_at/status + + with HarnessLog(db, "r1", register_run=False) as log: + log.query("SELECT 1") # 只读 + row1 = _read_run_row(db, "r1") + assert row1["started_at"] == row0["started_at"] + assert row1["status"] == row0["status"] +``` +> 依 `test_harness_log.py` 现有读行 helper；实现前读 `TestHarnessLogUpsert`（L196）对齐。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_log.py -k register_run_false -v` +Expected: FAIL（`unexpected keyword argument 'register_run'`）。 + +- [ ] **Step 3: HarnessLog 加 register_run 开关** + +`app/harness/log.py` `__init__` 签名加 `register_run: bool = True`（keyword）。把 L73-82 的 `_runs` upsert 包进 `if register_run:`；`register_run=False` 时跳过 upsert（仅 `_init_fixed_tables` 建表 + 连接，供只读查询）。`__exit__`/`close` 的 status 更新同样在 `register_run` 为 True 时才执行（避免只读关闭把 status 改 completed）。 + +- [ ] **Step 4: 只读调用点传 register_run=False** + +`app/harness/runner.py:923`（_init_gate 用 baseline_run_id 只读查 predictions）、`app/harness/pools.py:765,818`（build_or_load_pools 只读查 predictions）三处 `HarnessLog(...)` 调用加 `register_run=False`。 + +- [ ] **Step 5: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_log.py tests/unit/test_harness_pools.py -q` +Expected: 全 PASS。 + +- [ ] **Step 6: 提交** + +```bash +git add app/harness/log.py app/harness/runner.py app/harness/pools.py tests/unit/test_harness_log.py +git commit -m "fix: read-only baseline queries skip _runs upsert (register_run flag)" +``` + +--- + +## Task 9: dual_metric version_kind 口径修正 + +**Files:** +- Modify: `app/harness/runner.py:1444-1448`（第二次 write_dual_metric） +- Test: `tests/unit/test_harness_observation.py` 或 `test_harness_runner.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_observation.py` 追加（验证 slow_candidate 与 final 可区分）： +```python +def test_slow_candidate_kind_distinct_from_final(tmp_path): + from app.harness.observation import write_dual_metric, read_dual_metric + + db = str(tmp_path / "h.db") + write_dual_metric(db, run_id="r", epoch=1, version_kind="final", + skills_version="v1", prompts_version="v1", pool="val", + hard_acc=0.7, soft_score=None, mixed_score=None) + write_dual_metric(db, run_id="r", epoch=1, version_kind="slow_candidate", + skills_version="v2", prompts_version="v2", pool="val", + hard_acc=0.6, soft_score=None, mixed_score=None) + rows = read_dual_metric(db, run_id="r", epoch=1) + kinds = {r["version_kind"] for r in rows} + assert "final" in kinds and "slow_candidate" in kinds +``` +> 依 `test_harness_observation.py` 现有 read_dual_metric 签名；实现前读对齐。 + +- [ ] **Step 2: 运行确认失败/通过基线** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_observation.py -k slow_candidate -v` +Expected: 若 read_dual_metric 已支持则 PASS 基线；重点是 Step 3 改 runner 调用点。 + +- [ ] **Step 3: 改 runner 慢更新第二次写为 slow_candidate** + +`app/harness/runner.py` L1444-1448 的 `write_dual_metric(..., version_kind="final", ...)`（`_slow_update_cycle` Phase 8 的 R2 行）改为 `version_kind="slow_candidate"`，使被 revert 的慢更新候选不再占用 `final` 语义。确认 Phase 2 的第一次（L1399-1403）仍为 `final`（epoch 终值唯一）。 + +- [ ] **Step 4: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_observation.py tests/unit/test_harness_runner.py -q` +Expected: 全 PASS。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/runner.py tests/unit/test_harness_observation.py +git commit -m "fix: slow-update R2 dual_metric uses slow_candidate kind (not final)" +``` + +--- + +## Task 10: 离线 --retry-uncertain + docstring 修正 + +**Files:** +- Modify: `app/harness/video_split_cli.py`（build_arg_parser + run_pipeline） +- Modify: `app/harness/baseline_diagnosis.py`（done 集排除 uncertain + docstring） +- Test: `tests/unit/test_video_split_cli.py` 或 `test_baseline_diagnosis` + +- [ ] **Step 1: 写失败测试** + +定位 `baseline_diagnosis` 的 done_question_ids 计算函数（`app/harness/baseline_diagnosis.py`，grep `done_question_ids`），追加测试：`retry_uncertain=True` 时 tier=uncertain 的已落库题不计入 done（会被重诊）。 +```python +def test_retry_uncertain_excludes_uncertain_from_done(...): + # 构造 baseline_diagnosis 表含 tier=T2 与 tier=uncertain 行； + # done_ids(retry_uncertain=False) 含 uncertain 题； + # done_ids(retry_uncertain=True) 不含 uncertain 题 + ... +``` +> 实现前读 `baseline_diagnosis.py` 的 done 集查询函数签名对齐。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest -k retry_uncertain -v` +Expected: FAIL（无该参数）。 + +- [ ] **Step 3: 实现 retry_uncertain** + +`app/harness/baseline_diagnosis.py`：done 集查询函数加 `retry_uncertain: bool = False`，True 时 `WHERE ... AND tier != 'uncertain'`（uncertain 题不算完成，会被重诊）。修正模块 docstring 把"崩溃最多丢正在写的一行/逐行落库"表述改为实际的"run 末批量落库（Phase 3），崩溃丢本次 run 全部未落库结果，靠 Redis 缓存缓解重烧"。 +`app/harness/video_split_cli.py`：`build_arg_parser` 加 `--retry-uncertain`（store_true）；`run_pipeline` 透传到 `run_baseline_diagnosis` 的 done 集计算。 + +- [ ] **Step 4: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -q` +Expected: 全 PASS。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/video_split_cli.py app/harness/baseline_diagnosis.py tests/unit/ +git commit -m "feat: --retry-uncertain re-diagnoses uncertain rows; fix docstring" +``` + +--- + +## Self-Review（作者自查，执行者复核） + +- [ ] cache_salt 四处（protocols/llm/redis_cache/VLM 转发）贯通，默认 None 保持旧键。 +- [ ] TTL≤0 fail-loud + .env/.env.example 同步（.env 手动，runbook 提示）。 +- [ ] SSE 未收 [DONE] → _SseAnomaly（已在瞬时清单 → 重试 + 不写缓存）。 +- [ ] 瞬时清单用 TimeoutException+TransportError 两族基类覆盖断连族，不误纳 HTTPStatusError。 +- [ ] 熔断半开单探针：is_open 检查+标记原子（asyncio 单线程）。 +- [ ] Task 6 触及算法 #6，已设保真前置（逐行比对 TRM4，只改 INFRA 不写缓存 + 护栏前置）。 +- [ ] manifest 4 处原子写；只读查询 register_run=False 不改基线元数据。 + +## 核心算法保真校验结论 + +Task 6 触及算法 #6（块顺序验证），已在该 Task 设"保真前置"要求逐行比对 TRM4 `validate.py`，仅改 INFRA 隔离与护栏时序，不动配对翻转/基线快照/unit 折叠。其余 Task 均为治理栈/持久化层，不涉及核心算法。 + +## 验收标准 + +1. `pytest tests/unit/test_redis_cache.py tests/unit/test_governed_llm.py tests/unit/test_breaker.py tests/unit/test_streaming.py tests/unit/test_harness_validate.py tests/unit/test_harness_workspace.py tests/unit/test_harness_log.py tests/unit/test_harness_observation.py tests/unit/test_infra_settings.py` 全绿。 +2. 缓存加 salt 后跨 epoch 键不同、同 epoch 键相同。 +3. 熔断半开只放一个探针；SSE 截断进重试不写缓存。 +4. manifest 原子写；基线 _runs 元数据只读查询不被改写。 diff --git a/research-wiki/plans/preflight-wp1-asset-migration.md b/research-wiki/plans/preflight-wp1-asset-migration.md new file mode 100644 index 0000000..a7493e6 --- /dev/null +++ b/research-wiki/plans/preflight-wp1-asset-migration.md @@ -0,0 +1,9 @@ +--- +type: plan +node_id: plan:preflight-wp1-asset-migration +title: "WP1 资产迁移" +date: 2026-07-16 +--- + +# WP1 资产迁移 + diff --git a/research-wiki/plans/preflight-wp2-split-wiring.md b/research-wiki/plans/preflight-wp2-split-wiring.md new file mode 100644 index 0000000..20bffd8 --- /dev/null +++ b/research-wiki/plans/preflight-wp2-split-wiring.md @@ -0,0 +1,9 @@ +--- +type: plan +node_id: plan:preflight-wp2-split-wiring +title: "WP2 切分与接线" +date: 2026-07-16 +--- + +# WP2 切分与接线 + diff --git a/research-wiki/plans/preflight-wp3-train-loop.md b/research-wiki/plans/preflight-wp3-train-loop.md new file mode 100644 index 0000000..fdb0535 --- /dev/null +++ b/research-wiki/plans/preflight-wp3-train-loop.md @@ -0,0 +1,9 @@ +--- +type: plan +node_id: plan:preflight-wp3-train-loop +title: "WP3 训练循环与进化引擎" +date: 2026-07-16 +--- + +# WP3 训练循环与进化引擎 + diff --git a/research-wiki/plans/preflight-wp4-resilience.md b/research-wiki/plans/preflight-wp4-resilience.md new file mode 100644 index 0000000..8f02197 --- /dev/null +++ b/research-wiki/plans/preflight-wp4-resilience.md @@ -0,0 +1,9 @@ +--- +type: plan +node_id: plan:preflight-wp4-resilience +title: "WP4 韧性与持久化" +date: 2026-07-16 +--- + +# WP4 韧性与持久化 +