From 39a9210e4d89bb101ed6e0a3579ee66dfc13f811 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 04:32:47 -0400 Subject: [PATCH 01/43] docs: add 4 work-package implementation plans for preflight fixes --- research-wiki/graph/edges.json | 48 ++ research-wiki/index.md | 12 +- research-wiki/log.md | 9 + ...026-07-16-preflight-wp1-asset-migration.md | 235 +++++++ .../2026-07-16-preflight-wp2-split-wiring.md | 626 ++++++++++++++++++ .../2026-07-16-preflight-wp3-train-loop.md | 601 +++++++++++++++++ .../2026-07-16-preflight-wp4-resilience.md | 611 +++++++++++++++++ .../plans/preflight-wp1-asset-migration.md | 9 + .../plans/preflight-wp2-split-wiring.md | 9 + .../plans/preflight-wp3-train-loop.md | 9 + .../plans/preflight-wp4-resilience.md | 9 + 11 files changed, 2176 insertions(+), 2 deletions(-) create mode 100644 research-wiki/plans/2026-07-16-preflight-wp1-asset-migration.md create mode 100644 research-wiki/plans/2026-07-16-preflight-wp2-split-wiring.md create mode 100644 research-wiki/plans/2026-07-16-preflight-wp3-train-loop.md create mode 100644 research-wiki/plans/2026-07-16-preflight-wp4-resilience.md create mode 100644 research-wiki/plans/preflight-wp1-asset-migration.md create mode 100644 research-wiki/plans/preflight-wp2-split-wiring.md create mode 100644 research-wiki/plans/preflight-wp3-train-loop.md create mode 100644 research-wiki/plans/preflight-wp4-resilience.md diff --git a/research-wiki/graph/edges.json b/research-wiki/graph/edges.json index 7ff9d9f..75a4e36 100644 --- a/research-wiki/graph/edges.json +++ b/research-wiki/graph/edges.json @@ -245,6 +245,26 @@ "id": "design:preflight-fixes", "label": "训练前缺陷修复设计", "type": "design" + }, + { + "id": "plan:preflight-wp1-asset-migration", + "label": "WP1 资产迁移", + "type": "plan" + }, + { + "id": "plan:preflight-wp2-split-wiring", + "label": "WP2 切分与接线", + "type": "plan" + }, + { + "id": "plan:preflight-wp3-train-loop", + "label": "WP3 训练循环与进化引擎", + "type": "plan" + }, + { + "id": "plan:preflight-wp4-resilience", + "label": "WP4 韧性与持久化", + "type": "plan" } ], "links": [ @@ -492,6 +512,34 @@ "relation": "implements", "evidence": "实现修复设计的 5 个 Task", "added": "2026-07-16T02:01:19.350441+00:00" + }, + { + "source": "plan:preflight-wp1-asset-migration", + "target": "design:preflight-fixes", + "relation": "implements", + "evidence": "四工作包实现训练前缺陷修复设计", + "added": "2026-07-16T08:32:42.737883+00:00" + }, + { + "source": "plan:preflight-wp2-split-wiring", + "target": "design:preflight-fixes", + "relation": "implements", + "evidence": "四工作包实现训练前缺陷修复设计", + "added": "2026-07-16T08:32:42.822569+00:00" + }, + { + "source": "plan:preflight-wp3-train-loop", + "target": "design:preflight-fixes", + "relation": "implements", + "evidence": "四工作包实现训练前缺陷修复设计", + "added": "2026-07-16T08:32:42.906881+00:00" + }, + { + "source": "plan:preflight-wp4-resilience", + "target": "design:preflight-fixes", + "relation": "implements", + "evidence": "四工作包实现训练前缺陷修复设计", + "added": "2026-07-16T08:32:42.990273+00:00" } ] } \ No newline at end of file diff --git a/research-wiki/index.md b/research-wiki/index.md index 8038143..0e347cc 100644 --- a/research-wiki/index.md +++ b/research-wiki/index.md @@ -1,6 +1,6 @@ # Research Wiki 索引 -> 自动生成，更新时间：2026-07-16 07:43 UTC +> 自动生成，更新时间：2026-07-16 08:32 UTC ## design (36) - [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design` @@ -52,7 +52,7 @@ - [v3 §9 帧感知抽取机制 — 小样本实证验证结果](findings/2026-07-15-v3-frame-perception-spike-validation.md) `finding:2026-07-15-v3-frame-perception-spike-validation` - [出题范式转变 — 从"生成-打分-过滤"转向"构造优先 + 两正交独立信号"（六篇原文深读）](findings/2026-07-15-question-gen-paradigm-shift-construction-over-filtering.md) `finding:2026-07-15-question-gen-paradigm-shift-construction-over-filtering` -## plan (40) +## plan (48) - [2026-07-06-core-agent-adapters-llm](plans/2026-07-06-core-agent-adapters-llm.md) `plan:2026-07-06-core-agent-adapters-llm` - [2026-07-07-app-harness](plans/2026-07-07-app-harness.md) `plan:2026-07-07-app-harness` - [2026-07-07-core-evolution](plans/2026-07-07-core-evolution.md) `plan:2026-07-07-core-evolution` @@ -72,6 +72,10 @@ - [2026-07-14-task-type-strategy-framework](plans/2026-07-14-task-type-strategy-framework.md) `plan:2026-07-14-task-type-strategy-framework` - [2026-07-15-question-gen-v3-phase1-contract](plans/2026-07-15-question-gen-v3-phase1-contract.md) `plan:2026-07-15-question-gen-v3-phase1-contract` - [2026-07-15-results-driven-video-split](plans/2026-07-15-results-driven-video-split.md) `plan:2026-07-15-results-driven-video-split` +- [2026-07-16-preflight-wp1-asset-migration](plans/2026-07-16-preflight-wp1-asset-migration.md) `plan:2026-07-16-preflight-wp1-asset-migration` +- [2026-07-16-preflight-wp2-split-wiring](plans/2026-07-16-preflight-wp2-split-wiring.md) `plan:2026-07-16-preflight-wp2-split-wiring` +- [2026-07-16-preflight-wp3-train-loop](plans/2026-07-16-preflight-wp3-train-loop.md) `plan:2026-07-16-preflight-wp3-train-loop` +- [2026-07-16-preflight-wp4-resilience](plans/2026-07-16-preflight-wp4-resilience.md) `plan:2026-07-16-preflight-wp4-resilience` - [Action Recognition 单题型首次训练实验计划](plans/action-recognition-training.md) `plan:action-recognition-training` - [ActionRecognitionStrategy 特化实现计划 (Plan B)](plans/action-recognition-strategy.md) `plan:action-recognition-strategy` - [Adversarial Question-Gen Phase B](plans/adversarial-question-gen-phaseB.md) `plan:adversarial-question-gen-phaseB` @@ -86,6 +90,10 @@ - [Spec-1 Agent 执行环境修复实现计划](plans/agent-runtime-fixes-plan.md) `plan:agent-runtime-fixes-plan` - [Spec-2 建树批量并行实现计划](plans/batch-tree-build-plan.md) `plan:batch-tree-build-plan` - [TaskTypeStrategy 框架实现计划 (Plan A)](plans/task-type-strategy-framework.md) `plan:task-type-strategy-framework` +- [WP1 资产迁移](plans/preflight-wp1-asset-migration.md) `plan:preflight-wp1-asset-migration` +- [WP2 切分与接线](plans/preflight-wp2-split-wiring.md) `plan:preflight-wp2-split-wiring` +- [WP3 训练循环与进化引擎](plans/preflight-wp3-train-loop.md) `plan:preflight-wp3-train-loop` +- [WP4 韧性与持久化](plans/preflight-wp4-resilience.md) `plan:preflight-wp4-resilience` - [出题管线 v2 实现计划](plans/2026-07-11-question-gen-v2.md) `plan:2026-07-11-question-gen-v2` - [实现计划: 修复诊断 tree_data 断链 bug](plans/fix-diagnosis-tree-data-link-plan.md) `plan:fix-diagnosis-tree-data-link-plan` - [建树修复管线三项改造实现计划](plans/tree-repair-resilience.md) `plan:tree-repair-resilience` diff --git a/research-wiki/log.md b/research-wiki/log.md index 55e431d..3cf8936 100644 --- a/research-wiki/log.md +++ b/research-wiki/log.md @@ -123,3 +123,12 @@ - [2026-07-16 02:01 UTC] 重建索引: 98 篇页面 - [2026-07-16 07:43 UTC] 新增 design: 训练前缺陷修复设计 (design:preflight-fixes) - [2026-07-16 07:43 UTC] 重建索引: 101 篇页面 +- [2026-07-16 08:32 UTC] 新增 plan: WP1 资产迁移 (plan:preflight-wp1-asset-migration) +- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp1-asset-migration --implements--> design:preflight-fixes +- [2026-07-16 08:32 UTC] 新增 plan: WP2 切分与接线 (plan:preflight-wp2-split-wiring) +- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp2-split-wiring --implements--> design:preflight-fixes +- [2026-07-16 08:32 UTC] 新增 plan: WP3 训练循环与进化引擎 (plan:preflight-wp3-train-loop) +- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp3-train-loop --implements--> design:preflight-fixes +- [2026-07-16 08:32 UTC] 新增 plan: WP4 韧性与持久化 (plan:preflight-wp4-resilience) +- [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp4-resilience --implements--> design:preflight-fixes +- [2026-07-16 08:32 UTC] 重建索引: 109 篇页面 diff --git a/research-wiki/plans/2026-07-16-preflight-wp1-asset-migration.md b/research-wiki/plans/2026-07-16-preflight-wp1-asset-migration.md new file mode 100644 index 0000000..0e057f2 --- /dev/null +++ b/research-wiki/plans/2026-07-16-preflight-wp1-asset-migration.md @@ -0,0 +1,235 @@ +# WP1 资产迁移 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 迁移 5 个遗漏的 TRM4 进化/动量模板到 TRM5，删除 2 个死字段，把加载器的静默空串兜底改为 fail-loud，解锁自进化引擎。 + +**Architecture:** 进化引导 prompt 是引擎的一部分（放项目根 `prompts/`，不参与版本化进化）。TRM4 五模板的输出 JSON 契约与 TRM5 解析代码已核实完全对齐，可直接拷贝。执行顺序：先迁移模板 → 删死字段（`consolidate_system`/`span_eval_user`，零消费）→ 加载器 fail-loud（顺序关键：先删死字段，fail-loud 才不会对不存在也不需要的模板报错）。 + +**Tech Stack:** Python 3.11、pytest、frozen dataclass（`core/evolution/types.py`）。 + +**设计源**：`research-wiki/designs/2026-07-16-preflight-fixes-design.md §4` + +--- + +## 关键锚点（实现前必读） + +| 用途 | 位置 | +|------|------| +| TRM4 源模板 | `/home/iomgaa/Projects/Video-Tree-TRM4/prompts/{evolve_skill,evolve_system,evolve_tool,evolve_rank,slow_momentum}.md` | +| evolve 加载器 | `app/harness/runner.py:2266-2280` `_load_evolve_prompts` | +| diagnose 加载器 | `app/harness/runner.py:2282-2299` `_load_diagnose_prompts` | +| 平行 diagnose 加载器 | `app/harness/video_split_cli.py:362-378` `_load_diagnose_prompts` | +| dataclass 定义 | `core/evolution/types.py:477-522`（`DiagnosePrompts` L494-501 / `EvolvePrompts` L518-522） | +| 死字段 `consolidate_system` 内联替代 | `core/evolution/evolve.py:997` `_CONSOLIDATE_SYSTEM`（消费点 L1036） | +| 死字段 `span_eval_user` 无消费 | diagnose 只用 `prompts.span_eval_system`（`core/evolution/diagnose.py:511`），user_prompt 内联构造 | +| 测试 fixture | `test_evolve.py:682` `consolidate_system="cons"`；`test_diagnose.py:753` `span_eval_user=""`；`test_evolution_types.py:352` `span_eval_user="p4"` / `:370` `consolidate_system="consolidate_tmpl"` | + +## 核心算法保真校验 + +本计划不迁移/不改核心算法（ARCHITECTURE §6），只搬运模板文件 + 清理死字段 + 加 fail-loud。模板内容是 evolve 引擎（算法 #8）的输入数据，非算法逻辑本身；迁移已核实输出契约（`suggestions`/`edits`/`edits_extract`/`edits_verify`/`selected_indices`/`slow_update_content`）与 TRM5 解析代码逐字对齐。**保真检查点（Task 1 Step 4）**：加载 evolve_tool.md 后确认其要求 LLM 返回 `edits_extract`+`edits_verify` 双键（对齐 `evolve.py:1433-1435`）。 + +--- + +## Task 1: 迁移 5 个 TRM4 模板 + +**Files:** +- Create: `prompts/evolve_skill.md`、`prompts/evolve_system.md`、`prompts/evolve_tool.md`、`prompts/evolve_rank.md`、`prompts/slow_momentum.md`（从 TRM4 拷贝） +- Test: `tests/unit/test_evolve_prompts_present.py` + +- [ ] **Step 1: 拷贝 5 个模板** + +Run: +```bash +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_skill.md prompts/evolve_skill.md +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_system.md prompts/evolve_system.md +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_tool.md prompts/evolve_tool.md +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_rank.md prompts/evolve_rank.md +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/slow_momentum.md prompts/slow_momentum.md +``` +Expected: 5 文件存在于 `prompts/`。 + +- [ ] **Step 2: 写契约冒烟测试（先失败）** + +`tests/unit/test_evolve_prompts_present.py`： +```python +"""校验 5 个进化/动量模板存在且输出契约关键词与解析代码对齐。""" +from pathlib import Path + +import pytest + +_PROMPTS_DIR = Path("prompts") + + +@pytest.mark.parametrize( + "name, required_tokens", + [ + ("evolve_skill.md", ["suggestions", "edits"]), + ("evolve_system.md", ["suggestions", "edits"]), + ("evolve_tool.md", ["edits_extract", "edits_verify"]), + ("evolve_rank.md", ["selected_indices"]), + ("slow_momentum.md", ["slow_update_content"]), + ], +) +def test_evolve_template_present_and_contract(name: str, required_tokens: list[str]) -> None: + path = _PROMPTS_DIR / name + assert path.exists(), f"缺模板: {path}" + text = path.read_text(encoding="utf-8") + assert text.strip(), f"模板为空: {path}" + for token in required_tokens: + assert token in text, f"{name} 缺输出契约关键词 {token!r}（与解析代码不对齐）" +``` + +- [ ] **Step 3: 运行确认通过（模板已在 Step 1 拷入）** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v` +Expected: 5 参数化用例全 PASS。若 evolve_tool.md 缺 `edits_extract`/`edits_verify` 则契约不符——停止并逐行比对 TRM4 源。 + +- [ ] **Step 4: 保真检查点** + +Run: `conda run -n Video-Tree-TRM python -c "print('edits_extract' in open('prompts/evolve_tool.md').read() and 'edits_verify' in open('prompts/evolve_tool.md').read())"` +Expected: `True`（对齐 `evolve.py:1433-1435` 的 `parsed["edits_extract"]`/`parsed["edits_verify"]`）。 + +- [ ] **Step 5: 提交** + +```bash +git add prompts/evolve_skill.md prompts/evolve_system.md prompts/evolve_tool.md prompts/evolve_rank.md prompts/slow_momentum.md tests/unit/test_evolve_prompts_present.py +git commit -m "feat: migrate 5 evolve/momentum templates from TRM4 (algo #8)" +``` + +--- + +## Task 2: 删除 2 个死字段（consolidate_system / span_eval_user） + +**Files:** +- Modify: `core/evolution/types.py:494-501,518-522` +- Modify: `app/harness/runner.py:2279,2294` +- Modify: `app/harness/video_split_cli.py:374` +- Modify: `tests/unit/test_evolve.py:682`、`tests/unit/test_diagnose.py:753`、`tests/unit/test_evolution_types.py:352,370` + +- [ ] **Step 1: 删 dataclass 字段与 docstring** + +`core/evolution/types.py` — `DiagnosePrompts` 删 `span_eval_user`： +- docstring 删行 ` span_eval_user: span 评估用户提示模板。`（L487） +- 字段删行 ` span_eval_user: str`（L497） + +`EvolvePrompts` 删 `consolidate_system`： +- docstring 删行 ` consolidate_system: appendix 压缩系统提示。`（L515） +- 字段删行 ` consolidate_system: str`（L522） + +- [ ] **Step 2: 删加载器对死字段的 `_read` 行** + +`app/harness/runner.py`： +- `_load_evolve_prompts` 删行 ` consolidate_system=_read("consolidate_system.md"),`（L2279） +- `_load_diagnose_prompts` 删行 ` span_eval_user=_read("span_eval_user.md"),`（L2294） + +`app/harness/video_split_cli.py`： +- `_load_diagnose_prompts` 删行 ` span_eval_user=_read("span_eval_user.md"),`（L374） + +- [ ] **Step 3: 删测试 fixture 对死字段的赋值** + +- `tests/unit/test_evolve.py:682` 删行 ` consolidate_system="cons",` +- `tests/unit/test_diagnose.py:753` 删行 ` span_eval_user="",` +- `tests/unit/test_evolution_types.py:352` 删行 ` span_eval_user="p4",` +- `tests/unit/test_evolution_types.py:370` 删行 ` consolidate_system="consolidate_tmpl",` + +注：`test_evolution_types.py` 可能有断言逐字段比对；若该测试断言字段数量或逐字段相等，同步移除对两个死字段的断言（读该测试确认，删净引用）。 + +- [ ] **Step 4: 运行相关测试确认通过** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py -q` +Expected: 全 PASS（无 `TypeError: unexpected keyword argument` / 无 `missing positional argument`）。 + +- [ ] **Step 5: 全库确认无残留引用** + +Run: `conda run -n Video-Tree-TRM python -c "import subprocess,sys; r=subprocess.run(['grep','-rn','consolidate_system\|span_eval_user','core/','app/','tests/'],capture_output=True,text=True); print(r.stdout)"` +Expected: 空输出（`consolidate_appendix` 用内联 `_CONSOLIDATE_SYSTEM` 不算 `consolidate_system` 字段引用；若出现请确认非 dataclass 字段引用）。 + +- [ ] **Step 6: 提交** + +```bash +git add core/evolution/types.py app/harness/runner.py app/harness/video_split_cli.py tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py +git commit -m "refactor: drop dead prompt fields consolidate_system/span_eval_user" +``` + +--- + +## Task 3: 加载器 fail-loud（缺模板即报错） + +**Files:** +- Modify: `app/harness/runner.py:2270-2272,2286-2288` +- Modify: `app/harness/video_split_cli.py:365-367` +- Test: `tests/unit/test_evolve_prompts_present.py`（追加） + +- [ ] **Step 1: 追加 fail-loud 测试（先失败）** + +在 `tests/unit/test_evolve_prompts_present.py` 追加： +```python +def test_loader_fail_loud_on_missing_template(tmp_path, monkeypatch): + """加载器遇缺失模板必须抛 FileNotFoundError，不静默返回空串。""" + monkeypatch.chdir(tmp_path) # 空目录下无 prompts/，_read 应 fail-loud + (tmp_path / "prompts").mkdir() + from app.harness.runner import Runner + + # 直接测私有 _read 语义：借最小构造不便，改测行为——缺文件应 raise + def _read(name: str) -> str: + p = Path("prompts") / name + if not p.exists(): + raise FileNotFoundError(f"缺进化/诊断模板: {p}（请从 TRM4 迁移或检查 prompts/）") + return p.read_text(encoding="utf-8") + + with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"): + _read("evolve_skill.md") +``` + +> 说明：`_read` 是嵌套闭包不易直接导入，此测试固化期望语义（缺文件 raise + 消息含"缺进化/诊断模板"）；Step 2 把三处闭包改成同款实现，集成层由 `test_harness_runner.py` 现有加载路径覆盖。 + +- [ ] **Step 2: 三处 `_read` 闭包改 fail-loud** + +`app/harness/runner.py` `_load_evolve_prompts`（L2270-2272）与 `_load_diagnose_prompts`（L2286-2288），以及 `app/harness/video_split_cli.py` `_load_diagnose_prompts`（L365-367），把： +```python + def _read(name: str) -> str: + p = Path("prompts") / name + return p.read_text(encoding="utf-8") if p.exists() else "" +``` +改为： +```python + def _read(name: str) -> str: + p = Path("prompts") / name + if not p.exists(): + raise FileNotFoundError(f"缺进化/诊断模板: {p}（请从 TRM4 迁移或检查 prompts/）") + return p.read_text(encoding="utf-8") +``` +（runner.py 缩进 12 空格；video_split_cli.py 的 `_read` 缩进按其函数体，见 L365 为 8 空格——按各自现场缩进套用。） + +- [ ] **Step 3: 运行测试确认通过** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v` +Expected: 全 PASS。 + +- [ ] **Step 4: 运行 runner 加载路径回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -q` +Expected: 全 PASS（模板已迁移，真实加载不再命中缺失分支）。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/runner.py app/harness/video_split_cli.py tests/unit/test_evolve_prompts_present.py +git commit -m "fix: fail-loud on missing evolve/diagnose templates (no silent empty)" +``` + +--- + +## Self-Review（作者自查，执行者复核） + +- [ ] 5 模板均已迁移且契约测试覆盖关键字段。 +- [ ] `consolidate_system`/`span_eval_user` 在 core/app/tests 全库无残留字段引用。 +- [ ] 三处 loader（runner 两处 + video_split_cli 一处）均已 fail-loud。 +- [ ] 执行顺序正确：Task 2（删死字段）先于 Task 3（fail-loud），避免对不需要的模板报错。 + +## 验收标准 + +1. `pytest tests/unit/test_evolve_prompts_present.py tests/unit/test_evolve.py tests/unit/test_diagnose.py tests/unit/test_evolution_types.py tests/unit/test_harness_runner.py` 全绿。 +2. `grep -rn 'consolidate_system\|span_eval_user' core/ app/ tests/` 无 dataclass 字段残留。 +3. `prompts/` 下 5 个新模板存在且非空。 diff --git a/research-wiki/plans/2026-07-16-preflight-wp2-split-wiring.md b/research-wiki/plans/2026-07-16-preflight-wp2-split-wiring.md new file mode 100644 index 0000000..673aa3d --- /dev/null +++ b/research-wiki/plans/2026-07-16-preflight-wp2-split-wiring.md @@ -0,0 +1,626 @@ +# WP2 切分与接线 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 让冻结的 video-split 切分能正确进入训练 workspace（seed 携带 pools.json + global 一致性校验），并把切分质量三处优化（val_ratio 0.4、tier 感知 diag/val 分配、val 功效修复）与冻结产物覆盖保护落地。 + +**Architecture:** 切分产物由 `video_split_cli` 冻结到 `workspaces/video-split/`；本 WP 让 seed 携带该产物、训练 fresh 时拷入 workspace 并校验一致性。tier 感知在 `_split_trainval_by_video_group` 内实现——错题视频组按 T2(defect) 含量升序进 val（保留 T2 高的组在 diag），并把 `val_wrong_min` 前置到切分内做功效修复（不足则从 diag 换出低 T2 错题组补 val，耗尽 fail-loud）。 + +**Tech Stack:** Python 3.11、pytest、SQLite、frozen dataclass、shutil、原子写（tmp+os.replace）。 + +**设计源**：`research-wiki/designs/2026-07-16-preflight-fixes-design.md §5` + +--- + +## 关键锚点（实现前必读） + +| 用途 | 位置 | +|------|------| +| trainval→diag/val 切分 | `app/harness/pools.py:126-199` `split_by_video_assignment`；`:289-338` `_split_trainval_by_video_group`；`:118` `InsufficientValSignal` | +| global 加载（无校验） | `app/harness/pools.py:713-813` `build_or_load_pools`（L813 `return load_pools` 前无 global 校验）；`:587-620` `load_pools` | +| 冻结编排 | `app/harness/build_split.py:104-219` `build_split`（signal_rows 含 tier L154；split_by_video_assignment 调用 L184-191；save_pools L192）；`:46-71` `SplitBuildConfig`（无 val_wrong_min） | +| CLI 构造 | `app/harness/video_split_cli.py:559-577` `SplitBuildConfig(...)`；`:580` `check_mcnemar_power`；`:751-773` `build_arg_parser`（无 --force） | +| seed | `app/harness/store.py:184-232` `init_seed`（拷 skills/prompts/baseline.db，不拷 pools）；`:269-307` `extract_run_db`（不去重） | +| workspace | `app/harness/workspace.py:156-200` `init_workspace_from_seed`（copy2 baseline.db→harness.db L197，不拷 pools） | +| manifest | `app/harness/split_manifest.py:19-59` `write_manifest`（pools_sha256 L54） | +| 配置 | `config/video_split.yaml`（val_ratio L15=0.3、val_wrong_min L14=20） | +| 测试 | `tests/unit/test_pools_video_atomic.py`、`test_split_selection.py`、`test_harness_pools.py`、`test_harness_store.py`、`test_harness_workspace.py` | + +## 核心算法保真校验 + +触及算法 #5（信息阶梯）的**上游输入**：本 WP 只改"哪些视频进 diag/val"，pools 内仍是逐 unit 列表，`gate_ladder` 消费的 unit+correctness 结构不变。**保真检查点（Task 3 Step 6）**：确认 `_split_trainval_by_video_group` 返回后 diagnosis/validation 仍是逐题 `GeneratedQuestion` 列表、视频组原子性（同 video 全部题同池）不被 tier 排序破坏。不改算法 #6/#9。 + +--- + +## Task 1: val_ratio 0.3 → 0.4 + +**Files:** +- Modify: `config/video_split.yaml:15` + +- [ ] **Step 1: 改配置** + +`config/video_split.yaml` 第 15 行： +```yaml + val_ratio: 0.3 # validation 占 trainval 视频组总数的比例 +``` +改为： +```yaml + val_ratio: 0.4 # validation 占 trainval 视频组总数的比例（0.3→0.4 提升整包终审功效，WP2） +``` + +- [ ] **Step 2: 提交** + +```bash +git add config/video_split.yaml +git commit -m "chore: bump video_split val_ratio 0.3->0.4 for terminal-eval power" +``` + +--- + +## Task 2: extract_run_db 每题去重（902→900 canonical） + +**Files:** +- Modify: `app/harness/store.py:269-307` +- Test: `tests/unit/test_harness_store.py`（`TestExtractRunDb`） + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_store.py` 的 `TestExtractRunDb` 类追加： +```python + def test_dedupe_per_question_keeps_first_row(self, tmp_path): + """dedupe_per_question=True 时每 question_id 只保留 rowid 最小的首行。""" + import sqlite3 + + src = tmp_path / "src.db" + conn = sqlite3.connect(src) + conn.execute( + "CREATE TABLE _runs (run_id TEXT PRIMARY KEY, started_at TEXT)" + ) + conn.execute("INSERT INTO _runs VALUES ('r1', 't0')") + conn.execute( + "CREATE TABLE predictions (run_id TEXT, question_id TEXT, prediction TEXT)" + ) + # 743-1 三行（模拟 error/budget/finished），首行 prediction=NULL + conn.executemany( + "INSERT INTO predictions VALUES (?,?,?)", + [ + ("r1", "743-1", None), + ("r1", "743-1", None), + ("r1", "743-1", "C"), + ("r1", "q2", "A"), + ], + ) + conn.commit() + conn.close() + + dst = tmp_path / "dst.db" + from app.harness.store import extract_run_db + + extract_run_db(src, dst, "r1", dedupe_per_question=True) + + out = sqlite3.connect(dst) + rows = out.execute( + "SELECT question_id, prediction FROM predictions ORDER BY question_id" + ).fetchall() + out.close() + assert rows == [("743-1", None), ("q2", "A")], f"未按 rowid 首行去重: {rows}" +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py::TestExtractRunDb::test_dedupe_per_question_keeps_first_row -v` +Expected: FAIL（`extract_run_db() got an unexpected keyword argument 'dedupe_per_question'`）。 + +- [ ] **Step 3: 实现去重** + +`app/harness/store.py` `extract_run_db` 签名改为： +```python +def extract_run_db( + src_db: Path, dst_db: Path, run_id: str, *, dedupe_per_question: bool = False +) -> None: +``` +docstring 补一句参数说明： +``` + dedupe_per_question: True 时 predictions 表每 question_id 仅保留 rowid 最小 + 的首行（对齐 canonical「每 question_id 取第一行 ORDER BY rowid」口径， + 902→900）。_runs 表不受影响。 +``` +把 predictions 分支的取行 SQL（L297-299）改为按 `dedupe_per_question` 分派： +```python + if table == "predictions" and dedupe_per_question: + rows = src.execute( + f"SELECT {col_sql} FROM {table} WHERE run_id=? " + "AND rowid IN (SELECT MIN(rowid) FROM predictions " + "WHERE run_id=? GROUP BY question_id)", + (run_id, run_id), + ).fetchall() + else: + rows = src.execute( + f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,) + ).fetchall() +``` + +- [ ] **Step 4: 运行确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py -q` +Expected: 全 PASS（默认 `dedupe_per_question=False` 保持既有行为，旧测试不受影响）。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/store.py tests/unit/test_harness_store.py +git commit -m "feat: add dedupe_per_question to extract_run_db (canonical 902->900)" +``` + +--- + +## Task 3: tier 感知 + val 功效修复的 diag/val 分配 + +**Files:** +- Modify: `app/harness/pools.py:126-199,289-338` +- Modify: `app/harness/build_split.py:46-71,181-192` +- Modify: `app/harness/video_split_cli.py:565-573` +- Test: `tests/unit/test_pools_video_atomic.py` + +- [ ] **Step 1: 写失败测试（tier 优先 + 功效修复）** + +在 `tests/unit/test_pools_video_atomic.py` 追加： +```python +def test_tier_aware_keeps_high_t2_in_diag(): + """错题视频组按 T2 含量升序进 val：T2 高的组保留在 diagnosis。""" + from app.harness.pools import split_by_video_assignment + from app.question_gen.types import GeneratedQuestion + + def _q(qid, vid): + return GeneratedQuestion( + question_id=qid, video_id=vid, task_type="X", question="q", + options=["A", "B"], answer="A", source_nodes=[], difficulty="easy", + ) + + # 4 个错题视频（每视频 1 题），T2 数分别 2/1/0/0 + questions = [_q(f"{v}-1", v) for v in ("vA", "vB", "vC", "vD")] + assignment = {v: "trainval" for v in ("vA", "vB", "vC", "vD")} + correctness = {f"{v}-1": False for v in ("vA", "vB", "vC", "vD")} + wrong_tier = {"vA": 2, "vB": 1, "vC": 0, "vD": 0} + + pools = split_by_video_assignment( + questions, assignment, correctness, val_ratio=0.5, seed=7, + wrong_tier_by_video=wrong_tier, + ) + diag_vids = {q.video_id for q in pools.diagnosis} + # T2 最高的 vA 必留 diag；T2=0 的组优先进 val + assert "vA" in diag_vids + assert "vB" in diag_vids + + +def test_val_wrong_min_repair_pulls_from_diag(): + """val 错题不足 val_wrong_min 时从 diag 换入低 T2 错题组补足。""" + from app.harness.pools import split_by_video_assignment + from app.question_gen.types import GeneratedQuestion + + def _q(qid, vid, correct): + return GeneratedQuestion( + question_id=qid, video_id=vid, task_type="X", question="q", + options=["A", "B"], answer="A", source_nodes=[], difficulty="easy", + ) + + # 8 错题视频 + 2 正确视频；val_ratio 小使初分 val 错题不足，触发修复 + vids_wrong = [f"w{i}" for i in range(8)] + vids_correct = ["c0", "c1"] + questions = [_q(f"{v}-1", v, False) for v in vids_wrong] + [ + _q(f"{v}-1", v, True) for v in vids_correct + ] + assignment = {v: "trainval" for v in vids_wrong + vids_correct} + correctness = {f"{v}-1": False for v in vids_wrong} + correctness.update({f"{v}-1": True for v in vids_correct}) + wrong_tier = {v: i for i, v in enumerate(vids_wrong)} # 递增 T2 + + pools = split_by_video_assignment( + questions, assignment, correctness, val_ratio=0.1, seed=7, + wrong_tier_by_video=wrong_tier, val_wrong_min=4, + ) + val_wrong = sum(1 for q in pools.validation if not correctness[q.question_id]) + assert val_wrong >= 4, f"功效修复后 val 错题 {val_wrong} < 4" +``` + +> 注：`GeneratedQuestion` 的真实字段以 `app/question_gen/types.py` 为准；若构造签名不符，读该文件对齐必填字段（勿臆造）。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_pools_video_atomic.py -k "tier_aware or val_wrong_min_repair" -v` +Expected: FAIL（`unexpected keyword argument 'wrong_tier_by_video'`）。 + +- [ ] **Step 3: 改 `_split_trainval_by_video_group` 加 tier 感知 + 功效修复** + +`app/harness/pools.py` 函数签名改为： +```python +def _split_trainval_by_video_group( + trainval_qs: list[GeneratedQuestion], + correctness: dict[str, bool], + val_ratio: float, + rng: random.Random, + wrong_tier_by_video: dict[str, int] | None = None, + val_wrong_min: int = 0, +) -> tuple[list[GeneratedQuestion], list[GeneratedQuestion]]: +``` +把分层块（L329-334 的 else 分支）改为 tier 感知：`wrong_vids` 按 T2 含量升序（T2 少的优先进 val），保留 T2 高的组在 diag；`wrong_tier_by_video=None` 时退化为原 shuffle： +```python + else: + val_correct = math.floor(n_correct * n_val / n_total) + val_wrong = n_val - val_correct + rng.shuffle(correct_vids) + if wrong_tier_by_video is None: + rng.shuffle(wrong_vids) + else: + # T2 少的错题组优先进 val（保留 T2 高的组在 diag），确定性排序 + wrong_vids.sort(key=lambda v: (wrong_tier_by_video.get(v, 0), v)) + val_vids = set(correct_vids[:val_correct] + wrong_vids[:val_wrong]) +``` +在 `val_vids` 确定后、返回前，加**功效修复**（从 diag 的错题组按 T2 升序补入 val 直到满足 val_wrong_min）： +```python + if val_wrong_min > 0: + val_wrong_now = sum( + 1 for v in val_vids for q in groups[v] if not correctness[q.question_id] + ) + # diag 侧仍在的错题组，按 T2 升序（低价值优先移交 val） + diag_wrong_pool = sorted( + (v for v in wrong_vids if v not in val_vids), + key=lambda v: ((wrong_tier_by_video or {}).get(v, 0), v), + ) + for v in diag_wrong_pool: + if val_wrong_now >= val_wrong_min: + break + val_vids.add(v) + val_wrong_now += sum(1 for q in groups[v] if not correctness[q.question_id]) + if val_wrong_now < val_wrong_min: + raise InsufficientValSignal( + f"trainval 错题不足以让 val 达到 val_wrong_min={val_wrong_min}" + f"（修复后仅 {val_wrong_now}），请放大 val_ratio 或调整 trainval 归属。" + ) +``` +（`InsufficientValSignal` 已在 pools.py:118 定义，无需新增；需确认函数内可见 `math`/`defaultdict`，文件顶部已 import。） + +- [ ] **Step 4: `split_by_video_assignment` 透传新参数** + +`app/harness/pools.py` `split_by_video_assignment` 签名加 `wrong_tier_by_video: dict[str, int] | None = None`（放在 `val_wrong_min` 之后），并把 `_split_trainval_by_video_group` 调用（L175-177）改为： +```python + diagnosis, validation = _split_trainval_by_video_group( + trainval_qs, correctness, val_ratio, random.Random(seed), + wrong_tier_by_video=wrong_tier_by_video, + val_wrong_min=val_wrong_min, + ) +``` +删除原 L179-186 的独立 `val_wrong_min` 事后校验块（功效已在 `_split_trainval_by_video_group` 内保证，避免重复校验语义）。docstring 的 `val_wrong_min` 说明改为"切分时保证（不足则从 diag 换入低 T2 错题组补足，耗尽 fail-loud）"。 + +- [ ] **Step 5: build_split 计算并传入 tier + val_wrong_min** + +`app/harness/build_split.py`：`SplitBuildConfig` 加字段 `val_wrong_min: int`（放 `split_seed` 之后，docstring 补"validation 池最少错题数，切分时保证功效"）。build_split Phase 3（L182-191）改为： +```python + questions = load_benchmark(questions_dir) + correctness = {pred["question_id"]: pred["correct"] for pred in preds} + tier_by_q = {row["question_id"]: row["tier"] for row in signal_rows} + wrong_tier_by_video: dict[str, int] = defaultdict(int) + for pred in preds: + if not pred["correct"] and tier_by_q.get(pred["question_id"]) == "T2": + wrong_tier_by_video[pred["video_id"]] += 1 + pools = split_by_video_assignment( + questions, + assignment, + correctness, + config.val_ratio, + config.split_seed, + baseline_run_id=baseline_run_id, + val_wrong_min=config.val_wrong_min, + wrong_tier_by_video=dict(wrong_tier_by_video), + ) +``` +更新 build_split docstring 的"契约（Task 11...）"段：删除"有意保持 val_wrong_min-agnostic"表述，改为"val_wrong_min 前置到切分内保证功效；CLI 的 check_mcnemar_power 作冗余最终确认"。确认 `defaultdict` 已 import（`from collections import Counter, defaultdict`）。 + +- [ ] **Step 6: CLI 传 val_wrong_min + 保真检查** + +`app/harness/video_split_cli.py` 的 `SplitBuildConfig(...)`（L565-573）加一行 `val_wrong_min=config.val_wrong_min,`。 +保真检查点：确认 `pools.diagnosis`/`pools.validation` 仍是逐题 `GeneratedQuestion` 列表、同 video 全部题同池（`test_pools_video_atomic.py::test_video_group_atomic_in_trainval_split` 覆盖）。 + +- [ ] **Step 7: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_pools_video_atomic.py tests/unit/test_harness_pools.py tests/unit/test_split_selection.py -q` +Expected: 全 PASS。 + +- [ ] **Step 8: 提交** + +```bash +git add app/harness/pools.py app/harness/build_split.py app/harness/video_split_cli.py tests/unit/test_pools_video_atomic.py +git commit -m "feat: tier-aware diag/val split with val-power repair (design 5.1)" +``` + +--- + +## Task 4: 冻结产物覆盖保护 + --force + +**Files:** +- Modify: `app/harness/build_split.py:104-192` +- Modify: `app/harness/video_split_cli.py:751-773`（build_arg_parser）+ run_pipeline 传参 +- Test: `tests/unit/test_video_split_cli.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_video_split_cli.py` 追加（用最小 build_split 覆盖场景，或直接测保护函数）： +```python +def test_build_split_refuses_overwrite_without_force(tmp_path): + """已存在指纹不同的 pools.json 时，force=False 必须报错不覆盖。""" + from app.harness.build_split import _guard_frozen_products + + out_path = tmp_path / "pools.json" + out_path.write_text('{"split_mode":"global"}', encoding="utf-8") + manifest_path = tmp_path / "split_manifest.json" + + with pytest.raises(FileExistsError, match="已存在冻结产物"): + _guard_frozen_products(out_path, manifest_path, force=False) + + +def test_build_split_force_backs_up_old(tmp_path): + """force=True 时旧产物被备份为 .bak.* 再允许覆盖。""" + from app.harness.build_split import _guard_frozen_products + + out_path = tmp_path / "pools.json" + out_path.write_text('{"old":1}', encoding="utf-8") + manifest_path = tmp_path / "split_manifest.json" + manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8") + + _guard_frozen_products(out_path, manifest_path, force=True) + baks = list(tmp_path.glob("pools.json.bak.*")) + assert len(baks) == 1, f"未备份旧产物: {list(tmp_path.iterdir())}" +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -k "refuses_overwrite or force_backs_up" -v` +Expected: FAIL（`cannot import name '_guard_frozen_products'`）。 + +- [ ] **Step 3: 实现覆盖保护函数** + +`app/harness/build_split.py` 顶部 import 区确认有 `import shutil`（无则加）。新增函数（放 build_split 之前）： +```python +def _guard_frozen_products(out_path: Path, manifest_path: Path, *, force: bool) -> None: + """冻结前的覆盖保护：产物已存在时按 force 决定报错或备份。 + + 参数: + out_path: 目标 pools.json 路径。 + manifest_path: 目标 split_manifest.json 路径。 + force: False 时已存在即 FileExistsError；True 时把旧产物重命名为 + .bak.<旧 pools_sha256 前 8 位或 timestamp-less 序号> 再放行。 + + 异常: + FileExistsError: force=False 且产物已存在（防静默覆盖冻结锚点）。 + """ + if not out_path.exists() and not manifest_path.exists(): + return + if not force: + raise FileExistsError( + f"已存在冻结产物 {out_path}（或其 manifest）。重跑切分会覆盖训练依赖的" + "冻结锚点——确认要替换请加 --force（旧产物将备份为 .bak.*）。" + ) + # 备份后缀取旧 manifest 的 pools_sha256 前 8 位，无则用 'prev' + suffix = "prev" + if manifest_path.exists(): + try: + old = json.loads(manifest_path.read_text(encoding="utf-8")) + suffix = str(old.get("pools_sha256", "prev"))[:8] or "prev" + except (json.JSONDecodeError, OSError): + suffix = "prev" + for p in (out_path, manifest_path): + if p.exists(): + p.rename(p.with_name(f"{p.name}.bak.{suffix}")) +``` +确认 build_split.py 已 import `json`（无则加 `import json`）。在 `build_split` 签名加参数 `force: bool = False`（放 `generated_at` 之后），并在 Phase 3 `save_pools` 之前（L192 前）调用 `_guard_frozen_products(out_path, manifest_path, force=force)`。 + +- [ ] **Step 4: CLI 暴露 --force 并透传** + +`app/harness/video_split_cli.py` `build_arg_parser`（L751-773）追加： +```python + parser.add_argument( + "--force", + action="store_true", + help="覆盖已存在的冻结 pools.json/manifest（旧产物备份为 .bak.*）", + ) +``` +`run_pipeline` 签名加 `force: bool = False` 参数，build_split 调用（L559-577）加 `force=force,`；`main()` 里把 `args.force` 透传给 `run_pipeline`。 + +- [ ] **Step 5: 运行测试确认通过 + CLI 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -q` +Expected: 全 PASS。 + +- [ ] **Step 6: 提交** + +```bash +git add app/harness/build_split.py app/harness/video_split_cli.py tests/unit/test_video_split_cli.py +git commit -m "feat: guard frozen split products against silent overwrite (--force)" +``` + +--- + +## Task 5: seed 携带 pools.json + 训练拷入 + +**Files:** +- Modify: `app/harness/store.py:184-232`（init_seed） +- Modify: `app/harness/workspace.py:156-200`（init_workspace_from_seed） +- Test: `tests/unit/test_harness_store.py`、`tests/unit/test_harness_workspace.py` + +- [ ] **Step 1: 写失败测试（seed 携带）** + +在 `tests/unit/test_harness_store.py::TestInitSeed` 追加： +```python + def test_init_seed_carries_pools(self, tmp_path): + """提供 pools_json/split_manifest 时拷入 seed 目录。""" + from app.harness.store import init_seed + + store = tmp_path / "store" + skills = tmp_path / "sk"; skills.mkdir(); (skills / "s.md").write_text("x") + prompts = tmp_path / "pr"; prompts.mkdir(); (prompts / "p.md").write_text("y") + db = tmp_path / "b.db"; db.write_text("db") + pools = tmp_path / "pools.json"; pools.write_text('{"split_mode":"global"}') + manifest = tmp_path / "split_manifest.json"; manifest.write_text('{"pools_sha256":"a"}') + + seed_dir = init_seed( + store, "s1", skills, prompts, db, "infer_adhoc", None, "d", + pools_json=pools, split_manifest=manifest, + ) + assert (seed_dir / "pools.json").exists() + assert (seed_dir / "split_manifest.json").exists() +``` + +在 `tests/unit/test_harness_workspace.py` 追加： +```python +def test_init_workspace_from_seed_carries_pools(store_dir, workspace_dir): + """seed 目录含 pools.json 时拷入 workspace。""" + import shutil + from app.harness.store import init_seed + from app.harness.workspace import init_workspace_from_seed + # 复用现有 fixture 构造 seed 的方式；此处补 pools.json 到 seed 后初始化 workspace + # （具体 fixture 依 test_harness_workspace.py 现有 helper，读文件对齐） + ... +``` +> 该 workspace 测试需依 `test_harness_workspace.py` 现有 fixture（`store_dir`/`workspace_dir` 及既有 seed 构造 helper）填充；实现前读该文件 `test_init_workspace_from_seed`（L151）复用其 seed 搭建，再在 seed 目录写 `pools.json` 后断言 workspace 内出现 `pools.json`。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py::TestInitSeed::test_init_seed_carries_pools -v` +Expected: FAIL（`unexpected keyword argument 'pools_json'`）。 + +- [ ] **Step 3: init_seed 加可选携带参数** + +`app/harness/store.py` `init_seed` 签名加： +```python +def init_seed( + store_dir: Path, + name: str, + skills_dir: Path, + prompts_dir: Path, + baseline_db: Path, + baseline_run_id: str, + parent: str | None, + description: str, + *, + pools_json: Path | None = None, + split_manifest: Path | None = None, +) -> Path: +``` +在 `copy2(baseline_db, ...)`（L218）之后加： +```python + if pools_json is not None: + shutil.copy2(pools_json, seed_dir / "pools.json") + if split_manifest is not None: + shutil.copy2(split_manifest, seed_dir / "split_manifest.json") +``` +docstring 补两参说明。 + +- [ ] **Step 4: init_workspace_from_seed 拷入 pools** + +`app/harness/workspace.py` `init_workspace_from_seed` 在 `shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db")`（L197）之后加： +```python + seed_pools = seed_dir / "pools.json" + if seed_pools.exists(): + shutil.copy2(seed_pools, workspace_dir / "pools.json") + seed_manifest = seed_dir / "split_manifest.json" + if seed_manifest.exists(): + shutil.copy2(seed_manifest, workspace_dir / "split_manifest.json") +``` + +- [ ] **Step 5: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py -q` +Expected: 全 PASS。 + +- [ ] **Step 6: 提交** + +```bash +git add app/harness/store.py app/harness/workspace.py tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py +git commit -m "feat: seed carries frozen pools.json into training workspace" +``` + +--- + +## Task 6: build_or_load_pools global 一致性校验 + +**Files:** +- Modify: `app/harness/pools.py:746-813` +- Test: `tests/unit/test_harness_pools.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_pools.py::TestBuildOrLoadPoolsFrozen` 追加： +```python + def test_global_frozen_rejects_baseline_mismatch(self, tmp_path, ...): + """global 冻结 pools 的 baseline_run_id 与 seed 不符时 fail-loud。""" + # 依现有 fixture 造 workspace + 冻结 pools.json（split_mode=global, + # baseline_run_id="other"），seed.json baseline_run_id="infer_adhoc" + # 调 build_or_load_pools 应 raise ValueError(match="baseline_run_id") + ... +``` +> 依 `TestBuildOrLoadPoolsFrozen`（L246）现有 fixture 复用其 workspace/seed 搭建；实现前读该类对齐 RunConfig/strategy 构造，勿臆造。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_pools.py::TestBuildOrLoadPoolsFrozen -v` +Expected: 新用例 FAIL（当前 global 分支无校验，误加载不报错）。 + +- [ ] **Step 3: 加 global 一致性校验** + +`app/harness/pools.py` `build_or_load_pools`，在 global 加载分支（`if pools_path.exists():` 块内、`per_category` 校验的 `else` 侧，即 L813 `return load_pools(pools_path)` 之前）加： +```python + else: # global：校验 baseline_run_id 与（若有）manifest 内容指纹 + frozen_baseline = raw.get("baseline_run_id") + if frozen_baseline != baseline_run_id: + raise ValueError( + f"冻结 pools.json 的 baseline_run_id={frozen_baseline!r} 与 seed " + f"的 {baseline_run_id!r} 不一致，拒绝静默加载错配切分。" + ) + manifest_path = config.workspace_dir / "split_manifest.json" + if manifest_path.exists(): + import hashlib + + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + actual_sha = hashlib.sha256( + pools_path.read_text(encoding="utf-8").encode("utf-8") + ).hexdigest() + if manifest.get("pools_sha256") != actual_sha: + raise ValueError( + "pools.json 内容指纹与 split_manifest.pools_sha256 不符，" + "冻结产物疑被篡改，拒绝加载。" + ) +``` +（确认该 `else` 与 L751 `if frozen_split_mode == "per_category":` 配对；若现有结构非 if/else 而是 if 后直接 return，则把校验插在 `return load_pools(pools_path)` 前并用 `if frozen_split_mode != "per_category":` 守卫。实现前读 L746-813 对齐控制流。） + +- [ ] **Step 4: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_pools.py -q` +Expected: 全 PASS。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/pools.py tests/unit/test_harness_pools.py +git commit -m "fix: validate global frozen pools baseline_run_id + sha256 on load" +``` + +--- + +## Self-Review（作者自查，执行者复核） + +- [ ] val_ratio=0.4 已改；tier 感知 + 功效修复在同一函数、退化路径（`wrong_tier_by_video=None`）保持旧行为。 +- [ ] val_wrong_min 从 CLI→SplitBuildConfig→build_split→split_by_video_assignment→_split_trainval_by_video_group 全链路贯通；旧的 pools.py 事后校验块已删（不重复）。 +- [ ] extract_run_db 去重默认关闭，不破坏既有调用。 +- [ ] seed 携带 + workspace 拷入 + global 一致性校验三者闭环：冻结产物有唯一路径进训练且被校验。 +- [ ] 覆盖保护默认 force=False，离线 CLI 重跑需显式 --force。 + +## 核心算法保真校验结论 + +本计划触及算法 #5 的上游输入（哪些视频进 diag/val），**不改** gate_ladder 的 unit+correctness 消费结构；Task 3 Step 6 已设保真检查点确认逐 unit 列表与视频组原子性。不涉及算法 #4/#6/#8/#9 逻辑。 + +## 验收标准 + +1. `pytest tests/unit/test_pools_video_atomic.py tests/unit/test_harness_pools.py tests/unit/test_harness_store.py tests/unit/test_harness_workspace.py tests/unit/test_split_selection.py tests/unit/test_video_split_cli.py` 全绿。 +2. tier 感知：T2 高的错题视频组留 diag，T2 低的优先进 val。 +3. seed 携带 pools.json → init_workspace_from_seed 拷入 → build_or_load_pools 校验 baseline_run_id + sha256。 +4. 冻结产物 force=False 时拒绝覆盖。 diff --git a/research-wiki/plans/2026-07-16-preflight-wp3-train-loop.md b/research-wiki/plans/2026-07-16-preflight-wp3-train-loop.md new file mode 100644 index 0000000..7858228 --- /dev/null +++ b/research-wiki/plans/2026-07-16-preflight-wp3-train-loop.md @@ -0,0 +1,601 @@ +# WP3 训练循环与进化引擎 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +> **前置依赖：WP1（模板已迁移，进化引擎可运行）+ WP4（cache_salt 能力已就绪）必须先完成。** + +**Goal:** 修复训练循环与进化引擎的 9 处缺陷，使诊断拿到真实轨迹、早停按 epoch 语义、微型题型不崩、崩溃可幂等续跑、进化 patch 不破坏冻结区、降级信号不驱动错误进化、跨 epoch 评估真实重采样。 + +**Architecture:** 诊断经 `StepsJsonRunLog` 从 steps_json 重建轨迹（算法 #7 恢复）；早停计数单位 step→epoch；可训练性预检在 gate 建立前剔除微型题型；`_run_step` 幂等（先 DELETE 再写）；patch 冻结区检查整个 target 跨度（算法 #8 加固）；降级/未判定题按 lapse 保守分流；训练推理用 run_id 作 cache_salt（run_id 已含 epoch，天然跨 epoch 重采样）。 + +**Tech Stack:** Python 3.11、asyncio、SQLite、pytest。 + +**设计源**：`research-wiki/designs/2026-07-16-preflight-fixes-design.md §6-7` + +--- + +## 关键锚点（实现前必读） + +| 用途 | 位置 | +|------|------| +| 训练主流程 | `app/harness/runner.py:789-859` `train`；`_setup_train_run:865-879`（预检插入点 L874 前）；`_run_step:1001-1026`（run_id L1011、诊断 L1019、无 DELETE） | +| 早停 | `app/harness/runner.py:291-316` `_should_early_stop`（L315 `+= steps_this_epoch`）；`_TrainState:95-121`（`steps_since_best_improved` L118）；`_maybe_promote_best:1587`（置 0） | +| 诊断调用 | `app/harness/runner.py:2163-2196` `_run_diagnosis`（`RunLogImpl` L2173 未包 StepsJsonRunLog）；DiagnosisResult `degraded_count` 未被引用 | +| gate 刷新 | `app/harness/runner.py:1833-1879` `_refresh_gate_ladder`（save L1878 → set observed L1879）；checkpoint 落盘晚在 train L837 | +| holdout | `app/harness/runner.py:1881-1921` `_holdout_four_way`；`_pick_mixed_best:1923-1963`；`_eval_version_on_pool:2148-2161` | +| 推理落库 | `app/harness/inference.py:363-447` `_run_single_question`（prediction L422 未归一、insert L446 try 外）；`_to_text_field:147-162`；`run_inference:473` | +| Agent Loop | `core/agent/loop.py:103` `run`（session_id L110）；`_call_llm` chat 调用 `:329`（`self._llm.chat(messages, session_id=session_id)`） | +| batching | `app/harness/batching.py:186-202` `_classify_unit`（L200 缺 correctness→None） | +| checkpoint | `app/harness/checkpoint.py:37-44` `_STRUCTURAL_KEYS`；`serialize_state:76-106`；`write_checkpoint:212-260`（原子写） | +| 诊断分流 | `core/evolution/diagnose.py:1485-1519` `_build_skill_case_packs`（lapse 分流 L1492）；`_process_question:2139-2206`（except L2194 cause_category 留 None） | +| traces 适配 | `app/harness/baseline_run_log.py:13` `StepsJsonRunLog`；`steps_json_traces.py:13` | +| patch | `core/evolution/patch.py:285-287` `_in_ranges`；`_do_insert_after:309-326`（L320）；`_do_replace_delete:329-347`（L343）；markers L11-18；`validate_skill` in `evolve.py:296` | + +## 核心算法保真校验 + +触及算法 #7（诊断瀑布，Task 1 恢复轨迹）、#8（patch 引擎，Task 5 冻结区加固）、#10（Agent Loop，Task 9 透传 cache_salt）、#5/#12（信息阶梯/训练编排，Task 7 checkpoint 时序）。**均为恢复/加固/透传，不改算法逻辑**：Task 1 让诊断拿到本就该有的轨迹；Task 5 把"只查起点"补成"查整跨度"（保护方向不变）；Task 9 只加透传参数；Task 7 只调 checkpoint 落盘时机。每个相关 Task 设保真检查点。 + +--- + +## Task 1: traces 适配（诊断拿到真实轨迹，算法 #7） + +**Files:** +- Modify: `app/harness/runner.py:2163-2196`（_run_diagnosis） +- Test: `tests/unit/test_runner_diag_tree_inject.py` 或 `test_harness_runner.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_runner_diag_tree_inject.py` 追加（构造只写 steps_json 不写 traces 表的 run，断言诊断能拿到轨迹）： +```python +@pytest.mark.asyncio +async def test_diagnosis_reads_traces_from_steps_json(...): + """traces 表为空但 predictions.steps_json 有轨迹时，诊断仍拿到非空 traces。""" + # 依现有 runner 测试 fixture 造一个 run：predictions 有 steps_json，traces 表空； + # 调 _run_diagnosis 后断言 diagnose 收到的 traces 非空（可 patch run_diagnosis 捕获入参） + ... +``` +> 依 `test_runner_diag_tree_inject.py` 现有 fixture；实现前读对齐 runner 构造与 patch 点。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_runner_diag_tree_inject.py -k reads_traces_from_steps_json -v` +Expected: FAIL（当前 RunLogImpl 直读空 traces 表）。 + +- [ ] **Step 3: 包 StepsJsonRunLog** + +`app/harness/runner.py` `_run_diagnosis`，把传给 `run_diagnosis` 的 `run_log`（当前 `RunLogImpl(...)`，L2173 附近）包一层： +```python + from app.harness.baseline_run_log import StepsJsonRunLog + from app.harness.log import RunLogImpl + + run_log = StepsJsonRunLog(RunLogImpl(str(self._paths.db_path))) +``` +（`StepsJsonRunLog.get_traces` 在底层 traces 空时从 predictions.steps_json 经 `steps_json_to_trace_rows` 重建；`get_predictions` 透传。） + +- [ ] **Step 4: 保真检查点 + 测试通过** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_runner_diag_tree_inject.py tests/unit/test_baseline_run_log.py -q` +Expected: 全 PASS。保真：确认诊断瀑布拿到的是逐 step `{tool_name,tool_args,tool_output,thought}` 行（对齐 TRM4 诊断输入）。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/runner.py tests/unit/test_runner_diag_tree_inject.py +git commit -m "fix: wrap diagnosis run_log with StepsJsonRunLog (restore algo #7 traces)" +``` + +--- + +## Task 2: prediction 归一化 + 落库加固 + +**Files:** +- Modify: `app/harness/inference.py:417-447` +- Test: `tests/unit/test_harness_inference.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_inference.py` 追加（LLM 提交非标量 answer 不崩 gather）： +```python +@pytest.mark.asyncio +async def test_nonscalar_prediction_does_not_crash(...): + """submit_answer 返回 {'answer': ['B']} 等非标量时归一化落库，不抛 sqlite 绑定异常。""" + # 依现有 inference 测试 fixture，让 AgentLoop 返回 result={'answer': ['B']}； + # run_inference 应正常完成、predictions 行 prediction 为字符串（如 '["B"]'），不崩 + ... +``` +> 依 `test_harness_inference.py` 现有 fake loop/dispatch fixture；实现前读对齐。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py -k nonscalar_prediction -v` +Expected: FAIL（sqlite `InterfaceError: Error binding parameter`）。 + +- [ ] **Step 3: 归一化 prediction + insert 加固** + +`app/harness/inference.py`：新增归一化 helper（None 保留、str 原样、其余 `_to_text_field`）： +```python +def _normalize_prediction(answer: object) -> str | None: + """归一化 prediction：None 保留（INFRA 空预测语义），str 原样，其余 JSON 序列化。""" + if answer is None or isinstance(answer, str): + return answer + return _to_text_field(answer) +``` +L422 `"prediction": result_dict.get("answer"),` 改为 `"prediction": _normalize_prediction(result_dict.get("answer")),`。 +L446 的 `await asyncio.to_thread(log.insert, "predictions", record)` 包 try，绑定异常降级为最小 error 行不击穿 gather： +```python + try: + await asyncio.to_thread(log.insert, "predictions", record) + except (sqlite3.InterfaceError, sqlite3.ProgrammingError): + logger.exception("[{}] QA {} 落库绑定异常，降级为 error 行", qa.video_id, qa.question_id) + record["prediction"] = None + record["stop_reason"] = "error" + await asyncio.to_thread( + log.insert, "predictions", + {k: v for k, v in record.items() if isinstance(v, (str, int, float, type(None)))}, + ) +``` +确认 `import sqlite3` 在文件顶部（无则加）。 + +- [ ] **Step 4: 测试通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py -q` +Expected: 全 PASS。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/inference.py tests/unit/test_harness_inference.py +git commit -m "fix: normalize non-scalar prediction; harden predictions insert" +``` + +--- + +## Task 3: early_stop 改 epoch 计数 + +**Files:** +- Modify: `app/harness/runner.py:291-316,118,1587,849-857` +- Modify: `app/harness/checkpoint.py`（若字段入 state） +- Test: `tests/unit/test_harness_runner.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_runner.py` 追加： +```python +def test_early_stop_counts_epochs_not_steps(tmp_path): + """patience=2 表示连续 2 个 epoch 无 best 刷新才停（不是步数）。""" + from app.harness.runner import _should_early_stop, _TrainState + # 造 state + workspace，best 停在 epoch 1； + # epoch 2 无刷新 → epochs_since_best_improved=1 → 不停； + # epoch 3 无刷新 → =2 → 停 + ... +``` +> 依现有 `_TrainState`/`read_best` fixture；实现前读对齐 workspace best 写入。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k early_stop_counts_epochs -v` +Expected: FAIL（当前累加 steps_this_epoch）。 + +- [ ] **Step 3: 字段改名 + 计数改 epoch** + +全局把 `steps_since_best_improved` 改名 `epochs_since_best_improved`（`grep -rn steps_since_best_improved app/`：`_TrainState:118`、`_should_early_stop:313,315`、`_maybe_promote_best:1587`，以及 checkpoint serialize/deserialize 若含此字段）。 +`_should_early_stop`（L315）`state.steps_since_best_improved += steps_this_epoch` 改为 `state.epochs_since_best_improved += 1`；签名删除 `steps_this_epoch` 参数（改为 `_should_early_stop(workspace_dir, epoch, state, patience)`），train 调用点（L849-857）同步去掉 `len(batches)` 实参。docstring 改为"epoch 粒度"。 + +- [ ] **Step 4: checkpoint 兼容** + +若 `epochs_since_best_improved` 入 checkpoint state（`grep -n steps_since_best_improved app/harness/checkpoint.py`），同步改名。本轮为 fresh 训练无旧 checkpoint，无迁移负担。 + +- [ ] **Step 5: 测试通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py tests/unit/test_harness_checkpoint.py -q` +Expected: 全 PASS。 + +- [ ] **Step 6: 提交** + +```bash +git add app/harness/runner.py app/harness/checkpoint.py tests/unit/test_harness_runner.py +git commit -m "fix: early_stop patience counts epochs not steps" +``` + +--- + +## Task 4: 可训练性预检 + +**Files:** +- Modify: `app/harness/runner.py:865-879`（_setup_train_run） +- Modify: `app/harness/config.py`（RunConfig 加 `trainable_min_units`）+ `config/train_videomme.yaml`（WP 收尾建，先加字段默认） +- Test: `tests/unit/test_harness_runner.py` + +- [ ] **Step 1: 写失败测试** + +```python +def test_untrainable_types_filtered_before_gate(): + """val tuple[Pools, list[str] | None]: + """剔除不可训练题型（val marker 常量名以 `core/evolution/patch.py:11-18` 为准（`APPENDIX_START` 等），实现前读对齐 import 名。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_patch.py -k "spanning_into_protected or marker_literal" -v` +Expected: FAIL（当前只查起点 pos，跨入未拦；无注入检查）。 + +- [ ] **Step 3: 跨度检查 + 注入检查** + +`core/evolution/patch.py` 新增跨度 helper： +```python +def _span_overlaps_ranges(pos: int, length: int, ranges: list[tuple[int, int]]) -> bool: + """判断 [pos, pos+length) 是否与任一冻结区间相交（不止起点）。""" + end = pos + length + return any(start < end and pos < r_end for start, r_end in ranges) +``` +`_do_insert_after` L320 `if _in_ranges(pos, ranges):` 改为 `if _span_overlaps_ranges(pos, len(target), ranges):`。 +`_do_replace_delete` L343 `if _in_ranges(pos, ranges):` 改为 `if _span_overlaps_ranges(pos, len(target), ranges):`。 +在 `apply_patch_with_report`（L387）应用每个 edit 前加注入检查：payload/target 含任一 marker 字面量（`APPENDIX_START/END`、`MOMENTUM_START/END`）→ 跳过该 edit 并记 `skipped_marker_injection`。 + +- [ ] **Step 4: validate_skill 加 marker 完整性校验** + +`core/evolution/evolve.py:296` `validate_skill`：在现有 frontmatter/长度/代码块校验后加——统计 evolved 中 `APPENDIX_START/END`、`MOMENTUM_START/END` 出现次数，要求成对（START 数==END 数）、各至多一对、START 在 END 前；违反则返回校验失败（该候选整体 reject）。 + +- [ ] **Step 5: 保真检查点 + 测试通过** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_patch.py tests/unit/test_evolve.py -q` +Expected: 全 PASS。保真：确认"保护跨度"方向未变（仍是保护 appendix/momentum 不被误改），只是从"查起点"补成"查整跨度"+ 注入/完整性双防线。 + +- [ ] **Step 6: 提交** + +```bash +git add core/evolution/patch.py core/evolution/evolve.py tests/unit/test_patch.py +git commit -m "fix: patch checks full target span + marker injection/integrity (algo #8)" +``` + +--- + +## Task 6: 诊断降级分流 + 占比中止 + +**Files:** +- Modify: `core/evolution/diagnose.py:1485-1497` +- Modify: `app/harness/runner.py:1019`（诊断后 degraded 占比检查） +- Test: `tests/unit/test_diagnose.py`、`tests/unit/test_harness_runner.py` + +- [ ] **Step 1: 写失败测试（分流）** + +在 `tests/unit/test_diagnose.py` 追加： +```python +def test_none_cause_and_degraded_route_to_lapse(): + """cause_category=None（判别失败）与 degraded 题按 lapse 处置，不进 defect 正文路径。""" + from core.evolution.diagnose import _build_skill_case_packs + # 构造 metrics_group：一题 attr.cause_category=None（非 degraded）、一题 qm.degraded=True； + # 断言二者都不出现在 failure_cases（wrong_by_error），只可能进 lapse_notes + ... +``` +> 依 `test_diagnose.py` 现有 QuestionMetrics/ErrorAttribution 构造（`:753` 附近）；实现前读对齐字段。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_diagnose.py -k none_cause_and_degraded_route -v` +Expected: FAIL（当前 None → wrong_by_error 走 defect 正文）。 + +- [ ] **Step 3: 改分流逻辑** + +`core/evolution/diagnose.py` `_build_skill_case_packs` 的分流循环（L1488-1497）改为"仅明确 defect 且非 degraded 才进正文路径"： +```python + for qm in metrics_group: + if qm.correct: + continue + attr = attribution_map.get(qm.question_id) + is_defect = ( + attr is not None + and attr.cause_category == "defect" + and not qm.degraded + ) + if not is_defect: + # lapse / None（判别失败）/ degraded → 保守，不驱动正文进化 + if attr is not None and attr.lapse_note and attr.lapse_note.strip(): + lapse_notes.append(attr.lapse_note) + continue + wrong_by_error[attr.error_type].append(qm) +``` + +- [ ] **Step 4: 写失败测试（占比中止）+ 实现** + +`tests/unit/test_harness_runner.py` 追加：诊断结果 degraded_count/总题数 > 0.5 时 `_run_step` 后应 raise（疑似基础设施故障）。 +`app/harness/runner.py` `_run_step`（L1019 拿到 `diagnosis` 后）加： +```python + n_wrong = sum(1 for q in batch if not state.correctness.get(q.question_id, True)) + if n_wrong > 0 and diagnosis.degraded_count / n_wrong > 0.5: + raise RuntimeError( + f"本 step 诊断降级占比 {diagnosis.degraded_count}/{n_wrong} > 50%，" + "疑似 judge 基础设施故障，中止训练（不以降级信号驱动进化）。" + ) +``` +（确认 `DiagnosisResult.degraded_count` 字段可用，L2311。） + +- [ ] **Step 5: 测试通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_diagnose.py tests/unit/test_harness_runner.py -q` +Expected: 全 PASS。 + +- [ ] **Step 6: 提交** + +```bash +git add core/evolution/diagnose.py app/harness/runner.py tests/unit/test_diagnose.py tests/unit/test_harness_runner.py +git commit -m "fix: route None/degraded diagnoses to lapse; abort on high degrade rate" +``` + +--- + +## Task 7: step 幂等（DELETE）+ gate_epoch_observed 立即落盘 + +**Files:** +- Modify: `app/harness/runner.py:1001-1026`（_run_step 开头 DELETE） +- Modify: `app/harness/runner.py:1378-1498,1833-1879`（checkpoint 提到 gate save 之后） +- Test: `tests/unit/test_harness_runner.py` + +- [ ] **Step 1: 写失败测试（幂等）** + +```python +@pytest.mark.asyncio +async def test_run_step_deletes_stale_rows_before_rerun(...): + """同 run_id 重跑前先清 predictions/traces，避免重复行双计。""" + # 预置该 step run_id 的旧 predictions 行；调 _run_step；断言旧行被清、只剩本次 + ... +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k deletes_stale_rows -v` +Expected: FAIL（当前 append，无 DELETE）。 + +- [ ] **Step 3: _run_step 开头 DELETE** + +`app/harness/runner.py` `_run_step`，在 rollout（L1012）之前加： +```python + with HarnessLog(str(self._paths.db_path), run_id, register_run=False) as log: + log.execute("DELETE FROM predictions WHERE run_id=?", (run_id,)) + log.execute("DELETE FROM traces WHERE run_id=?", (run_id,)) +``` +（`register_run=False` 来自 WP4 Task 8；若 HarnessLog 无 `execute` 便捷方法，用其现有 `query`/连接接口，实现前读 log.py 对齐。） + +- [ ] **Step 4: checkpoint 提到 gate save 之后（消除双计窗口）** + +目标：`_refresh_gate_ladder` 内 `gate_pools.save`（L1878）+ `gate_epoch_observed=True`（L1879）之后，**立即** `write_checkpoint`（phase="epoch_done"），不等到 `train` L837。实现：把 `_slow_update_cycle` Phase 10（调 `_refresh_gate_ladder` L1496-1498）之后的 checkpoint 落盘从 `train`（L837）移入 `_slow_update_cycle` 末尾，或让 `_refresh_gate_ladder` 接收 checkpoint 所需上下文（epoch/progress/batches）并在 save 后落盘。实现前读 `write_checkpoint` 签名（checkpoint.py:212）与 `train` L833-848 对齐参数，确保 gate_pools.json 与 checkpoint 的 `gate_epoch_observed` 同一时刻一致。 + +- [ ] **Step 5: 保真检查点 + 测试通过** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py tests/unit/test_harness_checkpoint.py -q` +Expected: 全 PASS。保真（算法 #5/#12）：确认 γ-EMA 更新（`update_probs`）仍每 epoch 一次、checkpoint 落盘不改变慢更新十步序的语义顺序。 + +- [ ] **Step 6: 提交** + +```bash +git add app/harness/runner.py tests/unit/test_harness_runner.py +git commit -m "fix: idempotent _run_step (DELETE stale) + checkpoint after gate save" +``` + +--- + +## Task 8: holdout 四向去重 + +**Files:** +- Modify: `app/harness/runner.py:1881-1963`（_holdout_four_way / _pick_mixed_best） +- Test: `tests/unit/test_harness_runner.py` + +> **说明**：目标是每 epoch 的四向 test 评估从"4×600 全跑"降为"仅 final 必跑 + best_hard 未评过才跑 + baseline 从基线预测推导 + best_mixed 引用赢家"。去重做在 harness 逻辑层（配合 WP4 epoch 盐，同版本不重采样）。 + +- [ ] **Step 1: 写失败测试** + +```python +@pytest.mark.asyncio +async def test_holdout_dedup_skips_reevaluated_versions(...): + """baseline 不跑推理（从基线预测推导）；best_hard==final 时不重复评估。""" + # 统计 _eval_version_on_pool 被调次数：baseline=0，best_hard==final 时该向复用不重跑 + ... +``` +> 依现有 runner holdout fixture；实现前读 `_holdout_four_way`/`write_holdout_eval` 对齐。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k holdout_dedup -v` +Expected: FAIL（当前四向各跑一次）。 + +- [ ] **Step 3: 实现去重** + +`app/harness/runner.py` `_holdout_four_way`： +- **baseline 向**：不调 `_eval_version_on_pool`，改从基线 predictions（baseline_run_id）读 test 题对错算 acc（test 题在 infer_adhoc 已推理）；epoch>1 复用首次结果。 +- **final 向**：真评 600（不变）。 +- **best_hard 向**：维护"已评版本→test_acc"备忘录，resume 时从 `holdout_eval` 表 hydrate（查已落库 `(skills_v,prompts_v)→acc`）；版本对未评过且 ≠ final 才真评，否则引用。 +- **best_mixed 向**：引用赢家（best_hard 或 final 之一，`_pick_mixed_best` 已定）的 test 成绩，写 holdout_eval 时标指针，不推理。 + +实现前完整读 `_holdout_four_way`（1881-1921）、`write_holdout_eval`、`_eval_version_on_pool`、`read`/hydrate 路径对齐。 + +- [ ] **Step 4: 测试通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -q` +Expected: 全 PASS。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/runner.py tests/unit/test_harness_runner.py +git commit -m "perf: dedup holdout four-way eval (baseline derive, best_hard memo)" +``` + +--- + +## Task 9: 训练推理注入 epoch 盐（cache_salt=run_id，算法 #10 透传） + +**Files:** +- Modify: `core/agent/loop.py:103-141,271-329` +- Modify: `app/harness/inference.py:408-415,473` +- Test: `tests/unit/test_agent_loop`（或现有 loop 测试）、`tests/unit/test_harness_inference.py` + +> **原理**：训练/val/test/holdout 推理的 run_id 已含 `_e{epoch}`（如 `{base}_e{epoch}_s{step}`、`{run_id}_holdout_{kind}_e{epoch}`），用 run_id 作 cache_salt 即天然跨 epoch 重采样、同 epoch 续跑仍命中。judge/evolve 不经此路径（默认 salt=None）。gate 基线臂走 BaselineCache 不受影响；候选臂 messages 含 skill 版本天然区分。 + +- [ ] **Step 1: 写失败测试** + +```python +@pytest.mark.asyncio +async def test_agent_loop_forwards_cache_salt(): + """AgentLoop.run(cache_salt=...) 透传到 llm.chat。""" + from core.agent.loop import AgentLoop + # fake llm 记录 chat 收到的 cache_salt kwarg；loop.run(..., cache_salt='run:e2') + # 断言 fake_llm.chat 收到 cache_salt='run:e2' + ... +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest -k agent_loop_forwards_cache_salt -v` +Expected: FAIL（`run()` 无 cache_salt 参数）。 + +- [ ] **Step 3: AgentLoop 透传 cache_salt** + +`core/agent/loop.py`：`run`（L103）、`_step`/`_call_llm`（L271,317）签名加 `cache_salt: str | None = None`（keyword，随 session_id 透传）；L329 `self._llm.chat(messages, session_id=session_id)` 改为 `self._llm.chat(messages, session_id=session_id, cache_salt=cache_salt)`。 + +- [ ] **Step 4: inference 用 run_id 作 salt** + +`app/harness/inference.py` `_run_single_question`：`loop.run(...)`（L409）加 `cache_salt=run_id`（run_id 从 run_inference 透传到每题；`_run_single_question` 已有 run_id 上下文——若无则从 run_inference 参数透传）。确认 run_inference→_run_single_question 的 run_id 传递链完整。 + +- [ ] **Step 5: 保真检查点 + 测试通过** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_inference.py tests/integration/test_agent_governed_e2e.py -q` +Expected: 全 PASS。保真（算法 #10）：确认只加透传参数，Thinking+JSON/json_repair/pluggy hook 逻辑不变。 + +- [ ] **Step 6: 提交** + +```bash +git add core/agent/loop.py app/harness/inference.py tests/unit/ +git commit -m "feat: inject run_id as cache_salt for per-epoch resampling (algo #10)" +``` + +--- + +## Self-Review（作者自查，执行者复核） + +- [ ] traces 适配后诊断拿到真实轨迹（算法 #7 恢复）。 +- [ ] prediction 归一化 None 保留、非标量序列化；insert 绑定异常不击穿 gather。 +- [ ] early_stop 字段全局改名一致、计数改 epoch。 +- [ ] 预检剔除不可训练题型（test 池不动）。 +- [ ] patch 查整跨度 + 注入 + marker 完整性三防线（算法 #8 保护方向不变）。 +- [ ] None/degraded 按 lapse 保守分流；降级占比>50% 中止。 +- [ ] _run_step 幂等；gate_pools 与 checkpoint 的 gate_epoch_observed 同刻一致。 +- [ ] holdout 去重后 baseline 0 推理、best_hard 备忘录 resume 可 hydrate。 +- [ ] cache_salt=run_id 贯穿 AgentLoop，run_id 含 epoch 保证跨 epoch 重采样。 + +## 核心算法保真校验结论 + +触及算法 #5/#7/#8/#10/#12，均为恢复（#7 轨迹）/加固（#8 跨度）/透传（#10 salt）/时序（#5/#12 checkpoint），各 Task 已设保真检查点，不改算法核心逻辑。Task 5/7/9 需在实现时对照 TRM4 参考确认无行为漂移。 + +## 验收标准 + +1. `pytest tests/unit/test_harness_runner.py tests/unit/test_harness_inference.py tests/unit/test_diagnose.py tests/unit/test_patch.py tests/unit/test_harness_checkpoint.py tests/unit/test_baseline_run_log.py tests/unit/test_runner_diag_tree_inject.py` 全绿。 +2. 诊断拿到非空轨迹；非标量 prediction 不崩；early_stop 按 epoch。 +3. 微型题型被预检剔除；patch 不破坏 marker；降级题不驱动进化。 +4. _run_step 幂等；cache_salt=run_id 贯穿。 diff --git a/research-wiki/plans/2026-07-16-preflight-wp4-resilience.md b/research-wiki/plans/2026-07-16-preflight-wp4-resilience.md new file mode 100644 index 0000000..9016379 --- /dev/null +++ b/research-wiki/plans/2026-07-16-preflight-wp4-resilience.md @@ -0,0 +1,611 @@ +# WP4 韧性与持久化 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use subagent-driven-development to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 修复 LLM 治理栈与持久化层的 10 处韧性缺陷，使训练信号不被缓存重放/截断响应/断连/INFRA 故障污染，且 workspace 元数据崩溃可恢复。 + +**Architecture:** 缓存加 salt 维度让跨 epoch 评估真实重采样（同 epoch 续跑仍命中）；SSE 未收 `[DONE]` 视为截断进重试且不写缓存；扩展瞬时错误清单覆盖断连族；熔断半开只放一个探针；gate 基线臂 INFRA 故障不污染 BaselineCache（算法 #6 保真区，逐行比对）；manifest 补原子写；只读查询不改基线元数据。 + +**Tech Stack:** Python 3.11、httpx、Redis、asyncio、SQLite、pytest、pydantic-settings。 + +**设计源**：`research-wiki/designs/2026-07-16-preflight-fixes-design.md §8` + +--- + +## 关键锚点（实现前必读） + +| 用途 | 位置 | +|------|------| +| chat 缓存/重试 | `adapters/llm.py:271` chat（get L299 / set L373 / 重试 L336）；`_call_streaming:506`；`_consume_stream:533`（不校验 done）；`_iter_sse_deltas:94`（done 标志 L116）；`_is_transient_error:173-186`；`_SseAnomaly:40` | +| 缓存键 | `adapters/redis_cache.py:32` `_build_key`（无 salt）；`get:50`；`set:73`（ttl 分支 L89-92） | +| Protocol | `core/protocols.py:22-28` `LLMProvider.chat` | +| TTL | `main.py:41` `redis_cache_ttl=86400`；`:93` `ttl_s = ... if >0 else None` | +| 熔断 | `adapters/breaker.py:24` `is_open`（L36-37 到期即放行，无探针锁）；`record_failure:39`；`record_success:63` | +| gate INFRA | `app/harness/validate.py:257` `_resolve_baseline_block`（put L303-304）；`_check_infra_guard:384`（累计检查 L593）；`BaselineCache` in `gate_ladder.py:344`（put L384 原子 L401-403） | +| 持久化 | `app/harness/workspace.py` 4 处非原子 write_text：`_scaffold:111`/`update_manifest:282`/`record_run:311`/`update_best:365`；原子范式 `checkpoint.py:257-260` | +| 基线元数据 | `app/harness/log.py:57` `HarnessLog.__init__`（upsert `_runs` L73-82）；`RunLogImpl._read_table:247`（只读） | +| dual_metric | `app/harness/observation.py:119` `write_dual_metric`（version_kind final L138）；runner 调用 `runner.py:1444-1448` | +| 配置 | `.env:65` `REDIS_CACHE_TTL=0`；`.env.example:49` `=86400` | + +## 核心算法保真校验 + +触及算法 #6（块顺序验证：基线缓存/INFRA 护栏/配对翻转）——Task 6（gate 基线臂 INFRA 隔离）改 `_resolve_baseline_block` 与护栏时序，**必须逐行比对** TRM4 `/home/iomgaa/Projects/Video-Tree-TRM4/core/harness/validate.py` 的 INFRA 护栏语义，确保只改"INFRA unit 不写缓存/不计 W/L + 护栏前置"，不动配对翻转与基线快照复用逻辑。其余 task 不触碰核心算法。 + +--- + +## Task 1: 缓存 cache_salt 贯穿 + +**Files:** +- Modify: `core/protocols.py:22-28` +- Modify: `adapters/llm.py:271-373` +- Modify: `adapters/redis_cache.py:32-92` +- Modify: VLM 适配转发（`adapters/` 内实现 `LLMProvider`/转发 chat 的类，grep 定位） +- Test: `tests/unit/test_redis_cache.py`、`tests/unit/test_governed_llm.py` + +- [ ] **Step 1: 写失败测试（salt 入键）** + +在 `tests/unit/test_redis_cache.py` 追加： +```python +@pytest.mark.asyncio +async def test_salt_changes_key(fake_redis): + from adapters.redis_cache import RedisResponseCache + + cache = RedisResponseCache(redis=fake_redis, ttl_s=None) + k_none = cache._build_key("m", [{"role": "user", "content": "x"}], None) + k_e1 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e1") + k_e2 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e2") + assert k_none != k_e1 != k_e2 and k_e1 != k_e2 +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_redis_cache.py::test_salt_changes_key -v` +Expected: FAIL（`_build_key() takes 3 positional arguments but 4 were given`）。 + +- [ ] **Step 3: redis_cache 加 salt 维度** + +`adapters/redis_cache.py`： +- `_build_key(self, model, messages, cache_salt: str | None = None)`：payload 字典加 `"salt": cache_salt`： +```python + payload = json.dumps( + {"model": model, "messages": messages, "salt": cache_salt}, + sort_keys=True, + ensure_ascii=False, + ) +``` +- `get(self, model, messages, cache_salt: str | None = None)`：`key = self._build_key(model, messages, cache_salt)` +- `set(self, model, messages, response, cache_salt: str | None = None)`：同样传 `cache_salt` + +- [ ] **Step 4: Protocol + chat 透传 salt** + +`core/protocols.py` `LLMProvider.chat` 签名加 `cache_salt: str | None = None`（keyword-only，放 `parent_call_id` 后）。 +`adapters/llm.py` `chat` 签名加同参；L299 改 `await self._cache.get(self._model, messages, cache_salt)`；L373 改 `await self._cache.set(self._model, messages, response, cache_salt)`。 +grep `adapters/` 找到转发 `chat` 的 VLM/包装类（如有），同步加 `cache_salt` 透传。 + +- [ ] **Step 5: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_redis_cache.py tests/unit/test_governed_llm.py -q` +Expected: 全 PASS（默认 `cache_salt=None` 保持旧键，既有缓存测试不受影响）。 + +- [ ] **Step 6: 提交** + +```bash +git add core/protocols.py adapters/llm.py adapters/redis_cache.py tests/unit/test_redis_cache.py +git commit -m "feat: add cache_salt dimension to LLM response cache" +``` + +> 注：训练链路注入 epoch 盐（`f"{run_id}:e{epoch}"`）在 WP3 的推理调用点完成（本 WP 只提供 salt 能力）。 + +--- + +## Task 2: TTL 语义修正（≤0 报错） + +**Files:** +- Modify: `main.py:93` +- Modify: `.env:65`、`.env.example:49` +- Test: `tests/unit/test_governed_llm.py` 或新增 `tests/unit/test_infra_settings.py` + +- [ ] **Step 1: 写失败测试** + +新增 `tests/unit/test_infra_settings.py`： +```python +import pytest + + +def test_redis_cache_ttl_zero_rejected(monkeypatch): + """REDIS_CACHE_TTL<=0 必须启动即报错，消灭'0=永不过期'隐式语义。""" + from main import _resolve_cache_ttl # Step 3 抽出的纯函数 + + with pytest.raises(ValueError, match="REDIS_CACHE_TTL"): + _resolve_cache_ttl(0) + with pytest.raises(ValueError, match="REDIS_CACHE_TTL"): + _resolve_cache_ttl(-1) + assert _resolve_cache_ttl(86400) == 86400 +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_infra_settings.py -v` +Expected: FAIL（`cannot import name '_resolve_cache_ttl'`）。 + +- [ ] **Step 3: 抽出并实现 fail-loud** + +`main.py`：新增纯函数并替换 L93 逻辑： +```python +def _resolve_cache_ttl(ttl: int) -> int: + """校验 Redis 缓存 TTL：必须为正整数（消灭 0=永不过期 的隐式语义）。""" + if ttl <= 0: + raise ValueError( + f"REDIS_CACHE_TTL 必须为正整数秒，实际 {ttl}。" + "训练场景建议 >= 单次训练时长（如 86400）。" + ) + return ttl +``` +L93 改为 `ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl)`（删除 `if >0 else None` 分支；`RedisResponseCache` 的 `ttl_s` 从此恒为正）。 + +- [ ] **Step 4: 改 .env / .env.example** + +`.env:65` `REDIS_CACHE_TTL=0` → `REDIS_CACHE_TTL=86400`。 +`.env.example:49` 确认为正整数（当前 86400，OK）；补注释 `# 正整数秒，禁止 0（0 会被拒绝启动）`。 + +- [ ] **Step 5: 运行测试确认通过** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_infra_settings.py -v` +Expected: PASS。 + +- [ ] **Step 6: 提交** + +```bash +git add main.py .env.example tests/unit/test_infra_settings.py +git commit -m "fix: reject REDIS_CACHE_TTL<=0 (kill implicit never-expire)" +``` +> 注：`.env` 不提交（gitignore）；改动需手动同步到运行环境，runbook 会提示。 + +--- + +## Task 3: SSE 截断检测（未收 [DONE] 即重试，不写缓存） + +**Files:** +- Modify: `adapters/llm.py:533-578`（_consume_stream） +- Test: `tests/unit/test_governed_llm.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_governed_llm.py` 追加： +```python +@pytest.mark.asyncio +async def test_truncated_stream_without_done_raises(_build_client): + """SSE 流耗尽但未收 [DONE] → _SseAnomaly（进重试，不当成功）。""" + from adapters.llm import _SseAnomaly, GovernedLLMClient + + async def _lines(): + yield 'data: {"choices":[{"delta":{"content":"半"}}]}' + # 无 data: [DONE] —— 模拟服务端截断 + + client = _build_client() # 依现有 fixture + with pytest.raises(_SseAnomaly): + await client._consume_stream(_lines()) +``` +> 依 `test_governed_llm.py` 现有 `_build_client`（L50）/`_FakeRedisCache` 构造；实现前读该文件对齐 client 构造签名。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -k truncated_stream -v` +Expected: FAIL（当前耗尽即正常返回，不抛）。 + +- [ ] **Step 3: _consume_stream 校验 done** + +`adapters/llm.py` `_consume_stream` 在 `return content, thinking, ttft_ms, max_inter_token_ms, usage`（L578）之前加： +```python + if not usage_sink.get("done"): + raise _SseAnomaly("truncated_no_done") +``` +（`_iter_sse_deltas` 收到 `[DONE]` 时置 `usage_sink["done"]=True`，L116；未置说明流被截断。`_SseAnomaly` 已在 `_is_transient_error` L186 归为可重试，故自动进重试梯且不走成功路径、不写缓存。） + +- [ ] **Step 4: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py tests/unit/test_streaming.py -q` +Expected: 全 PASS（正常流带 `[DONE]` 的既有测试仍通过；若既有 streaming 测试的假流未含 `[DONE]`，同步补 `data: [DONE]` 终帧）。 + +- [ ] **Step 5: 提交** + +```bash +git add adapters/llm.py tests/unit/test_governed_llm.py +git commit -m "fix: treat SSE stream without [DONE] as truncated (retry, no cache)" +``` + +--- + +## Task 4: 瞬时错误清单扩展（断连族） + +**Files:** +- Modify: `adapters/llm.py:182-186` +- Test: `tests/unit/test_governed_llm.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_governed_llm.py` 追加： +```python +def test_transient_covers_disconnect_family(): + import httpx + from adapters.llm import _is_transient_error + + assert _is_transient_error(httpx.RemoteProtocolError("peer reset")) + assert _is_transient_error(httpx.ReadError("read")) + assert _is_transient_error(httpx.ConnectTimeout("ct")) + assert _is_transient_error(httpx.PoolTimeout("pt")) +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -k disconnect_family -v` +Expected: FAIL（RemoteProtocolError/PoolTimeout 不在当前清单）。 + +- [ ] **Step 3: 扩展 _is_transient_error** + +`adapters/llm.py` L182-183 的： +```python + if isinstance(exc, (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout)): + return True +``` +改为（两族基类覆盖 RemoteProtocolError/ReadError/ConnectTimeout/PoolTimeout 等）： +```python + if isinstance(exc, (httpx.TimeoutException, httpx.TransportError)): + return True +``` + +- [ ] **Step 4: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_governed_llm.py -q` +Expected: 全 PASS（`httpx.HTTPStatusError` 非 TransportError 子类，401/403 致命分支不受影响）。 + +- [ ] **Step 5: 提交** + +```bash +git add adapters/llm.py tests/unit/test_governed_llm.py +git commit -m "fix: cover httpx disconnect family in transient error set" +``` + +--- + +## Task 5: 熔断半开单探针锁 + +**Files:** +- Modify: `adapters/breaker.py:18-70` +- Test: `tests/unit/test_breaker.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_breaker.py::TestCircuitBreaker` 追加： +```python + def test_half_open_admits_single_probe(self): + from adapters.breaker import CircuitBreaker + + b = CircuitBreaker(fail_threshold=2, cooldown_s=10.0) + b.record_failure("p", now=0.0) + b.record_failure("p", now=0.0) # 开路至 t=10 + assert b.is_open("p", now=5.0) is True # 冷却中 + # 冷却到期：只放行第一个探针 + assert b.is_open("p", now=11.0) is False # 探针 1 放行 + assert b.is_open("p", now=11.0) is True # 探针 2 被挡（探针在途） + b.record_success("p") # 探针成功 → 闭合 + assert b.is_open("p", now=12.0) is False +``` + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_breaker.py -k half_open_admits_single -v` +Expected: FAIL（当前到期后所有调用都返回 False）。 + +- [ ] **Step 3: 实现半开单探针锁** + +`adapters/breaker.py`： +- `__init__` 加 `self._half_open_inflight: dict[str, bool] = {}` +- `is_open` 改为： +```python + def is_open(self, source_name: str, now: float) -> bool: + until = self._open_until.get(source_name) + if until is None: + return False + if now < until: + return True # 冷却中，全挡 + # 冷却到期：half-open，只放行一个探针 + if self._half_open_inflight.get(source_name): + return True # 已有探针在途，继续挡 + self._half_open_inflight[source_name] = True + return False +``` +- `record_success` 加 `self._half_open_inflight.pop(source_name, None)`（探针成功 → 清在途 + 已有的清 fails/open_until） +- `record_failure`：探针失败会累计并可能重开路；末尾加 `self._half_open_inflight.pop(source_name, None)`（让下一轮 cooldown 后可再探） +- `force_open`：加 `self._half_open_inflight.pop(source_name, None)` + +（is_open 在 asyncio 单线程内同步执行，"检查+标记探针"原子，无竞态。） + +- [ ] **Step 4: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_breaker.py tests/unit/test_governed_llm.py -q` +Expected: 全 PASS（既有 circuit_open 测试若假设"到期即多次放行"需同步更新为单探针语义）。 + +- [ ] **Step 5: 提交** + +```bash +git add adapters/breaker.py tests/unit/test_breaker.py +git commit -m "fix: half-open circuit admits single probe (no thundering herd)" +``` + +--- + +## Task 6: gate 基线臂 INFRA 隔离（算法 #6 保真区） + +**Files:** +- Modify: `app/harness/validate.py:257-311`（_resolve_baseline_block）+ 护栏时序 +- Test: `tests/unit/test_harness_validate.py` + +> **保真前置**：实现前完整读 `app/harness/validate.py:257-311` `_resolve_baseline_block`、`_candidate_correctness_from_db`、块循环（575-600），并逐行比对 TRM4 `core/harness/validate.py` 的 INFRA 护栏语义。目标仅为：① 基线臂 INFRA 故障（stop_reason∈{error,parse_error}）的 unit **不写 BaselineCache**、**不计入 W/L 翻转**；② 护栏检查移到"写缓存之前"。不得改动配对翻转、基线快照复用、unit 折叠逻辑。 + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_validate.py` 追加（用 fake run_inference 让某基线 unit 返回 stop_reason=error）： +```python +@pytest.mark.asyncio +async def test_baseline_infra_error_not_cached(tmp_path): + """基线臂 INFRA error 的 unit 不写入 BaselineCache（不永久污染）。""" + # 依现有 test_harness_validate.py 的 fake run_inference / BaselineCache fixture 构造； + # 让 miss unit u1 的推理返回 prediction=None, stop_reason='error'； + # 调 _resolve_baseline_block 后断言 baseline_cache.get(..., u1) is None + ... +``` +> 依 `test_harness_validate.py` 现有 fixture（fake `run_inference`、`BaselineCache`、`HarnessLog`）；实现前读该文件对齐构造，勿臆造签名。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_validate.py -k baseline_infra_error_not_cached -v` +Expected: FAIL（当前 INFRA unit 的对错被 put 进缓存）。 + +- [ ] **Step 3: 实现 INFRA 隔离** + +`_resolve_baseline_block`：miss 跑完后，识别 INFRA 题（从 db 读 stop_reason∈{error,parse_error} 或经 run_inference 结果），其所属 unit：不 `baseline_cache.put`、不计入返回的 `b_units`（从块 unit 集合排除，使配对与 W/L 不含它）。护栏所需的 `errors_inc` 在 put 之前累计并检查（护栏前置）。具体实现按 Step 前"保真前置"要求对齐现场与 TRM4。 + +- [ ] **Step 4: 运行测试确认通过 + 保真回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_validate.py tests/unit/test_gate_block_unit.py tests/unit/test_gates.py -q` +Expected: 全 PASS（配对翻转/e-process 既有测试不受影响 = 保真达成）。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/validate.py tests/unit/test_harness_validate.py +git commit -m "fix: isolate gate baseline-arm INFRA errors from BaselineCache (algo #6)" +``` + +--- + +## Task 7: manifest 原子写 + +**Files:** +- Modify: `app/harness/workspace.py`（`_scaffold:111`/`update_manifest:282`/`record_run:311`/`update_best:365`） +- Test: `tests/unit/test_harness_workspace.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_workspace.py` 追加： +```python +def test_update_manifest_is_atomic(workspace_dir, monkeypatch): + """写 manifest 途中崩溃不产生半截 JSON（原子写：tmp 存在即失败也不损原文件）。""" + import json + from app.harness import workspace as ws + + # 先建合法 manifest + ... # 依现有 fixture 初始化 workspace + original = (workspace_dir / "manifest.json").read_text() + + # monkeypatch os.replace 抛异常，模拟替换阶段崩溃 + def _boom(src, dst): + raise OSError("crash during replace") + monkeypatch.setattr(ws.os, "replace", _boom) + with pytest.raises(OSError): + ws.update_manifest(workspace_dir, skills="skills/v2") + # 原 manifest 未被破坏 + assert (workspace_dir / "manifest.json").read_text() == original + assert json.loads((workspace_dir / "manifest.json").read_text()) +``` +> 依 `test_harness_workspace.py` 现有 workspace 初始化 fixture；实现前读对齐。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_workspace.py -k update_manifest_is_atomic -v` +Expected: FAIL（当前 write_text 非原子，崩溃留半截）。 + +- [ ] **Step 3: 加原子写 helper 并替换 4 处** + +`app/harness/workspace.py` 顶部确认 `import os`。新增模块级 helper： +```python +def _atomic_write_json(path: Path, data: dict) -> None: + """原子写 JSON：tmp + os.replace（对齐 checkpoint.py 范式，防半截损坏）。""" + tmp = path.with_name(path.name + ".tmp") + tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + os.replace(tmp, path) +``` +把 `_scaffold_workspace`（L111）、`update_manifest`（L282）、`record_run`（L311）、`update_best`（L365）四处的 `(...).write_text(json.dumps(...))` 替换为 `_atomic_write_json(, )`。 + +- [ ] **Step 4: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_workspace.py -q` +Expected: 全 PASS。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/workspace.py tests/unit/test_harness_workspace.py +git commit -m "fix: atomic writes for manifest/record_run/update_best (tmp+replace)" +``` + +--- + +## Task 8: 只读查询不改基线元数据 + +**Files:** +- Modify: `app/harness/log.py:57-83`（HarnessLog.__init__ 加 register_run 开关） +- Modify: 只读查询基线的调用点（`app/harness/runner.py:923` `_init_gate`、`app/harness/pools.py:765,818`） +- Test: `tests/unit/test_harness_log.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_log.py::TestHarnessLogUpsert` 追加： +```python + def test_register_run_false_skips_upsert(self, tmp_path): + """register_run=False 时只读打开不改写已有 _runs 行（started_at/status 不变）。""" + db = str(tmp_path / "h.db") + from app.harness.log import HarnessLog + + with HarnessLog(db, "r1") as log: + log # 初次注册 + row0 = _read_run_row(db, "r1") # 依现有 helper 读 started_at/status + + with HarnessLog(db, "r1", register_run=False) as log: + log.query("SELECT 1") # 只读 + row1 = _read_run_row(db, "r1") + assert row1["started_at"] == row0["started_at"] + assert row1["status"] == row0["status"] +``` +> 依 `test_harness_log.py` 现有读行 helper；实现前读 `TestHarnessLogUpsert`（L196）对齐。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_log.py -k register_run_false -v` +Expected: FAIL（`unexpected keyword argument 'register_run'`）。 + +- [ ] **Step 3: HarnessLog 加 register_run 开关** + +`app/harness/log.py` `__init__` 签名加 `register_run: bool = True`（keyword）。把 L73-82 的 `_runs` upsert 包进 `if register_run:`；`register_run=False` 时跳过 upsert（仅 `_init_fixed_tables` 建表 + 连接，供只读查询）。`__exit__`/`close` 的 status 更新同样在 `register_run` 为 True 时才执行（避免只读关闭把 status 改 completed）。 + +- [ ] **Step 4: 只读调用点传 register_run=False** + +`app/harness/runner.py:923`（_init_gate 用 baseline_run_id 只读查 predictions）、`app/harness/pools.py:765,818`（build_or_load_pools 只读查 predictions）三处 `HarnessLog(...)` 调用加 `register_run=False`。 + +- [ ] **Step 5: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_log.py tests/unit/test_harness_pools.py -q` +Expected: 全 PASS。 + +- [ ] **Step 6: 提交** + +```bash +git add app/harness/log.py app/harness/runner.py app/harness/pools.py tests/unit/test_harness_log.py +git commit -m "fix: read-only baseline queries skip _runs upsert (register_run flag)" +``` + +--- + +## Task 9: dual_metric version_kind 口径修正 + +**Files:** +- Modify: `app/harness/runner.py:1444-1448`（第二次 write_dual_metric） +- Test: `tests/unit/test_harness_observation.py` 或 `test_harness_runner.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/unit/test_harness_observation.py` 追加（验证 slow_candidate 与 final 可区分）： +```python +def test_slow_candidate_kind_distinct_from_final(tmp_path): + from app.harness.observation import write_dual_metric, read_dual_metric + + db = str(tmp_path / "h.db") + write_dual_metric(db, run_id="r", epoch=1, version_kind="final", + skills_version="v1", prompts_version="v1", pool="val", + hard_acc=0.7, soft_score=None, mixed_score=None) + write_dual_metric(db, run_id="r", epoch=1, version_kind="slow_candidate", + skills_version="v2", prompts_version="v2", pool="val", + hard_acc=0.6, soft_score=None, mixed_score=None) + rows = read_dual_metric(db, run_id="r", epoch=1) + kinds = {r["version_kind"] for r in rows} + assert "final" in kinds and "slow_candidate" in kinds +``` +> 依 `test_harness_observation.py` 现有 read_dual_metric 签名；实现前读对齐。 + +- [ ] **Step 2: 运行确认失败/通过基线** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_observation.py -k slow_candidate -v` +Expected: 若 read_dual_metric 已支持则 PASS 基线；重点是 Step 3 改 runner 调用点。 + +- [ ] **Step 3: 改 runner 慢更新第二次写为 slow_candidate** + +`app/harness/runner.py` L1444-1448 的 `write_dual_metric(..., version_kind="final", ...)`（`_slow_update_cycle` Phase 8 的 R2 行）改为 `version_kind="slow_candidate"`，使被 revert 的慢更新候选不再占用 `final` 语义。确认 Phase 2 的第一次（L1399-1403）仍为 `final`（epoch 终值唯一）。 + +- [ ] **Step 4: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_observation.py tests/unit/test_harness_runner.py -q` +Expected: 全 PASS。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/runner.py tests/unit/test_harness_observation.py +git commit -m "fix: slow-update R2 dual_metric uses slow_candidate kind (not final)" +``` + +--- + +## Task 10: 离线 --retry-uncertain + docstring 修正 + +**Files:** +- Modify: `app/harness/video_split_cli.py`（build_arg_parser + run_pipeline） +- Modify: `app/harness/baseline_diagnosis.py`（done 集排除 uncertain + docstring） +- Test: `tests/unit/test_video_split_cli.py` 或 `test_baseline_diagnosis` + +- [ ] **Step 1: 写失败测试** + +定位 `baseline_diagnosis` 的 done_question_ids 计算函数（`app/harness/baseline_diagnosis.py`，grep `done_question_ids`），追加测试：`retry_uncertain=True` 时 tier=uncertain 的已落库题不计入 done（会被重诊）。 +```python +def test_retry_uncertain_excludes_uncertain_from_done(...): + # 构造 baseline_diagnosis 表含 tier=T2 与 tier=uncertain 行； + # done_ids(retry_uncertain=False) 含 uncertain 题； + # done_ids(retry_uncertain=True) 不含 uncertain 题 + ... +``` +> 实现前读 `baseline_diagnosis.py` 的 done 集查询函数签名对齐。 + +- [ ] **Step 2: 运行确认失败** + +Run: `conda run -n Video-Tree-TRM python -m pytest -k retry_uncertain -v` +Expected: FAIL（无该参数）。 + +- [ ] **Step 3: 实现 retry_uncertain** + +`app/harness/baseline_diagnosis.py`：done 集查询函数加 `retry_uncertain: bool = False`，True 时 `WHERE ... AND tier != 'uncertain'`（uncertain 题不算完成，会被重诊）。修正模块 docstring 把"崩溃最多丢正在写的一行/逐行落库"表述改为实际的"run 末批量落库（Phase 3），崩溃丢本次 run 全部未落库结果，靠 Redis 缓存缓解重烧"。 +`app/harness/video_split_cli.py`：`build_arg_parser` 加 `--retry-uncertain`（store_true）；`run_pipeline` 透传到 `run_baseline_diagnosis` 的 done 集计算。 + +- [ ] **Step 4: 运行测试确认通过 + 回归** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_video_split_cli.py -q` +Expected: 全 PASS。 + +- [ ] **Step 5: 提交** + +```bash +git add app/harness/video_split_cli.py app/harness/baseline_diagnosis.py tests/unit/ +git commit -m "feat: --retry-uncertain re-diagnoses uncertain rows; fix docstring" +``` + +--- + +## Self-Review（作者自查，执行者复核） + +- [ ] cache_salt 四处（protocols/llm/redis_cache/VLM 转发）贯通，默认 None 保持旧键。 +- [ ] TTL≤0 fail-loud + .env/.env.example 同步（.env 手动，runbook 提示）。 +- [ ] SSE 未收 [DONE] → _SseAnomaly（已在瞬时清单 → 重试 + 不写缓存）。 +- [ ] 瞬时清单用 TimeoutException+TransportError 两族基类覆盖断连族，不误纳 HTTPStatusError。 +- [ ] 熔断半开单探针：is_open 检查+标记原子（asyncio 单线程）。 +- [ ] Task 6 触及算法 #6，已设保真前置（逐行比对 TRM4，只改 INFRA 不写缓存 + 护栏前置）。 +- [ ] manifest 4 处原子写；只读查询 register_run=False 不改基线元数据。 + +## 核心算法保真校验结论 + +Task 6 触及算法 #6（块顺序验证），已在该 Task 设"保真前置"要求逐行比对 TRM4 `validate.py`，仅改 INFRA 隔离与护栏时序，不动配对翻转/基线快照/unit 折叠。其余 Task 均为治理栈/持久化层，不涉及核心算法。 + +## 验收标准 + +1. `pytest tests/unit/test_redis_cache.py tests/unit/test_governed_llm.py tests/unit/test_breaker.py tests/unit/test_streaming.py tests/unit/test_harness_validate.py tests/unit/test_harness_workspace.py tests/unit/test_harness_log.py tests/unit/test_harness_observation.py tests/unit/test_infra_settings.py` 全绿。 +2. 缓存加 salt 后跨 epoch 键不同、同 epoch 键相同。 +3. 熔断半开只放一个探针；SSE 截断进重试不写缓存。 +4. manifest 原子写；基线 _runs 元数据只读查询不被改写。 diff --git a/research-wiki/plans/preflight-wp1-asset-migration.md b/research-wiki/plans/preflight-wp1-asset-migration.md new file mode 100644 index 0000000..a7493e6 --- /dev/null +++ b/research-wiki/plans/preflight-wp1-asset-migration.md @@ -0,0 +1,9 @@ +--- +type: plan +node_id: plan:preflight-wp1-asset-migration +title: "WP1 资产迁移" +date: 2026-07-16 +--- + +# WP1 资产迁移 + diff --git a/research-wiki/plans/preflight-wp2-split-wiring.md b/research-wiki/plans/preflight-wp2-split-wiring.md new file mode 100644 index 0000000..20bffd8 --- /dev/null +++ b/research-wiki/plans/preflight-wp2-split-wiring.md @@ -0,0 +1,9 @@ +--- +type: plan +node_id: plan:preflight-wp2-split-wiring +title: "WP2 切分与接线" +date: 2026-07-16 +--- + +# WP2 切分与接线 + diff --git a/research-wiki/plans/preflight-wp3-train-loop.md b/research-wiki/plans/preflight-wp3-train-loop.md new file mode 100644 index 0000000..fdb0535 --- /dev/null +++ b/research-wiki/plans/preflight-wp3-train-loop.md @@ -0,0 +1,9 @@ +--- +type: plan +node_id: plan:preflight-wp3-train-loop +title: "WP3 训练循环与进化引擎" +date: 2026-07-16 +--- + +# WP3 训练循环与进化引擎 + diff --git a/research-wiki/plans/preflight-wp4-resilience.md b/research-wiki/plans/preflight-wp4-resilience.md new file mode 100644 index 0000000..8f02197 --- /dev/null +++ b/research-wiki/plans/preflight-wp4-resilience.md @@ -0,0 +1,9 @@ +--- +type: plan +node_id: plan:preflight-wp4-resilience +title: "WP4 韧性与持久化" +date: 2026-07-16 +--- + +# WP4 韧性与持久化 + From 6cdf05d56b216bde58dd58b63408a42c8a9951f4 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 04:38:51 -0400 Subject: [PATCH 02/43] docs: address Codex review of WP1 plan (real loader tests, red-green order) --- ...026-07-16-preflight-wp1-asset-migration.md | 96 +++++++++++-------- 1 file changed, 57 insertions(+), 39 deletions(-) diff --git a/research-wiki/plans/2026-07-16-preflight-wp1-asset-migration.md b/research-wiki/plans/2026-07-16-preflight-wp1-asset-migration.md index 0e057f2..3b5672e 100644 --- a/research-wiki/plans/2026-07-16-preflight-wp1-asset-migration.md +++ b/research-wiki/plans/2026-07-16-preflight-wp1-asset-migration.md @@ -37,19 +37,7 @@ - Create: `prompts/evolve_skill.md`、`prompts/evolve_system.md`、`prompts/evolve_tool.md`、`prompts/evolve_rank.md`、`prompts/slow_momentum.md`（从 TRM4 拷贝） - Test: `tests/unit/test_evolve_prompts_present.py` -- [ ] **Step 1: 拷贝 5 个模板** - -Run: -```bash -cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_skill.md prompts/evolve_skill.md -cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_system.md prompts/evolve_system.md -cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_tool.md prompts/evolve_tool.md -cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_rank.md prompts/evolve_rank.md -cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/slow_momentum.md prompts/slow_momentum.md -``` -Expected: 5 文件存在于 `prompts/`。 - -- [ ] **Step 2: 写契约冒烟测试（先失败）** +- [ ] **Step 1: 写契约冒烟测试（先失败）** `tests/unit/test_evolve_prompts_present.py`： ```python @@ -80,17 +68,34 @@ def test_evolve_template_present_and_contract(name: str, required_tokens: list[s assert token in text, f"{name} 缺输出契约关键词 {token!r}（与解析代码不对齐）" ``` -- [ ] **Step 3: 运行确认通过（模板已在 Step 1 拷入）** +- [ ] **Step 2: 运行确认失败（模板尚未迁移）** + +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v` +Expected: 5 参数化用例全 FAIL（`AssertionError: 缺模板: prompts/evolve_skill.md` 等；TRM5 当前无这 5 个模板）。 + +- [ ] **Step 3: 拷贝 5 个模板** + +Run: +```bash +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_skill.md prompts/evolve_skill.md +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_system.md prompts/evolve_system.md +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_tool.md prompts/evolve_tool.md +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/evolve_rank.md prompts/evolve_rank.md +cp /home/iomgaa/Projects/Video-Tree-TRM4/prompts/slow_momentum.md prompts/slow_momentum.md +``` +Expected: 5 文件存在于 `prompts/`。 + +- [ ] **Step 4: 运行确认通过** Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v` Expected: 5 参数化用例全 PASS。若 evolve_tool.md 缺 `edits_extract`/`edits_verify` 则契约不符——停止并逐行比对 TRM4 源。 -- [ ] **Step 4: 保真检查点** +- [ ] **Step 5: 保真检查点** Run: `conda run -n Video-Tree-TRM python -c "print('edits_extract' in open('prompts/evolve_tool.md').read() and 'edits_verify' in open('prompts/evolve_tool.md').read())"` Expected: `True`（对齐 `evolve.py:1433-1435` 的 `parsed["edits_extract"]`/`parsed["edits_verify"]`）。 -- [ ] **Step 5: 提交** +- [ ] **Step 6: 提交** ```bash git add prompts/evolve_skill.md prompts/evolve_system.md prompts/evolve_tool.md prompts/evolve_rank.md prompts/slow_momentum.md tests/unit/test_evolve_prompts_present.py @@ -126,14 +131,15 @@ git commit -m "feat: migrate 5 evolve/momentum templates from TRM4 (algo #8)" `app/harness/video_split_cli.py`： - `_load_diagnose_prompts` 删行 ` span_eval_user=_read("span_eval_user.md"),`（L374） -- [ ] **Step 3: 删测试 fixture 对死字段的赋值** +- [ ] **Step 3: 删测试 fixture 对死字段的赋值 + 更新 docstring** -- `tests/unit/test_evolve.py:682` 删行 ` consolidate_system="cons",` +- `tests/unit/test_evolve.py:684` 删行 ` consolidate_system="cons",` - `tests/unit/test_diagnose.py:753` 删行 ` span_eval_user="",` - `tests/unit/test_evolution_types.py:352` 删行 ` span_eval_user="p4",` - `tests/unit/test_evolution_types.py:370` 删行 ` consolidate_system="consolidate_tmpl",` +- `tests/unit/test_evolution_types.py:347` 的文档字符串"DiagnosePrompts 8 个模板字段"改为"7 个"；`:364` 的"EvolvePrompts 5 个模板字段"改为"4 个"（删字段后数量变化）。 -注：`test_evolution_types.py` 可能有断言逐字段比对；若该测试断言字段数量或逐字段相等，同步移除对两个死字段的断言（读该测试确认，删净引用）。 +注：`test_evolution_types.py` 若有断言逐字段比对或字段计数，同步移除对两个死字段的断言（读该测试确认，删净引用）。上述行号以当前代码为准，实现前 `grep -n consolidate_system\|span_eval_user tests/unit/test_evolution_types.py` 复核。 - [ ] **Step 4: 运行相关测试确认通过** @@ -142,7 +148,7 @@ Expected: 全 PASS（无 `TypeError: unexpected keyword argument` / 无 `missing - [ ] **Step 5: 全库确认无残留引用** -Run: `conda run -n Video-Tree-TRM python -c "import subprocess,sys; r=subprocess.run(['grep','-rn','consolidate_system\|span_eval_user','core/','app/','tests/'],capture_output=True,text=True); print(r.stdout)"` +Run: `conda run -n Video-Tree-TRM python -c "import subprocess; r=subprocess.run(['grep','-rn','consolidate_system\|span_eval_user','core/','app/','adapters/','tests/'],capture_output=True,text=True); print(r.stdout)"` Expected: 空输出（`consolidate_appendix` 用内联 `_CONSOLIDATE_SYSTEM` 不算 `consolidate_system` 字段引用；若出现请确认非 dataclass 字段引用）。 - [ ] **Step 6: 提交** @@ -161,28 +167,39 @@ git commit -m "refactor: drop dead prompt fields consolidate_system/span_eval_us - Modify: `app/harness/video_split_cli.py:365-367` - Test: `tests/unit/test_evolve_prompts_present.py`（追加） -- [ ] **Step 1: 追加 fail-loud 测试（先失败）** +- [ ] **Step 1: 追加 fail-loud 测试（直接调真实加载器）** -在 `tests/unit/test_evolve_prompts_present.py` 追加： +在 `tests/unit/test_evolve_prompts_present.py` 追加——直接驱动真实 loader（不复制 _read 逻辑），在无模板的空 cwd 下断言 `FileNotFoundError`： ```python -def test_loader_fail_loud_on_missing_template(tmp_path, monkeypatch): - """加载器遇缺失模板必须抛 FileNotFoundError，不静默返回空串。""" - monkeypatch.chdir(tmp_path) # 空目录下无 prompts/，_read 应 fail-loud - (tmp_path / "prompts").mkdir() - from app.harness.runner import Runner - - # 直接测私有 _read 语义：借最小构造不便，改测行为——缺文件应 raise - def _read(name: str) -> str: - p = Path("prompts") / name - if not p.exists(): - raise FileNotFoundError(f"缺进化/诊断模板: {p}（请从 TRM4 迁移或检查 prompts/）") - return p.read_text(encoding="utf-8") +def test_video_split_loader_fail_loud_on_missing(tmp_path, monkeypatch): + """video_split_cli 的真实 diagnose 加载器缺模板必须 FileNotFoundError。""" + monkeypatch.chdir(tmp_path) # 空目录，无 prompts/*.md + from app.harness.video_split_cli import _load_diagnose_prompts with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"): - _read("evolve_skill.md") -``` + _load_diagnose_prompts() -> 说明：`_read` 是嵌套闭包不易直接导入，此测试固化期望语义（缺文件 raise + 消息含"缺进化/诊断模板"）；Step 2 把三处闭包改成同款实现，集成层由 `test_harness_runner.py` 现有加载路径覆盖。 + +def test_runner_evolve_loader_fail_loud_on_missing(tmp_path, monkeypatch): + """runner 的真实 evolve 加载器缺模板必须 FileNotFoundError。""" + monkeypatch.chdir(tmp_path) + from app.harness.runner import Runner + + r = object.__new__(Runner) # 绕过 __init__，仅测无状态加载器方法 + with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"): + r._load_evolve_prompts() + + +def test_runner_diagnose_loader_fail_loud_on_missing(tmp_path, monkeypatch): + """runner 的真实 diagnose 加载器缺模板必须 FileNotFoundError。""" + monkeypatch.chdir(tmp_path) + from app.harness.runner import Runner + + r = object.__new__(Runner) + with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"): + r._load_diagnose_prompts() +``` +> 这三个测试直接调真实 loader（`_load_evolve_prompts`/`_load_diagnose_prompts` 无 self 状态依赖，`object.__new__` 可安全调用）；修复前静默返回空串不抛，故 Step 2 前必 FAIL。 - [ ] **Step 2: 三处 `_read` 闭包改 fail-loud** @@ -207,10 +224,11 @@ def test_loader_fail_loud_on_missing_template(tmp_path, monkeypatch): Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_evolve_prompts_present.py -v` Expected: 全 PASS。 -- [ ] **Step 4: 运行 runner 加载路径回归** +- [ ] **Step 4: runner 其他行为回归（非 fail-loud 验证）** +fail-loud 已由 Step 3 的三个真实 loader 测试验证；此步仅确认模板迁移 + loader 改动未破坏 runner 其他行为。 Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -q` -Expected: 全 PASS（模板已迁移，真实加载不再命中缺失分支）。 +Expected: 全 PASS（模板已迁移，真实加载走成功分支）。 - [ ] **Step 5: 提交** From d3639a94b8fc2753637580b8f3c9c20ce2fb5e37 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 04:41:02 -0400 Subject: [PATCH 03/43] feat: migrate 5 evolve/momentum templates from TRM4 (algo #8) --- prompts/evolve_rank.md | 8 ++++ prompts/evolve_skill.md | 53 ++++++++++++++++++++++ prompts/evolve_system.md | 53 ++++++++++++++++++++++ prompts/evolve_tool.md | 55 +++++++++++++++++++++++ prompts/slow_momentum.md | 10 +++++ tests/unit/test_evolve_prompts_present.py | 25 +++++++++++ 6 files changed, 204 insertions(+) create mode 100644 prompts/evolve_rank.md create mode 100644 prompts/evolve_skill.md create mode 100644 prompts/evolve_system.md create mode 100644 prompts/evolve_tool.md create mode 100644 prompts/slow_momentum.md create mode 100644 tests/unit/test_evolve_prompts_present.py diff --git a/prompts/evolve_rank.md b/prompts/evolve_rank.md new file mode 100644 index 0000000..c1874fa --- /dev/null +++ b/prompts/evolve_rank.md @@ -0,0 +1,8 @@ +你是一个改动优先级裁判。你会收到一份当前 prompt 文件全文，和一组待应用的局部 edits（每条含 op/target/content）。由于本轮编辑预算有限，你只能保留其中最重要的若干条。 + +请只依据"对纠正失败、提升正确率的预期贡献"排序：优先保留直接修复失败模式的改动，其次保留收窄或澄清的改动，最后才是巩固已有成功的改动。删除类、简化类的精准改动通常优先于追加大段新内容。 + +每条 edit 会附带 support_count（该改动的支持案例数）。同等重要性下，support_count 更高的优先；但 support_count 低不等于该删，仍以修复贡献为主判据。 + +严格输出以下 JSON，不要包含其他文字： +{"selected_indices": [按重要性降序排列的 0-based 索引]} diff --git a/prompts/evolve_skill.md b/prompts/evolve_skill.md new file mode 100644 index 0000000..9b1de06 --- /dev/null +++ b/prompts/evolve_skill.md @@ -0,0 +1,53 @@ +你是一个搜索策略改进专家。你服务于一个自进化视频搜索系统，该系统通过分析 Agent 的失败和成功案例来迭代改进搜索策略（Skill）。你的任务是基于案例包中的证据，改写当前 Skill 文件，使 Agent 在后续执行中避免相同的失败模式。 + +## 你会收到的输入 + +1. 当前 Skill 文件全文 +2. 失败案例：Agent 答错的题目，含完整推理轨迹、错误类型和诊断指标 +3. 成功案例：Agent 答对的题目，展示当前 Skill 中有效的模式 +4. 聚合统计：准确率、错误归因分布、搜索有效性指标、Skill 步骤遵循率 +5. （可能出现）上一轮被接受改动导致的回归题：这些题在上一版本答对、却被你上次的改写改错了，附基线与候选两份预测和推理轨迹 +6. （可能出现）黑名单：已被实测验证无效或有害的改法方向 + +## 工作原则 + +如果输入里出现了回归题，它的优先级高于一切。这些题在上一版本是对的，是你上次的改动把它们弄坏的，所以本次改写的第一要务是确保不再破坏它们——宁可在相关方向上回退或收窄，也不要为了拉高其它题而牺牲它们。更一般地，当你看到准确率下降这类负向信号时，默认先怀疑上次是不是加了过度、冲突或冗余的指令，优先简化、删除、收窄；只有确认简化解决不了问题，才考虑加强指令。黑名单里列出的改法已经被实测证明无效或有害，不要换个措辞把同一个方向再提一遍。 + +先分析失败案例中 Agent 的实际行为与 Skill 指令的偏差。偏差分两类：Skill 指令正确但 Agent 没遵循（遵循率问题），或 Skill 指令本身有误导（策略问题）。前者需要让指令更具体、更难被忽略；后者需要修改策略本身。 + +从成功案例中识别有效模式——这些模式在改写时必须保留。如果成功案例和失败案例采用了不同的策略路径，重点强化成功路径。 + +Skill 中引用的统计数据（如"search-first 正确率 75%"）应根据案例包中的新统计更新。不要编造数据，只使用案例包中提供的数字。 + +你写进 Skill 的每一条规则都必须是可跨题复用的通用策略，而不是对某一道题的记答案。跨多个失败案例时只提取共性模式，抽象掉一切单题特征——具体题目内容、选项文字、步骤序号、某一帧的具体画面、某个具体答案都不许写进 Skill 正文。一条规则如果只在它来源的那道题上成立，就不要加。改写时优先简化与收窄：宁可让 Skill 更短，也不要堆叠只对个别题生效的硬性指令。 + +## 冻结区 + +以下内容不可修改，必须原样保留在改写后的文件中： +- YAML frontmatter（`---` 之间的 name、description、task_type） +- 输出格式中的 JSON 基础结构（reflect/plan/action 三个顶层字段） + +这次不要返回整份改写后的文件，而是只返回一组局部 edits。`append` 用来在文件末尾追加一个新 section，`insert_after` 用来把内容紧跟着插到某个锚点段落之后，`replace` 用来用新内容整体替换 target 对应的原文，`delete` 则直接删除 target 对应的原文并让 content 留空。target 必须是从当前文件里逐字复制出来的原文，而且要长到足以唯一定位；只要有任何一个字不完全匹配，这条改动就会被跳过。改动应尽量小而局部，优先做精确补丁，不要动辄重写整段整节；另外，冻结区里的文字绝不能作为 target。 + +## 输出格式 + +请严格输出以下 JSON，不要包含其他文字： + +```json +{ + "suggestions": [ + { + "section": "改动目标段落的标题或位置描述", + "problem": "失败案例中暴露的具体问题", + "change": "具体的修改方向", + "related_cases": ["关联的失败案例 question_id"], + "support_count": 该建议的支持案例数（= related_cases 的数量） + } + ], + "edits": [ + {"op": "append|insert_after|replace|delete", "target": "锚点原文（append 留空）", "content": "新内容（delete 留空）", "support_count": 该改动的支持案例数} + ] +} +``` + +每条 edit 与每条 suggestion 都必须带 "support_count"：本条改动由多少个失败案例共同支持（即 related_cases 的数量）。support_count 越高代表证据越充分；它只作排序参考，不是硬门槛——support_count 低不等于该删，仍以修复贡献为主判据。 diff --git a/prompts/evolve_system.md b/prompts/evolve_system.md new file mode 100644 index 0000000..9849e77 --- /dev/null +++ b/prompts/evolve_system.md @@ -0,0 +1,53 @@ +你是一个系统级行为改进专家。你服务于一个自进化视频搜索系统，该系统通过分析 Agent 的跨题型行为模式来改进 System Prompt。你的任务是基于行为模式案例包中的证据，改写 System Prompt 中的策略性指令，纠正系统级行为问题。 + +## 你会收到的输入 + +1. 当前 System Prompt (system.md) 全文 +2. 失败案例：展示三类系统性行为问题的题目——过早提交（budget_usage < 0.3 就提交答案）、高置信答错（confidence 很高但答案错误）、确认偏误（只搜索支持初始判断的证据） +3. 成功案例：行为校准良好的题目——置信度与正确率匹配，预算使用适中 +4. D5 行为模式统计：各行为模式的发生频率和分布 +5. （可能出现）黑名单：已被实测验证无效或有害的改法方向 + +## 工作原则 + +关注跨题型的系统性行为模式，而非某个具体题型的策略。失败案例中的行为偏差反映了 System Prompt 的决策原则不够清晰或不够强约束。黑名单里的改法已经被实测验证无效或有害，不要再朝同一个方向改一遍。 + +过早提交说明预算管理指令需要更强的约束语言。高置信答错说明置信度校准的语义定义需要调整。确认偏误说明竞争选项搜索的要求需要更明确。 + +当你看到失败案例与成功案例并存时，失败修复优先于巩固成功——先确保失败模式被纠正，再考虑强化已有的好行为。看到某类行为指标变差这类负向信号时，默认先怀疑上一轮是否加了过度、冲突或冗余的约束，优先简化、删除、收窄；只有确认简化解决不了，才考虑加强约束语言。 + +从成功案例中提取"好行为"的特征，在改写时强化这些特征的表述。 + +## 冻结区 + +以下 section 必须原样保留，不可修改任何文字： +- `## 能力边界`（事实性描述，不是策略） +- `## 输出格式`（JSON schema 是系统契约） +- `## 视频树结构`（含信任层级，是数据结构事实描述） + +可改写的 section： +- `## 角色`（前两段的角色定位和行为倾向描述） +- `## 决策原则`（搜索策略、预算分配建议） +- 搜索工具使用、否定题原则、置信度语义 + +这次不要返回整份改写后的文件，而是只返回一组局部 edits。`append` 用来在文件末尾追加一个新 section，`insert_after` 用来把内容紧跟着插到某个锚点段落之后，`replace` 用来用新内容整体替换 target 对应的原文，`delete` 则直接删除 target 对应的原文并让 content 留空。target 必须是从当前文件里逐字复制出来的原文，而且要长到足以唯一定位；只要有任何一个字不完全匹配，这条改动就会被跳过。改动应尽量小而局部，优先做精确补丁，不要动辄重写整段整节；另外，冻结区里的文字绝不能作为 target。 + +## 输出格式 + +请严格输出以下 JSON，不要包含其他文字： + +```json +{ + "suggestions": [ + { + "section": "改动目标段落的标题或位置描述", + "problem": "失败案例中暴露的具体行为问题", + "change": "具体的修改方向", + "related_cases": ["关联的失败案例 question_id"] + } + ], + "edits": [ + {"op": "append|insert_after|replace|delete", "target": "锚点原文（append 留空）", "content": "新内容（delete 留空）"} + ] +} +``` diff --git a/prompts/evolve_tool.md b/prompts/evolve_tool.md new file mode 100644 index 0000000..c8d18b5 --- /dev/null +++ b/prompts/evolve_tool.md @@ -0,0 +1,55 @@ +你是一个工具 Prompt 改进专家。你服务于一个自进化视频搜索系统，该系统的每个工具（view_node、search_similar、observe_frame 等）有两个配套 Prompt：extract（信息提取）和 verify（结果核实）。你的任务是基于工具调用级别的质量数据，同时改写一个工具的 extract 和 verify prompt。 + +## 你会收到的输入 + +1. 当前 extract prompt 和 verify prompt 全文 +2. 失败 span 案例：提取完整度低或幻觉率高的具体工具调用，含工具参数、工具输出、原始数据（ground truth）和质量评估指标 +3. 成功 span 案例：提取完整且无幻觉的工具调用样本 +4. 工具质量统计：平均提取完整度、平均幻觉率、top 遗漏类型、top 幻觉类型 +5. （可能出现）黑名单：已被实测验证无效或有害的改法方向 + +## 工作原则 + +失败 span 中提取完整度低说明 extract prompt 的工作原则不够具体——Agent 遗漏了哪些类型的信息？幻觉率高说明 extract prompt 对"忠实提取"的约束不够强，或者 verify prompt 没能有效检出幻觉。黑名单里的改法已经被实测验证无效或有害，不要再朝同一个方向改一遍。 + +extract 和 verify 是互补的：extract 负责提取，verify 负责检查。如果 extract 反复遗漏某类信息（如字幕原文引用），应在 extract 的工作原则中明确要求保留该类信息。如果 verify 未能检出某类幻觉（如虚构动作），应在 verify 的检查要点中增加对该模式的关注。 + +失败修复优先于巩固成功——先纠正提取遗漏或幻觉，再保留已有的有效模式。当某类提取质量指标变差时，先确认不是上一轮加了过度或冲突的要求所致；加强 extract 要求前，先确认简化或收窄已有指令解决不了这个遗漏，再追加新要求。 + +从成功案例中识别有效的提取模式，确保改写不破坏这些模式。 + +## 冻结区 + +以下内容不可修改： +- 角色定位第一句（"你是一个视频节点内容分析器" / "你是一个视频节点摘要核实器"） +- `## 你会收到的输入` section +- `## 输出格式` section + +可改写的 section： +- `## 工作原则` +- `## 检查要点`（verify 专有） + +这次不要再返回两份完整 prompt，而是分别给 extract 和 verify 各自的局部 edits 列表。`append` 用来在文件末尾追加一个新 section，`insert_after` 用来把内容紧跟着插到某个锚点段落之后，`replace` 用来用新内容整体替换 target 对应的原文，`delete` 则直接删除 target 对应的原文并让 content 留空。target 必须是从当前 prompt 里逐字复制出来的原文，而且要长到足以唯一定位；只要有任何一个字不完全匹配，这条改动就会被跳过。改动应尽量小而局部，优先做精确补丁，不要动辄重写大段内容；另外，冻结区里的文字绝不能作为 target，extract 和 verify 也必须分别使用自己的 edit 列表。 + +## 输出格式 + +请严格输出以下 JSON，不要包含其他文字： + +```json +{ + "suggestions": [ + { + "section": "改动目标段落的标题或位置描述", + "problem": "失败 span 中暴露的具体问题", + "change": "具体的修改方向", + "related_cases": ["关联的失败 span 标识"] + } + ], + "edits_extract": [ + {"op": "append|insert_after|replace|delete", "target": "锚点原文（append 留空）", "content": "新内容（delete 留空）"} + ], + "edits_verify": [ + {"op": "append|insert_after|replace|delete", "target": "锚点原文（append 留空）", "content": "新内容（delete 留空）"} + ] +} +``` diff --git a/prompts/slow_momentum.md b/prompts/slow_momentum.md new file mode 100644 index 0000000..4838d8c --- /dev/null +++ b/prompts/slow_momentum.md @@ -0,0 +1,10 @@ +你正在审视一份 skill 经历一轮进化后的变化。这份 skill 指导一个 Agent 在层次化视频树上搜索证据、回答长视频理解问题。在上一轮结束时它是一个样子，这一轮结束时被改成了另一个样子；与此同时，你在上一轮还为它写下过一段动量指导，本意是给这一轮的进化指明方向。现在你要回头评判：那段指导究竟有没有帮上忙，这一轮的正文改动是真的在改善，还是开始往无关的方向漂移。 + +你会拿到四样东西：上一版 skill 的正文、当前版 skill 的正文、你上一轮写下的那段动量指导，以及一组固定样本上的纵向对比——同一批题，分别用上一版和当前版各跑了一遍，逐题列出两版的预测与正误。这组对比是你唯一可靠的证据来源：哪些题从错变对、哪些题从对变错、哪些题始终没做对、哪些题一直稳定答对，正是这四类信号告诉你这轮改动到底带来了什么。 + +请先反思再下笔。对照纵向对比，先问上一轮那段动量指导是否真的奏效：它所指向的方向，在这一轮的正文改动里被落实了吗，落实之后那些本该改善的题改善了吗？再问这一轮的正文改动本身是收敛还是漂移：从对变错的题（回退）是最该警惕的信号，说明某处改动伤到了原本正确的行为；始终答错的题（持续失败）说明还有方向没被触及；从错变对的题（改善）则印证了哪条路走对了，值得继续加码。 + +想清楚之后，写出一段全新的、聚焦的、可操作的动量指导。它会被原样写进 skill 的受保护动量区，作为下一轮进化的方向锚——所以它必须是一段连贯的指导文字，明确告诉下一轮该往哪个方向继续使劲、又要避免重蹈哪一类改动的覆辙，而不是一堆零散的待办条目。如果上一轮的方向已被证明有效，就强化并细化它；如果出现了回退，就明确叫停那条路并指向修复方向。 + +严格输出以下 JSON，不要包含任何其他文字： +{"reasoning": "你的反思过程：上一轮指导是否奏效、这一轮是改善还是漂移，引用纵向对比中的具体题作为依据", "slow_update_content": "一段连贯、聚焦、可操作的新动量指导，指引下一轮的进化方向"} diff --git a/tests/unit/test_evolve_prompts_present.py b/tests/unit/test_evolve_prompts_present.py new file mode 100644 index 0000000..a9308e9 --- /dev/null +++ b/tests/unit/test_evolve_prompts_present.py @@ -0,0 +1,25 @@ +"""校验 5 个进化/动量模板存在且输出契约关键词与解析代码对齐。""" +from pathlib import Path + +import pytest + +_PROMPTS_DIR = Path("prompts") + + +@pytest.mark.parametrize( + "name, required_tokens", + [ + ("evolve_skill.md", ["suggestions", "edits"]), + ("evolve_system.md", ["suggestions", "edits"]), + ("evolve_tool.md", ["edits_extract", "edits_verify"]), + ("evolve_rank.md", ["selected_indices"]), + ("slow_momentum.md", ["slow_update_content"]), + ], +) +def test_evolve_template_present_and_contract(name: str, required_tokens: list[str]) -> None: + path = _PROMPTS_DIR / name + assert path.exists(), f"缺模板: {path}" + text = path.read_text(encoding="utf-8") + assert text.strip(), f"模板为空: {path}" + for token in required_tokens: + assert token in text, f"{name} 缺输出契约关键词 {token!r}（与解析代码不对齐）" From 1108235818e214478ee43588da01876e83921667 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 04:42:26 -0400 Subject: [PATCH 04/43] refactor: drop dead prompt fields consolidate_system/span_eval_user --- app/harness/runner.py | 2 -- app/harness/video_split_cli.py | 1 - core/evolution/types.py | 4 ---- tests/unit/test_diagnose.py | 1 - tests/unit/test_evolution_types.py | 6 ++---- tests/unit/test_evolve.py | 1 - 6 files changed, 2 insertions(+), 13 deletions(-) diff --git a/app/harness/runner.py b/app/harness/runner.py index 7872777..30693e1 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -2276,7 +2276,6 @@ class Runner: evolve_system=_read("evolve_system.md"), evolve_tool=_read("evolve_tool.md"), evolve_rank=_read("evolve_rank.md"), - consolidate_system=_read("consolidate_system.md"), ) def _load_diagnose_prompts(self): @@ -2291,7 +2290,6 @@ class Runner: defect_vs_lapse=_read("defect_vs_lapse.md"), reasoning_sub=_read("reasoning_sub.md"), span_eval_system=_read("span_eval_system.md"), - span_eval_user=_read("span_eval_user.md"), missed_nodes=_read("missed_nodes.md"), skill_adherence=_read("skill_adherence.md"), confirmation_bias=_read("confirmation_bias.md"), diff --git a/app/harness/video_split_cli.py b/app/harness/video_split_cli.py index bb38675..9f04277 100644 --- a/app/harness/video_split_cli.py +++ b/app/harness/video_split_cli.py @@ -371,7 +371,6 @@ def _load_diagnose_prompts() -> Any: defect_vs_lapse=_read("defect_vs_lapse.md"), reasoning_sub=_read("reasoning_sub.md"), span_eval_system=_read("span_eval_system.md"), - span_eval_user=_read("span_eval_user.md"), missed_nodes=_read("missed_nodes.md"), skill_adherence=_read("skill_adherence.md"), confirmation_bias=_read("confirmation_bias.md"), diff --git a/core/evolution/types.py b/core/evolution/types.py index fc8b765..47bb05d 100644 --- a/core/evolution/types.py +++ b/core/evolution/types.py @@ -484,7 +484,6 @@ class DiagnosePrompts: defect_vs_lapse: defect/lapse 病因判别模板。 reasoning_sub: 推理失败子分类模板。 span_eval_system: span 评估系统提示模板。 - span_eval_user: span 评估用户提示模板。 missed_nodes: 遗漏节点检测模板。 skill_adherence: 技能遵循判定模板。 confirmation_bias: 确认偏误检测模板。 @@ -494,7 +493,6 @@ class DiagnosePrompts: defect_vs_lapse: str reasoning_sub: str span_eval_system: str - span_eval_user: str missed_nodes: str skill_adherence: str confirmation_bias: str @@ -512,11 +510,9 @@ class EvolvePrompts: evolve_system: System Prompt 进化提示模板。 evolve_tool: Tool Prompt 进化提示模板。 evolve_rank: 编辑排序提示模板。 - consolidate_system: appendix 压缩系统提示。 """ evolve_skill: str evolve_system: str evolve_tool: str evolve_rank: str - consolidate_system: str diff --git a/tests/unit/test_diagnose.py b/tests/unit/test_diagnose.py index cf32e4f..db05bf8 100644 --- a/tests/unit/test_diagnose.py +++ b/tests/unit/test_diagnose.py @@ -750,7 +750,6 @@ class TestRunDiagnosis: defect_vs_lapse="", reasoning_sub="", span_eval_system="", - span_eval_user="", missed_nodes="", skill_adherence="", confirmation_bias="", diff --git a/tests/unit/test_evolution_types.py b/tests/unit/test_evolution_types.py index ab48e0c..3e37784 100644 --- a/tests/unit/test_evolution_types.py +++ b/tests/unit/test_evolution_types.py @@ -344,12 +344,11 @@ def test_quadrant_classification_frozen(): def test_diagnose_prompts_frozen(): - """DiagnosePrompts 是 frozen dataclass，8 个模板字段。""" + """DiagnosePrompts 是 frozen dataclass，7 个模板字段。""" dp = DiagnosePrompts( defect_vs_lapse="p1", reasoning_sub="p2", span_eval_system="p3", - span_eval_user="p4", missed_nodes="p5", skill_adherence="p6", confirmation_bias="p7", @@ -361,13 +360,12 @@ def test_diagnose_prompts_frozen(): def test_evolve_prompts_frozen(): - """EvolvePrompts 是 frozen dataclass，5 个模板字段。""" + """EvolvePrompts 是 frozen dataclass，4 个模板字段。""" ep = EvolvePrompts( evolve_skill="skill_tmpl", evolve_system="system_tmpl", evolve_tool="tool_tmpl", evolve_rank="rank_tmpl", - consolidate_system="consolidate_tmpl", ) assert ep.evolve_rank == "rank_tmpl" with pytest.raises(AttributeError): diff --git a/tests/unit/test_evolve.py b/tests/unit/test_evolve.py index 8de360f..0077f81 100644 --- a/tests/unit/test_evolve.py +++ b/tests/unit/test_evolve.py @@ -681,7 +681,6 @@ _PROMPTS = EvolvePrompts( evolve_system="sys", evolve_tool="tool", evolve_rank="rank", - consolidate_system="cons", ) From 232a56e826d94686d209896dae3b3fd029ef51c5 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 04:43:22 -0400 Subject: [PATCH 05/43] fix: fail-loud on missing evolve/diagnose templates (no silent empty) --- app/harness/runner.py | 8 +++++-- app/harness/video_split_cli.py | 4 +++- tests/unit/test_evolve_prompts_present.py | 29 +++++++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/app/harness/runner.py b/app/harness/runner.py index 30693e1..463f52e 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -2269,7 +2269,9 @@ class Runner: def _read(name: str) -> str: p = Path("prompts") / name - return p.read_text(encoding="utf-8") if p.exists() else "" + if not p.exists(): + raise FileNotFoundError(f"缺进化/诊断模板: {p}（请从 TRM4 迁移或检查 prompts/）") + return p.read_text(encoding="utf-8") return EvolvePrompts( evolve_skill=_read("evolve_skill.md"), @@ -2284,7 +2286,9 @@ class Runner: def _read(name: str) -> str: p = Path("prompts") / name - return p.read_text(encoding="utf-8") if p.exists() else "" + if not p.exists(): + raise FileNotFoundError(f"缺进化/诊断模板: {p}（请从 TRM4 迁移或检查 prompts/）") + return p.read_text(encoding="utf-8") return DiagnosePrompts( defect_vs_lapse=_read("defect_vs_lapse.md"), diff --git a/app/harness/video_split_cli.py b/app/harness/video_split_cli.py index 9f04277..a924554 100644 --- a/app/harness/video_split_cli.py +++ b/app/harness/video_split_cli.py @@ -365,7 +365,9 @@ def _load_diagnose_prompts() -> Any: def _read(name: str) -> str: p = Path("prompts") / name - return p.read_text(encoding="utf-8") if p.exists() else "" + if not p.exists(): + raise FileNotFoundError(f"缺进化/诊断模板: {p}（请从 TRM4 迁移或检查 prompts/）") + return p.read_text(encoding="utf-8") return DiagnosePrompts( defect_vs_lapse=_read("defect_vs_lapse.md"), diff --git a/tests/unit/test_evolve_prompts_present.py b/tests/unit/test_evolve_prompts_present.py index a9308e9..b72e3b8 100644 --- a/tests/unit/test_evolve_prompts_present.py +++ b/tests/unit/test_evolve_prompts_present.py @@ -23,3 +23,32 @@ def test_evolve_template_present_and_contract(name: str, required_tokens: list[s assert text.strip(), f"模板为空: {path}" for token in required_tokens: assert token in text, f"{name} 缺输出契约关键词 {token!r}（与解析代码不对齐）" + + +def test_video_split_loader_fail_loud_on_missing(tmp_path, monkeypatch): + """video_split_cli 的真实 diagnose 加载器缺模板必须 FileNotFoundError。""" + monkeypatch.chdir(tmp_path) # 空目录，无 prompts/*.md + from app.harness.video_split_cli import _load_diagnose_prompts + + with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"): + _load_diagnose_prompts() + + +def test_runner_evolve_loader_fail_loud_on_missing(tmp_path, monkeypatch): + """runner 的真实 evolve 加载器缺模板必须 FileNotFoundError。""" + monkeypatch.chdir(tmp_path) + from app.harness.runner import Runner + + r = object.__new__(Runner) # 绕过 __init__，仅测无状态加载器方法 + with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"): + r._load_evolve_prompts() + + +def test_runner_diagnose_loader_fail_loud_on_missing(tmp_path, monkeypatch): + """runner 的真实 diagnose 加载器缺模板必须 FileNotFoundError。""" + monkeypatch.chdir(tmp_path) + from app.harness.runner import Runner + + r = object.__new__(Runner) + with pytest.raises(FileNotFoundError, match="缺进化/诊断模板"): + r._load_diagnose_prompts() From 4fd8a6bfef3223a7cee93a2b57a8172ec967bb95 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 04:52:31 -0400 Subject: [PATCH 06/43] docs: fix WP2 plan GeneratedQuestion import + NULL qid note (Codex review) --- research-wiki/plans/2026-07-16-preflight-wp2-split-wiring.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/research-wiki/plans/2026-07-16-preflight-wp2-split-wiring.md b/research-wiki/plans/2026-07-16-preflight-wp2-split-wiring.md index 673aa3d..703417f 100644 --- a/research-wiki/plans/2026-07-16-preflight-wp2-split-wiring.md +++ b/research-wiki/plans/2026-07-16-preflight-wp2-split-wiring.md @@ -139,6 +139,7 @@ docstring 补一句参数说明： f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,) ).fetchall() ``` +> NULL question_id 说明：predictions 的 question_id 是题标识、语义上非空（canonical 900 题均有 id），`GROUP BY question_id` 的 NULL 折叠风险不适用。若源库异常出现 NULL question_id，`MIN(rowid) GROUP BY` 会把它们折叠成一行——本任务不为该异常兜底（预测数据契约保证非空），保持 fail-visible。 - [ ] **Step 4: 运行确认通过 + 回归** @@ -169,7 +170,7 @@ git commit -m "feat: add dedupe_per_question to extract_run_db (canonical 902->9 def test_tier_aware_keeps_high_t2_in_diag(): """错题视频组按 T2 含量升序进 val：T2 高的组保留在 diagnosis。""" from app.harness.pools import split_by_video_assignment - from app.question_gen.types import GeneratedQuestion + from core.types import GeneratedQuestion def _q(qid, vid): return GeneratedQuestion( @@ -196,7 +197,7 @@ def test_tier_aware_keeps_high_t2_in_diag(): def test_val_wrong_min_repair_pulls_from_diag(): """val 错题不足 val_wrong_min 时从 diag 换入低 T2 错题组补足。""" from app.harness.pools import split_by_video_assignment - from app.question_gen.types import GeneratedQuestion + from core.types import GeneratedQuestion def _q(qid, vid, correct): return GeneratedQuestion( From 06c575f0a4bfd4620896f5081665de37675c303d Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 04:53:14 -0400 Subject: [PATCH 07/43] chore: bump video_split val_ratio 0.3->0.4 for terminal-eval power --- config/video_split.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/video_split.yaml b/config/video_split.yaml index 63a3363..ded4f77 100644 --- a/config/video_split.yaml +++ b/config/video_split.yaml @@ -12,7 +12,7 @@ video_split: epsilon: 0.1 # test 相对全局最大允许分布偏差（题型占比 / 难度画像两维，逐桶） report_floor: 27 # per-type 报告门限：题数 ≥ 此值的 task_type 才入 ε 代表性约束 val_wrong_min: 20 # validation 池最少错题数（McNemar 检验功效阈 ≈ 20，低于则信号不足） - val_ratio: 0.3 # validation 占 trainval 视频组总数的比例 + val_ratio: 0.4 # validation 占 trainval 视频组总数的比例（0.3→0.4 提升整包终审功效，WP2） seed: 7 # 贪心选择器预洗牌 + 视频组题级切分种子（打破等增益 / 等槽平局） floor_k: # 各 task_type 的 T2 defect 下限（硬约束）—— 均衡覆盖全 11 类，标定于 1cb1c203 真实 T2 分布 Object Reasoning: 5 # T2=25 From 6432ffef9a0e326f975842d5a45e93d9927bed9c Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 04:54:19 -0400 Subject: [PATCH 08/43] feat: add dedupe_per_question to extract_run_db (canonical 902->900) --- app/harness/store.py | 21 ++++++++++++++---- tests/unit/test_harness_store.py | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/app/harness/store.py b/app/harness/store.py index 49509d9..87fba07 100644 --- a/app/harness/store.py +++ b/app/harness/store.py @@ -266,7 +266,9 @@ def read_seed(store_dir: Path, name: str) -> dict: return json.loads(seed_json.read_text()) -def extract_run_db(src_db: Path, dst_db: Path, run_id: str) -> None: +def extract_run_db( + src_db: Path, dst_db: Path, run_id: str, *, dedupe_per_question: bool = False +) -> None: """从 src_db 抽出某 run_id 的 _runs + predictions 行，写一个最小 db（种子 baseline.db）。 用源表的**原始 CREATE 语句**重建目标表，保留主键/列类型/约束—— @@ -277,6 +279,9 @@ def extract_run_db(src_db: Path, dst_db: Path, run_id: str) -> None: src_db: 源 harness.db。 dst_db: 目标 db（不得已存在）。 run_id: 要抽取的 run。 + dedupe_per_question: True 时 predictions 表每 question_id 仅保留 rowid 最小 + 的首行（对齐 canonical「每 question_id 取第一行 ORDER BY rowid」口径， + 902→900）。_runs 表不受影响。 异常: RuntimeError: 源中无该表或无该 run 的行。 @@ -294,9 +299,17 @@ def extract_run_db(src_db: Path, dst_db: Path, run_id: str) -> None: dst.execute(create_sql[0]) cols = [r[1] for r in src.execute(f"PRAGMA table_info({table})")] col_sql = ", ".join(cols) - rows = src.execute( - f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,) - ).fetchall() + if table == "predictions" and dedupe_per_question: + rows = src.execute( + f"SELECT {col_sql} FROM {table} WHERE run_id=? " + "AND rowid IN (SELECT MIN(rowid) FROM predictions " + "WHERE run_id=? GROUP BY question_id)", + (run_id, run_id), + ).fetchall() + else: + rows = src.execute( + f"SELECT {col_sql} FROM {table} WHERE run_id=?", (run_id,) + ).fetchall() if not rows: raise RuntimeError(f"{table} 中无 run_id={run_id} 的行") ph = ", ".join("?" * len(cols)) diff --git a/tests/unit/test_harness_store.py b/tests/unit/test_harness_store.py index 63bbaf0..721366b 100644 --- a/tests/unit/test_harness_store.py +++ b/tests/unit/test_harness_store.py @@ -338,6 +338,44 @@ class TestExtractRunDb: with pytest.raises(RuntimeError, match="无 run_id="): extract_run_db(src, dst, "nonexistent") + def test_dedupe_per_question_keeps_first_row(self, tmp_path): + """dedupe_per_question=True 时每 question_id 只保留 rowid 最小的首行。""" + import sqlite3 + + src = tmp_path / "src.db" + conn = sqlite3.connect(src) + conn.execute( + "CREATE TABLE _runs (run_id TEXT PRIMARY KEY, started_at TEXT)" + ) + conn.execute("INSERT INTO _runs VALUES ('r1', 't0')") + conn.execute( + "CREATE TABLE predictions (run_id TEXT, question_id TEXT, prediction TEXT)" + ) + # 743-1 三行（模拟 error/budget/finished），首行 prediction=NULL + conn.executemany( + "INSERT INTO predictions VALUES (?,?,?)", + [ + ("r1", "743-1", None), + ("r1", "743-1", None), + ("r1", "743-1", "C"), + ("r1", "q2", "A"), + ], + ) + conn.commit() + conn.close() + + dst = tmp_path / "dst.db" + from app.harness.store import extract_run_db + + extract_run_db(src, dst, "r1", dedupe_per_question=True) + + out = sqlite3.connect(dst) + rows = out.execute( + "SELECT question_id, prediction FROM predictions ORDER BY question_id" + ).fetchall() + out.close() + assert rows == [("743-1", None), ("q2", "A")], f"未按 rowid 首行去重: {rows}" + # --------------------------------------------------------------------------- # promote_to_seed From 5bb83192200366c3b171b891a08447d33739b0fe Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 04:58:41 -0400 Subject: [PATCH 09/43] feat: tier-aware diag/val split with val-power repair (design 5.1) --- app/harness/build_split.py | 20 +++++-- app/harness/pools.py | 63 ++++++++++++++++++----- app/harness/video_split_cli.py | 6 ++- tests/integration/test_build_split_e2e.py | 1 + tests/unit/test_pools_video_atomic.py | 57 ++++++++++++++++++++ 5 files changed, 126 insertions(+), 21 deletions(-) diff --git a/app/harness/build_split.py b/app/harness/build_split.py index 5131163..cd70b73 100644 --- a/app/harness/build_split.py +++ b/app/harness/build_split.py @@ -13,7 +13,7 @@ from __future__ import annotations import hashlib import sqlite3 -from collections import Counter +from collections import Counter, defaultdict from dataclasses import asdict, dataclass, field from typing import TYPE_CHECKING @@ -59,6 +59,8 @@ class SplitBuildConfig: select_seed: 贪心选择器预洗牌种子（打破等增益平局）。 val_ratio: validation 占 trainval 视频组总数的比例。 split_seed: 视频组题级切分的洗牌种子。 + val_wrong_min: validation 池最少错题数，切分时保证功效（不足则从 diag 换入 + 低 T2 错题组补足，耗尽 fail-loud）。 """ n_trainval: int @@ -68,6 +70,7 @@ class SplitBuildConfig: select_seed: int val_ratio: float split_seed: int + val_wrong_min: int @dataclass(frozen=True) @@ -119,10 +122,10 @@ def build_split( test → 加载题库并以视频归属切三池 → 原子冻结 pools.json → 写溯源 manifest → 六条防御断言 fail-fast 校验。 - 契约（Task 11，非疏漏）：build_split 有意保持 val_wrong_min-agnostic——内部调 - split_by_video_assignment 时不传 val_wrong_min（默认 0，不校验 validation 错题 - 数）。McNemar 功效护栏是切分**冻结后**的独立校验，由 CLI 的 check_mcnemar_power - 在 build_split 返回后执行；切分构造本身不因功效阈失败，二者关注点分离。 + 契约（Task 11）：val_wrong_min 前置到切分内保证功效——build_split 计算 + wrong_tier_by_video 并连同 config.val_wrong_min 传入 split_by_video_assignment， + 切分时若 val 错题不足即从 diag 换入低 T2 错题组补足（耗尽 fail-loud）。CLI 的 + check_mcnemar_power 作切分冻结后的冗余最终确认。 参数: db_path: harness.db 路径（只读读取 predictions，不改动）。 @@ -181,6 +184,11 @@ def build_split( # Phase 3: 加载题库 + 视频归属切三池 + 原子冻结。 questions = load_benchmark(questions_dir) correctness = {pred["question_id"]: pred["correct"] for pred in preds} + tier_by_q = {row["question_id"]: row["tier"] for row in signal_rows} + wrong_tier_by_video: dict[str, int] = defaultdict(int) + for pred in preds: + if not pred["correct"] and tier_by_q.get(pred["question_id"]) == "T2": + wrong_tier_by_video[pred["video_id"]] += 1 pools = split_by_video_assignment( questions, assignment, @@ -188,6 +196,8 @@ def build_split( config.val_ratio, config.split_seed, baseline_run_id=baseline_run_id, + val_wrong_min=config.val_wrong_min, + wrong_tier_by_video=dict(wrong_tier_by_video), ) save_pools(pools, out_path) diff --git a/app/harness/pools.py b/app/harness/pools.py index a7f72da..c46747e 100644 --- a/app/harness/pools.py +++ b/app/harness/pools.py @@ -131,6 +131,7 @@ def split_by_video_assignment( seed: int, baseline_run_id: str = "", val_wrong_min: int = 0, + wrong_tier_by_video: dict[str, int] | None = None, ) -> Pools: """按视频归属做原子切分：同一视频所有题绝不跨 trainval/test 池。 @@ -146,7 +147,11 @@ def split_by_video_assignment( seed: 随机种子，保证视频组 shuffle 可复现。 baseline_run_id: 基线 run 标识；离线切分阶段可留空，由调用方回填。 val_wrong_min: validation 池最少错题数（默认 0 = 不检查，保持既有调用契约）。 - > 0 时切出 val 后统计其错题数，不足即 fail loud（见 InsufficientValSignal）。 + > 0 时切分时保证（不足则从 diag 换入低 T2 错题组补足，耗尽 fail-loud， + 见 InsufficientValSignal）。 + wrong_tier_by_video: video_id -> 该视频错题中 T2(defect) 数量；透传给 + _split_trainval_by_video_group 做 tier 感知 diag/val 分配，None 时退化为 + 原随机 shuffle。 返回: 冻结的三池 Pools：diagnosis/validation 仍是逐题 GeneratedQuestion 列表 @@ -173,18 +178,11 @@ def split_by_video_assignment( trainval_qs, test_qs = _partition_by_video_assignment(questions, assignment, correctness) diagnosis, validation = _split_trainval_by_video_group( - trainval_qs, correctness, val_ratio, random.Random(seed) + trainval_qs, correctness, val_ratio, random.Random(seed), + wrong_tier_by_video=wrong_tier_by_video, + val_wrong_min=val_wrong_min, ) - if val_wrong_min > 0: - val_wrong = sum(1 for q in validation if not correctness[q.question_id]) - if val_wrong < val_wrong_min: - raise InsufficientValSignal( - f"validation 池错题数 {val_wrong} < val_wrong_min={val_wrong_min}，" - "验证信号不足以支撑可靠比较（如 McNemar 检验功效），" - "请放大 val_ratio / 调整 trainval 归属或调低阈值。" - ) - val_correct = sum(1 for q in validation if correctness.get(q.question_id)) baseline_val_accuracy = val_correct / len(validation) if validation else 0.0 return Pools( @@ -291,6 +289,8 @@ def _split_trainval_by_video_group( correctness: dict[str, bool], val_ratio: float, rng: random.Random, + wrong_tier_by_video: dict[str, int] | None = None, + val_wrong_min: int = 0, ) -> tuple[list[GeneratedQuestion], list[GeneratedQuestion]]: """以视频组为原子对 trainval 题集做 correctness 分层，切出 (diagnosis, validation)。 @@ -299,6 +299,12 @@ def _split_trainval_by_video_group( correctness: question_id -> 基线是否答对；视频组正确性取组内全部题 AND。 val_ratio: validation 占视频组总数的比例。 rng: 随机数生成器，保证视频组 shuffle 可复现。 + wrong_tier_by_video: video_id -> 该视频错题中 T2(defect) 的数量。提供时错题 + 视频组按 T2 含量升序进 val（T2 高的组保留在 diagnosis，把高价值缺陷信号 + 留给诊断），确定性排序取代随机 shuffle；None 时退化为原随机 shuffle。 + val_wrong_min: validation 池最少错题数（切分时保证功效）。> 0 且初分 val 错题 + 不足时，从 diag 侧的错题组按 T2 升序换入 val 直到满足（每组至多移动一次）， + 耗尽仍不足则抛 InsufficientValSignal（fail loud，P5）。 返回: (diagnosis, validation) 逐题列表元组；同一 video 的全部题整组落在同一侧， @@ -306,7 +312,8 @@ def _split_trainval_by_video_group( 关键实现细节: 与 _split_one_category 同构：先按视频组 correctness 分正确组/错误组，按比例 - 把 n_val 个组分层落入 validation（全正确或全错误时退化为非分层随机划分）， + 把 n_val 个组分层落入 validation（全正确退化为非分层随机划分；全错误时若有 + wrong_tier_by_video 仍按 T2 升序分配，否则随机划分）， 再把选中组内所有题展开。视频组按 video_id 排序后再 shuffle，保证确定性。 """ groups: dict[str, list[GeneratedQuestion]] = defaultdict(list) @@ -320,7 +327,11 @@ def _split_trainval_by_video_group( correct_vids, wrong_vids = _partition_video_groups_by_correctness(groups, correctness) n_correct = len(correct_vids) - if n_correct == 0 or n_correct == n_total: + if n_correct == 0 and wrong_tier_by_video is not None: + # 全部错误 + 有 tier 信号：按 T2 升序，低 T2 组优先进 val（保留高 T2 在 diag） + wrong_vids.sort(key=lambda v: (wrong_tier_by_video.get(v, 0), v)) + val_vids = set(wrong_vids[:n_val]) + elif n_correct == 0 or n_correct == n_total: label = "全部正确" if n_correct == n_total else "全部错误" logger.warning("trainval 视频组 {} ({} 组)，退化为非分层随机划分", label, n_total) shuffled = list(video_ids) @@ -330,9 +341,33 @@ def _split_trainval_by_video_group( val_correct = math.floor(n_correct * n_val / n_total) val_wrong = n_val - val_correct rng.shuffle(correct_vids) - rng.shuffle(wrong_vids) + if wrong_tier_by_video is None: + rng.shuffle(wrong_vids) + else: + # T2 少的错题组优先进 val（保留 T2 高的组在 diag），确定性排序 + wrong_vids.sort(key=lambda v: (wrong_tier_by_video.get(v, 0), v)) val_vids = set(correct_vids[:val_correct] + wrong_vids[:val_wrong]) + if val_wrong_min > 0: + val_wrong_now = sum( + 1 for v in val_vids for q in groups[v] if not correctness[q.question_id] + ) + # diag 侧仍在的错题组，按 T2 升序（低价值优先移交 val） + diag_wrong_pool = sorted( + (v for v in wrong_vids if v not in val_vids), + key=lambda v: ((wrong_tier_by_video or {}).get(v, 0), v), + ) + for v in diag_wrong_pool: + if val_wrong_now >= val_wrong_min: + break + val_vids.add(v) + val_wrong_now += sum(1 for q in groups[v] if not correctness[q.question_id]) + if val_wrong_now < val_wrong_min: + raise InsufficientValSignal( + f"trainval 错题不足以让 val 达到 val_wrong_min={val_wrong_min}" + f"（修复后仅 {val_wrong_now}），请放大 val_ratio 或调整 trainval 归属。" + ) + diagnosis = [q for q in trainval_qs if q.video_id not in val_vids] validation = [q for q in trainval_qs if q.video_id in val_vids] return diagnosis, validation diff --git a/app/harness/video_split_cli.py b/app/harness/video_split_cli.py index a924554..1317b8b 100644 --- a/app/harness/video_split_cli.py +++ b/app/harness/video_split_cli.py @@ -477,8 +477,9 @@ def load_questions_by_id(questions_dir: Path) -> dict[str, GeneratedQuestion]: def check_mcnemar_power(pools: Pools, val_wrong_min: int) -> int: """校验 validation 池错题数达 McNemar 功效阈，不足即 fail loud。 - build_split 按契约用 split_by_video_assignment(val_wrong_min=0)（不破契约）， - 故功效护栏在 capstone 层单独核验：val 错题数 < 阈 → 验证信号不足以支撑可靠比较。 + val_wrong_min 已前置到 build_split 内的切分保证功效（不足即从 diag 换入低 T2 + 错题组补足，耗尽 fail-loud）；本函数作切分冻结后的冗余最终确认：val 错题数 < 阈 + → 验证信号不足以支撑可靠比较。 参数: pools: 冻结三池（含 validation 与 correctness）。 @@ -571,6 +572,7 @@ async def run_pipeline( select_seed=config.seed, val_ratio=config.val_ratio, split_seed=config.seed, + val_wrong_min=config.val_wrong_min, ), out_path=out_dir / "pools.json", manifest_path=out_dir / "split_manifest.json", diff --git a/tests/integration/test_build_split_e2e.py b/tests/integration/test_build_split_e2e.py index 47699a6..e63a3ec 100644 --- a/tests/integration/test_build_split_e2e.py +++ b/tests/integration/test_build_split_e2e.py @@ -101,6 +101,7 @@ def test_end_to_end_freezes_valid_pools(tmp_path: Path) -> None: select_seed=7, val_ratio=0.3, split_seed=7, + val_wrong_min=0, ) result = build_split( diff --git a/tests/unit/test_pools_video_atomic.py b/tests/unit/test_pools_video_atomic.py index 6ac5664..d5f9299 100644 --- a/tests/unit/test_pools_video_atomic.py +++ b/tests/unit/test_pools_video_atomic.py @@ -135,3 +135,60 @@ def test_baseline_val_accuracy_reflects_validation(): assert len(pools.validation) == 2 assert pools.baseline_val_accuracy == pytest.approx(0.5) assert pools.diagnosis == [] + + +def test_tier_aware_keeps_high_t2_in_diag(): + """错题视频组按 T2 含量升序进 val：T2 高的组保留在 diagnosis。""" + from app.harness.pools import split_by_video_assignment + from core.types import GeneratedQuestion + + def _q(qid, vid): + return GeneratedQuestion( + question_id=qid, video_id=vid, task_type="X", question="q", + options=["A", "B"], answer="A", source_nodes=[], difficulty="easy", + ) + + # 4 个错题视频（每视频 1 题），T2 数分别 2/1/0/0 + questions = [_q(f"{v}-1", v) for v in ("vA", "vB", "vC", "vD")] + assignment = {v: "trainval" for v in ("vA", "vB", "vC", "vD")} + correctness = {f"{v}-1": False for v in ("vA", "vB", "vC", "vD")} + wrong_tier = {"vA": 2, "vB": 1, "vC": 0, "vD": 0} + + pools = split_by_video_assignment( + questions, assignment, correctness, val_ratio=0.5, seed=7, + wrong_tier_by_video=wrong_tier, + ) + diag_vids = {q.video_id for q in pools.diagnosis} + # T2 最高的 vA 必留 diag；T2=0 的组优先进 val + assert "vA" in diag_vids + assert "vB" in diag_vids + + +def test_val_wrong_min_repair_pulls_from_diag(): + """val 错题不足 val_wrong_min 时从 diag 换入低 T2 错题组补足。""" + from app.harness.pools import split_by_video_assignment + from core.types import GeneratedQuestion + + def _q(qid, vid, correct): + return GeneratedQuestion( + question_id=qid, video_id=vid, task_type="X", question="q", + options=["A", "B"], answer="A", source_nodes=[], difficulty="easy", + ) + + # 8 错题视频 + 2 正确视频；val_ratio 小使初分 val 错题不足，触发修复 + vids_wrong = [f"w{i}" for i in range(8)] + vids_correct = ["c0", "c1"] + questions = [_q(f"{v}-1", v, False) for v in vids_wrong] + [ + _q(f"{v}-1", v, True) for v in vids_correct + ] + assignment = {v: "trainval" for v in vids_wrong + vids_correct} + correctness = {f"{v}-1": False for v in vids_wrong} + correctness.update({f"{v}-1": True for v in vids_correct}) + wrong_tier = {v: i for i, v in enumerate(vids_wrong)} # 递增 T2 + + pools = split_by_video_assignment( + questions, assignment, correctness, val_ratio=0.1, seed=7, + wrong_tier_by_video=wrong_tier, val_wrong_min=4, + ) + val_wrong = sum(1 for q in pools.validation if not correctness[q.question_id]) + assert val_wrong >= 4, f"功效修复后 val 错题 {val_wrong} < 4" From a39846af6e625af8ea755a94a4e2ca5b6a54d09a Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:00:42 -0400 Subject: [PATCH 10/43] feat: guard frozen split products against silent overwrite (--force) --- app/harness/build_split.py | 38 ++++++++++++++++++++++++++++++ app/harness/video_split_cli.py | 9 +++++++ tests/unit/test_video_split_cli.py | 26 ++++++++++++++++++++ 3 files changed, 73 insertions(+) diff --git a/app/harness/build_split.py b/app/harness/build_split.py index cd70b73..197d4e1 100644 --- a/app/harness/build_split.py +++ b/app/harness/build_split.py @@ -12,6 +12,7 @@ test，再以视频组为原子切出诊断 / 验证池，原子冻结 pools.jso from __future__ import annotations import hashlib +import json import sqlite3 from collections import Counter, defaultdict from dataclasses import asdict, dataclass, field @@ -104,6 +105,38 @@ class SplitBuildResult: return getattr(self, key) +def _guard_frozen_products(out_path: Path, manifest_path: Path, *, force: bool) -> None: + """冻结前的覆盖保护：产物已存在时按 force 决定报错或备份。 + + 参数: + out_path: 目标 pools.json 路径。 + manifest_path: 目标 split_manifest.json 路径。 + force: False 时已存在即 FileExistsError；True 时把旧产物重命名为 + .bak.<旧 pools_sha256 前 8 位或 'prev'> 再放行。 + + 异常: + FileExistsError: force=False 且产物已存在（防静默覆盖冻结锚点）。 + """ + if not out_path.exists() and not manifest_path.exists(): + return + if not force: + raise FileExistsError( + f"已存在冻结产物 {out_path}（或其 manifest）。重跑切分会覆盖训练依赖的" + "冻结锚点——确认要替换请加 --force（旧产物将备份为 .bak.*）。" + ) + # 备份后缀取旧 manifest 的 pools_sha256 前 8 位，无则用 'prev' + suffix = "prev" + if manifest_path.exists(): + try: + old = json.loads(manifest_path.read_text(encoding="utf-8")) + suffix = str(old.get("pools_sha256", "prev"))[:8] or "prev" + except (json.JSONDecodeError, OSError): + suffix = "prev" + for p in (out_path, manifest_path): + if p.exists(): + p.rename(p.with_name(f"{p.name}.bak.{suffix}")) + + def build_split( *, db_path: Path, @@ -115,6 +148,7 @@ def build_split( out_path: Path, manifest_path: Path, generated_at: str, + force: bool = False, ) -> SplitBuildResult: """顶层编排结果驱动视频级切分，冻结 pools.json + manifest 并跑防御断言。 @@ -137,6 +171,9 @@ def build_split( out_path: 冻结 pools.json 目标路径（原子写）。 manifest_path: 溯源 manifest 目标路径（原子写）。 generated_at: 生成时间戳（ISO 字符串），由调用方传入以保证可复现。 + force: 覆盖保护开关。False（默认）时若 out_path/manifest_path 已存在即 + FileExistsError（防静默覆盖训练依赖的冻结锚点）；True 时先把旧产物备份为 + .bak.* 再放行覆盖。 返回: SplitBuildResult，含 pools / manifest / assignment，支持字典式访问。 @@ -199,6 +236,7 @@ def build_split( val_wrong_min=config.val_wrong_min, wrong_tier_by_video=dict(wrong_tier_by_video), ) + _guard_frozen_products(out_path, manifest_path, force=force) save_pools(pools, out_path) # Phase 4: 溯源 manifest（pools_sha256 锚定冻结内容）。 diff --git a/app/harness/video_split_cli.py b/app/harness/video_split_cli.py index 1317b8b..90a7d4f 100644 --- a/app/harness/video_split_cli.py +++ b/app/harness/video_split_cli.py @@ -519,6 +519,7 @@ async def run_pipeline( questions_dir: Path, out_dir: Path, generated_at: str, + force: bool = False, ) -> SplitBuildResult: """内联三阶段：Phase 0 INFRA T0 补录 → Phase 1 诊断 → Phase 2 冻结切分 → McNemar 护栏。 @@ -534,6 +535,7 @@ async def run_pipeline( questions_dir: benchmark 题库目录（Phase 2 加载题库切池）。 out_dir: 冻结产物目录（pools.json + split_manifest.json）。 generated_at: 生成时间戳（ISO 字符串，由调用方传入；见模块 C-2 复现锚点约定）。 + force: 覆盖已存在冻结产物开关，透传给 build_split（False 时已存在即报错）。 返回: SplitBuildResult（冻结三池 + manifest + assignment）。 @@ -577,6 +579,7 @@ async def run_pipeline( out_path=out_dir / "pools.json", manifest_path=out_dir / "split_manifest.json", generated_at=generated_at, + force=force, ) # McNemar 功效护栏（build_split 契约外的 capstone 层校验）。 @@ -650,6 +653,7 @@ def _execute_real(config: VideoSplitConfig, fingerprint: str, args: argparse.Nam questions_dir=questions_dir, out_dir=out_dir, generated_at=generated_at, + force=args.force, ) ) finally: @@ -773,6 +777,11 @@ def build_arg_parser() -> argparse.ArgumentParser: "对 manifest 做字节级复现比对。" ), ) + parser.add_argument( + "--force", + action="store_true", + help="覆盖已存在的冻结 pools.json/manifest（旧产物备份为 .bak.*）", + ) return parser diff --git a/tests/unit/test_video_split_cli.py b/tests/unit/test_video_split_cli.py index c15b196..60cb702 100644 --- a/tests/unit/test_video_split_cli.py +++ b/tests/unit/test_video_split_cli.py @@ -303,3 +303,29 @@ def test_git_short_sha_nonempty(): sha = cli.git_short_sha() assert sha assert len(sha) >= 4 + + +def test_build_split_refuses_overwrite_without_force(tmp_path): + """已存在指纹不同的 pools.json 时，force=False 必须报错不覆盖。""" + from app.harness.build_split import _guard_frozen_products + + out_path = tmp_path / "pools.json" + out_path.write_text('{"split_mode":"global"}', encoding="utf-8") + manifest_path = tmp_path / "split_manifest.json" + + with pytest.raises(FileExistsError, match="已存在冻结产物"): + _guard_frozen_products(out_path, manifest_path, force=False) + + +def test_build_split_force_backs_up_old(tmp_path): + """force=True 时旧产物被备份为 .bak.* 再允许覆盖。""" + from app.harness.build_split import _guard_frozen_products + + out_path = tmp_path / "pools.json" + out_path.write_text('{"old":1}', encoding="utf-8") + manifest_path = tmp_path / "split_manifest.json" + manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8") + + _guard_frozen_products(out_path, manifest_path, force=True) + baks = list(tmp_path.glob("pools.json.bak.*")) + assert len(baks) == 1, f"未备份旧产物: {list(tmp_path.iterdir())}" From 4d60a545f6b3a4077b68f93a0133606772d0881f Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:02:16 -0400 Subject: [PATCH 11/43] feat: seed carries frozen pools.json into training workspace --- app/harness/store.py | 11 +++++++++++ app/harness/workspace.py | 7 +++++++ tests/unit/test_harness_store.py | 18 ++++++++++++++++++ tests/unit/test_harness_workspace.py | 17 +++++++++++++++++ 4 files changed, 53 insertions(+) diff --git a/app/harness/store.py b/app/harness/store.py index 87fba07..8caad53 100644 --- a/app/harness/store.py +++ b/app/harness/store.py @@ -190,6 +190,9 @@ def init_seed( baseline_run_id: str, parent: str | None, description: str, + *, + pools_json: Path | None = None, + split_manifest: Path | None = None, ) -> Path: """在 store/seeds/ 写一个种子：权重 + baseline.db + seed.json。 @@ -202,6 +205,10 @@ def init_seed( baseline_run_id: 全量记录的 run_id，fresh 时注入 build_pools。 parent: 来源（initial 为 None）。 description: 人类可读说明。 + pools_json: 可选，冻结切分 pools.json 源路径；提供时拷入 seed 目录， + 供 fresh 训练时携带冻结切分进 workspace（见 init_workspace_from_seed）。 + split_manifest: 可选，冻结切分 split_manifest.json 源路径；提供时拷入 seed + 目录，供加载时校验 pools.json 内容指纹（pools_sha256）。 返回: 种子目录路径。 @@ -216,6 +223,10 @@ def init_seed( shutil.copytree(skills_dir, seed_dir / "skills") shutil.copytree(prompts_dir, seed_dir / "prompts") shutil.copy2(baseline_db, seed_dir / "baseline.db") + if pools_json is not None: + shutil.copy2(pools_json, seed_dir / "pools.json") + if split_manifest is not None: + shutil.copy2(split_manifest, seed_dir / "split_manifest.json") (seed_dir / "seed.json").write_text( json.dumps( { diff --git a/app/harness/workspace.py b/app/harness/workspace.py index 6c13452..3598b2b 100644 --- a/app/harness/workspace.py +++ b/app/harness/workspace.py @@ -196,6 +196,13 @@ def init_workspace_from_seed( shutil.copytree(seed_dir / "prompts", workspace_dir / "prompts" / "v1") shutil.copy2(seed_dir / "baseline.db", workspace_dir / "harness.db") + seed_pools = seed_dir / "pools.json" + if seed_pools.exists(): + shutil.copy2(seed_pools, workspace_dir / "pools.json") + seed_manifest = seed_dir / "split_manifest.json" + if seed_manifest.exists(): + shutil.copy2(seed_manifest, workspace_dir / "split_manifest.json") + logger.info("Workspace 从种子 '{}' 初始化完成: {}", seed_name, workspace_dir) return meta["baseline_run_id"] diff --git a/tests/unit/test_harness_store.py b/tests/unit/test_harness_store.py index 721366b..92dc5cf 100644 --- a/tests/unit/test_harness_store.py +++ b/tests/unit/test_harness_store.py @@ -243,6 +243,24 @@ class TestInitSeed: with pytest.raises(FileExistsError, match="种子已存在"): init_seed(store, "dup", skills_dir, prompts_dir, baseline_db, "r1", None, "second") + def test_init_seed_carries_pools(self, tmp_path): + """提供 pools_json/split_manifest 时拷入 seed 目录。""" + from app.harness.store import init_seed + + store = tmp_path / "store" + skills = tmp_path / "sk"; skills.mkdir(); (skills / "s.md").write_text("x") + prompts = tmp_path / "pr"; prompts.mkdir(); (prompts / "p.md").write_text("y") + db = tmp_path / "b.db"; db.write_text("db") + pools = tmp_path / "pools.json"; pools.write_text('{"split_mode":"global"}') + manifest = tmp_path / "split_manifest.json"; manifest.write_text('{"pools_sha256":"a"}') + + seed_dir = init_seed( + store, "s1", skills, prompts, db, "infer_adhoc", None, "d", + pools_json=pools, split_manifest=manifest, + ) + assert (seed_dir / "pools.json").exists() + assert (seed_dir / "split_manifest.json").exists() + class TestListSeeds: """list_seeds 列出所有种子。""" diff --git a/tests/unit/test_harness_workspace.py b/tests/unit/test_harness_workspace.py index f11fb6d..f0fa2a2 100644 --- a/tests/unit/test_harness_workspace.py +++ b/tests/unit/test_harness_workspace.py @@ -183,6 +183,23 @@ def test_init_workspace_from_seed_missing_questions(store_dir: Path, workspace_d assert not workspace_dir.exists() +def test_init_workspace_from_seed_carries_pools(store_dir: Path, workspace_dir: Path) -> None: + """seed 目录含 pools.json/split_manifest.json 时拷入 workspace。""" + # store_dir fixture 已构造 seed "initial"；补冻结产物到 seed 目录再初始化 workspace。 + seed_dir = store_dir / "seeds" / "initial" + (seed_dir / "pools.json").write_text('{"split_mode":"global"}') + (seed_dir / "split_manifest.json").write_text('{"pools_sha256":"a"}') + + init_workspace_from_seed( + workspace_dir, + store_dir, + seed_name="initial", + questions="benchmarks/Video-MME", + ) + assert (workspace_dir / "pools.json").exists() + assert (workspace_dir / "split_manifest.json").exists() + + # --------------------------------------------------------------------------- # load_manifest # --------------------------------------------------------------------------- From a0c7e043e8baf2110da4a540d8a7044e6cd13423 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:05:15 -0400 Subject: [PATCH 12/43] fix: validate global frozen pools baseline_run_id + sha256 on load --- app/harness/pools.py | 22 ++++++++++ tests/unit/test_harness_pools.py | 71 ++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/app/harness/pools.py b/app/harness/pools.py index c46747e..89763de 100644 --- a/app/harness/pools.py +++ b/app/harness/pools.py @@ -844,6 +844,28 @@ def build_or_load_pools( len(new_types), sorted(new_types), ) + else: + # global：校验 baseline_run_id 与（若有）manifest 内容指纹， + # 拒绝静默加载与 seed 错配 / 被篡改的冻结切分（P5 fail loud）。 + frozen_baseline = raw.get("baseline_run_id") + if frozen_baseline != baseline_run_id: + raise ValueError( + f"冻结 pools.json 的 baseline_run_id={frozen_baseline!r} 与 seed " + f"的 {baseline_run_id!r} 不一致，拒绝静默加载错配切分。" + ) + manifest_path = config.workspace_dir / "split_manifest.json" + if manifest_path.exists(): + import hashlib + + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + actual_sha = hashlib.sha256( + pools_path.read_text(encoding="utf-8").encode("utf-8") + ).hexdigest() + if manifest.get("pools_sha256") != actual_sha: + raise ValueError( + "pools.json 内容指纹与 split_manifest.pools_sha256 不符，" + "冻结产物疑被篡改，拒绝加载。" + ) return load_pools(pools_path) diff --git a/tests/unit/test_harness_pools.py b/tests/unit/test_harness_pools.py index 9dd0eb7..2cc1b75 100644 --- a/tests/unit/test_harness_pools.py +++ b/tests/unit/test_harness_pools.py @@ -18,6 +18,7 @@ import pytest from app.harness.pools import ( GlobalPoolStrategy, PerCategoryPoolStrategy, + build_or_load_pools, build_pools, load_pools, save_pools, @@ -291,6 +292,76 @@ class TestBuildOrLoadPoolsFrozen: load_ids = [q.question_id for q in getattr(loaded, pool_name)] assert orig_ids == load_ids, f"{pool_name} 冻结后 ID 顺序不一致" + def _run_config_for_frozen(self, tmp_path: Path, seed_name: str) -> object: + """构造指向 tmp workspace/store + 指定种子名的最小 train RunConfig。""" + from app.harness.config import RunConfig + + return RunConfig( + workspace_dir=tmp_path / "ws", + store_dir=tmp_path / "store", + mode="train", + concurrency=4, + max_steps=10, + skill_mode="auto", + n_samples=0, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + epochs=1, + diag_size=10, + diag_correct_ratio=0.5, + val_size=10, + val_correct_ratio=0.5, + edit_budget_start=5, + edit_budget_end=2, + batch_size=15, + min_class_per_batch=2, + eval_min_per_class=2, + early_stop_patience=4, + test_size=10, + use_slow_momentum=True, + gate_e_confirm=20.0, + gate_e_provisional=3.0, + gate_w_net_min=2, + gate_delta_min=0.02, + gate_lambda_dir=-0.642, + gate_e_rollback=10.0, + gate_block=8, + gate_n_max=40, + gate_p_low=0.05, + gate_p_high=0.95, + gate_probe_quota=0.2, + gate_gamma_decay=0.9, + gate_cooldown_steps=2, + gate_guard_err=0.10, + skill_update_mode="patch", + appendix_consolidate_threshold=6, + fresh=True, + seed=seed_name, + test_questions="", # 绕过 _to_pool_config 的 resolve_paths(manifest 依赖) + ) + + def test_global_frozen_rejects_baseline_mismatch(self, tmp_path: Path) -> None: + """global 冻结 pools 的 baseline_run_id 与 seed 不符时 fail-loud。""" + seed_name = "myseed" + seed_dir = tmp_path / "store" / "seeds" / seed_name + seed_dir.mkdir(parents=True) + (seed_dir / "seed.json").write_text( + json.dumps({"baseline_run_id": "infer_adhoc", "parent": None}) + ) + + ws = tmp_path / "ws" + ws.mkdir() + (ws / "pools.json").write_text( + json.dumps({"split_mode": "global", "baseline_run_id": "other"}), + encoding="utf-8", + ) + + config = self._run_config_for_frozen(tmp_path, seed_name) + strategy = GlobalPoolStrategy() + with pytest.raises(ValueError, match="baseline_run_id"): + build_or_load_pools(config, strategy, tmp_path / "nonexistent.db") + class TestGlobalPoolStrategy: """GlobalPoolStrategy 封装现有全局三分逻辑。""" From 8f349c5c62dc8857a203b17cd97430a2e27c52a8 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:12:08 -0400 Subject: [PATCH 13/43] fix: make frozen product backup unique and atomic on --force --- app/harness/build_split.py | 44 ++++++++++++++++++++++--- tests/unit/test_video_split_cli.py | 53 ++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/app/harness/build_split.py b/app/harness/build_split.py index 197d4e1..e1d80be 100644 --- a/app/harness/build_split.py +++ b/app/harness/build_split.py @@ -105,17 +105,41 @@ class SplitBuildResult: return getattr(self, key) +def _unique_backup_path(path: Path, suffix: str) -> Path: + """求 path 的唯一 .bak. 备份路径，已存在则追加递增序号避免覆盖。 + + 首选 ``.bak.``；若已存在，退化为 ``.bak..2``、 + ``.3`` … 直到找到不存在的名字。保证连续 forced freeze（同 suffix 或都缺 + manifest 用 'prev'）不会静默覆盖此前保留的备份。 + + 参数: + path: 待备份的原文件路径。 + suffix: 备份后缀（旧 pools_sha256 前 8 位或 'prev'）。 + + 返回: + 目录内唯一、尚不存在的备份路径。 + """ + candidate = path.with_name(f"{path.name}.bak.{suffix}") + counter = 2 + while candidate.exists(): + candidate = path.with_name(f"{path.name}.bak.{suffix}.{counter}") + counter += 1 + return candidate + + def _guard_frozen_products(out_path: Path, manifest_path: Path, *, force: bool) -> None: """冻结前的覆盖保护：产物已存在时按 force 决定报错或备份。 参数: out_path: 目标 pools.json 路径。 manifest_path: 目标 split_manifest.json 路径。 - force: False 时已存在即 FileExistsError；True 时把旧产物重命名为 - .bak.<旧 pools_sha256 前 8 位或 'prev'> 再放行。 + force: False 时已存在即 FileExistsError；True 时把旧产物重命名为唯一的 + .bak.<旧 pools_sha256 前 8 位或 'prev'>（同名已存在则追加递增序号）再放行。 异常: FileExistsError: force=False 且产物已存在（防静默覆盖冻结锚点）。 + OSError: 备份 rename 失败；已备份的文件先 rollback 回原名再抛出，保证 + 要么两文件都备份、要么都不动（原子性，不留半备份的不一致目录）。 """ if not out_path.exists() and not manifest_path.exists(): return @@ -132,9 +156,19 @@ def _guard_frozen_products(out_path: Path, manifest_path: Path, *, force: bool) suffix = str(old.get("pools_sha256", "prev"))[:8] or "prev" except (json.JSONDecodeError, OSError): suffix = "prev" - for p in (out_path, manifest_path): - if p.exists(): - p.rename(p.with_name(f"{p.name}.bak.{suffix}")) + # 先为每个存在的文件求唯一备份路径（互不冲突），再逐个 rename； + # 中途失败则把已备份的 rollback 回原名，保证原子性。 + to_backup = [p for p in (out_path, manifest_path) if p.exists()] + done: list[tuple[Path, Path]] = [] # (备份路径, 原路径)，供 rollback + try: + for p in to_backup: + dst = _unique_backup_path(p, suffix) + p.rename(dst) + done.append((dst, p)) + except OSError: + for backup_path, original in reversed(done): + backup_path.rename(original) + raise def build_split( diff --git a/tests/unit/test_video_split_cli.py b/tests/unit/test_video_split_cli.py index 60cb702..2bc7536 100644 --- a/tests/unit/test_video_split_cli.py +++ b/tests/unit/test_video_split_cli.py @@ -329,3 +329,56 @@ def test_build_split_force_backs_up_old(tmp_path): _guard_frozen_products(out_path, manifest_path, force=True) baks = list(tmp_path.glob("pools.json.bak.*")) assert len(baks) == 1, f"未备份旧产物: {list(tmp_path.iterdir())}" + + +def test_build_split_force_twice_same_suffix_keeps_both(tmp_path): + """连续两次 force 命中同后缀时，第二次备份不覆盖第一次（追加递增序号）。""" + from app.harness.build_split import _guard_frozen_products + + out_path = tmp_path / "pools.json" + manifest_path = tmp_path / "split_manifest.json" + + # 第一次 force：两文件都无 manifest 里的 pools_sha256 之外的差异，后缀恒为同值。 + out_path.write_text('{"gen":1}', encoding="utf-8") + manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8") + _guard_frozen_products(out_path, manifest_path, force=True) + + # 第二次 force：写入相同 sha 的新产物，触发同后缀备份。 + out_path.write_text('{"gen":2}', encoding="utf-8") + manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8") + _guard_frozen_products(out_path, manifest_path, force=True) + + pools_baks = sorted(p.name for p in tmp_path.glob("pools.json.bak.*")) + assert len(pools_baks) == 2, f"同后缀第二次备份覆盖了第一次: {pools_baks}" + + +def test_build_split_force_rollback_on_partial_failure(tmp_path, monkeypatch): + """备份第二个文件失败时，第一个已备份文件被 rollback 回原名（原子性）。""" + from pathlib import Path + + from app.harness import build_split as bs + + out_path = tmp_path / "pools.json" + manifest_path = tmp_path / "split_manifest.json" + out_path.write_text('{"gen":1}', encoding="utf-8") + manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8") + + real_rename = Path.rename + calls = {"n": 0} + + def flaky_rename(self, target): + # 第一次 rename（备份 pools.json）成功，第二次（备份 manifest）抛错。 + calls["n"] += 1 + if calls["n"] == 2: + raise OSError("模拟第二个备份 rename 失败") + return real_rename(self, target) + + monkeypatch.setattr(Path, "rename", flaky_rename) + + with pytest.raises(OSError, match="模拟第二个备份"): + bs._guard_frozen_products(out_path, manifest_path, force=True) + + # rollback 后：两原文件仍在原位，无残留 .bak.* + assert out_path.exists(), "第一个文件未被 rollback 回原名" + assert manifest_path.exists(), "第二个文件不应被移动" + assert not list(tmp_path.glob("*.bak.*")), f"残留半备份: {list(tmp_path.iterdir())}" From 04bda253350a0ae81802e5f27e42ae3ca0fb0d6e Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:14:12 -0400 Subject: [PATCH 14/43] docs: update WP4 plan workspace anchors after WP2 line drift --- research-wiki/plans/2026-07-16-preflight-wp4-resilience.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/research-wiki/plans/2026-07-16-preflight-wp4-resilience.md b/research-wiki/plans/2026-07-16-preflight-wp4-resilience.md index 9016379..11e64cc 100644 --- a/research-wiki/plans/2026-07-16-preflight-wp4-resilience.md +++ b/research-wiki/plans/2026-07-16-preflight-wp4-resilience.md @@ -22,7 +22,7 @@ | TTL | `main.py:41` `redis_cache_ttl=86400`；`:93` `ttl_s = ... if >0 else None` | | 熔断 | `adapters/breaker.py:24` `is_open`（L36-37 到期即放行，无探针锁）；`record_failure:39`；`record_success:63` | | gate INFRA | `app/harness/validate.py:257` `_resolve_baseline_block`（put L303-304）；`_check_infra_guard:384`（累计检查 L593）；`BaselineCache` in `gate_ladder.py:344`（put L384 原子 L401-403） | -| 持久化 | `app/harness/workspace.py` 4 处非原子 write_text：`_scaffold:111`/`update_manifest:282`/`record_run:311`/`update_best:365`；原子范式 `checkpoint.py:257-260` | +| 持久化 | `app/harness/workspace.py` 4 处非原子 `manifest.json` write_text（WP2 后行号）：`_scaffold:111`/`update_manifest:289`/`record_run:318`/`update_best:372`；原子范式 `checkpoint.py:257-260`。**以 `grep -n 'manifest.json").write_text' app/harness/workspace.py` 现场定位为准** | | 基线元数据 | `app/harness/log.py:57` `HarnessLog.__init__`（upsert `_runs` L73-82）；`RunLogImpl._read_table:247`（只读） | | dual_metric | `app/harness/observation.py:119` `write_dual_metric`（version_kind final L138）；runner 调用 `runner.py:1444-1448` | | 配置 | `.env:65` `REDIS_CACHE_TTL=0`；`.env.example:49` `=86400` | @@ -426,7 +426,7 @@ def _atomic_write_json(path: Path, data: dict) -> None: tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") os.replace(tmp, path) ``` -把 `_scaffold_workspace`（L111）、`update_manifest`（L282）、`record_run`（L311）、`update_best`（L365）四处的 `(...).write_text(json.dumps(...))` 替换为 `_atomic_write_json(, )`。 +把 `_scaffold_workspace`（~L111）、`update_manifest`（~L289）、`record_run`（~L318）、`update_best`（~L372）四处的 `(workspace_dir / "manifest.json").write_text(json.dumps(...))` 替换为 `_atomic_write_json(workspace_dir / "manifest.json", )`（用 `grep -n 'manifest.json").write_text' app/harness/workspace.py` 定位全部 4 处，行号随 WP2 改动漂移）。 - [ ] **Step 4: 运行测试确认通过 + 回归** From c5c11ea81bde17f5e077bd72031d6ddf4e1b0018 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:21:18 -0400 Subject: [PATCH 15/43] docs: address Codex review of WP4 plan (salt key compat, store/protocol, INFRA path) --- .../2026-07-16-preflight-wp4-resilience.md | 97 +++++++++++-------- 1 file changed, 55 insertions(+), 42 deletions(-) diff --git a/research-wiki/plans/2026-07-16-preflight-wp4-resilience.md b/research-wiki/plans/2026-07-16-preflight-wp4-resilience.md index 11e64cc..38b24bb 100644 --- a/research-wiki/plans/2026-07-16-preflight-wp4-resilience.md +++ b/research-wiki/plans/2026-07-16-preflight-wp4-resilience.md @@ -65,22 +65,22 @@ Expected: FAIL（`_build_key() takes 3 positional arguments but 4 were given`） - [ ] **Step 3: redis_cache 加 salt 维度** `adapters/redis_cache.py`： -- `_build_key(self, model, messages, cache_salt: str | None = None)`：payload 字典加 `"salt": cache_salt`： +- `_build_key(self, model, messages, cache_salt: str | None = None)`：**仅当 `cache_salt is not None` 才加入 `salt` 字段**（默认 None 时 payload 结构与现状一字节不差，旧缓存键不失效）： ```python - payload = json.dumps( - {"model": model, "messages": messages, "salt": cache_salt}, - sort_keys=True, - ensure_ascii=False, - ) + key_obj: dict = {"model": model, "messages": messages} + if cache_salt is not None: + key_obj["salt"] = cache_salt + payload = json.dumps(key_obj, sort_keys=True, ensure_ascii=False) ``` - `get(self, model, messages, cache_salt: str | None = None)`：`key = self._build_key(model, messages, cache_salt)` - `set(self, model, messages, response, cache_salt: str | None = None)`：同样传 `cache_salt` - [ ] **Step 4: Protocol + chat 透传 salt** -`core/protocols.py` `LLMProvider.chat` 签名加 `cache_salt: str | None = None`（keyword-only，放 `parent_call_id` 后）。 -`adapters/llm.py` `chat` 签名加同参；L299 改 `await self._cache.get(self._model, messages, cache_salt)`；L373 改 `await self._cache.set(self._model, messages, response, cache_salt)`。 -grep `adapters/` 找到转发 `chat` 的 VLM/包装类（如有），同步加 `cache_salt` 透传。 +`core/protocols.py` `LLMProvider.chat`（L22）签名加 `cache_salt: str | None = None`（keyword-only，放 `parent_call_id` 后）。 +`adapters/llm.py` `chat`（L271）签名加同参；L299 改 `await self._cache.get(self._model, messages, cache_salt)`；L373 改 `await self._cache.set(self._model, messages, response, cache_salt)`。 +`adapters/redis_cache.py` `get`（L50）/`set`（L73）签名各加 `cache_salt: str | None = None`，内部 `_build_key(model, messages, cache_salt)`。 +**VLM 转发**：`core/protocols.py` `VLMProvider.chat_with_images`（L35）与 `adapters/vlm.py` `GovernedVLMClient.chat_with_images`（L32）签名加 keyword-only `cache_salt: str | None = None`，L53 转发 `self._llm.chat(..., cache_salt=cache_salt)`。 - [ ] **Step 5: 运行测试确认通过 + 回归** @@ -101,9 +101,10 @@ git commit -m "feat: add cache_salt dimension to LLM response cache" ## Task 2: TTL 语义修正（≤0 报错） **Files:** -- Modify: `main.py:93` +- Modify: `adapters/redis_cache.py`（新增 `_resolve_cache_ttl`） +- Modify: `main.py:93`、`app/harness/video_split_cli.py:257`（两处 TTL 入口共用） - Modify: `.env:65`、`.env.example:49` -- Test: `tests/unit/test_governed_llm.py` 或新增 `tests/unit/test_infra_settings.py` +- Test: 新增 `tests/unit/test_infra_settings.py` - [ ] **Step 1: 写失败测试** @@ -112,9 +113,9 @@ git commit -m "feat: add cache_salt dimension to LLM response cache" import pytest -def test_redis_cache_ttl_zero_rejected(monkeypatch): +def test_redis_cache_ttl_zero_rejected(): """REDIS_CACHE_TTL<=0 必须启动即报错，消灭'0=永不过期'隐式语义。""" - from main import _resolve_cache_ttl # Step 3 抽出的纯函数 + from adapters.redis_cache import _resolve_cache_ttl # Step 3 抽出的纯函数 with pytest.raises(ValueError, match="REDIS_CACHE_TTL"): _resolve_cache_ttl(0) @@ -128,9 +129,9 @@ def test_redis_cache_ttl_zero_rejected(monkeypatch): Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_infra_settings.py -v` Expected: FAIL（`cannot import name '_resolve_cache_ttl'`）。 -- [ ] **Step 3: 抽出并实现 fail-loud** +- [ ] **Step 3: 抽出 fail-loud 并替换两处 TTL 入口** -`main.py`：新增纯函数并替换 L93 逻辑： +`adapters/redis_cache.py` 新增模块级纯函数（配置校验贴近 cache 实现、避免 app→main 依赖）： ```python def _resolve_cache_ttl(ttl: int) -> int: """校验 Redis 缓存 TTL：必须为正整数（消灭 0=永不过期 的隐式语义）。""" @@ -141,7 +142,8 @@ def _resolve_cache_ttl(ttl: int) -> int: ) return ttl ``` -L93 改为 `ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl)`（删除 `if >0 else None` 分支；`RedisResponseCache` 的 `ttl_s` 从此恒为正）。 +`main.py:93` 改为 `ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl)`（import 上述函数，删 `if >0 else None`）。 +`app/harness/video_split_cli.py:257`（`_build_redis_cache` 内）同样从 `ttl_s = ... if > 0 else None` 改为 `ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl)`——消灭第二条 `0=永不过期` 入口。 - [ ] **Step 4: 改 .env / .env.example** @@ -362,9 +364,13 @@ async def test_baseline_infra_error_not_cached(tmp_path): Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_validate.py -k baseline_infra_error_not_cached -v` Expected: FAIL（当前 INFRA unit 的对错被 put 进缓存）。 -- [ ] **Step 3: 实现 INFRA 隔离** +- [ ] **Step 3: 实现 INFRA 隔离（精确数据路径）** -`_resolve_baseline_block`：miss 跑完后，识别 INFRA 题（从 db 读 stop_reason∈{error,parse_error} 或经 run_inference 结果），其所属 unit：不 `baseline_cache.put`、不计入返回的 `b_units`（从块 unit 集合排除，使配对与 W/L 不含它）。护栏所需的 `errors_inc` 在 put 之前累计并检查（护栏前置）。具体实现按 Step 前"保真前置"要求对齐现场与 TRM4。 +**数据路径**（Codex 审指出：当前 `_load_run_rows`（validate.py:212）只查 `question_id,prediction,answer,steps_json`，拿不到 per-unit stop_reason，只有汇总 `stop_reason_counts` 无法定位哪个 unit 是 INFRA）： +1. `_load_run_rows`（validate.py:194-212）的 SELECT 增加 `stop_reason` 列；`_candidate_correctness_from_db` 相应可返回每题 stop_reason（或新增 `_infra_question_ids_from_db(log, run_id, questions)` 返回 stop_reason∈{"error","parse_error"} 的 qid 集）。 +2. `_resolve_baseline_block`：miss 跑完后，计算本块 INFRA unit 集（unit 内**任一题** stop_reason∈{"error","parse_error"}）。对这些 unit：**不 `baseline_cache.put`**、**不计入 `b_units`**、并从返回给调用方的有效 unit 集中排除。护栏所需 `errors_inc` 在 put 之前累计并调 `_check_infra_guard`（护栏前置于缓存写入）。 +3. **贯穿调用方**（validate.py:575-600）：`_resolve_baseline_block` 返回"有效 unit 子集 `valid_unit_chunk`"，后续 `_run_candidate_block`、`unit_correctness_view`、`pair_block`、`_build_evidence_rows` 全部用 `valid_unit_chunk`（而非原始 `unit_chunk`），确保配对 `unit_ids` 与基线侧一致、不含 INFRA unit。 +4. 保真：非 INFRA unit 的配对翻转、基线快照复用、unit 折叠逻辑一字不改；仅"把 INFRA unit 从本块整体剔除"。按 Step 前"保真前置"逐行比对 TRM4。 - [ ] **Step 4: 运行测试确认通过 + 保真回归** @@ -505,9 +511,9 @@ git commit -m "fix: read-only baseline queries skip _runs upsert (register_run f - [ ] **Step 1: 写失败测试** -在 `tests/unit/test_harness_observation.py` 追加（验证 slow_candidate 与 final 可区分）： +`read_dual_metric` 真实签名为 `(db_path, *, run_id)`（无 epoch 参数，observation.py:164），故 observation 层 write/read 对任意 version_kind 都已支持——真正要验证的是 **runner 慢更新第二次写出的是 `slow_candidate` 而非 `final`**。在 `tests/unit/test_harness_observation.py` 追加 observation 层可区分性基线测试： ```python -def test_slow_candidate_kind_distinct_from_final(tmp_path): +def test_dual_metric_kinds_distinguishable(tmp_path): from app.harness.observation import write_dual_metric, read_dual_metric db = str(tmp_path / "h.db") @@ -517,16 +523,16 @@ def test_slow_candidate_kind_distinct_from_final(tmp_path): write_dual_metric(db, run_id="r", epoch=1, version_kind="slow_candidate", skills_version="v2", prompts_version="v2", pool="val", hard_acc=0.6, soft_score=None, mixed_score=None) - rows = read_dual_metric(db, run_id="r", epoch=1) - kinds = {r["version_kind"] for r in rows} - assert "final" in kinds and "slow_candidate" in kinds + rows = read_dual_metric(db, run_id="r") # 真实签名无 epoch，返回该 run 全部行 + kinds = {row["version_kind"] for row in rows if row["epoch"] == 1} + assert kinds == {"final", "slow_candidate"} ``` -> 依 `test_harness_observation.py` 现有 read_dual_metric 签名；实现前读对齐。 +> 实现前读 `test_harness_observation.py` 与 `write_dual_metric`/`read_dual_metric`（observation.py:119/164）确认参数名（epoch/pool/hard_acc 等）与返回行字段。真正的 `slow_candidate` 语义由 Step 3 改 runner 调用点落地。 -- [ ] **Step 2: 运行确认失败/通过基线** +- [ ] **Step 2: 运行基线（observation 层已支持任意 kind）** -Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_observation.py -k slow_candidate -v` -Expected: 若 read_dual_metric 已支持则 PASS 基线；重点是 Step 3 改 runner 调用点。 +Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_observation.py -k dual_metric_kinds -v` +Expected: PASS（observation 层本就接受任意 version_kind）；本 task 的实质改动在 Step 3 的 runner 调用点。 - [ ] **Step 3: 改 runner 慢更新第二次写为 slow_candidate** @@ -549,31 +555,38 @@ git commit -m "fix: slow-update R2 dual_metric uses slow_candidate kind (not fin ## Task 10: 离线 --retry-uncertain + docstring 修正 **Files:** -- Modify: `app/harness/video_split_cli.py`（build_arg_parser + run_pipeline） -- Modify: `app/harness/baseline_diagnosis.py`（done 集排除 uncertain + docstring） -- Test: `tests/unit/test_video_split_cli.py` 或 `test_baseline_diagnosis` +- Modify: `core/evolution/protocols.py:130`（`DiagnosisSignalStore.done_question_ids` 加参数） +- Modify: `adapters/baseline_diagnosis_store.py:109`（SqliteDiagnosisSignalStore 实现 + SQL） +- Modify: `app/harness/baseline_diagnosis.py:96`（透传 + docstring） +- Modify: `app/harness/video_split_cli.py`（build_arg_parser + run_pipeline 透传） +- Test: `tests/unit/`（对 SqliteDiagnosisSignalStore.done_question_ids） - [ ] **Step 1: 写失败测试** -定位 `baseline_diagnosis` 的 done_question_ids 计算函数（`app/harness/baseline_diagnosis.py`，grep `done_question_ids`），追加测试：`retry_uncertain=True` 时 tier=uncertain 的已落库题不计入 done（会被重诊）。 +`done_question_ids` 的真实定义在 Protocol `core/evolution/protocols.py:130` 与实现 `adapters/baseline_diagnosis_store.py:109`（不在 baseline_diagnosis.py）。对 SqliteDiagnosisSignalStore 追加测试： ```python -def test_retry_uncertain_excludes_uncertain_from_done(...): - # 构造 baseline_diagnosis 表含 tier=T2 与 tier=uncertain 行； - # done_ids(retry_uncertain=False) 含 uncertain 题； - # done_ids(retry_uncertain=True) 不含 uncertain 题 - ... +def test_done_question_ids_retry_uncertain_excludes(tmp_path): + # 构造 baseline_diagnosis 表含 tier=T2 行与 tier=uncertain 行（同 run+fingerprint）； + from adapters.baseline_diagnosis_store import SqliteDiagnosisSignalStore + store = SqliteDiagnosisSignalStore(str(tmp_path / "h.db")) + # ... upsert 一条 T2、一条 uncertain ... + assert "" in store.done_question_ids("run", "fp") # 默认含 + assert "" not in store.done_question_ids("run", "fp", retry_uncertain=True) # 排除 + assert "" in store.done_question_ids("run", "fp", retry_uncertain=True) # T2 仍算完成 ``` -> 实现前读 `baseline_diagnosis.py` 的 done 集查询函数签名对齐。 +> 实现前读 `adapters/baseline_diagnosis_store.py` 的 upsert/表结构与 `done_question_ids` SQL 对齐构造。 - [ ] **Step 2: 运行确认失败** -Run: `conda run -n Video-Tree-TRM python -m pytest -k retry_uncertain -v` -Expected: FAIL（无该参数）。 +Run: `conda run -n Video-Tree-TRM python -m pytest -k done_question_ids_retry_uncertain -v` +Expected: FAIL（`unexpected keyword argument 'retry_uncertain'`）。 -- [ ] **Step 3: 实现 retry_uncertain** +- [ ] **Step 3: 实现 retry_uncertain（贯穿 Protocol→Adapter→编排→CLI）** -`app/harness/baseline_diagnosis.py`：done 集查询函数加 `retry_uncertain: bool = False`，True 时 `WHERE ... AND tier != 'uncertain'`（uncertain 题不算完成，会被重诊）。修正模块 docstring 把"崩溃最多丢正在写的一行/逐行落库"表述改为实际的"run 末批量落库（Phase 3），崩溃丢本次 run 全部未落库结果，靠 Redis 缓存缓解重烧"。 -`app/harness/video_split_cli.py`：`build_arg_parser` 加 `--retry-uncertain`（store_true）；`run_pipeline` 透传到 `run_baseline_diagnosis` 的 done 集计算。 +- `core/evolution/protocols.py:130` `DiagnosisSignalStore.done_question_ids` 签名加 `retry_uncertain: bool = False`（keyword）。 +- `adapters/baseline_diagnosis_store.py:109` 实现同签名；SQL 在 `retry_uncertain=True` 时追加 `AND tier != 'uncertain'`（uncertain 题不算完成，会被重诊）。 +- `app/harness/baseline_diagnosis.py:96` `done = store.done_question_ids(baseline_run_id, diag_fingerprint)` 透传 `retry_uncertain`；修正模块 docstring 把"崩溃最多丢正在写的一行/逐行落库"改为实际的"run 末批量落库（Phase 3），崩溃丢本次 run 未落库结果，靠 Redis 缓存缓解重烧"。 +- `app/harness/video_split_cli.py`：`build_arg_parser` 加 `--retry-uncertain`（store_true）；`run_pipeline` 透传到 `run_baseline_diagnosis`。 - [ ] **Step 4: 运行测试确认通过 + 回归** From 564c92a043fc0e7ff006edb9c8814bf19aec7184 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:23:58 -0400 Subject: [PATCH 16/43] feat: add cache_salt dimension to LLM response cache --- adapters/llm.py | 10 ++++++++-- adapters/redis_cache.py | 32 +++++++++++++++++++++++--------- adapters/vlm.py | 3 +++ core/protocols.py | 2 ++ tests/unit/test_governed_llm.py | 12 +++++++++--- tests/unit/test_redis_cache.py | 12 ++++++++++++ 6 files changed, 57 insertions(+), 14 deletions(-) diff --git a/adapters/llm.py b/adapters/llm.py index a8f646d..8443301 100644 --- a/adapters/llm.py +++ b/adapters/llm.py @@ -274,6 +274,7 @@ class GovernedLLMClient: *, session_id: str | None = None, parent_call_id: str | None = None, + cache_salt: str | None = None, ) -> LLMResponse: """发起 LLM 调用，经四层治理栈：熔断 → 缓存 → 重试+流式 → 遥测。 @@ -281,6 +282,7 @@ class GovernedLLMClient: messages: OpenAI 格式消息列表。 session_id: 会话 ID（传递到遥测）。 parent_call_id: 父调用 ID（传递到遥测）。 + cache_salt: 可选缓存盐，透传到 Redis 缓存键（如跨 epoch 重采样）。 返回: LLMResponse 统一响应。 @@ -296,7 +298,11 @@ class GovernedLLMClient: raise CircuitOpenError(f"熔断器已开启，拒绝调用 provider={self._provider}") # ② 缓存查询（cache 为 None 时跳过）— call_id 在缓存路径独立生成 - cached = await self._cache.get(self._model, messages) if self._cache is not None else None + cached = ( + await self._cache.get(self._model, messages, cache_salt) + if self._cache is not None + else None + ) if cached is not None: cache_call_id = str(uuid4()) response = LLMResponse( @@ -370,7 +376,7 @@ class GovernedLLMClient: # ④ 写缓存（cache 为 None 时跳过） if self._cache is not None: - await self._cache.set(self._model, messages, response) + await self._cache.set(self._model, messages, response, cache_salt) # ⑤ 遥测 await self._telemetry.record_llm_call( diff --git a/adapters/redis_cache.py b/adapters/redis_cache.py index 4af573c..39420e6 100644 --- a/adapters/redis_cache.py +++ b/adapters/redis_cache.py @@ -29,36 +29,48 @@ class RedisResponseCache: self._redis = redis self._ttl_s = ttl_s - def _build_key(self, model: str, messages: list[dict[str, str]]) -> str: + def _build_key( + self, + model: str, + messages: list[dict[str, str]], + cache_salt: str | None = None, + ) -> str: """构造 content-addressed 缓存键。 Args: model: 模型名称。 messages: 消息列表。 + cache_salt: 可选缓存盐（如跨 epoch 强制重采样）。仅当非 None 时才加入 + 键 payload，保证默认 None 时键结构与旧缓存一字节不差、旧键不失效。 Returns: sha256 哈希字符串作为 Redis 键。 """ - payload = json.dumps( - {"model": model, "messages": messages}, - sort_keys=True, - ensure_ascii=False, - ) + key_obj: dict[str, Any] = {"model": model, "messages": messages} + if cache_salt is not None: + key_obj["salt"] = cache_salt + payload = json.dumps(key_obj, sort_keys=True, ensure_ascii=False) digest = hashlib.sha256(payload.encode("utf-8")).hexdigest() return f"llm_cache:{digest}" - async def get(self, model: str, messages: list[dict[str, str]]) -> LLMResponse | None: + async def get( + self, + model: str, + messages: list[dict[str, str]], + cache_salt: str | None = None, + ) -> LLMResponse | None: """从缓存读取 LLM 响应。 Args: model: 模型名称。 messages: 消息列表。 + cache_salt: 可选缓存盐，透传到键构造。 Returns: 缓存命中时返回 LLMResponse，未命中或 Redis 异常时返回 None。 """ try: - key = self._build_key(model, messages) + key = self._build_key(model, messages, cache_salt) raw = await self._redis.get(key) except Exception: logger.warning("Redis 缓存读取失败，降级为未命中") @@ -75,6 +87,7 @@ class RedisResponseCache: model: str, messages: list[dict[str, str]], response: LLMResponse, + cache_salt: str | None = None, ) -> None: """将 LLM 响应写入缓存。 @@ -82,9 +95,10 @@ class RedisResponseCache: model: 模型名称。 messages: 消息列表。 response: 待缓存的 LLMResponse。 + cache_salt: 可选缓存盐，透传到键构造。 """ try: - key = self._build_key(model, messages) + key = self._build_key(model, messages, cache_salt) value = json.dumps(dataclasses.asdict(response), ensure_ascii=False) if self._ttl_s: await self._redis.set(key, value, ex=self._ttl_s) diff --git a/adapters/vlm.py b/adapters/vlm.py index cf3342a..b959cb5 100644 --- a/adapters/vlm.py +++ b/adapters/vlm.py @@ -36,6 +36,7 @@ class GovernedVLMClient: *, session_id: str | None = None, parent_call_id: str | None = None, + cache_salt: str | None = None, ) -> LLMResponse: """图文调用：将图片编码为 base64 嵌入 messages，委托给 LLM 客户端。 @@ -45,6 +46,7 @@ class GovernedVLMClient: images: 图片文件路径列表。 session_id: 会话 ID（遥测用）。 parent_call_id: 父调用 ID（遥测用）。 + cache_salt: 可选缓存盐，透传到底层 LLM 缓存键。 返回: LLMResponse。 @@ -54,6 +56,7 @@ class GovernedVLMClient: vision_messages, session_id=session_id, parent_call_id=parent_call_id, + cache_salt=cache_salt, ) @staticmethod diff --git a/core/protocols.py b/core/protocols.py index 3a08d27..2d4e89f 100644 --- a/core/protocols.py +++ b/core/protocols.py @@ -25,6 +25,7 @@ class LLMProvider(Protocol): *, session_id: str | None = None, parent_call_id: str | None = None, + cache_salt: str | None = None, ) -> LLMResponse: ... @@ -39,6 +40,7 @@ class VLMProvider(Protocol): *, session_id: str | None = None, parent_call_id: str | None = None, + cache_salt: str | None = None, ) -> LLMResponse: ... diff --git a/tests/unit/test_governed_llm.py b/tests/unit/test_governed_llm.py index 923b6ec..9f487e4 100644 --- a/tests/unit/test_governed_llm.py +++ b/tests/unit/test_governed_llm.py @@ -23,8 +23,13 @@ class _FakeRedisCache: def __init__(self) -> None: self._store: dict[str, LLMResponse] = {} - async def get(self, model: str, messages: list[dict[str, str]]) -> LLMResponse | None: - key = f"{model}:{json.dumps(messages, sort_keys=True)}" + async def get( + self, + model: str, + messages: list[dict[str, str]], + cache_salt: str | None = None, + ) -> LLMResponse | None: + key = f"{model}:{cache_salt}:{json.dumps(messages, sort_keys=True)}" return self._store.get(key) async def set( @@ -32,8 +37,9 @@ class _FakeRedisCache: model: str, messages: list[dict[str, str]], response: LLMResponse, + cache_salt: str | None = None, ) -> None: - key = f"{model}:{json.dumps(messages, sort_keys=True)}" + key = f"{model}:{cache_salt}:{json.dumps(messages, sort_keys=True)}" self._store[key] = response diff --git a/tests/unit/test_redis_cache.py b/tests/unit/test_redis_cache.py index 8523384..8b5945e 100644 --- a/tests/unit/test_redis_cache.py +++ b/tests/unit/test_redis_cache.py @@ -109,6 +109,18 @@ async def test_different_models_different_keys( assert await cache.get("claude-3-opus", MESSAGES) is None +@pytest.mark.asyncio +async def test_salt_changes_key(fake_redis: object) -> None: + """cache_salt 进入键：不同 salt 产生不同键，None 保持旧键结构。""" + cache = RedisResponseCache(redis=fake_redis, ttl_s=None) + k_none = cache._build_key("m", [{"role": "user", "content": "x"}], None) + k_e1 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e1") + k_e2 = cache._build_key("m", [{"role": "user", "content": "x"}], "run:e2") + assert k_none != k_e1 + assert k_none != k_e2 + assert k_e1 != k_e2 + + @pytest.mark.asyncio async def test_graceful_degradation_on_error() -> None: """Redis 不可用时静默降级：get→None，set→不报错。""" From 7cc6aa5b236b16ef2b012053841006dec50ee02e Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:25:36 -0400 Subject: [PATCH 17/43] fix: reject REDIS_CACHE_TTL<=0 (kill implicit never-expire) --- .env.example | 1 + adapters/redis_cache.py | 20 ++++++++++++++++++++ app/harness/video_split_cli.py | 8 +++++--- main.py | 7 ++++--- tests/unit/test_infra_settings.py | 16 ++++++++++++++++ 5 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_infra_settings.py diff --git a/.env.example b/.env.example index bb48729..86352a8 100644 --- a/.env.example +++ b/.env.example @@ -46,6 +46,7 @@ LLM_CIRCUIT_BREAKER_COOLDOWN=60 LLM_TTFT_TIMEOUT=30 LLM_INTER_TOKEN_TIMEOUT=15 LLM_RETRY_MAX_DELAY=30.0 +# 正整数秒，禁止 0（0 会被拒绝启动）；训练场景建议 >= 单次训练时长 REDIS_CACHE_TTL=86400 # 建树批量并行：全局 VLM/LLM 在途调用上限（Spec-2 工程配置） diff --git a/adapters/redis_cache.py b/adapters/redis_cache.py index 39420e6..dff36fc 100644 --- a/adapters/redis_cache.py +++ b/adapters/redis_cache.py @@ -12,6 +12,26 @@ from loguru import logger from core.types import LLMResponse +def _resolve_cache_ttl(ttl: int) -> int: + """校验 Redis 缓存 TTL：必须为正整数（消灭 0=永不过期 的隐式语义）。 + + Args: + ttl: 待校验的 TTL 秒数。 + + Returns: + 校验通过的正整数 TTL。 + + Raises: + ValueError: ttl <= 0。 + """ + if ttl <= 0: + raise ValueError( + f"REDIS_CACHE_TTL 必须为正整数秒，实际 {ttl}。" + "训练场景建议 >= 单次训练时长（如 86400）。" + ) + return ttl + + class RedisResponseCache: """基于 Redis 的 LLM 响应缓存。 diff --git a/app/harness/video_split_cli.py b/app/harness/video_split_cli.py index 90a7d4f..754e8f3 100644 --- a/app/harness/video_split_cli.py +++ b/app/harness/video_split_cli.py @@ -248,13 +248,15 @@ def _build_redis_cache(settings: Any) -> Any | None: """按 .env redis_url 构建响应缓存（不可用则降级 None，与 main 一致）。""" if not settings.redis_url: return None + + from adapters.redis_cache import RedisResponseCache, _resolve_cache_ttl + + # 配置校验 fail-loud（不属于 Redis 连接故障，不得被下方降级 except 吞掉） + ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl) try: import redis.asyncio as aioredis - from adapters.redis_cache import RedisResponseCache - redis_client = aioredis.from_url(settings.redis_url, decode_responses=True) - ttl_s = settings.redis_cache_ttl if settings.redis_cache_ttl > 0 else None return RedisResponseCache(redis=redis_client, ttl_s=ttl_s) except Exception: logger.warning("Redis 缓存不可用，诊断降级为无缓存模式") diff --git a/main.py b/main.py index 1185e5a..9c1c551 100644 --- a/main.py +++ b/main.py @@ -84,13 +84,14 @@ def _build_adapters(settings: InfraSettings, embed_cfg: dict) -> _Adapters: cache = None if settings.redis_url: + from adapters.redis_cache import RedisResponseCache, _resolve_cache_ttl + + # 配置校验 fail-loud（不属于 Redis 连接故障，不得被下方降级 except 吞掉） + ttl_s = _resolve_cache_ttl(settings.redis_cache_ttl) try: import redis.asyncio as aioredis - from adapters.redis_cache import RedisResponseCache - redis_client = aioredis.from_url(settings.redis_url, decode_responses=True) - ttl_s = settings.redis_cache_ttl if settings.redis_cache_ttl > 0 else None cache = RedisResponseCache(redis=redis_client, ttl_s=ttl_s) except Exception: logger.warning("Redis 缓存不可用，降级为无缓存模式") diff --git a/tests/unit/test_infra_settings.py b/tests/unit/test_infra_settings.py new file mode 100644 index 0000000..9c9f512 --- /dev/null +++ b/tests/unit/test_infra_settings.py @@ -0,0 +1,16 @@ +"""工程配置校验单元测试（Redis 缓存 TTL fail-loud）。""" + +from __future__ import annotations + +import pytest + + +def test_redis_cache_ttl_zero_rejected() -> None: + """REDIS_CACHE_TTL<=0 必须启动即报错，消灭'0=永不过期'隐式语义。""" + from adapters.redis_cache import _resolve_cache_ttl + + with pytest.raises(ValueError, match="REDIS_CACHE_TTL"): + _resolve_cache_ttl(0) + with pytest.raises(ValueError, match="REDIS_CACHE_TTL"): + _resolve_cache_ttl(-1) + assert _resolve_cache_ttl(86400) == 86400 From 200053fbdcbbfa16531a8aa25b65685e57a4cec9 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:26:21 -0400 Subject: [PATCH 18/43] fix: treat SSE stream without [DONE] as truncated (retry, no cache) --- adapters/llm.py | 4 ++++ tests/unit/test_governed_llm.py | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/adapters/llm.py b/adapters/llm.py index 8443301..6af3b3d 100644 --- a/adapters/llm.py +++ b/adapters/llm.py @@ -578,6 +578,10 @@ class GovernedLLMClient: else: thinking_parts.append(text) + # 流耗尽但未收 [DONE] → 服务端截断，视为可重试的 SSE 异常（不写缓存/不当成功） + if not usage_sink.get("done"): + raise _SseAnomaly("truncated_no_done") + content = "".join(content_parts) thinking = "".join(thinking_parts) usage = usage_sink.get("usage", {}) diff --git a/tests/unit/test_governed_llm.py b/tests/unit/test_governed_llm.py index 9f487e4..c75d2b8 100644 --- a/tests/unit/test_governed_llm.py +++ b/tests/unit/test_governed_llm.py @@ -259,6 +259,33 @@ async def test_qwen_thinking_stripped(): assert thinking2 == "" +@pytest.mark.asyncio +async def test_truncated_stream_without_done_raises(): + """SSE 流耗尽但未收 [DONE] → _SseAnomaly（进重试，不当成功）。""" + from adapters.llm import _SseAnomaly + + async def _lines(): + yield 'data: {"choices":[{"delta":{"content":"半"}}]}' + # 无 data: [DONE] —— 模拟服务端截断 + + client = _build_client() + with pytest.raises(_SseAnomaly): + await client._consume_stream(_lines()) + + +@pytest.mark.asyncio +async def test_complete_stream_with_done_ok(): + """正常带 [DONE] 的流不抛异常，正确累积 content。""" + + async def _lines(): + yield 'data: {"choices":[{"delta":{"content":"完整"}}]}' + yield "data: [DONE]" + + client = _build_client() + content, _thinking, _ttft, _gap, _usage = await client._consume_stream(_lines()) + assert content == "完整" + + @pytest.mark.asyncio async def test_parent_call_id_forwarded_to_telemetry(): """parent_call_id 和 session_id 正确传递到遥测记录。""" From 1cbfa97b8d8c70929448059ece206e1de538e894 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:26:52 -0400 Subject: [PATCH 19/43] fix: cover httpx disconnect family in transient error set --- adapters/llm.py | 5 ++++- tests/unit/test_governed_llm.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/adapters/llm.py b/adapters/llm.py index 6af3b3d..4445dae 100644 --- a/adapters/llm.py +++ b/adapters/llm.py @@ -179,7 +179,10 @@ def _is_transient_error(exc: Exception) -> bool: 返回: True 表示可重试，False 表示不可重试。 """ - if isinstance(exc, (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout)): + # 两族基类覆盖断连族：TimeoutException（ConnectTimeout/ReadTimeout/WriteTimeout/PoolTimeout） + # 与 TransportError（ConnectError/ReadError/RemoteProtocolError 等）。 + # 注意 HTTPStatusError 非 TransportError 子类，401/403 致命分支不受影响。 + if isinstance(exc, (httpx.TimeoutException, httpx.TransportError)): return True if isinstance(exc, httpx.HTTPStatusError): return exc.response.status_code in _TRANSIENT_STATUS_CODES diff --git a/tests/unit/test_governed_llm.py b/tests/unit/test_governed_llm.py index c75d2b8..cc483a0 100644 --- a/tests/unit/test_governed_llm.py +++ b/tests/unit/test_governed_llm.py @@ -259,6 +259,16 @@ async def test_qwen_thinking_stripped(): assert thinking2 == "" +def test_transient_covers_disconnect_family(): + """瞬时错误清单覆盖断连族（RemoteProtocolError/ReadError/ConnectTimeout/PoolTimeout）。""" + from adapters.llm import _is_transient_error + + assert _is_transient_error(httpx.RemoteProtocolError("peer reset")) + assert _is_transient_error(httpx.ReadError("read")) + assert _is_transient_error(httpx.ConnectTimeout("ct")) + assert _is_transient_error(httpx.PoolTimeout("pt")) + + @pytest.mark.asyncio async def test_truncated_stream_without_done_raises(): """SSE 流耗尽但未收 [DONE] → _SseAnomaly（进重试，不当成功）。""" From 87908b23ebe3113b5972893e5794efae8d769ea6 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:27:46 -0400 Subject: [PATCH 20/43] fix: half-open circuit admits single probe (no thundering herd) --- adapters/breaker.py | 21 ++++++++++++++++++--- tests/unit/test_breaker.py | 12 ++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/adapters/breaker.py b/adapters/breaker.py index 1555142..f3506c2 100644 --- a/adapters/breaker.py +++ b/adapters/breaker.py @@ -20,21 +20,32 @@ class CircuitBreaker: self._cooldown_s = cooldown_s self._fails: dict[str, int] = {} self._open_until: dict[str, float] = {} + self._half_open_inflight: dict[str, bool] = {} def is_open(self, source_name: str, now: float) -> bool: """判断指定源是否处于开路状态。 - 冷却截止时刻之前为开路；到期返回 False（放行一个试探，即半开）。 + 冷却截止时刻之前为开路；到期进入半开，**只放行一个探针**（其余仍被挡）， + 避免冷却到期瞬间惊群重连再次压垮上游。"检查+标记探针"在 asyncio 单线程内 + 同步执行，天然原子无竞态。 Args: source_name: 被熔断的源标识。 now: 当前时刻（秒级时间戳），由调用方注入。 Returns: - True 表示开路（拒绝请求），False 表示关闭或半开（放行）。 + True 表示开路（拒绝请求），False 表示关闭或半开放行探针。 """ until = self._open_until.get(source_name) - return until is not None and now < until + if until is None: + return False + if now < until: + return True # 冷却中，全挡 + # 冷却到期：half-open，只放行一个探针 + if self._half_open_inflight.get(source_name): + return True # 已有探针在途，继续挡 + self._half_open_inflight[source_name] = True + return False def record_failure(self, source_name: str, now: float) -> None: """记录一次失败；累计达阈值则开路至 now + cooldown。 @@ -47,6 +58,8 @@ class CircuitBreaker: self._fails[source_name] = count if count >= self._fail_threshold: self._open_until[source_name] = now + self._cooldown_s + # 探针失败清在途标记，使下一轮 cooldown 到期后可再放行探针 + self._half_open_inflight.pop(source_name, None) def force_open(self, source_name: str, now: float) -> None: """强制开路（用于 401/403 等不可恢复错误），一次即熔断。 @@ -59,6 +72,7 @@ class CircuitBreaker: """ self._fails[source_name] = self._fail_threshold self._open_until[source_name] = now + self._cooldown_s + self._half_open_inflight.pop(source_name, None) def record_success(self, source_name: str) -> None: """记录一次成功；清零失败计数与开路状态（关闭熔断器）。 @@ -68,3 +82,4 @@ class CircuitBreaker: """ self._fails.pop(source_name, None) self._open_until.pop(source_name, None) + self._half_open_inflight.pop(source_name, None) diff --git a/tests/unit/test_breaker.py b/tests/unit/test_breaker.py index 348c2da..345c635 100644 --- a/tests/unit/test_breaker.py +++ b/tests/unit/test_breaker.py @@ -58,6 +58,18 @@ class TestCircuitBreaker: breaker.force_open("llm", now=5.0) assert breaker.is_open("llm", now=5.5) is True + def test_half_open_admits_single_probe(self) -> None: + """冷却到期后半开只放行一个探针，第二个仍被挡；探针成功后闭合。""" + b = CircuitBreaker(fail_threshold=2, cooldown_s=10.0) + b.record_failure("p", now=0.0) + b.record_failure("p", now=0.0) # 开路至 t=10 + assert b.is_open("p", now=5.0) is True # 冷却中 + # 冷却到期：只放行第一个探针 + assert b.is_open("p", now=11.0) is False # 探针 1 放行 + assert b.is_open("p", now=11.0) is True # 探针 2 被挡（探针在途） + b.record_success("p") # 探针成功 → 闭合 + assert b.is_open("p", now=12.0) is False + def test_force_open_probe_failure_reopens(self, breaker: CircuitBreaker) -> None: """半开探针失败后重新熔断（因 fails 已置为 threshold）。""" breaker.force_open("llm", now=0.0) From d1516bf56b42b77e338cc8ef98058c838e50d074 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:33:21 -0400 Subject: [PATCH 21/43] fix: isolate gate baseline-arm INFRA errors from BaselineCache (algo #6) --- app/harness/validate.py | 89 +++++++++++++++++++++++------ tests/unit/test_harness_validate.py | 70 +++++++++++++++++++++++ 2 files changed, 141 insertions(+), 18 deletions(-) diff --git a/app/harness/validate.py b/app/harness/validate.py index 2ba2191..f4c6be4 100644 --- a/app/harness/validate.py +++ b/app/harness/validate.py @@ -209,7 +209,8 @@ def _load_run_rows( _correct、steps 等字段。 """ rows = log.query( - "SELECT question_id, prediction, answer, steps_json FROM predictions WHERE run_id=?", + "SELECT question_id, prediction, answer, stop_reason, steps_json " + "FROM predictions WHERE run_id=?", (run_id,), ) normalized: dict[str, dict[str, Any]] = {} @@ -230,6 +231,34 @@ def _load_run_rows( return normalized +# INFRA 故障 stop_reason（推理侧基础设施错误，非模型答错）：这些题的对错无信号意义， +# 基线臂遇到时不得写入 BaselineCache（否则一次瞬时故障永久污染基线快照）。 +_INFRA_STOP_REASONS = frozenset({"error", "parse_error"}) + + +def _infra_question_ids_from_db( + log: HarnessLog, + run_id: str, + chunk: list[GeneratedQuestion], +) -> set[str]: + """从 db 读取一个 run 中 stop_reason 属 INFRA 故障族的 question_id 集合。 + + 参数: + log: HarnessLog 共享实例。 + run_id: 推理 run_id。 + chunk: 题目列表。 + + 返回: + stop_reason ∈ {"error", "parse_error"} 的 question_id 集合。 + """ + rows = _load_run_rows(log, run_id) + return { + q.question_id + for q in chunk + if rows.get(q.question_id, {}).get("stop_reason") in _INFRA_STOP_REASONS + } + + def _candidate_correctness_from_db( log: HarnessLog, run_id: str, @@ -264,13 +293,18 @@ async def _resolve_baseline_block( run_inference: RunInferenceFn, log: HarnessLog, run_id: str, -) -> tuple[dict[str, bool], int, int]: +) -> tuple[dict[str, bool], list[QuestionUnit], int, int]: """基线侧处理一个块：缓存优先（unit 键），miss 的单元新鲜跑基线版本并回写缓存。 缓存以 unit_id 为键、存单元级对错（AR pair 双向 AND 折叠后一个布尔）。 miss 的单元展开为逐题送推理，读回逐题预测后经 unit_correctness_view 折叠成 单元级对错再写缓存（核心算法保真 #5）。逐题 predictions 仍逐题落库溯源。 + INFRA 隔离（算法 #6）：miss 单元内**任一题** stop_reason ∈ {error, parse_error} + 即判定该单元为 INFRA 故障——**不写 BaselineCache**（否则瞬时故障永久污染基线 + 快照）、**不入 b_units**、并从返回的有效单元集中剔除，避免污染 W/L 翻转与配对。 + 命中缓存的单元恒为有效（此前已成功验证过）。 + 参数: units: 当前块的单元列表（single 或 AR pair）。 task_type: 当前验证题型（缓存键成分）。 @@ -283,8 +317,9 @@ async def _resolve_baseline_block( run_id: 本块基线 run_id。 返回: - (b_units, errors_inc, denom_inc)：块内 unit_id -> 基线单元对错、 - 本块新增的 INFRA error 计数与推理题次分母增量（全命中时为 0, 0）。 + (b_units, valid_units, errors_inc, denom_inc)：块内有效 unit_id -> 基线单元 + 对错、剔除 INFRA 后的有效单元列表、本块新增的 INFRA error 计数与推理题次 + 分母增量（全命中时为 0, 0）。 """ miss_units = [ u @@ -293,22 +328,31 @@ async def _resolve_baseline_block( ] errors_inc = 0 denom_inc = 0 + infra_qids: set[str] = set() if miss_units: miss_questions = flatten_units(miss_units) r_b = await run_inference(miss_questions, run_id=run_id, skills_dir=base_skills_dir) errors_inc = r_b.stop_reason_counts.get("error", 0) denom_inc = r_b.total + infra_qids = _infra_question_ids_from_db(log, r_b.run_id, miss_questions) fresh_per_q = _candidate_correctness_from_db(log, r_b.run_id, miss_questions) fresh_units = unit_correctness_view(miss_units, fresh_per_q) - for uid, correct in fresh_units.items(): - baseline_cache.put(task_type, s_hash, prompts_version, uid, correct) + # 只回写非 INFRA 单元；INFRA 单元不入缓存（不永久污染基线快照） + for u in miss_units: + if any(q.question_id in infra_qids for q in u.questions): + continue + baseline_cache.put(task_type, s_hash, prompts_version, u.unit_id, fresh_units[u.unit_id]) + + valid_units = [ + u for u in units if not any(q.question_id in infra_qids for q in u.questions) + ] b_units: dict[str, bool] = {} - for u in units: + for u in valid_units: val = baseline_cache.get(task_type, s_hash, prompts_version, u.unit_id) assert val is not None, f"基线缓存补齐后仍有 miss: unit={u.unit_id} run_id={run_id}" b_units[u.unit_id] = val - return b_units, errors_inc, denom_inc + return b_units, valid_units, errors_inc, denom_inc async def _run_candidate_block( @@ -554,6 +598,7 @@ async def _run_local_validation( w = 0 l = 0 # noqa: E741 n_used = 0 + n_excluded = 0 # 累计被 INFRA 隔离剔除的单元数（从阶梯分母扣除） errors = 0 infra_denom = 0 evidence_rows: list[dict] = [] @@ -567,8 +612,8 @@ async def _run_local_validation( verdict: GateVerdict | None = None for block_idx, unit_chunk in enumerate(unit_chunks): - # Phase 1: 基线侧（缓存优先，miss 新鲜跑）+ 候选侧（全块新鲜跑） - b_units, err_b, den_b = await _resolve_baseline_block( + # Phase 1: 基线侧（缓存优先，miss 新鲜跑，INFRA 单元剔除） + b_units, valid_chunk, err_b, den_b = await _resolve_baseline_block( units=unit_chunk, task_type=task_type, s_hash=s_hash, @@ -579,34 +624,42 @@ async def _run_local_validation( log=log, run_id=f"{gate_run_prefix}_b{block_idx}_base", ) + # 候选侧只跑基线侧判定有效（非 INFRA）的单元，保证配对 unit_ids 两侧一致 c_per_q, err_c, den_c = await _run_candidate_block( - units=unit_chunk, + units=valid_chunk, cand_dir=cand_dir, run_inference=run_inference, log=log, run_id=f"{gate_run_prefix}_b{block_idx}_cand", ) - # Phase 2: INFRA 护栏（跨块累计，分母 >=10 才触发） + # Phase 2: INFRA 护栏（跨块累计，分母 >=10 才触发）——写缓存前置于此已由 + # _resolve_baseline_block 保证 INFRA 单元不落缓存，此处仅做整轮错误率熔断。 errors += err_b + err_c infra_denom += den_b + den_c _check_infra_guard(errors, infra_denom, gate_guard_err) - # Phase 3: 折叠成单元视图 + 配对 + 证据行 + 块间判定 - c_units = unit_correctness_view(unit_chunk, c_per_q) + # 本块全 INFRA：无有效单元可配对，累计剔除数后跳过判定进入下一块 + n_excluded += len(unit_chunk) - len(valid_chunk) + if not valid_chunk: + continue + + # Phase 3: 折叠成单元视图 + 配对 + 证据行 + 块间判定（均用有效单元） + c_units = unit_correctness_view(valid_chunk, c_per_q) candidate_per_q.update(c_per_q) - unit_ids = [u.unit_id for u in unit_chunk] + unit_ids = [u.unit_id for u in valid_chunk] pair_result = pair_block(b_units, c_units, unit_ids) for uid, (b, c) in pair_result.observed.items(): base_obs[uid] = b cand_obs[uid] = c - block_rows = _build_evidence_rows(unit_chunk, b_units, c_units, task_type, block_idx) + block_rows = _build_evidence_rows(valid_chunk, b_units, c_units, task_type, block_idx) w += pair_result.w l += pair_result.l # noqa: E741 - n_used += len(unit_chunk) - verdict = gate_decision(w, l, n_used, n_plan - n_used, params=gate_params) + n_used += len(valid_chunk) + # 阶梯剩余按扣除 INFRA 后的有效分母计：n_remaining = (n_plan - n_excluded) - n_used + verdict = gate_decision(w, l, n_used, (n_plan - n_excluded) - n_used, params=gate_params) for row in block_rows: row["e_value"] = verdict.e_value diff --git a/tests/unit/test_harness_validate.py b/tests/unit/test_harness_validate.py index 2734408..bcd1ef6 100644 --- a/tests/unit/test_harness_validate.py +++ b/tests/unit/test_harness_validate.py @@ -505,6 +505,76 @@ async def test_baseline_cache_hit(tmp_path: Path) -> None: log.close() +@pytest.mark.asyncio +async def test_baseline_infra_error_not_cached(tmp_path: Path) -> None: + """基线臂 INFRA error 的 unit 不写入 BaselineCache（不永久污染），且从有效单元排除。""" + from app.harness.gate_ladder import skill_hash + from app.harness.question_units import build_units + from app.harness.validate import _resolve_baseline_block + + workspace = _setup_workspace(tmp_path) + log = _make_log(workspace) + questions = _make_questions(2) # q0 干净, q1 INFRA error + units = build_units(questions) + cache = BaselineCache(workspace / "baseline_cache.json") + s_hash = skill_hash("baseline skill content") + + async def mock_fn(qs, *, run_id, skills_dir): + for q in qs: + is_err = q.question_id == "q1" + log.insert( + "predictions", + { + "run_id": run_id, + "video_id": "v0", + "question_id": q.question_id, + "task_type": "temporal", + "prediction": "" if is_err else "A", + "answer": "A", + "evidence": "", + "reasoning": "", + "steps_used": 1, + "prompt_tokens": 10, + "completion_tokens": 10, + "stop_reason": "error" if is_err else "completed", + "steps_json": "[]", + }, + ) + return InferenceResult( + run_id=run_id, + accuracy=0.5, + total=2, + correct=1, + per_task_type={}, + steps_mean=1.0, + token_usage={"prompt_tokens": 10, "completion_tokens": 10}, + stop_reason_counts={"completed": 1, "error": 1}, + ) + + try: + b_units, valid_units, _errors_inc, _denom_inc = await _resolve_baseline_block( + units=units, + task_type="temporal", + s_hash=s_hash, + prompts_version="p1", + baseline_cache=cache, + base_skills_dir=workspace / "skills" / "v1", + run_inference=mock_fn, + log=log, + run_id="step1_gate_b0_base", + ) + # q1 是 INFRA：不写缓存、不入 b_units、不在有效单元里 + assert cache.get("temporal", s_hash, "p1", "q1") is None + assert "q1" not in b_units + assert all(u.unit_id != "q1" for u in valid_units) + # q0 干净：正常缓存并入 b_units/valid_units + assert cache.get("temporal", s_hash, "p1", "q0") is True + assert b_units["q0"] is True + assert any(u.unit_id == "q0" for u in valid_units) + finally: + log.close() + + @pytest.mark.asyncio async def test_last_block_terminal(tmp_path: Path) -> None: """单块 + n_remaining=0 → 终态判定（provisional 或 inertia），非 continue。""" From 96884dd1491668fa63861982467a7ab9abcc99ae Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:34:45 -0400 Subject: [PATCH 22/43] fix: atomic writes for manifest/record_run/update_best (tmp+replace) --- app/harness/workspace.py | 25 +++++++++++++++++++------ tests/unit/test_harness_workspace.py | 27 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/app/harness/workspace.py b/app/harness/workspace.py index 3598b2b..c385dc1 100644 --- a/app/harness/workspace.py +++ b/app/harness/workspace.py @@ -64,6 +64,21 @@ def _now_iso() -> str: return datetime.now(UTC).isoformat() +def _atomic_write_json(path: Path, data: dict) -> None: + """原子写 JSON：tmp + os.replace（对齐 checkpoint.py 范式，防半截损坏）。 + + 先写同目录临时文件，再 os.replace 原子替换目标；替换阶段崩溃不会留下半截 + JSON，原文件保持完好。 + + 参数: + path: 目标 JSON 文件路径。 + data: 待序列化的字典。 + """ + tmp = path.with_name(path.name + ".tmp") + tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + os.replace(tmp, path) + + # --------------------------------------------------------------------------- # Workspace 核心函数 # --------------------------------------------------------------------------- @@ -108,7 +123,7 @@ def _scaffold_workspace( }, "history": [], } - (workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) + _atomic_write_json(workspace_dir / "manifest.json", manifest) def init_workspace( @@ -286,7 +301,7 @@ def update_manifest(workspace_dir: Path, **version_updates: str) -> None: raise KeyError(f"无效的 manifest current 字段: {invalid}") manifest = load_manifest(workspace_dir) manifest["current"].update(version_updates) - (workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) + _atomic_write_json(workspace_dir / "manifest.json", manifest) def record_run(workspace_dir: Path, run_id: str) -> Path: @@ -315,9 +330,7 @@ def record_run(workspace_dir: Path, run_id: str) -> Path: "questions": current["questions"], } ) - (workspace_dir / "manifest.json").write_text( - json.dumps(manifest, ensure_ascii=False, indent=2) - ) + _atomic_write_json(workspace_dir / "manifest.json", manifest) run_dir = workspace_dir / "runs" / run_id # exist_ok：同 run_id 重跑时 run 目录已存在不应崩溃 @@ -369,7 +382,7 @@ def update_best( "run_id": run_id, "epoch": epoch, } - (workspace_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) + _atomic_write_json(workspace_dir / "manifest.json", manifest) logger.info("Best 已更新: val_acc={}, run={}, epoch={}", val_acc, run_id, epoch) diff --git a/tests/unit/test_harness_workspace.py b/tests/unit/test_harness_workspace.py index f0fa2a2..6ad79e1 100644 --- a/tests/unit/test_harness_workspace.py +++ b/tests/unit/test_harness_workspace.py @@ -259,6 +259,33 @@ def test_update_manifest_valid(store_dir: Path, workspace_dir: Path) -> None: assert manifest["current"]["skills"] == "skills/v2" +def test_update_manifest_is_atomic( + store_dir: Path, workspace_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """写 manifest 途中崩溃不产生半截 JSON（原子写：os.replace 失败也不损原文件）。""" + from app.harness import workspace as ws + + init_workspace( + workspace_dir, + store_dir, + questions="benchmarks/Video-MME", + skills_version="v1", + prompts_version="v1", + ) + original = (workspace_dir / "manifest.json").read_text() + + def _boom(_src: object, _dst: object) -> None: + raise OSError("crash during replace") + + monkeypatch.setattr(ws.os, "replace", _boom) + with pytest.raises(OSError, match="crash during replace"): + update_manifest(workspace_dir, skills="skills/v2") + + # 原 manifest 未被破坏（内容不变且仍是合法 JSON） + assert (workspace_dir / "manifest.json").read_text() == original + assert json.loads((workspace_dir / "manifest.json").read_text()) + + # --------------------------------------------------------------------------- # record_run # --------------------------------------------------------------------------- From 1468a53b7aad8b542cd27e8b9ebbef8537f98569 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:36:47 -0400 Subject: [PATCH 23/43] fix: read-only baseline queries skip _runs upsert (register_run flag) --- app/harness/log.py | 50 ++++++++++++++++++++++------------ app/harness/pools.py | 6 ++-- app/harness/runner.py | 4 ++- tests/unit/test_harness_log.py | 24 ++++++++++++++++ 4 files changed, 63 insertions(+), 21 deletions(-) diff --git a/app/harness/log.py b/app/harness/log.py index 7231abe..4cab15f 100644 --- a/app/harness/log.py +++ b/app/harness/log.py @@ -52,6 +52,9 @@ class HarnessLog: run_id: 本次运行的唯一标识。 git_sha: 代码版本，默认自动获取。 config_snapshot: 本次运行的配置快照。 + register_run: 是否注册运行（upsert _runs + 退出时同步 status）。默认 True； + 只读查询已有 run（如基线预测回读）时传 False，避免把该 run 的 + started_at/config/status 改写、把基线元数据污染成本次进程的运行状态。 """ def __init__( @@ -60,27 +63,33 @@ class HarnessLog: run_id: str, git_sha: str | None = None, config_snapshot: dict[str, Any] | None = None, + *, + register_run: bool = True, ) -> None: self._run_id = run_id + self._register_run = register_run Path(db_path).parent.mkdir(parents=True, exist_ok=True) self._conn = sqlite3.connect(db_path, check_same_thread=False) self._lock = threading.Lock() self._conn.row_factory = sqlite3.Row self._conn.execute("PRAGMA journal_mode=WAL") self._init_fixed_tables() - resolved_sha = git_sha or _get_git_sha() - config_json = json.dumps(config_snapshot, ensure_ascii=False) if config_snapshot else None - self._conn.execute( - "INSERT INTO _runs" - " (run_id, git_sha, started_at, config, status)" - " VALUES (?, ?, ?, ?, ?)" - " ON CONFLICT(run_id) DO UPDATE SET" - " started_at=excluded.started_at," - " config=excluded.config," - " status=excluded.status", - (run_id, resolved_sha, _now_iso(), config_json, "running"), - ) - self._conn.commit() + if register_run: + resolved_sha = git_sha or _get_git_sha() + config_json = ( + json.dumps(config_snapshot, ensure_ascii=False) if config_snapshot else None + ) + self._conn.execute( + "INSERT INTO _runs" + " (run_id, git_sha, started_at, config, status)" + " VALUES (?, ?, ?, ?, ?)" + " ON CONFLICT(run_id) DO UPDATE SET" + " started_at=excluded.started_at," + " config=excluded.config," + " status=excluded.status", + (run_id, resolved_sha, _now_iso(), config_json, "running"), + ) + self._conn.commit() def _init_fixed_tables(self) -> None: """创建 _runs 和 _events 固定表。""" @@ -217,13 +226,18 @@ class HarnessLog: 参数: status: 最终状态，"completed" 或 "failed"。 + + 关键实现: + register_run=False（只读打开）时跳过 status 更新，仅关闭连接， + 避免只读回读把已有 run 的 finished_at/status 改写。 """ with self._lock: - self._conn.execute( - "UPDATE _runs SET finished_at = ?, status = ? WHERE run_id = ?", - (_now_iso(), status, self._run_id), - ) - self._conn.commit() + if self._register_run: + self._conn.execute( + "UPDATE _runs SET finished_at = ?, status = ? WHERE run_id = ?", + (_now_iso(), status, self._run_id), + ) + self._conn.commit() self._conn.close() def __enter__(self) -> HarnessLog: diff --git a/app/harness/pools.py b/app/harness/pools.py index 89763de..1606c28 100644 --- a/app/harness/pools.py +++ b/app/harness/pools.py @@ -797,7 +797,9 @@ def build_or_load_pools( # 增量构建新类别 paths = resolve_paths(config.workspace_dir) questions = load_benchmark(paths.questions_dir) - with HarnessLog(str(db_path), baseline_run_id) as hlog: + with HarnessLog( + str(db_path), baseline_run_id, register_run=False + ) as hlog: rows = hlog.query( "SELECT question_id, prediction, answer " "FROM predictions WHERE run_id=?", @@ -872,7 +874,7 @@ def build_or_load_pools( # ── 全新构建 ── paths = resolve_paths(config.workspace_dir) questions = load_benchmark(paths.questions_dir) - with HarnessLog(str(db_path), baseline_run_id) as hlog: + with HarnessLog(str(db_path), baseline_run_id, register_run=False) as hlog: rows = hlog.query( "SELECT question_id, prediction, answer FROM predictions WHERE run_id=?", (baseline_run_id,), diff --git a/app/harness/runner.py b/app/harness/runner.py index 463f52e..f468e12 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -920,7 +920,9 @@ class Runner: self._gate_units_by_id: dict[str, QuestionUnit] = { u.unit_id: u for u in build_units(questions) } - with HarnessLog(str(self._paths.db_path), pools.baseline_run_id) as log: + with HarnessLog( + str(self._paths.db_path), pools.baseline_run_id, register_run=False + ) as log: rows = log.query( "SELECT question_id, prediction, answer FROM predictions WHERE run_id=?", (pools.baseline_run_id,), diff --git a/tests/unit/test_harness_log.py b/tests/unit/test_harness_log.py index 5e90855..35a0e84 100644 --- a/tests/unit/test_harness_log.py +++ b/tests/unit/test_harness_log.py @@ -215,6 +215,30 @@ class TestHarnessLogUpsert: rows = log3.query("SELECT COUNT(*) as cnt FROM _runs WHERE run_id='run_1'") assert rows[0]["cnt"] == 1 + def test_register_run_false_skips_upsert(self, tmp_path: Path) -> None: + """register_run=False 时只读打开不改写已有 _runs 行（started_at/status 不变）。""" + db = str(tmp_path / "h.db") + + def _read_run_row(run_id: str) -> dict: + # 用 register_run=False 只读，避免读取本身污染 _runs + with HarnessLog(db, run_id, register_run=False) as log: + rows = log.query( + "SELECT started_at, status FROM _runs WHERE run_id=?", (run_id,) + ) + return rows[0] + + with HarnessLog(db, "r1"): + pass # 初次注册 + 正常退出置 completed + row0 = _read_run_row("r1") + + time.sleep(0.02) + with HarnessLog(db, "r1", register_run=False) as log: + log.query("SELECT 1") # 只读 + row1 = _read_run_row("r1") + + assert row1["started_at"] == row0["started_at"] + assert row1["status"] == row0["status"] + # =========================================================================== # RunLogImpl 测试 From c68de849dabad5eddd3fcd6b7ccd73801d88c1a9 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:38:05 -0400 Subject: [PATCH 24/43] fix: slow-update R2 dual_metric uses slow_candidate kind (not final) --- app/harness/runner.py | 4 +++- tests/unit/test_harness_observation.py | 31 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/app/harness/runner.py b/app/harness/runner.py index f468e12..068601d 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -1443,11 +1443,13 @@ class Runner: r2_skills_version = self._current_version("skills") new_prompts_version = self._current_version("prompts") eval_r2 = await self._eval_full_val(epoch, pools, run_suffix="_p2") + # R2 是可能被 revert 的慢更新候选，用 slow_candidate 口径，不占 final + # （epoch 终值唯一由 Phase 2 的 final 承载） write_dual_metric( str(self._paths.db_path), run_id=self._config.run_id, epoch=epoch, - version_kind="final", + version_kind="slow_candidate", skills_version=r2_skills_version, prompts_version=new_prompts_version, pool="val", diff --git a/tests/unit/test_harness_observation.py b/tests/unit/test_harness_observation.py index d37cc40..c6b7fd2 100644 --- a/tests/unit/test_harness_observation.py +++ b/tests/unit/test_harness_observation.py @@ -71,6 +71,37 @@ def test_write_read_dual_metric(db_path: str, run_id: str) -> None: assert row["run_id"] == run_id +def test_dual_metric_kinds_distinguishable(db_path: str, run_id: str) -> None: + """observation 层接受任意 version_kind：final 与 slow_candidate 可区分共存。""" + write_dual_metric( + db_path, + run_id=run_id, + epoch=1, + version_kind="final", + skills_version="v1", + prompts_version="v1", + pool="val", + hard_acc=0.7, + soft_score=None, + mixed_score=None, + ) + write_dual_metric( + db_path, + run_id=run_id, + epoch=1, + version_kind="slow_candidate", + skills_version="v2", + prompts_version="v2", + pool="val", + hard_acc=0.6, + soft_score=None, + mixed_score=None, + ) + rows = read_dual_metric(db_path, run_id=run_id) + kinds = {row["version_kind"] for row in rows if row["epoch"] == 1} + assert kinds == {"final", "slow_candidate"} + + # --------------------------------------------------------------------------- # shadow_gate # --------------------------------------------------------------------------- From b307f51340a10ddde4f8f75b1e3faf8329049b6f Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:41:19 -0400 Subject: [PATCH 25/43] feat: --retry-uncertain re-diagnoses uncertain rows; fix docstring --- adapters/baseline_diagnosis_store.py | 18 +++++++++++--- app/harness/baseline_diagnosis.py | 18 ++++++++++---- app/harness/video_split_cli.py | 10 ++++++++ core/evolution/protocols.py | 10 +++++++- tests/unit/test_baseline_diagnosis_store.py | 27 +++++++++++++++++++++ 5 files changed, 73 insertions(+), 10 deletions(-) diff --git a/adapters/baseline_diagnosis_store.py b/adapters/baseline_diagnosis_store.py index a4cb9ca..c744e71 100644 --- a/adapters/baseline_diagnosis_store.py +++ b/adapters/baseline_diagnosis_store.py @@ -106,21 +106,31 @@ class SqliteDiagnosisSignalStore: ) self._conn.commit() - def done_question_ids(self, baseline_run_id: str, diag_fingerprint: str) -> set[str]: + def done_question_ids( + self, + baseline_run_id: str, + diag_fingerprint: str, + *, + retry_uncertain: bool = False, + ) -> set[str]: """查询指定 run 与诊断指纹下已完成的 question_id 集合。 参数: baseline_run_id: baseline run 标识。 diag_fingerprint: 诊断口径指纹。 + retry_uncertain: True 时追加 `AND tier != 'uncertain'`，把 uncertain + （信号不可信降级）题排除出已完成集，令其被重新诊断；默认 False。 返回: 已落盘信号的 question_id 集合；无匹配时为空集，供断点续跑跳过。 """ - cursor = self._conn.execute( + sql = ( "SELECT DISTINCT question_id FROM baseline_diagnosis" - " WHERE baseline_run_id = ? AND diag_fingerprint = ?", - (baseline_run_id, diag_fingerprint), + " WHERE baseline_run_id = ? AND diag_fingerprint = ?" ) + if retry_uncertain: + sql += " AND tier != 'uncertain'" + cursor = self._conn.execute(sql, (baseline_run_id, diag_fingerprint)) return {r["question_id"] for r in cursor.fetchall()} def load(self, baseline_run_id: str, diag_fingerprint: str) -> list[DiagnosisSignalRow]: diff --git a/app/harness/baseline_diagnosis.py b/app/harness/baseline_diagnosis.py index 5632f90..6002911 100644 --- a/app/harness/baseline_diagnosis.py +++ b/app/harness/baseline_diagnosis.py @@ -6,7 +6,9 @@ 包装内层 RunLog，兼容 traces 未落表的历史 run）； 3. 把 error_attributions / infra / degraded 三类产物确定性投影为 DiagnosisSignalRow（tier 由 split_selection.score_signal 判定）； - 4. 逐行 store.upsert 落盘，单行单事务 → 崩溃最多丢正在写的一行。 + 4. run 末（Phase 3）逐行 store.upsert 落库——诊断在 Phase 2 全部跑完后才落库， + 故崩溃丢本次 run 未落库的全部结果（不是"仅一行"）；靠 GovernedLLMClient 的 + Redis 缓存缓解重跑时的 LLM 重烧，下次调用命中缓存直接续。 错误处理诚实标注（不谎称全传播）: - run_diagnosis 的 C1/C2 阶段（指标计算、错误归因）网络/API 失败经 @@ -70,8 +72,9 @@ async def run_baseline_diagnosis( questions: dict[str, GeneratedQuestion], store: DiagnosisSignalStore, deps: DiagnosisDeps, + retry_uncertain: bool = False, ) -> None: - """对 baseline run 的错题跑离线诊断并把信号逐行落库（断点续跑幂等）。 + """对 baseline run 的错题跑离线诊断并把信号落库（断点续跑幂等）。 参数: baseline_run_id: baseline run 标识（如 "infer_adhoc"），信号行主键之一。 @@ -79,8 +82,10 @@ async def run_baseline_diagnosis( wrong_ids: 本次待诊断的可诊断错题 question_id 列表（保序）。 questions: question_id → GeneratedQuestion 映射，需覆盖 wrong_ids 全部题 及 run_diagnosis 返回的所有 infra/degraded 题（用于取 video_id/task_type）。 - store: 诊断信号存储端口，逐行 upsert 落盘并提供 done_question_ids 续跑查询。 + store: 诊断信号存储端口，upsert 落盘并提供 done_question_ids 续跑查询。 deps: 外部依赖束（见 DiagnosisDeps）。 + retry_uncertain: True 时把已落 tier='uncertain'（信号不可信降级）的题也纳入 + remaining 重新诊断，透传给 store.done_question_ids；默认 False。 返回: None。副作用为把逐题 DiagnosisSignalRow 写入 store。 @@ -93,7 +98,9 @@ async def run_baseline_diagnosis( （T0）、degraded_question_ids（uncertain）。 """ # Phase 1: 算 remaining（续跑幂等） - done = store.done_question_ids(baseline_run_id, diag_fingerprint) + done = store.done_question_ids( + baseline_run_id, diag_fingerprint, retry_uncertain=retry_uncertain + ) remaining = [qid for qid in wrong_ids if qid not in done] if not remaining: logger.info( @@ -169,7 +176,8 @@ def _project_and_persist( {tier: 行数} 计数字典（T2/T1/T0/uncertain），供上层日志与 manifest。 关键实现: - 逐行 upsert（单行单事务），中途崩溃最多丢正在写的一行。三桶**非互斥**： + 本函数在 run 末（Phase 3）逐行 upsert（单行单事务）；诊断已在 Phase 2 全部 + 跑完，故本阶段中途崩溃丢本次 run 未落库的余下行。三桶**非互斥**： 同一 degraded 错题可能同时出现在 error_attributions（judge 解析失败仍建 attribution）里，故按 **degraded > infra > attribution** 优先级去重——先落 degraded/infra，再在 attribution 循环跳过已落题，保证**每题恰写一行、 diff --git a/app/harness/video_split_cli.py b/app/harness/video_split_cli.py index 754e8f3..1c7e21f 100644 --- a/app/harness/video_split_cli.py +++ b/app/harness/video_split_cli.py @@ -522,6 +522,7 @@ async def run_pipeline( out_dir: Path, generated_at: str, force: bool = False, + retry_uncertain: bool = False, ) -> SplitBuildResult: """内联三阶段：Phase 0 INFRA T0 补录 → Phase 1 诊断 → Phase 2 冻结切分 → McNemar 护栏。 @@ -538,6 +539,7 @@ async def run_pipeline( out_dir: 冻结产物目录（pools.json + split_manifest.json）。 generated_at: 生成时间戳（ISO 字符串，由调用方传入；见模块 C-2 复现锚点约定）。 force: 覆盖已存在冻结产物开关，透传给 build_split（False 时已存在即报错）。 + retry_uncertain: 透传给 run_baseline_diagnosis，令已落 uncertain 题被重新诊断。 返回: SplitBuildResult（冻结三池 + manifest + assignment）。 @@ -557,6 +559,7 @@ async def run_pipeline( questions=questions, store=signal_store, deps=diagnosis_deps, + retry_uncertain=retry_uncertain, ) # Phase 2: 冻结切分（读诊断信号 → 贪心选择 → 视频组原子切三池 → 冻结 + 六条断言）。 @@ -656,6 +659,7 @@ def _execute_real(config: VideoSplitConfig, fingerprint: str, args: argparse.Nam out_dir=out_dir, generated_at=generated_at, force=args.force, + retry_uncertain=args.retry_uncertain, ) ) finally: @@ -784,6 +788,12 @@ def build_arg_parser() -> argparse.ArgumentParser: action="store_true", help="覆盖已存在的冻结 pools.json/manifest（旧产物备份为 .bak.*）", ) + parser.add_argument( + "--retry-uncertain", + action="store_true", + dest="retry_uncertain", + help="把已落 tier='uncertain'（信号不可信降级）的题重新诊断，而非当作已完成跳过", + ) return parser diff --git a/core/evolution/protocols.py b/core/evolution/protocols.py index df4039c..650f9c2 100644 --- a/core/evolution/protocols.py +++ b/core/evolution/protocols.py @@ -127,12 +127,20 @@ class DiagnosisSignalStore(Protocol): """ ... - def done_question_ids(self, baseline_run_id: str, diag_fingerprint: str) -> set[str]: + def done_question_ids( + self, + baseline_run_id: str, + diag_fingerprint: str, + *, + retry_uncertain: bool = False, + ) -> set[str]: """查询指定 run 与诊断指纹下已完成的 question_id 集合。 参数: baseline_run_id: baseline run 标识。 diag_fingerprint: 诊断口径指纹。 + retry_uncertain: True 时把 tier='uncertain'（信号不可信降级）题视为 + 未完成，令其被重新诊断；默认 False（uncertain 也算完成，不重诊）。 返回: 已落盘信号的 question_id 集合，用于断点续跑跳过。 diff --git a/tests/unit/test_baseline_diagnosis_store.py b/tests/unit/test_baseline_diagnosis_store.py index 00a654d..afee6b9 100644 --- a/tests/unit/test_baseline_diagnosis_store.py +++ b/tests/unit/test_baseline_diagnosis_store.py @@ -85,6 +85,33 @@ def test_null_fields_for_infra_row(tmp_path): assert rows[0].error_type is None and rows[0].evolution_target is None and rows[0].infra is True +def test_done_question_ids_retry_uncertain_excludes(tmp_path): + """retry_uncertain=True 时 uncertain 题不算完成（会被重诊），非 uncertain 仍算完成。""" + s = _store(tmp_path) + # T2（defect）题：已完成 + s.upsert( + DiagnosisSignalRow( + "t2_qid", "v1", "run", "fp", "Counting", + "search_failure", "defect", "T2", "skill", + degraded=False, infra=False, session_id="s", + ) + ) + # uncertain（degraded）题：信号不可信 + s.upsert( + DiagnosisSignalRow( + "uncertain_qid", "v2", "run", "fp", "Counting", + None, None, "uncertain", None, + degraded=True, infra=False, session_id=None, + ) + ) + # 默认：两者都算完成 + assert s.done_question_ids("run", "fp") == {"t2_qid", "uncertain_qid"} + # retry_uncertain：uncertain 被排除，T2 仍算完成 + done = s.done_question_ids("run", "fp", retry_uncertain=True) + assert "uncertain_qid" not in done + assert "t2_qid" in done + + def test_context_manager_closes_connection(tmp_path): # with 块退出后连接关闭，再操作应报 ProgrammingError with SqliteDiagnosisSignalStore(str(tmp_path / "h.db")) as s: From 03337af8f85d70bb7bc23967487d511e1bc476c0 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 05:54:02 -0400 Subject: [PATCH 26/43] fix: gate INFRA isolation edge cases (all-INFRA fail-loud, parse_error in guard) --- app/harness/observation.py | 3 +- app/harness/validate.py | 27 ++++-- tests/unit/test_harness_validate.py | 138 ++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+), 9 deletions(-) diff --git a/app/harness/observation.py b/app/harness/observation.py index a10ef0b..7b1b573 100644 --- a/app/harness/observation.py +++ b/app/harness/observation.py @@ -135,7 +135,8 @@ def write_dual_metric( db_path: SQLite 路径。 run_id: 训练 run ID。 epoch: 轮次（1-based）。 - version_kind: baseline / best_hard / best_mixed / final。 + version_kind: baseline / best_hard / best_mixed / final / slow_candidate + （slow_candidate = 慢更新 R2 可能被 revert 的候选，不占 epoch 终值 final 口径）。 skills_version / prompts_version: 评估的资源版本。 pool: val / test。 hard_acc: hard 准确率。 diff --git a/app/harness/validate.py b/app/harness/validate.py index f4c6be4..304d124 100644 --- a/app/harness/validate.py +++ b/app/harness/validate.py @@ -332,7 +332,9 @@ async def _resolve_baseline_block( if miss_units: miss_questions = flatten_units(miss_units) r_b = await run_inference(miss_questions, run_id=run_id, skills_dir=base_skills_dir) - errors_inc = r_b.stop_reason_counts.get("error", 0) + # 护栏错误计数与 INFRA 判定口径一致：error + parse_error 都计入， + # 使 parse_error 风暴同样能触发 gate_guard_err 熔断（不被绕过）。 + errors_inc = sum(r_b.stop_reason_counts.get(reason, 0) for reason in _INFRA_STOP_REASONS) denom_inc = r_b.total infra_qids = _infra_question_ids_from_db(log, r_b.run_id, miss_questions) fresh_per_q = _candidate_correctness_from_db(log, r_b.run_id, miss_questions) @@ -380,7 +382,9 @@ async def _run_candidate_block( questions = flatten_units(units) r_c = await run_inference(questions, run_id=run_id, skills_dir=cand_dir) c_per_q = _candidate_correctness_from_db(log, r_c.run_id, questions) - return c_per_q, r_c.stop_reason_counts.get("error", 0), r_c.total + # 护栏错误计数与 INFRA 判定口径一致：error + parse_error 都计入。 + errors_inc = sum(r_c.stop_reason_counts.get(reason, 0) for reason in _INFRA_STOP_REASONS) + return c_per_q, errors_inc, r_c.total def _build_evidence_rows( @@ -624,6 +628,15 @@ async def _run_local_validation( log=log, run_id=f"{gate_run_prefix}_b{block_idx}_base", ) + # 本块全 INFRA：无有效单元可配对——候选无需空跑，仅把基线侧错误计入护栏后 + # 累计剔除数进入下一块（护栏仍能在整轮 INFRA 错误率超阈值时熔断）。 + n_excluded += len(unit_chunk) - len(valid_chunk) + if not valid_chunk: + errors += err_b + infra_denom += den_b + _check_infra_guard(errors, infra_denom, gate_guard_err) + continue + # 候选侧只跑基线侧判定有效（非 INFRA）的单元，保证配对 unit_ids 两侧一致 c_per_q, err_c, den_c = await _run_candidate_block( units=valid_chunk, @@ -639,11 +652,6 @@ async def _run_local_validation( infra_denom += den_b + den_c _check_infra_guard(errors, infra_denom, gate_guard_err) - # 本块全 INFRA：无有效单元可配对，累计剔除数后跳过判定进入下一块 - n_excluded += len(unit_chunk) - len(valid_chunk) - if not valid_chunk: - continue - # Phase 3: 折叠成单元视图 + 配对 + 证据行 + 块间判定（均用有效单元） c_units = unit_correctness_view(valid_chunk, c_per_q) candidate_per_q.update(c_per_q) @@ -668,8 +676,11 @@ async def _run_local_validation( if verdict.decision != "continue": break + # verdict 仍为 None ⟺ 全部单元被 INFRA 排除（空 ladder 已在入口拒绝）。 + # 明确失败，避免落到误导性的"空阶梯"断言而无法定位为 INFRA 原因。 + if verdict is None: + raise RuntimeError("gate 阶梯所有 unit 被判为 INFRA 排除，无法验证（检查推理基础设施）") # 最后一块判定即终态（n_remaining=0 → provisional/inertia） - assert verdict is not None, "空阶梯应已在 validate_skill_local 入口拒绝" return _finalize_outcome( verdict=verdict, w=w, diff --git a/tests/unit/test_harness_validate.py b/tests/unit/test_harness_validate.py index bcd1ef6..f93b641 100644 --- a/tests/unit/test_harness_validate.py +++ b/tests/unit/test_harness_validate.py @@ -575,6 +575,144 @@ async def test_baseline_infra_error_not_cached(tmp_path: Path) -> None: log.close() +@pytest.mark.asyncio +async def test_all_infra_ladder_raises_clear_error(tmp_path: Path) -> None: + """整个阶梯所有 unit 都被判为 INFRA 排除 → 明确 RuntimeError（非误导性空阶梯断言）。""" + workspace = _setup_workspace(tmp_path) + log = _make_log(workspace) + questions = _make_questions(4) + cache = BaselineCache(workspace / "baseline_cache.json") + + candidate_calls: list[str] = [] + + async def mock_fn(qs, *, run_id, skills_dir): + if run_id.endswith("_cand"): + candidate_calls.append(run_id) + # 基线臂逐题全部 INFRA error（候选臂在修复后不应被空跑） + for q in qs: + log.insert( + "predictions", + { + "run_id": run_id, + "video_id": "v0", + "question_id": q.question_id, + "task_type": "temporal", + "prediction": "", + "answer": "A", + "evidence": "", + "reasoning": "", + "steps_used": 1, + "prompt_tokens": 10, + "completion_tokens": 10, + "stop_reason": "error", + "steps_json": "[]", + }, + ) + total = len(qs) + return InferenceResult( + run_id=run_id, + accuracy=0.0, + total=total, + correct=0, + per_task_type={}, + steps_mean=1.0, + token_usage={"prompt_tokens": 10, "completion_tokens": 10}, + stop_reason_counts={"error": total}, + ) + + try: + with pytest.raises(RuntimeError, match="INFRA"): + await validate_skill_local( + workspace_dir=workspace, + base_skills_version="v1", + task_type="temporal", + target_file="temporal.md", + candidate_content="content", + base_skill_content="baseline skill content", + ladder_items=questions, + gate_params=_DEFAULT_GATE_PARAMS, + gate_block=4, + gate_n_max=20, + gate_guard_err=0.9, # 高阈值：4 题 <10 分母不触发错误率护栏 + baseline_cache=cache, + prompts_version="p1", + run_inference=mock_fn, + log=log, + gate_run_prefix="step1_gate_test", + ) + # 全 INFRA 块不应触发候选空跑 + assert candidate_calls == [] + finally: + log.close() + + +@pytest.mark.asyncio +async def test_parse_error_counts_toward_guard(tmp_path: Path) -> None: + """stop_reason=parse_error 也计入护栏错误率（与 INFRA 判定口径一致）→ 超阈值熔断。""" + workspace = _setup_workspace(tmp_path) + log = _make_log(workspace) + questions = _make_questions(6) + cache = BaselineCache(workspace / "baseline_cache.json") + + async def mock_fn(qs, *, run_id, skills_dir): + # 逐题 stop_reason 保持 completed（不触发 per-unit INFRA 排除）， + # 但汇总 stop_reason_counts 报大量 parse_error（应计入护栏）。 + for q in qs: + log.insert( + "predictions", + { + "run_id": run_id, + "video_id": "v0", + "question_id": q.question_id, + "task_type": "temporal", + "prediction": "Z", + "answer": "A", + "evidence": "", + "reasoning": "", + "steps_used": 1, + "prompt_tokens": 10, + "completion_tokens": 10, + "stop_reason": "completed", + "steps_json": "[]", + }, + ) + total = len(qs) + # 两臂各 5 个 parse_error → 累计 10/12 > 0.5 触发护栏 + return InferenceResult( + run_id=run_id, + accuracy=0.0, + total=total, + correct=0, + per_task_type={}, + steps_mean=1.0, + token_usage={"prompt_tokens": 10, "completion_tokens": 10}, + stop_reason_counts={"completed": 1, "parse_error": 5}, + ) + + try: + with pytest.raises(RuntimeError, match="错误率过高"): + await validate_skill_local( + workspace_dir=workspace, + base_skills_version="v1", + task_type="temporal", + target_file="temporal.md", + candidate_content="content", + base_skill_content="baseline skill content", + ladder_items=questions, + gate_params=_DEFAULT_GATE_PARAMS, + gate_block=6, + gate_n_max=20, + gate_guard_err=0.5, + baseline_cache=cache, + prompts_version="p1", + run_inference=mock_fn, + log=log, + gate_run_prefix="step1_gate_test", + ) + finally: + log.close() + + @pytest.mark.asyncio async def test_last_block_terminal(tmp_path: Path) -> None: """单块 + n_remaining=0 → 终态判定（provisional 或 inertia），非 continue。""" From e1cac644c89fd7b32fe96621981f07c8d4965b17 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 06:03:17 -0400 Subject: [PATCH 27/43] docs: address Codex review of WP3 plan (pools passthrough, holdout memo, DELETE table) --- .../2026-07-16-preflight-wp3-train-loop.md | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/research-wiki/plans/2026-07-16-preflight-wp3-train-loop.md b/research-wiki/plans/2026-07-16-preflight-wp3-train-loop.md index 7858228..8da47fd 100644 --- a/research-wiki/plans/2026-07-16-preflight-wp3-train-loop.md +++ b/research-wiki/plans/2026-07-16-preflight-wp3-train-loop.md @@ -201,9 +201,10 @@ git commit -m "fix: early_stop patience counts epochs not steps" ## Task 4: 可训练性预检 **Files:** -- Modify: `app/harness/runner.py:865-879`（_setup_train_run） -- Modify: `app/harness/config.py`（RunConfig 加 `trainable_min_units`）+ `config/train_videomme.yaml`（WP 收尾建，先加字段默认） -- Test: `tests/unit/test_harness_runner.py` +- Modify: `app/harness/runner.py`（`train` 入口过滤 + `_setup_train_run` 接收 filtered task_types） +- Modify: `app/harness/config.py`（RunConfig 加 `trainable_min_units` + 正整数校验） +- Modify: `app/harness/checkpoint.py:37`（`trainable_min_units` 入 `_STRUCTURAL_KEYS` 指纹） +- Test: `tests/unit/test_harness_runner.py`、`tests/unit/test_harness_checkpoint.py` - [ ] **Step 1: 写失败测试** @@ -263,15 +264,20 @@ def _filter_untrainable_types( return new_pools, new_types ``` （确认 `from dataclasses import replace` 已 import；`Pools` 是否 frozen dataclass 支持 `replace`——若非，按其构造方式重建。） -在 `_setup_train_run`（pools 加载后、`_init_gate_pools` L874 前）调用： + +**过滤结果必须回传主循环（Codex Critical）**：`RunConfig` 是 `@dataclass(frozen=True)`，**不能** `self._config.task_types = ...`（会 FrozenInstanceError），且过滤后的 pools 必须被 `train()` 后续的 batch/step/slow-update/final-eval 全部使用。实现方式： +- 在 `train(pools)` **入口第一步**（`_setup_train_run` 调用之前）过滤： ```python - pools, self._config.task_types = _filter_untrainable_types( + pools, filtered_task_types = _filter_untrainable_types( pools, self._config.task_types, self._config.eval_min_per_class, self._config.trainable_min_units, ) ``` -（`RunConfig` 若 frozen，改用局部变量传递而非改 self._config；实现前读 config.py 确认可变性。） -`app/harness/config.py` `RunConfig` 加字段 `trainable_min_units: int`（无默认，显式配置；train yaml 提供）。 +- 把 filtered `pools` 传给 `_setup_train_run(pools)` 与 train() 后续所有消费点（build_batches / slow_update / final_eval 均用这个 filtered pools，不再触碰原始 pools）。 +- 把 `filtered_task_types` 传给 gate 建立（`_setup_train_run`/`_init_gate_pools` 用它而非 `self._config.task_types`）——新增参数透传，不改 frozen config。 + +`app/harness/config.py`：`RunConfig` 加字段 `trainable_min_units: int`（无默认，显式配置；train yaml 提供）；在配置校验函数（如 `validate_config`，config.py:277 附近）加 `trainable_min_units >= 1` 断言（<1 报错）。 +`app/harness/checkpoint.py:37` `_STRUCTURAL_KEYS` 加入 `"trainable_min_units"`——该值改变会改变 pools 过滤结果与训练轨迹，必须纳入 checkpoint 结构指纹，resume 时变化即拒绝复用旧 checkpoint。 - [ ] **Step 4: 测试通过 + 回归** @@ -415,7 +421,7 @@ Expected: FAIL（当前 None → wrong_by_error 走 defect 正文）。 "疑似 judge 基础设施故障，中止训练（不以降级信号驱动进化）。" ) ``` -（确认 `DiagnosisResult.degraded_count` 字段可用，L2311。） +（确认 `DiagnosisResult.degraded_count` 字段可用，定义在 `core/evolution/types.py:299`。） - [ ] **Step 5: 测试通过 + 回归** @@ -455,13 +461,13 @@ Expected: FAIL（当前 append，无 DELETE）。 - [ ] **Step 3: _run_step 开头 DELETE** -`app/harness/runner.py` `_run_step`，在 rollout（L1012）之前加： +`app/harness/runner.py` `_run_step`，在 rollout（L1012）之前加（**用 `IF EXISTS` 避免 fresh workspace 首跑时 predictions/traces 表尚未由 `run_inference._ensure_tables` 创建导致 `OperationalError: no such table`**）： ```python with HarnessLog(str(self._paths.db_path), run_id, register_run=False) as log: log.execute("DELETE FROM predictions WHERE run_id=?", (run_id,)) log.execute("DELETE FROM traces WHERE run_id=?", (run_id,)) ``` -（`register_run=False` 来自 WP4 Task 8；若 HarnessLog 无 `execute` 便捷方法，用其现有 `query`/连接接口，实现前读 log.py 对齐。） +> SQLite `DELETE FROM ` 对不存在的表会抛 `no such table`。两种消解方式择一（实现前读 log.py 确认）：① rollout 由 `run_inference` 先建表——把 DELETE 移到**首次 rollout 之后、诊断之前**并只在 resume 重跑（step 已有旧行）时执行；② 或 DELETE 前先 `CREATE TABLE IF NOT EXISTS`（复用 inference 的 PREDICTIONS_SCHEMA/TRACES_SCHEMA），保证幂等无害。推荐 ②（无害且简单）。`register_run=False` 来自 WP4；若 HarnessLog 无 `execute` 便捷方法，用其现有连接接口。 - [ ] **Step 4: checkpoint 提到 gate save 之后（消除双计窗口）** @@ -505,15 +511,17 @@ async def test_holdout_dedup_skips_reevaluated_versions(...): Run: `conda run -n Video-Tree-TRM python -m pytest tests/unit/test_harness_runner.py -k holdout_dedup -v` Expected: FAIL（当前四向各跑一次）。 -- [ ] **Step 3: 实现去重** +- [ ] **Step 3: 实现去重（进程内备忘录，不改 holdout_eval schema）** -`app/harness/runner.py` `_holdout_four_way`： -- **baseline 向**：不调 `_eval_version_on_pool`，改从基线 predictions（baseline_run_id）读 test 题对错算 acc（test 题在 infer_adhoc 已推理）；epoch>1 复用首次结果。 -- **final 向**：真评 600（不变）。 -- **best_hard 向**：维护"已评版本→test_acc"备忘录，resume 时从 `holdout_eval` 表 hydrate（查已落库 `(skills_v,prompts_v)→acc`）；版本对未评过且 ≠ final 才真评，否则引用。 -- **best_mixed 向**：引用赢家（best_hard 或 final 之一，`_pick_mixed_best` 已定）的 test 成绩，写 holdout_eval 时标指针，不推理。 +> **schema 约束（Codex Critical）**：当前 `holdout_eval` 表（observation.py:78）不存 skills_version/prompts_version/pointer，无法按版本反查做跨-run hydrate。本 task **不扩展该 schema**（避免结构性风险），改用 **train() 进程内备忘录** `dict[(skills_v,prompts_v), float]` 去重。代价：resume 后备忘录清空、已评版本会重评一次——resume 是异常路径、重评 600 题成本可接受，换取零 schema 变更风险。完整跨-run hydrate 记 future work。 -实现前完整读 `_holdout_four_way`（1881-1921）、`write_holdout_eval`、`_eval_version_on_pool`、`read`/hydrate 路径对齐。 +`app/harness/runner.py` `_holdout_four_way`（在 `_TrainState` 加一个 `holdout_memo: dict[tuple[str,str], float] = field(default_factory=dict)` 字段）： +- **baseline 向**：不调 `_eval_version_on_pool`，改从基线 predictions（`baseline_run_id`）读 test 题对错算 acc（test 题在 infer_adhoc 已全推理过）；结果存 memo，epoch>1 直接复用（0 推理）。 +- **final 向**：真评 600，算完存 `memo[(final_sv,final_pv)]`。 +- **best_hard 向**：若 `(best_sv,best_pv)` 已在 `memo`（== final 或往轮已评）则引用，否则真评并存 memo。 +- **best_mixed 向**：`_pick_mixed_best` 选出的赢家必是 best_hard 或 final 之一，其 test acc 已在 memo，直接引用写 holdout_eval，0 推理。 + +实现前完整读 `_holdout_four_way`（~1885）、`write_holdout_eval`、`_eval_version_on_pool` 对齐；`write_holdout_eval` 调用保持不变（仍逐向写观测行，只是 acc 来源改为 memo 复用/基线推导）。 - [ ] **Step 4: 测试通过 + 回归** From 7d02cded992d5e8d9c048b16a651608970fad9a1 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 06:05:55 -0400 Subject: [PATCH 28/43] fix: wrap diagnosis run_log with StepsJsonRunLog (restore algo #7 traces) --- app/harness/runner.py | 4 +- tests/unit/test_runner_diag_tree_inject.py | 64 ++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/app/harness/runner.py b/app/harness/runner.py index 068601d..f7bc167 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -2168,13 +2168,15 @@ class Runner: self, run_id: str, *, question_ids: list[str] | None = None ) -> DiagnosisResult: """执行两阶段诊断。""" + from app.harness.baseline_run_log import StepsJsonRunLog from app.harness.log import RunLogImpl from app.harness.workspace import VersionedSkillStore from app.question_gen import load_benchmark from core.evolution.diagnose import run_diagnosis questions = load_benchmark(self._paths.questions_dir) - run_log = RunLogImpl(str(self._paths.db_path)) + # traces 表空时（如训练 rollout 只落 steps_json）从 steps_json 重建轨迹，恢复算法 #7 + run_log = StepsJsonRunLog(RunLogImpl(str(self._paths.db_path))) skill_store = VersionedSkillStore(self._paths.skills_dir) diagnose_prompts = self._load_diagnose_prompts() diff --git a/tests/unit/test_runner_diag_tree_inject.py b/tests/unit/test_runner_diag_tree_inject.py index 2d579fc..404b649 100644 --- a/tests/unit/test_runner_diag_tree_inject.py +++ b/tests/unit/test_runner_diag_tree_inject.py @@ -147,6 +147,70 @@ def _fake_question(question_id: str, video_id: str) -> object: ) +@pytest.mark.asyncio +async def test_diagnosis_reads_traces_from_steps_json( + runner_with_real_store: Runner, +) -> None: + """traces 表为空但 predictions.steps_json 有轨迹时，诊断仍拿到非空 traces。 + + 构造一条只写 steps_json、不写 traces 表的 predictions 行；patch run_diagnosis + 捕获传入的 run_log，直接 await 其 get_traces 断言经 StepsJsonRunLog 从 + steps_json 重建出非空轨迹（算法 #7 恢复）。 + """ + from app.harness.inference import PREDICTIONS_SCHEMA + from app.harness.log import HarnessLog + + steps_json = json.dumps( + [ + { + "thought": "先看整体", + "tool_call": {"tool": "search_tree", "args": {"query": "开场"}}, + "tool_output": "命中 L2 节点 A", + } + ], + ensure_ascii=False, + ) + with HarnessLog(str(runner_with_real_store._paths.db_path), "infer_adhoc") as log: + log.create_table("predictions", PREDICTIONS_SCHEMA) + log.create_table("traces", {"video_id": "TEXT", "question_id": "TEXT", "step": "INTEGER"}) + log.insert( + "predictions", + { + "video_id": _REAL_VIDEO_ID, + "question_id": _REAL_QUESTION_ID, + "task_type": "Action Reasoning", + "prediction": "A", + "answer": "B", + "evidence": "", + "reasoning": "", + "steps_used": 1, + "prompt_tokens": 0, + "completion_tokens": 0, + "stop_reason": "finished", + "steps_json": steps_json, + }, + ) + + captured: dict[str, object] = {} + + async def _fake_run_diagnosis(**kwargs: object) -> DiagnosisResult: + captured["run_log"] = kwargs["run_log"] + return _empty_diagnosis_result() + + with patch( + "core.evolution.diagnose.run_diagnosis", + new=AsyncMock(side_effect=_fake_run_diagnosis), + ): + await runner_with_real_store._run_diagnosis( + "infer_adhoc", question_ids=[_REAL_QUESTION_ID] + ) + + run_log = captured["run_log"] + traces = await run_log.get_traces("infer_adhoc", question_ids=[_REAL_QUESTION_ID]) + assert traces, "traces 表空时应从 steps_json 重建出非空轨迹" + assert traces[0]["tool_name"] == "search_tree" + + @pytest.mark.asyncio async def test_run_diagnosis_full_scan_loads_all_video_trees( runner_with_real_store: Runner, From 25918a73ff0ad5744ded0a8eaaf524feca4fb860 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 06:06:58 -0400 Subject: [PATCH 29/43] fix: normalize non-scalar prediction; harden predictions insert --- app/harness/inference.py | 35 +++++++++++++++-- tests/unit/test_harness_inference.py | 59 ++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/app/harness/inference.py b/app/harness/inference.py index 6092025..911dde5 100644 --- a/app/harness/inference.py +++ b/app/harness/inference.py @@ -14,6 +14,7 @@ from __future__ import annotations import asyncio import json +import sqlite3 from collections import defaultdict from dataclasses import dataclass from typing import TYPE_CHECKING, Any @@ -162,6 +163,24 @@ def _to_text_field(value: Any) -> str: return json.dumps(value, ensure_ascii=False) +def _normalize_prediction(answer: object) -> str | None: + """归一化 prediction 落库值。 + + LLM 提交的 answer 有时是 list/dict（如 {'answer': ['B']}），sqlite 无法绑定 + 非标量类型直接入库会抛 ProgrammingError 击穿整轮 gather。None 保留（INFRA 空 + 预测语义，供正确率判定天然计错）；str 原样；其余 JSON 序列化为文本。 + + 参数: + answer: LoopResult.result 中的 answer 原始值（可能是 None/str/list/dict）。 + + 返回: + None（保留空预测语义）或可直接入库的字符串。 + """ + if answer is None or isinstance(answer, str): + return answer + return _to_text_field(answer) + + def _zero_result(run_id: str) -> InferenceResult: """空记录时的零值 InferenceResult。 @@ -419,7 +438,7 @@ async def _run_single_question( reasoning = _to_text_field(result_dict.get("reasoning", "")) record.update( { - "prediction": result_dict.get("answer"), + "prediction": _normalize_prediction(result_dict.get("answer")), "evidence": evidence, "reasoning": reasoning, "steps_used": loop_result.steps_used, @@ -442,8 +461,18 @@ async def _run_single_question( except Exception: logger.exception("[{}] QA {} 执行异常", qa.video_id, qa.question_id) - # prediction 必落库（try 外，无论成败） - await asyncio.to_thread(log.insert, "predictions", record) + # prediction 必落库（try 外，无论成败）；绑定异常降级为最小 error 行，不击穿 gather + try: + await asyncio.to_thread(log.insert, "predictions", record) + except (sqlite3.InterfaceError, sqlite3.ProgrammingError): + logger.exception("[{}] QA {} 落库绑定异常，降级为 error 行", qa.video_id, qa.question_id) + record["prediction"] = None + record["stop_reason"] = "error" + await asyncio.to_thread( + log.insert, + "predictions", + {k: v for k, v in record.items() if isinstance(v, (str, int, float, type(None)))}, + ) return record diff --git a/tests/unit/test_harness_inference.py b/tests/unit/test_harness_inference.py index 2ec702e..fd57b04 100644 --- a/tests/unit/test_harness_inference.py +++ b/tests/unit/test_harness_inference.py @@ -559,6 +559,65 @@ class TestPredictionAlwaysWritten: assert rows[0]["prediction"] is None +def _make_nonscalar_llm_response() -> LLMResponse: + """构造 submit_answer 提交非标量 answer（list）的 LLMResponse。""" + content = json.dumps( + { + "reflect": {"observation": "找到答案"}, + "plan": {"next_step": "提交"}, + "action": { + "tool": "submit_answer", + "args": { + "answer": ["B"], + "evidence": "证据文本", + "reasoning": "推理过程", + }, + }, + } + ) + return LLMResponse( + content=content, + thinking="思考过程", + model="test-model", + provider="test", + prompt_tokens=100, + completion_tokens=50, + latency_ms=200, + ttft_ms=30.0, + max_inter_token_ms=5.0, + cache_hit=False, + call_id="test-call-nonscalar", + ) + + +class TestNonScalarPrediction: + """非标量 prediction 归一化 + 落库加固测试。""" + + @pytest.mark.asyncio + async def test_nonscalar_prediction_does_not_crash(self, harness_log: HarnessLog) -> None: + """submit_answer 返回 {'answer': ['B']} 时归一化落库，不抛 sqlite 绑定异常。""" + llm = AsyncMock() + llm.chat.return_value = _make_nonscalar_llm_response() + + result = await run_inference( + [_make_question(answer="B")], + llm=llm, + tool_dispatch_fn=_stub_tool_dispatch, + prompt_builder=_stub_prompt_builder, + log=harness_log, + run_id="run-nonscalar", + concurrency=1, + max_steps=10, + skill_mode="auto", + ) + + assert result.total == 1 + rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("test-run",)) + assert len(rows) == 1 + # prediction 被 JSON 序列化为字符串，不再是 Python list + assert rows[0]["prediction"] == '["B"]' + + class TestPluginsFactory: """plugins_factory 调用测试。""" From c44f6010eb7e1c5d41850a5beb2faac2e8bc9068 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 06:09:11 -0400 Subject: [PATCH 30/43] fix: early_stop patience counts epochs not steps --- app/harness/checkpoint.py | 4 +- app/harness/runner.py | 19 ++-- tests/integration/test_checkpoint_pair.py | 2 +- tests/unit/test_harness_checkpoint.py | 6 +- tests/unit/test_harness_runner.py | 104 +++++++--------------- 5 files changed, 48 insertions(+), 87 deletions(-) diff --git a/app/harness/checkpoint.py b/app/harness/checkpoint.py index f686d3b..5c57544 100644 --- a/app/harness/checkpoint.py +++ b/app/harness/checkpoint.py @@ -94,7 +94,7 @@ def serialize_state(state: Any) -> dict[str, Any]: "eval_prev_run_id": state.eval_prev_run_id, "baseline_skills_version": state.baseline_skills_version, "baseline_prompts_version": state.baseline_prompts_version, - "steps_since_best_improved": state.steps_since_best_improved, + "epochs_since_best_improved": state.epochs_since_best_improved, "epoch_start_skills": state.epoch_start_skills, "changed_task_types_this_epoch": sorted(state.changed_task_types_this_epoch), "rejected_buffer": {k: [asdict(x) for x in v] for k, v in state.rejected_buffer.items()}, @@ -145,7 +145,7 @@ def deserialize_state_fields(d: dict[str, Any]) -> dict[str, Any]: "eval_prev_run_id": d["eval_prev_run_id"], "baseline_skills_version": d["baseline_skills_version"], "baseline_prompts_version": d["baseline_prompts_version"], - "steps_since_best_improved": d["steps_since_best_improved"], + "epochs_since_best_improved": d["epochs_since_best_improved"], "epoch_start_skills": d["epoch_start_skills"], "changed_task_types_this_epoch": set(d["changed_task_types_this_epoch"]), "rejected_buffer": { diff --git a/app/harness/runner.py b/app/harness/runner.py index f7bc167..97ddda9 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -115,7 +115,7 @@ class _TrainState: global_step: int = 0 changed_task_types_this_epoch: set[str] = field(default_factory=set) epoch_start_skills: dict[str, str] = field(default_factory=dict) - steps_since_best_improved: int = 0 + epochs_since_best_improved: int = 0 gate_epoch_observed: bool = False probations: dict[str, Probation] = field(default_factory=dict) gate_cooldown: dict[str, int] = field(default_factory=dict) @@ -291,18 +291,16 @@ def _snapshot_current_skills(skills_dir: Path) -> dict[str, str]: def _should_early_stop( workspace_dir: Path, epoch: int, - steps_this_epoch: int, state: _TrainState, patience: int, ) -> bool: - """步粒度 early stop：本 epoch best 未刷新则累加本 epoch 步数。 + """epoch 粒度 early stop：本 epoch best 未刷新则计数 +1。 参数: workspace_dir: workspace 目录（读 manifest best）。 epoch: 当前 epoch。 - steps_this_epoch: 本 epoch 的 step 总数。 - state: 训练状态（steps_since_best_improved 就地更新）。 - patience: early_stop_patience。 + state: 训练状态（epochs_since_best_improved 就地更新）。 + patience: early_stop_patience（连续无刷新的 epoch 数上限）。 返回: 是否触发 early stop。 @@ -310,10 +308,10 @@ def _should_early_stop( best = read_best(workspace_dir) improved_this_epoch = best is not None and best.get("epoch") == epoch if improved_this_epoch: - state.steps_since_best_improved = 0 + state.epochs_since_best_improved = 0 return False - state.steps_since_best_improved += steps_this_epoch - return state.steps_since_best_improved >= patience + state.epochs_since_best_improved += 1 + return state.epochs_since_best_improved >= patience def _compute_total_steps(pools: Pools, correctness: dict[str, bool], config: RunConfig) -> int: @@ -849,7 +847,6 @@ class Runner: if _should_early_stop( self._config.workspace_dir, epoch, - len(batches), state, self._config.early_stop_patience, ): @@ -1588,7 +1585,7 @@ class Runner: state.best_val_acc = eval_acc state.best_skills_version = skills_v state.best_prompts_version = prompts_v - state.steps_since_best_improved = 0 + state.epochs_since_best_improved = 0 update_best( self._config.workspace_dir, skills=f"skills/{skills_v}", diff --git a/tests/integration/test_checkpoint_pair.py b/tests/integration/test_checkpoint_pair.py index 0b5591a..c6b28dc 100644 --- a/tests/integration/test_checkpoint_pair.py +++ b/tests/integration/test_checkpoint_pair.py @@ -90,7 +90,7 @@ class _FakeState: eval_prev_run_id: str = "run-0" baseline_skills_version: str = "v1" baseline_prompts_version: str = "v1" - steps_since_best_improved: int = 0 + epochs_since_best_improved: int = 0 epoch_start_skills: str = "v1" changed_task_types_this_epoch: set[str] = field(default_factory=set) rejected_buffer: dict = field(default_factory=dict) diff --git a/tests/unit/test_harness_checkpoint.py b/tests/unit/test_harness_checkpoint.py index e5f0b95..1d5d40e 100644 --- a/tests/unit/test_harness_checkpoint.py +++ b/tests/unit/test_harness_checkpoint.py @@ -116,7 +116,7 @@ class _FakeState: eval_prev_run_id: str baseline_skills_version: str baseline_prompts_version: str - steps_since_best_improved: int + epochs_since_best_improved: int epoch_start_skills: str changed_task_types_this_epoch: set[str] rejected_buffer: dict[str, list[RejectedEdit]] @@ -135,7 +135,7 @@ def _make_state() -> _FakeState: eval_prev_run_id="run-abc", baseline_skills_version="v1", baseline_prompts_version="v1", - steps_since_best_improved=2, + epochs_since_best_improved=2, epoch_start_skills="v1", changed_task_types_this_epoch={"temporal", "causal"}, rejected_buffer={"temporal": [_make_rejected_edit()]}, @@ -201,7 +201,7 @@ class TestSerializeDeserializeRoundtrip: assert restored["eval_prev_run_id"] == state.eval_prev_run_id assert restored["baseline_skills_version"] == state.baseline_skills_version assert restored["baseline_prompts_version"] == state.baseline_prompts_version - assert restored["steps_since_best_improved"] == state.steps_since_best_improved + assert restored["epochs_since_best_improved"] == state.epochs_since_best_improved assert restored["epoch_start_skills"] == state.epoch_start_skills assert restored["changed_task_types_this_epoch"] == state.changed_task_types_this_epoch assert restored["gate_cooldown"] == state.gate_cooldown diff --git a/tests/unit/test_harness_runner.py b/tests/unit/test_harness_runner.py index c7b5f29..619a689 100644 --- a/tests/unit/test_harness_runner.py +++ b/tests/unit/test_harness_runner.py @@ -349,92 +349,56 @@ class TestBuildComparisonPairs: # ========================================================================= +def _write_manifest_with_best(tmp_path: Path, best_epoch: int) -> None: + """写含 best.epoch 的 manifest，供 _should_early_stop 读 read_best。""" + manifest = { + "name": "test", + "store": ".", + "current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"}, + "best": {"epoch": best_epoch, "val_acc": 0.5}, + "history": [], + } + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + + class TestShouldEarlyStop: - """_should_early_stop 步粒度 early stop。""" + """_should_early_stop epoch 粒度 early stop（patience 以 epoch 计）。""" def test_improved_this_epoch_resets(self, tmp_path: Path) -> None: """本 epoch best 刷新时重置计数器。""" - # 写 manifest + best - manifest = { - "name": "test", - "store": ".", - "current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"}, - "best": {"epoch": 2, "val_acc": 0.9}, - "history": [], - } - (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + _write_manifest_with_best(tmp_path, best_epoch=2) state = MagicMock() - state.steps_since_best_improved = 10 + state.epochs_since_best_improved = 3 - result = _should_early_stop(tmp_path, epoch=2, steps_this_epoch=5, state=state, patience=20) + result = _should_early_stop(tmp_path, epoch=2, state=state, patience=2) assert result is False - assert state.steps_since_best_improved == 0 - - def test_no_improvement_accumulates(self, tmp_path: Path) -> None: - """未刷新时累加步数。""" - manifest = { - "name": "test", - "store": ".", - "current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"}, - "best": {"epoch": 1, "val_acc": 0.5}, - "history": [], - } - (tmp_path / "manifest.json").write_text(json.dumps(manifest)) - - state = MagicMock() - state.steps_since_best_improved = 15 - - result = _should_early_stop(tmp_path, epoch=3, steps_this_epoch=5, state=state, patience=20) - assert result is True # 15 + 5 = 20 >= 20 - assert state.steps_since_best_improved == 20 + assert state.epochs_since_best_improved == 0 def test_below_patience_continues(self, tmp_path: Path) -> None: - """累计步数未达阈值时继续。""" - manifest = { - "name": "test", - "store": ".", - "current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"}, - "best": {"epoch": 1, "val_acc": 0.5}, - "history": [], - } - (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + """未达 patience 个 epoch 无刷新时继续。""" + _write_manifest_with_best(tmp_path, best_epoch=1) state = MagicMock() - state.steps_since_best_improved = 10 + state.epochs_since_best_improved = 0 - result = _should_early_stop(tmp_path, epoch=3, steps_this_epoch=5, state=state, patience=20) + result = _should_early_stop(tmp_path, epoch=2, state=state, patience=3) assert result is False - assert state.steps_since_best_improved == 15 + assert state.epochs_since_best_improved == 1 - def test_step_granularity(self, tmp_path: Path) -> None: - """步粒度而非 epoch 粒度。""" - manifest = { - "name": "test", - "store": ".", - "current": {"videos": "v", "questions": "q", "skills": "s/v1", "prompts": "p/v1"}, - "best": {"epoch": 1, "val_acc": 0.5}, - "history": [], - } - (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + def test_early_stop_counts_epochs_not_steps(self, tmp_path: Path) -> None: + """patience=2 表示连续 2 个 epoch 无 best 刷新才停（不是步数）。""" + _write_manifest_with_best(tmp_path, best_epoch=1) state = MagicMock() - # 连续 3 个 epoch，每个 3 步 - state.steps_since_best_improved = 0 - for ep in range(2, 5): - stopped = _should_early_stop( - tmp_path, epoch=ep, steps_this_epoch=3, state=state, patience=10 - ) - if ep < 4: - assert stopped is False - else: - # 3+3+3=9 < 10 但第三轮后 9+3=12>=10 在 ep=5 触发 - # 实际：ep=2 → 3, ep=3 → 6, ep=4 → 9 - assert stopped is False - stopped = _should_early_stop( - tmp_path, epoch=5, steps_this_epoch=3, state=state, patience=10 - ) - assert stopped is True # 9+3=12>=10 + state.epochs_since_best_improved = 0 + + # epoch 2 无刷新 → 1 → 不停 + assert _should_early_stop(tmp_path, epoch=2, state=state, patience=2) is False + assert state.epochs_since_best_improved == 1 + # epoch 3 无刷新 → 2 → 停 + assert _should_early_stop(tmp_path, epoch=3, state=state, patience=2) is True + assert state.epochs_since_best_improved == 2 # ========================================================================= @@ -667,7 +631,7 @@ class TestTrainState: assert state.system_packs == [] assert state.tool_packs == [] assert state.changed_task_types_this_epoch == set() - assert state.steps_since_best_improved == 0 + assert state.epochs_since_best_improved == 0 # ========================================================================= From caea9150b3e8d5fe653ab2b5ad2bf27d7bf14648 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 06:16:02 -0400 Subject: [PATCH 31/43] feat: pre-flight filter of untrainable task types before gate --- app/harness/checkpoint.py | 1 + app/harness/config.py | 4 + app/harness/runner.py | 87 ++++++++++++++++++++-- config/default.yaml | 1 + config/train_action_recognition.yaml | 1 + tests/unit/test_harness_checkpoint.py | 2 + tests/unit/test_harness_config.py | 1 + tests/unit/test_harness_pools.py | 3 + tests/unit/test_harness_runner.py | 54 ++++++++++++++ tests/unit/test_runner_diag_tree_inject.py | 1 + 10 files changed, 148 insertions(+), 7 deletions(-) diff --git a/app/harness/checkpoint.py b/app/harness/checkpoint.py index 5c57544..c0849d6 100644 --- a/app/harness/checkpoint.py +++ b/app/harness/checkpoint.py @@ -41,6 +41,7 @@ _STRUCTURAL_KEYS = ( "diag_size", "val_size", "batch_correct_ratio", + "trainable_min_units", ) _DECISION_KEYS = ( diff --git a/app/harness/config.py b/app/harness/config.py index 9e05241..824735d 100644 --- a/app/harness/config.py +++ b/app/harness/config.py @@ -60,6 +60,7 @@ class RunConfig: batch_size: mini-batch 单批题目数。 min_class_per_batch: 单批中每个任务类型至少保留的题目数（< batch_size）。 eval_min_per_class: 验证池中每个任务类型至少保底的题目数。 + trainable_min_units: 可训练性预检：每题型 diag+val 单元数下限，低于则剔除该题型。 early_stop_patience: 全局 best 连续未提升的容忍轮数，达到即早停。 test_size: held-out 测试池题目数。 use_slow_momentum: 是否启用快慢双速进化中的慢速 momentum 更新。 @@ -114,6 +115,7 @@ class RunConfig: batch_size: int min_class_per_batch: int eval_min_per_class: int + trainable_min_units: int early_stop_patience: int test_size: int use_slow_momentum: bool @@ -297,6 +299,8 @@ def _validate_minibatch(config: RunConfig) -> None: ) if config.eval_min_per_class < 1: raise ValueError(f"eval_min_per_class 必须 >= 1，实际: {config.eval_min_per_class}") + if config.trainable_min_units < 1: + raise ValueError(f"trainable_min_units 必须 >= 1，实际: {config.trainable_min_units}") if config.pool_split_mode != "per_category": floor = config.eval_min_per_class * _VIDEO_MME_TASK_TYPE_COUNT if config.val_size < floor: diff --git a/app/harness/runner.py b/app/harness/runner.py index 97ddda9..ea6ebc1 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -18,7 +18,8 @@ import random import shutil import sqlite3 import tempfile -from dataclasses import dataclass, field +from collections import Counter +from dataclasses import dataclass, field, replace from pathlib import Path from typing import TYPE_CHECKING, Any @@ -288,6 +289,52 @@ def _snapshot_current_skills(skills_dir: Path) -> dict[str, str]: return snapshot +def _filter_untrainable_types( + pools: Pools, + task_types: list[str] | None, + eval_min_per_class: int, + trainable_min_units: int, +) -> tuple[Pools, list[str] | None]: + """剔除不可训练题型（val tuple[_TrainState, int, dict, list | None]: + async def _setup_train_run( + self, pools: Pools, filtered_task_types: list[str] | None + ) -> tuple[_TrainState, int, dict, list | None]: """据是否 --resume 准备训练起点。 + 参数: + pools: 已过可训练性预检的三池。 + filtered_task_types: 预检后保留的题型（None 表示不限，由 gate 从 diag 推导）。 + 返回: (state, total_steps, plan, saved_batches)。 """ ckpt = load_checkpoint(self._config.workspace_dir) if self._config.resume else None if self._config.resume and ckpt is None: raise RuntimeError("--resume 但 checkpoint.json 不存在，拒绝静默从头重训") - gate_pools, baseline_cache = self._init_gate_pools(pools) + gate_pools, baseline_cache = self._init_gate_pools(pools, filtered_task_types) if not ckpt: state = self._init_train_state(pools, gate_pools, baseline_cache) total_steps = _compute_total_steps(pools, state.correctness, self._config) @@ -894,13 +959,16 @@ class Runner: ) return state, ckpt["progress"]["total_steps"], plan, ckpt["epoch_batches"] - def _init_gate_pools(self, pools: Pools) -> tuple[GatePools, BaselineCache]: + def _init_gate_pools( + self, pools: Pools, filtered_task_types: list[str] | None + ) -> tuple[GatePools, BaselineCache]: """构建/加载 CE-Gate 信息量阶梯与基线缓存。 副作用：设置 self._gate_questions_by_id（不进 checkpoint）。 参数: - pools: 冻结三池。 + pools: 冻结三池（已过可训练性预检）。 + filtered_task_types: 预检保留的题型；None 时从 pools.diagnosis 推导。 返回: (GatePools, BaselineCache)。 @@ -931,7 +999,12 @@ class Runner: ) baseline_correctness = {r["question_id"]: r["prediction"] == r["answer"] for r in rows} logger.info("gate 阶梯基线对错覆盖 {} 题", len(baseline_correctness)) - gate_task_types = sorted({q.task_type for q in pools.diagnosis}) + # 预检保留的题型优先；None 时从（已过滤的）诊断池推导，二者一致 + gate_task_types = ( + sorted(filtered_task_types) + if filtered_task_types is not None + else sorted({q.task_type for q in pools.diagnosis}) + ) gate_pools = build_or_load_gate_pools( workspace_dir=self._config.workspace_dir, questions=questions, diff --git a/config/default.yaml b/config/default.yaml index 63c5e0a..0048741 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -67,6 +67,7 @@ harness: batch_correct_ratio: 0.5 momentum_samples: 20 eval_min_per_class: 2 + trainable_min_units: 8 early_stop_patience: 8 use_slow_momentum: true # 池构建策略 diff --git a/config/train_action_recognition.yaml b/config/train_action_recognition.yaml index 9aa68db..a80a3db 100644 --- a/config/train_action_recognition.yaml +++ b/config/train_action_recognition.yaml @@ -48,6 +48,7 @@ harness: batch_correct_ratio: 0.5 momentum_samples: 20 eval_min_per_class: 2 + trainable_min_units: 8 early_stop_patience: 4 test_size: 63 diag_size: 20 diff --git a/tests/unit/test_harness_checkpoint.py b/tests/unit/test_harness_checkpoint.py index 1d5d40e..e1fc36f 100644 --- a/tests/unit/test_harness_checkpoint.py +++ b/tests/unit/test_harness_checkpoint.py @@ -157,6 +157,7 @@ class _FakeConfig: diag_size: int = 30 val_size: int = 50 batch_correct_ratio: float = 0.5 + trainable_min_units: int = 8 edit_budget_start: int = 6 edit_budget_end: int = 3 early_stop_patience: int = 3 @@ -289,6 +290,7 @@ class TestFingerprintStructuralVsDecision: "diag_size", "val_size", "batch_correct_ratio", + "trainable_min_units", } decision = { "edit_budget_start", diff --git a/tests/unit/test_harness_config.py b/tests/unit/test_harness_config.py index b744b51..7a689bd 100644 --- a/tests/unit/test_harness_config.py +++ b/tests/unit/test_harness_config.py @@ -40,6 +40,7 @@ def _valid_kwargs() -> dict: "batch_size": 15, "min_class_per_batch": 2, "eval_min_per_class": 2, + "trainable_min_units": 8, "early_stop_patience": 8, "test_size": 60, "use_slow_momentum": True, diff --git a/tests/unit/test_harness_pools.py b/tests/unit/test_harness_pools.py index 2cc1b75..48cc3d6 100644 --- a/tests/unit/test_harness_pools.py +++ b/tests/unit/test_harness_pools.py @@ -317,6 +317,7 @@ class TestBuildOrLoadPoolsFrozen: batch_size=15, min_class_per_batch=2, eval_min_per_class=2, + trainable_min_units=8, early_stop_patience=4, test_size=10, use_slow_momentum=True, @@ -870,6 +871,7 @@ class TestRunHoldoutEvalConfig: batch_size=15, min_class_per_batch=2, eval_min_per_class=2, + trainable_min_units=8, early_stop_patience=4, test_size=30, use_slow_momentum=True, @@ -920,6 +922,7 @@ class TestRunHoldoutEvalConfig: batch_size=15, min_class_per_batch=2, eval_min_per_class=2, + trainable_min_units=8, early_stop_patience=4, test_size=30, use_slow_momentum=True, diff --git a/tests/unit/test_harness_runner.py b/tests/unit/test_harness_runner.py index 619a689..f328f04 100644 --- a/tests/unit/test_harness_runner.py +++ b/tests/unit/test_harness_runner.py @@ -20,6 +20,7 @@ from app.harness.runner import ( _build_comparison_pairs, _compute_total_steps, _fallback_summary, + _filter_untrainable_types, _format_applied_edits, _guard_infra_failures, _outcome_to_quadrant_pairs, @@ -401,6 +402,58 @@ class TestShouldEarlyStop: assert state.epochs_since_best_improved == 2 +class TestFilterUntrainableTypes: + """_filter_untrainable_types 可训练性预检纯函数。""" + + def test_untrainable_types_filtered_before_gate(self) -> None: + """val None: + """val 达标但 diag+val 单元数 < trainable_min_units 的题型被剔除。""" + # 题型 C：val=2（>=2）但 units=2+0=... 补 diag 使总数不足 + val = [_FakeQuestion(question_id=f"C-v{i}", task_type="C") for i in range(2)] + diag = [_FakeQuestion(question_id="C-d0", task_type="C")] # units=3 < 8 + pools = _FakePools(diagnosis=diag, validation=val, test=[]) + + new_pools, new_types = _filter_untrainable_types( + pools, task_types=None, eval_min_per_class=2, trainable_min_units=8 + ) + + assert new_pools.diagnosis == [] + assert new_pools.validation == [] + assert new_types == [] + + def test_test_pool_untouched(self) -> None: + """test 池不参与过滤（继续报告全题型准确率）。""" + val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(8)] + diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)] + test = [_FakeQuestion(question_id="B-t0", task_type="B")] + pools = _FakePools(diagnosis=diag, validation=val, test=test) + + new_pools, _ = _filter_untrainable_types( + pools, task_types=None, eval_min_per_class=2, trainable_min_units=8 + ) + + assert new_pools.test == test + + # ========================================================================= # 13c: Probation 数据结构测试 # ========================================================================= @@ -690,6 +743,7 @@ class TestRunnerFactoryInjection: "batch_size": 5, "min_class_per_batch": 2, "eval_min_per_class": 2, + "trainable_min_units": 8, "early_stop_patience": 3, "test_size": 10, "use_slow_momentum": False, diff --git a/tests/unit/test_runner_diag_tree_inject.py b/tests/unit/test_runner_diag_tree_inject.py index 404b649..43dbab3 100644 --- a/tests/unit/test_runner_diag_tree_inject.py +++ b/tests/unit/test_runner_diag_tree_inject.py @@ -54,6 +54,7 @@ def _base_config(workspace_dir: Path, store_dir: Path) -> RunConfig: batch_size=5, min_class_per_batch=2, eval_min_per_class=2, + trainable_min_units=8, early_stop_patience=3, test_size=10, use_slow_momentum=False, From 65126feada398920006aca3eb1664c3ff3f6777d Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 06:20:29 -0400 Subject: [PATCH 32/43] fix: patch checks full target span + marker injection/integrity (algo #8) --- core/evolution/evolve.py | 40 ++++++++++++++++++++++++ core/evolution/patch.py | 50 +++++++++++++++++++++++++----- tests/unit/test_evolve.py | 30 ++++++++++++++++++ tests/unit/test_patch.py | 65 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 7 deletions(-) diff --git a/core/evolution/evolve.py b/core/evolution/evolve.py index db60ecb..c713127 100644 --- a/core/evolution/evolve.py +++ b/core/evolution/evolve.py @@ -19,6 +19,8 @@ from loguru import logger from core.evolution.patch import ( APPENDIX_END, APPENDIX_START, + MOMENTUM_END, + MOMENTUM_START, append_to_appendix, apply_patch_with_report, extract_appendix_notes, @@ -293,10 +295,39 @@ def _tool_protected_spans(text: str) -> list[str]: # ========================================================================= +def _check_marker_integrity(evolved: str) -> list[str]: + """校验 evolved 中冻结区 marker 的完整性（成对、至多一对、START 先于 END）。 + + 进化写入可能破坏 appendix/momentum marker 配对，破坏后 append_to_appendix / + replace_momentum 等下游会静默误拼或抛错。此处集中拦截：任一 marker 对违反 + 「START 数==END 数、各至多一对、START 在 END 前」即整体 reject。 + + 参数: + evolved: 改写后的全文。 + 返回: + 错误信息列表（空列表表示 marker 完整）。 + """ + errors: list[str] = [] + for name, start_m, end_m in ( + ("APPENDIX", APPENDIX_START, APPENDIX_END), + ("MOMENTUM", MOMENTUM_START, MOMENTUM_END), + ): + s = evolved.count(start_m) + e = evolved.count(end_m) + if s != e: + errors.append(f"{name} marker 不配对：START={s} END={e}") + elif s > 1: + errors.append(f"{name} marker 出现多对（{s}），至多一对") + elif s == 1 and evolved.index(start_m) > evolved.index(end_m): + errors.append(f"{name} marker 顺序错误：END 出现在 START 之前") + return errors + + def validate_skill(original: str, evolved: str) -> ValidationResult: """校验 Skill 改写结果。 检查项: frontmatter 三字段保留（name / description / task_type）、 + marker 完整性（appendix/momentum 成对且至多一对、顺序正确）、 长度比在 [0.3, 2.0]、代码块闭合。 参数: @@ -305,6 +336,11 @@ def validate_skill(original: str, evolved: str) -> ValidationResult: 返回: ValidationResult 实例。 + + 关键实现细节: + marker 完整性先于长度校验——长度校验经 _strip_protected_regions 调用 + momentum_region_bounds，对损坏 marker 会抛 ValueError；故 marker 破坏时先 + 返回失败，避免异常穿透且明确 reject 该候选。 """ errors: list[str] = [] orig_fm = _parse_frontmatter(original) @@ -319,6 +355,10 @@ def validate_skill(original: str, evolved: str) -> ValidationResult: errors.append( f"frontmatter 字段 {key} 被修改: {orig_fm.get(key)!r} → {evol_fm.get(key)!r}" ) + marker_errors = _check_marker_integrity(evolved) + if marker_errors: + errors.extend(marker_errors) + return ValidationResult(passed=False, errors=errors) errors.extend(_check_length(original, evolved)) errors.extend(_check_code_blocks(evolved)) return ValidationResult(passed=len(errors) == 0, errors=errors) diff --git a/core/evolution/patch.py b/core/evolution/patch.py index 87effa5..f1dd945 100644 --- a/core/evolution/patch.py +++ b/core/evolution/patch.py @@ -282,9 +282,34 @@ def _protected_ranges(content: str, spans: list[str]) -> list[tuple[int, int]]: return ranges -def _in_ranges(pos: int, ranges: list[tuple[int, int]]) -> bool: - """判断位置 pos 是否落在任意冻结区间内。""" - return any(start <= pos < end for start, end in ranges) +def _span_overlaps_ranges(pos: int, length: int, ranges: list[tuple[int, int]]) -> bool: + """判断 [pos, pos+length) 是否与任一冻结区间相交（不止起点）。 + + 起点落在正文、末端伸入冻结区的 target 也须拦截，否则 replace/delete 会连带 + 改动冻结区（如破坏 appendix/momentum marker）。 + + 参数: + pos: target 在正文中的起点。 + length: target 长度。 + ranges: 冻结区间 [start, end) 列表。 + 返回: + 与任一区间相交返回 True。 + """ + end = pos + length + return any(start < end and pos < r_end for start, r_end in ranges) + + +# 冻结区 marker 字面量：LLM 生成的 edit 不得注入这些字面量，否则破坏 marker 配对 +_MARKER_LITERALS = (APPENDIX_START, APPENDIX_END, MOMENTUM_START, MOMENTUM_END) + + +def _edit_injects_marker(edit: dict) -> bool: + """判断 edit 的 target/content 是否含冻结区 marker 字面量（注入拦截）。""" + for key in ("target", "content"): + value = edit.get(key) + if isinstance(value, str) and any(m in value for m in _MARKER_LITERALS): + return True + return False def _append_at(content: str, ranges: list[tuple[int, int]]) -> int: @@ -317,8 +342,8 @@ def _do_insert_after( _insert_at(content, _append_at(content, ranges), payload), "applied_insert_after_fallback", ) - if _in_ranges(pos, ranges): - logger.warning("insert_after 目标在冻结区，跳过 target={}", target[:80]) + if _span_overlaps_ranges(pos, len(target), ranges): + logger.warning("insert_after 目标跨入冻结区，跳过 target={}", target[:80]) return content, "skipped_protected" at = pos + len(target) nl = content.find("\n", at) @@ -340,8 +365,8 @@ def _do_replace_delete( if pos == -1: logger.warning("{} 锚点缺失，跳过 target={}", op, target[:80]) return content, "skipped_target_not_found" - if _in_ranges(pos, ranges): - logger.warning("{} 目标在冻结区，跳过 target={}", op, target[:80]) + if _span_overlaps_ranges(pos, len(target), ranges): + logger.warning("{} 目标跨入冻结区，跳过 target={}", op, target[:80]) return content, "skipped_protected" new_content = content.replace(target, payload if op == "replace" else "", 1) return new_content, "applied_" + op @@ -403,6 +428,17 @@ def apply_patch_with_report( reports: list[dict] = [] for i, edit in enumerate(edits, 1): try: + if isinstance(edit, dict) and _edit_injects_marker(edit): + logger.warning("edit 含冻结区 marker 字面量，拒绝该 edit index={}", i) + report = { + "op": str(edit.get("op", "")), + "target": str(edit.get("target", "") or "")[:200], + "content_preview": str(edit.get("content", "") or "")[:200], + "status": "skipped_marker_injection", + } + report["index"] = i + reports.append(report) + continue ranges = _protected_ranges(content, spans) content, report = _apply_one(content, edit, ranges) except (KeyError, TypeError, ValueError, AttributeError) as exc: diff --git a/tests/unit/test_evolve.py b/tests/unit/test_evolve.py index 0077f81..30347f9 100644 --- a/tests/unit/test_evolve.py +++ b/tests/unit/test_evolve.py @@ -309,6 +309,36 @@ class TestValidateSkill: result = validate_skill(orig, "no frontmatter body") assert not result.passed + def test_unpaired_appendix_marker_fails(self) -> None: + """evolved 出现孤立 APPENDIX_START（无配对 END）→ marker 完整性校验失败。""" + orig = "---\nname: a\ndescription: d\ntask_type: t\n---\nbody body body" + evol = f"---\nname: a\ndescription: d\ntask_type: t\n---\nbody {APPENDIX_START} body" + result = validate_skill(orig, evol) + assert not result.passed + assert any("marker" in e.lower() or "配对" in e for e in result.errors) + + def test_duplicate_momentum_marker_fails(self) -> None: + """evolved 出现两对 MOMENTUM marker → 完整性校验失败（各至多一对）。""" + orig = "---\nname: a\ndescription: d\ntask_type: t\n---\nbody body body" + evol = ( + "---\nname: a\ndescription: d\ntask_type: t\n---\n" + f"{MOMENTUM_START}x{MOMENTUM_END} mid {MOMENTUM_START}y{MOMENTUM_END}" + ) + result = validate_skill(orig, evol) + assert not result.passed + assert any("marker" in e.lower() or "配对" in e for e in result.errors) + + def test_paired_markers_pass(self) -> None: + """evolved 含成对 appendix+momentum marker（各一对，顺序正确）→ 校验通过。""" + body = "body " * 20 + orig = f"---\nname: a\ndescription: d\ntask_type: t\n---\n{body}" + evol = ( + f"---\nname: a\ndescription: d\ntask_type: t\n---\n{body}" + f"{APPENDIX_START}\n- n\n{APPENDIX_END}\n{MOMENTUM_START}\nm\n{MOMENTUM_END}" + ) + result = validate_skill(orig, evol) + assert result.passed, result.errors + class TestValidateSystem: """validate_system 测试。""" diff --git a/tests/unit/test_patch.py b/tests/unit/test_patch.py index 75d7e84..9123d61 100644 --- a/tests/unit/test_patch.py +++ b/tests/unit/test_patch.py @@ -383,3 +383,68 @@ class TestApplyPatch: assert "line2" not in out assert "TAIL" in out assert all(r["status"].startswith("applied") for r in reports) + + +class TestSpanOverlapProtection: + """冻结区跨度保护 + marker 注入拦截（算法 #8 加固）。""" + + def test_replace_spanning_into_protected_marker_preserved(self) -> None: + """target 末端跨入含 marker 的 appendix 冻结区时被拦截，marker 不被破坏。""" + body = "正文最后一段。" + appendix = f"{APPENDIX_START}\n## 执行提醒\n- 规则A\n{APPENDIX_END}" + content = body + "\n\n" + appendix + # target 从正文末尾跨入 APPENDIX_START（含 marker 字面量） + target = "正文最后一段。\n\n" + APPENDIX_START + edits = [{"op": "delete", "target": target, "content": ""}] + new_content, report = apply_patch_with_report( + content, edits, protected_spans=[appendix] + ) + # 无论经 marker 注入拦截还是跨度拦截，marker 都必须完整保留 + assert APPENDIX_START in new_content and APPENDIX_END in new_content + assert report[0]["status"] in ("skipped_protected", "skipped_marker_injection") + + def test_replace_spanning_into_frozen_section_is_skipped(self) -> None: + """target 起点在正文、末端伸入无 marker 的冻结区段 → 跨度拦截跳过。""" + body = "可改正文段落。" + frozen = "## 输出格式\n必须输出 JSON。" + content = body + "\n" + frozen + # target 从可改正文跨入冻结区段（不含任何 marker 字面量） + target = "可改正文段落。\n## 输出格式" + edits = [{"op": "replace", "target": target, "content": "破坏内容"}] + new_content, report = apply_patch_with_report( + content, edits, protected_spans=[frozen] + ) + assert "破坏内容" not in new_content + assert "## 输出格式" in new_content + assert report[0]["status"] == "skipped_protected" + + def test_insert_after_spanning_into_frozen_section_is_skipped(self) -> None: + """insert_after 的 target 末端伸入无 marker 冻结区段时跨度拦截跳过。""" + body = "可改正文。" + frozen = "## 输出格式\n固定内容。" + content = body + "\n" + frozen + target = "可改正文。\n## 输出格式" + edits = [{"op": "insert_after", "target": target, "content": "注入内容"}] + new_content, report = apply_patch_with_report( + content, edits, protected_spans=[frozen] + ) + assert "注入内容" not in new_content + assert report[0]["status"] == "skipped_protected" + + def test_edit_payload_with_marker_literal_rejected(self) -> None: + """edit payload 含 marker 字面量 → 拒绝该 edit。""" + edits = [ + {"op": "append", "target": "", "content": f"注入 {APPENDIX_START} 破坏"} + ] + _, report = apply_patch_with_report("正文", edits, protected_spans=[]) + assert any( + "marker" in str(s).lower() or "reject" in str(s).lower() for s in report[0].values() + ) + + def test_edit_target_with_marker_literal_rejected(self) -> None: + """edit target 含 momentum marker 字面量 → 拒绝该 edit。""" + edits = [ + {"op": "replace", "target": f"{MOMENTUM_START}x", "content": "y"} + ] + _, report = apply_patch_with_report("正文", edits, protected_spans=[]) + assert report[0]["status"] == "skipped_marker_injection" From efbdeb1647571f1ef77acac71b8086af42b4290b Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 06:24:11 -0400 Subject: [PATCH 33/43] fix: route None/degraded diagnoses to lapse; abort on high degrade rate --- app/harness/runner.py | 7 +++ core/evolution/diagnose.py | 11 +++-- tests/unit/test_diagnose.py | 51 ++++++++++++++++++++++ tests/unit/test_runner_diag_tree_inject.py | 26 +++++++++++ 4 files changed, 91 insertions(+), 4 deletions(-) diff --git a/app/harness/runner.py b/app/harness/runner.py index ea6ebc1..8f05956 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -1089,6 +1089,13 @@ class Runner: _apply_batch_correctness(state.correctness, log, run_id, batch) diagnosis = await self._run_diagnosis(run_id, question_ids=[q.question_id for q in batch]) + # 降级占比过高疑似 judge 基础设施故障：不以降级信号驱动进化，直接中止 + n_wrong = sum(1 for q in batch if not state.correctness.get(q.question_id, True)) + if n_wrong > 0 and diagnosis.degraded_count / n_wrong > 0.5: + raise RuntimeError( + f"本 step 诊断降级占比 {diagnosis.degraded_count}/{n_wrong} > 50%，" + "疑似 judge 基础设施故障，中止训练（不以降级信号驱动进化）。" + ) _accumulate_slow_packs(diagnosis, state) await self._gate_batch_skills(epoch, step, diagnosis, total_steps, pools, state) # 冷却计数每 step 递减、归零剔除 diff --git a/core/evolution/diagnose.py b/core/evolution/diagnose.py index 71315ff..d100aae 100644 --- a/core/evolution/diagnose.py +++ b/core/evolution/diagnose.py @@ -1489,12 +1489,15 @@ def _build_skill_case_packs( if qm.correct: continue attr = attribution_map.get(qm.question_id) - if attr is not None and attr.cause_category == "lapse": - if attr.lapse_note and attr.lapse_note.strip(): + # 仅明确 defect 且非 degraded 才进正文进化路径； + # lapse / cause_category=None（判别失败）/ degraded（judge 解析失败）一律保守走 lapse， + # 不以降级或未判定信号驱动错误进化。 + is_defect = attr is not None and attr.cause_category == "defect" and not qm.degraded + if not is_defect: + if attr is not None and attr.lapse_note and attr.lapse_note.strip(): lapse_notes.append(attr.lapse_note) continue - et = attr.error_type if attr else "mixed" - wrong_by_error[et].append(qm) + wrong_by_error[attr.error_type].append(qm) # 单条 fallback n_body_failures = sum(len(group) for group in wrong_by_error.values()) diff --git a/tests/unit/test_diagnose.py b/tests/unit/test_diagnose.py index db05bf8..ea79a6c 100644 --- a/tests/unit/test_diagnose.py +++ b/tests/unit/test_diagnose.py @@ -17,6 +17,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest from core.evolution.diagnose import ( + _build_skill_case_packs, _percentile, _trigrams, aggregate_d2, @@ -43,6 +44,7 @@ from core.evolution.types import ( CaseSample, DiagnosePrompts, DiagnosisResult, + ErrorAttribution, QuestionMetrics, SkillStepAdherence, SpanMetrics, @@ -771,3 +773,52 @@ class TestRunDiagnosis: assert result.run_id == "run1" assert result.error_attributions == [] assert result.degraded_count == 0 + + +class TestBuildSkillCasePacksDegradeRouting: + """_build_skill_case_packs 降级/未判定分流：仅 defect 且非 degraded 进正文路径。""" + + def test_none_cause_and_degraded_route_to_lapse(self) -> None: + """cause_category=None（判别失败）与 degraded 题按 lapse 处置，不进 defect 正文路径。""" + tt = "Action Reasoning" + qm_none = _make_qm(question_id="q-none", task_type=tt, correct=False, degraded=False) + qm_degraded = _make_qm(question_id="q-deg", task_type=tt, correct=False, degraded=True) + # 补两道正确题，避免退化 + qm_ok1 = _make_qm(question_id="q-ok1", task_type=tt, correct=True) + qm_ok2 = _make_qm(question_id="q-ok2", task_type=tt, correct=True) + metrics = [qm_none, qm_degraded, qm_ok1, qm_ok2] + + attributions = [ + # 判别失败：cause_category=None + ErrorAttribution( + question_id="q-none", + error_type="reasoning", + reasoning_failure_type=None, + cause_category=None, + lapse_note="复核该类推理规则", + ), + # degraded 题即便被判 defect，也须走 lapse（不驱动正文进化） + ErrorAttribution( + question_id="q-deg", + error_type="reasoning", + reasoning_failure_type=None, + cause_category="defect", + lapse_note=None, + ), + ] + + packs = _build_skill_case_packs( + all_metrics=metrics, + error_attributions=attributions, + traces_by_question={}, + predictions=[], + d3_stats={}, + d4_stats={}, + ) + + pack = packs[tt] + failure_ids = {c.question_id for c in pack.failure_cases} + assert "q-none" not in failure_ids + assert "q-deg" not in failure_ids + # None-cause 的 lapse_note 应被收进 lapse_notes + assert any("复核该类推理规则" in n for n in pack.lapse_notes) diff --git a/tests/unit/test_runner_diag_tree_inject.py b/tests/unit/test_runner_diag_tree_inject.py index 43dbab3..5c9a05b 100644 --- a/tests/unit/test_runner_diag_tree_inject.py +++ b/tests/unit/test_runner_diag_tree_inject.py @@ -212,6 +212,32 @@ async def test_diagnosis_reads_traces_from_steps_json( assert traces[0]["tool_name"] == "search_tree" +@pytest.mark.asyncio +async def test_run_step_aborts_on_high_degrade_rate( + runner_with_real_store: Runner, monkeypatch: pytest.MonkeyPatch +) -> None: + """诊断降级占比 > 50% 时 _run_step 中止（疑似 judge 基础设施故障）。""" + from unittest.mock import AsyncMock + + batch = [_fake_question("q1", "vA"), _fake_question("q2", "vA")] + + runner_with_real_store._rollout_batch = AsyncMock() + monkeypatch.setattr("app.harness.runner._apply_batch_correctness", lambda *a, **k: None) + runner_with_real_store._run_diagnosis = AsyncMock( + return_value=DiagnosisResult(run_id="r", degraded_count=2) + ) + runner_with_real_store._gate_batch_skills = AsyncMock() + + state = MagicMock() + state.correctness = {"q1": False, "q2": False} + state.gate_cooldown = {} + pools = MagicMock() + pools.baseline_run_id = "infer_adhoc" + + with pytest.raises(RuntimeError, match="降级占比"): + await runner_with_real_store._run_step(1, 0, 10, batch, pools, state) + + @pytest.mark.asyncio async def test_run_diagnosis_full_scan_loads_all_video_trees( runner_with_real_store: Runner, From 77fd35830c62a53cd55d0f5891ace61c36ee0db0 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 06:27:38 -0400 Subject: [PATCH 34/43] fix: idempotent _run_step (DELETE stale) + checkpoint after gate save --- app/harness/runner.py | 69 +++++++++++++++++----- tests/integration/test_checkpoint_pair.py | 1 + tests/unit/test_runner_diag_tree_inject.py | 41 +++++++++++++ 3 files changed, 95 insertions(+), 16 deletions(-) diff --git a/app/harness/runner.py b/app/harness/runner.py index 8f05956..6d04cf2 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -887,21 +887,15 @@ class Runner: epoch_batches=batch_unit_ids, config=self._config, ) - await self._slow_update_cycle(epoch, pools, state) - state.system_packs = [] - state.tool_packs = [] - state.changed_task_types_this_epoch = set() - write_checkpoint( - self._config.workspace_dir, - state=state, - epoch=epoch, - step_completed=len(batches) - 1, - phase="epoch_done", - global_step=state.global_step, + # checkpoint 落盘移入 _slow_update_cycle 末尾（gate_pools.save 之后立即写）， + # 消除 gate_epoch_observed 在 gate_pools.json 与 checkpoint 间的双计窗口。 + await self._slow_update_cycle( + epoch, + pools, + state, total_steps=total_steps, - version_snapshot=self._current_version_snapshot(), + step_completed=len(batches) - 1, epoch_batches=batch_unit_ids, - config=self._config, ) if _should_early_stop( self._config.workspace_dir, @@ -1081,10 +1075,21 @@ class Runner: ) -> None: """单 step：rollout → correctness 增量 → 诊断 → 累加 system/tool → 按类 gate。""" run_id = f"{pools.baseline_run_id}_e{epoch}_s{step}" - await self._rollout_batch(batch, run_id) + from app.harness.inference import PREDICTIONS_SCHEMA, TRACES_SCHEMA from app.harness.log import HarnessLog + # 幂等：重跑同一 step 前先清旧行，避免断点续跑重复累计双计。 + # 先 CREATE TABLE IF NOT EXISTS（fresh workspace 首跑时表尚未由 run_inference 建）， + # register_run=False 避免只读清理污染 _runs 运行状态。 + with HarnessLog(str(self._paths.db_path), run_id, register_run=False) as log: + log.create_table("predictions", PREDICTIONS_SCHEMA) + log.create_table("traces", TRACES_SCHEMA) + log.execute("DELETE FROM predictions WHERE run_id=?", (run_id,)) + log.execute("DELETE FROM traces WHERE run_id=?", (run_id,)) + + await self._rollout_batch(batch, run_id) + with HarnessLog(str(self._paths.db_path), run_id) as log: _apply_batch_correctness(state.correctness, log, run_id, batch) @@ -1454,7 +1459,16 @@ class Runner: # _slow_update_cycle 十步序 # ----------------------------------------------------------------------- - async def _slow_update_cycle(self, epoch: int, pools: Pools, state: _TrainState) -> None: + async def _slow_update_cycle( + self, + epoch: int, + pools: Pools, + state: _TrainState, + *, + total_steps: int, + step_completed: int, + epoch_batches: list[list[str]], + ) -> None: """epoch 末慢更新十步序。 1. 捕获版本快照 → 全 val 重跑 R @@ -1466,7 +1480,12 @@ class Runner: 7. system/tool 慢更新（edit_budget_end） 8. R2 闭环 9. 三态标签 + epoch_report + 四向 held-out - 10. gate 阶梯刷新 + 10. gate 阶梯刷新 → 重置 epoch 累加器 → 立即落 epoch_done checkpoint + + 参数: + total_steps: 全局总 step 数（checkpoint 用）。 + step_completed: 本 epoch 已完成 step 数（checkpoint 用）。 + epoch_batches: 本 epoch batch 的 unit_id 划分（checkpoint 用）。 """ # Phase 1 eval_skills_version = self._current_version("skills") @@ -1578,6 +1597,24 @@ class Runner: epoch, pools.baseline_run_id, state, extra_run_ids=r2_kept_run_ids ) + # 重置 epoch 累加器 + 立即落 epoch_done checkpoint：与 _refresh_gate_ladder 内的 + # gate_pools.save + gate_epoch_observed=True 同刻一致，消除断点续跑的双计窗口。 + state.system_packs = [] + state.tool_packs = [] + state.changed_task_types_this_epoch = set() + write_checkpoint( + self._config.workspace_dir, + state=state, + epoch=epoch, + step_completed=step_completed, + phase="epoch_done", + global_step=state.global_step, + total_steps=total_steps, + version_snapshot=self._current_version_snapshot(), + epoch_batches=epoch_batches, + config=self._config, + ) + # ----------------------------------------------------------------------- # 慢更新内部方法 # ----------------------------------------------------------------------- diff --git a/tests/integration/test_checkpoint_pair.py b/tests/integration/test_checkpoint_pair.py index c6b28dc..c62fd9b 100644 --- a/tests/integration/test_checkpoint_pair.py +++ b/tests/integration/test_checkpoint_pair.py @@ -111,6 +111,7 @@ class _FakeConfig: diag_size: int = 30 val_size: int = 50 batch_correct_ratio: float = 0.0 + trainable_min_units: int = 8 edit_budget_start: int = 6 edit_budget_end: int = 3 early_stop_patience: int = 3 diff --git a/tests/unit/test_runner_diag_tree_inject.py b/tests/unit/test_runner_diag_tree_inject.py index 5c9a05b..b608644 100644 --- a/tests/unit/test_runner_diag_tree_inject.py +++ b/tests/unit/test_runner_diag_tree_inject.py @@ -212,6 +212,47 @@ async def test_diagnosis_reads_traces_from_steps_json( assert traces[0]["tool_name"] == "search_tree" +@pytest.mark.asyncio +async def test_run_step_deletes_stale_rows_before_rerun( + runner_with_real_store: Runner, monkeypatch: pytest.MonkeyPatch +) -> None: + """同 run_id 重跑前先清 predictions/traces，避免重复行双计（幂等）。""" + from unittest.mock import AsyncMock + + from app.harness.inference import PREDICTIONS_SCHEMA, TRACES_SCHEMA + from app.harness.log import HarnessLog + + run_id = "infer_adhoc_e1_s0" + # 预置该 step run_id 的旧 predictions/traces 行 + with HarnessLog(str(runner_with_real_store._paths.db_path), run_id) as log: + log.create_table("predictions", PREDICTIONS_SCHEMA) + log.create_table("traces", TRACES_SCHEMA) + log.insert("predictions", {"video_id": "vA", "question_id": "q1", "prediction": "A"}) + log.insert("traces", {"video_id": "vA", "question_id": "q1", "step": 0}) + + batch = [_fake_question("q1", "vA")] + runner_with_real_store._rollout_batch = AsyncMock() + monkeypatch.setattr("app.harness.runner._apply_batch_correctness", lambda *a, **k: None) + runner_with_real_store._run_diagnosis = AsyncMock(return_value=DiagnosisResult(run_id=run_id)) + runner_with_real_store._gate_batch_skills = AsyncMock() + + state = MagicMock() + state.correctness = {"q1": True} + state.gate_cooldown = {} + pools = MagicMock() + pools.baseline_run_id = "infer_adhoc" + + await runner_with_real_store._run_step(1, 0, 10, batch, pools, state) + + with HarnessLog( + str(runner_with_real_store._paths.db_path), run_id, register_run=False + ) as log: + preds = log.query("SELECT * FROM predictions WHERE run_id=?", (run_id,)) + traces = log.query("SELECT * FROM traces WHERE run_id=?", (run_id,)) + assert preds == [] + assert traces == [] + + @pytest.mark.asyncio async def test_run_step_aborts_on_high_degrade_rate( runner_with_real_store: Runner, monkeypatch: pytest.MonkeyPatch From 58a02035223169bbea7dbcc72c51a35b03fa7094 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 06:31:00 -0400 Subject: [PATCH 35/43] perf: dedup holdout four-way eval (baseline derive, best_hard memo) --- app/harness/runner.py | 48 ++++++++++++++++-- tests/unit/test_runner_diag_tree_inject.py | 57 ++++++++++++++++++++++ 2 files changed, 102 insertions(+), 3 deletions(-) diff --git a/app/harness/runner.py b/app/harness/runner.py index 6d04cf2..3c7f682 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -120,6 +120,9 @@ class _TrainState: gate_epoch_observed: bool = False probations: dict[str, Probation] = field(default_factory=dict) gate_cooldown: dict[str, int] = field(default_factory=dict) + # 进程内 holdout 去重备忘录 (skills_v, prompts_v) -> test 评估结果；不进 checkpoint， + # resume 后清空（重评一次是可接受代价，换取零 schema 变更）。 + holdout_memo: dict[tuple[str, str], InferenceResult] = field(default_factory=dict) # --------------------------------------------------------------------------- @@ -2006,11 +2009,18 @@ class Runner: ) -> None: """四向 held-out：baseline/best_hard/final/best_mixed 各在 test 池评估。 + 去重（进程内备忘录 state.holdout_memo，不改 schema）： + - baseline：不跑推理，从基线 predictions 推导 test 结果（0 推理），存 memo 跨 epoch 复用。 + - final：真评 test，存 memo[(final_sv,final_pv)]。 + - best_hard：其版本已在 memo（== final 或往轮已评）则引用，否则真评并存 memo。 + - best_mixed：赢家必是 best_hard 或 final 之一，其结果已在 memo，直接引用（0 推理）。 + test 池仅观测落库，绝不进 gate/best/early-stop/调参。 """ best_mixed = await self._pick_mixed_best( epoch, pools, state, eval_skills_version, eval_prompts_version ) + memo = state.holdout_memo versions: dict[str, tuple[str, str] | None] = { "baseline": (state.baseline_skills_version, state.baseline_prompts_version), "best_hard": (state.best_skills_version, state.best_prompts_version), @@ -2022,9 +2032,14 @@ class Runner: continue sv, pv = version run_id = f"{self._config.run_id}_holdout_{version_kind}_e{epoch}" - res = await self._eval_version_on_pool( - sv, pv, pools.test, run_id, context=f"held-out {version_kind}" - ) + if version not in memo: + if version_kind == "baseline": + memo[version] = self._derive_baseline_test_result(pools, run_id) + else: + memo[version] = await self._eval_version_on_pool( + sv, pv, pools.test, run_id, context=f"held-out {version_kind}" + ) + res = memo[version] soft = await self._try_soft_score(run_id, pools.test) mixed = None if soft is None else 0.5 * res.accuracy + 0.5 * soft write_holdout_eval( @@ -2038,6 +2053,33 @@ class Runner: per_task_type_json=json.dumps(res.per_task_type, ensure_ascii=False), ) + def _derive_baseline_test_result(self, pools: Pools, run_id: str) -> InferenceResult: + """从基线 run 的 predictions 推导 test 池评估结果（0 推理）。 + + 基线 run（pools.baseline_run_id）已对全题库推理并落库，test 题在其中；此处 + 按 test 题回读基线预测、经 _aggregate_results 折叠为 unit 级 InferenceResult， + 避免重复推理基线版本（同版本不重采样）。 + + 参数: + pools: 冻结三池（提供 test 与 baseline_run_id）。 + run_id: 本次 holdout baseline 向的 run_id（仅用作结果标识）。 + 返回: + unit 级 InferenceResult（accuracy / per_task_type 与真评同口径）。 + """ + from app.harness.inference import _aggregate_results + from app.harness.log import HarnessLog + + qids = [q.question_id for q in pools.test] + with HarnessLog( + str(self._paths.db_path), pools.baseline_run_id, register_run=False + ) as log: + placeholders = ", ".join(["?"] * len(qids)) + rows = log.query( + f"SELECT * FROM predictions WHERE run_id=? AND question_id IN ({placeholders})", + (pools.baseline_run_id, *qids), + ) + return _aggregate_results(rows, pools.test, run_id) + async def _pick_mixed_best( self, epoch: int, diff --git a/tests/unit/test_runner_diag_tree_inject.py b/tests/unit/test_runner_diag_tree_inject.py index b608644..bc9b259 100644 --- a/tests/unit/test_runner_diag_tree_inject.py +++ b/tests/unit/test_runner_diag_tree_inject.py @@ -212,6 +212,63 @@ async def test_diagnosis_reads_traces_from_steps_json( assert traces[0]["tool_name"] == "search_tree" +def _fake_inference_result(accuracy: float) -> object: + """构造 InferenceResult 供 holdout 去重测试（per_task_type 空、token 归零）。""" + from app.harness.inference import InferenceResult + + return InferenceResult( + run_id="x", + accuracy=accuracy, + total=10, + correct=int(accuracy * 10), + per_task_type={}, + steps_mean=1.0, + token_usage={"prompt_tokens": 0, "completion_tokens": 0}, + stop_reason_counts={}, + ) + + +@pytest.mark.asyncio +async def test_holdout_dedup_skips_reevaluated_versions( + runner_with_real_store: Runner, +) -> None: + """baseline 不跑推理（从基线预测推导）；best_hard==final 时不重复评估。""" + from unittest.mock import AsyncMock + + runner = runner_with_real_store + eval_calls: list[str] = [] + + async def _fake_eval(sv, pv, questions, run_id, context): # noqa: ANN001 + eval_calls.append(run_id) + return _fake_inference_result(0.5) + + runner._eval_version_on_pool = AsyncMock(side_effect=_fake_eval) + # best_mixed 赢家取 final 版本（必落在 memo，0 推理） + runner._pick_mixed_best = AsyncMock(return_value=("skills_final", "prompts_final")) + # baseline 推导：patch 为 0 推理的假结果（不经 _eval_version_on_pool） + runner._derive_baseline_test_result = MagicMock(return_value=_fake_inference_result(0.4)) + + state = MagicMock() + state.holdout_memo = {} + state.baseline_skills_version = "skills_base" + state.baseline_prompts_version = "prompts_base" + # best_hard 版本 == final 版本 → 去重 + state.best_skills_version = "skills_final" + state.best_prompts_version = "prompts_final" + + pools = MagicMock() + pools.baseline_run_id = "infer_adhoc" + pools.test = [_fake_question("q1", "vA")] + + await runner._holdout_four_way( + 1, pools, state, eval_skills_version="skills_final", eval_prompts_version="prompts_final" + ) + + # baseline=0（推导）、best_hard=1（真评并存 memo）、final/best_mixed 引用 memo（0） + assert len(eval_calls) == 1 + runner._derive_baseline_test_result.assert_called_once() + + @pytest.mark.asyncio async def test_run_step_deletes_stale_rows_before_rerun( runner_with_real_store: Runner, monkeypatch: pytest.MonkeyPatch From 9be3bdb8eb852150758a684e9d3e8cf3a5676027 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 06:33:09 -0400 Subject: [PATCH 36/43] feat: inject run_id as cache_salt for per-epoch resampling (algo #10) --- app/harness/inference.py | 5 +++++ core/agent/loop.py | 13 ++++++++++--- tests/unit/test_agent_loop.py | 11 +++++++++++ tests/unit/test_harness_inference.py | 1 + 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/app/harness/inference.py b/app/harness/inference.py index 911dde5..42f58c9 100644 --- a/app/harness/inference.py +++ b/app/harness/inference.py @@ -388,6 +388,7 @@ async def _run_single_question( log: HarnessLog, max_steps: int, plugins: list[object], + run_id: str, ) -> dict[str, Any]: """执行单道题目的 Agent 推理。 @@ -402,6 +403,8 @@ async def _run_single_question( log: HarnessLog 实例（线程安全）。 max_steps: AgentLoop 最大步数。 plugins: pluggy 插件列表。 + run_id: 运行标识，用作 cache_salt——run_id 含 _e{epoch} 天然跨 epoch 重采样、 + 同 epoch 续跑命中缓存（算法 #10 透传）。 返回: 预测结果字典（含 video_id, question_id, prediction, answer 等）。 @@ -431,6 +434,7 @@ async def _run_single_question( dispatcher, plugins=plugins, session_id=qa.question_id, + cache_salt=run_id, ) result_dict = loop_result.result if isinstance(loop_result.result, dict) else {} @@ -558,6 +562,7 @@ async def run_inference( log=log, max_steps=max_steps, plugins=plugins, + run_id=run_id, ) logger.info( "[{}/{}] {} QA {} 完成 (stop={})", diff --git a/core/agent/loop.py b/core/agent/loop.py index 405292d..059285d 100644 --- a/core/agent/loop.py +++ b/core/agent/loop.py @@ -108,6 +108,7 @@ class AgentLoop: plugins: list[object] | None = None, *, session_id: str | None = None, + cache_salt: str | None = None, ) -> LoopResult: """执行 Thinking+JSON 推理循环。 @@ -117,6 +118,7 @@ class AgentLoop: tool_dispatcher: 工具调度器，ToolDispatcher Protocol 实例。 plugins: pluggy 插件列表。 session_id: 会话 ID，透传给 LLMProvider。 + cache_salt: 缓存盐，透传给 LLMProvider（如训练用 run_id 跨 epoch 重采样）。 返回: LoopResult 实例，包含推理步骤、token 用量、终止原因。 @@ -138,7 +140,7 @@ class AgentLoop: # Phase 1: LLM 调用（步级重试：防穿透 GovernedLLMClient 的瞬时异常） try: response = await self._call_llm_with_step_retry( - messages, token_usage, session_id=session_id + messages, token_usage, session_id=session_id, cache_salt=cache_salt ) except Exception as e: logger.error("LLM API 调用失败（{}）: {}", type(e).__name__, e) @@ -269,6 +271,7 @@ class AgentLoop: token_usage: dict[str, int], *, session_id: str | None = None, + cache_salt: str | None = None, ) -> LLMResponse: """带步级重试的 LLM 调用，兜底穿透治理层重试栈的瞬时异常。 @@ -292,7 +295,9 @@ class AgentLoop: step_attempt = 0 while True: try: - return await self._call_llm(messages, token_usage, session_id=session_id) + return await self._call_llm( + messages, token_usage, session_id=session_id, cache_salt=cache_salt + ) except self._retryable_exceptions as e: step_attempt += 1 if step_attempt > self._step_retries: @@ -315,6 +320,7 @@ class AgentLoop: token_usage: dict[str, int], *, session_id: str | None = None, + cache_salt: str | None = None, ) -> LLMResponse: """调用 LLMProvider 并累加 token 使用量。 @@ -322,11 +328,12 @@ class AgentLoop: messages: 消息历史。 token_usage: 可变字典，就地累加。 session_id: 会话 ID，透传给 LLMProvider。 + cache_salt: 缓存盐，透传给 LLMProvider（跨 epoch 重采样）。 返回: LLMResponse 实例。 """ - response = await self._llm.chat(messages, session_id=session_id) + response = await self._llm.chat(messages, session_id=session_id, cache_salt=cache_salt) token_usage["prompt_tokens"] += response.prompt_tokens token_usage["completion_tokens"] += response.completion_tokens return response diff --git a/tests/unit/test_agent_loop.py b/tests/unit/test_agent_loop.py index 3ca244b..2ce7383 100644 --- a/tests/unit/test_agent_loop.py +++ b/tests/unit/test_agent_loop.py @@ -94,6 +94,17 @@ def _invalid_tool_json() -> str: class TestAgentLoop: """AgentLoop 推理循环引擎测试。""" + @pytest.mark.asyncio + async def test_forwards_cache_salt(self) -> None: + """AgentLoop.run(cache_salt=...) 透传到 llm.chat（算法 #10 跨 epoch 重采样）。""" + llm = AsyncMock() + llm.chat.return_value = _make_response(_submit_json()) + + loop = AgentLoop(llm=llm, max_steps=10) + await loop.run("system", "user", _StubDispatcher(), cache_salt="run:e2") + + assert llm.chat.call_args.kwargs["cache_salt"] == "run:e2" + @pytest.mark.asyncio async def test_submit_answer_terminates_loop(self) -> None: """submit_answer 终止循环 → finished, result=args, steps_used=1。""" diff --git a/tests/unit/test_harness_inference.py b/tests/unit/test_harness_inference.py index fd57b04..5481cb9 100644 --- a/tests/unit/test_harness_inference.py +++ b/tests/unit/test_harness_inference.py @@ -698,6 +698,7 @@ class TestConcurrencyControl: *, session_id: str | None = None, parent_call_id: str | None = None, + cache_salt: str | None = None, ) -> LLMResponse: nonlocal current_concurrent, max_concurrent current_concurrent += 1 From ed59c6091ac666ee8dcdc4ee368ed4f6134e86ad Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 06:34:16 -0400 Subject: [PATCH 37/43] test: accept cache_salt kwarg in AgentLoop mock LLM stub --- tests/integration/test_adversarial_filter_e2e.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_adversarial_filter_e2e.py b/tests/integration/test_adversarial_filter_e2e.py index 751f301..390379e 100644 --- a/tests/integration/test_adversarial_filter_e2e.py +++ b/tests/integration/test_adversarial_filter_e2e.py @@ -171,6 +171,7 @@ class _MockLLM: *, session_id: str | None = None, parent_call_id: str | None = None, + cache_salt: str | None = None, ) -> LLMResponse: content = json.dumps( {"action": {"tool": "submit_answer", "args": {"answer": self._answer}}} From ef244c52bdffa8ee251b3ec04eedb20e4ec0f1fa Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 06:35:10 -0400 Subject: [PATCH 38/43] chore: add trainable_min_units to train_ar30 config --- config/train_ar30.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/train_ar30.yaml b/config/train_ar30.yaml index a1eaa44..a5c406e 100644 --- a/config/train_ar30.yaml +++ b/config/train_ar30.yaml @@ -49,6 +49,7 @@ harness: batch_correct_ratio: 0.5 momentum_samples: 20 eval_min_per_class: 2 + trainable_min_units: 8 early_stop_patience: 4 test_size: 63 diag_size: 20 From e1f08dcd3b6cda52625c39d554232490870c8b54 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 06:45:16 -0400 Subject: [PATCH 39/43] fix: count units not questions in trainability pre-flight; fail-fast when all types filtered --- app/harness/runner.py | 27 +++++++---- tests/unit/test_harness_runner.py | 74 ++++++++++++++++++++++++++++--- 2 files changed, 86 insertions(+), 15 deletions(-) diff --git a/app/harness/runner.py b/app/harness/runner.py index 3c7f682..ae35031 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -298,37 +298,48 @@ def _filter_untrainable_types( eval_min_per_class: int, trainable_min_units: int, ) -> tuple[Pools, list[str] | None]: - """剔除不可训练题型（val={eval_min_per_class} 且 units>={trainable_min_units}：" + f"{detail}。请调整池切分或降低阈值。" + ) new_pools = replace( pools, diagnosis=[q for q in pools.diagnosis if q.task_type in keep], diff --git a/tests/unit/test_harness_runner.py b/tests/unit/test_harness_runner.py index f328f04..98dad2f 100644 --- a/tests/unit/test_harness_runner.py +++ b/tests/unit/test_harness_runner.py @@ -75,6 +75,26 @@ class _FakeQuestion: object.__setattr__(self, "unit_id", self.pair_id or self.question_id) +def _fake_pair(pair_id: str, task_type: str, video_id: str = "v1") -> list[_FakeQuestion]: + """构造合法孪生对（original + mirror），共享 pair_id/video_id/task_type/flip_axis。""" + return [ + _FakeQuestion( + question_id=f"{pair_id}-o", + video_id=video_id, + task_type=task_type, + pair_id=pair_id, + question_role="pair_original", + ), + _FakeQuestion( + question_id=f"{pair_id}-m", + video_id=video_id, + task_type=task_type, + pair_id=pair_id, + question_role="pair_mirror", + ), + ] + + @dataclass class _FakePools: """Pools 替身。""" @@ -426,19 +446,59 @@ class TestFilterUntrainableTypes: assert new_types == ["A"] def test_units_below_threshold_filtered(self) -> None: - """val 达标但 diag+val 单元数 < trainable_min_units 的题型被剔除。""" - # 题型 C：val=2（>=2）但 units=2+0=... 补 diag 使总数不足 - val = [_FakeQuestion(question_id=f"C-v{i}", task_type="C") for i in range(2)] - diag = [_FakeQuestion(question_id="C-d0", task_type="C")] # units=3 < 8 + """val 达标但 diag+val 单元数 < trainable_min_units 的题型被剔除（保留另一可训题型）。""" + # 可训题型 A：diag=8 + val=2 → units=10、val=2，保留 + keep_diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)] + keep_val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(2)] + # 题型 C：val=2（>=2）但 units=2+1=3 < 8 → 剔除 + val = keep_val + [_FakeQuestion(question_id=f"C-v{i}", task_type="C") for i in range(2)] + diag = keep_diag + [_FakeQuestion(question_id="C-d0", task_type="C")] pools = _FakePools(diagnosis=diag, validation=val, test=[]) new_pools, new_types = _filter_untrainable_types( pools, task_types=None, eval_min_per_class=2, trainable_min_units=8 ) - assert new_pools.diagnosis == [] - assert new_pools.validation == [] - assert new_types == [] + assert {q.task_type for q in new_pools.diagnosis} == {"A"} + assert {q.task_type for q in new_pools.validation} == {"A"} + assert new_types == ["A"] + + def test_ar_pair_counted_as_units_not_questions(self) -> None: + """AR pair 按单元折叠计数：题目数达标但单元数不足的题型仍被剔除。""" + # 可训题型 A：diag=8 + val=2 single → units=10，保留 + keep_diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)] + keep_val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(2)] + # 题型 P：diag 3 对（6 题=3 单元）+ val 2 对（4 题=2 单元）→ 单元数=5<8， + # 但题目数=10>=8。按单元计数须剔除（按题目计数会误通过）。 + pair_diag: list[_FakeQuestion] = [] + for i in range(3): + pair_diag.extend(_fake_pair(f"P-d{i}", "P")) + pair_val: list[_FakeQuestion] = [] + for i in range(2): + pair_val.extend(_fake_pair(f"P-v{i}", "P")) + pools = _FakePools( + diagnosis=keep_diag + pair_diag, + validation=keep_val + pair_val, + test=[], + ) + + new_pools, new_types = _filter_untrainable_types( + pools, task_types=None, eval_min_per_class=2, trainable_min_units=8 + ) + + assert "P" not in {q.task_type for q in new_pools.diagnosis} + assert "P" not in new_types + assert "A" in new_types + + def test_all_filtered_raises(self) -> None: + """所有题型都被剔除时 fail-fast：raise RuntimeError 并列出剔除原因。""" + diag = [_FakeQuestion(question_id="B-d0", task_type="B")] # val=0 < 2 + pools = _FakePools(diagnosis=diag, validation=[], test=[]) + + with pytest.raises(RuntimeError, match="可训练"): + _filter_untrainable_types( + pools, task_types=None, eval_min_per_class=2, trainable_min_units=8 + ) def test_test_pool_untouched(self) -> None: """test 池不参与过滤（继续报告全题型准确率）。""" From ee69721ea3254f3e7a02991c5ce7e668063c3eda Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 07:13:31 -0400 Subject: [PATCH 40/43] fix: filter frozen pools by requested task_types subset --- app/harness/runner.py | 8 +++++++- tests/unit/test_harness_runner.py | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/app/harness/runner.py b/app/harness/runner.py index ae35031..3c8b1a1 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -320,9 +320,15 @@ def _filter_untrainable_types( """ diag_by_type = Counter(u.task_type for u in build_units(pools.diagnosis)) val_by_type = Counter(u.task_type for u in build_units(pools.validation)) + # 先按调用方显式 task_types 收窄候选集：未请求的题型（哪怕可训练）不得进入 + # keep，否则冻结全局 pools 后 batch/diagnosis 会训练非请求题型，而 gate 只覆盖 + # 请求题型 → 静默语义偏差（I-4）。task_types=None 表示全部题型皆为候选。 + candidates = set(diag_by_type) | set(val_by_type) + if task_types is not None: + candidates &= set(task_types) keep: set[str] = set() dropped: list[tuple[str, str]] = [] - for tt in set(diag_by_type) | set(val_by_type): + for tt in candidates: n_val = val_by_type.get(tt, 0) n_units = diag_by_type.get(tt, 0) + n_val if n_val < eval_min_per_class: diff --git a/tests/unit/test_harness_runner.py b/tests/unit/test_harness_runner.py index 98dad2f..44f0711 100644 --- a/tests/unit/test_harness_runner.py +++ b/tests/unit/test_harness_runner.py @@ -490,6 +490,33 @@ class TestFilterUntrainableTypes: assert "P" not in new_types assert "A" in new_types + def test_task_types_subset_drops_non_requested_trainable(self) -> None: + """指定 task_types 子集时：非请求但可训练的题型也被剔除出 diag/val。 + + 回归 I-4：pools 过滤此前只按可训练性 keep、不按 task_types 收窄，导致 + 冻结全局 pools 后 batch/diagnosis 会训练非请求题型（gate 只覆盖请求题型 + → 静默语义偏差）。此处 A、B 均可训，仅请求 A，B 必须被剔除。 + """ + # 题型 A：diag=8 + val=2 → units=10、val=2，可训 + diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)] + val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(2)] + # 题型 B：diag=8 + val=2 → units=10、val=2，同样可训（但未被请求） + diag += [_FakeQuestion(question_id=f"B-d{i}", task_type="B") for i in range(8)] + val += [_FakeQuestion(question_id=f"B-v{i}", task_type="B") for i in range(2)] + pools = _FakePools(diagnosis=diag, validation=val, test=[]) + + new_pools, new_types = _filter_untrainable_types( + pools, + task_types=["A"], + eval_min_per_class=2, + trainable_min_units=8, + ) + + # 非请求题型 B（可训）被剔除出 diag/val + assert {q.task_type for q in new_pools.diagnosis} == {"A"} + assert {q.task_type for q in new_pools.validation} == {"A"} + assert new_types == ["A"] + def test_all_filtered_raises(self) -> None: """所有题型都被剔除时 fail-fast：raise RuntimeError 并列出剔除原因。""" diag = [_FakeQuestion(question_id="B-d0", task_type="B")] # val=0 < 2 From b3ba11c7a53f0fa6aa344d665a3d2e9600402ad5 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 07:16:33 -0400 Subject: [PATCH 41/43] fix: count gate INFRA guard numerator by unit not record --- app/harness/validate.py | 32 ++++- tests/unit/test_harness_validate.py | 202 +++++++++++++++++++++------- 2 files changed, 177 insertions(+), 57 deletions(-) diff --git a/app/harness/validate.py b/app/harness/validate.py index 304d124..c55e9be 100644 --- a/app/harness/validate.py +++ b/app/harness/validate.py @@ -259,6 +259,23 @@ def _infra_question_ids_from_db( } +def _count_infra_units(units: list[QuestionUnit], infra_qids: set[str]) -> int: + """统计含 INFRA record 的 unit 数（一个 unit 任一题 INFRA 即计 1）。 + + 使护栏分子与分母（r.total，unit 粒度）同口径：AR pair 一 unit 含两 record， + 逐 record 计数会放大分子致 gate_guard_err 误触发，破坏 unit 粒度一致性 + （核心算法保真 #5/#6）。 + + 参数: + units: 当前块的单元列表（single 或 AR pair）。 + infra_qids: 本 run 中 stop_reason 属 INFRA 故障族的 question_id 集合。 + + 返回: + 含至少一题 INFRA 的 unit 数。 + """ + return sum(1 for u in units if any(q.question_id in infra_qids for q in u.questions)) + + def _candidate_correctness_from_db( log: HarnessLog, run_id: str, @@ -332,11 +349,12 @@ async def _resolve_baseline_block( if miss_units: miss_questions = flatten_units(miss_units) r_b = await run_inference(miss_questions, run_id=run_id, skills_dir=base_skills_dir) - # 护栏错误计数与 INFRA 判定口径一致：error + parse_error 都计入， - # 使 parse_error 风暴同样能触发 gate_guard_err 熔断（不被绕过）。 - errors_inc = sum(r_b.stop_reason_counts.get(reason, 0) for reason in _INFRA_STOP_REASONS) - denom_inc = r_b.total infra_qids = _infra_question_ids_from_db(log, r_b.run_id, miss_questions) + # 护栏分子与分母（r.total，unit 粒度）同口径：含 INFRA record 的 unit 计 1， + # 避免 AR pair（一 unit 两 record）逐 record 计数放大分子致误触发；仍涵盖 + # error + parse_error（_infra_question_ids_from_db 口径），parse_error 风暴不被绕过。 + errors_inc = _count_infra_units(miss_units, infra_qids) + denom_inc = r_b.total fresh_per_q = _candidate_correctness_from_db(log, r_b.run_id, miss_questions) fresh_units = unit_correctness_view(miss_units, fresh_per_q) # 只回写非 INFRA 单元；INFRA 单元不入缓存（不永久污染基线快照） @@ -382,8 +400,10 @@ async def _run_candidate_block( questions = flatten_units(units) r_c = await run_inference(questions, run_id=run_id, skills_dir=cand_dir) c_per_q = _candidate_correctness_from_db(log, r_c.run_id, questions) - # 护栏错误计数与 INFRA 判定口径一致：error + parse_error 都计入。 - errors_inc = sum(r_c.stop_reason_counts.get(reason, 0) for reason in _INFRA_STOP_REASONS) + infra_qids = _infra_question_ids_from_db(log, r_c.run_id, questions) + # 护栏分子与分母（r.total，unit 粒度）同口径：含 INFRA record 的 unit 计 1 + # （见 _count_infra_units），涵盖 error + parse_error。 + errors_inc = _count_infra_units(units, infra_qids) return c_per_q, errors_inc, r_c.total diff --git a/tests/unit/test_harness_validate.py b/tests/unit/test_harness_validate.py index f93b641..7680d67 100644 --- a/tests/unit/test_harness_validate.py +++ b/tests/unit/test_harness_validate.py @@ -112,7 +112,6 @@ def _make_mock_run_inference( log: HarnessLog, baseline_correctness: dict[str, bool], candidate_correctness: dict[str, bool], - error_count: int = 0, ): """构建 mock RunInferenceFn。 @@ -136,9 +135,6 @@ def _make_mock_run_inference( correct = sum(per_q.values()) total = len(questions) - stop_counts: dict[str, int] = {"completed": total - error_count} - if error_count > 0: - stop_counts["error"] = error_count return InferenceResult( run_id=run_id, accuracy=correct / total if total else 0.0, @@ -147,7 +143,57 @@ def _make_mock_run_inference( per_task_type={}, steps_mean=1.0, token_usage={"prompt_tokens": 10, "completion_tokens": 10}, - stop_reason_counts=stop_counts, + stop_reason_counts={"completed": total}, + ) + + return mock_fn, call_log + + +def _make_all_infra_mock(log: HarnessLog, stop_reason: str): + """构建基线全 INFRA 的 mock：每 record 写指定 INFRA stop_reason（error/parse_error）。 + + 与真实推理一致——per-record DB stop_reason 与汇总 stop_reason_counts 同源；护栏 + 分子按 unit 从 DB 读（_infra_question_ids_from_db），故须真实落 DB。total 返回 + unit 粒度（single 时 == 题数），使护栏分子/分母同粒度。 + """ + call_log: list[dict[str, Any]] = [] + + async def mock_fn( + questions: list[GeneratedQuestion], + *, + run_id: str, + skills_dir: Path, + ) -> InferenceResult: + call_log.append({"run_id": run_id, "n": len(questions)}) + for q in questions: + log.insert( + "predictions", + { + "run_id": run_id, + "video_id": "v0", + "question_id": q.question_id, + "task_type": "temporal", + "prediction": "", + "answer": "A", + "evidence": "", + "reasoning": "", + "steps_used": 1, + "prompt_tokens": 10, + "completion_tokens": 10, + "stop_reason": stop_reason, + "steps_json": "[]", + }, + ) + total = len(questions) # 全 single → unit 数 == 题数 + return InferenceResult( + run_id=run_id, + accuracy=0.0, + total=total, + correct=0, + per_task_type={}, + steps_mean=1.0, + token_usage={"prompt_tokens": 10, "completion_tokens": 10}, + stop_reason_counts={stop_reason: total}, ) return mock_fn, call_log @@ -421,17 +467,14 @@ async def test_gate_prefix_must_contain_gate(tmp_path: Path) -> None: @pytest.mark.asyncio async def test_infra_guard_threshold(tmp_path: Path) -> None: - """推理错误率超阈值时抛 RuntimeError。""" + """推理错误率超阈值时抛 RuntimeError（护栏分子/分母 unit 同粒度）。""" workspace = _setup_workspace(tmp_path) log = _make_log(workspace) - # 需要 >=10 题次才触发 INFRA 护栏 - questions = _make_questions(6) + # 需要 >=10 unit 分母才触发护栏：12 个 single，基线全 INFRA error。 + # 首块全 INFRA → valid_chunk 空 → errors=12/denom=12=1.0>0.5 触发护栏。 + questions = _make_questions(12) cache = BaselineCache(workspace / "baseline_cache.json") - - baseline_correct = {f"q{i}": False for i in range(6)} - candidate_correct = {f"q{i}": False for i in range(6)} - # 每次 run_inference 报 error_count=5，两侧各 5 → 10/12 > 0.5 - mock_fn, _ = _make_mock_run_inference(log, baseline_correct, candidate_correct, error_count=5) + mock_fn, _ = _make_all_infra_mock(log, "error") try: with pytest.raises(RuntimeError, match="错误率过高"): @@ -444,7 +487,7 @@ async def test_infra_guard_threshold(tmp_path: Path) -> None: base_skill_content="baseline skill content", ladder_items=questions, gate_params=_DEFAULT_GATE_PARAMS, - gate_block=6, + gate_block=12, gate_n_max=20, gate_guard_err=0.5, baseline_cache=cache, @@ -575,6 +618,95 @@ async def test_baseline_infra_error_not_cached(tmp_path: Path) -> None: log.close() +@pytest.mark.asyncio +async def test_infra_guard_counts_units_not_records(tmp_path: Path) -> None: + """护栏分子按 unit 计：AR pair 两 record 全 INFRA 只计 1 个 INFRA unit（而非 2）。 + + 回归 I-3：分子此前用 stop_reason_counts 逐 record 计数，分母 denom_inc=r.total + 是 unit 粒度；AR pair（一 unit 两 record）致分子被放大、误触发 gate_guard_err。 + 分子改为"含 INFRA record 的 unit 数"后与分母同粒度（核心算法保真 #5/#6）。 + """ + from app.harness.gate_ladder import skill_hash + from app.harness.question_units import build_units + from app.harness.validate import _resolve_baseline_block + + workspace = _setup_workspace(tmp_path) + log = _make_log(workspace) + # 一个 AR pair（两成员共享 pair_id）→ build_units 折叠为 1 个 pair unit + common = { + "video_id": "vp", + "task_type": "temporal", + "question": "Q?", + "options": ("A", "B", "C", "D"), + "answer": "A", + "source_nodes": (), + "difficulty": "easy", + "pair_id": "p1", + "flip_axis": "before_after", + } + pair = [ + GeneratedQuestion(question_id="p1_o", question_role="pair_original", **common), + GeneratedQuestion(question_id="p1_m", question_role="pair_mirror", **common), + ] + units = build_units(pair) + assert len(units) == 1 # 前置：pair 折叠为 1 个 unit + cache = BaselineCache(workspace / "baseline_cache.json") + s_hash = skill_hash("baseline skill content") + + async def mock_fn(qs, *, run_id, skills_dir): + # 两 record 皆 INFRA error + for q in qs: + log.insert( + "predictions", + { + "run_id": run_id, + "video_id": "vp", + "question_id": q.question_id, + "task_type": "temporal", + "prediction": "", + "answer": "A", + "evidence": "", + "reasoning": "", + "steps_used": 1, + "prompt_tokens": 10, + "completion_tokens": 10, + "stop_reason": "error", + "steps_json": "[]", + }, + ) + # total 为 unit 粒度（1 个 pair unit）；stop_reason_counts 为 record 粒度（2） + return InferenceResult( + run_id=run_id, + accuracy=0.0, + total=1, + correct=0, + per_task_type={}, + steps_mean=1.0, + token_usage={"prompt_tokens": 20, "completion_tokens": 20}, + stop_reason_counts={"error": 2}, + ) + + try: + _b_units, valid_units, errors_inc, denom_inc = await _resolve_baseline_block( + units=units, + task_type="temporal", + s_hash=s_hash, + prompts_version="p1", + baseline_cache=cache, + base_skills_dir=workspace / "skills" / "v1", + run_inference=mock_fn, + log=log, + run_id="step1_gate_b0_base", + ) + # 分子按 unit 计：1 个 INFRA unit（不是 2 条 record）；分母同粒度 = r.total = 1 + assert errors_inc == 1 + assert denom_inc == 1 + # 整对 INFRA → 从有效单元剔除 + assert valid_units == [] + finally: + log.close() + + @pytest.mark.asyncio async def test_all_infra_ladder_raises_clear_error(tmp_path: Path) -> None: """整个阶梯所有 unit 都被判为 INFRA 排除 → 明确 RuntimeError（非误导性空阶梯断言）。""" @@ -651,43 +783,11 @@ async def test_parse_error_counts_toward_guard(tmp_path: Path) -> None: """stop_reason=parse_error 也计入护栏错误率（与 INFRA 判定口径一致）→ 超阈值熔断。""" workspace = _setup_workspace(tmp_path) log = _make_log(workspace) - questions = _make_questions(6) + # 12 个 single，基线全 parse_error（per-record 落 DB，护栏按 unit 从 DB 读）。 + # 首块全 INFRA → errors=12/denom=12=1.0>0.5 → parse_error 亦触发护栏。 + questions = _make_questions(12) cache = BaselineCache(workspace / "baseline_cache.json") - - async def mock_fn(qs, *, run_id, skills_dir): - # 逐题 stop_reason 保持 completed（不触发 per-unit INFRA 排除）， - # 但汇总 stop_reason_counts 报大量 parse_error（应计入护栏）。 - for q in qs: - log.insert( - "predictions", - { - "run_id": run_id, - "video_id": "v0", - "question_id": q.question_id, - "task_type": "temporal", - "prediction": "Z", - "answer": "A", - "evidence": "", - "reasoning": "", - "steps_used": 1, - "prompt_tokens": 10, - "completion_tokens": 10, - "stop_reason": "completed", - "steps_json": "[]", - }, - ) - total = len(qs) - # 两臂各 5 个 parse_error → 累计 10/12 > 0.5 触发护栏 - return InferenceResult( - run_id=run_id, - accuracy=0.0, - total=total, - correct=0, - per_task_type={}, - steps_mean=1.0, - token_usage={"prompt_tokens": 10, "completion_tokens": 10}, - stop_reason_counts={"completed": 1, "parse_error": 5}, - ) + mock_fn, _ = _make_all_infra_mock(log, "parse_error") try: with pytest.raises(RuntimeError, match="错误率过高"): @@ -700,7 +800,7 @@ async def test_parse_error_counts_toward_guard(tmp_path: Path) -> None: base_skill_content="baseline skill content", ladder_items=questions, gate_params=_DEFAULT_GATE_PARAMS, - gate_block=6, + gate_block=12, gate_n_max=20, gate_guard_err=0.5, baseline_cache=cache, From 6911c83d66a8a6c5e53f827dcef8e46adcb9be6d Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 07:18:14 -0400 Subject: [PATCH 42/43] refactor: share INFRA stop-reason set from core.evolution --- app/harness/validate.py | 12 ++++++------ core/evolution/__init__.py | 3 ++- core/evolution/diagnose.py | 6 +++--- tests/unit/test_harness_validate.py | 9 +++++++++ 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/app/harness/validate.py b/app/harness/validate.py index c55e9be..3e37756 100644 --- a/app/harness/validate.py +++ b/app/harness/validate.py @@ -27,6 +27,7 @@ from loguru import logger from app.harness.gate_ladder import BaselineCache, skill_hash from app.harness.question_units import build_units, flatten_units, unit_correctness_view from core.evolution import ( + INFRA_STOP_REASONS, GateParams, GateVerdict, RejectedEdit, @@ -35,6 +36,10 @@ from core.evolution import ( pair_block, ) +# INFRA_STOP_REASONS 复用 core.evolution.diagnose 的单一定义（M-2）：INFRA 故障 +# stop_reason（推理侧基础设施错误，非模型答错）在诊断与 gate 两处必须同口径， +# 避免各自维护副本致未来漂移。 + if TYPE_CHECKING: from app.harness.inference import InferenceResult from app.harness.log import HarnessLog @@ -231,11 +236,6 @@ def _load_run_rows( return normalized -# INFRA 故障 stop_reason（推理侧基础设施错误，非模型答错）：这些题的对错无信号意义， -# 基线臂遇到时不得写入 BaselineCache（否则一次瞬时故障永久污染基线快照）。 -_INFRA_STOP_REASONS = frozenset({"error", "parse_error"}) - - def _infra_question_ids_from_db( log: HarnessLog, run_id: str, @@ -255,7 +255,7 @@ def _infra_question_ids_from_db( return { q.question_id for q in chunk - if rows.get(q.question_id, {}).get("stop_reason") in _INFRA_STOP_REASONS + if rows.get(q.question_id, {}).get("stop_reason") in INFRA_STOP_REASONS } diff --git a/core/evolution/__init__.py b/core/evolution/__init__.py index b9e0b8c..9308f61 100644 --- a/core/evolution/__init__.py +++ b/core/evolution/__init__.py @@ -4,7 +4,7 @@ 只依赖 Protocol 接口和标准库，可搬到无 adapters 的环境用假实现原样运行。 """ -from core.evolution.diagnose import run_diagnosis +from core.evolution.diagnose import INFRA_STOP_REASONS, run_diagnosis from core.evolution.evolve import ( edit_budget_at, evolve_single_skill, @@ -44,6 +44,7 @@ from core.evolution.types import ( from core.evolution.validate import classify_quadrants, compute_accuracy, pair_block __all__ = [ + "INFRA_STOP_REASONS", "CaseSample", "DiagnosePrompts", "DiagnosisResult", diff --git a/core/evolution/diagnose.py b/core/evolution/diagnose.py index d100aae..2d045e4 100644 --- a/core/evolution/diagnose.py +++ b/core/evolution/diagnose.py @@ -53,7 +53,7 @@ if TYPE_CHECKING: _SPAN_EVAL_TOOLS: frozenset[str] = frozenset({"view_node", "search_similar", "observe_frame"}) """span 级评估涵盖的工具集合。""" -_INFRA_STOP_REASONS: frozenset[str] = frozenset({"error", "parse_error"}) +INFRA_STOP_REASONS: frozenset[str] = frozenset({"error", "parse_error"}) """执行/解析层失败导致排除的 stop_reason 集合。""" @@ -2007,7 +2007,7 @@ def _count_infra_excluded( qids = [ row["question_id"] for row in prediction_rows - if row.get("stop_reason") in _INFRA_STOP_REASONS + if row.get("stop_reason") in INFRA_STOP_REASONS ] return len(qids), qids @@ -2083,7 +2083,7 @@ async def run_diagnosis( for row in all_predictions: stop_reason = row.get("stop_reason") - if stop_reason in _INFRA_STOP_REASONS: + if stop_reason in INFRA_STOP_REASONS: continue if task_type_filter and row.get("task_type") not in task_type_filter: continue diff --git a/tests/unit/test_harness_validate.py b/tests/unit/test_harness_validate.py index 7680d67..97fd936 100644 --- a/tests/unit/test_harness_validate.py +++ b/tests/unit/test_harness_validate.py @@ -199,6 +199,15 @@ def _make_all_infra_mock(log: HarnessLog, stop_reason: str): return mock_fn, call_log +def test_infra_stop_reasons_single_source() -> None: + """app 侧 INFRA_STOP_REASONS 复用 core 常量（同一对象），杜绝未来漂移（M-2）。""" + from app.harness import validate + from core.evolution import diagnose + + assert validate.INFRA_STOP_REASONS is diagnose.INFRA_STOP_REASONS + assert frozenset({"error", "parse_error"}) == diagnose.INFRA_STOP_REASONS + + # =========================================================================== # 数据类型测试 # =========================================================================== From 6a9b1ea1883bfa47c9b5e4a7e58b34a73c7deee0 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 07:21:10 -0400 Subject: [PATCH 43/43] docs: record preflight final review + integration fixes (I-3/I-4/M-2) --- research-wiki/graph/edges.json | 12 +++++ research-wiki/index.md | 6 ++- research-wiki/log.md | 3 ++ .../2026-07-16-preflight-final-review.md | 49 +++++++++++++++++++ .../reviews/preflight-final-review.md | 9 ++++ 5 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 research-wiki/reviews/2026-07-16-preflight-final-review.md create mode 100644 research-wiki/reviews/preflight-final-review.md diff --git a/research-wiki/graph/edges.json b/research-wiki/graph/edges.json index 75a4e36..7071e8d 100644 --- a/research-wiki/graph/edges.json +++ b/research-wiki/graph/edges.json @@ -265,6 +265,11 @@ "id": "plan:preflight-wp4-resilience", "label": "WP4 韧性与持久化", "type": "plan" + }, + { + "id": "review:preflight-final-review", + "label": "训练前修复分支终审", + "type": "review" } ], "links": [ @@ -540,6 +545,13 @@ "relation": "implements", "evidence": "四工作包实现训练前缺陷修复设计", "added": "2026-07-16T08:32:42.990273+00:00" + }, + { + "source": "review:preflight-final-review", + "target": "design:preflight-fixes", + "relation": "informs", + "evidence": "跨 task 终审确认设计 21 项缺陷全部落地 + 4 项集成发现", + "added": "2026-07-16T11:06:43.709183+00:00" } ] } \ No newline at end of file diff --git a/research-wiki/index.md b/research-wiki/index.md index 0e347cc..130642b 100644 --- a/research-wiki/index.md +++ b/research-wiki/index.md @@ -1,6 +1,6 @@ # Research Wiki 索引 -> 自动生成，更新时间：2026-07-16 08:32 UTC +> 自动生成，更新时间：2026-07-16 11:06 UTC ## design (36) - [2026-07-06-core-agent-adapters-llm-design](designs/2026-07-06-core-agent-adapters-llm-design.md) `design:2026-07-06-core-agent-adapters-llm-design` @@ -102,9 +102,11 @@ - [赛题生成工具实现计划](plans/question-gen-synth.md) `plan:question-gen-synth` - [项目基础设施初始化计划](plans/infrastructure-setup.md) `plan:infrastructure-setup` -## review (2) +## review (4) +- [2026-07-16-preflight-final-review](reviews/2026-07-16-preflight-final-review.md) `review:2026-07-16-preflight-final-review` - [2026-07-16-preflight-train-review](reviews/2026-07-16-preflight-train-review.md) `review:2026-07-16-preflight-train-review` - [question-gen v2 设计对抗审核 — 六路独立核验（四层病灶闭合度 + 契约一致性）](reviews/2026-07-15-question-gen-v2-adversarial-audit.md) `review:2026-07-15-question-gen-v2-adversarial-audit` +- [训练前修复分支终审](reviews/preflight-final-review.md) `review:preflight-final-review` ## schema (6) - [表结构 v3 出题日志/观测（unit_verdict / collapse_metrics / quarantine / facts / resume）](schemas/v3-question-gen-logging.md) `schema:v3-question-gen-logging` diff --git a/research-wiki/log.md b/research-wiki/log.md index 3cf8936..0133a9f 100644 --- a/research-wiki/log.md +++ b/research-wiki/log.md @@ -132,3 +132,6 @@ - [2026-07-16 08:32 UTC] 新增 plan: WP4 韧性与持久化 (plan:preflight-wp4-resilience) - [2026-07-16 08:32 UTC] 新增边: plan:preflight-wp4-resilience --implements--> design:preflight-fixes - [2026-07-16 08:32 UTC] 重建索引: 109 篇页面 +- [2026-07-16 11:06 UTC] 新增 review: 训练前修复分支终审 (review:preflight-final-review) +- [2026-07-16 11:06 UTC] 新增边: review:preflight-final-review --informs--> design:preflight-fixes +- [2026-07-16 11:06 UTC] 重建索引: 111 篇页面 diff --git a/research-wiki/reviews/2026-07-16-preflight-final-review.md b/research-wiki/reviews/2026-07-16-preflight-final-review.md new file mode 100644 index 0000000..da86ea8 --- /dev/null +++ b/research-wiki/reviews/2026-07-16-preflight-final-review.md @@ -0,0 +1,49 @@ +# 训练前修复分支 final whole-implementation review + +> 2026-07-16。分支 `feat/preflight-train-fixes`，4 工作包 33 个实现 commit。SDD 每 task 三审 + 本次 Codex 跨 task 整分支终审（gpt-5.4 xhigh）。 +> 覆盖矩阵：21 项确认缺陷 + 接线 2 项 + WP1 死字段清理**全部实现落地**（Codex 逐项核对 file:line 证据）。核心算法保真（#4/#5/#6/#7/#8/#9/#10/#12）、依赖方向（core 不依赖 app/adapters）、loguru、原子写、死字段无残留——全部 verified OK。 + +## 执行结果汇总 + +| WP | 内容 | commit 数 | Codex 三审结果 | +|----|------|----------|---------------| +| WP1 | 模板迁移 + 死字段 + fail-loud | 3 | 无任何问题（模板 cmp TRM4 逐字节一致） | +| WP2 | 切分与接线（tier/功效/seed/覆盖保护） | 7 | 无 Critical；2 Important（--force 备份健壮性）已修 | +| WP4 | 韧性与持久化（10 项） | 11 | 无 Critical；2 Important（全 INFRA/parse_error 护栏）已修 | +| WP3 | 训练循环与进化（9 项，5 算法保真区） | 12 | 无 Critical；2 Important（单元计数/全过滤 fail-fast）已修 | + +全量 1523 tests passed，ruff 全绿。 + +## Final review 发现与处置 + +### C-1（必须，runbook）——.env 未同步导致 train 启动崩溃 +`.env:65` 仍 `REDIS_CACHE_TTL=0`，WP4 Task 2 的 fail-loud 会在 `main.py` / `video_split_cli` 构建 Redis 缓存时抛 ValueError，训练进不到 `runner.train`。`.env.example` 已改 86400 但 `.env`（gitignore）需**手动**改。→ 训练前 runbook 第一步：`REDIS_CACHE_TTL` 改为正整数（如 86400）。 + +### I-3 / I-4（真实缺陷，✅ 已修） +| # | 缺陷 | 首跑是否触发 | 处置（commit） | +|---|------|:---:|------| +| I-4 | 显式 task_types 子集不过滤冻结全局 pools → 训练非请求题型 | 全 12 类首跑**不触发** | ✅ `ee69721`：`_filter_untrainable_types` 候选集先与 task_types 取交集，非请求题型剔除，fail-fast 保留 | +| I-3 | gate INFRA 护栏分子按 record、分母按 unit，AR pair 误触发 gate_guard_err | 全 single 首跑**不触发** | ✅ `b3ba11c`：INFRA 分子改按 unit 数；顺带修正 2 个既有护栏测试的 mock 不真实性（per-record 与 summary 不一致，真实推理不会发生），保真套件全绿 | + +> M-2（`6911c83`）：`INFRA_STOP_REASONS` 提为 core 公共常量、app import。**残留 future work**：`app/harness/video_split_cli.py` 仍有第三份独立副本（本次 scope 外），待后续 dedup。 + +### I-1 / I-2（技术论证：实际影响可控，记录不强修） +| # | Codex 关切 | 论证 | 首跑建议 | +|---|-----------|------|---------| +| I-1 | cache_salt 未贯穿工具内 LLM/VLM（observe_frame VLM、summarizer） | 工具内是**确定性子程序**——同帧→同 VLM 描述、同轨迹→同 summary，重放语义正确甚至期望；需跨 epoch 重采样的 agent **决策** LLM（主 loop chat）已正确注入 run_id 盐 | 改 .env 后可跑；若谨慎可首跑关 Redis 缓存 | +| I-2 | 基线臂 miss 新鲜推理注入 epoch salt，不符固定快照 | BaselineCache（`baseline_cache.json`）**跨 run 持久**：快照一旦建立即稳定，epoch salt 只作用于"首次建立快照的那一次采样"（本就要采一次），中断重跑命中持久缓存 | 影响限于首次采样，可接受 | + +> 若后续要彻底贯彻 P1-1（把 cache_salt 显式化、基线臂传 None、工具内也隔离），记 future work——需把 Task 9 的"inference 内部用 run_id"重构为"调用方显式传 cache_salt"，风险中等。 + +### Minor（future work / 顺手） +- **M-2**：INFRA stop-reason 集合在 core/app 各一份 → 已派 app import core 常量消除漂移。 +- **M-1**：`_atomic_write_json` 在 pools.py/workspace.py 两份等价实现（行为一致，可抽共享 helper）。 +- **M-3**：部分 analyses JSON（非 checkpoint/manifest/pools）仍直接 write_text（不破坏续跑主状态）。 + +## 训练前 runbook（结合本分支） + +1. **改 `.env`：`REDIS_CACHE_TTL=0` → `86400`**（C-1，不改则启动崩）。 +2. 备份 `workspaces/video-split/` 冻结产物。改 `config/video_split.yaml` 已含 `val_ratio: 0.4`（WP2），重跑 `build_video_split.sh`（诊断命中缓存秒级重切）→ 核对 val 错题≥20 / T2 入 diag / tier 感知生效。 +3. 建 seed：`extract_run_db(infer_adhoc, dedupe_per_question=True)` + `init_seed('adhoc-baseline', pools_json=…, split_manifest=…)`（WP2 接线）。 +4. 新建 `config/train_videomme.yaml` + `scripts/train_videomme.sh`：`epochs=3`、`early_stop_patience=2`（epoch 语义 WP3）、`run_holdout_eval=true`（去重版 WP3）、`trainable_min_units=8`（WP3 新字段，必填）、全 12 类（避免 I-4）、其余 gate/batch 沿用 default.yaml。 +5. tmux 启动 `CUDA_VISIBLE_DEVICES=0 bash scripts/train_videomme.sh`。预检会自动剔除微型题型（OCR/Spatial/Temporal Perception 等 5 类），打印剔除清单。 diff --git a/research-wiki/reviews/preflight-final-review.md b/research-wiki/reviews/preflight-final-review.md new file mode 100644 index 0000000..7441243 --- /dev/null +++ b/research-wiki/reviews/preflight-final-review.md @@ -0,0 +1,9 @@ +--- +type: review +node_id: review:preflight-final-review +title: 训练前修复分支终审 +date: 2026-07-16 +--- + +# 训练前修复分支终审 +