feat: add offline baseline diagnosis orchestration

2026-07-15 12:22:07 -04:00
parent 3d8bd75372
commit 53989078a0
2 changed files with 504 additions and 0 deletions
@@ -0,0 +1,264 @@
+"""离线诊断编排集成测试（LLM 类：MD 产出到 tests/outputs/）。
+
+用 fake run_diagnosis + fake deps 覆盖编排契约，不实际调 LLM/VLM：
+    1. 续跑幂等 —— 已落盘题跳过，第二次无剩余则 run_diagnosis 收到空列表。
+    2. 投影正确 —— defect→T2、lapse→T1，evolution_target 由 error_type 派生。
+
+测试结束把编排过程（remaining、各 tier 计数、投影样例）写入
+tests/outputs/test_baseline_diagnosis/<test>_<固定 ts>.md（CLAUDE.md §4.6）。
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from adapters.baseline_diagnosis_store import SqliteDiagnosisSignalStore
+from app.harness.baseline_diagnosis import DiagnosisDeps, run_baseline_diagnosis
+
+# 固定时间戳：库代码不用 datetime.now，测试传入固定值保证 MD 可复现。
+_FIXED_TS = "20260715_000000"
+_OUTPUT_DIR = Path(__file__).resolve().parents[1] / "outputs" / "test_baseline_diagnosis"
+
+
+class _FakeRunLog:
+    """内层 RunLog 假实现：predictions/traces 均返回空，编排不真正诊断。"""
+
+    async def get_predictions(self, run_id, *, question_ids=None):
+        return []
+
+    async def get_traces(self, run_id, *, question_ids=None):
+        return []
+
+
+class _FakeLLM: ...
+
+
+class _FakeSkillStore: ...
+
+
+def _mk_q(qid):
+    """构造最小可用 GeneratedQuestion（补齐必填 source_nodes/difficulty）。"""
+    from core.types import GeneratedQuestion
+
+    return GeneratedQuestion(
+        question_id=qid,
+        video_id="v",
+        task_type="Counting Problem",
+        question="",
+        options=("A", "B", "C", "D"),
+        answer="A",
+        source_nodes=(),
+        difficulty="easy",
+    )
+
+
+def _deps(monkeypatch, calls):
+    """构造 DiagnosisDeps 并 monkeypatch run_diagnosis 记录每次 question_ids。"""
+
+    async def fake_run_diagnosis(
+        run_id,
+        questions,
+        tree_data,
+        llm,
+        run_log,
+        skill_store,
+        prompts,
+        *,
+        concurrency,
+        question_ids=None,
+        task_types=None,
+        only_incorrect=False,
+    ):
+        calls.append(tuple(question_ids or []))
+        from core.evolution.types import DiagnosisResult, ErrorAttribution
+
+        # 仅对本次传入的题产出归因，续跑时空列表 → 无归因。
+        attributions = []
+        if "q1" in (question_ids or []):
+            attributions.append(ErrorAttribution("q1", "search_failure", None, "defect"))
+        if "q2" in (question_ids or []):
+            attributions.append(ErrorAttribution("q2", "mixed", None, "lapse"))
+        return DiagnosisResult(
+            run_id=run_id,
+            error_attributions=attributions,
+            infra_question_ids=[],
+            degraded_question_ids=[],
+        )
+
+    monkeypatch.setattr("app.harness.baseline_diagnosis.run_diagnosis", fake_run_diagnosis)
+    return DiagnosisDeps(
+        run_log=_FakeRunLog(),
+        llm=_FakeLLM(),
+        skill_store=_FakeSkillStore(),
+        prompts=object(),
+        tree_data={},
+        concurrency=2,
+    )
+
+
+def _write_md(test_name: str, lines: list[str]) -> Path:
+    """把编排过程写入 tests/outputs 下的 MD（人类可读结构化）。"""
+    _OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+    path = _OUTPUT_DIR / f"{test_name}_{_FIXED_TS}.md"
+    path.write_text("\n".join(lines), encoding="utf-8")
+    return path
+
+
+@pytest.mark.asyncio
+async def test_resume_skips_done(tmp_path, monkeypatch):
+    """续跑幂等 + 投影正确：首次全诊断落库，第二次无剩余；tier 投影符合分层。"""
+    calls: list[tuple[str, ...]] = []
+    deps = _deps(monkeypatch, calls)
+    store = SqliteDiagnosisSignalStore(str(tmp_path / "h.db"))
+    q_by_id = {"q1": _mk_q("q1"), "q2": _mk_q("q2")}
+
+    # 第一次：两题都是剩余，全部诊断落库。
+    await run_baseline_diagnosis(
+        baseline_run_id="infer_adhoc",
+        diag_fingerprint="fp",
+        wrong_ids=["q1", "q2"],
+        questions=q_by_id,
+        store=store,
+        deps=deps,
+    )
+    assert store.done_question_ids("infer_adhoc", "fp") == {"q1", "q2"}
+    assert calls[0] == ("q1", "q2")
+
+    # 投影正确性：q1 defect→T2、q2 lapse→T1，evolution_target 由 error_type 派生。
+    rows = {r.question_id: r for r in store.load("infer_adhoc", "fp")}
+    assert rows["q1"].tier == "T2"
+    assert rows["q1"].cause_category == "defect"
+    assert rows["q1"].evolution_target == "skill"  # search_failure → skill
+    assert rows["q1"].error_type == "search_failure"
+    assert rows["q1"].infra is False
+    assert rows["q1"].degraded is False
+    assert rows["q2"].tier == "T1"
+    assert rows["q2"].cause_category == "lapse"
+    assert rows["q2"].evolution_target == "system"  # mixed → system
+
+    # 第二次：两题已完成 → remaining 为空。编排按规约直接 return（续跑幂等，
+    # 不浪费 LLM 诊断调用），故不会向 run_diagnosis 新增调用。
+    n_calls_before = len(calls)
+    await run_baseline_diagnosis(
+        baseline_run_id="infer_adhoc",
+        diag_fingerprint="fp",
+        wrong_ids=["q1", "q2"],
+        questions=q_by_id,
+        store=store,
+        deps=deps,
+    )
+    assert len(calls) == n_calls_before  # 第二次无剩余 → 未触发诊断
+
+    md_path = _write_md(
+        "test_resume_skips_done",
+        [
+            "# 离线诊断编排：续跑幂等 + 投影正确",
+            "",
+            "## 任务描述",
+            "对 infer_adhoc 的错题跑离线诊断，投影为逐题信号行并落库；验证续跑幂等。",
+            "",
+            "## run_diagnosis 每次收到的 question_ids",
+            f"- 第 1 次: {calls[0]}",
+            "- 第 2 次: 未触发（remaining 为空，编排直接 return）",
+            "",
+            "## 落库信号投影样例",
+            "| question_id | tier | cause_category | error_type | evolution_target |",
+            "|---|---|---|---|---|",
+            f"| q1 | {rows['q1'].tier} | {rows['q1'].cause_category} | "
+            f"{rows['q1'].error_type} | {rows['q1'].evolution_target} |",
+            f"| q2 | {rows['q2'].tier} | {rows['q2'].cause_category} | "
+            f"{rows['q2'].error_type} | {rows['q2'].evolution_target} |",
+            "",
+            "## tier 计数",
+            "- T2: 1（defect，可训练核心）",
+            "- T1: 1（lapse，低信号）",
+            "",
+            "## 结论",
+            "首次两题全落库，第二次 remaining 为空 → 续跑幂等成立；tier/evolution_target 投影正确。",
+        ],
+    )
+    assert md_path.exists()
+    store.close()
+
+
+@pytest.mark.asyncio
+async def test_infra_and_degraded_projection(tmp_path, monkeypatch):
+    """INFRA→T0、degraded→uncertain 的投影：对应字段置位、error_type/target 为 None。"""
+    calls: list[tuple[str, ...]] = []
+
+    async def fake_run_diagnosis(
+        run_id,
+        questions,
+        tree_data,
+        llm,
+        run_log,
+        skill_store,
+        prompts,
+        *,
+        concurrency,
+        question_ids=None,
+        task_types=None,
+        only_incorrect=False,
+    ):
+        calls.append(tuple(question_ids or []))
+        from core.evolution.types import DiagnosisResult
+
+        return DiagnosisResult(
+            run_id=run_id,
+            error_attributions=[],
+            infra_question_ids=["q3"],
+            degraded_question_ids=["q4"],
+        )
+
+    monkeypatch.setattr("app.harness.baseline_diagnosis.run_diagnosis", fake_run_diagnosis)
+    deps = DiagnosisDeps(
+        run_log=_FakeRunLog(),
+        llm=_FakeLLM(),
+        skill_store=_FakeSkillStore(),
+        prompts=object(),
+        tree_data={},
+        concurrency=2,
+    )
+    store = SqliteDiagnosisSignalStore(str(tmp_path / "h.db"))
+    q_by_id = {"q3": _mk_q("q3"), "q4": _mk_q("q4")}
+
+    await run_baseline_diagnosis(
+        baseline_run_id="infer_adhoc",
+        diag_fingerprint="fp",
+        wrong_ids=["q3", "q4"],
+        questions=q_by_id,
+        store=store,
+        deps=deps,
+    )
+
+    rows = {r.question_id: r for r in store.load("infer_adhoc", "fp")}
+    assert rows["q3"].tier == "T0"
+    assert rows["q3"].infra is True
+    assert rows["q3"].error_type is None
+    assert rows["q3"].evolution_target is None
+    assert rows["q4"].tier == "uncertain"
+    assert rows["q4"].degraded is True
+    assert rows["q4"].error_type is None
+    assert store.done_question_ids("infer_adhoc", "fp") == {"q3", "q4"}
+
+    md_path = _write_md(
+        "test_infra_and_degraded_projection",
+        [
+            "# 离线诊断编排：INFRA / degraded 投影",
+            "",
+            "## 落库信号投影样例",
+            "| question_id | tier | infra | degraded | error_type | evolution_target |",
+            "|---|---|---|---|---|---|",
+            f"| q3 | {rows['q3'].tier} | {rows['q3'].infra} | {rows['q3'].degraded} | "
+            f"{rows['q3'].error_type} | {rows['q3'].evolution_target} |",
+            f"| q4 | {rows['q4'].tier} | {rows['q4'].infra} | {rows['q4'].degraded} | "
+            f"{rows['q4'].error_type} | {rows['q4'].evolution_target} |",
+            "",
+            "## 结论",
+            "INFRA→T0（infra 置位）、degraded→uncertain（degraded 置位），error_type/target 均 None。",
+        ],
+    )
+    assert md_path.exists()
+    store.close()