355 lines
12 KiB
Python
355 lines
12 KiB
Python
"""离线诊断编排集成测试（LLM 类：MD 产出到 tests/outputs/）。
|
||
|
||
用 fake run_diagnosis + fake deps 覆盖编排契约，不实际调 LLM/VLM：
|
||
1. 续跑幂等 —— 已落盘题跳过；第二次 remaining 为空则直接早返回、不调用
|
||
run_diagnosis。
|
||
2. 投影正确 —— defect→T2、lapse→T1，evolution_target 由 error_type 派生。
|
||
3. 三桶去重 —— 同题同时在 degraded 与 attributions 时按 degraded>infra>
|
||
attribution 优先级只落 1 行、tier=uncertain。
|
||
|
||
测试结束把编排过程（remaining、各 tier 计数、投影样例）写入
|
||
tests/outputs/test_baseline_diagnosis/<test>_<固定 ts>.md（CLAUDE.md §4.6）。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from adapters.baseline_diagnosis_store import SqliteDiagnosisSignalStore
|
||
from app.harness.baseline_diagnosis import DiagnosisDeps, run_baseline_diagnosis
|
||
|
||
# 固定时间戳：库代码不用 datetime.now，测试传入固定值保证 MD 可复现。
|
||
_FIXED_TS = "20260715_000000"
|
||
_OUTPUT_DIR = Path(__file__).resolve().parents[1] / "outputs" / "test_baseline_diagnosis"
|
||
|
||
|
||
class _FakeRunLog:
|
||
"""内层 RunLog 假实现：predictions/traces 均返回空，编排不真正诊断。"""
|
||
|
||
async def get_predictions(self, run_id, *, question_ids=None):
|
||
return []
|
||
|
||
async def get_traces(self, run_id, *, question_ids=None):
|
||
return []
|
||
|
||
|
||
class _FakeLLM: ...
|
||
|
||
|
||
class _FakeSkillStore: ...
|
||
|
||
|
||
def _mk_q(qid):
|
||
"""构造最小可用 GeneratedQuestion（补齐必填 source_nodes/difficulty）。"""
|
||
from core.types import GeneratedQuestion
|
||
|
||
return GeneratedQuestion(
|
||
question_id=qid,
|
||
video_id="v",
|
||
task_type="Counting Problem",
|
||
question="",
|
||
options=("A", "B", "C", "D"),
|
||
answer="A",
|
||
source_nodes=(),
|
||
difficulty="easy",
|
||
)
|
||
|
||
|
||
def _deps(monkeypatch, calls):
|
||
"""构造 DiagnosisDeps 并 monkeypatch run_diagnosis 记录每次 question_ids。"""
|
||
|
||
async def fake_run_diagnosis(
|
||
run_id,
|
||
questions,
|
||
tree_data,
|
||
llm,
|
||
run_log,
|
||
skill_store,
|
||
prompts,
|
||
*,
|
||
concurrency,
|
||
question_ids=None,
|
||
task_types=None,
|
||
only_incorrect=False,
|
||
):
|
||
calls.append(tuple(question_ids or []))
|
||
from core.evolution.types import DiagnosisResult, ErrorAttribution
|
||
|
||
# 仅对本次传入的题产出归因，续跑时空列表 → 无归因。
|
||
attributions = []
|
||
if "q1" in (question_ids or []):
|
||
attributions.append(ErrorAttribution("q1", "search_failure", None, "defect"))
|
||
if "q2" in (question_ids or []):
|
||
attributions.append(ErrorAttribution("q2", "mixed", None, "lapse"))
|
||
return DiagnosisResult(
|
||
run_id=run_id,
|
||
error_attributions=attributions,
|
||
infra_question_ids=[],
|
||
degraded_question_ids=[],
|
||
)
|
||
|
||
monkeypatch.setattr("app.harness.baseline_diagnosis.run_diagnosis", fake_run_diagnosis)
|
||
return DiagnosisDeps(
|
||
run_log=_FakeRunLog(),
|
||
llm=_FakeLLM(),
|
||
skill_store=_FakeSkillStore(),
|
||
prompts=object(),
|
||
tree_data={},
|
||
concurrency=2,
|
||
)
|
||
|
||
|
||
def _write_md(test_name: str, lines: list[str]) -> Path:
|
||
"""把编排过程写入 tests/outputs 下的 MD（人类可读结构化）。"""
|
||
_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||
path = _OUTPUT_DIR / f"{test_name}_{_FIXED_TS}.md"
|
||
path.write_text("\n".join(lines), encoding="utf-8")
|
||
return path
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_resume_skips_done(tmp_path, monkeypatch):
|
||
"""续跑幂等 + 投影正确：首次全诊断落库，第二次无剩余；tier 投影符合分层。"""
|
||
calls: list[tuple[str, ...]] = []
|
||
deps = _deps(monkeypatch, calls)
|
||
store = SqliteDiagnosisSignalStore(str(tmp_path / "h.db"))
|
||
q_by_id = {"q1": _mk_q("q1"), "q2": _mk_q("q2")}
|
||
|
||
# 第一次：两题都是剩余，全部诊断落库。
|
||
await run_baseline_diagnosis(
|
||
baseline_run_id="infer_adhoc",
|
||
diag_fingerprint="fp",
|
||
wrong_ids=["q1", "q2"],
|
||
questions=q_by_id,
|
||
store=store,
|
||
deps=deps,
|
||
)
|
||
assert store.done_question_ids("infer_adhoc", "fp") == {"q1", "q2"}
|
||
assert calls[0] == ("q1", "q2")
|
||
|
||
# 投影正确性：q1 defect→T2、q2 lapse→T1，evolution_target 由 error_type 派生。
|
||
rows = {r.question_id: r for r in store.load("infer_adhoc", "fp")}
|
||
assert rows["q1"].tier == "T2"
|
||
assert rows["q1"].cause_category == "defect"
|
||
assert rows["q1"].evolution_target == "skill" # search_failure → skill
|
||
assert rows["q1"].error_type == "search_failure"
|
||
assert rows["q1"].infra is False
|
||
assert rows["q1"].degraded is False
|
||
assert rows["q2"].tier == "T1"
|
||
assert rows["q2"].cause_category == "lapse"
|
||
assert rows["q2"].evolution_target == "system" # mixed → system
|
||
|
||
# 第二次：两题已完成 → remaining 为空。编排按规约直接 return（续跑幂等，
|
||
# 不浪费 LLM 诊断调用），故不会向 run_diagnosis 新增调用。
|
||
n_calls_before = len(calls)
|
||
await run_baseline_diagnosis(
|
||
baseline_run_id="infer_adhoc",
|
||
diag_fingerprint="fp",
|
||
wrong_ids=["q1", "q2"],
|
||
questions=q_by_id,
|
||
store=store,
|
||
deps=deps,
|
||
)
|
||
assert len(calls) == n_calls_before # 第二次无剩余 → 未触发诊断
|
||
|
||
md_path = _write_md(
|
||
"test_resume_skips_done",
|
||
[
|
||
"# 离线诊断编排：续跑幂等 + 投影正确",
|
||
"",
|
||
"## 任务描述",
|
||
"对 infer_adhoc 的错题跑离线诊断，投影为逐题信号行并落库；验证续跑幂等。",
|
||
"",
|
||
"## run_diagnosis 每次收到的 question_ids",
|
||
f"- 第 1 次: {calls[0]}",
|
||
"- 第 2 次: 未触发（remaining 为空，编排直接 return）",
|
||
"",
|
||
"## 落库信号投影样例",
|
||
"| question_id | tier | cause_category | error_type | evolution_target |",
|
||
"|---|---|---|---|---|",
|
||
f"| q1 | {rows['q1'].tier} | {rows['q1'].cause_category} | "
|
||
f"{rows['q1'].error_type} | {rows['q1'].evolution_target} |",
|
||
f"| q2 | {rows['q2'].tier} | {rows['q2'].cause_category} | "
|
||
f"{rows['q2'].error_type} | {rows['q2'].evolution_target} |",
|
||
"",
|
||
"## tier 计数",
|
||
"- T2: 1（defect，可训练核心）",
|
||
"- T1: 1（lapse，低信号）",
|
||
"",
|
||
"## 结论",
|
||
"首次两题全落库，第二次 remaining 为空 → 续跑幂等成立；tier/evolution_target 投影正确。",
|
||
],
|
||
)
|
||
assert md_path.exists()
|
||
store.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_infra_and_degraded_projection(tmp_path, monkeypatch):
|
||
"""INFRA→T0、degraded→uncertain 的投影：对应字段置位、error_type/target 为 None。"""
|
||
calls: list[tuple[str, ...]] = []
|
||
|
||
async def fake_run_diagnosis(
|
||
run_id,
|
||
questions,
|
||
tree_data,
|
||
llm,
|
||
run_log,
|
||
skill_store,
|
||
prompts,
|
||
*,
|
||
concurrency,
|
||
question_ids=None,
|
||
task_types=None,
|
||
only_incorrect=False,
|
||
):
|
||
calls.append(tuple(question_ids or []))
|
||
from core.evolution.types import DiagnosisResult
|
||
|
||
return DiagnosisResult(
|
||
run_id=run_id,
|
||
error_attributions=[],
|
||
infra_question_ids=["q3"],
|
||
degraded_question_ids=["q4"],
|
||
)
|
||
|
||
monkeypatch.setattr("app.harness.baseline_diagnosis.run_diagnosis", fake_run_diagnosis)
|
||
deps = DiagnosisDeps(
|
||
run_log=_FakeRunLog(),
|
||
llm=_FakeLLM(),
|
||
skill_store=_FakeSkillStore(),
|
||
prompts=object(),
|
||
tree_data={},
|
||
concurrency=2,
|
||
)
|
||
store = SqliteDiagnosisSignalStore(str(tmp_path / "h.db"))
|
||
q_by_id = {"q3": _mk_q("q3"), "q4": _mk_q("q4")}
|
||
|
||
await run_baseline_diagnosis(
|
||
baseline_run_id="infer_adhoc",
|
||
diag_fingerprint="fp",
|
||
wrong_ids=["q3", "q4"],
|
||
questions=q_by_id,
|
||
store=store,
|
||
deps=deps,
|
||
)
|
||
|
||
rows = {r.question_id: r for r in store.load("infer_adhoc", "fp")}
|
||
assert rows["q3"].tier == "T0"
|
||
assert rows["q3"].infra is True
|
||
assert rows["q3"].error_type is None
|
||
assert rows["q3"].evolution_target is None
|
||
assert rows["q4"].tier == "uncertain"
|
||
assert rows["q4"].degraded is True
|
||
assert rows["q4"].error_type is None
|
||
assert store.done_question_ids("infer_adhoc", "fp") == {"q3", "q4"}
|
||
|
||
md_path = _write_md(
|
||
"test_infra_and_degraded_projection",
|
||
[
|
||
"# 离线诊断编排：INFRA / degraded 投影",
|
||
"",
|
||
"## 落库信号投影样例",
|
||
"| question_id | tier | infra | degraded | error_type | evolution_target |",
|
||
"|---|---|---|---|---|---|",
|
||
f"| q3 | {rows['q3'].tier} | {rows['q3'].infra} | {rows['q3'].degraded} | "
|
||
f"{rows['q3'].error_type} | {rows['q3'].evolution_target} |",
|
||
f"| q4 | {rows['q4'].tier} | {rows['q4'].infra} | {rows['q4'].degraded} | "
|
||
f"{rows['q4'].error_type} | {rows['q4'].evolution_target} |",
|
||
"",
|
||
"## 结论",
|
||
"INFRA→T0（infra 置位）、degraded→uncertain（degraded 置位），error_type/target 均 None。",
|
||
],
|
||
)
|
||
assert md_path.exists()
|
||
store.close()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_degraded_overrides_attribution(tmp_path, monkeypatch):
|
||
"""三桶去重：同题同时在 degraded 与 attributions 时只落 1 行且 tier=uncertain。
|
||
|
||
judge 解析失败会生成 degraded metrics，若仍是错题还会建 attribution，
|
||
故同一 question_id 可同时出现在两桶。按 degraded>infra>attribution 优先级，
|
||
该题必须只落 1 行、tier=uncertain（degraded 置位），attribution 行被跳过。
|
||
"""
|
||
calls: list[tuple[str, ...]] = []
|
||
|
||
async def fake_run_diagnosis(
|
||
run_id,
|
||
questions,
|
||
tree_data,
|
||
llm,
|
||
run_log,
|
||
skill_store,
|
||
prompts,
|
||
*,
|
||
concurrency,
|
||
question_ids=None,
|
||
task_types=None,
|
||
only_incorrect=False,
|
||
):
|
||
calls.append(tuple(question_ids or []))
|
||
from core.evolution.types import DiagnosisResult, ErrorAttribution
|
||
|
||
# q5 同时出现在 attributions（judge 解析失败仍建归因）与 degraded_question_ids。
|
||
return DiagnosisResult(
|
||
run_id=run_id,
|
||
error_attributions=[ErrorAttribution("q5", "reasoning_failure", None, "defect")],
|
||
infra_question_ids=[],
|
||
degraded_question_ids=["q5"],
|
||
)
|
||
|
||
monkeypatch.setattr("app.harness.baseline_diagnosis.run_diagnosis", fake_run_diagnosis)
|
||
deps = DiagnosisDeps(
|
||
run_log=_FakeRunLog(),
|
||
llm=_FakeLLM(),
|
||
skill_store=_FakeSkillStore(),
|
||
prompts=object(),
|
||
tree_data={},
|
||
concurrency=2,
|
||
)
|
||
store = SqliteDiagnosisSignalStore(str(tmp_path / "h.db"))
|
||
q_by_id = {"q5": _mk_q("q5")}
|
||
|
||
await run_baseline_diagnosis(
|
||
baseline_run_id="infer_adhoc",
|
||
diag_fingerprint="fp",
|
||
wrong_ids=["q5"],
|
||
questions=q_by_id,
|
||
store=store,
|
||
deps=deps,
|
||
)
|
||
|
||
all_rows = store.load("infer_adhoc", "fp")
|
||
assert len(all_rows) == 1 # 只落 1 行（无同 PK 双写）
|
||
row = all_rows[0]
|
||
assert row.question_id == "q5"
|
||
assert row.tier == "uncertain" # degraded 优先级最高
|
||
assert row.degraded is True
|
||
assert row.error_type is None # 走 degraded 投影而非 attribution
|
||
assert row.evolution_target is None
|
||
|
||
md_path = _write_md(
|
||
"test_degraded_overrides_attribution",
|
||
[
|
||
"# 离线诊断编排：三桶优先级去重",
|
||
"",
|
||
"## 任务描述",
|
||
"q5 同时出现在 error_attributions（defect）与 degraded_question_ids，"
|
||
"验证按 degraded>infra>attribution 优先级只落 1 行。",
|
||
"",
|
||
"## 落库结果",
|
||
"| question_id | 落库行数 | tier | degraded | error_type |",
|
||
"|---|---|---|---|---|",
|
||
f"| q5 | {len(all_rows)} | {row.tier} | {row.degraded} | {row.error_type} |",
|
||
"",
|
||
"## 结论",
|
||
"degraded 优先级最高，attribution 行被跳过 → 每题恰写一行、counts 恰计一次。",
|
||
],
|
||
)
|
||
assert md_path.exists()
|
||
store.close()
|