fix: route None/degraded diagnoses to lapse; abort on high degrade rate

This commit is contained in:
2026-07-16 06:24:11 -04:00
parent 65126feada
commit efbdeb1647
4 changed files with 91 additions and 4 deletions
+51
View File
@@ -17,6 +17,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from core.evolution.diagnose import (
_build_skill_case_packs,
_percentile,
_trigrams,
aggregate_d2,
@@ -43,6 +44,7 @@ from core.evolution.types import (
CaseSample,
DiagnosePrompts,
DiagnosisResult,
ErrorAttribution,
QuestionMetrics,
SkillStepAdherence,
SpanMetrics,
@@ -771,3 +773,52 @@ class TestRunDiagnosis:
assert result.run_id == "run1"
assert result.error_attributions == []
assert result.degraded_count == 0
class TestBuildSkillCasePacksDegradeRouting:
"""_build_skill_case_packs 降级/未判定分流:仅 defect 且非 degraded 进正文路径。"""
def test_none_cause_and_degraded_route_to_lapse(self) -> None:
"""cause_category=None(判别失败)与 degraded 题按 lapse 处置,不进 defect 正文路径。"""
tt = "Action Reasoning"
qm_none = _make_qm(question_id="q-none", task_type=tt, correct=False, degraded=False)
qm_degraded = _make_qm(question_id="q-deg", task_type=tt, correct=False, degraded=True)
# 补两道正确题,避免退化
qm_ok1 = _make_qm(question_id="q-ok1", task_type=tt, correct=True)
qm_ok2 = _make_qm(question_id="q-ok2", task_type=tt, correct=True)
metrics = [qm_none, qm_degraded, qm_ok1, qm_ok2]
attributions = [
# 判别失败:cause_category=None
ErrorAttribution(
question_id="q-none",
error_type="reasoning",
reasoning_failure_type=None,
cause_category=None,
lapse_note="复核该类推理规则",
),
# degraded 题即便被判 defect,也须走 lapse(不驱动正文进化)
ErrorAttribution(
question_id="q-deg",
error_type="reasoning",
reasoning_failure_type=None,
cause_category="defect",
lapse_note=None,
),
]
packs = _build_skill_case_packs(
all_metrics=metrics,
error_attributions=attributions,
traces_by_question={},
predictions=[],
d3_stats={},
d4_stats={},
)
pack = packs[tt]
failure_ids = {c.question_id for c in pack.failure_cases}
assert "q-none" not in failure_ids
assert "q-deg" not in failure_ids
# None-cause 的 lapse_note 应被收进 lapse_notes
assert any("复核该类推理规则" in n for n in pack.lapse_notes)
@@ -212,6 +212,32 @@ async def test_diagnosis_reads_traces_from_steps_json(
assert traces[0]["tool_name"] == "search_tree"
@pytest.mark.asyncio
async def test_run_step_aborts_on_high_degrade_rate(
runner_with_real_store: Runner, monkeypatch: pytest.MonkeyPatch
) -> None:
"""诊断降级占比 > 50% 时 _run_step 中止(疑似 judge 基础设施故障)。"""
from unittest.mock import AsyncMock
batch = [_fake_question("q1", "vA"), _fake_question("q2", "vA")]
runner_with_real_store._rollout_batch = AsyncMock()
monkeypatch.setattr("app.harness.runner._apply_batch_correctness", lambda *a, **k: None)
runner_with_real_store._run_diagnosis = AsyncMock(
return_value=DiagnosisResult(run_id="r", degraded_count=2)
)
runner_with_real_store._gate_batch_skills = AsyncMock()
state = MagicMock()
state.correctness = {"q1": False, "q2": False}
state.gate_cooldown = {}
pools = MagicMock()
pools.baseline_run_id = "infer_adhoc"
with pytest.raises(RuntimeError, match="降级占比"):
await runner_with_real_store._run_step(1, 0, 10, batch, pools, state)
@pytest.mark.asyncio
async def test_run_diagnosis_full_scan_loads_all_video_trees(
runner_with_real_store: Runner,