From efbdeb1647571f1ef77acac71b8086af42b4290b Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 16 Jul 2026 06:24:11 -0400 Subject: [PATCH] fix: route None/degraded diagnoses to lapse; abort on high degrade rate --- app/harness/runner.py | 7 +++ core/evolution/diagnose.py | 11 +++-- tests/unit/test_diagnose.py | 51 ++++++++++++++++++++++ tests/unit/test_runner_diag_tree_inject.py | 26 +++++++++++ 4 files changed, 91 insertions(+), 4 deletions(-) diff --git a/app/harness/runner.py b/app/harness/runner.py index ea6ebc1..8f05956 100644 --- a/app/harness/runner.py +++ b/app/harness/runner.py @@ -1089,6 +1089,13 @@ class Runner: _apply_batch_correctness(state.correctness, log, run_id, batch) diagnosis = await self._run_diagnosis(run_id, question_ids=[q.question_id for q in batch]) + # 降级占比过高疑似 judge 基础设施故障：不以降级信号驱动进化，直接中止 + n_wrong = sum(1 for q in batch if not state.correctness.get(q.question_id, True)) + if n_wrong > 0 and diagnosis.degraded_count / n_wrong > 0.5: + raise RuntimeError( + f"本 step 诊断降级占比 {diagnosis.degraded_count}/{n_wrong} > 50%，" + "疑似 judge 基础设施故障，中止训练（不以降级信号驱动进化）。" + ) _accumulate_slow_packs(diagnosis, state) await self._gate_batch_skills(epoch, step, diagnosis, total_steps, pools, state) # 冷却计数每 step 递减、归零剔除 diff --git a/core/evolution/diagnose.py b/core/evolution/diagnose.py index 71315ff..d100aae 100644 --- a/core/evolution/diagnose.py +++ b/core/evolution/diagnose.py @@ -1489,12 +1489,15 @@ def _build_skill_case_packs( if qm.correct: continue attr = attribution_map.get(qm.question_id) - if attr is not None and attr.cause_category == "lapse": - if attr.lapse_note and attr.lapse_note.strip(): + # 仅明确 defect 且非 degraded 才进正文进化路径； + # lapse / cause_category=None（判别失败）/ degraded（judge 解析失败）一律保守走 lapse， + # 不以降级或未判定信号驱动错误进化。 + is_defect = attr is not None and attr.cause_category == "defect" and not qm.degraded + if not is_defect: + if attr is not None and attr.lapse_note and attr.lapse_note.strip(): lapse_notes.append(attr.lapse_note) continue - et = attr.error_type if attr else "mixed" - wrong_by_error[et].append(qm) + wrong_by_error[attr.error_type].append(qm) # 单条 fallback n_body_failures = sum(len(group) for group in wrong_by_error.values()) diff --git a/tests/unit/test_diagnose.py b/tests/unit/test_diagnose.py index db05bf8..ea79a6c 100644 --- a/tests/unit/test_diagnose.py +++ b/tests/unit/test_diagnose.py @@ -17,6 +17,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest from core.evolution.diagnose import ( + _build_skill_case_packs, _percentile, _trigrams, aggregate_d2, @@ -43,6 +44,7 @@ from core.evolution.types import ( CaseSample, DiagnosePrompts, DiagnosisResult, + ErrorAttribution, QuestionMetrics, SkillStepAdherence, SpanMetrics, @@ -771,3 +773,52 @@ class TestRunDiagnosis: assert result.run_id == "run1" assert result.error_attributions == [] assert result.degraded_count == 0 + + +class TestBuildSkillCasePacksDegradeRouting: + """_build_skill_case_packs 降级/未判定分流：仅 defect 且非 degraded 进正文路径。""" + + def test_none_cause_and_degraded_route_to_lapse(self) -> None: + """cause_category=None（判别失败）与 degraded 题按 lapse 处置，不进 defect 正文路径。""" + tt = "Action Reasoning" + qm_none = _make_qm(question_id="q-none", task_type=tt, correct=False, degraded=False) + qm_degraded = _make_qm(question_id="q-deg", task_type=tt, correct=False, degraded=True) + # 补两道正确题，避免退化 + qm_ok1 = _make_qm(question_id="q-ok1", task_type=tt, correct=True) + qm_ok2 = _make_qm(question_id="q-ok2", task_type=tt, correct=True) + metrics = [qm_none, qm_degraded, qm_ok1, qm_ok2] + + attributions = [ + # 判别失败：cause_category=None + ErrorAttribution( + question_id="q-none", + error_type="reasoning", + reasoning_failure_type=None, + cause_category=None, + lapse_note="复核该类推理规则", + ), + # degraded 题即便被判 defect，也须走 lapse（不驱动正文进化） + ErrorAttribution( + question_id="q-deg", + error_type="reasoning", + reasoning_failure_type=None, + cause_category="defect", + lapse_note=None, + ), + ] + + packs = _build_skill_case_packs( + all_metrics=metrics, + error_attributions=attributions, + traces_by_question={}, + predictions=[], + d3_stats={}, + d4_stats={}, + ) + + pack = packs[tt] + failure_ids = {c.question_id for c in pack.failure_cases} + assert "q-none" not in failure_ids + assert "q-deg" not in failure_ids + # None-cause 的 lapse_note 应被收进 lapse_notes + assert any("复核该类推理规则" in n for n in pack.lapse_notes) diff --git a/tests/unit/test_runner_diag_tree_inject.py b/tests/unit/test_runner_diag_tree_inject.py index 43dbab3..5c9a05b 100644 --- a/tests/unit/test_runner_diag_tree_inject.py +++ b/tests/unit/test_runner_diag_tree_inject.py @@ -212,6 +212,32 @@ async def test_diagnosis_reads_traces_from_steps_json( assert traces[0]["tool_name"] == "search_tree" +@pytest.mark.asyncio +async def test_run_step_aborts_on_high_degrade_rate( + runner_with_real_store: Runner, monkeypatch: pytest.MonkeyPatch +) -> None: + """诊断降级占比 > 50% 时 _run_step 中止（疑似 judge 基础设施故障）。""" + from unittest.mock import AsyncMock + + batch = [_fake_question("q1", "vA"), _fake_question("q2", "vA")] + + runner_with_real_store._rollout_batch = AsyncMock() + monkeypatch.setattr("app.harness.runner._apply_batch_correctness", lambda *a, **k: None) + runner_with_real_store._run_diagnosis = AsyncMock( + return_value=DiagnosisResult(run_id="r", degraded_count=2) + ) + runner_with_real_store._gate_batch_skills = AsyncMock() + + state = MagicMock() + state.correctness = {"q1": False, "q2": False} + state.gate_cooldown = {} + pools = MagicMock() + pools.baseline_run_id = "infer_adhoc" + + with pytest.raises(RuntimeError, match="降级占比"): + await runner_with_real_store._run_step(1, 0, 10, batch, pools, state) + + @pytest.mark.asyncio async def test_run_diagnosis_full_scan_loads_all_video_trees( runner_with_real_store: Runner,