fix: route None/degraded diagnoses to lapse; abort on high degrade rate
This commit is contained in:
@@ -1089,6 +1089,13 @@ class Runner:
|
||||
_apply_batch_correctness(state.correctness, log, run_id, batch)
|
||||
|
||||
diagnosis = await self._run_diagnosis(run_id, question_ids=[q.question_id for q in batch])
|
||||
# 降级占比过高疑似 judge 基础设施故障:不以降级信号驱动进化,直接中止
|
||||
n_wrong = sum(1 for q in batch if not state.correctness.get(q.question_id, True))
|
||||
if n_wrong > 0 and diagnosis.degraded_count / n_wrong > 0.5:
|
||||
raise RuntimeError(
|
||||
f"本 step 诊断降级占比 {diagnosis.degraded_count}/{n_wrong} > 50%,"
|
||||
"疑似 judge 基础设施故障,中止训练(不以降级信号驱动进化)。"
|
||||
)
|
||||
_accumulate_slow_packs(diagnosis, state)
|
||||
await self._gate_batch_skills(epoch, step, diagnosis, total_steps, pools, state)
|
||||
# 冷却计数每 step 递减、归零剔除
|
||||
|
||||
@@ -1489,12 +1489,15 @@ def _build_skill_case_packs(
|
||||
if qm.correct:
|
||||
continue
|
||||
attr = attribution_map.get(qm.question_id)
|
||||
if attr is not None and attr.cause_category == "lapse":
|
||||
if attr.lapse_note and attr.lapse_note.strip():
|
||||
# 仅明确 defect 且非 degraded 才进正文进化路径;
|
||||
# lapse / cause_category=None(判别失败)/ degraded(judge 解析失败)一律保守走 lapse,
|
||||
# 不以降级或未判定信号驱动错误进化。
|
||||
is_defect = attr is not None and attr.cause_category == "defect" and not qm.degraded
|
||||
if not is_defect:
|
||||
if attr is not None and attr.lapse_note and attr.lapse_note.strip():
|
||||
lapse_notes.append(attr.lapse_note)
|
||||
continue
|
||||
et = attr.error_type if attr else "mixed"
|
||||
wrong_by_error[et].append(qm)
|
||||
wrong_by_error[attr.error_type].append(qm)
|
||||
|
||||
# 单条 fallback
|
||||
n_body_failures = sum(len(group) for group in wrong_by_error.values())
|
||||
|
||||
@@ -17,6 +17,7 @@ from unittest.mock import AsyncMock, MagicMock
|
||||
import pytest
|
||||
|
||||
from core.evolution.diagnose import (
|
||||
_build_skill_case_packs,
|
||||
_percentile,
|
||||
_trigrams,
|
||||
aggregate_d2,
|
||||
@@ -43,6 +44,7 @@ from core.evolution.types import (
|
||||
CaseSample,
|
||||
DiagnosePrompts,
|
||||
DiagnosisResult,
|
||||
ErrorAttribution,
|
||||
QuestionMetrics,
|
||||
SkillStepAdherence,
|
||||
SpanMetrics,
|
||||
@@ -771,3 +773,52 @@ class TestRunDiagnosis:
|
||||
assert result.run_id == "run1"
|
||||
assert result.error_attributions == []
|
||||
assert result.degraded_count == 0
|
||||
|
||||
|
||||
class TestBuildSkillCasePacksDegradeRouting:
|
||||
"""_build_skill_case_packs 降级/未判定分流:仅 defect 且非 degraded 进正文路径。"""
|
||||
|
||||
def test_none_cause_and_degraded_route_to_lapse(self) -> None:
|
||||
"""cause_category=None(判别失败)与 degraded 题按 lapse 处置,不进 defect 正文路径。"""
|
||||
tt = "Action Reasoning"
|
||||
qm_none = _make_qm(question_id="q-none", task_type=tt, correct=False, degraded=False)
|
||||
qm_degraded = _make_qm(question_id="q-deg", task_type=tt, correct=False, degraded=True)
|
||||
# 补两道正确题,避免退化
|
||||
qm_ok1 = _make_qm(question_id="q-ok1", task_type=tt, correct=True)
|
||||
qm_ok2 = _make_qm(question_id="q-ok2", task_type=tt, correct=True)
|
||||
metrics = [qm_none, qm_degraded, qm_ok1, qm_ok2]
|
||||
|
||||
attributions = [
|
||||
# 判别失败:cause_category=None
|
||||
ErrorAttribution(
|
||||
question_id="q-none",
|
||||
error_type="reasoning",
|
||||
reasoning_failure_type=None,
|
||||
cause_category=None,
|
||||
lapse_note="复核该类推理规则",
|
||||
),
|
||||
# degraded 题即便被判 defect,也须走 lapse(不驱动正文进化)
|
||||
ErrorAttribution(
|
||||
question_id="q-deg",
|
||||
error_type="reasoning",
|
||||
reasoning_failure_type=None,
|
||||
cause_category="defect",
|
||||
lapse_note=None,
|
||||
),
|
||||
]
|
||||
|
||||
packs = _build_skill_case_packs(
|
||||
all_metrics=metrics,
|
||||
error_attributions=attributions,
|
||||
traces_by_question={},
|
||||
predictions=[],
|
||||
d3_stats={},
|
||||
d4_stats={},
|
||||
)
|
||||
|
||||
pack = packs[tt]
|
||||
failure_ids = {c.question_id for c in pack.failure_cases}
|
||||
assert "q-none" not in failure_ids
|
||||
assert "q-deg" not in failure_ids
|
||||
# None-cause 的 lapse_note 应被收进 lapse_notes
|
||||
assert any("复核该类推理规则" in n for n in pack.lapse_notes)
|
||||
|
||||
@@ -212,6 +212,32 @@ async def test_diagnosis_reads_traces_from_steps_json(
|
||||
assert traces[0]["tool_name"] == "search_tree"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_step_aborts_on_high_degrade_rate(
|
||||
runner_with_real_store: Runner, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""诊断降级占比 > 50% 时 _run_step 中止(疑似 judge 基础设施故障)。"""
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
batch = [_fake_question("q1", "vA"), _fake_question("q2", "vA")]
|
||||
|
||||
runner_with_real_store._rollout_batch = AsyncMock()
|
||||
monkeypatch.setattr("app.harness.runner._apply_batch_correctness", lambda *a, **k: None)
|
||||
runner_with_real_store._run_diagnosis = AsyncMock(
|
||||
return_value=DiagnosisResult(run_id="r", degraded_count=2)
|
||||
)
|
||||
runner_with_real_store._gate_batch_skills = AsyncMock()
|
||||
|
||||
state = MagicMock()
|
||||
state.correctness = {"q1": False, "q2": False}
|
||||
state.gate_cooldown = {}
|
||||
pools = MagicMock()
|
||||
pools.baseline_run_id = "infer_adhoc"
|
||||
|
||||
with pytest.raises(RuntimeError, match="降级占比"):
|
||||
await runner_with_real_store._run_step(1, 0, 10, batch, pools, state)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_diagnosis_full_scan_loads_all_video_trees(
|
||||
runner_with_real_store: Runner,
|
||||
|
||||
Reference in New Issue
Block a user