fix: gate INFRA isolation edge cases (all-INFRA fail-loud, parse_error in guard)

2026-07-16 05:54:02 -04:00
parent b307f51340
commit 03337af8f8
3 changed files with 159 additions and 9 deletions
@@ -575,6 +575,144 @@ async def test_baseline_infra_error_not_cached(tmp_path: Path) -> None:
        log.close()


+@pytest.mark.asyncio
+async def test_all_infra_ladder_raises_clear_error(tmp_path: Path) -> None:
+    """整个阶梯所有 unit 都被判为 INFRA 排除 → 明确 RuntimeError（非误导性空阶梯断言）。"""
+    workspace = _setup_workspace(tmp_path)
+    log = _make_log(workspace)
+    questions = _make_questions(4)
+    cache = BaselineCache(workspace / "baseline_cache.json")
+
+    candidate_calls: list[str] = []
+
+    async def mock_fn(qs, *, run_id, skills_dir):
+        if run_id.endswith("_cand"):
+            candidate_calls.append(run_id)
+        # 基线臂逐题全部 INFRA error（候选臂在修复后不应被空跑）
+        for q in qs:
+            log.insert(
+                "predictions",
+                {
+                    "run_id": run_id,
+                    "video_id": "v0",
+                    "question_id": q.question_id,
+                    "task_type": "temporal",
+                    "prediction": "",
+                    "answer": "A",
+                    "evidence": "",
+                    "reasoning": "",
+                    "steps_used": 1,
+                    "prompt_tokens": 10,
+                    "completion_tokens": 10,
+                    "stop_reason": "error",
+                    "steps_json": "[]",
+                },
+            )
+        total = len(qs)
+        return InferenceResult(
+            run_id=run_id,
+            accuracy=0.0,
+            total=total,
+            correct=0,
+            per_task_type={},
+            steps_mean=1.0,
+            token_usage={"prompt_tokens": 10, "completion_tokens": 10},
+            stop_reason_counts={"error": total},
+        )
+
+    try:
+        with pytest.raises(RuntimeError, match="INFRA"):
+            await validate_skill_local(
+                workspace_dir=workspace,
+                base_skills_version="v1",
+                task_type="temporal",
+                target_file="temporal.md",
+                candidate_content="content",
+                base_skill_content="baseline skill content",
+                ladder_items=questions,
+                gate_params=_DEFAULT_GATE_PARAMS,
+                gate_block=4,
+                gate_n_max=20,
+                gate_guard_err=0.9,  # 高阈值：4 题 <10 分母不触发错误率护栏
+                baseline_cache=cache,
+                prompts_version="p1",
+                run_inference=mock_fn,
+                log=log,
+                gate_run_prefix="step1_gate_test",
+            )
+        # 全 INFRA 块不应触发候选空跑
+        assert candidate_calls == []
+    finally:
+        log.close()
+
+
+@pytest.mark.asyncio
+async def test_parse_error_counts_toward_guard(tmp_path: Path) -> None:
+    """stop_reason=parse_error 也计入护栏错误率（与 INFRA 判定口径一致）→ 超阈值熔断。"""
+    workspace = _setup_workspace(tmp_path)
+    log = _make_log(workspace)
+    questions = _make_questions(6)
+    cache = BaselineCache(workspace / "baseline_cache.json")
+
+    async def mock_fn(qs, *, run_id, skills_dir):
+        # 逐题 stop_reason 保持 completed（不触发 per-unit INFRA 排除），
+        # 但汇总 stop_reason_counts 报大量 parse_error（应计入护栏）。
+        for q in qs:
+            log.insert(
+                "predictions",
+                {
+                    "run_id": run_id,
+                    "video_id": "v0",
+                    "question_id": q.question_id,
+                    "task_type": "temporal",
+                    "prediction": "Z",
+                    "answer": "A",
+                    "evidence": "",
+                    "reasoning": "",
+                    "steps_used": 1,
+                    "prompt_tokens": 10,
+                    "completion_tokens": 10,
+                    "stop_reason": "completed",
+                    "steps_json": "[]",
+                },
+            )
+        total = len(qs)
+        # 两臂各 5 个 parse_error → 累计 10/12 > 0.5 触发护栏
+        return InferenceResult(
+            run_id=run_id,
+            accuracy=0.0,
+            total=total,
+            correct=0,
+            per_task_type={},
+            steps_mean=1.0,
+            token_usage={"prompt_tokens": 10, "completion_tokens": 10},
+            stop_reason_counts={"completed": 1, "parse_error": 5},
+        )
+
+    try:
+        with pytest.raises(RuntimeError, match="错误率过高"):
+            await validate_skill_local(
+                workspace_dir=workspace,
+                base_skills_version="v1",
+                task_type="temporal",
+                target_file="temporal.md",
+                candidate_content="content",
+                base_skill_content="baseline skill content",
+                ladder_items=questions,
+                gate_params=_DEFAULT_GATE_PARAMS,
+                gate_block=6,
+                gate_n_max=20,
+                gate_guard_err=0.5,
+                baseline_cache=cache,
+                prompts_version="p1",
+                run_inference=mock_fn,
+                log=log,
+                gate_run_prefix="step1_gate_test",
+            )
+    finally:
+        log.close()
+
+
@pytest.mark.asyncio
 async def test_last_block_terminal(tmp_path: Path) -> None:
    """单块 + n_remaining=0 → 终态判定（provisional 或 inertia），非 continue。"""