fix(question_gen): gates reject_reason returns raw reason; short-circuit skips LLM
- GateReport.reject_reason now returns gate.reason directly (no [name] prefix) - verbatim short-circuit sets other 3 gates to SKIP without calling LLM - test_high_verbatim_shortcircuits asserts zero LLM calls and SKIP verdicts Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
+15
-22
@@ -131,14 +131,9 @@ class GateReport:
|
||||
@property
|
||||
def reject_reason(self) -> str | None:
|
||||
"""首个 FAIL 门的 reason,全部通过返回 None。"""
|
||||
for name, gate in [
|
||||
("key_verify", self.key_verify),
|
||||
("blind_answer", self.blind_answer),
|
||||
("multi_true", self.multi_true),
|
||||
("leak_test", self.leak_test),
|
||||
]:
|
||||
for gate in (self.key_verify, self.blind_answer, self.multi_true, self.leak_test):
|
||||
if gate.verdict == GateVerdict.FAIL:
|
||||
return f"[{name}] {gate.reason}"
|
||||
return gate.reason
|
||||
return None
|
||||
|
||||
|
||||
@@ -419,29 +414,27 @@ async def run_gates(
|
||||
返回:
|
||||
GateReport 四门汇总。
|
||||
"""
|
||||
# Phase 1: verbatim 前置短路
|
||||
# Phase 1: verbatim 前置短路 — 不调 LLM,直接返回
|
||||
if postprocess.verbatim_ratio > _VERBATIM_THRESHOLD:
|
||||
logger.info(
|
||||
"verbatim_ratio={:.3f} > {:.1f},key_verify 短路 FAIL",
|
||||
"verbatim_ratio={:.3f} > {:.1f},key_verify 短路 FAIL,其余三门 SKIP",
|
||||
postprocess.verbatim_ratio,
|
||||
_VERBATIM_THRESHOLD,
|
||||
)
|
||||
key_verify_result = GateResult(
|
||||
verdict=GateVerdict.FAIL,
|
||||
reason=f"verbatim_ratio={postprocess.verbatim_ratio:.3f} exceeds threshold {_VERBATIM_THRESHOLD}",
|
||||
skip_result = GateResult(
|
||||
verdict=GateVerdict.SKIP,
|
||||
reason="skipped due to verbatim short-circuit",
|
||||
raw_response="",
|
||||
)
|
||||
# 其余三门仍然并发执行(收集完整诊断信息)
|
||||
blind_result, multi_result, leak_result = await asyncio.gather(
|
||||
_gate_blind_answer(candidate, llm, session_id=session_id),
|
||||
_gate_multi_true(candidate, tree, llm, session_id=session_id),
|
||||
_gate_leak_test(candidate, family_spec, llm, session_id=session_id),
|
||||
)
|
||||
return GateReport(
|
||||
key_verify=key_verify_result,
|
||||
blind_answer=blind_result,
|
||||
multi_true=multi_result,
|
||||
leak_test=leak_result,
|
||||
key_verify=GateResult(
|
||||
verdict=GateVerdict.FAIL,
|
||||
reason=f"verbatim_ratio={postprocess.verbatim_ratio:.3f} exceeds threshold {_VERBATIM_THRESHOLD}",
|
||||
raw_response="",
|
||||
),
|
||||
blind_answer=skip_result,
|
||||
multi_true=skip_result,
|
||||
leak_test=skip_result,
|
||||
)
|
||||
|
||||
# Phase 2: 四门并发执行
|
||||
|
||||
@@ -293,7 +293,7 @@ class TestRunGates:
|
||||
|
||||
@pytest.mark.asyncio()
|
||||
async def test_high_verbatim_shortcircuits(self) -> None:
|
||||
"""verbatim_ratio > 0.5 → key_verify 直接 FAIL,不调用 LLM。"""
|
||||
"""verbatim_ratio > 0.5 → key_verify 直接 FAIL,其余 SKIP,不调用 LLM。"""
|
||||
llm = MockLLM([_make_llm_response("pass", "should not be called")])
|
||||
candidate = _make_candidate()
|
||||
tree = _make_tree()
|
||||
@@ -310,3 +310,9 @@ class TestRunGates:
|
||||
assert report.passed is False
|
||||
assert report.key_verify.verdict == GateVerdict.FAIL
|
||||
assert "verbatim" in report.key_verify.reason.lower()
|
||||
# 其余三门应为 SKIP
|
||||
assert report.blind_answer.verdict == GateVerdict.SKIP
|
||||
assert report.multi_true.verdict == GateVerdict.SKIP
|
||||
assert report.leak_test.verdict == GateVerdict.SKIP
|
||||
# LLM 不应被调用
|
||||
assert len(llm.calls) == 0
|
||||
|
||||
Reference in New Issue
Block a user