fix(question_gen): gates reject_reason returns raw reason; short-circuit skips LLM
- GateReport.reject_reason now returns gate.reason directly (no [name] prefix) - verbatim short-circuit sets other 3 gates to SKIP without calling LLM - test_high_verbatim_shortcircuits asserts zero LLM calls and SKIP verdicts Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
+15
-22
@@ -131,14 +131,9 @@ class GateReport:
|
|||||||
@property
|
@property
|
||||||
def reject_reason(self) -> str | None:
|
def reject_reason(self) -> str | None:
|
||||||
"""首个 FAIL 门的 reason,全部通过返回 None。"""
|
"""首个 FAIL 门的 reason,全部通过返回 None。"""
|
||||||
for name, gate in [
|
for gate in (self.key_verify, self.blind_answer, self.multi_true, self.leak_test):
|
||||||
("key_verify", self.key_verify),
|
|
||||||
("blind_answer", self.blind_answer),
|
|
||||||
("multi_true", self.multi_true),
|
|
||||||
("leak_test", self.leak_test),
|
|
||||||
]:
|
|
||||||
if gate.verdict == GateVerdict.FAIL:
|
if gate.verdict == GateVerdict.FAIL:
|
||||||
return f"[{name}] {gate.reason}"
|
return gate.reason
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
@@ -419,29 +414,27 @@ async def run_gates(
|
|||||||
返回:
|
返回:
|
||||||
GateReport 四门汇总。
|
GateReport 四门汇总。
|
||||||
"""
|
"""
|
||||||
# Phase 1: verbatim 前置短路
|
# Phase 1: verbatim 前置短路 — 不调 LLM,直接返回
|
||||||
if postprocess.verbatim_ratio > _VERBATIM_THRESHOLD:
|
if postprocess.verbatim_ratio > _VERBATIM_THRESHOLD:
|
||||||
logger.info(
|
logger.info(
|
||||||
"verbatim_ratio={:.3f} > {:.1f},key_verify 短路 FAIL",
|
"verbatim_ratio={:.3f} > {:.1f},key_verify 短路 FAIL,其余三门 SKIP",
|
||||||
postprocess.verbatim_ratio,
|
postprocess.verbatim_ratio,
|
||||||
_VERBATIM_THRESHOLD,
|
_VERBATIM_THRESHOLD,
|
||||||
)
|
)
|
||||||
key_verify_result = GateResult(
|
skip_result = GateResult(
|
||||||
|
verdict=GateVerdict.SKIP,
|
||||||
|
reason="skipped due to verbatim short-circuit",
|
||||||
|
raw_response="",
|
||||||
|
)
|
||||||
|
return GateReport(
|
||||||
|
key_verify=GateResult(
|
||||||
verdict=GateVerdict.FAIL,
|
verdict=GateVerdict.FAIL,
|
||||||
reason=f"verbatim_ratio={postprocess.verbatim_ratio:.3f} exceeds threshold {_VERBATIM_THRESHOLD}",
|
reason=f"verbatim_ratio={postprocess.verbatim_ratio:.3f} exceeds threshold {_VERBATIM_THRESHOLD}",
|
||||||
raw_response="",
|
raw_response="",
|
||||||
)
|
),
|
||||||
# 其余三门仍然并发执行(收集完整诊断信息)
|
blind_answer=skip_result,
|
||||||
blind_result, multi_result, leak_result = await asyncio.gather(
|
multi_true=skip_result,
|
||||||
_gate_blind_answer(candidate, llm, session_id=session_id),
|
leak_test=skip_result,
|
||||||
_gate_multi_true(candidate, tree, llm, session_id=session_id),
|
|
||||||
_gate_leak_test(candidate, family_spec, llm, session_id=session_id),
|
|
||||||
)
|
|
||||||
return GateReport(
|
|
||||||
key_verify=key_verify_result,
|
|
||||||
blind_answer=blind_result,
|
|
||||||
multi_true=multi_result,
|
|
||||||
leak_test=leak_result,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Phase 2: 四门并发执行
|
# Phase 2: 四门并发执行
|
||||||
|
|||||||
@@ -293,7 +293,7 @@ class TestRunGates:
|
|||||||
|
|
||||||
@pytest.mark.asyncio()
|
@pytest.mark.asyncio()
|
||||||
async def test_high_verbatim_shortcircuits(self) -> None:
|
async def test_high_verbatim_shortcircuits(self) -> None:
|
||||||
"""verbatim_ratio > 0.5 → key_verify 直接 FAIL,不调用 LLM。"""
|
"""verbatim_ratio > 0.5 → key_verify 直接 FAIL,其余 SKIP,不调用 LLM。"""
|
||||||
llm = MockLLM([_make_llm_response("pass", "should not be called")])
|
llm = MockLLM([_make_llm_response("pass", "should not be called")])
|
||||||
candidate = _make_candidate()
|
candidate = _make_candidate()
|
||||||
tree = _make_tree()
|
tree = _make_tree()
|
||||||
@@ -310,3 +310,9 @@ class TestRunGates:
|
|||||||
assert report.passed is False
|
assert report.passed is False
|
||||||
assert report.key_verify.verdict == GateVerdict.FAIL
|
assert report.key_verify.verdict == GateVerdict.FAIL
|
||||||
assert "verbatim" in report.key_verify.reason.lower()
|
assert "verbatim" in report.key_verify.reason.lower()
|
||||||
|
# 其余三门应为 SKIP
|
||||||
|
assert report.blind_answer.verdict == GateVerdict.SKIP
|
||||||
|
assert report.multi_true.verdict == GateVerdict.SKIP
|
||||||
|
assert report.leak_test.verdict == GateVerdict.SKIP
|
||||||
|
# LLM 不应被调用
|
||||||
|
assert len(llm.calls) == 0
|
||||||
|
|||||||
Reference in New Issue
Block a user