From 76f719018c24f3ef7fea2466e36d975255916a7e Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 14 Jul 2026 14:33:22 -0400 Subject: [PATCH] feat: loosen multi_true gate to qualifier-scoped correctness --- store/prompts/question_gen/gate_multi_true.md | 15 +++++++++++---- tests/unit/test_gate_multi_true_rubric.py | 13 +++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_gate_multi_true_rubric.py diff --git a/store/prompts/question_gen/gate_multi_true.md b/store/prompts/question_gen/gate_multi_true.md index e020e60..a36927b 100644 --- a/store/prompts/question_gen/gate_multi_true.md +++ b/store/prompts/question_gen/gate_multi_true.md @@ -16,10 +16,17 @@ You are checking whether a multiple-choice question has **more than one plausibl ## Instructions -1. Read the source material and the question carefully. -2. For each option, assess whether it could be considered a correct or plausible answer given the source. -3. If exactly ONE option is clearly correct → verdict "pass". -4. If TWO or more options are plausibly correct → verdict "fail". +1. Read the source material and the question carefully. Note the question's + explicit qualifiers (subject, timing, manner, object). +2. For each option, assess whether it is **fully correct under the question's + qualifiers** — not merely whether it has some partial or local support in + the source. +3. A good hard-negative distractor MAY have partial/local evidence in the video + yet be FALSE under the question's constraints. Such an option is NOT a + second correct answer — do NOT fail the question for it. +4. Verdict "fail" ONLY IF two or more options are each fully correct under the + question's qualifiers (a genuine ambiguity). +5. Otherwise verdict "pass". ## Response Format (strict JSON) diff --git a/tests/unit/test_gate_multi_true_rubric.py b/tests/unit/test_gate_multi_true_rubric.py new file mode 100644 index 0000000..5d0aa50 --- /dev/null +++ b/tests/unit/test_gate_multi_true_rubric.py @@ -0,0 +1,13 @@ +"""multi_true rubric 已松绑为题干限定判据。""" + +from pathlib import Path + +_PROMPT = Path(__file__).resolve().parents[2] / "store" / "prompts" / "question_gen" / "gate_multi_true.md" + + +def test_rubric_uses_qualifier_constraint(): + text = _PROMPT.read_text(encoding="utf-8") + # 新 rubric 必须提到"题干限定下同时为真才 fail"（qualifier / under the question's constraints） + assert "under the question" in text.lower() or "qualifier" in text.lower() + # 必须显式放行"有局部真实证据但在限定下为假"的干扰项 + assert "partial" in text.lower() or "locally" in text.lower()