From 8a54055d028d7bad152cb9ac2b8b4e4cbc4c13a2 Mon Sep 17 00:00:00 2001
From: iomgaa <iomgaaycz@gmail.com>
Date: Tue, 14 Jul 2026 14:06:07 -0400
Subject: [PATCH] feat: add grounded distractor selector with visual scoring

---
 app/question_gen/distractor_selector.py       | 270 ++++++++++++++++++
 .../question_gen/ar_distractor_pool.md        |  23 ++
 .../question_gen/ar_distractor_score.md       |  23 ++
 tests/unit/test_distractor_selector.py        |  93 ++++++
 4 files changed, 409 insertions(+)
 create mode 100644 app/question_gen/distractor_selector.py
 create mode 100644 store/prompts/question_gen/ar_distractor_pool.md
 create mode 100644 store/prompts/question_gen/ar_distractor_score.md
 create mode 100644 tests/unit/test_distractor_selector.py

diff --git a/app/question_gen/distractor_selector.py b/app/question_gen/distractor_selector.py
new file mode 100644
index 0000000..6e3bf2a
--- /dev/null
+++ b/app/question_gen/distractor_selector.py
@@ -0,0 +1,270 @@
+"""Grounded 干扰项 selector — 候选池 + VLM 视觉打分 + 区间选择（仅 AR 路径）。
+
+把干扰项从"VLM 主观写得像"下沉到机制层：VLM 生成 N 个候选干扰项，再对
+候选 + 正解逐一打"视觉可信度"分，按 [正解分-δ_high, 正解分-δ_low] 区间
+选 3 个 grounded near-miss，从机制上消灭 Easy-Options Bias。
+
+设计: research-wiki/designs/2026-07-14-grounded-question-gen-phaseA-design.md §3
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from json_repair import repair_json
+from loguru import logger
+
+if TYPE_CHECKING:
+    from app.question_gen.sampler_v2 import MaterialContext
+    from core.protocols import VLMProvider
+
+_PROMPTS_DIR = Path(__file__).resolve().parent.parent.parent / "store" / "prompts" / "question_gen"
+
+
+@dataclass(frozen=True)
+class SelectorConfig:
+    """selector 科研参数。
+
+    属性:
+        candidate_pool_size: 首轮候选干扰项数 N。
+        delta_low: 干扰项视觉分与正解的最小差（上界，太近=真歧义）。
+        delta_high: 干扰项视觉分与正解的最大差（下界，太低=负空间）。
+        max_delta_relax: δ_high 放宽次数上限（退火）。
+        delta_relax_step: 每次放宽 δ_high 的增量。
+    """
+
+    candidate_pool_size: int
+    delta_low: float
+    delta_high: float
+    max_delta_relax: int = 2
+    delta_relax_step: float = 0.1
+
+
+@dataclass(frozen=True)
+class SelectorOutcome:
+    """selector 产出。observation 始终存在（含 hard-fail），供 run_store 落库。
+
+    属性:
+        observation: 打分观测 dict（correct_score/chosen/pool_size/anneal_rounds/hard_fail）。
+        options: 重组四选项（A=正解），hard-fail 时为 None。
+        answer: 正解字母（恒 "A"），hard-fail 时为 None。
+    """
+
+    observation: dict
+    options: tuple[str, ...] | None = None
+    answer: str | None = None
+
+    @property
+    def hard_fail(self) -> bool:
+        """是否硬失败（凑不齐 3 个 grounded 干扰项）。"""
+        return self.options is None
+
+
+def _select_in_interval(
+    correct_score: float,
+    candidates: list[str],
+    candidate_scores: list[float],
+    delta_low: float,
+    delta_high: float,
+) -> list[str] | None:
+    """从候选中选 3 个视觉分落 [correct-δ_high, correct-δ_low] 区间的干扰项。
+
+    落区间者按分数降序取前 3（分数越高越接近正解=越难）。不足 3 个返回 None。
+
+    参数:
+        correct_score: 正解视觉可信度分。
+        candidates: 候选干扰项文本列表。
+        candidate_scores: 与 candidates 对齐的视觉分列表。
+        delta_low: 最小差（上界 = correct - delta_low）。
+        delta_high: 最大差（下界 = correct - delta_high）。
+
+    返回:
+        选中的 3 个候选文本（降序）或 None（不足 3 个）。
+    """
+    upper = correct_score - delta_low
+    lower = correct_score - delta_high
+    eligible = [
+        (c, s)
+        for c, s in zip(candidates, candidate_scores, strict=True)
+        if lower <= s <= upper
+    ]
+    if len(eligible) < 3:
+        return None
+    eligible.sort(key=lambda cs: cs[1], reverse=True)
+    return [c for c, _ in eligible[:3]]
+
+
+def _load_prompt(name: str) -> str:
+    path = _PROMPTS_DIR / name
+    if not path.exists():
+        msg = f"Prompt 模板不存在: {path}"
+        raise FileNotFoundError(msg)
+    return path.read_text(encoding="utf-8")
+
+
+def _material_context_block(question: str, correct_text: str, material: MaterialContext) -> str:
+    parts = [f"## Question\n{question}", f"## Correct Answer\n{correct_text}"]
+    if material.subtitle_sentences:
+        parts.append("## Subtitles")
+        parts.extend(f"  - {s}" for s in material.subtitle_sentences)
+    if getattr(material, "cross_l2_texts", None):
+        parts.append("## Cross-Segment Context")
+        parts.extend(f"  - {t}" for t in material.cross_l2_texts)
+    return "\n".join(parts)
+
+
+def _parse_json_object(raw: str) -> dict:
+    content = raw.strip()
+    if "```" in content:
+        for part in content.split("```"):
+            stripped = part.strip()
+            if stripped.startswith("json"):
+                stripped = stripped[4:].strip()
+            if stripped.startswith("{"):
+                content = stripped
+                break
+    data = json.loads(repair_json(content, return_objects=False))
+    if not isinstance(data, dict):
+        msg = f"selector 响应顶层非 JSON 对象: {type(data).__name__}"
+        raise ValueError(msg)
+    return data
+
+
+async def _generate_pool(
+    vlm: VLMProvider, question: str, correct_text: str,
+    material: MaterialContext, n: int, *, session_id: str,
+) -> list[str]:
+    """VLM 生成 n 个候选干扰项文本。"""
+    system = _load_prompt("ar_distractor_pool.md")
+    user = _material_context_block(question, correct_text, material) + f"\n## N\nGenerate exactly {n} distractors."
+    messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
+    resp = await vlm.chat_with_images(messages, list(material.frame_paths), session_id=session_id)
+    data = _parse_json_object(resp.content)
+    raw = data.get("distractors", [])
+    if not isinstance(raw, list):
+        return []
+    # 防御：去空、去重、剔除与正解字面相同者
+    seen: set[str] = set()
+    out: list[str] = []
+    for item in raw:
+        text = str(item).strip()
+        if not text or text == correct_text.strip() or text in seen:
+            continue
+        seen.add(text)
+        out.append(text)
+    return out
+
+
+async def _score_options(
+    vlm: VLMProvider, question: str, options: list[str],
+    material: MaterialContext, *, session_id: str,
+) -> list[float]:
+    """VLM 对 options（首个为正解）逐一打视觉可信度分 [0,1]，返回对齐分数列表。"""
+    system = _load_prompt("ar_distractor_score.md")
+    numbered = "\n".join(f"{i}. {opt}" for i, opt in enumerate(options, 1))
+    user = f"## Question\n{question}\n\n## Candidates\n{numbered}"
+    messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
+    resp = await vlm.chat_with_images(messages, list(material.frame_paths), session_id=session_id)
+    data = _parse_json_object(resp.content)
+    scores_raw = data.get("scores", [])
+    if not isinstance(scores_raw, list) or len(scores_raw) != len(options):
+        msg = f"打分数量({len(scores_raw) if isinstance(scores_raw, list) else 'NA'}) != 选项数({len(options)})"
+        raise ValueError(msg)
+    return [max(0.0, min(1.0, float(s))) for s in scores_raw]
+
+
+async def build_grounded_options(
+    vlm: VLMProvider,
+    question: str,
+    correct_text: str,
+    material: MaterialContext,
+    config: SelectorConfig,
+    *,
+    session_id: str,
+) -> SelectorOutcome:
+    """生成候选池 → 视觉打分 → 区间选 3 干扰项 → 重组四选项。
+
+    退火（凑不齐 3 个时按序）：① 追加 N 个候选使池达 2N 再打分；② 逐步放宽
+    δ_high（纯重选，不再调 VLM）；③ 仍不足则 hard_fail（调用方走重出）。
+
+    参数:
+        vlm: VLM 端口。
+        question: 题干。
+        correct_text: 正解文本（无字母前缀）。
+        material: 采样素材（提供 frame_paths / subtitles）。
+        config: selector 科研参数。
+        session_id: 遥测会话 ID。
+
+    返回:
+        SelectorOutcome。成功时 options=A 正解+3 grounded 干扰项；hard_fail
+        时 options=None，但 observation 始终存在供落库。
+    """
+    candidates = await _generate_pool(
+        vlm, question, correct_text, material, config.candidate_pool_size, session_id=session_id
+    )
+    # options[0] 恒为正解
+    scored = await _score_options(vlm, question, [correct_text, *candidates], material, session_id=session_id)
+    correct_score, cand_scores = scored[0], scored[1:]
+    anneal_rounds = 0
+
+    chosen = _select_in_interval(
+        correct_score, candidates, cand_scores, config.delta_low, config.delta_high
+    )
+
+    # 退火 1: 追加 N 个候选使池达 2N（仅对新增候选打分，正解分保持首轮值）
+    if chosen is None:
+        anneal_rounds += 1
+        more = await _generate_pool(
+            vlm, question, correct_text, material, config.candidate_pool_size, session_id=session_id
+        )
+        more = [m for m in more if m not in candidates]
+        if more:
+            more_scores = await _score_options(
+                vlm, question, [correct_text, *more], material, session_id=session_id
+            )
+            candidates = candidates + more
+            cand_scores = cand_scores + more_scores[1:]
+        chosen = _select_in_interval(
+            correct_score, candidates, cand_scores, config.delta_low, config.delta_high
+        )
+
+    # 退火 2: 放宽 δ_high（下界下移，纳入更低分候选），δ_low 不动
+    relax = 0
+    delta_high = config.delta_high
+    while chosen is None and relax < config.max_delta_relax:
+        relax += 1
+        anneal_rounds += 1
+        delta_high = delta_high + config.delta_relax_step
+        chosen = _select_in_interval(
+            correct_score, candidates, cand_scores, config.delta_low, delta_high
+        )
+
+    hard_fail = chosen is None
+    observation = {
+        "correct_score": correct_score,
+        "chosen": [
+            cand_scores[candidates.index(c)] for c in (chosen or [])
+        ],
+        "pool_size": len(candidates),
+        "anneal_rounds": anneal_rounds,
+        "delta_high_final": delta_high,
+        "hard_fail": hard_fail,
+    }
+    if hard_fail:
+        logger.warning(
+            "grounded selector 硬失败: correct={:.3f}, pool={}, anneal={}",
+            correct_score, len(candidates), anneal_rounds,
+        )
+        # observation 仍返回，供 pipeline 落 selector_scores（设计 §3.3 退化观测）
+        return SelectorOutcome(observation=observation)
+
+    options = (
+        f"A. {correct_text}",
+        f"B. {chosen[0]}",
+        f"C. {chosen[1]}",
+        f"D. {chosen[2]}",
+    )
+    return SelectorOutcome(observation=observation, options=options, answer="A")
diff --git a/store/prompts/question_gen/ar_distractor_pool.md b/store/prompts/question_gen/ar_distractor_pool.md
new file mode 100644
index 0000000..4c19781
--- /dev/null
+++ b/store/prompts/question_gen/ar_distractor_pool.md
@@ -0,0 +1,23 @@
+You generate hard-negative distractor options for a video Action Recognition
+multiple-choice question.
+
+## Given
+- The question, the correct answer, subtitle context, and video frames.
+
+## Rules
+- Produce distractors that are **grounded near-misses**: each MUST describe an
+  action/entity that genuinely appears in the video, differing from the correct
+  answer in exactly ONE dimension (timing, subject, manner, or object).
+- NEVER invent events absent from the video ("negative space"). A distractor
+  that names something not shown is a failure.
+- Each distractor must be a plausible answer to the question for someone who
+  only skimmed the video.
+- Keep each distractor parallel in structure and length to the correct answer.
+
+## Output
+Respond with ONLY a JSON object:
+```json
+{"distractors": ["...", "...", "..."]}
+```
+Return exactly N distractors (N is given in the request). No option-letter
+prefixes, just the raw text.
diff --git a/store/prompts/question_gen/ar_distractor_score.md b/store/prompts/question_gen/ar_distractor_score.md
new file mode 100644
index 0000000..4243b7e
--- /dev/null
+++ b/store/prompts/question_gen/ar_distractor_score.md
@@ -0,0 +1,23 @@
+You are a strict visual grader for a video Action Recognition question.
+
+## Given
+- The question, video frames, and a numbered list of candidate answer texts
+  (the first is the true answer; the rest are distractor candidates — but you
+  are NOT told which is which).
+
+## Task
+For EACH candidate, judge how visually credible it is as an answer given ONLY
+the frames — i.e. how strongly the frames could be read as supporting it.
+Score in [0.0, 1.0]: 1.0 = frames strongly depict this; 0.0 = frames show no
+trace of it (pure negative space).
+
+Judge visual groundedness ONLY. Do NOT reward the option for being the
+"correct" answer — a good distractor is visually credible yet wrong.
+
+## Output
+Respond with ONLY a JSON object mapping 1-based index to score, same order as
+input:
+```json
+{"scores": [0.9, 0.7, 0.6, 0.3, 0.85]}
+```
+Return exactly as many scores as candidates, in order.
diff --git a/tests/unit/test_distractor_selector.py b/tests/unit/test_distractor_selector.py
new file mode 100644
index 0000000..cf340f0
--- /dev/null
+++ b/tests/unit/test_distractor_selector.py
@@ -0,0 +1,93 @@
+"""distractor_selector 区间选择纯逻辑与编排（mock VLM）测试。"""
+
+import pytest
+
+from app.question_gen.distractor_selector import (
+    SelectorConfig,
+    _select_in_interval,
+    build_grounded_options,
+)
+from core.types import LLMResponse
+
+
+def test_select_three_in_interval_by_highest_score():
+    # correct=0.90, 区间 = [0.90-0.35, 0.90-0.05] = [0.55, 0.85]
+    cands = ["a", "b", "c", "d", "e"]
+    scores = [0.84, 0.70, 0.60, 0.50, 0.88]  # e=0.88 太接近(>0.85)剔除, d=0.50 太低剔除
+    chosen = _select_in_interval(0.90, cands, scores, delta_low=0.05, delta_high=0.35)
+    assert chosen == ["a", "b", "c"]  # 落区间的按分数降序取 3（最难）
+
+
+def test_select_returns_none_when_fewer_than_three():
+    cands = ["a", "b"]
+    scores = [0.80, 0.70]
+    assert _select_in_interval(0.90, cands, scores, 0.05, 0.35) is None
+
+
+def test_select_excludes_out_of_band():
+    cands = ["hi", "lo", "ok1", "ok2", "ok3"]
+    scores = [0.89, 0.10, 0.80, 0.75, 0.70]  # hi>上界, lo<下界
+    chosen = _select_in_interval(0.90, cands, scores, 0.05, 0.35)
+    assert chosen == ["ok1", "ok2", "ok3"]
+
+
+class _FakeVLM:
+    """按队列返回预设响应的 mock VLM。"""
+
+    def __init__(self, responses: list[str]):
+        self._responses = list(responses)
+        self.calls = 0
+
+    async def chat_with_images(self, messages, images, *, session_id=None, parent_call_id=None):
+        self.calls += 1
+        content = self._responses.pop(0)
+        return LLMResponse(
+            content=content, thinking="", model="fake", provider="fake",
+            prompt_tokens=0, completion_tokens=0, latency_ms=0,
+            ttft_ms=None, max_inter_token_ms=None, cache_hit=False, call_id="c",
+        )
+
+
+class _Material:
+    subtitle_sentences = ["厨师先炒后蒸"]
+    frame_paths = ["/f1.jpg", "/f2.jpg"]
+    cross_l2_texts: list = []
+    source_nodes = ("n1",)
+
+
+@pytest.mark.asyncio
+async def test_build_grounded_options_happy_path():
+    pool = '{"distractors": ["炒", "煮", "炸", "烤"]}'
+    scores = '{"scores": [0.90, 0.80, 0.70, 0.60, 0.20]}'  # 正解0.90; 炒0.80 煮0.70 炸0.60 落区间, 烤0.20 剔除
+    vlm = _FakeVLM([pool, scores])
+    cfg = SelectorConfig(candidate_pool_size=4, delta_low=0.05, delta_high=0.35)
+    out = await build_grounded_options(
+        vlm=vlm, question="厨师最终用哪种方式？", correct_text="蒸",
+        material=_Material(), config=cfg, session_id="s",
+    )
+    assert out.hard_fail is False
+    assert out.answer == "A"
+    assert out.options[0] == "A. 蒸"
+    assert {o[3:] for o in out.options[1:]} == {"炒", "煮", "炸"}
+    assert out.observation["hard_fail"] is False
+
+
+@pytest.mark.asyncio
+async def test_build_grounded_options_hard_fail_keeps_observation():
+    # 所有候选都在负空间（分数极低），退火后仍不足 3 个 → hard_fail。
+    # VLM 只被调 2 次（首轮 pool+score）+ 1 次退火 pool + 1 次退火 score = 4 次；
+    # δ_high 放宽轮次是纯重选，不调 VLM。退火 pool 打分含正解，共 4 个分数。
+    pool = '{"distractors": ["x", "y", "z"]}'
+    scores = '{"scores": [0.90, 0.05, 0.04, 0.03]}'
+    pool2 = '{"distractors": ["p", "q", "r"]}'
+    scores2 = '{"scores": [0.90, 0.05, 0.04, 0.03]}'
+    vlm = _FakeVLM([pool, scores, pool2, scores2])
+    cfg = SelectorConfig(candidate_pool_size=3, delta_low=0.05, delta_high=0.35)
+    out = await build_grounded_options(
+        vlm=vlm, question="?", correct_text="蒸",
+        material=_Material(), config=cfg, session_id="s",
+    )
+    assert out.hard_fail is True
+    assert out.options is None
+    assert out.observation["hard_fail"] is True
+    assert out.observation["pool_size"] == 6  # 首轮 3 + 退火追加 3