From b13eab0659b30e3f298e3fc2d1b3ea2d2510139d Mon Sep 17 00:00:00 2001 From: iomgaa Date: Tue, 14 Jul 2026 14:15:50 -0400 Subject: [PATCH] feat: wire grounded selector into AR slot processing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 Task 5 的 grounded selector 织入 AR 出题路径（Phase 3.5，位于 record_item 与 postprocess 之间），仅在 strategy.uses_grounded_selector 为真时进入。observation 始终落库（含 hard-fail），硬失败走重出。 PipelineConfig 新增 candidate_pool_size/selector_delta_low/ selector_delta_high 三参，YAML 与 CLI seed override 同步。 --- app/question_gen/pipeline_v2.py | 105 ++++++++++++++++++++ config/question_gen_ar30.yaml | 14 +++ tests/integration/test_pipeline_v2.py | 7 ++ tests/unit/test_pipeline_selector_wiring.py | 42 ++++++++ tools/generate_questions.py | 3 + 5 files changed, 171 insertions(+) create mode 100644 config/question_gen_ar30.yaml create mode 100644 tests/unit/test_pipeline_selector_wiring.py diff --git a/app/question_gen/pipeline_v2.py b/app/question_gen/pipeline_v2.py index 16c1393..5237dae 100644 --- a/app/question_gen/pipeline_v2.py +++ b/app/question_gen/pipeline_v2.py @@ -24,6 +24,7 @@ from __future__ import annotations import asyncio +import json import random import uuid from dataclasses import dataclass @@ -98,6 +99,9 @@ class PipelineConfig: concurrency: 并发 slot 数上限。 seed: 随机种子。 output_dir: 输出目录。 + candidate_pool_size: grounded selector 首轮候选干扰项数 N。 + selector_delta_low: 干扰项视觉分与正解的最小差（区间上界）。 + selector_delta_high: 干扰项视觉分与正解的最大差（区间下界）。 """ per_type: int @@ -107,6 +111,9 @@ class PipelineConfig: concurrency: int seed: int output_dir: Path + candidate_pool_size: int = 24 + selector_delta_low: float = 0.05 + selector_delta_high: float = 0.35 # --------------------------------------------------------------------------- @@ -150,6 +157,9 @@ def load_pipeline_config(yaml_path: Path) -> PipelineConfig: concurrency=int(section["concurrency"]), seed=int(section["seed"]), output_dir=Path(section["output_dir"]), + candidate_pool_size=int(section.get("candidate_pool_size", 24)), + selector_delta_low=float(section.get("selector_delta_low", 0.05)), + selector_delta_high=float(section.get("selector_delta_high", 0.35)), ) @@ -281,6 +291,56 @@ def _to_generated_question( ) +def _extract_correct_text(options: tuple[str, ...], answer: str) -> str: + """从四选项中取正解文本（去掉 "X. " 字母前缀）。 + + 参数: + options: 选项元组，格式 ("A. ...", "B. ...", ...)。 + answer: 正解字母（大小写不敏感）。 + + 返回: + 正解选项去前缀后的文本。 + + 异常: + ValueError: answer 对应索引超出选项范围。 + """ + idx = ord(answer.strip().upper()) - ord("A") + if not 0 <= idx < len(options): + msg = f"answer '{answer}' 超出选项范围 (n={len(options)})" + raise ValueError(msg) + opt = options[idx] + prefix = f"{answer.strip().upper()}. " + return opt[len(prefix) :] if opt.startswith(prefix) else opt + + +def _replace_candidate_options( + candidate: CandidateQuestion, options: tuple[str, ...], answer: str +) -> CandidateQuestion: + """用 selector 重组的选项/答案替换候选（CandidateQuestion frozen）。 + + 参数: + candidate: 原候选题目。 + options: grounded selector 重组后的四选项。 + answer: 重组后的正解字母（恒 "A"）。 + + 返回: + 仅替换 options/answer、其余字段照搬的新 CandidateQuestion。 + """ + return CandidateQuestion( + question_id=candidate.question_id, + video_id=candidate.video_id, + task_type=candidate.task_type, + skill_target=candidate.skill_target, + question=candidate.question, + options=options, + answer=answer, + source_nodes=candidate.source_nodes, + difficulty=candidate.difficulty, + subtitle_sentences=candidate.subtitle_sentences, + frame_paths=candidate.frame_paths, + ) + + # --------------------------------------------------------------------------- # 单 Slot 处理（重出循环） # --------------------------------------------------------------------------- @@ -428,6 +488,51 @@ async def _process_one_slot( sub_pattern=sub_pattern.name if sub_pattern else None, ) + # Phase 3.5: grounded selector（仅 AR 路径） + if strategy.uses_grounded_selector: + from app.question_gen.distractor_selector import ( + SelectorConfig, + build_grounded_options, + ) + + correct_text = _extract_correct_text(candidate.options, candidate.answer) + selector_cfg = SelectorConfig( + candidate_pool_size=config.candidate_pool_size, + delta_low=config.selector_delta_low, + delta_high=config.selector_delta_high, + ) + try: + outcome = await build_grounded_options( + vlm=vlm, + question=candidate.question, + correct_text=correct_text, + material=material, + config=selector_cfg, + session_id=session_id, + ) + except (ValueError, FileNotFoundError) as e: + logger.warning( + "slot {} selector 异常 (attempt {}): {}", slot.slot_id, attempt, e + ) + prev_reason = f"selector_error: {e}" + continue + + # observation 始终落库（含 hard-fail），供 EOB 退化观测与调参 + store.update_selector_scores( + item_id, json.dumps(outcome.observation, ensure_ascii=False) + ) + + if outcome.hard_fail: + prev_reason = "grounded 干扰项不足（selector 硬失败）" + store.mark_item_rejected(item_id, prev_reason) + logger.info("slot {} selector 硬失败 (attempt {})", slot.slot_id, attempt) + continue + + # 用 grounded 四选项替换候选（frozen → 构造新实例） + candidate = _replace_candidate_options( + candidate, outcome.options, outcome.answer + ) + # Phase 4: 后处理 source_texts = list(material.subtitle_sentences) pp = run_postprocess( diff --git a/config/question_gen_ar30.yaml b/config/question_gen_ar30.yaml new file mode 100644 index 0000000..856fdb3 --- /dev/null +++ b/config/question_gen_ar30.yaml @@ -0,0 +1,14 @@ +# AR 专项出题配置：使用 ActionRecognitionStrategy + 6 SubPattern 靶向生成 +# 目标：生成 30 道 Action Recognition 题 + +question_gen_v2: + per_type: 30 # 只跑 AR 一类，30 题 + retry_limit: 15 # AR 约束更严，给更多重试机会 + heavy_sample_rate: 0.0 # 不需要重量抽检 + dedup_threshold: 0.85 + concurrency: 8 # AR 需要帧，适度并发 + seed: 2024 + output_dir: "store/questions/generated-ar30" + candidate_pool_size: 24 # grounded selector 首轮候选干扰项数 N + selector_delta_low: 0.05 # 干扰项视觉分与正解的最小差（区间上界） + selector_delta_high: 0.35 # 干扰项视觉分与正解的最大差（区间下界） diff --git a/tests/integration/test_pipeline_v2.py b/tests/integration/test_pipeline_v2.py index bb1221b..fbaf5dc 100644 --- a/tests/integration/test_pipeline_v2.py +++ b/tests/integration/test_pipeline_v2.py @@ -194,8 +194,15 @@ class MockVLM: parent_call_id: str | None = None, ) -> LLMResponse: prompt_text = str(messages) + system_text = messages[0].get("content", "") if messages else "" if "verdict" in prompt_text.lower(): return _make_llm_response(self._gate_response) + if "distractor" in system_text.lower() and "grader" not in system_text.lower(): + # 候选池请求：返回 4 个 grounded 干扰项 + return _make_llm_response('{"distractors": ["蒸", "煮", "炸", "烤"]}') + if "grader" in system_text.lower(): + # 打分请求：正解高分、3 个落区间、1 个负空间 + return _make_llm_response('{"scores": [0.90, 0.80, 0.70, 0.60, 0.20]}') idx = min(self._gen_count, len(self._responses) - 1) self._gen_count += 1 return _make_llm_response(self._responses[idx]) diff --git a/tests/unit/test_pipeline_selector_wiring.py b/tests/unit/test_pipeline_selector_wiring.py new file mode 100644 index 0000000..1cf30a1 --- /dev/null +++ b/tests/unit/test_pipeline_selector_wiring.py @@ -0,0 +1,42 @@ +"""selector 织入辅助：正解文本提取 + 分流。""" + +import pytest + +from app.question_gen.pipeline_v2 import _extract_correct_text + + +def test_extract_correct_text_strips_prefix(): + options = ("A. 蒸", "B. 炒", "C. 煮", "D. 炸") + assert _extract_correct_text(options, "C") == "煮" + + +def test_extract_correct_text_handles_lowercase_answer(): + options = ("A. run", "B. walk", "C. jump", "D. sit") + assert _extract_correct_text(options, "b") == "walk" + + +def test_extract_correct_text_out_of_range_raises(): + options = ("A. a", "B. b", "C. c", "D. d") + with pytest.raises(ValueError): + _extract_correct_text(options, "E") + + +def test_replace_candidate_options(): + from app.question_gen.generator_v2 import CandidateQuestion + from app.question_gen.pipeline_v2 import _replace_candidate_options + + c = CandidateQuestion( + question_id="q", + video_id="v", + task_type="Action Recognition", + skill_target="M1_AR", + question="?", + options=("A. a", "B. b", "C. c", "D. d"), + answer="A", + source_nodes=("n1",), + difficulty="hard", + ) + new = _replace_candidate_options(c, ("A. 蒸", "B. 炒", "C. 煮", "D. 炸"), "A") + assert new.options == ("A. 蒸", "B. 炒", "C. 煮", "D. 炸") + assert new.question == "?" # 其余字段不变 + assert new.source_nodes == ("n1",) diff --git a/tools/generate_questions.py b/tools/generate_questions.py index b1fd760..acfdb7d 100644 --- a/tools/generate_questions.py +++ b/tools/generate_questions.py @@ -912,6 +912,9 @@ async def _run_generate_v2(args: argparse.Namespace) -> None: concurrency=config.concurrency, seed=args.seed, output_dir=config.output_dir, + candidate_pool_size=config.candidate_pool_size, + selector_delta_low=config.selector_delta_low, + selector_delta_high=config.selector_delta_high, ) logger.info("seed 覆盖为: {}", args.seed)