feat: wire grounded selector into AR slot processing
将 Task 5 的 grounded selector 织入 AR 出题路径(Phase 3.5,位于 record_item 与 postprocess 之间),仅在 strategy.uses_grounded_selector 为真时进入。observation 始终落库(含 hard-fail),硬失败走重出。 PipelineConfig 新增 candidate_pool_size/selector_delta_low/ selector_delta_high 三参,YAML 与 CLI seed override 同步。
This commit is contained in:
@@ -24,6 +24,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import random
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
@@ -98,6 +99,9 @@ class PipelineConfig:
|
||||
concurrency: 并发 slot 数上限。
|
||||
seed: 随机种子。
|
||||
output_dir: 输出目录。
|
||||
candidate_pool_size: grounded selector 首轮候选干扰项数 N。
|
||||
selector_delta_low: 干扰项视觉分与正解的最小差(区间上界)。
|
||||
selector_delta_high: 干扰项视觉分与正解的最大差(区间下界)。
|
||||
"""
|
||||
|
||||
per_type: int
|
||||
@@ -107,6 +111,9 @@ class PipelineConfig:
|
||||
concurrency: int
|
||||
seed: int
|
||||
output_dir: Path
|
||||
candidate_pool_size: int = 24
|
||||
selector_delta_low: float = 0.05
|
||||
selector_delta_high: float = 0.35
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -150,6 +157,9 @@ def load_pipeline_config(yaml_path: Path) -> PipelineConfig:
|
||||
concurrency=int(section["concurrency"]),
|
||||
seed=int(section["seed"]),
|
||||
output_dir=Path(section["output_dir"]),
|
||||
candidate_pool_size=int(section.get("candidate_pool_size", 24)),
|
||||
selector_delta_low=float(section.get("selector_delta_low", 0.05)),
|
||||
selector_delta_high=float(section.get("selector_delta_high", 0.35)),
|
||||
)
|
||||
|
||||
|
||||
@@ -281,6 +291,56 @@ def _to_generated_question(
|
||||
)
|
||||
|
||||
|
||||
def _extract_correct_text(options: tuple[str, ...], answer: str) -> str:
|
||||
"""从四选项中取正解文本(去掉 "X. " 字母前缀)。
|
||||
|
||||
参数:
|
||||
options: 选项元组,格式 ("A. ...", "B. ...", ...)。
|
||||
answer: 正解字母(大小写不敏感)。
|
||||
|
||||
返回:
|
||||
正解选项去前缀后的文本。
|
||||
|
||||
异常:
|
||||
ValueError: answer 对应索引超出选项范围。
|
||||
"""
|
||||
idx = ord(answer.strip().upper()) - ord("A")
|
||||
if not 0 <= idx < len(options):
|
||||
msg = f"answer '{answer}' 超出选项范围 (n={len(options)})"
|
||||
raise ValueError(msg)
|
||||
opt = options[idx]
|
||||
prefix = f"{answer.strip().upper()}. "
|
||||
return opt[len(prefix) :] if opt.startswith(prefix) else opt
|
||||
|
||||
|
||||
def _replace_candidate_options(
|
||||
candidate: CandidateQuestion, options: tuple[str, ...], answer: str
|
||||
) -> CandidateQuestion:
|
||||
"""用 selector 重组的选项/答案替换候选(CandidateQuestion frozen)。
|
||||
|
||||
参数:
|
||||
candidate: 原候选题目。
|
||||
options: grounded selector 重组后的四选项。
|
||||
answer: 重组后的正解字母(恒 "A")。
|
||||
|
||||
返回:
|
||||
仅替换 options/answer、其余字段照搬的新 CandidateQuestion。
|
||||
"""
|
||||
return CandidateQuestion(
|
||||
question_id=candidate.question_id,
|
||||
video_id=candidate.video_id,
|
||||
task_type=candidate.task_type,
|
||||
skill_target=candidate.skill_target,
|
||||
question=candidate.question,
|
||||
options=options,
|
||||
answer=answer,
|
||||
source_nodes=candidate.source_nodes,
|
||||
difficulty=candidate.difficulty,
|
||||
subtitle_sentences=candidate.subtitle_sentences,
|
||||
frame_paths=candidate.frame_paths,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 单 Slot 处理(重出循环)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -428,6 +488,51 @@ async def _process_one_slot(
|
||||
sub_pattern=sub_pattern.name if sub_pattern else None,
|
||||
)
|
||||
|
||||
# Phase 3.5: grounded selector(仅 AR 路径)
|
||||
if strategy.uses_grounded_selector:
|
||||
from app.question_gen.distractor_selector import (
|
||||
SelectorConfig,
|
||||
build_grounded_options,
|
||||
)
|
||||
|
||||
correct_text = _extract_correct_text(candidate.options, candidate.answer)
|
||||
selector_cfg = SelectorConfig(
|
||||
candidate_pool_size=config.candidate_pool_size,
|
||||
delta_low=config.selector_delta_low,
|
||||
delta_high=config.selector_delta_high,
|
||||
)
|
||||
try:
|
||||
outcome = await build_grounded_options(
|
||||
vlm=vlm,
|
||||
question=candidate.question,
|
||||
correct_text=correct_text,
|
||||
material=material,
|
||||
config=selector_cfg,
|
||||
session_id=session_id,
|
||||
)
|
||||
except (ValueError, FileNotFoundError) as e:
|
||||
logger.warning(
|
||||
"slot {} selector 异常 (attempt {}): {}", slot.slot_id, attempt, e
|
||||
)
|
||||
prev_reason = f"selector_error: {e}"
|
||||
continue
|
||||
|
||||
# observation 始终落库(含 hard-fail),供 EOB 退化观测与调参
|
||||
store.update_selector_scores(
|
||||
item_id, json.dumps(outcome.observation, ensure_ascii=False)
|
||||
)
|
||||
|
||||
if outcome.hard_fail:
|
||||
prev_reason = "grounded 干扰项不足(selector 硬失败)"
|
||||
store.mark_item_rejected(item_id, prev_reason)
|
||||
logger.info("slot {} selector 硬失败 (attempt {})", slot.slot_id, attempt)
|
||||
continue
|
||||
|
||||
# 用 grounded 四选项替换候选(frozen → 构造新实例)
|
||||
candidate = _replace_candidate_options(
|
||||
candidate, outcome.options, outcome.answer
|
||||
)
|
||||
|
||||
# Phase 4: 后处理
|
||||
source_texts = list(material.subtitle_sentences)
|
||||
pp = run_postprocess(
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
# AR 专项出题配置:使用 ActionRecognitionStrategy + 6 SubPattern 靶向生成
|
||||
# 目标:生成 30 道 Action Recognition 题
|
||||
|
||||
question_gen_v2:
|
||||
per_type: 30 # 只跑 AR 一类,30 题
|
||||
retry_limit: 15 # AR 约束更严,给更多重试机会
|
||||
heavy_sample_rate: 0.0 # 不需要重量抽检
|
||||
dedup_threshold: 0.85
|
||||
concurrency: 8 # AR 需要帧,适度并发
|
||||
seed: 2024
|
||||
output_dir: "store/questions/generated-ar30"
|
||||
candidate_pool_size: 24 # grounded selector 首轮候选干扰项数 N
|
||||
selector_delta_low: 0.05 # 干扰项视觉分与正解的最小差(区间上界)
|
||||
selector_delta_high: 0.35 # 干扰项视觉分与正解的最大差(区间下界)
|
||||
@@ -194,8 +194,15 @@ class MockVLM:
|
||||
parent_call_id: str | None = None,
|
||||
) -> LLMResponse:
|
||||
prompt_text = str(messages)
|
||||
system_text = messages[0].get("content", "") if messages else ""
|
||||
if "verdict" in prompt_text.lower():
|
||||
return _make_llm_response(self._gate_response)
|
||||
if "distractor" in system_text.lower() and "grader" not in system_text.lower():
|
||||
# 候选池请求:返回 4 个 grounded 干扰项
|
||||
return _make_llm_response('{"distractors": ["蒸", "煮", "炸", "烤"]}')
|
||||
if "grader" in system_text.lower():
|
||||
# 打分请求:正解高分、3 个落区间、1 个负空间
|
||||
return _make_llm_response('{"scores": [0.90, 0.80, 0.70, 0.60, 0.20]}')
|
||||
idx = min(self._gen_count, len(self._responses) - 1)
|
||||
self._gen_count += 1
|
||||
return _make_llm_response(self._responses[idx])
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""selector 织入辅助:正解文本提取 + 分流。"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.question_gen.pipeline_v2 import _extract_correct_text
|
||||
|
||||
|
||||
def test_extract_correct_text_strips_prefix():
|
||||
options = ("A. 蒸", "B. 炒", "C. 煮", "D. 炸")
|
||||
assert _extract_correct_text(options, "C") == "煮"
|
||||
|
||||
|
||||
def test_extract_correct_text_handles_lowercase_answer():
|
||||
options = ("A. run", "B. walk", "C. jump", "D. sit")
|
||||
assert _extract_correct_text(options, "b") == "walk"
|
||||
|
||||
|
||||
def test_extract_correct_text_out_of_range_raises():
|
||||
options = ("A. a", "B. b", "C. c", "D. d")
|
||||
with pytest.raises(ValueError):
|
||||
_extract_correct_text(options, "E")
|
||||
|
||||
|
||||
def test_replace_candidate_options():
|
||||
from app.question_gen.generator_v2 import CandidateQuestion
|
||||
from app.question_gen.pipeline_v2 import _replace_candidate_options
|
||||
|
||||
c = CandidateQuestion(
|
||||
question_id="q",
|
||||
video_id="v",
|
||||
task_type="Action Recognition",
|
||||
skill_target="M1_AR",
|
||||
question="?",
|
||||
options=("A. a", "B. b", "C. c", "D. d"),
|
||||
answer="A",
|
||||
source_nodes=("n1",),
|
||||
difficulty="hard",
|
||||
)
|
||||
new = _replace_candidate_options(c, ("A. 蒸", "B. 炒", "C. 煮", "D. 炸"), "A")
|
||||
assert new.options == ("A. 蒸", "B. 炒", "C. 煮", "D. 炸")
|
||||
assert new.question == "?" # 其余字段不变
|
||||
assert new.source_nodes == ("n1",)
|
||||
@@ -912,6 +912,9 @@ async def _run_generate_v2(args: argparse.Namespace) -> None:
|
||||
concurrency=config.concurrency,
|
||||
seed=args.seed,
|
||||
output_dir=config.output_dir,
|
||||
candidate_pool_size=config.candidate_pool_size,
|
||||
selector_delta_low=config.selector_delta_low,
|
||||
selector_delta_high=config.selector_delta_high,
|
||||
)
|
||||
logger.info("seed 覆盖为: {}", args.seed)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user