fix: degrade distractor pool gracefully on malformed VLM response
This commit is contained in:
@@ -53,7 +53,7 @@ class SelectorOutcome:
|
|||||||
answer: 正解字母(恒 "A"),hard-fail 时为 None。
|
answer: 正解字母(恒 "A"),hard-fail 时为 None。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
observation: dict
|
observation: dict[str, object]
|
||||||
options: tuple[str, ...] | None = None
|
options: tuple[str, ...] | None = None
|
||||||
answer: str | None = None
|
answer: str | None = None
|
||||||
|
|
||||||
@@ -142,7 +142,12 @@ async def _generate_pool(
|
|||||||
user = _material_context_block(question, correct_text, material) + f"\n## N\nGenerate exactly {n} distractors."
|
user = _material_context_block(question, correct_text, material) + f"\n## N\nGenerate exactly {n} distractors."
|
||||||
messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
|
messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
|
||||||
resp = await vlm.chat_with_images(messages, list(material.frame_paths), session_id=session_id)
|
resp = await vlm.chat_with_images(messages, list(material.frame_paths), session_id=session_id)
|
||||||
data = _parse_json_object(resp.content)
|
# graceful 降级:解析爆炸时返回空池,让退火/hard_fail 接管(区别于契约违反的打分校验)
|
||||||
|
try:
|
||||||
|
data = _parse_json_object(resp.content)
|
||||||
|
except (ValueError, json.JSONDecodeError) as exc:
|
||||||
|
logger.warning("grounded selector 候选池响应解析失败,降级为空池: {}", exc)
|
||||||
|
return []
|
||||||
raw = data.get("distractors", [])
|
raw = data.get("distractors", [])
|
||||||
if not isinstance(raw, list):
|
if not isinstance(raw, list):
|
||||||
return []
|
return []
|
||||||
@@ -187,7 +192,7 @@ async def build_grounded_options(
|
|||||||
) -> SelectorOutcome:
|
) -> SelectorOutcome:
|
||||||
"""生成候选池 → 视觉打分 → 区间选 3 干扰项 → 重组四选项。
|
"""生成候选池 → 视觉打分 → 区间选 3 干扰项 → 重组四选项。
|
||||||
|
|
||||||
退火(凑不齐 3 个时按序):① 追加 N 个候选使池达 2N 再打分;② 逐步放宽
|
退火(凑不齐 3 个时按序):① 最多追加 N 个新候选(去重后实际增量可能更少)再打分;② 逐步放宽
|
||||||
δ_high(纯重选,不再调 VLM);③ 仍不足则 hard_fail(调用方走重出)。
|
δ_high(纯重选,不再调 VLM);③ 仍不足则 hard_fail(调用方走重出)。
|
||||||
|
|
||||||
参数:
|
参数:
|
||||||
@@ -214,7 +219,7 @@ async def build_grounded_options(
|
|||||||
correct_score, candidates, cand_scores, config.delta_low, config.delta_high
|
correct_score, candidates, cand_scores, config.delta_low, config.delta_high
|
||||||
)
|
)
|
||||||
|
|
||||||
# 退火 1: 追加 N 个候选使池达 2N(仅对新增候选打分,正解分保持首轮值)
|
# 退火 1: 最多追加 N 个新候选(去重后实际增量可能更少),仅对新增候选打分,正解分保持首轮值
|
||||||
if chosen is None:
|
if chosen is None:
|
||||||
anneal_rounds += 1
|
anneal_rounds += 1
|
||||||
more = await _generate_pool(
|
more = await _generate_pool(
|
||||||
@@ -243,7 +248,7 @@ async def build_grounded_options(
|
|||||||
)
|
)
|
||||||
|
|
||||||
hard_fail = chosen is None
|
hard_fail = chosen is None
|
||||||
observation = {
|
observation: dict[str, object] = {
|
||||||
"correct_score": correct_score,
|
"correct_score": correct_score,
|
||||||
"chosen": [
|
"chosen": [
|
||||||
cand_scores[candidates.index(c)] for c in (chosen or [])
|
cand_scores[candidates.index(c)] for c in (chosen or [])
|
||||||
|
|||||||
@@ -55,6 +55,14 @@ class _Material:
|
|||||||
source_nodes = ("n1",)
|
source_nodes = ("n1",)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_generate_pool_degrades_on_malformed_response():
|
||||||
|
from app.question_gen.distractor_selector import _generate_pool
|
||||||
|
vlm = _FakeVLM(['["not", "a", "dict"]']) # 顶层是 list 不是 dict
|
||||||
|
out = await _generate_pool(vlm, "?", "蒸", _Material(), 4, session_id="s")
|
||||||
|
assert out == []
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_build_grounded_options_happy_path():
|
async def test_build_grounded_options_happy_path():
|
||||||
pool = '{"distractors": ["炒", "煮", "炸", "烤"]}'
|
pool = '{"distractors": ["炒", "煮", "炸", "烤"]}'
|
||||||
|
|||||||
Reference in New Issue
Block a user