fix: degrade distractor pool gracefully on malformed VLM response

This commit is contained in:
2026-07-14 14:11:13 -04:00
parent 8a54055d02
commit d0194f5840
2 changed files with 18 additions and 5 deletions
+9 -4
View File
@@ -53,7 +53,7 @@ class SelectorOutcome:
answer: 正解字母(恒 "A"),hard-fail 时为 None。 answer: 正解字母(恒 "A"),hard-fail 时为 None。
""" """
observation: dict observation: dict[str, object]
options: tuple[str, ...] | None = None options: tuple[str, ...] | None = None
answer: str | None = None answer: str | None = None
@@ -142,7 +142,12 @@ async def _generate_pool(
user = _material_context_block(question, correct_text, material) + f"\n## N\nGenerate exactly {n} distractors." user = _material_context_block(question, correct_text, material) + f"\n## N\nGenerate exactly {n} distractors."
messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
resp = await vlm.chat_with_images(messages, list(material.frame_paths), session_id=session_id) resp = await vlm.chat_with_images(messages, list(material.frame_paths), session_id=session_id)
# graceful 降级:解析爆炸时返回空池,让退火/hard_fail 接管(区别于契约违反的打分校验)
try:
data = _parse_json_object(resp.content) data = _parse_json_object(resp.content)
except (ValueError, json.JSONDecodeError) as exc:
logger.warning("grounded selector 候选池响应解析失败,降级为空池: {}", exc)
return []
raw = data.get("distractors", []) raw = data.get("distractors", [])
if not isinstance(raw, list): if not isinstance(raw, list):
return [] return []
@@ -187,7 +192,7 @@ async def build_grounded_options(
) -> SelectorOutcome: ) -> SelectorOutcome:
"""生成候选池 → 视觉打分 → 区间选 3 干扰项 → 重组四选项。 """生成候选池 → 视觉打分 → 区间选 3 干扰项 → 重组四选项。
退火(凑不齐 3 个时按序):① 追加 N 个候选使池达 2N 再打分;② 逐步放宽 退火(凑不齐 3 个时按序):① 最多追加 N 个候选(去重后实际增量可能更少)再打分;② 逐步放宽
δ_high(纯重选,不再调 VLM);③ 仍不足则 hard_fail(调用方走重出)。 δ_high(纯重选,不再调 VLM);③ 仍不足则 hard_fail(调用方走重出)。
参数: 参数:
@@ -214,7 +219,7 @@ async def build_grounded_options(
correct_score, candidates, cand_scores, config.delta_low, config.delta_high correct_score, candidates, cand_scores, config.delta_low, config.delta_high
) )
# 退火 1: 追加 N 个候选使池达 2N仅对新增候选打分,正解分保持首轮值 # 退火 1: 最多追加 N 个候选(去重后实际增量可能更少),仅对新增候选打分,正解分保持首轮值
if chosen is None: if chosen is None:
anneal_rounds += 1 anneal_rounds += 1
more = await _generate_pool( more = await _generate_pool(
@@ -243,7 +248,7 @@ async def build_grounded_options(
) )
hard_fail = chosen is None hard_fail = chosen is None
observation = { observation: dict[str, object] = {
"correct_score": correct_score, "correct_score": correct_score,
"chosen": [ "chosen": [
cand_scores[candidates.index(c)] for c in (chosen or []) cand_scores[candidates.index(c)] for c in (chosen or [])
+8
View File
@@ -55,6 +55,14 @@ class _Material:
source_nodes = ("n1",) source_nodes = ("n1",)
@pytest.mark.asyncio
async def test_generate_pool_degrades_on_malformed_response():
from app.question_gen.distractor_selector import _generate_pool
vlm = _FakeVLM(['["not", "a", "dict"]']) # 顶层是 list 不是 dict
out = await _generate_pool(vlm, "?", "", _Material(), 4, session_id="s")
assert out == []
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_build_grounded_options_happy_path(): async def test_build_grounded_options_happy_path():
pool = '{"distractors": ["", "", "", ""]}' pool = '{"distractors": ["", "", "", ""]}'