diff --git a/research-wiki/plans/2026-07-14-grounded-question-gen-phaseA-plan.md b/research-wiki/plans/2026-07-14-grounded-question-gen-phaseA-plan.md index 7f3c037..272ceb6 100644 --- a/research-wiki/plans/2026-07-14-grounded-question-gen-phaseA-plan.md +++ b/research-wiki/plans/2026-07-14-grounded-question-gen-phaseA-plan.md @@ -219,11 +219,27 @@ def _to_generated_question( Run: `conda run -n Video-Tree-TRM pytest tests/unit/test_generated_question_sub_pattern.py -v` Expected: PASS -- [ ] **Step 7: 写 JSON 持久化测试** +- [ ] **Step 7: 写共享序列化测试** -在同文件追加，验证 `_on_accept` 与 `_append_to_json` 写出 `sub_pattern` 键。因 `_on_accept` 是闭包，测试改为直接测公共写函数 `_append_to_json`（同格式）： +`_append_to_json` 与 `_on_accept` 是两条独立写路径（后者才写 `accepted_questions.json`），各自维护一份 entry dict——易漏改一处而测试不红。抽共享函数 `_question_to_entry(q) -> dict` 供两处复用，直接测它保证两条路径都含 `sub_pattern`： ```python +def test_question_to_entry_includes_sub_pattern(): + from tools.generate_questions import _question_to_entry + q = GeneratedQuestion( + question_id="v1_Action Recognition_0001", video_id="v1", + task_type="Action Recognition", question="?", + options=("A. 蒸", "B. 炒", "C. 煮", "D. 炸"), answer="A", + source_nodes=("n1",), difficulty="hard", + family="ACTION_RECOGNITION", skill_target="M1_AR", + sub_pattern="temporal_reasoning_failure", + ) + entry = _question_to_entry(q) + assert entry["sub_pattern"] == "temporal_reasoning_failure" + assert entry["question_id"] == "v1_Action Recognition_0001" + assert entry["options"] == ["A. 蒸", "B. 炒", "C. 煮", "D. 炸"] + + def test_append_to_json_writes_sub_pattern(tmp_path): from tools.generate_questions import _append_to_json q = GeneratedQuestion( @@ -242,16 +258,19 @@ def test_append_to_json_writes_sub_pattern(tmp_path): - [ ] **Step 8: 跑测试确认失败** -Run: `conda run -n Video-Tree-TRM pytest tests/unit/test_generated_question_sub_pattern.py::test_append_to_json_writes_sub_pattern -v` -Expected: FAIL（`_append_to_json` 的 entry dict 无 `sub_pattern`） +Run: `conda run -n Video-Tree-TRM pytest tests/unit/test_generated_question_sub_pattern.py -k "entry or append" -v` +Expected: FAIL（`_question_to_entry` 不存在） -- [ ] **Step 9: 改 `tools/generate_questions.py` 两处写 dict** +- [ ] **Step 9: 抽共享 `_question_to_entry` + 两处复用** -`_append_to_json` 的 `entry` dict（约 344 行）与 `_on_accept` 内的 append dict（约 1123 行），各补一行 `"sub_pattern": q.sub_pattern,`（放在 `"skill_target"` 之后）。示例（`_append_to_json`）： +`tools/generate_questions.py`，在 `_append_to_json` 之前加共享函数： ```python - entry = { +def _question_to_entry(question: GeneratedQuestion) -> dict: + """将题目序列化为 JSON entry（_append_to_json 与 _on_accept 共用）。""" + return { "question_id": question.question_id, + "video_id": question.video_id, "task_type": question.task_type, "question": question.question, "options": list(question.options), @@ -264,7 +283,9 @@ Expected: FAIL（`_append_to_json` 的 entry dict 无 `sub_pattern`） } ``` -`_on_accept` 内同理，在 `"skill_target": q.skill_target,` 后加 `"sub_pattern": q.sub_pattern,`。 +`_append_to_json` 内 `entry = {...}` 整体替换为 `entry = _question_to_entry(question)`。`_on_accept` 内 `existing.append({...})` 整体替换为 `existing.append(_question_to_entry(q))`。 + +> 注：`_append_to_json` 原 entry 不含 `video_id` 键（按 video 分文件），改用共享函数后会多出 `video_id` 键——无害（下游按需取键），且与 `accepted_questions.json` 格式统一。若下游有严格 schema 校验，保留两函数但都调用 `_question_to_entry` 后 `entry.pop("video_id", None)`；实现时确认下游读取无强约束即可直接统一。 - [ ] **Step 10: 跑测试确认通过** @@ -363,6 +384,7 @@ git commit -m "feat: add uses_grounded_selector strategy switch (AR only)" **Files:** - Modify: `app/question_gen/run_store.py`（`_DDL_ITEMS` 加列 + 幂等 ALTER TABLE + 新方法） +- Modify: `research-wiki/schemas/question-gen-items.md`（登记 `selector_scores` 列 + JSON 结构） - Test: `tests/unit/test_run_store_selector_scores.py`（新建） - [ ] **Step 1: 写失败测试** @@ -467,10 +489,14 @@ Expected: FAIL Run: `conda run -n Video-Tree-TRM pytest tests/unit/test_run_store_selector_scores.py -v` Expected: PASS -- [ ] **Step 6: 提交** +- [ ] **Step 6: 登记 schema 文档** + +`research-wiki/schemas/question-gen-items.md` 的列清单补一行 `selector_scores TEXT`，并说明其 JSON 结构：`{correct_score: float, chosen: float[], pool_size: int, anneal_rounds: int, delta_high_final: float, hard_fail: bool}`。若文档用表格，追加一行；保持与既有 `sub_pattern` 条目同风格。 + +- [ ] **Step 7: 提交** ```bash -git add app/question_gen/run_store.py tests/unit/test_run_store_selector_scores.py +git add app/question_gen/run_store.py research-wiki/schemas/question-gen-items.md tests/unit/test_run_store_selector_scores.py git commit -m "feat: add selector_scores observation column to question_gen_items" ``` @@ -574,17 +600,22 @@ class SelectorConfig: @dataclass(frozen=True) class SelectorOutcome: - """selector 产出。 + """selector 产出。observation 始终存在（含 hard-fail），供 run_store 落库。 属性: - options: 重组后的四选项（A=正解），格式 ("A. ...", "B. ...", ...)。 - answer: 正解字母（恒为 "A"，后续 postprocess 洗牌）。 - observation: 打分观测 dict（落 run_store selector_scores）。 + observation: 打分观测 dict（correct_score/chosen/pool_size/anneal_rounds/hard_fail）。 + options: 重组四选项（A=正解），hard-fail 时为 None。 + answer: 正解字母（恒 "A"），hard-fail 时为 None。 """ - options: tuple[str, ...] - answer: str observation: dict + options: tuple[str, ...] | None = None + answer: str | None = None + + @property + def hard_fail(self) -> bool: + """是否硬失败（凑不齐 3 个 grounded 干扰项）。""" + return self.options is None def _select_in_interval( @@ -736,7 +767,7 @@ async def test_build_grounded_options_happy_path(): vlm=vlm, question="厨师最终用哪种方式？", correct_text="蒸", material=_Material(), config=cfg, session_id="s", ) - assert out is not None + assert out.hard_fail is False assert out.answer == "A" assert out.options[0] == "A. 蒸" assert {o[3:] for o in out.options[1:]} == {"炒", "煮", "炸"} @@ -744,22 +775,27 @@ async def test_build_grounded_options_happy_path(): @pytest.mark.asyncio -async def test_build_grounded_options_hard_fail_returns_none(): +async def test_build_grounded_options_hard_fail_keeps_observation(): from app.question_gen.distractor_selector import ( SelectorConfig, build_grounded_options, ) - # 所有候选都在负空间（分数极低），退火后仍不足 3 个 → None + # 所有候选都在负空间（分数极低），退火后仍不足 3 个 → hard_fail。 + # VLM 只被调 2 次（首轮 pool+score）+ 1 次退火 pool + 1 次退火 score = 4 次； + # δ_high 放宽轮次是纯重选，不调 VLM。退火 pool 打分含正解，共 4 个分数。 pool = '{"distractors": ["x", "y", "z"]}' scores = '{"scores": [0.90, 0.05, 0.04, 0.03]}' pool2 = '{"distractors": ["p", "q", "r"]}' - scores2 = '{"scores": [0.90, 0.05, 0.04, 0.03, 0.05, 0.04, 0.03]}' - vlm = _FakeVLM([pool, scores, pool2, scores2, pool2, scores2]) + scores2 = '{"scores": [0.90, 0.05, 0.04, 0.03]}' + vlm = _FakeVLM([pool, scores, pool2, scores2]) cfg = SelectorConfig(candidate_pool_size=3, delta_low=0.05, delta_high=0.35) out = await build_grounded_options( vlm=vlm, question="?", correct_text="蒸", material=_Material(), config=cfg, session_id="s", ) - assert out is None + assert out.hard_fail is True + assert out.options is None + assert out.observation["hard_fail"] is True + assert out.observation["pool_size"] == 6 # 首轮 3 + 退火追加 3 ``` - [ ] **Step 8: 跑测试确认失败** @@ -859,11 +895,11 @@ async def build_grounded_options( config: SelectorConfig, *, session_id: str, -) -> SelectorOutcome | None: +) -> SelectorOutcome: """生成候选池 → 视觉打分 → 区间选 3 干扰项 → 重组四选项。 - 退火（凑不齐 3 个时按序）：① 扩池 N→2N 再打分；② 逐步放宽 δ_high； - ③ 仍不足返回 None（调用方走重出）。 + 退火（凑不齐 3 个时按序）：① 追加 N 个候选使池达 2N 再打分；② 逐步放宽 + δ_high（纯重选，不再调 VLM）；③ 仍不足则 hard_fail（调用方走重出）。 参数: vlm: VLM 端口。 @@ -874,7 +910,8 @@ async def build_grounded_options( session_id: 遥测会话 ID。 返回: - SelectorOutcome（A=正解 + 3 grounded 干扰项）或 None（硬失败）。 + SelectorOutcome。成功时 options=A 正解+3 grounded 干扰项；hard_fail + 时 options=None，但 observation 始终存在供落库。 """ candidates = await _generate_pool( vlm, question, correct_text, material, config.candidate_pool_size, session_id=session_id @@ -888,7 +925,7 @@ async def build_grounded_options( correct_score, candidates, cand_scores, config.delta_low, config.delta_high ) - # 退火 1: 扩池一轮 N→2N + # 退火 1: 追加 N 个候选使池达 2N（仅对新增候选打分，正解分保持首轮值） if chosen is None: anneal_rounds += 1 more = await _generate_pool( @@ -932,7 +969,8 @@ async def build_grounded_options( "grounded selector 硬失败: correct={:.3f}, pool={}, anneal={}", correct_score, len(candidates), anneal_rounds, ) - return None + # observation 仍返回，供 pipeline 落 selector_scores（设计 §3.3 退化观测） + return SelectorOutcome(observation=observation) options = ( f"A. {correct_text}", @@ -940,7 +978,7 @@ async def build_grounded_options( f"C. {chosen[1]}", f"D. {chosen[2]}", ) - return SelectorOutcome(options=options, answer="A", observation=observation) + return SelectorOutcome(observation=observation, options=options, answer="A") ``` - [ ] **Step 10: 跑全模块测试确认通过** @@ -988,6 +1026,25 @@ git commit -m "feat: add grounded distractor selector with visual scoring" selector_delta_high=float(section.get("selector_delta_high", 0.35)), ``` +**同步修 CLI seed override**：`tools/generate_questions.py:900` 的 `--seed` 覆盖手工重建 `PipelineConfig`，只复制旧字段会把 selector 三参重置为默认。补三行： + +```python + config = PipelineConfig( + per_type=config.per_type, + retry_limit=config.retry_limit, + heavy_sample_rate=config.heavy_sample_rate, + dedup_threshold=config.dedup_threshold, + concurrency=config.concurrency, + seed=args.seed, + output_dir=config.output_dir, + candidate_pool_size=config.candidate_pool_size, + selector_delta_low=config.selector_delta_low, + selector_delta_high=config.selector_delta_high, + ) +``` + +> 更稳健的等价写法是 `dataclasses.replace(config, seed=args.seed)`；若采用请在文件顶部 `import dataclasses` 或 `from dataclasses import replace`。二选一即可，实现时保持一致。 + - [ ] **Step 2: 写失败测试（正解文本提取 + 织入分流）** 新建 `tests/unit/test_pipeline_selector_wiring.py`。先测纯辅助 `_extract_correct_text`： @@ -1069,13 +1126,17 @@ def _extract_correct_text(options: tuple[str, ...], answer: str) -> str: prev_reason = f"selector_error: {e}" continue - if outcome is None: + # observation 始终落库（含 hard-fail），供 EOB 退化观测与调参 + store.update_selector_scores( + item_id, json.dumps(outcome.observation, ensure_ascii=False) + ) + + if outcome.hard_fail: prev_reason = "grounded 干扰项不足（selector 硬失败）" store.mark_item_rejected(item_id, prev_reason) logger.info("slot {} selector 硬失败 (attempt {})", slot.slot_id, attempt) continue - store.update_selector_scores(item_id, json.dumps(outcome.observation, ensure_ascii=False)) # 用 grounded 四选项替换候选（frozen → 构造新实例） candidate = _replace_candidate_options(candidate, outcome.options, outcome.answer) ``` @@ -1137,15 +1198,38 @@ Expected: PASS selector_delta_high: 0.35 ``` -- [ ] **Step 9: 全量出题相关单测回归** +- [ ] **Step 9: 更新 AR 集成测试 MockVLM（selector 启用后必须能应答 pool/score）** -Run: `conda run -n Video-Tree-TRM pytest tests/unit/test_generate_questions.py tests/unit/test_families.py tests/unit/test_gate.py -v` -Expected: PASS（非 AR 路径不受影响） +`tests/integration/test_pipeline_v2.py` 的 `MockVLM.chat_with_images` 现只区分门控（含 "verdict"）与生成。selector 启用后 AR 路径会额外发 pool 请求（system prompt 含 "distractor"）和 score 请求（含 "grader" / "scores"）。若不识别，pool 会解析成候选 JSON → `_generate_pool` 得空列表 → hard-fail → AR slot 全拒，破坏既有断言。改 `chat_with_images` 顶部按 system prompt 关键词分流： -- [ ] **Step 10: 提交** +```python + async def chat_with_images(self, messages, images, *, session_id=None, parent_call_id=None): + prompt_text = str(messages) + system_text = messages[0].get("content", "") if messages else "" + if "verdict" in prompt_text.lower(): + return _make_llm_response(self._gate_response) + if "distractor" in system_text.lower() and "grader" not in system_text.lower(): + # 候选池请求：返回 4 个 grounded 干扰项 + return _make_llm_response('{"distractors": ["蒸", "煮", "炸", "烤"]}') + if "grader" in system_text.lower(): + # 打分请求：正解高分、3 个落区间、1 个负空间 + return _make_llm_response('{"scores": [0.90, 0.80, 0.70, 0.60, 0.20]}') + idx = min(self._gen_count, len(self._responses) - 1) + self._gen_count += 1 + return _make_llm_response(self._responses[idx]) +``` + +> 打分响应长度需匹配"正解 + 候选数"。若某测试自定义候选池大小，须相应调整该 mock（打分列表长度 = pool 返回的干扰项数 + 1）。默认候选 JSON 4 个 → 打分 5 个，与上面一致。 + +- [ ] **Step 10: 全量出题相关单测 + AR 集成回归** + +Run: `conda run -n Video-Tree-TRM pytest tests/integration/test_pipeline_v2.py tests/unit/test_generate_questions.py tests/unit/test_families.py tests/unit/test_gates.py -v` +Expected: PASS（AR 集成经 selector 仍通过；非 AR 路径不受影响） + +- [ ] **Step 11: 提交** ```bash -git add app/question_gen/pipeline_v2.py config/question_gen_ar30.yaml tests/unit/test_pipeline_selector_wiring.py +git add app/question_gen/pipeline_v2.py config/question_gen_ar30.yaml tools/generate_questions.py tests/unit/test_pipeline_selector_wiring.py tests/integration/test_pipeline_v2.py git commit -m "feat: wire grounded selector into AR slot processing" ``` @@ -1272,6 +1356,8 @@ git commit -m "feat: enforce single-dimension counterfactual in AR distractor ru 放行"错误选项有局部真实证据、但在题干限定（同主体/时点/方式/对象）下为假"的近似干扰项——否则 grounded 干扰项会被 multi_true 误毙。 +> **这是本计划唯一一处授权的公共路径行为变更**（用户在 brainstorming 明确答复"全局松绑"，见设计 §3.5）。它作用于全部 12 题型的 multi_true 门。**无回归破坏风险**：`_gate_multi_true`（`app/question_gen/gates.py:363`）加载 prompt 后调 `llm.chat`，`tests/unit/test_gates.py` 用 mock LLM 返回固定 verdict、不校验 prompt 内容，故 rubric 文案变更不会使既有非 AR 门控测试变红。 + **Files:** - Modify: `store/prompts/question_gen/gate_multi_true.md` - Test: `tests/unit/test_gate_multi_true_rubric.py`（新建） @@ -1326,8 +1412,8 @@ Expected: PASS - [ ] **Step 5: 回归 gate 测试** -Run: `conda run -n Video-Tree-TRM pytest tests/unit/test_gate.py -v` -Expected: PASS +Run: `conda run -n Video-Tree-TRM pytest tests/unit/test_gates.py -v` +Expected: PASS（mock LLM，rubric 文案变更不影响判定断言） - [ ] **Step 6: 提交**