"""翻转门四路径:passed / filtered_no_flip / flip_skipped / 镜像不入库。""" import json import pytest from app.question_gen.adversarial_config import AdversarialFilterConfig from app.question_gen.adversarial_filter import ( agent_config_fingerprint, question_hash, run_flip_gate, write_final_bank, ) from app.question_gen.run_store import QuestionGenStore from core.types import GeneratedQuestion, LLMResponse class _FakeAgent: """复用 Task 6 语义;带 skill_mode 属性(AgentRunner Protocol 要求)。""" def __init__(self, preds, model="m1", skill_mode="auto"): self._preds = preds self.model = model self.skill_mode = skill_mode self.calls: list[str] = [] async def predict(self, questions, *, max_steps, run_id): self.calls.extend(q.question_id for q in questions) return {q.question_id: self._preds.get(q.question_id) for q in questions} class _FakeVLM: def __init__(self, content): self._content = content async def chat_with_images(self, messages, images, *, session_id=None, parent_call_id=None): return LLMResponse( content=self._content, thinking="", model="fake", provider="fake", prompt_tokens=0, completion_tokens=0, latency_ms=0, ttft_ms=None, max_inter_token_ms=None, cache_hit=False, call_id="c", ) class _FakeMaterial: subtitle_sentences = ["先炒后蒸"] frame_paths = ["/f1.jpg"] def _q(qid, sub="temporal_reasoning_failure"): return GeneratedQuestion( question_id=qid, video_id="v1", task_type="Action Recognition", question="X 之前做了什么?", options=("A. 蒸", "B. 炒", "C. 煮", "D. 炸"), answer="A", source_nodes=("n1",), difficulty="hard", sub_pattern=sub, ) def _fp(): return agent_config_fingerprint(skill_mode="auto", max_steps=40, model="m1") @pytest.fixture(autouse=True) def _stub_material(monkeypatch): """隔离建树素材重建,直接给镜像生成喂假素材。""" monkeypatch.setattr( "app.question_gen.adversarial_filter._rebuild_material", lambda tree, source_nodes: _FakeMaterial(), ) def _preset_cheat(store, q, pred="A"): """预置作弊门 P 预测行(翻转门须复用它,不重跑 agent)。""" store.record_verdict( question_id=q.question_id, question_hash=question_hash(q), stage="cheat", round=0, agent_prediction=pred, agent_correct=False, verdict="passed", pair_id=None, agent_config=_fp(), ) @pytest.mark.asyncio async def test_flip_gate_answer_flips_passed(tmp_path): store = QuestionGenStore(str(tmp_path / "q.db")) q = _q("hard") _preset_cheat(store, q, pred="A") # P canonical="蒸" agent = _FakeAgent({"hard_mirror": "A"}) # 镜像洗牌后 A=炒 → canonical≠蒸 vlm = _FakeVLM( json.dumps( { "mirror": { "question": "X 之后?", "options": ["A. 炒", "B. 蒸", "C. 煮", "D. 炸"], "answer": "A", } }, ensure_ascii=False, ) ) kept = await run_flip_gate( [q], agent=agent, vlm=vlm, store=store, trees={"v1": object()}, config=AdversarialFilterConfig(), round_no=0, run_id="r0", session_id="s", ) assert {x.question_id for x in kept} == {"hard"} assert agent.calls == ["hard_mirror"] # C2: 原题 P 未被重跑,只跑镜像 cheat = store._conn.execute( "SELECT verdict FROM adversarial_verdicts WHERE question_id='hard' AND stage='cheat'" ).fetchone()[0] assert cheat == "passed" # flip_original + flip_mirror 两条独立 stage 行,共用同一 pair_id(非空),verdict=passed by_stage = { r[0]: (r[1], r[2]) for r in store._conn.execute( "SELECT stage, verdict, pair_id FROM adversarial_verdicts " "WHERE question_id='hard' AND stage IN ('flip_original', 'flip_mirror')" ) } assert set(by_stage) == {"flip_original", "flip_mirror"} assert by_stage["flip_original"][0] == "passed" assert by_stage["flip_mirror"][0] == "passed" assert by_stage["flip_original"][1] # pair_id 非空 assert by_stage["flip_original"][1] == by_stage["flip_mirror"][1] # 共用同一 pair_id assert "hard" in store.final_passed_question_ids({"hard": question_hash(q)}, _fp()) store.close() @pytest.mark.asyncio async def test_flip_gate_same_answer_filtered(tmp_path): store = QuestionGenStore(str(tmp_path / "q.db")) q = _q("stick") _preset_cheat(store, q, pred="A") # P canonical="蒸" agent = _FakeAgent({"stick_mirror": "A"}) # 镜像 A=蒸 → canonical 与 P 相同 vlm = _FakeVLM( json.dumps( { "mirror": { "question": "X 之后?", "options": ["A. 蒸", "B. 炒", "C. 煮", "D. 炸"], "answer": "B", } }, ensure_ascii=False, ) ) kept = await run_flip_gate( [q], agent=agent, vlm=vlm, store=store, trees={"v1": object()}, config=AdversarialFilterConfig(), round_no=0, run_id="r0", session_id="s", ) assert kept == [] # 未随问题翻转 → 剔除 cheat = store._conn.execute( "SELECT verdict FROM adversarial_verdicts WHERE question_id='stick' AND stage='cheat'" ).fetchone()[0] assert cheat == "filtered_no_flip" # cheat 行被改写 → final 不含它 # flip_original + flip_mirror 两条独立 stage 行,共用同一 pair_id(非空),verdict 一致 by_stage = { r[0]: (r[1], r[2]) for r in store._conn.execute( "SELECT stage, verdict, pair_id FROM adversarial_verdicts " "WHERE question_id='stick' AND stage IN ('flip_original', 'flip_mirror')" ) } assert set(by_stage) == {"flip_original", "flip_mirror"} assert by_stage["flip_original"][0] == "filtered_no_flip" assert by_stage["flip_mirror"][0] == "filtered_no_flip" assert by_stage["flip_original"][1] # pair_id 非空 assert by_stage["flip_original"][1] == by_stage["flip_mirror"][1] # 共用同一 pair_id # 改写后的 cheat 行不再计入终判 passed(双保险:cheat 改判 + filtered_no_flip 排除) assert "stick" not in store.final_passed_question_ids( {"stick": question_hash(q)}, _fp() ) store.close() @pytest.mark.asyncio async def test_flip_gate_invalid_mirror_skipped_but_kept(tmp_path): store = QuestionGenStore(str(tmp_path / "q.db")) q = _q("murky") _preset_cheat(store, q, pred="A") agent = _FakeAgent({}) # 镜像造不出 → agent 不该被调用 vlm = _FakeVLM('{"mirror": null}') kept = await run_flip_gate( [q], agent=agent, vlm=vlm, store=store, trees={"v1": object()}, config=AdversarialFilterConfig(), round_no=0, run_id="r0", session_id="s", ) assert {x.question_id for x in kept} == {"murky"} # 退回只经作弊门,保留不误杀 assert agent.calls == [] # 镜像 None → 未跑 agent cheat = store._conn.execute( "SELECT verdict FROM adversarial_verdicts WHERE question_id='murky' AND stage='cheat'" ).fetchone()[0] assert cheat == "passed" store.close() @pytest.mark.asyncio async def test_flip_gate_mirror_ok_but_agent_refuses_skipped(tmp_path): """镜像生成成功但 agent 对镜像 Q 拒答(None) → flip_skipped,survivor 保留不误杀。 与 invalid_mirror 用例互补:此处镜像题**造得出来**(走到 agent),只因 agent 无 有效预测→canonical(Q)=None→保守 flip_skipped,绝不据此误杀原题。 """ store = QuestionGenStore(str(tmp_path / "q.db")) q = _q("refuse") _preset_cheat(store, q, pred="A") # P canonical="蒸" agent = _FakeAgent({}) # 镜像题无预测 → predict 返回 None(拒答) vlm = _FakeVLM( json.dumps( { "mirror": { "question": "X 之后?", "options": ["A. 炒", "B. 蒸", "C. 煮", "D. 炸"], "answer": "A", } }, ensure_ascii=False, ) ) kept = await run_flip_gate( [q], agent=agent, vlm=vlm, store=store, trees={"v1": object()}, config=AdversarialFilterConfig(), round_no=0, run_id="r0", session_id="s", ) assert {x.question_id for x in kept} == {"refuse"} # 保守保留,不误杀 assert agent.calls == ["refuse_mirror"] # 镜像被跑一次,C2:原题 P 未被重跑 by_stage = { r[0]: r[1] for r in store._conn.execute( "SELECT stage, verdict FROM adversarial_verdicts " "WHERE question_id='refuse' AND stage IN ('flip_original', 'flip_mirror')" ) } assert by_stage == {"flip_original": "flip_skipped", "flip_mirror": "flip_skipped"} cheat = store._conn.execute( "SELECT verdict FROM adversarial_verdicts WHERE question_id='refuse' AND stage='cheat'" ).fetchone()[0] assert cheat == "passed" # cheat 行保持 passed(保留只经作弊门) # flip_skipped 不算 filtered_no_flip → 仍计入终判 passed(保留题最终可入库) assert "refuse" in store.final_passed_question_ids( {"refuse": question_hash(q)}, _fp() ) store.close() @pytest.mark.asyncio async def test_flip_gate_mirror_excluded_and_unsupported_passes(tmp_path): store = QuestionGenStore(str(tmp_path / "q.db")) q = _q("hard") # 支持 flip npq = _q("plain", sub="premature_evidence_anchoring") # 不支持 flip _preset_cheat(store, q, pred="A") _preset_cheat(store, npq, pred="B") agent = _FakeAgent({"hard_mirror": "A"}) vlm = _FakeVLM( json.dumps( { "mirror": { "question": "X 之后?", "options": ["A. 炒", "B. 蒸", "C. 煮", "D. 炸"], "answer": "A", } }, ensure_ascii=False, ) ) kept = await run_flip_gate( [q, npq], agent=agent, vlm=vlm, store=store, trees={"v1": object()}, config=AdversarialFilterConfig(), round_no=0, run_id="r0", session_id="s", ) assert {x.question_id for x in kept} == {"hard", "plain"} # 不支持 flip 直接 passed assert agent.calls == ["hard_mirror"] # 不支持 flip 的题不跑 agent/VLM out = tmp_path / "final.json" write_final_bank(out, store, {"hard": q, "plain": npq}, _fp()) ids = [d["question_id"] for d in json.loads(out.read_text(encoding="utf-8"))] assert "hard_mirror" not in ids and set(ids) == {"hard", "plain"} # 镜像不入题库 store.close()