feat: add adversarial round loop with backfill iteration and difficulty report

2026-07-14 16:40:05 -04:00
parent c1565a01c2
commit 73d0bb9190
2 changed files with 305 additions and 1 deletions
@@ -0,0 +1,165 @@
+"""Task 9：final 全量重写 + 补生成迭代循环 + 难度报告。
+
+覆盖：
+- write_final_bank 仅含当前 hash+config 过两门的题、stale-config 旧行被排除、原子重写。
+- run_adversarial_rounds 缺额驱动：deficit≤0 或 round≥max → 停止（fake backfill 计数）。
+- _report_difficulty：agent 正确率 > 阈值 → WARNING。
+"""
+
+import json
+
+import pytest
+from loguru import logger
+
+from app.question_gen.adversarial_config import AdversarialFilterConfig
+from app.question_gen.adversarial_filter import (
+    _report_difficulty,
+    question_hash,
+    run_adversarial_rounds,
+    write_final_bank,
+)
+from app.question_gen.run_store import QuestionGenStore
+from core.types import GeneratedQuestion
+
+
+def _q(qid, sub=None):
+    """构造一条 AR 题；sub=None → 不支持 flip，翻转门直接放行（无需 VLM/树）。"""
+    return GeneratedQuestion(
+        question_id=qid,
+        video_id="v1",
+        task_type="Action Recognition",
+        question=f"{qid} 之前做了什么？",
+        options=("A. 蒸", "B. 炒", "C. 煮", "D. 炸"),
+        answer="A",
+        source_nodes=("n1",),
+        difficulty="hard",
+        sub_pattern=sub,
+    )
+
+
+class _FakeAgent:
+    """完整 agent 试答桩：对每题返回固定预测（AgentRunner Protocol 要求 model/skill_mode）。"""
+
+    def __init__(self, pred="B", model="m1", skill_mode="auto"):
+        self._pred = pred
+        self.model = model
+        self.skill_mode = skill_mode
+        self.calls: list[str] = []
+
+    async def predict(self, questions, *, max_steps, run_id):
+        self.calls.extend(q.question_id for q in questions)
+        return {q.question_id: self._pred for q in questions}
+
+
+class _CountingBackfill:
+    """补生成回调桩：记录调用次数，按工厂返回新题。"""
+
+    def __init__(self, factory):
+        self.calls = 0
+        self._factory = factory
+
+    async def __call__(self, deficit, round_no, existing):
+        self.calls += 1
+        return self._factory(deficit, round_no)
+
+
+def test_write_final_bank_only_passed(tmp_path):
+    store = QuestionGenStore(str(tmp_path / "q.db"))
+    q1, q2 = _q("q1"), _q("q2")
+    store.record_verdict(
+        question_id="q1", question_hash=question_hash(q1), stage="cheat",
+        round=0, agent_prediction="B", agent_correct=False,
+        verdict="passed", pair_id=None, agent_config="c",
+    )
+    store.record_verdict(
+        question_id="q2", question_hash=question_hash(q2), stage="cheat",
+        round=0, agent_prediction="A", agent_correct=True,
+        verdict="filtered_too_easy", pair_id=None, agent_config="c",
+    )
+    out = tmp_path / "accepted_questions_final.json"
+    write_final_bank(out, store, {"q1": q1, "q2": q2}, "c")
+    data = json.loads(out.read_text(encoding="utf-8"))
+    assert [d["question_id"] for d in data] == ["q1"]
+    store.close()
+
+
+def test_write_final_bank_excludes_stale_config(tmp_path):
+    store = QuestionGenStore(str(tmp_path / "q.db"))
+    q1 = _q("q1")
+    store.record_verdict(
+        question_id="q1", question_hash=question_hash(q1), stage="cheat",
+        round=0, agent_prediction="B", agent_correct=False,
+        verdict="passed", pair_id=None, agent_config="OLD",
+    )
+    out = tmp_path / "accepted_questions_final.json"
+    write_final_bank(out, store, {"q1": q1}, "NEW")
+    assert json.loads(out.read_text(encoding="utf-8")) == []
+    store.close()
+
+
+def test_difficulty_warns_above_threshold(tmp_path):
+    store = QuestionGenStore(str(tmp_path / "q.db"))
+    for i in range(4):  # 4 题全对 = 正确率 1.0 > 阈值 0.85 → 必触发告警
+        store.record_verdict(
+            question_id=f"q{i}", question_hash=str(i), stage="cheat",
+            round=0, agent_prediction="A", agent_correct=True,
+            verdict="filtered_too_easy", pair_id=None, agent_config="c",
+        )
+    # loguru 不走标准 logging，用项目既定 sink 捕获模式（见 test_pool_strategy）。
+    captured: list[str] = []
+    sink_id = logger.add(lambda msg: captured.append(str(msg)), level="WARNING")
+    try:
+        _report_difficulty(store, round_no=0, threshold=0.85)
+    finally:
+        logger.remove(sink_id)
+    assert any("太简单" in m or "简单" in m for m in captured), f"未捕获告警: {captured}"
+    store.close()
+
+
+@pytest.mark.asyncio
+async def test_rounds_stop_when_deficit_met(tmp_path):
+    """首轮即达标（passed≥target）→ 不调 backfill，迭代立即停止。"""
+    store = QuestionGenStore(str(tmp_path / "q.db"))
+    agent = _FakeAgent(pred="B")  # 答错 → 过作弊门；sub=None → 过翻转门
+    backfill = _CountingBackfill(lambda d, r: [])
+    final_path = tmp_path / "accepted_questions_final.json"
+    await run_adversarial_rounds(
+        [_q("q0")],
+        agent=agent,
+        vlm=object(),
+        store=store,
+        trees={},
+        config=AdversarialFilterConfig(adversarial_max_rounds=5),
+        final_path=final_path,
+        target=1,
+        backfill=backfill,
+        session_id="s",
+    )
+    assert backfill.calls == 0
+    assert [d["question_id"] for d in json.loads(final_path.read_text(encoding="utf-8"))] == ["q0"]
+    store.close()
+
+
+@pytest.mark.asyncio
+async def test_rounds_backfill_then_stop_at_max(tmp_path):
+    """缺额 > 0 → 调 backfill 补生成；达轮次上限即停（不无限循环）。"""
+    store = QuestionGenStore(str(tmp_path / "q.db"))
+    agent = _FakeAgent(pred="B")
+    backfill = _CountingBackfill(lambda d, r: [_q(f"bf{r}_{i}") for i in range(d)])
+    final_path = tmp_path / "accepted_questions_final.json"
+    await run_adversarial_rounds(
+        [_q("q0")],
+        agent=agent,
+        vlm=object(),
+        store=store,
+        trees={},
+        config=AdversarialFilterConfig(adversarial_max_rounds=2),
+        final_path=final_path,
+        target=99,  # 永远达不到 → 靠 max_rounds 终止
+        backfill=backfill,
+        session_id="s",
+    )
+    assert backfill.calls == 1  # round0 补生成一次；round1 达上限 break，不再补
+    ids = {d["question_id"] for d in json.loads(final_path.read_text(encoding="utf-8"))}
+    assert "q0" in ids and any(x.startswith("bf0_") for x in ids)
+    store.close()