feat: wire real agent runner, backfill assembly and adversarial-filter CLI

2026-07-14 16:58:32 -04:00
parent 73d0bb9190
commit 1d222d9f18
3 changed files with 734 additions and 1 deletions
@@ -0,0 +1,277 @@
+"""Task 10：run_adversarial_filter 顶层入口 + _RealAgentRunner 真实装配 e2e。
+
+覆盖两条路径：
+- run_adversarial_filter 编排（mock agent + mock backfill）：过滤 filter_task_types、
+  非 AR 题不进 agent 门、final 仅含 passed、断点续跑不重跑已判题。
+- _RealAgentRunner 真实装配 smoke（I6）：predict 经 run_inference 落 predictions 表再
+  读回（LLM mock，路径真穿过 HarnessLog / RunLogImpl，非假 runner 短路）。
+"""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+from unittest.mock import AsyncMock, MagicMock
+
+import numpy as np
+import pytest
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+from app.harness.deps_router import InferenceDepsRouter
+from app.question_gen.adversarial_config import AdversarialFilterConfig
+from app.question_gen.adversarial_filter import _RealAgentRunner, run_adversarial_filter
+from app.question_gen.run_store import QuestionGenStore
+from core.types import GeneratedQuestion, LLMResponse
+
+
+def _ar_q(qid: str, video_id: str = "v1") -> dict:
+    """构造一条 AR 题 JSON 记录；sub_pattern 缺省 → 不支持 flip，翻转门直接放行。"""
+    return {
+        "question_id": qid,
+        "video_id": video_id,
+        "task_type": "Action Recognition",
+        "question": f"{qid} 之前做了什么？",
+        "options": ["A. 蒸", "B. 炒", "C. 煮", "D. 炸"],
+        "answer": "A",
+        "source_nodes": ["n1"],
+        "difficulty": "hard",
+    }
+
+
+def _non_ar_q(qid: str, video_id: str = "v1") -> dict:
+    """构造一条非 AR 题 JSON 记录（不应进 agent 门）。"""
+    return {
+        "question_id": qid,
+        "video_id": video_id,
+        "task_type": "Object Recognition",
+        "question": f"{qid} 里的物体是什么？",
+        "options": ["A. 锅", "B. 碗", "C. 盘", "D. 勺"],
+        "answer": "A",
+        "source_nodes": ["n2"],
+        "difficulty": "hard",
+    }
+
+
+class _FakeAgent:
+    """完整 agent 试答桩：对每题返回固定预测，记录被调用的 question_id。"""
+
+    def __init__(self, pred: str = "B", model: str = "m1", skill_mode: str = "auto") -> None:
+        self._pred = pred
+        self.model = model
+        self.skill_mode = skill_mode
+        self.calls: list[str] = []
+
+    async def predict(
+        self, questions: list[GeneratedQuestion], *, max_steps: int, run_id: str
+    ) -> dict[str, str]:
+        self.calls.extend(q.question_id for q in questions)
+        return {q.question_id: self._pred for q in questions}
+
+
+class _NoBackfill:
+    """补生成回调桩：记录调用次数，永远返回空（首轮即达标时不应被调用）。"""
+
+    def __init__(self) -> None:
+        self.calls = 0
+
+    async def __call__(
+        self, deficit: int, round_no: int, existing: dict[str, GeneratedQuestion]
+    ) -> list[GeneratedQuestion]:
+        self.calls += 1
+        return []
+
+
+def _write_accepted(path: Path, records: list[dict]) -> None:
+    """写 accepted_questions.json（Phase A 产物形态：JSON 列表）。"""
+    path.write_text(json.dumps(records, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+@pytest.mark.asyncio
+async def test_filter_only_processes_ar_and_writes_passed(tmp_path: Path) -> None:
+    """非 AR 题不进 agent 门；final 仅含 passed AR 题。"""
+    accepted = tmp_path / "accepted_questions.json"
+    _write_accepted(accepted, [_ar_q("q1"), _ar_q("q2"), _non_ar_q("obj1")])
+    final_path = tmp_path / "accepted_questions_final.json"
+    store = QuestionGenStore(str(tmp_path / "q.db"))
+    agent = _FakeAgent(pred="B")  # 答错 → 过作弊门；sub_pattern=None → 过翻转门
+    backfill = _NoBackfill()
+
+    await run_adversarial_filter(
+        accepted_path=accepted,
+        final_path=final_path,
+        agent=agent,
+        vlm=object(),
+        trees={},
+        store=store,
+        filter_config=AdversarialFilterConfig(adversarial_max_rounds=3),
+        backfill=backfill,
+        session_id="s",
+    )
+
+    # 只有 AR 题进 agent 门
+    assert set(agent.calls) == {"q1", "q2"}
+    # 非 AR 题不出现在 verdicts 表
+    rows = store._conn.execute(
+        "SELECT question_id FROM adversarial_verdicts WHERE question_id=?", ("obj1",)
+    ).fetchall()
+    assert rows == []
+    # final 仅含 passed AR 题
+    data = json.loads(final_path.read_text(encoding="utf-8"))
+    assert {d["question_id"] for d in data} == {"q1", "q2"}
+    assert backfill.calls == 0  # 首轮即达标（target=2, passed=2）
+    store.close()
+
+
+@pytest.mark.asyncio
+async def test_filter_resume_does_not_rerun_judged(tmp_path: Path) -> None:
+    """断点续跑：第二次调用不重跑已判题（agent 调用计数不变）。"""
+    accepted = tmp_path / "accepted_questions.json"
+    _write_accepted(accepted, [_ar_q("q1"), _ar_q("q2")])
+    final_path = tmp_path / "accepted_questions_final.json"
+    store = QuestionGenStore(str(tmp_path / "q.db"))
+    agent = _FakeAgent(pred="B")
+    backfill = _NoBackfill()
+
+    async def _run() -> None:
+        await run_adversarial_filter(
+            accepted_path=accepted,
+            final_path=final_path,
+            agent=agent,
+            vlm=object(),
+            trees={},
+            store=store,
+            filter_config=AdversarialFilterConfig(adversarial_max_rounds=3),
+            backfill=backfill,
+            session_id="s",
+        )
+
+    await _run()
+    first_calls = list(agent.calls)
+    await _run()
+    assert agent.calls == first_calls  # 第二次未新增 agent 调用
+    store.close()
+
+
+# ---------------------------------------------------------------------------
+# I6：_RealAgentRunner 真实装配 smoke（LLM mock，路径真穿过 predictions 表）
+# ---------------------------------------------------------------------------
+
+
+class _MockLLM:
+    """最小 LLM 桩：一步即产出 submit_answer，让真实 AgentLoop 稳定收敛。"""
+
+    def __init__(self, answer: str) -> None:
+        self._answer = answer
+
+    async def chat(
+        self,
+        messages: list[dict[str, Any]],
+        *,
+        session_id: str | None = None,
+        parent_call_id: str | None = None,
+    ) -> LLMResponse:
+        content = json.dumps(
+            {"action": {"tool": "submit_answer", "args": {"answer": self._answer}}}
+        )
+        return LLMResponse(
+            content=content,
+            thinking="",
+            model="mock",
+            provider="mock",
+            prompt_tokens=1,
+            completion_tokens=1,
+            latency_ms=1,
+            ttft_ms=None,
+            max_inter_token_ms=None,
+            cache_hit=False,
+            call_id="mock-call",
+        )
+
+
+def _build_real_router(tmp_path: Path) -> InferenceDepsRouter:
+    """构建真实 InferenceDepsRouter（仅 embed/vlm 打桩，router/deps 装配全真实）。"""
+    vid = "smoke_vid"
+    vid_dir = tmp_path / "videos" / vid
+    (vid_dir / "frames").mkdir(parents=True)
+    minimal_tree = {
+        "metadata": {"source_path": "test", "modality": "video"},
+        "roots": [
+            {
+                "id": "L1_000",
+                "card": {
+                    "scene_summary": "s",
+                    "main_setting": "s",
+                    "key_entities": [],
+                    "main_actions": [],
+                    "topic_keywords": [],
+                    "visible_text": [],
+                    "temporal_flow": "s",
+                },
+                "time_range": [0, 10],
+                "children": [],
+            }
+        ],
+    }
+    (vid_dir / "tree.json").write_text(json.dumps(minimal_tree))
+    prompts_dir = tmp_path / "prompts"
+    prompts_dir.mkdir()
+    (prompts_dir / "system.md").write_text("You are a search agent.")
+
+    fake_embed = MagicMock()
+    fake_embed.dim = 4
+    fake_embed.embed = lambda t: np.zeros((1, 4), dtype=np.float32)
+
+    return InferenceDepsRouter(
+        store_dir=tmp_path,
+        embed_provider=fake_embed,
+        llm=_MockLLM(answer="B"),
+        vlm=AsyncMock(),
+        ocr=None,
+        default_prompts_dir=prompts_dir,
+        default_skills_dir=None,
+        skill_mode="none",
+        verify_vision=False,
+        anchor=False,
+        assemble_mode="ids",
+    )
+
+
+def _smoke_q(qid: str) -> GeneratedQuestion:
+    """构造 smoke 题（video_id 对应真实 router 的树 fixture）。"""
+    return GeneratedQuestion(
+        question_id=qid,
+        video_id="smoke_vid",
+        task_type="Action Recognition",
+        question="他之前做了什么？",
+        options=("A. 蒸", "B. 炒", "C. 煮", "D. 炸"),
+        answer="A",
+        source_nodes=("L1_000",),
+        difficulty="hard",
+    )
+
+
+@pytest.mark.asyncio
+async def test_real_agent_runner_predict_roundtrips_predictions(tmp_path: Path) -> None:
+    """真实装配 smoke：predict 经 run_inference 落 predictions 表再读回（LLM mock）。"""
+    router = _build_real_router(tmp_path)
+    runner = _RealAgentRunner(
+        llm=_MockLLM(answer="B"),
+        tool_dispatch_fn=router.create_dispatch(),
+        prompt_builder=router.create_prompt_builder(),
+        db_path=str(tmp_path / "harness.db"),
+        concurrency=1,
+        skill_mode="none",
+        model="mock",
+    )
+    preds = await runner.predict([_smoke_q("smoke")], max_steps=2, run_id="smoke_r0")
+    assert preds["smoke"] == "B"  # 真的从 predictions 表读回，非 mock 直返
+
+    # 断言确实写进了 predictions 表（穿过 HarnessLog / RunLogImpl）
+    from app.harness.log import RunLogImpl
+
+    rows = await RunLogImpl(str(tmp_path / "harness.db")).get_predictions(
+        "smoke_r0", question_ids=["smoke"]
+    )
+    assert rows and rows[0]["prediction"] == "B"