feat: wire real agent runner, backfill assembly and adversarial-filter CLI
This commit is contained in:
@@ -9,6 +9,7 @@ shortcut：作弊门（agent 秒杀=太简单，剔除）+ 翻转门（agent 答
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import enum
|
||||
import hashlib
|
||||
import json
|
||||
@@ -22,11 +23,16 @@ from loguru import logger
|
||||
from core.types import GeneratedQuestion
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
import numpy as np
|
||||
|
||||
from app.question_gen.adversarial_config import AdversarialFilterConfig
|
||||
from app.question_gen.pipeline_v2 import PipelineConfig
|
||||
from app.question_gen.run_store import QuestionGenStore
|
||||
from app.question_gen.sampler_v2 import MaterialContext
|
||||
from app.tree.index import TreeIndex
|
||||
from core.protocols import VLMProvider
|
||||
from core.protocols import LLMProvider, VLMProvider
|
||||
|
||||
_VALID_LETTERS = ("A", "B", "C", "D")
|
||||
|
||||
@@ -757,3 +763,278 @@ async def run_adversarial_rounds(
|
||||
all_questions[q.question_id] = q
|
||||
pending = new_qs # 只对新补的题重新过滤（已判题走续跑）
|
||||
logger.info("对抗过滤结束: final={} 题（target={}）", passed_now, target)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 10: 真实 agent 装配 — _RealAgentRunner（run_inference + RunLogImpl 回读）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _RealAgentRunner:
|
||||
"""AgentRunner 实现 — 复用 run_inference + RunLogImpl 读回预测。
|
||||
|
||||
每次 predict 开一次 HarnessLog（按 run_id 幂等 upsert），跑完整 agent 推理把
|
||||
prediction 落 predictions 表，再用只读 RunLogImpl 按 (run_id, question_ids) 读回。
|
||||
prediction 是 submit_answer 的答案字母（A/B/C/D）；agent 报错/未提交时该行
|
||||
prediction 为 None，据此保守处理（不误判）。
|
||||
|
||||
参数:
|
||||
llm: 推理 LLMProvider（共享注入）。
|
||||
tool_dispatch_fn: InferenceDepsRouter.create_dispatch() 返回的调度闭包。
|
||||
prompt_builder: InferenceDepsRouter.create_prompt_builder() 返回的构建闭包。
|
||||
db_path: HarnessLog / RunLogImpl 的 sqlite 路径。
|
||||
concurrency: run_inference 并发数（asyncio.Semaphore 容量）。
|
||||
skill_mode: skill 模式（"auto"/"manual"/"none"）——同时是指纹来源，须与
|
||||
router 的 skill_mode 一致，保证续跑/失效口径稳定。
|
||||
model: 推理模型名——指纹来源之一。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
llm: LLMProvider,
|
||||
tool_dispatch_fn: Callable[..., object],
|
||||
prompt_builder: Callable[[GeneratedQuestion], tuple[str, str]],
|
||||
db_path: str,
|
||||
concurrency: int,
|
||||
skill_mode: str,
|
||||
model: str,
|
||||
) -> None:
|
||||
self._llm = llm
|
||||
self._dispatch = tool_dispatch_fn
|
||||
self._builder = prompt_builder
|
||||
self._db_path = db_path
|
||||
self._concurrency = concurrency
|
||||
self.skill_mode = skill_mode
|
||||
self.model = model
|
||||
|
||||
async def predict(
|
||||
self,
|
||||
questions: list[GeneratedQuestion],
|
||||
*,
|
||||
max_steps: int,
|
||||
run_id: str,
|
||||
) -> dict[str, str | None]:
|
||||
"""跑完整 agent，回读 predictions 表，返回 question_id → 预测字母（无预测 None）。
|
||||
|
||||
参数:
|
||||
questions: 待推理题目列表。
|
||||
max_steps: AgentLoop 单题最大步数。
|
||||
run_id: 本次推理 run 标识（predictions 表按此过滤回读）。
|
||||
|
||||
返回:
|
||||
question_id → 预测答案字母（缺失/未提交为 None）。
|
||||
"""
|
||||
from app.harness.inference import run_inference
|
||||
from app.harness.log import HarnessLog, RunLogImpl
|
||||
|
||||
with HarnessLog(self._db_path, run_id) as log:
|
||||
await run_inference(
|
||||
questions,
|
||||
llm=self._llm,
|
||||
tool_dispatch_fn=self._dispatch,
|
||||
prompt_builder=self._builder,
|
||||
log=log,
|
||||
run_id=run_id,
|
||||
concurrency=self._concurrency,
|
||||
max_steps=max_steps,
|
||||
skill_mode=self.skill_mode,
|
||||
)
|
||||
rows = await RunLogImpl(self._db_path).get_predictions(
|
||||
run_id, question_ids=[q.question_id for q in questions]
|
||||
)
|
||||
return {r["question_id"]: r["prediction"] for r in rows}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 10: 真实 backfill 装配 — 缺额驱动 run_pipeline_v2
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _max_existing_seq(existing: dict[str, GeneratedQuestion]) -> int:
|
||||
"""从已有题 question_id（格式 "{video_id}_{task_type}_{seq:04d}"）取最大 seq。
|
||||
|
||||
只解析末段为纯数字的 id；无可解析 id → 0（新 run 从 seq_offset+1 起编，防撞。）
|
||||
"""
|
||||
max_seq = 0
|
||||
for qid in existing:
|
||||
tail = qid.rsplit("_", 1)[-1]
|
||||
if tail.isdigit():
|
||||
max_seq = max(max_seq, int(tail))
|
||||
return max_seq
|
||||
|
||||
|
||||
def build_backfill(
|
||||
*,
|
||||
trees: dict[str, TreeIndex],
|
||||
vlm: VLMProvider,
|
||||
llm: LLMProvider,
|
||||
embed_fn: Callable[[str], np.ndarray],
|
||||
store: QuestionGenStore,
|
||||
pipeline_config: PipelineConfig,
|
||||
filter_task_types: tuple[str, ...],
|
||||
session_id: str,
|
||||
) -> BackfillFn:
|
||||
"""组装真实 backfill 回调 — 缺额驱动 run_pipeline_v2，返回新增 AR 题。
|
||||
|
||||
闭包捕获 run_pipeline_v2 全部依赖。每轮按缺额 deficit 用 dataclasses.replace 生成
|
||||
per_type=deficit 的新 config（PipelineConfig frozen，绝不原地改），并传：
|
||||
- initial_used_node_ids = 已有题 source_nodes 并集（避开已用节点）；
|
||||
- initial_embed_pool = 已有题 embedding（跨 run 去重）；
|
||||
- seq_offset = 已有题最大 seq（新题续编，防撞 question_id）。
|
||||
仅补 filter_task_types（当前锁定为 AR 单类），返回 PipelineResult.accepted。
|
||||
|
||||
参数:
|
||||
trees: video_id → 三层树索引（生成素材来源）。
|
||||
vlm: VLM 端口。
|
||||
llm: LLM 端口（门控用）。
|
||||
embed_fn: 文本嵌入函数。
|
||||
store: 出题持久化。
|
||||
pipeline_config: ar30 原配置（除 per_type 外全部继承）。
|
||||
filter_task_types: 补生成的题型（仅这些）。
|
||||
session_id: 遥测会话 ID（透传给管线子调用）。
|
||||
|
||||
返回:
|
||||
BackfillFn 闭包。
|
||||
"""
|
||||
from app.question_gen.pipeline_v2 import run_pipeline_v2
|
||||
|
||||
video_ids = list(trees.keys())
|
||||
|
||||
async def _backfill(
|
||||
deficit: int,
|
||||
round_no: int,
|
||||
existing: dict[str, GeneratedQuestion],
|
||||
) -> list[GeneratedQuestion]:
|
||||
"""按缺额补生成 deficit 道 AR 题。"""
|
||||
run_cfg = dataclasses.replace(pipeline_config, per_type=deficit)
|
||||
used_node_ids: set[str] = set()
|
||||
for q in existing.values():
|
||||
used_node_ids.update(q.source_nodes)
|
||||
embed_pool = [embed_fn(q.question).flatten() for q in existing.values()]
|
||||
seq_offset = _max_existing_seq(existing)
|
||||
logger.info(
|
||||
"backfill round={}: deficit={}, seq_offset={}, used_nodes={}",
|
||||
round_no,
|
||||
deficit,
|
||||
seq_offset,
|
||||
len(used_node_ids),
|
||||
)
|
||||
result = await run_pipeline_v2(
|
||||
video_ids=video_ids,
|
||||
trees=trees,
|
||||
vlm=vlm,
|
||||
llm=llm,
|
||||
embed_fn=embed_fn,
|
||||
store=store,
|
||||
config=run_cfg,
|
||||
task_types=list(filter_task_types),
|
||||
initial_used_node_ids=used_node_ids,
|
||||
initial_embed_pool=embed_pool,
|
||||
seq_offset=seq_offset,
|
||||
)
|
||||
return result.accepted
|
||||
|
||||
return _backfill
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 10: 顶层入口 — run_adversarial_filter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_filter_questions(
|
||||
accepted_path: Path, filter_task_types: tuple[str, ...]
|
||||
) -> list[GeneratedQuestion]:
|
||||
"""从 accepted_questions.json 读题并过滤到 filter_task_types（只读，不改 Phase A）。
|
||||
|
||||
accepted_questions.json 是 Phase A 产物（JSON 列表）；仅 filter_task_types 的题
|
||||
进 agent 门，其余原样留在 Phase A 文件中（本层不触碰）。
|
||||
|
||||
参数:
|
||||
accepted_path: accepted_questions.json 路径。
|
||||
filter_task_types: 需过滤的题型集合。
|
||||
|
||||
返回:
|
||||
过滤后的题目列表（保持文件内顺序）。
|
||||
|
||||
异常:
|
||||
FileNotFoundError: 文件不存在。
|
||||
ValueError: JSON 结构非列表。
|
||||
"""
|
||||
if not accepted_path.exists():
|
||||
raise FileNotFoundError(f"accepted_questions.json 不存在: {accepted_path}")
|
||||
raw = json.loads(accepted_path.read_text(encoding="utf-8"))
|
||||
if not isinstance(raw, list):
|
||||
raise ValueError(f"accepted_questions.json 顶层结构应为列表: {accepted_path}")
|
||||
types = set(filter_task_types)
|
||||
questions: list[GeneratedQuestion] = []
|
||||
for item in raw:
|
||||
if item.get("task_type") not in types:
|
||||
continue
|
||||
questions.append(
|
||||
GeneratedQuestion(
|
||||
question_id=item["question_id"],
|
||||
video_id=item["video_id"],
|
||||
task_type=item["task_type"],
|
||||
question=item["question"],
|
||||
options=tuple(item["options"]),
|
||||
answer=item["answer"],
|
||||
source_nodes=tuple(item.get("source_nodes", ())),
|
||||
difficulty=item.get("difficulty", "medium"),
|
||||
family=item.get("family"),
|
||||
skill_target=item.get("skill_target"),
|
||||
difficulty_steps=item.get("difficulty_steps"),
|
||||
sub_pattern=item.get("sub_pattern"),
|
||||
)
|
||||
)
|
||||
return questions
|
||||
|
||||
|
||||
async def run_adversarial_filter(
|
||||
*,
|
||||
accepted_path: Path,
|
||||
final_path: Path,
|
||||
agent: AgentRunner,
|
||||
vlm: VLMProvider,
|
||||
trees: dict[str, TreeIndex],
|
||||
store: QuestionGenStore,
|
||||
filter_config: AdversarialFilterConfig,
|
||||
backfill: BackfillFn,
|
||||
session_id: str,
|
||||
) -> None:
|
||||
"""Phase B 顶层入口：读 accepted_questions.json 过滤 AR，跑两门 + 补生成迭代。
|
||||
|
||||
路径隔离：只处理 filter_task_types 的题，accepted_questions.json 只读，Phase A
|
||||
状态机不受影响。target 锁定为首轮过滤出的 AR 题数（缺额 = target - passed）。
|
||||
|
||||
参数:
|
||||
accepted_path: Phase A 产物 accepted_questions.json 路径（只读）。
|
||||
final_path: 最终题库 JSON 路径（每轮全量原子重写）。
|
||||
agent: 完整 agent 试答端口（Task 10 的 _RealAgentRunner，单测可 mock）。
|
||||
vlm: 镜像题生成 VLM 端口。
|
||||
trees: video_id → 三层树索引（重建镜像素材 / 补生成用）。
|
||||
store: verdict 持久化。
|
||||
filter_config: 后置对抗过滤配置（题型 / 轮次 / max_steps / 难度阈值）。
|
||||
backfill: 补生成回调（CLI 用 build_backfill 组装真实实现，单测可 mock）。
|
||||
session_id: 遥测会话 ID（派生各门 run_id）。
|
||||
"""
|
||||
initial_questions = _load_filter_questions(accepted_path, filter_config.filter_task_types)
|
||||
target = len(initial_questions)
|
||||
logger.info(
|
||||
"对抗过滤启动: 过滤题型={}, 首轮 AR 题数={}",
|
||||
filter_config.filter_task_types,
|
||||
target,
|
||||
)
|
||||
await run_adversarial_rounds(
|
||||
initial_questions,
|
||||
agent=agent,
|
||||
vlm=vlm,
|
||||
store=store,
|
||||
trees=trees,
|
||||
config=filter_config,
|
||||
final_path=final_path,
|
||||
target=target,
|
||||
backfill=backfill,
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user