feat: add cheater gate with resume-safe survivor recovery

This commit is contained in:
2026-07-14 16:00:14 -04:00
parent 24ed7ca322
commit d4e9852864
2 changed files with 206 additions and 0 deletions
+119
View File
@@ -12,6 +12,14 @@ from __future__ import annotations
import enum
import hashlib
import json
from typing import TYPE_CHECKING, Protocol
from loguru import logger
if TYPE_CHECKING:
from app.question_gen.adversarial_config import AdversarialFilterConfig
from app.question_gen.run_store import QuestionGenStore
from core.types import GeneratedQuestion
_VALID_LETTERS = ("A", "B", "C", "D")
@@ -94,3 +102,114 @@ def judge_flip(*, p_text: str | None, q_text: str | None) -> FlipDecision:
if p_text.strip() != q_text.strip():
return FlipDecision.PASSED
return FlipDecision.FILTERED_NO_FLIP
class AgentRunner(Protocol):
"""完整 inference agent 试答端口 — Phase B 只依赖此接口(便于 mock)。
实现见 Task 10 的 _RealAgentRunner(复用 run_inference + RunLogImpl)。
"""
model: str
skill_mode: str # 从首次定义即入 Protocol,保证 Task 6/8/10 指纹口径一致
async def predict(
self,
questions: list[GeneratedQuestion],
*,
max_steps: int,
run_id: str,
) -> dict[str, str | None]:
"""跑完整 agent,返回 question_id → 预测答案字母(无预测为 None)。"""
...
def _cheat_hash(question: GeneratedQuestion) -> str:
"""按散参数签名计算作弊门题面 hashquestion/options/answer)。"""
return question_hash(question.question, question.options, question.answer)
def _recover_survivors(
questions: list[GeneratedQuestion],
store: QuestionGenStore,
cfg_fp: str,
) -> list[GeneratedQuestion]:
"""从已落 cheat verdict 恢复"agent 答错"的题(续跑,不重跑 agent)。"""
survivors: list[GeneratedQuestion] = []
for q in questions:
rows = store._conn.execute(
"SELECT agent_correct FROM adversarial_verdicts "
"WHERE question_id=? AND question_hash=? AND stage='cheat' AND agent_config=?",
(q.question_id, _cheat_hash(q), cfg_fp),
).fetchall()
if rows and rows[0][0] == 0:
survivors.append(q)
return survivors
async def run_cheater_gate(
questions: list[GeneratedQuestion],
*,
agent: AgentRunner,
store: QuestionGenStore,
config: AdversarialFilterConfig,
round_no: int,
run_id: str,
) -> list[GeneratedQuestion]:
"""作弊门:完整 agent 试答;答对→filtered_too_easy,答错→cheat passed 待翻转。
续跑:已在当前 hash+config 有 cheat verdict 的题跳过重跑。agent_config 变
化时先作废该题旧 verdict。预测立即落表(崩溃不丢)。
参数:
questions: 待判定的 AR 题列表。
agent: 完整 agent 试答端口。
store: verdict 持久化。
config: 过滤配置(提供 max_steps)。
round_no: 当前轮次。
run_id: agent 推理 run 标识。
返回:
agent 答错的题(进翻转门)——含"已完成续跑恢复的存活者""本轮新判答错者"
两部分合并。答错题的 cheat 预测字母已落 adversarial_verdicts 表
stage='cheat'),翻转门经 `_read_cheat_prediction` 从表读取复用(不重跑)。
"""
cfg_fp = agent_config_fingerprint(
skill_mode=agent.skill_mode,
max_steps=config.adversarial_agent_max_steps,
model=agent.model,
)
todo: list[GeneratedQuestion] = []
completed: list[GeneratedQuestion] = []
for q in questions:
h = _cheat_hash(q)
store.invalidate_stale_config(q.question_id, cfg_fp)
if "cheat" in store.completed_stages(q.question_id, h, cfg_fp):
completed.append(q)
else:
todo.append(q)
# C1: 无条件先从已完成题恢复存活者(agent 答错),再对未判题跑 agent 追加。
# 两者都流向翻转门——绝不因 todo 非空而丢掉已完成的存活者(混合续跑正确性)。
survivors: list[GeneratedQuestion] = _recover_survivors(completed, store, cfg_fp)
if todo:
preds = await agent.predict(
todo, max_steps=config.adversarial_agent_max_steps, run_id=run_id
)
for q in todo:
pred = preds.get(q.question_id)
correct = pred is not None and pred.strip().upper() == q.answer.strip().upper()
verdict = "filtered_too_easy" if correct else "passed"
store.record_verdict(
question_id=q.question_id, question_hash=_cheat_hash(q), stage="cheat",
round=round_no, agent_prediction=pred, agent_correct=correct,
verdict=verdict, pair_id=None, agent_config=cfg_fp,
)
if not correct:
survivors.append(q)
logger.info(
"作弊门: {} 题(续跑复用 {},新判 {})→ 存活 {}",
len(questions), len(completed), len(todo), len(survivors),
)
return survivors