366 lines
14 KiB
Python
366 lines
14 KiB
Python
"""inference pair-level 双向 AND 聚合单元测试（Task 6）。
|
||
|
||
覆盖 question-gen v3 Phase 1 Task 6 的核心契约：
|
||
- 逐题推理不变：每条 GeneratedQuestion 照常各答一次、per-question prediction
|
||
仍逐题落 predictions 表（保留逐题溯源）。
|
||
- pair 按 pair_id 收齐 original + mirror 后合成 1 条 unit-level 记录，
|
||
pair 正确 = (P.pred==P.answer) AND (Q.pred==Q.answer)（双向 AND）。
|
||
- InferenceResult.total / correct / per_task_type 全部按 unit 粒度
|
||
（single 计 1，pair 计 1）。
|
||
- 孤儿 pair（收不齐 2 条）→ 告警 + 剔除该 unit、不计入 total（不静默）。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from typing import Any
|
||
from unittest.mock import AsyncMock
|
||
|
||
import pytest
|
||
from loguru import logger
|
||
|
||
from app.harness.inference import _aggregate_results, run_inference
|
||
from app.harness.log import HarnessLog
|
||
from core.types import GeneratedQuestion, LLMResponse
|
||
|
||
# ── 测试基础设施 ──────────────────────────────────────────────────
|
||
|
||
|
||
def _make_question(
|
||
question_id: str,
|
||
*,
|
||
task_type: str = "Action Reasoning",
|
||
answer: str = "B",
|
||
pair_id: str | None = None,
|
||
question_role: str = "single",
|
||
flip_axis: str | None = None,
|
||
video_id: str = "v1",
|
||
) -> GeneratedQuestion:
|
||
"""构造测试题目；pair_id 非空时视为孪生对成员。"""
|
||
return GeneratedQuestion(
|
||
question_id=question_id,
|
||
video_id=video_id,
|
||
task_type=task_type,
|
||
question="测试问题",
|
||
options=("A. 选项A", "B. 选项B", "C. 选项C", "D. 选项D"),
|
||
answer=answer,
|
||
source_nodes=("L1_001",),
|
||
difficulty="medium",
|
||
pair_id=pair_id,
|
||
question_role=question_role,
|
||
flip_axis=flip_axis if pair_id else None,
|
||
)
|
||
|
||
|
||
def _make_record(
|
||
question_id: str,
|
||
*,
|
||
prediction: str | None,
|
||
answer: str = "B",
|
||
task_type: str = "Action Reasoning",
|
||
steps_used: int = 2,
|
||
prompt_tokens: int = 100,
|
||
completion_tokens: int = 50,
|
||
stop_reason: str = "finished",
|
||
) -> dict[str, Any]:
|
||
"""构造与题目匹配的 prediction record（键与 _run_single_question 一致）。"""
|
||
return {
|
||
"question_id": question_id,
|
||
"prediction": prediction,
|
||
"answer": answer,
|
||
"task_type": task_type,
|
||
"steps_used": steps_used,
|
||
"prompt_tokens": prompt_tokens,
|
||
"completion_tokens": completion_tokens,
|
||
"stop_reason": stop_reason,
|
||
}
|
||
|
||
|
||
def _make_llm_response(answer: str = "B") -> LLMResponse:
|
||
"""构造 submit_answer 场景的 LLMResponse。"""
|
||
content = json.dumps(
|
||
{
|
||
"reflect": {"observation": "找到答案"},
|
||
"plan": {"next_step": "提交"},
|
||
"action": {
|
||
"tool": "submit_answer",
|
||
"args": {"answer": answer, "evidence": "证据", "reasoning": "推理"},
|
||
},
|
||
}
|
||
)
|
||
return LLMResponse(
|
||
content=content,
|
||
thinking="思考",
|
||
model="test-model",
|
||
provider="test",
|
||
prompt_tokens=100,
|
||
completion_tokens=50,
|
||
latency_ms=200,
|
||
ttft_ms=30.0,
|
||
max_inter_token_ms=5.0,
|
||
cache_hit=False,
|
||
call_id="test-call-001",
|
||
)
|
||
|
||
|
||
async def _stub_tool_dispatch(
|
||
tool_name: str, args: dict[str, Any], *, context: dict[str, Any]
|
||
) -> str:
|
||
"""测试用工具调度函数。"""
|
||
if tool_name == "submit_answer":
|
||
return "答案已提交"
|
||
raise ValueError(f"未知工具: {tool_name}")
|
||
|
||
|
||
def _stub_prompt_builder(qa: GeneratedQuestion) -> tuple[str, str]:
|
||
"""测试用 prompt 构建函数。"""
|
||
return "系统提示词", f"用户问题: {qa.question}"
|
||
|
||
|
||
@pytest.fixture
|
||
def harness_log(tmp_path: Any, request: Any) -> HarnessLog:
|
||
"""创建临时 HarnessLog 实例。"""
|
||
db_path = str(tmp_path / f"harness_{id(request)}.db")
|
||
log = HarnessLog(db_path, "test-run")
|
||
yield log
|
||
log.close()
|
||
|
||
|
||
# ── unit 粒度聚合（_aggregate_results 直测） ─────────────────────────
|
||
|
||
|
||
class TestUnitLevelAggregation:
|
||
"""_aggregate_results 按 unit 粒度聚合测试。"""
|
||
|
||
def test_single_units_counted_per_question(self) -> None:
|
||
"""全 single：total = single 数，correct 逐题判定。"""
|
||
questions = [
|
||
_make_question("s1", answer="B"),
|
||
_make_question("s2", answer="A"),
|
||
]
|
||
records = [
|
||
_make_record("s1", prediction="B", answer="B"),
|
||
_make_record("s2", prediction="C", answer="A"),
|
||
]
|
||
result = _aggregate_results(records, questions, "run-single")
|
||
assert result.total == 2
|
||
assert result.correct == 1
|
||
assert abs(result.accuracy - 0.5) < 1e-9
|
||
|
||
def test_pair_both_correct_is_one_correct_unit(self) -> None:
|
||
"""pair 两题皆对 → 1 个 unit、correct=1。"""
|
||
questions = [
|
||
_make_question("po", answer="B", pair_id="p", question_role="pair_original"),
|
||
_make_question("pm", answer="A", pair_id="p", question_role="pair_mirror"),
|
||
]
|
||
records = [
|
||
_make_record("po", prediction="B", answer="B"),
|
||
_make_record("pm", prediction="A", answer="A"),
|
||
]
|
||
result = _aggregate_results(records, questions, "run-pair-ok")
|
||
assert result.total == 1
|
||
assert result.correct == 1
|
||
assert result.accuracy == 1.0
|
||
|
||
def test_pair_one_wrong_fails_by_and(self) -> None:
|
||
"""pair 一题错 → 双向 AND 判 unit 错，correct=0。"""
|
||
questions = [
|
||
_make_question("po", answer="B", pair_id="p", question_role="pair_original"),
|
||
_make_question("pm", answer="A", pair_id="p", question_role="pair_mirror"),
|
||
]
|
||
records = [
|
||
_make_record("po", prediction="B", answer="B"), # 对
|
||
_make_record("pm", prediction="D", answer="A"), # 错
|
||
]
|
||
result = _aggregate_results(records, questions, "run-pair-half")
|
||
assert result.total == 1
|
||
assert result.correct == 0
|
||
assert result.accuracy == 0.0
|
||
|
||
def test_mixed_single_and_pair_unit_total(self) -> None:
|
||
"""single + pair 混合：total = single 数 + pair 数（pair 计 1）。"""
|
||
questions = [
|
||
_make_question("s1", answer="B"),
|
||
_make_question("po", answer="B", pair_id="p", question_role="pair_original"),
|
||
_make_question("pm", answer="A", pair_id="p", question_role="pair_mirror"),
|
||
]
|
||
records = [
|
||
_make_record("s1", prediction="B", answer="B"),
|
||
_make_record("po", prediction="B", answer="B"),
|
||
_make_record("pm", prediction="A", answer="A"),
|
||
]
|
||
result = _aggregate_results(records, questions, "run-mixed")
|
||
assert result.total == 2 # 1 single + 1 pair
|
||
assert result.correct == 2
|
||
|
||
def test_per_task_type_by_unit(self) -> None:
|
||
"""per_task_type 按 unit 计数：pair 归入其 task_type 计 1 个 unit。"""
|
||
questions = [
|
||
_make_question("s1", task_type="SP", answer="B"),
|
||
_make_question(
|
||
"po", task_type="AR", answer="B", pair_id="p", question_role="pair_original"
|
||
),
|
||
_make_question(
|
||
"pm", task_type="AR", answer="A", pair_id="p", question_role="pair_mirror"
|
||
),
|
||
]
|
||
records = [
|
||
_make_record("s1", prediction="B", answer="B", task_type="SP"),
|
||
_make_record("po", prediction="B", answer="B", task_type="AR"),
|
||
_make_record("pm", prediction="C", answer="A", task_type="AR"), # pair 错
|
||
]
|
||
result = _aggregate_results(records, questions, "run-tt")
|
||
assert result.per_task_type["AR"]["total"] == 1 # pair 计 1 个 unit
|
||
assert result.per_task_type["AR"]["correct"] == 0
|
||
assert result.per_task_type["SP"]["total"] == 1
|
||
assert result.per_task_type["SP"]["correct"] == 1
|
||
|
||
def test_orphan_pair_dropped_and_warned(self) -> None:
|
||
"""孤儿 pair（收不齐 2 条）→ 告警 + 剔除、不计入 total。"""
|
||
questions = [
|
||
_make_question("s1", answer="B"),
|
||
_make_question(
|
||
"po", answer="B", pair_id="orphan", question_role="pair_original"
|
||
), # 缺 mirror
|
||
]
|
||
records = [
|
||
_make_record("s1", prediction="B", answer="B"),
|
||
_make_record("po", prediction="B", answer="B"),
|
||
]
|
||
captured: list[str] = []
|
||
sink_id = logger.add(captured.append, level="WARNING", format="{message}")
|
||
try:
|
||
result = _aggregate_results(records, questions, "run-orphan")
|
||
finally:
|
||
logger.remove(sink_id)
|
||
|
||
assert result.total == 1 # 仅 single，孤儿 pair 被剔除
|
||
assert result.correct == 1
|
||
assert any("orphan" in msg for msg in captured), "孤儿 pair 未告警（静默）"
|
||
|
||
def test_pair_with_extra_illegal_role_dropped(self) -> None:
|
||
"""pair_id 下混入额外非法 role 记录（total>2）→ 整对剔除、不计入 total。"""
|
||
questions = [
|
||
_make_question("s1", answer="B"),
|
||
_make_question("po", answer="B", pair_id="p", question_role="pair_original"),
|
||
_make_question("pm", answer="A", pair_id="p", question_role="pair_mirror"),
|
||
# 共享 pair_id 的额外非法记录（重复 original 角色）
|
||
_make_question("px", answer="C", pair_id="p", question_role="pair_original"),
|
||
]
|
||
records = [
|
||
_make_record("s1", prediction="B", answer="B"),
|
||
_make_record("po", prediction="B", answer="B"),
|
||
_make_record("pm", prediction="A", answer="A"),
|
||
_make_record("px", prediction="C", answer="C"),
|
||
]
|
||
captured: list[str] = []
|
||
sink_id = logger.add(captured.append, level="WARNING", format="{message}")
|
||
try:
|
||
result = _aggregate_results(records, questions, "run-illegal-role")
|
||
finally:
|
||
logger.remove(sink_id)
|
||
|
||
assert result.total == 1 # 仅 single 存活，非法配对整对剔除
|
||
assert result.correct == 1
|
||
assert any("total=3" in msg for msg in captured), "非法配对未告警（静默）"
|
||
|
||
def test_unit_missing_prediction_raises_descriptive_error(self) -> None:
|
||
"""unit 缺 prediction → 描述性 ValueError（fail-loud，非静默、非裸 KeyError）。
|
||
|
||
故意破坏聚合不变量（questions 含 s2 但 records 无 s2），验证带上下文报错。
|
||
"""
|
||
questions = [
|
||
_make_question("s1", answer="B"),
|
||
_make_question("s2", answer="A"),
|
||
]
|
||
records = [_make_record("s1", prediction="B", answer="B")] # 缺 s2 的 record
|
||
with pytest.raises(ValueError, match=r"s2.*缺 prediction.*聚合不变量被破坏"):
|
||
_aggregate_results(records, questions, "run-broken-invariant")
|
||
|
||
def test_empty_records_returns_zero(self) -> None:
|
||
"""空 records/questions → 零值结果。"""
|
||
result = _aggregate_results([], [], "run-empty")
|
||
assert result.total == 0
|
||
assert result.correct == 0
|
||
assert result.accuracy == 0.0
|
||
assert result.per_task_type == {}
|
||
|
||
def test_token_and_steps_span_all_records(self) -> None:
|
||
"""token/steps 诊断字段覆盖全部 record（含 pair 两条）。"""
|
||
questions = [
|
||
_make_question("po", answer="B", pair_id="p", question_role="pair_original"),
|
||
_make_question("pm", answer="A", pair_id="p", question_role="pair_mirror"),
|
||
]
|
||
records = [
|
||
_make_record("po", prediction="B", answer="B", steps_used=3, prompt_tokens=100),
|
||
_make_record("pm", prediction="A", answer="A", steps_used=1, prompt_tokens=200),
|
||
]
|
||
result = _aggregate_results(records, questions, "run-diag")
|
||
assert result.token_usage["prompt_tokens"] == 300
|
||
assert abs(result.steps_mean - 2.0) < 1e-9 # (3+1)/2 record 粒度
|
||
|
||
|
||
# ── run_inference 端到端：逐题溯源 + pair 聚合 ──────────────────────
|
||
|
||
|
||
class TestRunInferencePairEndToEnd:
|
||
"""run_inference pair 端到端：逐题落库不变 + unit 级聚合。"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_pair_predictions_persisted_per_question(self, harness_log: HarnessLog) -> None:
|
||
"""pair 两题各自逐题落 predictions（保留逐题溯源），聚合按 unit。"""
|
||
llm = AsyncMock()
|
||
llm.chat.return_value = _make_llm_response(answer="B")
|
||
|
||
questions = [
|
||
_make_question("po", answer="B", pair_id="p", question_role="pair_original"),
|
||
_make_question("pm", answer="B", pair_id="p", question_role="pair_mirror"),
|
||
]
|
||
result = await run_inference(
|
||
questions,
|
||
llm=llm,
|
||
tool_dispatch_fn=_stub_tool_dispatch,
|
||
prompt_builder=_stub_prompt_builder,
|
||
log=harness_log,
|
||
run_id="run-pair-e2e",
|
||
concurrency=2,
|
||
max_steps=10,
|
||
skill_mode="auto",
|
||
)
|
||
|
||
# unit 级：1 个 pair unit，两题皆对 → correct=1
|
||
assert result.total == 1
|
||
assert result.correct == 1
|
||
|
||
# 逐题溯源：predictions 表两条 record 都在（record 显式携带传入的 run_id）
|
||
rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("run-pair-e2e",))
|
||
qids = {r["question_id"] for r in rows}
|
||
assert qids == {"po", "pm"}
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_orphan_pair_excluded_single_survives(self, harness_log: HarnessLog) -> None:
|
||
"""run_inference 中孤儿 pair 被剔除、single 仍计入。"""
|
||
llm = AsyncMock()
|
||
llm.chat.return_value = _make_llm_response(answer="B")
|
||
|
||
questions = [
|
||
_make_question("s1", answer="B"),
|
||
_make_question("po", answer="B", pair_id="orphan", question_role="pair_original"),
|
||
]
|
||
result = await run_inference(
|
||
questions,
|
||
llm=llm,
|
||
tool_dispatch_fn=_stub_tool_dispatch,
|
||
prompt_builder=_stub_prompt_builder,
|
||
log=harness_log,
|
||
run_id="run-orphan-e2e",
|
||
concurrency=2,
|
||
max_steps=10,
|
||
skill_mode="auto",
|
||
)
|
||
|
||
assert result.total == 1 # single 存活，孤儿剔除
|
||
# 逐题溯源：孤儿题仍逐题落库（推理不变；record 显式携带传入的 run_id）
|
||
rows = harness_log.query("SELECT * FROM predictions WHERE run_id = ?", ("run-orphan-e2e",))
|
||
assert {r["question_id"] for r in rows} == {"s1", "po"}
|