iomgaa
b497db97ba
Codex Task 8 审查修复： - Important 1: _satisfy_floors 每步移入前检查预算，floor 需求超 n_trainval 时 抛 InfeasibleSplitError（fail loud），保证 trainval 永不超额挤占 test；补预算超限测试。 - Important 2: 确定性测试末尾用 _epsilon_ok 断言产出 test 真满足 ε（两维偏差回归护栏）， 并断言 trainval <= n_trainval。 - Minor: fixture docstring 注明结构真实 / signal 二次构造，正式运行由真实诊断替换。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
240 lines
8.8 KiB
Python
240 lines
8.8 KiB
Python
import sqlite3
|
||
from collections import Counter
|
||
from itertools import cycle
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from app.harness.split_selection import (
|
||
build_video_records,
|
||
cell_of,
|
||
evolution_target_of,
|
||
score_signal,
|
||
)
|
||
|
||
# T2 信号构造用的错误类别循环源（4 类真实 error_type，制造多样格子）。
|
||
_ERROR_TYPES = ["extraction_failure", "search_failure", "reasoning_failure", "mixed"]
|
||
# 二次构造 T2 信号覆盖的高信号题型（floor / diversity 需有料）。
|
||
_SIGNAL_TYPES = {"Counting Problem", "Action Reasoning"}
|
||
_HARNESS_DB = Path(__file__).resolve().parents[2] / "workspaces" / "default" / "harness.db"
|
||
|
||
|
||
def _load_adhoc_predictions() -> list[dict]:
|
||
"""从真实 harness.db 的 infer_adhoc run 读取 900 条预测（按 question_id 去重）。
|
||
|
||
返回:
|
||
preds 列表，每行含 video_id / question_id / task_type / correct，
|
||
correct = (prediction 非空 且 prediction == answer)。
|
||
"""
|
||
con = sqlite3.connect(str(_HARNESS_DB))
|
||
try:
|
||
cur = con.execute(
|
||
"SELECT question_id, video_id, task_type, prediction, answer "
|
||
"FROM predictions WHERE run_id = 'infer_adhoc'"
|
||
)
|
||
seen: set[str] = set()
|
||
preds: list[dict] = []
|
||
for question_id, video_id, task_type, prediction, answer in cur.fetchall():
|
||
if question_id in seen:
|
||
continue
|
||
seen.add(question_id)
|
||
preds.append(
|
||
{
|
||
"video_id": video_id,
|
||
"question_id": question_id,
|
||
"task_type": task_type,
|
||
"correct": prediction is not None and prediction == answer,
|
||
}
|
||
)
|
||
return preds
|
||
finally:
|
||
con.close()
|
||
|
||
|
||
def _real_shaped_video_records() -> list:
|
||
"""用真实预测结构 + 二次构造 T2 信号构建 300 个 VideoRecord。
|
||
|
||
注意：视频 type / difficulty 结构真实（取自真实 infer_adhoc 预测），signal 叠加是二次构造
|
||
（pre-diagnosis 阶段固有限制，真实诊断尚未跑）；正式运行时 T2 信号由真实诊断产物替换。
|
||
这里对高信号题型（Counting / Action Reasoning）的错题标 T2 defect，error_type 循环取 4 类，
|
||
使 floor 约束有料、多样性格子有区分度。
|
||
|
||
返回:
|
||
全部 300 个真实结构的 VideoRecord 列表。
|
||
"""
|
||
preds = _load_adhoc_predictions()
|
||
err = cycle(_ERROR_TYPES)
|
||
signal_rows: list[dict] = []
|
||
for pred in preds:
|
||
if pred["task_type"] in _SIGNAL_TYPES and not pred["correct"]:
|
||
signal_rows.append(
|
||
{
|
||
"question_id": pred["question_id"],
|
||
"task_type": pred["task_type"],
|
||
"error_type": next(err),
|
||
"tier": "T2",
|
||
}
|
||
)
|
||
return build_video_records(preds, signal_rows)
|
||
|
||
|
||
def _count_questions_by_type(videos: list) -> dict:
|
||
"""统计各 task_type 的视频承载数（含该题型的视频数），供 derive_reportable_types。
|
||
|
||
VideoRecord 只留题型去重集 type_set，故这里按"含该题型的视频数"聚合，
|
||
作为长尾报告门限的代理度量。
|
||
|
||
参数:
|
||
videos: VideoRecord 列表。
|
||
|
||
返回:
|
||
{task_type: 含该题型的视频数}。
|
||
"""
|
||
counter: Counter = Counter()
|
||
for video in videos:
|
||
counter.update(video.type_set)
|
||
return dict(counter)
|
||
|
||
|
||
def test_evolution_target_mapping():
|
||
assert evolution_target_of("extraction_failure") == "tool"
|
||
assert evolution_target_of("search_failure") == "skill"
|
||
assert evolution_target_of("reasoning_failure") == "skill"
|
||
assert evolution_target_of("mixed") == "system"
|
||
|
||
|
||
def test_evolution_target_unknown_raises():
|
||
with pytest.raises(ValueError):
|
||
evolution_target_of("unknown_type")
|
||
|
||
|
||
def test_cell_is_task_type_x_error_type():
|
||
assert cell_of("Counting Problem", "search_failure") == ("Counting Problem", "search_failure")
|
||
|
||
|
||
def test_tiers():
|
||
assert score_signal(cause_category="defect", infra=False, degraded=False).tier == "T2"
|
||
assert score_signal(cause_category="lapse", infra=False, degraded=False).tier == "T1"
|
||
assert (
|
||
score_signal(cause_category="defect", infra=True, degraded=False).tier == "T0"
|
||
) # INFRA 先判
|
||
assert score_signal(cause_category=None, infra=False, degraded=True).tier == "uncertain"
|
||
|
||
|
||
def test_build_video_records_covers_all_videos_with_difficulty_and_types():
|
||
from app.harness.split_selection import build_video_records
|
||
|
||
preds = [
|
||
{
|
||
"video_id": "v1",
|
||
"question_id": "v1-1",
|
||
"task_type": "Counting Problem",
|
||
"correct": False,
|
||
},
|
||
{"video_id": "v1", "question_id": "v1-2", "task_type": "Action Reasoning", "correct": True},
|
||
{"video_id": "v1", "question_id": "v1-3", "task_type": "OCR Problems", "correct": True},
|
||
{"video_id": "v2", "question_id": "v2-1", "task_type": "Counting Problem", "correct": True},
|
||
{"video_id": "v2", "question_id": "v2-2", "task_type": "Counting Problem", "correct": True},
|
||
{"video_id": "v2", "question_id": "v2-3", "task_type": "Counting Problem", "correct": True},
|
||
]
|
||
signal_rows = [
|
||
{
|
||
"question_id": "v1-1",
|
||
"task_type": "Counting Problem",
|
||
"error_type": "search_failure",
|
||
"tier": "T2",
|
||
}
|
||
]
|
||
recs = build_video_records(preds, signal_rows)
|
||
assert {r.video_id for r in recs} == {"v1", "v2"} # 全视频（含零信号 v2）
|
||
v1 = next(r for r in recs if r.video_id == "v1")
|
||
v2 = next(r for r in recs if r.video_id == "v2")
|
||
assert v1.n_correct == 2 and v1.difficulty == 1 # 3题对2 → 难度桶=1错
|
||
assert v2.difficulty == 0 and v2.cells == set() # 零信号视频无 T2 格子
|
||
assert v1.cells == {("Counting Problem", "search_failure")}
|
||
assert v1.type_set == {"Counting Problem", "Action Reasoning", "OCR Problems"}
|
||
assert v1.wrong_by_type == {"Counting Problem": 1} # T2 计数供 floor
|
||
|
||
|
||
def test_select_split_video_disjoint_and_floor_and_deterministic():
|
||
from app.harness.split_selection import (
|
||
SelectConfig,
|
||
_epsilon_ok,
|
||
derive_reportable_types,
|
||
select_split,
|
||
)
|
||
|
||
videos = _real_shaped_video_records()
|
||
total_by_type = _count_questions_by_type(videos)
|
||
cfg = SelectConfig(
|
||
n_trainval=100,
|
||
floor_k={"Counting Problem": 3},
|
||
epsilon=0.1,
|
||
reportable_types=derive_reportable_types(total_by_type, report_floor=27),
|
||
seed=7,
|
||
)
|
||
a = select_split(videos, config=cfg)
|
||
b = select_split(videos, config=cfg)
|
||
assert set(a.trainval) & set(a.test) == set() # 视频级不相交
|
||
assert set(a.trainval) | set(a.test) == {v.video_id for v in videos} # 补集覆盖全集
|
||
assert a.trainval == b.trainval # 同 seed 同 config → 同解
|
||
trainval_ids = set(a.trainval)
|
||
counting_defects = sum(
|
||
v.wrong_by_type.get("Counting Problem", 0) for v in videos if v.video_id in trainval_ids
|
||
)
|
||
assert counting_defects >= 3 # floor 硬约束满足
|
||
assert len(a.trainval) <= cfg.n_trainval # trainval 永不超预算
|
||
test_ids = set(a.test)
|
||
test_records = [v for v in videos if v.video_id in test_ids]
|
||
assert _epsilon_ok(test_records, videos, cfg) # 产出 test 真满足 ε（回归护栏）
|
||
|
||
|
||
def test_infeasible_floor_vs_epsilon_raises():
|
||
from app.harness.split_selection import (
|
||
InfeasibleSplitError,
|
||
SelectConfig,
|
||
select_split,
|
||
)
|
||
|
||
videos = _real_shaped_video_records()
|
||
with pytest.raises(InfeasibleSplitError): # 极小 ε + 高 floor → 死锁
|
||
select_split(
|
||
videos,
|
||
config=SelectConfig(
|
||
n_trainval=2,
|
||
floor_k={"OCR Problems": 50},
|
||
epsilon=0.001,
|
||
reportable_types=set(),
|
||
seed=1,
|
||
),
|
||
)
|
||
|
||
|
||
def test_infeasible_floor_exceeds_n_trainval_budget_raises():
|
||
from app.harness.split_selection import (
|
||
InfeasibleSplitError,
|
||
SelectConfig,
|
||
select_split,
|
||
)
|
||
|
||
# floor 需求（Action Reasoning 48 缺陷）远超 n_trainval=1 预算：单视频至多带 1 题 AR 缺陷，
|
||
# 无法在 1 个 trainval 名额内满足 floor=40 → 抛 InfeasibleSplitError（预算不足）。
|
||
videos = _real_shaped_video_records()
|
||
with pytest.raises(InfeasibleSplitError):
|
||
select_split(
|
||
videos,
|
||
config=SelectConfig(
|
||
n_trainval=1,
|
||
floor_k={"Action Reasoning": 40},
|
||
epsilon=1.0,
|
||
reportable_types=set(),
|
||
seed=3,
|
||
),
|
||
)
|
||
|
||
|
||
def test_derive_reportable_types():
|
||
from app.harness.split_selection import derive_reportable_types
|
||
|
||
assert derive_reportable_types({"A": 30, "B": 10}, report_floor=27) == {"A"}
|