64 lines
2.7 KiB
Python
64 lines
2.7 KiB
Python
import pytest
|
||
|
||
from app.harness.split_selection import cell_of, evolution_target_of, score_signal
|
||
|
||
|
||
def test_evolution_target_mapping():
|
||
assert evolution_target_of("extraction_failure") == "tool"
|
||
assert evolution_target_of("search_failure") == "skill"
|
||
assert evolution_target_of("reasoning_failure") == "skill"
|
||
assert evolution_target_of("mixed") == "system"
|
||
|
||
|
||
def test_evolution_target_unknown_raises():
|
||
with pytest.raises(ValueError):
|
||
evolution_target_of("unknown_type")
|
||
|
||
|
||
def test_cell_is_task_type_x_error_type():
|
||
assert cell_of("Counting Problem", "search_failure") == ("Counting Problem", "search_failure")
|
||
|
||
|
||
def test_tiers():
|
||
assert score_signal(cause_category="defect", infra=False, degraded=False).tier == "T2"
|
||
assert score_signal(cause_category="lapse", infra=False, degraded=False).tier == "T1"
|
||
assert (
|
||
score_signal(cause_category="defect", infra=True, degraded=False).tier == "T0"
|
||
) # INFRA 先判
|
||
assert score_signal(cause_category=None, infra=False, degraded=True).tier == "uncertain"
|
||
|
||
|
||
def test_build_video_records_covers_all_videos_with_difficulty_and_types():
|
||
from app.harness.split_selection import build_video_records
|
||
|
||
preds = [
|
||
{
|
||
"video_id": "v1",
|
||
"question_id": "v1-1",
|
||
"task_type": "Counting Problem",
|
||
"correct": False,
|
||
},
|
||
{"video_id": "v1", "question_id": "v1-2", "task_type": "Action Reasoning", "correct": True},
|
||
{"video_id": "v1", "question_id": "v1-3", "task_type": "OCR Problems", "correct": True},
|
||
{"video_id": "v2", "question_id": "v2-1", "task_type": "Counting Problem", "correct": True},
|
||
{"video_id": "v2", "question_id": "v2-2", "task_type": "Counting Problem", "correct": True},
|
||
{"video_id": "v2", "question_id": "v2-3", "task_type": "Counting Problem", "correct": True},
|
||
]
|
||
signal_rows = [
|
||
{
|
||
"question_id": "v1-1",
|
||
"task_type": "Counting Problem",
|
||
"error_type": "search_failure",
|
||
"tier": "T2",
|
||
}
|
||
]
|
||
recs = build_video_records(preds, signal_rows)
|
||
assert {r.video_id for r in recs} == {"v1", "v2"} # 全视频（含零信号 v2）
|
||
v1 = next(r for r in recs if r.video_id == "v1")
|
||
v2 = next(r for r in recs if r.video_id == "v2")
|
||
assert v1.n_correct == 2 and v1.difficulty == 1 # 3题对2 → 难度桶=1错
|
||
assert v2.difficulty == 0 and v2.cells == set() # 零信号视频无 T2 格子
|
||
assert v1.cells == {("Counting Problem", "search_failure")}
|
||
assert v1.type_set == {"Counting Problem", "Action Reasoning", "OCR Problems"}
|
||
assert v1.wrong_by_type == {"Counting Problem": 1} # T2 计数供 floor
|