diff --git a/app/harness/split_selection.py b/app/harness/split_selection.py index 56d0cc6..3cd6ad3 100644 --- a/app/harness/split_selection.py +++ b/app/harness/split_selection.py @@ -7,7 +7,7 @@ build_video_records / select_split。 from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field _EVOLUTION_TARGET = { "extraction_failure": "tool", @@ -94,3 +94,95 @@ def score_signal(*, cause_category: str | None, infra: bool, degraded: bool) -> if cause_category == "lapse": return SignalLabel(tier="T1") return SignalLabel(tier="uncertain") + + +@dataclass(frozen=True) +class VideoRecord: + """全视频画像单元(贪心选择器 Task 8 的输入单元)。 + + 覆盖全部视频(含全对、零诊断信号的视频),既承载 test 代表性所需的难度/题型画像, + 也叠加 T2 可训练缺陷的多样性格子,供选择器算覆盖与补集。 + + 字段: + video_id: 视频唯一标识。 + type_set: 该视频所有题的 task_type 集合(去重,画像用)。 + n_correct: 该视频答对题数。 + difficulty: 难度画像桶 = 错题数 = 题数 - n_correct。 + cells: 仅 tier=="T2" 信号行投影的 (task_type, error_type) 主格子并集(去重)。 + wrong_by_type: 各 task_type 的 T2 计数,供选择器 floor 约束(普通 dict)。 + + 实现细节: + frozen 生成的 __hash__ 会遍历各字段;wrong_by_type 为不可哈希 dict, + 故显式标注 hash=False 将其排除出哈希,避免 VideoRecord 入 set/dict 键时报错, + 仍保留其参与相等性比较。 + """ + + video_id: str + type_set: frozenset[str] + n_correct: int + difficulty: int + cells: frozenset[tuple[str, str]] + wrong_by_type: dict[str, int] = field(hash=False) + + +def build_video_records(preds: list[dict], signal_rows: list[dict]) -> list[VideoRecord]: + """由全量 predictions 与诊断信号行构建全视频 VideoRecord 列表。 + + 先按 video_id 聚合全部 predictions(覆盖全对、零信号视频),再叠加仅 tier=="T2" + 的诊断信号为多样性格子与 wrong_by_type 计数。非 T2 信号行(T0/T1/uncertain) + 不计入格子与计数。 + + 参数: + preds: 全量预测行,每行含 video_id / question_id / task_type / correct。 + 每视频含其全部题(不限于错题),correct 为布尔答对标记。 + signal_rows: 诊断信号行,每行含 question_id / task_type / error_type / tier。 + 诊断只覆盖错题子集,正确题无对应信号行属正常,不视为错误。 + + 返回: + 全部视频的 VideoRecord 列表,按视频在 preds 中首次出现顺序排列。 + 无任何 T2 信号的视频其 cells 为空 frozenset、wrong_by_type 为空 dict。 + + 实现细节: + signal_rows 的 question_id 若不在 preds 中则忽略(诊断可能滞后于当前预测集, + 非数据损坏),不 fail-fast;缺失必需键则按 KeyError 直接暴露(不静默兜底)。 + + 异常: + KeyError: preds 或 signal_rows 行缺少必需键(校验前置,防脏数据静默通过)。 + """ + # Phase 1: 按 video_id 聚合 preds(保持首次出现顺序)。 + signal_by_qid = {row["question_id"]: row for row in signal_rows} + aggregates: dict[str, dict] = {} + for pred in preds: + video_id = pred["video_id"] + bucket = aggregates.setdefault( + video_id, {"types": set(), "question_ids": [], "n_correct": 0} + ) + bucket["types"].add(pred["task_type"]) + bucket["question_ids"].append(pred["question_id"]) + if pred["correct"]: + bucket["n_correct"] += 1 + + # Phase 2: 逐视频叠加 T2 信号为格子与 wrong_by_type。 + records: list[VideoRecord] = [] + for video_id, bucket in aggregates.items(): + cells: set[tuple[str, str]] = set() + wrong_by_type: dict[str, int] = {} + for question_id in bucket["question_ids"]: + row = signal_by_qid.get(question_id) + if row is None or row["tier"] != "T2": + continue + task_type = row["task_type"] + cells.add(cell_of(task_type, row["error_type"])) + wrong_by_type[task_type] = wrong_by_type.get(task_type, 0) + 1 + n_questions = len(bucket["question_ids"]) + records.append( + VideoRecord( + video_id=video_id, + type_set=frozenset(bucket["types"]), + n_correct=bucket["n_correct"], + difficulty=n_questions - bucket["n_correct"], + cells=frozenset(cells), + wrong_by_type=wrong_by_type, + ) + ) + return records diff --git a/tests/unit/test_split_selection.py b/tests/unit/test_split_selection.py index 8f750ee..9e24245 100644 --- a/tests/unit/test_split_selection.py +++ b/tests/unit/test_split_selection.py @@ -26,3 +26,38 @@ def test_tiers(): score_signal(cause_category="defect", infra=True, degraded=False).tier == "T0" ) # INFRA 先判 assert score_signal(cause_category=None, infra=False, degraded=True).tier == "uncertain" + + +def test_build_video_records_covers_all_videos_with_difficulty_and_types(): + from app.harness.split_selection import build_video_records + + preds = [ + { + "video_id": "v1", + "question_id": "v1-1", + "task_type": "Counting Problem", + "correct": False, + }, + {"video_id": "v1", "question_id": "v1-2", "task_type": "Action Reasoning", "correct": True}, + {"video_id": "v1", "question_id": "v1-3", "task_type": "OCR Problems", "correct": True}, + {"video_id": "v2", "question_id": "v2-1", "task_type": "Counting Problem", "correct": True}, + {"video_id": "v2", "question_id": "v2-2", "task_type": "Counting Problem", "correct": True}, + {"video_id": "v2", "question_id": "v2-3", "task_type": "Counting Problem", "correct": True}, + ] + signal_rows = [ + { + "question_id": "v1-1", + "task_type": "Counting Problem", + "error_type": "search_failure", + "tier": "T2", + } + ] + recs = build_video_records(preds, signal_rows) + assert {r.video_id for r in recs} == {"v1", "v2"} # 全视频(含零信号 v2) + v1 = next(r for r in recs if r.video_id == "v1") + v2 = next(r for r in recs if r.video_id == "v2") + assert v1.n_correct == 2 and v1.difficulty == 1 # 3题对2 → 难度桶=1错 + assert v2.difficulty == 0 and v2.cells == set() # 零信号视频无 T2 格子 + assert v1.cells == {("Counting Problem", "search_failure")} + assert v1.type_set == {"Counting Problem", "Action Reasoning", "OCR Problems"} + assert v1.wrong_by_type == {"Counting Problem": 1} # T2 计数供 floor