fix: extend correctness fail-fast to test-side pool questions (P5)

This commit is contained in:
2026-07-15 12:22:11 -04:00
parent 53989078a0
commit fd907aab46
2 changed files with 33 additions and 7 deletions
+12 -7
View File
@@ -143,7 +143,8 @@ def split_by_video_assignment(
异常:
ValueError: assignment 缺失某题 video_idfail-fast 不静默丢题)、
assignment 取值非法、correctness 缺失 trainval 题、或 val_ratio 越界。
assignment 取值非法、correctness 缺失任一参与 Pools 的题(trainval 或
test)、或 val_ratio 越界。
关键实现细节:
视频组 correctness 取组内全部题的 AND(组内均答对才记为 correct 组),
@@ -170,8 +171,7 @@ def split_by_video_assignment(
baseline_run_id=baseline_run_id,
baseline_val_accuracy=baseline_val_accuracy,
correctness={
q.question_id: correctness.get(q.question_id, False)
for q in test_qs + validation + diagnosis
q.question_id: correctness[q.question_id] for q in test_qs + validation + diagnosis
},
)
@@ -186,24 +186,29 @@ def _partition_by_video_assignment(
参数:
questions: 题目全集。
assignment: video_id -> "trainval" | "test" 归属字典。
correctness: question_id -> 基线是否答对;仅对 trainval 题强制完整。
correctness: question_id -> 基线是否答对;对全部参与 Pools 的题(trainval
与 test 双侧)强制完整。
返回:
(trainval_qs, test_qs) 逐题列表元组,划分依据每题的 video_id 归属。
异常:
ValueError: assignment 取值非法、缺失某题 video_id、或 correctness 缺失
trainval 题(fail-fast,不静默丢题)。
任一参与 Pools 的题(trainval 或 testfail-fast,不静默丢题)。
"""
_assert_valid_assignment(questions, assignment)
trainval_qs = [q for q in questions if assignment[q.video_id] == "trainval"]
test_qs = [q for q in questions if assignment[q.video_id] == "test"]
missing_correctness = [q.question_id for q in trainval_qs if q.question_id not in correctness]
# Pools.correctness 会为 test + validation + diagnosis 全体写入基线对错,
# 故 test 侧同样必须有 correctness,缺失即报错而非静默兜底 False(P5)。
missing_correctness = [
q.question_id for q in trainval_qs + test_qs if q.question_id not in correctness
]
if missing_correctness:
raise ValueError(
f"correctness 缺失 {len(missing_correctness)} trainval 题: {missing_correctness[:5]}"
f"correctness 缺失 {len(missing_correctness)} 道题: {missing_correctness[:5]}"
)
return trainval_qs, test_qs