chore: snapshot in-progress question-gen work before preflight fixes
This commit is contained in:
@@ -81,18 +81,11 @@ RETRIEVAL_FAMILY = QuestionFamilySpec(
|
||||
),
|
||||
legal_task_types=frozenset(
|
||||
[
|
||||
"Action Recognition",
|
||||
"Action Reasoning",
|
||||
"Action Prediction",
|
||||
"Action Sequence",
|
||||
"Object Recognition",
|
||||
"Object Reasoning",
|
||||
"Object Interaction",
|
||||
"Scene Understanding",
|
||||
"Event Reasoning",
|
||||
"Causal Reasoning",
|
||||
"Temporal Reasoning",
|
||||
"Spatial Reasoning",
|
||||
"Action Recognition",
|
||||
"Attribute Perception",
|
||||
"OCR Problems",
|
||||
]
|
||||
),
|
||||
leak_profile=LeakTestProfile(
|
||||
@@ -116,10 +109,7 @@ REASONING_FAMILY = QuestionFamilySpec(
|
||||
[
|
||||
"Action Reasoning",
|
||||
"Object Reasoning",
|
||||
"Event Reasoning",
|
||||
"Causal Reasoning",
|
||||
"Temporal Reasoning",
|
||||
"Spatial Reasoning",
|
||||
"Information Synopsis",
|
||||
]
|
||||
),
|
||||
leak_profile=LeakTestProfile(
|
||||
@@ -141,10 +131,10 @@ ENUMERATION_FAMILY = QuestionFamilySpec(
|
||||
),
|
||||
legal_task_types=frozenset(
|
||||
[
|
||||
"Action Sequence",
|
||||
"Object Recognition",
|
||||
"Object Interaction",
|
||||
"Scene Understanding",
|
||||
"Counting Problem",
|
||||
"Temporal Reasoning",
|
||||
"Temporal Perception",
|
||||
"Information Synopsis",
|
||||
]
|
||||
),
|
||||
leak_profile=LeakTestProfile(
|
||||
@@ -166,10 +156,10 @@ VISUAL_FAMILY = QuestionFamilySpec(
|
||||
),
|
||||
legal_task_types=frozenset(
|
||||
[
|
||||
"Object Recognition",
|
||||
"Scene Understanding",
|
||||
"Attribute Perception",
|
||||
"Counting Problem",
|
||||
"OCR Problems",
|
||||
"Action Recognition",
|
||||
"Spatial Reasoning",
|
||||
]
|
||||
),
|
||||
leak_profile=LeakTestProfile(
|
||||
@@ -191,9 +181,8 @@ SPATIAL_FAMILY = QuestionFamilySpec(
|
||||
),
|
||||
legal_task_types=frozenset(
|
||||
[
|
||||
"Spatial Perception",
|
||||
"Spatial Reasoning",
|
||||
"Object Interaction",
|
||||
"Scene Understanding",
|
||||
]
|
||||
),
|
||||
leak_profile=LeakTestProfile(
|
||||
|
||||
+10
-15
@@ -124,19 +124,6 @@ def stratified_sample(
|
||||
return flatten_units(sampled)
|
||||
|
||||
|
||||
def _unit_correct(unit: QuestionUnit, correctness: dict[str, bool]) -> bool:
|
||||
"""单元级正确性:成员全部答对才算对(缺失按 False,宽松口径)。
|
||||
|
||||
参数:
|
||||
unit: 目标单元(single 1 题,pair 2 题)。
|
||||
correctness: question_id -> 基线是否答对。
|
||||
|
||||
返回:
|
||||
pair 走双向 AND、single 即单题正确性;任一成员缺失或答错即 False。
|
||||
"""
|
||||
return all(correctness.get(q.question_id, False) for q in unit.questions)
|
||||
|
||||
|
||||
def _ratio_stratified_sample(
|
||||
pool: list[QuestionUnit],
|
||||
correctness: dict[str, bool],
|
||||
@@ -158,9 +145,17 @@ def _ratio_stratified_sample(
|
||||
|
||||
异常:
|
||||
ValueError: 对单元或错单元层不足。
|
||||
|
||||
关键实现:
|
||||
unit_correctness 采用函数内延迟导入:loader 属 question_gen,
|
||||
question_units 属 harness,模块级导入将触发循环依赖(沿用 build_units /
|
||||
flatten_units 的既有做法)。以 strict=False 保持"缺基线对错即视为未答对"的
|
||||
原 loose 语义不变。
|
||||
"""
|
||||
correct = [u for u in pool if _unit_correct(u, correctness)]
|
||||
wrong = [u for u in pool if not _unit_correct(u, correctness)]
|
||||
from app.harness.question_units import unit_correctness
|
||||
|
||||
correct = [u for u in pool if unit_correctness(u, correctness, strict=False)]
|
||||
wrong = [u for u in pool if not unit_correctness(u, correctness, strict=False)]
|
||||
n_correct = round(size * correct_ratio)
|
||||
n_wrong = size - n_correct
|
||||
if len(correct) < n_correct or len(wrong) < n_wrong:
|
||||
|
||||
Reference in New Issue
Block a user