chore: snapshot in-progress question-gen work before preflight fixes
This commit is contained in:
@@ -27,7 +27,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from app.harness.question_units import build_units
|
||||
from app.harness.question_units import build_units, unit_correctness
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
@@ -65,20 +65,6 @@ class LadderEntry:
|
||||
p_hat: float
|
||||
|
||||
|
||||
def _unit_correct(unit: QuestionUnit, correctness: dict[str, bool]) -> bool:
|
||||
"""单元级正确性：AR pair 双向 AND，single 即单题；单元错 = 任一成员错。
|
||||
|
||||
参数:
|
||||
unit: 目标单元。
|
||||
correctness: question_id -> 是否答对（缺项按未答对处理，与迁移前
|
||||
correctness.get(qid, False) 的默认语义一致，不改判定）。
|
||||
|
||||
返回:
|
||||
单元内所有成员均答对时 True，否则 False。
|
||||
"""
|
||||
return all(correctness.get(q.question_id, False) for q in unit.questions)
|
||||
|
||||
|
||||
def build_cold_entries(
|
||||
units: list[QuestionUnit],
|
||||
correctness: dict[str, bool],
|
||||
@@ -90,7 +76,7 @@ def build_cold_entries(
|
||||
参数:
|
||||
units: 该题型的全部候选单元（已排除 test 池；AR pair 已折叠成单元）。
|
||||
correctness: question_id -> 种子基线是否答对（900 题全量逐题对错）。
|
||||
单元级对错由 _unit_correct 折叠（任一成员错 → 单元错）。
|
||||
单元级对错由 unit_correctness(strict=False) 折叠（任一成员错 → 单元错）。
|
||||
probe_quota: 从错 unit 中随机抽出插到梯尾的探针比例（防"解锁新能力"盲区）。
|
||||
seed: 洗牌种子，保证确定性重建。
|
||||
|
||||
@@ -104,8 +90,8 @@ def build_cold_entries(
|
||||
错错对 2:1 交错（一方耗尽后顺排另一方）-> 探针追加尾部。
|
||||
"""
|
||||
rng = random.Random(seed)
|
||||
wrong = [u for u in units if not _unit_correct(u, correctness)]
|
||||
right = [u for u in units if _unit_correct(u, correctness)]
|
||||
wrong = [u for u in units if not unit_correctness(u, correctness, strict=False)]
|
||||
right = [u for u in units if unit_correctness(u, correctness, strict=False)]
|
||||
rng.shuffle(wrong)
|
||||
rng.shuffle(right)
|
||||
|
||||
@@ -125,7 +111,7 @@ def build_cold_entries(
|
||||
interleaved.extend(probes)
|
||||
|
||||
def _p0(u: QuestionUnit) -> float:
|
||||
return 2 / 3 if _unit_correct(u, correctness) else 1 / 3
|
||||
return 2 / 3 if unit_correctness(u, correctness, strict=False) else 1 / 3
|
||||
|
||||
return [LadderEntry(u.unit_id, _p0(u)) for u in interleaved]
|
||||
|
||||
|
||||
@@ -96,24 +96,33 @@ def validate_units(units: list[QuestionUnit]) -> list[QuestionUnit]:
|
||||
return units
|
||||
|
||||
|
||||
def unit_correctness(unit: QuestionUnit, per_q: dict[str, bool]) -> bool:
|
||||
def unit_correctness(unit: QuestionUnit, per_q: dict[str, bool], *, strict: bool = True) -> bool:
|
||||
"""计算单元级正确性：AR pair 走双向 AND，single 即单题正确性。
|
||||
|
||||
参数:
|
||||
unit: 目标单元。
|
||||
per_q: 题目 question_id → 该题是否作答正确的映射。
|
||||
strict: 缺键策略。True（默认）时以 per_q[q.question_id] 取值，缺任一题
|
||||
触发 KeyError（防静默兜底，强制上游先补齐全部单题结果）；False 时以
|
||||
per_q.get(q.question_id, False) 取值，缺键计 False（宽松口径，供池
|
||||
构建 / gate 冷启动 / 采样等"缺基线对错即视为未答对"的调用点复用）。
|
||||
|
||||
返回:
|
||||
单元内所有题目均正确时为 True，否则 False。
|
||||
|
||||
关键实现:
|
||||
直接以 per_q[q.question_id] 取值，缺任一题触发 KeyError（防静默兜底），
|
||||
强制上游先补齐全部单题结果再计单元正确性。
|
||||
pool 构建（pools）、gate 冷启动（gate_ladder）、分层采样（loader）三处
|
||||
原各自持有的 loose 版 _unit_correct 副本统一收敛到本函数 strict=False 分支，
|
||||
消除重复逻辑与 missing-key 策略分叉。
|
||||
"""
|
||||
return all(per_q[q.question_id] for q in unit.questions)
|
||||
if strict:
|
||||
return all(per_q[q.question_id] for q in unit.questions)
|
||||
return all(per_q.get(q.question_id, False) for q in unit.questions)
|
||||
|
||||
|
||||
def unit_correctness_view(units: list[QuestionUnit], per_q: dict[str, bool]) -> dict[str, bool]:
|
||||
def unit_correctness_view(
|
||||
units: list[QuestionUnit], per_q: dict[str, bool], *, strict: bool = True
|
||||
) -> dict[str, bool]:
|
||||
"""把逐题对错折叠成单元级视图：unit_id → 单元是否整体正确。
|
||||
|
||||
进化引擎（gate e-process / quadrant / probation / pair_block / compute_accuracy）
|
||||
@@ -123,13 +132,15 @@ def unit_correctness_view(units: list[QuestionUnit], per_q: dict[str, bool]) ->
|
||||
参数:
|
||||
units: 目标单元列表（single 或 pair）。
|
||||
per_q: 题目 question_id → 该题是否作答正确（唯一逐题溯源来源）。
|
||||
strict: 缺键策略，透传给 unit_correctness。True（默认）缺任一题 raise
|
||||
KeyError；False 缺键计 False（宽松口径）。
|
||||
|
||||
返回:
|
||||
unit_id → 单元级正确性。single 的 unit_id 等于其 question_id，
|
||||
pair 的 unit_id 等于共享 pair_id。
|
||||
|
||||
关键实现:
|
||||
逐单元复用 unit_correctness（内部以 per_q[q.question_id] 取值，缺任一题
|
||||
触发 KeyError），禁静默兜底、强制上游先补齐全部单题结果。
|
||||
逐单元复用 unit_correctness（strict 透传），默认 strict 禁静默兜底、
|
||||
强制上游先补齐全部单题结果。
|
||||
"""
|
||||
return {u.unit_id: unit_correctness(u, per_q) for u in units}
|
||||
return {u.unit_id: unit_correctness(u, per_q, strict=strict) for u in units}
|
||||
|
||||
@@ -81,18 +81,11 @@ RETRIEVAL_FAMILY = QuestionFamilySpec(
|
||||
),
|
||||
legal_task_types=frozenset(
|
||||
[
|
||||
"Action Recognition",
|
||||
"Action Reasoning",
|
||||
"Action Prediction",
|
||||
"Action Sequence",
|
||||
"Object Recognition",
|
||||
"Object Reasoning",
|
||||
"Object Interaction",
|
||||
"Scene Understanding",
|
||||
"Event Reasoning",
|
||||
"Causal Reasoning",
|
||||
"Temporal Reasoning",
|
||||
"Spatial Reasoning",
|
||||
"Action Recognition",
|
||||
"Attribute Perception",
|
||||
"OCR Problems",
|
||||
]
|
||||
),
|
||||
leak_profile=LeakTestProfile(
|
||||
@@ -116,10 +109,7 @@ REASONING_FAMILY = QuestionFamilySpec(
|
||||
[
|
||||
"Action Reasoning",
|
||||
"Object Reasoning",
|
||||
"Event Reasoning",
|
||||
"Causal Reasoning",
|
||||
"Temporal Reasoning",
|
||||
"Spatial Reasoning",
|
||||
"Information Synopsis",
|
||||
]
|
||||
),
|
||||
leak_profile=LeakTestProfile(
|
||||
@@ -141,10 +131,10 @@ ENUMERATION_FAMILY = QuestionFamilySpec(
|
||||
),
|
||||
legal_task_types=frozenset(
|
||||
[
|
||||
"Action Sequence",
|
||||
"Object Recognition",
|
||||
"Object Interaction",
|
||||
"Scene Understanding",
|
||||
"Counting Problem",
|
||||
"Temporal Reasoning",
|
||||
"Temporal Perception",
|
||||
"Information Synopsis",
|
||||
]
|
||||
),
|
||||
leak_profile=LeakTestProfile(
|
||||
@@ -166,10 +156,10 @@ VISUAL_FAMILY = QuestionFamilySpec(
|
||||
),
|
||||
legal_task_types=frozenset(
|
||||
[
|
||||
"Object Recognition",
|
||||
"Scene Understanding",
|
||||
"Attribute Perception",
|
||||
"Counting Problem",
|
||||
"OCR Problems",
|
||||
"Action Recognition",
|
||||
"Spatial Reasoning",
|
||||
]
|
||||
),
|
||||
leak_profile=LeakTestProfile(
|
||||
@@ -191,9 +181,8 @@ SPATIAL_FAMILY = QuestionFamilySpec(
|
||||
),
|
||||
legal_task_types=frozenset(
|
||||
[
|
||||
"Spatial Perception",
|
||||
"Spatial Reasoning",
|
||||
"Object Interaction",
|
||||
"Scene Understanding",
|
||||
]
|
||||
),
|
||||
leak_profile=LeakTestProfile(
|
||||
|
||||
+10
-15
@@ -124,19 +124,6 @@ def stratified_sample(
|
||||
return flatten_units(sampled)
|
||||
|
||||
|
||||
def _unit_correct(unit: QuestionUnit, correctness: dict[str, bool]) -> bool:
|
||||
"""单元级正确性：成员全部答对才算对（缺失按 False，宽松口径）。
|
||||
|
||||
参数:
|
||||
unit: 目标单元（single 1 题，pair 2 题）。
|
||||
correctness: question_id -> 基线是否答对。
|
||||
|
||||
返回:
|
||||
pair 走双向 AND、single 即单题正确性；任一成员缺失或答错即 False。
|
||||
"""
|
||||
return all(correctness.get(q.question_id, False) for q in unit.questions)
|
||||
|
||||
|
||||
def _ratio_stratified_sample(
|
||||
pool: list[QuestionUnit],
|
||||
correctness: dict[str, bool],
|
||||
@@ -158,9 +145,17 @@ def _ratio_stratified_sample(
|
||||
|
||||
异常:
|
||||
ValueError: 对单元或错单元层不足。
|
||||
|
||||
关键实现:
|
||||
unit_correctness 采用函数内延迟导入：loader 属 question_gen，
|
||||
question_units 属 harness，模块级导入将触发循环依赖（沿用 build_units /
|
||||
flatten_units 的既有做法）。以 strict=False 保持"缺基线对错即视为未答对"的
|
||||
原 loose 语义不变。
|
||||
"""
|
||||
correct = [u for u in pool if _unit_correct(u, correctness)]
|
||||
wrong = [u for u in pool if not _unit_correct(u, correctness)]
|
||||
from app.harness.question_units import unit_correctness
|
||||
|
||||
correct = [u for u in pool if unit_correctness(u, correctness, strict=False)]
|
||||
wrong = [u for u in pool if not unit_correctness(u, correctness, strict=False)]
|
||||
n_correct = round(size * correct_ratio)
|
||||
n_wrong = size - n_correct
|
||||
if len(correct) < n_correct or len(wrong) < n_wrong:
|
||||
|
||||
Reference in New Issue
Block a user