fix: guard floor phase against n_trainval budget + assert epsilon on output
Codex Task 8 审查修复: - Important 1: _satisfy_floors 每步移入前检查预算,floor 需求超 n_trainval 时 抛 InfeasibleSplitError(fail loud),保证 trainval 永不超额挤占 test;补预算超限测试。 - Important 2: 确定性测试末尾用 _epsilon_ok 断言产出 test 真满足 ε(两维偏差回归护栏), 并断言 trainval <= n_trainval。 - Minor: fixture docstring 注明结构真实 / signal 二次构造,正式运行由真实诊断替换。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -409,6 +409,9 @@ def _satisfy_floors(
|
|||||||
每轮取未达标类型的缺口,候选 = 能填 ≥1 槽 且 移入后 test 仍满足 ε 的视频;候选为空即
|
每轮取未达标类型的缺口,候选 = 能填 ≥1 槽 且 移入后 test 仍满足 ε 的视频;候选为空即
|
||||||
死锁抛 InfeasibleSplitError;否则选填槽最多者(等槽数按预洗牌顺序取首个,确定性)。
|
死锁抛 InfeasibleSplitError;否则选填槽最多者(等槽数按预洗牌顺序取首个,确定性)。
|
||||||
|
|
||||||
|
n_trainval 是硬预算:floor 需求超出预算(尚有缺口却已达 n_trainval)也判不可行 fail loud,
|
||||||
|
保证返回的 trainval 永不超过 n_trainval(不因硬约束悄悄超额、挤占 test)。
|
||||||
|
|
||||||
参数:
|
参数:
|
||||||
selected: 当前 trainval(就地追加)。
|
selected: 当前 trainval(就地追加)。
|
||||||
pool: 当前剩余池 = test 补集(就地移除)。
|
pool: 当前剩余池 = test 补集(就地移除)。
|
||||||
@@ -416,12 +419,17 @@ def _satisfy_floors(
|
|||||||
config: 选择配置。
|
config: 选择配置。
|
||||||
|
|
||||||
异常:
|
异常:
|
||||||
InfeasibleSplitError: 存在未达标类型但无候选可在不破 ε 下填补。
|
InfeasibleSplitError: 存在未达标类型但无候选可在不破 ε 下填补,
|
||||||
|
或 floor 需求超过 n_trainval 预算。
|
||||||
"""
|
"""
|
||||||
while True:
|
while True:
|
||||||
deficits = _unmet_floors(selected, config.floor_k)
|
deficits = _unmet_floors(selected, config.floor_k)
|
||||||
if not deficits:
|
if not deficits:
|
||||||
return
|
return
|
||||||
|
if len(selected) >= config.n_trainval:
|
||||||
|
raise InfeasibleSplitError(
|
||||||
|
f"floor 需求超过 n_trainval={config.n_trainval} 预算,剩余缺口: {dict(deficits)}"
|
||||||
|
)
|
||||||
candidates = [
|
candidates = [
|
||||||
video
|
video
|
||||||
for video in pool
|
for video in pool
|
||||||
@@ -501,7 +509,8 @@ def select_split(videos: list[VideoRecord], *, config: SelectConfig) -> SplitAss
|
|||||||
SplitAssignment,trainval 按选择顺序、test 按 videos 原始顺序。
|
SplitAssignment,trainval 按选择顺序、test 按 videos 原始顺序。
|
||||||
|
|
||||||
异常:
|
异常:
|
||||||
InfeasibleSplitError: videos 为空,或 floor 与 ε 死锁无法满足。
|
InfeasibleSplitError: videos 为空、floor 与 ε 死锁无法满足、
|
||||||
|
或 floor 需求超过 n_trainval 预算。
|
||||||
"""
|
"""
|
||||||
if not videos:
|
if not videos:
|
||||||
raise InfeasibleSplitError("videos 为空,无法执行切分")
|
raise InfeasibleSplitError("videos 为空,无法执行切分")
|
||||||
|
|||||||
@@ -54,8 +54,9 @@ def _load_adhoc_predictions() -> list[dict]:
|
|||||||
def _real_shaped_video_records() -> list:
|
def _real_shaped_video_records() -> list:
|
||||||
"""用真实预测结构 + 二次构造 T2 信号构建 300 个 VideoRecord。
|
"""用真实预测结构 + 二次构造 T2 信号构建 300 个 VideoRecord。
|
||||||
|
|
||||||
题型 / 难度画像取自真实 infer_adhoc 预测;因真实诊断尚未跑,T2 信号是二次构造:
|
注意:视频 type / difficulty 结构真实(取自真实 infer_adhoc 预测),signal 叠加是二次构造
|
||||||
对高信号题型(Counting / Action Reasoning)的错题标 T2 defect,error_type 循环取 4 类,
|
(pre-diagnosis 阶段固有限制,真实诊断尚未跑);正式运行时 T2 信号由真实诊断产物替换。
|
||||||
|
这里对高信号题型(Counting / Action Reasoning)的错题标 T2 defect,error_type 循环取 4 类,
|
||||||
使 floor 约束有料、多样性格子有区分度。
|
使 floor 约束有料、多样性格子有区分度。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
@@ -158,6 +159,7 @@ def test_build_video_records_covers_all_videos_with_difficulty_and_types():
|
|||||||
def test_select_split_video_disjoint_and_floor_and_deterministic():
|
def test_select_split_video_disjoint_and_floor_and_deterministic():
|
||||||
from app.harness.split_selection import (
|
from app.harness.split_selection import (
|
||||||
SelectConfig,
|
SelectConfig,
|
||||||
|
_epsilon_ok,
|
||||||
derive_reportable_types,
|
derive_reportable_types,
|
||||||
select_split,
|
select_split,
|
||||||
)
|
)
|
||||||
@@ -181,6 +183,10 @@ def test_select_split_video_disjoint_and_floor_and_deterministic():
|
|||||||
v.wrong_by_type.get("Counting Problem", 0) for v in videos if v.video_id in trainval_ids
|
v.wrong_by_type.get("Counting Problem", 0) for v in videos if v.video_id in trainval_ids
|
||||||
)
|
)
|
||||||
assert counting_defects >= 3 # floor 硬约束满足
|
assert counting_defects >= 3 # floor 硬约束满足
|
||||||
|
assert len(a.trainval) <= cfg.n_trainval # trainval 永不超预算
|
||||||
|
test_ids = set(a.test)
|
||||||
|
test_records = [v for v in videos if v.video_id in test_ids]
|
||||||
|
assert _epsilon_ok(test_records, videos, cfg) # 产出 test 真满足 ε(回归护栏)
|
||||||
|
|
||||||
|
|
||||||
def test_infeasible_floor_vs_epsilon_raises():
|
def test_infeasible_floor_vs_epsilon_raises():
|
||||||
@@ -204,6 +210,29 @@ def test_infeasible_floor_vs_epsilon_raises():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_infeasible_floor_exceeds_n_trainval_budget_raises():
|
||||||
|
from app.harness.split_selection import (
|
||||||
|
InfeasibleSplitError,
|
||||||
|
SelectConfig,
|
||||||
|
select_split,
|
||||||
|
)
|
||||||
|
|
||||||
|
# floor 需求(Action Reasoning 48 缺陷)远超 n_trainval=1 预算:单视频至多带 1 题 AR 缺陷,
|
||||||
|
# 无法在 1 个 trainval 名额内满足 floor=40 → 抛 InfeasibleSplitError(预算不足)。
|
||||||
|
videos = _real_shaped_video_records()
|
||||||
|
with pytest.raises(InfeasibleSplitError):
|
||||||
|
select_split(
|
||||||
|
videos,
|
||||||
|
config=SelectConfig(
|
||||||
|
n_trainval=1,
|
||||||
|
floor_k={"Action Reasoning": 40},
|
||||||
|
epsilon=1.0,
|
||||||
|
reportable_types=set(),
|
||||||
|
seed=3,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_derive_reportable_types():
|
def test_derive_reportable_types():
|
||||||
from app.harness.split_selection import derive_reportable_types
|
from app.harness.split_selection import derive_reportable_types
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user