feat: pre-flight filter of untrainable task types before gate
This commit is contained in:
@@ -157,6 +157,7 @@ class _FakeConfig:
|
||||
diag_size: int = 30
|
||||
val_size: int = 50
|
||||
batch_correct_ratio: float = 0.5
|
||||
trainable_min_units: int = 8
|
||||
edit_budget_start: int = 6
|
||||
edit_budget_end: int = 3
|
||||
early_stop_patience: int = 3
|
||||
@@ -289,6 +290,7 @@ class TestFingerprintStructuralVsDecision:
|
||||
"diag_size",
|
||||
"val_size",
|
||||
"batch_correct_ratio",
|
||||
"trainable_min_units",
|
||||
}
|
||||
decision = {
|
||||
"edit_budget_start",
|
||||
|
||||
@@ -40,6 +40,7 @@ def _valid_kwargs() -> dict:
|
||||
"batch_size": 15,
|
||||
"min_class_per_batch": 2,
|
||||
"eval_min_per_class": 2,
|
||||
"trainable_min_units": 8,
|
||||
"early_stop_patience": 8,
|
||||
"test_size": 60,
|
||||
"use_slow_momentum": True,
|
||||
|
||||
@@ -317,6 +317,7 @@ class TestBuildOrLoadPoolsFrozen:
|
||||
batch_size=15,
|
||||
min_class_per_batch=2,
|
||||
eval_min_per_class=2,
|
||||
trainable_min_units=8,
|
||||
early_stop_patience=4,
|
||||
test_size=10,
|
||||
use_slow_momentum=True,
|
||||
@@ -870,6 +871,7 @@ class TestRunHoldoutEvalConfig:
|
||||
batch_size=15,
|
||||
min_class_per_batch=2,
|
||||
eval_min_per_class=2,
|
||||
trainable_min_units=8,
|
||||
early_stop_patience=4,
|
||||
test_size=30,
|
||||
use_slow_momentum=True,
|
||||
@@ -920,6 +922,7 @@ class TestRunHoldoutEvalConfig:
|
||||
batch_size=15,
|
||||
min_class_per_batch=2,
|
||||
eval_min_per_class=2,
|
||||
trainable_min_units=8,
|
||||
early_stop_patience=4,
|
||||
test_size=30,
|
||||
use_slow_momentum=True,
|
||||
|
||||
@@ -20,6 +20,7 @@ from app.harness.runner import (
|
||||
_build_comparison_pairs,
|
||||
_compute_total_steps,
|
||||
_fallback_summary,
|
||||
_filter_untrainable_types,
|
||||
_format_applied_edits,
|
||||
_guard_infra_failures,
|
||||
_outcome_to_quadrant_pairs,
|
||||
@@ -401,6 +402,58 @@ class TestShouldEarlyStop:
|
||||
assert state.epochs_since_best_improved == 2
|
||||
|
||||
|
||||
class TestFilterUntrainableTypes:
|
||||
"""_filter_untrainable_types 可训练性预检纯函数。"""
|
||||
|
||||
def test_untrainable_types_filtered_before_gate(self) -> None:
|
||||
"""val<eval_min_per_class 或 非test单元<trainable_min_units 的题型被剔除。"""
|
||||
# 题型 A:diag=5 + val=5 → units=10、val=5,可训
|
||||
diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(5)]
|
||||
val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(5)]
|
||||
# 题型 B:val=0(<eval_min_per_class)不可训
|
||||
diag += [_FakeQuestion(question_id=f"B-d{i}", task_type="B") for i in range(2)]
|
||||
pools = _FakePools(diagnosis=diag, validation=val, test=[])
|
||||
|
||||
new_pools, new_types = _filter_untrainable_types(
|
||||
pools,
|
||||
task_types=["A", "B"],
|
||||
eval_min_per_class=2,
|
||||
trainable_min_units=8,
|
||||
)
|
||||
|
||||
assert {q.task_type for q in new_pools.diagnosis} == {"A"}
|
||||
assert {q.task_type for q in new_pools.validation} == {"A"}
|
||||
assert new_types == ["A"]
|
||||
|
||||
def test_units_below_threshold_filtered(self) -> None:
|
||||
"""val 达标但 diag+val 单元数 < trainable_min_units 的题型被剔除。"""
|
||||
# 题型 C:val=2(>=2)但 units=2+0=... 补 diag 使总数不足
|
||||
val = [_FakeQuestion(question_id=f"C-v{i}", task_type="C") for i in range(2)]
|
||||
diag = [_FakeQuestion(question_id="C-d0", task_type="C")] # units=3 < 8
|
||||
pools = _FakePools(diagnosis=diag, validation=val, test=[])
|
||||
|
||||
new_pools, new_types = _filter_untrainable_types(
|
||||
pools, task_types=None, eval_min_per_class=2, trainable_min_units=8
|
||||
)
|
||||
|
||||
assert new_pools.diagnosis == []
|
||||
assert new_pools.validation == []
|
||||
assert new_types == []
|
||||
|
||||
def test_test_pool_untouched(self) -> None:
|
||||
"""test 池不参与过滤(继续报告全题型准确率)。"""
|
||||
val = [_FakeQuestion(question_id=f"A-v{i}", task_type="A") for i in range(8)]
|
||||
diag = [_FakeQuestion(question_id=f"A-d{i}", task_type="A") for i in range(8)]
|
||||
test = [_FakeQuestion(question_id="B-t0", task_type="B")]
|
||||
pools = _FakePools(diagnosis=diag, validation=val, test=test)
|
||||
|
||||
new_pools, _ = _filter_untrainable_types(
|
||||
pools, task_types=None, eval_min_per_class=2, trainable_min_units=8
|
||||
)
|
||||
|
||||
assert new_pools.test == test
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 13c: Probation 数据结构测试
|
||||
# =========================================================================
|
||||
@@ -690,6 +743,7 @@ class TestRunnerFactoryInjection:
|
||||
"batch_size": 5,
|
||||
"min_class_per_batch": 2,
|
||||
"eval_min_per_class": 2,
|
||||
"trainable_min_units": 8,
|
||||
"early_stop_patience": 3,
|
||||
"test_size": 10,
|
||||
"use_slow_momentum": False,
|
||||
|
||||
@@ -54,6 +54,7 @@ def _base_config(workspace_dir: Path, store_dir: Path) -> RunConfig:
|
||||
batch_size=5,
|
||||
min_class_per_batch=2,
|
||||
eval_min_per_class=2,
|
||||
trainable_min_units=8,
|
||||
early_stop_patience=3,
|
||||
test_size=10,
|
||||
use_slow_momentum=False,
|
||||
|
||||
Reference in New Issue
Block a user