feat(harness): add task_types, pool_split_mode, train_ratio, test_questions to RunConfig
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -600,3 +600,76 @@ def test_video_mme_task_type_count_is_12():
|
||||
from app.harness.config import _VIDEO_MME_TASK_TYPE_COUNT
|
||||
|
||||
assert _VIDEO_MME_TASK_TYPE_COUNT == 12
|
||||
|
||||
|
||||
# ────────────────────── test_run_config_new_fields ─────────────────────────
|
||||
|
||||
|
||||
class TestRunConfigNewFields:
|
||||
"""RunConfig 新增字段校验(task_types / pool_split_mode / train_ratio / test_questions)。"""
|
||||
|
||||
def test_pool_split_mode_valid(self) -> None:
|
||||
"""pool_split_mode 合法值应通过校验。"""
|
||||
for mode in ("global", "per_category"):
|
||||
cfg = _make_config(pool_split_mode=mode)
|
||||
_validate(cfg)
|
||||
|
||||
def test_pool_split_mode_invalid(self) -> None:
|
||||
"""pool_split_mode 非法值应报错。"""
|
||||
cfg = _make_config(pool_split_mode="invalid")
|
||||
with pytest.raises(ValueError, match="pool_split_mode"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_task_types_tuple(self) -> None:
|
||||
"""task_types 接受 tuple。"""
|
||||
cfg = _make_config(task_types=("short", "medium"))
|
||||
_validate(cfg)
|
||||
assert cfg.task_types == ("short", "medium")
|
||||
|
||||
def test_task_types_none(self) -> None:
|
||||
"""task_types 默认 None。"""
|
||||
cfg = _make_config()
|
||||
assert cfg.task_types is None
|
||||
|
||||
def test_train_ratio_valid(self) -> None:
|
||||
"""train_ratio 在 (0, 1) 内应通过。"""
|
||||
cfg = _make_config(train_ratio=0.5)
|
||||
_validate(cfg)
|
||||
|
||||
def test_train_ratio_zero_rejected(self) -> None:
|
||||
"""train_ratio <= 0 应报错。"""
|
||||
cfg = _make_config(train_ratio=0.0)
|
||||
with pytest.raises(ValueError, match="train_ratio"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_train_ratio_one_rejected(self) -> None:
|
||||
"""train_ratio >= 1 应报错。"""
|
||||
cfg = _make_config(train_ratio=1.0)
|
||||
with pytest.raises(ValueError, match="train_ratio"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_train_ratio_negative_rejected(self) -> None:
|
||||
"""train_ratio < 0 应报错。"""
|
||||
cfg = _make_config(train_ratio=-0.1)
|
||||
with pytest.raises(ValueError, match="train_ratio"):
|
||||
_validate(cfg)
|
||||
|
||||
def test_test_questions_default(self) -> None:
|
||||
"""test_questions 默认值为 benchmarks/Video-MME。"""
|
||||
cfg = _make_config()
|
||||
assert cfg.test_questions == "benchmarks/Video-MME"
|
||||
|
||||
def test_per_category_skips_val_size_check(self) -> None:
|
||||
"""per_category 模式跳过 val_size >= eval_min_per_class * 12 的校验。
|
||||
|
||||
val_size=1 在 global 模式会因低于 floor 而报错,
|
||||
但 per_category 模式应跳过该检查。
|
||||
"""
|
||||
cfg = _make_config(pool_split_mode="per_category", val_size=1)
|
||||
_validate(cfg)
|
||||
|
||||
def test_global_mode_enforces_val_size_check(self) -> None:
|
||||
"""global 模式仍然执行 val_size floor 检查。"""
|
||||
cfg = _make_config(pool_split_mode="global", eval_min_per_class=3, val_size=30)
|
||||
with pytest.raises(ValueError, match="val_size"):
|
||||
_validate(cfg)
|
||||
|
||||
Reference in New Issue
Block a user