268 lines
10 KiB
Python
268 lines
10 KiB
Python
"""app/harness/batching.py 的单元测试。
|
||
|
||
覆盖 FFD + round-robin mini-batch 构建的核心场景：
|
||
确定性、小类不拆、大类 round-robin、正确率混合、空错题、参数校验、
|
||
correctness False vs None 精确匹配。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import pytest
|
||
|
||
from core.types import GeneratedQuestion
|
||
from app.harness.batching import (
|
||
build_batches,
|
||
_validate_params,
|
||
_select_mixed_by_task_type,
|
||
)
|
||
|
||
import random
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 辅助构造
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _make_q(
|
||
qid: str,
|
||
task_type: str = "default",
|
||
video_id: str = "v1",
|
||
) -> GeneratedQuestion:
|
||
"""构造最小 GeneratedQuestion 用于测试。"""
|
||
return GeneratedQuestion(
|
||
question_id=qid,
|
||
video_id=video_id,
|
||
task_type=task_type,
|
||
question=f"question_{qid}",
|
||
options=("A. a", "B. b", "C. c", "D. d"),
|
||
answer="A",
|
||
source_nodes=("n1",),
|
||
difficulty="medium",
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# test_build_batches_deterministic
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TestBuildBatchesDeterministic:
|
||
"""相同输入 + 相同 seed 产出完全一致的切分。"""
|
||
|
||
def test_same_seed_same_result(self) -> None:
|
||
items = [_make_q(f"q{i}", task_type=f"type_{i % 3}") for i in range(20)]
|
||
correctness = {f"q{i}": False for i in range(20)}
|
||
r1 = build_batches(items, correctness, batch_size=5, min_class_per_batch=2, seed=42)
|
||
r2 = build_batches(items, correctness, batch_size=5, min_class_per_batch=2, seed=42)
|
||
assert r1 == r2
|
||
|
||
def test_different_seed_may_differ(self) -> None:
|
||
"""不同 seed 结果应不同（极大概率，用大量样本保证）。"""
|
||
items = [_make_q(f"q{i}", task_type=f"type_{i % 5}") for i in range(50)]
|
||
correctness = {f"q{i}": False for i in range(50)}
|
||
r1 = build_batches(items, correctness, batch_size=10, min_class_per_batch=3, seed=1)
|
||
r2 = build_batches(items, correctness, batch_size=10, min_class_per_batch=3, seed=99)
|
||
# 至少 batch 内容不同（不比较结构，只比较 selected_count 一致性）
|
||
assert r1[1] == r2[1] # 总题数一致
|
||
# 但 batch 内部排列几乎必然不同
|
||
flat1 = [q.question_id for b in r1[0] for q in b]
|
||
flat2 = [q.question_id for b in r2[0] for q in b]
|
||
assert flat1 != flat2
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# test_small_class_not_split
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TestSmallClassNotSplit:
|
||
"""小类（≤ min_class_per_batch）整组不拆，锁在同一 batch。"""
|
||
|
||
def test_small_group_stays_together(self) -> None:
|
||
# 2 道题属于 small_type（≤ min_class=3），应在同一 batch
|
||
items = [
|
||
_make_q("s1", task_type="small_type"),
|
||
_make_q("s2", task_type="small_type"),
|
||
# 大类 8 道题
|
||
*[_make_q(f"big{i}", task_type="big_type") for i in range(8)],
|
||
]
|
||
correctness = {q.question_id: False for q in items}
|
||
batches, count = build_batches(
|
||
items, correctness, batch_size=5, min_class_per_batch=3, seed=0
|
||
)
|
||
assert count == 10
|
||
# 找到包含 small_type 的 batch
|
||
small_batch = [
|
||
b for b in batches
|
||
if any(q.task_type == "small_type" for q in b)
|
||
]
|
||
assert len(small_batch) == 1 # 整组在同一个 batch
|
||
small_ids = {q.question_id for q in small_batch[0] if q.task_type == "small_type"}
|
||
assert small_ids == {"s1", "s2"}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# test_large_class_round_robin
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TestLargeClassRoundRobin:
|
||
"""大类样本 round-robin 散布到多个 batch，不集中于单一 batch。"""
|
||
|
||
def test_large_group_distributed(self) -> None:
|
||
# 12 道大类题，batch_size=4，min_class=2 → 大类 > 2 → round-robin
|
||
items = [_make_q(f"q{i}", task_type="large_type") for i in range(12)]
|
||
correctness = {q.question_id: False for q in items}
|
||
batches, count = build_batches(
|
||
items, correctness, batch_size=4, min_class_per_batch=2, seed=7
|
||
)
|
||
assert count == 12
|
||
assert len(batches) >= 3 # ceil(12/4) = 3
|
||
# 每个 batch 不超过 batch_size
|
||
for b in batches:
|
||
assert len(b) <= 4
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# test_correct_ratio_mixing
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TestCorrectRatioMixing:
|
||
"""correct_ratio > 0 时混入正确题。"""
|
||
|
||
def test_mixed_includes_correct(self) -> None:
|
||
items = [
|
||
_make_q("e1", task_type="t1"),
|
||
_make_q("e2", task_type="t1"),
|
||
_make_q("c1", task_type="t1"),
|
||
_make_q("c2", task_type="t1"),
|
||
_make_q("c3", task_type="t1"),
|
||
]
|
||
correctness = {"e1": False, "e2": False, "c1": True, "c2": True, "c3": True}
|
||
batches, count = build_batches(
|
||
items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
|
||
correct_ratio=0.5,
|
||
)
|
||
# correct_ratio=0.5 → 错:正 = 1:1 → 2 错 + 2 正 = 4 题
|
||
assert count == 4
|
||
all_ids = {q.question_id for b in batches for q in b}
|
||
assert {"e1", "e2"}.issubset(all_ids) # 错题全部
|
||
correct_in = all_ids - {"e1", "e2"}
|
||
assert len(correct_in) == 2 # 采样 2 个正确题
|
||
assert correct_in.issubset({"c1", "c2", "c3"})
|
||
|
||
def test_ratio_zero_pure_errors(self) -> None:
|
||
items = [
|
||
_make_q("e1", task_type="t1"),
|
||
_make_q("c1", task_type="t1"),
|
||
]
|
||
correctness = {"e1": False, "c1": True}
|
||
batches, count = build_batches(
|
||
items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
|
||
correct_ratio=0.0,
|
||
)
|
||
assert count == 1
|
||
assert batches[0][0].question_id == "e1"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# test_no_wrong_answers_empty
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TestNoWrongAnswersEmpty:
|
||
"""无错题时返回空列表。"""
|
||
|
||
def test_all_correct_returns_empty(self) -> None:
|
||
items = [_make_q(f"q{i}") for i in range(5)]
|
||
correctness = {f"q{i}": True for i in range(5)}
|
||
batches, count = build_batches(
|
||
items, correctness, batch_size=3, min_class_per_batch=1, seed=0,
|
||
)
|
||
assert batches == []
|
||
assert count == 0
|
||
|
||
def test_empty_items_returns_empty(self) -> None:
|
||
batches, count = build_batches(
|
||
[], {}, batch_size=3, min_class_per_batch=1, seed=0,
|
||
)
|
||
assert batches == []
|
||
assert count == 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# test_validate_params_strict
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TestValidateParamsStrict:
|
||
"""参数校验：batch_size < 1、min_class < 1、min_class >= batch_size 都报错。"""
|
||
|
||
def test_batch_size_zero(self) -> None:
|
||
with pytest.raises(ValueError, match="batch_size 必须 >= 1"):
|
||
_validate_params(0, 1)
|
||
|
||
def test_batch_size_negative(self) -> None:
|
||
with pytest.raises(ValueError, match="batch_size 必须 >= 1"):
|
||
_validate_params(-1, 1)
|
||
|
||
def test_min_class_zero(self) -> None:
|
||
with pytest.raises(ValueError, match="min_class_per_batch 必须 >= 1"):
|
||
_validate_params(5, 0)
|
||
|
||
def test_min_class_equals_batch_size(self) -> None:
|
||
with pytest.raises(ValueError, match="min_class_per_batch 必须严格 < batch_size"):
|
||
_validate_params(5, 5)
|
||
|
||
def test_min_class_exceeds_batch_size(self) -> None:
|
||
with pytest.raises(ValueError, match="min_class_per_batch 必须严格 < batch_size"):
|
||
_validate_params(3, 5)
|
||
|
||
def test_valid_params_no_error(self) -> None:
|
||
_validate_params(5, 3) # 不抛异常
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# test_correctness_false_vs_none
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TestCorrectnessFalseVsNone:
|
||
"""correctness.get(qid) is False 精确匹配：None（未知题）不算错题。"""
|
||
|
||
def test_none_excluded_from_errors(self) -> None:
|
||
items = [
|
||
_make_q("wrong", task_type="t1"),
|
||
_make_q("right", task_type="t1"),
|
||
_make_q("unknown", task_type="t1"),
|
||
]
|
||
# wrong=False（错题），right=True（正确题），unknown 不在 correctness（None）
|
||
correctness: dict[str, bool] = {"wrong": False, "right": True}
|
||
batches, count = build_batches(
|
||
items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
|
||
correct_ratio=0.0,
|
||
)
|
||
# 仅 wrong 进入 batch，unknown 不算错题
|
||
assert count == 1
|
||
assert batches[0][0].question_id == "wrong"
|
||
|
||
def test_explicit_false_only(self) -> None:
|
||
"""直接测试 _select_mixed_by_task_type 内部逻辑。"""
|
||
items = [
|
||
_make_q("f1", task_type="t1"),
|
||
_make_q("n1", task_type="t1"), # None（未知）
|
||
_make_q("t1", task_type="t1"), # True（正确）
|
||
]
|
||
correctness: dict[str, bool] = {"f1": False, "t1": True}
|
||
rng = random.Random(0)
|
||
result = _select_mixed_by_task_type(items, correctness, 0.0, rng)
|
||
assert "t1" in result
|
||
assert len(result["t1"]) == 1
|
||
assert result["t1"][0].question_id == "f1"
|
||
|
||
def test_none_not_treated_as_correct(self) -> None:
|
||
"""None（未知）不进正确组，不被 correct_ratio 采样。"""
|
||
items = [
|
||
_make_q("err", task_type="t1"),
|
||
_make_q("unk", task_type="t1"),
|
||
]
|
||
correctness: dict[str, bool] = {"err": False}
|
||
rng = random.Random(0)
|
||
result = _select_mixed_by_task_type(items, correctness, 0.5, rng)
|
||
# 只有 err 一题错题，unk 不在 correctness 中 → get 返回 None → 不进 correct 组
|
||
assert len(result["t1"]) == 1 # 只有错题，无正确题可混入
|