feat(harness): batching.py — FFD + round-robin mini-batch (#10 算法保真)

2026-07-07 12:47:33 -04:00
parent d349fe1148
commit 9800fef37a
2 changed files with 509 additions and 0 deletions
@@ -0,0 +1,267 @@
+"""app/harness/batching.py 的单元测试。
+
+覆盖 FFD + round-robin mini-batch 构建的核心场景：
+确定性、小类不拆、大类 round-robin、正确率混合、空错题、参数校验、
+correctness False vs None 精确匹配。
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from core.types import GeneratedQuestion
+from app.harness.batching import (
+    build_batches,
+    _validate_params,
+    _select_mixed_by_task_type,
+)
+
+import random
+
+
+# ---------------------------------------------------------------------------
+# 辅助构造
+# ---------------------------------------------------------------------------
+
+def _make_q(
+    qid: str,
+    task_type: str = "default",
+    video_id: str = "v1",
+) -> GeneratedQuestion:
+    """构造最小 GeneratedQuestion 用于测试。"""
+    return GeneratedQuestion(
+        question_id=qid,
+        video_id=video_id,
+        task_type=task_type,
+        question=f"question_{qid}",
+        options=("A. a", "B. b", "C. c", "D. d"),
+        answer="A",
+        source_nodes=("n1",),
+        difficulty="medium",
+    )
+
+
+# ---------------------------------------------------------------------------
+# test_build_batches_deterministic
+# ---------------------------------------------------------------------------
+
+class TestBuildBatchesDeterministic:
+    """相同输入 + 相同 seed 产出完全一致的切分。"""
+
+    def test_same_seed_same_result(self) -> None:
+        items = [_make_q(f"q{i}", task_type=f"type_{i % 3}") for i in range(20)]
+        correctness = {f"q{i}": False for i in range(20)}
+        r1 = build_batches(items, correctness, batch_size=5, min_class_per_batch=2, seed=42)
+        r2 = build_batches(items, correctness, batch_size=5, min_class_per_batch=2, seed=42)
+        assert r1 == r2
+
+    def test_different_seed_may_differ(self) -> None:
+        """不同 seed 结果应不同（极大概率，用大量样本保证）。"""
+        items = [_make_q(f"q{i}", task_type=f"type_{i % 5}") for i in range(50)]
+        correctness = {f"q{i}": False for i in range(50)}
+        r1 = build_batches(items, correctness, batch_size=10, min_class_per_batch=3, seed=1)
+        r2 = build_batches(items, correctness, batch_size=10, min_class_per_batch=3, seed=99)
+        # 至少 batch 内容不同（不比较结构，只比较 selected_count 一致性）
+        assert r1[1] == r2[1]  # 总题数一致
+        # 但 batch 内部排列几乎必然不同
+        flat1 = [q.question_id for b in r1[0] for q in b]
+        flat2 = [q.question_id for b in r2[0] for q in b]
+        assert flat1 != flat2
+
+
+# ---------------------------------------------------------------------------
+# test_small_class_not_split
+# ---------------------------------------------------------------------------
+
+class TestSmallClassNotSplit:
+    """小类（≤ min_class_per_batch）整组不拆，锁在同一 batch。"""
+
+    def test_small_group_stays_together(self) -> None:
+        # 2 道题属于 small_type（≤ min_class=3），应在同一 batch
+        items = [
+            _make_q("s1", task_type="small_type"),
+            _make_q("s2", task_type="small_type"),
+            # 大类 8 道题
+            *[_make_q(f"big{i}", task_type="big_type") for i in range(8)],
+        ]
+        correctness = {q.question_id: False for q in items}
+        batches, count = build_batches(
+            items, correctness, batch_size=5, min_class_per_batch=3, seed=0
+        )
+        assert count == 10
+        # 找到包含 small_type 的 batch
+        small_batch = [
+            b for b in batches
+            if any(q.task_type == "small_type" for q in b)
+        ]
+        assert len(small_batch) == 1  # 整组在同一个 batch
+        small_ids = {q.question_id for q in small_batch[0] if q.task_type == "small_type"}
+        assert small_ids == {"s1", "s2"}
+
+
+# ---------------------------------------------------------------------------
+# test_large_class_round_robin
+# ---------------------------------------------------------------------------
+
+class TestLargeClassRoundRobin:
+    """大类样本 round-robin 散布到多个 batch，不集中于单一 batch。"""
+
+    def test_large_group_distributed(self) -> None:
+        # 12 道大类题，batch_size=4，min_class=2 → 大类 > 2 → round-robin
+        items = [_make_q(f"q{i}", task_type="large_type") for i in range(12)]
+        correctness = {q.question_id: False for q in items}
+        batches, count = build_batches(
+            items, correctness, batch_size=4, min_class_per_batch=2, seed=7
+        )
+        assert count == 12
+        assert len(batches) >= 3  # ceil(12/4) = 3
+        # 每个 batch 不超过 batch_size
+        for b in batches:
+            assert len(b) <= 4
+
+
+# ---------------------------------------------------------------------------
+# test_correct_ratio_mixing
+# ---------------------------------------------------------------------------
+
+class TestCorrectRatioMixing:
+    """correct_ratio > 0 时混入正确题。"""
+
+    def test_mixed_includes_correct(self) -> None:
+        items = [
+            _make_q("e1", task_type="t1"),
+            _make_q("e2", task_type="t1"),
+            _make_q("c1", task_type="t1"),
+            _make_q("c2", task_type="t1"),
+            _make_q("c3", task_type="t1"),
+        ]
+        correctness = {"e1": False, "e2": False, "c1": True, "c2": True, "c3": True}
+        batches, count = build_batches(
+            items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
+            correct_ratio=0.5,
+        )
+        # correct_ratio=0.5 → 错:正 = 1:1 → 2 错 + 2 正 = 4 题
+        assert count == 4
+        all_ids = {q.question_id for b in batches for q in b}
+        assert {"e1", "e2"}.issubset(all_ids)  # 错题全部
+        correct_in = all_ids - {"e1", "e2"}
+        assert len(correct_in) == 2  # 采样 2 个正确题
+        assert correct_in.issubset({"c1", "c2", "c3"})
+
+    def test_ratio_zero_pure_errors(self) -> None:
+        items = [
+            _make_q("e1", task_type="t1"),
+            _make_q("c1", task_type="t1"),
+        ]
+        correctness = {"e1": False, "c1": True}
+        batches, count = build_batches(
+            items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
+            correct_ratio=0.0,
+        )
+        assert count == 1
+        assert batches[0][0].question_id == "e1"
+
+
+# ---------------------------------------------------------------------------
+# test_no_wrong_answers_empty
+# ---------------------------------------------------------------------------
+
+class TestNoWrongAnswersEmpty:
+    """无错题时返回空列表。"""
+
+    def test_all_correct_returns_empty(self) -> None:
+        items = [_make_q(f"q{i}") for i in range(5)]
+        correctness = {f"q{i}": True for i in range(5)}
+        batches, count = build_batches(
+            items, correctness, batch_size=3, min_class_per_batch=1, seed=0,
+        )
+        assert batches == []
+        assert count == 0
+
+    def test_empty_items_returns_empty(self) -> None:
+        batches, count = build_batches(
+            [], {}, batch_size=3, min_class_per_batch=1, seed=0,
+        )
+        assert batches == []
+        assert count == 0
+
+
+# ---------------------------------------------------------------------------
+# test_validate_params_strict
+# ---------------------------------------------------------------------------
+
+class TestValidateParamsStrict:
+    """参数校验：batch_size < 1、min_class < 1、min_class >= batch_size 都报错。"""
+
+    def test_batch_size_zero(self) -> None:
+        with pytest.raises(ValueError, match="batch_size 必须 >= 1"):
+            _validate_params(0, 1)
+
+    def test_batch_size_negative(self) -> None:
+        with pytest.raises(ValueError, match="batch_size 必须 >= 1"):
+            _validate_params(-1, 1)
+
+    def test_min_class_zero(self) -> None:
+        with pytest.raises(ValueError, match="min_class_per_batch 必须 >= 1"):
+            _validate_params(5, 0)
+
+    def test_min_class_equals_batch_size(self) -> None:
+        with pytest.raises(ValueError, match="min_class_per_batch 必须严格 < batch_size"):
+            _validate_params(5, 5)
+
+    def test_min_class_exceeds_batch_size(self) -> None:
+        with pytest.raises(ValueError, match="min_class_per_batch 必须严格 < batch_size"):
+            _validate_params(3, 5)
+
+    def test_valid_params_no_error(self) -> None:
+        _validate_params(5, 3)  # 不抛异常
+
+
+# ---------------------------------------------------------------------------
+# test_correctness_false_vs_none
+# ---------------------------------------------------------------------------
+
+class TestCorrectnessFalseVsNone:
+    """correctness.get(qid) is False 精确匹配：None（未知题）不算错题。"""
+
+    def test_none_excluded_from_errors(self) -> None:
+        items = [
+            _make_q("wrong", task_type="t1"),
+            _make_q("right", task_type="t1"),
+            _make_q("unknown", task_type="t1"),
+        ]
+        # wrong=False（错题），right=True（正确题），unknown 不在 correctness（None）
+        correctness: dict[str, bool] = {"wrong": False, "right": True}
+        batches, count = build_batches(
+            items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
+            correct_ratio=0.0,
+        )
+        # 仅 wrong 进入 batch，unknown 不算错题
+        assert count == 1
+        assert batches[0][0].question_id == "wrong"
+
+    def test_explicit_false_only(self) -> None:
+        """直接测试 _select_mixed_by_task_type 内部逻辑。"""
+        items = [
+            _make_q("f1", task_type="t1"),
+            _make_q("n1", task_type="t1"),  # None（未知）
+            _make_q("t1", task_type="t1"),  # True（正确）
+        ]
+        correctness: dict[str, bool] = {"f1": False, "t1": True}
+        rng = random.Random(0)
+        result = _select_mixed_by_task_type(items, correctness, 0.0, rng)
+        assert "t1" in result
+        assert len(result["t1"]) == 1
+        assert result["t1"][0].question_id == "f1"
+
+    def test_none_not_treated_as_correct(self) -> None:
+        """None（未知）不进正确组，不被 correct_ratio 采样。"""
+        items = [
+            _make_q("err", task_type="t1"),
+            _make_q("unk", task_type="t1"),
+        ]
+        correctness: dict[str, bool] = {"err": False}
+        rng = random.Random(0)
+        result = _select_mixed_by_task_type(items, correctness, 0.5, rng)
+        # 只有 err 一题错题，unk 不在 correctness 中 → get 返回 None → 不进 correct 组
+        assert len(result["t1"]) == 1  # 只有错题，无正确题可混入