feat(harness): batching.py — FFD + round-robin mini-batch (#10 算法保真)

app/harness/batching.py

@@ -0,0 +1,242 @@
+"""混合 mini-batch 切分：大类打散、小类整锁，供 runner 每 step 处理一个 batch。"""
+
+from __future__ import annotations
+
+import math
+import random
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from core.types import GeneratedQuestion
+
+
+def build_batches(
+    items: list[GeneratedQuestion],
+    correctness: dict[str, bool],
+    batch_size: int,
+    min_class_per_batch: int,
+    seed: int,
+    correct_ratio: float = 0.0,
+) -> tuple[list[list[GeneratedQuestion]], int]:
+    """把诊断池里的题目切成多个混合 mini-batch。
+
+    当 ``correct_ratio > 0`` 时，按题型为每组错题配比一定数量的正确题，使 batch
+    包含正误混合样本（"动量"机制）；``correct_ratio <= 0`` 时退化为纯错题模式。
+
+    参数:
+        items: 候选题目全集。
+        correctness: question_id -> 基线是否答对。
+        batch_size: 单个 batch 的样本数上限（> 0）。
+        min_class_per_batch: 小类判定阈值——题目数 ≤ 此值的题型整组锁进单一
+            batch（> 0）。
+        seed: 随机种子，保证相同输入产出完全一致的切分。
+        correct_ratio: 正确题占比（0.0 ~ 1.0）。0.0 = 纯错题；0.5 = 错题:正确题 = 1:1。
+    返回:
+        (非空 mini-batch 列表, selected_count)；无错题时返回 ([], 0)。
+        selected_count 是所有 batch 中题目总数。
+    异常:
+        ValueError: batch_size 或 min_class_per_batch < 1, 或
+            min_class_per_batch >= batch_size（破坏小类整组装箱不超容的前提）。
+    关键实现细节:
+        装箱顺序为「先小类后大类」。小类整组用 first-fit-decreasing 装箱：按组大小
+        降序处理（同大小再按 task_type 排序保证确定性），每组放进第一个剩余容量足够
+        的 batch；若现有 batch 都装不下就新开一个空 batch——因小类组大小
+        ≤ min_class_per_batch < batch_size，新空 batch 必能容纳，故小类装箱永不抛
+        ValueError，且保证整组不拆。再把大类样本（seed 确定性 shuffle 后）round-robin
+        分发到所有现存 batch 填充剩余容量。这样小类聚集于单 batch、大类散布多 batch
+        且与小类共箱，自然产生多类混合 batch（纯类切片会被 multiclass 断言拒绝）。
+        nb = ceil(总题数/batch_size) 是初始 batch 数下界估计而非硬上限：小类装箱可能
+        新开 bin 使实际 batch 数超过 nb。每次新开 bin 都意味着总容量随之增加，故总容量
+        恒 ≥ 总题数，大类 round-robin 跳过满箱后仍能放下全部样本，不会违反 batch_size
+        上限。题型按名称排序处理以保证跨运行确定性，不依赖 dict 遍历顺序。
+    """
+    _validate_params(batch_size, min_class_per_batch)
+
+    rng = random.Random(seed)
+    grouped = _select_mixed_by_task_type(items, correctness, correct_ratio, rng)
+    total = sum(len(g) for g in grouped.values())
+    if total == 0:
+        return [], 0
+
+    nb = max(1, math.ceil(total / batch_size))
+    batches: list[list[GeneratedQuestion]] = [[] for _ in range(nb)]
+
+    small, large = _split_by_size(grouped, min_class_per_batch)
+    for group in _small_groups_decreasing(small):
+        _pack_small_class(batches, group, batch_size)
+    _distribute_large_classes(batches, large, batch_size, rng)
+
+    result = [b for b in batches if b]
+    selected_count = sum(len(b) for b in result)
+    return result, selected_count
+
+
+def _validate_params(batch_size: int, min_class_per_batch: int) -> None:
+    """校验切分参数，非法值直接报错而非用默认值掩盖。
+
+    除各自 >= 1 外，强制 min_class_per_batch < batch_size：小类组大小 ≤
+    min_class_per_batch，唯有此前提成立才能保证小类整组放入单一 batch 而不超容；否则
+    _pack_small_class 新开的 bin 会装入超 batch_size 的整组，静默违反容量合约。此约束
+    与 config._validate_minibatch 一致，是 build_batches 对自身前提的防御性自校验（P5）。
+    """
+    if batch_size < 1:
+        raise ValueError(f"batch_size 必须 >= 1, 实为 {batch_size}")
+    if min_class_per_batch < 1:
+        raise ValueError(f"min_class_per_batch 必须 >= 1, 实为 {min_class_per_batch}")
+    if min_class_per_batch >= batch_size:
+        raise ValueError(
+            f"min_class_per_batch 必须严格 < batch_size, 否则无法保证小类整组放入单一 "
+            f"batch 不超容; 实为 min_class_per_batch={min_class_per_batch}, "
+            f"batch_size={batch_size}"
+        )
+
+
+def _split_by_size(
+    grouped: dict[str, list[GeneratedQuestion]],
+    min_class_per_batch: int,
+) -> tuple[dict[str, list[GeneratedQuestion]], dict[str, list[GeneratedQuestion]]]:
+    """按错题数把题型分为小类（≤ 阈值）与大类（> 阈值）两组。"""
+    small = {t: g for t, g in grouped.items() if len(g) <= min_class_per_batch}
+    large = {t: g for t, g in grouped.items() if len(g) > min_class_per_batch}
+    return small, large
+
+
+def _select_mixed_by_task_type(
+    items: list[GeneratedQuestion],
+    correctness: dict[str, bool],
+    correct_ratio: float,
+    rng: random.Random,
+) -> dict[str, list[GeneratedQuestion]]:
+    """按题型分组，为每组错题按比例采样正确题混入。
+
+    只对有错题的题型做混合——无错题的题型不进 batch，即使有正确题。
+    ``correct_ratio <= 0`` 时退化为纯错题模式（向后兼容）。
+
+    参数:
+        items: 候选题目全集。
+        correctness: question_id -> 基线是否答对。
+        correct_ratio: 正确题占比（0.0 ~ 1.0）。
+        rng: 随机数发生器，用于采样正确题。
+    返回:
+        task_type -> 该题型的混合题目列表（错题全部 + 按比例采样的正确题）。
+    """
+    errors_by_type: dict[str, list[GeneratedQuestion]] = {}
+    correct_by_type: dict[str, list[GeneratedQuestion]] = {}
+    for q in items:
+        qid = q.question_id
+        if correctness.get(qid) is False:
+            errors_by_type.setdefault(q.task_type, []).append(q)
+        elif correctness.get(qid, False):
+            correct_by_type.setdefault(q.task_type, []).append(q)
+
+    if correct_ratio <= 0:
+        return errors_by_type
+
+    # 为每个有错题的 task_type 混入正确题
+    grouped: dict[str, list[GeneratedQuestion]] = {}
+    for task_type in sorted(errors_by_type):
+        errs = errors_by_type[task_type]
+        n_correct = round(len(errs) * correct_ratio / (1 - correct_ratio))
+        available = correct_by_type.get(task_type, [])
+        sampled = (
+            list(available)
+            if len(available) <= n_correct
+            else rng.sample(available, n_correct)
+        )
+        grouped[task_type] = errs + sampled
+
+    return grouped
+
+
+def _small_groups_decreasing(
+    small: dict[str, list[GeneratedQuestion]],
+) -> list[list[GeneratedQuestion]]:
+    """按组大小降序、同大小按 task_type 升序排出小类组（first-fit-decreasing 顺序）。
+
+    参数:
+        small: task_type -> 小类错题列表。
+    返回:
+        排好序的小类组列表；降序处理可降低碎片，确定性 tie-break 保证跨运行一致。
+    """
+    return [small[t] for t in sorted(small, key=lambda t: (-len(small[t]), t))]
+
+
+def _pack_small_class(
+    batches: list[list[GeneratedQuestion]],
+    group: list[GeneratedQuestion],
+    batch_size: int,
+) -> None:
+    """用 first-fit 把一个小类整组放入首个容得下的 batch，装不下则新开 bin（就地修改）。
+
+    因小类组大小 ≤ min_class_per_batch < batch_size，新开的空 batch 必能容纳整组，
+    故此函数永不抛 ValueError，且整组不拆。
+
+    参数:
+        batches: 当前各 batch（就地追加，必要时 append 新空 batch）。
+        group: 待锁定的小类错题（整组不拆）。
+        batch_size: 单 batch 容量上限。
+    """
+    for b in batches:
+        if len(b) + len(group) <= batch_size:
+            b.extend(group)
+            return
+    batches.append(list(group))
+
+
+def _distribute_large_classes(
+    batches: list[list[GeneratedQuestion]],
+    large: dict[str, list[GeneratedQuestion]],
+    batch_size: int,
+    rng: random.Random,
+) -> None:
+    """将各大类样本 shuffle 后 round-robin 分发到所有现存 batch（就地修改）。
+
+    参数:
+        batches: 当前各 batch（含小类装箱可能新开的 bin，就地追加）。
+        large: task_type -> 大类错题列表。
+        batch_size: 单 batch 容量上限。
+        rng: 复用的随机数发生器，保证 shuffle 确定性。
+    异常:
+        ValueError: 所有 batch 均满仍有样本未放置（总容量估算异常，合法输入不可达）。
+    关键实现细节:
+        轮转范围是「所有现存 batch」而非固定 nb 个——小类装箱新开的 bin 也参与分发。
+        总容量 = 现存 batch 数 × batch_size，每次新开 bin 都同步抬高总容量，故总容量恒
+        ≥ 总错题数，防御性 ValueError 在合法输入下不可达。全局指针在所有大类样本间持续
+        轮转（不为每类重置），满箱即跳过，使大类充分散布并与已锁定的小类共箱。题型按名称
+        排序以保证分发顺序确定。
+    """
+    nb = len(batches)
+    pointer = 0
+    for task_type in sorted(large):
+        group = list(large[task_type])
+        rng.shuffle(group)
+        for q in group:
+            pointer = _place_round_robin(batches, q, pointer, batch_size, nb)
+
+
+def _place_round_robin(
+    batches: list[list[GeneratedQuestion]],
+    q: GeneratedQuestion,
+    pointer: int,
+    batch_size: int,
+    nb: int,
+) -> int:
+    """从 pointer 起找第一个未满 batch 放入 q，返回下一次起始指针。
+
+    参数:
+        batches: 当前各 batch（就地追加）。
+        q: 待放置的样本。
+        pointer: 本次轮转起始 batch 下标。
+        batch_size: 单 batch 容量上限。
+        nb: batch 总数。
+    返回:
+        下一次轮转的起始指针（已前移一位）。
+    异常:
+        ValueError: 扫描一轮所有 batch 均满（总容量估算异常）。
+    """
+    for offset in range(nb):
+        idx = (pointer + offset) % nb
+        if len(batches[idx]) < batch_size:
+            batches[idx].append(q)
+            return (idx + 1) % nb
+    raise ValueError("所有 batch 均满仍有样本待放置, 总容量估算异常")

tests/unit/test_harness_batching.py

@@ -0,0 +1,267 @@
+"""app/harness/batching.py 的单元测试。
+
+覆盖 FFD + round-robin mini-batch 构建的核心场景：
+确定性、小类不拆、大类 round-robin、正确率混合、空错题、参数校验、
+correctness False vs None 精确匹配。
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from core.types import GeneratedQuestion
+from app.harness.batching import (
+    build_batches,
+    _validate_params,
+    _select_mixed_by_task_type,
+)
+
+import random
+
+
+# ---------------------------------------------------------------------------
+# 辅助构造
+# ---------------------------------------------------------------------------
+
+def _make_q(
+    qid: str,
+    task_type: str = "default",
+    video_id: str = "v1",
+) -> GeneratedQuestion:
+    """构造最小 GeneratedQuestion 用于测试。"""
+    return GeneratedQuestion(
+        question_id=qid,
+        video_id=video_id,
+        task_type=task_type,
+        question=f"question_{qid}",
+        options=("A. a", "B. b", "C. c", "D. d"),
+        answer="A",
+        source_nodes=("n1",),
+        difficulty="medium",
+    )
+
+
+# ---------------------------------------------------------------------------
+# test_build_batches_deterministic
+# ---------------------------------------------------------------------------
+
+class TestBuildBatchesDeterministic:
+    """相同输入 + 相同 seed 产出完全一致的切分。"""
+
+    def test_same_seed_same_result(self) -> None:
+        items = [_make_q(f"q{i}", task_type=f"type_{i % 3}") for i in range(20)]
+        correctness = {f"q{i}": False for i in range(20)}
+        r1 = build_batches(items, correctness, batch_size=5, min_class_per_batch=2, seed=42)
+        r2 = build_batches(items, correctness, batch_size=5, min_class_per_batch=2, seed=42)
+        assert r1 == r2
+
+    def test_different_seed_may_differ(self) -> None:
+        """不同 seed 结果应不同（极大概率，用大量样本保证）。"""
+        items = [_make_q(f"q{i}", task_type=f"type_{i % 5}") for i in range(50)]
+        correctness = {f"q{i}": False for i in range(50)}
+        r1 = build_batches(items, correctness, batch_size=10, min_class_per_batch=3, seed=1)
+        r2 = build_batches(items, correctness, batch_size=10, min_class_per_batch=3, seed=99)
+        # 至少 batch 内容不同（不比较结构，只比较 selected_count 一致性）
+        assert r1[1] == r2[1]  # 总题数一致
+        # 但 batch 内部排列几乎必然不同
+        flat1 = [q.question_id for b in r1[0] for q in b]
+        flat2 = [q.question_id for b in r2[0] for q in b]
+        assert flat1 != flat2
+
+
+# ---------------------------------------------------------------------------
+# test_small_class_not_split
+# ---------------------------------------------------------------------------
+
+class TestSmallClassNotSplit:
+    """小类（≤ min_class_per_batch）整组不拆，锁在同一 batch。"""
+
+    def test_small_group_stays_together(self) -> None:
+        # 2 道题属于 small_type（≤ min_class=3），应在同一 batch
+        items = [
+            _make_q("s1", task_type="small_type"),
+            _make_q("s2", task_type="small_type"),
+            # 大类 8 道题
+            *[_make_q(f"big{i}", task_type="big_type") for i in range(8)],
+        ]
+        correctness = {q.question_id: False for q in items}
+        batches, count = build_batches(
+            items, correctness, batch_size=5, min_class_per_batch=3, seed=0
+        )
+        assert count == 10
+        # 找到包含 small_type 的 batch
+        small_batch = [
+            b for b in batches
+            if any(q.task_type == "small_type" for q in b)
+        ]
+        assert len(small_batch) == 1  # 整组在同一个 batch
+        small_ids = {q.question_id for q in small_batch[0] if q.task_type == "small_type"}
+        assert small_ids == {"s1", "s2"}
+
+
+# ---------------------------------------------------------------------------
+# test_large_class_round_robin
+# ---------------------------------------------------------------------------
+
+class TestLargeClassRoundRobin:
+    """大类样本 round-robin 散布到多个 batch，不集中于单一 batch。"""
+
+    def test_large_group_distributed(self) -> None:
+        # 12 道大类题，batch_size=4，min_class=2 → 大类 > 2 → round-robin
+        items = [_make_q(f"q{i}", task_type="large_type") for i in range(12)]
+        correctness = {q.question_id: False for q in items}
+        batches, count = build_batches(
+            items, correctness, batch_size=4, min_class_per_batch=2, seed=7
+        )
+        assert count == 12
+        assert len(batches) >= 3  # ceil(12/4) = 3
+        # 每个 batch 不超过 batch_size
+        for b in batches:
+            assert len(b) <= 4
+
+
+# ---------------------------------------------------------------------------
+# test_correct_ratio_mixing
+# ---------------------------------------------------------------------------
+
+class TestCorrectRatioMixing:
+    """correct_ratio > 0 时混入正确题。"""
+
+    def test_mixed_includes_correct(self) -> None:
+        items = [
+            _make_q("e1", task_type="t1"),
+            _make_q("e2", task_type="t1"),
+            _make_q("c1", task_type="t1"),
+            _make_q("c2", task_type="t1"),
+            _make_q("c3", task_type="t1"),
+        ]
+        correctness = {"e1": False, "e2": False, "c1": True, "c2": True, "c3": True}
+        batches, count = build_batches(
+            items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
+            correct_ratio=0.5,
+        )
+        # correct_ratio=0.5 → 错:正 = 1:1 → 2 错 + 2 正 = 4 题
+        assert count == 4
+        all_ids = {q.question_id for b in batches for q in b}
+        assert {"e1", "e2"}.issubset(all_ids)  # 错题全部
+        correct_in = all_ids - {"e1", "e2"}
+        assert len(correct_in) == 2  # 采样 2 个正确题
+        assert correct_in.issubset({"c1", "c2", "c3"})
+
+    def test_ratio_zero_pure_errors(self) -> None:
+        items = [
+            _make_q("e1", task_type="t1"),
+            _make_q("c1", task_type="t1"),
+        ]
+        correctness = {"e1": False, "c1": True}
+        batches, count = build_batches(
+            items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
+            correct_ratio=0.0,
+        )
+        assert count == 1
+        assert batches[0][0].question_id == "e1"
+
+
+# ---------------------------------------------------------------------------
+# test_no_wrong_answers_empty
+# ---------------------------------------------------------------------------
+
+class TestNoWrongAnswersEmpty:
+    """无错题时返回空列表。"""
+
+    def test_all_correct_returns_empty(self) -> None:
+        items = [_make_q(f"q{i}") for i in range(5)]
+        correctness = {f"q{i}": True for i in range(5)}
+        batches, count = build_batches(
+            items, correctness, batch_size=3, min_class_per_batch=1, seed=0,
+        )
+        assert batches == []
+        assert count == 0
+
+    def test_empty_items_returns_empty(self) -> None:
+        batches, count = build_batches(
+            [], {}, batch_size=3, min_class_per_batch=1, seed=0,
+        )
+        assert batches == []
+        assert count == 0
+
+
+# ---------------------------------------------------------------------------
+# test_validate_params_strict
+# ---------------------------------------------------------------------------
+
+class TestValidateParamsStrict:
+    """参数校验：batch_size < 1、min_class < 1、min_class >= batch_size 都报错。"""
+
+    def test_batch_size_zero(self) -> None:
+        with pytest.raises(ValueError, match="batch_size 必须 >= 1"):
+            _validate_params(0, 1)
+
+    def test_batch_size_negative(self) -> None:
+        with pytest.raises(ValueError, match="batch_size 必须 >= 1"):
+            _validate_params(-1, 1)
+
+    def test_min_class_zero(self) -> None:
+        with pytest.raises(ValueError, match="min_class_per_batch 必须 >= 1"):
+            _validate_params(5, 0)
+
+    def test_min_class_equals_batch_size(self) -> None:
+        with pytest.raises(ValueError, match="min_class_per_batch 必须严格 < batch_size"):
+            _validate_params(5, 5)
+
+    def test_min_class_exceeds_batch_size(self) -> None:
+        with pytest.raises(ValueError, match="min_class_per_batch 必须严格 < batch_size"):
+            _validate_params(3, 5)
+
+    def test_valid_params_no_error(self) -> None:
+        _validate_params(5, 3)  # 不抛异常
+
+
+# ---------------------------------------------------------------------------
+# test_correctness_false_vs_none
+# ---------------------------------------------------------------------------
+
+class TestCorrectnessFalseVsNone:
+    """correctness.get(qid) is False 精确匹配：None（未知题）不算错题。"""
+
+    def test_none_excluded_from_errors(self) -> None:
+        items = [
+            _make_q("wrong", task_type="t1"),
+            _make_q("right", task_type="t1"),
+            _make_q("unknown", task_type="t1"),
+        ]
+        # wrong=False（错题），right=True（正确题），unknown 不在 correctness（None）
+        correctness: dict[str, bool] = {"wrong": False, "right": True}
+        batches, count = build_batches(
+            items, correctness, batch_size=10, min_class_per_batch=2, seed=0,
+            correct_ratio=0.0,
+        )
+        # 仅 wrong 进入 batch，unknown 不算错题
+        assert count == 1
+        assert batches[0][0].question_id == "wrong"
+
+    def test_explicit_false_only(self) -> None:
+        """直接测试 _select_mixed_by_task_type 内部逻辑。"""
+        items = [
+            _make_q("f1", task_type="t1"),
+            _make_q("n1", task_type="t1"),  # None（未知）
+            _make_q("t1", task_type="t1"),  # True（正确）
+        ]
+        correctness: dict[str, bool] = {"f1": False, "t1": True}
+        rng = random.Random(0)
+        result = _select_mixed_by_task_type(items, correctness, 0.0, rng)
+        assert "t1" in result
+        assert len(result["t1"]) == 1
+        assert result["t1"][0].question_id == "f1"
+
+    def test_none_not_treated_as_correct(self) -> None:
+        """None（未知）不进正确组，不被 correct_ratio 采样。"""
+        items = [
+            _make_q("err", task_type="t1"),
+            _make_q("unk", task_type="t1"),
+        ]
+        correctness: dict[str, bool] = {"err": False}
+        rng = random.Random(0)
+        result = _select_mixed_by_task_type(items, correctness, 0.5, rng)
+        # 只有 err 一题错题，unk 不在 correctness 中 → get 返回 None → 不进 correct 组
+        assert len(result["t1"]) == 1  # 只有错题，无正确题可混入