feat(question_gen): stratified_sample — 分层采样 + 题型保底
算法 100% 保真 TRM4: task_types 过滤、correctness.get(id, False) 语义、 对题在前返回顺序、min_per_class 遍历 pool 全部题型(含稀疏类)。 所有参数显式传入,无默认值。 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -7,7 +7,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.question_gen.loader import load_benchmark
|
||||
from app.question_gen.loader import load_benchmark, stratified_sample
|
||||
from core.types import GeneratedQuestion
|
||||
|
||||
|
||||
@@ -127,3 +127,235 @@ class TestLoadBenchmark:
|
||||
(tmp_path / "vid.json").write_text(json.dumps(data), encoding="utf-8")
|
||||
with pytest.raises(KeyError):
|
||||
load_benchmark(tmp_path)
|
||||
|
||||
|
||||
def _make_questions(n: int, task_type: str = "T") -> list[GeneratedQuestion]:
|
||||
"""辅助函数:批量构造题目。"""
|
||||
return [
|
||||
GeneratedQuestion(
|
||||
question_id=f"{task_type}-{i}",
|
||||
video_id="v1",
|
||||
task_type=task_type,
|
||||
question=f"Q{i}?",
|
||||
options=("A", "B", "C", "D"),
|
||||
answer="A",
|
||||
source_nodes=(),
|
||||
difficulty="medium",
|
||||
)
|
||||
for i in range(n)
|
||||
]
|
||||
|
||||
|
||||
class TestStratifiedSample:
|
||||
def test_natural_distribution(self) -> None:
|
||||
questions = _make_questions(20)
|
||||
result = stratified_sample(
|
||||
questions=questions,
|
||||
correctness={},
|
||||
size=10,
|
||||
correct_ratio=None,
|
||||
task_types=None,
|
||||
seed=42,
|
||||
min_per_class=None,
|
||||
)
|
||||
assert len(result) == 10
|
||||
|
||||
def test_natural_distribution_pool_insufficient(self) -> None:
|
||||
questions = _make_questions(5)
|
||||
with pytest.raises(ValueError, match="自然分布采样不足"):
|
||||
stratified_sample(
|
||||
questions=questions,
|
||||
correctness={},
|
||||
size=10,
|
||||
correct_ratio=None,
|
||||
task_types=None,
|
||||
seed=42,
|
||||
min_per_class=None,
|
||||
)
|
||||
|
||||
def test_ratio_stratified(self) -> None:
|
||||
questions = _make_questions(20)
|
||||
correctness = {f"T-{i}": i < 10 for i in range(20)}
|
||||
result = stratified_sample(
|
||||
questions=questions,
|
||||
correctness=correctness,
|
||||
size=10,
|
||||
correct_ratio=0.6,
|
||||
task_types=None,
|
||||
seed=42,
|
||||
min_per_class=None,
|
||||
)
|
||||
assert len(result) == 10
|
||||
correct_count = sum(1 for q in result if correctness.get(q.question_id, False))
|
||||
assert correct_count == 6
|
||||
|
||||
def test_ratio_stratified_correct_first(self) -> None:
|
||||
questions = _make_questions(20)
|
||||
correctness = {f"T-{i}": i < 10 for i in range(20)}
|
||||
result = stratified_sample(
|
||||
questions=questions,
|
||||
correctness=correctness,
|
||||
size=10,
|
||||
correct_ratio=0.5,
|
||||
task_types=None,
|
||||
seed=42,
|
||||
min_per_class=None,
|
||||
)
|
||||
n_correct = round(10 * 0.5)
|
||||
for q in result[:n_correct]:
|
||||
assert correctness.get(q.question_id, False) is True
|
||||
for q in result[n_correct:]:
|
||||
assert correctness.get(q.question_id, False) is False
|
||||
|
||||
def test_ratio_stratified_pool_insufficient(self) -> None:
|
||||
questions = _make_questions(10)
|
||||
correctness = {f"T-{i}": True for i in range(10)}
|
||||
with pytest.raises(ValueError, match="分层不足"):
|
||||
stratified_sample(
|
||||
questions=questions,
|
||||
correctness=correctness,
|
||||
size=10,
|
||||
correct_ratio=0.5,
|
||||
task_types=None,
|
||||
seed=42,
|
||||
min_per_class=None,
|
||||
)
|
||||
|
||||
def test_task_types_filter(self) -> None:
|
||||
q_a = _make_questions(10, task_type="TypeA")
|
||||
q_b = _make_questions(10, task_type="TypeB")
|
||||
result = stratified_sample(
|
||||
questions=q_a + q_b,
|
||||
correctness={},
|
||||
size=5,
|
||||
correct_ratio=None,
|
||||
task_types=["TypeA"],
|
||||
seed=42,
|
||||
min_per_class=None,
|
||||
)
|
||||
assert all(q.task_type == "TypeA" for q in result)
|
||||
|
||||
def test_unknown_correctness_treated_as_wrong(self) -> None:
|
||||
questions = _make_questions(20)
|
||||
correctness = {f"T-{i}": True for i in range(10)}
|
||||
result = stratified_sample(
|
||||
questions=questions,
|
||||
correctness=correctness,
|
||||
size=10,
|
||||
correct_ratio=0.5,
|
||||
task_types=None,
|
||||
seed=42,
|
||||
min_per_class=None,
|
||||
)
|
||||
n_correct = round(10 * 0.5)
|
||||
for q in result[:n_correct]:
|
||||
assert q.question_id in correctness
|
||||
|
||||
def test_seed_reproducibility(self) -> None:
|
||||
questions = _make_questions(20)
|
||||
r1 = stratified_sample(
|
||||
questions=questions,
|
||||
correctness={},
|
||||
size=10,
|
||||
correct_ratio=None,
|
||||
task_types=None,
|
||||
seed=123,
|
||||
min_per_class=None,
|
||||
)
|
||||
r2 = stratified_sample(
|
||||
questions=questions,
|
||||
correctness={},
|
||||
size=10,
|
||||
correct_ratio=None,
|
||||
task_types=None,
|
||||
seed=123,
|
||||
min_per_class=None,
|
||||
)
|
||||
assert [q.question_id for q in r1] == [q.question_id for q in r2]
|
||||
|
||||
def test_different_seeds_differ(self) -> None:
|
||||
questions = _make_questions(20)
|
||||
r1 = stratified_sample(
|
||||
questions=questions,
|
||||
correctness={},
|
||||
size=10,
|
||||
correct_ratio=None,
|
||||
task_types=None,
|
||||
seed=1,
|
||||
min_per_class=None,
|
||||
)
|
||||
r2 = stratified_sample(
|
||||
questions=questions,
|
||||
correctness={},
|
||||
size=10,
|
||||
correct_ratio=None,
|
||||
task_types=None,
|
||||
seed=2,
|
||||
min_per_class=None,
|
||||
)
|
||||
assert [q.question_id for q in r1] != [q.question_id for q in r2]
|
||||
|
||||
def test_min_per_class_backfill(self) -> None:
|
||||
q_a = _make_questions(10, task_type="TypeA")
|
||||
q_b = _make_questions(10, task_type="TypeB")
|
||||
all_q = q_a + q_b
|
||||
correctness = {q.question_id: True for q in q_a[:5]}
|
||||
result = stratified_sample(
|
||||
questions=all_q,
|
||||
correctness=correctness,
|
||||
size=3,
|
||||
correct_ratio=None,
|
||||
task_types=None,
|
||||
seed=42,
|
||||
min_per_class=2,
|
||||
)
|
||||
type_counts: dict[str, int] = {}
|
||||
for q in result:
|
||||
type_counts[q.task_type] = type_counts.get(q.task_type, 0) + 1
|
||||
assert type_counts.get("TypeA", 0) >= 2
|
||||
assert type_counts.get("TypeB", 0) >= 2
|
||||
|
||||
def test_min_per_class_partial_backfill(self) -> None:
|
||||
q_sparse = _make_questions(1, task_type="Sparse")
|
||||
q_main = _make_questions(10, task_type="Main")
|
||||
result = stratified_sample(
|
||||
questions=q_sparse + q_main,
|
||||
correctness={},
|
||||
size=5,
|
||||
correct_ratio=None,
|
||||
task_types=None,
|
||||
seed=42,
|
||||
min_per_class=3,
|
||||
)
|
||||
sparse_in_result = [q for q in result if q.task_type == "Sparse"]
|
||||
assert len(sparse_in_result) == 1
|
||||
|
||||
def test_min_per_class_no_duplicates(self) -> None:
|
||||
q_a = _make_questions(5, task_type="TypeA")
|
||||
q_b = _make_questions(5, task_type="TypeB")
|
||||
result = stratified_sample(
|
||||
questions=q_a + q_b,
|
||||
correctness={},
|
||||
size=3,
|
||||
correct_ratio=None,
|
||||
task_types=None,
|
||||
seed=42,
|
||||
min_per_class=2,
|
||||
)
|
||||
ids = [q.question_id for q in result]
|
||||
assert len(ids) == len(set(ids))
|
||||
|
||||
def test_backfill_enumerates_all_pool_types(self) -> None:
|
||||
q_main = _make_questions(10, task_type="Main")
|
||||
q_rare = _make_questions(3, task_type="Rare")
|
||||
result = stratified_sample(
|
||||
questions=q_main + q_rare,
|
||||
correctness={},
|
||||
size=2,
|
||||
correct_ratio=None,
|
||||
task_types=None,
|
||||
seed=0,
|
||||
min_per_class=1,
|
||||
)
|
||||
types_in_result = {q.task_type for q in result}
|
||||
assert "Rare" in types_in_result
|
||||
|
||||
Reference in New Issue
Block a user