feat: tier-aware diag/val split with val-power repair (design 5.1)

This commit is contained in:
2026-07-16 04:58:41 -04:00
parent 6432ffef9a
commit 5bb8319220
5 changed files with 126 additions and 21 deletions
+57
View File
@@ -135,3 +135,60 @@ def test_baseline_val_accuracy_reflects_validation():
assert len(pools.validation) == 2
assert pools.baseline_val_accuracy == pytest.approx(0.5)
assert pools.diagnosis == []
def test_tier_aware_keeps_high_t2_in_diag():
"""错题视频组按 T2 含量升序进 val:T2 高的组保留在 diagnosis。"""
from app.harness.pools import split_by_video_assignment
from core.types import GeneratedQuestion
def _q(qid, vid):
return GeneratedQuestion(
question_id=qid, video_id=vid, task_type="X", question="q",
options=["A", "B"], answer="A", source_nodes=[], difficulty="easy",
)
# 4 个错题视频(每视频 1 题),T2 数分别 2/1/0/0
questions = [_q(f"{v}-1", v) for v in ("vA", "vB", "vC", "vD")]
assignment = {v: "trainval" for v in ("vA", "vB", "vC", "vD")}
correctness = {f"{v}-1": False for v in ("vA", "vB", "vC", "vD")}
wrong_tier = {"vA": 2, "vB": 1, "vC": 0, "vD": 0}
pools = split_by_video_assignment(
questions, assignment, correctness, val_ratio=0.5, seed=7,
wrong_tier_by_video=wrong_tier,
)
diag_vids = {q.video_id for q in pools.diagnosis}
# T2 最高的 vA 必留 diagT2=0 的组优先进 val
assert "vA" in diag_vids
assert "vB" in diag_vids
def test_val_wrong_min_repair_pulls_from_diag():
"""val 错题不足 val_wrong_min 时从 diag 换入低 T2 错题组补足。"""
from app.harness.pools import split_by_video_assignment
from core.types import GeneratedQuestion
def _q(qid, vid, correct):
return GeneratedQuestion(
question_id=qid, video_id=vid, task_type="X", question="q",
options=["A", "B"], answer="A", source_nodes=[], difficulty="easy",
)
# 8 错题视频 + 2 正确视频;val_ratio 小使初分 val 错题不足,触发修复
vids_wrong = [f"w{i}" for i in range(8)]
vids_correct = ["c0", "c1"]
questions = [_q(f"{v}-1", v, False) for v in vids_wrong] + [
_q(f"{v}-1", v, True) for v in vids_correct
]
assignment = {v: "trainval" for v in vids_wrong + vids_correct}
correctness = {f"{v}-1": False for v in vids_wrong}
correctness.update({f"{v}-1": True for v in vids_correct})
wrong_tier = {v: i for i, v in enumerate(vids_wrong)} # 递增 T2
pools = split_by_video_assignment(
questions, assignment, correctness, val_ratio=0.1, seed=7,
wrong_tier_by_video=wrong_tier, val_wrong_min=4,
)
val_wrong = sum(1 for q in pools.validation if not correctness[q.question_id])
assert val_wrong >= 4, f"功效修复后 val 错题 {val_wrong} < 4"