iomgaa
8fef7ced42
C-1: persist_infra_t0_rows 补 INFRA/空预测错题的 T0 信号行（不进诊断故须单独落库），run_pipeline 加 Phase 0，dry-run 用假数据走通。 C-2: CLI 加 --generated-at，真实运行默认盖真实 UTC now，可显式固定以字节级复现 manifest。 I-1: coverage_report 增 evolution_target_distribution（T2 信号按 tool/skill/system 计数）。 I-2: 删除 PoolConfig 死字段 n_trainval/floor_k/epsilon/report_floor/val_wrong_min（grep 确认无消费者，视频级切分用独立 VideoSplitConfig/SplitBuildConfig/SelectConfig）。 I-3: 抽共享 load_canonical_predictions(db_path, run_id)，CLI 与 build_split 共用；消除 canonical 取行 + correct 判定重复。 M-1: build_split docstring 注明 val_wrong_min-agnostic 契约（McNemar 护栏由 CLI 冻结后执行，Task 11 契约）。
69 lines
2.8 KiB
Python
69 lines
2.8 KiB
Python
"""视频级切分科研旋钮单元测试：诊断指纹 + val_wrong_min 功效护栏。
|
||
|
||
覆盖：
|
||
- diag_fingerprint 对 (prompt 版本 / 模型 / 代码版本) 三元组确定且敏感；
|
||
- split_by_video_assignment 的 val_wrong_min 门控 fail loud（验证信号不足即报错）；
|
||
- val_wrong_min 默认 0 时行为与 Task 11 现有调用完全一致（不回归）。
|
||
|
||
注：结果驱动视频级切分不复用 PoolConfig——它有独立的 VideoSplitConfig /
|
||
SplitBuildConfig / SelectConfig，故 PoolConfig 不承载视频级切分旋钮（无死配置面）。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import pytest
|
||
|
||
from app.harness.pools import InsufficientValSignal, split_by_video_assignment
|
||
from app.harness.split_selection import diag_fingerprint
|
||
from core.types import GeneratedQuestion
|
||
|
||
|
||
def _q(qid: str, vid: str, tt: str = "Counting Problem") -> GeneratedQuestion:
|
||
"""构造最小可用题目（补齐 GeneratedQuestion 的必填 source_nodes/difficulty）。"""
|
||
return GeneratedQuestion(
|
||
question_id=qid,
|
||
video_id=vid,
|
||
task_type=tt,
|
||
question="",
|
||
options=("A", "B", "C", "D"),
|
||
answer="A",
|
||
source_nodes=(),
|
||
difficulty="easy",
|
||
)
|
||
|
||
|
||
def test_diag_fingerprint_deterministic_and_sensitive():
|
||
"""诊断指纹对相同三元组稳定，对任一分量变化敏感。"""
|
||
a = diag_fingerprint("p1", "deepseek-v4", "abc123")
|
||
assert a == diag_fingerprint("p1", "deepseek-v4", "abc123") # 确定性
|
||
assert a != diag_fingerprint("p2", "deepseek-v4", "abc123") # prompt 变则变
|
||
assert a != diag_fingerprint("p1", "kimi", "abc123") # model 变则变
|
||
assert a != diag_fingerprint("p1", "deepseek-v4", "def456") # 代码版本变则变
|
||
assert len(a) == 16 # sha256 截断 16 位十六进制
|
||
|
||
|
||
def test_val_wrong_min_enforced():
|
||
"""val 错题数 < val_wrong_min 时 fail loud（InsufficientValSignal），不静默兜底。"""
|
||
qs = [_q("v1-1", "v1"), _q("v1-2", "v1"), _q("v1-3", "v1")]
|
||
correctness = {q.question_id: True for q in qs} # 全对 → val 无错题
|
||
with pytest.raises(InsufficientValSignal): # val 错题 < val_wrong_min
|
||
split_by_video_assignment(
|
||
qs,
|
||
{"v1": "trainval"},
|
||
correctness=correctness,
|
||
val_ratio=0.5,
|
||
seed=0,
|
||
val_wrong_min=5,
|
||
)
|
||
|
||
|
||
def test_val_wrong_min_default_zero_no_regression():
|
||
"""val_wrong_min 默认 0 时不检查错题数，保持 Task 11 现有调用契约不破。"""
|
||
qs = [_q("v1-1", "v1"), _q("v2-1", "v2")]
|
||
assignment = {"v1": "trainval", "v2": "trainval"}
|
||
correctness = {"v1-1": True, "v2-1": True} # 全对但默认不触发护栏
|
||
pools = split_by_video_assignment(
|
||
qs, assignment, correctness=correctness, val_ratio=1.0, seed=0
|
||
)
|
||
assert len(pools.validation) == 2 # 未抛异常，正常返回
|