feat(tools): generate_questions.py calibrate 子命令
- Fisher exact test + effect size 组合判定(PASS/WARN/FAIL) - 按 video_id 分组推理,避免跨视频树错用 - baseline 支持从 DB 读取或自动跑推理 - 对比表输出 + 退出码控制 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -11,6 +11,7 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
# 确保项目根目录在 sys.path 中
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
@@ -20,10 +21,13 @@ if str(PROJECT_ROOT) not in sys.path:
|
||||
from core.types import GeneratedQuestion
|
||||
from tools.generate_questions import (
|
||||
_append_to_json,
|
||||
_calibrate_exit_code,
|
||||
_judge_task_type,
|
||||
_load_or_init_progress,
|
||||
_rebuild_embedding_pool,
|
||||
_save_progress,
|
||||
_select_exemplars,
|
||||
_validate_calibrate_args,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -320,3 +324,94 @@ class TestAppendToJson:
|
||||
v2_data = json.loads((tmp_path / "v2.json").read_text())
|
||||
assert len(v1_data) == 1
|
||||
assert len(v2_data) == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestCalibrateJudgment
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCalibrateJudgment:
|
||||
"""_judge_task_type 校准判定测试。"""
|
||||
|
||||
def test_pass_when_delta_small(self) -> None:
|
||||
"""差值在容忍范围内判定为 PASS。"""
|
||||
verdict = _judge_task_type(
|
||||
bench_correct=60,
|
||||
bench_total=100,
|
||||
gen_correct=12,
|
||||
gen_total=20,
|
||||
tolerance=0.10,
|
||||
alpha=0.05,
|
||||
)
|
||||
assert verdict == "PASS"
|
||||
|
||||
def test_fail_when_delta_large_and_significant(self) -> None:
|
||||
"""差值超阈值且统计显著判定为 FAIL。"""
|
||||
verdict = _judge_task_type(
|
||||
bench_correct=144,
|
||||
bench_total=240,
|
||||
gen_correct=6,
|
||||
gen_total=20,
|
||||
tolerance=0.10,
|
||||
alpha=0.05,
|
||||
)
|
||||
assert verdict == "FAIL"
|
||||
|
||||
def test_warn_when_delta_large_but_not_significant(self) -> None:
|
||||
"""差值超阈值但不统计显著判定为 WARN。"""
|
||||
verdict = _judge_task_type(
|
||||
bench_correct=2,
|
||||
bench_total=3,
|
||||
gen_correct=8,
|
||||
gen_total=20,
|
||||
tolerance=0.10,
|
||||
alpha=0.05,
|
||||
)
|
||||
assert verdict == "WARN"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestCalibrateIntegration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCalibrateIntegration:
|
||||
"""calibrate 辅助函数集成测试。"""
|
||||
|
||||
def test_baseline_params_must_be_paired(self) -> None:
|
||||
"""baseline 参数必须成对出现。"""
|
||||
with pytest.raises(ValueError, match="成对"):
|
||||
_validate_calibrate_args(baseline_db="some.db", baseline_run_id=None)
|
||||
|
||||
def test_baseline_params_both_none_ok(self) -> None:
|
||||
"""两个参数都为 None 不报错。"""
|
||||
_validate_calibrate_args(baseline_db=None, baseline_run_id=None)
|
||||
|
||||
def test_baseline_params_both_provided_ok(self) -> None:
|
||||
"""两个参数都提供不报错。"""
|
||||
_validate_calibrate_args(baseline_db="some.db", baseline_run_id="run-001")
|
||||
|
||||
def test_baseline_run_id_only_raises(self) -> None:
|
||||
"""只提供 run_id 也报错。"""
|
||||
with pytest.raises(ValueError, match="成对"):
|
||||
_validate_calibrate_args(baseline_db=None, baseline_run_id="run-001")
|
||||
|
||||
def test_has_fail_returns_exit_code_1(self) -> None:
|
||||
"""存在 FAIL 时返回退出码 1。"""
|
||||
verdicts = {"Object Recognition": "PASS", "Action Reasoning": "FAIL"}
|
||||
assert _calibrate_exit_code(verdicts) == 1
|
||||
|
||||
def test_all_pass_or_warn_returns_exit_code_0(self) -> None:
|
||||
"""全部 PASS 或 WARN 时返回退出码 0。"""
|
||||
verdicts = {"Object Recognition": "PASS", "Spatial Perception": "WARN"}
|
||||
assert _calibrate_exit_code(verdicts) == 0
|
||||
|
||||
def test_all_pass_returns_exit_code_0(self) -> None:
|
||||
"""全部 PASS 时返回退出码 0。"""
|
||||
verdicts = {"Object Recognition": "PASS", "Action Reasoning": "PASS"}
|
||||
assert _calibrate_exit_code(verdicts) == 0
|
||||
|
||||
def test_empty_verdicts_returns_exit_code_0(self) -> None:
|
||||
"""空 verdicts 时返回退出码 0。"""
|
||||
assert _calibrate_exit_code({}) == 0
|
||||
|
||||
Reference in New Issue
Block a user