feat(question_gen): add generate-v2 CLI subcommand and experiment script
- Add generate-v2 subparser with --config, --store-dir, --db-path, --seed, and --dry-run arguments to tools/generate_questions.py - Implement _run_generate_v2 async handler: config loading, video discovery, DI client construction, TreeIndex loading, pipeline invocation, and result persistence - Add scripts/generate_questions_v2.sh following build_trees.sh conventions (source .env, conda run python path, MODE=mock support) - Update app/question_gen/__init__.py to export full v2 public API: run_pipeline_v2, PipelineConfig, PipelineResult, QuestionFamilySpec, ALL_FAMILIES, CandidateQuestion, generate_one_v2, GateReport, run_gates - Add QuestionGenStore.load_progress() for pipeline resumption - Add integration tests for CLI help and dry-run behavior - Update test_question_gen_api to match expanded __all__ Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,102 @@
|
||||
"""generate-v2 CLI 子命令集成测试 — 验证 CLI 入口、dry-run 模式与参数解析。
|
||||
|
||||
测试策略:
|
||||
- test_subcommand_help: 验证子命令注册成功、--help 返回码 0 且包含 --config
|
||||
- test_dry_run: 验证 dry-run 模式不调用 LLM/VLM,仅输出统计信息
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
|
||||
class TestCLIGenerateV2:
|
||||
"""generate-v2 子命令 CLI 集成测试。"""
|
||||
|
||||
def test_subcommand_help(self) -> None:
|
||||
"""验证 generate-v2 子命令已注册,--help 正确退出并包含 --config 参数说明。"""
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(PROJECT_ROOT / "tools" / "generate_questions.py"),
|
||||
"generate-v2",
|
||||
"--help",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
assert result.returncode == 0, f"stderr: {result.stderr}"
|
||||
assert "--config" in result.stdout
|
||||
assert "--store-dir" in result.stdout
|
||||
assert "--dry-run" in result.stdout
|
||||
|
||||
def test_dry_run(self, tmp_path: Path) -> None:
|
||||
"""验证 dry-run 模式:加载配置、计算 slot,但不调用 LLM/VLM。
|
||||
|
||||
通过创建最小 config + 临时 store-dir 来验证 dry-run 的快速退出行为。
|
||||
"""
|
||||
# 创建最小配置文件
|
||||
config_path = tmp_path / "config.yaml"
|
||||
config_path.write_text(
|
||||
"""\
|
||||
question_gen_v2:
|
||||
family_ratios:
|
||||
retrieval: 0.30
|
||||
reasoning: 0.25
|
||||
enumeration: 0.20
|
||||
visual: 0.15
|
||||
spatial: 0.10
|
||||
gate:
|
||||
blind_answer_model: "mock"
|
||||
leak_test_model: "mock"
|
||||
key_verify_model: "mock"
|
||||
multi_true_model: "mock"
|
||||
dedup_threshold: 0.85
|
||||
retry_limit: 3
|
||||
heavy_sample_rate: 0.15
|
||||
heavy_agent_model: "mock"
|
||||
output_dir: "{output_dir}"
|
||||
per_type: 2
|
||||
concurrency: 2
|
||||
seed: 42
|
||||
""".format(output_dir=str(tmp_path / "output")),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
# 创建 store-dir 结构:至少一个带 tree.json 的视频目录
|
||||
store_dir = tmp_path / "store"
|
||||
videos_dir = store_dir / "videos"
|
||||
video_dir = videos_dir / "test_video_001"
|
||||
video_dir.mkdir(parents=True)
|
||||
# 最小 tree.json(仅需存在)
|
||||
(video_dir / "tree.json").write_text("{}", encoding="utf-8")
|
||||
|
||||
db_path = tmp_path / "question_gen.db"
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(PROJECT_ROOT / "tools" / "generate_questions.py"),
|
||||
"generate-v2",
|
||||
"--config",
|
||||
str(config_path),
|
||||
"--store-dir",
|
||||
str(store_dir),
|
||||
"--db-path",
|
||||
str(db_path),
|
||||
"--dry-run",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
assert result.returncode == 0, f"stderr: {result.stderr}"
|
||||
# dry-run 应输出 slot 统计信息到 stderr(loguru 输出)
|
||||
assert "dry-run" in result.stderr.lower() or "slot" in result.stderr.lower()
|
||||
# 确保没有创建 telemetry db(意味着未初始化 LLM 客户端)
|
||||
# DB 可能会被创建用于 store,但不应有 telemetry 调用
|
||||
@@ -35,13 +35,24 @@ class TestQuestionGenPublicAPI:
|
||||
assert hasattr(mod, "stratified_sample")
|
||||
|
||||
def test_all_exports(self) -> None:
|
||||
"""__all__ 包含预期的公开 API(loader + synthesizer)。"""
|
||||
"""__all__ 包含预期的公开 API(v1 + v2)。"""
|
||||
mod = importlib.import_module("app.question_gen")
|
||||
assert set(mod.__all__) == {
|
||||
# v1 接口
|
||||
"load_benchmark",
|
||||
"stratified_sample",
|
||||
"TASK_TYPE_LEVEL_MAP",
|
||||
"AnchorContext",
|
||||
"generate_one",
|
||||
"sample_anchor",
|
||||
# v2 接口
|
||||
"run_pipeline_v2",
|
||||
"PipelineConfig",
|
||||
"PipelineResult",
|
||||
"QuestionFamilySpec",
|
||||
"ALL_FAMILIES",
|
||||
"CandidateQuestion",
|
||||
"generate_one_v2",
|
||||
"GateReport",
|
||||
"run_gates",
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user