feat(question_gen): add generate-v2 CLI subcommand and experiment script

- Add generate-v2 subparser with --config, --store-dir, --db-path,
  --seed, and --dry-run arguments to tools/generate_questions.py
- Implement _run_generate_v2 async handler: config loading, video
  discovery, DI client construction, TreeIndex loading, pipeline
  invocation, and result persistence
- Add scripts/generate_questions_v2.sh following build_trees.sh
  conventions (source .env, conda run python path, MODE=mock support)
- Update app/question_gen/__init__.py to export full v2 public API:
  run_pipeline_v2, PipelineConfig, PipelineResult, QuestionFamilySpec,
  ALL_FAMILIES, CandidateQuestion, generate_one_v2, GateReport, run_gates
- Add QuestionGenStore.load_progress() for pipeline resumption
- Add integration tests for CLI help and dry-run behavior
- Update test_question_gen_api to match expanded __all__

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-12 00:04:42 -04:00
parent f46e87258c
commit eecb86e27a
6 changed files with 533 additions and 14 deletions
+102
View File
@@ -0,0 +1,102 @@
"""generate-v2 CLI 子命令集成测试 — 验证 CLI 入口、dry-run 模式与参数解析。
测试策略:
- test_subcommand_help: 验证子命令注册成功、--help 返回码 0 且包含 --config
- test_dry_run: 验证 dry-run 模式不调用 LLM/VLM,仅输出统计信息
"""
from __future__ import annotations
import subprocess
import sys
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
class TestCLIGenerateV2:
"""generate-v2 子命令 CLI 集成测试。"""
def test_subcommand_help(self) -> None:
"""验证 generate-v2 子命令已注册,--help 正确退出并包含 --config 参数说明。"""
result = subprocess.run(
[
sys.executable,
str(PROJECT_ROOT / "tools" / "generate_questions.py"),
"generate-v2",
"--help",
],
capture_output=True,
text=True,
timeout=30,
)
assert result.returncode == 0, f"stderr: {result.stderr}"
assert "--config" in result.stdout
assert "--store-dir" in result.stdout
assert "--dry-run" in result.stdout
def test_dry_run(self, tmp_path: Path) -> None:
"""验证 dry-run 模式:加载配置、计算 slot,但不调用 LLM/VLM。
通过创建最小 config + 临时 store-dir 来验证 dry-run 的快速退出行为。
"""
# 创建最小配置文件
config_path = tmp_path / "config.yaml"
config_path.write_text(
"""\
question_gen_v2:
family_ratios:
retrieval: 0.30
reasoning: 0.25
enumeration: 0.20
visual: 0.15
spatial: 0.10
gate:
blind_answer_model: "mock"
leak_test_model: "mock"
key_verify_model: "mock"
multi_true_model: "mock"
dedup_threshold: 0.85
retry_limit: 3
heavy_sample_rate: 0.15
heavy_agent_model: "mock"
output_dir: "{output_dir}"
per_type: 2
concurrency: 2
seed: 42
""".format(output_dir=str(tmp_path / "output")),
encoding="utf-8",
)
# 创建 store-dir 结构:至少一个带 tree.json 的视频目录
store_dir = tmp_path / "store"
videos_dir = store_dir / "videos"
video_dir = videos_dir / "test_video_001"
video_dir.mkdir(parents=True)
# 最小 tree.json(仅需存在)
(video_dir / "tree.json").write_text("{}", encoding="utf-8")
db_path = tmp_path / "question_gen.db"
result = subprocess.run(
[
sys.executable,
str(PROJECT_ROOT / "tools" / "generate_questions.py"),
"generate-v2",
"--config",
str(config_path),
"--store-dir",
str(store_dir),
"--db-path",
str(db_path),
"--dry-run",
],
capture_output=True,
text=True,
timeout=30,
)
assert result.returncode == 0, f"stderr: {result.stderr}"
# dry-run 应输出 slot 统计信息到 stderrloguru 输出)
assert "dry-run" in result.stderr.lower() or "slot" in result.stderr.lower()
# 确保没有创建 telemetry db(意味着未初始化 LLM 客户端)
# DB 可能会被创建用于 store,但不应有 telemetry 调用
+12 -1
View File
@@ -35,13 +35,24 @@ class TestQuestionGenPublicAPI:
assert hasattr(mod, "stratified_sample")
def test_all_exports(self) -> None:
"""__all__ 包含预期的公开 APIloader + synthesizer)。"""
"""__all__ 包含预期的公开 APIv1 + v2)。"""
mod = importlib.import_module("app.question_gen")
assert set(mod.__all__) == {
# v1 接口
"load_benchmark",
"stratified_sample",
"TASK_TYPE_LEVEL_MAP",
"AnchorContext",
"generate_one",
"sample_anchor",
# v2 接口
"run_pipeline_v2",
"PipelineConfig",
"PipelineResult",
"QuestionFamilySpec",
"ALL_FAMILIES",
"CandidateQuestion",
"generate_one_v2",
"GateReport",
"run_gates",
}