iomgaa/Video-Tree-TRM5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(pipeline): replace QuestionFamilySpec with TaskTypeStrategy

Browse Source 
- SlotAssignment: remove family field, strategy looked up at process time
- PipelineConfig: remove family_ratios field
- _assign_slots: remove family_ratios and rng params (pure deterministic)
- _process_one_slot: use get_strategy() for sampling, generation, gates
- Add sub_pattern support (level/constraint override, instruction injection)
- Add strategy.extra_gates() check after standard gates
- load_pipeline_config: stop reading family_ratios from YAML
- Update tools/generate_questions.py seed override and dry-run log
- Update all integration tests to match new API

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
iomgaa
2026-07-14 06:05:23 -04:00
parent b9616de21e
commit 36c712defa
4 changed files with 190 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/integration/test_cli_generate_v2.py
-12
View File
@@ -45,21 +45,9 @@ class TestCLIGenerateV2:
config_path.write_text(
"""\
question_gen_v2:
family_ratios:
retrieval: 0.30
reasoning: 0.25
enumeration: 0.20
visual: 0.15
spatial: 0.10
gate:
blind_answer_model: "mock"
leak_test_model: "mock"
key_verify_model: "mock"
multi_true_model: "mock"
dedup_threshold: 0.85
retry_limit: 3
heavy_sample_rate: 0.15
heavy_agent_model: "mock"
output_dir: "{output_dir}"
per_type: 2
concurrency: 2
tests/integration/test_pipeline_v2.py
+34 -77
View File
@@ -16,7 +16,6 @@ if TYPE_CHECKING:
import numpy as np
import pytest
from app.question_gen.families import ALL_FAMILIES
from app.question_gen.pipeline_v2 import (
PipelineConfig,
PipelineResult,
@@ -170,11 +169,21 @@ def _make_tree() -> TreeIndex:
class MockVLM:
"""受控 VLM mock — 每次调用返回候选题 JSON。"""
"""受控 VLM mock — 自动区分生成请求和门控请求。
def __init__(self, responses: list[str] | None = None) -> None:
检测 prompt 中是否包含 'verdict' 关键词判断请求类型：
- 门控请求 → 返回 gate_response（默认 pass）
- 生成请求 → 按序返回 candidate JSON
"""
def __init__(
self,
responses: list[str] | None = None,
gate_response: str | None = None,
) -> None:
self._responses = responses or [_make_candidate_json()]
self._call_count = 0
self._gate_response = gate_response or _make_gate_pass_response()
self._gen_count = 0
async def chat_with_images(
self,
@@ -184,8 +193,11 @@ class MockVLM:
session_id: str | None = None,
parent_call_id: str | None = None,
) -> LLMResponse:
idx = min(self._call_count, len(self._responses) - 1)
self._call_count += 1
prompt_text = str(messages)
if "verdict" in prompt_text.lower():
return _make_llm_response(self._gate_response)
idx = min(self._gen_count, len(self._responses) - 1)
self._gen_count += 1
return _make_llm_response(self._responses[idx])
@@ -223,13 +235,6 @@ def tree() -> TreeIndex:
@pytest.fixture
def default_config(tmp_path: Path) -> PipelineConfig:
return PipelineConfig(
family_ratios={
"RETRIEVAL": 0.30,
"REASONING": 0.25,
"ENUMERATION": 0.20,
"VISUAL": 0.15,
"SPATIAL": 0.10,
},
per_type=2,
retry_limit=3,
heavy_sample_rate=0.15,
@@ -237,13 +242,6 @@ def default_config(tmp_path: Path) -> PipelineConfig:
concurrency=2,
seed=42,
output_dir=tmp_path / "output",
gate_models={
"blind_answer_model": "gpt-4.1-mini",
"leak_test_model": "gpt-4.1-mini",
"key_verify_model": "gpt-4.1-mini",
"multi_true_model": "gpt-4.1-mini",
},
heavy_agent_model="gpt-4.1-mini",
)
@@ -268,63 +266,39 @@ class TestSlotAssignment:
def test_per_type_count(self):
"""验证生成的 slot 总数 = len(task_types) * per_type。"""
video_ids = ["vid_001", "vid_002"]
task_types = ["Action Recognition", "Object Recognition", "Causal Reasoning"]
task_types = ["Action Recognition", "Object Recognition", "Counting Problem"]
per_type = 4
family_ratios = {"RETRIEVAL": 0.5, "VISUAL": 0.5}
rng = random.Random(42)
slots = _assign_slots(video_ids, task_types, per_type, family_ratios, rng)
slots = _assign_slots(video_ids, task_types, per_type)
assert len(slots) == len(task_types) * per_type
def test_family_distribution(self):
"""验证家族分配来自 get_family_for_slot（合法 family 与 task_type 匹配）。"""
video_ids = ["vid_001", "vid_002", "vid_003"]
task_types = ["Action Recognition"]
per_type = 20
family_ratios = {"RETRIEVAL": 0.5, "VISUAL": 0.5}
rng = random.Random(42)
slots = _assign_slots(video_ids, task_types, per_type, family_ratios, rng)
for slot in slots:
assert slot.task_type == "Action Recognition"
# Family 必须是 task_type 合法的家族之一
family = slot.family
assert slot.task_type in family.legal_task_types
def test_round_robin_across_videos(self):
"""验证 slot 在视频间轮转分配。"""
video_ids = ["vid_A", "vid_B"]
task_types = ["Action Recognition"]
per_type = 4
family_ratios = {"RETRIEVAL": 1.0}
rng = random.Random(42)
slots = _assign_slots(video_ids, task_types, per_type, family_ratios, rng)
slots = _assign_slots(video_ids, task_types, per_type)
video_assignments = [s.video_id for s in slots]
# 应该轮转分配
assert video_assignments.count("vid_A") == 2
assert video_assignments.count("vid_B") == 2
def test_deterministic_with_seed(self):
"""相同 seed 产出相同 slot 序列。"""
def test_deterministic(self):
"""相同参数产出相同 slot 序列（无随机性）。"""
video_ids = ["vid_001", "vid_002"]
task_types = ["Action Recognition", "Causal Reasoning"]
task_types = ["Action Recognition", "Temporal Reasoning"]
per_type = 3
family_ratios = {"RETRIEVAL": 0.5, "REASONING": 0.5}
rng1 = random.Random(99)
slots1 = _assign_slots(video_ids, task_types, per_type, family_ratios, rng1)
rng2 = random.Random(99)
slots2 = _assign_slots(video_ids, task_types, per_type, family_ratios, rng2)
slots1 = _assign_slots(video_ids, task_types, per_type)
slots2 = _assign_slots(video_ids, task_types, per_type)
for s1, s2 in zip(slots1, slots2, strict=True):
assert s1.slot_id == s2.slot_id
assert s1.video_id == s2.video_id
assert s1.family.name == s2.family.name
assert s1.task_type == s2.task_type
# ---------------------------------------------------------------------------
@@ -344,12 +318,10 @@ class TestProcessOneSlot:
used_node_ids: set[str] = set()
rng = random.Random(42)
family = ALL_FAMILIES[0] # RETRIEVAL
slot = SlotAssignment(
slot_id="slot_001",
video_id="vid_L1_000",
task_type="Action Recognition",
family=family,
seq=1,
)
@@ -374,7 +346,11 @@ class TestProcessOneSlot:
assert result is not None
assert result.question_id
assert result.skill_target == family.skill_target
# strategy 根据 task_type 决定 skill_target
from app.question_gen.strategy import get_strategy
expected_strategy = get_strategy("Action Recognition")
assert result.skill_target == expected_strategy.skill_target
@pytest.mark.asyncio
async def test_retry_on_fail(self, tree, default_config, store, tmp_path):
@@ -402,12 +378,10 @@ class TestProcessOneSlot:
used_node_ids: set[str] = set()
rng = random.Random(42)
family = ALL_FAMILIES[0] # RETRIEVAL
slot = SlotAssignment(
slot_id="slot_002",
video_id="vid_L1_000",
task_type="Action Recognition",
family=family,
seq=2,
)
@@ -444,12 +418,10 @@ class TestProcessOneSlot:
used_node_ids: set[str] = set()
rng = random.Random(42)
family = ALL_FAMILIES[0] # RETRIEVAL
slot = SlotAssignment(
slot_id="slot_003",
video_id="vid_L1_000",
task_type="Action Recognition",
family=family,
seq=3,
)
@@ -490,7 +462,6 @@ class TestPipelineV2:
llm = MockLLM([_make_gate_pass_response()] * 200)
config = PipelineConfig(
family_ratios=default_config.family_ratios,
per_type=2,
retry_limit=2,
heavy_sample_rate=0.5, # 高比例便于测试
@@ -498,8 +469,6 @@ class TestPipelineV2:
concurrency=2,
seed=42,
output_dir=tmp_path / "out",
gate_models=default_config.gate_models,
heavy_agent_model="gpt-4.1-mini",
)
# 只用一个 task_type 确保树能满足采样
@@ -525,7 +494,6 @@ class TestPipelineV2:
llm = MockLLM([_make_gate_pass_response()] * 100)
config = PipelineConfig(
family_ratios=default_config.family_ratios,
per_type=2,
retry_limit=2,
heavy_sample_rate=0.0, # 不做 heavy check
@@ -533,18 +501,13 @@ class TestPipelineV2:
concurrency=2,
seed=42,
output_dir=tmp_path / "out",
gate_models=default_config.gate_models,
heavy_agent_model="gpt-4.1-mini",
)
# 用相同 seed 计算 slot_ids（_assign_slots 是确定性的）
rng_preview = random.Random(config.seed)
# _assign_slots 现在是确定性的（无随机性）
preview_slots = _assign_slots(
["vid_L1_000"],
["Action Recognition"],
config.per_type,
config.family_ratios,
rng_preview,
)
# 构造 progress 标记所有 slot 已完成
progress = {s.slot_id: "accepted" for s in preview_slots}
@@ -565,7 +528,7 @@ class TestPipelineV2:
)
# 所有 slot 在 progress 中 → VLM 零调用
assert vlm2._call_count == 0
assert vlm2._gen_count == 0
assert len(result2.accepted) == 0
@pytest.mark.asyncio
@@ -586,7 +549,6 @@ class TestPipelineV2:
vlm = MockVLM([_make_candidate_json(f"Q{i}?") for i in range(20)])
config = PipelineConfig(
family_ratios=default_config.family_ratios,
per_type=2,
retry_limit=2,
heavy_sample_rate=1.0, # 100% 抽检
@@ -594,8 +556,6 @@ class TestPipelineV2:
concurrency=2,
seed=42,
output_dir=tmp_path / "out",
gate_models=default_config.gate_models,
heavy_agent_model="gpt-4.1-mini",
)
result = await run_pipeline_v2(
@@ -620,7 +580,6 @@ class TestPipelineV2:
llm = MockLLM([_make_gate_pass_response()] * 50)
config = PipelineConfig(
family_ratios=default_config.family_ratios,
per_type=2,
retry_limit=2,
heavy_sample_rate=0.0,
@@ -628,8 +587,6 @@ class TestPipelineV2:
concurrency=1,
seed=42,
output_dir=tmp_path / "out",
gate_models=default_config.gate_models,
heavy_agent_model="gpt-4.1-mini",
)
result = await run_pipeline_v2(