iomgaa
d0a8019fe1
Self-contained strategy targeting 6 Agent failure modes in Action Recognition: premature_evidence_anchoring, temporal_reasoning_failure, semantic_rigidity, fine_grained_visual_action, cross_segment_entity_tracking, evidence_gap_confabulation. - L2 default sampling (upgrade from L3) with 3 patterns overriding to L1 - Weighted random SubPattern selection (0.20/0.20/0.15/0.15/0.15/0.15) - Each SubPattern includes instruction, examples, distractor rules - Satisfies TaskTypeStrategy Protocol without extending BaseTaskTypeStrategy - 32 unit tests covering all properties, definitions, and selection behavior Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
263 lines
11 KiB
Python
263 lines
11 KiB
Python
"""ActionRecognitionStrategy 单元测试。
|
||
|
||
验证策略属性、SubPattern 定义、加权随机选择行为。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import random
|
||
from collections import Counter
|
||
|
||
import pytest
|
||
|
||
from app.question_gen.families import SamplingConstraint
|
||
from app.question_gen.strategy import SubPattern, TaskTypeStrategy
|
||
from app.question_gen.strategy_action_recognition import (
|
||
AR_SUB_PATTERNS,
|
||
ActionRecognitionStrategy,
|
||
)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# SubPattern 分组
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_L1_PATTERN_NAMES = frozenset(
|
||
{
|
||
"premature_evidence_anchoring",
|
||
"temporal_reasoning_failure",
|
||
"cross_segment_entity_tracking",
|
||
}
|
||
)
|
||
_L2_PATTERN_NAMES = frozenset(
|
||
{
|
||
"semantic_rigidity",
|
||
"fine_grained_visual_action",
|
||
"evidence_gap_confabulation",
|
||
}
|
||
)
|
||
_ALL_PATTERN_NAMES = _L1_PATTERN_NAMES | _L2_PATTERN_NAMES
|
||
|
||
|
||
class TestStrategyProperties:
|
||
"""策略静态属性与 Protocol 符合性。"""
|
||
|
||
@pytest.fixture()
|
||
def strategy(self) -> ActionRecognitionStrategy:
|
||
"""创建策略实例。"""
|
||
return ActionRecognitionStrategy()
|
||
|
||
def test_implements_protocol(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""ActionRecognitionStrategy 满足 TaskTypeStrategy Protocol。"""
|
||
assert isinstance(strategy, TaskTypeStrategy)
|
||
|
||
def test_task_type(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""task_type 为 'Action Recognition'。"""
|
||
assert strategy.task_type == "Action Recognition"
|
||
|
||
def test_strategy_name(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""strategy_name 为 'ACTION_RECOGNITION'。"""
|
||
assert strategy.strategy_name == "ACTION_RECOGNITION"
|
||
|
||
def test_skill_target(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""skill_target 为 'M1_AR'。"""
|
||
assert strategy.skill_target == "M1_AR"
|
||
|
||
def test_sampling_level(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""sampling_level 为 2（L2）。"""
|
||
assert strategy.sampling_level == 2
|
||
|
||
def test_sampling_constraint(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""采样约束匹配设计值。"""
|
||
expected = SamplingConstraint(
|
||
min_subtitles=3,
|
||
min_l3_nodes=5,
|
||
require_frames=True,
|
||
cross_l2_span=True,
|
||
)
|
||
assert strategy.sampling_constraint == expected
|
||
|
||
def test_prompt_template(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""prompt_template 为 'action_recognition.md'。"""
|
||
assert strategy.prompt_template == "action_recognition.md"
|
||
|
||
def test_leak_probe_template(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""leak_probe_template 为 'gate_leak_retrieval.md'。"""
|
||
assert strategy.leak_probe_template == "gate_leak_retrieval.md"
|
||
|
||
def test_extra_gates_empty(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""extra_gates 返回空列表。"""
|
||
assert strategy.extra_gates(None) == []
|
||
|
||
|
||
class TestSubPatternDefinitions:
|
||
"""SubPattern 定义的完整性与正确性。"""
|
||
|
||
def test_ar_sub_patterns_count(self) -> None:
|
||
"""AR_SUB_PATTERNS 包含 6 个子模式。"""
|
||
assert len(AR_SUB_PATTERNS) == 6
|
||
|
||
def test_ar_sub_patterns_is_tuple(self) -> None:
|
||
"""AR_SUB_PATTERNS 为 tuple 类型。"""
|
||
assert isinstance(AR_SUB_PATTERNS, tuple)
|
||
|
||
def test_all_are_sub_pattern_instances(self) -> None:
|
||
"""所有元素均为 SubPattern 实例。"""
|
||
for sp in AR_SUB_PATTERNS:
|
||
assert isinstance(sp, SubPattern)
|
||
|
||
def test_all_names_present(self) -> None:
|
||
"""6 个子模式名称完整覆盖。"""
|
||
names = {sp.name for sp in AR_SUB_PATTERNS}
|
||
assert names == _ALL_PATTERN_NAMES
|
||
|
||
def test_l1_patterns_have_level_override_1(self) -> None:
|
||
"""L1 子模式的 sampling_level_override 为 1。"""
|
||
for sp in AR_SUB_PATTERNS:
|
||
if sp.name in _L1_PATTERN_NAMES:
|
||
assert sp.sampling_level_override == 1, (
|
||
f"{sp.name} 应为 L1 (override=1)，实际 {sp.sampling_level_override}"
|
||
)
|
||
|
||
def test_l2_patterns_have_no_level_override(self) -> None:
|
||
"""L2 子模式的 sampling_level_override 为 None。"""
|
||
for sp in AR_SUB_PATTERNS:
|
||
if sp.name in _L2_PATTERN_NAMES:
|
||
assert sp.sampling_level_override is None, (
|
||
f"{sp.name} 应为 L2 (override=None)，实际 {sp.sampling_level_override}"
|
||
)
|
||
|
||
def test_weights_sum_to_one(self) -> None:
|
||
"""所有子模式权重之和为 1.0。"""
|
||
total = sum(sp.weight for sp in AR_SUB_PATTERNS)
|
||
assert abs(total - 1.0) < 1e-9
|
||
|
||
def test_individual_weights(self) -> None:
|
||
"""各子模式权重匹配设计值。"""
|
||
weight_map = {sp.name: sp.weight for sp in AR_SUB_PATTERNS}
|
||
assert abs(weight_map["premature_evidence_anchoring"] - 0.20) < 1e-9
|
||
assert abs(weight_map["temporal_reasoning_failure"] - 0.20) < 1e-9
|
||
assert abs(weight_map["semantic_rigidity"] - 0.15) < 1e-9
|
||
assert abs(weight_map["fine_grained_visual_action"] - 0.15) < 1e-9
|
||
assert abs(weight_map["cross_segment_entity_tracking"] - 0.15) < 1e-9
|
||
assert abs(weight_map["evidence_gap_confabulation"] - 0.15) < 1e-9
|
||
|
||
def test_all_have_nonempty_instruction(self) -> None:
|
||
"""每个子模式有非空 instruction。"""
|
||
for sp in AR_SUB_PATTERNS:
|
||
assert sp.instruction.strip(), f"{sp.name} instruction 为空"
|
||
|
||
def test_all_have_nonempty_distractor_rules(self) -> None:
|
||
"""每个子模式有非空 distractor_rules。"""
|
||
for sp in AR_SUB_PATTERNS:
|
||
assert sp.distractor_rules.strip(), f"{sp.name} distractor_rules 为空"
|
||
|
||
def test_all_have_positive_examples(self) -> None:
|
||
"""每个子模式有至少 1 个正面示例。"""
|
||
for sp in AR_SUB_PATTERNS:
|
||
assert len(sp.positive_examples) >= 1, f"{sp.name} 缺少 positive_examples"
|
||
|
||
def test_all_have_negative_examples(self) -> None:
|
||
"""每个子模式有至少 1 个反面示例。"""
|
||
for sp in AR_SUB_PATTERNS:
|
||
assert len(sp.negative_examples) >= 1, f"{sp.name} 缺少 negative_examples"
|
||
|
||
def test_positive_examples_have_required_keys(self) -> None:
|
||
"""正面示例包含 question, answer, why 字段。"""
|
||
required = {"question", "answer", "why"}
|
||
for sp in AR_SUB_PATTERNS:
|
||
for ex in sp.positive_examples:
|
||
missing = required - set(ex.keys())
|
||
assert not missing, f"{sp.name} 正面示例缺少字段: {missing}"
|
||
|
||
def test_negative_examples_have_required_keys(self) -> None:
|
||
"""反面示例包含 question, why 字段。"""
|
||
required = {"question", "why"}
|
||
for sp in AR_SUB_PATTERNS:
|
||
for ex in sp.negative_examples:
|
||
missing = required - set(ex.keys())
|
||
assert not missing, f"{sp.name} 反面示例缺少字段: {missing}"
|
||
|
||
def test_all_constraint_overrides_none(self) -> None:
|
||
"""所有子模式的 constraint_override 为 None。"""
|
||
for sp in AR_SUB_PATTERNS:
|
||
assert sp.constraint_override is None, f"{sp.name} constraint_override 应为 None"
|
||
|
||
|
||
class TestSelectSubPattern:
|
||
"""select_sub_pattern 加权随机选择行为。"""
|
||
|
||
@pytest.fixture()
|
||
def strategy(self) -> ActionRecognitionStrategy:
|
||
"""创建策略实例。"""
|
||
return ActionRecognitionStrategy()
|
||
|
||
def test_returns_sub_pattern(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""select_sub_pattern 返回 SubPattern（不是 None）。"""
|
||
rng = random.Random(42)
|
||
result = strategy.select_sub_pattern(rng)
|
||
assert isinstance(result, SubPattern)
|
||
|
||
def test_deterministic_with_same_seed(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""相同种子产生相同结果。"""
|
||
results_a = [strategy.select_sub_pattern(random.Random(99)).name for _ in range(20)]
|
||
results_b = [strategy.select_sub_pattern(random.Random(99)).name for _ in range(20)]
|
||
assert results_a == results_b
|
||
|
||
def test_covers_all_patterns(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""足够多次采样覆盖全部 6 个子模式。"""
|
||
rng = random.Random(12345)
|
||
seen = {strategy.select_sub_pattern(rng).name for _ in range(500)}
|
||
assert seen == _ALL_PATTERN_NAMES
|
||
|
||
def test_distribution_roughly_matches_weights(
|
||
self, strategy: ActionRecognitionStrategy
|
||
) -> None:
|
||
"""采样分布大致匹配权重（容差 +-0.05）。"""
|
||
rng = random.Random(42)
|
||
n = 5000
|
||
counter: Counter[str] = Counter()
|
||
for _ in range(n):
|
||
counter[strategy.select_sub_pattern(rng).name] += 1
|
||
|
||
weight_map = {sp.name: sp.weight for sp in AR_SUB_PATTERNS}
|
||
for name, expected_w in weight_map.items():
|
||
observed_ratio = counter[name] / n
|
||
assert abs(observed_ratio - expected_w) < 0.05, (
|
||
f"{name}: expected ~{expected_w:.2f}, got {observed_ratio:.3f}"
|
||
)
|
||
|
||
|
||
class TestBuildPromptContext:
|
||
"""build_prompt_context 输出验证。"""
|
||
|
||
@pytest.fixture()
|
||
def strategy(self) -> ActionRecognitionStrategy:
|
||
"""创建策略实例。"""
|
||
return ActionRecognitionStrategy()
|
||
|
||
def test_returns_dict_with_required_keys(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""返回包含 family_name, prompt_template, sub_pattern 的字典。"""
|
||
sp = AR_SUB_PATTERNS[0]
|
||
ctx = strategy.build_prompt_context(material=None, sub_pattern=sp)
|
||
assert "family_name" in ctx
|
||
assert "prompt_template" in ctx
|
||
assert "sub_pattern" in ctx
|
||
|
||
def test_family_name_value(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""family_name 为 ACTION_RECOGNITION。"""
|
||
sp = AR_SUB_PATTERNS[0]
|
||
ctx = strategy.build_prompt_context(material=None, sub_pattern=sp)
|
||
assert ctx["family_name"] == "ACTION_RECOGNITION"
|
||
|
||
def test_prompt_template_value(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""prompt_template 为 action_recognition.md。"""
|
||
sp = AR_SUB_PATTERNS[0]
|
||
ctx = strategy.build_prompt_context(material=None, sub_pattern=sp)
|
||
assert ctx["prompt_template"] == "action_recognition.md"
|
||
|
||
def test_sub_pattern_name(self, strategy: ActionRecognitionStrategy) -> None:
|
||
"""sub_pattern 字段等于子模式名称。"""
|
||
for sp in AR_SUB_PATTERNS:
|
||
ctx = strategy.build_prompt_context(material=None, sub_pattern=sp)
|
||
assert ctx["sub_pattern"] == sp.name
|