778 lines
26 KiB
Python
778 lines
26 KiB
Python
"""core/evolution/evolve.py 单元测试。
|
||
|
||
覆盖验证函数、编辑预算退火、resolve_skill_file、内部辅助函数、
|
||
受保护区构建、JSON 解析、rank_and_clip、格式化工具。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
from unittest.mock import AsyncMock, MagicMock
|
||
|
||
import pytest
|
||
|
||
from core.evolution.evolve import (
|
||
_check_code_blocks,
|
||
_check_length,
|
||
_extract_section,
|
||
_format_case_samples,
|
||
_format_rejected_edits,
|
||
_format_spans,
|
||
_parse_frontmatter,
|
||
_parse_llm_json,
|
||
_select_top_edits,
|
||
_skill_protected_spans,
|
||
_strip_appendix_region,
|
||
_strip_momentum_region,
|
||
_strip_protected_regions,
|
||
_system_protected_spans,
|
||
_tool_protected_spans,
|
||
consolidate_appendix,
|
||
edit_budget_at,
|
||
evolve_single_skill,
|
||
evolve_single_tool,
|
||
evolve_system_prompt,
|
||
rank_and_clip,
|
||
resolve_skill_file,
|
||
validate_skill,
|
||
validate_system,
|
||
validate_tool,
|
||
)
|
||
from core.evolution.patch import (
|
||
APPENDIX_END,
|
||
APPENDIX_START,
|
||
MOMENTUM_END,
|
||
MOMENTUM_START,
|
||
)
|
||
from core.evolution.types import (
|
||
EvolvePrompts,
|
||
RejectedEdit,
|
||
SkillCasePack,
|
||
SystemCasePack,
|
||
ToolCasePack,
|
||
)
|
||
from core.types import LLMResponse
|
||
|
||
# =========================================================================
|
||
# A. 内部辅助函数
|
||
# =========================================================================
|
||
|
||
|
||
class TestParseFrontmatter:
|
||
"""_parse_frontmatter 测试。"""
|
||
|
||
def test_valid_frontmatter(self) -> None:
|
||
text = "---\nname: test\ndescription: d\n---\nbody"
|
||
result = _parse_frontmatter(text)
|
||
assert result == {"name": "test", "description": "d"}
|
||
|
||
def test_no_frontmatter(self) -> None:
|
||
assert _parse_frontmatter("no frontmatter here") is None
|
||
|
||
def test_invalid_yaml(self) -> None:
|
||
text = "---\n: : invalid\n---\nbody"
|
||
assert _parse_frontmatter(text) is None
|
||
|
||
def test_empty_frontmatter(self) -> None:
|
||
text = "---\n\n---\nbody"
|
||
result = _parse_frontmatter(text)
|
||
assert result is None # yaml.safe_load("") returns None
|
||
|
||
|
||
class TestStripAppendixRegion:
|
||
"""_strip_appendix_region 测试。"""
|
||
|
||
def test_no_markers(self) -> None:
|
||
text = "hello world"
|
||
assert _strip_appendix_region(text) == text
|
||
|
||
def test_with_markers(self) -> None:
|
||
text = f"before\n{APPENDIX_START}\nappendix stuff\n{APPENDIX_END}\nafter"
|
||
result = _strip_appendix_region(text)
|
||
assert "appendix stuff" not in result
|
||
assert "before" in result
|
||
assert "after" in result
|
||
|
||
def test_only_start_marker(self) -> None:
|
||
text = f"before\n{APPENDIX_START}\nno end marker"
|
||
assert _strip_appendix_region(text) == text
|
||
|
||
def test_only_end_marker(self) -> None:
|
||
text = f"before\n{APPENDIX_END}\nno start marker"
|
||
assert _strip_appendix_region(text) == text
|
||
|
||
|
||
class TestStripMomentumRegion:
|
||
"""_strip_momentum_region 测试。"""
|
||
|
||
def test_no_markers(self) -> None:
|
||
text = "hello world"
|
||
assert _strip_momentum_region(text) == text
|
||
|
||
def test_with_markers(self) -> None:
|
||
text = f"before\n{MOMENTUM_START}\nmomentum stuff\n{MOMENTUM_END}\nafter"
|
||
result = _strip_momentum_region(text)
|
||
assert "momentum stuff" not in result
|
||
assert "before" in result
|
||
assert "after" in result
|
||
|
||
def test_damaged_markers_raise(self) -> None:
|
||
text = f"before\n{MOMENTUM_START}\nno end marker"
|
||
with pytest.raises(ValueError, match="momentum marker 损坏"):
|
||
_strip_momentum_region(text)
|
||
|
||
|
||
class TestStripProtectedRegions:
|
||
"""_strip_protected_regions 测试（先 appendix 后 momentum）。"""
|
||
|
||
def test_both_regions(self) -> None:
|
||
text = (
|
||
f"body\n"
|
||
f"{APPENDIX_START}\nappendix\n{APPENDIX_END}\n"
|
||
f"{MOMENTUM_START}\nmomentum\n{MOMENTUM_END}\n"
|
||
f"tail"
|
||
)
|
||
result = _strip_protected_regions(text)
|
||
assert "appendix" not in result
|
||
assert "momentum" not in result
|
||
assert "body" in result
|
||
assert "tail" in result
|
||
|
||
|
||
class TestCheckLength:
|
||
"""_check_length 测试。"""
|
||
|
||
def test_normal_ratio(self) -> None:
|
||
orig = "x" * 100
|
||
evol = "x" * 120
|
||
assert _check_length(orig, evol) == []
|
||
|
||
def test_too_long(self) -> None:
|
||
orig = "x" * 100
|
||
evol = "x" * 300
|
||
errors = _check_length(orig, evol)
|
||
assert len(errors) == 1
|
||
assert "超限" in errors[0]
|
||
|
||
def test_too_short(self) -> None:
|
||
orig = "x" * 100
|
||
evol = "x" * 10
|
||
errors = _check_length(orig, evol)
|
||
assert len(errors) == 1
|
||
assert "不足" in errors[0]
|
||
|
||
def test_orig_empty(self) -> None:
|
||
assert _check_length("", "something") == []
|
||
|
||
|
||
class TestCheckCodeBlocks:
|
||
"""_check_code_blocks 测试。"""
|
||
|
||
def test_even_count(self) -> None:
|
||
text = "```python\ncode\n```"
|
||
assert _check_code_blocks(text) == []
|
||
|
||
def test_odd_count(self) -> None:
|
||
text = "```python\ncode"
|
||
errors = _check_code_blocks(text)
|
||
assert len(errors) == 1
|
||
assert "未闭合" in errors[0]
|
||
|
||
def test_no_blocks(self) -> None:
|
||
assert _check_code_blocks("no code blocks") == []
|
||
|
||
|
||
class TestExtractSection:
|
||
"""_extract_section 测试。"""
|
||
|
||
def test_found(self) -> None:
|
||
text = "intro\n## 能力边界\nfrozen content\n## other\nmore"
|
||
result = _extract_section(text, "能力边界")
|
||
assert result is not None
|
||
assert "frozen content" in result
|
||
assert "## 能力边界" in result
|
||
|
||
def test_not_found(self) -> None:
|
||
text = "intro\n## other\nmore"
|
||
assert _extract_section(text, "不存在") is None
|
||
|
||
def test_last_section(self) -> None:
|
||
text = "intro\n## 最后段\ncontent at end"
|
||
result = _extract_section(text, "最后段")
|
||
assert result is not None
|
||
assert "content at end" in result
|
||
|
||
|
||
# =========================================================================
|
||
# B. 受保护区构建
|
||
# =========================================================================
|
||
|
||
|
||
class TestSkillProtectedSpans:
|
||
"""_skill_protected_spans 测试。"""
|
||
|
||
def test_with_frontmatter(self) -> None:
|
||
text = "---\nname: test\n---\nbody"
|
||
spans = _skill_protected_spans(text)
|
||
assert any("---" in s for s in spans)
|
||
|
||
def test_with_appendix(self) -> None:
|
||
text = f"body\n{APPENDIX_START}\nnotes\n{APPENDIX_END}"
|
||
spans = _skill_protected_spans(text)
|
||
assert any(APPENDIX_START in s for s in spans)
|
||
|
||
def test_with_momentum(self) -> None:
|
||
text = f"body\n{MOMENTUM_START}\nmomentum\n{MOMENTUM_END}"
|
||
spans = _skill_protected_spans(text)
|
||
assert any(MOMENTUM_START in s for s in spans)
|
||
|
||
def test_empty(self) -> None:
|
||
assert _skill_protected_spans("plain text") == []
|
||
|
||
|
||
class TestSystemProtectedSpans:
|
||
"""_system_protected_spans 测试。"""
|
||
|
||
def test_frozen_sections(self) -> None:
|
||
text = "intro\n## 能力边界\ncontent1\n## 输出格式\ncontent2\n## 视频树结构\ncontent3\n## other\nmore"
|
||
spans = _system_protected_spans(text)
|
||
assert len(spans) == 3
|
||
|
||
def test_with_appendix(self) -> None:
|
||
text = f"body\n{APPENDIX_START}\nnotes\n{APPENDIX_END}"
|
||
spans = _system_protected_spans(text)
|
||
assert len(spans) == 1
|
||
|
||
|
||
class TestToolProtectedSpans:
|
||
"""_tool_protected_spans 测试。"""
|
||
|
||
def test_output_format_section(self) -> None:
|
||
text = "intro\n## 输出格式\nformat\n## other\nmore"
|
||
spans = _tool_protected_spans(text)
|
||
assert any("输出格式" in s for s in spans)
|
||
|
||
def test_no_output_format(self) -> None:
|
||
text = "intro\n## other\nmore"
|
||
spans = _tool_protected_spans(text)
|
||
assert len(spans) == 0
|
||
|
||
|
||
# =========================================================================
|
||
# C. 验证函数
|
||
# =========================================================================
|
||
|
||
|
||
class TestValidateSkill:
|
||
"""validate_skill 测试。"""
|
||
|
||
def test_identical_passes(self) -> None:
|
||
content = "---\nname: test\ndescription: d\ntask_type: t\n---\nbody"
|
||
assert validate_skill(content, content).passed
|
||
|
||
def test_changed_frontmatter_fails(self) -> None:
|
||
orig = "---\nname: a\ndescription: d\ntask_type: t\n---\nbody"
|
||
evol = "---\nname: b\ndescription: d\ntask_type: t\n---\nbody"
|
||
result = validate_skill(orig, evol)
|
||
assert not result.passed
|
||
assert any("name" in e for e in result.errors)
|
||
|
||
def test_length_ratio_too_short_fails(self) -> None:
|
||
orig = "---\nname: a\ndescription: d\ntask_type: t\n---\n" + "x" * 1000
|
||
evol = "---\nname: a\ndescription: d\ntask_type: t\n---\nshort"
|
||
result = validate_skill(orig, evol)
|
||
assert not result.passed
|
||
assert any("不足" in e for e in result.errors)
|
||
|
||
def test_length_ratio_too_long_fails(self) -> None:
|
||
orig = "---\nname: a\ndescription: d\ntask_type: t\n---\nshort"
|
||
evol = "---\nname: a\ndescription: d\ntask_type: t\n---\n" + "x" * 1000
|
||
result = validate_skill(orig, evol)
|
||
assert not result.passed
|
||
assert any("超限" in e for e in result.errors)
|
||
|
||
def test_unclosed_code_block_fails(self) -> None:
|
||
orig = "---\nname: a\ndescription: d\ntask_type: t\n---\nbody"
|
||
evol = "---\nname: a\ndescription: d\ntask_type: t\n---\nbody\n```"
|
||
result = validate_skill(orig, evol)
|
||
assert not result.passed
|
||
assert any("未闭合" in e for e in result.errors)
|
||
|
||
def test_missing_original_frontmatter(self) -> None:
|
||
result = validate_skill("no frontmatter", "no frontmatter")
|
||
assert not result.passed
|
||
assert any("原文缺少" in e for e in result.errors)
|
||
|
||
def test_missing_evolved_frontmatter(self) -> None:
|
||
orig = "---\nname: a\ndescription: d\ntask_type: t\n---\nbody"
|
||
result = validate_skill(orig, "no frontmatter body")
|
||
assert not result.passed
|
||
|
||
|
||
class TestValidateSystem:
|
||
"""validate_system 测试。"""
|
||
|
||
def test_identical_passes(self) -> None:
|
||
content = "intro\n## 能力边界\nfrozen\n## 输出格式\nfrozen2\n## other\nbody"
|
||
assert validate_system(content, content).passed
|
||
|
||
def test_changed_frozen_section_fails(self) -> None:
|
||
orig = "intro\n## 能力边界\noriginal\n## other\nbody"
|
||
evol = "intro\n## 能力边界\nchanged\n## other\nbody"
|
||
result = validate_system(orig, evol)
|
||
assert not result.passed
|
||
assert any("能力边界" in e for e in result.errors)
|
||
|
||
def test_missing_frozen_section_fails(self) -> None:
|
||
orig = "intro\n## 能力边界\nfrozen\n## other\nbody"
|
||
evol = "intro\n## other\nbody that is similar length content padding"
|
||
result = validate_system(orig, evol)
|
||
assert not result.passed
|
||
assert any("缺失" in e for e in result.errors)
|
||
|
||
def test_no_frozen_sections_passes(self) -> None:
|
||
content = "intro\n## other section\nbody content"
|
||
assert validate_system(content, content).passed
|
||
|
||
def test_code_block_check(self) -> None:
|
||
content = "## 能力边界\nfrozen\n## other\nbody\n```unclosed"
|
||
result = validate_system(content, content)
|
||
assert not result.passed
|
||
|
||
|
||
class TestValidateTool:
|
||
"""validate_tool 测试。"""
|
||
|
||
def test_identical_passes(self) -> None:
|
||
extract = "## 输出格式\nfixed\n## other\nbody"
|
||
verify = "## 输出格式\nfixed2\n## other\nbody2"
|
||
assert validate_tool(extract, extract, verify, verify).passed
|
||
|
||
def test_no_code_block_check(self) -> None:
|
||
"""validate_tool 不检查代码块闭合（与 skill/system 不同）。"""
|
||
extract = "## 输出格式\nfixed\n```\nunclosed"
|
||
assert validate_tool(extract, extract, "v", "v").passed
|
||
|
||
def test_changed_output_format_fails(self) -> None:
|
||
orig = "## 输出格式\noriginal format\n## other\nbody"
|
||
evol = "## 输出格式\nchanged format\n## other\nbody"
|
||
result = validate_tool(orig, evol, "v", "v")
|
||
assert not result.passed
|
||
assert any("输出格式" in e for e in result.errors)
|
||
|
||
def test_verify_output_format_checked_too(self) -> None:
|
||
extract = "## 输出格式\nfixed\n## other\nbody"
|
||
orig_verify = "## 输出格式\nfixed_v\n## other\nbody_v"
|
||
evol_verify = "## 输出格式\nchanged_v\n## other\nbody_v"
|
||
result = validate_tool(extract, extract, orig_verify, evol_verify)
|
||
assert not result.passed
|
||
|
||
|
||
# =========================================================================
|
||
# D. 纯数学
|
||
# =========================================================================
|
||
|
||
|
||
class TestEditBudget:
|
||
"""edit_budget_at 测试。"""
|
||
|
||
def test_start_at_zero(self) -> None:
|
||
assert edit_budget_at(0, 100, 5, 2) == 5
|
||
|
||
def test_end_at_total(self) -> None:
|
||
assert edit_budget_at(100, 100, 5, 2) == 2
|
||
|
||
def test_total_steps_one(self) -> None:
|
||
assert edit_budget_at(0, 1, 5, 2) == 5
|
||
|
||
def test_start_less_than_end_asserts(self) -> None:
|
||
with pytest.raises(AssertionError):
|
||
edit_budget_at(0, 100, 2, 5)
|
||
|
||
def test_mid_step(self) -> None:
|
||
result = edit_budget_at(50, 100, 5, 2)
|
||
assert 2 <= result <= 5
|
||
|
||
def test_beyond_total_clamped(self) -> None:
|
||
"""global_step 超过 total_steps 时被钳住在 end。"""
|
||
assert edit_budget_at(200, 100, 5, 2) == 2
|
||
|
||
def test_equal_start_end(self) -> None:
|
||
assert edit_budget_at(50, 100, 3, 3) == 3
|
||
|
||
def test_total_steps_zero(self) -> None:
|
||
"""total_steps <= 1 直接返回 start。"""
|
||
assert edit_budget_at(0, 0, 5, 2) == 5
|
||
|
||
|
||
# =========================================================================
|
||
# E. JSON 解析
|
||
# =========================================================================
|
||
|
||
|
||
class TestParseLlmJson:
|
||
"""_parse_llm_json 测试。"""
|
||
|
||
def test_plain_json(self) -> None:
|
||
raw = '{"key": "value"}'
|
||
assert _parse_llm_json(raw) == {"key": "value"}
|
||
|
||
def test_fenced_json(self) -> None:
|
||
raw = 'text before\n```json\n{"key": "value"}\n```\ntext after'
|
||
assert _parse_llm_json(raw) == {"key": "value"}
|
||
|
||
def test_non_dict_returns_none(self) -> None:
|
||
raw = "[1, 2, 3]"
|
||
assert _parse_llm_json(raw) is None
|
||
|
||
def test_invalid_json_returns_none(self) -> None:
|
||
raw = "not json at all"
|
||
assert _parse_llm_json(raw) is None
|
||
|
||
def test_empty_string(self) -> None:
|
||
assert _parse_llm_json("") is None
|
||
|
||
|
||
# =========================================================================
|
||
# F. rank_and_clip
|
||
# =========================================================================
|
||
|
||
|
||
class TestSelectTopEdits:
|
||
"""_select_top_edits 测试。"""
|
||
|
||
def test_valid_indices(self) -> None:
|
||
edits = [{"op": "a"}, {"op": "b"}, {"op": "c"}]
|
||
result = _select_top_edits([2, 0], edits, 2)
|
||
assert result == [{"op": "c"}, {"op": "a"}]
|
||
|
||
def test_dedup(self) -> None:
|
||
edits = [{"op": "a"}, {"op": "b"}]
|
||
result = _select_top_edits([0, 0, 1], edits, 3)
|
||
assert len(result) == 2
|
||
|
||
def test_out_of_bounds_skipped(self) -> None:
|
||
edits = [{"op": "a"}, {"op": "b"}]
|
||
result = _select_top_edits([5, 0, -1], edits, 3)
|
||
assert result == [{"op": "a"}]
|
||
|
||
def test_bool_excluded(self) -> None:
|
||
"""type(True) is int 返回 True 但 type(idx) is int 排除 bool。"""
|
||
edits = [{"op": "a"}, {"op": "b"}]
|
||
result = _select_top_edits([True, False, 0], edits, 3)
|
||
assert result == [{"op": "a"}]
|
||
|
||
def test_max_edits_limit(self) -> None:
|
||
edits = [{"op": "a"}, {"op": "b"}, {"op": "c"}]
|
||
result = _select_top_edits([0, 1, 2], edits, 2)
|
||
assert len(result) == 2
|
||
|
||
|
||
class TestRankAndClip:
|
||
"""rank_and_clip 测试。"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_within_budget_passthrough(self) -> None:
|
||
"""edits <= max_edits 时原样返回。"""
|
||
llm = AsyncMock()
|
||
edits = [{"op": "a"}, {"op": "b"}]
|
||
result, info = await rank_and_clip(llm, "content", edits, 5, "test")
|
||
assert result == edits
|
||
assert info["triggered"] is False
|
||
llm.chat.assert_not_called()
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_llm_rank_success(self) -> None:
|
||
"""LLM 成功返回索引时按优先级裁剪。"""
|
||
llm = AsyncMock()
|
||
response = AsyncMock()
|
||
response.content = json.dumps({"selected_indices": [2, 0]})
|
||
llm.chat.return_value = response
|
||
|
||
edits = [{"op": "a"}, {"op": "b"}, {"op": "c"}]
|
||
result, info = await rank_and_clip(llm, "content", edits, 2, "test")
|
||
assert len(result) == 2
|
||
assert result[0] == {"op": "c"}
|
||
assert info["triggered"] is True
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_llm_failure_fallback(self) -> None:
|
||
"""LLM 失败时退化为按原序取前 max_edits 条。"""
|
||
llm = AsyncMock()
|
||
llm.chat.side_effect = Exception("LLM down")
|
||
|
||
edits = [{"op": "a"}, {"op": "b"}, {"op": "c"}]
|
||
result, info = await rank_and_clip(llm, "content", edits, 2, "test")
|
||
assert len(result) == 2
|
||
assert result == [{"op": "a"}, {"op": "b"}]
|
||
assert info["triggered"] is True
|
||
|
||
|
||
# =========================================================================
|
||
# G. resolve_skill_file
|
||
# =========================================================================
|
||
|
||
|
||
class TestResolveSkillFile:
|
||
"""resolve_skill_file 测试。"""
|
||
|
||
def test_direct_match(self) -> None:
|
||
class FakeStore:
|
||
def list_skill_files(self) -> list[str]:
|
||
return ["action-reasoning.md", "default-strategy.md"]
|
||
|
||
def read_skill(self, f: str) -> str:
|
||
return ""
|
||
|
||
assert resolve_skill_file(FakeStore(), "Action Reasoning") == "action-reasoning.md"
|
||
|
||
def test_fallback_to_default(self) -> None:
|
||
class FakeStore:
|
||
def list_skill_files(self) -> list[str]:
|
||
return ["default-strategy.md"]
|
||
|
||
def read_skill(self, f: str) -> str:
|
||
return ""
|
||
|
||
assert resolve_skill_file(FakeStore(), "Unknown Type") == "default-strategy.md"
|
||
|
||
def test_case_insensitive(self) -> None:
|
||
class FakeStore:
|
||
def list_skill_files(self) -> list[str]:
|
||
return ["temporal-reasoning.md"]
|
||
|
||
def read_skill(self, f: str) -> str:
|
||
return ""
|
||
|
||
assert resolve_skill_file(FakeStore(), "Temporal Reasoning") == "temporal-reasoning.md"
|
||
|
||
|
||
# =========================================================================
|
||
# H. 格式化辅助
|
||
# =========================================================================
|
||
|
||
|
||
class TestFormatCaseSamples:
|
||
"""_format_case_samples 测试。"""
|
||
|
||
def test_basic_format(self) -> None:
|
||
cases = [
|
||
{
|
||
"question_id": "q1",
|
||
"question": "What?",
|
||
"options": ["A", "B"],
|
||
"answer": "A",
|
||
"prediction": "B",
|
||
"error_type": "wrong",
|
||
"selection_reason": "test",
|
||
"trace": [],
|
||
}
|
||
]
|
||
result = _format_case_samples(cases)
|
||
assert "q1" in result
|
||
assert "What?" in result
|
||
|
||
def test_trace_truncation(self) -> None:
|
||
cases = [
|
||
{
|
||
"question_id": "q1",
|
||
"question": "Q",
|
||
"options": [],
|
||
"answer": "A",
|
||
"prediction": "B",
|
||
"error_type": "e",
|
||
"selection_reason": "s",
|
||
"trace": [
|
||
{
|
||
"step": 1,
|
||
"tool_name": "t",
|
||
"tool_args": {},
|
||
"tool_output": "x" * 600,
|
||
}
|
||
],
|
||
}
|
||
]
|
||
result = _format_case_samples(cases)
|
||
assert "..." in result
|
||
|
||
|
||
class TestFormatSpans:
|
||
"""_format_spans 测试。"""
|
||
|
||
def test_basic_format(self) -> None:
|
||
spans = [
|
||
{
|
||
"step": 1,
|
||
"tool_name": "extract",
|
||
"tool_args": {"query": "test"},
|
||
"tool_output": "result",
|
||
"extraction_completeness": 0.9,
|
||
"hallucination_rate": 0.1,
|
||
}
|
||
]
|
||
result = _format_spans(spans)
|
||
assert "extract" in result
|
||
assert "0.9" in result
|
||
|
||
def test_output_truncation(self) -> None:
|
||
spans = [
|
||
{
|
||
"step": 1,
|
||
"tool_name": "t",
|
||
"tool_args": {},
|
||
"tool_output": "x" * 600,
|
||
"extraction_completeness": 0.5,
|
||
"hallucination_rate": 0.0,
|
||
}
|
||
]
|
||
result = _format_spans(spans)
|
||
assert "..." in result
|
||
|
||
|
||
class TestFormatRejectedEdits:
|
||
"""_format_rejected_edits 测试。"""
|
||
|
||
def test_with_gate_evidence(self) -> None:
|
||
edits = [
|
||
RejectedEdit(
|
||
target_file="skill.md",
|
||
target_type="skill",
|
||
change_summary="changed X",
|
||
delta=-0.05,
|
||
source_version="v2",
|
||
epoch=3,
|
||
gate_w=5,
|
||
gate_l=8,
|
||
gate_e_value=0.42,
|
||
gate_delta_shrunk=-0.123,
|
||
)
|
||
]
|
||
result = _format_rejected_edits(edits)
|
||
assert "W=5" in result
|
||
assert "L=8" in result
|
||
assert "E=0.42" in result
|
||
assert "δ̂=-0.123" in result
|
||
|
||
def test_without_gate_evidence(self) -> None:
|
||
edits = [
|
||
RejectedEdit(
|
||
target_file="skill.md",
|
||
target_type="skill",
|
||
change_summary="changed X",
|
||
delta=-0.05,
|
||
source_version="v2",
|
||
epoch=3,
|
||
)
|
||
]
|
||
result = _format_rejected_edits(edits)
|
||
assert "skill.md" in result
|
||
assert "changed X" in result
|
||
assert "W=" not in result
|
||
|
||
|
||
# =========================================================================
|
||
# I. 进化入口函数测试（Task 8）
|
||
# =========================================================================
|
||
|
||
|
||
_PROMPTS = EvolvePrompts(
|
||
evolve_skill="sk",
|
||
evolve_system="sys",
|
||
evolve_tool="tool",
|
||
evolve_rank="rank",
|
||
consolidate_system="cons",
|
||
)
|
||
|
||
|
||
def _make_fake_llm(response_content: str) -> AsyncMock:
|
||
"""构造返回固定 LLMResponse 的模拟 LLM。"""
|
||
mock = AsyncMock()
|
||
mock.chat.return_value = LLMResponse(
|
||
content=response_content,
|
||
thinking="",
|
||
model="test",
|
||
provider="test",
|
||
prompt_tokens=0,
|
||
completion_tokens=0,
|
||
latency_ms=0,
|
||
ttft_ms=None,
|
||
max_inter_token_ms=None,
|
||
cache_hit=False,
|
||
call_id="test-id",
|
||
)
|
||
return mock
|
||
|
||
|
||
class TestEvolveSingleSkill:
|
||
"""evolve_single_skill 测试。"""
|
||
|
||
def test_empty_pack_skipped(self) -> None:
|
||
"""空案例包（无失败、无 lapse）导致无 applied edits → rejected。"""
|
||
pack = SkillCasePack(
|
||
task_type="test",
|
||
target_file="test.md",
|
||
stats={},
|
||
failure_cases=[],
|
||
success_cases=[],
|
||
lapse_notes=[],
|
||
)
|
||
store = MagicMock()
|
||
store.read_skill.return_value = "---\nname: t\ndescription: d\ntask_type: t\n---\nbody"
|
||
store.list_skill_files.return_value = ["test.md"]
|
||
llm = _make_fake_llm('{"suggestions":[],"edits":[]}')
|
||
record = asyncio.run(evolve_single_skill(llm, pack, store, _PROMPTS, "v1", 5, 6))
|
||
assert record.status in ("rejected", "skipped")
|
||
|
||
|
||
class TestEvolveSystemPrompt:
|
||
"""evolve_system_prompt 测试。"""
|
||
|
||
def test_no_failures_returns_skipped(self) -> None:
|
||
"""空 failure_cases + 空 edits → 无 applied → rejected。"""
|
||
pack = SystemCasePack(stats={}, failure_cases=[], success_cases=[])
|
||
store = MagicMock()
|
||
store.read_prompt.return_value = (
|
||
"## 能力边界\nfixed\n## 输出格式\nfixed\n## 视频树结构\nfixed\nbody"
|
||
)
|
||
llm = _make_fake_llm('{"suggestions":[],"edits":[]}')
|
||
record = asyncio.run(evolve_system_prompt(llm, pack, store, _PROMPTS, "v1", 5))
|
||
assert record.status in ("rejected", "skipped")
|
||
|
||
|
||
class TestEvolveSingleTool:
|
||
"""evolve_single_tool 测试。"""
|
||
|
||
def test_evolved_content_is_json(self) -> None:
|
||
"""即使 rejected，evolved_content 仍是合法 JSON 含 extract/verify。"""
|
||
pack = ToolCasePack(
|
||
tool_name="view_node",
|
||
target_files=["view_node_extract.md", "view_node_verify.md"],
|
||
stats={},
|
||
failure_spans=[],
|
||
success_spans=[],
|
||
)
|
||
store = MagicMock()
|
||
store.read_prompt.return_value = "## 输出格式\nfixed\nbody"
|
||
llm = _make_fake_llm('{"suggestions":[],"edits":[]}')
|
||
record = asyncio.run(evolve_single_tool(llm, pack, store, _PROMPTS, "v1", 5))
|
||
parsed = json.loads(record.evolved_content)
|
||
assert "extract" in parsed and "verify" in parsed
|
||
|
||
|
||
class TestConsolidateAppendix:
|
||
"""consolidate_appendix 测试。"""
|
||
|
||
def test_single_note_passthrough(self) -> None:
|
||
"""G1 守卫：单条 note 直接返回，不调 LLM。"""
|
||
llm = _make_fake_llm("")
|
||
result = asyncio.run(consolidate_appendix(llm, ["note1"]))
|
||
assert result == ["note1"]
|
||
|
||
def test_exception_returns_original(self) -> None:
|
||
"""G3 守卫：LLM 异常时降级返回原 notes。"""
|
||
llm = AsyncMock()
|
||
llm.chat.side_effect = RuntimeError("boom")
|
||
result = asyncio.run(consolidate_appendix(llm, ["a", "b", "c"]))
|
||
assert result == ["a", "b", "c"]
|