feat(evolution): evolve.py per-target evolution — skill/system/tool (#9)
This commit is contained in:
+804
-28
@@ -12,19 +12,32 @@ from __future__ import annotations
|
||||
import json
|
||||
import re
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from core.evolution.patch import (
|
||||
APPENDIX_END,
|
||||
APPENDIX_START,
|
||||
append_to_appendix,
|
||||
apply_patch_with_report,
|
||||
extract_appendix_notes,
|
||||
momentum_region_bounds,
|
||||
replace_appendix_notes,
|
||||
)
|
||||
from core.evolution.types import EvolutionRecord
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from core.evolution.protocols import SkillStore
|
||||
from core.evolution.types import RejectedEdit
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from core.evolution.protocols import PromptStore, SkillStore
|
||||
from core.evolution.types import (
|
||||
EvolvePrompts,
|
||||
RejectedEdit,
|
||||
SkillCasePack,
|
||||
SystemCasePack,
|
||||
ToolCasePack,
|
||||
)
|
||||
from core.protocols import LLMProvider
|
||||
|
||||
# =========================================================================
|
||||
@@ -133,13 +146,9 @@ def _check_length(original: str, evolved: str) -> list[str]:
|
||||
ratio = len(evol_body) / orig_len
|
||||
evol_len = len(evol_body)
|
||||
if ratio > 2.0:
|
||||
errors.append(
|
||||
f"长度超限: {evol_len} 字符是原文 {orig_len} 的 {ratio:.1f} 倍 (上限 2.0)"
|
||||
)
|
||||
errors.append(f"长度超限: {evol_len} 字符是原文 {orig_len} 的 {ratio:.1f} 倍 (上限 2.0)")
|
||||
if ratio < 0.3:
|
||||
errors.append(
|
||||
f"长度不足: {evol_len} 字符是原文 {orig_len} 的 {ratio:.1f} 倍 (下限 0.3)"
|
||||
)
|
||||
errors.append(f"长度不足: {evol_len} 字符是原文 {orig_len} 的 {ratio:.1f} 倍 (下限 0.3)")
|
||||
return errors
|
||||
|
||||
|
||||
@@ -250,8 +259,7 @@ def _system_protected_spans(text: str) -> list[str]:
|
||||
spans: list[str] = [
|
||||
section
|
||||
for section in (
|
||||
_extract_section(text, name)
|
||||
for name in ("能力边界", "输出格式", "视频树结构")
|
||||
_extract_section(text, name) for name in ("能力边界", "输出格式", "视频树结构")
|
||||
)
|
||||
if section
|
||||
]
|
||||
@@ -309,8 +317,7 @@ def validate_skill(original: str, evolved: str) -> ValidationResult:
|
||||
for key in ("name", "description", "task_type"):
|
||||
if orig_fm.get(key) != evol_fm.get(key):
|
||||
errors.append(
|
||||
f"frontmatter 字段 {key} 被修改: "
|
||||
f"{orig_fm.get(key)!r} → {evol_fm.get(key)!r}"
|
||||
f"frontmatter 字段 {key} 被修改: {orig_fm.get(key)!r} → {evol_fm.get(key)!r}"
|
||||
)
|
||||
errors.extend(_check_length(original, evolved))
|
||||
errors.extend(_check_code_blocks(evolved))
|
||||
@@ -406,9 +413,7 @@ def edit_budget_at(global_step: int, total_steps: int, start: int, end: int) ->
|
||||
异常:
|
||||
AssertionError: start < end。
|
||||
"""
|
||||
assert start >= end, (
|
||||
f"edit_budget_at 要求 start >= end,实际 start={start}, end={end}"
|
||||
)
|
||||
assert start >= end, f"edit_budget_at 要求 start >= end,实际 start={start}, end={end}"
|
||||
if total_steps <= 1:
|
||||
return start
|
||||
t = min(global_step, total_steps) / total_steps
|
||||
@@ -675,27 +680,19 @@ def _format_spans(spans: list[dict[str, Any]]) -> str:
|
||||
for span in spans:
|
||||
lines.append(f"### step {span.get('step', '?')}")
|
||||
lines.append(f"- tool_name: {span.get('tool_name', '')}")
|
||||
lines.append(
|
||||
f"- tool_args: {json.dumps(span.get('tool_args', {}), ensure_ascii=False)}"
|
||||
)
|
||||
lines.append(f"- tool_args: {json.dumps(span.get('tool_args', {}), ensure_ascii=False)}")
|
||||
output_text = str(span.get("tool_output", ""))
|
||||
if len(output_text) > 500:
|
||||
output_text = output_text[:500] + "..."
|
||||
lines.append(f"- tool_output: {output_text}")
|
||||
lines.append(
|
||||
f"- extraction_completeness: {span.get('extraction_completeness', '')}"
|
||||
)
|
||||
lines.append(f"- extraction_completeness: {span.get('extraction_completeness', '')}")
|
||||
lines.append(f"- hallucination_rate: {span.get('hallucination_rate', '')}")
|
||||
missed = span.get("missed_info_tags", [])
|
||||
if missed:
|
||||
lines.append(
|
||||
f"- missed_info_tags: {json.dumps(missed, ensure_ascii=False)}"
|
||||
)
|
||||
lines.append(f"- missed_info_tags: {json.dumps(missed, ensure_ascii=False)}")
|
||||
hall_tags = span.get("hallucination_tags", [])
|
||||
if hall_tags:
|
||||
lines.append(
|
||||
f"- hallucination_tags: {json.dumps(hall_tags, ensure_ascii=False)}"
|
||||
)
|
||||
lines.append(f"- hallucination_tags: {json.dumps(hall_tags, ensure_ascii=False)}")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -722,3 +719,782 @@ def _format_rejected_edits(rejected: list[RejectedEdit]) -> str:
|
||||
)
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# I. 进化循环内部类型
|
||||
# =========================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class _PatchEvolutionAttempt:
|
||||
"""单次补丁式进化尝试的中间结果。
|
||||
|
||||
属性:
|
||||
evolved_content: 改写后内容。
|
||||
validation: 校验结果。
|
||||
suggestions: LLM 输出的改动建议列表。
|
||||
edits: LLM 输出的补丁列表。
|
||||
apply_report: 补丁逐条应用状态。
|
||||
clip_info: 超预算裁剪信息。
|
||||
"""
|
||||
|
||||
evolved_content: str
|
||||
validation: ValidationResult
|
||||
suggestions: list[dict[str, Any]] = field(default_factory=list)
|
||||
edits: list[dict[str, Any]] = field(default_factory=list)
|
||||
apply_report: list[dict[str, Any]] = field(default_factory=list)
|
||||
clip_info: dict[str, Any] = field(default_factory=lambda: {"triggered": False, "clipped": 0})
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# J. 进化循环辅助函数
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def _count_applied_reports(reports: list[dict[str, Any]]) -> int:
|
||||
"""统计补丁报告中成功应用的条数。
|
||||
|
||||
以 ``status`` 前缀 ``"applied"`` 为判据,涵盖 applied_append /
|
||||
applied_replace / applied_rewrite 等所有成功状态。
|
||||
|
||||
参数:
|
||||
reports: apply_patch_with_report 或合成报告的列表。
|
||||
|
||||
返回:
|
||||
成功应用的条数。
|
||||
"""
|
||||
return sum(1 for r in reports if r["status"].startswith("applied"))
|
||||
|
||||
|
||||
def _with_report_source(reports: list[dict[str, Any]], source: str) -> list[dict[str, Any]]:
|
||||
"""给补丁报告补上来源字段(extract / verify 标注)。
|
||||
|
||||
参数:
|
||||
reports: 原始补丁报告列表。
|
||||
source: 来源标签("extract" / "verify")。
|
||||
|
||||
返回:
|
||||
每条追加 ``"source"`` 字段的新列表。
|
||||
"""
|
||||
return [{**r, "source": source} for r in reports]
|
||||
|
||||
|
||||
def _append_retry_messages(
|
||||
messages: list[dict[str, Any]],
|
||||
raw_content: str,
|
||||
feedback: str,
|
||||
) -> None:
|
||||
"""向对话中追加一次失败后的重试反馈(assistant + user)。
|
||||
|
||||
参数:
|
||||
messages: 当前对话消息列表(原地修改)。
|
||||
raw_content: 上一轮 LLM 原始输出。
|
||||
feedback: 给 LLM 的纠正反馈。
|
||||
"""
|
||||
messages.append({"role": "assistant", "content": raw_content})
|
||||
messages.append({"role": "user", "content": feedback})
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# K. 补丁进化循环
|
||||
# =========================================================================
|
||||
|
||||
|
||||
async def _run_patch_evolution_loop(
|
||||
*,
|
||||
llm: LLMProvider,
|
||||
messages: list[dict[str, Any]],
|
||||
attempts: list[dict[str, Any]],
|
||||
target_file: str,
|
||||
target_type: str,
|
||||
original_content: str,
|
||||
source_version: str,
|
||||
log_target: str,
|
||||
attempt_builder: Callable[[dict[str, Any]], Awaitable[_PatchEvolutionAttempt]],
|
||||
) -> EvolutionRecord:
|
||||
"""执行带补丁应用与 no-op 重试的两轮进化循环。
|
||||
|
||||
恰好 2 次尝试(range(2)),三种失败模式各有对应重试提示:
|
||||
1. JSON 解析失败 → "你的输出不是合法 JSON,请重新输出"
|
||||
2. 0 条 applied 补丁 → "你的 edit 的 target 都没在原文中匹配到…"
|
||||
3. 校验失败 → 具体校验错误文本
|
||||
|
||||
首轮失败追加重试提示继续第二轮;第二轮失败直接 reject。
|
||||
校验通过立即返回 accepted。
|
||||
|
||||
参数:
|
||||
llm: LLM 调用端口。
|
||||
messages: 对话消息列表(原地修改,追加重试上下文)。
|
||||
attempts: 尝试摘要列表(原地追加)。
|
||||
target_file: 目标文件名。
|
||||
target_type: 目标类型(skill / system / tool)。
|
||||
original_content: 改写前原文。
|
||||
source_version: 改写前版本号。
|
||||
log_target: 日志标签。
|
||||
attempt_builder: 异步构建尝试的回调,接收 parsed JSON dict。
|
||||
|
||||
返回:
|
||||
EvolutionRecord 实例。
|
||||
"""
|
||||
for attempt_idx in range(2):
|
||||
response = await llm.chat(messages)
|
||||
raw_content = response.content
|
||||
attempts.append({"attempt": attempt_idx + 1, "raw_length": len(raw_content)})
|
||||
parsed = _parse_llm_json(raw_content)
|
||||
|
||||
# 失败模式 1:JSON 解析失败
|
||||
if parsed is None:
|
||||
logger.warning("{} 进化 LLM 响应 JSON 解析失败: {}", target_type, log_target)
|
||||
if attempt_idx == 0:
|
||||
_append_retry_messages(
|
||||
messages,
|
||||
raw_content,
|
||||
"你的输出不是合法 JSON,请重新输出。",
|
||||
)
|
||||
continue
|
||||
return EvolutionRecord(
|
||||
target_file=target_file,
|
||||
target_type=target_type,
|
||||
original_content=original_content,
|
||||
evolved_content=original_content,
|
||||
reason="LLM 响应 JSON 解析失败",
|
||||
status="rejected",
|
||||
source_version=source_version,
|
||||
attempts=attempts,
|
||||
validation_errors=["JSON 解析失败"],
|
||||
)
|
||||
|
||||
attempt = await attempt_builder(parsed)
|
||||
|
||||
# 失败模式 2:0 条 applied 补丁
|
||||
if _count_applied_reports(attempt.apply_report) == 0:
|
||||
if attempt_idx == 0:
|
||||
_append_retry_messages(
|
||||
messages,
|
||||
raw_content,
|
||||
"你的 edit 的 target 都没在原文中匹配到,请逐字摘抄原文锚点后重输。",
|
||||
)
|
||||
continue
|
||||
return EvolutionRecord(
|
||||
target_file=target_file,
|
||||
target_type=target_type,
|
||||
original_content=original_content,
|
||||
evolved_content=original_content,
|
||||
reason="补丁无有效改动(target 全未匹配)",
|
||||
status="rejected",
|
||||
source_version=source_version,
|
||||
suggestions=attempt.suggestions,
|
||||
attempts=attempts,
|
||||
edits=attempt.edits,
|
||||
apply_report=attempt.apply_report,
|
||||
clip_info=attempt.clip_info,
|
||||
)
|
||||
|
||||
# 成功:校验通过
|
||||
if attempt.validation.passed:
|
||||
return EvolutionRecord(
|
||||
target_file=target_file,
|
||||
target_type=target_type,
|
||||
original_content=original_content,
|
||||
evolved_content=attempt.evolved_content,
|
||||
reason="验证通过",
|
||||
status="accepted",
|
||||
source_version=source_version,
|
||||
suggestions=attempt.suggestions,
|
||||
attempts=attempts,
|
||||
edits=attempt.edits,
|
||||
apply_report=attempt.apply_report,
|
||||
clip_info=attempt.clip_info,
|
||||
)
|
||||
|
||||
# 失败模式 3:校验失败
|
||||
error_feedback = "\n".join(attempt.validation.errors)
|
||||
if attempt_idx == 0:
|
||||
_append_retry_messages(
|
||||
messages,
|
||||
raw_content,
|
||||
f"验证失败,请修正后重新输出:\n{error_feedback}",
|
||||
)
|
||||
continue
|
||||
return EvolutionRecord(
|
||||
target_file=target_file,
|
||||
target_type=target_type,
|
||||
original_content=original_content,
|
||||
evolved_content=original_content,
|
||||
reason="验证失败(重试后仍未通过)",
|
||||
status="rejected",
|
||||
source_version=source_version,
|
||||
suggestions=attempt.suggestions,
|
||||
attempts=attempts,
|
||||
validation_errors=attempt.validation.errors,
|
||||
edits=attempt.edits,
|
||||
apply_report=attempt.apply_report,
|
||||
clip_info=attempt.clip_info,
|
||||
)
|
||||
|
||||
# 兜底(正常流程不可达)
|
||||
return EvolutionRecord(
|
||||
target_file=target_file,
|
||||
target_type=target_type,
|
||||
original_content=original_content,
|
||||
evolved_content=original_content,
|
||||
reason="未知错误",
|
||||
status="rejected",
|
||||
source_version=source_version,
|
||||
attempts=attempts,
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# L. Lapse-only 尝试构建
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def _build_lapse_only_attempt(
|
||||
original_content: str,
|
||||
lapse_notes: list[str],
|
||||
) -> _PatchEvolutionAttempt:
|
||||
"""构造「仅 appendix 更新」尝试:无 defect edit,只把 lapse 提醒落进受保护区。
|
||||
|
||||
LLM 没给 defect edit 但有 lapse 提醒时,正常补丁路径会因 0 条 applied 报告被
|
||||
``_run_patch_evolution_loop`` 判为 no-op 而丢弃。这里给出一条 ``applied_append``
|
||||
合成报告(前缀 ``applied`` 使 ``_count_applied_reports > 0``),令该记录走
|
||||
accepted 路径、appendix 真正落盘。
|
||||
|
||||
参数:
|
||||
original_content: 改写前全文。
|
||||
lapse_notes: 待落 appendix 的 LAPSE 提醒。
|
||||
|
||||
返回:
|
||||
含 appendix 更新内容与合成 apply_report 的 _PatchEvolutionAttempt。
|
||||
"""
|
||||
evolved_content = append_to_appendix(original_content, lapse_notes)
|
||||
apply_report = [
|
||||
{
|
||||
"op": "append",
|
||||
"target": "",
|
||||
"content_preview": "appendix LAPSE 提醒",
|
||||
"status": "applied_append",
|
||||
"index": 1,
|
||||
}
|
||||
]
|
||||
return _PatchEvolutionAttempt(
|
||||
evolved_content=evolved_content,
|
||||
validation=validate_skill(original_content, evolved_content),
|
||||
suggestions=[],
|
||||
edits=[],
|
||||
apply_report=apply_report,
|
||||
clip_info={"triggered": False, "clipped": 0},
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# M. Appendix consolidation
|
||||
# =========================================================================
|
||||
|
||||
|
||||
_CONSOLIDATE_SYSTEM = (
|
||||
"你在压缩一个 agent skill 的「执行提醒 appendix」。每条提醒都重申一条 skill 已有"
|
||||
"规则、是 agent 没遵循的点。你的任务是周期性压缩:去重、合并近义、精简措辞,但"
|
||||
"保留每条的可执行性。禁止发明新规则;禁止写入任何具体题目/选项/实体名等案例事实。"
|
||||
"只返回 JSON。"
|
||||
)
|
||||
|
||||
|
||||
async def consolidate_appendix(llm: LLMProvider, notes: list[str]) -> list[str]:
|
||||
"""LLM 压缩 appendix notes(去重/合并/精简),失败永不丢内容。
|
||||
|
||||
四关守卫(对标 TRM4 consolidate_appendix):
|
||||
G1. clean 后 <2 条直接短路返回(无需压缩,不调 LLM)。
|
||||
G2. 只接受「非空且 len(compacted) <= len(clean)」的压缩结果。
|
||||
G3. 任何异常(解析/空/网络)→ 返回 clean(绝不丢 appendix)。
|
||||
G4. 在调用方 _append_lapse_with_consolidation 中:
|
||||
len(compacted) >= len(notes) → 拒绝等长压缩。
|
||||
|
||||
参数:
|
||||
llm: LLM 调用端口。
|
||||
notes: 待压缩的 appendix 提醒列表。
|
||||
|
||||
返回:
|
||||
压缩后的提醒列表;任何守卫未通过时返回 clean 后的原 notes。
|
||||
"""
|
||||
# G1:clean 后不足 2 条,直接返回
|
||||
clean = [str(n).strip() for n in (notes or []) if str(n).strip()]
|
||||
if len(clean) < 2:
|
||||
return clean
|
||||
|
||||
numbered = "\n".join(f"{i}. {n}" for i, n in enumerate(clean, 1))
|
||||
user = (
|
||||
f"## 当前执行提醒(共 {len(clean)} 条)\n{numbered}\n\n"
|
||||
"压缩为更短的列表,不丢失可执行信息;合并重复与近义;保持每条简短具体可复用。"
|
||||
'只返回 JSON:{ "appendix_notes": ["压缩后提醒1", "压缩后提醒2"] }'
|
||||
)
|
||||
try:
|
||||
response = await llm.chat(
|
||||
[
|
||||
{"role": "system", "content": _CONSOLIDATE_SYSTEM},
|
||||
{"role": "user", "content": user},
|
||||
]
|
||||
)
|
||||
parsed = _parse_llm_json(response.content)
|
||||
compacted = [
|
||||
str(n).strip() for n in (parsed or {}).get("appendix_notes", []) if str(n).strip()
|
||||
]
|
||||
# G2:非空且确实压缩了
|
||||
if compacted and len(compacted) <= len(clean):
|
||||
return compacted
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# G3:任何失败降级为保留原 notes。设计授权的优雅降级(非 P5 违规):
|
||||
# consolidation 是纯优化,失败不应中断 evolve 或丢 appendix;记 warning 非静默。
|
||||
logger.warning("appendix consolidation 失败,保留原 notes:{}", exc)
|
||||
return clean
|
||||
|
||||
|
||||
async def _append_lapse_with_consolidation(
|
||||
text: str,
|
||||
lapse_notes: list[str],
|
||||
llm: LLMProvider,
|
||||
consolidate_threshold: int,
|
||||
) -> str:
|
||||
"""把 lapse 提醒追加进 appendix,超阈值时触发 LLM consolidation。
|
||||
|
||||
回写侧二确认——即便 consolidate_appendix 守卫已保证 <=,这里再校验「确实
|
||||
变短」才 replace,避免等长压缩带来无意义改写抖动(守卫 G4)。
|
||||
|
||||
参数:
|
||||
text: 待追加的 skill 全文(正文已改完)。
|
||||
lapse_notes: 本轮待落 appendix 的 LAPSE 提醒。
|
||||
llm: LLM 调用端口,供 consolidation 使用。
|
||||
consolidate_threshold: appendix note 条数 >= 此值时触发压缩。
|
||||
|
||||
返回:
|
||||
追加(必要时压缩)后的 skill 全文。
|
||||
"""
|
||||
after = append_to_appendix(text, lapse_notes)
|
||||
notes = extract_appendix_notes(after)
|
||||
if len(notes) >= consolidate_threshold:
|
||||
compacted = await consolidate_appendix(llm, notes)
|
||||
# G4:压缩结果必须严格变短才替换
|
||||
if len(compacted) < len(notes):
|
||||
after = replace_appendix_notes(after, compacted)
|
||||
return after
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# N. 整篇重写
|
||||
# =========================================================================
|
||||
|
||||
|
||||
_REWRITE_SYSTEM = (
|
||||
"你负责根据改动建议整篇重写 Agent Skill 文件。保留 frontmatter(---...---)中的 "
|
||||
"name / description / task_type 不变。保持文件精简,重写后长度不得超过原文。"
|
||||
'只返回 JSON:{ "rewritten": "重写后的完整文件内容" }'
|
||||
)
|
||||
|
||||
|
||||
async def rewrite_from_suggestions(
|
||||
llm: LLMProvider,
|
||||
original: str,
|
||||
suggestions: list[dict[str, Any]],
|
||||
) -> str:
|
||||
"""从抽象 suggestion 整篇重写 Skill;校验失败回退原文(skill 不变)。
|
||||
|
||||
硬约束由系统提示下达 + 本函数校验双重保证。三条拒绝条件任一触发
|
||||
即返回原文(保守不改):
|
||||
1. 解析失败(JSON / rewritten 字段缺失或非字符串)
|
||||
2. 重写后长度 > 原文
|
||||
3. validate_skill 校验不过
|
||||
|
||||
仅捕获 ValueError / KeyError / TypeError / AttributeError;API 错误向上传播。
|
||||
|
||||
参数:
|
||||
llm: LLM 调用端口。
|
||||
original: 改写前 Skill 文件全文。
|
||||
suggestions: 抽象改动建议列表。
|
||||
|
||||
返回:
|
||||
校验通过的重写全文;任一守卫未通过时返回 original。
|
||||
"""
|
||||
sugg_text = "\n".join(f"- {s.get('change', '')}" for s in (suggestions or []))
|
||||
user_msg = f"## 当前 Skill 文件\n\n{original}\n\n## 改动建议\n\n{sugg_text or '(无)'}"
|
||||
try:
|
||||
response = await llm.chat(
|
||||
[
|
||||
{"role": "system", "content": _REWRITE_SYSTEM},
|
||||
{"role": "user", "content": user_msg},
|
||||
]
|
||||
)
|
||||
parsed = _parse_llm_json(response.content)
|
||||
rewritten = (parsed or {}).get("rewritten")
|
||||
if not isinstance(rewritten, str) or not rewritten.strip():
|
||||
raise ValueError("rewrite 未返回非空 rewritten 字符串")
|
||||
except (ValueError, KeyError, TypeError, AttributeError):
|
||||
logger.warning("rewrite 解析失败,回退原文")
|
||||
return original
|
||||
|
||||
# 拒绝条件 2:不许变长
|
||||
if len(rewritten) > len(original):
|
||||
logger.warning("rewrite 变长({}->{}),回退原文", len(original), len(rewritten))
|
||||
return original
|
||||
# 拒绝条件 3:冻结区/格式校验
|
||||
if not validate_skill(original, rewritten).passed:
|
||||
logger.warning("rewrite 校验未过(冻结区/格式),回退原文")
|
||||
return original
|
||||
return rewritten
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# O. 单目标进化函数
|
||||
# =========================================================================
|
||||
|
||||
|
||||
async def evolve_single_skill(
|
||||
llm: LLMProvider,
|
||||
pack: SkillCasePack,
|
||||
skill_store: SkillStore,
|
||||
prompts: EvolvePrompts,
|
||||
source_version: str,
|
||||
edit_budget: int,
|
||||
consolidate_threshold: int,
|
||||
*,
|
||||
skill_update_mode: Literal["patch", "rewrite"] = "patch",
|
||||
rejected: list[RejectedEdit] | None = None,
|
||||
) -> EvolutionRecord:
|
||||
"""进化单个 Skill 文件。
|
||||
|
||||
三分支构建:
|
||||
A. lapse-only:无 defect edit + 有 lapse_notes → 仅 appendix 更新。
|
||||
B. rewrite:mode="rewrite" + 有 edit → 整篇重写;失败回退 A 或 no-op。
|
||||
C. patch(默认):rank_and_clip → apply_patch_with_report。
|
||||
分支 B/C 完成后,若有 lapse_notes,追加 appendix(超阈值 consolidation)。
|
||||
|
||||
用户消息结构(按顺序):
|
||||
1. (可选)黑名单
|
||||
2. 当前 Skill 文件原文
|
||||
3. 聚合统计 JSON
|
||||
4. 失败案例
|
||||
5. 成功案例
|
||||
|
||||
参数:
|
||||
llm: LLM 调用端口。
|
||||
pack: 该题型的案例包。
|
||||
skill_store: 版本化技能读取端口。
|
||||
prompts: 进化模板束。
|
||||
source_version: 改写前版本号。
|
||||
edit_budget: per-target 编辑预算上限。
|
||||
consolidate_threshold: appendix note 条数 >= 此值触发 consolidation。
|
||||
skill_update_mode: 正文更新模式,"patch"(局部 edit)/ "rewrite"(整篇重写)。
|
||||
rejected: 已验证无效的历史改法列表。
|
||||
|
||||
返回:
|
||||
EvolutionRecord 实例。
|
||||
"""
|
||||
target_file = pack.target_file
|
||||
original_content = skill_store.read_skill(target_file)
|
||||
|
||||
# 构建用户消息
|
||||
stats_json = json.dumps(pack.stats, ensure_ascii=False, indent=2)
|
||||
user_msg = (
|
||||
f"## 当前 Skill 文件\n\n{original_content}\n\n"
|
||||
f"## 聚合统计\n\n```json\n{stats_json}\n```\n\n"
|
||||
f"## 失败案例\n\n{_format_case_samples(pack.failure_cases)}\n\n"
|
||||
f"## 成功案例\n\n{_format_case_samples(pack.success_cases)}"
|
||||
)
|
||||
rejected = rejected or []
|
||||
if rejected:
|
||||
user_msg = (
|
||||
"## 已验证无效的改法(黑名单,勿重复)\n\n"
|
||||
+ _format_rejected_edits(rejected)
|
||||
+ "\n\n"
|
||||
+ user_msg
|
||||
)
|
||||
|
||||
messages: list[dict[str, Any]] = [
|
||||
{"role": "system", "content": prompts.evolve_skill},
|
||||
{"role": "user", "content": user_msg},
|
||||
]
|
||||
attempts: list[dict[str, Any]] = []
|
||||
|
||||
async def _build_attempt(parsed: dict[str, Any]) -> _PatchEvolutionAttempt:
|
||||
suggestions = parsed.get("suggestions", [])
|
||||
edits = parsed.get("edits", [])
|
||||
|
||||
# 分支 A:lapse-only(无 defect edit、仅有 lapse 提醒)
|
||||
if not edits and pack.lapse_notes:
|
||||
return _build_lapse_only_attempt(original_content, pack.lapse_notes)
|
||||
|
||||
# 分支 B:rewrite 模式(有 defect edits 时整篇重写)
|
||||
if skill_update_mode == "rewrite" and edits:
|
||||
rewritten = await rewrite_from_suggestions(llm, original_content, suggestions)
|
||||
if rewritten == original_content:
|
||||
# 重写校验失败/变长/解析失败 → 正文无改动
|
||||
if pack.lapse_notes:
|
||||
return _build_lapse_only_attempt(original_content, pack.lapse_notes)
|
||||
return _PatchEvolutionAttempt(
|
||||
evolved_content=original_content,
|
||||
validation=validate_skill(original_content, original_content),
|
||||
suggestions=suggestions,
|
||||
edits=[],
|
||||
apply_report=[],
|
||||
clip_info={"triggered": False, "clipped": 0},
|
||||
)
|
||||
evolved_content = rewritten
|
||||
apply_report = [
|
||||
{
|
||||
"op": "rewrite",
|
||||
"target": "",
|
||||
"content_preview": "整篇重写",
|
||||
"status": "applied_rewrite",
|
||||
"index": 1,
|
||||
}
|
||||
]
|
||||
clip_info: dict[str, Any] = {"triggered": False, "clipped": 0}
|
||||
|
||||
# 分支 C:patch 模式(默认)
|
||||
else:
|
||||
edits, clip_info = await rank_and_clip(
|
||||
llm,
|
||||
original_content,
|
||||
edits,
|
||||
edit_budget,
|
||||
"skill",
|
||||
rank_prompt=prompts.evolve_rank,
|
||||
)
|
||||
evolved_content, apply_report = apply_patch_with_report(
|
||||
original_content,
|
||||
edits,
|
||||
protected_spans=_skill_protected_spans(original_content),
|
||||
)
|
||||
|
||||
# 分支 B/C 完成后:lapse 提醒追加 + consolidation
|
||||
if pack.lapse_notes:
|
||||
evolved_content = await _append_lapse_with_consolidation(
|
||||
evolved_content,
|
||||
pack.lapse_notes,
|
||||
llm,
|
||||
consolidate_threshold,
|
||||
)
|
||||
|
||||
return _PatchEvolutionAttempt(
|
||||
evolved_content=evolved_content,
|
||||
validation=validate_skill(original_content, evolved_content),
|
||||
suggestions=suggestions,
|
||||
edits=edits,
|
||||
apply_report=apply_report,
|
||||
clip_info=clip_info,
|
||||
)
|
||||
|
||||
return await _run_patch_evolution_loop(
|
||||
llm=llm,
|
||||
messages=messages,
|
||||
attempts=attempts,
|
||||
target_file=target_file,
|
||||
target_type="skill",
|
||||
original_content=original_content,
|
||||
source_version=source_version,
|
||||
log_target=target_file,
|
||||
attempt_builder=_build_attempt,
|
||||
)
|
||||
|
||||
|
||||
async def evolve_system_prompt(
|
||||
llm: LLMProvider,
|
||||
pack: SystemCasePack,
|
||||
prompt_store: PromptStore,
|
||||
prompts: EvolvePrompts,
|
||||
source_version: str,
|
||||
edit_budget: int,
|
||||
) -> EvolutionRecord:
|
||||
"""进化 System Prompt。
|
||||
|
||||
无 lapse notes、无 appendix consolidation、无 rewrite 模式。
|
||||
使用 system protected spans 保护冻结区。
|
||||
用户消息中统计标题为「D5 行为模式统计」。
|
||||
|
||||
参数:
|
||||
llm: LLM 调用端口。
|
||||
pack: 跨题型行为模式案例包。
|
||||
prompt_store: 版本化提示词读取端口。
|
||||
prompts: 进化模板束。
|
||||
source_version: 改写前版本号。
|
||||
edit_budget: per-target 编辑预算上限。
|
||||
|
||||
返回:
|
||||
EvolutionRecord 实例。
|
||||
"""
|
||||
target_file = "system.md"
|
||||
original_content = prompt_store.read_prompt(target_file)
|
||||
|
||||
stats_json = json.dumps(pack.stats, ensure_ascii=False, indent=2)
|
||||
user_msg = (
|
||||
f"## 当前 System Prompt\n\n{original_content}\n\n"
|
||||
f"## D5 行为模式统计\n\n```json\n{stats_json}\n```\n\n"
|
||||
f"## 失败案例\n\n{_format_case_samples(pack.failure_cases)}\n\n"
|
||||
f"## 成功案例\n\n{_format_case_samples(pack.success_cases)}"
|
||||
)
|
||||
|
||||
messages: list[dict[str, Any]] = [
|
||||
{"role": "system", "content": prompts.evolve_system},
|
||||
{"role": "user", "content": user_msg},
|
||||
]
|
||||
attempts: list[dict[str, Any]] = []
|
||||
|
||||
async def _build_attempt(parsed: dict[str, Any]) -> _PatchEvolutionAttempt:
|
||||
suggestions = parsed.get("suggestions", [])
|
||||
edits = parsed.get("edits", [])
|
||||
edits, clip_info = await rank_and_clip(
|
||||
llm,
|
||||
original_content,
|
||||
edits,
|
||||
edit_budget,
|
||||
"system",
|
||||
rank_prompt=prompts.evolve_rank,
|
||||
)
|
||||
evolved_content, apply_report = apply_patch_with_report(
|
||||
original_content,
|
||||
edits,
|
||||
protected_spans=_system_protected_spans(original_content),
|
||||
)
|
||||
return _PatchEvolutionAttempt(
|
||||
evolved_content=evolved_content,
|
||||
validation=validate_system(original_content, evolved_content),
|
||||
suggestions=suggestions,
|
||||
edits=edits,
|
||||
apply_report=apply_report,
|
||||
clip_info=clip_info,
|
||||
)
|
||||
|
||||
return await _run_patch_evolution_loop(
|
||||
llm=llm,
|
||||
messages=messages,
|
||||
attempts=attempts,
|
||||
target_file=target_file,
|
||||
target_type="system",
|
||||
original_content=original_content,
|
||||
source_version=source_version,
|
||||
log_target=target_file,
|
||||
attempt_builder=_build_attempt,
|
||||
)
|
||||
|
||||
|
||||
async def evolve_single_tool(
|
||||
llm: LLMProvider,
|
||||
pack: ToolCasePack,
|
||||
prompt_store: PromptStore,
|
||||
prompts: EvolvePrompts,
|
||||
source_version: str,
|
||||
edit_budget: int,
|
||||
) -> EvolutionRecord:
|
||||
"""进化单个工具的 extract + verify prompt。
|
||||
|
||||
extract 与 verify 的 edits 合并到 SHARED 预算池(打 ``_src`` 标签),
|
||||
整体 rank_and_clip 到 edit_budget 后按 ``_src`` 拆回各自文件应用。
|
||||
``evolved_content`` 以 ``json.dumps({"extract": ..., "verify": ...})`` 存储。
|
||||
``target_file`` 固定为 ``{tool_name}_extract.md``。
|
||||
apply_report 每条带 ``"source"`` 注解("extract" / "verify")。
|
||||
|
||||
参数:
|
||||
llm: LLM 调用端口。
|
||||
pack: 该工具的案例包。
|
||||
prompt_store: 版本化提示词读取端口。
|
||||
prompts: 进化模板束。
|
||||
source_version: 改写前版本号。
|
||||
edit_budget: per-target 编辑预算上限(extract + verify 共享)。
|
||||
|
||||
返回:
|
||||
EvolutionRecord 实例。
|
||||
"""
|
||||
tool_name = pack.tool_name
|
||||
target_file = f"{tool_name}_extract.md"
|
||||
orig_extract = prompt_store.read_prompt(f"{tool_name}_extract.md")
|
||||
orig_verify = prompt_store.read_prompt(f"{tool_name}_verify.md")
|
||||
original_combined = json.dumps(
|
||||
{"extract": orig_extract, "verify": orig_verify},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
stats_json = json.dumps(pack.stats, ensure_ascii=False, indent=2)
|
||||
user_msg = (
|
||||
f"## 当前 extract prompt\n\n{orig_extract}\n\n"
|
||||
f"## 当前 verify prompt\n\n{orig_verify}\n\n"
|
||||
f"## 工具质量统计\n\n```json\n{stats_json}\n```\n\n"
|
||||
f"## 失败 span 案例\n\n{_format_spans(pack.failure_spans)}\n\n"
|
||||
f"## 成功 span 案例\n\n{_format_spans(pack.success_spans)}"
|
||||
)
|
||||
|
||||
messages: list[dict[str, Any]] = [
|
||||
{"role": "system", "content": prompts.evolve_tool},
|
||||
{"role": "user", "content": user_msg},
|
||||
]
|
||||
attempts: list[dict[str, Any]] = []
|
||||
|
||||
async def _build_attempt(parsed: dict[str, Any]) -> _PatchEvolutionAttempt:
|
||||
suggestions = parsed.get("suggestions", [])
|
||||
edits_extract = parsed.get("edits_extract", [])
|
||||
edits_verify = parsed.get("edits_verify", [])
|
||||
|
||||
# 合并打来源标记,对单 tool target 整体裁到 edit_budget
|
||||
pool: list[dict[str, Any]] = [{**e, "_src": "extract"} for e in edits_extract] + [
|
||||
{**e, "_src": "verify"} for e in edits_verify
|
||||
]
|
||||
|
||||
pool, clip_info = await rank_and_clip(
|
||||
llm,
|
||||
original_combined,
|
||||
pool,
|
||||
edit_budget,
|
||||
"tool",
|
||||
rank_prompt=prompts.evolve_rank,
|
||||
)
|
||||
|
||||
# 按 _src 拆回,并剥离 _src 字段
|
||||
extract_kept = [
|
||||
{k: v for k, v in e.items() if k != "_src"} for e in pool if e["_src"] == "extract"
|
||||
]
|
||||
verify_kept = [
|
||||
{k: v for k, v in e.items() if k != "_src"} for e in pool if e["_src"] == "verify"
|
||||
]
|
||||
|
||||
# 分别应用,使用各自的 tool protected spans
|
||||
evolved_extract, extract_report = apply_patch_with_report(
|
||||
orig_extract,
|
||||
extract_kept,
|
||||
protected_spans=_tool_protected_spans(orig_extract),
|
||||
)
|
||||
evolved_verify, verify_report = apply_patch_with_report(
|
||||
orig_verify,
|
||||
verify_kept,
|
||||
protected_spans=_tool_protected_spans(orig_verify),
|
||||
)
|
||||
|
||||
# 报告注解来源
|
||||
apply_report = _with_report_source(extract_report, "extract") + _with_report_source(
|
||||
verify_report, "verify"
|
||||
)
|
||||
|
||||
evolved_combined = json.dumps(
|
||||
{"extract": evolved_extract, "verify": evolved_verify},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
validation = validate_tool(orig_extract, evolved_extract, orig_verify, evolved_verify)
|
||||
return _PatchEvolutionAttempt(
|
||||
evolved_content=evolved_combined,
|
||||
validation=validation,
|
||||
suggestions=suggestions,
|
||||
edits=extract_kept + verify_kept,
|
||||
apply_report=apply_report,
|
||||
clip_info=clip_info,
|
||||
)
|
||||
|
||||
return await _run_patch_evolution_loop(
|
||||
llm=llm,
|
||||
messages=messages,
|
||||
attempts=attempts,
|
||||
target_file=target_file,
|
||||
target_type="tool",
|
||||
original_content=original_combined,
|
||||
source_version=source_version,
|
||||
log_target=tool_name,
|
||||
attempt_builder=_build_attempt,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user