feat(agent): step-level retry for transient LLM errors (20s/40s backoff)
核心算法 #10(Agent Loop):仅加固异常路径的韧性兜底,不改变 解析协议、hook 时序与步数语义。benchmark 错题 796-3 显示一次 SSL BAD_RECORD_MAC 穿透 GovernedLLMClient 重试栈后废掉 13 步 已积累上下文;本次在 run() Phase 1 增加步级重试(默认 2 次, 20s/40s 退避),可重试异常限定 (TimeoutError, OSError),非可 重试异常照旧 fail-fast 整题终止,行为与现状一致(P5 显式异常)。 为满足 radon C 级复杂度约束,重试循环抽取为私有方法 _call_llm_with_step_retry,行为不变。
This commit is contained in:
+64
-2
@@ -17,6 +17,7 @@ TRM4 → TRM5 有意变更(非简化):
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
@@ -71,6 +72,12 @@ class AgentLoop:
|
|||||||
llm: LLMProvider 实例(Protocol 类型化,提供 async chat 方法)。
|
llm: LLMProvider 实例(Protocol 类型化,提供 async chat 方法)。
|
||||||
max_steps: 最大有效步数(每次成功工具调用计一步)。
|
max_steps: 最大有效步数(每次成功工具调用计一步)。
|
||||||
max_retries: JSON 解析连续失败的最大容忍次数。
|
max_retries: JSON 解析连续失败的最大容忍次数。
|
||||||
|
step_retries: LLM 瞬时异常的步级重试次数(不含首次调用)。
|
||||||
|
step_retry_delays: 步级重试的退避秒数序列,超出部分取末值。
|
||||||
|
retryable_exceptions: 可重试异常元组,默认 (TimeoutError, OSError)——
|
||||||
|
ssl.SSLError/ConnectionError 均为 OSError 子类,覆盖穿透
|
||||||
|
GovernedLLMClient 内部重试栈的瞬时异常;openai API 类异常由
|
||||||
|
治理层负责,core 不依赖 SDK。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -78,10 +85,17 @@ class AgentLoop:
|
|||||||
llm: LLMProvider,
|
llm: LLMProvider,
|
||||||
max_steps: int,
|
max_steps: int,
|
||||||
max_retries: int = 3,
|
max_retries: int = 3,
|
||||||
|
*,
|
||||||
|
step_retries: int = 2,
|
||||||
|
step_retry_delays: tuple[float, ...] = (20.0, 40.0),
|
||||||
|
retryable_exceptions: tuple[type[BaseException], ...] = (TimeoutError, OSError),
|
||||||
) -> None:
|
) -> None:
|
||||||
self._llm = llm
|
self._llm = llm
|
||||||
self._max_steps = max_steps
|
self._max_steps = max_steps
|
||||||
self._max_retries = max_retries
|
self._max_retries = max_retries
|
||||||
|
self._step_retries = step_retries
|
||||||
|
self._step_retry_delays = step_retry_delays
|
||||||
|
self._retryable_exceptions = retryable_exceptions
|
||||||
|
|
||||||
async def run(
|
async def run(
|
||||||
self,
|
self,
|
||||||
@@ -118,9 +132,11 @@ class AgentLoop:
|
|||||||
while step_count < self._max_steps:
|
while step_count < self._max_steps:
|
||||||
await _call_hook(pm.hook.before_step, iteration=iteration, messages=messages)
|
await _call_hook(pm.hook.before_step, iteration=iteration, messages=messages)
|
||||||
|
|
||||||
# Phase 1: LLM 调用
|
# Phase 1: LLM 调用(步级重试:防穿透 GovernedLLMClient 的瞬时异常)
|
||||||
try:
|
try:
|
||||||
response = await self._call_llm(messages, token_usage, session_id=session_id)
|
response = await self._call_llm_with_step_retry(
|
||||||
|
messages, token_usage, session_id=session_id
|
||||||
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error("LLM API 调用失败: {}", e)
|
logger.error("LLM API 调用失败: {}", e)
|
||||||
result = LoopResult(
|
result = LoopResult(
|
||||||
@@ -244,6 +260,52 @@ class AgentLoop:
|
|||||||
pm.register(plugin)
|
pm.register(plugin)
|
||||||
return pm
|
return pm
|
||||||
|
|
||||||
|
async def _call_llm_with_step_retry(
|
||||||
|
self,
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
token_usage: dict[str, int],
|
||||||
|
*,
|
||||||
|
session_id: str | None = None,
|
||||||
|
) -> LLMResponse:
|
||||||
|
"""带步级重试的 LLM 调用,兜底穿透治理层重试栈的瞬时异常。
|
||||||
|
|
||||||
|
可重试异常(self._retryable_exceptions)按 self._step_retry_delays
|
||||||
|
退避后重发,重试预算(self._step_retries)耗尽后抛出最后一次异常;
|
||||||
|
非可重试异常直接穿透(fail-fast)。asyncio.CancelledError 继承
|
||||||
|
BaseException,天然不被捕获——取消信号照常传播。失败尝试的 error
|
||||||
|
遥测由 GovernedLLMClient 内部负责,此处仅 loguru 记录。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
messages: 消息历史。
|
||||||
|
token_usage: 可变字典,就地累加。
|
||||||
|
session_id: 会话 ID,透传给 LLMProvider。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
LLMResponse 实例。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
重试耗尽或非可重试时,原样抛出底层异常。
|
||||||
|
"""
|
||||||
|
step_attempt = 0
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
return await self._call_llm(messages, token_usage, session_id=session_id)
|
||||||
|
except self._retryable_exceptions as e:
|
||||||
|
step_attempt += 1
|
||||||
|
if step_attempt > self._step_retries:
|
||||||
|
raise
|
||||||
|
delay = self._step_retry_delays[
|
||||||
|
min(step_attempt - 1, len(self._step_retry_delays) - 1)
|
||||||
|
]
|
||||||
|
logger.warning(
|
||||||
|
"LLM 瞬时异常,步级重试 {}/{}({}s 后重发): {}",
|
||||||
|
step_attempt,
|
||||||
|
self._step_retries,
|
||||||
|
delay,
|
||||||
|
e,
|
||||||
|
)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
|
||||||
async def _call_llm(
|
async def _call_llm(
|
||||||
self,
|
self,
|
||||||
messages: list[dict[str, Any]],
|
messages: list[dict[str, Any]],
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ thinking 捕获、token 累加、call_id 透传、pluggy hook。
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import ssl
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
from unittest.mock import AsyncMock
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
@@ -305,3 +306,60 @@ class TestParseNormalization:
|
|||||||
def test_empty_content_still_rejected(self) -> None:
|
def test_empty_content_still_rejected(self) -> None:
|
||||||
"""空 content 照旧拒绝,归一化不改变该边界。"""
|
"""空 content 照旧拒绝,归一化不改变该边界。"""
|
||||||
assert self._parse("") is None
|
assert self._parse("") is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── A2 步级重试测试(Spec-1)──────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestStepLevelRetry:
|
||||||
|
"""LLM 瞬时异常的步级重试:可重试元组 / 退避 / fail-fast。"""
|
||||||
|
|
||||||
|
def _make_loop(self, chat_side_effects: list) -> AgentLoop:
|
||||||
|
"""构造 chat 按序抛异常/返回响应的 AgentLoop。"""
|
||||||
|
llm = AsyncMock()
|
||||||
|
llm.chat = AsyncMock(side_effect=chat_side_effects)
|
||||||
|
return AgentLoop(llm=llm, max_steps=10)
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_transient_error_retried_then_succeeds(self, monkeypatch) -> None:
|
||||||
|
"""SSL/超时瞬时异常按 20s/40s 退避重试,第三次成功 → finished。"""
|
||||||
|
delays: list[float] = []
|
||||||
|
|
||||||
|
async def _fake_sleep(seconds: float) -> None:
|
||||||
|
delays.append(seconds)
|
||||||
|
|
||||||
|
monkeypatch.setattr("core.agent.loop.asyncio.sleep", _fake_sleep)
|
||||||
|
loop = self._make_loop(
|
||||||
|
[
|
||||||
|
ssl.SSLError("SSLV3_ALERT_BAD_RECORD_MAC"),
|
||||||
|
TimeoutError("watchdog"),
|
||||||
|
_make_response(_submit_json()),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
result = await loop.run("sys", "user", _StubDispatcher())
|
||||||
|
assert result.stop_reason == "finished"
|
||||||
|
assert delays == [20.0, 40.0]
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_retry_exhausted_terminates_with_error(self, monkeypatch) -> None:
|
||||||
|
"""重试预算(首次 + 2 次)耗尽仍失败 → stop_reason=error。"""
|
||||||
|
|
||||||
|
async def _fake_sleep(seconds: float) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
monkeypatch.setattr("core.agent.loop.asyncio.sleep", _fake_sleep)
|
||||||
|
loop = self._make_loop([TimeoutError("t1"), TimeoutError("t2"), TimeoutError("t3")])
|
||||||
|
result = await loop.run("sys", "user", _StubDispatcher())
|
||||||
|
assert result.stop_reason == "error"
|
||||||
|
assert loop._llm.chat.await_count == 3 # 首次 + 2 次重试
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_non_retryable_fails_fast(self, monkeypatch) -> None:
|
||||||
|
"""非可重试异常不退避不重发,首次即终止 → stop_reason=error。"""
|
||||||
|
sleep_mock = AsyncMock()
|
||||||
|
monkeypatch.setattr("core.agent.loop.asyncio.sleep", sleep_mock)
|
||||||
|
loop = self._make_loop([RuntimeError("programming bug")])
|
||||||
|
result = await loop.run("sys", "user", _StubDispatcher())
|
||||||
|
assert result.stop_reason == "error"
|
||||||
|
sleep_mock.assert_not_awaited()
|
||||||
|
assert loop._llm.chat.await_count == 1
|
||||||
|
|||||||
Reference in New Issue
Block a user