feat(agent): step-level retry for transient LLM errors (20s/40s backoff)
核心算法 #10（Agent Loop）：仅加固异常路径的韧性兜底，不改变 解析协议、hook 时序与步数语义。benchmark 错题 796-3 显示一次 SSL BAD_RECORD_MAC 穿透 GovernedLLMClient 重试栈后废掉 13 步 已积累上下文；本次在 run() Phase 1 增加步级重试（默认 2 次， 20s/40s 退避），可重试异常限定 (TimeoutError, OSError)，非可 重试异常照旧 fail-fast 整题终止，行为与现状一致（P5 显式异常）。 为满足 radon C 级复杂度约束，重试循环抽取为私有方法 _call_llm_with_step_retry，行为不变。
This commit is contained in:
+64
-2
@@ -17,6 +17,7 @@ TRM4 → TRM5 有意变更（非简化）:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
from typing import TYPE_CHECKING, Any
|
||||
@@ -71,6 +72,12 @@ class AgentLoop:
|
||||
llm: LLMProvider 实例（Protocol 类型化，提供 async chat 方法）。
|
||||
max_steps: 最大有效步数（每次成功工具调用计一步）。
|
||||
max_retries: JSON 解析连续失败的最大容忍次数。
|
||||
step_retries: LLM 瞬时异常的步级重试次数（不含首次调用）。
|
||||
step_retry_delays: 步级重试的退避秒数序列，超出部分取末值。
|
||||
retryable_exceptions: 可重试异常元组，默认 (TimeoutError, OSError)——
|
||||
ssl.SSLError/ConnectionError 均为 OSError 子类，覆盖穿透
|
||||
GovernedLLMClient 内部重试栈的瞬时异常；openai API 类异常由
|
||||
治理层负责，core 不依赖 SDK。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -78,10 +85,17 @@ class AgentLoop:
|
||||
llm: LLMProvider,
|
||||
max_steps: int,
|
||||
max_retries: int = 3,
|
||||
*,
|
||||
step_retries: int = 2,
|
||||
step_retry_delays: tuple[float, ...] = (20.0, 40.0),
|
||||
retryable_exceptions: tuple[type[BaseException], ...] = (TimeoutError, OSError),
|
||||
) -> None:
|
||||
self._llm = llm
|
||||
self._max_steps = max_steps
|
||||
self._max_retries = max_retries
|
||||
self._step_retries = step_retries
|
||||
self._step_retry_delays = step_retry_delays
|
||||
self._retryable_exceptions = retryable_exceptions
|
||||
|
||||
async def run(
|
||||
self,
|
||||
@@ -118,9 +132,11 @@ class AgentLoop:
|
||||
while step_count < self._max_steps:
|
||||
await _call_hook(pm.hook.before_step, iteration=iteration, messages=messages)
|
||||
|
||||
# Phase 1: LLM 调用
|
||||
# Phase 1: LLM 调用（步级重试：防穿透 GovernedLLMClient 的瞬时异常）
|
||||
try:
|
||||
response = await self._call_llm(messages, token_usage, session_id=session_id)
|
||||
response = await self._call_llm_with_step_retry(
|
||||
messages, token_usage, session_id=session_id
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("LLM API 调用失败: {}", e)
|
||||
result = LoopResult(
|
||||
@@ -244,6 +260,52 @@ class AgentLoop:
|
||||
pm.register(plugin)
|
||||
return pm
|
||||
|
||||
async def _call_llm_with_step_retry(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
token_usage: dict[str, int],
|
||||
*,
|
||||
session_id: str | None = None,
|
||||
) -> LLMResponse:
|
||||
"""带步级重试的 LLM 调用，兜底穿透治理层重试栈的瞬时异常。
|
||||
|
||||
可重试异常（self._retryable_exceptions）按 self._step_retry_delays
|
||||
退避后重发，重试预算（self._step_retries）耗尽后抛出最后一次异常；
|
||||
非可重试异常直接穿透（fail-fast）。asyncio.CancelledError 继承
|
||||
BaseException，天然不被捕获——取消信号照常传播。失败尝试的 error
|
||||
遥测由 GovernedLLMClient 内部负责，此处仅 loguru 记录。
|
||||
|
||||
参数:
|
||||
messages: 消息历史。
|
||||
token_usage: 可变字典，就地累加。
|
||||
session_id: 会话 ID，透传给 LLMProvider。
|
||||
|
||||
返回:
|
||||
LLMResponse 实例。
|
||||
|
||||
异常:
|
||||
重试耗尽或非可重试时，原样抛出底层异常。
|
||||
"""
|
||||
step_attempt = 0
|
||||
while True:
|
||||
try:
|
||||
return await self._call_llm(messages, token_usage, session_id=session_id)
|
||||
except self._retryable_exceptions as e:
|
||||
step_attempt += 1
|
||||
if step_attempt > self._step_retries:
|
||||
raise
|
||||
delay = self._step_retry_delays[
|
||||
min(step_attempt - 1, len(self._step_retry_delays) - 1)
|
||||
]
|
||||
logger.warning(
|
||||
"LLM 瞬时异常，步级重试 {}/{}（{}s 后重发）: {}",
|
||||
step_attempt,
|
||||
self._step_retries,
|
||||
delay,
|
||||
e,
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
async def _call_llm(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
|
||||
@@ -8,6 +8,7 @@ thinking 捕获、token 累加、call_id 透传、pluggy hook。
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import ssl
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
@@ -305,3 +306,60 @@ class TestParseNormalization:
|
||||
def test_empty_content_still_rejected(self) -> None:
|
||||
"""空 content 照旧拒绝，归一化不改变该边界。"""
|
||||
assert self._parse("") is None
|
||||
|
||||
|
||||
# ── A2 步级重试测试（Spec-1）──────────────────────────────────
|
||||
|
||||
|
||||
class TestStepLevelRetry:
|
||||
"""LLM 瞬时异常的步级重试：可重试元组 / 退避 / fail-fast。"""
|
||||
|
||||
def _make_loop(self, chat_side_effects: list) -> AgentLoop:
|
||||
"""构造 chat 按序抛异常/返回响应的 AgentLoop。"""
|
||||
llm = AsyncMock()
|
||||
llm.chat = AsyncMock(side_effect=chat_side_effects)
|
||||
return AgentLoop(llm=llm, max_steps=10)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transient_error_retried_then_succeeds(self, monkeypatch) -> None:
|
||||
"""SSL/超时瞬时异常按 20s/40s 退避重试，第三次成功 → finished。"""
|
||||
delays: list[float] = []
|
||||
|
||||
async def _fake_sleep(seconds: float) -> None:
|
||||
delays.append(seconds)
|
||||
|
||||
monkeypatch.setattr("core.agent.loop.asyncio.sleep", _fake_sleep)
|
||||
loop = self._make_loop(
|
||||
[
|
||||
ssl.SSLError("SSLV3_ALERT_BAD_RECORD_MAC"),
|
||||
TimeoutError("watchdog"),
|
||||
_make_response(_submit_json()),
|
||||
]
|
||||
)
|
||||
result = await loop.run("sys", "user", _StubDispatcher())
|
||||
assert result.stop_reason == "finished"
|
||||
assert delays == [20.0, 40.0]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_exhausted_terminates_with_error(self, monkeypatch) -> None:
|
||||
"""重试预算（首次 + 2 次）耗尽仍失败 → stop_reason=error。"""
|
||||
|
||||
async def _fake_sleep(seconds: float) -> None:
|
||||
pass
|
||||
|
||||
monkeypatch.setattr("core.agent.loop.asyncio.sleep", _fake_sleep)
|
||||
loop = self._make_loop([TimeoutError("t1"), TimeoutError("t2"), TimeoutError("t3")])
|
||||
result = await loop.run("sys", "user", _StubDispatcher())
|
||||
assert result.stop_reason == "error"
|
||||
assert loop._llm.chat.await_count == 3 # 首次 + 2 次重试
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_retryable_fails_fast(self, monkeypatch) -> None:
|
||||
"""非可重试异常不退避不重发，首次即终止 → stop_reason=error。"""
|
||||
sleep_mock = AsyncMock()
|
||||
monkeypatch.setattr("core.agent.loop.asyncio.sleep", sleep_mock)
|
||||
loop = self._make_loop([RuntimeError("programming bug")])
|
||||
result = await loop.run("sys", "user", _StubDispatcher())
|
||||
assert result.stop_reason == "error"
|
||||
sleep_mock.assert_not_awaited()
|
||||
assert loop._llm.chat.await_count == 1
|
||||
|
||||
Reference in New Issue
Block a user