From e3184c11f9adce3f2e8c05cd284d7a972311e341 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Sat, 11 Jul 2026 08:37:35 -0400 Subject: [PATCH] feat(agent): step-level retry for transient LLM errors (20s/40s backoff) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 核心算法 #10（Agent Loop）：仅加固异常路径的韧性兜底，不改变 解析协议、hook 时序与步数语义。benchmark 错题 796-3 显示一次 SSL BAD_RECORD_MAC 穿透 GovernedLLMClient 重试栈后废掉 13 步 已积累上下文；本次在 run() Phase 1 增加步级重试（默认 2 次， 20s/40s 退避），可重试异常限定 (TimeoutError, OSError)，非可 重试异常照旧 fail-fast 整题终止，行为与现状一致（P5 显式异常）。 为满足 radon C 级复杂度约束，重试循环抽取为私有方法 _call_llm_with_step_retry，行为不变。 --- core/agent/loop.py | 66 +++++++++++++++++++++++++++++++++-- tests/unit/test_agent_loop.py | 58 ++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/core/agent/loop.py b/core/agent/loop.py index e98cf20..c6a8d53 100644 --- a/core/agent/loop.py +++ b/core/agent/loop.py @@ -17,6 +17,7 @@ TRM4 → TRM5 有意变更（非简化）: from __future__ import annotations +import asyncio import json import re from typing import TYPE_CHECKING, Any @@ -71,6 +72,12 @@ class AgentLoop: llm: LLMProvider 实例（Protocol 类型化，提供 async chat 方法）。 max_steps: 最大有效步数（每次成功工具调用计一步）。 max_retries: JSON 解析连续失败的最大容忍次数。 + step_retries: LLM 瞬时异常的步级重试次数（不含首次调用）。 + step_retry_delays: 步级重试的退避秒数序列，超出部分取末值。 + retryable_exceptions: 可重试异常元组，默认 (TimeoutError, OSError)—— + ssl.SSLError/ConnectionError 均为 OSError 子类，覆盖穿透 + GovernedLLMClient 内部重试栈的瞬时异常；openai API 类异常由 + 治理层负责，core 不依赖 SDK。 """ def __init__( @@ -78,10 +85,17 @@ class AgentLoop: llm: LLMProvider, max_steps: int, max_retries: int = 3, + *, + step_retries: int = 2, + step_retry_delays: tuple[float, ...] = (20.0, 40.0), + retryable_exceptions: tuple[type[BaseException], ...] = (TimeoutError, OSError), ) -> None: self._llm = llm self._max_steps = max_steps self._max_retries = max_retries + self._step_retries = step_retries + self._step_retry_delays = step_retry_delays + self._retryable_exceptions = retryable_exceptions async def run( self, @@ -118,9 +132,11 @@ class AgentLoop: while step_count < self._max_steps: await _call_hook(pm.hook.before_step, iteration=iteration, messages=messages) - # Phase 1: LLM 调用 + # Phase 1: LLM 调用（步级重试：防穿透 GovernedLLMClient 的瞬时异常） try: - response = await self._call_llm(messages, token_usage, session_id=session_id) + response = await self._call_llm_with_step_retry( + messages, token_usage, session_id=session_id + ) except Exception as e: logger.error("LLM API 调用失败: {}", e) result = LoopResult( @@ -244,6 +260,52 @@ class AgentLoop: pm.register(plugin) return pm + async def _call_llm_with_step_retry( + self, + messages: list[dict[str, Any]], + token_usage: dict[str, int], + *, + session_id: str | None = None, + ) -> LLMResponse: + """带步级重试的 LLM 调用，兜底穿透治理层重试栈的瞬时异常。 + + 可重试异常（self._retryable_exceptions）按 self._step_retry_delays + 退避后重发，重试预算（self._step_retries）耗尽后抛出最后一次异常； + 非可重试异常直接穿透（fail-fast）。asyncio.CancelledError 继承 + BaseException，天然不被捕获——取消信号照常传播。失败尝试的 error + 遥测由 GovernedLLMClient 内部负责，此处仅 loguru 记录。 + + 参数: + messages: 消息历史。 + token_usage: 可变字典，就地累加。 + session_id: 会话 ID，透传给 LLMProvider。 + + 返回: + LLMResponse 实例。 + + 异常: + 重试耗尽或非可重试时，原样抛出底层异常。 + """ + step_attempt = 0 + while True: + try: + return await self._call_llm(messages, token_usage, session_id=session_id) + except self._retryable_exceptions as e: + step_attempt += 1 + if step_attempt > self._step_retries: + raise + delay = self._step_retry_delays[ + min(step_attempt - 1, len(self._step_retry_delays) - 1) + ] + logger.warning( + "LLM 瞬时异常，步级重试 {}/{}（{}s 后重发）: {}", + step_attempt, + self._step_retries, + delay, + e, + ) + await asyncio.sleep(delay) + async def _call_llm( self, messages: list[dict[str, Any]], diff --git a/tests/unit/test_agent_loop.py b/tests/unit/test_agent_loop.py index 343914b..fe00b06 100644 --- a/tests/unit/test_agent_loop.py +++ b/tests/unit/test_agent_loop.py @@ -8,6 +8,7 @@ thinking 捕获、token 累加、call_id 透传、pluggy hook。 from __future__ import annotations import json +import ssl from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock @@ -305,3 +306,60 @@ class TestParseNormalization: def test_empty_content_still_rejected(self) -> None: """空 content 照旧拒绝，归一化不改变该边界。""" assert self._parse("") is None + + +# ── A2 步级重试测试（Spec-1）────────────────────────────────── + + +class TestStepLevelRetry: + """LLM 瞬时异常的步级重试：可重试元组 / 退避 / fail-fast。""" + + def _make_loop(self, chat_side_effects: list) -> AgentLoop: + """构造 chat 按序抛异常/返回响应的 AgentLoop。""" + llm = AsyncMock() + llm.chat = AsyncMock(side_effect=chat_side_effects) + return AgentLoop(llm=llm, max_steps=10) + + @pytest.mark.asyncio + async def test_transient_error_retried_then_succeeds(self, monkeypatch) -> None: + """SSL/超时瞬时异常按 20s/40s 退避重试，第三次成功 → finished。""" + delays: list[float] = [] + + async def _fake_sleep(seconds: float) -> None: + delays.append(seconds) + + monkeypatch.setattr("core.agent.loop.asyncio.sleep", _fake_sleep) + loop = self._make_loop( + [ + ssl.SSLError("SSLV3_ALERT_BAD_RECORD_MAC"), + TimeoutError("watchdog"), + _make_response(_submit_json()), + ] + ) + result = await loop.run("sys", "user", _StubDispatcher()) + assert result.stop_reason == "finished" + assert delays == [20.0, 40.0] + + @pytest.mark.asyncio + async def test_retry_exhausted_terminates_with_error(self, monkeypatch) -> None: + """重试预算（首次 + 2 次）耗尽仍失败 → stop_reason=error。""" + + async def _fake_sleep(seconds: float) -> None: + pass + + monkeypatch.setattr("core.agent.loop.asyncio.sleep", _fake_sleep) + loop = self._make_loop([TimeoutError("t1"), TimeoutError("t2"), TimeoutError("t3")]) + result = await loop.run("sys", "user", _StubDispatcher()) + assert result.stop_reason == "error" + assert loop._llm.chat.await_count == 3 # 首次 + 2 次重试 + + @pytest.mark.asyncio + async def test_non_retryable_fails_fast(self, monkeypatch) -> None: + """非可重试异常不退避不重发，首次即终止 → stop_reason=error。""" + sleep_mock = AsyncMock() + monkeypatch.setattr("core.agent.loop.asyncio.sleep", sleep_mock) + loop = self._make_loop([RuntimeError("programming bug")]) + result = await loop.run("sys", "user", _StubDispatcher()) + assert result.stop_reason == "error" + sleep_mock.assert_not_awaited() + assert loop._llm.chat.await_count == 1