iomgaa/PolyGateway
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases 9 Wiki Activity

feat: add dual-condition stall detection and quiet accounting degradation

Browse Source
This commit is contained in:
iomgaa
2026-07-21 00:43:39 -04:00
parent 63f2cc294e
commit 0e22fcf433
3 changed files with 323 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/polygateway/middleware/retry.py
+58 -15
View File
@@ -23,6 +23,7 @@ from loguru import logger
from polygateway.errors import (
AllSourcesExhausted,
CircuitOpenError,
GovernanceBackendError,
PolyGatewayError,
RequestRejectedError,
ResultInvalidError,
@@ -55,6 +56,22 @@ if TYPE_CHECKING:
)
def backoff_delay(
policy: RetryPolicy,
fails: int,
exc: BaseException | None,
rng: Callable[[], float],
) -> float:
"""指数退避+jitter,与 Retry-After 提示取大(ARCH §7.2;VT jitter 系数)。
模块级纯函数: RetryMW 与 EmbeddingClient(M2 §7)共用同一公式。
"""
base = min(policy.backoff_base_s * (2 ** (fails - 1)), policy.backoff_max_s)
delay = base * (0.5 + rng())
retry_after = getattr(exc, "retry_after_s", None) or 0.0
return max(delay, retry_after)
def _failure_reason(exc: PolyGatewayError) -> str:
"""失败原因归类(CHS governance.py:169 同款)。"""
if isinstance(exc, SourceDeadError):
@@ -118,10 +135,11 @@ class RetryMW:
raise AllSourcesExhausted(scope=self._scope, reason="no_sources", retry_after_s=0.0)
fails = 0
reasons: dict[str, str] = {}
entered_at = self._now() # 调用级累计计时,循环内不重置(CHS governance.py:207)
while True:
picked, gate_rejections = await self._pick_runnable(reasons)
if picked is None:
await self._on_no_runnable(gate_rejections, reasons)
await self._on_no_runnable(gate_rejections, reasons, entered_at)
continue
outcome = await self._attempt(request, *picked, reasons)
if isinstance(outcome, LLMResponse):
@@ -170,7 +188,9 @@ class RetryMW:
await self._settle_and_release(permit, 0)
return None, gate_rejections
async def _on_no_runnable(self, gate_rejections: int, reasons: dict[str, str]) -> None:
async def _on_no_runnable(
self, gate_rejections: int, reasons: dict[str, str], entered_at: float
) -> None:
if gate_rejections == len(self._sources):
names = tuple(s.name for s in self._sources)
raise CircuitOpenError(
@@ -185,7 +205,19 @@ class RetryMW:
retry_after_s=self._bp.poll_interval_s,
per_source_reasons=reasons,
)
await self._sleep(self._bp.poll_interval_s)
# 双条件 stall 判死(CHS governance.py:270-281): 本地累计等待与全局
# 无进展**同时**超窗才判死——本地 monotonic 与后端时钟刻意不混用。
stall = self._bp.stall_window_s
if self._now() - entered_at > stall and await self._quota.progress_age_s() > stall:
names = tuple(s.name for s in self._sources)
raise AllSourcesExhausted(
scope=self._scope,
reason="stalled",
retry_after_s=await self._breaker.retry_after_s(names),
per_source_reasons=reasons,
)
# jitter ∈ [0.5p, 1.0p] 防惊群(CHS governance.py:283-285)
await self._sleep(self._bp.poll_interval_s * (0.5 + 0.5 * self._rng()))
# —— 单次尝试(CHS run 200-268)——
@@ -209,8 +241,8 @@ class RetryMW:
call_id=call_id,
)
actual = result.prompt_tokens + result.completion_tokens
await self._breaker.record_success(entry)
await self._quota.mark_progress()
await self._record_quietly(self._breaker.record_success(entry))
await self._record_quietly(self._quota.mark_progress())
response = self._build_response(source, result, call_id, started)
await self._emit(request, source, call_id, started, response=response)
return response
@@ -220,19 +252,19 @@ class RetryMW:
raise
except ResultInvalidError as exc:
# 坏结果 ≠ 坏服务: 熔断记成功,异常上抛消耗业务失败预算(§6.3)
await self._breaker.record_success(entry)
await self._record_quietly(self._breaker.record_success(entry))
await self._emit(request, source, call_id, started, error=exc)
raise
except asyncio.CancelledError:
if entry.is_probe:
await self._breaker.release_probe(entry)
await self._record_quietly(self._breaker.release_probe(entry))
await self._emit(request, source, call_id, started, error="cancelled")
raise
except (SourceDeadError, TransientError) as exc:
dead = isinstance(exc, SourceDeadError)
reason = _failure_reason(exc)
reasons[source.name] = reason
await self._breaker.record_failure(entry, reason, dead)
await self._record_quietly(self._breaker.record_failure(entry, reason, dead))
if not dead:
actual = source.est_tokens # 保守: 失败请求可能已被网关计费(CHS 同款)
await self._emit(request, source, call_id, started, error=exc)
@@ -245,18 +277,29 @@ class RetryMW:
) -> None:
provider_responded = exc.source_name == source.name and exc.status_code is not None
if provider_responded:
await self._breaker.record_success(entry) # 网关健康地拒了坏请求
# 网关健康地拒了坏请求
await self._record_quietly(self._breaker.record_success(entry))
elif entry.is_probe:
await self._breaker.release_probe(entry)
await self._record_quietly(self._breaker.release_probe(entry))
async def _record_quietly(self, write_back: Awaitable[object]) -> None:
"""记账侧写回(record_*/mark_progress/release_probe)降级执行(设计 §10)。
调用已真实完成: 后端失败若冒泡会丢弃真实成功响应或掩盖原始尝试
异常,故 warning 降级(ARCH §7.3 勘误,CHS 全 fail-closed 的有意反转);
取消照常穿透。
"""
try:
await write_back
except asyncio.CancelledError:
raise
except GovernanceBackendError as exc:
logger.warning("治理记账写回降级(不冒泡): {}", exc)
# —— 辅助 ——
def _backoff_delay(self, fails: int, exc: PolyGatewayError) -> float:
"""指数退避+jitter,与 Retry-After 提示取大(ARCH §7.2;VT jitter 系数)。"""
base = min(self._retry.backoff_base_s * (2 ** (fails - 1)), self._retry.backoff_max_s)
delay = base * (0.5 + self._rng())
retry_after = getattr(exc, "retry_after_s", None) or 0.0
return max(delay, retry_after)
return backoff_delay(self._retry, fails, exc, self._rng)
def _build_response(
self, source: SourceConfig, result: TransportResult, call_id: str, started: float