iomgaa/PolyGateway
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases 9 Wiki Activity

fix: reparent governance backend failures under GatewayUnavailableError (issue #7)

Browse Source 
A fail-closed limiter or breaker backend means the scope cannot emit a
single request, which is exactly scope-level unavailability. But the error
sat directly under PolyGatewayError, so a caller writing only
`except GatewayUnavailableError` dropped it into the catch-all branch:
Redis blips once and a backlog of tasks burns its business failure budget
into the dead letter queue, over a fault a restart would clear.

Three gate paths leak to callers rather than being absorbed by
_record_quietly (try_acquire, try_enter, progress_age_s); each is now
pinned by a test, since none of them had one before.

The two unknown-source sites move to SourceNotConfiguredError instead of
following along. They report a misconfigured source name, not an outage,
and letting them into the retryable family would be the mirror of the bug
being fixed here: the task would retry forever and never surface.
This commit is contained in:
iomgaa
2026-08-06 04:53:52 -04:00
parent dd540496a1
commit 45073486a7
13 changed files with 179 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/polygateway/backends/memory/limiter.py
+2 -2
View File
@@ -15,7 +15,7 @@ import time
import uuid
from typing import TYPE_CHECKING
from polygateway.errors import GovernanceBackendError
from polygateway.errors import SourceNotConfiguredError
from polygateway.types import GlobalLimits, SourceConfig, SourceStats
if TYPE_CHECKING:
@@ -89,7 +89,7 @@ class InMemoryLimiter:
def _cfg(self, source_key: str) -> SourceConfig:
cfg = self._sources.get(source_key)
if cfg is None:
raise GovernanceBackendError(f"未知源 {source_key!r}(scope={self._scope})")
raise SourceNotConfiguredError(f"未知源 {source_key!r}(scope={self._scope})")
return cfg
def _window(self) -> int:
src/polygateway/backends/redis/breaker.py
+5 -5
View File
@@ -367,7 +367,7 @@ class RedisGate:
keys=[self._key(source_name)], args=[owner, self._probe_ttl_ms]
)
except RedisError as exc:
raise GovernanceBackendError(f"熔断后端 try_enter 失败: {exc}") from exc
raise GovernanceBackendError(f"熔断后端 try_enter 失败: {exc}", scope=self._scope) from exc
return self._decision(source_name, result)
async def record_success(
@@ -385,7 +385,7 @@ class RedisGate:
try:
result = await self._success_lua(keys=[self._key(entry.source_name)], args=args)
except RedisError as exc:
raise GovernanceBackendError(f"熔断后端 record_success 失败: {exc}") from exc
raise GovernanceBackendError(f"熔断后端 record_success 失败: {exc}", scope=self._scope) from exc
return self._update(result)
async def record_failure(
@@ -407,7 +407,7 @@ class RedisGate:
try:
result = await self._failure_lua(keys=[self._key(entry.source_name)], args=args)
except RedisError as exc:
raise GovernanceBackendError(f"熔断后端 record_failure 失败: {exc}") from exc
raise GovernanceBackendError(f"熔断后端 record_failure 失败: {exc}", scope=self._scope) from exc
return self._update(result)
async def release_probe(self, entry: GateDecision) -> GateUpdate:
@@ -419,7 +419,7 @@ class RedisGate:
keys=[self._key(entry.source_name)], args=[entry.epoch, entry.probe_owner]
)
except RedisError as exc:
raise GovernanceBackendError(f"熔断后端 release_probe 失败: {exc}") from exc
raise GovernanceBackendError(f"熔断后端 release_probe 失败: {exc}", scope=self._scope) from exc
return self._update(result)
async def retry_after_s(self, sources: tuple[str, ...]) -> float:
@@ -429,7 +429,7 @@ class RedisGate:
try:
result = await self._retry_after_lua(keys=[self._key(s) for s in sources])
except RedisError as exc:
raise GovernanceBackendError(f"熔断后端 retry_after_s 失败: {exc}") from exc
raise GovernanceBackendError(f"熔断后端 retry_after_s 失败: {exc}", scope=self._scope) from exc
return int(result) / 1000.0
async def aclose(self) -> None:
src/polygateway/backends/redis/limiter.py
+8 -8
View File
@@ -22,7 +22,7 @@ from typing import TYPE_CHECKING
from loguru import logger
from redis.exceptions import RedisError
from polygateway.errors import GovernanceBackendError
from polygateway.errors import GovernanceBackendError, SourceNotConfiguredError
from polygateway.types import GlobalLimits, SourceConfig, SourceStats
if TYPE_CHECKING:
@@ -195,7 +195,7 @@ class RedisLimiter:
def _cfg(self, source_key: str) -> SourceConfig:
cfg = self._sources.get(source_key)
if cfg is None:
raise GovernanceBackendError(f"未知源 {source_key!r}(scope={self._scope})")
raise SourceNotConfiguredError(f"未知源 {source_key!r}(scope={self._scope})")
return cfg
def _lease_keys(self, source_key: str) -> tuple[str, str]:
@@ -247,7 +247,7 @@ class RedisLimiter:
],
)
except RedisError as exc:
raise GovernanceBackendError(f"限流后端 try_acquire 失败: {exc}") from exc
raise GovernanceBackendError(f"限流后端 try_acquire 失败: {exc}", scope=self._scope) from exc
if ok != 1:
return None
return _RedisPermit(self, source_key, lease_id, est_tokens, window)
@@ -265,14 +265,14 @@ class RedisLimiter:
try:
await self._release_lua(keys=[gl, sl], args=[lease_id])
except RedisError as exc:
raise GovernanceBackendError(f"限流后端 release 失败: {exc}") from exc
raise GovernanceBackendError(f"限流后端 release 失败: {exc}", scope=self._scope) from exc
async def _settle_tpm(self, source_key: str, delta: int, window: int) -> None:
wk = self._window_keys(source_key, window)
try:
await self._settle_lua(keys=[wk["g_tpm"], wk["s_tpm"]], args=[delta, _WINDOW_TTL_S])
except RedisError as exc:
raise GovernanceBackendError(f"限流后端 settle 失败: {exc}") from exc
raise GovernanceBackendError(f"限流后端 settle 失败: {exc}", scope=self._scope) from exc
async def source_stats(self, source_key: str) -> SourceStats:
"""当前窗口快照;读侧 clamp ≥0(展示口径,存储保留负值)。"""
@@ -283,7 +283,7 @@ class RedisLimiter:
wk = self._window_keys(source_key, window)
res = await self._stats_lua(keys=[sl, wk["s_rpm"], wk["s_tpm"]])
except RedisError as exc:
raise GovernanceBackendError(f"限流后端 source_stats 失败: {exc}") from exc
raise GovernanceBackendError(f"限流后端 source_stats 失败: {exc}", scope=self._scope) from exc
return SourceStats(
inflight=int(res[0]),
rpm_used=max(0, int(res[1])),
@@ -295,14 +295,14 @@ class RedisLimiter:
try:
await self._progress_mark_lua(keys=[self._progress_key()], args=[_PROGRESS_TTL_S])
except RedisError as exc:
raise GovernanceBackendError(f"限流后端 mark_progress 失败: {exc}") from exc
raise GovernanceBackendError(f"限流后端 mark_progress 失败: {exc}", scope=self._scope) from exc
async def progress_age_s(self) -> float:
"""距上次全局成功的秒数;仅键缺失(-1)= 从未进展 → inf(CHS limiter.py:208)。"""
try:
res = await self._progress_age_lua(keys=[self._progress_key()])
except RedisError as exc:
raise GovernanceBackendError(f"限流后端 progress_age_s 失败: {exc}") from exc
raise GovernanceBackendError(f"限流后端 progress_age_s 失败: {exc}", scope=self._scope) from exc
return float("inf") if int(res) == -1 else int(res) / 1000.0
async def aclose(self) -> None:
src/polygateway/embedding.py
+2 -2
View File
@@ -120,8 +120,8 @@ class EmbeddingClient:
# 否则遥测会记录一个从未发出的采样参数(issue #4 决策 G)
self._sources = strip_unsupported_extra_body(list(sources), path="embedding")
self._selector = selector
self._quota = QuotaGate(limiter)
self._breaker = BreakerGate(breaker)
self._quota = QuotaGate(limiter, scope=self._scope)
self._breaker = BreakerGate(breaker, scope=self._scope)
self._transport = transport
self._retry = retry
self._bp = backpressure
src/polygateway/errors.py
+26 -2
View File
@@ -150,5 +150,29 @@ class SourceNotConfiguredError(PolyGatewayError):
"""
class GovernanceBackendError(PolyGatewayError):
"""限流/熔断状态后端自身故障: 必须报错而非放行(防击穿网关,降级方向铁律)。"""
class GovernanceBackendError(GatewayUnavailableError):
"""限流/熔断状态后端自身故障: 必须报错而非放行(防击穿网关,降级方向铁律)。
继承 `GatewayUnavailableError`(issue #7): fail-closed 意味着整个 scope 一个
请求都发不出去,语义上即 scope 级不可用。此前它是 `PolyGatewayError` 的直接
子类,只写 `except GatewayUnavailableError` 的调用方接不住,后果是"Redis 抖
一下 → 积压任务消耗业务失败预算 → 进死信",而那是运维重启即可恢复的故障。
"""
def __init__(
self,
message: str,
*,
scope: str,
retry_after_s: float = GOVERNANCE_BACKEND_RETRY_AFTER_S,
source_name: str | None = None,
) -> None:
super().__init__(
scope=scope,
reason="governance_backend_down",
retry_after_s=retry_after_s,
source_name=source_name,
)
# 父类会把 message 覆写为 "{scope} 网关暂时不可用: {reason}",而各构造点
# 携带的诊断串(如"限流后端 try_acquire 失败: ...")是排障主线索,必须保住
self.args = (message,)
src/polygateway/middleware/breaker.py
+8 -6
View File
@@ -14,8 +14,10 @@ if TYPE_CHECKING:
class BreakerGate:
"""RetryMW 面向熔断后端的唯一入口;包装一切后端异常。"""
def __init__(self, gate: ProviderGate) -> None:
def __init__(self, gate: ProviderGate, *, scope: str) -> None:
self._gate = gate
# 后端故障即 scope 级不可用,异常须携 scope 供调用方定位(issue #7 §3.3)
self._scope = scope
async def try_enter(self, source: SourceConfig, owner: str) -> GateDecision:
try:
@@ -23,7 +25,7 @@ class BreakerGate:
except GovernanceBackendError:
raise
except Exception as exc:
raise GovernanceBackendError(f"熔断后端故障(try_enter): {exc}") from exc
raise GovernanceBackendError(f"熔断后端故障(try_enter): {exc}", scope=self._scope) from exc
async def record_success(
self, entry: GateDecision, *, count_attempt: bool = True
@@ -33,7 +35,7 @@ class BreakerGate:
except GovernanceBackendError:
raise
except Exception as exc:
raise GovernanceBackendError(f"熔断后端故障(record_success): {exc}") from exc
raise GovernanceBackendError(f"熔断后端故障(record_success): {exc}", scope=self._scope) from exc
async def record_failure(
self, entry: GateDecision, reason: str, force_open: bool
@@ -43,7 +45,7 @@ class BreakerGate:
except GovernanceBackendError:
raise
except Exception as exc:
raise GovernanceBackendError(f"熔断后端故障(record_failure): {exc}") from exc
raise GovernanceBackendError(f"熔断后端故障(record_failure): {exc}", scope=self._scope) from exc
async def release_probe(self, entry: GateDecision) -> GateUpdate:
try:
@@ -51,7 +53,7 @@ class BreakerGate:
except GovernanceBackendError:
raise
except Exception as exc:
raise GovernanceBackendError(f"熔断后端故障(release_probe): {exc}") from exc
raise GovernanceBackendError(f"熔断后端故障(release_probe): {exc}", scope=self._scope) from exc
async def retry_after_s(self, sources: tuple[str, ...]) -> float:
try:
@@ -59,4 +61,4 @@ class BreakerGate:
except GovernanceBackendError:
raise
except Exception as exc:
raise GovernanceBackendError(f"熔断后端故障(retry_after_s): {exc}") from exc
raise GovernanceBackendError(f"熔断后端故障(retry_after_s): {exc}", scope=self._scope) from exc
src/polygateway/middleware/ratelimit.py
+7 -5
View File
@@ -18,8 +18,10 @@ if TYPE_CHECKING:
class QuotaGate:
"""RetryMW 面向限流后端的唯一入口;包装一切后端异常。"""
def __init__(self, limiter: RateLimiter) -> None:
def __init__(self, limiter: RateLimiter, *, scope: str) -> None:
self._limiter = limiter
# 后端故障即 scope 级不可用,异常须携 scope 供调用方定位(issue #7 §3.3)
self._scope = scope
async def try_acquire(self, source: SourceConfig) -> Permit | None:
try:
@@ -27,7 +29,7 @@ class QuotaGate:
except GovernanceBackendError:
raise
except Exception as exc:
raise GovernanceBackendError(f"限流后端故障(try_acquire): {exc}") from exc
raise GovernanceBackendError(f"限流后端故障(try_acquire): {exc}", scope=self._scope) from exc
async def stats(self, source: SourceConfig) -> SourceStats:
try:
@@ -35,7 +37,7 @@ class QuotaGate:
except GovernanceBackendError:
raise
except Exception as exc:
raise GovernanceBackendError(f"限流后端故障(source_stats): {exc}") from exc
raise GovernanceBackendError(f"限流后端故障(source_stats): {exc}", scope=self._scope) from exc
async def mark_progress(self) -> None:
try:
@@ -43,7 +45,7 @@ class QuotaGate:
except GovernanceBackendError:
raise
except Exception as exc:
raise GovernanceBackendError(f"限流后端故障(mark_progress): {exc}") from exc
raise GovernanceBackendError(f"限流后端故障(mark_progress): {exc}", scope=self._scope) from exc
async def progress_age_s(self) -> float:
try:
@@ -51,4 +53,4 @@ class QuotaGate:
except GovernanceBackendError:
raise
except Exception as exc:
raise GovernanceBackendError(f"限流后端故障(progress_age_s): {exc}") from exc
raise GovernanceBackendError(f"限流后端故障(progress_age_s): {exc}", scope=self._scope) from exc
src/polygateway/middleware/retry.py
+2 -2
View File
@@ -183,8 +183,8 @@ class RetryMW:
self._scope = scope
self._sources = list(sources)
self._selector = selector
self._quota = QuotaGate(limiter)
self._breaker = BreakerGate(gate)
self._quota = QuotaGate(limiter, scope=self._scope)
self._breaker = BreakerGate(gate, scope=self._scope)
self._transport = transport
self._retry = retry
self._bp = backpressure
src/polygateway/ocr.py
+2 -2
View File
@@ -119,8 +119,8 @@ class OcrClient:
self._sources = strip_unsupported_extra_body(list(sources), path="OCR")
self._selector = selector
self._feed_health = isinstance(selector, OutcomeAwareSelector)
self._quota = QuotaGate(limiter)
self._breaker = BreakerGate(breaker)
self._quota = QuotaGate(limiter, scope=self._scope)
self._breaker = BreakerGate(breaker, scope=self._scope)
self._transport = transport
self._retry = retry
self._bp = backpressure