fix: reparent governance backend failures under GatewayUnavailableError (issue #7)

A fail-closed limiter or breaker backend means the scope cannot emit a
single request, which is exactly scope-level unavailability. But the error
sat directly under PolyGatewayError, so a caller writing only
`except GatewayUnavailableError` dropped it into the catch-all branch:
Redis blips once and a backlog of tasks burns its business failure budget
into the dead letter queue, over a fault a restart would clear.

Three gate paths leak to callers rather than being absorbed by
_record_quietly (try_acquire, try_enter, progress_age_s); each is now
pinned by a test, since none of them had one before.

The two unknown-source sites move to SourceNotConfiguredError instead of
following along. They report a misconfigured source name, not an outage,
and letting them into the retryable family would be the mirror of the bug
being fixed here: the task would retry forever and never surface.
This commit is contained in:
2026-08-06 04:53:52 -04:00
parent dd540496a1
commit 45073486a7
13 changed files with 179 additions and 49 deletions
+2 -2
View File
@@ -15,7 +15,7 @@ import time
import uuid import uuid
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from polygateway.errors import GovernanceBackendError from polygateway.errors import SourceNotConfiguredError
from polygateway.types import GlobalLimits, SourceConfig, SourceStats from polygateway.types import GlobalLimits, SourceConfig, SourceStats
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -89,7 +89,7 @@ class InMemoryLimiter:
def _cfg(self, source_key: str) -> SourceConfig: def _cfg(self, source_key: str) -> SourceConfig:
cfg = self._sources.get(source_key) cfg = self._sources.get(source_key)
if cfg is None: if cfg is None:
raise GovernanceBackendError(f"未知源 {source_key!r}(scope={self._scope})") raise SourceNotConfiguredError(f"未知源 {source_key!r}(scope={self._scope})")
return cfg return cfg
def _window(self) -> int: def _window(self) -> int:
+5 -5
View File
@@ -367,7 +367,7 @@ class RedisGate:
keys=[self._key(source_name)], args=[owner, self._probe_ttl_ms] keys=[self._key(source_name)], args=[owner, self._probe_ttl_ms]
) )
except RedisError as exc: except RedisError as exc:
raise GovernanceBackendError(f"熔断后端 try_enter 失败: {exc}") from exc raise GovernanceBackendError(f"熔断后端 try_enter 失败: {exc}", scope=self._scope) from exc
return self._decision(source_name, result) return self._decision(source_name, result)
async def record_success( async def record_success(
@@ -385,7 +385,7 @@ class RedisGate:
try: try:
result = await self._success_lua(keys=[self._key(entry.source_name)], args=args) result = await self._success_lua(keys=[self._key(entry.source_name)], args=args)
except RedisError as exc: except RedisError as exc:
raise GovernanceBackendError(f"熔断后端 record_success 失败: {exc}") from exc raise GovernanceBackendError(f"熔断后端 record_success 失败: {exc}", scope=self._scope) from exc
return self._update(result) return self._update(result)
async def record_failure( async def record_failure(
@@ -407,7 +407,7 @@ class RedisGate:
try: try:
result = await self._failure_lua(keys=[self._key(entry.source_name)], args=args) result = await self._failure_lua(keys=[self._key(entry.source_name)], args=args)
except RedisError as exc: except RedisError as exc:
raise GovernanceBackendError(f"熔断后端 record_failure 失败: {exc}") from exc raise GovernanceBackendError(f"熔断后端 record_failure 失败: {exc}", scope=self._scope) from exc
return self._update(result) return self._update(result)
async def release_probe(self, entry: GateDecision) -> GateUpdate: async def release_probe(self, entry: GateDecision) -> GateUpdate:
@@ -419,7 +419,7 @@ class RedisGate:
keys=[self._key(entry.source_name)], args=[entry.epoch, entry.probe_owner] keys=[self._key(entry.source_name)], args=[entry.epoch, entry.probe_owner]
) )
except RedisError as exc: except RedisError as exc:
raise GovernanceBackendError(f"熔断后端 release_probe 失败: {exc}") from exc raise GovernanceBackendError(f"熔断后端 release_probe 失败: {exc}", scope=self._scope) from exc
return self._update(result) return self._update(result)
async def retry_after_s(self, sources: tuple[str, ...]) -> float: async def retry_after_s(self, sources: tuple[str, ...]) -> float:
@@ -429,7 +429,7 @@ class RedisGate:
try: try:
result = await self._retry_after_lua(keys=[self._key(s) for s in sources]) result = await self._retry_after_lua(keys=[self._key(s) for s in sources])
except RedisError as exc: except RedisError as exc:
raise GovernanceBackendError(f"熔断后端 retry_after_s 失败: {exc}") from exc raise GovernanceBackendError(f"熔断后端 retry_after_s 失败: {exc}", scope=self._scope) from exc
return int(result) / 1000.0 return int(result) / 1000.0
async def aclose(self) -> None: async def aclose(self) -> None:
+8 -8
View File
@@ -22,7 +22,7 @@ from typing import TYPE_CHECKING
from loguru import logger from loguru import logger
from redis.exceptions import RedisError from redis.exceptions import RedisError
from polygateway.errors import GovernanceBackendError from polygateway.errors import GovernanceBackendError, SourceNotConfiguredError
from polygateway.types import GlobalLimits, SourceConfig, SourceStats from polygateway.types import GlobalLimits, SourceConfig, SourceStats
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -195,7 +195,7 @@ class RedisLimiter:
def _cfg(self, source_key: str) -> SourceConfig: def _cfg(self, source_key: str) -> SourceConfig:
cfg = self._sources.get(source_key) cfg = self._sources.get(source_key)
if cfg is None: if cfg is None:
raise GovernanceBackendError(f"未知源 {source_key!r}(scope={self._scope})") raise SourceNotConfiguredError(f"未知源 {source_key!r}(scope={self._scope})")
return cfg return cfg
def _lease_keys(self, source_key: str) -> tuple[str, str]: def _lease_keys(self, source_key: str) -> tuple[str, str]:
@@ -247,7 +247,7 @@ class RedisLimiter:
], ],
) )
except RedisError as exc: except RedisError as exc:
raise GovernanceBackendError(f"限流后端 try_acquire 失败: {exc}") from exc raise GovernanceBackendError(f"限流后端 try_acquire 失败: {exc}", scope=self._scope) from exc
if ok != 1: if ok != 1:
return None return None
return _RedisPermit(self, source_key, lease_id, est_tokens, window) return _RedisPermit(self, source_key, lease_id, est_tokens, window)
@@ -265,14 +265,14 @@ class RedisLimiter:
try: try:
await self._release_lua(keys=[gl, sl], args=[lease_id]) await self._release_lua(keys=[gl, sl], args=[lease_id])
except RedisError as exc: except RedisError as exc:
raise GovernanceBackendError(f"限流后端 release 失败: {exc}") from exc raise GovernanceBackendError(f"限流后端 release 失败: {exc}", scope=self._scope) from exc
async def _settle_tpm(self, source_key: str, delta: int, window: int) -> None: async def _settle_tpm(self, source_key: str, delta: int, window: int) -> None:
wk = self._window_keys(source_key, window) wk = self._window_keys(source_key, window)
try: try:
await self._settle_lua(keys=[wk["g_tpm"], wk["s_tpm"]], args=[delta, _WINDOW_TTL_S]) await self._settle_lua(keys=[wk["g_tpm"], wk["s_tpm"]], args=[delta, _WINDOW_TTL_S])
except RedisError as exc: except RedisError as exc:
raise GovernanceBackendError(f"限流后端 settle 失败: {exc}") from exc raise GovernanceBackendError(f"限流后端 settle 失败: {exc}", scope=self._scope) from exc
async def source_stats(self, source_key: str) -> SourceStats: async def source_stats(self, source_key: str) -> SourceStats:
"""当前窗口快照;读侧 clamp ≥0(展示口径,存储保留负值)。""" """当前窗口快照;读侧 clamp ≥0(展示口径,存储保留负值)。"""
@@ -283,7 +283,7 @@ class RedisLimiter:
wk = self._window_keys(source_key, window) wk = self._window_keys(source_key, window)
res = await self._stats_lua(keys=[sl, wk["s_rpm"], wk["s_tpm"]]) res = await self._stats_lua(keys=[sl, wk["s_rpm"], wk["s_tpm"]])
except RedisError as exc: except RedisError as exc:
raise GovernanceBackendError(f"限流后端 source_stats 失败: {exc}") from exc raise GovernanceBackendError(f"限流后端 source_stats 失败: {exc}", scope=self._scope) from exc
return SourceStats( return SourceStats(
inflight=int(res[0]), inflight=int(res[0]),
rpm_used=max(0, int(res[1])), rpm_used=max(0, int(res[1])),
@@ -295,14 +295,14 @@ class RedisLimiter:
try: try:
await self._progress_mark_lua(keys=[self._progress_key()], args=[_PROGRESS_TTL_S]) await self._progress_mark_lua(keys=[self._progress_key()], args=[_PROGRESS_TTL_S])
except RedisError as exc: except RedisError as exc:
raise GovernanceBackendError(f"限流后端 mark_progress 失败: {exc}") from exc raise GovernanceBackendError(f"限流后端 mark_progress 失败: {exc}", scope=self._scope) from exc
async def progress_age_s(self) -> float: async def progress_age_s(self) -> float:
"""距上次全局成功的秒数;仅键缺失(-1)= 从未进展 → inf(CHS limiter.py:208)。""" """距上次全局成功的秒数;仅键缺失(-1)= 从未进展 → inf(CHS limiter.py:208)。"""
try: try:
res = await self._progress_age_lua(keys=[self._progress_key()]) res = await self._progress_age_lua(keys=[self._progress_key()])
except RedisError as exc: except RedisError as exc:
raise GovernanceBackendError(f"限流后端 progress_age_s 失败: {exc}") from exc raise GovernanceBackendError(f"限流后端 progress_age_s 失败: {exc}", scope=self._scope) from exc
return float("inf") if int(res) == -1 else int(res) / 1000.0 return float("inf") if int(res) == -1 else int(res) / 1000.0
async def aclose(self) -> None: async def aclose(self) -> None:
+2 -2
View File
@@ -120,8 +120,8 @@ class EmbeddingClient:
# 否则遥测会记录一个从未发出的采样参数(issue #4 决策 G) # 否则遥测会记录一个从未发出的采样参数(issue #4 决策 G)
self._sources = strip_unsupported_extra_body(list(sources), path="embedding") self._sources = strip_unsupported_extra_body(list(sources), path="embedding")
self._selector = selector self._selector = selector
self._quota = QuotaGate(limiter) self._quota = QuotaGate(limiter, scope=self._scope)
self._breaker = BreakerGate(breaker) self._breaker = BreakerGate(breaker, scope=self._scope)
self._transport = transport self._transport = transport
self._retry = retry self._retry = retry
self._bp = backpressure self._bp = backpressure
+26 -2
View File
@@ -150,5 +150,29 @@ class SourceNotConfiguredError(PolyGatewayError):
""" """
class GovernanceBackendError(PolyGatewayError): class GovernanceBackendError(GatewayUnavailableError):
"""限流/熔断状态后端自身故障: 必须报错而非放行(防击穿网关,降级方向铁律)。""" """限流/熔断状态后端自身故障: 必须报错而非放行(防击穿网关,降级方向铁律)。
继承 `GatewayUnavailableError`(issue #7): fail-closed 意味着整个 scope 一个
请求都发不出去,语义上即 scope 级不可用。此前它是 `PolyGatewayError` 的直接
子类,只写 `except GatewayUnavailableError` 的调用方接不住,后果是"Redis 抖
一下 → 积压任务消耗业务失败预算 → 进死信",而那是运维重启即可恢复的故障。
"""
def __init__(
self,
message: str,
*,
scope: str,
retry_after_s: float = GOVERNANCE_BACKEND_RETRY_AFTER_S,
source_name: str | None = None,
) -> None:
super().__init__(
scope=scope,
reason="governance_backend_down",
retry_after_s=retry_after_s,
source_name=source_name,
)
# 父类会把 message 覆写为 "{scope} 网关暂时不可用: {reason}",而各构造点
# 携带的诊断串(如"限流后端 try_acquire 失败: ...")是排障主线索,必须保住
self.args = (message,)
+8 -6
View File
@@ -14,8 +14,10 @@ if TYPE_CHECKING:
class BreakerGate: class BreakerGate:
"""RetryMW 面向熔断后端的唯一入口;包装一切后端异常。""" """RetryMW 面向熔断后端的唯一入口;包装一切后端异常。"""
def __init__(self, gate: ProviderGate) -> None: def __init__(self, gate: ProviderGate, *, scope: str) -> None:
self._gate = gate self._gate = gate
# 后端故障即 scope 级不可用,异常须携 scope 供调用方定位(issue #7 §3.3)
self._scope = scope
async def try_enter(self, source: SourceConfig, owner: str) -> GateDecision: async def try_enter(self, source: SourceConfig, owner: str) -> GateDecision:
try: try:
@@ -23,7 +25,7 @@ class BreakerGate:
except GovernanceBackendError: except GovernanceBackendError:
raise raise
except Exception as exc: except Exception as exc:
raise GovernanceBackendError(f"熔断后端故障(try_enter): {exc}") from exc raise GovernanceBackendError(f"熔断后端故障(try_enter): {exc}", scope=self._scope) from exc
async def record_success( async def record_success(
self, entry: GateDecision, *, count_attempt: bool = True self, entry: GateDecision, *, count_attempt: bool = True
@@ -33,7 +35,7 @@ class BreakerGate:
except GovernanceBackendError: except GovernanceBackendError:
raise raise
except Exception as exc: except Exception as exc:
raise GovernanceBackendError(f"熔断后端故障(record_success): {exc}") from exc raise GovernanceBackendError(f"熔断后端故障(record_success): {exc}", scope=self._scope) from exc
async def record_failure( async def record_failure(
self, entry: GateDecision, reason: str, force_open: bool self, entry: GateDecision, reason: str, force_open: bool
@@ -43,7 +45,7 @@ class BreakerGate:
except GovernanceBackendError: except GovernanceBackendError:
raise raise
except Exception as exc: except Exception as exc:
raise GovernanceBackendError(f"熔断后端故障(record_failure): {exc}") from exc raise GovernanceBackendError(f"熔断后端故障(record_failure): {exc}", scope=self._scope) from exc
async def release_probe(self, entry: GateDecision) -> GateUpdate: async def release_probe(self, entry: GateDecision) -> GateUpdate:
try: try:
@@ -51,7 +53,7 @@ class BreakerGate:
except GovernanceBackendError: except GovernanceBackendError:
raise raise
except Exception as exc: except Exception as exc:
raise GovernanceBackendError(f"熔断后端故障(release_probe): {exc}") from exc raise GovernanceBackendError(f"熔断后端故障(release_probe): {exc}", scope=self._scope) from exc
async def retry_after_s(self, sources: tuple[str, ...]) -> float: async def retry_after_s(self, sources: tuple[str, ...]) -> float:
try: try:
@@ -59,4 +61,4 @@ class BreakerGate:
except GovernanceBackendError: except GovernanceBackendError:
raise raise
except Exception as exc: except Exception as exc:
raise GovernanceBackendError(f"熔断后端故障(retry_after_s): {exc}") from exc raise GovernanceBackendError(f"熔断后端故障(retry_after_s): {exc}", scope=self._scope) from exc
+7 -5
View File
@@ -18,8 +18,10 @@ if TYPE_CHECKING:
class QuotaGate: class QuotaGate:
"""RetryMW 面向限流后端的唯一入口;包装一切后端异常。""" """RetryMW 面向限流后端的唯一入口;包装一切后端异常。"""
def __init__(self, limiter: RateLimiter) -> None: def __init__(self, limiter: RateLimiter, *, scope: str) -> None:
self._limiter = limiter self._limiter = limiter
# 后端故障即 scope 级不可用,异常须携 scope 供调用方定位(issue #7 §3.3)
self._scope = scope
async def try_acquire(self, source: SourceConfig) -> Permit | None: async def try_acquire(self, source: SourceConfig) -> Permit | None:
try: try:
@@ -27,7 +29,7 @@ class QuotaGate:
except GovernanceBackendError: except GovernanceBackendError:
raise raise
except Exception as exc: except Exception as exc:
raise GovernanceBackendError(f"限流后端故障(try_acquire): {exc}") from exc raise GovernanceBackendError(f"限流后端故障(try_acquire): {exc}", scope=self._scope) from exc
async def stats(self, source: SourceConfig) -> SourceStats: async def stats(self, source: SourceConfig) -> SourceStats:
try: try:
@@ -35,7 +37,7 @@ class QuotaGate:
except GovernanceBackendError: except GovernanceBackendError:
raise raise
except Exception as exc: except Exception as exc:
raise GovernanceBackendError(f"限流后端故障(source_stats): {exc}") from exc raise GovernanceBackendError(f"限流后端故障(source_stats): {exc}", scope=self._scope) from exc
async def mark_progress(self) -> None: async def mark_progress(self) -> None:
try: try:
@@ -43,7 +45,7 @@ class QuotaGate:
except GovernanceBackendError: except GovernanceBackendError:
raise raise
except Exception as exc: except Exception as exc:
raise GovernanceBackendError(f"限流后端故障(mark_progress): {exc}") from exc raise GovernanceBackendError(f"限流后端故障(mark_progress): {exc}", scope=self._scope) from exc
async def progress_age_s(self) -> float: async def progress_age_s(self) -> float:
try: try:
@@ -51,4 +53,4 @@ class QuotaGate:
except GovernanceBackendError: except GovernanceBackendError:
raise raise
except Exception as exc: except Exception as exc:
raise GovernanceBackendError(f"限流后端故障(progress_age_s): {exc}") from exc raise GovernanceBackendError(f"限流后端故障(progress_age_s): {exc}", scope=self._scope) from exc
+2 -2
View File
@@ -183,8 +183,8 @@ class RetryMW:
self._scope = scope self._scope = scope
self._sources = list(sources) self._sources = list(sources)
self._selector = selector self._selector = selector
self._quota = QuotaGate(limiter) self._quota = QuotaGate(limiter, scope=self._scope)
self._breaker = BreakerGate(gate) self._breaker = BreakerGate(gate, scope=self._scope)
self._transport = transport self._transport = transport
self._retry = retry self._retry = retry
self._bp = backpressure self._bp = backpressure
+2 -2
View File
@@ -119,8 +119,8 @@ class OcrClient:
self._sources = strip_unsupported_extra_body(list(sources), path="OCR") self._sources = strip_unsupported_extra_body(list(sources), path="OCR")
self._selector = selector self._selector = selector
self._feed_health = isinstance(selector, OutcomeAwareSelector) self._feed_health = isinstance(selector, OutcomeAwareSelector)
self._quota = QuotaGate(limiter) self._quota = QuotaGate(limiter, scope=self._scope)
self._breaker = BreakerGate(breaker) self._breaker = BreakerGate(breaker, scope=self._scope)
self._transport = transport self._transport = transport
self._retry = retry self._retry = retry
self._bp = backpressure self._bp = backpressure
@@ -16,7 +16,11 @@ import pytest
from polygateway.backends.redis.breaker import RedisGate from polygateway.backends.redis.breaker import RedisGate
from polygateway.backends.redis.limiter import RedisLimiter from polygateway.backends.redis.limiter import RedisLimiter
from polygateway.client import GatewayClient from polygateway.client import GatewayClient
from polygateway.errors import AllSourcesExhausted, GovernanceBackendError from polygateway.errors import (
AllSourcesExhausted,
GatewayUnavailableError,
GovernanceBackendError,
)
from polygateway.sources import RoundRobinSelector from polygateway.sources import RoundRobinSelector
from polygateway.types import ( from polygateway.types import (
BackpressurePolicy, BackpressurePolicy,
@@ -225,7 +229,11 @@ async def test_cancel_in_flight_releases_lease(clients):
async def test_redis_down_admission_fails_closed(): async def test_redis_down_admission_fails_closed():
"""Redis 不可达 → 准入侧抛 GovernanceBackendError,绝不放行(库铁律)。""" """Redis 不可达 → 准入侧报错绝不放行(库铁律),且以 scope 级形态到达调用方。
issue #7: 调用方只写 `except GatewayUnavailableError` 就该覆盖后端故障——
真实 Redis 掉线是这条链路唯一的端到端证据,故断言收紧到 scope 级语义。
"""
import redis.asyncio as aioredis import redis.asyncio as aioredis
dead = aioredis.from_url( dead = aioredis.from_url(
@@ -240,9 +248,12 @@ async def test_redis_down_admission_fails_closed():
lease_ttl_s=30.0, lease_ttl_s=30.0,
) )
gate = RedisGate(config=_CFG, redis=dead, scope="t-dead") gate = RedisGate(config=_CFG, redis=dead, scope="t-dead")
with pytest.raises(GovernanceBackendError): for call in (limiter.try_acquire("s1", 0), gate.try_enter("s1", "w")):
await limiter.try_acquire("s1", 0) with pytest.raises(GatewayUnavailableError) as ei:
with pytest.raises(GovernanceBackendError): await call
await gate.try_enter("s1", "w") assert isinstance(ei.value, GovernanceBackendError)
assert ei.value.reason == "governance_backend_down"
assert ei.value.scope == "t-dead"
assert ei.value.retry_after_s > 0
finally: finally:
await dead.aclose() await dead.aclose()
+77 -6
View File
@@ -11,7 +11,13 @@ import pytest
from polygateway.backends.memory.breaker import InMemoryGate from polygateway.backends.memory.breaker import InMemoryGate
from polygateway.backends.memory.limiter import InMemoryLimiter from polygateway.backends.memory.limiter import InMemoryLimiter
from polygateway.errors import AllSourcesExhausted, GovernanceBackendError, TransientError from polygateway.errors import (
AllSourcesExhausted,
GatewayUnavailableError,
GovernanceBackendError,
SourceNotConfiguredError,
TransientError,
)
from polygateway.middleware.retry import RetryMW, backoff_delay from polygateway.middleware.retry import RetryMW, backoff_delay
from polygateway.sources import RoundRobinSelector, SourceCooldownMemo from polygateway.sources import RoundRobinSelector, SourceCooldownMemo
from polygateway.types import ( from polygateway.types import (
@@ -173,17 +179,17 @@ class TestStallQuadrants:
class _GateSuccessBroken(InMemoryGate): class _GateSuccessBroken(InMemoryGate):
async def record_success(self, entry): async def record_success(self, entry):
raise GovernanceBackendError("redis 抖动") raise GovernanceBackendError("redis 抖动", scope="llm")
class _GateFailureBroken(InMemoryGate): class _GateFailureBroken(InMemoryGate):
async def record_failure(self, entry, reason, force_open): async def record_failure(self, entry, reason, force_open):
raise GovernanceBackendError("redis 抖动") raise GovernanceBackendError("redis 抖动", scope="llm")
class _LimiterProgressBroken(InMemoryLimiter): class _LimiterProgressBroken(InMemoryLimiter):
async def mark_progress(self): async def mark_progress(self):
raise GovernanceBackendError("redis 抖动") raise GovernanceBackendError("redis 抖动", scope="llm")
class TestAccountingDegradation: class TestAccountingDegradation:
@@ -252,6 +258,71 @@ class TestQuotaGateProgressAge:
async def progress_age_s(self): async def progress_age_s(self):
raise OSError("down") raise OSError("down")
assert await QuotaGate(_L()).progress_age_s() == 12.5 assert await QuotaGate(_L(), scope="llm").progress_age_s() == 12.5
with pytest.raises(GovernanceBackendError): with pytest.raises(GovernanceBackendError):
await QuotaGate(_Broken()).progress_age_s() await QuotaGate(_Broken(), scope="llm").progress_age_s()
class TestUnknownSourceIsAssemblyDefect:
"""未知源 = 限流后端的源名单与治理循环对不上,是装配缺陷不是后端故障。
两个后端行为必须一致(Redis 版对应用例在 `test_redis_key_layout.py::
TestConversions::test_unknown_source_rejected`);内存版此前无覆盖,
该分支从未被测过(issue #7 §3.4)。
"""
def test_memory_limiter_rejects_unknown_source(self):
limiter = InMemoryLimiter(
scope="llm", sources={"s1": make_source("s1")}, global_limits=_NO_GLOBAL
)
with pytest.raises(SourceNotConfiguredError) as ei:
limiter._cfg("nope")
# 关键: 若归入 scope 级家族,配置写错的任务会永远延期重投、永不进死信
assert not isinstance(ei.value, GatewayUnavailableError)
class TestGateFailuresReachCallersAsScopeLevel:
"""三条闸门泄漏路径必须以 scope 级不可用的形态到达调用方(issue #7)。
记账路径由 `_record_quietly` 降级为 warning,但闸门路径没有那层包裹,会一路
抛给调用方。只写 `except GatewayUnavailableError` 的调用方此前接不住,后果
是 Redis 抖一下就让积压任务烧掉业务失败预算进死信——而那是运维重启即可恢复
的故障。三条路径逐一钉住,防止将来任何一条被漏掉。
"""
async def test_try_acquire_failure_is_scope_level(self):
from polygateway.middleware.ratelimit import QuotaGate
class _Broken:
async def try_acquire(self, name, est):
raise OSError("down")
with pytest.raises(GatewayUnavailableError) as ei:
await QuotaGate(_Broken(), scope="LLM").try_acquire(make_source("s1"))
assert ei.value.scope == "llm"
assert ei.value.reason == "governance_backend_down"
assert ei.value.retry_after_s > 0 # 0 会让积压任务零延迟冲击已挂的后端
async def test_try_enter_failure_is_scope_level(self):
from polygateway.middleware.breaker import BreakerGate
class _Broken:
async def try_enter(self, name, owner):
raise OSError("down")
with pytest.raises(GatewayUnavailableError) as ei:
await BreakerGate(_Broken(), scope="LLM").try_enter(make_source("s1"), "owner")
assert ei.value.scope == "llm"
assert ei.value.reason == "governance_backend_down"
async def test_progress_age_failure_is_scope_level(self):
from polygateway.middleware.ratelimit import QuotaGate
class _Broken:
async def progress_age_s(self):
raise OSError("down")
with pytest.raises(GatewayUnavailableError) as ei:
await QuotaGate(_Broken(), scope="LLM").progress_age_s()
assert ei.value.scope == "llm"
assert ei.value.reason == "governance_backend_down"
+18 -1
View File
@@ -89,10 +89,27 @@ class TestGatewayUnavailable:
class TestBackendFailure: class TestBackendFailure:
def test_governance_backend_error_is_not_transient(self): def test_governance_backend_error_is_not_transient(self):
"""限流/熔断后端故障必须报错不放行,且不落入可重试分类。""" """限流/熔断后端故障必须报错不放行,且不落入可重试分类。"""
exc = GovernanceBackendError("redis down") exc = GovernanceBackendError("redis down", scope="llm")
assert isinstance(exc, PolyGatewayError) assert isinstance(exc, PolyGatewayError)
assert not isinstance(exc, TransientError) assert not isinstance(exc, TransientError)
def test_is_scope_level_unavailability(self):
"""fail-closed 时整个 scope 一个请求都发不出去,调用方一条 except 应覆盖(issue #7)。"""
exc = GovernanceBackendError("限流后端 try_acquire 失败: boom", scope="LLM")
assert isinstance(exc, GatewayUnavailableError)
assert exc.reason == "governance_backend_down"
assert exc.scope == "llm" # 与既有 scope 级异常同款: 归一化小写
assert exc.retry_after_s == GOVERNANCE_BACKEND_RETRY_AFTER_S
def test_diagnostic_message_survives_reparenting(self):
"""父类把 message 覆写为模板串,而各构造点的诊断串是排障主线索(§3.5)。"""
exc = GovernanceBackendError("熔断后端 try_enter 失败: boom", scope="llm")
assert str(exc) == "熔断后端 try_enter 失败: boom"
def test_retry_after_overridable(self):
exc = GovernanceBackendError("redis down", scope="llm", retry_after_s=30.0)
assert exc.retry_after_s == 30.0
class TestSourceNotConfigured: class TestSourceNotConfigured:
"""装配缺陷有意留在 scope 级家族之外(issue #7 §3.4,Q1 人类拍板)。""" """装配缺陷有意留在 scope 级家族之外(issue #7 §3.4,Q1 人类拍板)。"""
+5 -2
View File
@@ -68,10 +68,13 @@ class TestConversions:
_limiter(lease_ttl_s=0) _limiter(lease_ttl_s=0)
def test_unknown_source_rejected(self): def test_unknown_source_rejected(self):
from polygateway.errors import GovernanceBackendError """未知源是装配缺陷,不是后端故障(issue #7 §3.4)。"""
from polygateway.errors import GatewayUnavailableError, SourceNotConfiguredError
with pytest.raises(GovernanceBackendError): with pytest.raises(SourceNotConfiguredError) as ei:
_limiter()._cfg("nope") _limiter()._cfg("nope")
# 关键: 若归入 scope 级家族,配置写错的任务会永远延期重投、永不进死信
assert not isinstance(ei.value, GatewayUnavailableError)
class TestLuaFidelity: class TestLuaFidelity: