fix: reparent governance backend failures under GatewayUnavailableError (issue #7)

A fail-closed limiter or breaker backend means the scope cannot emit a
single request, which is exactly scope-level unavailability. But the error
sat directly under PolyGatewayError, so a caller writing only
`except GatewayUnavailableError` dropped it into the catch-all branch:
Redis blips once and a backlog of tasks burns its business failure budget
into the dead letter queue, over a fault a restart would clear.

Three gate paths leak to callers rather than being absorbed by
_record_quietly (try_acquire, try_enter, progress_age_s); each is now
pinned by a test, since none of them had one before.

The two unknown-source sites move to SourceNotConfiguredError instead of
following along. They report a misconfigured source name, not an outage,
and letting them into the retryable family would be the mirror of the bug
being fixed here: the task would retry forever and never surface.
This commit is contained in:
2026-08-06 04:53:52 -04:00
parent dd540496a1
commit 45073486a7
13 changed files with 179 additions and 49 deletions
+77 -6
View File
@@ -11,7 +11,13 @@ import pytest
from polygateway.backends.memory.breaker import InMemoryGate
from polygateway.backends.memory.limiter import InMemoryLimiter
from polygateway.errors import AllSourcesExhausted, GovernanceBackendError, TransientError
from polygateway.errors import (
AllSourcesExhausted,
GatewayUnavailableError,
GovernanceBackendError,
SourceNotConfiguredError,
TransientError,
)
from polygateway.middleware.retry import RetryMW, backoff_delay
from polygateway.sources import RoundRobinSelector, SourceCooldownMemo
from polygateway.types import (
@@ -173,17 +179,17 @@ class TestStallQuadrants:
class _GateSuccessBroken(InMemoryGate):
async def record_success(self, entry):
raise GovernanceBackendError("redis 抖动")
raise GovernanceBackendError("redis 抖动", scope="llm")
class _GateFailureBroken(InMemoryGate):
async def record_failure(self, entry, reason, force_open):
raise GovernanceBackendError("redis 抖动")
raise GovernanceBackendError("redis 抖动", scope="llm")
class _LimiterProgressBroken(InMemoryLimiter):
async def mark_progress(self):
raise GovernanceBackendError("redis 抖动")
raise GovernanceBackendError("redis 抖动", scope="llm")
class TestAccountingDegradation:
@@ -252,6 +258,71 @@ class TestQuotaGateProgressAge:
async def progress_age_s(self):
raise OSError("down")
assert await QuotaGate(_L()).progress_age_s() == 12.5
assert await QuotaGate(_L(), scope="llm").progress_age_s() == 12.5
with pytest.raises(GovernanceBackendError):
await QuotaGate(_Broken()).progress_age_s()
await QuotaGate(_Broken(), scope="llm").progress_age_s()
class TestUnknownSourceIsAssemblyDefect:
"""未知源 = 限流后端的源名单与治理循环对不上,是装配缺陷不是后端故障。
两个后端行为必须一致(Redis 版对应用例在 `test_redis_key_layout.py::
TestConversions::test_unknown_source_rejected`);内存版此前无覆盖,
该分支从未被测过(issue #7 §3.4)。
"""
def test_memory_limiter_rejects_unknown_source(self):
limiter = InMemoryLimiter(
scope="llm", sources={"s1": make_source("s1")}, global_limits=_NO_GLOBAL
)
with pytest.raises(SourceNotConfiguredError) as ei:
limiter._cfg("nope")
# 关键: 若归入 scope 级家族,配置写错的任务会永远延期重投、永不进死信
assert not isinstance(ei.value, GatewayUnavailableError)
class TestGateFailuresReachCallersAsScopeLevel:
"""三条闸门泄漏路径必须以 scope 级不可用的形态到达调用方(issue #7)。
记账路径由 `_record_quietly` 降级为 warning,但闸门路径没有那层包裹,会一路
抛给调用方。只写 `except GatewayUnavailableError` 的调用方此前接不住,后果
是 Redis 抖一下就让积压任务烧掉业务失败预算进死信——而那是运维重启即可恢复
的故障。三条路径逐一钉住,防止将来任何一条被漏掉。
"""
async def test_try_acquire_failure_is_scope_level(self):
from polygateway.middleware.ratelimit import QuotaGate
class _Broken:
async def try_acquire(self, name, est):
raise OSError("down")
with pytest.raises(GatewayUnavailableError) as ei:
await QuotaGate(_Broken(), scope="LLM").try_acquire(make_source("s1"))
assert ei.value.scope == "llm"
assert ei.value.reason == "governance_backend_down"
assert ei.value.retry_after_s > 0 # 0 会让积压任务零延迟冲击已挂的后端
async def test_try_enter_failure_is_scope_level(self):
from polygateway.middleware.breaker import BreakerGate
class _Broken:
async def try_enter(self, name, owner):
raise OSError("down")
with pytest.raises(GatewayUnavailableError) as ei:
await BreakerGate(_Broken(), scope="LLM").try_enter(make_source("s1"), "owner")
assert ei.value.scope == "llm"
assert ei.value.reason == "governance_backend_down"
async def test_progress_age_failure_is_scope_level(self):
from polygateway.middleware.ratelimit import QuotaGate
class _Broken:
async def progress_age_s(self):
raise OSError("down")
with pytest.raises(GatewayUnavailableError) as ei:
await QuotaGate(_Broken(), scope="LLM").progress_age_s()
assert ei.value.scope == "llm"
assert ei.value.reason == "governance_backend_down"