fix: reparent governance backend failures under GatewayUnavailableError (issue #7)
A fail-closed limiter or breaker backend means the scope cannot emit a single request, which is exactly scope-level unavailability. But the error sat directly under PolyGatewayError, so a caller writing only `except GatewayUnavailableError` dropped it into the catch-all branch: Redis blips once and a backlog of tasks burns its business failure budget into the dead letter queue, over a fault a restart would clear. Three gate paths leak to callers rather than being absorbed by _record_quietly (try_acquire, try_enter, progress_age_s); each is now pinned by a test, since none of them had one before. The two unknown-source sites move to SourceNotConfiguredError instead of following along. They report a misconfigured source name, not an outage, and letting them into the retryable family would be the mirror of the bug being fixed here: the task would retry forever and never surface.
This commit is contained in:
@@ -16,7 +16,11 @@ import pytest
|
||||
from polygateway.backends.redis.breaker import RedisGate
|
||||
from polygateway.backends.redis.limiter import RedisLimiter
|
||||
from polygateway.client import GatewayClient
|
||||
from polygateway.errors import AllSourcesExhausted, GovernanceBackendError
|
||||
from polygateway.errors import (
|
||||
AllSourcesExhausted,
|
||||
GatewayUnavailableError,
|
||||
GovernanceBackendError,
|
||||
)
|
||||
from polygateway.sources import RoundRobinSelector
|
||||
from polygateway.types import (
|
||||
BackpressurePolicy,
|
||||
@@ -225,7 +229,11 @@ async def test_cancel_in_flight_releases_lease(clients):
|
||||
|
||||
|
||||
async def test_redis_down_admission_fails_closed():
|
||||
"""Redis 不可达 → 准入侧抛 GovernanceBackendError,绝不放行(库铁律)。"""
|
||||
"""Redis 不可达 → 准入侧报错绝不放行(库铁律),且以 scope 级形态到达调用方。
|
||||
|
||||
issue #7: 调用方只写 `except GatewayUnavailableError` 就该覆盖后端故障——
|
||||
真实 Redis 掉线是这条链路唯一的端到端证据,故断言收紧到 scope 级语义。
|
||||
"""
|
||||
import redis.asyncio as aioredis
|
||||
|
||||
dead = aioredis.from_url(
|
||||
@@ -240,9 +248,12 @@ async def test_redis_down_admission_fails_closed():
|
||||
lease_ttl_s=30.0,
|
||||
)
|
||||
gate = RedisGate(config=_CFG, redis=dead, scope="t-dead")
|
||||
with pytest.raises(GovernanceBackendError):
|
||||
await limiter.try_acquire("s1", 0)
|
||||
with pytest.raises(GovernanceBackendError):
|
||||
await gate.try_enter("s1", "w")
|
||||
for call in (limiter.try_acquire("s1", 0), gate.try_enter("s1", "w")):
|
||||
with pytest.raises(GatewayUnavailableError) as ei:
|
||||
await call
|
||||
assert isinstance(ei.value, GovernanceBackendError)
|
||||
assert ei.value.reason == "governance_backend_down"
|
||||
assert ei.value.scope == "t-dead"
|
||||
assert ei.value.retry_after_s > 0
|
||||
finally:
|
||||
await dead.aclose()
|
||||
|
||||
@@ -11,7 +11,13 @@ import pytest
|
||||
|
||||
from polygateway.backends.memory.breaker import InMemoryGate
|
||||
from polygateway.backends.memory.limiter import InMemoryLimiter
|
||||
from polygateway.errors import AllSourcesExhausted, GovernanceBackendError, TransientError
|
||||
from polygateway.errors import (
|
||||
AllSourcesExhausted,
|
||||
GatewayUnavailableError,
|
||||
GovernanceBackendError,
|
||||
SourceNotConfiguredError,
|
||||
TransientError,
|
||||
)
|
||||
from polygateway.middleware.retry import RetryMW, backoff_delay
|
||||
from polygateway.sources import RoundRobinSelector, SourceCooldownMemo
|
||||
from polygateway.types import (
|
||||
@@ -173,17 +179,17 @@ class TestStallQuadrants:
|
||||
|
||||
class _GateSuccessBroken(InMemoryGate):
|
||||
async def record_success(self, entry):
|
||||
raise GovernanceBackendError("redis 抖动")
|
||||
raise GovernanceBackendError("redis 抖动", scope="llm")
|
||||
|
||||
|
||||
class _GateFailureBroken(InMemoryGate):
|
||||
async def record_failure(self, entry, reason, force_open):
|
||||
raise GovernanceBackendError("redis 抖动")
|
||||
raise GovernanceBackendError("redis 抖动", scope="llm")
|
||||
|
||||
|
||||
class _LimiterProgressBroken(InMemoryLimiter):
|
||||
async def mark_progress(self):
|
||||
raise GovernanceBackendError("redis 抖动")
|
||||
raise GovernanceBackendError("redis 抖动", scope="llm")
|
||||
|
||||
|
||||
class TestAccountingDegradation:
|
||||
@@ -252,6 +258,71 @@ class TestQuotaGateProgressAge:
|
||||
async def progress_age_s(self):
|
||||
raise OSError("down")
|
||||
|
||||
assert await QuotaGate(_L()).progress_age_s() == 12.5
|
||||
assert await QuotaGate(_L(), scope="llm").progress_age_s() == 12.5
|
||||
with pytest.raises(GovernanceBackendError):
|
||||
await QuotaGate(_Broken()).progress_age_s()
|
||||
await QuotaGate(_Broken(), scope="llm").progress_age_s()
|
||||
|
||||
|
||||
class TestUnknownSourceIsAssemblyDefect:
|
||||
"""未知源 = 限流后端的源名单与治理循环对不上,是装配缺陷不是后端故障。
|
||||
|
||||
两个后端行为必须一致(Redis 版对应用例在 `test_redis_key_layout.py::
|
||||
TestConversions::test_unknown_source_rejected`);内存版此前无覆盖,
|
||||
该分支从未被测过(issue #7 §3.4)。
|
||||
"""
|
||||
|
||||
def test_memory_limiter_rejects_unknown_source(self):
|
||||
limiter = InMemoryLimiter(
|
||||
scope="llm", sources={"s1": make_source("s1")}, global_limits=_NO_GLOBAL
|
||||
)
|
||||
with pytest.raises(SourceNotConfiguredError) as ei:
|
||||
limiter._cfg("nope")
|
||||
# 关键: 若归入 scope 级家族,配置写错的任务会永远延期重投、永不进死信
|
||||
assert not isinstance(ei.value, GatewayUnavailableError)
|
||||
|
||||
|
||||
class TestGateFailuresReachCallersAsScopeLevel:
|
||||
"""三条闸门泄漏路径必须以 scope 级不可用的形态到达调用方(issue #7)。
|
||||
|
||||
记账路径由 `_record_quietly` 降级为 warning,但闸门路径没有那层包裹,会一路
|
||||
抛给调用方。只写 `except GatewayUnavailableError` 的调用方此前接不住,后果
|
||||
是 Redis 抖一下就让积压任务烧掉业务失败预算进死信——而那是运维重启即可恢复
|
||||
的故障。三条路径逐一钉住,防止将来任何一条被漏掉。
|
||||
"""
|
||||
|
||||
async def test_try_acquire_failure_is_scope_level(self):
|
||||
from polygateway.middleware.ratelimit import QuotaGate
|
||||
|
||||
class _Broken:
|
||||
async def try_acquire(self, name, est):
|
||||
raise OSError("down")
|
||||
|
||||
with pytest.raises(GatewayUnavailableError) as ei:
|
||||
await QuotaGate(_Broken(), scope="LLM").try_acquire(make_source("s1"))
|
||||
assert ei.value.scope == "llm"
|
||||
assert ei.value.reason == "governance_backend_down"
|
||||
assert ei.value.retry_after_s > 0 # 0 会让积压任务零延迟冲击已挂的后端
|
||||
|
||||
async def test_try_enter_failure_is_scope_level(self):
|
||||
from polygateway.middleware.breaker import BreakerGate
|
||||
|
||||
class _Broken:
|
||||
async def try_enter(self, name, owner):
|
||||
raise OSError("down")
|
||||
|
||||
with pytest.raises(GatewayUnavailableError) as ei:
|
||||
await BreakerGate(_Broken(), scope="LLM").try_enter(make_source("s1"), "owner")
|
||||
assert ei.value.scope == "llm"
|
||||
assert ei.value.reason == "governance_backend_down"
|
||||
|
||||
async def test_progress_age_failure_is_scope_level(self):
|
||||
from polygateway.middleware.ratelimit import QuotaGate
|
||||
|
||||
class _Broken:
|
||||
async def progress_age_s(self):
|
||||
raise OSError("down")
|
||||
|
||||
with pytest.raises(GatewayUnavailableError) as ei:
|
||||
await QuotaGate(_Broken(), scope="LLM").progress_age_s()
|
||||
assert ei.value.scope == "llm"
|
||||
assert ei.value.reason == "governance_backend_down"
|
||||
|
||||
@@ -89,10 +89,27 @@ class TestGatewayUnavailable:
|
||||
class TestBackendFailure:
|
||||
def test_governance_backend_error_is_not_transient(self):
|
||||
"""限流/熔断后端故障必须报错不放行,且不落入可重试分类。"""
|
||||
exc = GovernanceBackendError("redis down")
|
||||
exc = GovernanceBackendError("redis down", scope="llm")
|
||||
assert isinstance(exc, PolyGatewayError)
|
||||
assert not isinstance(exc, TransientError)
|
||||
|
||||
def test_is_scope_level_unavailability(self):
|
||||
"""fail-closed 时整个 scope 一个请求都发不出去,调用方一条 except 应覆盖(issue #7)。"""
|
||||
exc = GovernanceBackendError("限流后端 try_acquire 失败: boom", scope="LLM")
|
||||
assert isinstance(exc, GatewayUnavailableError)
|
||||
assert exc.reason == "governance_backend_down"
|
||||
assert exc.scope == "llm" # 与既有 scope 级异常同款: 归一化小写
|
||||
assert exc.retry_after_s == GOVERNANCE_BACKEND_RETRY_AFTER_S
|
||||
|
||||
def test_diagnostic_message_survives_reparenting(self):
|
||||
"""父类把 message 覆写为模板串,而各构造点的诊断串是排障主线索(§3.5)。"""
|
||||
exc = GovernanceBackendError("熔断后端 try_enter 失败: boom", scope="llm")
|
||||
assert str(exc) == "熔断后端 try_enter 失败: boom"
|
||||
|
||||
def test_retry_after_overridable(self):
|
||||
exc = GovernanceBackendError("redis down", scope="llm", retry_after_s=30.0)
|
||||
assert exc.retry_after_s == 30.0
|
||||
|
||||
|
||||
class TestSourceNotConfigured:
|
||||
"""装配缺陷有意留在 scope 级家族之外(issue #7 §3.4,Q1 人类拍板)。"""
|
||||
|
||||
@@ -68,10 +68,13 @@ class TestConversions:
|
||||
_limiter(lease_ttl_s=0)
|
||||
|
||||
def test_unknown_source_rejected(self):
|
||||
from polygateway.errors import GovernanceBackendError
|
||||
"""未知源是装配缺陷,不是后端故障(issue #7 §3.4)。"""
|
||||
from polygateway.errors import GatewayUnavailableError, SourceNotConfiguredError
|
||||
|
||||
with pytest.raises(GovernanceBackendError):
|
||||
with pytest.raises(SourceNotConfiguredError) as ei:
|
||||
_limiter()._cfg("nope")
|
||||
# 关键: 若归入 scope 级家族,配置写错的任务会永远延期重投、永不进死信
|
||||
assert not isinstance(ei.value, GatewayUnavailableError)
|
||||
|
||||
|
||||
class TestLuaFidelity:
|
||||
|
||||
Reference in New Issue
Block a user