fix: reparent governance backend failures under GatewayUnavailableError (issue #7)
A fail-closed limiter or breaker backend means the scope cannot emit a single request, which is exactly scope-level unavailability. But the error sat directly under PolyGatewayError, so a caller writing only `except GatewayUnavailableError` dropped it into the catch-all branch: Redis blips once and a backlog of tasks burns its business failure budget into the dead letter queue, over a fault a restart would clear. Three gate paths leak to callers rather than being absorbed by _record_quietly (try_acquire, try_enter, progress_age_s); each is now pinned by a test, since none of them had one before. The two unknown-source sites move to SourceNotConfiguredError instead of following along. They report a misconfigured source name, not an outage, and letting them into the retryable family would be the mirror of the bug being fixed here: the task would retry forever and never surface.
This commit is contained in:
@@ -15,7 +15,7 @@ import time
|
|||||||
import uuid
|
import uuid
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from polygateway.errors import GovernanceBackendError
|
from polygateway.errors import SourceNotConfiguredError
|
||||||
from polygateway.types import GlobalLimits, SourceConfig, SourceStats
|
from polygateway.types import GlobalLimits, SourceConfig, SourceStats
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -89,7 +89,7 @@ class InMemoryLimiter:
|
|||||||
def _cfg(self, source_key: str) -> SourceConfig:
|
def _cfg(self, source_key: str) -> SourceConfig:
|
||||||
cfg = self._sources.get(source_key)
|
cfg = self._sources.get(source_key)
|
||||||
if cfg is None:
|
if cfg is None:
|
||||||
raise GovernanceBackendError(f"未知源 {source_key!r}(scope={self._scope})")
|
raise SourceNotConfiguredError(f"未知源 {source_key!r}(scope={self._scope})")
|
||||||
return cfg
|
return cfg
|
||||||
|
|
||||||
def _window(self) -> int:
|
def _window(self) -> int:
|
||||||
|
|||||||
@@ -367,7 +367,7 @@ class RedisGate:
|
|||||||
keys=[self._key(source_name)], args=[owner, self._probe_ttl_ms]
|
keys=[self._key(source_name)], args=[owner, self._probe_ttl_ms]
|
||||||
)
|
)
|
||||||
except RedisError as exc:
|
except RedisError as exc:
|
||||||
raise GovernanceBackendError(f"熔断后端 try_enter 失败: {exc}") from exc
|
raise GovernanceBackendError(f"熔断后端 try_enter 失败: {exc}", scope=self._scope) from exc
|
||||||
return self._decision(source_name, result)
|
return self._decision(source_name, result)
|
||||||
|
|
||||||
async def record_success(
|
async def record_success(
|
||||||
@@ -385,7 +385,7 @@ class RedisGate:
|
|||||||
try:
|
try:
|
||||||
result = await self._success_lua(keys=[self._key(entry.source_name)], args=args)
|
result = await self._success_lua(keys=[self._key(entry.source_name)], args=args)
|
||||||
except RedisError as exc:
|
except RedisError as exc:
|
||||||
raise GovernanceBackendError(f"熔断后端 record_success 失败: {exc}") from exc
|
raise GovernanceBackendError(f"熔断后端 record_success 失败: {exc}", scope=self._scope) from exc
|
||||||
return self._update(result)
|
return self._update(result)
|
||||||
|
|
||||||
async def record_failure(
|
async def record_failure(
|
||||||
@@ -407,7 +407,7 @@ class RedisGate:
|
|||||||
try:
|
try:
|
||||||
result = await self._failure_lua(keys=[self._key(entry.source_name)], args=args)
|
result = await self._failure_lua(keys=[self._key(entry.source_name)], args=args)
|
||||||
except RedisError as exc:
|
except RedisError as exc:
|
||||||
raise GovernanceBackendError(f"熔断后端 record_failure 失败: {exc}") from exc
|
raise GovernanceBackendError(f"熔断后端 record_failure 失败: {exc}", scope=self._scope) from exc
|
||||||
return self._update(result)
|
return self._update(result)
|
||||||
|
|
||||||
async def release_probe(self, entry: GateDecision) -> GateUpdate:
|
async def release_probe(self, entry: GateDecision) -> GateUpdate:
|
||||||
@@ -419,7 +419,7 @@ class RedisGate:
|
|||||||
keys=[self._key(entry.source_name)], args=[entry.epoch, entry.probe_owner]
|
keys=[self._key(entry.source_name)], args=[entry.epoch, entry.probe_owner]
|
||||||
)
|
)
|
||||||
except RedisError as exc:
|
except RedisError as exc:
|
||||||
raise GovernanceBackendError(f"熔断后端 release_probe 失败: {exc}") from exc
|
raise GovernanceBackendError(f"熔断后端 release_probe 失败: {exc}", scope=self._scope) from exc
|
||||||
return self._update(result)
|
return self._update(result)
|
||||||
|
|
||||||
async def retry_after_s(self, sources: tuple[str, ...]) -> float:
|
async def retry_after_s(self, sources: tuple[str, ...]) -> float:
|
||||||
@@ -429,7 +429,7 @@ class RedisGate:
|
|||||||
try:
|
try:
|
||||||
result = await self._retry_after_lua(keys=[self._key(s) for s in sources])
|
result = await self._retry_after_lua(keys=[self._key(s) for s in sources])
|
||||||
except RedisError as exc:
|
except RedisError as exc:
|
||||||
raise GovernanceBackendError(f"熔断后端 retry_after_s 失败: {exc}") from exc
|
raise GovernanceBackendError(f"熔断后端 retry_after_s 失败: {exc}", scope=self._scope) from exc
|
||||||
return int(result) / 1000.0
|
return int(result) / 1000.0
|
||||||
|
|
||||||
async def aclose(self) -> None:
|
async def aclose(self) -> None:
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ from typing import TYPE_CHECKING
|
|||||||
from loguru import logger
|
from loguru import logger
|
||||||
from redis.exceptions import RedisError
|
from redis.exceptions import RedisError
|
||||||
|
|
||||||
from polygateway.errors import GovernanceBackendError
|
from polygateway.errors import GovernanceBackendError, SourceNotConfiguredError
|
||||||
from polygateway.types import GlobalLimits, SourceConfig, SourceStats
|
from polygateway.types import GlobalLimits, SourceConfig, SourceStats
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -195,7 +195,7 @@ class RedisLimiter:
|
|||||||
def _cfg(self, source_key: str) -> SourceConfig:
|
def _cfg(self, source_key: str) -> SourceConfig:
|
||||||
cfg = self._sources.get(source_key)
|
cfg = self._sources.get(source_key)
|
||||||
if cfg is None:
|
if cfg is None:
|
||||||
raise GovernanceBackendError(f"未知源 {source_key!r}(scope={self._scope})")
|
raise SourceNotConfiguredError(f"未知源 {source_key!r}(scope={self._scope})")
|
||||||
return cfg
|
return cfg
|
||||||
|
|
||||||
def _lease_keys(self, source_key: str) -> tuple[str, str]:
|
def _lease_keys(self, source_key: str) -> tuple[str, str]:
|
||||||
@@ -247,7 +247,7 @@ class RedisLimiter:
|
|||||||
],
|
],
|
||||||
)
|
)
|
||||||
except RedisError as exc:
|
except RedisError as exc:
|
||||||
raise GovernanceBackendError(f"限流后端 try_acquire 失败: {exc}") from exc
|
raise GovernanceBackendError(f"限流后端 try_acquire 失败: {exc}", scope=self._scope) from exc
|
||||||
if ok != 1:
|
if ok != 1:
|
||||||
return None
|
return None
|
||||||
return _RedisPermit(self, source_key, lease_id, est_tokens, window)
|
return _RedisPermit(self, source_key, lease_id, est_tokens, window)
|
||||||
@@ -265,14 +265,14 @@ class RedisLimiter:
|
|||||||
try:
|
try:
|
||||||
await self._release_lua(keys=[gl, sl], args=[lease_id])
|
await self._release_lua(keys=[gl, sl], args=[lease_id])
|
||||||
except RedisError as exc:
|
except RedisError as exc:
|
||||||
raise GovernanceBackendError(f"限流后端 release 失败: {exc}") from exc
|
raise GovernanceBackendError(f"限流后端 release 失败: {exc}", scope=self._scope) from exc
|
||||||
|
|
||||||
async def _settle_tpm(self, source_key: str, delta: int, window: int) -> None:
|
async def _settle_tpm(self, source_key: str, delta: int, window: int) -> None:
|
||||||
wk = self._window_keys(source_key, window)
|
wk = self._window_keys(source_key, window)
|
||||||
try:
|
try:
|
||||||
await self._settle_lua(keys=[wk["g_tpm"], wk["s_tpm"]], args=[delta, _WINDOW_TTL_S])
|
await self._settle_lua(keys=[wk["g_tpm"], wk["s_tpm"]], args=[delta, _WINDOW_TTL_S])
|
||||||
except RedisError as exc:
|
except RedisError as exc:
|
||||||
raise GovernanceBackendError(f"限流后端 settle 失败: {exc}") from exc
|
raise GovernanceBackendError(f"限流后端 settle 失败: {exc}", scope=self._scope) from exc
|
||||||
|
|
||||||
async def source_stats(self, source_key: str) -> SourceStats:
|
async def source_stats(self, source_key: str) -> SourceStats:
|
||||||
"""当前窗口快照;读侧 clamp ≥0(展示口径,存储保留负值)。"""
|
"""当前窗口快照;读侧 clamp ≥0(展示口径,存储保留负值)。"""
|
||||||
@@ -283,7 +283,7 @@ class RedisLimiter:
|
|||||||
wk = self._window_keys(source_key, window)
|
wk = self._window_keys(source_key, window)
|
||||||
res = await self._stats_lua(keys=[sl, wk["s_rpm"], wk["s_tpm"]])
|
res = await self._stats_lua(keys=[sl, wk["s_rpm"], wk["s_tpm"]])
|
||||||
except RedisError as exc:
|
except RedisError as exc:
|
||||||
raise GovernanceBackendError(f"限流后端 source_stats 失败: {exc}") from exc
|
raise GovernanceBackendError(f"限流后端 source_stats 失败: {exc}", scope=self._scope) from exc
|
||||||
return SourceStats(
|
return SourceStats(
|
||||||
inflight=int(res[0]),
|
inflight=int(res[0]),
|
||||||
rpm_used=max(0, int(res[1])),
|
rpm_used=max(0, int(res[1])),
|
||||||
@@ -295,14 +295,14 @@ class RedisLimiter:
|
|||||||
try:
|
try:
|
||||||
await self._progress_mark_lua(keys=[self._progress_key()], args=[_PROGRESS_TTL_S])
|
await self._progress_mark_lua(keys=[self._progress_key()], args=[_PROGRESS_TTL_S])
|
||||||
except RedisError as exc:
|
except RedisError as exc:
|
||||||
raise GovernanceBackendError(f"限流后端 mark_progress 失败: {exc}") from exc
|
raise GovernanceBackendError(f"限流后端 mark_progress 失败: {exc}", scope=self._scope) from exc
|
||||||
|
|
||||||
async def progress_age_s(self) -> float:
|
async def progress_age_s(self) -> float:
|
||||||
"""距上次全局成功的秒数;仅键缺失(-1)= 从未进展 → inf(CHS limiter.py:208)。"""
|
"""距上次全局成功的秒数;仅键缺失(-1)= 从未进展 → inf(CHS limiter.py:208)。"""
|
||||||
try:
|
try:
|
||||||
res = await self._progress_age_lua(keys=[self._progress_key()])
|
res = await self._progress_age_lua(keys=[self._progress_key()])
|
||||||
except RedisError as exc:
|
except RedisError as exc:
|
||||||
raise GovernanceBackendError(f"限流后端 progress_age_s 失败: {exc}") from exc
|
raise GovernanceBackendError(f"限流后端 progress_age_s 失败: {exc}", scope=self._scope) from exc
|
||||||
return float("inf") if int(res) == -1 else int(res) / 1000.0
|
return float("inf") if int(res) == -1 else int(res) / 1000.0
|
||||||
|
|
||||||
async def aclose(self) -> None:
|
async def aclose(self) -> None:
|
||||||
|
|||||||
@@ -120,8 +120,8 @@ class EmbeddingClient:
|
|||||||
# 否则遥测会记录一个从未发出的采样参数(issue #4 决策 G)
|
# 否则遥测会记录一个从未发出的采样参数(issue #4 决策 G)
|
||||||
self._sources = strip_unsupported_extra_body(list(sources), path="embedding")
|
self._sources = strip_unsupported_extra_body(list(sources), path="embedding")
|
||||||
self._selector = selector
|
self._selector = selector
|
||||||
self._quota = QuotaGate(limiter)
|
self._quota = QuotaGate(limiter, scope=self._scope)
|
||||||
self._breaker = BreakerGate(breaker)
|
self._breaker = BreakerGate(breaker, scope=self._scope)
|
||||||
self._transport = transport
|
self._transport = transport
|
||||||
self._retry = retry
|
self._retry = retry
|
||||||
self._bp = backpressure
|
self._bp = backpressure
|
||||||
|
|||||||
@@ -150,5 +150,29 @@ class SourceNotConfiguredError(PolyGatewayError):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
class GovernanceBackendError(PolyGatewayError):
|
class GovernanceBackendError(GatewayUnavailableError):
|
||||||
"""限流/熔断状态后端自身故障: 必须报错而非放行(防击穿网关,降级方向铁律)。"""
|
"""限流/熔断状态后端自身故障: 必须报错而非放行(防击穿网关,降级方向铁律)。
|
||||||
|
|
||||||
|
继承 `GatewayUnavailableError`(issue #7): fail-closed 意味着整个 scope 一个
|
||||||
|
请求都发不出去,语义上即 scope 级不可用。此前它是 `PolyGatewayError` 的直接
|
||||||
|
子类,只写 `except GatewayUnavailableError` 的调用方接不住,后果是"Redis 抖
|
||||||
|
一下 → 积压任务消耗业务失败预算 → 进死信",而那是运维重启即可恢复的故障。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
message: str,
|
||||||
|
*,
|
||||||
|
scope: str,
|
||||||
|
retry_after_s: float = GOVERNANCE_BACKEND_RETRY_AFTER_S,
|
||||||
|
source_name: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(
|
||||||
|
scope=scope,
|
||||||
|
reason="governance_backend_down",
|
||||||
|
retry_after_s=retry_after_s,
|
||||||
|
source_name=source_name,
|
||||||
|
)
|
||||||
|
# 父类会把 message 覆写为 "{scope} 网关暂时不可用: {reason}",而各构造点
|
||||||
|
# 携带的诊断串(如"限流后端 try_acquire 失败: ...")是排障主线索,必须保住
|
||||||
|
self.args = (message,)
|
||||||
|
|||||||
@@ -14,8 +14,10 @@ if TYPE_CHECKING:
|
|||||||
class BreakerGate:
|
class BreakerGate:
|
||||||
"""RetryMW 面向熔断后端的唯一入口;包装一切后端异常。"""
|
"""RetryMW 面向熔断后端的唯一入口;包装一切后端异常。"""
|
||||||
|
|
||||||
def __init__(self, gate: ProviderGate) -> None:
|
def __init__(self, gate: ProviderGate, *, scope: str) -> None:
|
||||||
self._gate = gate
|
self._gate = gate
|
||||||
|
# 后端故障即 scope 级不可用,异常须携 scope 供调用方定位(issue #7 §3.3)
|
||||||
|
self._scope = scope
|
||||||
|
|
||||||
async def try_enter(self, source: SourceConfig, owner: str) -> GateDecision:
|
async def try_enter(self, source: SourceConfig, owner: str) -> GateDecision:
|
||||||
try:
|
try:
|
||||||
@@ -23,7 +25,7 @@ class BreakerGate:
|
|||||||
except GovernanceBackendError:
|
except GovernanceBackendError:
|
||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise GovernanceBackendError(f"熔断后端故障(try_enter): {exc}") from exc
|
raise GovernanceBackendError(f"熔断后端故障(try_enter): {exc}", scope=self._scope) from exc
|
||||||
|
|
||||||
async def record_success(
|
async def record_success(
|
||||||
self, entry: GateDecision, *, count_attempt: bool = True
|
self, entry: GateDecision, *, count_attempt: bool = True
|
||||||
@@ -33,7 +35,7 @@ class BreakerGate:
|
|||||||
except GovernanceBackendError:
|
except GovernanceBackendError:
|
||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise GovernanceBackendError(f"熔断后端故障(record_success): {exc}") from exc
|
raise GovernanceBackendError(f"熔断后端故障(record_success): {exc}", scope=self._scope) from exc
|
||||||
|
|
||||||
async def record_failure(
|
async def record_failure(
|
||||||
self, entry: GateDecision, reason: str, force_open: bool
|
self, entry: GateDecision, reason: str, force_open: bool
|
||||||
@@ -43,7 +45,7 @@ class BreakerGate:
|
|||||||
except GovernanceBackendError:
|
except GovernanceBackendError:
|
||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise GovernanceBackendError(f"熔断后端故障(record_failure): {exc}") from exc
|
raise GovernanceBackendError(f"熔断后端故障(record_failure): {exc}", scope=self._scope) from exc
|
||||||
|
|
||||||
async def release_probe(self, entry: GateDecision) -> GateUpdate:
|
async def release_probe(self, entry: GateDecision) -> GateUpdate:
|
||||||
try:
|
try:
|
||||||
@@ -51,7 +53,7 @@ class BreakerGate:
|
|||||||
except GovernanceBackendError:
|
except GovernanceBackendError:
|
||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise GovernanceBackendError(f"熔断后端故障(release_probe): {exc}") from exc
|
raise GovernanceBackendError(f"熔断后端故障(release_probe): {exc}", scope=self._scope) from exc
|
||||||
|
|
||||||
async def retry_after_s(self, sources: tuple[str, ...]) -> float:
|
async def retry_after_s(self, sources: tuple[str, ...]) -> float:
|
||||||
try:
|
try:
|
||||||
@@ -59,4 +61,4 @@ class BreakerGate:
|
|||||||
except GovernanceBackendError:
|
except GovernanceBackendError:
|
||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise GovernanceBackendError(f"熔断后端故障(retry_after_s): {exc}") from exc
|
raise GovernanceBackendError(f"熔断后端故障(retry_after_s): {exc}", scope=self._scope) from exc
|
||||||
|
|||||||
@@ -18,8 +18,10 @@ if TYPE_CHECKING:
|
|||||||
class QuotaGate:
|
class QuotaGate:
|
||||||
"""RetryMW 面向限流后端的唯一入口;包装一切后端异常。"""
|
"""RetryMW 面向限流后端的唯一入口;包装一切后端异常。"""
|
||||||
|
|
||||||
def __init__(self, limiter: RateLimiter) -> None:
|
def __init__(self, limiter: RateLimiter, *, scope: str) -> None:
|
||||||
self._limiter = limiter
|
self._limiter = limiter
|
||||||
|
# 后端故障即 scope 级不可用,异常须携 scope 供调用方定位(issue #7 §3.3)
|
||||||
|
self._scope = scope
|
||||||
|
|
||||||
async def try_acquire(self, source: SourceConfig) -> Permit | None:
|
async def try_acquire(self, source: SourceConfig) -> Permit | None:
|
||||||
try:
|
try:
|
||||||
@@ -27,7 +29,7 @@ class QuotaGate:
|
|||||||
except GovernanceBackendError:
|
except GovernanceBackendError:
|
||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise GovernanceBackendError(f"限流后端故障(try_acquire): {exc}") from exc
|
raise GovernanceBackendError(f"限流后端故障(try_acquire): {exc}", scope=self._scope) from exc
|
||||||
|
|
||||||
async def stats(self, source: SourceConfig) -> SourceStats:
|
async def stats(self, source: SourceConfig) -> SourceStats:
|
||||||
try:
|
try:
|
||||||
@@ -35,7 +37,7 @@ class QuotaGate:
|
|||||||
except GovernanceBackendError:
|
except GovernanceBackendError:
|
||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise GovernanceBackendError(f"限流后端故障(source_stats): {exc}") from exc
|
raise GovernanceBackendError(f"限流后端故障(source_stats): {exc}", scope=self._scope) from exc
|
||||||
|
|
||||||
async def mark_progress(self) -> None:
|
async def mark_progress(self) -> None:
|
||||||
try:
|
try:
|
||||||
@@ -43,7 +45,7 @@ class QuotaGate:
|
|||||||
except GovernanceBackendError:
|
except GovernanceBackendError:
|
||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise GovernanceBackendError(f"限流后端故障(mark_progress): {exc}") from exc
|
raise GovernanceBackendError(f"限流后端故障(mark_progress): {exc}", scope=self._scope) from exc
|
||||||
|
|
||||||
async def progress_age_s(self) -> float:
|
async def progress_age_s(self) -> float:
|
||||||
try:
|
try:
|
||||||
@@ -51,4 +53,4 @@ class QuotaGate:
|
|||||||
except GovernanceBackendError:
|
except GovernanceBackendError:
|
||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise GovernanceBackendError(f"限流后端故障(progress_age_s): {exc}") from exc
|
raise GovernanceBackendError(f"限流后端故障(progress_age_s): {exc}", scope=self._scope) from exc
|
||||||
|
|||||||
@@ -183,8 +183,8 @@ class RetryMW:
|
|||||||
self._scope = scope
|
self._scope = scope
|
||||||
self._sources = list(sources)
|
self._sources = list(sources)
|
||||||
self._selector = selector
|
self._selector = selector
|
||||||
self._quota = QuotaGate(limiter)
|
self._quota = QuotaGate(limiter, scope=self._scope)
|
||||||
self._breaker = BreakerGate(gate)
|
self._breaker = BreakerGate(gate, scope=self._scope)
|
||||||
self._transport = transport
|
self._transport = transport
|
||||||
self._retry = retry
|
self._retry = retry
|
||||||
self._bp = backpressure
|
self._bp = backpressure
|
||||||
|
|||||||
@@ -119,8 +119,8 @@ class OcrClient:
|
|||||||
self._sources = strip_unsupported_extra_body(list(sources), path="OCR")
|
self._sources = strip_unsupported_extra_body(list(sources), path="OCR")
|
||||||
self._selector = selector
|
self._selector = selector
|
||||||
self._feed_health = isinstance(selector, OutcomeAwareSelector)
|
self._feed_health = isinstance(selector, OutcomeAwareSelector)
|
||||||
self._quota = QuotaGate(limiter)
|
self._quota = QuotaGate(limiter, scope=self._scope)
|
||||||
self._breaker = BreakerGate(breaker)
|
self._breaker = BreakerGate(breaker, scope=self._scope)
|
||||||
self._transport = transport
|
self._transport = transport
|
||||||
self._retry = retry
|
self._retry = retry
|
||||||
self._bp = backpressure
|
self._bp = backpressure
|
||||||
|
|||||||
@@ -16,7 +16,11 @@ import pytest
|
|||||||
from polygateway.backends.redis.breaker import RedisGate
|
from polygateway.backends.redis.breaker import RedisGate
|
||||||
from polygateway.backends.redis.limiter import RedisLimiter
|
from polygateway.backends.redis.limiter import RedisLimiter
|
||||||
from polygateway.client import GatewayClient
|
from polygateway.client import GatewayClient
|
||||||
from polygateway.errors import AllSourcesExhausted, GovernanceBackendError
|
from polygateway.errors import (
|
||||||
|
AllSourcesExhausted,
|
||||||
|
GatewayUnavailableError,
|
||||||
|
GovernanceBackendError,
|
||||||
|
)
|
||||||
from polygateway.sources import RoundRobinSelector
|
from polygateway.sources import RoundRobinSelector
|
||||||
from polygateway.types import (
|
from polygateway.types import (
|
||||||
BackpressurePolicy,
|
BackpressurePolicy,
|
||||||
@@ -225,7 +229,11 @@ async def test_cancel_in_flight_releases_lease(clients):
|
|||||||
|
|
||||||
|
|
||||||
async def test_redis_down_admission_fails_closed():
|
async def test_redis_down_admission_fails_closed():
|
||||||
"""Redis 不可达 → 准入侧抛 GovernanceBackendError,绝不放行(库铁律)。"""
|
"""Redis 不可达 → 准入侧报错绝不放行(库铁律),且以 scope 级形态到达调用方。
|
||||||
|
|
||||||
|
issue #7: 调用方只写 `except GatewayUnavailableError` 就该覆盖后端故障——
|
||||||
|
真实 Redis 掉线是这条链路唯一的端到端证据,故断言收紧到 scope 级语义。
|
||||||
|
"""
|
||||||
import redis.asyncio as aioredis
|
import redis.asyncio as aioredis
|
||||||
|
|
||||||
dead = aioredis.from_url(
|
dead = aioredis.from_url(
|
||||||
@@ -240,9 +248,12 @@ async def test_redis_down_admission_fails_closed():
|
|||||||
lease_ttl_s=30.0,
|
lease_ttl_s=30.0,
|
||||||
)
|
)
|
||||||
gate = RedisGate(config=_CFG, redis=dead, scope="t-dead")
|
gate = RedisGate(config=_CFG, redis=dead, scope="t-dead")
|
||||||
with pytest.raises(GovernanceBackendError):
|
for call in (limiter.try_acquire("s1", 0), gate.try_enter("s1", "w")):
|
||||||
await limiter.try_acquire("s1", 0)
|
with pytest.raises(GatewayUnavailableError) as ei:
|
||||||
with pytest.raises(GovernanceBackendError):
|
await call
|
||||||
await gate.try_enter("s1", "w")
|
assert isinstance(ei.value, GovernanceBackendError)
|
||||||
|
assert ei.value.reason == "governance_backend_down"
|
||||||
|
assert ei.value.scope == "t-dead"
|
||||||
|
assert ei.value.retry_after_s > 0
|
||||||
finally:
|
finally:
|
||||||
await dead.aclose()
|
await dead.aclose()
|
||||||
|
|||||||
@@ -11,7 +11,13 @@ import pytest
|
|||||||
|
|
||||||
from polygateway.backends.memory.breaker import InMemoryGate
|
from polygateway.backends.memory.breaker import InMemoryGate
|
||||||
from polygateway.backends.memory.limiter import InMemoryLimiter
|
from polygateway.backends.memory.limiter import InMemoryLimiter
|
||||||
from polygateway.errors import AllSourcesExhausted, GovernanceBackendError, TransientError
|
from polygateway.errors import (
|
||||||
|
AllSourcesExhausted,
|
||||||
|
GatewayUnavailableError,
|
||||||
|
GovernanceBackendError,
|
||||||
|
SourceNotConfiguredError,
|
||||||
|
TransientError,
|
||||||
|
)
|
||||||
from polygateway.middleware.retry import RetryMW, backoff_delay
|
from polygateway.middleware.retry import RetryMW, backoff_delay
|
||||||
from polygateway.sources import RoundRobinSelector, SourceCooldownMemo
|
from polygateway.sources import RoundRobinSelector, SourceCooldownMemo
|
||||||
from polygateway.types import (
|
from polygateway.types import (
|
||||||
@@ -173,17 +179,17 @@ class TestStallQuadrants:
|
|||||||
|
|
||||||
class _GateSuccessBroken(InMemoryGate):
|
class _GateSuccessBroken(InMemoryGate):
|
||||||
async def record_success(self, entry):
|
async def record_success(self, entry):
|
||||||
raise GovernanceBackendError("redis 抖动")
|
raise GovernanceBackendError("redis 抖动", scope="llm")
|
||||||
|
|
||||||
|
|
||||||
class _GateFailureBroken(InMemoryGate):
|
class _GateFailureBroken(InMemoryGate):
|
||||||
async def record_failure(self, entry, reason, force_open):
|
async def record_failure(self, entry, reason, force_open):
|
||||||
raise GovernanceBackendError("redis 抖动")
|
raise GovernanceBackendError("redis 抖动", scope="llm")
|
||||||
|
|
||||||
|
|
||||||
class _LimiterProgressBroken(InMemoryLimiter):
|
class _LimiterProgressBroken(InMemoryLimiter):
|
||||||
async def mark_progress(self):
|
async def mark_progress(self):
|
||||||
raise GovernanceBackendError("redis 抖动")
|
raise GovernanceBackendError("redis 抖动", scope="llm")
|
||||||
|
|
||||||
|
|
||||||
class TestAccountingDegradation:
|
class TestAccountingDegradation:
|
||||||
@@ -252,6 +258,71 @@ class TestQuotaGateProgressAge:
|
|||||||
async def progress_age_s(self):
|
async def progress_age_s(self):
|
||||||
raise OSError("down")
|
raise OSError("down")
|
||||||
|
|
||||||
assert await QuotaGate(_L()).progress_age_s() == 12.5
|
assert await QuotaGate(_L(), scope="llm").progress_age_s() == 12.5
|
||||||
with pytest.raises(GovernanceBackendError):
|
with pytest.raises(GovernanceBackendError):
|
||||||
await QuotaGate(_Broken()).progress_age_s()
|
await QuotaGate(_Broken(), scope="llm").progress_age_s()
|
||||||
|
|
||||||
|
|
||||||
|
class TestUnknownSourceIsAssemblyDefect:
|
||||||
|
"""未知源 = 限流后端的源名单与治理循环对不上,是装配缺陷不是后端故障。
|
||||||
|
|
||||||
|
两个后端行为必须一致(Redis 版对应用例在 `test_redis_key_layout.py::
|
||||||
|
TestConversions::test_unknown_source_rejected`);内存版此前无覆盖,
|
||||||
|
该分支从未被测过(issue #7 §3.4)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_memory_limiter_rejects_unknown_source(self):
|
||||||
|
limiter = InMemoryLimiter(
|
||||||
|
scope="llm", sources={"s1": make_source("s1")}, global_limits=_NO_GLOBAL
|
||||||
|
)
|
||||||
|
with pytest.raises(SourceNotConfiguredError) as ei:
|
||||||
|
limiter._cfg("nope")
|
||||||
|
# 关键: 若归入 scope 级家族,配置写错的任务会永远延期重投、永不进死信
|
||||||
|
assert not isinstance(ei.value, GatewayUnavailableError)
|
||||||
|
|
||||||
|
|
||||||
|
class TestGateFailuresReachCallersAsScopeLevel:
|
||||||
|
"""三条闸门泄漏路径必须以 scope 级不可用的形态到达调用方(issue #7)。
|
||||||
|
|
||||||
|
记账路径由 `_record_quietly` 降级为 warning,但闸门路径没有那层包裹,会一路
|
||||||
|
抛给调用方。只写 `except GatewayUnavailableError` 的调用方此前接不住,后果
|
||||||
|
是 Redis 抖一下就让积压任务烧掉业务失败预算进死信——而那是运维重启即可恢复
|
||||||
|
的故障。三条路径逐一钉住,防止将来任何一条被漏掉。
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def test_try_acquire_failure_is_scope_level(self):
|
||||||
|
from polygateway.middleware.ratelimit import QuotaGate
|
||||||
|
|
||||||
|
class _Broken:
|
||||||
|
async def try_acquire(self, name, est):
|
||||||
|
raise OSError("down")
|
||||||
|
|
||||||
|
with pytest.raises(GatewayUnavailableError) as ei:
|
||||||
|
await QuotaGate(_Broken(), scope="LLM").try_acquire(make_source("s1"))
|
||||||
|
assert ei.value.scope == "llm"
|
||||||
|
assert ei.value.reason == "governance_backend_down"
|
||||||
|
assert ei.value.retry_after_s > 0 # 0 会让积压任务零延迟冲击已挂的后端
|
||||||
|
|
||||||
|
async def test_try_enter_failure_is_scope_level(self):
|
||||||
|
from polygateway.middleware.breaker import BreakerGate
|
||||||
|
|
||||||
|
class _Broken:
|
||||||
|
async def try_enter(self, name, owner):
|
||||||
|
raise OSError("down")
|
||||||
|
|
||||||
|
with pytest.raises(GatewayUnavailableError) as ei:
|
||||||
|
await BreakerGate(_Broken(), scope="LLM").try_enter(make_source("s1"), "owner")
|
||||||
|
assert ei.value.scope == "llm"
|
||||||
|
assert ei.value.reason == "governance_backend_down"
|
||||||
|
|
||||||
|
async def test_progress_age_failure_is_scope_level(self):
|
||||||
|
from polygateway.middleware.ratelimit import QuotaGate
|
||||||
|
|
||||||
|
class _Broken:
|
||||||
|
async def progress_age_s(self):
|
||||||
|
raise OSError("down")
|
||||||
|
|
||||||
|
with pytest.raises(GatewayUnavailableError) as ei:
|
||||||
|
await QuotaGate(_Broken(), scope="LLM").progress_age_s()
|
||||||
|
assert ei.value.scope == "llm"
|
||||||
|
assert ei.value.reason == "governance_backend_down"
|
||||||
|
|||||||
@@ -89,10 +89,27 @@ class TestGatewayUnavailable:
|
|||||||
class TestBackendFailure:
|
class TestBackendFailure:
|
||||||
def test_governance_backend_error_is_not_transient(self):
|
def test_governance_backend_error_is_not_transient(self):
|
||||||
"""限流/熔断后端故障必须报错不放行,且不落入可重试分类。"""
|
"""限流/熔断后端故障必须报错不放行,且不落入可重试分类。"""
|
||||||
exc = GovernanceBackendError("redis down")
|
exc = GovernanceBackendError("redis down", scope="llm")
|
||||||
assert isinstance(exc, PolyGatewayError)
|
assert isinstance(exc, PolyGatewayError)
|
||||||
assert not isinstance(exc, TransientError)
|
assert not isinstance(exc, TransientError)
|
||||||
|
|
||||||
|
def test_is_scope_level_unavailability(self):
|
||||||
|
"""fail-closed 时整个 scope 一个请求都发不出去,调用方一条 except 应覆盖(issue #7)。"""
|
||||||
|
exc = GovernanceBackendError("限流后端 try_acquire 失败: boom", scope="LLM")
|
||||||
|
assert isinstance(exc, GatewayUnavailableError)
|
||||||
|
assert exc.reason == "governance_backend_down"
|
||||||
|
assert exc.scope == "llm" # 与既有 scope 级异常同款: 归一化小写
|
||||||
|
assert exc.retry_after_s == GOVERNANCE_BACKEND_RETRY_AFTER_S
|
||||||
|
|
||||||
|
def test_diagnostic_message_survives_reparenting(self):
|
||||||
|
"""父类把 message 覆写为模板串,而各构造点的诊断串是排障主线索(§3.5)。"""
|
||||||
|
exc = GovernanceBackendError("熔断后端 try_enter 失败: boom", scope="llm")
|
||||||
|
assert str(exc) == "熔断后端 try_enter 失败: boom"
|
||||||
|
|
||||||
|
def test_retry_after_overridable(self):
|
||||||
|
exc = GovernanceBackendError("redis down", scope="llm", retry_after_s=30.0)
|
||||||
|
assert exc.retry_after_s == 30.0
|
||||||
|
|
||||||
|
|
||||||
class TestSourceNotConfigured:
|
class TestSourceNotConfigured:
|
||||||
"""装配缺陷有意留在 scope 级家族之外(issue #7 §3.4,Q1 人类拍板)。"""
|
"""装配缺陷有意留在 scope 级家族之外(issue #7 §3.4,Q1 人类拍板)。"""
|
||||||
|
|||||||
@@ -68,10 +68,13 @@ class TestConversions:
|
|||||||
_limiter(lease_ttl_s=0)
|
_limiter(lease_ttl_s=0)
|
||||||
|
|
||||||
def test_unknown_source_rejected(self):
|
def test_unknown_source_rejected(self):
|
||||||
from polygateway.errors import GovernanceBackendError
|
"""未知源是装配缺陷,不是后端故障(issue #7 §3.4)。"""
|
||||||
|
from polygateway.errors import GatewayUnavailableError, SourceNotConfiguredError
|
||||||
|
|
||||||
with pytest.raises(GovernanceBackendError):
|
with pytest.raises(SourceNotConfiguredError) as ei:
|
||||||
_limiter()._cfg("nope")
|
_limiter()._cfg("nope")
|
||||||
|
# 关键: 若归入 scope 级家族,配置写错的任务会永远延期重投、永不进死信
|
||||||
|
assert not isinstance(ei.value, GatewayUnavailableError)
|
||||||
|
|
||||||
|
|
||||||
class TestLuaFidelity:
|
class TestLuaFidelity:
|
||||||
|
|||||||
Reference in New Issue
Block a user