fix: reparent governance backend failures under GatewayUnavailableError (issue #7)
A fail-closed limiter or breaker backend means the scope cannot emit a single request, which is exactly scope-level unavailability. But the error sat directly under PolyGatewayError, so a caller writing only `except GatewayUnavailableError` dropped it into the catch-all branch: Redis blips once and a backlog of tasks burns its business failure budget into the dead letter queue, over a fault a restart would clear. Three gate paths leak to callers rather than being absorbed by _record_quietly (try_acquire, try_enter, progress_age_s); each is now pinned by a test, since none of them had one before. The two unknown-source sites move to SourceNotConfiguredError instead of following along. They report a misconfigured source name, not an outage, and letting them into the retryable family would be the mirror of the bug being fixed here: the task would retry forever and never surface.
This commit is contained in:
@@ -14,8 +14,10 @@ if TYPE_CHECKING:
|
||||
class BreakerGate:
|
||||
"""RetryMW 面向熔断后端的唯一入口;包装一切后端异常。"""
|
||||
|
||||
def __init__(self, gate: ProviderGate) -> None:
|
||||
def __init__(self, gate: ProviderGate, *, scope: str) -> None:
|
||||
self._gate = gate
|
||||
# 后端故障即 scope 级不可用,异常须携 scope 供调用方定位(issue #7 §3.3)
|
||||
self._scope = scope
|
||||
|
||||
async def try_enter(self, source: SourceConfig, owner: str) -> GateDecision:
|
||||
try:
|
||||
@@ -23,7 +25,7 @@ class BreakerGate:
|
||||
except GovernanceBackendError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise GovernanceBackendError(f"熔断后端故障(try_enter): {exc}") from exc
|
||||
raise GovernanceBackendError(f"熔断后端故障(try_enter): {exc}", scope=self._scope) from exc
|
||||
|
||||
async def record_success(
|
||||
self, entry: GateDecision, *, count_attempt: bool = True
|
||||
@@ -33,7 +35,7 @@ class BreakerGate:
|
||||
except GovernanceBackendError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise GovernanceBackendError(f"熔断后端故障(record_success): {exc}") from exc
|
||||
raise GovernanceBackendError(f"熔断后端故障(record_success): {exc}", scope=self._scope) from exc
|
||||
|
||||
async def record_failure(
|
||||
self, entry: GateDecision, reason: str, force_open: bool
|
||||
@@ -43,7 +45,7 @@ class BreakerGate:
|
||||
except GovernanceBackendError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise GovernanceBackendError(f"熔断后端故障(record_failure): {exc}") from exc
|
||||
raise GovernanceBackendError(f"熔断后端故障(record_failure): {exc}", scope=self._scope) from exc
|
||||
|
||||
async def release_probe(self, entry: GateDecision) -> GateUpdate:
|
||||
try:
|
||||
@@ -51,7 +53,7 @@ class BreakerGate:
|
||||
except GovernanceBackendError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise GovernanceBackendError(f"熔断后端故障(release_probe): {exc}") from exc
|
||||
raise GovernanceBackendError(f"熔断后端故障(release_probe): {exc}", scope=self._scope) from exc
|
||||
|
||||
async def retry_after_s(self, sources: tuple[str, ...]) -> float:
|
||||
try:
|
||||
@@ -59,4 +61,4 @@ class BreakerGate:
|
||||
except GovernanceBackendError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise GovernanceBackendError(f"熔断后端故障(retry_after_s): {exc}") from exc
|
||||
raise GovernanceBackendError(f"熔断后端故障(retry_after_s): {exc}", scope=self._scope) from exc
|
||||
|
||||
@@ -18,8 +18,10 @@ if TYPE_CHECKING:
|
||||
class QuotaGate:
|
||||
"""RetryMW 面向限流后端的唯一入口;包装一切后端异常。"""
|
||||
|
||||
def __init__(self, limiter: RateLimiter) -> None:
|
||||
def __init__(self, limiter: RateLimiter, *, scope: str) -> None:
|
||||
self._limiter = limiter
|
||||
# 后端故障即 scope 级不可用,异常须携 scope 供调用方定位(issue #7 §3.3)
|
||||
self._scope = scope
|
||||
|
||||
async def try_acquire(self, source: SourceConfig) -> Permit | None:
|
||||
try:
|
||||
@@ -27,7 +29,7 @@ class QuotaGate:
|
||||
except GovernanceBackendError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise GovernanceBackendError(f"限流后端故障(try_acquire): {exc}") from exc
|
||||
raise GovernanceBackendError(f"限流后端故障(try_acquire): {exc}", scope=self._scope) from exc
|
||||
|
||||
async def stats(self, source: SourceConfig) -> SourceStats:
|
||||
try:
|
||||
@@ -35,7 +37,7 @@ class QuotaGate:
|
||||
except GovernanceBackendError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise GovernanceBackendError(f"限流后端故障(source_stats): {exc}") from exc
|
||||
raise GovernanceBackendError(f"限流后端故障(source_stats): {exc}", scope=self._scope) from exc
|
||||
|
||||
async def mark_progress(self) -> None:
|
||||
try:
|
||||
@@ -43,7 +45,7 @@ class QuotaGate:
|
||||
except GovernanceBackendError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise GovernanceBackendError(f"限流后端故障(mark_progress): {exc}") from exc
|
||||
raise GovernanceBackendError(f"限流后端故障(mark_progress): {exc}", scope=self._scope) from exc
|
||||
|
||||
async def progress_age_s(self) -> float:
|
||||
try:
|
||||
@@ -51,4 +53,4 @@ class QuotaGate:
|
||||
except GovernanceBackendError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise GovernanceBackendError(f"限流后端故障(progress_age_s): {exc}") from exc
|
||||
raise GovernanceBackendError(f"限流后端故障(progress_age_s): {exc}", scope=self._scope) from exc
|
||||
|
||||
@@ -183,8 +183,8 @@ class RetryMW:
|
||||
self._scope = scope
|
||||
self._sources = list(sources)
|
||||
self._selector = selector
|
||||
self._quota = QuotaGate(limiter)
|
||||
self._breaker = BreakerGate(gate)
|
||||
self._quota = QuotaGate(limiter, scope=self._scope)
|
||||
self._breaker = BreakerGate(gate, scope=self._scope)
|
||||
self._transport = transport
|
||||
self._retry = retry
|
||||
self._bp = backpressure
|
||||
|
||||
Reference in New Issue
Block a user