feat: add failure-rate breaker channel with exponential reopen backoff
Dual-channel opening: consecutive failures (CHS-compatible, no streak bump) plus windowed failure rate (two 30s buckets, min_calls guard). 429s bypass both channels as backpressure, and a probe hitting 429 releases instead of holding the lease. Open duration doubles per rate/probe reopen up to max_cooldown_s, decaying after stable CLOSED. record_success gains count_attempt so bad-result successes stay out of the window.
This commit is contained in:
@@ -4,6 +4,12 @@
|
||||
契约形态承 CHS `provider_gate.py`: 半开探针是**带 TTL 的租约**(持有者
|
||||
死亡后可被接管,防"探针永远在路上"死锁),写回经 epoch fencing 拒绝
|
||||
旧世代污染。epoch 在每次进入 OPEN 时递增。时钟构造注入,纯确定性可测。
|
||||
|
||||
M2.5 双通道(设计 2026-07-21-m25 §3.1): 在连续失败通道之外加失败率
|
||||
通道(双 30s 桶窗口,样本 ≥ min_calls 且失败率 ≥ fail_rate 即开路);
|
||||
429(rate_limited)是背压不是故障,两通道均不计;开路时长按 reopen_streak
|
||||
指数递增封顶 max_cooldown_s——连续通道触发的开路不递增 streak(误熔
|
||||
健康源的代价封顶为单次 cooldown_s)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -22,7 +28,7 @@ if TYPE_CHECKING:
|
||||
|
||||
@dataclass
|
||||
class _SourceGate:
|
||||
"""单源的门控可变状态。"""
|
||||
"""单源的门控可变状态(窗口/退避域见设计 §3.1 状态机域清单)。"""
|
||||
|
||||
state: GateState = GateState.CLOSED
|
||||
fails: int = 0
|
||||
@@ -31,6 +37,14 @@ class _SourceGate:
|
||||
probe_owner: str | None = None
|
||||
probe_expires: float = 0.0
|
||||
reasons: dict[str, str] = field(default_factory=dict)
|
||||
# 失败率窗口: 双半窗桶轮换(win_id = now // (window_s/2))
|
||||
win_id: int = -1
|
||||
a0: int = 0 # 当前桶 attempts / failures
|
||||
f0: int = 0
|
||||
a1: int = 0 # 上一桶
|
||||
f1: int = 0
|
||||
reopen_streak: int = 0
|
||||
closed_since: float | None = None
|
||||
|
||||
|
||||
class InMemoryGate:
|
||||
@@ -46,6 +60,38 @@ class InMemoryGate:
|
||||
raise ValueError("source_name 不能为空")
|
||||
return self._gates.setdefault(source_name, _SourceGate())
|
||||
|
||||
# —— M2.5 失败率窗口 ——
|
||||
|
||||
def _rotate_window(self, g: _SourceGate) -> None:
|
||||
"""按半窗粒度轮换双桶;跨两桶以上的空窗直接清零。"""
|
||||
half = self._cfg.window_s / 2.0
|
||||
wid = int(self._now() // half)
|
||||
if wid == g.win_id:
|
||||
return
|
||||
if wid == g.win_id + 1:
|
||||
g.a1, g.f1 = g.a0, g.f0
|
||||
else:
|
||||
g.a1, g.f1 = 0, 0
|
||||
g.a0, g.f0 = 0, 0
|
||||
g.win_id = wid
|
||||
|
||||
def _window_add(self, g: _SourceGate, *, failed: bool) -> None:
|
||||
self._rotate_window(g)
|
||||
g.a0 += 1
|
||||
if failed:
|
||||
g.f0 += 1
|
||||
|
||||
def _rate_channel_open(self, g: _SourceGate) -> bool:
|
||||
self._rotate_window(g)
|
||||
attempts = g.a0 + g.a1
|
||||
if attempts < self._cfg.min_calls:
|
||||
return False
|
||||
return (g.f0 + g.f1) / attempts >= self._cfg.fail_rate
|
||||
|
||||
def _cooldown_eff(self, g: _SourceGate) -> float:
|
||||
streak = max(1, g.reopen_streak)
|
||||
return min(self._cfg.cooldown_s * (2 ** (streak - 1)), self._cfg.max_cooldown_s)
|
||||
|
||||
def _grant_probe(self, g: _SourceGate, source_name: str, owner: str) -> GateDecision:
|
||||
g.state = GateState.HALF_OPEN
|
||||
g.probe_owner = owner
|
||||
@@ -123,18 +169,35 @@ class InMemoryGate:
|
||||
else 0.0,
|
||||
)
|
||||
|
||||
def _open(self, g: _SourceGate, reason: str) -> None:
|
||||
def _open(self, g: _SourceGate, reason: str, *, bump_streak: bool) -> None:
|
||||
if bump_streak:
|
||||
g.reopen_streak += 1
|
||||
g.state = GateState.OPEN
|
||||
g.epoch += 1 # 世代推进: 旧 entry 的迟到写回自此被 fencing 拒绝
|
||||
g.open_until = self._now() + self._cfg.cooldown_s
|
||||
g.open_until = self._now() + self._cooldown_eff(g)
|
||||
g.fails = max(g.fails, self._cfg.fail_threshold)
|
||||
g.probe_owner = None
|
||||
g.probe_expires = 0.0
|
||||
g.closed_since = None
|
||||
|
||||
async def record_success(self, entry: GateDecision) -> GateUpdate:
|
||||
async def record_success(
|
||||
self, entry: GateDecision, *, count_attempt: bool = True
|
||||
) -> GateUpdate:
|
||||
g = self._gate(entry.source_name)
|
||||
if not self._fenced(g, entry):
|
||||
return self._snapshot(g, applied=False)
|
||||
was_probe = entry.is_probe
|
||||
if count_attempt:
|
||||
self._window_add(g, failed=False)
|
||||
# streak 衰减: CLOSED 稳定满 2×cooldown_eff 后的首次成功归零(设计 §3.1)
|
||||
if (
|
||||
g.reopen_streak > 0
|
||||
and g.closed_since is not None
|
||||
and self._now() - g.closed_since >= 2 * self._cooldown_eff(g)
|
||||
):
|
||||
g.reopen_streak = 0
|
||||
if was_probe:
|
||||
g.closed_since = self._now() # 仅探针转 CLOSED 时写,普通成功不刷新
|
||||
g.state = GateState.CLOSED
|
||||
g.fails = 0
|
||||
g.probe_owner = None
|
||||
@@ -147,12 +210,25 @@ class InMemoryGate:
|
||||
g = self._gate(entry.source_name)
|
||||
if not self._fenced(g, entry):
|
||||
return self._snapshot(g, applied=False)
|
||||
if entry.is_probe or force_open:
|
||||
self._open(g, reason) # 探针失败重开 / SourceDead 一击即熔
|
||||
if reason == "rate_limited" and not force_open:
|
||||
# 429 = 背压不是故障(设计 §3.1): 两通道均不计,选源层软处理;
|
||||
# 探针撞 429 按无果归还语义放下家,不挂租约
|
||||
if entry.is_probe:
|
||||
g.state = GateState.OPEN
|
||||
g.open_until = self._now()
|
||||
g.probe_owner = None
|
||||
g.probe_expires = 0.0
|
||||
return self._snapshot(g, applied=True)
|
||||
if entry.is_probe or force_open:
|
||||
# 探针失败重开递增 streak;SourceDead 一击即熔不递增
|
||||
self._open(g, reason, bump_streak=entry.is_probe)
|
||||
return self._snapshot(g, applied=True)
|
||||
self._window_add(g, failed=True)
|
||||
g.fails += 1
|
||||
if g.fails >= self._cfg.fail_threshold:
|
||||
self._open(g, reason)
|
||||
if self._rate_channel_open(g):
|
||||
self._open(g, reason, bump_streak=True)
|
||||
elif g.fails >= self._cfg.fail_threshold:
|
||||
self._open(g, reason, bump_streak=False) # 连续通道不递增(C1 封顶)
|
||||
return self._snapshot(g, applied=True)
|
||||
|
||||
async def release_probe(self, entry: GateDecision) -> GateUpdate:
|
||||
|
||||
Reference in New Issue
Block a user