feat: add failure-rate breaker channel with exponential reopen backoff

Dual-channel opening: consecutive failures (CHS-compatible, no streak
bump) plus windowed failure rate (two 30s buckets, min_calls guard).
429s bypass both channels as backpressure, and a probe hitting 429
releases instead of holding the lease. Open duration doubles per
rate/probe reopen up to max_cooldown_s, decaying after stable CLOSED.
record_success gains count_attempt so bad-result successes stay out of
the window.
This commit is contained in:
2026-07-21 09:08:14 -04:00
parent 72b25724d8
commit c7ccb5798c
4 changed files with 426 additions and 22 deletions
@@ -235,3 +235,95 @@ async def test_rpm_window_rollover_resets_quota(redis_client):
fresh = await limiter.try_acquire("s1", 0)
assert fresh is not None
await fresh.release()
# —— M2.5 双通道/退避变体(小 cooldown 配置控时长;语义与契约用例同源)——
_M25_CFG = BreakerConfig(
fail_threshold=100,
cooldown_s=4.0,
probe_ttl_s=8.0,
min_calls=4,
fail_rate=0.9,
window_s=8.0,
max_cooldown_s=16.0,
)
_M25_CONSEC_CFG = BreakerConfig(fail_threshold=3, cooldown_s=4.0, probe_ttl_s=8.0)
def _m25_gate(redis_client, cfg=_M25_CFG) -> RedisGate:
return RedisGate(config=cfg, redis=redis_client, scope=f"t{uuid4().hex[:8]}")
async def _fail_n(gate, n, reason="timeout"):
update = None
for _ in range(n):
entry = await gate.try_enter("s1", "w")
assert entry.allowed
update = await gate.record_failure(entry, reason, False)
return update
@pytestmark_slow
async def test_variant_backoff_doubles_and_caps(redis_client):
gate = _m25_gate(redis_client)
update = await _fail_n(gate, 4) # 率通道首开(4/4 ≥ 0.9): cooldown_eff = 4
assert update.state is GateState.OPEN
assert 0 < await gate.retry_after_s(("s1",)) <= 4.0
await asyncio.sleep(5)
probe = await gate.try_enter("s1", "w1")
assert probe.is_probe
await gate.record_failure(probe, "timeout", False) # streak 2 → 8
wait = await gate.retry_after_s(("s1",))
assert 4.0 < wait <= 8.0
await asyncio.sleep(9)
probe = await gate.try_enter("s1", "w1")
await gate.record_failure(probe, "timeout", False) # streak 3 → 16(封顶)
wait = await gate.retry_after_s(("s1",))
assert 8.0 < wait <= 16.0
@pytestmark_slow
async def test_variant_streak_survives_close_then_decays(redis_client):
gate = _m25_gate(redis_client)
await _fail_n(gate, 4) # streak 1(4s)
await asyncio.sleep(5)
probe = await gate.try_enter("s1", "w1")
await gate.record_failure(probe, "timeout", False) # streak 2(8s)
await asyncio.sleep(9)
probe = await gate.try_enter("s1", "w1")
await gate.record_success(probe) # 转 CLOSED,streak 不清零
await _fail_n(gate, 9) # 窗口 1 成功 + 9 失败 = 0.9 率开: streak 3 → 16
assert await gate.retry_after_s(("s1",)) > 8.0
await asyncio.sleep(17)
probe = await gate.try_enter("s1", "w1")
await gate.record_success(probe) # closed_since 落点
await asyncio.sleep(2 * 16.0 + 1)
entry = await gate.try_enter("s1", "w")
await gate.record_success(entry) # CLOSED 稳定期满 → streak 衰减归零
await _fail_n(gate, 9) # 再开回基础档
assert 0 < await gate.retry_after_s(("s1",)) <= 4.0
@pytestmark_slow
async def test_variant_consecutive_open_does_not_bump_streak(redis_client):
gate = _m25_gate(redis_client, _M25_CONSEC_CFG)
await _fail_n(gate, 3) # 连续通道开路(不递增 streak)
await asyncio.sleep(5)
probe = await gate.try_enter("s1", "w1")
await gate.record_success(probe)
await _fail_n(gate, 3) # 再次连续开路: 仍是基础档
assert 0 < await gate.retry_after_s(("s1",)) <= 4.0
@pytestmark_slow
async def test_variant_probe_rate_limited_releases_not_hangs(redis_client):
gate = _m25_gate(redis_client)
await _fail_n(gate, 4)
await asyncio.sleep(5)
probe = await gate.try_enter("s1", "w1")
assert probe.is_probe
update = await gate.record_failure(probe, "rate_limited", False)
assert update.applied
nxt = await gate.try_enter("s1", "w2")
assert nxt.allowed and nxt.is_probe # 立即可再探,不等 probe_ttl