feat: add failure-rate breaker channel with exponential reopen backoff
Dual-channel opening: consecutive failures (CHS-compatible, no streak bump) plus windowed failure rate (two 30s buckets, min_calls guard). 429s bypass both channels as backpressure, and a probe hitting 429 releases instead of holding the lease. Open duration doubles per rate/probe reopen up to max_cooldown_s, decaying after stable CLOSED. record_success gains count_attempt so bad-result successes stay out of the window.
This commit is contained in:
@@ -235,3 +235,95 @@ async def test_rpm_window_rollover_resets_quota(redis_client):
|
||||
fresh = await limiter.try_acquire("s1", 0)
|
||||
assert fresh is not None
|
||||
await fresh.release()
|
||||
|
||||
|
||||
# —— M2.5 双通道/退避变体(小 cooldown 配置控时长;语义与契约用例同源)——
|
||||
|
||||
_M25_CFG = BreakerConfig(
|
||||
fail_threshold=100,
|
||||
cooldown_s=4.0,
|
||||
probe_ttl_s=8.0,
|
||||
min_calls=4,
|
||||
fail_rate=0.9,
|
||||
window_s=8.0,
|
||||
max_cooldown_s=16.0,
|
||||
)
|
||||
_M25_CONSEC_CFG = BreakerConfig(fail_threshold=3, cooldown_s=4.0, probe_ttl_s=8.0)
|
||||
|
||||
|
||||
def _m25_gate(redis_client, cfg=_M25_CFG) -> RedisGate:
|
||||
return RedisGate(config=cfg, redis=redis_client, scope=f"t{uuid4().hex[:8]}")
|
||||
|
||||
|
||||
async def _fail_n(gate, n, reason="timeout"):
|
||||
update = None
|
||||
for _ in range(n):
|
||||
entry = await gate.try_enter("s1", "w")
|
||||
assert entry.allowed
|
||||
update = await gate.record_failure(entry, reason, False)
|
||||
return update
|
||||
|
||||
|
||||
@pytestmark_slow
|
||||
async def test_variant_backoff_doubles_and_caps(redis_client):
|
||||
gate = _m25_gate(redis_client)
|
||||
update = await _fail_n(gate, 4) # 率通道首开(4/4 ≥ 0.9): cooldown_eff = 4
|
||||
assert update.state is GateState.OPEN
|
||||
assert 0 < await gate.retry_after_s(("s1",)) <= 4.0
|
||||
await asyncio.sleep(5)
|
||||
probe = await gate.try_enter("s1", "w1")
|
||||
assert probe.is_probe
|
||||
await gate.record_failure(probe, "timeout", False) # streak 2 → 8
|
||||
wait = await gate.retry_after_s(("s1",))
|
||||
assert 4.0 < wait <= 8.0
|
||||
await asyncio.sleep(9)
|
||||
probe = await gate.try_enter("s1", "w1")
|
||||
await gate.record_failure(probe, "timeout", False) # streak 3 → 16(封顶)
|
||||
wait = await gate.retry_after_s(("s1",))
|
||||
assert 8.0 < wait <= 16.0
|
||||
|
||||
|
||||
@pytestmark_slow
|
||||
async def test_variant_streak_survives_close_then_decays(redis_client):
|
||||
gate = _m25_gate(redis_client)
|
||||
await _fail_n(gate, 4) # streak 1(4s)
|
||||
await asyncio.sleep(5)
|
||||
probe = await gate.try_enter("s1", "w1")
|
||||
await gate.record_failure(probe, "timeout", False) # streak 2(8s)
|
||||
await asyncio.sleep(9)
|
||||
probe = await gate.try_enter("s1", "w1")
|
||||
await gate.record_success(probe) # 转 CLOSED,streak 不清零
|
||||
await _fail_n(gate, 9) # 窗口 1 成功 + 9 失败 = 0.9 率开: streak 3 → 16
|
||||
assert await gate.retry_after_s(("s1",)) > 8.0
|
||||
await asyncio.sleep(17)
|
||||
probe = await gate.try_enter("s1", "w1")
|
||||
await gate.record_success(probe) # closed_since 落点
|
||||
await asyncio.sleep(2 * 16.0 + 1)
|
||||
entry = await gate.try_enter("s1", "w")
|
||||
await gate.record_success(entry) # CLOSED 稳定期满 → streak 衰减归零
|
||||
await _fail_n(gate, 9) # 再开回基础档
|
||||
assert 0 < await gate.retry_after_s(("s1",)) <= 4.0
|
||||
|
||||
|
||||
@pytestmark_slow
|
||||
async def test_variant_consecutive_open_does_not_bump_streak(redis_client):
|
||||
gate = _m25_gate(redis_client, _M25_CONSEC_CFG)
|
||||
await _fail_n(gate, 3) # 连续通道开路(不递增 streak)
|
||||
await asyncio.sleep(5)
|
||||
probe = await gate.try_enter("s1", "w1")
|
||||
await gate.record_success(probe)
|
||||
await _fail_n(gate, 3) # 再次连续开路: 仍是基础档
|
||||
assert 0 < await gate.retry_after_s(("s1",)) <= 4.0
|
||||
|
||||
|
||||
@pytestmark_slow
|
||||
async def test_variant_probe_rate_limited_releases_not_hangs(redis_client):
|
||||
gate = _m25_gate(redis_client)
|
||||
await _fail_n(gate, 4)
|
||||
await asyncio.sleep(5)
|
||||
probe = await gate.try_enter("s1", "w1")
|
||||
assert probe.is_probe
|
||||
update = await gate.record_failure(probe, "rate_limited", False)
|
||||
assert update.applied
|
||||
nxt = await gate.try_enter("s1", "w2")
|
||||
assert nxt.allowed and nxt.is_probe # 立即可再探,不等 probe_ttl
|
||||
|
||||
Reference in New Issue
Block a user