feat: add failure-rate breaker channel with exponential reopen backoff

Dual-channel opening: consecutive failures (CHS-compatible, no streak bump) plus windowed failure rate (two 30s buckets, min_calls guard). 429s bypass both channels as backpressure, and a probe hitting 429 releases instead of holding the lease. Open duration doubles per rate/probe reopen up to max_cooldown_s, decaying after stable CLOSED. record_success gains count_attempt so bad-result successes stay out of the window.
2026-07-21 09:08:14 -04:00
parent 72b25724d8
commit c7ccb5798c
4 changed files with 426 additions and 22 deletions
@@ -133,6 +133,147 @@ class TestEpochFencing:
        assert not (await gate.record_success(stale_probe)).applied


+_RATE_CFG = BreakerConfig(
+    fail_threshold=100,  # 连续通道抬高失声,单测率通道
+    cooldown_s=60.0,
+    probe_ttl_s=120.0,
+    min_calls=4,
+    fail_rate=0.9,
+    window_s=60.0,
+    max_cooldown_s=240.0,
+)
+
+
+async def _fail(gate, source="s1", reason="timeout", n=1):
+    update = None
+    for _ in range(n):
+        entry = await gate.try_enter(source, "w")
+        assert entry.allowed
+        update = await gate.record_failure(entry, reason, False)
+    return update
+
+
+class TestRateChannel:
+    """M2.5 失败率通道(设计 §3.1;全部确定性序列)。"""
+
+    async def test_rate_opens_at_min_calls(self, gate_factory):
+        # 病灶 1 回归: 高失败率源在 min_calls 样本处开路(连续通道静默)
+        gate = gate_factory(_RATE_CFG)
+        await _fail(gate, n=3)  # attempts 3 < min_calls 4 → 仍 CLOSED
+        assert (await gate.try_enter("s1", "w")).allowed
+        update = await _fail(gate, n=1)  # attempts 4, 失败率 1.0 ≥ 0.9
+        assert update.state is GateState.OPEN
+        assert not (await gate.try_enter("s1", "w")).allowed
+
+    async def test_below_min_calls_never_rate_opens(self, gate_factory):
+        gate = gate_factory(_RATE_CFG)
+        await _fail(gate, n=3)
+        assert (await gate.try_enter("s1", "w")).allowed
+
+    async def test_real_success_dilutes_window(self, gate_factory):
+        # 真实成功计入 attempts: 3 失败 + 1 成功 + 1 失败 = 4/5 = 0.8 < 0.9 不开
+        gate = gate_factory(_RATE_CFG)
+        await _fail(gate, n=3)
+        entry = await gate.try_enter("s1", "w")
+        await gate.record_success(entry)
+        update = await _fail(gate, n=1)
+        assert update.state is GateState.CLOSED
+        assert (await gate.try_enter("s1", "w")).allowed
+
+    async def test_result_invalid_not_counted(self, gate_factory):
+        # 坏结果 ≠ 坏服务: count_attempt=False 完全不动窗口 →
+        # 3 失败 + 1 不计成功 + 1 失败 = attempts 4, 失败率 1.0 → 开路
+        gate = gate_factory(_RATE_CFG)
+        await _fail(gate, n=3)
+        entry = await gate.try_enter("s1", "w")
+        await gate.record_success(entry, count_attempt=False)
+        update = await _fail(gate, n=1)
+        assert update.state is GateState.OPEN
+
+    async def test_rate_limited_bypasses_both_channels(self, gate_factory):
+        # C1 对抗: 429 是背压不是故障——连续通道(阈值 3)与率通道都不吃
+        cfg = BreakerConfig(fail_threshold=3, cooldown_s=60.0, probe_ttl_s=120.0, min_calls=4)
+        gate = gate_factory(cfg)
+        await _fail(gate, reason="rate_limited", n=10)
+        assert (await gate.try_enter("s1", "w")).allowed
+        # timeout 脉冲照常走连续通道开路
+        update = await _fail(gate, reason="timeout", n=3)
+        assert update.state is GateState.OPEN
+
+    async def test_probe_rate_limited_releases_not_hangs(self, gate_factory, clock):
+        # 探针撞 429: 非故障证据也非成功——按无果归还语义放下家接管,
+        # 不得把探针租约挂到 TTL(否则源被锁死 probe_ttl_s)
+        gate = gate_factory(_RATE_CFG)
+        await _fail(gate, n=4)
+        clock.advance(61)
+        probe = await gate.try_enter("s1", "w1")
+        assert probe.is_probe
+        update = await gate.record_failure(probe, "rate_limited", False)
+        assert update.applied
+        nxt = await gate.try_enter("s1", "w2")
+        assert nxt.allowed and nxt.is_probe  # 立即可再探,而非等 probe_ttl
+
+    async def test_rate_open_failure_count_capped(self, gate_factory):
+        gate = gate_factory(_RATE_CFG)
+        await _fail(gate, n=3)
+        update = await _fail(gate, n=1)
+        assert update.state is GateState.OPEN
+        assert update.failure_count == _RATE_CFG.fail_threshold  # 顶格语义沿用
+
+
+class TestReopenBackoff:
+    """开路时长指数递增与衰减(设计 §3.1;memory 假时钟,redis 走真实等待变体)。"""
+
+    async def test_backoff_doubles_and_caps(self, gate_factory, clock):
+        gate = gate_factory(_RATE_CFG)
+        await _fail(gate, n=4)  # 率通道首开: cooldown_eff = 60
+        assert 0 < await gate.retry_after_s(("s1",)) <= 60.0
+        clock.advance(61)
+        probe = await gate.try_enter("s1", "w1")
+        assert probe.is_probe
+        await gate.record_failure(probe, "timeout", False)  # 探针失败重开: streak 2 → 120
+        wait = await gate.retry_after_s(("s1",))
+        assert 60.0 < wait <= 120.0
+        clock.advance(121)
+        probe = await gate.try_enter("s1", "w1")
+        await gate.record_failure(probe, "timeout", False)  # streak 3 → 240(封顶)
+        wait = await gate.retry_after_s(("s1",))
+        assert 120.0 < wait <= 240.0
+
+    async def test_streak_survives_close_then_decays(self, gate_factory, clock):
+        gate = gate_factory(_RATE_CFG)
+        await _fail(gate, n=4)
+        clock.advance(61)
+        probe = await gate.try_enter("s1", "w1")
+        await gate.record_failure(probe, "timeout", False)  # streak 2
+        clock.advance(121)
+        probe = await gate.try_enter("s1", "w1")
+        await gate.record_success(probe)  # 转 CLOSED,streak 不清零
+        # 窗口含探针成功 1 次,9 失败 → 9/10 = 0.9 率开;streak 递增至封顶
+        await _fail(gate, n=9)
+        assert await gate.retry_after_s(("s1",)) > 60.0
+        # 衰减: CLOSED 稳定 2×cooldown_eff 后首次 record_success 归零
+        clock.advance(241)
+        probe = await gate.try_enter("s1", "w1")
+        await gate.record_success(probe)
+        clock.advance(2 * 240.0 + 1)
+        entry = await gate.try_enter("s1", "w")
+        await gate.record_success(entry)  # 触发衰减
+        await _fail(gate, n=9)  # 再开(9/10)回到基础档
+        assert 0 < await gate.retry_after_s(("s1",)) <= 60.0
+
+    async def test_consecutive_open_does_not_bump_streak(self, gate_factory, clock):
+        # C1 对抗: 连续通道误熔健康源的代价封顶为单次 cooldown_s
+        cfg = BreakerConfig(fail_threshold=3, cooldown_s=60.0, probe_ttl_s=120.0)
+        gate = gate_factory(cfg)
+        await _fail(gate, reason="timeout", n=3)  # 连续通道开路
+        clock.advance(61)
+        probe = await gate.try_enter("s1", "w1")
+        await gate.record_success(probe)  # 恢复
+        await _fail(gate, reason="timeout", n=3)  # 再次连续开路
+        assert 0 < await gate.retry_after_s(("s1",)) <= 60.0  # 无翻倍
+
+
 class TestRetryAfter:
    async def test_retry_after_semantics(self, gate_factory, clock):
        gate = gate_factory(_CFG)
@@ -235,3 +235,95 @@ async def test_rpm_window_rollover_resets_quota(redis_client):
    fresh = await limiter.try_acquire("s1", 0)
    assert fresh is not None
    await fresh.release()
+
+
+# —— M2.5 双通道/退避变体(小 cooldown 配置控时长;语义与契约用例同源)——
+
+_M25_CFG = BreakerConfig(
+    fail_threshold=100,
+    cooldown_s=4.0,
+    probe_ttl_s=8.0,
+    min_calls=4,
+    fail_rate=0.9,
+    window_s=8.0,
+    max_cooldown_s=16.0,
+)
+_M25_CONSEC_CFG = BreakerConfig(fail_threshold=3, cooldown_s=4.0, probe_ttl_s=8.0)
+
+
+def _m25_gate(redis_client, cfg=_M25_CFG) -> RedisGate:
+    return RedisGate(config=cfg, redis=redis_client, scope=f"t{uuid4().hex[:8]}")
+
+
+async def _fail_n(gate, n, reason="timeout"):
+    update = None
+    for _ in range(n):
+        entry = await gate.try_enter("s1", "w")
+        assert entry.allowed
+        update = await gate.record_failure(entry, reason, False)
+    return update
+
+
+@pytestmark_slow
+async def test_variant_backoff_doubles_and_caps(redis_client):
+    gate = _m25_gate(redis_client)
+    update = await _fail_n(gate, 4)  # 率通道首开(4/4 ≥ 0.9): cooldown_eff = 4
+    assert update.state is GateState.OPEN
+    assert 0 < await gate.retry_after_s(("s1",)) <= 4.0
+    await asyncio.sleep(5)
+    probe = await gate.try_enter("s1", "w1")
+    assert probe.is_probe
+    await gate.record_failure(probe, "timeout", False)  # streak 2 → 8
+    wait = await gate.retry_after_s(("s1",))
+    assert 4.0 < wait <= 8.0
+    await asyncio.sleep(9)
+    probe = await gate.try_enter("s1", "w1")
+    await gate.record_failure(probe, "timeout", False)  # streak 3 → 16(封顶)
+    wait = await gate.retry_after_s(("s1",))
+    assert 8.0 < wait <= 16.0
+
+
+@pytestmark_slow
+async def test_variant_streak_survives_close_then_decays(redis_client):
+    gate = _m25_gate(redis_client)
+    await _fail_n(gate, 4)  # streak 1(4s)
+    await asyncio.sleep(5)
+    probe = await gate.try_enter("s1", "w1")
+    await gate.record_failure(probe, "timeout", False)  # streak 2(8s)
+    await asyncio.sleep(9)
+    probe = await gate.try_enter("s1", "w1")
+    await gate.record_success(probe)  # 转 CLOSED,streak 不清零
+    await _fail_n(gate, 9)  # 窗口 1 成功 + 9 失败 = 0.9 率开: streak 3 → 16
+    assert await gate.retry_after_s(("s1",)) > 8.0
+    await asyncio.sleep(17)
+    probe = await gate.try_enter("s1", "w1")
+    await gate.record_success(probe)  # closed_since 落点
+    await asyncio.sleep(2 * 16.0 + 1)
+    entry = await gate.try_enter("s1", "w")
+    await gate.record_success(entry)  # CLOSED 稳定期满 → streak 衰减归零
+    await _fail_n(gate, 9)  # 再开回基础档
+    assert 0 < await gate.retry_after_s(("s1",)) <= 4.0
+
+
+@pytestmark_slow
+async def test_variant_consecutive_open_does_not_bump_streak(redis_client):
+    gate = _m25_gate(redis_client, _M25_CONSEC_CFG)
+    await _fail_n(gate, 3)  # 连续通道开路(不递增 streak)
+    await asyncio.sleep(5)
+    probe = await gate.try_enter("s1", "w1")
+    await gate.record_success(probe)
+    await _fail_n(gate, 3)  # 再次连续开路: 仍是基础档
+    assert 0 < await gate.retry_after_s(("s1",)) <= 4.0
+
+
+@pytestmark_slow
+async def test_variant_probe_rate_limited_releases_not_hangs(redis_client):
+    gate = _m25_gate(redis_client)
+    await _fail_n(gate, 4)
+    await asyncio.sleep(5)
+    probe = await gate.try_enter("s1", "w1")
+    assert probe.is_probe
+    update = await gate.record_failure(probe, "rate_limited", False)
+    assert update.applied
+    nxt = await gate.try_enter("s1", "w2")
+    assert nxt.allowed and nxt.is_probe  # 立即可再探,不等 probe_ttl