370 lines
17 KiB
Python
370 lines
17 KiB
Python
"""熔断后端契约测试(状态机 + 半开单探针租约 + epoch fencing)。
|
||
|
||
蓝本: VT adapters/breaker.py 状态机语义 + CHS provider_gate.py 契约;
|
||
M2 的 Redis 实现复用本套件。
|
||
"""
|
||
|
||
import pytest
|
||
|
||
from polygateway.ports import GateState
|
||
from polygateway.types import BreakerConfig
|
||
|
||
_CFG = BreakerConfig(fail_threshold=3, cooldown_s=60.0, probe_ttl_s=120.0)
|
||
|
||
|
||
async def _open_gate(gate, source="s1"):
|
||
"""连续失败到阈值,打开熔断,返回最后一次 Update。"""
|
||
update = None
|
||
for _ in range(_CFG.fail_threshold):
|
||
entry = await gate.try_enter(source, "w")
|
||
assert entry.allowed
|
||
update = await gate.record_failure(entry, "network_error", False)
|
||
assert update.state is GateState.OPEN
|
||
return update
|
||
|
||
|
||
class TestStateMachine:
|
||
async def test_closed_admits_and_success_resets(self, gate_factory):
|
||
gate = gate_factory(_CFG)
|
||
entry = await gate.try_enter("s1", "w")
|
||
assert entry.allowed and entry.state is GateState.CLOSED and not entry.is_probe
|
||
update = await gate.record_success(entry)
|
||
assert update.applied and update.failure_count == 0
|
||
|
||
async def test_threshold_opens_and_rejects(self, gate_factory):
|
||
gate = gate_factory(_CFG)
|
||
await _open_gate(gate)
|
||
entry = await gate.try_enter("s1", "w")
|
||
assert not entry.allowed and entry.state is GateState.OPEN
|
||
assert entry.retry_after_s > 0
|
||
|
||
async def test_success_before_threshold_resets_count(self, gate_factory):
|
||
gate = gate_factory(_CFG)
|
||
for _ in range(_CFG.fail_threshold - 1):
|
||
entry = await gate.try_enter("s1", "w")
|
||
await gate.record_failure(entry, "timeout", False)
|
||
entry = await gate.try_enter("s1", "w")
|
||
update = await gate.record_success(entry)
|
||
assert update.applied and update.failure_count == 0
|
||
assert (await gate.try_enter("s1", "w")).allowed
|
||
|
||
async def test_force_open_single_strike(self, gate_factory):
|
||
gate = gate_factory(_CFG)
|
||
entry = await gate.try_enter("s1", "w")
|
||
update = await gate.record_failure(entry, "source_dead", True)
|
||
assert update.state is GateState.OPEN
|
||
assert not (await gate.try_enter("s1", "w")).allowed
|
||
|
||
|
||
class TestHalfOpenProbe:
|
||
async def test_cooldown_grants_single_probe(self, gate_factory, clock):
|
||
gate = gate_factory(_CFG)
|
||
await _open_gate(gate)
|
||
clock.advance(_CFG.cooldown_s + 1)
|
||
probe = await gate.try_enter("s1", "w1")
|
||
assert probe.allowed and probe.is_probe and probe.probe_owner == "w1"
|
||
# 第二个进入者被拒(防惊群)
|
||
second = await gate.try_enter("s1", "w2")
|
||
assert not second.allowed and second.state is GateState.HALF_OPEN
|
||
|
||
async def test_probe_success_closes(self, gate_factory, clock):
|
||
gate = gate_factory(_CFG)
|
||
await _open_gate(gate)
|
||
clock.advance(_CFG.cooldown_s + 1)
|
||
probe = await gate.try_enter("s1", "w1")
|
||
update = await gate.record_success(probe)
|
||
assert update.applied and update.state is GateState.CLOSED
|
||
assert (await gate.try_enter("s1", "w2")).allowed
|
||
|
||
async def test_probe_failure_reopens(self, gate_factory, clock):
|
||
gate = gate_factory(_CFG)
|
||
await _open_gate(gate)
|
||
clock.advance(_CFG.cooldown_s + 1)
|
||
probe = await gate.try_enter("s1", "w1")
|
||
update = await gate.record_failure(probe, "network_error", False)
|
||
assert update.state is GateState.OPEN
|
||
assert not (await gate.try_enter("s1", "w2")).allowed
|
||
|
||
async def test_probe_lease_expiry_allows_takeover(self, gate_factory, clock):
|
||
"""探针持有者死亡 → 租约过期后新 caller 接管探针,防死锁。"""
|
||
gate = gate_factory(_CFG)
|
||
await _open_gate(gate)
|
||
clock.advance(_CFG.cooldown_s + 1)
|
||
stale = await gate.try_enter("s1", "dead-worker")
|
||
assert stale.is_probe
|
||
clock.advance(_CFG.probe_ttl_s + 1)
|
||
takeover = await gate.try_enter("s1", "w2")
|
||
assert takeover.allowed and takeover.is_probe and takeover.probe_owner == "w2"
|
||
|
||
async def test_release_probe_hands_back_and_idempotent(self, gate_factory, clock):
|
||
gate = gate_factory(_CFG)
|
||
await _open_gate(gate)
|
||
clock.advance(_CFG.cooldown_s + 1)
|
||
probe = await gate.try_enter("s1", "w1")
|
||
update = await gate.release_probe(probe)
|
||
assert update.applied
|
||
# 幂等: 第二次释放不再生效
|
||
assert not (await gate.release_probe(probe)).applied
|
||
# 源保持可接管状态: 下一 caller 拿到探针
|
||
nxt = await gate.try_enter("s1", "w2")
|
||
assert nxt.allowed and nxt.is_probe
|
||
|
||
|
||
class TestEpochFencing:
|
||
async def test_stale_epoch_write_rejected(self, gate_factory, clock):
|
||
"""entry 取得后世代已推进(他人触发开路)→ 迟到写回被 fencing 拒绝。"""
|
||
gate = gate_factory(_CFG)
|
||
stale_entry = await gate.try_enter("s1", "slow-worker")
|
||
await _open_gate(gate) # 他人连续失败 → 开路,epoch 推进
|
||
late = await gate.record_success(stale_entry)
|
||
assert not late.applied
|
||
# 门仍是 OPEN,未被迟到的成功污染
|
||
assert not (await gate.try_enter("s1", "w")).allowed
|
||
|
||
async def test_stale_probe_owner_rejected(self, gate_factory, clock):
|
||
gate = gate_factory(_CFG)
|
||
await _open_gate(gate)
|
||
clock.advance(_CFG.cooldown_s + 1)
|
||
stale_probe = await gate.try_enter("s1", "dead-worker")
|
||
clock.advance(_CFG.probe_ttl_s + 1)
|
||
takeover = await gate.try_enter("s1", "w2")
|
||
assert takeover.is_probe
|
||
# 死亡探针的迟到写回被拒(owner 已易主)
|
||
assert not (await gate.record_success(stale_probe)).applied
|
||
|
||
|
||
_RATE_CFG = BreakerConfig(
|
||
fail_threshold=100, # 连续通道抬高失声,单测率通道
|
||
cooldown_s=60.0,
|
||
probe_ttl_s=120.0,
|
||
min_calls=4,
|
||
fail_rate=0.9,
|
||
window_s=60.0,
|
||
max_cooldown_s=240.0,
|
||
)
|
||
|
||
|
||
async def _fail(gate, source="s1", reason="timeout", n=1):
|
||
update = None
|
||
for _ in range(n):
|
||
entry = await gate.try_enter(source, "w")
|
||
assert entry.allowed
|
||
update = await gate.record_failure(entry, reason, False)
|
||
return update
|
||
|
||
|
||
class TestRateChannel:
|
||
"""M2.5 失败率通道(设计 §3.1;全部确定性序列)。"""
|
||
|
||
async def test_rate_opens_at_min_calls(self, gate_factory):
|
||
# 病灶 1 回归: 高失败率源在 min_calls 样本处开路(连续通道静默)
|
||
gate = gate_factory(_RATE_CFG)
|
||
await _fail(gate, n=3) # attempts 3 < min_calls 4 → 仍 CLOSED
|
||
assert (await gate.try_enter("s1", "w")).allowed
|
||
update = await _fail(gate, n=1) # attempts 4, 失败率 1.0 ≥ 0.9
|
||
assert update.state is GateState.OPEN
|
||
assert not (await gate.try_enter("s1", "w")).allowed
|
||
|
||
async def test_below_min_calls_never_rate_opens(self, gate_factory):
|
||
gate = gate_factory(_RATE_CFG)
|
||
await _fail(gate, n=3)
|
||
assert (await gate.try_enter("s1", "w")).allowed
|
||
|
||
async def test_real_success_dilutes_window(self, gate_factory):
|
||
# 真实成功计入 attempts: 3 失败 + 1 成功 + 1 失败 = 4/5 = 0.8 < 0.9 不开
|
||
gate = gate_factory(_RATE_CFG)
|
||
await _fail(gate, n=3)
|
||
entry = await gate.try_enter("s1", "w")
|
||
await gate.record_success(entry)
|
||
update = await _fail(gate, n=1)
|
||
assert update.state is GateState.CLOSED
|
||
assert (await gate.try_enter("s1", "w")).allowed
|
||
|
||
async def test_result_invalid_not_counted(self, gate_factory):
|
||
# 坏结果 ≠ 坏服务: count_attempt=False 完全不动窗口 →
|
||
# 3 失败 + 1 不计成功 + 1 失败 = attempts 4, 失败率 1.0 → 开路
|
||
gate = gate_factory(_RATE_CFG)
|
||
await _fail(gate, n=3)
|
||
entry = await gate.try_enter("s1", "w")
|
||
await gate.record_success(entry, count_attempt=False)
|
||
update = await _fail(gate, n=1)
|
||
assert update.state is GateState.OPEN
|
||
|
||
async def test_rate_limited_bypasses_both_channels(self, gate_factory):
|
||
# C1 对抗: 429 是背压不是故障——连续通道(阈值 3)与率通道都不吃
|
||
cfg = BreakerConfig(fail_threshold=3, cooldown_s=60.0, probe_ttl_s=120.0, min_calls=4)
|
||
gate = gate_factory(cfg)
|
||
await _fail(gate, reason="rate_limited", n=10)
|
||
assert (await gate.try_enter("s1", "w")).allowed
|
||
# timeout 脉冲照常走连续通道开路
|
||
update = await _fail(gate, reason="timeout", n=3)
|
||
assert update.state is GateState.OPEN
|
||
|
||
async def test_probe_rate_limited_releases_not_hangs(self, gate_factory, clock):
|
||
# 探针撞 429: 非故障证据也非成功——按无果归还语义放下家接管,
|
||
# 不得把探针租约挂到 TTL(否则源被锁死 probe_ttl_s)
|
||
gate = gate_factory(_RATE_CFG)
|
||
await _fail(gate, n=4)
|
||
clock.advance(61)
|
||
probe = await gate.try_enter("s1", "w1")
|
||
assert probe.is_probe
|
||
update = await gate.record_failure(probe, "rate_limited", False)
|
||
assert update.applied
|
||
nxt = await gate.try_enter("s1", "w2")
|
||
assert nxt.allowed and nxt.is_probe # 立即可再探,而非等 probe_ttl
|
||
|
||
async def test_rate_open_failure_count_capped(self, gate_factory):
|
||
gate = gate_factory(_RATE_CFG)
|
||
await _fail(gate, n=3)
|
||
update = await _fail(gate, n=1)
|
||
assert update.state is GateState.OPEN
|
||
assert update.failure_count == _RATE_CFG.fail_threshold # 顶格语义沿用
|
||
|
||
|
||
class TestReopenBackoff:
|
||
"""开路时长指数递增与衰减(设计 §3.1;memory 假时钟,redis 走真实等待变体)。"""
|
||
|
||
async def test_backoff_doubles_and_caps(self, gate_factory, clock):
|
||
gate = gate_factory(_RATE_CFG)
|
||
await _fail(gate, n=4) # 率通道首开: cooldown_eff = 60
|
||
assert 0 < await gate.retry_after_s(("s1",)) <= 60.0
|
||
clock.advance(61)
|
||
probe = await gate.try_enter("s1", "w1")
|
||
assert probe.is_probe
|
||
await gate.record_failure(probe, "timeout", False) # 探针失败重开: streak 2 → 120
|
||
wait = await gate.retry_after_s(("s1",))
|
||
assert 60.0 < wait <= 120.0
|
||
clock.advance(121)
|
||
probe = await gate.try_enter("s1", "w1")
|
||
await gate.record_failure(probe, "timeout", False) # streak 3 → 240(封顶)
|
||
wait = await gate.retry_after_s(("s1",))
|
||
assert 120.0 < wait <= 240.0
|
||
|
||
async def test_streak_survives_close_then_decays(self, gate_factory, clock):
|
||
gate = gate_factory(_RATE_CFG)
|
||
await _fail(gate, n=4)
|
||
clock.advance(61)
|
||
probe = await gate.try_enter("s1", "w1")
|
||
await gate.record_failure(probe, "timeout", False) # streak 2
|
||
clock.advance(121)
|
||
probe = await gate.try_enter("s1", "w1")
|
||
await gate.record_success(probe) # 转 CLOSED,streak 不清零
|
||
# 窗口含探针成功 1 次,9 失败 → 9/10 = 0.9 率开;streak 递增至封顶
|
||
await _fail(gate, n=9)
|
||
assert await gate.retry_after_s(("s1",)) > 60.0
|
||
# 衰减: CLOSED 稳定 2×cooldown_eff 后首次 record_success 归零
|
||
clock.advance(241)
|
||
probe = await gate.try_enter("s1", "w1")
|
||
await gate.record_success(probe)
|
||
clock.advance(2 * 240.0 + 1)
|
||
entry = await gate.try_enter("s1", "w")
|
||
await gate.record_success(entry) # 触发衰减
|
||
await _fail(gate, n=9) # 再开(9/10)回到基础档
|
||
assert 0 < await gate.retry_after_s(("s1",)) <= 60.0
|
||
|
||
async def test_consecutive_open_does_not_bump_streak(self, gate_factory, clock):
|
||
# C1 对抗: 连续通道误熔健康源的代价封顶为单次 cooldown_s
|
||
cfg = BreakerConfig(fail_threshold=3, cooldown_s=60.0, probe_ttl_s=120.0)
|
||
gate = gate_factory(cfg)
|
||
await _fail(gate, reason="timeout", n=3) # 连续通道开路
|
||
clock.advance(61)
|
||
probe = await gate.try_enter("s1", "w1")
|
||
await gate.record_success(probe) # 恢复
|
||
await _fail(gate, reason="timeout", n=3) # 再次连续开路
|
||
assert 0 < await gate.retry_after_s(("s1",)) <= 60.0 # 无翻倍
|
||
|
||
|
||
class TestRetryAfter:
|
||
async def test_retry_after_semantics(self, gate_factory, clock):
|
||
gate = gate_factory(_CFG)
|
||
assert await gate.retry_after_s(("s1",)) == 0.0 # 健康 → 0
|
||
await _open_gate(gate)
|
||
wait = await gate.retry_after_s(("s1",))
|
||
assert 0 < wait <= _CFG.cooldown_s
|
||
clock.advance(_CFG.cooldown_s + 1)
|
||
assert await gate.retry_after_s(("s1",)) == 0.0 # 冷却到期 → 0
|
||
with pytest.raises(ValueError):
|
||
await gate.retry_after_s(())
|
||
|
||
async def test_retry_after_takes_min_across_sources(self, gate_factory, clock):
|
||
gate = gate_factory(_CFG)
|
||
await _open_gate(gate, "s1") # s1 开路;s2 健康
|
||
assert await gate.retry_after_s(("s1", "s2")) == 0.0
|
||
|
||
async def test_half_open_rejection_reports_no_certain_wait(self, gate_factory, clock):
|
||
"""探针在途时被拒 → 0.0(issue #14): 探针随时可能出结果,不存在确定时刻。
|
||
|
||
旧行为返回探针租约剩余,而租约长度是**死锁保护参数**(派生自
|
||
`2 × 最慢源 timeout`),与"这个源多久能恢复"没有因果关系。现场
|
||
`TIMEOUT_S=300` 时它是 600s,而冷却期只有 60s。
|
||
"""
|
||
gate = gate_factory(_CFG)
|
||
await _open_gate(gate)
|
||
clock.advance(_CFG.cooldown_s + 1)
|
||
probe = await gate.try_enter("s1", "w1")
|
||
assert probe.is_probe
|
||
blocked = await gate.try_enter("s1", "w2")
|
||
assert not blocked.allowed and blocked.state is GateState.HALF_OPEN
|
||
assert blocked.retry_after_s == 0.0
|
||
|
||
async def test_probe_grant_reports_no_certain_wait(self, gate_factory, clock):
|
||
"""准入被允许 → 恒 0.0(现在就能试);此前 redis 侧返回探针 TTL。"""
|
||
gate = gate_factory(_CFG)
|
||
await _open_gate(gate)
|
||
clock.advance(_CFG.cooldown_s + 1)
|
||
probe = await gate.try_enter("s1", "w1")
|
||
assert probe.allowed and probe.is_probe
|
||
assert probe.retry_after_s == 0.0
|
||
|
||
async def test_retry_after_zero_while_probe_in_flight(self, gate_factory, clock):
|
||
"""集合查询同口径: 探针在途的源不贡献等待时间。"""
|
||
gate = gate_factory(_CFG)
|
||
await _open_gate(gate)
|
||
clock.advance(_CFG.cooldown_s + 1)
|
||
assert (await gate.try_enter("s1", "w1")).is_probe
|
||
assert await gate.retry_after_s(("s1",)) == 0.0
|
||
|
||
async def test_fenced_write_in_half_open_reports_no_certain_wait(self, gate_factory, clock):
|
||
"""写回被 fencing 拒时的快照同口径;此前 redis 侧返回探针租约剩余。"""
|
||
gate = gate_factory(_CFG)
|
||
stale = await gate.try_enter("s1", "slow-worker") # epoch 0 的旧 entry
|
||
await _open_gate(gate) # 他人开路,epoch 推进
|
||
clock.advance(_CFG.cooldown_s + 1)
|
||
assert (await gate.try_enter("s1", "w1")).is_probe # 门此刻 HALF_OPEN
|
||
update = await gate.record_success(stale)
|
||
assert not update.applied and update.state is GateState.HALF_OPEN
|
||
assert update.retry_after_s == 0.0
|
||
|
||
|
||
class TestConsecutiveSuppression:
|
||
"""迭代 6: 窗口证据充足且健康时,连败是噪声,不开路(设计 §3.39)。"""
|
||
|
||
async def test_streak_suppressed_on_evidently_healthy_source(self, gate_factory):
|
||
# 20 成功垫底(窗口样本充足、失败率低)后 3 连败(阈值 3)→ 不开路
|
||
cfg = BreakerConfig(fail_threshold=3, cooldown_s=60.0, probe_ttl_s=120.0, min_calls=10)
|
||
gate = gate_factory(cfg)
|
||
for _ in range(20):
|
||
entry = await gate.try_enter("s1", "w")
|
||
await gate.record_success(entry)
|
||
await _fail(gate, reason="timeout", n=3)
|
||
assert (await gate.try_enter("s1", "w")).allowed
|
||
|
||
async def test_streak_fires_when_window_insufficient(self, gate_factory):
|
||
# 冷启动(窗口样本不足)3 连败照常开路——连续通道本职保留
|
||
cfg = BreakerConfig(fail_threshold=3, cooldown_s=60.0, probe_ttl_s=120.0, min_calls=10)
|
||
gate = gate_factory(cfg)
|
||
update = await _fail(gate, reason="timeout", n=3)
|
||
assert update.state is GateState.OPEN
|
||
|
||
async def test_warm_source_going_dead_caught_by_rate(self, gate_factory):
|
||
# 温热源猝死: 连败被抑制,但失败率窗口随失败累积必然接管
|
||
cfg = BreakerConfig(
|
||
fail_threshold=3, cooldown_s=60.0, probe_ttl_s=120.0, min_calls=4, fail_rate=0.6
|
||
)
|
||
gate = gate_factory(cfg)
|
||
for _ in range(3):
|
||
entry = await gate.try_enter("s1", "w")
|
||
await gate.record_success(entry)
|
||
update = await _fail(gate, reason="timeout", n=5) # 5/8 = 0.625 ≥ 0.6
|
||
assert update.state is GateState.OPEN
|