feat: let circuit_open=wait queue instead of killing the call

on_no_runnable now dispatches on why every source was rejected instead
of falling through two serial branches. Under wait, a fully open circuit
sleeps out the cooldown and comes back for another round; the breaker's
protection is untouched (still not a single request leaves during the
wait, so no quota or money burns) -- what changes is whether the caller
dies on the spot or queues.

Dispatching is not cosmetic. Left serial, wait would fall into the quota
branch and a caller with quota_full=fail_fast would get a
quota_exhausted error while its quota was in fact fine.

_nap sleeps to the cooldown deadline rather than polling every 10ms,
which for a 60s cooldown is 6000 round trips per in-flight call on the
Redis backend. Jitter is added on top instead of scaling the wait, since
waking early before a known deadline just earns another rejection. Both
arms clamp to the remaining stall budget, so the worst case per call is
stall_window plus one poll and does not drift with max_cooldown_s. The
clamp's lower bound is the jitter itself, not poll_interval -- the
latter would have lifted the existing [0.5p, 1.0p] quota polling.
This commit is contained in:
2026-08-20 00:27:00 -04:00
parent eb956b2cdf
commit 6edf4ac9de
2 changed files with 198 additions and 17 deletions
+143
View File
@@ -13,6 +13,7 @@ from polygateway.backends.memory.breaker import InMemoryGate
from polygateway.backends.memory.limiter import InMemoryLimiter
from polygateway.errors import (
AllSourcesExhausted,
CircuitOpenError,
GatewayUnavailableError,
GovernanceBackendError,
SourceNotConfiguredError,
@@ -62,6 +63,7 @@ def _mw(
sleep,
rng=lambda: 0.0,
quota_full="wait",
circuit_open="fail_fast",
gate=None,
transport=None,
emitter=None,
@@ -76,6 +78,7 @@ def _mw(
retry=RetryPolicy(max_attempts=3, backoff_base_s=2.0, backoff_max_s=30.0),
backpressure=BackpressurePolicy(stall_window_s=_STALL, poll_interval_s=0.01),
quota_full=quota_full,
circuit_open=circuit_open,
cooldown_memo=SourceCooldownMemo(now=clock),
emitter=emitter,
now=clock,
@@ -593,3 +596,143 @@ class TestGateFailuresReachCallersAsScopeLevel:
await QuotaGate(_Broken(), scope="LLM").progress_age_s()
assert ei.value.scope == "llm"
assert ei.value.reason == "governance_backend_down"
class TestCircuitOpenPolicy:
"""issue #14: 熔断全拒时是当场判死还是等冷却过去。
缺省 fail_fast 即历史行为(TestStallQuadrants 等既有用例照旧覆盖);
本类钉的是 wait 档,以及两条策略互不串线。
"""
@staticmethod
async def _opened_gate(clock, cfg=_BREAKER):
gate = InMemoryGate(config=cfg, now=clock)
for _ in range(cfg.fail_threshold):
entry = await gate.try_enter("s1", "w")
await gate.record_failure(entry, "network_error", False)
return gate
@staticmethod
def _free_limiter(clock, src):
return InMemoryLimiter(
scope="llm",
sources={"s1": src},
global_limits=_NO_GLOBAL,
lease_ttl_s=10_000.0,
now=clock,
)
async def test_fail_fast_is_the_default(self):
"""缺省档逐字保持历史行为: 全源开路当场抛 CircuitOpenError。"""
clock = FakeClock()
src = make_source()
mw = _mw(
[src], self._free_limiter(clock, src), [], clock=clock,
sleep=BoundedSleep(), gate=await self._opened_gate(clock),
)
with pytest.raises(CircuitOpenError) as ei:
await mw(_REQ)
assert ei.value.reason == "circuit_open"
async def test_wait_sleeps_out_the_cooldown_instead_of_dying(self):
"""wait 档: 睡到冷却结束再来一轮,拿到探针后正常返回。
睡的是**冷却剩余**而不是 poll_interval——60 秒冷却用 10ms 轮询要空转
6000 次,memory 后端只是查字典,Redis 后端则是 6000 次往返 × 每个在途调用。
"""
clock = FakeClock()
src = make_source()
sleep = BoundedSleep()
async def advance(_n):
clock.advance(sleep.delays[-1])
sleep._side_effect = advance
mw = _mw(
[src], self._free_limiter(clock, src), [_ok()], clock=clock,
sleep=sleep, gate=await self._opened_gate(clock), circuit_open="wait",
)
resp = await mw(_REQ)
assert resp.content == "ok"
# 一觉睡到冷却结束(jitter 上加,rng=0 → +0.5×poll),不是 poll 空转
assert sleep.delays[0] == pytest.approx(_BREAKER.cooldown_s + 0.005)
async def test_wait_does_not_leak_into_the_quota_branch(self):
"""两条策略互不串线: circuit_open=wait 配 quota_full=fail_fast 时,
熔断等待**不得**被当成配额耗尽上报——串线会让调用方拿到一个
reason=quota_exhausted 的异常,而配额其实是满的。"""
clock = FakeClock()
src = make_source()
sleep = BoundedSleep()
async def advance(_n):
clock.advance(sleep.delays[-1])
sleep._side_effect = advance
mw = _mw(
[src], self._free_limiter(clock, src), [_ok()], clock=clock, sleep=sleep,
gate=await self._opened_gate(clock), quota_full="fail_fast", circuit_open="wait",
)
assert (await mw(_REQ)).content == "ok"
async def test_wait_still_dies_when_cooldown_outlasts_the_stall_budget(self):
"""等待有可解释的上界: 冷却比 stall 预算还长时,在窗口耗尽处判死。
单次睡眠夹到剩余 stall 预算,故最坏墙钟 = stall_window + 一个 poll,
不随 max_cooldown_s 漂移。
"""
clock = FakeClock()
src = make_source()
long_cooldown = BreakerConfig(
fail_threshold=3, cooldown_s=1000.0, probe_ttl_s=2000.0, max_cooldown_s=1000.0
)
sleep = BoundedSleep()
async def advance(_n):
clock.advance(sleep.delays[-1])
sleep._side_effect = advance
mw = _mw(
[src], self._free_limiter(clock, src), [], clock=clock, sleep=sleep,
gate=await self._opened_gate(clock, long_cooldown), circuit_open="wait",
)
with pytest.raises(AllSourcesExhausted) as ei:
await mw(_REQ)
assert ei.value.reason == "stalled"
assert ei.value.per_source_reasons == {"s1": "circuit_open"}
assert sleep.delays[0] == pytest.approx(_STALL + 0.01) # 夹到预算 + 一个 poll
async def test_wait_loop_stays_cancellable(self):
"""取消穿透(铁律): 熔断等待中的取消不得被吞。"""
clock = FakeClock()
src = make_source()
mw = _mw(
[src], self._free_limiter(clock, src), [], clock=clock,
sleep=asyncio.sleep, gate=await self._opened_gate(clock), circuit_open="wait",
)
task = asyncio.create_task(mw(_REQ))
await asyncio.sleep(0.03)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
async def test_half_open_rejection_does_not_blacklist_a_recovered_source(self):
"""issue #14 §1.3 回归: 探针成功后本进程立即可再选该源。
此前 HALF_OPEN 拒绝把探针租约(派生自 2 × timeout,现场 600s)写进冷却
备忘,而 `set_until` 取更晚者、不可回退——门恢复 CLOSED 之后本进程仍
跳过该源整整一个租约,单源下每次调用照旧判死。多源部署同样中招,只是
被别的源接住流量掩盖了。
"""
clock = FakeClock()
cfg = BreakerConfig(fail_threshold=3, cooldown_s=60.0, probe_ttl_s=600.0)
gate = await self._opened_gate(clock, cfg)
memo = SourceCooldownMemo(now=clock)
clock.advance(cfg.cooldown_s + 1)
probe = await gate.try_enter("s1", "w1")
blocked = await gate.try_enter("s1", "w2") # 并发调用撞上在途探针
assert not blocked.allowed
memo.set_until("s1", clock() + blocked.retry_after_s) # 准入路径的写法
await gate.record_success(probe) # 探针成功 → 门恢复 CLOSED
assert not memo.active("s1")