iomgaa
8edd3fb2cd
retry_after_s never had a written definition, so each backend improvised and they drifted apart. It now answers exactly one question: how long until a retry is *certainly* worth attempting. OPEN has such a moment (the cooldown deadline); HALF_OPEN does not, because the probe can come back at any time -- so it reports 0.0, which already means "retry now" elsewhere in the library. Six exits are brought in line. The half-open rejection is the one issue 14 reported: it returned the probe lease remainder, a deadlock-guard value derived from 2x the slowest timeout, so a 60s cooldown told callers to wait 600s. Worse, retry.py fed that number into the source cooldown memo, whose set_until only moves forward -- a source stayed skipped in-process for the whole lease even after its probe succeeded and the gate closed. That now writes an already-expired deadline, so the memo goes back to recording only real OPEN cooldowns. The other five were pre-existing memory/redis divergences hidden by a contract-test blind spot (the suite pinned that a second caller gets rejected, never what number it got): redis reported the probe TTL on grant and the lease remainder on fenced-out writes, where memory has always reported 0. Contract cases now pin all four half-open exits on both backends, with 1:1 real-wait variants for redis since the fake-clock ones skip there.
276 lines
11 KiB
Python
276 lines
11 KiB
Python
"""进程内熔断门: CHS gate 契约的内存实现(D3 双后端)。
|
||
|
||
状态机蓝本 VT `adapters/breaker.py`(闭路→阈值开路→冷却半开→单探针),
|
||
契约形态承 CHS `provider_gate.py`: 半开探针是**带 TTL 的租约**(持有者
|
||
死亡后可被接管,防"探针永远在路上"死锁),写回经 epoch fencing 拒绝
|
||
旧世代污染。epoch 在每次进入 OPEN 时递增。时钟构造注入,纯确定性可测。
|
||
|
||
M2.5 双通道(设计 2026-07-21-m25 §3.1): 在连续失败通道之外加失败率
|
||
通道(双 30s 桶窗口,样本 ≥ min_calls 且失败率 ≥ fail_rate 即开路);
|
||
429(rate_limited)是背压不是故障,两通道均不计;开路时长按 reopen_streak
|
||
指数递增封顶 max_cooldown_s——连续通道触发的开路不递增 streak(误熔
|
||
健康源的代价封顶为单次 cooldown_s)。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from typing import TYPE_CHECKING
|
||
|
||
from polygateway.ports import GateDecision, GateState, GateUpdate
|
||
|
||
if TYPE_CHECKING:
|
||
from collections.abc import Callable
|
||
|
||
from polygateway.types import BreakerConfig
|
||
|
||
|
||
@dataclass
|
||
class _SourceGate:
|
||
"""单源的门控可变状态(窗口/退避域见设计 §3.1 状态机域清单)。"""
|
||
|
||
state: GateState = GateState.CLOSED
|
||
fails: int = 0
|
||
epoch: int = 0
|
||
open_until: float = 0.0
|
||
probe_owner: str | None = None
|
||
probe_expires: float = 0.0
|
||
reasons: dict[str, str] = field(default_factory=dict)
|
||
# 失败率窗口: 双半窗桶轮换(win_id = now // (window_s/2))
|
||
win_id: int = -1
|
||
a0: int = 0 # 当前桶 attempts / failures
|
||
f0: int = 0
|
||
a1: int = 0 # 上一桶
|
||
f1: int = 0
|
||
reopen_streak: int = 0
|
||
closed_since: float | None = None
|
||
|
||
|
||
class InMemoryGate:
|
||
"""按 source_name 分别计数的进程内熔断门。"""
|
||
|
||
def __init__(self, *, config: BreakerConfig, now: Callable[[], float] = time.monotonic) -> None:
|
||
self._cfg = config
|
||
self._now = now
|
||
self._gates: dict[str, _SourceGate] = {}
|
||
|
||
def _gate(self, source_name: str) -> _SourceGate:
|
||
if not source_name.strip():
|
||
raise ValueError("source_name 不能为空")
|
||
return self._gates.setdefault(source_name, _SourceGate())
|
||
|
||
# —— M2.5 失败率窗口 ——
|
||
|
||
def _rotate_window(self, g: _SourceGate) -> None:
|
||
"""按半窗粒度轮换双桶;跨两桶以上的空窗直接清零。"""
|
||
half = self._cfg.window_s / 2.0
|
||
wid = int(self._now() // half)
|
||
if wid == g.win_id:
|
||
return
|
||
if wid == g.win_id + 1:
|
||
g.a1, g.f1 = g.a0, g.f0
|
||
else:
|
||
g.a1, g.f1 = 0, 0
|
||
g.a0, g.f0 = 0, 0
|
||
g.win_id = wid
|
||
|
||
def _window_add(self, g: _SourceGate, *, failed: bool) -> None:
|
||
self._rotate_window(g)
|
||
g.a0 += 1
|
||
if failed:
|
||
g.f0 += 1
|
||
|
||
def _window_evidently_healthy(self, g: _SourceGate) -> bool:
|
||
"""窗口样本充足且失败率低于阈值 = 有充分健康证据(迭代 6 抑制判据)。"""
|
||
self._rotate_window(g)
|
||
attempts = g.a0 + g.a1
|
||
if attempts < self._cfg.min_calls:
|
||
return False
|
||
return (g.f0 + g.f1) / attempts < self._cfg.fail_rate
|
||
|
||
def _rate_channel_open(self, g: _SourceGate) -> bool:
|
||
self._rotate_window(g)
|
||
attempts = g.a0 + g.a1
|
||
if attempts < self._cfg.min_calls:
|
||
return False
|
||
return (g.f0 + g.f1) / attempts >= self._cfg.fail_rate
|
||
|
||
def _cooldown_eff(self, g: _SourceGate) -> float:
|
||
streak = max(1, g.reopen_streak)
|
||
return min(self._cfg.cooldown_s * (2 ** (streak - 1)), self._cfg.max_cooldown_s)
|
||
|
||
def _remaining(self, g: _SourceGate) -> float:
|
||
"""距离**确定**可再试的时刻还有多久(issue #14 的契约定义)。
|
||
|
||
OPEN 的冷却截止是确定时刻;HALF_OPEN 下探针随时可能出结果,**不存在**
|
||
确定时刻,故 `0.0`——`0 = 可立即重试` 是库既有约定。此前这里返回探针
|
||
租约剩余,而租约长度是死锁保护参数(派生自 `2 × 最慢源 timeout`),与
|
||
"源多久能恢复"无因果关系;它还被喂进源冷却备忘,而备忘 `set_until`
|
||
取更晚者不可回退,于是门恢复 CLOSED 后本进程仍跳过该源整整一个租约。
|
||
|
||
三个出口(`try_enter` 拒绝、`_snapshot`、`retry_after_s`)共用本方法,
|
||
避免同一语义在三处各算一遍而漂移。
|
||
"""
|
||
if g.state is GateState.OPEN:
|
||
return max(0.0, g.open_until - self._now())
|
||
return 0.0
|
||
|
||
def _grant_probe(self, g: _SourceGate, source_name: str, owner: str) -> GateDecision:
|
||
g.state = GateState.HALF_OPEN
|
||
g.probe_owner = owner
|
||
g.probe_expires = self._now() + self._cfg.probe_ttl_s
|
||
return GateDecision(
|
||
source_name=source_name,
|
||
allowed=True,
|
||
state=GateState.HALF_OPEN,
|
||
epoch=g.epoch,
|
||
is_probe=True,
|
||
probe_owner=owner,
|
||
retry_after_s=0.0,
|
||
)
|
||
|
||
async def try_enter(self, source_name: str, owner: str) -> GateDecision:
|
||
"""健康普通准入;冷却到期/探针租约过期时原子授予唯一探针。"""
|
||
if not owner.strip():
|
||
raise ValueError("owner 不能为空")
|
||
g = self._gate(source_name)
|
||
now = self._now()
|
||
if g.state is GateState.CLOSED:
|
||
return GateDecision(
|
||
source_name=source_name,
|
||
allowed=True,
|
||
state=GateState.CLOSED,
|
||
epoch=g.epoch,
|
||
is_probe=False,
|
||
probe_owner=None,
|
||
retry_after_s=0.0,
|
||
)
|
||
if g.state is GateState.OPEN:
|
||
if now >= g.open_until:
|
||
return self._grant_probe(g, source_name, owner)
|
||
return GateDecision(
|
||
source_name=source_name,
|
||
allowed=False,
|
||
state=GateState.OPEN,
|
||
epoch=g.epoch,
|
||
is_probe=False,
|
||
probe_owner=None,
|
||
retry_after_s=self._remaining(g),
|
||
)
|
||
# HALF_OPEN: 探针在途;租约过期则接管,否则拒绝(防惊群)
|
||
if now >= g.probe_expires:
|
||
return self._grant_probe(g, source_name, owner)
|
||
return GateDecision(
|
||
source_name=source_name,
|
||
allowed=False,
|
||
state=GateState.HALF_OPEN,
|
||
epoch=g.epoch,
|
||
is_probe=False,
|
||
probe_owner=None,
|
||
retry_after_s=self._remaining(g),
|
||
)
|
||
|
||
def _fenced(self, g: _SourceGate, entry: GateDecision) -> bool:
|
||
"""写回资格: 世代一致;探针写回还要求 owner 仍在位(CHS fencing 同款)。"""
|
||
if not entry.allowed:
|
||
raise ValueError("被拒决定不得写回")
|
||
if entry.epoch != g.epoch:
|
||
return False
|
||
return not (
|
||
entry.is_probe
|
||
and (g.state is not GateState.HALF_OPEN or g.probe_owner != entry.probe_owner)
|
||
)
|
||
|
||
def _snapshot(self, g: _SourceGate, applied: bool) -> GateUpdate:
|
||
return GateUpdate(
|
||
applied=applied,
|
||
state=g.state,
|
||
epoch=g.epoch,
|
||
failure_count=g.fails,
|
||
retry_after_s=self._remaining(g),
|
||
)
|
||
|
||
def _open(self, g: _SourceGate, reason: str, *, bump_streak: bool) -> None:
|
||
if bump_streak:
|
||
g.reopen_streak += 1
|
||
g.state = GateState.OPEN
|
||
g.epoch += 1 # 世代推进: 旧 entry 的迟到写回自此被 fencing 拒绝
|
||
g.open_until = self._now() + self._cooldown_eff(g)
|
||
g.fails = max(g.fails, self._cfg.fail_threshold)
|
||
g.probe_owner = None
|
||
g.probe_expires = 0.0
|
||
g.closed_since = None
|
||
|
||
async def record_success(
|
||
self, entry: GateDecision, *, count_attempt: bool = True
|
||
) -> GateUpdate:
|
||
g = self._gate(entry.source_name)
|
||
if not self._fenced(g, entry):
|
||
return self._snapshot(g, applied=False)
|
||
was_probe = entry.is_probe
|
||
if count_attempt:
|
||
self._window_add(g, failed=False)
|
||
# streak 衰减: CLOSED 稳定满 2×cooldown_eff 后的首次成功归零(设计 §3.1)
|
||
if (
|
||
g.reopen_streak > 0
|
||
and g.closed_since is not None
|
||
and self._now() - g.closed_since >= 2 * self._cooldown_eff(g)
|
||
):
|
||
g.reopen_streak = 0
|
||
if was_probe:
|
||
g.closed_since = self._now() # 仅探针转 CLOSED 时写,普通成功不刷新
|
||
g.state = GateState.CLOSED
|
||
g.fails = 0
|
||
g.probe_owner = None
|
||
g.probe_expires = 0.0
|
||
return self._snapshot(g, applied=True)
|
||
|
||
async def record_failure(
|
||
self, entry: GateDecision, reason: str, force_open: bool
|
||
) -> GateUpdate:
|
||
g = self._gate(entry.source_name)
|
||
if not self._fenced(g, entry):
|
||
return self._snapshot(g, applied=False)
|
||
if reason == "rate_limited" and not force_open:
|
||
# 429 = 背压不是故障(设计 §3.1): 两通道均不计,选源层软处理;
|
||
# 探针撞 429 按无果归还语义放下家,不挂租约
|
||
if entry.is_probe:
|
||
g.state = GateState.OPEN
|
||
g.open_until = self._now()
|
||
g.probe_owner = None
|
||
g.probe_expires = 0.0
|
||
return self._snapshot(g, applied=True)
|
||
if entry.is_probe or force_open:
|
||
# 探针失败重开递增 streak;SourceDead 一击即熔不递增
|
||
self._open(g, reason, bump_streak=entry.is_probe)
|
||
return self._snapshot(g, applied=True)
|
||
self._window_add(g, failed=True)
|
||
g.fails += 1
|
||
if self._rate_channel_open(g):
|
||
self._open(g, reason, bump_streak=True)
|
||
elif g.fails >= self._cfg.fail_threshold and not self._window_evidently_healthy(g):
|
||
# 连续通道不递增(C1 封顶);窗口证据充足且健康时连败是噪声,
|
||
# 不误熔"当前最好的源"(迭代 6,第八轮实证: 源1 被 5 连空补全误关 60s)
|
||
self._open(g, reason, bump_streak=False)
|
||
return self._snapshot(g, applied=True)
|
||
|
||
async def release_probe(self, entry: GateDecision) -> GateUpdate:
|
||
"""探针无果归还(如取消): 源保持可接管状态让下一 caller 接手;幂等。"""
|
||
if not (entry.allowed and entry.is_probe and entry.probe_owner is not None):
|
||
raise ValueError("release_probe 只接受在途探针决定")
|
||
g = self._gate(entry.source_name)
|
||
if not self._fenced(g, entry):
|
||
return self._snapshot(g, applied=False)
|
||
g.state = GateState.OPEN
|
||
g.open_until = self._now() # 立即可被下一 caller 以探针身份接管
|
||
g.probe_owner = None
|
||
g.probe_expires = 0.0
|
||
return self._snapshot(g, applied=True)
|
||
|
||
async def retry_after_s(self, sources: tuple[str, ...]) -> float:
|
||
"""集合中最早可尝试时间;健康/到期返回 0。"""
|
||
if not sources:
|
||
raise ValueError("sources 不能为空")
|
||
return min(self._remaining(self._gate(name)) for name in sources)
|