refactor: share one admission path across the three governance loops
_pick_runnable and _on_no_runnable lived in three copies (retry.py, embedding.py, ocr.py), the latter two being verbatim subsets of the first. Admission semantics keep evolving -- issue #8 changed the stall accounting, M2.5 added the AIMD pacer, issue #14 is about to add a wait policy -- and every round had to be applied three times. SourceAdmission now owns picking a runnable source and deciding what happens when none is available. The three loops keep their QuotaGate, BreakerGate and pacer references because _attempt still needs them for write-back and pacer.leave(); those instances are shared, not rebuilt (a second pacer would split the in-flight counter). The cooldown memo moves in wholesale since only admission consumes it. Behaviour is unchanged: pick differs from the old chat copy only by the pacer None-guards, on_no_runnable is verbatim identical, and the suite reports the same 967 passed / 21 skipped / 32 deselected as before. The one visible change is the settle-and-release warning text, which had three variants ("permit", "embedding permit", "OCR permit") and is now one. Tests importing _demote_call_failures follow it to its new home.
This commit is contained in:
@@ -23,7 +23,6 @@ from loguru import logger
|
||||
|
||||
from polygateway.errors import (
|
||||
AllSourcesExhausted,
|
||||
CircuitOpenError,
|
||||
GovernanceBackendError,
|
||||
PolyGatewayError,
|
||||
RequestRejectedError,
|
||||
@@ -32,10 +31,11 @@ from polygateway.errors import (
|
||||
SourceNotConfiguredError,
|
||||
TransientError,
|
||||
)
|
||||
from polygateway.middleware.admission import SourceAdmission, settle_and_release
|
||||
from polygateway.middleware.breaker import BreakerGate
|
||||
from polygateway.middleware.ratelimit import QuotaGate
|
||||
from polygateway.ports import OutcomeAwareSelector
|
||||
from polygateway.sources import AdaptivePacer, SourceCooldownMemo
|
||||
from polygateway.sources import AdaptivePacer
|
||||
from polygateway.streaming import StreamLivenessTimeout
|
||||
from polygateway.types import LLMResponse
|
||||
|
||||
@@ -50,6 +50,7 @@ if TYPE_CHECKING:
|
||||
SourceSelector,
|
||||
Transport,
|
||||
)
|
||||
from polygateway.sources import SourceCooldownMemo
|
||||
from polygateway.types import (
|
||||
BackpressurePolicy,
|
||||
ChatRequest,
|
||||
@@ -134,70 +135,6 @@ class StallClock:
|
||||
self._productive_s += self._now() - started
|
||||
|
||||
|
||||
def _demote_call_failures(
|
||||
ordered: list[SourceConfig],
|
||||
attempt_fails: dict[str, int],
|
||||
health: Callable[[str], float] | None,
|
||||
) -> list[SourceConfig]:
|
||||
"""调用内降权(设计 §3.3/§3.36): 失败 ≥2 次且存在可信替代才让位。
|
||||
|
||||
可信替代 = 某未失败候选 health ≥ 0.5 × 失败源 health——异构池里健康源
|
||||
偶发失败不该被推向已知坏源(第三轮教训: 期望成功率 83% vs 10%)。
|
||||
无健康视图(round_robin 等)保持无条件降权(冷启动保护)。
|
||||
"""
|
||||
demoted = [s for s in ordered if attempt_fails.get(s.name, 0) >= 2]
|
||||
if not demoted or len(demoted) == len(ordered):
|
||||
return ordered
|
||||
if health is None:
|
||||
return _move_to_tail(ordered, demoted)
|
||||
return _health_gated_reorder(ordered, demoted, attempt_fails, health)
|
||||
|
||||
|
||||
def _move_to_tail(ordered: list[SourceConfig], demoted: list[SourceConfig]) -> list[SourceConfig]:
|
||||
"""无健康视图: 无条件移尾(冷启动保护原语义)。"""
|
||||
names = {d.name for d in demoted}
|
||||
return [s for s in ordered if s.name not in names] + demoted
|
||||
|
||||
|
||||
def _health_gated_reorder(
|
||||
ordered: list[SourceConfig],
|
||||
demoted: list[SourceConfig],
|
||||
attempt_fails: dict[str, int],
|
||||
health: Callable[[str], float],
|
||||
) -> list[SourceConfig]:
|
||||
"""健康门槛降权: 无可信替代则原地重试;有则插到可信替代之后。"""
|
||||
demoted = _credible_demotions(ordered, demoted, attempt_fails, health)
|
||||
if not demoted:
|
||||
return ordered
|
||||
names = {d.name for d in demoted}
|
||||
rest = [s for s in ordered if s.name not in names]
|
||||
return _insert_after_credible(rest, demoted, health)
|
||||
|
||||
|
||||
def _insert_after_credible(
|
||||
rest: list[SourceConfig],
|
||||
demoted: list[SourceConfig],
|
||||
health: Callable[[str], float],
|
||||
) -> list[SourceConfig]:
|
||||
"""插入位置(第四轮教训): 被降权源排在可信替代之后、不可信源之前——
|
||||
可信替代被限流闸/熔断跳过时,下一候选是失败源本身而非垃圾源。"""
|
||||
bar = 0.5 * max(health(d.name) for d in demoted)
|
||||
credible = [s for s in rest if health(s.name) >= bar]
|
||||
junk = [s for s in rest if health(s.name) < bar]
|
||||
return credible + demoted + junk
|
||||
|
||||
|
||||
def _credible_demotions(
|
||||
ordered: list[SourceConfig],
|
||||
demoted: list[SourceConfig],
|
||||
attempt_fails: dict[str, int],
|
||||
health: Callable[[str], float],
|
||||
) -> list[SourceConfig]:
|
||||
"""健康门槛过滤: 仅当存在"健康分 ≥ 失败源一半"的未失败候选,让位才有意义。"""
|
||||
alts = [o for o in ordered if attempt_fails.get(o.name, 0) < 2]
|
||||
return [s for s in demoted if any(health(o.name) >= 0.5 * health(s.name) for o in alts)]
|
||||
|
||||
|
||||
def _failure_reason(exc: PolyGatewayError) -> str:
|
||||
"""失败原因归类(CHS governance.py:169 同款)。"""
|
||||
if isinstance(exc, SourceDeadError):
|
||||
@@ -252,23 +189,36 @@ class RetryMW:
|
||||
raise ValueError(f"quota_full 必须是 wait|fail_fast: {quota_full!r}")
|
||||
self._scope = scope
|
||||
self._sources = list(sources)
|
||||
self._selector = selector
|
||||
# 记账写回与 pacer 结算仍在 `_attempt` 内,故这三者由本类持有并与
|
||||
# `SourceAdmission` **共享同一实例**(pacer 有在途计数,不可分裂)
|
||||
self._quota = QuotaGate(limiter, scope=self._scope)
|
||||
self._breaker = BreakerGate(gate, scope=self._scope)
|
||||
self._transport = transport
|
||||
self._retry = retry
|
||||
self._bp = backpressure
|
||||
self._quota_full = quota_full
|
||||
self._memo = cooldown_memo or SourceCooldownMemo(now=now)
|
||||
# M2.5: 选源器可选健康喂数端口,构造期 isinstance 判定一次(设计 §3.2)
|
||||
self._outcome_sink = selector if isinstance(selector, OutcomeAwareSelector) else None
|
||||
self._health_view = self._outcome_sink.health if self._outcome_sink else None
|
||||
# M2.5 §3.35: AIMD 自适应并发——429 收紧、成功回涨,超限调用排队不烧预算
|
||||
self._pacer = pacer or AdaptivePacer(ceiling=64.0)
|
||||
self._emitter = emitter
|
||||
self._now = now
|
||||
self._sleep = sleep
|
||||
self._rng = rng
|
||||
# 准入编排三条循环共用一份(issue #14);冷却备忘由它独占
|
||||
self._admission = SourceAdmission(
|
||||
scope=self._scope,
|
||||
sources=self._sources,
|
||||
selector=selector,
|
||||
quota=self._quota,
|
||||
breaker=self._breaker,
|
||||
backpressure=backpressure,
|
||||
quota_full=quota_full,
|
||||
memo=cooldown_memo,
|
||||
pacer=self._pacer,
|
||||
health_view=self._outcome_sink.health if self._outcome_sink else None,
|
||||
now=now,
|
||||
sleep=sleep,
|
||||
rng=rng,
|
||||
)
|
||||
|
||||
async def __call__(self, request: ChatRequest) -> LLMResponse:
|
||||
"""执行治理调用;scope 级失败按 §6.1 携结构化字段上抛。"""
|
||||
@@ -283,16 +233,16 @@ class RetryMW:
|
||||
clock = StallClock(self._now)
|
||||
while True:
|
||||
# 调用级时间上限(迭代 5): 429 免预算后的兜底,防饱和期无限循环
|
||||
if await self._stalled(clock):
|
||||
if await self._admission.stalled(clock):
|
||||
raise AllSourcesExhausted(
|
||||
scope=self._scope,
|
||||
reason="stalled",
|
||||
retry_after_s=self._retry.backoff_base_s,
|
||||
per_source_reasons=reasons,
|
||||
)
|
||||
picked, gate_rejections = await self._pick_runnable(reasons, attempt_fails)
|
||||
picked, gate_rejections = await self._admission.pick(reasons, attempt_fails)
|
||||
if picked is None:
|
||||
await self._on_no_runnable(gate_rejections, reasons, clock)
|
||||
await self._admission.on_no_runnable(gate_rejections, reasons, clock)
|
||||
continue
|
||||
async with clock.attempting() as attempt:
|
||||
outcome = await self._attempt(request, *picked, reasons, attempt_fails)
|
||||
@@ -314,87 +264,6 @@ class RetryMW:
|
||||
if not outcome.immediate:
|
||||
await self._sleep(self._backoff_delay(max(fails, 1), outcome.exc))
|
||||
|
||||
# —— 选源与准入(CHS _pick_runnable 120-167)——
|
||||
|
||||
async def _pick_runnable(
|
||||
self, reasons: dict[str, str], attempt_fails: dict[str, int]
|
||||
) -> tuple[tuple[SourceConfig, Permit, GateDecision] | None, int]:
|
||||
stats = {s.name: await self._quota.stats(s) for s in self._sources}
|
||||
gate_rejections = 0
|
||||
ordered = _demote_call_failures(
|
||||
self._selector.order(self._sources, stats), attempt_fails, self._health_view
|
||||
)
|
||||
for cand in ordered:
|
||||
if self._memo.active(cand.name):
|
||||
# 冷却备忘跳过也计入拒绝数,保住 circuit_open 判据(CHS 同款)
|
||||
gate_rejections += 1
|
||||
reasons[cand.name] = "cooldown"
|
||||
continue
|
||||
if not self._pacer.admit(cand.name):
|
||||
# AIMD 超限: 不计 gate_rejections → 走 quota-wait 排队,不误判熔断
|
||||
reasons.setdefault(cand.name, "adaptive_paced")
|
||||
continue
|
||||
permit = await self._quota.try_acquire(cand)
|
||||
if permit is None:
|
||||
reasons.setdefault(cand.name, "rate_limited")
|
||||
continue
|
||||
entry = None
|
||||
try:
|
||||
entry = await self._breaker.try_enter(cand, uuid.uuid4().hex)
|
||||
finally:
|
||||
# try_enter 未归还 entry(异常/取消)→ 释放已占 permit,不吞任何异常
|
||||
if entry is None:
|
||||
await self._settle_and_release(permit, 0)
|
||||
if entry.allowed:
|
||||
self._pacer.enter(cand.name)
|
||||
return (cand, permit, entry), gate_rejections
|
||||
gate_rejections += 1
|
||||
reasons[cand.name] = "circuit_open"
|
||||
# 开路源本地记冷却,避免每轮白烧 RPM 探测(CHS governance.py:107)
|
||||
self._memo.set_until(cand.name, self._now() + entry.retry_after_s)
|
||||
await self._settle_and_release(permit, 0)
|
||||
return None, gate_rejections
|
||||
|
||||
# —— 背压与 stall 判死(CHS governance.py:270-285)——
|
||||
|
||||
async def _stalled(self, clock: StallClock) -> bool:
|
||||
"""双条件 stall 判死(CHS governance.py:270-281): 本地累计等待与全局
|
||||
无进展**同时**超窗才判死——本地 monotonic 与后端时钟刻意不混用。
|
||||
|
||||
本地一侧只计非生产性等待(issue #8,见 `StallClock`)。短路顺序有意为之:
|
||||
本地未超窗就不问后端,省一次 Redis 往返。
|
||||
"""
|
||||
stall = self._bp.stall_window_s
|
||||
return clock.stalled_s() > stall and await self._quota.progress_age_s() > stall
|
||||
|
||||
async def _on_no_runnable(
|
||||
self, gate_rejections: int, reasons: dict[str, str], clock: StallClock
|
||||
) -> None:
|
||||
if gate_rejections == len(self._sources):
|
||||
names = tuple(s.name for s in self._sources)
|
||||
raise CircuitOpenError(
|
||||
scope=self._scope,
|
||||
retry_after_s=await self._breaker.retry_after_s(names),
|
||||
per_source_reasons=reasons,
|
||||
)
|
||||
if self._quota_full == "fail_fast":
|
||||
raise AllSourcesExhausted(
|
||||
scope=self._scope,
|
||||
reason="quota_exhausted",
|
||||
retry_after_s=self._bp.poll_interval_s,
|
||||
per_source_reasons=reasons,
|
||||
)
|
||||
if await self._stalled(clock):
|
||||
names = tuple(s.name for s in self._sources)
|
||||
raise AllSourcesExhausted(
|
||||
scope=self._scope,
|
||||
reason="stalled",
|
||||
retry_after_s=await self._breaker.retry_after_s(names),
|
||||
per_source_reasons=reasons,
|
||||
)
|
||||
# jitter ∈ [0.5p, 1.0p] 防惊群(CHS governance.py:283-285)
|
||||
await self._sleep(self._bp.poll_interval_s * (0.5 + 0.5 * self._rng()))
|
||||
|
||||
# —— 单次尝试(CHS run 200-268)——
|
||||
|
||||
async def _attempt(
|
||||
@@ -460,7 +329,7 @@ class RetryMW:
|
||||
return _Failed(exc, immediate=dead)
|
||||
finally:
|
||||
self._pacer.leave(source.name)
|
||||
await self._settle_and_release(permit, actual)
|
||||
await settle_and_release(permit, actual)
|
||||
|
||||
async def _on_rejected(
|
||||
self, exc: RequestRejectedError, source: SourceConfig, entry: GateDecision
|
||||
@@ -523,18 +392,6 @@ class RetryMW:
|
||||
reasoning_tokens=result.reasoning_tokens,
|
||||
)
|
||||
|
||||
async def _settle_and_release(self, permit: Permit, actual: int) -> None:
|
||||
"""finally 专用: settle 后必 release;失败降级 warning,绝不掩盖主异常/取消。"""
|
||||
try:
|
||||
try:
|
||||
await permit.settle(actual)
|
||||
finally:
|
||||
await permit.release()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning("permit 结算/释放失败(不掩盖主异常): {}", exc)
|
||||
|
||||
async def _emit(
|
||||
self,
|
||||
request: ChatRequest,
|
||||
|
||||
Reference in New Issue
Block a user