feat: feed selector health and demote in-call failed sources
RetryMW keeps a per-call failure map (local, never instance state): a source failing twice in one call yields to the next candidate. Attempt outcomes feed OutcomeAwareSelector behind a swallow-and-warn guard; ResultInvalid and provider-rejected paths record success with count_attempt=False so the breaker window stays clean. Same accounting applied in EmbeddingClient.
This commit is contained in:
@@ -32,6 +32,7 @@ from polygateway.errors import (
|
||||
)
|
||||
from polygateway.middleware.breaker import BreakerGate
|
||||
from polygateway.middleware.ratelimit import QuotaGate
|
||||
from polygateway.ports import OutcomeAwareSelector
|
||||
from polygateway.sources import SourceCooldownMemo
|
||||
from polygateway.streaming import StreamLivenessTimeout
|
||||
from polygateway.types import LLMResponse
|
||||
@@ -72,6 +73,16 @@ def backoff_delay(
|
||||
return max(delay, retry_after)
|
||||
|
||||
|
||||
def _demote_call_failures(
|
||||
ordered: list[SourceConfig], attempt_fails: dict[str, int]
|
||||
) -> list[SourceConfig]:
|
||||
"""调用内降权(设计 §3.3): 本调用失败 ≥2 次的源移尾;全失败则保持原序。"""
|
||||
demoted = [s for s in ordered if attempt_fails.get(s.name, 0) >= 2]
|
||||
if not demoted or len(demoted) == len(ordered):
|
||||
return ordered
|
||||
return [s for s in ordered if attempt_fails.get(s.name, 0) < 2] + demoted
|
||||
|
||||
|
||||
def _failure_reason(exc: PolyGatewayError) -> str:
|
||||
"""失败原因归类(CHS governance.py:169 同款)。"""
|
||||
if isinstance(exc, SourceDeadError):
|
||||
@@ -124,6 +135,8 @@ class RetryMW:
|
||||
self._bp = backpressure
|
||||
self._quota_full = quota_full
|
||||
self._memo = cooldown_memo or SourceCooldownMemo(now=now)
|
||||
# M2.5: 选源器可选健康喂数端口,构造期 isinstance 判定一次(设计 §3.2)
|
||||
self._outcome_sink = selector if isinstance(selector, OutcomeAwareSelector) else None
|
||||
self._emitter = emitter
|
||||
self._now = now
|
||||
self._sleep = sleep
|
||||
@@ -135,13 +148,15 @@ class RetryMW:
|
||||
raise AllSourcesExhausted(scope=self._scope, reason="no_sources", retry_after_s=0.0)
|
||||
fails = 0
|
||||
reasons: dict[str, str] = {}
|
||||
# 调用内失败计数(设计 §3.3): 局部状态,调用结束即弃;严禁实例属性(并发共享)
|
||||
attempt_fails: dict[str, int] = {}
|
||||
entered_at = self._now() # 调用级累计计时,循环内不重置(CHS governance.py:207)
|
||||
while True:
|
||||
picked, gate_rejections = await self._pick_runnable(reasons)
|
||||
picked, gate_rejections = await self._pick_runnable(reasons, attempt_fails)
|
||||
if picked is None:
|
||||
await self._on_no_runnable(gate_rejections, reasons, entered_at)
|
||||
continue
|
||||
outcome = await self._attempt(request, *picked, reasons)
|
||||
outcome = await self._attempt(request, *picked, reasons, attempt_fails)
|
||||
if isinstance(outcome, LLMResponse):
|
||||
return outcome
|
||||
fails += 1
|
||||
@@ -158,11 +173,12 @@ class RetryMW:
|
||||
# —— 选源与准入(CHS _pick_runnable 120-167)——
|
||||
|
||||
async def _pick_runnable(
|
||||
self, reasons: dict[str, str]
|
||||
self, reasons: dict[str, str], attempt_fails: dict[str, int]
|
||||
) -> tuple[tuple[SourceConfig, Permit, GateDecision] | None, int]:
|
||||
stats = {s.name: await self._quota.stats(s) for s in self._sources}
|
||||
gate_rejections = 0
|
||||
for cand in self._selector.order(self._sources, stats):
|
||||
ordered = _demote_call_failures(self._selector.order(self._sources, stats), attempt_fails)
|
||||
for cand in ordered:
|
||||
if self._memo.active(cand.name):
|
||||
# 冷却备忘跳过也计入拒绝数,保住 circuit_open 判据(CHS 同款)
|
||||
gate_rejections += 1
|
||||
@@ -228,6 +244,7 @@ class RetryMW:
|
||||
permit: Permit,
|
||||
entry: GateDecision,
|
||||
reasons: dict[str, str],
|
||||
attempt_fails: dict[str, int],
|
||||
) -> LLMResponse | _Failed:
|
||||
call_id = str(uuid.uuid4())
|
||||
started = self._now()
|
||||
@@ -243,6 +260,7 @@ class RetryMW:
|
||||
actual = result.prompt_tokens + result.completion_tokens
|
||||
await self._record_quietly(self._breaker.record_success(entry))
|
||||
await self._record_quietly(self._quota.mark_progress())
|
||||
self._feed_outcome(source.name, ok=True)
|
||||
response = self._build_response(source, result, call_id, started)
|
||||
await self._emit(request, source, call_id, started, response=response)
|
||||
return response
|
||||
@@ -251,8 +269,8 @@ class RetryMW:
|
||||
await self._emit(request, source, call_id, started, error=exc)
|
||||
raise
|
||||
except ResultInvalidError as exc:
|
||||
# 坏结果 ≠ 坏服务: 熔断记成功,异常上抛消耗业务失败预算(§6.3)
|
||||
await self._record_quietly(self._breaker.record_success(entry))
|
||||
# 坏结果 ≠ 坏服务: 熔断记成功但不计窗口样本,亦不喂健康分(M2.5 §3.1)
|
||||
await self._record_quietly(self._breaker.record_success(entry, count_attempt=False))
|
||||
await self._emit(request, source, call_id, started, error=exc)
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
@@ -264,6 +282,8 @@ class RetryMW:
|
||||
dead = isinstance(exc, SourceDeadError)
|
||||
reason = _failure_reason(exc)
|
||||
reasons[source.name] = reason
|
||||
attempt_fails[source.name] = attempt_fails.get(source.name, 0) + 1
|
||||
self._feed_outcome(source.name, ok=False)
|
||||
await self._record_quietly(self._breaker.record_failure(entry, reason, dead))
|
||||
if not dead:
|
||||
actual = source.est_tokens # 保守: 失败请求可能已被网关计费(CHS 同款)
|
||||
@@ -277,8 +297,8 @@ class RetryMW:
|
||||
) -> None:
|
||||
provider_responded = exc.source_name == source.name and exc.status_code is not None
|
||||
if provider_responded:
|
||||
# 网关健康地拒了坏请求
|
||||
await self._record_quietly(self._breaker.record_success(entry))
|
||||
# 网关健康地拒了坏请求: 记成功但不计窗口样本(M2.5 §3.1)
|
||||
await self._record_quietly(self._breaker.record_success(entry, count_attempt=False))
|
||||
elif entry.is_probe:
|
||||
await self._record_quietly(self._breaker.release_probe(entry))
|
||||
|
||||
@@ -296,6 +316,15 @@ class RetryMW:
|
||||
except GovernanceBackendError as exc:
|
||||
logger.warning("治理记账写回降级(不冒泡): {}", exc)
|
||||
|
||||
def _feed_outcome(self, source_name: str, ok: bool) -> None:
|
||||
"""健康喂数降级执行: 选源器异常不得打断真实成功/失败的主路径(设计 §4)。"""
|
||||
if self._outcome_sink is None:
|
||||
return
|
||||
try:
|
||||
self._outcome_sink.record_outcome(source_name, ok)
|
||||
except Exception as exc:
|
||||
logger.warning("选源健康喂数失败(降级不冒泡): {}", exc)
|
||||
|
||||
# —— 辅助 ——
|
||||
|
||||
def _backoff_delay(self, fails: int, exc: PolyGatewayError) -> float:
|
||||
|
||||
Reference in New Issue
Block a user