feat: feed selector health and demote in-call failed sources

RetryMW keeps a per-call failure map (local, never instance state):
a source failing twice in one call yields to the next candidate.
Attempt outcomes feed OutcomeAwareSelector behind a swallow-and-warn
guard; ResultInvalid and provider-rejected paths record success with
count_attempt=False so the breaker window stays clean. Same accounting
applied in EmbeddingClient.
This commit is contained in:
2026-07-21 09:55:08 -04:00
parent e69dc05fd5
commit 4c2a148db9
5 changed files with 139 additions and 13 deletions
+4 -2
View File
@@ -25,9 +25,11 @@ class BreakerGate:
except Exception as exc:
raise GovernanceBackendError(f"熔断后端故障(try_enter): {exc}") from exc
async def record_success(self, entry: GateDecision) -> GateUpdate:
async def record_success(
self, entry: GateDecision, *, count_attempt: bool = True
) -> GateUpdate:
try:
return await self._gate.record_success(entry)
return await self._gate.record_success(entry, count_attempt=count_attempt)
except GovernanceBackendError:
raise
except Exception as exc:
+37 -8
View File
@@ -32,6 +32,7 @@ from polygateway.errors import (
)
from polygateway.middleware.breaker import BreakerGate
from polygateway.middleware.ratelimit import QuotaGate
from polygateway.ports import OutcomeAwareSelector
from polygateway.sources import SourceCooldownMemo
from polygateway.streaming import StreamLivenessTimeout
from polygateway.types import LLMResponse
@@ -72,6 +73,16 @@ def backoff_delay(
return max(delay, retry_after)
def _demote_call_failures(
ordered: list[SourceConfig], attempt_fails: dict[str, int]
) -> list[SourceConfig]:
"""调用内降权(设计 §3.3): 本调用失败 ≥2 次的源移尾;全失败则保持原序。"""
demoted = [s for s in ordered if attempt_fails.get(s.name, 0) >= 2]
if not demoted or len(demoted) == len(ordered):
return ordered
return [s for s in ordered if attempt_fails.get(s.name, 0) < 2] + demoted
def _failure_reason(exc: PolyGatewayError) -> str:
"""失败原因归类(CHS governance.py:169 同款)。"""
if isinstance(exc, SourceDeadError):
@@ -124,6 +135,8 @@ class RetryMW:
self._bp = backpressure
self._quota_full = quota_full
self._memo = cooldown_memo or SourceCooldownMemo(now=now)
# M2.5: 选源器可选健康喂数端口,构造期 isinstance 判定一次(设计 §3.2)
self._outcome_sink = selector if isinstance(selector, OutcomeAwareSelector) else None
self._emitter = emitter
self._now = now
self._sleep = sleep
@@ -135,13 +148,15 @@ class RetryMW:
raise AllSourcesExhausted(scope=self._scope, reason="no_sources", retry_after_s=0.0)
fails = 0
reasons: dict[str, str] = {}
# 调用内失败计数(设计 §3.3): 局部状态,调用结束即弃;严禁实例属性(并发共享)
attempt_fails: dict[str, int] = {}
entered_at = self._now() # 调用级累计计时,循环内不重置(CHS governance.py:207)
while True:
picked, gate_rejections = await self._pick_runnable(reasons)
picked, gate_rejections = await self._pick_runnable(reasons, attempt_fails)
if picked is None:
await self._on_no_runnable(gate_rejections, reasons, entered_at)
continue
outcome = await self._attempt(request, *picked, reasons)
outcome = await self._attempt(request, *picked, reasons, attempt_fails)
if isinstance(outcome, LLMResponse):
return outcome
fails += 1
@@ -158,11 +173,12 @@ class RetryMW:
# —— 选源与准入(CHS _pick_runnable 120-167)——
async def _pick_runnable(
self, reasons: dict[str, str]
self, reasons: dict[str, str], attempt_fails: dict[str, int]
) -> tuple[tuple[SourceConfig, Permit, GateDecision] | None, int]:
stats = {s.name: await self._quota.stats(s) for s in self._sources}
gate_rejections = 0
for cand in self._selector.order(self._sources, stats):
ordered = _demote_call_failures(self._selector.order(self._sources, stats), attempt_fails)
for cand in ordered:
if self._memo.active(cand.name):
# 冷却备忘跳过也计入拒绝数,保住 circuit_open 判据(CHS 同款)
gate_rejections += 1
@@ -228,6 +244,7 @@ class RetryMW:
permit: Permit,
entry: GateDecision,
reasons: dict[str, str],
attempt_fails: dict[str, int],
) -> LLMResponse | _Failed:
call_id = str(uuid.uuid4())
started = self._now()
@@ -243,6 +260,7 @@ class RetryMW:
actual = result.prompt_tokens + result.completion_tokens
await self._record_quietly(self._breaker.record_success(entry))
await self._record_quietly(self._quota.mark_progress())
self._feed_outcome(source.name, ok=True)
response = self._build_response(source, result, call_id, started)
await self._emit(request, source, call_id, started, response=response)
return response
@@ -251,8 +269,8 @@ class RetryMW:
await self._emit(request, source, call_id, started, error=exc)
raise
except ResultInvalidError as exc:
# 坏结果 ≠ 坏服务: 熔断记成功,异常上抛消耗业务失败预算(§6.3)
await self._record_quietly(self._breaker.record_success(entry))
# 坏结果 ≠ 坏服务: 熔断记成功但不计窗口样本,亦不喂健康分(M2.5 §3.1)
await self._record_quietly(self._breaker.record_success(entry, count_attempt=False))
await self._emit(request, source, call_id, started, error=exc)
raise
except asyncio.CancelledError:
@@ -264,6 +282,8 @@ class RetryMW:
dead = isinstance(exc, SourceDeadError)
reason = _failure_reason(exc)
reasons[source.name] = reason
attempt_fails[source.name] = attempt_fails.get(source.name, 0) + 1
self._feed_outcome(source.name, ok=False)
await self._record_quietly(self._breaker.record_failure(entry, reason, dead))
if not dead:
actual = source.est_tokens # 保守: 失败请求可能已被网关计费(CHS 同款)
@@ -277,8 +297,8 @@ class RetryMW:
) -> None:
provider_responded = exc.source_name == source.name and exc.status_code is not None
if provider_responded:
# 网关健康地拒了坏请求
await self._record_quietly(self._breaker.record_success(entry))
# 网关健康地拒了坏请求: 记成功但不计窗口样本(M2.5 §3.1)
await self._record_quietly(self._breaker.record_success(entry, count_attempt=False))
elif entry.is_probe:
await self._record_quietly(self._breaker.release_probe(entry))
@@ -296,6 +316,15 @@ class RetryMW:
except GovernanceBackendError as exc:
logger.warning("治理记账写回降级(不冒泡): {}", exc)
def _feed_outcome(self, source_name: str, ok: bool) -> None:
"""健康喂数降级执行: 选源器异常不得打断真实成功/失败的主路径(设计 §4)。"""
if self._outcome_sink is None:
return
try:
self._outcome_sink.record_outcome(source_name, ok)
except Exception as exc:
logger.warning("选源健康喂数失败(降级不冒泡): {}", exc)
# —— 辅助 ——
def _backoff_delay(self, fails: int, exc: PolyGatewayError) -> float: