feat: exempt 429 pushback from retry budget with stall ceiling
Round 6 hit account-level rate throttling the concurrency AIMD cannot absorb: at 26 req/min the gateway still returned 16% 429s and each one burned a third of the retry budget. Retry-After-guided 429s now back off without consuming attempts (gRPC pushback semantics); the retry loop gains a per-call ceiling using the same dual-condition stall verdict as quota-wait (local window exceeded AND no global progress).
This commit is contained in:
@@ -210,6 +210,16 @@ class RetryMW:
|
||||
attempt_fails: dict[str, int] = {}
|
||||
entered_at = self._now() # 调用级累计计时,循环内不重置(CHS governance.py:207)
|
||||
while True:
|
||||
# 调用级时间上限(迭代 5): 429 免预算后的兜底,防饱和期无限循环。
|
||||
# 与 _on_no_runnable 同款双条件(CHS 口径): 本地超窗且全局无进展才判死
|
||||
stall = self._bp.stall_window_s
|
||||
if self._now() - entered_at > stall and await self._quota.progress_age_s() > stall:
|
||||
raise AllSourcesExhausted(
|
||||
scope=self._scope,
|
||||
reason="stalled",
|
||||
retry_after_s=self._retry.backoff_base_s,
|
||||
per_source_reasons=reasons,
|
||||
)
|
||||
picked, gate_rejections = await self._pick_runnable(reasons, attempt_fails)
|
||||
if picked is None:
|
||||
await self._on_no_runnable(gate_rejections, reasons, entered_at)
|
||||
@@ -217,7 +227,10 @@ class RetryMW:
|
||||
outcome = await self._attempt(request, *picked, reasons, attempt_fails)
|
||||
if isinstance(outcome, LLMResponse):
|
||||
return outcome
|
||||
fails += 1
|
||||
# 429 = 服务端调度指令(gRPC pushback 语义,迭代 5): 按 Retry-After
|
||||
# 退避但不消耗重试预算——饱和窗口里等待而非死亡;其余失败照常计数
|
||||
if _failure_reason(outcome.exc) != "rate_limited":
|
||||
fails += 1
|
||||
if fails >= self._retry.max_attempts:
|
||||
raise AllSourcesExhausted(
|
||||
scope=self._scope,
|
||||
@@ -226,7 +239,7 @@ class RetryMW:
|
||||
per_source_reasons=reasons,
|
||||
) from outcome.exc
|
||||
if not outcome.immediate:
|
||||
await self._sleep(self._backoff_delay(fails, outcome.exc))
|
||||
await self._sleep(self._backoff_delay(max(fails, 1), outcome.exc))
|
||||
|
||||
# —— 选源与准入(CHS _pick_runnable 120-167)——
|
||||
|
||||
|
||||
Reference in New Issue
Block a user