fix: bill only non-productive waiting against the chat stall budget
Issue #8: with timeout_s >= stall_window_s a single timed-out request exhausted the stall window before the second attempt was even dispatched, so LLM_MAX_RETRIES never applied and the whole scope was declared dead. Root cause is that real attempts and non-productive waiting charged the same wall clock, while the stall budget is the smaller of the two. The new StallClock subtracts attempt time from the stall account, leaving the two budgets orthogonal: attempts bill max_attempts, waiting bills stall_window_s. The dual-condition verdict, the inf semantics of progress_age_s, the 429 exemption and the error surface are untouched. The productive boundary is _attempt itself, telemetry included, so a slow recorder cannot push a call into a stalled verdict.
This commit is contained in:
@@ -11,6 +11,7 @@ httpx 是库的核心依赖而非实现层内部件,不违反"middleware 只依
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
@@ -39,7 +40,7 @@ from polygateway.streaming import StreamLivenessTimeout
|
||||
from polygateway.types import LLMResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Awaitable, Callable
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable
|
||||
|
||||
from polygateway.ports import (
|
||||
GateDecision,
|
||||
@@ -74,6 +75,44 @@ def backoff_delay(
|
||||
return max(delay, retry_after)
|
||||
|
||||
|
||||
class StallClock:
|
||||
"""调用级 stall 计时器: 只累计非生产性等待(issue #8 设计 §3.1)。
|
||||
|
||||
stall 预算治理的是"无人治理的等待"——429 退避、配额 wait 轮询、熔断冷却;
|
||||
真实尝试的耗时已由重试预算 `max_attempts` 治理,必须从 stall 账里扣除。
|
||||
两者重叠计费正是 issue #8 的根因: stall 预算(默认 300s)小于重试预算
|
||||
(3 × timeout_s),必然先耗尽,于是重试预算在超时场景下永远用不上。
|
||||
|
||||
"生产性"的边界即 `_attempt` 的边界,含该次尝试的记账与遥测收尾——它们是
|
||||
"尝试已有结论"之后的动作,不是在等待重试机会;把它们计入 stall 会让遥测
|
||||
抖动参与判死。
|
||||
|
||||
每次调用创建一个实例。严禁提升为实例属性: 并发调用共享会互相污染计时。
|
||||
模块级共享单元, EmbeddingClient 与 OcrClient 复用(同 `backoff_delay`)。
|
||||
"""
|
||||
|
||||
__slots__ = ("_now", "_entered_at", "_productive_s")
|
||||
|
||||
def __init__(self, now: Callable[[], float]) -> None:
|
||||
self._now = now
|
||||
self._entered_at = now()
|
||||
self._productive_s = 0.0
|
||||
|
||||
def stalled_s(self) -> float:
|
||||
"""非生产性等待累计秒数 = 调用总耗时 - 真实尝试耗时。"""
|
||||
return self._now() - self._entered_at - self._productive_s
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def attempting(self) -> AsyncIterator[None]:
|
||||
"""包裹一次真实尝试,其耗时记为生产性。"""
|
||||
started = self._now()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
# 只做算术, 不吞任何异常——CancelledError 逐字穿透(库铁律)
|
||||
self._productive_s += self._now() - started
|
||||
|
||||
|
||||
def _demote_call_failures(
|
||||
ordered: list[SourceConfig],
|
||||
attempt_fails: dict[str, int],
|
||||
@@ -209,12 +248,14 @@ class RetryMW:
|
||||
reasons: dict[str, str] = {}
|
||||
# 调用内失败计数(设计 §3.3): 局部状态,调用结束即弃;严禁实例属性(并发共享)
|
||||
attempt_fails: dict[str, int] = {}
|
||||
entered_at = self._now() # 调用级累计计时,循环内不重置(CHS governance.py:207)
|
||||
# 调用级累计计时,循环内不重置(CHS governance.py:207);issue #8 起只计
|
||||
# 非生产性等待——真实尝试由重试预算治理,不再重复烧 stall 预算
|
||||
clock = StallClock(self._now)
|
||||
while True:
|
||||
# 调用级时间上限(迭代 5): 429 免预算后的兜底,防饱和期无限循环。
|
||||
# 与 _on_no_runnable 同款双条件(CHS 口径): 本地超窗且全局无进展才判死
|
||||
stall = self._bp.stall_window_s
|
||||
if self._now() - entered_at > stall and await self._quota.progress_age_s() > stall:
|
||||
if clock.stalled_s() > stall and await self._quota.progress_age_s() > stall:
|
||||
raise AllSourcesExhausted(
|
||||
scope=self._scope,
|
||||
reason="stalled",
|
||||
@@ -223,9 +264,10 @@ class RetryMW:
|
||||
)
|
||||
picked, gate_rejections = await self._pick_runnable(reasons, attempt_fails)
|
||||
if picked is None:
|
||||
await self._on_no_runnable(gate_rejections, reasons, entered_at)
|
||||
await self._on_no_runnable(gate_rejections, reasons, clock)
|
||||
continue
|
||||
outcome = await self._attempt(request, *picked, reasons, attempt_fails)
|
||||
async with clock.attempting():
|
||||
outcome = await self._attempt(request, *picked, reasons, attempt_fails)
|
||||
if isinstance(outcome, LLMResponse):
|
||||
return outcome
|
||||
# 429 = 服务端调度指令(gRPC pushback 语义,迭代 5): 按 Retry-After
|
||||
@@ -284,7 +326,7 @@ class RetryMW:
|
||||
return None, gate_rejections
|
||||
|
||||
async def _on_no_runnable(
|
||||
self, gate_rejections: int, reasons: dict[str, str], entered_at: float
|
||||
self, gate_rejections: int, reasons: dict[str, str], clock: StallClock
|
||||
) -> None:
|
||||
if gate_rejections == len(self._sources):
|
||||
names = tuple(s.name for s in self._sources)
|
||||
@@ -303,7 +345,7 @@ class RetryMW:
|
||||
# 双条件 stall 判死(CHS governance.py:270-281): 本地累计等待与全局
|
||||
# 无进展**同时**超窗才判死——本地 monotonic 与后端时钟刻意不混用。
|
||||
stall = self._bp.stall_window_s
|
||||
if self._now() - entered_at > stall and await self._quota.progress_age_s() > stall:
|
||||
if clock.stalled_s() > stall and await self._quota.progress_age_s() > stall:
|
||||
names = tuple(s.name for s in self._sources)
|
||||
raise AllSourcesExhausted(
|
||||
scope=self._scope,
|
||||
|
||||
Reference in New Issue
Block a user