fix: bill only non-productive waiting against the chat stall budget

Issue #8: with timeout_s >= stall_window_s a single timed-out request
exhausted the stall window before the second attempt was even dispatched,
so LLM_MAX_RETRIES never applied and the whole scope was declared dead.

Root cause is that real attempts and non-productive waiting charged the
same wall clock, while the stall budget is the smaller of the two. The new
StallClock subtracts attempt time from the stall account, leaving the two
budgets orthogonal: attempts bill max_attempts, waiting bills
stall_window_s. The dual-condition verdict, the inf semantics of
progress_age_s, the 429 exemption and the error surface are untouched.

The productive boundary is _attempt itself, telemetry included, so a slow
recorder cannot push a call into a stalled verdict.
This commit is contained in:
2026-08-06 09:20:21 -04:00
parent 573e505a4b
commit 02c3d06ec6
2 changed files with 227 additions and 10 deletions
+49 -7
View File
@@ -11,6 +11,7 @@ httpx 是库的核心依赖而非实现层内部件,不违反"middleware 只依
from __future__ import annotations
import asyncio
import contextlib
import random
import time
import uuid
@@ -39,7 +40,7 @@ from polygateway.streaming import StreamLivenessTimeout
from polygateway.types import LLMResponse
if TYPE_CHECKING:
from collections.abc import Awaitable, Callable
from collections.abc import AsyncIterator, Awaitable, Callable
from polygateway.ports import (
GateDecision,
@@ -74,6 +75,44 @@ def backoff_delay(
return max(delay, retry_after)
class StallClock:
"""调用级 stall 计时器: 只累计非生产性等待(issue #8 设计 §3.1)。
stall 预算治理的是"无人治理的等待"——429 退避、配额 wait 轮询、熔断冷却;
真实尝试的耗时已由重试预算 `max_attempts` 治理,必须从 stall 账里扣除。
两者重叠计费正是 issue #8 的根因: stall 预算(默认 300s)小于重试预算
(3 × timeout_s),必然先耗尽,于是重试预算在超时场景下永远用不上。
"生产性"的边界即 `_attempt` 的边界,含该次尝试的记账与遥测收尾——它们是
"尝试已有结论"之后的动作,不是在等待重试机会;把它们计入 stall 会让遥测
抖动参与判死。
每次调用创建一个实例。严禁提升为实例属性: 并发调用共享会互相污染计时。
模块级共享单元, EmbeddingClient 与 OcrClient 复用(同 `backoff_delay`)。
"""
__slots__ = ("_now", "_entered_at", "_productive_s")
def __init__(self, now: Callable[[], float]) -> None:
self._now = now
self._entered_at = now()
self._productive_s = 0.0
def stalled_s(self) -> float:
"""非生产性等待累计秒数 = 调用总耗时 - 真实尝试耗时。"""
return self._now() - self._entered_at - self._productive_s
@contextlib.asynccontextmanager
async def attempting(self) -> AsyncIterator[None]:
"""包裹一次真实尝试,其耗时记为生产性。"""
started = self._now()
try:
yield
finally:
# 只做算术, 不吞任何异常——CancelledError 逐字穿透(库铁律)
self._productive_s += self._now() - started
def _demote_call_failures(
ordered: list[SourceConfig],
attempt_fails: dict[str, int],
@@ -209,12 +248,14 @@ class RetryMW:
reasons: dict[str, str] = {}
# 调用内失败计数(设计 §3.3): 局部状态,调用结束即弃;严禁实例属性(并发共享)
attempt_fails: dict[str, int] = {}
entered_at = self._now() # 调用级累计计时,循环内不重置(CHS governance.py:207)
# 调用级累计计时,循环内不重置(CHS governance.py:207);issue #8 起只计
# 非生产性等待——真实尝试由重试预算治理,不再重复烧 stall 预算
clock = StallClock(self._now)
while True:
# 调用级时间上限(迭代 5): 429 免预算后的兜底,防饱和期无限循环。
# 与 _on_no_runnable 同款双条件(CHS 口径): 本地超窗且全局无进展才判死
stall = self._bp.stall_window_s
if self._now() - entered_at > stall and await self._quota.progress_age_s() > stall:
if clock.stalled_s() > stall and await self._quota.progress_age_s() > stall:
raise AllSourcesExhausted(
scope=self._scope,
reason="stalled",
@@ -223,9 +264,10 @@ class RetryMW:
)
picked, gate_rejections = await self._pick_runnable(reasons, attempt_fails)
if picked is None:
await self._on_no_runnable(gate_rejections, reasons, entered_at)
await self._on_no_runnable(gate_rejections, reasons, clock)
continue
outcome = await self._attempt(request, *picked, reasons, attempt_fails)
async with clock.attempting():
outcome = await self._attempt(request, *picked, reasons, attempt_fails)
if isinstance(outcome, LLMResponse):
return outcome
# 429 = 服务端调度指令(gRPC pushback 语义,迭代 5): 按 Retry-After
@@ -284,7 +326,7 @@ class RetryMW:
return None, gate_rejections
async def _on_no_runnable(
self, gate_rejections: int, reasons: dict[str, str], entered_at: float
self, gate_rejections: int, reasons: dict[str, str], clock: StallClock
) -> None:
if gate_rejections == len(self._sources):
names = tuple(s.name for s in self._sources)
@@ -303,7 +345,7 @@ class RetryMW:
# 双条件 stall 判死(CHS governance.py:270-281): 本地累计等待与全局
# 无进展**同时**超窗才判死——本地 monotonic 与后端时钟刻意不混用。
stall = self._bp.stall_window_s
if self._now() - entered_at > stall and await self._quota.progress_age_s() > stall:
if clock.stalled_s() > stall and await self._quota.progress_age_s() > stall:
names = tuple(s.name for s in self._sources)
raise AllSourcesExhausted(
scope=self._scope,