fix: bill only non-productive waiting against the chat stall budget
Issue #8: with timeout_s >= stall_window_s a single timed-out request exhausted the stall window before the second attempt was even dispatched, so LLM_MAX_RETRIES never applied and the whole scope was declared dead. Root cause is that real attempts and non-productive waiting charged the same wall clock, while the stall budget is the smaller of the two. The new StallClock subtracts attempt time from the stall account, leaving the two budgets orthogonal: attempts bill max_attempts, waiting bills stall_window_s. The dual-condition verdict, the inf semantics of progress_age_s, the 429 exemption and the error surface are untouched. The productive boundary is _attempt itself, telemetry included, so a slow recorder cannot push a call into a stalled verdict.
This commit is contained in:
@@ -53,19 +53,31 @@ class BoundedSleep:
|
||||
await self._side_effect(len(self.delays))
|
||||
|
||||
|
||||
def _mw(sources, limiter, script, *, clock, sleep, rng=lambda: 0.0, quota_full="wait", gate=None):
|
||||
def _mw(
|
||||
sources,
|
||||
limiter,
|
||||
script,
|
||||
*,
|
||||
clock,
|
||||
sleep,
|
||||
rng=lambda: 0.0,
|
||||
quota_full="wait",
|
||||
gate=None,
|
||||
transport=None,
|
||||
emitter=None,
|
||||
):
|
||||
return RetryMW(
|
||||
scope="llm",
|
||||
sources=sources,
|
||||
selector=RoundRobinSelector(),
|
||||
limiter=limiter,
|
||||
gate=gate or InMemoryGate(config=_BREAKER, now=clock),
|
||||
transport=FakeTransport(script),
|
||||
transport=transport or FakeTransport(script),
|
||||
retry=RetryPolicy(max_attempts=3, backoff_base_s=2.0, backoff_max_s=30.0),
|
||||
backpressure=BackpressurePolicy(stall_window_s=_STALL, poll_interval_s=0.01),
|
||||
quota_full=quota_full,
|
||||
cooldown_memo=SourceCooldownMemo(now=clock),
|
||||
emitter=None,
|
||||
emitter=emitter,
|
||||
now=clock,
|
||||
sleep=sleep,
|
||||
rng=rng,
|
||||
@@ -178,6 +190,169 @@ class TestStallQuadrants:
|
||||
await task
|
||||
|
||||
|
||||
class ClockAdvancingTransport:
|
||||
"""按脚本 [(推进秒数, 动作), ...] 执行: 在一次尝试内部推进时钟, 模拟真实耗时。
|
||||
|
||||
动作语义同 `FakeTransport`(异常即抛、"hang" 即挂起、其余为返回值)。
|
||||
stall 口径的关键区分在于"时间花在哪", 故必须能让时钟只在 transport 内前进。
|
||||
"""
|
||||
|
||||
def __init__(self, script, clock):
|
||||
self.script = list(script)
|
||||
self.clock = clock
|
||||
self.calls = []
|
||||
|
||||
async def complete(self, *, messages, source, stream, overlay, call_id):
|
||||
self.calls.append((source.name, call_id))
|
||||
advance, action = self.script.pop(0)
|
||||
self.clock.advance(advance)
|
||||
if isinstance(action, Exception):
|
||||
raise action
|
||||
if action == "hang":
|
||||
await asyncio.Event().wait()
|
||||
return action
|
||||
|
||||
|
||||
class _SlowEmitter:
|
||||
"""遥测收尾中推进时钟: 钉住"遥测耗时属生产性"(设计 §3.1 边界声明)。"""
|
||||
|
||||
def __init__(self, clock, advance):
|
||||
self._clock = clock
|
||||
self._advance = advance
|
||||
|
||||
async def emit_attempt(self, *args, **kwargs):
|
||||
self._clock.advance(self._advance)
|
||||
|
||||
|
||||
class TestStallBudget:
|
||||
"""stall 预算只计非生产性等待(issue #8 设计 §3.1)。
|
||||
|
||||
根因是两个预算重叠计费: 真实尝试的耗时同时烧重试预算与 stall 预算,
|
||||
而 stall 预算更小必然先耗尽, 于是 max_attempts 在超时场景下永不生效。
|
||||
"""
|
||||
|
||||
def _free_limiter(self, clock):
|
||||
src = make_source()
|
||||
limiter = InMemoryLimiter(
|
||||
scope="llm", sources={"s1": src}, global_limits=_NO_GLOBAL, now=clock
|
||||
)
|
||||
return src, limiter
|
||||
|
||||
async def test_single_timeout_does_not_exhaust_stall_budget(self):
|
||||
"""timeout_s == stall_window_s 时, 一次超时不得判死——重试预算须真实可用。"""
|
||||
clock = FakeClock()
|
||||
src, limiter = self._free_limiter(clock)
|
||||
# 第一次尝试耗满 300s 超时后失败, 第二次立即成功
|
||||
transport = ClockAdvancingTransport(
|
||||
[(_STALL + 1, TransientError("timeout", status_code=504)), (0.0, _ok())], clock
|
||||
)
|
||||
mw = _mw([src], limiter, [], clock=clock, sleep=BoundedSleep(), transport=transport)
|
||||
resp = await mw(_REQ)
|
||||
assert resp.content == "ok"
|
||||
assert len(transport.calls) == 2 # 第二次尝试确实发出了
|
||||
|
||||
async def test_productive_time_excluded_from_stall(self):
|
||||
"""连续多次长尝试也不烧 stall 预算: 它们烧的是重试预算。"""
|
||||
clock = FakeClock()
|
||||
src, limiter = self._free_limiter(clock)
|
||||
transport = ClockAdvancingTransport(
|
||||
[
|
||||
(_STALL + 100, TransientError("slow", status_code=500)),
|
||||
(_STALL + 100, TransientError("slow", status_code=500)),
|
||||
(0.0, _ok()),
|
||||
],
|
||||
clock,
|
||||
)
|
||||
mw = _mw([src], limiter, [], clock=clock, sleep=BoundedSleep(), transport=transport)
|
||||
resp = await mw(_REQ)
|
||||
assert resp.content == "ok"
|
||||
|
||||
async def test_telemetry_time_counts_as_productive(self):
|
||||
"""遥测收尾属 `_attempt` 边界内: 遥测抖动不得参与判死(设计 §3.1)。
|
||||
|
||||
必须走**失败**路径才有判别力: 成功后直接 return, 循环开头的 stall
|
||||
判定根本不会再执行。此处让首次尝试快速失败、而遥测收尾慢得超窗,
|
||||
下一轮循环开头即检验遥测耗时有没有被算进 stall 账。
|
||||
"""
|
||||
clock = FakeClock()
|
||||
src, limiter = self._free_limiter(clock)
|
||||
transport = ClockAdvancingTransport(
|
||||
[(0.1, TransientError("boom", status_code=500)), (0.0, _ok())], clock
|
||||
)
|
||||
mw = _mw(
|
||||
[src],
|
||||
limiter,
|
||||
[],
|
||||
clock=clock,
|
||||
sleep=BoundedSleep(),
|
||||
transport=transport,
|
||||
emitter=_SlowEmitter(clock, _STALL + 100),
|
||||
)
|
||||
resp = await mw(_REQ)
|
||||
assert resp.content == "ok"
|
||||
|
||||
async def test_nonproductive_wait_still_triggers_stall(self):
|
||||
"""兜底未被削弱: 纯轮询等待累满窗口仍判死。"""
|
||||
clock = FakeClock()
|
||||
src, limiter = _blocked_limiter(clock)
|
||||
_held = await limiter.try_acquire("s1", 0)
|
||||
|
||||
async def advance(_n):
|
||||
clock.advance(_STALL + 100)
|
||||
|
||||
mw = _mw([src], limiter, [], clock=clock, sleep=BoundedSleep(advance))
|
||||
with pytest.raises(AllSourcesExhausted) as ei:
|
||||
await mw(_REQ)
|
||||
assert ei.value.reason == "stalled"
|
||||
|
||||
async def test_saturation_429_still_stalls(self):
|
||||
"""429 免预算不烧 fails, 主循环兜底须仍能判死而非无限循环(设计 §3.5)。"""
|
||||
clock = FakeClock()
|
||||
src, limiter = self._free_limiter(clock)
|
||||
# 429 往返本身极快(生产性可忽略), 退避 sleep 才是非生产性的大头
|
||||
transport = ClockAdvancingTransport(
|
||||
[(0.1, TransientError("429", status_code=429)) for _ in range(10)], clock
|
||||
)
|
||||
|
||||
async def advance(_n):
|
||||
clock.advance(_STALL)
|
||||
|
||||
mw = _mw(
|
||||
[src], limiter, [], clock=clock, sleep=BoundedSleep(advance), transport=transport
|
||||
)
|
||||
with pytest.raises(AllSourcesExhausted) as ei:
|
||||
await mw(_REQ)
|
||||
assert ei.value.reason == "stalled" # 不是 retry_exhausted: 429 确实没烧重试预算
|
||||
|
||||
async def test_cancel_inside_attempt_pierces(self):
|
||||
"""取消发生在 `attempting()` 包裹内仍逐字穿透(库铁律)。"""
|
||||
clock = FakeClock()
|
||||
src, limiter = self._free_limiter(clock)
|
||||
transport = ClockAdvancingTransport([(0.0, "hang")], clock)
|
||||
mw = _mw([src], limiter, [], clock=clock, sleep=asyncio.sleep, transport=transport)
|
||||
task = asyncio.create_task(mw(_REQ))
|
||||
while not transport.calls:
|
||||
await asyncio.sleep(0.01)
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert (await limiter.source_stats("s1")).inflight == 0 # permit 在 finally 释放
|
||||
|
||||
async def test_concurrent_calls_do_not_share_clock(self):
|
||||
"""StallClock 必须是调用级局部状态: 一路长尝试不得污染另一路的 stall 账。"""
|
||||
clock = FakeClock()
|
||||
src = make_source(max_concurrency=2)
|
||||
limiter = InMemoryLimiter(
|
||||
scope="llm", sources={"s1": src}, global_limits=_NO_GLOBAL, now=clock
|
||||
)
|
||||
slow = ClockAdvancingTransport([(_STALL + 100, _ok("slow"))], clock)
|
||||
fast = ClockAdvancingTransport([(0.0, _ok("fast"))], clock)
|
||||
mw_slow = _mw([src], limiter, [], clock=clock, sleep=BoundedSleep(), transport=slow)
|
||||
mw_fast = _mw([src], limiter, [], clock=clock, sleep=BoundedSleep(), transport=fast)
|
||||
results = await asyncio.gather(mw_slow(_REQ), mw_fast(_REQ))
|
||||
assert {r.content for r in results} == {"slow", "fast"}
|
||||
|
||||
|
||||
class _GateSuccessBroken(InMemoryGate):
|
||||
async def record_success(self, entry):
|
||||
raise GovernanceBackendError("redis 抖动", scope="llm")
|
||||
|
||||
Reference in New Issue
Block a user