From 0a6d6225db835c64266a6f4c6bdcd314e0914fba Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 10 Sep 2026 13:16:38 -0400 Subject: [PATCH 1/7] feat: add a required first-token event to the transport port Transport.complete gains the keyword-only first_token_event (no default, per the port convention): streaming sets it on the first delta, the non-streaming path accepts it but never sets it, None means the caller does not observe the first token. All fake/wrapping transports and the three direct call sites follow the signature; the e2e wrapper forwards. Red-green evidence: tests/outputs/137/t1/ (batch A TypeError red, then 147 file tests + 1550 unit tests green). --- src/polygateway/middleware/retry.py | 2 + src/polygateway/ports.py | 6 ++ src/polygateway/transports/openai_compat.py | 24 ++++++- tests/e2e/conftest.py | 3 + .../test_redis_cross_connection.py | 4 +- tests/unit/test_backpressure.py | 4 +- tests/unit/test_client.py | 4 +- tests/unit/test_live_evidence.py | 2 + tests/unit/test_openai_compat.py | 65 ++++++++++++++++++- tests/unit/test_ports.py | 4 +- tests/unit/test_retry.py | 4 +- tests/unit/test_usage_source_domain.py | 1 + 12 files changed, 114 insertions(+), 9 deletions(-) diff --git a/src/polygateway/middleware/retry.py b/src/polygateway/middleware/retry.py index 9e24d96..9a8a782 100644 --- a/src/polygateway/middleware/retry.py +++ b/src/polygateway/middleware/retry.py @@ -297,6 +297,8 @@ class RetryMW: # 逐次尝试原样重传: 换源不改变调用方要的档位(源级默认由 transport # 自己按选中的源解析,两者在 effective_effort 里汇合) reasoning_effort=request.reasoning_effort, + # T3 对冲编排接线前恒为 None: 调用方不观测首 token(计划 §3.1) + first_token_event=None, ) if result.usage_source == "unavailable": # 用量不可得时按入场预扣量结算(delta==0),否则押金会被整笔退回, diff --git a/src/polygateway/ports.py b/src/polygateway/ports.py index a8397d8..7b595a1 100644 --- a/src/polygateway/ports.py +++ b/src/polygateway/ports.py @@ -5,6 +5,7 @@ 时间量纲一律**秒**(CHS Redis 实现内部的毫秒换算是后端私事,不进契约)。 """ +import asyncio from collections.abc import Awaitable, Callable from dataclasses import dataclass from enum import StrEnum @@ -46,6 +47,10 @@ class Transport(Protocol): 该参数**不设默认值**,与 `TelemetryRecorder.record_llm_call` 同一既有约定: 库外无第三方实现者,写全签名的成本为零,而默认值会把"某一层漏传"变成静默的 "调用方没表态"——一次本该报错的漏配就此变成一次悄悄涨价的调用。 + + `first_token_event` 同一约定(1.3.7 对冲 H2): `None` = 调用方不观测首 token + (未启用对冲);非流式实现**永不置位**(物理上无中途信号,事件自然退化为纯时间 + 阈值),流式实现在首个增量(内容或思考)到达时置位。 """ async def complete( @@ -57,6 +62,7 @@ class Transport(Protocol): overlay: dict[str, Any], call_id: str, reasoning_effort: Effort | None, + first_token_event: asyncio.Event | None, ) -> TransportResult: ... diff --git a/src/polygateway/transports/openai_compat.py b/src/polygateway/transports/openai_compat.py index e09572c..554141b 100644 --- a/src/polygateway/transports/openai_compat.py +++ b/src/polygateway/transports/openai_compat.py @@ -46,6 +46,7 @@ from polygateway.types import ( ) if TYPE_CHECKING: + import asyncio from collections.abc import AsyncIterator, Callable, Mapping _THINK_PATTERN = re.compile(r"(.*?)", re.DOTALL) @@ -432,11 +433,14 @@ class OpenAICompatTransport: overlay: dict[str, Any], call_id: str, reasoning_effort: Effort | None, + first_token_event: asyncio.Event | None, ) -> TransportResult: """一次原始调用;HTTP/线路/流式异常按 ARCH §6.2 翻译为领域错误。 `reasoning_effort` 是**请求级**档位(`None` = 不表态);它与源级配置的优先级 在 `_build_payload` 里由 `effective_effort` 裁定,本层只负责把它送到。 + `first_token_event` 为对冲处置位: `None` = 不观测首 token;仅流式路径 + (`_complete_stream`)在首个增量到达时置位,非流式路径收它但永不置位。 """ profile = get_provider(source.provider, registry=self._registry) try: @@ -461,9 +465,13 @@ class OpenAICompatTransport: ctx: dict[str, Any] = {"source_name": source.name, "operation": "chat"} try: if stream: - result = await self._complete_stream(client, url, payload, source, profile) + result = await self._complete_stream( + client, url, payload, source, profile, first_token_event + ) else: - result = await self._complete_once(client, url, payload, source, profile) + result = await self._complete_once( + client, url, payload, source, profile, first_token_event + ) except StreamLivenessTimeout as exc: raise TransientError(f"{source.name} 流活性超时({exc.kind})", **ctx) from exc except httpx.TimeoutException as exc: @@ -549,6 +557,7 @@ class OpenAICompatTransport: payload: dict[str, Any], source: SourceConfig, profile: ProviderProfile, + first_token_event: asyncio.Event | None, ) -> TransportResult: started = time.monotonic() async with client.stream("POST", url, json=payload) as resp: @@ -573,6 +582,10 @@ class OpenAICompatTransport: now = time.monotonic() if ttft_ms is None: ttft_ms = (now - started) * 1000 + # 首个增量即对冲语义上的"首 token"(思考增量同样是存活证据, + # 与看门狗活性口径一致);None = 调用方未启用对冲,零分支成本 + if first_token_event is not None: + first_token_event.set() else: max_gap = max(max_gap, (now - last) * 1000) last = now @@ -650,8 +663,13 @@ class OpenAICompatTransport: payload: dict[str, Any], source: SourceConfig, profile: ProviderProfile, + first_token_event: asyncio.Event | None, ) -> TransportResult: - """非流式快路径(三项目均无,库新增): 单 JSON 响应,仅 total 超时。""" + """非流式快路径(三项目均无,库新增): 单 JSON 响应,仅 total 超时。 + + 接收 `first_token_event` 但**永不置位**: 非流式无中途信号,事件自然退化 + 为纯时间阈值(对冲只能靠 `hedge_after_s` 触发)。 + """ resp = await client.post(url, json=payload) if resp.status_code != 200: raise _status_to_error( diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index c5a65cd..5c94c22 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,5 +1,6 @@ """测试侧独立 HTTP 取证装配；无环境自读取或成功 SSE 预读。""" +import asyncio from collections.abc import AsyncIterator, Iterator, Mapping from contextlib import AsyncExitStack, asynccontextmanager, contextmanager from contextvars import ContextVar @@ -290,6 +291,7 @@ class ObservedTransport: overlay: dict[str, Any], call_id: str, reasoning_effort: Effort | None, + first_token_event: asyncio.Event | None, ) -> TransportResult: """与生产端口逐参数同签名。""" with self._capture.attempt_context(call_id): @@ -301,6 +303,7 @@ class ObservedTransport: overlay=overlay, call_id=call_id, reasoning_effort=reasoning_effort, + first_token_event=first_token_event, ) async def embed( diff --git a/tests/integration/test_redis_cross_connection.py b/tests/integration/test_redis_cross_connection.py index 5405154..ff980ba 100644 --- a/tests/integration/test_redis_cross_connection.py +++ b/tests/integration/test_redis_cross_connection.py @@ -75,7 +75,9 @@ class ScriptedTransport: # 取消用例的确定性窗口(同 test_retry FakeTransport): 进入挂起即置位 self.entered = asyncio.Event() - async def complete(self, *, messages, source, stream, overlay, call_id, reasoning_effort): + async def complete( + self, *, messages, source, stream, overlay, call_id, reasoning_effort, first_token_event + ): self.calls.append(source.name) if self.hang: self.entered.set() diff --git a/tests/unit/test_backpressure.py b/tests/unit/test_backpressure.py index 479ea05..3186e1d 100644 --- a/tests/unit/test_backpressure.py +++ b/tests/unit/test_backpressure.py @@ -210,7 +210,9 @@ class ClockAdvancingTransport: self.clock = clock self.calls = [] - async def complete(self, *, messages, source, stream, overlay, call_id, reasoning_effort): + async def complete( + self, *, messages, source, stream, overlay, call_id, reasoning_effort, first_token_event + ): self.calls.append((source.name, call_id)) advance, action = self.script.pop(0) self.clock.advance(advance) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 5d8cc80..47f5b0e 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1738,7 +1738,9 @@ class _ClockJumpTransport: self._jump = jump self.calls = [] - async def complete(self, *, messages, source, stream, overlay, call_id, reasoning_effort): + async def complete( + self, *, messages, source, stream, overlay, call_id, reasoning_effort, first_token_event + ): self.calls.append(call_id) self._clock.advance(self._jump) return _ok() diff --git a/tests/unit/test_live_evidence.py b/tests/unit/test_live_evidence.py index 1c75e2d..c58238f 100644 --- a/tests/unit/test_live_evidence.py +++ b/tests/unit/test_live_evidence.py @@ -277,6 +277,7 @@ async def _complete(observed, *, call_id="a", stream=False): overlay={}, call_id=call_id, reasoning_effort=None, + first_token_event=None, ) @@ -1266,6 +1267,7 @@ async def test_structured_first_attempt_requires_exact_initial_messages(): overlay={}, call_id="first", reasoning_effort=None, + first_token_event=None, ) event = capture.attempts(session_id="first", parent_call_id="parent")[0].http[0] assert not request_is_valid(event) diff --git a/tests/unit/test_openai_compat.py b/tests/unit/test_openai_compat.py index 47102aa..1fcecd8 100644 --- a/tests/unit/test_openai_compat.py +++ b/tests/unit/test_openai_compat.py @@ -3,6 +3,7 @@ SSE 帧样本按三项目真实网关响应形态二次构造(OpenAI 兼容 chunk 结构)。 """ +import asyncio import json import httpx @@ -79,7 +80,9 @@ def _transport_for(handler, *, registry=None): ) -async def _complete(transport, source, *, stream=True, overlay=None, reasoning_effort=None): +async def _complete( + transport, source, *, stream=True, overlay=None, reasoning_effort=None, first_token_event=None +): return await transport.complete( messages=[{"role": "user", "content": "hi"}], source=source, @@ -87,6 +90,7 @@ async def _complete(transport, source, *, stream=True, overlay=None, reasoning_e overlay=overlay or {}, call_id="cid-1", reasoning_effort=reasoning_effort, + first_token_event=first_token_event, ) @@ -214,6 +218,65 @@ class TestStreamHappyPath: assert await _recorded_cost(result, source) is None +class TestFirstTokenEvent: + """首 token 处置位(1.3.7 对冲 H2): 流式置位、非流式永不置位、None 不观测。""" + + async def test_stream_sets_first_token_event(self): + """流式首 token(内容或思考增量)到达即置位——对冲触发窗的取消信号。""" + + def handler(request): + return _sse_stream( + _chunk(reasoning="ponder"), _chunk(content="hi"), _chunk(usage=_USAGE) + ) + + event = asyncio.Event() + result = await _complete(_transport_for(handler), _source(), first_token_event=event) + assert result.content == "hi" + assert event.is_set() + + async def test_non_stream_never_sets_first_token_event(self): + """非流式物理上无中途信号: 即使调用方给了事件,本路径也永不置位。""" + + def handler(request): + return httpx.Response( + 200, json={"choices": [{"message": {"content": "42"}}], "usage": _USAGE} + ) + + event = asyncio.Event() + result = await _complete( + _transport_for(handler), _source(), stream=False, first_token_event=event + ) + assert result.content == "42" + assert not event.is_set() + + async def test_none_first_token_event_keeps_behavior(self): + """`None` = 调用方不观测首 token(未启用对冲): 行为与旧版逐字相同。""" + + def handler(request): + return _sse_stream(_chunk(content="ok"), _chunk(usage=_USAGE)) + + result = await _complete(_transport_for(handler), _source(), first_token_event=None) + assert result.content == "ok" + assert result.ttft_ms is not None + + async def test_first_token_event_is_required_keyword(self): + """端口必填约定: 漏传必须 TypeError——默认值会把"漏传"伪装成"不观测"。""" + + def handler(request): + return _sse_stream(_chunk(content="ok"), _chunk(usage=_USAGE)) + + transport = _transport_for(handler) + with pytest.raises(TypeError): + await transport.complete( + messages=[{"role": "user", "content": "hi"}], + source=_source(), + stream=True, + overlay={}, + call_id="cid-1", + reasoning_effort=None, + ) + + class TestMissingDoneSemantics: def _no_done_handler(self, request): return _sse_stream(_chunk(content="partial"), _chunk(usage=_USAGE), done=False) diff --git a/tests/unit/test_ports.py b/tests/unit/test_ports.py index 57beecc..0eb0efb 100644 --- a/tests/unit/test_ports.py +++ b/tests/unit/test_ports.py @@ -72,7 +72,9 @@ class _DummyMw: class _DummyTransport: - async def complete(self, *, messages, source, stream, overlay, call_id, reasoning_effort): + async def complete( + self, *, messages, source, stream, overlay, call_id, reasoning_effort, first_token_event + ): raise NotImplementedError diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 214b487..309502f 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -79,7 +79,9 @@ class FakeTransport: # 取消用例的确定性窗口: 进入 hang 分支即置位, 用例据此取消而非 sleep 猜时长 self.entered = asyncio.Event() - async def complete(self, *, messages, source, stream, overlay, call_id, reasoning_effort): + async def complete( + self, *, messages, source, stream, overlay, call_id, reasoning_effort, first_token_event + ): self.calls.append((source.name, call_id)) self.efforts.append(reasoning_effort) action = self.script.pop(0) diff --git a/tests/unit/test_usage_source_domain.py b/tests/unit/test_usage_source_domain.py index 36fff7a..61c6b37 100644 --- a/tests/unit/test_usage_source_domain.py +++ b/tests/unit/test_usage_source_domain.py @@ -139,6 +139,7 @@ async def test_salvage_override_stays_in_domain(usage): overlay={}, call_id="cid", reasoning_effort=None, + first_token_event=None, ) assert result.usage_source in USAGE_SOURCES From 463eca380d5bd187152bc89b7033b95d16ee4912 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 10 Sep 2026 13:28:51 -0400 Subject: [PATCH 2/7] feat: expose bare generation time and hedge flags in CallStats CallStats gains hedges/generation_ms/hedge_won (all defaulted, appended after total_latency_ms); _CallContext counts them via record_generation (overwrite for chat/OCR, accumulate for embedding batches) and register_hedge, and snapshot carries them out. All three _attempt implementations time only the transport call itself on the same injected clock as total_latency_ms; the chat sink is recorded by the orchestrator so hedge winner attribution stays with T3. Hedge counters stay 0/False until the T3 orchestration lands. Red-green evidence: tests/outputs/137/t2/ (10 new tests AttributeError red, then green; full unit+contracts 1621 passed). --- src/polygateway/embedding.py | 5 ++ src/polygateway/middleware/retry.py | 17 +++++- src/polygateway/ocr.py | 5 ++ src/polygateway/types.py | 37 +++++++++++- tests/unit/test_client.py | 60 ++++++++++++++++++++ tests/unit/test_embedding.py | 14 +++++ tests/unit/test_ocr_client.py | 15 +++++ tests/unit/test_retry.py | 87 ++++++++++++++++++++++++++++- tests/unit/test_types.py | 49 ++++++++++++++++ 9 files changed, 283 insertions(+), 6 deletions(-) diff --git a/src/polygateway/embedding.py b/src/polygateway/embedding.py index a2b0684..a37f794 100644 --- a/src/polygateway/embedding.py +++ b/src/polygateway/embedding.py @@ -370,7 +370,11 @@ class EmbeddingClient: # 登记在 transport 调用**之前**(同 RetryMW): 失败与取消的尝试也真的发出去了 context.register_attempt() try: + # 裸生成时间(1.3.7 H8): 计时只包 transport 调用本身(同 RetryMW 口径), + # 与本链路 total_latency_ms 同一只注入钟;分批累加在成功分支登记 + gen_started = self._now() result = await self._transport.embed(texts=batch, source=source, call_id=call_id) + gen_ms = int((self._now() - gen_started) * 1000) if self._expected_dim is not None and result.dim != self._expected_dim: raise ResultInvalidError( f"{source.name} 维度 {result.dim} 不符期望 {self._expected_dim}", @@ -384,6 +388,7 @@ class EmbeddingClient: actual = result.prompt_tokens # 真实 usage 恰为 0 也是已知事实, 后续取消不得改写成 est settlement_known = True + context.record_generation(gen_ms, accumulate=True) await self._record_quietly(self._breaker.record_success(entry)) await self._record_quietly(self._quota.mark_progress()) latency_ms = int((self._now() - started) * 1000) diff --git a/src/polygateway/middleware/retry.py b/src/polygateway/middleware/retry.py index 9a8a782..1c5d81d 100644 --- a/src/polygateway/middleware/retry.py +++ b/src/polygateway/middleware/retry.py @@ -246,12 +246,20 @@ class RetryMW: await self._admission.on_no_runnable(gate_rejections, reasons, clock) continue async with clock.attempting() as attempt: - outcome = await self._attempt(request, *picked, reasons, attempt_fails) + # 每轮一个 sink: 裸生成时间由编排裁定归属(T2 单路 = 成功那轮; + # T3 对冲 = 赢家那一路),`_attempt` 只负责把本次 transport 耗时投进来 + generation_sink: list[int] = [] + outcome = await self._attempt( + request, *picked, reasons, attempt_fails, generation_sink=generation_sink + ) rate_limited = _is_rate_limited(outcome) if rate_limited: # 免了重试预算就得进 stall 账,否则这段耗时无人治理(见 StallClock) attempt.refund() if isinstance(outcome, LLMResponse): + # 与 register_attempt 同款 None 守卫: 库内现场构造的请求跳过登记 + if request.call_context is not None: + request.call_context.record_generation(generation_sink[0], accumulate=False) return outcome if not rate_limited: fails += 1 @@ -275,6 +283,8 @@ class RetryMW: entry: GateDecision, reasons: dict[str, str], attempt_fails: dict[str, int], + *, + generation_sink: list[int], ) -> LLMResponse | _Failed: call_id = str(uuid.uuid4()) started = self._now() @@ -288,6 +298,10 @@ class RetryMW: if request.call_context is not None: request.call_context.register_attempt() try: + # 裸生成时间(1.3.7 H8): 计时只包 transport 调用本身,起点紧贴调用前、 + # 终点为返回后首句(中间无 await);取消落进来时 transport 未返回,不计。 + # 与该链路 total_latency_ms 同一只注入钟,差值(波动开销)才有意义 + gen_started = self._now() result = await self._transport.complete( messages=request.messages, source=source, @@ -300,6 +314,7 @@ class RetryMW: # T3 对冲编排接线前恒为 None: 调用方不观测首 token(计划 §3.1) first_token_event=None, ) + generation_sink.append(int((self._now() - gen_started) * 1000)) if result.usage_source == "unavailable": # 用量不可得时按入场预扣量结算(delta==0),否则押金会被整笔退回, # 对"从不返回 usage 帧"的源等于 TPM 闸失效(设计 §3.2 #9) diff --git a/src/polygateway/ocr.py b/src/polygateway/ocr.py index 4c6bbca..a0f862c 100644 --- a/src/polygateway/ocr.py +++ b/src/polygateway/ocr.py @@ -395,11 +395,16 @@ class OcrClient: # layout 的 POST + ZIP GET 在同一次 `_invoke` 内,故这里只登记 **1** 次 context.register_attempt() try: + # 裸生成时间(1.3.7 H8): 计时只包 transport 调用本身(同 RetryMW 口径); + # layout 的 POST + ZIP GET 在同一次 `_invoke` 内,属同一段生成耗时 + gen_started = self._now() result = await self._invoke(kind, image, source, call_id) + gen_ms = int((self._now() - gen_started) * 1000) await self._record_quietly(self._breaker.record_success(entry)) await self._record_quietly(self._quota.mark_progress()) self._feed_outcome(source.name, ok=True) latency_ms = int((self._now() - started) * 1000) + context.record_generation(gen_ms, accumulate=False) await self._emit( kind, operation, diff --git a/src/polygateway/types.py b/src/polygateway/types.py index 8800036..a9499aa 100644 --- a/src/polygateway/types.py +++ b/src/polygateway/types.py @@ -317,23 +317,42 @@ class CallStats: 含缓存 IO、退避等待、准入等待、重问、分批与内联记账。 "总耗时减最后一次尝试耗时"**不等于**纯等待(含其他本地工作)。""" + hedges: int = 0 + """本次逻辑调用实际并发发出的对冲路数(触发但准入失败静默不计);1.3.6 及以前恒 0。""" + generation_ms: int = 0 + """裸生成时间: 赢家/成功那次 transport 调用的墙钟时长(口径见设计 §4.5 H8)。""" + hedge_won: bool = False + """赢家是否为对冲路;无对冲恒 False。""" + class _CallContext: """私有可变逻辑调用上下文: 只持计数、单调时钟与终态去重位,不做 I/O。 - **每调用一个实例**的单任务对象: chat 重试、结构化重问、embedding 分批 - 都在同一任务内串行推进,故计数无需锁。**严禁提升为 client 实例属性** + **每逻辑调用一个实例**,可多任务并发登记(对冲);全部方法无 await, + 事件循环内任务安全。**严禁提升为 client 实例属性** ——那会让同一 client 的并发调用互相串掉计数与逻辑 ID(库铁律"纯 asyncio 中立"、 VT `evolve_llm = llm` 教训的同一形态)。 """ - __slots__ = ("_attempts", "_now", "_started", "_terminal_claimed", "logical_call_id") + __slots__ = ( + "_attempts", + "_generation_ms", + "_hedge_won", + "_hedges", + "_now", + "_started", + "_terminal_claimed", + "logical_call_id", + ) def __init__(self, *, now: Callable[[], float]) -> None: self.logical_call_id = str(uuid.uuid4()) self._now = now self._started = now() self._attempts = 0 + self._generation_ms = 0 + self._hedges = 0 + self._hedge_won = False self._terminal_claimed = False def register_attempt(self) -> None: @@ -344,12 +363,24 @@ class _CallContext: """ self._attempts += 1 + def record_generation(self, elapsed_ms: int, *, accumulate: bool) -> None: + """chat/OCR 覆盖(结构化重问最后一轮为准);embedding 分批累加。""" + self._generation_ms = self._generation_ms + elapsed_ms if accumulate else elapsed_ms + + def register_hedge(self, *, hedge_won: bool) -> None: + """对冲路实际发出即计数;赢家裁定后一次性登记。""" + self._hedges += 1 + self._hedge_won = hedge_won + def snapshot(self) -> CallStats: """同步冻结当前快照;**绝不 await**,可多次调用。""" return CallStats( logical_call_id=self.logical_call_id, attempts=self._attempts, total_latency_ms=int((self._now() - self._started) * 1000), + hedges=self._hedges, + generation_ms=self._generation_ms, + hedge_won=self._hedge_won, ) def claim_terminal(self) -> bool: diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 47f5b0e..063644a 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -144,6 +144,66 @@ class TestChatEndToEnd: await client.chat([{"role": "user", "content": "hi"}], structured="json") +class TestGenerationMsClient: + """裸生成时间的 client 级口径(1.3.7 批次 C2/F)。 + + `_ScriptedGenClockTransport` 在每次 transport 调用内推进注入钟, + 使"时间花在哪"可断言(同 test_backpressure.ClockAdvancingTransport 范式)。 + """ + + _MSG = [{"role": "user", "content": "hi"}] + + async def test_generation_ms_structured_last_round_wins(self): + """结构化重问覆盖而非累加: 首轮坏 JSON 推进 1s,重问轮推进 0.25s → 250。 + + 推进量取二进制可精确表示值: int 截断下非精确值会因浮点误差少 1ms。 + """ + from pydantic import BaseModel + + class Answer(BaseModel): + answer: int + + clock = _StatsClock() + transport = _ScriptedGenClockTransport( + [_ok("not json at all"), _ok('{"answer": 1}')], [1.0, 0.25], clock + ) + async with _client(transport=transport, structured_max_retries=1, now=clock) as client: + resp = await client.chat(self._MSG, structured=Answer) + assert resp.content == '{"answer": 1}' and len(transport.calls) == 2 + assert resp.call_stats is not None + assert resp.call_stats.attempts == 2 + assert resp.call_stats.generation_ms == 250 + + async def test_cache_hit_generation_ms_zero(self): + """缓存命中不产生 transport 调用: generation_ms 恒 0(0 是实测,非"未知")。""" + client = _client(cache=InMemoryCache(), cache_namespace="proj", cache_ttl_s=3600) + async with client: + first = await client.chat(self._MSG) + second = await client.chat(self._MSG) + assert first.cache_hit is False and second.cache_hit is True + assert second.call_stats is not None + assert second.call_stats.generation_ms == 0 + assert second.call_stats.attempts == 0 + + +class _ScriptedGenClockTransport: + """脚本化假 transport: 每次成功调用在返回前按脚本推进注入钟(批次 C2/F)。""" + + def __init__(self, results, advances, clock): + self._results = list(results) + self._advances = list(advances) + self._clock = clock + self.calls = [] + + async def complete( + self, *, messages, source, stream, overlay, call_id, reasoning_effort, first_token_event + ): + self.calls.append(call_id) + result = self._results.pop(0) + self._clock.advance(self._advances.pop(0)) + return result + + class TestSamplingOverlay: """调用级采样参数入口(issue #4 Task 3)。""" diff --git a/tests/unit/test_embedding.py b/tests/unit/test_embedding.py index 430231d..ab12fc1 100644 --- a/tests/unit/test_embedding.py +++ b/tests/unit/test_embedding.py @@ -319,6 +319,20 @@ class TestEmbedBatching: assert resp.cost is None +class TestEmbedGenerationMs: + """裸生成时间按批累加(1.3.7 H8): generation_ms = 各批 transport 耗时之和。""" + + async def test_generation_ms_sums_batch_transports(self): + # 0.25s 为二进制可精确表示值: int 截断下非精确值会因浮点误差少 1ms + clock = FakeClock() + transport = _ClockAdvancingEmbedTransport([(0.25, "ok"), (0.25, "ok")], clock) + client, _ = _embed_client([_src()], [], transport=transport, now=clock) + resp = await client.embed(["a", "bb", "ccc", "dddd"]) + assert resp.call_stats is not None + assert resp.call_stats.attempts == 2 + assert resp.call_stats.generation_ms == 500 + + class TestEmbedPostProcess: async def test_normalize_l2(self): raw = EmbeddingTransportResult( diff --git a/tests/unit/test_ocr_client.py b/tests/unit/test_ocr_client.py index a72caae..8660e33 100644 --- a/tests/unit/test_ocr_client.py +++ b/tests/unit/test_ocr_client.py @@ -196,6 +196,21 @@ class TestSuccessPaths: await client.parse_layout(b"") +class TestOcrGenerationMs: + """OCR 裸生成时间单次覆盖(1.3.7 H8): 计时只包 transport 调用本身。""" + + async def test_generation_ms_single_transport_call(self): + # 0.5s 为二进制可精确表示值: int 截断下非精确值会因浮点误差少 1ms + clock = FakeClock() + transport = ClockAdvancingOcrTransport([(0.5, "text")], clock) + client, _, _ = _client([_src()], [], now=clock, transport=transport) + r = await client.recognize_text(b"jpg") + assert r.text == "LINE-1" + assert r.call_stats is not None + assert r.call_stats.attempts == 1 + assert r.call_stats.generation_ms == 500 + + class TestFailover: async def test_transient_retries_with_backoff(self): sleeps = [] diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 309502f..3a2f951 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -93,6 +93,39 @@ class FakeTransport: return action +class _GenClockTransport: + """委托 FakeTransport 的薄包装: 每次调用返回前按脚本推进注入钟(1.3.7 批次 C)。 + + generation_ms 的口径是"只计 transport 调用本身",故推进必须发生在被包 + transport 内部;退避耗时由用例自带的 sleep 闭包推进,与本包装无关。 + """ + + def __init__(self, script, advances, clock): + self._inner = FakeTransport(script) + self._advances = list(advances) + self._clock = clock + + @property + def calls(self): + return self._inner.calls + + async def complete( + self, *, messages, source, stream, overlay, call_id, reasoning_effort, first_token_event + ): + advance = self._advances.pop(0) + result = await self._inner.complete( + messages=messages, + source=source, + stream=stream, + overlay=overlay, + call_id=call_id, + reasoning_effort=reasoning_effort, + first_token_event=first_token_event, + ) + self._clock.advance(advance) + return result + + class HangingGate(InMemoryGate): """在指定记账写回处永久挂起的门控: 把"取消落在某个 await 上"变成确定性事件。 @@ -141,6 +174,8 @@ def _harness( selector=None, pacer=None, gate=None, + transport=None, + sleep=None, ): clock = clock or FakeClock() limiter = InMemoryLimiter( @@ -151,8 +186,8 @@ def _harness( now=clock, ) gate = gate if gate is not None else InMemoryGate(config=_BREAKER, now=clock) - transport = FakeTransport(script) - sleep = FakeSleep() + transport = transport if transport is not None else FakeTransport(script) + sleep = sleep if sleep is not None else FakeSleep() mw = RetryMW( scope="llm", sources=sources, @@ -916,6 +951,54 @@ class TestRateLimitPushback: assert len(transport.calls) == 3 +class TestGenerationMs: + """裸生成时间(1.3.7 H8): 只计 transport 调用本身,不含退避/准入/遥测收尾。""" + + def _ctx(self, clock): + from polygateway.types import _CallContext + + return _CallContext(now=clock) + + async def test_generation_ms_excludes_backoff_and_admission(self): + """[Transient, ok] 脚本: 退避推进 5s、成功次 transport 推进 0.25s。 + + generation_ms 恒等于成功次 transport 的 250ms;若口径混入了退避, + 它会涨到 5250ms 量级——与 total_latency_ms 的下界断言互为对偶。 + 推进量取二进制可精确表示值(0.25/5.0): int 截断下 0.2 之类会因浮点 + 误差落到 199,断言随之抖动(同 test_types 既有用例只用 1.5/2.0 的惯例)。 + """ + clock = FakeClock() + + async def advancing_sleep(seconds): + clock.advance(seconds) + + transport = _GenClockTransport( + [TransientError("boom", source_name="a"), _ok()], [0.0, 0.25], clock + ) + mw, *_ = _harness( + [_src("a")], + [], + clock=clock, + transport=transport, + sleep=advancing_sleep, + rng=lambda: 2.0, # backoff = 2.0 * (0.5 + 2.0) = 5.0s + ) + ctx = self._ctx(clock) + resp = await mw(dataclasses.replace(_REQ, call_context=ctx)) + assert resp.content == "ok" and len(transport.calls) == 2 + stats = ctx.snapshot() + assert stats.generation_ms == 250 + assert stats.total_latency_ms >= 5250 + + async def test_generation_ms_zero_hedge_flags_without_hedging(self): + """无对冲时 hedges/hedge_won 恒 0/False(对冲登记是 T3 的事)。""" + mw, *_ = _harness([_src("a")], [_ok()]) + ctx = self._ctx(FakeClock()) + await mw(dataclasses.replace(_REQ, call_context=ctx)) + stats = ctx.snapshot() + assert stats.hedges == 0 and stats.hedge_won is False + + class TestLogicalAttemptCounting: """尝试登记在 transport 调用**之前**(1.3.5 设计 §4)。 diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index 4459383..d9e1863 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -622,6 +622,55 @@ class TestCallStatsAndContext: with pytest.raises(dataclasses.FrozenInstanceError): stats.attempts = 3 + def test_callstats_hedge_fields_default(self): + """1.3.7 三字段全带默认值: 仅旧三参数构造不炸,无对冲恒 0/0/False。""" + from polygateway.types import CallStats + + stats = CallStats(logical_call_id="lc-1", attempts=2, total_latency_ms=15) + assert stats.hedges == 0 + assert stats.generation_ms == 0 + assert stats.hedge_won is False + explicit = CallStats( + logical_call_id="lc-2", + attempts=2, + total_latency_ms=15, + hedges=1, + generation_ms=42, + hedge_won=True, + ) + assert (explicit.hedges, explicit.generation_ms, explicit.hedge_won) == (1, 42, True) + + def test_callcontext_record_generation_overwrite_and_accumulate(self): + """chat/OCR 覆盖(结构化重问最后一轮为准);embedding 分批累加。""" + from polygateway.types import _CallContext + + ctx = _CallContext(now=_FakeMonotonic()) + ctx.record_generation(100, accumulate=False) + ctx.record_generation(30, accumulate=False) + assert ctx.snapshot().generation_ms == 30 + ctx.record_generation(50, accumulate=True) + assert ctx.snapshot().generation_ms == 80 + + def test_callcontext_register_hedge_counts(self): + """对冲路实际发出即计数;赢家裁定后一次性登记赢家身份。""" + from polygateway.types import _CallContext + + ctx = _CallContext(now=_FakeMonotonic()) + ctx.register_hedge(hedge_won=False) + ctx.register_hedge(hedge_won=True) + stats = ctx.snapshot() + assert stats.hedges == 2 and stats.hedge_won is True + + def test_snapshot_includes_hedge_fields(self): + """快照把三字段带出: 裸生成时间与对冲计数不停留在内部状态里。""" + from polygateway.types import _CallContext + + ctx = _CallContext(now=_FakeMonotonic()) + ctx.record_generation(200, accumulate=False) + ctx.register_hedge(hedge_won=True) + stats = ctx.snapshot() + assert (stats.hedges, stats.generation_ms, stats.hedge_won) == (1, 200, True) + def test_context_counts_attempts_and_freezes_elapsed(self): """快照是同步冻结的时间切片: 登记两次尝试后耗时按注入钟折算成毫秒。""" from polygateway.types import _CallContext From adc069447a485d9272ee9631a8f06bfa21269c89 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 10 Sep 2026 14:00:50 -0400 Subject: [PATCH 3/7] feat: add opt-in cross-source hedged requests for chat --- src/polygateway/client.py | 17 +- src/polygateway/config.py | 129 ++++++- src/polygateway/middleware/admission.py | 17 +- src/polygateway/middleware/retry.py | 224 +++++++++++- tests/unit/test_client.py | 26 ++ tests/unit/test_config.py | 162 +++++++++ tests/unit/test_hedge.py | 456 ++++++++++++++++++++++++ 7 files changed, 1018 insertions(+), 13 deletions(-) create mode 100644 tests/unit/test_hedge.py diff --git a/src/polygateway/client.py b/src/polygateway/client.py index 6c884f8..9b2ea10 100644 --- a/src/polygateway/client.py +++ b/src/polygateway/client.py @@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Any, Literal from polygateway.backends.memory.breaker import InMemoryGate from polygateway.backends.memory.cache import InMemoryCache from polygateway.backends.memory.limiter import InMemoryLimiter -from polygateway.config import GatewaySettings +from polygateway.config import GatewaySettings, check_hedge_assembly from polygateway.deadline import ensure_call_deadline, with_call_deadline from polygateway.errors import PolyGatewayError from polygateway.middleware.base import compose @@ -237,6 +237,8 @@ class GatewayClient: structured_escalation: StructuredOutputStrategy | None = None, structured_max_retries: int = 1, call_deadline_s: float | None = None, + hedge_after_s: float | None = None, + hedge_max_extra: int = 1, now: Any = time.monotonic, sleep: Any = asyncio.sleep, rng: Any = random.random, @@ -245,6 +247,15 @@ class GatewayClient: self._call_deadline_s = ensure_call_deadline( call_deadline_s, "GatewayClient(call_deadline_s=...)" ) + # 对冲守卫与 GatewaySettings 共用同一份(issue #24 H4): 直接构造这条路 + # 不经过 settings,值域/交叉守卫若只挂在 settings 上就会被它绕过 + self._hedge_after_s = check_hedge_assembly( + hedge_after_s=hedge_after_s, + hedge_max_extra=hedge_max_extra, + sources=sources, + call_deadline_s=self._call_deadline_s, + origin="GatewayClient(hedge_after_s=...)", + ) emitter = ( TelemetryEmitter(telemetry, scope=scope, pricing=pricing, text_cap=text_cap) if telemetry is not None @@ -267,6 +278,8 @@ class GatewayClient: ceiling=float(max([64, *(s.max_concurrency for s in sources if s.max_concurrency)])) ), emitter=emitter, + # max_extra 不下传(issue #24 H5): v1 编排固定单路对冲,无消费者 + hedge_after_s=self._hedge_after_s, now=now, sleep=sleep, rng=rng, @@ -509,6 +522,8 @@ class GatewayClient: structured_escalation=escalation, structured_max_retries=settings.structured_max_retries, call_deadline_s=settings.call_deadline_s, + hedge_after_s=settings.hedge_after_s, + hedge_max_extra=settings.hedge_max_extra, ) _mark_owned_components(client, limiter=limiter, breaker=breaker, telemetry=telemetry) client._owns_cache = cache is None # 缓存后端可以是 None(backend=none),helper 会跳过 diff --git a/src/polygateway/config.py b/src/polygateway/config.py index b665103..511d2fc 100644 --- a/src/polygateway/config.py +++ b/src/polygateway/config.py @@ -30,7 +30,7 @@ from polygateway.types import ( ) if TYPE_CHECKING: - from collections.abc import Mapping + from collections.abc import Mapping, Sequence # FIELD → (SourceConfig 属性, 类型);CHS config.py:95-104 全集 + M1 新增 _SOURCE_FIELDS: dict[str, tuple[str, str]] = { @@ -54,7 +54,7 @@ _SOURCE_FIELDS: dict[str, tuple[str, str]] = { "TRUST_ENV": ("trust_env", "bool"), "EXTRA_BODY": ("extra_body", "json"), } -_RESERVED_SEGMENTS = frozenset({"GLOBAL", "RETRY", "BREAKER", "BACKPRESSURE"}) +_RESERVED_SEGMENTS = frozenset({"GLOBAL", "RETRY", "BREAKER", "BACKPRESSURE", "HEDGE"}) _SELECTORS = frozenset({"round_robin", "least_inflight", "health_aware"}) _QUOTA_FULL = frozenset({"wait", "fail_fast"}) # 熔断全拒时的处置(issue #14);值域与 _QUOTA_FULL 相同但语义不同——配额满是 @@ -188,6 +188,12 @@ class GatewaySettings: # 与 `OcrSettings.gateway` 自动继承。值域由 `_validate_call_deadline` 把关, # 直接构造、`dataclasses.replace` 与 env 三条路一致 call_deadline_s: float | None = None + # 长尾对冲(issue #24 H4): 挂起超阈值时并发向异源再发一次,先回者赢、输家取消。 + # 缺省 None = 关闭,行为逐字等于 1.3.6。`hedge_max_extra` 值域 [1,3] 为 H5 梯次 + # 预留,v1 仅单路生效(>1 装配期 warning);**不下传 RetryMW**。值域与交叉守卫 + # 由 `check_hedge_assembly` 单一定义点把关,直接构造/replace/env 三路一致 + hedge_after_s: float | None = None + hedge_max_extra: int = 1 def __post_init__(self) -> None: self._normalize() @@ -199,6 +205,7 @@ class GatewaySettings: self._validate_stall() self._validate_probe() self._validate_call_deadline() + self._validate_hedge() def _normalize(self) -> None: """把 `from_env` 一直在做的规范化补到构造路上,两条路必须产出同一个值。 @@ -365,6 +372,24 @@ class GatewaySettings: ensure_call_deadline(self.call_deadline_s, "GatewaySettings.call_deadline_s"), ) + def _validate_hedge(self) -> None: + """对冲装配守卫(issue #24 H4): 与期限同款,盖住直接构造与 replace 两条路。 + + env 路的值域错误已在 `_load_hedge` 里带真实键名报过;此处对合法值是幂等 + 空操作,交叉守卫(阈值 vs timeout/deadline/ttft、单源)只在这里有一处。 + """ + object.__setattr__( + self, + "hedge_after_s", + check_hedge_assembly( + hedge_after_s=self.hedge_after_s, + hedge_max_extra=self.hedge_max_extra, + sources=self.sources, + call_deadline_s=self.call_deadline_s, + origin="GatewaySettings.hedge_after_s", + ), + ) + @classmethod def from_env( cls, @@ -394,6 +419,7 @@ class GatewaySettings: quota_full=_load_choice(env, f"{scope_u}__QUOTA_FULL", _QUOTA_FULL, "wait"), circuit_open=_load_choice(env, f"{scope_u}__CIRCUIT_OPEN", _CIRCUIT_OPEN, "fail_fast"), call_deadline_s=_load_call_deadline(scope_u, env), + **_load_hedge(scope_u, env), **_load_pgw(env), ) @@ -716,6 +742,105 @@ def _load_call_deadline(scope: str, env: Mapping[str, str]) -> float | None: return ensure_call_deadline(_cast(found[1], "float", found[0]), found[0]) +def _load_hedge(scope: str, env: Mapping[str, str]) -> dict[str, object]: + """读 `{SCOPE}__HEDGE__AFTER_S`/`{SCOPE}__HEDGE__MAX_EXTRA`(issue #24 H4)。 + + 两键均为 3 段键(`split("__")` 长度 3 ≠ 4),`_load_sources` 的段数判据天然 + 跳过它们;`HEDGE` 已进 `_RESERVED_SEGMENTS`,4 段的 `{SCOPE}__HEDGE__{N}__*` + 也不会被当成 provider 段造出源。`AFTER_S` 未设 = 关闭(缺省逐字等于 1.3.6); + `MAX_EXTRA` 未设 = 1。origin 传实际命中键名(同 `_load_call_deadline` 纪律); + 值域的交叉守卫(ttft/单源/max_extra 生效口径)归 `check_hedge_assembly` 一处。 + + Args: + scope: 已大写的 scope 名。 + env: 已合并的环境映射。 + + Returns: + `{"hedge_after_s": float | None, "hedge_max_extra": int}`,直传构造器。 + """ + found_after = _first(env, f"{scope}__HEDGE__AFTER_S") + after = ( + ensure_call_deadline(_cast(found_after[1], "float", found_after[0]), found_after[0]) + if found_after + else None + ) + found_extra = _first(env, f"{scope}__HEDGE__MAX_EXTRA") + extra = int(_cast(found_extra[1], "int", found_extra[0])) if found_extra else 1 + return {"hedge_after_s": after, "hedge_max_extra": extra} + + +def check_hedge_assembly( + *, + hedge_after_s: float | None, + hedge_max_extra: int, + sources: Sequence[SourceConfig], + call_deadline_s: float | None, + origin: str, +) -> float | None: + """对冲装配守卫的唯一事实源(issue #24 设计 §5 全表,H4 批准)。 + + `GatewaySettings.__post_init__` 与 `GatewayClient.__init__` 调同一份,两条装配 + 路的值域/交叉守卫不漂移。返回归一化后的 `hedge_after_s`(None 或有限正数, + 复用 `ensure_call_deadline` 的值域校验);`hedge_after_s is None`(未启用)时 + 值域归一化后直接返回,交叉守卫不查——它们没有可校验的对象。 + + Args: + hedge_after_s: 对冲触发阈值(秒);None = 关闭。 + hedge_max_extra: 每次逻辑调用最多并发对冲路数;v1 仅单路生效(H5)。 + sources: 本 scope 的源集合(交叉守卫要读 timeout_s/ttft_timeout_s)。 + call_deadline_s: 调用期限(秒);与对冲的组合守卫见设计 §6。 + origin: after_s 值域报错的定位串(env 键名 / `GatewaySettings.hedge_after_s` / + `GatewayClient(hedge_after_s=...)`);跨字段守卫与 warning 沿用本模块先例, + 消息自带字段名与 env 键型,不挂 origin。 + + Raises: + ValueError: max_extra 非 int/bool 或出 [1,3];阈值 ≥ 最小源 timeout_s(永不 + 可能触发);阈值 ≥ call_deadline_s(期限先于对冲触发,对冲形同虚设)。 + """ + if isinstance(hedge_max_extra, bool) or not isinstance(hedge_max_extra, int): + raise ValueError( + f"hedge_max_extra({{SCOPE}}__HEDGE__MAX_EXTRA)必须是 int: {hedge_max_extra!r}" + ) + if not 1 <= hedge_max_extra <= 3: + raise ValueError( + f"hedge_max_extra({{SCOPE}}__HEDGE__MAX_EXTRA)须在 [1,3]: {hedge_max_extra};" + "每次逻辑调用最多并发对冲路数,v1 仅单路生效" + ) + after = ensure_call_deadline(hedge_after_s, origin) + if after is None: + return None + min_timeout = min(s.timeout_s for s in sources) + if after >= min_timeout: + raise ValueError( + f"hedge_after_s({{SCOPE}}__HEDGE__AFTER_S={after})须 < 最小源 timeout_s" + f"({min_timeout});对冲永不可能触发,配置即错误" + ) + if call_deadline_s is not None and after >= call_deadline_s: + raise ValueError( + f"hedge_after_s({after})须 < call_deadline_s({call_deadline_s});" + "期限会先于对冲触发,对冲形同虚设(设计 §6)" + ) + ttfts = [s.ttft_timeout_s for s in sources if s.ttft_timeout_s is not None] + if ttfts and after >= min(ttfts): + logger.warning( + "hedge_after_s({}) ≥ 最小源 ttft_timeout_s({}): 流式挂起会被 TTFT 看门狗" + "先行切断,对冲对流式形同虚设(非流式仍有效)", + after, + min(ttfts), + ) + if len(sources) == 1: + logger.warning( + "单源 scope 配置了对冲阈值 hedge_after_s={};运行期拿不到异源候选,对冲自然静默", + after, + ) + if hedge_max_extra > 1: + logger.warning( + "hedge_max_extra={} 已接受,但 v1 仅单路对冲生效(梯次追加为 H5 预留)", + hedge_max_extra, + ) + return after + + @dataclass(frozen=True) class EmbeddingSettings: """Embedding scope 装配配置(M2 §7): 复用 GatewaySettings + embedding 专用键。 diff --git a/src/polygateway/middleware/admission.py b/src/polygateway/middleware/admission.py index 21ac9ba..c579b76 100644 --- a/src/polygateway/middleware/admission.py +++ b/src/polygateway/middleware/admission.py @@ -169,15 +169,28 @@ class SourceAdmission: # —— 选源与准入(CHS _pick_runnable 120-167)—— async def pick( - self, reasons: dict[str, str], attempt_fails: dict[str, int] + self, + reasons: dict[str, str], + attempt_fails: dict[str, int], + *, + exclude: frozenset[str] | None = None, ) -> tuple[tuple[SourceConfig, Permit, GateDecision] | None, int]: - """挑出第一个过闸的候选;返回 (选中三元组 | None, 熔断类拒绝计数)。""" + """挑出第一个过闸的候选;返回 (选中三元组 | None, 熔断类拒绝计数)。 + + `exclude` 是对冲编排的私有排除参数(issue #24): 以在途源名为排除集, + 保证对冲路落在**异源**。被排除不是源的拒绝——不计 gate_rejections、 + 不写 reasons,否则会污染 `on_no_runnable` 的分派判据与 per_source_reasons + 对账。拿不到候选时返回 None,是否放弃由调用方决定(对冲方静默等原路, + **严禁**对这个 None 调 `on_no_runnable`)。 + """ stats = {s.name: await self._quota.stats(s) for s in self._sources} gate_rejections = 0 ordered = _demote_call_failures( self._selector.order(self._sources, stats), attempt_fails, self._health_view ) for cand in ordered: + if exclude and cand.name in exclude: + continue # 对冲排除在途源: 不是拒绝, 不计数不写原因(见 docstring) if self._memo.active(cand.name): # 冷却备忘跳过也计入拒绝数,保住 circuit_open 判据(CHS 同款) gate_rejections += 1 diff --git a/src/polygateway/middleware/retry.py b/src/polygateway/middleware/retry.py index 1c5d81d..022f817 100644 --- a/src/polygateway/middleware/retry.py +++ b/src/polygateway/middleware/retry.py @@ -164,6 +164,17 @@ def _is_rate_limited(outcome: LLMResponse | _Failed) -> bool: return isinstance(outcome, _Failed) and _failure_reason(outcome.exc) == "rate_limited" +_HEDGE_LOSER_ATTR = "_polygateway_hedge_loser" +"""编排在 cancel() 之前给输家任务置位的标记;_attempt 读它选遥测标签。""" + + +def _combine_failures(primary_f: _Failed, hedge_f: _Failed) -> _Failed: + """任一非 429 优先(计预算);两路皆 429 才按 429 免预算退还 stall 账;同类取原路。""" + if _failure_reason(primary_f.exc) == "rate_limited" != _failure_reason(hedge_f.exc): + return hedge_f + return primary_f + + class RetryMW: """尝试编排器;时钟/睡眠/随机全部注入,纯确定性可测(P6)。""" @@ -183,6 +194,7 @@ class RetryMW: cooldown_memo: SourceCooldownMemo | None = None, pacer: AdaptivePacer | None = None, emitter: TelemetryEmitter | None = None, + hedge_after_s: float | None = None, now: Callable[[], float] = time.monotonic, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, rng: Callable[[], float] = random.random, @@ -200,6 +212,10 @@ class RetryMW: # M2.5 §3.35: AIMD 自适应并发——429 收紧、成功回涨,超限调用排队不烧预算 self._pacer = pacer or AdaptivePacer(ceiling=64.0) self._emitter = emitter + # 对冲触发阈值(issue #24): None = 关闭(__call__ 逐字走 1.3.6 单路路径)。 + # 值域/交叉守卫在装配层(config.check_hedge_assembly),本类不重复校验; + # hedge_max_extra 不下传——v1 编排固定单路对冲(H5) + self._hedge_after_s = hedge_after_s self._now = now self._sleep = sleep self._rng = rng @@ -249,16 +265,27 @@ class RetryMW: # 每轮一个 sink: 裸生成时间由编排裁定归属(T2 单路 = 成功那轮; # T3 对冲 = 赢家那一路),`_attempt` 只负责把本次 transport 耗时投进来 generation_sink: list[int] = [] - outcome = await self._attempt( - request, *picked, reasons, attempt_fails, generation_sink=generation_sink - ) + if self._hedge_after_s is None: + # 默认关闭: 逐字 1.3.6 单路路径(默认关闭回归门据此成立) + outcome = await self._attempt( + request, + *picked, + reasons, + attempt_fails, + first_token_event=None, + generation_sink=generation_sink, + ) + else: + # 对冲轮 = 一次"超级尝试": 编排内部自建 sink 并按赢家归属登记 + outcome = await self._attempt_hedged(request, picked, reasons, attempt_fails) rate_limited = _is_rate_limited(outcome) if rate_limited: # 免了重试预算就得进 stall 账,否则这段耗时无人治理(见 StallClock) attempt.refund() if isinstance(outcome, LLMResponse): - # 与 register_attempt 同款 None 守卫: 库内现场构造的请求跳过登记 - if request.call_context is not None: + # 与 register_attempt 同款 None 守卫: 库内现场构造的请求跳过登记; + # 对冲轮由 _attempt_hedged 按赢家归属登记,此处外层 sink 恒为空 + if request.call_context is not None and generation_sink: request.call_context.record_generation(generation_sink[0], accumulate=False) return outcome if not rate_limited: @@ -284,6 +311,7 @@ class RetryMW: reasons: dict[str, str], attempt_fails: dict[str, int], *, + first_token_event: asyncio.Event | None, generation_sink: list[int], ) -> LLMResponse | _Failed: call_id = str(uuid.uuid4()) @@ -311,8 +339,9 @@ class RetryMW: # 逐次尝试原样重传: 换源不改变调用方要的档位(源级默认由 transport # 自己按选中的源解析,两者在 effective_effort 里汇合) reasoning_effort=request.reasoning_effort, - # T3 对冲编排接线前恒为 None: 调用方不观测首 token(计划 §3.1) - first_token_event=None, + # 对冲编排(计划 §3.4): 原路携带首 token 事件,对冲路恒 None(v1 单路, + # 不再梯次);未启用对冲时 __call__ 传 None = 调用方不观测首 token + first_token_event=first_token_event, ) generation_sink.append(int((self._now() - gen_started) * 1000)) if result.usage_source == "unavailable": @@ -347,7 +376,15 @@ class RetryMW: actual = source.effective_est_tokens() if entry.is_probe: await self._record_quietly(self._breaker.release_probe(entry)) - await self._emit(request, source, call_id, started, error="cancelled") + # 对冲输家在 cancel() 前被编排置位标记(计划 §3.4): 据此区分"被对冲 + # 淘汰"与"外部取消",零新遥测列(设计 §4.5);外部取消与对冲取消同时 + # 到达的竞速可能误贴——记账方向一致(est 保留),属已批准的可接受残留 + label = ( + "hedge_cancelled" + if getattr(asyncio.current_task(), _HEDGE_LOSER_ATTR, False) + else "cancelled" + ) + await self._emit(request, source, call_id, started, error=label) raise except (SourceDeadError, TransientError) as exc: dead = isinstance(exc, SourceDeadError) @@ -369,6 +406,177 @@ class RetryMW: self._pacer.leave(source.name) await settle_and_release(permit, actual) + # —— 对冲编排(issue #24 设计 §4.6;默认关闭,`hedge_after_s is None` 不进这里)—— + + async def _attempt_hedged( + self, + request: ChatRequest, + picked: tuple[SourceConfig, Permit, GateDecision], + reasons: dict[str, str], + attempt_fails: dict[str, int], + ) -> LLMResponse | _Failed: + """一次"超级尝试": 原路 + (触发后)异源对冲路,FIRST_COMPLETED 竞速。 + + 计时只用事件循环相对时长(`asyncio.wait` timeout),绝不读注入 `now` + (设计 §4.1 时钟纪律);两路各持各的 permit,结算/遥测/熔断写回全部沿用 + `_attempt` 既有路径,本方法只做编排与赢家裁定。 + """ + # Phase 1 启动原路: 事件与 sink 每轮新建(局部状态,严禁实例属性) + source, permit, entry = picked + first_token: asyncio.Event = asyncio.Event() + sink_p: list[int] = [] + primary = asyncio.create_task( + self._attempt( + request, + source, + permit, + entry, + reasons, + attempt_fails, + first_token_event=first_token, + generation_sink=sink_p, + ) + ) + hedge: asyncio.Task[LLMResponse | _Failed] | None = None + try: + # Phase 2 触发窗: 只认"阈值到 + 首 token 未至 + 原路在途"(loop 相对时长) + if not await self._past_hedge_window(primary, first_token): + # 原路已了结/首 token 已至: 等价于未配置对冲 + outcome = await primary + if isinstance(outcome, LLMResponse): + self._record_generation(request, sink_p) + return outcome + # Phase 3 异源准入(完整 pick 路径,无旁路): 拿不到候选 = 静默等原路 + # (对冲是优化不是权利;严禁对这个 None 调 on_no_runnable,见 §3.5) + hedge_picked, _ = await self._admission.pick( + reasons, attempt_fails, exclude=frozenset({source.name}) + ) + if hedge_picked is None: + outcome = await primary + if isinstance(outcome, LLMResponse): + self._record_generation(request, sink_p) + return outcome + # Phase 4 启动对冲路(v1 单路,H5): 对冲路恒传 first_token_event=None, + # 不再触发梯次对冲 + sink_h: list[int] = [] + hedge = asyncio.create_task( + self._attempt( + request, + *hedge_picked, + reasons, + attempt_fails, + first_token_event=None, + generation_sink=sink_h, + ) + ) + # Phase 5 赢家裁定与收口 + done, pending = await asyncio.wait( + {primary, hedge}, return_when=asyncio.FIRST_COMPLETED + ) + if pending and not any(self._succeeded(t, done) for t in done): + # 先了结的是失败: 等另一路的结论再裁定(它可能后发先至; + # 原路先败、对冲在途时不重试,设计 §4.6) + more, pending = await asyncio.wait(pending) + done |= more + winner = ( + primary + if self._succeeded(primary, done) + else hedge + if self._succeeded(hedge, done) + else None + ) + if winner is None: + primary_exc = primary.exception() + hedge_exc = hedge.exception() # 两路都取回,不留 never-retrieved 告警 + # 非可重试族(RequestRejected/ResultInvalid)穿透,与单路同口径 + if primary_exc is not None: + raise primary_exc + if hedge_exc is not None: + raise hedge_exc + # 两败(H6): 汇合为一次失败,重试预算只计一次;路数照登(无赢家) + outcome = _combine_failures(primary.result(), hedge.result()) + self._register_hedge(request, hedge_won=False, winner_sink=None) + return outcome + # 两路同时成功的竞速 → 原路优先(保守不弃原路成果),由上面 primary + # 先判实现;竞速落选者已完成,不置标记——它没被取消,行是正常行 + loser = hedge if winner is primary else primary + if not loser.done(): + # 先置标记再 cancel: _attempt 的取消分支据标记选 hedge_cancelled + setattr(loser, _HEDGE_LOSER_ATTR, True) + loser.cancel() + # 收口: gather 等输家 finally 的结算/遥测跑完才返回(快照含输家, + # attempts==2);对已完成的输家顺带取回结果,不留 never-retrieved 告警 + await asyncio.gather(loser, return_exceptions=True) + self._register_hedge( + request, + hedge_won=winner is hedge, + winner_sink=sink_h if winner is hedge else sink_p, + ) + return winner.result() + except BaseException: + # 取消穿透与准入冒泡(GovernanceBackendError)同路: 两任务(存在且未完者) + # 同消,尽力收口后原异常上抛;不 shield,收口 await 允许被再取消(136 清理纪律) + tasks = [t for t in (primary, hedge) if t is not None and not t.done()] + for t in tasks: + t.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + raise + + async def _past_hedge_window(self, primary: asyncio.Task, first_token: asyncio.Event) -> bool: + """对冲触发窗: 阈值到 + 首 token 未至 + 原路在途,三者齐备才放行对冲。 + + 只用事件循环相对时长(`asyncio.wait` timeout),绝不读注入 `now`——测试 + 伪造注入钟跳变不得触发对冲(设计 §4.1)。waiter 任务必须收口,且**不得吞 + 外部取消**(finally 里 await 已取消的 waiter 会接住一个 CancelledError, + 须靠 cancelling() 区分它是 waiter 自己的还是外面打进来的)。 + """ + waiter = asyncio.create_task(first_token.wait()) + try: + await asyncio.wait( + {primary, waiter}, timeout=self._hedge_after_s, return_when=asyncio.FIRST_COMPLETED + ) + finally: + waiter.cancel() + try: + await waiter + except asyncio.CancelledError: + if asyncio.current_task().cancelling(): # 外部取消,穿透 + raise + return not primary.done() and not first_token.is_set() + + @staticmethod + def _succeeded(task: asyncio.Task, done: set[asyncio.Task]) -> bool: + """赢家裁定: 已完成、未被取消、无异常且结果为 LLMResponse。""" + return ( + task in done + and not task.cancelled() + and task.exception() is None + and isinstance(task.result(), LLMResponse) + ) + + @staticmethod + def _record_generation(request: ChatRequest, sink: list[int]) -> None: + """record_generation 的共享守卫: 上下文缺失(库内现场构造)或空 sink 均跳过。""" + if request.call_context is not None and sink: + request.call_context.record_generation(sink[0], accumulate=False) + + @staticmethod + def _register_hedge( + request: ChatRequest, *, hedge_won: bool, winner_sink: list[int] | None + ) -> None: + """对冲登记(赢家裁定后一次性;同 register_attempt 的 None 守卫)。 + + hedges 计"实际并发发出的对冲路数"——触发但准入失败的静默不计(没走到 + 这里);两败轮次无赢家: 路数照登(hedge_won=False),裸生成时间无归属不记。 + """ + context = request.call_context + if context is None: + return + if winner_sink: + context.record_generation(winner_sink[0], accumulate=False) + context.register_hedge(hedge_won=hedge_won) + async def _on_rejected( self, exc: RequestRejectedError, source: SourceConfig, entry: GateDecision ) -> None: diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 063644a..d2b180f 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -144,6 +144,32 @@ class TestChatEndToEnd: await client.chat([{"role": "user", "content": "hi"}], structured="json") +class TestHedgeParams: + """对冲两个 keyword-only 参数的 client 入口校验(issue #24 H4;计划批次 H)。 + + 编排行为本身见 tests/unit/test_hedge.py;这里只钉装配面: 入口即校、 + 值域/交叉守卫与 settings 共用同一份 `check_hedge_assembly`。 + """ + + def test_client_hedge_params_entry_validation(self): + # 值域: 0/负/非有限当场 ValueError(不经 settings 那道守卫) + with pytest.raises(ValueError, match=r"GatewayClient\(hedge_after_s"): + _client(hedge_after_s=0) + with pytest.raises(ValueError, match="hedge_max_extra"): + _client(hedge_max_extra=0) + with pytest.raises(ValueError, match="hedge_max_extra"): + _client(hedge_max_extra=True) # bool 不是 int 档位 + # 交叉: 阈值 ≥ 源 timeout_s(10)= 对冲永不可能触发 + with pytest.raises(ValueError, match="timeout_s"): + _client(hedge_after_s=99.0) + # 合法值透传到 RetryMW(单源 scope 的装配 warning 是预期噪音,不断言) + client = _client(hedge_after_s=0.05) + assert client._hedge_after_s == 0.05 + assert client._terminal._hedge_after_s == 0.05 + # 未启用(缺省): 行为逐字等于 1.3.6 + assert _client()._terminal._hedge_after_s is None + + class TestGenerationMsClient: """裸生成时间的 client 级口径(1.3.7 批次 C2/F)。 diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 5a7ec82..71ac727 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1096,3 +1096,165 @@ class TestCallDeadlineConfig: with pytest.raises(ValueError, match=r"GatewayClient\(call_deadline_s"): _client(call_deadline_s=0) + + +class TestHedgeConfig: + """`{SCOPE}__HEDGE__AFTER_S`/`{SCOPE}__HEDGE__MAX_EXTRA` 两键与装配守卫(issue #24 H4)。 + + 对冲默认关闭: 键未设 = None/1,行为逐字等于 1.3.6。守卫四路覆盖 + (env/直接构造/dataclasses.replace/client 直传),单一定义点是 + `config.check_hedge_assembly`。 + """ + + def _two_source_env(self, **overrides): + """双源 env(同 provider 避免注册表依赖): 隔离单源 warning 的干扰。""" + return _env( + **{ + "LLM__QWEN__2__BASE_URL": "https://gw-b.example/v1", + "LLM__QWEN__2__API_KEY": "sk-b", + "LLM__QWEN__2__MODEL": "qwen-plus", + "LLM__QWEN__2__TIMEOUT_S": "90", + **overrides, + } + ) + + def test_hedge_keys_from_env_skip_source_loader(self): + """两键为 3 段键,天然不被 `_load_sources` 当源字段;`HEDGE` 进保留段防 4 段撞名。""" + env = self._two_source_env(**{"LLM__HEDGE__AFTER_S": "8", "LLM__HEDGE__MAX_EXTRA": "2"}) + with _captured_warnings(): # max_extra>1 的 v1 单路 warning, 本用例不断言它 + s = GatewaySettings.from_env("LLM", env=env) + assert s.hedge_after_s == 8.0 + assert s.hedge_max_extra == 2 + assert {src.name for src in s.sources} == {"qwen_1", "qwen_2"} # HEDGE 键未造源 + # `HEDGE` 在保留段: `LLM__HEDGE__1__*` 四段键不得造出一个名为 hedge_1 的源 + env_collision = _env( + **{ + "LLM__HEDGE__1__BASE_URL": "https://gw-c.example/v1", + "LLM__HEDGE__1__API_KEY": "sk-c", + "LLM__HEDGE__1__MODEL": "m-c", + "LLM__HEDGE__1__TIMEOUT_S": "60", + } + ) + s2 = GatewaySettings.from_env("LLM", env=env_collision) + assert [src.name for src in s2.sources] == ["qwen_1"] + + def test_hedge_keys_unset_mean_disabled(self): + """默认关闭: 两键未设 = None/1,且不产生任何 warning。""" + with _captured_warnings() as warnings: + s = GatewaySettings.from_env("LLM", env=_env()) + assert s.hedge_after_s is None and s.hedge_max_extra == 1 + assert not warnings + + def test_hedge_after_s_domain_four_paths(self): + """非法值四条装配路全部当场 ValueError(消息须定位得到是哪个键/参数)。""" + from tests.unit.test_client import _client + + # 路 1: env(origin 是实际命中的键名) + with pytest.raises(ValueError, match="LLM__HEDGE__AFTER_S"): + GatewaySettings.from_env("LLM", env=_env(**{"LLM__HEDGE__AFTER_S": "0"})) + with pytest.raises(ValueError, match="LLM__HEDGE__AFTER_S"): + GatewaySettings.from_env("LLM", env=_env(**{"LLM__HEDGE__AFTER_S": "abc"})) + base = GatewaySettings.from_env("LLM", env=_env()) + # 路 2: 直接构造 + fields = {f.name: getattr(base, f.name) for f in dataclasses.fields(base)} + with pytest.raises(ValueError, match="hedge_after_s"): + GatewaySettings(**{**fields, "hedge_after_s": float("nan")}) + # 路 3: dataclasses.replace + with pytest.raises(ValueError, match="hedge_after_s"): + dataclasses.replace(base, hedge_after_s=-1) + # 路 4: client 直传(不经 settings 那道守卫) + with pytest.raises(ValueError, match=r"GatewayClient\(hedge_after_s"): + _client(hedge_after_s=0) + + def test_hedge_guard_below_min_timeout_raises(self): + """阈值 ≥ 最小源 timeout_s = 对冲永不可能触发,装配期炸掉(ValueError)。""" + env = self._two_source_env(**{"LLM__HEDGE__AFTER_S": "90"}) # min(timeout)=90 + with pytest.raises(ValueError, match="timeout_s"): + GatewaySettings.from_env("LLM", env=env) + # 边界内侧合法(89 < 90) + s = GatewaySettings.from_env( + "LLM", env=self._two_source_env(**{"LLM__HEDGE__AFTER_S": "89"}) + ) + assert s.hedge_after_s == 89.0 + + def test_hedge_guard_ttft_warns(self): + """阈值 ≥ 最小已设 ttft_timeout_s: 流式被看门狗先切,装配期 warning 而非 ValueError。""" + env = self._two_source_env( + **{ + "LLM__QWEN__1__TTFT_TIMEOUT_S": "30", + "LLM__QWEN__1__INTER_TOKEN_TIMEOUT_S": "15", + "LLM__HEDGE__AFTER_S": "35", # ≥ ttft 30, < timeout 90 + } + ) + with _captured_warnings() as warnings: + s = GatewaySettings.from_env("LLM", env=env) + assert s.hedge_after_s == 35.0 # warning 不是拒绝: 非流式仍有效 + assert any("ttft_timeout_s" in m for m in warnings) + # 阈值低于看门狗时不告警 + with _captured_warnings() as warnings2: + GatewaySettings.from_env( + "LLM", + env=self._two_source_env( + **{ + "LLM__QWEN__1__TTFT_TIMEOUT_S": "30", + "LLM__QWEN__1__INTER_TOKEN_TIMEOUT_S": "15", + "LLM__HEDGE__AFTER_S": "25", + } + ), + ) + assert not warnings2 + + def test_hedge_guard_single_source_warns(self): + """单源 scope 设阈值: 装配期 warning 放行,运行期拿不到候选自然静默。""" + with _captured_warnings() as warnings: + s = GatewaySettings.from_env("LLM", env=_env(**{"LLM__HEDGE__AFTER_S": "8"})) + assert s.hedge_after_s == 8.0 + assert any("单源" in m for m in warnings) + + def test_hedge_guard_deadline_conflict_raises(self): + """阈值 ≥ call_deadline_s: 期限先于对冲触发,对冲形同虚设 → ValueError(§6)。""" + env = self._two_source_env(**{"LLM__HEDGE__AFTER_S": "35", "LLM__CALL_DEADLINE_S": "30"}) + with pytest.raises(ValueError, match="call_deadline_s"): + GatewaySettings.from_env("LLM", env=env) + # 边界值(恰好相等)同样拒绝 + with pytest.raises(ValueError, match="call_deadline_s"): + GatewaySettings.from_env( + "LLM", + env=self._two_source_env( + **{"LLM__HEDGE__AFTER_S": "30", "LLM__CALL_DEADLINE_S": "30"} + ), + ) + # 阈值 < 期限是合法组合 + s = GatewaySettings.from_env( + "LLM", + env=self._two_source_env(**{"LLM__HEDGE__AFTER_S": "29", "LLM__CALL_DEADLINE_S": "30"}), + ) + assert s.hedge_after_s == 29.0 and s.call_deadline_s == 30.0 + + def test_hedge_max_extra_v1_cap(self): + """max_extra 值域 [1,3] 的四路校验;>1 已接受但 warning 声明 v1 仅单路生效(H5)。""" + # 域外值无条件拒绝(即使对冲未启用: 非法值没有"惰性"豁免) + with pytest.raises(ValueError, match="hedge_max_extra"): + GatewaySettings.from_env("LLM", env=_env(**{"LLM__HEDGE__MAX_EXTRA": "0"})) + with pytest.raises(ValueError, match="LLM__HEDGE__MAX_EXTRA"): + GatewaySettings.from_env("LLM", env=_env(**{"LLM__HEDGE__MAX_EXTRA": "x"})) + with pytest.raises(ValueError, match="hedge_max_extra"): + GatewaySettings.from_env("LLM", env=_env(**{"LLM__HEDGE__MAX_EXTRA": "4"})) + base = GatewaySettings.from_env("LLM", env=_env()) + with pytest.raises(ValueError, match="hedge_max_extra"): + dataclasses.replace(base, hedge_max_extra=0) + # 2/3 接受 + warning: v1 运行期恒单路(对冲任务不再携带首 token 观测, + # 行为面由 test_hedge.py 的 ft_events 断言钉住),梯次追加为 H5 预留 + env = self._two_source_env(**{"LLM__HEDGE__AFTER_S": "8", "LLM__HEDGE__MAX_EXTRA": "2"}) + with _captured_warnings() as warnings: + s = GatewaySettings.from_env("LLM", env=env) + assert s.hedge_max_extra == 2 + assert any("单路" in m for m in warnings) + + def test_from_settings_propagates_hedge_to_client(self): + """from_settings 透传: RetryMW 拿到归一化阈值;max_extra 不下传(v1 无消费者)。""" + env = self._two_source_env(**{"LLM__HEDGE__AFTER_S": "8"}) + s = GatewaySettings.from_env("LLM", env=env) + client = GatewayClient.from_settings(s) + assert client._hedge_after_s == 8.0 + assert client._terminal._hedge_after_s == 8.0 diff --git a/tests/unit/test_hedge.py b/tests/unit/test_hedge.py new file mode 100644 index 0000000..06fe8af --- /dev/null +++ b/tests/unit/test_hedge.py @@ -0,0 +1,456 @@ +"""对冲编排测试(issue #24 设计 §4.6 / 计划批次 G)。 + +设施纪律(计划 §5): 两源 scope、事件驱动假 transport(每源一对 entered/release +Event + 可脚本化"先置首 token 再挂起")、**真实 loop 钟**(不注入 FakeClock)、 +`hedge_after_s=0.05`、断言容差 4–10×;取消窗口用 entered 双 Event 栅栏钉死, +禁 sleep 撞窗口;计时只断言下界与相对比较,不断言精确值。 +""" + +import asyncio +import time + +import pytest + +from polygateway import CallDeadlineExceeded +from polygateway.backends.memory.breaker import InMemoryGate +from polygateway.backends.memory.limiter import InMemoryLimiter +from polygateway.errors import AllSourcesExhausted, TransientError +from polygateway.middleware.retry import RetryMW +from polygateway.sources import SourceCooldownMemo +from polygateway.types import ( + BackpressurePolicy, + BreakerConfig, + ChatRequest, + GlobalLimits, + RetryPolicy, + _CallContext, +) +from tests.unit.test_retry import RecordingSelector, StaticSelector, _ok, _src + +_BREAKER = BreakerConfig(fail_threshold=3, cooldown_s=60.0, probe_ttl_s=120.0) +_NO_GLOBAL = GlobalLimits(max_concurrency=0, rpm=0, tpm=0) +_HEDGE_AFTER_S = 0.05 # 真实 loop 钟阈值;断言上界取 10×(0.5s) + + +class HedgeTransport: + """事件驱动假 transport: 按源剧本精确控制首 token 置位与完成时刻。 + + 剧本动作(每源一条队列,耗尽后重复最后一项——429 风暴用例需无限供应): + ("hang",) — 置位该源 entered,挂起直到该源 release(或被取消) + ("token_then_hang",) — 先置 first_token_event 再 hang(首 token 已至) + ("succeed", content) — 立即成功 + ("succeed_after", delay, content)— 真实 loop 钟睡 delay 后成功 + ("fail_after", delay, factory) — 睡 delay 后抛 `factory()` 新造的异常 + """ + + def __init__(self, scripts: dict[str, list[tuple]]): + self._scripts = {name: list(actions) for name, actions in scripts.items()} + self.calls: list[str] = [] + # 逐次记录收到的 first_token_event 身份(H5: 对冲路恒为 None,不再梯次) + self.ft_events: list[object] = [] + self.entered = {name: asyncio.Event() for name in scripts} + self.release = {name: asyncio.Event() for name in scripts} + + async def complete( + self, *, messages, source, stream, overlay, call_id, reasoning_effort, first_token_event + ): + self.calls.append(source.name) + self.ft_events.append(first_token_event) + actions = self._scripts[source.name] + action = actions.pop(0) if len(actions) > 1 else actions[0] + kind = action[0] + if kind == "hang": + self.entered[source.name].set() + await self.release[source.name].wait() + return _ok(f"ok-{source.name}") + if kind == "token_then_hang": + if first_token_event is not None: + first_token_event.set() + self.entered[source.name].set() + await self.release[source.name].wait() + return _ok(f"ok-{source.name}") + if kind == "succeed": + return _ok(action[1]) + if kind == "succeed_after": + await asyncio.sleep(action[1]) + return _ok(action[2]) + if kind == "fail_after": + await asyncio.sleep(action[1]) + raise action[2]() + raise AssertionError(f"未知剧本动作: {action!r}") + + +class RecordingEmitter: + """逐次遥测假 emitter: 记录每行的源/错误标签/逻辑调用 ID/attempt call_id。""" + + def __init__(self): + self.rows: list[dict] = [] + + async def emit_attempt( + self, + *, + request, + source, + call_id, + latency_ms, + response, + error, + reasoning_applies, + operation, + ): + self.rows.append( + { + "source": source.name, + "call_id": call_id, + "logical_call_id": request.call_context.logical_call_id + if request.call_context is not None + else None, + "error": error, + } + ) + + +def _harness( + sources, + transport, + *, + hedge_after_s=_HEDGE_AFTER_S, + max_attempts=3, + emitter=None, + selector=None, + limiter=None, + gate=None, + stall_window_s=300.0, +): + """真实 loop 钟装配(对冲计时纪律: 只用 loop 相对时长,不注入 FakeClock)。""" + limiter = limiter or InMemoryLimiter( + scope="llm", + sources={s.name: s for s in sources}, + global_limits=_NO_GLOBAL, + lease_ttl_s=100.0, + ) + gate = gate or InMemoryGate(config=_BREAKER) + mw = RetryMW( + scope="llm", + sources=sources, + # 固定配置序: 原路恒为 s1、对冲路恒为 s2,断言不依赖选源器内部状态 + selector=selector if selector is not None else StaticSelector(), + limiter=limiter, + gate=gate, + transport=transport, + retry=RetryPolicy(max_attempts=max_attempts, backoff_base_s=0.01, backoff_max_s=0.05), + backpressure=BackpressurePolicy(stall_window_s=stall_window_s, poll_interval_s=0.01), + quota_full="wait", + cooldown_memo=SourceCooldownMemo(), + emitter=emitter, + hedge_after_s=hedge_after_s, + ) + return mw, limiter, gate + + +def _req(*, stream=False, ctx=None): + return ChatRequest( + messages=[{"role": "user", "content": "hi"}], stream=stream, call_context=ctx + ) + + +def _ctx(): + return _CallContext(now=time.monotonic) + + +class TestHedgeTrigger: + """触发两形态(验收矩阵 ①): 非流式纯时间阈值;流式以首 token 未至为判据。""" + + async def test_non_stream_triggers_hedge_and_fast_leg_wins(self): + """s1 挂起、s2 即时成功: 对冲截断长尾,赢家为对冲路(⑩: 计时不含触发前等待)。""" + s1, s2 = _src("s1"), _src("s2") + transport = HedgeTransport({"s1": [("hang",)], "s2": [("succeed_after", 0.02, "fast")]}) + mw, _, _ = _harness([s1, s2], transport) + ctx = _ctx() + started = time.monotonic() + # wait_for 是防挂安全带(红相位无实现时 5s 判负),不是计时断言 + resp = await asyncio.wait_for(mw(_req(stream=False, ctx=ctx)), timeout=5) + elapsed = time.monotonic() - started + assert resp.content == "fast" and resp.source_name == "s2" + assert elapsed < 10 * _HEDGE_AFTER_S # 挂起路被对冲截断,而非等到释放 + stats = ctx.snapshot() + assert stats.hedges == 1 and stats.hedge_won is True + # 赢家裸生成时间: 下界 = s2 实际 transport 耗时(0.02s 留截断余量), + # 且严格小于总时长(不含 0.05s 触发窗等待) + assert stats.generation_ms >= 15 + assert stats.generation_ms < stats.total_latency_ms + + async def test_stream_triggers_only_when_first_token_absent(self): + """两例(①): 首 token 未至 → 触发;先置首 token 再挂起 → 不触发。""" + # 例一: 流式但首 token 未至,阈值到 → 对冲触发 + s1, s2 = _src("s1"), _src("s2") + transport = HedgeTransport({"s1": [("hang",)], "s2": [("succeed", "fast")]}) + mw, _, _ = _harness([s1, s2], transport) + ctx = _ctx() + resp = await asyncio.wait_for(mw(_req(stream=True, ctx=ctx)), timeout=5) + assert resp.source_name == "s2" + stats = ctx.snapshot() + assert stats.hedges == 1 and stats.hedge_won is True and stats.attempts == 2 + + # 例二: 首 token 已至(假 transport 先 set 再挂起)→ 不触发,误杀慢生成即此处 + transport2 = HedgeTransport({"s1": [("token_then_hang",)], "s2": [("succeed", "other")]}) + mw2, _, _ = _harness([_src("s1"), _src("s2")], transport2) + ctx2 = _ctx() + + async def release_later(): + await asyncio.sleep(4 * _HEDGE_AFTER_S) # 4× 余量确认窗口已过 + transport2.release["s1"].set() + + releaser = asyncio.create_task(release_later()) + resp2 = await asyncio.wait_for(mw2(_req(stream=True, ctx=ctx2)), timeout=5) + await releaser + assert resp2.source_name == "s1" + assert transport2.calls == ["s1"] # 对冲从未发出 + stats2 = ctx2.snapshot() + assert stats2.hedges == 0 and stats2.hedge_won is False and stats2.attempts == 1 + + +class TestHedgeRouting: + """异源排除与静默放弃(验收矩阵 ②③)。""" + + async def test_hedge_goes_to_other_source(self): + """对冲请求落在另一源;两 attempt 行共享同一 logical_call_id(②⑤)。""" + emitter = RecordingEmitter() + transport = HedgeTransport({"s1": [("hang",)], "s2": [("succeed", "hedged")]}) + mw, _, _ = _harness([_src("s1"), _src("s2")], transport, emitter=emitter) + ctx = _ctx() + resp = await asyncio.wait_for(mw(_req(ctx=ctx)), timeout=5) + assert resp.source_name == "s2" + assert transport.calls == ["s1", "s2"] # 第二请求落在异源 + # H5 接缝: 原路携带首 token 观测,对冲路恒 None(v1 单路,不再梯次) + assert [e is not None for e in transport.ft_events] == [True, False] + assert len(emitter.rows) == 2 + assert {r["logical_call_id"] for r in emitter.rows} == {ctx.logical_call_id} + assert emitter.rows[0]["call_id"] != emitter.rows[1]["call_id"] # 各 attempt 独立 ID + + async def test_hedge_silent_when_no_candidate(self): + """异源配额被占满 → 准入失败静默放弃: 不对冲、不抛错、原请求照等(③)。""" + s1 = _src("s1") + s2 = _src("s2", max_concurrency=1) + limiter = InMemoryLimiter( + scope="llm", sources={"s1": s1, "s2": s2}, global_limits=_NO_GLOBAL, lease_ttl_s=100.0 + ) + held = await limiter.try_acquire("s2", 0) # 外部预占满 s2 并发 + assert held is not None + try: + transport = HedgeTransport({"s1": [("hang",)], "s2": [("succeed", "x")]}) + mw, _, _ = _harness([s1, s2], transport, limiter=limiter) + ctx = _ctx() + task = asyncio.ensure_future(mw(_req(ctx=ctx))) + await transport.entered["s1"].wait() + # 4× 余量: 给对冲窗与那次注定失败的准入留足发生时间 + await asyncio.sleep(4 * _HEDGE_AFTER_S) + assert transport.calls == ["s1"] # 对冲静默未发出 + transport.release["s1"].set() + resp = await asyncio.wait_for(task, timeout=5) + assert resp.source_name == "s1" + stats = ctx.snapshot() + assert stats.hedges == 0 and stats.attempts == 1 and stats.hedge_won is False + finally: + await held.release() + + async def test_hedge_silent_when_single_source(self): + """单源 scope: 运行期拿不到异源候选自然静默,行为与不配阈值逐字相同(②)。""" + transport = HedgeTransport({"s1": [("hang",)]}) + mw, _, _ = _harness([_src("s1")], transport) + ctx = _ctx() + task = asyncio.ensure_future(mw(_req(ctx=ctx))) + await transport.entered["s1"].wait() + await asyncio.sleep(4 * _HEDGE_AFTER_S) # 窗口已过,仍无候选 + assert transport.calls == ["s1"] + transport.release["s1"].set() + resp = await asyncio.wait_for(task, timeout=5) + assert resp.content == "ok-s1" + stats = ctx.snapshot() + assert stats.hedges == 0 and stats.attempts == 1 and stats.hedge_won is False + + +class TestHedgeSettlementAndSignals: + """赢输记账与熔断/健康信号(验收矩阵 ④⑤;设计 §3 关键判断: 挂起 ≠ 源死亡)。""" + + async def test_winner_settles_actual_loser_keeps_est(self): + """赢家按真实 usage 结算;输家取消落 1.3.6 S3 格: est 预扣保留(④)。""" + s1 = _src("s1", tpm=1000, est_tokens=400) + s2 = _src("s2", tpm=1000, est_tokens=400) + transport = HedgeTransport({"s1": [("hang",)], "s2": [("succeed", "win")]}) + mw, limiter, _ = _harness([s1, s2], transport) + resp = await asyncio.wait_for(mw(_req()), timeout=5) + assert resp.source_name == "s2" + winner_stats = await limiter.source_stats("s2") + loser_stats = await limiter.source_stats("s1") + assert winner_stats.tpm_used == 15 # 预扣 400,实测 10+5 → settle 后只记 15 + assert loser_stats.tpm_used == 400 # 输家 est 保留(可能被上游计费,保守下限) + assert winner_stats.inflight == 0 and loser_stats.inflight == 0 + + async def test_loser_row_labelled_hedge_cancelled(self): + """输家 attempt 行 error=='hedge_cancelled',赢家行无 error,同行逻辑调用(④⑤)。 + + 终态行是 client 级语义且成功调用本就不写终态行(emit_terminal_once 只在 + 异常/取消路径触发),MW 级可观测面即这两条 attempt 行。 + """ + emitter = RecordingEmitter() + transport = HedgeTransport({"s1": [("hang",)], "s2": [("succeed", "win")]}) + mw, _, _ = _harness([_src("s1"), _src("s2")], transport, emitter=emitter) + ctx = _ctx() + await asyncio.wait_for(mw(_req(ctx=ctx)), timeout=5) + assert len(emitter.rows) == 2 + loser = next(r for r in emitter.rows if r["source"] == "s1") + winner = next(r for r in emitter.rows if r["source"] == "s2") + assert loser["error"] == "hedge_cancelled" + assert winner["error"] is None + assert loser["logical_call_id"] == winner["logical_call_id"] == ctx.logical_call_id + + async def test_loser_does_not_feed_breaker(self): + """输家取消不喂熔断失败计数、不喂健康分;赢家照常 record_success(④)。""" + selector = RecordingSelector() + gate = InMemoryGate(config=_BREAKER) + transport = HedgeTransport({"s1": [("hang",)], "s2": [("succeed", "win")]}) + mw, _, _ = _harness([_src("s1"), _src("s2")], transport, selector=selector, gate=gate) + await asyncio.wait_for(mw(_req()), timeout=5) + gate_s1 = gate._gates["s1"] + assert gate_s1.a0 + gate_s1.a1 == 0 # 熔断失败率窗口无样本 + assert selector.outcomes == [("s2", True)] # 健康喂数只有赢家的成功 + assert (await gate.try_enter("s1", "w")).allowed # 挂起源未被标记 + + async def test_attempts_two_and_no_task_leak(self): + """快照 attempts==2(含输家);返回后无本调用残留任务(⑤)。""" + before = asyncio.all_tasks() + transport = HedgeTransport({"s1": [("hang",)], "s2": [("succeed", "win")]}) + mw, _, _ = _harness([_src("s1"), _src("s2")], transport) + ctx = _ctx() + await asyncio.wait_for(mw(_req(ctx=ctx)), timeout=5) + assert ctx.snapshot().attempts == 2 + assert asyncio.all_tasks() == before + + +class TestHedgeCancellation: + """取消穿透(⑦)与期限组合(⑧): 两任务同消、不 shield、不留后台任务。""" + + async def test_external_cancel_cancels_both_legs(self): + """两路均在途时外部取消: CancelledError 上抛,两 permit 释放。 + + 输家标记只在赢家产生后才置位——外部取消下没有赢家,两行都是普通 + "cancelled"(⑦;竞速误贴属设计 §4.5 已批准残留)。 + """ + emitter = RecordingEmitter() + s1, s2 = _src("s1"), _src("s2") + transport = HedgeTransport({"s1": [("hang",)], "s2": [("hang",)]}) + mw, limiter, _ = _harness([s1, s2], transport, emitter=emitter) + task = asyncio.ensure_future(mw(_req())) + # 双 Event 栅栏: 确认对冲已触发、两路均在途,再取消(禁 sleep 猜窗口) + await transport.entered["s1"].wait() + await transport.entered["s2"].wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert {r["source"]: r["error"] for r in emitter.rows} == { + "s1": "cancelled", + "s2": "cancelled", + } + assert (await limiter.source_stats("s1")).inflight == 0 + assert (await limiter.source_stats("s2")).inflight == 0 + + async def test_deadline_cuts_hedged_tree(self): + """client 级 call_deadline_s=0.2 + 两路挂起 → CallDeadlineExceeded,permit 全释放(⑧)。""" + from tests.unit.test_client import _client + + s1, s2 = _src("s1"), _src("s2") + limiter = InMemoryLimiter( + scope="llm", sources={"s1": s1, "s2": s2}, global_limits=_NO_GLOBAL + ) + transport = HedgeTransport({"s1": [("hang",)], "s2": [("hang",)]}) + client = _client( + sources=[s1, s2], + transport=transport, + limiter=limiter, + call_deadline_s=0.2, + hedge_after_s=_HEDGE_AFTER_S, + ) + async with client: + with pytest.raises(CallDeadlineExceeded): + await client.chat([{"role": "user", "content": "hi"}], stream=False) + assert transport.calls == ["s1", "s2"] # 期限截止前对冲确已触发 + assert (await limiter.source_stats("s1")).inflight == 0 + assert (await limiter.source_stats("s2")).inflight == 0 + + +class TestHedgeWinnerAdjudication: + """赢家裁定(⑩): 取快者,含原路后发先至的对称面。""" + + async def test_primary_late_success_wins_back(self): + """s1 挂 0.3s(6× 阈值)后成功、s2 对冲路在途: 原路先完成 → 原路赢。""" + emitter = RecordingEmitter() + transport = HedgeTransport({"s1": [("succeed_after", 0.3, "late")], "s2": [("hang",)]}) + mw, _, _ = _harness([_src("s1"), _src("s2")], transport, emitter=emitter) + ctx = _ctx() + resp = await asyncio.wait_for(mw(_req(ctx=ctx)), timeout=5) + assert resp.content == "late" and resp.source_name == "s1" + stats = ctx.snapshot() + assert stats.hedges == 1 and stats.hedge_won is False + # 裸生成时间为原路那次 transport 时长(≈300ms,只断言下界与相对关系) + assert 250 <= stats.generation_ms <= stats.total_latency_ms + loser = next(r for r in emitter.rows if r["source"] == "s2") + assert loser["error"] == "hedge_cancelled" # 在途对冲路被裁为输家 + + +class TestHedgeFailureCombination: + """两败汇合(H6): 只计一次重试预算;429 分账逐字沿用 attempt 级机制。""" + + async def test_both_fail_counts_budget_once(self): + """两路 Transient: max_attempts=2 时恰进第二轮(两败只计一次),第二轮两败后才耗尽。""" + transport = HedgeTransport( + { + "s1": [("fail_after", 0.1, lambda: TransientError("p", source_name="s1"))], + "s2": [("fail_after", 0.12, lambda: TransientError("h", source_name="s2"))], + } + ) + mw, _, _ = _harness([_src("s1"), _src("s2")], transport, max_attempts=2) + ctx = _ctx() + with pytest.raises(AllSourcesExhausted) as ei: + await mw(_req(ctx=ctx)) + assert ei.value.reason == "retry_exhausted" + # 若两败计两次预算,第一轮即耗尽,这些调用根本不会发生 + assert transport.calls == ["s1", "s2", "s1", "s2"] + # 两败轮次同样登记对冲路数(设计 §4.5: 实际并发发出即计) + assert ctx.snapshot().hedges == 2 + + async def test_both_429_refund_no_budget(self): + """两路皆 429: 免预算且耗时退 stall 账——小 stall 窗下终局 stalled 而非耗尽。""" + + def _429(): + return TransientError("throttled", status_code=429, retry_after_s=0.01) + + transport = HedgeTransport( + {"s1": [("fail_after", 0.1, _429)], "s2": [("fail_after", 0.12, _429)]} + ) + mw, _, _ = _harness([_src("s1"), _src("s2")], transport, max_attempts=1, stall_window_s=0.3) + with pytest.raises(AllSourcesExhausted) as ei: + await mw(_req()) + # max_attempts=1: 任一路计预算都会当场 retry_exhausted; + # 两 429 免预算 → 循环到 stall 窗口判死 + assert ei.value.reason == "stalled" + + async def test_mixed_429_and_failure_counts_budget(self): + """一路 429 一路 Transient → 计一次预算、不退还 stall 账(_combine_failures)。""" + transport = HedgeTransport( + { + "s1": [ + ( + "fail_after", + 0.1, + lambda: TransientError("rl", status_code=429, retry_after_s=0.01), + ) + ], + "s2": [("fail_after", 0.12, lambda: TransientError("boom", source_name="s2"))], + } + ) + mw, _, _ = _harness([_src("s1"), _src("s2")], transport, max_attempts=1, stall_window_s=0.3) + with pytest.raises(AllSourcesExhausted) as ei: + await mw(_req()) + assert ei.value.reason == "retry_exhausted" + assert transport.calls == ["s1", "s2"] # 恰一轮两路: 计一次预算即耗尽 From fe616cf91d18084e230fb8c0342313305f1d7830 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 10 Sep 2026 14:04:31 -0400 Subject: [PATCH 4/7] docs: add approved hedged requests design and implementation plan --- .../2026-09-10-24-hedged-requests-design.md | 189 ++++++++++ .../plans/2026-09-10-24-hedged-requests.md | 338 ++++++++++++++++++ 2 files changed, 527 insertions(+) create mode 100644 research-wiki/designs/2026-09-10-24-hedged-requests-design.md create mode 100644 research-wiki/plans/2026-09-10-24-hedged-requests.md diff --git a/research-wiki/designs/2026-09-10-24-hedged-requests-design.md b/research-wiki/designs/2026-09-10-24-hedged-requests-design.md new file mode 100644 index 0000000..488d345 --- /dev/null +++ b/research-wiki/designs/2026-09-10-24-hedged-requests-design.md @@ -0,0 +1,189 @@ +# 长尾对冲请求(issue #24)设计 + +- 状态: **已批准**(2026-09-10 人类批准 §9 全部批准项 H1–H7,含增补 H8:`CallStats` 增 `generation_ms`/`hedge_won` 裸生成时间字段);本文件只做设计与权衡,不含实现 +- 基线: main `166b286` / 1.3.6(已发布,含 `call_deadline_s` 与取消结算修复) +- 输入: issue #24 原文(非流式挂起后正常 200:20 次里 5 次超 60s、中位 15.1s、真实负载 13% 调用吃掉 71% 模型总时间、慢调用输出中位 186 token——在等不在生成、90–96s 窄峰疑似源侧固定机制) +- 关联: `designs/2026-09-09-136-call-budgets-design.md`(期限与取消结算,本设计直接站在其 S3 格上);ARCH §6.4 取消语义、§7.2 重试、§7.3 限流;`designs/2026-08-06-issue8-stall-budget-design.md` + +## 1. 目标与非目标 + +| 项 | 内容 | +| --- | --- | +| 目标 1 | 非流式请求挂起超过阈值时,并发向**另一个等价源**再发一次,先回者赢,输家取消——把 p99 从"挂起时长"压到"阈值 + 健康源耗时" | +| 目标 2 | 流式请求以 **TTFT 未至**为触发判据(不误杀慢生成);非流式无 TTFT 可观测,用总时长阈值 | +| 目标 3 | 默认关闭;开启后的一切行为(配额、熔断、遥测、结算)可观测、可对账 | +| 非目标 A | embedding / OCR 不做对冲(无 TTFT 概念、issue 未涉、无配置面) | +| 非目标 B | 不做滚动分位数触发(`AFTER_PERCENTILE`);不做同源对冲;不加遥测新列(默认档) | +| 非目标 C | 不改 `call_deadline_s`/`deadline.py`/限流 Lua/429 分账;不改既有四分类 | + +## 2. 现状核实(现读 1.3.6 源码,不引用旧报告) + +| 事实 | 证据 | 对本设计的意义 | +| --- | --- | --- | +| 非流式路径是单 JSON 响应,**仅 total 超时**,无中途进度信号 | `transports/openai_compat.py:646-654`(`_complete_once` docstring 原文)、`:684 ttft_ms=None` | 非流式的对冲触发**物理上只有总时长阈值**一种;"挂起 vs 慢生成"在非流式不可分,只能靠阈值取值与成本上限控制误对冲 | +| 流式首 token 观测点已存在 | `openai_compat.py:569-575`(`ttft_ms` 首次赋值处) | TTFT 事件信号只需在该点 `event.set()`,探测成本近零 | +| `Transport.complete` 是公共端口签名,库内唯一实现 | `ports.py:51-60` | 加首 token 事件参数 = **公共端口签名变更**,必须进批准项(H2) | +| 每次尝试 = 选源 → 熔断门 → 限流 permit → transport,全在 `_attempt` 内 | `middleware/retry.py:276-353`;准入编排 `middleware/admission.py:171-213`(`pick`) | 对冲 = **并发跑第二次 `_attempt`**,配额/熔断/结算/pacer 全部复用,无需发明第二套准入 | +| 取消路径结算: `settlement_known=False` 时保留预扣 est | `retry.py:327-331`(1.3.6 S3 格);finally 结算 `:352-353` → `admission.py:46-60` | 对冲输家走取消路径,**结算语义现成**:额外成本上限 = 一份 est 预扣滞留 | +| 取消的 attempt 行记 `error="cancelled"` 后穿透 | `retry.py:332-334` | 输家遥测只需换一个区分字符串,零新列(§4.5) | +| `_CallContext` 承诺"每调用一个实例的**单任务**对象,计数无需锁" | `types.py:321-337`;`register_attempt` `:339-345`、`claim_terminal` `:354-360` 均为无 await 同步方法 | 对冲引入第二个并发任务,该 docstring 承诺须修订;同步方法在事件循环内天然任务安全(无 await 间隙),机制零改动 | +| 成功响应在**返回前**冻结 `call_stats` 快照 | `client.py:442`(`dataclasses.replace(response, call_stats=context.snapshot())`) | 输家取消收口必须**先于**快照,否则 `attempts` 漏计输家(§4.5) | +| 期限包整棵树,缺省 None 不进上下文 | `deadline.py:64-86`;接入点 `client.py:419-421`;配置 `config.py:190`、`:695-715` | deadline 与 hedge 正交组合,`deadline.py` 零改动(§6) | +| 429 免重试预算且耗时退 stall 账 | `retry.py:158-164`、`:250-257` | 对冲轮内某任务 429 的免预算语义沿用 attempt 级既有机制(§4.6) | +| 配置键两段/三段式天然跳过 `_load_sources`;保留段防撞名 | `config.py:401-407`(len==4 判定)、`:57`(`_RESERVED_SEGMENTS`) | 新键 `{SCOPE}__HEDGE__AFTER_S` 为 3 段,天然不被当源字段;`HEDGE` 须加进保留段(§5) | + +## 3. 备选方案与权衡 + +| 维度 | **方案 A:并发对冲(推荐)** | 方案 B:取消式投机重试 | 方案 C:仅流式对冲,非流式只靠 deadline | +| --- | --- | --- | --- | +| 做法 | 阈值到 → 并发向异源发第二次 `_attempt`,`asyncio.wait(FIRST_COMPLETED)`,赢家返回、输家 `cancel()` 并 await 收口 | 阈值到 → 取消在途 attempt,按可重试失败走既有换源重试循环 | 只对 `stream=True` 做 TTFT 对冲;非流式维持 1.3.6 现状(期限切长尾) | +| 挂起请求的信号处理 | 输家只是"被取消",**不喂熔断/健康分**——挂起的请求最终正常 200,记 failure 是错误信号(源没坏,是这一跳排队) | 必须新造一类"挂起失败":复用 Transient 会把未死源喂进熔断失败计数(`retry.py:340/346-348`),污染熔断与健康分;新造免预算类别 = 又一类四分类外特例 | 同 A(但只覆盖流式) | +| 重试预算 | 对冲不消耗 `max_attempts`——它是"一次尝试的加速形态" | 消耗预算(3 次挂起即 `AllSourcesExhausted`),除非新造免预算类 | 同 A | +| 尾部赢面 | 原请求"后发先至"时仍可用其成果;尾部的尾部 = min(两路) | 原请求成果恒被丢弃;延迟恒 = 阈值 + 重试耗时 | 流式同 A;非流式尾部 = 期限(更晚失败,不是更快成功) | +| 成本 | 对冲窗口内两路并发,输家可能被上游计费 + 一份 est 预扣滞留 | 取消更早(阈值即取消),已计费浪费**更少** | 最低(覆盖面也最小) | +| 实现量 | 大:对冲编排 + 端口加参 + 并发收口 + 遥测区分 | 约为 A 的 1/3:阈值计时器 + 取消 + 失败归类 | 中:同 A 但免非流式分支 | +| 解决 issue 现场 | 是(issue 复现即非流式) | 是 | **否**——issue 的现场就是非流式,等于没解决 | + +**关键判断**: B 的性价比看似更高(issue 数据显示挂起峰在 90s+,原请求几乎不可能后发先至),但它要回答一个 A 不用回答的问题——"挂起中的源该不该记失败"。记,则熔断/健康分被一次排队事件污染(双峰窄峰指向源侧固定机制,不是源死亡);不记,则要在四分类外新造语义。A 让输家落进 1.3.6 已有的取消路径,**零新分类语义**,且对冲拿不到配额时自然静默(饱和期不添乱)。C 不解决原问题,仅列为范围收缩的退路。 + +**推荐 A**;B 作为"预算敏感且接受熔断语义代价"的降级备选保留在批准项(H1)中由人类定夺。 + +## 4. 方案 A 的具体形态 + +### 4.1 触发条件(设计问题 1) + +| 调用形态 | 触发判据 | 机制 | +| --- | --- | --- | +| 流式 | 已过 `hedge_after_s` **且首 token 事件未置位** | `Transport.complete` 加 keyword-only 参数 `first_token_event: asyncio.Event \| None`(必填,不设默认值,与端口既有约定同款);`OpenAICompatTransport` 在 `openai_compat.py:573-575` 首 token 处 `set()`。阈值计时器 = 等待该事件,超时即触发 | +| 非流式 | 已过 `hedge_after_s`(纯总时长阈值) | `_complete_once` 物理上无中途信号(`openai_compat.py:646-654`),事件**永不置位**直到完成——同一套"等事件超时"机制自然退化为时间阈值,**零分支** | +| 分位数触发 | **v1 不做** | 滚动分位数需要 per-source 状态窗口,跨进程部署还得进 Redis;issue 的双峰形态(主峰 0–10s vs 挂起峰 90s+)用绝对阈值区分度已足够。保留为未来扩展 | + +- "非流式不对冲只做 deadline"已被方案 C 覆盖并否决(不解决 issue 现场);但**配置层面允许只对流式生效**——`stream=False` 的调用方若不接受误对冲成本,可不配阈值。 +- 误对冲的代价有界:最多 `hedge_max_extra` 次额外请求/逻辑调用,输家记账见 §4.4。 +- 时钟纪律同 deadline(136 设计 §5.1):只用**相对时长 + 事件循环钟**,不读注入 `now`——测试伪造注入钟跳变不得触发对冲,验收矩阵钉住。 + +### 4.2 对冲目标 = 异源(设计问题 2) + +| 决策 | 取法 | 理由 | +| --- | --- | --- | +| 同源 or 异源 | **异源,且仅异源** | issue 观测的 90–96s 固定窗口窄峰指向源侧机制;同源对冲 = 给同一队列再排一个号,徒增成本 | +| 等价源定义 | 同 scope 内 `SourceAdmission.pick` 正常排序选出的下一个候选——等价性由 **scope 语义**承诺(同 scope 源本就可互换,同 model 集合是常态),对冲层不发明新的等价概念 | 复用既有选源排序、冷却备忘、调用内降权(`admission.py:63-127`),不新建"等价类"配置维度 | +| 排除当前源 | `pick` 加**私有**排除参数(如 `exclude: frozenset[str]`);对冲任务以在途源名为排除集 | 改动收敛在 middleware 内部,不碰公共端口 | +| 无候选可用 | 单源 scope / 其余源全冷却、开路、配额满 → **放弃本次对冲**,继续等原请求 | 对冲是优化不是权利;单源 scope 配了阈值 = 装配期 warning、运行期自然静默(§5) | +| 同源对冲开关 | 不做(YAGNI) | 配置面少一个维度;真出现"源内分片排队"形态再立 issue | + +### 4.3 准入不独立:对冲走完整准入(设计问题 3) + +对冲请求**照常走** QuotaGate + BreakerGate + pacer + 冷却备忘(`admission.py:171-213` 的完整 `pick` 路径),**不给旁路**: + +| 情形 | 行为 | 对齐 | +| --- | --- | --- | +| 配额满 / 被熔断 / pacer 超限 | 放弃本次对冲,原请求继续等(不抛错、不排队硬等) | 对冲若绕闸,源挂起风暴时并发翻倍打进正在排队的网关——正是限流铁律要防的击穿;拿不到配额时自然静默,饱和期不添乱 | +| 限流/熔断后端不可用 | `try_acquire`/`try_enter` 抛 `GovernanceBackendError`,照常冒泡 | 铁律"后端不可用 → 报错而非放行",对冲分支不新增降级面 | +| permit 持有 | 赢家输家各持各的 permit,各自 `finally` 结算释放(`retry.py:352-353`) | 与两个独立并发调用完全同构,限流契约零改动 | + +### 4.4 成本与取消记账(设计问题 4) + +| 角色 | 结算 | 依据 | +| --- | --- | --- | +| 赢家(先成功) | 正常成功路径:`settle(实际 usage)`;`usage_source="unavailable"` 时按 est | `retry.py:300-308` 既有分支,零改动 | +| 输家(被取消) | 落 1.3.6 取消 S3 格:transport 在途、结算未定 → `settle(est)`(**保留预扣,不退款**) | `retry.py:327-331`;上游可能已对输家计费,est 保留是保守下限——与期限到期同口径,文档明写"对冲掉的那次可能已计费" | +| 输家(取消前已真失败) | 走既有失败分支结算(dead=0/瞬时=est) | `retry.py:346-347`,取消落点决定取值,S5 机制已覆盖 | +| 对冲轮内 429 | attempt 级免预算 + stall 退还照既有机制 | `retry.py:158-164` | + +**计费对账口径**:一次逻辑调用对冲一次的最大额外成本 = 一份 est 预扣滞留(窗口过期自动释放)+ 输家已被上游计费的不可观测部分。下游要能算出"对冲浪费多少钱"——靠 §4.5 的遥测区分,而不是新记账通道。 + +### 4.5 并发安全与遥测区分(设计问题 5) + +| 关注点 | 设计 | +| --- | --- | +| `_CallContext` 共享 | 两个对冲任务共享同一个 context(同一逻辑调用)。`register_attempt`/`claim_terminal`/`snapshot` 均为**无 await 同步方法**(`types.py:339-360`),事件循环内任务并发调用天然安全,机制零改动;但 `types.py:321-337` docstring 的"单任务对象"承诺须修订为"单逻辑调用、可多任务并发登记"。增补 H8 后 context 再持 `_hedges`/`_generation_ms`/`_hedge_won` 三个计数与 `register_hedge`/`record_generation` 两个同步方法,任务安全性与 `register_attempt` 同款 | +| 逻辑调用 ID | 不变:两任务共享 `logical_call_id`;各 attempt 独立 `call_id`(uuid4,`retry.py:279`) | +| 快照时点 | 赢家产生 → 输家 `cancel()` 并 **await 收口完毕**(输家 finally 的结算/遥测跑完)→ 才允许 `client.py:442` 的快照返回。`attempts` 因此恒含输家(=2),`total_latency_ms` 含输家清理耗时——与 deadline"返回时刻 = 期限 + 清理耗时"同口径 | +| 任务泄漏 | 编排用 `asyncio.wait(FIRST_COMPLETED)` + 显式收口;取消优先铁律不变:外部取消到达时两任务都被取消并穿透,不 shield、不留后台任务(ARCH §6.4) | +| 遥测行区分(默认档,零新列零 DDL) | 输家 attempt 行 `error="hedge_cancelled"`(与既有 `"cancelled"` 同通道,`retry.py:327-334` 同款字符串);赢家 attempt 行照常;`CallStats` **只增**三字段(`types.py:296-318` 既有快照对象,经 `LLMResponse.call_stats` 既有通道带出,`types.py:425`;三字段全带默认值,1.3.6 及以前构造的 `CallStats(...)` 位置调用不炸):`hedges: int = 0`(本次调用**实际并发发出**的对冲路数;触发但准入失败静默不计)、`generation_ms: int = 0`(裸生成时间,口径见下行)、`hedge_won: bool = False`(赢家是否对冲路) | +| `generation_ms` 口径(增补 H8) | **赢家那次 transport 调用的墙钟时长**(HTTP 发出到响应收完):chat/对冲 = 赢家那次;无对冲 = 成功那次 attempt;结构化重问 = 最后一轮(覆盖语义,每轮成功覆写);embedding = 各批 transport 时长之和(累加语义);OCR = 单次;缓存命中 = 0(未产生 transport 调用,0 是实测而非"未知")。**排除** admission 排队/backoff/对冲触发前等待/清理遥测;计时点收敛在三条链路 `_attempt` 的 transport 调用两侧,用该链路既有注入钟(与 `total_latency_ms` 同钟,差值才有意义);对冲编排裁定赢家后才写入 context,输家(含两路同时完成的竞速落选者)的值一律丢弃 | +| 对前端有用的对冲参数(增补 H8 取舍) | **纳入** `hedges` + `hedge_won` + `generation_ms`(经 CallStats 既有通道带出,零遥测新列);**不纳入**每路 attempt 分别耗时/输家身份——那是运维诊断面,遥测 DB attempt 行已有 `hedge_cancelled` 标签与同 `logical_call_id` 可 join 还原,不重复进公开响应。`generation_ms` 与 `total_latency_ms` 的**差值即波动开销**(等待/退避/准入/对冲触发前耗损),前端可直接展示"在等不在生成" | +| `hedge_cancelled` 标签机制 | 编排在 `cancel()` **之前**给输家任务置位标记(如 `task._polygateway_hedge_loser = True`);`_attempt` 的 CancelledError 分支读标记选 `"hedge_cancelled"`/`"cancelled"`。外部取消与对冲取消竞速时可能误贴——两任务同消、记账方向一致(est 保留),标签误贴不造成结算或熔断错误,属可接受并明写 | +| 遥测行区分(备选调) | attempt 表加 `hedge_role` 列(`NULL/'primary'/'hedge'`,PG/SQLite 各一次 DDL)——遥测列变更代价有 issue #12/#13/#15 教训,v1 不推荐;列进批准项(H3)由人类定夺 | +| 熔断/健康信号 | 输家取消**不喂**失败、赢家照常记成功——挂起不是源死亡证据(§3 关键判断) | + +### 4.6 编排形态与失败汇合 + +- 对冲轮 = 一次"超级尝试":首个成功即本轮结果;**两任务都失败**才进既有重试循环,且 `fails += 1` 只计一次(对冲是加速形态,不是两次独立尝试;429 的免预算/refund 仍在 attempt 级生效)。 +- 原 attempt 先失败、对冲在途 → 直接等对冲结果,不重试;对冲先失败、原 attempt 在途 → 继续等原 attempt(等价于未触发对冲)。 +- `retry` 循环骨架(`retry.py:228-263`)、退避、stall 判定全部不变;变化收敛在"单轮尝试的内部从单任务变任务组"。 + +## 5. 配置面(设计问题 6;默认必须关闭) + +| 键 | 值域 | 缺省 | 说明 | +| --- | --- | --- | --- | +| `{SCOPE}__HEDGE__AFTER_S` | 有限正数秒,复用 `ensure_call_deadline` 同款值域校验(`deadline.py:20-42`) | **未设 = 关闭** | 3 段键天然跳过 `_load_sources`(`config.py:401-407`);`HEDGE` 加进 `_RESERVED_SEGMENTS`(`config.py:57`)防 provider 段撞名 | +| `{SCOPE}__HEDGE__MAX_EXTRA` | int ∈ [1,3] | 1 | 每次逻辑调用最多并发对冲几路;>1 仅对冲再挂起时梯次追加 | + +装配路径与期限同款(136 设计 §4.3 形态):`GatewaySettings` 末尾追加两字段 + `__post_init__` 新守卫(`config.py:190-201` 同列);`GatewayClient.__init__` keyword-only 参数**入口即校**(`client.py:239-241` 同列);`from_settings` 透传,`from_env` 无签名变化。 + +| 守卫 | 判定 | 理由 | +| --- | --- | --- | +| `hedge_after_s ≥ min(源 timeout_s)` | `ValueError` | 对冲永不可能触发,配置即错误(与 `_validate_probe` 同款装配期炸掉哲学,`config.py:346-355`) | +| `hedge_after_s ≥ min(ttft_timeout_s)`(仅设有该键的源) | 装配期 **warning** | 流式档挂起已被 TTFT 看门狗先行切断(`openai_compat.py:561-566`),对冲形同虚设;非流式仍有效,故不升 ValueError(与 `stall_window_s ≥ max(ttft_timeout_s)` 同型交叉守卫先例,`config.py:336-344`) | +| 单源 scope 设了阈值 | 装配期 **warning**,允许 | 源集合可运行期之外的配置演进;运行期拿不到候选自然静默(§4.2) | +| `hedge_after_s ≥ call_deadline_s`(两者皆设) | `ValueError` | 期限先于对冲触发,对冲形同虚设(§6) | +| chat() per-call 覆盖参数 | **不提供** | 对冲阈值是源/渠道特性,不是任务特性(期限有 per-call 是因为任务耐心不同);需要不同阈值就装配两个 client | + +## 6. 与 `call_deadline_s` 的关系(设计问题 7) + +| 维度 | hedge | deadline | +| --- | --- | --- | +| 语义 | **提前换路**:提高 deadline 内拿到结果的概率 | **最终保险**:超过耐心即终止(治理等待) | +| 层级 | RetryMW 单轮尝试内部 | 公开边界包整棵树(`client.py:419-421`),对冲编排在树内,`deadline.py` 零改动 | +| 独立配置 | 可只配 hedge(无期限) | 可只配 deadline(1.3.6 现状) | +| 组合 | `hedge_after_s < call_deadline_s`(装配守卫强制);典型:`timeout_s=300, hedge_after_s=8, call_deadline_s=120` | 输家取消的清理耗时不受期限管辖,沿用"返回时刻 = 期限 + 清理耗时"措辞(136 设计 §5.3);**per-call 覆盖** `chat(call_deadline_s=X)` 使 X < hedge_after_s 时,该次调用对冲不触发(deadline 先切整棵树),属合法语义不告警——装配守卫只管默认值,per-call 是调用方的当次选择 | + +两者回答不同问题:deadline 让长尾**更早失败**,hedge 让调用**更快成功**——文档不得混写(136 设计 §11 已立此措辞纪律)。 + +## 7. 变更点清单(反 gold-plating;实施前置零) + +| 类别 | 内容 | +| --- | --- | +| 改动 | `middleware/retry.py`(单轮尝试 → 任务组编排 + 对冲计时 + 输家 `hedge_cancelled` 遥测 + `_attempt` transport 级计时点);`middleware/admission.py`(`pick` 加私有排除参数);`ports.py` + `transports/openai_compat.py`(`complete` 加 `first_token_event` 必填 kw,流式首 token 处置位);`config.py`(两键 + loader + 两守卫 + 保留段);`types.py`(`CallStats` 增三字段 `hedges`/`generation_ms`/`hedge_won` + `_CallContext` docstring 修订与计数方法);`client.py`(构造参数 + 透传);`embedding.py`/`ocr.py`(`_attempt` 加 transport 级计时点——只计时不对冲,非目标 A 不变) | +| 新增文件 | 无(编排收敛在 retry.py;若超 150 行可拆 `middleware/hedge.py`,实施期定) | +| 直接复用 | 取消结算 S3 格、`settle_and_release` 单一出口、准入全链路、`asyncio.timeout` 范式、假 transport/FakeClock 测试设施、限流契约套件(Lua 不改) | +| 明确不做 | 不改四分类/熔断语义/429 分账/限流 Lua/`deadline.py`;不加遥测列(默认档);不做 embedding/OCR/分位数/同源对冲/per-call 参数;不引入 shield/后台任务 | + +## 8. 测试策略(设计问题 8:事件驱动,不 sleep 撞窗口) + +| 原则 | 做法 | +| --- | --- | +| 事件驱动假 transport | 两个 `asyncio.Event`(`first_token`/`complete`)精确控制 TTFT 与完成时刻;触发判定 = "事件未置位且计时器到期",从不真睡出长尾 | +| 真实 loop 钟 + 余量 | 对冲阈值取 0.05s 级、断言容差 4–10×(`tests/unit/test_streaming.py` 既有范式,136 设计 §10 已验证稳定,不标 slow) | +| 先失败后通过 | 同一挂起场景:无对冲 → 总时长 = 挂起时长(红);开启 → 总时长 ≈ 阈值 + 快源耗时(绿) | +| 注入钟纪律 | 伪造注入 `now` 跳变 10^6 秒不得触发对冲(对冲计时只用 loop 相对时长) | + +验收矩阵(离线、不触网):① 触发两形态(流式 TTFT 未至触发/已至不触发;非流式纯时间触发);② 异源排除(断言第二请求落在另一源;无候选静默);③ 准入失败静默(配额满 → 不对冲,原请求照等);④ 赢输记账(赢家 settle 实际 usage、输家 settle est、`tpm_used` 断言);⑤ 并发安全(`attempts==2`、终态行恰 1 条、`logical_call_id` 一致、无任务泄漏告警);⑥ 默认关闭回归(现有全套件不改一行断言全绿);⑦ 外部取消穿透(两任务同消、`CancelledError` 上抛);⑧ 与 deadline 组合(期限切断含对冲的整棵树);⑨ 配置守卫四路(env/直接构造/replace/client 直传);⑩ 裸生成时间断言(增补 H8):**赢家计时不含等待**(对冲赢家的 `generation_ms` ≈ 赢家路 transport 时长,不含触发前等待/admission/backoff);**对冲赢家取快者**(对冲路赢 → `hedge_won=True` 且为对冲路时长;原路后发先至 → `hedge_won=False` 且为原路时长);**embedding 为批次和**(N 批各自 transport 时长累加);**缓存命中为 0**(第二次同 key 调用 `generation_ms == 0` 且 `attempts == 0`)。 + +## 9. 集中人类批准项 + +**状态: H1–H7 与增补 H8 全部已于 2026-09-10 获人类批准**(H1 取方案 A;H3 取零新列档;H5 取"v1 只允许 1")。 + +| # | 决策 | 推荐 | 备选代价 | +| --- | --- | --- | --- | +| H1 | 方案选型 | **A(并发对冲)** | B 省 2/3 实现量,但须新造"挂起失败"语义且污染或不污染熔断二选一;C 不解决 issue 现场 | +| H2 | `Transport.complete` 加 `first_token_event` 必填 kw(公共端口签名变更) | 批准 | 不加则流式只能用纯时间阈值,误对冲慢生成(issue 明示的反面) | +| H3 | 遥测区分档位 | 零新列(`hedge_cancelled` 字符串 + `CallStats` 三字段,含 H8) | `hedge_role` 列更规整但要 PG/SQLite 双 DDL + 迁移纪律 | +| H4 | 配置键名/值域/守卫(§5 全表,含交叉守卫 ValueError) | 按 §5 | 交叉守卫降为 warning 则错配静默 | +| H5 | `hedge_max_extra > 1` 的梯次对冲 | v1 只允许 1(键存在但上限 1 也接受) | 直接放开到 3 省一次版本,但多路对冲洗掉信号 | +| H6 | 两败计一次重试预算 | 批准 | 计两次会让对冲调用更快耗尽预算,语义说不过去 | +| H7 | embedding/OCR/分位数/同源对冲/per-call 参数全部不进本版 | 批准 | 任一纳入都是公共面扩大,需单独论证 | +| H8(增补) | `CallStats` 再增 `generation_ms`(裸生成时间,口径见 §4.5)与 `hedge_won` 两字段;retry/embedding/ocr 三条链路 `_attempt` 加 transport 级计时点 | 批准(2026-09-10,随 H1–H7 同日) | 不加则前端拿不到"在等不在生成"的量化口径,issue #24 的现场观测(13% 调用吃掉 71% 模型总时间)无法在产品面复现;每路 attempt 分别耗时与输家身份走遥测 DB join 还原,不进公开响应 | + +## 10. 残余风险(诚实标注) + +| 项 | 状态 | +| --- | --- | +| 输家取消能否止住上游计费 | 无一手证据(与 136 设计 §12 同款):est 保留只是闸内保守记账,**不是**上游真实计费的计量;文档只写"可能已计费",不写"对冲浪费上限 = est" | +| 非流式误对冲慢生成 | 物理不可分(无中途信号);只能靠阈值取值(建议 > 源 p50 数倍)与 `hedge_max_extra` 上限控制;分位数触发是未来缓解 | +| 对冲流量放大 | 开启后挂起窗口内 in-flight 翻倍;准入全走闸意味着饱和期自然静默,但**配置者须理解**对冲 = 用配额换延迟 | +| "挂起不喂熔断"的反向代价 | 一个持续挂起的源不会因对冲输家而被熔断标记;源级淘汰仍靠既有失败/超时路径——这是有意选择(§3),但运维上"挂起率"只能靠 `hedge_cancelled` 遥测行统计 | +| 两任务共享 `_CallContext` 的承诺修订 | docstring 级变更;若未来给 context 加带 await 的方法,须重审任务安全 | +| 多路对冲(H5 若放开) | 信号冲刷与成本上界均未论证,v1 不碰 | diff --git a/research-wiki/plans/2026-09-10-24-hedged-requests.md b/research-wiki/plans/2026-09-10-24-hedged-requests.md new file mode 100644 index 0000000..932c3ac --- /dev/null +++ b/research-wiki/plans/2026-09-10-24-hedged-requests.md @@ -0,0 +1,338 @@ +--- +type: plan +node_id: plan:2026-09-10-24-hedged-requests +title: "issue #24 长尾对冲请求与裸生成时间实施计划" +date: 2026-09-10 +--- + +# issue #24 长尾对冲请求与裸生成时间实施计划 + +> 设计:`research-wiki/designs/2026-09-10-24-hedged-requests-design.md`,**人类于 2026-09-10 正式批准**(§9 H1–H7 及增补 H8 全数获批;H1 取方案 A 并发对冲,H3 取零新列档,H5 取 v1 只允许单路)。 +> 计划审核门:Claude 自审 + 独立模型审查;plan 无人类门,审毕直接执行。 +> 目标:① chat 链路可选对冲(挂起超阈值时并发向**异源**再发一次,先回者赢、输家取消);② `CallStats` 增 `hedges`/`generation_ms`/`hedge_won`,三条链路 `_attempt` 加 transport 级计时;③ 默认关闭,缺省行为逐字等于 1.3.6。 +> 方案:设计 §3 方案 A——对冲轮 = 一次"超级尝试",复用完整准入(QuotaGate + BreakerGate + pacer + 冷却备忘),输家落 1.3.6 取消 S3 格结算,零新错误分类。 +> 技术:Python 3.12+、asyncio 任务组编排(`asyncio.wait(FIRST_COMPLETED)` + 显式收口)、frozen dataclass、pytest + 事件驱动假 transport + 真实 loop 钟(4–10× 余量)、ruff、import-linter。 +> 基线 HEAD:`166b286`(main,1.3.6 已发布);分支 `feature/24-hedged-requests`。 + +**范围纪律**: 不改四分类/熔断语义/429 分账/限流 Lua/`deadline.py`/`Permit` 端口;不做 embedding/OCR 对冲、分位数触发、同源对冲、per-call 对冲参数;不引入 shield/后台任务;不加遥测新列(`hedge_cancelled` 字符串 + `CallStats` 三字段经既有通道带出)。 + +## 1. 边界、授权与执行纪律 + +| 项目 | 固定边界 | +| --- | --- | +| 唯一 writer | 一工作区一 writer;父会话负责前台委派与审核派发。1.3.X 合并/发布授权沿用;跨到 1.4 或新公共面变化须停下确认 | +| 公共面 | 只做设计 §9 已批准项:`Transport.complete` 加 `first_token_event` 必填 kw(H2)、两配置键 `{SCOPE}__HEDGE__AFTER_S`/`{SCOPE}__HEDGE__MAX_EXTRA`(H4)、`GatewayClient.__init__` 两 keyword-only 参数、`CallStats` 三字段(H3+H8)、输家 `hedge_cancelled` 标签。**不新增其它键/端口方法/遥测列/异常类** | +| 对冲边界 | 仅 chat(RetryMW);EmbeddingClient/OcrClient 不加对冲参数(非目标 A),但它们的 `_attempt` 照样加 generation 计时点(H8);v1 单路对冲(H5) | +| 记账边界 | 输家取消走既有取消路径:`settlement_known=False` → `settle(est)` 保留预扣(`retry.py:327-331`);输家**不** `record_failure`、不喂熔断/健康分(挂起 ≠ 源死亡);两任务都失败才进重试且 `fails += 1` 只计一次(H6) | +| 取消铁律 | 外部取消到达时两任务同消并穿透;不 shield、不留后台任务;收口 await 允许被再取消(同 136 清理纪律) | +| 降级方向 | 限流/熔断后端不可用 → 照常冒泡(fail-closed);准入失败 → **静默放弃对冲**,原请求继续等(不抛错、不硬等);遥测仍 warning 降级 | +| 时钟纪律 | 对冲触发只用**相对时长 + 事件循环钟**(`asyncio.wait` timeout),绝不读注入 `now`;`generation_ms` 计时用该链路既有注入钟(与 `total_latency_ms` 同钟,差值才有意义;生产即 `time.monotonic`) | +| 证据纪律 | 不打印 `.env`/token/Authorization;不提交 `.pi/`、`tests/outputs/`;测试事件驱动,**禁 sleep 撞窗口**;对冲阈值用 0.05s 级真实 loop 钟,断言容差 4–10×(`tests/unit/test_streaming.py` 既有范式,不标 slow) | + +Skill 纪律:T1–T3 行为变更执行 `test-driven-development`(先红后绿证据落在本会话工具输出);每次提交执行 `commit`;T4 前执行 `requesting-code-review` 与 `verification-before-completion`;异常先 `systematic-debugging`。 + +**保真校验**: 本计划不涉及 `reference/` 参考实现迁移(对冲编排为 D13 自研语义,蓝本即本库 1.3.6 的准入/结算/取消机制),保真校验不适用。 + +## 2. 文件职责与不变接缝 + +| 动作 | 精确路径 | 职责 | +| --- | --- | --- | +| 修改 | `src/polygateway/ports.py` | `Transport.complete`(:51-60)加 `first_token_event` 必填 kw;顶部加 `import asyncio`(stdlib,不违 P7) | +| 修改 | `src/polygateway/transports/openai_compat.py` | `complete`(:426-436)加参透传;`_complete_stream`(:552 起)首 token 处(:573-575)置位;`_complete_once`(:646 起)接收但**永不置位**(docstring 明写) | +| 修改 | `src/polygateway/middleware/retry.py` | `_attempt`(:270-353)加 `first_token_event`/`generation_sink` 私有 kw + transport 级计时;`__call__`(:224-267)单轮尝试 → 任务组编排;`_past_hedge_window`/`_attempt_hedged`/`_combine_failures` 新方法;CancelledError 分支(:327-334)`hedge_cancelled` 标签;`__init__` 加 `hedge_after_s` | +| 修改 | `src/polygateway/middleware/admission.py` | `pick`(:171-213)加 keyword-only `exclude: frozenset[str] \| None = None` | +| 修改 | `src/polygateway/types.py` | `CallStats`(:296-318)增三字段(全带默认值,追加在 `total_latency_ms` 后);`_CallContext`(:321-360)docstring 修订 + 三个计数 + `record_generation`/`register_hedge`;`snapshot`(:348-353)填三字段 | +| 修改 | `src/polygateway/config.py` | `_RESERVED_SEGMENTS`(:57)加 `"HEDGE"`;`GatewaySettings` 字段(:190 后)加 `hedge_after_s`/`hedge_max_extra`;`__post_init__`(:201 后)加 `_validate_hedge()`;新增 `_load_hedge`(:695 `_load_call_deadline` 之后)与模块级 `check_hedge_assembly` 守卫;`from_env`(:396 同列)透传 | +| 修改 | `src/polygateway/client.py` | `__init__`(:239 后)加两 keyword-only 参数,入口即校(复用 `check_hedge_assembly`);RetryMW 构造(:254-273)传 `hedge_after_s`;`from_settings`(:511 同列)透传 | +| 修改 | `src/polygateway/embedding.py` | `_attempt`(:351 起,transport 调用 :373)两侧计时 + 成功分支 `record_generation(accumulate=True)` | +| 修改 | `src/polygateway/ocr.py` | `_attempt`(:376 起,`_invoke` 调用 :397)两侧计时 + 成功分支 `record_generation(accumulate=False)` | +| 新建 | `tests/unit/test_hedge.py` | 对冲编排全部用例(批次 G) | +| 修改 | `tests/unit/test_openai_compat.py` | 批次 A(端口加参);`_complete` helper(:82-90)同步签名 | +| 修改 | `tests/unit/test_retry.py` `test_types.py` `test_embedding.py` `test_ocr_client.py` `test_client.py` `test_config.py` | 批次 B–F、H;`test_retry.py:82` FakeTransport 签名同步 | +| 修改 | `tests/unit/test_backpressure.py:213` `tests/unit/test_ports.py:75` `tests/unit/test_client.py:1741` `tests/integration/test_redis_cross_connection.py:78` `tests/e2e/conftest.py:284-303` | fake/包装 transport 签名同步(e2e 包装**转发** `first_token_event`) | +| 修改 | `tests/unit/test_live_evidence.py:270-279`(`_complete` helper)+`:1258`(直调 `ObservedTransport(...).complete(...)`);`tests/unit/test_usage_source_domain.py:135`(直调真实 `OpenAICompatTransport.complete`) | **调用方**同步(独立审 B1): helper 加 `first_token_event=None` 转发、两直调传 `None`;不传则必填 kw 报 `TypeError`,unit 门必红 | +| 修改 | `CHANGELOG.md`、`README.md`、`.env.example` | 新键、三字段、对外承诺措辞(§4 T4) | +| 新建 | `research-wiki/findings/2026-09-10-24-hedged-requests-validation.md` | 红绿、命令、豁免索引,≤300 行 | + +**不改**:`errors.py`(零新异常)/`deadline.py`/`telemetry/schema.py`(零新列)/`middleware/{ratelimit,breaker,structured,cache,telemetry}.py`/`backends/**`(含全部 Lua)/`tests/contracts/**`;`embedding.py`/`ocr.py` 除计时点外一字不动;`StallClock`/`backoff_delay`/`settle_and_release` 逐字不动。若必须突破本清单,先说明最小原因交父会话核定。 + +## 3. 跨任务接口(可执行定义,禁止占位) + +### 3.1 T1:`Transport.complete` 加首 token 事件(H2) + +`ports.py:51-60` 签名改为(顺序追加在 `reasoning_effort` 后,**必填、不设默认值**,与端口既有约定同款): + +```python +async def complete( + self, *, messages: list[dict[str, Any]], source: SourceConfig, stream: bool, + overlay: dict[str, Any], call_id: str, reasoning_effort: Effort | None, + first_token_event: asyncio.Event | None, +) -> TransportResult: ... +``` + +docstring 补两句:`None` = 调用方不观测首 token(未启用对冲);非流式实现**永不置位**(物理上无中途信号,事件自然退化为纯时间阈值)。`OpenAICompatTransport.complete`(:426-436)加同款必填 kw 并透传两条路径;`_complete_stream` 在 :573-575 `if ttft_ms is None:` 块内加: + +```python +if first_token_event is not None: + first_token_event.set() +``` + +`_complete_once`(:646)接收该参数但永不置位,docstring 明写"非流式无中途信号"。`retry.py:291-300` 调用处 T1 先传字面 `first_token_event=None`(必填参数不传即全库 TypeError;T3 换成真事件)。 + +假 transport 同步纪律(`test_retry.py:71-73` 既有注释的同款):签名加 `first_token_event`,**不给默认值**;除 `test_hedge.py` 外所有 fake 忽略该参数即可。e2e 包装(`tests/e2e/conftest.py:284-303`)必须**转发**给被包 transport。 + +### 3.2 T2:`CallStats` 三字段与 `_CallContext` 计数(H3+H8) + +`types.py` `CallStats`(:296-318)在 `total_latency_ms` 后追加: + +```python +hedges: int = 0 +"""本次逻辑调用实际并发发出的对冲路数(触发但准入失败静默不计);1.3.6 及以前恒 0。""" +generation_ms: int = 0 +"""裸生成时间: 赢家/成功那次 transport 调用的墙钟时长(口径见设计 §4.5 H8)。""" +hedge_won: bool = False +"""赢家是否为对冲路;无对冲恒 False。""" +``` + +`_CallContext`(:321-360):docstring "每调用一个实例的**单任务**对象" 修订为 "每逻辑调用一个实例,**可多任务并发登记**(对冲);全部方法无 await,事件循环内任务安全";`__slots__` 与 `__init__` 加 `_generation_ms: int`/`_hedges: int`/`_hedge_won: bool`;新增两个同步方法: + +```python +def record_generation(self, elapsed_ms: int, *, accumulate: bool) -> None: + """chat/OCR 覆盖(结构化重问最后一轮为准);embedding 分批累加。""" + self._generation_ms = self._generation_ms + elapsed_ms if accumulate else elapsed_ms + +def register_hedge(self, *, hedge_won: bool) -> None: + """对冲路实际发出即计数;赢家裁定后一次性登记。""" + self._hedges += 1 + self._hedge_won = hedge_won +``` + +`snapshot`(:348-353)按字段名填 `hedges=self._hedges, generation_ms=self._generation_ms, hedge_won=self._hedge_won`。 + +### 3.3 T2:三条链路 `_attempt` 的 transport 级计时点 + +统一形态:计时**只包 transport 调用本身**,用该链路既有注入钟 `self._now`(生产 = `time.monotonic`);起点紧贴调用前、终点在返回后首句,**中间无 await**(取消落进来时 transport 未返回,本就不计)。 + +| 链路 | 计时点 | 记录点 | +| --- | --- | --- | +| chat(`retry.py:291-300`) | `gen_started = self._now()` 紧贴 `await self._transport.complete(...)` 前;返回后首句 `generation_sink.append(int((self._now() - gen_started) * 1000))` | **不在 `_attempt` 内记录**——对冲赢家归属由编排裁定。`_attempt` 加私有 kw `generation_sink: list[int]`;`__call__`(:248-256)每轮建 sink,`outcome` 为 `LLMResponse` 且 `call_context` 非 None 时 `record_generation(sink[0], accumulate=False)`(与 `register_attempt` 同款 None 守卫) | +| embedding(`embedding.py:373`) | 同款两侧包 `await self._transport.embed(...)` | 成功分支(`:386 settlement_known = True` 之后)`context.record_generation(gen_ms, accumulate=True)`——分批累加 | +| ocr(`ocr.py:397`) | 同款两侧包 `await self._invoke(...)` | 成功分支 `context.record_generation(gen_ms, accumulate=False)` | + +缓存命中/空输入不产生 transport 调用 → `generation_ms` 恒 0(0 是实测,不违 `types.py` "None 表未知" 惯例——本字段语义是时长不是用量)。 + +### 3.4 T3:对冲编排(retry.py,设计 §4.6 的唯一实现形态) + +`RetryMW.__init__` 加 keyword-only `hedge_after_s: float | None = None`(存 `self._hedge_after_s`;`hedge_max_extra` **不下传**——v1 编排固定单路,H5;配置面值域与 v1 生效口径由 §3.6 守卫负责)。模块级: + +```python +_HEDGE_LOSER_ATTR = "_polygateway_hedge_loser" +"""编排在 cancel() 之前给输家任务置位的标记;_attempt 读它选遥测标签。""" +``` + +`__call__`(:244-256)循环体内:`self._hedge_after_s is None` → **逐字旧路径**(`_attempt` 传 `first_token_event=None`,单任务,默认关闭回归门据此成立);否则 `outcome = await self._attempt_hedged(request, picked, reasons, attempt_fails)`,其后的 `_is_rate_limited`/refund/`fails` 计数机制一字不动。 + +`_attempt_hedged` 编排(Phase 注释组织;`_attempt` 相应加 `first_token_event` kw 取代 T1 的字面 None): + +```python +# Phase 1 启动原路: 事件与 sink 每轮新建(局部状态,严禁实例属性) +source, permit, entry = picked +first_token: asyncio.Event = asyncio.Event() +sink_p: list[int] = [] +primary = asyncio.create_task( + self._attempt(request, source, permit, entry, reasons, attempt_fails, + first_token_event=first_token, generation_sink=sink_p) +) +# Phase 2 触发窗: 只认"阈值到 + 首 token 未至 + 原路在途"(loop 相对时长,不读注入 now) +if not await self._past_hedge_window(primary, first_token): + return await primary # 原路已了结/首 token 已至: 等价于未配置对冲 +# Phase 3 异源准入(完整 pick 路径, 无旁路): 拿不到候选 = 静默等原路 +hedge_picked, _ = await self._admission.pick( + reasons, attempt_fails, exclude=frozenset({source.name}) +) +if hedge_picked is None: + return await primary +``` + +Phase 4-5(赢家裁定与收口)规则: + +```python +sink_h: list[int] = [] +hedge = asyncio.create_task( + self._attempt(request, *hedge_picked, reasons, attempt_fails, + first_token_event=None, generation_sink=sink_h) # v1 单路: 对冲路不再触发梯次 +) +done, pending = await asyncio.wait({primary, hedge}, return_when=asyncio.FIRST_COMPLETED) +``` + +- **裁定**:done 中有成功(LLMResponse)即赢家;两路同时成功(竞速)→ **原路优先**(`hedge_won=False`,保守不弃原路成果);done 全是失败且 pending 非空 → 等 pending 了结后再裁定。 +- **收口**:赢家产生后,对 pending 中的输家先 `setattr(task, _HEDGE_LOSER_ATTR, True)` 再 `task.cancel()`,然后 `await asyncio.gather(*pending, return_exceptions=True)`——输家 finally 的结算/遥测跑完才返回(快照含输家,`attempts==2`;`client.py:442` 快照在返回后,天然在收口之后)。两路同时完成的竞速落选者**不置标记**(它没被取消,attempt 行是正常成功/失败行)。 +- **登记**:对冲路实际发出(Phase 3 之后)即计一次;赢家裁定后 `context.record_generation(赢家 sink[0], accumulate=False)` + `context.register_hedge(hedge_won=winner is hedge)`;**两败轮次(无赢家)同样照登** `register_hedge(hedge_won=False)`——设计 §4.5 已批准口径是"实际并发发出即计,触发但准入失败静默不计";`call_context is None` 时跳过(同 `register_attempt` 守卫)。 +- **两败汇合**(H6,只计一次预算;deterministic): + +```python +def _combine_failures(primary_f: _Failed, hedge_f: _Failed) -> _Failed: + """任一非 429 优先(计预算);两路皆 429 才按 429 免预算退还 stall 账;同类取原路。""" + if _failure_reason(primary_f.exc) == "rate_limited" != _failure_reason(hedge_f.exc): + return hedge_f + return primary_f +``` + +- **取消穿透**:Phase 2-5 全程包 `except BaseException`(含 `CancelledError` 与准入冒泡的 `GovernanceBackendError`)→ 两任务(存在者)`cancel()` + `gather(return_exceptions=True)` 尽力收口后 `raise` 原异常;收口 await 允许被再取消,不 shield。 + +`_past_hedge_window` 实现红线(waiter 任务必须收口,且**不得吞外部取消**): + +```python +async def _past_hedge_window(self, primary: asyncio.Task, first_token: asyncio.Event) -> bool: + waiter = asyncio.create_task(first_token.wait()) + try: + await asyncio.wait({primary, waiter}, timeout=self._hedge_after_s, + return_when=asyncio.FIRST_COMPLETED) + finally: + waiter.cancel() + try: + await waiter + except asyncio.CancelledError: + if asyncio.current_task().cancelling(): # 外部取消,穿透 + raise + return not primary.done() and not first_token.is_set() +``` + +输家标签:`_attempt` 的 CancelledError 分支(:327-334)把 `error="cancelled"` 换成按标记选择——`label = "hedge_cancelled" if getattr(asyncio.current_task(), _HEDGE_LOSER_ATTR, False) else "cancelled"`。竞速误贴(外部取消与对冲取消同时到达)记账方向一致(est 保留),属设计 §4.5 已批准的可接受残留。 + +### 3.5 T3:`admission.pick` 私有排除参数(设计 §4.2) + +`pick`(:171-173)签名加 keyword-only `exclude: frozenset[str] | None = None`;候选循环**首部**加: + +```python +if exclude and cand.name in exclude: + continue # 被排除不是源的拒绝: 不计 gate_rejections、不写 reasons +``` + +理由:写进 `reasons`/`gate_rejections` 会污染 `on_no_runnable`(:224)的分派判据与 `per_source_reasons` 对账。对冲调用方拿到 `None` 的处置是静默等原路(§3.4 Phase 3),**严禁**对它调 `on_no_runnable`(那会按 quota/circuit 策略抛错或睡觉,语义全错)。三条既有调用方(`retry.py:244`、`embedding.py:332`、`ocr.py:348` 所在循环)不传该参数,行为逐字不变。 + +### 3.6 T3:配置两键 + 两守卫 + client 透传(H4) + +`config.py` 改动(单一定义点纪律,值域/交叉守卫只写一份): + +| 项 | 精确定义 | +| --- | --- | +| 保留段 | :57 `_RESERVED_SEGMENTS` 加 `"HEDGE"`(防 provider 段撞名);两键均 3 段,`:405` 的 `len(parts) != 4` 判据天然跳过 `_load_sources` | +| loader | 新增 `_load_hedge(scope, env)`(:695 `_load_call_deadline` 之后):`AFTER_S` 用 `_first` + `_cast(..., "float", ...)` + `ensure_call_deadline`(origin 传实际命中键名,同 `_load_call_deadline` 纪律);`MAX_EXTRA` 用 `_first` + `_cast(..., "int", ...)`,未设 = 1;返回 `{"hedge_after_s": ..., "hedge_max_extra": ...}` | +| 字段 | `GatewaySettings` :190 后追加 `hedge_after_s: float \| None = None`、`hedge_max_extra: int = 1` | +| 守卫 | 新增模块级 `check_hedge_assembly(*, hedge_after_s, hedge_max_extra, sources, call_deadline_s, origin)`,返回归一化后的 `hedge_after_s`;`GatewaySettings.__post_init__`(:201 后)加 `_validate_hedge()` 调它并 `object.__setattr__` 写回归一化值(同 `_validate_call_deadline` 形态);`GatewayClient.__init__` 调同一份(client.py:22 已 import config,合法) | +| from_env | :396 同列加 `**_load_hedge(scope_u, env)` | + +`check_hedge_assembly` 守卫全表(设计 §5;`hedge_after_s is None` 时值域归一化后直接返回,交叉守卫不查): + +| 守卫 | 判定 | +| --- | --- | +| 值域 | `ensure_call_deadline(hedge_after_s, origin)`(None 或有限正数,复用 `deadline.py` 同款校验) | +| `hedge_after_s ≥ min(源 timeout_s)` | `ValueError`(对冲永不可能触发,配置即错误) | +| `hedge_after_s ≥ call_deadline_s`(两者皆设) | `ValueError`(期限先于对冲触发) | +| `hedge_after_s ≥ min(已设 ttft_timeout_s)` | 装配期 **warning**(流式档被 TTFT 看门狗先行切断;非流式仍有效,不升 ValueError) | +| 单源 scope 设了阈值 | 装配期 **warning**,允许(运行期拿不到候选自然静默) | +| `hedge_max_extra` | 非 int/bool 或不在 [1,3] → `ValueError`;**>1 → warning**"v1 仅单路对冲生效,梯次追加为 H5 预留"(值域按设计 §5 表放到 3,运行期 H5 只允许 1,warning 保 fail-loud 不静默) | + +`client.py`:`__init__` :239 后加 keyword-only `hedge_after_s: float | None = None, hedge_max_extra: int = 1`,在 :245-247 期限校验同列调 `check_hedge_assembly(hedge_after_s=..., hedge_max_extra=..., sources=sources, call_deadline_s=self._call_deadline_s, origin="GatewayClient(...)")`;RetryMW 构造(:254-273)传 `hedge_after_s=` 归一化值(**max_extra 不下传**,§3.4);`from_settings` :511 同列传 `settings.hedge_after_s`/`settings.hedge_max_extra`。`EmbeddingClient`/`OcrClient` 不加对冲参数;它们的 settings 嵌 `GatewaySettings` 故守卫照常跑(文档明写对冲键只对 chat 生效)。 + +## 4. 任务与提交点(4 个原子提交) + +### T0:设计增补并入与本计划(本任务,无代码) + +产出:设计文档 §4.5/§7/§8/§9 增补(已完成)+ 本计划。不提交代码。 + +### T1 → 提交 1 `feat: add a required first-token event to the transport port` + +1. **先红**:批次 A(`tests/unit/test_openai_compat.py` 四用例),确认失败为 `TypeError`(签名无此 kw)而非断言值不符。 +2. 按 §3.1 改 `ports.py`、`openai_compat.py`、`retry.py:291-300` 传 None;同步六处 fake/包装(§2 表)+ `test_live_evidence.py`/`test_usage_source_domain.py` 三处调用点(§2 表末行);`_complete` helper(:82-90)加 `first_token_event=None` 默认转发(测试设施,与生产端口的"必填无默认"约定不冲突——生产端口不变)。 +3. **后绿**:批次 A 通过;`pytest tests/unit -q` 全绿且不改一行既有断言。 +4. 暂存:`src/polygateway/ports.py`、`src/polygateway/transports/openai_compat.py`、`src/polygateway/middleware/retry.py`、六个测试文件。 + +### T2 → 提交 2 `feat: expose bare generation time and hedge flags in CallStats` + +1. **先红**:批次 B(`test_types.py` 三用例,字段不存在 → `TypeError`/`AttributeError`)+ C-F(各链路计时断言,字段恒 0 → 断言失败)。 +2. 按 §3.2 改 `types.py`;按 §3.3 改三条 `_attempt` 与 `retry.py::__call__` sink 接线;**对冲计数本提交保持 0/False**(T3 才登记)。 +3. **后绿**:批次 B–F 通过;`pytest tests/unit tests/contracts -q` 全绿不改既有断言。 +4. 暂存:`src/polygateway/types.py`、`middleware/retry.py`、`embedding.py`、`ocr.py`、五个测试文件。 + +### T3 → 提交 3 `feat: add opt-in cross-source hedged requests for chat` + +1. **先红**:批次 G(`test_hedge.py`,对冲未实现 → 挂起用例超时或 `hedges==0` 断言失败)+ H(`test_config.py`,键未识 → `ValueError`/`None` 断言失败)。 +2. 按 §3.5 改 `admission.py` → §3.6 改 `config.py`/`client.py` → §3.4 改 `retry.py` 编排。 +3. **后绿**:批次 G/H 通过;**默认关闭回归门**:`pytest tests/unit tests/contracts -q` 全绿且不改一行既有断言(批次 I);`make lint` 通过。 +4. 暂存:`src/polygateway/middleware/{retry,admission}.py`、`src/polygateway/{config,client}.py`、`tests/unit/test_hedge.py`、`tests/unit/test_config.py`。 + +### T4 → 提交 4 `docs: document hedged requests and bare generation time` + +1. `CHANGELOG.md` 未发布段:对冲三句强制措辞——**默认关闭,开启即用配额换延迟**(挂起窗口内 in-flight 翻倍);**输家可能已被上游计费**(est 保留只是闸内保守记账,非上游计量);**对冲只对 chat 生效,embedding/OCR 仅获得 `generation_ms` 计时**。另记 `CallStats` 三字段口径(`generation_ms` 与 `total_latency_ms` 差值 = 波动开销)与 `hedge_cancelled` 标签的遥测 join 用法。 +2. `README.md`:能力表加"长尾对冲(可选)"一行(三句措辞同上);配置键清单加两键与值域/守卫;`CallStats` 说明处加三字段。 +3. `.env.example`:`LLM__CALL_DEADLINE_S` 注释行后加 `# LLM__HEDGE__AFTER_S=` 与 `# LLM__HEDGE__MAX_EXTRA=1`(缺省关闭,说明触发语义与成本含义)。 +4. `research-wiki/findings/2026-09-10-24-hedged-requests-validation.md`:红绿证据、命令与退出码、豁免索引(含 H5 的 v1 单路口径与竞速误贴残留)。 +5. wiki 注册本计划与 findings(add_entity/add_edge/rebuild_index);独立验证(全新上下文 verifier)与整分支审查在本提交前完成;版本 bump/发布**不在本计划内**。 + +## 5. 测试矩阵 → 任务映射 + +**设施复用核对(动手前必读)**:`tests/unit/test_retry.py:120-127` 的 `FakeSleep` 只记录不推进时钟——退避推进须用例自带 `async def sleep(s): clock.advance(s)` 闭包;`tests/unit/test_embedding.py:217` 已有 `_ClockAdvancingEmbedTransport`(尝试内推进时钟),批次 D 直接复用;`tests/unit/test_config.py:31-33` 已有 loguru WARNING 捕获 fixture,warning 断言用它(caplog 抓不到 loguru);`tests/unit/test_client.py:134-139` 已有 `InMemoryCache` 命中回路,批次 F 复用;对冲编排用例(`test_hedge.py`)用**真实 loop 钟**(不注入 FakeClock),阈值 0.05s、断言容差 4–10×;取消窗口用 `entered` Event 范式(`test_retry.py:79-89` 既有),禁 sleep 撞窗口。 + +| 批次 | 断言(→ 任务) | 落点(精确测试名) | +| --- | --- | --- | +| A 端口加参 | 流式首 token 置位事件;非流式永不置位;`None` 不观测行为不变;漏传 → `TypeError`(钉住必填)(→T1) | `test_openai_compat.py::test_stream_sets_first_token_event`、`test_non_stream_never_sets_first_token_event`、`test_none_first_token_event_keeps_behavior`、`test_first_token_event_is_required_keyword` | +| B 三字段 | 三字段默认值(0/0/False);仅旧三参数构造 `CallStats(...)` 不炸;`record_generation` 覆盖/累加语义;`register_hedge` 计数;`snapshot` 带出三字段(→T2) | `test_types.py::test_callstats_hedge_fields_default`、`test_callcontext_record_generation_overwrite_and_accumulate`、`test_callcontext_register_hedge_counts`、`test_snapshot_includes_hedge_fields` | +| C chat 计时 | 脚本 [Transient, ok]:退避推进时钟 5s、成功次 transport 推进 0.2s → `generation_ms == 200` 且 `total_latency_ms ≥ 5200`(证明排除 backoff);无对冲时 `hedges == 0`/`hedge_won is False`(→T2) | `test_retry.py::test_generation_ms_excludes_backoff_and_admission`、`test_generation_ms_zero_hedge_flags_without_hedging`(配 `_GenClockTransport` 薄包装:委托 FakeTransport 并在返回前 `clock.advance(delta)`) | +| C2 重问覆盖 | 结构化首轮坏 JSON(transport 推进 1s)、重问轮好 JSON(推进 0.2s)→ `generation_ms == 200`(最后一轮覆盖,非累加)(→T2) | `test_client.py::test_generation_ms_structured_last_round_wins`(复用 `_client(structured_strategy=...)` 与 :129-132 范式;若 `_client` 未暴露 `now` 注入,按其既有模式补 keyword 参数——测试设施非公共面) | +| D embedding 计时 | 两批各推进 0.3s → `generation_ms == 600`(批次和)(→T2) | `test_embedding.py::test_generation_ms_sums_batch_transports`(复用 `_ClockAdvancingEmbedTransport`) | +| E ocr 计时 | 单次 transport 推进 0.4s → `generation_ms == 400`(→T2) | `test_ocr_client.py::test_generation_ms_single_transport_call`(同款薄包装,推进 `test_ocr_client.py:344` 那份本地 FakeClock——历史坑:不是 contracts 那份) | +| F 缓存命中 | 第二次同 key 调用 `generation_ms == 0` 且 `attempts == 0` 且 `cache_hit is True`(→T2) | `test_client.py::test_cache_hit_generation_ms_zero`(复用 :134-139 回路) | +| G 对冲编排 | 见下表(→T3) | `tests/unit/test_hedge.py`(新建) | +| H 配置守卫 | 两键 env 解析(3 段键不被当源字段、`HEDGE` 在保留段);非法值四路(env/直接构造/`dataclasses.replace`/client 直传)→ `ValueError`;`after_s ≥ min(timeout_s)` → ValueError;`after_s ≥ min(ttft)` → warning;单源 → warning;`after_s ≥ call_deadline_s` → ValueError;`max_extra` 0/"x" → ValueError、2/3 → warning 且生效 1(→T3) | `test_config.py::test_hedge_keys_from_env_skip_source_loader`、`test_hedge_after_s_domain_four_paths`、`test_hedge_guard_below_min_timeout_raises`、`test_hedge_guard_ttft_warns`、`test_hedge_guard_single_source_warns`、`test_hedge_guard_deadline_conflict_raises`、`test_hedge_max_extra_v1_cap`;client 直传入口校验 `test_client.py::test_client_hedge_params_entry_validation` | +| I 默认关闭回归 | 不配对冲键时 `tests/unit` + `tests/contracts` 全绿,**不改一行既有断言**(→T3 门) | 全套件 | + +批次 G(`test_hedge.py`)用例全表——设施:两源 scope(`s1`/`s2`),event-driven 假 transport(每源一对 `entered`/`release` Event + 可脚本化"先置 first_token 再挂起"),真实 loop 钟,`hedge_after_s=0.05`: + +| 测试名 | 断言(设计 §8 矩阵编号) | +| --- | --- | +| `test_non_stream_triggers_hedge_and_fast_leg_wins` | s1 挂起、s2 即时成功:总时长 < 10× 阈值(①);`hedges==1`、`hedge_won is True`;`generation_ms < total_latency_ms` 且 ≥ s2 实际 transport 耗时下界(⑩:不含触发前等待) | +| `test_stream_triggers_only_when_first_token_absent` | 两例:首 token 未至 → 触发;假 transport 先 `first_token_event.set()` 再挂起 → **不触发**,`attempts==1`(①) | +| `test_hedge_goes_to_other_source` | 对冲请求落在 s2(transport.calls 断言);`logical_call_id` 两行一致(②⑤) | +| `test_hedge_silent_when_no_candidate` | s2 permit 预占满 → 不对冲:`hedges==0`、`attempts==1`、原请求放行后正常成功(③) | +| `test_hedge_silent_when_single_source` | 单源 scope:运行期自然静默,行为与不配阈值逐字相同(②) | +| `test_winner_settles_actual_loser_keeps_est` | memory limiter:赢家源 `tpm_used == 真实 usage`,输家源 `tpm_used == est`(S3 格);调用结束后两源 `inflight == 0`(④) | +| `test_loser_row_labelled_hedge_cancelled` | 假 emitter:输家 attempt 行 `error=="hedge_cancelled"`,赢家行无 error;两行 `logical_call_id` 相同;无 `terminal_failure` 行(④⑤) | +| `test_loser_does_not_feed_breaker` | memory gate:挂起源 `failure_count` 不变、健康喂数无 `ok=False`;赢家照常 `record_success`(④,§3 关键判断) | +| `test_attempts_two_and_no_task_leak` | `call_stats.attempts == 2`;返回后 `asyncio.all_tasks()` 无本调用残留任务(⑤) | +| `test_external_cancel_cancels_both_legs` | 两路均挂起,`entered` 双置位后 `task.cancel()`:`CancelledError` 上抛;两行 attempt 均 `"cancelled"`(标记只在赢家产生后置,外部取消无 `hedge_cancelled`);两 permit 释放(⑦) | +| `test_deadline_cuts_hedged_tree` | client 级 `call_deadline_s=0.2` + 两路挂起 → `CallDeadlineExceeded`;两 permit 释放(⑧) | +| `test_primary_late_success_wins_back` | s1 挂 0.3s 后成功、s2 对冲路挂起:对冲已触发但原路先完成 → `hedge_won is False`、`generation_ms` 为原路时长、s2 行 `hedge_cancelled`(⑩ 取快者) | +| `test_both_fail_counts_budget_once` | 两路 Transient:`max_attempts=2` 时恰进第二轮(两败只计一次);最终 `retry_exhausted` 在第二轮两败后(H6) | +| `test_both_429_refund_no_budget` | 两路 429:不耗预算(`max_attempts=1` 不抛 `retry_exhausted`),stall 账退还——小 `stall_window_s` 下终局 `reason=="stalled"` 而非 `"retry_exhausted"` | +| `test_mixed_429_and_failure_counts_budget` | 一路 429 一路 Transient → 计一次预算、不退还 stall 账(§3.4 `_combine_failures`) | + +命令(全部 `conda run -n PolyGateway`,禁接管道):`pytest tests/unit/test_openai_compat.py -q`、`pytest tests/unit -q`、`pytest tests/unit/test_hedge.py -q`、`pytest tests/unit tests/contracts -q`、`make lint`。真实 Redis/网关 slow 用例本计划不新增、不跑,由发布清单第 4 步按 diff 交集选子集(本 diff 触及 retry/限流结算路径,Redis 时间语义变体届时在交集内)。 + +## 6. 阻塞矩阵与交接 + +| 触发条件 | 处置 | +| --- | --- | +| 需要新增本计划外的公共键/端口方法/遥测列/异常类 | **停下上报**(设计 §9 边界之外即未批准) | +| `asyncio.current_task().cancelling()` 在目标 Python 版本语义不符 | 3.12 语义同 136 探针已验证的取消计数;若实测不符,改用"取消标志位置于 `_attempt_hedged` 局部"方案并记入 findings,不得吞取消 | +| 两路同时成功的竞速在测试中无法确定性构造 | 用双 Event 栅栏(两 transport 都等同一放行事件)构造;仍不可得则记入 findings 豁免索引,不得删"原路优先"断言 | +| 批次 G 计时断言在 CI 机器抖动 | 只断言下界与相对比较(`generation_ms < total_latency_ms`、总时长 < 10× 阈值),不断言精确值;精确值断言只在注入钟批次(C–F) | +| 想顺手让对冲路再触发梯次对冲 | **不做**(H5:v1 单路;`hedge` 任务恒传 `first_token_event=None`) | +| 想把输家记进熔断/健康分 | **不记**(§3 关键判断:挂起 ≠ 源死亡);运维面靠 `hedge_cancelled` 遥测行统计 | +| `hedge_max_extra > 1` 应 warning 还是 ValueError 存疑 | 本计划取 warning(§5 值域 [1,3] 与 H5 "v1 只允许 1" 的并存解);若人类审定应 ValueError,改 `check_hedge_assembly` 一处 + 批次 H 一条断言 | +| 发现 embedding/OCR 也想加对冲参数 | **不加**(非目标 A,H7);登记为后续 issue | + +交接物:4 个提交、1 份 findings、CHANGELOG 未发布段。版本号 bump、tag、构建、上传 registry 与 wiki 同步**不在本计划内**,按 CLAUDE.md §4.4.1 另行执行。 + +## 7. 自审 + +| 检查 | 结论 | +| --- | --- | +| 路径/行号/签名是否可执行无 TBD | 是——接入点均现读:`ports.py:51-60`、`openai_compat.py:426/552/573-575/646`、`retry.py:244/248-256/270/291-300/327-334`、`admission.py:171-213/224`、`types.py:296-318/321-360`、`config.py:57/190/201/396/405/695`、`client.py:239/245-247/254-273/511`、`embedding.py:373/386`、`ocr.py:397`、六个 fake transport 精确行号 | +| 是否复用而非重造 | 是——取消结算 S3 格、准入全链路、`settle_and_release`、`asyncio.wait` 范式、`ensure_call_deadline` 值域校验、`_ClockAdvancingEmbedTransport`/loguru 捕获 fixture/InMemoryCache 回路全部复用;新增仅 1 测试文件 + 2 配置键 + 3 字段 | +| 先失败后通过证据点 | 是——T1 批次 A(TypeError)、T2 批次 B–F(字段缺失/恒 0)、T3 批次 G/H(未实现/键未识)均先红 | +| 取消与降级铁律 | 取消穿透路径显式收口不吞没;准入失败静默(设计 §4.3 批准);后端不可用照常冒泡;无 shield/后台任务 | +| 反 gold-plating | 四分类/熔断语义/429 分账/Lua/deadline.py/遥测列/embedding-OCR 对冲/分位数/同源/per-call 参数一律不碰;`hedge_max_extra` 不下传 RetryMW(v1 无消费者) | +| 跨任务签名一致 | `first_token_event`(T1 端口 → T3 接线)、`generation_sink`/`record_generation`(T2 定义,T3 编排消费)、`check_hedge_assembly`(config 定义,client 消费)三处接缝均在 §3 写出实际代码 | +| 残余诚实标注 | 输家 est 保留 ≠ 上游真实计费计量;非流式误对冲慢生物理不可分;竞速误贴标签可接受(设计 §4.5);`hedge_max_extra` v1 生效口径取 warning(§6 阻塞矩阵已列复核点) | From 0572611af7f50dc700828a8f23730fe6e7ab7fdb Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 10 Sep 2026 14:12:30 -0400 Subject: [PATCH 5/7] docs: document hedged requests and bare generation time --- .env.example | 6 ++ CHANGELOG.md | 42 +++++++++++ README.md | 3 +- ...026-09-10-24-hedged-requests-validation.md | 71 +++++++++++++++++++ 4 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 research-wiki/findings/2026-09-10-24-hedged-requests-validation.md diff --git a/.env.example b/.env.example index 73167b7..076c85c 100644 --- a/.env.example +++ b/.env.example @@ -74,6 +74,12 @@ LLM_CIRCUIT_BREAKER_COOLDOWN=60 # 或 LLM__BREAKER__COOLDOWN_S # ── 不是单次 HTTP 超时(那是 TIMEOUT_S)。清理仍在 finally 跑完: 返回时刻 = 期限 + 清理耗时, # ── 且到期 ≠ 未产出、≠ 未计费。到期抛 CallDeadlineExceeded(不属四分类、 # ── 不属 GatewayUnavailableError 族、无 retry_after_s);非法值(0/负/nan/inf)装配期报错 ── +# LLM__HEDGE__AFTER_S= # 长尾对冲触发阈值(秒);缺省不设 = 关闭(仅 chat 生效) +# LLM__HEDGE__MAX_EXTRA=1 # 每次逻辑调用最多对冲路数;v1 仅单路生效(>1 仅装配期 warning) +# ── 触发语义: 流式 = 超阈值且首 token 未至(不误杀慢生成);非流式 = 纯总时长阈值(无中途信号, +# ── 「挂起 vs 慢生成」物理不可分,建议取源 p50 的数倍)。对冲向异源并发再发一次,走完整限流/熔断准入, +# ── 拿不到配额静默放弃。成本含义: 开启即用配额换延迟——触发窗口内 in-flight 翻倍,输家被取消后按 est +# ── 保留预扣(不喂熔断),且输家可能已被上游计费、取消止不住。须与 CALL_DEADLINE_S 组合时强制阈值 < 期限 ── # ══ 装配选择(PGW_*)══ PGW_LIMITER_BACKEND=memory # memory | redis(redis 需 REDIS_URL;多进程 worker 必须 redis) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6770b4..cce0c3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,47 @@ # Changelog +## 未发布 + +给 chat 链路加了一条**默认关闭**的长尾对冲(issue #24):一次尝试挂起超过阈值时,并发向**另一个等价源**再发一次,先回者赢、输家取消。另给 `CallStats` 追加三个字段,其中 `generation_ms`(裸生成时间)对四条链路全部生效,与是否开启对冲无关。 + +### 关于对冲,请先读这三句 + +| # | 承诺 | 展开 | +| --- | --- | --- | +| 1 | **默认关闭,开启即用配额换延迟** | 不配 `{SCOPE}__HEDGE__AFTER_S` 时行为逐字等于 1.3.6(默认关闭回归门: 既有 unit/contracts 全套件一行断言未改)。开启后触发窗口内 in-flight 翻倍——两路各自走完整准入(配额闸/熔断门/pacer),拿不到配额**静默放弃**、原请求继续等,饱和期不添乱 | +| 2 | **输家可能已被上游计费,取消止不住** | 输家落既有取消路径按 est **保留预扣**(闸内保守记账,不是上游真实计费的计量);取消能止住等待,止不住上游已经烧掉的钱。对账靠遥测: 输家 attempt 行 `error="hedge_cancelled"`,与赢家行共享同一 `logical_call_id` 可 join | +| 3 | **对冲只对 chat 生效** | `EmbeddingClient`/`OcrClient` 不加对冲参数;它们本版的收益是 `CallStats.generation_ms` 计时(见下) | + +### 触发语义 + +| 调用形态 | 触发判据 | +| --- | --- | +| 流式 | 已过 `hedge_after_s` **且首 token 未至**——慢生成不会被误对冲 | +| 非流式 | 已过 `hedge_after_s`,纯总时长阈值。非流式无中途信号,「挂起 vs 慢生成」物理不可分,只能靠阈值取值控制误对冲(建议取源 p50 的数倍) | + +输家取消**不喂熔断/健康分**(挂起 ≠ 源死亡);两路都失败才进既有重试循环且**只计一次重试预算**(对冲是一次尝试的加速形态,不是两次独立尝试;两路皆 429 时按既有规则免预算并退 stall 账,一路 429 一路真失败则计一次不退还)。 + +### CallStats 三新字段(全带默认值,1.3.6 的构造方式不炸) + +| 字段 | 口径 | +| --- | --- | +| `generation_ms: int = 0` | 裸生成时间: 赢家/成功那次 transport 调用的墙钟,**排除** admission 排队、重试退避、对冲触发前等待与清理遥测;与 `total_latency_ms` 的差值即「在等不在生成」的波动开销。结构化重问取最后一轮(覆盖),embedding 为各批 transport 之和(累加),缓存命中恒 0(未产生 transport 调用,0 是实测) | +| `hedges: int = 0` | 实际并发发出的对冲路数(触发但准入失败静默不计);未启用对冲恒 0 | +| `hedge_won: bool = False` | 赢家是否为对冲路;无对冲恒 False | + +### 与 `call_deadline_s` 的关系 + +两者正交、可独立配置: deadline 让长尾**更早失败**,hedge 让调用**更快成功**。组合时装配守卫强制 `hedge_after_s < call_deadline_s`(违反即 `ValueError`);`hedge_after_s ≥ min(源 timeout_s)` 同样装配期报错(对冲永不可能触发);`hedge_after_s ≥ min(已设 ttft_timeout_s)` 与单源 scope 设阈值降为装配期 **warning**(流式档已被看门狗先行切断 / 运行期自然静默)。per-call `chat(call_deadline_s=X)` 使 X < 对冲阈值时该次调用对冲不触发,属合法语义不告警。 + +### 配置面 + +| 键 | 值域 | 缺省 | +| --- | --- | --- | +| `{SCOPE}__HEDGE__AFTER_S` | 有限正数秒(复用期限同款值域校验) | 未设 = 关闭 | +| `{SCOPE}__HEDGE__MAX_EXTRA` | int ∈ [1,3] | 1;**v1 仅单路对冲生效**,>1 接受但装配期 warning(梯次追加为预留) | + +`GatewayClient(...)` 直传两 keyword-only 参数(`hedge_after_s` / `hedge_max_extra`)走同一份装配守卫;`from_settings`/`from_env` 照常透传。embedding/OCR 的 settings 嵌 `GatewaySettings` 故守卫照常跑,但对冲键对这两条链路不生效。 + ## 1.3.6(2026-09-10) 给一次逻辑调用加了一条**可选**墙钟硬边界(issue #22),并修好取消路径的 TPM 结算与 `Retry-After` 非有限值防御。 diff --git a/README.md b/README.md index 401c322..24a9e09 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,13 @@ | 自适应并发 | AIMD:429 削减、成功缓升,防止打爆上游 | | 背压与判死 | 配额满与熔断开路**各自**可选等待或快速失败(`QUOTA_FULL` / `CIRCUIT_OPEN`,两键不可互相替代);等待期按双条件判死(本地非生产性等待与全局无进展**同时**超窗)。stall 窗口只计**非生产性**等待(429 退避/配额轮询/熔断冷却),与 `TIMEOUT_S` 无耦合 | | 调用期限 | 一次逻辑调用可选一条**墙钟硬边界**(`{SCOPE}__CALL_DEADLINE_S` 或 `chat(call_deadline_s=...)`,缺省不启用):治理的是**等待**——重试退避、配额轮询、熔断冷却、结构化重问与 embedding 分批共享同一份期限。三条须知:①**返回时刻 = 期限 + 清理耗时**(遥测/结算/缓存收尾在 `finally` 里跑完,允许超期;实测构造达期限的 5–7 倍),库只承诺切断等待、不给返回上界;②**到期 ≠ 未产出、≠ 未计费**,上游可能已算完并计费;③**不配置即逐字保持 1.3.5 语义**(纯 429 序列仍可能长等、有限大 `Retry-After` 仍照睡)。到期抛 `CallDeadlineExceeded`,**不属四分类、不属 `GatewayUnavailableError` 族** | +| 长尾对冲 | **默认关闭**的可选加速(`{SCOPE}__HEDGE__AFTER_S`,仅 chat):一次尝试挂起超阈值时向**异源**并发再发一次,先回者赢、输家取消。流式以「首 token 未至」为触发判据(不误杀慢生成),非流式只有纯总时长阈值;对冲走完整限流/熔断准入,拿不到配额静默放弃。成本须知:开启即用配额换延迟(触发窗口内 in-flight 翻倍),输家取消按 est 保留预扣且**可能已被上游计费**(取消止不住上游);输家不喂熔断,遥测 attempt 行标 `error="hedge_cancelled"` 可按 `logical_call_id` join 对账;两路皆败只计一次重试预算;与 `call_deadline_s` 正交组合(装配守卫强制对冲阈值 < 期限) | 响应缓存 | Redis/内存;key 含 model + messages 摘要 + namespace(缓存隔离单位)+ salt + 采样参数 + 请求级推理档位(同 messages 跑 low 与 max 不互相命中),多模态 content 先摘要再 hash(防毒化);可 per-call 绕过(科研重采样) | | 流式看门狗 | TTFT / inter-token / 总超时三层活性;thinking token 刷活性不计结果;截断流(缺 `[DONE]`)判瞬时不入缓存 | | 推理可观测性 | "这次到底推理没推理"由多信号裁定(推理正文压倒 usage 明细),三态落在 `LLMResponse.thinking_observation`:`observed` / `absent` / `unknown`——**`unknown` 是"本次判不出",不是"没推理"**;本次实发档位与实测观测矛盾时按 `(源, 模型, 生效档位)` 各告警一次(能力表过期、开启未生效、注入了却观测不到;同一模型的 low 与 max 是两个独立的矛盾,不共用节流键);裁定结果随遥测落库 | | 推理档位 | 推理是**八档**(`none`/`auto`/`minimal`/`low`/`medium`/`high`/`xhigh`/`max`)而非开关:源级 `REASONING_EFFORT` + 请求级 `chat(reasoning_effort=...)`,`ENABLE_THINKING` 保留为语法糖;库带 24 条能力表(逐条 evidence 自报实测/文档推定),档位打空**默认报错并给出该模型最省的可用档与该配的键**,要静默映射需显式配 `EFFORT_FALLBACK=nearest`;实发档随 `LLMResponse.applied_effort` 与遥测落库 | | 遥测与成本 | 每次调用(含缓存命中与失败)必录 36 字段;三类行(`event_kind` = `attempt` / `cache_hit` / `terminal_failure`)加逐源诊断列(`http_status_code` / `error_type` / `cause_type` / `error_body`);SQLite / Postgres 后端(表已存在时**不需要** schema 建表权限,最小权限账号可直接用);按价格表折算成本落库(注意 `LLMResponse.cost` 本身恒为 `None`,成本只进遥测);多模态内容摘要落库不存原图 | -| 逻辑调用统计 | 治理单位是**一次逻辑调用**而非一次尝试:四种响应(chat / embedding / OCR 两种)带 `call_stats`(`logical_call_id` / `attempts` / `total_latency_ms`),重试、换源、结构化重问、embedding 分批共享同一逻辑 ID;每次**领域失败**另落一条 `terminal_failure` 行,失败调用数从此是一条 `WHERE event_kind = 'terminal_failure'`,详见[1.3.5 逻辑调用统计与失败诊断](#135-逻辑调用统计与失败诊断) | +| 逻辑调用统计 | 治理单位是**一次逻辑调用**而非一次尝试:四种响应(chat / embedding / OCR 两种)带 `call_stats`(`logical_call_id` / `attempts` / `total_latency_ms`,及裸生成时间 `generation_ms`——赢家那次 transport 调用的墙钟,排除准入排队/退避/触发前等待,与 `total_latency_ms` 的差值即「在等不在生成」——与对冲计数 `hedges` / `hedge_won`),重试、换源、结构化重问、embedding 分批共享同一逻辑 ID;每次**领域失败**另落一条 `terminal_failure` 行,失败调用数从此是一条 `WHERE event_kind = 'terminal_failure'`,详见[1.3.5 逻辑调用统计与失败诊断](#135-逻辑调用统计与失败诊断) | | 遥测的资源与降级 | Postgres 池**闲时占 0 条连接**、忙时上限可配(`PGW_TELEMETRY_PG_POOL_MAX`,缺省 4),每次写入有硬预算(`PGW_TELEMETRY_PG_WRITE_TIMEOUT_S`,缺省 5s);后端不可用是**可恢复的降级**(冷却 60s 后自动重试,DBA 建完表/放开权限即自愈),永久失能只留给 DSN 本身写错;降级状态可编程查询——`client.telemetry_status` 给出 `degraded`/`fatal`/`reason`/`dropped_rows` 等只读快照,不必再靠人工对账。**对账要同时看 `degraded` 与 `dropped_rows`**: 池饱和超预算丢的行走行级丢弃,`degraded` 保持 `False`(后端没挂,是本进程并发超了),只按 `degraded` 告警会看不见这一类丢行——而它恰是 `pool_max` 配小了的唯一信号 | | 调用方维度 | 每次调用可带 `tenant_id`(遥测表的真实列,可挂 RLS、可建复合索引)与 `meta`(≤16 个自定义 KV);四个公共方法全覆盖,校验超限即报错;**库只交付列,不启用 RLS、不建索引** | | 遥测表治理 | `llm_calls` 是**下游的表**:PG 侧缺省**不再自动 `ALTER` 补列**(`PGW_TELEMETRY_SCHEMA_MODE` 三态,不设则 sqlite→auto、postgres→manual),manual 档点名缺列并按现有列裁剪写入;`telemetry_schema_sql(backend)` 自取可粘进迁移文件的建表/补列 SQL;`PGW_TELEMETRY_TEXT_CAP` 限正文长度(**不设 = 存全文**);保留期与访问控制走[生产部署 DDL 模板](#生产部署-ddl-模板postgresql)加 `tools/telemetry_retention.py` | diff --git a/research-wiki/findings/2026-09-10-24-hedged-requests-validation.md b/research-wiki/findings/2026-09-10-24-hedged-requests-validation.md new file mode 100644 index 0000000..8c0d28c --- /dev/null +++ b/research-wiki/findings/2026-09-10-24-hedged-requests-validation.md @@ -0,0 +1,71 @@ +--- +type: finding +node_id: finding:2026-09-10-24-hedged-requests-validation +title: "issue #24 长尾对冲请求与裸生成时间验证报告" +date: 2026-09-10 +--- + +# issue #24 长尾对冲请求与裸生成时间验证报告 + +> 计划:`research-wiki/plans/2026-09-10-24-hedged-requests.md`;设计:`research-wiki/designs/2026-09-10-24-hedged-requests-design.md`(H1–H8 全数获批)。 +> 分支 `feature/1.3.7-hedged-requests`;基线 main `166b286`(1.3.6);代码提交 `0a6d622`(T1)/`463eca3`(T2)/`adc0694`(T3),文档提交见本文件 git 历史。 +> 所有命令在 `PolyGateway` conda 环境执行,未接管道(退出码不失真)。 + +## 1. 红绿证据索引 + +证据目录 `tests/outputs/137/{t1,t2,t3}`(不提交,本地留存);每份日志末尾带 `EXIT_CODE=` 行。 + +| 任务 | 相位 | 证据文件 | 退出码 | 结果与失败形态 | +| --- | --- | --- | --- | --- | +| T1 端口事件 | 红(批次 A) | `t1/red-batch-a.log` | 1 | 4 failed,143 deselected——失败均为 `TypeError`(端口签名无 `first_token_event` 必填 kw),非断言值不符 | +| T1 | 绿(批次 A) | `t1/green-batch-a.log` | 0 | 147 passed | +| T1 | 绿(unit 全套) | `t1/green-unit-full.log` | 0 | 1550 passed,既有断言一行未改 | +| T2 CallStats 三字段 | 红(批次 B–F) | `t2/red-batch-b-f.log` | 1 | 10 failed,373 deselected——字段不存在(`TypeError`/`AttributeError`)与计时字段恒 0 | +| T2 | 绿(批次 B–F) | `t2/green-batch-b-f.log` | 0 | 10 passed,373 deselected | +| T2 | 绿(unit+contracts) | `t2/green-unit-contracts.log` | 0 | 1621 passed,17 skipped | +| T3 对冲编排 | 红(编排+配置守卫,计划批次 G/H) | `t3/red_batches_d_h.txt` | 1 | 25 failed——14×RetryMW 缺 `hedge_after_s`、2×GatewayClient 缺参、4×GatewaySettings 缺属性、4×守卫未抛 ValueError、1×client 缺属性,均为未实现形态 | +| T3 | 绿(同上 25 用例) | `t3/green_batches_d_h_run1.txt` | 0 | 25 passed,2.55s | +| T3 | 绿(批次 I 默认关闭回归) | `t3/green_full_unit_contracts.txt` | 0 | **1646 passed** = 基线 1621 + 新增 25,17 skipped,48s;既有断言一行未改 | +| T3 | lint | `t3/lint.txt` | 0 | `make lint`(ruff --fix + import-linter)通过,Contracts: 1 kept,0 broken | + +T3 各相位命令原文与退出码另见 `t3/commands.md`(该文件表头"批次 D–H"为执行批次流水号,对应计划 §5 的批次 G 对冲编排 + H 配置守卫,25 用例 = 15 编排 + 8 配置 + 2 client 入口校验)。 + +## 2. 批次与提交映射 + +| 提交 | 计划任务 | 覆盖批次(计划 §5) | 关键断言 | +| --- | --- | --- | --- | +| `0a6d622` | T1 端口事件(H2) | A | 流式首 token 置位事件;非流式永不置位;`None` 不观测行为不变;漏传必填 kw 即 `TypeError` | +| `463eca3` | T2 三字段与计时(H3+H8) | B–F | 三字段默认 0/0/False;chat 计时排除退避与准入;结构化重问取最后一轮;embedding 批次累加;OCR 单次;缓存命中恒 0 | +| `adc0694` | T3 对冲编排(H1/H4/H5/H6) | G、H、I | 触发两形态;异源排除;准入失败静默;赢家 settle 实际/输家 settle est;`hedge_cancelled` 标签;输家不喂熔断;`attempts==2` 无任务泄漏;外部取消两路穿透;deadline 切断对冲树;原路后发先至;两败计一次预算;两 429 免预算退 stall;混合失败计一次;配置守卫四路 | + +## 3. 实测核对(文档承诺 vs 运行实测,本文件交付时复核) + +| 承诺 | 核对方式 | 实测结果 | +| --- | --- | --- | +| `CallStats` 三字段默认值 | `CallStats(logical_call_id='x', attempts=1, total_latency_ms=5)` 仅旧三参构造 | `hedges=0`、`generation_ms=0`、`hedge_won=False`,构造不炸 | +| `hedge_won` 语义 | 现读 `middleware/retry.py:498-578` | 赢家裁定后 `hedge_won=winner is hedge`;两败轮次照登 `hedge_won=False` 且裸生成时间无归属不记;触发但准入失败不计 `hedges` | +| 两键 env 解析 | `GatewaySettings.from_env(env=...)` 注入两源 + `LLM__HEDGE__AFTER_S=8`/`MAX_EXTRA=2` | 解析出 `hedge_after_s=8.0`/`hedge_max_extra=2`,两源照常加载(3 段键未被当源字段);`MAX_EXTRA=2` 触发装配期 warning「v1 仅单路对冲生效」;不设两键时 `None`/`1` | +| 版本号不动 | `pyproject.toml` 与 `src/polygateway/__init__.py` | 两处均 `1.3.6`,本计划不 bump、不 tag、不发布 | +| 文档行号 | README/CHANGELOG/.env.example 接入点 | 均按交付时现读行号接入,未沿用计划旧行号 | + +## 4. 豁免索引(未跑项与去向) + +| 未跑项 | 理由 | 去向 | +| --- | --- | --- | +| `tests/e2e/`(slow) | 本计划不新增、不跑真实网关用例;`tests/e2e/conftest.py` 包装 transport 已同步转发 `first_token_event` | 发布清单第 4 步按 diff 交集选子集(本 diff 触及公开入口与 e2e 设施,e2e 冒烟届时在交集内) | +| Redis 契约/integration(slow) | T1–T3 未触碰限流 Lua、`Permit` 端口、`backends/**` 与 `tests/contracts/**`(计划 §2 不改清单);对冲结算复用既有 `settle_and_release` 路径,无新 Lua 行为可测 | 同按发布交集规则判断;diff 已触及 retry/限流结算路径,**Redis 时间语义变体届时在交集内**(计划 §5 末句已登记) | +| `test_thinking_live.py` 全模型矩阵 | 未动 `thinking.py`/能力注册表 | 不在交集,复用最近一次有效矩阵证据 | + +## 5. 残余复述(设计 §10 与计划 §6 的已批准口径) + +| 项 | 口径 | +| --- | --- | +| 非流式「挂起 vs 慢生成」 | 物理不可分(无中途信号),只能靠阈值取值(建议源 p50 数倍)与 `hedge_max_extra` 上限控制误对冲;分位数触发为未来扩展 | +| 输家取消止不住上游计费 | est 保留只是闸内保守记账,不是上游真实计量的计量;文档只写「可能已计费」,不写「浪费上限 = est」 | +| 竞速误贴标签 | 外部取消与对冲取消同时到达时,输家行可能误贴 `cancelled`/`hedge_cancelled`;两任务同消、记账方向一致(est 保留),不造成结算或熔断错误,可接受 | +| `hedge_max_extra` v1 单路 | 值域 [1,3] 接受,>1 仅装配期 warning,运行期恒单路(对冲任务恒传 `first_token_event=None`,不再触发梯次);若未来审定应为 `ValueError`,改 `check_hedge_assembly` 一处 + 批次 H 一条断言 | +| 挂起源不喂熔断的反向代价 | 持续挂起的源不会因对冲输家被熔断标记;「挂起率」只能靠 `hedge_cancelled` 遥测行统计(同 `logical_call_id` join 还原) | + +## 6. 结论 + +T1–T3 全部行为变更具备先红后绿证据(§1),默认关闭回归门成立(1646 passed 且既有断言一行未改),lint 与 import-linter 契约通过。文档承诺经运行实测核对(§3)。版本 bump、tag、发布与 slow/e2e 交集子集**不在本计划内**,按 CLAUDE.md §4.4.1 另行执行。 From 9f7d4071200a0fb889b68379ddc0a1635aebd46f Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 10 Sep 2026 14:47:40 -0400 Subject: [PATCH 6/7] fix: skip hedge dispatch when primary finishes during admission Address branch review findings 1-4 on feature/1.3.7-hedged-requests: - src/polygateway/middleware/retry.py: recheck primary.done() after hedge admission in _attempt_hedged; release the hedge permit via settle_and_release(permit, 0) (release_probe for probe entries) and adjudicate the primary directly instead of firing a billable HTTP request that would be cancelled immediately - src/polygateway/config.py: check_hedge_assembly raises a hedge-located ValueError for empty sources instead of a bare min() error - tests/unit/test_hedge.py: pin that an injected FakeClock jump of 10^6 seconds does not trigger hedging (design section 8); pin pick(exclude) counting no gate_rejections and leaving reasons untouched; pin silent hedge abandonment when the candidate circuit is open; deterministic regression for the admission-window race (BlockingLimiter harness) - tests/unit/test_config.py: assert the empty-sources guard message locates the hedge key Red-to-green evidence in tests/outputs/137/review-fixes/ --- src/polygateway/config.py | 7 ++ src/polygateway/middleware/retry.py | 16 ++++ tests/unit/test_config.py | 25 +++++ tests/unit/test_hedge.py | 140 +++++++++++++++++++++++++++- 4 files changed, 187 insertions(+), 1 deletion(-) diff --git a/src/polygateway/config.py b/src/polygateway/config.py index 511d2fc..f2ff270 100644 --- a/src/polygateway/config.py +++ b/src/polygateway/config.py @@ -809,6 +809,13 @@ def check_hedge_assembly( after = ensure_call_deadline(hedge_after_s, origin) if after is None: return None + if not sources: + # 直传路(GatewayClient(sources=[], hedge_after_s=...))没有 settings 的 + # 非空守卫先行拦截;报错必须定位到 hedge,而不是裸 min() 空序列异常 + raise ValueError( + "hedge_after_s({SCOPE}__HEDGE__AFTER_S)的交叉守卫要求 sources 不能为空;" + "对冲已启用但没有可校验的源" + ) min_timeout = min(s.timeout_s for s in sources) if after >= min_timeout: raise ValueError( diff --git a/src/polygateway/middleware/retry.py b/src/polygateway/middleware/retry.py index 022f817..6540679 100644 --- a/src/polygateway/middleware/retry.py +++ b/src/polygateway/middleware/retry.py @@ -456,6 +456,22 @@ class RetryMW: if isinstance(outcome, LLMResponse): self._record_generation(request, sink_p) return outcome + # Phase 3.5 准入后复查: 原路可能在对冲准入的 await 期间已了结——此时 + # 一个对冲请求都不发(那是白付一次真实计费请求 + 一份 est 滞留 + 一条 + # hedge_cancelled 行)。按既有语义释放刚拿到的对冲准入(probe 须 + # release_probe,顺序同 _attempt 取消分支),直接裁定原路结果 + if primary.done(): + hedge_source, hedge_permit, hedge_entry = hedge_picked + try: + if hedge_entry.is_probe: + await self._record_quietly(self._breaker.release_probe(hedge_entry)) + finally: + self._pacer.leave(hedge_source.name) + await settle_and_release(hedge_permit, 0) + outcome = await primary + if isinstance(outcome, LLMResponse): + self._record_generation(request, sink_p) + return outcome # Phase 4 启动对冲路(v1 单路,H5): 对冲路恒传 first_token_event=None, # 不再触发梯次对冲 sink_h: list[int] = [] diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 71ac727..396442e 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1166,6 +1166,31 @@ class TestHedgeConfig: with pytest.raises(ValueError, match=r"GatewayClient\(hedge_after_s"): _client(hedge_after_s=0) + def test_hedge_guard_empty_sources_raises_with_hedge_location(self): + """直传路空 sources + 设阈值: ValueError 且消息定位到 hedge(不是裸 min() 报错)。""" + from polygateway.config import check_hedge_assembly + + with pytest.raises(ValueError, match="sources") as ei: + check_hedge_assembly( + hedge_after_s=8, + hedge_max_extra=1, + sources=[], + call_deadline_s=None, + origin="GatewayClient(hedge_after_s=8)", + ) + assert "hedge_after_s" in str(ei.value) # 定位得到是哪个键 + # 未启用对冲(None)时空 sources 直接放行: 交叉守卫没有可校验的对象 + assert ( + check_hedge_assembly( + hedge_after_s=None, + hedge_max_extra=1, + sources=[], + call_deadline_s=None, + origin="test", + ) + is None + ) + def test_hedge_guard_below_min_timeout_raises(self): """阈值 ≥ 最小源 timeout_s = 对冲永不可能触发,装配期炸掉(ValueError)。""" env = self._two_source_env(**{"LLM__HEDGE__AFTER_S": "90"}) # min(timeout)=90 diff --git a/tests/unit/test_hedge.py b/tests/unit/test_hedge.py index 06fe8af..d419414 100644 --- a/tests/unit/test_hedge.py +++ b/tests/unit/test_hedge.py @@ -25,6 +25,7 @@ from polygateway.types import ( RetryPolicy, _CallContext, ) +from tests.contracts.conftest import FakeClock from tests.unit.test_retry import RecordingSelector, StaticSelector, _ok, _src _BREAKER = BreakerConfig(fail_threshold=3, cooldown_s=60.0, probe_ttl_s=120.0) @@ -80,6 +81,38 @@ class HedgeTransport: raise AssertionError(f"未知剧本动作: {action!r}") +class BlockingLimiter: + """限流包装: 挂起指定源的 try_acquire 直到测试放行(确定性复现"对冲准入挂起")。 + + `acquiring` 置位 = 对冲准入已停在该源闸内;`allow` 置位后才继续。只拦对冲 + 会走到的源,原路准入不受影响;其余方法逐字委托内层 InMemoryLimiter。 + """ + + def __init__(self, inner: InMemoryLimiter, block_source: str): + self._inner = inner + self._block_source = block_source + self.acquiring = asyncio.Event() + self.allow = asyncio.Event() + + async def try_acquire(self, source_key, est_tokens): + if source_key == self._block_source: + self.acquiring.set() + await self.allow.wait() + return await self._inner.try_acquire(source_key, est_tokens) + + async def acquire(self, source_key, est_tokens): + return await self._inner.acquire(source_key, est_tokens) + + async def source_stats(self, source_key): + return await self._inner.source_stats(source_key) + + async def mark_progress(self): + return await self._inner.mark_progress() + + async def progress_age_s(self): + return await self._inner.progress_age_s() + + class RecordingEmitter: """逐次遥测假 emitter: 记录每行的源/错误标签/逻辑调用 ID/attempt call_id。""" @@ -121,8 +154,13 @@ def _harness( limiter=None, gate=None, stall_window_s=300.0, + now=None, ): - """真实 loop 钟装配(对冲计时纪律: 只用 loop 相对时长,不注入 FakeClock)。""" + """真实 loop 钟装配(对冲计时纪律: 只用 loop 相对时长,不注入 FakeClock)。 + + `now` 仅供"注入钟与对冲触发正交"用例注入 FakeClock——触发路径结构性不读 + 它,注入只是为了证明这一点。 + """ limiter = limiter or InMemoryLimiter( scope="llm", sources={s.name: s for s in sources}, @@ -144,6 +182,7 @@ def _harness( cooldown_memo=SourceCooldownMemo(), emitter=emitter, hedge_after_s=hedge_after_s, + **({"now": now} if now is not None else {}), ) return mw, limiter, gate @@ -209,6 +248,30 @@ class TestHedgeTrigger: stats2 = ctx2.snapshot() assert stats2.hedges == 0 and stats2.hedge_won is False and stats2.attempts == 1 + async def test_injected_clock_jump_does_not_trigger_hedge(self): + """对冲触发只认真实 loop 钟: 注入钟跳 10^6 秒不得触发对冲(设计 §8 验收矩阵)。 + + s1 挂起剧本 + 触发窗内注入钟拨快 10^6 秒: 若触发路径误读注入钟,对冲会 + **立即**发出;断言对冲实际发出时刻不早于真实 loop 阈值(下界断言,不断 + 精确值),形态同 test_client.py:1974 deadline 的注入钟对应用例。 + """ + clock = FakeClock() + transport = HedgeTransport({"s1": [("hang",)], "s2": [("succeed", "hedged")]}) + mw, _, _ = _harness([_src("s1"), _src("s2")], transport, now=clock) + ctx = _ctx() + task = asyncio.ensure_future(mw(_req(ctx=ctx))) + await transport.entered["s1"].wait() # 原路在途,触发窗计时中 + clock.advance(1_000_000.0) # 跳变落在窗内: 误读注入钟即立刻触发 + started = time.monotonic() + resp = await asyncio.wait_for(task, timeout=5) + elapsed = time.monotonic() - started + assert resp.source_name == "s2" # 对冲确由真实 loop 阈值触发并截断长尾 + assert transport.calls == ["s1", "s2"] + # 下界留 20% 调度余量;误读注入钟的触发是毫秒级,与此差一个数量级以上 + assert elapsed >= _HEDGE_AFTER_S * 0.8 + stats = ctx.snapshot() + assert stats.hedges == 1 and stats.hedge_won is True + class TestHedgeRouting: """异源排除与静默放弃(验收矩阵 ②③)。""" @@ -254,6 +317,81 @@ class TestHedgeRouting: finally: await held.release() + async def test_pick_exclude_all_is_not_a_rejection(self): + """exclude 覆盖全源 → 返回 None 且 gate_rejections==0、reasons 不写(排除 ≠ 拒绝)。 + + admission 级直接钉(admission.py:192 `continue` 语义): 若未来重构把排除计入 + gate_rejections,`on_no_runnable` 的"全源熔断类拒绝"判据会被污染,此钉当场报警。 + """ + transport = HedgeTransport({"s1": [("succeed", "x")], "s2": [("succeed", "y")]}) + mw, _, _ = _harness([_src("s1"), _src("s2")], transport) + reasons = {"prior": "rate_limited"} # 既有原因须原样保留 + picked, gate_rejections = await mw._admission.pick( + reasons, {}, exclude=frozenset({"s1", "s2"}) + ) + assert picked is None + assert gate_rejections == 0 + assert reasons == {"prior": "rate_limited"} + + async def test_hedge_silent_when_candidate_circuit_open(self): + """对冲候选被熔断开路 → 静默放弃: 不对冲、不抛错、原请求照等(②③的另一形态)。 + + 现有限流闸用例只钉了"配额占满"一条静默路径;开路/pacer 拒绝走 pick 的另一 + 分支(gate_rejections 计数、reasons 写 circuit_open、settle_and_release 后 + 返回 None),同样不得发出对冲请求。 + """ + gate = InMemoryGate(config=_BREAKER) + entry = await gate.try_enter("s2", "test-owner") + assert entry.allowed + await gate.record_failure(entry, "source_dead", True) # SourceDead 一击即熔 + transport = HedgeTransport({"s1": [("hang",)], "s2": [("succeed", "x")]}) + mw, _, _ = _harness([_src("s1"), _src("s2")], transport, gate=gate) + ctx = _ctx() + task = asyncio.ensure_future(mw(_req(ctx=ctx))) + await transport.entered["s1"].wait() + # 4× 余量: 给对冲窗与那次注定被开路拒绝的准入留足发生时间 + await asyncio.sleep(4 * _HEDGE_AFTER_S) + assert transport.calls == ["s1"] # 对冲静默未发出 + transport.release["s1"].set() + resp = await asyncio.wait_for(task, timeout=5) + assert resp.source_name == "s1" + stats = ctx.snapshot() + assert stats.hedges == 0 and stats.attempts == 1 and stats.hedge_won is False + + async def test_primary_done_during_hedge_admission_sends_no_hedge(self): + """原路在对冲准入 await 期间已完成: 释放对冲准入直接裁定,一个对冲请求都不发。 + + 剧本钉死窗口(禁 sleep 猜): s2 的 try_acquire 挂起(对冲准入停在闸内) → + 放行 s1 → 轮询 s1 inflight 归零(_attempt finally 结算完,primary 必 done) + → 此刻才放行对冲准入。pick 返回时原路已了结,编排必须不落 create_task。 + """ + s1 = _src("s1", tpm=1000, est_tokens=400) + s2 = _src("s2", tpm=1000, est_tokens=400) + inner = InMemoryLimiter( + scope="llm", + sources={"s1": s1, "s2": s2}, + global_limits=_NO_GLOBAL, + lease_ttl_s=100.0, + ) + limiter = BlockingLimiter(inner, "s2") + transport = HedgeTransport({"s1": [("hang",)], "s2": [("succeed", "hedge")]}) + mw, _, _ = _harness([s1, s2], transport, limiter=limiter) + ctx = _ctx() + task = asyncio.ensure_future(mw(_req(ctx=ctx))) + await transport.entered["s1"].wait() # 原路在途 + await limiter.acquiring.wait() # 对冲准入停在 s2 闸内(触发窗已过) + transport.release["s1"].set() # 原路放行完成 + while (await inner.source_stats("s1")).inflight != 0: + await asyncio.sleep(0.001) # 结算完 = primary 已 done(同一任务步内返回) + limiter.allow.set() # pick 此刻才返回: primary.done() 已成立 + resp = await asyncio.wait_for(task, timeout=5) + assert resp.content == "ok-s1" and resp.source_name == "s1" + assert transport.calls == ["s1"] # 对冲 HTTP 从未发出(未修前这里会看到 s2) + stats = ctx.snapshot() + assert stats.hedges == 0 and stats.attempts == 1 and stats.hedge_won is False + s2_stats = await inner.source_stats("s2") + assert s2_stats.inflight == 0 and s2_stats.tpm_used == 0 # 对冲准入按 0 结算释放 + async def test_hedge_silent_when_single_source(self): """单源 scope: 运行期拿不到异源候选自然静默,行为与不配阈值逐字相同(②)。""" transport = HedgeTransport({"s1": [("hang",)]}) From c60011b0346a75a4e9b7f70610a13ddb627488ee Mon Sep 17 00:00:00 2001 From: iomgaa Date: Thu, 10 Sep 2026 14:51:53 -0400 Subject: [PATCH 7/7] chore: release 1.3.7 --- CHANGELOG.md | 2 +- README.md | 2 +- pyproject.toml | 2 +- research-wiki/ROADMAP.md | 2 ++ src/polygateway/__init__.py | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cce0c3f..4c0c0a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 未发布 +## 1.3.7(2026-09-10) 给 chat 链路加了一条**默认关闭**的长尾对冲(issue #24):一次尝试挂起超过阈值时,并发向**另一个等价源**再发一次,先回者赢、输家取消。另给 `CallStats` 追加三个字段,其中 `generation_ms`(裸生成时间)对四条链路全部生效,与是否开启对冲无关。 diff --git a/README.md b/README.md index 24a9e09..99e4a0e 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ stats.total_latency_ms # 含缓存 IO、退避、准入等待、重 ```bash pip install --extra-index-url https://gitea.iomgaa.online/api/packages/iomgaa/pypi/simple/ \ - "polygateway[redis,postgres,structured]>=1.3.6,<2" + "polygateway[redis,postgres,structured]>=1.3.7,<2" ``` 核心仅依赖 `httpx` + `pydantic`;按需选 extras: diff --git a/pyproject.toml b/pyproject.toml index acabea8..1a283d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "polygateway" -version = "1.3.6" +version = "1.3.7" description = "PolyGateway：实验室统一的大语言模型（LLM/VLM/OCR）调度与中转库——多源、限流、重试、熔断、缓存、遥测" # registry 包页面的正文只认这一项:缺了页面就是一片空白(1.1.2 的教训,twine 会警告 # long_description missing 但不阻塞上传)。README 在打包时被固化进产物,发布后再改无效。 diff --git a/research-wiki/ROADMAP.md b/research-wiki/ROADMAP.md index e80cc2b..03e0e1a 100644 --- a/research-wiki/ROADMAP.md +++ b/research-wiki/ROADMAP.md @@ -68,6 +68,8 @@ 音频端口实现(D10)、SDK transport(openai/anthropic 原生协议,D2 预留)、GLM OCR invoker(D9 预留)、内网 pip index(Q1)、多项目共用 Redis 的 namespace 治理、harness-eval 评估流水线激活。 +**已交付增补(2026-09-10)**: 长尾对冲(issue #24,1.3.7)——chat 单路并发对冲,触发=流式首 token 未至/非流式纯时长阈值,异源走完整准入,默认关闭;`CallStats` 增 `generation_ms`(裸生成时间)/`hedges`/`hedge_won`。设计 designs/2026-09-10-24-hedged-requests-design.md(人类已批准 H1-H8),实施 plans/2026-09-10-24-hedged-requests.md。后续储备:梯次多路对冲(H5 预留)、分位数触发、embedding/OCR 对冲(现不做)。 + ## 7. 开放决策依赖 | 决策 | 阻塞点 | 需拍板时间 | diff --git a/src/polygateway/__init__.py b/src/polygateway/__init__.py index a6225bf..2df241a 100644 --- a/src/polygateway/__init__.py +++ b/src/polygateway/__init__.py @@ -52,7 +52,7 @@ from polygateway.types import ( ThinkingObservation, ) -__version__ = "1.3.6" +__version__ = "1.3.7" __all__ = [ "DEFAULT_PROFILES",