feat: add opt-in cross-source hedged requests for chat
This commit is contained in:
+127
-2
@@ -30,7 +30,7 @@ from polygateway.types import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
||||
# FIELD → (SourceConfig 属性, 类型);CHS config.py:95-104 全集 + M1 新增
|
||||
_SOURCE_FIELDS: dict[str, tuple[str, str]] = {
|
||||
@@ -54,7 +54,7 @@ _SOURCE_FIELDS: dict[str, tuple[str, str]] = {
|
||||
"TRUST_ENV": ("trust_env", "bool"),
|
||||
"EXTRA_BODY": ("extra_body", "json"),
|
||||
}
|
||||
_RESERVED_SEGMENTS = frozenset({"GLOBAL", "RETRY", "BREAKER", "BACKPRESSURE"})
|
||||
_RESERVED_SEGMENTS = frozenset({"GLOBAL", "RETRY", "BREAKER", "BACKPRESSURE", "HEDGE"})
|
||||
_SELECTORS = frozenset({"round_robin", "least_inflight", "health_aware"})
|
||||
_QUOTA_FULL = frozenset({"wait", "fail_fast"})
|
||||
# 熔断全拒时的处置(issue #14);值域与 _QUOTA_FULL 相同但语义不同——配额满是
|
||||
@@ -188,6 +188,12 @@ class GatewaySettings:
|
||||
# 与 `OcrSettings.gateway` 自动继承。值域由 `_validate_call_deadline` 把关,
|
||||
# 直接构造、`dataclasses.replace` 与 env 三条路一致
|
||||
call_deadline_s: float | None = None
|
||||
# 长尾对冲(issue #24 H4): 挂起超阈值时并发向异源再发一次,先回者赢、输家取消。
|
||||
# 缺省 None = 关闭,行为逐字等于 1.3.6。`hedge_max_extra` 值域 [1,3] 为 H5 梯次
|
||||
# 预留,v1 仅单路生效(>1 装配期 warning);**不下传 RetryMW**。值域与交叉守卫
|
||||
# 由 `check_hedge_assembly` 单一定义点把关,直接构造/replace/env 三路一致
|
||||
hedge_after_s: float | None = None
|
||||
hedge_max_extra: int = 1
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self._normalize()
|
||||
@@ -199,6 +205,7 @@ class GatewaySettings:
|
||||
self._validate_stall()
|
||||
self._validate_probe()
|
||||
self._validate_call_deadline()
|
||||
self._validate_hedge()
|
||||
|
||||
def _normalize(self) -> None:
|
||||
"""把 `from_env` 一直在做的规范化补到构造路上,两条路必须产出同一个值。
|
||||
@@ -365,6 +372,24 @@ class GatewaySettings:
|
||||
ensure_call_deadline(self.call_deadline_s, "GatewaySettings.call_deadline_s"),
|
||||
)
|
||||
|
||||
def _validate_hedge(self) -> None:
|
||||
"""对冲装配守卫(issue #24 H4): 与期限同款,盖住直接构造与 replace 两条路。
|
||||
|
||||
env 路的值域错误已在 `_load_hedge` 里带真实键名报过;此处对合法值是幂等
|
||||
空操作,交叉守卫(阈值 vs timeout/deadline/ttft、单源)只在这里有一处。
|
||||
"""
|
||||
object.__setattr__(
|
||||
self,
|
||||
"hedge_after_s",
|
||||
check_hedge_assembly(
|
||||
hedge_after_s=self.hedge_after_s,
|
||||
hedge_max_extra=self.hedge_max_extra,
|
||||
sources=self.sources,
|
||||
call_deadline_s=self.call_deadline_s,
|
||||
origin="GatewaySettings.hedge_after_s",
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_env(
|
||||
cls,
|
||||
@@ -394,6 +419,7 @@ class GatewaySettings:
|
||||
quota_full=_load_choice(env, f"{scope_u}__QUOTA_FULL", _QUOTA_FULL, "wait"),
|
||||
circuit_open=_load_choice(env, f"{scope_u}__CIRCUIT_OPEN", _CIRCUIT_OPEN, "fail_fast"),
|
||||
call_deadline_s=_load_call_deadline(scope_u, env),
|
||||
**_load_hedge(scope_u, env),
|
||||
**_load_pgw(env),
|
||||
)
|
||||
|
||||
@@ -716,6 +742,105 @@ def _load_call_deadline(scope: str, env: Mapping[str, str]) -> float | None:
|
||||
return ensure_call_deadline(_cast(found[1], "float", found[0]), found[0])
|
||||
|
||||
|
||||
def _load_hedge(scope: str, env: Mapping[str, str]) -> dict[str, object]:
|
||||
"""读 `{SCOPE}__HEDGE__AFTER_S`/`{SCOPE}__HEDGE__MAX_EXTRA`(issue #24 H4)。
|
||||
|
||||
两键均为 3 段键(`split("__")` 长度 3 ≠ 4),`_load_sources` 的段数判据天然
|
||||
跳过它们;`HEDGE` 已进 `_RESERVED_SEGMENTS`,4 段的 `{SCOPE}__HEDGE__{N}__*`
|
||||
也不会被当成 provider 段造出源。`AFTER_S` 未设 = 关闭(缺省逐字等于 1.3.6);
|
||||
`MAX_EXTRA` 未设 = 1。origin 传实际命中键名(同 `_load_call_deadline` 纪律);
|
||||
值域的交叉守卫(ttft/单源/max_extra 生效口径)归 `check_hedge_assembly` 一处。
|
||||
|
||||
Args:
|
||||
scope: 已大写的 scope 名。
|
||||
env: 已合并的环境映射。
|
||||
|
||||
Returns:
|
||||
`{"hedge_after_s": float | None, "hedge_max_extra": int}`,直传构造器。
|
||||
"""
|
||||
found_after = _first(env, f"{scope}__HEDGE__AFTER_S")
|
||||
after = (
|
||||
ensure_call_deadline(_cast(found_after[1], "float", found_after[0]), found_after[0])
|
||||
if found_after
|
||||
else None
|
||||
)
|
||||
found_extra = _first(env, f"{scope}__HEDGE__MAX_EXTRA")
|
||||
extra = int(_cast(found_extra[1], "int", found_extra[0])) if found_extra else 1
|
||||
return {"hedge_after_s": after, "hedge_max_extra": extra}
|
||||
|
||||
|
||||
def check_hedge_assembly(
|
||||
*,
|
||||
hedge_after_s: float | None,
|
||||
hedge_max_extra: int,
|
||||
sources: Sequence[SourceConfig],
|
||||
call_deadline_s: float | None,
|
||||
origin: str,
|
||||
) -> float | None:
|
||||
"""对冲装配守卫的唯一事实源(issue #24 设计 §5 全表,H4 批准)。
|
||||
|
||||
`GatewaySettings.__post_init__` 与 `GatewayClient.__init__` 调同一份,两条装配
|
||||
路的值域/交叉守卫不漂移。返回归一化后的 `hedge_after_s`(None 或有限正数,
|
||||
复用 `ensure_call_deadline` 的值域校验);`hedge_after_s is None`(未启用)时
|
||||
值域归一化后直接返回,交叉守卫不查——它们没有可校验的对象。
|
||||
|
||||
Args:
|
||||
hedge_after_s: 对冲触发阈值(秒);None = 关闭。
|
||||
hedge_max_extra: 每次逻辑调用最多并发对冲路数;v1 仅单路生效(H5)。
|
||||
sources: 本 scope 的源集合(交叉守卫要读 timeout_s/ttft_timeout_s)。
|
||||
call_deadline_s: 调用期限(秒);与对冲的组合守卫见设计 §6。
|
||||
origin: after_s 值域报错的定位串(env 键名 / `GatewaySettings.hedge_after_s` /
|
||||
`GatewayClient(hedge_after_s=...)`);跨字段守卫与 warning 沿用本模块先例,
|
||||
消息自带字段名与 env 键型,不挂 origin。
|
||||
|
||||
Raises:
|
||||
ValueError: max_extra 非 int/bool 或出 [1,3];阈值 ≥ 最小源 timeout_s(永不
|
||||
可能触发);阈值 ≥ call_deadline_s(期限先于对冲触发,对冲形同虚设)。
|
||||
"""
|
||||
if isinstance(hedge_max_extra, bool) or not isinstance(hedge_max_extra, int):
|
||||
raise ValueError(
|
||||
f"hedge_max_extra({{SCOPE}}__HEDGE__MAX_EXTRA)必须是 int: {hedge_max_extra!r}"
|
||||
)
|
||||
if not 1 <= hedge_max_extra <= 3:
|
||||
raise ValueError(
|
||||
f"hedge_max_extra({{SCOPE}}__HEDGE__MAX_EXTRA)须在 [1,3]: {hedge_max_extra};"
|
||||
"每次逻辑调用最多并发对冲路数,v1 仅单路生效"
|
||||
)
|
||||
after = ensure_call_deadline(hedge_after_s, origin)
|
||||
if after is None:
|
||||
return None
|
||||
min_timeout = min(s.timeout_s for s in sources)
|
||||
if after >= min_timeout:
|
||||
raise ValueError(
|
||||
f"hedge_after_s({{SCOPE}}__HEDGE__AFTER_S={after})须 < 最小源 timeout_s"
|
||||
f"({min_timeout});对冲永不可能触发,配置即错误"
|
||||
)
|
||||
if call_deadline_s is not None and after >= call_deadline_s:
|
||||
raise ValueError(
|
||||
f"hedge_after_s({after})须 < call_deadline_s({call_deadline_s});"
|
||||
"期限会先于对冲触发,对冲形同虚设(设计 §6)"
|
||||
)
|
||||
ttfts = [s.ttft_timeout_s for s in sources if s.ttft_timeout_s is not None]
|
||||
if ttfts and after >= min(ttfts):
|
||||
logger.warning(
|
||||
"hedge_after_s({}) ≥ 最小源 ttft_timeout_s({}): 流式挂起会被 TTFT 看门狗"
|
||||
"先行切断,对冲对流式形同虚设(非流式仍有效)",
|
||||
after,
|
||||
min(ttfts),
|
||||
)
|
||||
if len(sources) == 1:
|
||||
logger.warning(
|
||||
"单源 scope 配置了对冲阈值 hedge_after_s={};运行期拿不到异源候选,对冲自然静默",
|
||||
after,
|
||||
)
|
||||
if hedge_max_extra > 1:
|
||||
logger.warning(
|
||||
"hedge_max_extra={} 已接受,但 v1 仅单路对冲生效(梯次追加为 H5 预留)",
|
||||
hedge_max_extra,
|
||||
)
|
||||
return after
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EmbeddingSettings:
|
||||
"""Embedding scope 装配配置(M2 §7): 复用 GatewaySettings + embedding 专用键。
|
||||
|
||||
Reference in New Issue
Block a user