Files
PolyGateway/src/polygateway/errors.py
T
iomgaa 9474c76ab0 feat: add an optional per-call wall-clock deadline
Give one logical call an optional hard wall-clock boundary (issue #22).
Leaving it unset keeps 1.3.5 behaviour verbatim: the timeout context is
never entered when deadline_s is None.

- new deadline.py: ensure_call_deadline() range check (None or a finite
  positive number; bool/0/nan/inf and out-of-range ints are rejected as
  ValueError so OverflowError never leaks) plus with_call_deadline(),
  which distinguishes an expiry from a TimeoutError raised by the body
  or its cleanup via a local-variable identity comparison rather than
  cm.expired() alone
- new CallDeadlineExceeded: deliberately outside the four categories and
  not a GatewayUnavailableError, and carries no retry_after_s
- new {SCOPE}__CALL_DEADLINE_S key, guarded on the env, direct
  construction and dataclasses.replace paths
- three clients take a call_deadline_s constructor argument and a
  keyword-only per-call override on chat/embed/recognize_text/
  parse_layout; None inherits the assembled value
- validation runs before the awaitable is created, so an illegal value
  cannot strand an un-awaited coroutine
- one embed call shares a single deadline across all of its batches
- import-linter gains a polygateway.deadline layer

- cover where the deadline lands: backoff sleep, admission polling,
  the structured re-ask ladder and embedding's batch loop, plus the
  empty-texts early return that stays outside it
- cover what an expiry costs: exactly one terminal_failure row carrying
  error_type=CallDeadlineExceeded, a cancelled attempt row sharing its
  logical_call_id, cleanup that outlives the deadline (lower bound only)
  and an already-billed success being discarded
- pin the injected clock as orthogonal: a 10^6 second jump never expires
  a call, yet total_latency_ms still reads that clock
2026-09-10 04:14:44 -04:00

240 lines
10 KiB
Python

"""错误四分类与 scope 级不可用语义(M1 设计 §3;ARCH §6)。
分类决定治理行为(重试/换源/熔断计数),库内禁止绕过分类做 ad-hoc 判断。
构造形态承 CHS `app/domain/errors.py` 的 ProviderError 一族。
"""
SCOPE_REASONS = frozenset(
{
"circuit_open",
"retry_exhausted",
"stalled",
"quota_exhausted",
"no_sources",
"governance_backend_down", # issue #7: 限流/熔断后端故障(fail-closed → 整个 scope 发不出请求)
}
)
GOVERNANCE_BACKEND_RETRY_AFTER_S = 5.0
"""治理后端故障的建议重投间隔(秒)。
**不是环境配置项**——后端恢复时间物理上不可知(不同于熔断冷却有确定到期时刻),
故取一个保守固定值;下游有自己的退避策略时可忽略本字段。取 0 会让积压任务零延迟
同时冲击已挂掉的后端,把一次故障放大成一场风暴(issue #7 §3.2)。
"""
SOURCE_REASONS = frozenset(
{
"network_error",
"timeout",
"rate_limited",
"source_dead",
"circuit_open",
"cooldown",
"adaptive_paced", # M2.5 §3.35: AIMD 超限排队(quota-wait 通道)
}
)
class PolyGatewayError(Exception):
"""库内一切领域错误的基类,携带来源上下文便于遥测与日志定位。
`body_text` 是**非 2xx 响应体的摘要**——网关拒绝这次调用时说的话(issue #10)。
它与 `ResultInvalidError.raw_text` 是两回事,严禁混用:
============== ==================================================
``body_text`` **非 2xx** 的 HTTP 错误响应体: 对方**拒绝**的理由
``raw_text`` **2xx** 但内容不可解析时的模型输出原文
============== ==================================================
加在基类而非某个子类,是因为这些错误全部由同一个 HTTP 响应翻译而来——
"对方说了什么"与"它属于哪一类"正交。scope 级错误(`GatewayUnavailableError`
一族)继承到的恒空值不是噪音,而是"没有单一响应体可言"的如实表达。
**内容已由 transport 层截断**(`transports/_http_errors.summarize_body`),
且可能包含网关对请求的回显——库不做脱敏: 它不知道下游哪些字段敏感,
猜测式脱敏只会同时丢掉诊断价值与安全性。
本字段是**旁路数据**,不参与任何治理判定(重试/换源/熔断计数/限流结算)。
"""
def __init__(
self,
message: str,
*,
source_name: str | None = None,
status_code: int | None = None,
operation: str | None = None,
body_text: str = "",
) -> None:
super().__init__(message)
self.source_name = source_name
self.status_code = status_code
self.operation = operation
self.body_text = body_text
class TransientError(PolyGatewayError):
"""瞬时错误(超时/5xx/429/网络抖动/SSE 异常): 退避后可重试、可换源、计熔断。"""
def __init__(self, message: str, *, retry_after_s: float | None = None, **kwargs) -> None:
super().__init__(message, **kwargs)
self.retry_after_s = retry_after_s
class SourceDeadError(PolyGatewayError):
"""源死亡(401/403/欠费): 不重试,立即换源,该源 force_open。"""
class RequestRejectedError(PolyGatewayError):
"""请求被拒(400/坏输入): 不重试不换源,直接上抛。
**经中转部署时请注意**(issue #10 下游实测): 第三方 API 中转服务自身抖动
时也会回 400,从状态码上与供应商说"你的输入有问题"无法区分。下游曾观测到
同一份字节(sha256 一致)重发 15 次全部成功,且失败那次 `prompt_tokens=0`、
耗时远低于任何成功调用——请求在推理开始前就被挡了。本库仍按确定性失败处理
(对直连供应商而言重试只会白烧配额),批处理场景的下游宜自备兜底分类;
`body_text` 即为此提供判据: 中转抖动的响应体与供应商的 `invalid_request_error`
形态不同。
"""
class ResultInvalidError(PolyGatewayError):
"""坏结果 ≠ 坏服务: 调用成功但内容不可解析;熔断记成功,不入 transport 重试。"""
def __init__(
self,
message: str,
*,
raw_text: str = "",
repair_error: str | None = None,
validation_errors: tuple[str, ...] = (),
**kwargs,
) -> None:
super().__init__(message, **kwargs)
self.raw_text = raw_text
self.repair_error = repair_error
self.validation_errors = tuple(validation_errors)
class GatewayUnavailableError(PolyGatewayError):
"""scope 级暂时不可用: 库的**调用级**预算已经耗尽。
**职责边界(issue #14)**: 调用级的重试、退避、换源、等待冷却全部在库内,
不需要下游再写一层——两边各写一份必然漂移(库调了退避曲线而下游不知道,
下游改了等待上限而库的遥测算不进去),漂移之后"这次调用到底等了多久、
试了几次"就没有单一事实源答得出来。本异常表示那份预算(重试预算或 stall
预算)已经用完。下游据此再投是**任务级重试**,与调用级重试语义不同,由
业务自行在库外包(ARCH §7.2 单层重试原则)。
熔断开路时是当场抛本类还是先等冷却过去,由 `{SCOPE}__CIRCUIT_OPEN`
决定(缺省 fail_fast;单源 scope 建议配 wait)。
`retry_after_s` 非可选,语义是"距离**确定**可再试的时刻还有多久";
`0` 表示不存在确定的等待时刻(可立即重试),承 CHS ProviderUnavailableError。
"""
def __init__(
self,
*,
scope: str,
reason: str,
retry_after_s: float,
per_source_reasons: dict[str, str] | None = None,
source_name: str | None = None,
) -> None:
if not scope.strip():
raise ValueError("scope 不能为空")
if reason not in SCOPE_REASONS:
raise ValueError(f"未知 scope 级 reason: {reason!r}(允许: {sorted(SCOPE_REASONS)})")
if retry_after_s < 0:
raise ValueError("retry_after_s 不能为负")
reasons = dict(per_source_reasons or {})
for src, src_reason in reasons.items():
if src_reason not in SOURCE_REASONS:
raise ValueError(
f"源 {src!r} 的 reason 非法: {src_reason!r}(允许: {sorted(SOURCE_REASONS)})"
)
super().__init__(f"{scope.lower()} 网关暂时不可用: {reason}", source_name=source_name)
self.scope = scope.lower()
self.reason = reason
self.retry_after_s = retry_after_s
self.per_source_reasons = reasons
class CircuitOpenError(GatewayUnavailableError):
"""全部候选源被熔断门拒绝;reason 恒为 circuit_open。"""
def __init__(
self,
*,
scope: str,
retry_after_s: float,
per_source_reasons: dict[str, str] | None = None,
) -> None:
super().__init__(
scope=scope,
reason="circuit_open",
retry_after_s=retry_after_s,
per_source_reasons=per_source_reasons,
)
class AllSourcesExhausted(GatewayUnavailableError): # noqa: N818 — ARCH §6.1 冻结的公共名
"""重试预算耗尽 / 无可用源 / 配额 fail-fast 等 scope 级失败。"""
class SourceNotConfiguredError(PolyGatewayError):
"""源名不在限流后端的配置字典中: 装配缺陷,正常不可达。
**有意不在** `GatewayUnavailableError` 之下: 它不是"暂时不可用"而是"配置写
错了",必须消耗失败预算进死信让人看见;归入可重投家族会让配置错误的任务永远
重投、永不告警——正是 issue #7 要修的那个 bug 的镜像(§3.4)。
"""
class GovernanceBackendError(GatewayUnavailableError):
"""限流/熔断状态后端自身故障: 必须报错而非放行(防击穿网关,降级方向铁律)。
继承 `GatewayUnavailableError`(issue #7): fail-closed 意味着整个 scope 一个
请求都发不出去,语义上即 scope 级不可用。此前它是 `PolyGatewayError` 的直接
子类,只写 `except GatewayUnavailableError` 的调用方接不住,后果是"Redis 抖
一下 → 积压任务消耗业务失败预算 → 进死信",而那是运维重启即可恢复的故障。
"""
def __init__(
self,
message: str,
*,
scope: str,
retry_after_s: float = GOVERNANCE_BACKEND_RETRY_AFTER_S,
source_name: str | None = None,
) -> None:
super().__init__(
scope=scope,
reason="governance_backend_down",
retry_after_s=retry_after_s,
source_name=source_name,
)
# 父类会把 message 覆写为 "{scope} 网关暂时不可用: {reason}",而各构造点
# 携带的诊断串(如"限流后端 try_acquire 失败: ...")是排障主线索,必须保住
self.args = (message,)
class CallDeadlineExceeded(PolyGatewayError): # noqa: N818 — 设计 §9 人类批准的公共名
"""调用方设定的整体调用期限到期; 不是网关不可用、也不是源故障。
刻意**不属**四分类、**不进** `SCOPE_REASONS`、**不继承** `GatewayUnavailableError`:
它描述的是调用方自己的耐心边界, 与"对方怎么了"正交——按四分类之一上报会让
下游的重试/换源/熔断逻辑对着一次本地超时做治理决策(库铁律「错误分类驱动」)。
也刻意**没有** `retry_after_s`: 期限到期不含"何时可再试"的信息, 给 `0.0`
会按既定语义指示下游立刻重打一条可能已经饱和的通道。
**到期不等于未产出、未计费**: 期限治理的是等待, 在途请求可能已经发出、
已被上游计费, 清理仍在 `finally` 里完成, 故返回时刻 = 期限 + 清理耗时。
"""
def __init__(self, *, scope: str, deadline_s: float) -> None:
super().__init__(f"{scope} 调用期限 {deadline_s}s 到期")
self.scope = scope
self.deadline_s = deadline_s