feat: add gateway client with env-driven assembly

Includes config aggregation for multi-source env keys, from_env and
from_settings factories with explicit shared-backend injection,
gather_bounded, top-level exports, tightened import-linter layers with
the gate removed from the Makefile, and the finalized .env.example.
This commit is contained in:
2026-07-20 07:47:05 -04:00
parent 936895919c
commit 7b9815f4bc
30 changed files with 1701 additions and 253 deletions
+41 -3
View File
@@ -1,7 +1,45 @@
"""PolyGateway实验室统一的大语言模型LLM/VLM/OCR调度与中转库。
"""PolyGateway: 实验室统一的大语言模型(LLM/VLM/OCR)调度与中转库。
治理单位是一次模型调用多源选择、限流、错误分类与重试、熔断、响应缓存、
流式活性看门狗、遥测(含成本)。架构单一事实源见 research-wiki/ARCHITECTURE.md。
治理单位是一次模型调用: 多源选择、限流、错误分类与重试、熔断、响应缓存、
流式活性看门狗、遥测。架构单一事实源见 research-wiki/ARCHITECTURE.md。
顶层导出即公共 API 面: 错误四分类必须从这里引用(ARCH §5.1 约定②)。
"""
from polygateway.client import GatewayClient, gather_bounded
from polygateway.config import GatewaySettings
from polygateway.errors import (
AllSourcesExhausted,
CircuitOpenError,
GatewayUnavailableError,
GovernanceBackendError,
PolyGatewayError,
RequestRejectedError,
ResultInvalidError,
SourceDeadError,
TransientError,
)
from polygateway.providers import DEFAULT_PROFILES, ProviderProfile, register_provider
from polygateway.types import LLMResponse, SourceConfig
__version__ = "0.1.0"
__all__ = [
"DEFAULT_PROFILES",
"AllSourcesExhausted",
"CircuitOpenError",
"GatewayClient",
"GatewaySettings",
"GatewayUnavailableError",
"GovernanceBackendError",
"LLMResponse",
"PolyGatewayError",
"ProviderProfile",
"RequestRejectedError",
"ResultInvalidError",
"SourceConfig",
"SourceDeadError",
"TransientError",
"__version__",
"gather_bounded",
"register_provider",
]
+36 -11
View File
@@ -51,8 +51,13 @@ class InMemoryGate:
g.probe_owner = owner
g.probe_expires = self._now() + self._cfg.probe_ttl_s
return GateDecision(
source_name=source_name, allowed=True, state=GateState.HALF_OPEN,
epoch=g.epoch, is_probe=True, probe_owner=owner, retry_after_s=0.0,
source_name=source_name,
allowed=True,
state=GateState.HALF_OPEN,
epoch=g.epoch,
is_probe=True,
probe_owner=owner,
retry_after_s=0.0,
)
async def try_enter(self, source_name: str, owner: str) -> GateDecision:
@@ -63,23 +68,36 @@ class InMemoryGate:
now = self._now()
if g.state is GateState.CLOSED:
return GateDecision(
source_name=source_name, allowed=True, state=GateState.CLOSED,
epoch=g.epoch, is_probe=False, probe_owner=None, retry_after_s=0.0,
source_name=source_name,
allowed=True,
state=GateState.CLOSED,
epoch=g.epoch,
is_probe=False,
probe_owner=None,
retry_after_s=0.0,
)
if g.state is GateState.OPEN:
if now >= g.open_until:
return self._grant_probe(g, source_name, owner)
return GateDecision(
source_name=source_name, allowed=False, state=GateState.OPEN,
epoch=g.epoch, is_probe=False, probe_owner=None,
source_name=source_name,
allowed=False,
state=GateState.OPEN,
epoch=g.epoch,
is_probe=False,
probe_owner=None,
retry_after_s=g.open_until - now,
)
# HALF_OPEN: 探针在途;租约过期则接管,否则拒绝(防惊群)
if now >= g.probe_expires:
return self._grant_probe(g, source_name, owner)
return GateDecision(
source_name=source_name, allowed=False, state=GateState.HALF_OPEN,
epoch=g.epoch, is_probe=False, probe_owner=None,
source_name=source_name,
allowed=False,
state=GateState.HALF_OPEN,
epoch=g.epoch,
is_probe=False,
probe_owner=None,
retry_after_s=g.probe_expires - now,
)
@@ -96,8 +114,13 @@ class InMemoryGate:
def _snapshot(self, g: _SourceGate, applied: bool) -> GateUpdate:
return GateUpdate(
applied=applied, state=g.state, epoch=g.epoch, failure_count=g.fails,
retry_after_s=max(0.0, g.open_until - self._now()) if g.state is GateState.OPEN else 0.0,
applied=applied,
state=g.state,
epoch=g.epoch,
failure_count=g.fails,
retry_after_s=max(0.0, g.open_until - self._now())
if g.state is GateState.OPEN
else 0.0,
)
def _open(self, g: _SourceGate, reason: str) -> None:
@@ -118,7 +141,9 @@ class InMemoryGate:
g.probe_expires = 0.0
return self._snapshot(g, applied=True)
async def record_failure(self, entry: GateDecision, reason: str, force_open: bool) -> GateUpdate:
async def record_failure(
self, entry: GateDecision, reason: str, force_open: bool
) -> GateUpdate:
g = self._gate(entry.source_name)
if not self._fenced(g, entry):
return self._snapshot(g, applied=False)
+3 -1
View File
@@ -27,7 +27,9 @@ _WINDOW_S = 60.0
class _MemoryPermit:
"""入场许可;release/settle 幂等(CHS _RedisPermit 同款 flag 语义)。"""
def __init__(self, limiter: InMemoryLimiter, source: str, lease_id: str, est: int, window: int) -> None:
def __init__(
self, limiter: InMemoryLimiter, source: str, lease_id: str, est: int, window: int
) -> None:
self._limiter = limiter
self._source = source
self._lease_id = lease_id
+302
View File
@@ -0,0 +1,302 @@
"""GatewayClient: 唯一组装层(D1 组装点)+ from_env/from_settings 工厂 + gather_bounded。
90% 用户三行起步: `client = GatewayClient.from_env(); resp = await client.chat(messages)`。
多逻辑角色 = 每角色调一次 `from_env(scope=...)`;共享限流/熔断状态 = 自建
后端实例(源表取并集)显式传入多次调用——共享必须显式,禁止隐式全局
(VT `evolve_llm = llm` 教训;ARCH §7.7 R5)。
"""
from __future__ import annotations
import asyncio
import random
import time
from typing import TYPE_CHECKING, Any, Literal, TypeVar
from polygateway.backends.memory.breaker import InMemoryGate
from polygateway.backends.memory.cache import InMemoryCache
from polygateway.backends.memory.limiter import InMemoryLimiter
from polygateway.config import GatewaySettings
from polygateway.middleware.base import compose
from polygateway.middleware.cache import CacheMW
from polygateway.middleware.retry import RetryMW
from polygateway.middleware.structured import StructuredMW
from polygateway.middleware.telemetry import TelemetryEmitter, TelemetryMW
from polygateway.providers import get_provider
from polygateway.sources import LeastInflightSelector, RoundRobinSelector, SourceCooldownMemo
from polygateway.transports.openai_compat import OpenAICompatTransport
from polygateway.types import ChatRequest, LLMResponse
if TYPE_CHECKING:
from collections.abc import Awaitable, Iterable, Mapping
from pydantic import BaseModel
from polygateway.ports import (
CacheBackend,
Middleware,
ProviderGate,
RateLimiter,
SourceSelector,
StructuredOutputStrategy,
TelemetryRecorder,
Transport,
)
from polygateway.providers import ProviderProfile
from polygateway.types import (
BackpressurePolicy,
RetryPolicy,
SourceConfig,
)
_T = TypeVar("_T")
class GatewayClient:
"""统一治理入口;构造函数全量注入(测试/高级),工厂覆盖 90% 场景。"""
def __init__(
self,
*,
scope: str,
sources: list[SourceConfig],
selector: SourceSelector,
limiter: RateLimiter,
gate: ProviderGate,
transport: Transport,
retry: RetryPolicy,
backpressure: BackpressurePolicy,
quota_full: str = "wait",
telemetry: TelemetryRecorder | None = None,
cache: CacheBackend | None = None,
cache_namespace: str | None = None,
cache_ttl_s: int | None = None,
structured_strategy: StructuredOutputStrategy | None = None,
structured_escalation: StructuredOutputStrategy | None = None,
structured_max_retries: int = 1,
now: Any = time.monotonic,
sleep: Any = asyncio.sleep,
rng: Any = random.random,
) -> None:
emitter = TelemetryEmitter(telemetry) if telemetry is not None else None
terminal = RetryMW(
scope=scope,
sources=sources,
selector=selector,
limiter=limiter,
gate=gate,
transport=transport,
retry=retry,
backpressure=backpressure,
quota_full=quota_full,
cooldown_memo=SourceCooldownMemo(now=now),
emitter=emitter,
now=now,
sleep=sleep,
rng=rng,
)
middlewares: list[Middleware] = []
if emitter is not None:
middlewares.append(TelemetryMW(emitter, now=now))
if cache is not None:
if cache_namespace is None or cache_ttl_s is None:
raise ValueError("启用缓存必须提供 cache_namespace 与 cache_ttl_s")
# 多源 scope 的 key 身份 = 排序去重的 model 合集;源集合变化 → 一次性冷启动
fingerprint = ",".join(sorted({s.model for s in sources}))
middlewares.append(
CacheMW(
backend=cache,
model_fingerprint=fingerprint,
default_namespace=cache_namespace,
ttl_s=cache_ttl_s,
strategy=structured_strategy,
)
)
if structured_strategy is not None:
middlewares.append(
StructuredMW(
strategy=structured_strategy,
max_retries=structured_max_retries,
escalation=structured_escalation,
)
)
self._structured_available = structured_strategy is not None
self._handler = compose(middlewares, terminal)
self._transport = transport
self._telemetry = telemetry
self._cache = cache
self._closed = False
async def chat(
self,
messages: list[dict[str, Any]],
*,
session_id: str | None = None,
parent_call_id: str | None = None,
cache_salt: str | None = None,
cache_namespace: str | None = None,
structured: type[BaseModel] | Literal["json"] | None = None,
stream: bool = True,
) -> LLMResponse:
"""一次治理调用(签名冻结,ARCH §5.2;与三项目 LLMProvider 协议兼容)。"""
if structured is not None and not self._structured_available:
raise ImportError(
"结构化输出未启用: 安装 pip install 'polygateway[structured]' 后重新装配"
)
request = ChatRequest(
messages=messages,
session_id=session_id,
parent_call_id=parent_call_id,
cache_salt=cache_salt,
cache_namespace=cache_namespace,
structured=structured,
stream=stream,
)
return await self._handler(request)
async def aclose(self) -> None:
"""幂等释放: transport 连接池、遥测连接、缓存客户端。"""
if self._closed:
return
self._closed = True
transport_aclose = getattr(self._transport, "aclose", None)
if transport_aclose is not None:
await transport_aclose()
telemetry_close = getattr(self._telemetry, "close", None)
if telemetry_close is not None:
telemetry_close()
cache_aclose = getattr(self._cache, "aclose", None)
if cache_aclose is not None:
await cache_aclose()
async def __aenter__(self) -> GatewayClient:
return self
async def __aexit__(self, *exc_info: object) -> None:
await self.aclose()
# —— 工厂 ——
@classmethod
def from_settings(
cls,
settings: GatewaySettings,
*,
limiter: RateLimiter | None = None,
gate: ProviderGate | None = None,
cache: CacheBackend | None = None,
telemetry: TelemetryRecorder | None = None,
registry: Mapping[str, ProviderProfile] | None = None,
) -> GatewayClient:
"""按配置装配;显式传入的后端实例即共享(None 项按配置自建私有实例)。"""
sources = list(settings.sources)
profiles = [get_provider(s.provider, registry=registry) for s in sources]
strategy, escalation = _build_structured(profiles)
return cls(
scope=settings.scope,
sources=sources,
selector=_build_selector(settings.selector),
limiter=limiter
or InMemoryLimiter(
scope=settings.scope,
sources={s.name: s for s in sources},
global_limits=settings.global_limits,
lease_ttl_s=settings.lease_ttl_s,
),
gate=gate or InMemoryGate(config=settings.breaker),
transport=OpenAICompatTransport(registry=registry),
retry=settings.retry,
backpressure=settings.backpressure,
quota_full=settings.quota_full,
telemetry=telemetry if telemetry is not None else _build_telemetry(settings),
cache=cache if cache is not None else _build_cache(settings),
cache_namespace=settings.cache_namespace,
cache_ttl_s=settings.cache_ttl_s,
structured_strategy=strategy,
structured_escalation=escalation,
structured_max_retries=settings.structured_max_retries,
)
@classmethod
def from_env(
cls,
scope: str = "LLM",
*,
limiter: RateLimiter | None = None,
gate: ProviderGate | None = None,
cache: CacheBackend | None = None,
telemetry: TelemetryRecorder | None = None,
registry: Mapping[str, ProviderProfile] | None = None,
env: Mapping[str, str] | None = None,
) -> GatewayClient:
"""从 .env/环境变量装配一个 scope 的 client(键名清单见 .env.example)。"""
return cls.from_settings(
GatewaySettings.from_env(scope, env=env),
limiter=limiter,
gate=gate,
cache=cache,
telemetry=telemetry,
registry=registry,
)
def _build_selector(name: str) -> SourceSelector:
return RoundRobinSelector() if name == "round_robin" else LeastInflightSelector()
def _build_cache(settings: GatewaySettings) -> CacheBackend | None:
if settings.cache_backend == "none":
return None
if settings.cache_backend == "memory":
return InMemoryCache()
from polygateway.backends.redis_cache import RedisCache
assert settings.redis_url is not None # 内部不变量: config 已校验
return RedisCache.from_url(settings.redis_url)
def _build_telemetry(settings: GatewaySettings) -> TelemetryRecorder | None:
if settings.telemetry_backend == "none":
return None
from polygateway.telemetry.sqlite import SQLiteRecorder
assert settings.telemetry_sqlite_path is not None # 内部不变量: config 已校验
return SQLiteRecorder(settings.telemetry_sqlite_path)
def _build_structured(
profiles: list[ProviderProfile],
) -> tuple[StructuredOutputStrategy | None, StructuredOutputStrategy | None]:
"""按注册表能力选策略(设计 §5): 全员支持原生 schema 才用 NativeSchema。
json_repair 缺失(未装 structured extra)→ 返回 (None, None),
chat(structured=...) 时显式报缺 extra,绝不静默跳过校验。
"""
try:
from polygateway.structured.json_repair import JsonRepairStrategy
from polygateway.structured.native_schema import NativeSchemaStrategy
except ImportError:
return None, None
try:
native_all = all(p.supports_native_schema for p in profiles)
if native_all:
return NativeSchemaStrategy(), NativeSchemaStrategy()
return JsonRepairStrategy(), None
except ImportError:
return None, None
async def gather_bounded(aws: Iterable[Awaitable[_T]], *, concurrency: int) -> list[_T]:
"""有界并发 gather(D5 便利函数,替代 VT 手搓 semaphore+gather 样板)。
语义与 `asyncio.gather` 默认一致: 结果保序、首个异常上抛;仅增加并发上限。
"""
if concurrency < 1:
raise ValueError("concurrency 必须 ≥ 1")
sem = asyncio.Semaphore(concurrency)
async def _run(aw: Awaitable[_T]) -> _T:
async with sem:
return await aw
return await asyncio.gather(*(_run(aw) for aw in aws))
+313
View File
@@ -0,0 +1,313 @@
"""配置聚合与装配期校验(M1 设计 §8;ARCH §9)。
键族: 多源 `{SCOPE}__{PROVIDER}__{N}__{FIELD}`、scope 韧性键、平铺 LLM_*
简写(单 scope 迁移零改名)、PGW_* 装配键。缺关键配置直接报错,严禁默认
值兜底;scope 键与平铺键并存时 scope 键优先。
实现说明: 多源键族是动态键名,pydantic-settings 的静态字段模型无法表达,
故用其底层 python-dotenv 读 `.env` 并与 os.environ 合并(环境变量优先),
fail-loud 校验语义与 pydantic-settings 一致。
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import TYPE_CHECKING
from dotenv import dotenv_values
from polygateway.types import (
BackpressurePolicy,
BreakerConfig,
GlobalLimits,
RetryPolicy,
SourceConfig,
)
if TYPE_CHECKING:
from collections.abc import Mapping
# FIELD → (SourceConfig 属性, 类型);CHS config.py:95-104 全集 + M1 新增
_SOURCE_FIELDS: dict[str, tuple[str, str]] = {
"BASE_URL": ("base_url", "str"),
"API_KEY": ("api_key", "str"),
"MODEL": ("model", "str"),
"MAX_CONCURRENCY": ("max_concurrency", "int"),
"RPM": ("rpm", "int"),
"TPM": ("tpm", "int"),
"EST_TOKENS": ("est_tokens", "int"),
"TIMEOUT_S": ("timeout_s", "float"),
"TTFT_TIMEOUT_S": ("ttft_timeout_s", "float"),
"INTER_TOKEN_TIMEOUT_S": ("inter_token_timeout_s", "float"),
"ENABLE_THINKING": ("enable_thinking", "bool"),
"MISSING_DONE": ("missing_done", "str"),
"TRUST_ENV": ("trust_env", "bool"),
}
_RESERVED_SEGMENTS = frozenset({"GLOBAL", "RETRY", "BREAKER", "BACKPRESSURE"})
_SELECTORS = frozenset({"round_robin", "least_inflight"})
_QUOTA_FULL = frozenset({"wait", "fail_fast"})
# 背压默认(M1 仅 poll 生效;CHS _BACKOFF_S=0.05 同源)
_DEFAULT_STALL_WINDOW_S = 300.0
_DEFAULT_POLL_INTERVAL_S = 0.05
_DEFAULT_LEASE_TTL_S = 1500.0 # CHS _DEFAULT_LEASE_TTL_MS 同源
def _cast(raw: str, kind: str, key: str) -> object:
try:
if kind == "int":
return int(raw)
if kind == "float":
return float(raw)
if kind == "bool":
lowered = raw.strip().lower()
if lowered in ("1", "true", "yes", "on"):
return True
if lowered in ("0", "false", "no", "off"):
return False
raise ValueError(f"非法布尔值: {raw!r}")
return raw
except ValueError as exc:
raise ValueError(f"配置 {key} 解析失败: {exc}") from exc
def _first(env: Mapping[str, str], *keys: str) -> tuple[str, str] | None:
for key in keys:
raw = env.get(key)
if raw is not None and raw != "":
return key, raw
return None
def _require(env: Mapping[str, str], *keys: str) -> tuple[str, str]:
found = _first(env, *keys)
if found is None:
raise ValueError(f"缺关键配置: 需设置 {''.join(keys)}")
return found
@dataclass(frozen=True)
class GatewaySettings:
"""一个 scope 的完整装配配置;构造经 from_env 聚合并通过全部守卫。"""
scope: str
sources: tuple[SourceConfig, ...]
global_limits: GlobalLimits
retry: RetryPolicy
breaker: BreakerConfig
backpressure: BackpressurePolicy
selector: str
quota_full: str
limiter_backend: str
breaker_backend: str
cache_backend: str
cache_namespace: str | None
cache_ttl_s: int | None
telemetry_backend: str
telemetry_sqlite_path: str | None
redis_url: str | None
structured_max_retries: int
lease_ttl_s: float
@classmethod
def from_env(
cls,
scope: str = "LLM",
env: Mapping[str, str] | None = None,
*,
env_file: str = ".env",
) -> GatewaySettings:
"""聚合 env(缺省 .env + os.environ,后者优先)并执行装配守卫。"""
if env is None:
env = {
k: v for k, v in {**dotenv_values(env_file), **os.environ}.items() if v is not None
}
scope_u = scope.upper()
sources = _load_sources(scope_u, env)
global_limits = _load_global_limits(scope_u, env)
retry = _load_retry(scope_u, env)
breaker = _load_breaker(scope_u, env, sources, global_limits)
settings = cls(
scope=scope_u.lower(),
sources=tuple(sources),
global_limits=global_limits,
retry=retry,
breaker=breaker,
backpressure=_load_backpressure(scope_u, env),
selector=_load_choice(env, f"{scope_u}__SELECTOR", _SELECTORS, "round_robin"),
quota_full=_load_choice(env, f"{scope_u}__QUOTA_FULL", _QUOTA_FULL, "wait"),
**_load_pgw(env),
)
_guard_lease(settings)
return settings
def _load_sources(scope: str, env: Mapping[str, str]) -> list[SourceConfig]:
specs: dict[tuple[str, str], dict[str, object]] = {}
for key, raw in env.items():
parts = key.split("__")
if len(parts) != 4 or parts[0] != scope or parts[1] in _RESERVED_SEGMENTS:
continue
_, provider, index, field = parts
if field not in _SOURCE_FIELDS:
raise ValueError(f"未知源配置字段 {field}(键 {key});允许: {sorted(_SOURCE_FIELDS)}")
attr, kind = _SOURCE_FIELDS[field]
specs.setdefault((provider.lower(), index), {})[attr] = _cast(raw, kind, key)
if not specs:
raise ValueError(f"scope {scope} 未配置任何源({scope}__{{PROVIDER}}__{{N}}__{{FIELD}})")
return [
_build_source(scope, provider, index, spec, env)
for (provider, index), spec in sorted(specs.items())
]
def _build_source(
scope: str, provider: str, index: str, spec: dict[str, object], env: Mapping[str, str]
) -> SourceConfig:
prefix = f"{scope}__{provider.upper()}__{index}"
for field in ("BASE_URL", "API_KEY", "MODEL"):
attr, _ = _SOURCE_FIELDS[field]
if attr not in spec:
raise ValueError(f"缺关键配置: {prefix}__{field}")
if "timeout_s" not in spec:
key, raw = _require(env, f"{prefix}__TIMEOUT_S", "LLM_TIMEOUT")
spec["timeout_s"] = _cast(raw, "float", key)
# 平铺看门狗键作为缺省(成对才生效,遵守 SourceConfig 不变式)
if "ttft_timeout_s" not in spec and "inter_token_timeout_s" not in spec:
flat = _first(env, "LLM_TTFT_TIMEOUT")
flat_inter = _first(env, "LLM_INTER_TOKEN_TIMEOUT")
if flat and flat_inter:
spec["ttft_timeout_s"] = _cast(flat[1], "float", flat[0])
spec["inter_token_timeout_s"] = _cast(flat_inter[1], "float", flat_inter[0])
return SourceConfig(name=f"{provider}_{index}", provider=provider, **spec)
def _load_global_limits(scope: str, env: Mapping[str, str]) -> GlobalLimits:
def read(field: str) -> int:
found = _first(env, f"{scope}__GLOBAL__{field}")
return int(_cast(found[1], "int", found[0])) if found else 0
return GlobalLimits(max_concurrency=read("MAX_CONCURRENCY"), rpm=read("RPM"), tpm=read("TPM"))
def _load_retry(scope: str, env: Mapping[str, str]) -> RetryPolicy:
key_a, attempts = _require(env, f"{scope}__RETRY__MAX_ATTEMPTS", "LLM_MAX_RETRIES")
key_b, base = _require(env, f"{scope}__RETRY__BACKOFF_BASE_S", "LLM_RETRY_BASE_DELAY")
key_m, max_d = _require(env, f"{scope}__RETRY__BACKOFF_MAX_S", "LLM_RETRY_MAX_DELAY")
return RetryPolicy(
max_attempts=int(_cast(attempts, "int", key_a)),
backoff_base_s=float(_cast(base, "float", key_b)),
backoff_max_s=float(_cast(max_d, "float", key_m)),
)
def _load_breaker(
scope: str, env: Mapping[str, str], sources: list[SourceConfig], global_limits: GlobalLimits
) -> BreakerConfig:
key_t, thr = _require(env, f"{scope}__BREAKER__FAIL_THRESHOLD", "LLM_CIRCUIT_BREAKER_THRESHOLD")
key_c, cool = _require(env, f"{scope}__BREAKER__COOLDOWN_S", "LLM_CIRCUIT_BREAKER_COOLDOWN")
threshold = int(_cast(thr, "int", key_t))
cooldown_s = float(_cast(cool, "float", key_c))
# 有效阈值 = max(配置值, 并发×2)——三项目 .env 注释的手动约定入库(设计 §9 行 4)
concurrency = global_limits.max_concurrency or max(
(s.max_concurrency for s in sources), default=0
)
if concurrency > 0:
threshold = max(threshold, concurrency * 2)
probe = _first(env, f"{scope}__BREAKER__PROBE_TTL_S")
if probe is not None:
probe_ttl_s = float(_cast(probe[1], "float", probe[0]))
else:
# 派生规则: 探针租约须撑过一次最慢调用,且不短于冷却期
probe_ttl_s = max(2 * max(s.timeout_s for s in sources), cooldown_s)
return BreakerConfig(fail_threshold=threshold, cooldown_s=cooldown_s, probe_ttl_s=probe_ttl_s)
def _load_backpressure(scope: str, env: Mapping[str, str]) -> BackpressurePolicy:
stall = _first(env, f"{scope}__BACKPRESSURE__STALL_WINDOW_S")
poll = _first(env, f"{scope}__BACKPRESSURE__POLL_INTERVAL_S")
return BackpressurePolicy(
stall_window_s=float(_cast(stall[1], "float", stall[0]))
if stall
else _DEFAULT_STALL_WINDOW_S,
poll_interval_s=float(_cast(poll[1], "float", poll[0]))
if poll
else _DEFAULT_POLL_INTERVAL_S,
)
def _load_choice(env: Mapping[str, str], key: str, allowed: frozenset[str], default: str) -> str:
found = _first(env, key)
value = found[1] if found else default
if value not in allowed:
raise ValueError(f"配置 {key} 非法值 {value!r};允许: {sorted(allowed)}")
return value
def _load_pgw(env: Mapping[str, str]) -> dict[str, object]:
limiter_backend = _load_choice(
env, "PGW_LIMITER_BACKEND", frozenset({"memory", "redis"}), "memory"
)
breaker_backend = _load_choice(
env, "PGW_BREAKER_BACKEND", frozenset({"memory", "redis"}), "memory"
)
if "redis" in (limiter_backend, breaker_backend):
raise ValueError("限流/熔断 Redis 后端在 M2 交付;M1 仅支持 memory")
_, cache_backend = _require(env, "PGW_CACHE_BACKEND")
_, telemetry_backend = _require(env, "PGW_TELEMETRY_BACKEND")
if cache_backend not in ("redis", "memory", "none"):
raise ValueError(f"PGW_CACHE_BACKEND 非法值 {cache_backend!r}")
if telemetry_backend not in ("sqlite", "none"):
raise ValueError(f"PGW_TELEMETRY_BACKEND 非法值 {telemetry_backend!r}(postgres 在 M2)")
redis_url = env.get("REDIS_URL") or None
return {
"limiter_backend": limiter_backend,
"breaker_backend": breaker_backend,
"cache_backend": cache_backend,
**_load_cache_keys(env, cache_backend, redis_url),
"telemetry_backend": telemetry_backend,
"telemetry_sqlite_path": _require(env, "PGW_TELEMETRY_SQLITE_PATH")[1]
if telemetry_backend == "sqlite"
else None,
"redis_url": redis_url,
"structured_max_retries": _load_structured_retries(env),
"lease_ttl_s": _load_lease_ttl(env),
}
def _load_cache_keys(
env: Mapping[str, str], cache_backend: str, redis_url: str | None
) -> dict[str, object]:
if cache_backend == "none":
return {"cache_namespace": None, "cache_ttl_s": None}
_, namespace = _require(env, "PGW_CACHE_NAMESPACE")
key, ttl_raw = _require(env, "PGW_CACHE_TTL_S")
ttl = int(_cast(ttl_raw, "int", key))
if ttl <= 0:
raise ValueError("PGW_CACHE_TTL_S(TTL)必须 > 0,禁止永不过期")
if cache_backend == "redis" and redis_url is None:
raise ValueError("缺关键配置: PGW_CACHE_BACKEND=redis 需设置 REDIS_URL")
return {"cache_namespace": namespace, "cache_ttl_s": ttl}
def _load_structured_retries(env: Mapping[str, str]) -> int:
found = _first(env, "PGW_STRUCTURED_MAX_RETRIES")
value = int(_cast(found[1], "int", found[0])) if found else 1
if value < 0:
raise ValueError("PGW_STRUCTURED_MAX_RETRIES 不能为负")
return value
def _guard_lease(settings: GatewaySettings) -> None:
"""装配守卫: 调用超时须 ≤ permit 租约 TTL,防租约先于请求过期(ARCH §7.3)。"""
slowest = max(s.timeout_s for s in settings.sources)
if slowest > settings.lease_ttl_s:
raise ValueError(
f"源最大 timeout_s({slowest})超过 permit 租约 TTL({settings.lease_ttl_s});"
f"调大 PGW_LEASE_TTL_S 或调小超时"
)
def _load_lease_ttl(env: Mapping[str, str]) -> float:
found = _first(env, "PGW_LEASE_TTL_S")
return float(_cast(found[1], "float", found[0])) if found else _DEFAULT_LEASE_TTL_S
+3 -1
View File
@@ -87,7 +87,9 @@ class GatewayUnavailableError(PolyGatewayError):
reasons = dict(per_source_reasons or {})
for src, src_reason in reasons.items():
if src_reason not in SOURCE_REASONS:
raise ValueError(f"{src!r} 的 reason 非法: {src_reason!r}(允许: {sorted(SOURCE_REASONS)})")
raise ValueError(
f"{src!r} 的 reason 非法: {src_reason!r}(允许: {sorted(SOURCE_REASONS)})"
)
super().__init__(f"{scope.lower()} 网关暂时不可用: {reason}", source_name=source_name)
self.scope = scope.lower()
self.reason = reason
+3 -1
View File
@@ -33,7 +33,9 @@ class BreakerGate:
except Exception as exc:
raise GovernanceBackendError(f"熔断后端故障(record_success): {exc}") from exc
async def record_failure(self, entry: GateDecision, reason: str, force_open: bool) -> GateUpdate:
async def record_failure(
self, entry: GateDecision, reason: str, force_open: bool
) -> GateUpdate:
try:
return await self._gate.record_failure(entry, reason, force_open)
except GovernanceBackendError:
+6 -2
View File
@@ -111,8 +111,12 @@ class CacheMW:
fields = {k: v for k, v in data.items() if k in _RESPONSE_FIELDS}
structured_data = self._rebuild_structured(fields.get("content", ""), request)
fields.update(
cache_hit=True, latency_ms=0, ttft_ms=None, max_inter_token_ms=None,
call_id=str(uuid.uuid4()), structured_data=structured_data,
cache_hit=True,
latency_ms=0,
ttft_ms=None,
max_inter_token_ms=None,
call_id=str(uuid.uuid4()),
structured_data=structured_data,
)
return LLMResponse(**fields)
except Exception as exc:
+35 -14
View File
@@ -129,7 +129,8 @@ class RetryMW:
fails += 1
if fails >= self._retry.max_attempts:
raise AllSourcesExhausted(
scope=self._scope, reason="retry_exhausted",
scope=self._scope,
reason="retry_exhausted",
retry_after_s=self._retry.backoff_base_s,
per_source_reasons=reasons,
) from outcome.exc
@@ -177,8 +178,10 @@ class RetryMW:
)
if self._quota_full == "fail_fast":
raise AllSourcesExhausted(
scope=self._scope, reason="quota_exhausted",
retry_after_s=self._bp.poll_interval_s, per_source_reasons=reasons,
scope=self._scope,
reason="quota_exhausted",
retry_after_s=self._bp.poll_interval_s,
per_source_reasons=reasons,
)
await self._sleep(self._bp.poll_interval_s)
@@ -197,8 +200,11 @@ class RetryMW:
actual = 0
try:
result = await self._transport.complete(
messages=request.messages, source=source,
stream=request.stream, overlay=request.overlay, call_id=call_id,
messages=request.messages,
source=source,
stream=request.stream,
overlay=request.overlay,
call_id=call_id,
)
actual = result.prompt_tokens + result.completion_tokens
await self._breaker.record_success(entry)
@@ -254,13 +260,20 @@ class RetryMW:
self, source: SourceConfig, result: TransportResult, call_id: str, started: float
) -> LLMResponse:
return LLMResponse(
content=result.content, thinking=result.thinking,
model=source.model, provider=source.provider,
prompt_tokens=result.prompt_tokens, completion_tokens=result.completion_tokens,
content=result.content,
thinking=result.thinking,
model=source.model,
provider=source.provider,
prompt_tokens=result.prompt_tokens,
completion_tokens=result.completion_tokens,
latency_ms=int((self._now() - started) * 1000),
ttft_ms=result.ttft_ms, max_inter_token_ms=result.max_inter_token_ms,
cache_hit=False, call_id=call_id,
source_name=source.name, cost=None, usage_source=result.usage_source,
ttft_ms=result.ttft_ms,
max_inter_token_ms=result.max_inter_token_ms,
cache_hit=False,
call_id=call_id,
source_name=source.name,
cost=None,
usage_source=result.usage_source,
)
async def _settle_and_release(self, permit: Permit, actual: int) -> None:
@@ -276,15 +289,23 @@ class RetryMW:
logger.warning("permit 结算/释放失败(不掩盖主异常): {}", exc)
async def _emit(
self, request: ChatRequest, source: SourceConfig, call_id: str, started: float,
*, response: LLMResponse | None = None, error: object | None = None,
self,
request: ChatRequest,
source: SourceConfig,
call_id: str,
started: float,
*,
response: LLMResponse | None = None,
error: object | None = None,
) -> None:
"""逐次遥测(经注入的单一 Emitter);遥测失败不得影响调用(铁律)。"""
if self._emitter is None:
return
try:
await self._emitter.emit_attempt(
request=request, source=source, call_id=call_id,
request=request,
source=source,
call_id=call_id,
latency_ms=int((self._now() - started) * 1000),
response=response,
error=None if error is None else str(error),
+8 -4
View File
@@ -166,15 +166,19 @@ class TelemetryMW:
response = await call_next(request)
except (GatewayUnavailableError, GovernanceBackendError) as exc:
await self._emitter.emit_terminal_failure(
request=request, call_id=str(uuid.uuid4()),
latency_ms=int((self._now() - started) * 1000), error=str(exc),
request=request,
call_id=str(uuid.uuid4()),
latency_ms=int((self._now() - started) * 1000),
error=str(exc),
)
raise
except asyncio.CancelledError:
# 尽力而为: 取消也留痕(§5.1 约定④);随后立即重抛
await self._emitter.emit_terminal_failure(
request=request, call_id=str(uuid.uuid4()),
latency_ms=int((self._now() - started) * 1000), error="cancelled",
request=request,
call_id=str(uuid.uuid4()),
latency_ms=int((self._now() - started) * 1000),
error="cancelled",
)
raise
if response.cache_hit:
+1 -3
View File
@@ -49,9 +49,7 @@ class JsonRepairStrategy:
try:
data = json.loads(repair_json(stripped))
except (json.JSONDecodeError, ValueError) as exc:
raise ResultInvalidError(
"JSON 修复失败", raw_text=text, repair_error=str(exc)
) from exc
raise ResultInvalidError("JSON 修复失败", raw_text=text, repair_error=str(exc)) from exc
if self._normalize is not None:
data = self._normalize(data)
return data
+18 -4
View File
@@ -39,10 +39,24 @@ CREATE TABLE IF NOT EXISTS llm_calls (
"""
_COLUMNS = (
"call_id", "parent_call_id", "session_id", "model", "provider", "source_name",
"messages", "response", "thinking", "prompt_tokens", "completion_tokens",
"usage_source", "latency_ms", "ttft_ms", "max_inter_token_ms", "cache_hit",
"error", "cost",
"call_id",
"parent_call_id",
"session_id",
"model",
"provider",
"source_name",
"messages",
"response",
"thinking",
"prompt_tokens",
"completion_tokens",
"usage_source",
"latency_ms",
"ttft_ms",
"max_inter_token_ms",
"cache_hit",
"error",
"cost",
)
_INSERT = (
+55 -19
View File
@@ -67,7 +67,9 @@ async def _iter_sse_deltas(
try:
chunk = json.loads(data)
except json.JSONDecodeError as exc:
raise TransientError(f"SSE 帧畸形(malformed_json): {data[:80]!r}", operation="chat") from exc
raise TransientError(
f"SSE 帧畸形(malformed_json): {data[:80]!r}", operation="chat"
) from exc
delta = _sse_delta(chunk, usage_sink)
if delta is not None:
yield delta
@@ -95,12 +97,16 @@ def _translate_429(source: SourceConfig, body_text: str, headers: Mapping[str, s
if err_type == "insufficient_quota":
return SourceDeadError(
f"{source.name} 配额耗尽(insufficient_quota)",
source_name=source.name, status_code=429, operation="chat",
source_name=source.name,
status_code=429,
operation="chat",
)
return TransientError(
f"{source.name} 限速: 429",
retry_after_s=_parse_retry_after(headers.get("retry-after")),
source_name=source.name, status_code=429, operation="chat",
source_name=source.name,
status_code=429,
operation="chat",
)
@@ -164,8 +170,13 @@ class OpenAICompatTransport:
return client
def _build_payload(
self, *, messages: list[dict[str, Any]], source: SourceConfig,
profile: ProviderProfile, stream: bool, overlay: dict[str, Any],
self,
*,
messages: list[dict[str, Any]],
source: SourceConfig,
profile: ProviderProfile,
stream: bool,
overlay: dict[str, Any],
) -> dict[str, Any]:
payload: dict[str, Any] = {"model": source.model, "messages": messages, "stream": stream}
if stream:
@@ -178,8 +189,13 @@ class OpenAICompatTransport:
return payload
async def complete(
self, *, messages: list[dict[str, Any]], source: SourceConfig,
stream: bool, overlay: dict[str, Any], call_id: str,
self,
*,
messages: list[dict[str, Any]],
source: SourceConfig,
stream: bool,
overlay: dict[str, Any],
call_id: str,
) -> TransportResult:
"""一次原始调用;HTTP/线路/流式异常按 ARCH §6.2 翻译为领域错误。"""
profile = get_provider(source.provider, registry=self._registry)
@@ -202,8 +218,12 @@ class OpenAICompatTransport:
raise TransientError(f"{source.name} 网络错误: {exc}", **ctx) from exc
async def _complete_stream(
self, client: httpx.AsyncClient, url: str, payload: dict[str, Any],
source: SourceConfig, profile: ProviderProfile,
self,
client: httpx.AsyncClient,
url: str,
payload: dict[str, Any],
source: SourceConfig,
profile: ProviderProfile,
) -> TransportResult:
started = time.monotonic()
async with client.stream("POST", url, json=payload) as resp:
@@ -236,15 +256,22 @@ class OpenAICompatTransport:
if salvaged:
usage_source = "estimated" # 打捞路径强制 estimated(设计 §6)
return TransportResult(
content=content, thinking=thinking, prompt_tokens=prompt,
completion_tokens=completion, usage_source=usage_source,
ttft_ms=ttft_ms, max_inter_token_ms=(max_gap if ttft_ms is not None else None),
content=content,
thinking=thinking,
prompt_tokens=prompt,
completion_tokens=completion,
usage_source=usage_source,
ttft_ms=ttft_ms,
max_inter_token_ms=(max_gap if ttft_ms is not None else None),
raw={"usage": sink.get("usage")},
)
def _check_done(
self, sink: dict[str, Any], content_parts: list[str],
thinking_parts: list[str], source: SourceConfig,
self,
sink: dict[str, Any],
content_parts: list[str],
thinking_parts: list[str],
source: SourceConfig,
) -> bool:
"""缺 [DONE] 语义(设计 §6): 零内容恒 retry;有内容按 missing_done 策略。"""
if sink.get("done"):
@@ -268,8 +295,12 @@ class OpenAICompatTransport:
return content, thinking
async def _complete_once(
self, client: httpx.AsyncClient, url: str, payload: dict[str, Any],
source: SourceConfig, profile: ProviderProfile,
self,
client: httpx.AsyncClient,
url: str,
payload: dict[str, Any],
source: SourceConfig,
profile: ProviderProfile,
) -> TransportResult:
"""非流式快路径(三项目均无,库新增): 单 JSON 响应,仅 total 超时。"""
resp = await client.post(url, json=payload)
@@ -292,9 +323,14 @@ class OpenAICompatTransport:
)
prompt, completion, usage_source = _resolve_usage(body.get("usage") or {}, source)
return TransportResult(
content=content, thinking=thinking, prompt_tokens=prompt,
completion_tokens=completion, usage_source=usage_source,
ttft_ms=None, max_inter_token_ms=None, raw={"usage": body.get("usage")},
content=content,
thinking=thinking,
prompt_tokens=prompt,
completion_tokens=completion,
usage_source=usage_source,
ttft_ms=None,
max_inter_token_ms=None,
raw={"usage": body.get("usage")},
)
async def aclose(self) -> None:
+3 -1
View File
@@ -112,7 +112,9 @@ class SourceConfig:
if not getattr(self, attr).strip():
raise ValueError(f"SourceConfig.{attr} 不能为空")
if self.missing_done not in _MISSING_DONE_DOMAIN:
raise ValueError(f"missing_done 必须是 {sorted(_MISSING_DONE_DOMAIN)}: {self.missing_done!r}")
raise ValueError(
f"missing_done 必须是 {sorted(_MISSING_DONE_DOMAIN)}: {self.missing_done!r}"
)
def _validate_gates(self) -> None:
if self.timeout_s <= 0: