diff --git a/CHANGELOG.md b/CHANGELOG.md index 66698c5..9a170ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 1.0.2(2026-07-30) + +1.0.1 的续作:那一版把三条跨字段守卫收进构造期后,独立验证发现 `from_env` 上还留着同一类的 15 条校验,一并收拢。 + +### 修复 + +- **后端选择与条件必填项在任何构造路径上都校验。** 以下此前只有 `from_env` 拦得住,`from_settings()` 与直接构造一律放行:`limiter_backend`/`breaker_backend`/`cache_backend`/`telemetry_backend`/`selector`/`quota_full` 六个字段的合法域;取 `redis` 的后端必须有 `redis_url`;启用缓存必须有 `cache_namespace` 与正 `cache_ttl_s`;`telemetry_backend` 取 `sqlite`/`postgres` 时对应的路径/DSN 必填;`structured_max_retries` 非负;`scope` 非空。 +- **`client.py` 五处断言的前提现在真的成立。** `assert settings.redis_url is not None # 内部不变量: config 已校验` 之类的注释此前在 `from_settings` 路上是假的:断言开启时抛不含任何字段信息的 `AssertionError`,`python -O` 下断言被移除、错误退化为 redis 库抛出的连接串解析异常。注释已改为点明由哪个校验方法保证。 +- **手工构造时的 Postgres DSN 会剥掉 SQLAlchemy 驱动后缀。** `postgresql+asyncpg://…` 中的 `+asyncpg` asyncpg 不认;`from_env` 一直会剥,直接构造那条路此前不剥,DSN 会一路带到首次写遥测时才炸。现在两条路产出一致,且构造期剥的时候会发一条 warning——库动了调用方给的值,不该静默。经 `from_env` 装配不受影响也不会有这条 warning。 + +### 行为收紧(下游请读) + +同 1.0.1:经 `from_env()` 装配的调用方**不受影响**。手工构造 `GatewaySettings` 或对它 `dataclasses.replace` 的调用方,若配置组合非法,现在会在构造期抛 `ValueError` 并点出字段名,而不是留到运行时表现为静默不建后端、裸 `AssertionError` 或第三方库的天书报错。 + ## 1.0.1(2026-07-30) ### 修复 diff --git a/pyproject.toml b/pyproject.toml index 31ec7fe..5406590 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "polygateway" -version = "1.0.1" +version = "1.0.2" description = "PolyGateway：实验室统一的大语言模型（LLM/VLM/OCR）调度与中转库——多源、限流、重试、熔断、缓存、遥测" requires-python = ">=3.11" dependencies = [ diff --git a/research-wiki/designs/2026-07-30-settings-invariants-round-2-design.md b/research-wiki/designs/2026-07-30-settings-invariants-round-2-design.md index 13188ea..5968c1a 100644 --- a/research-wiki/designs/2026-07-30-settings-invariants-round-2-design.md +++ b/research-wiki/designs/2026-07-30-settings-invariants-round-2-design.md @@ -79,10 +79,15 @@ assert settings.redis_url is not None # 内部不变量: config 已校验 | 选项 | 权衡 | |---|---| -| **A. 构造期校验,含 `+driver` 即报错(推荐)** | 显式优于隐式;frozen dataclass 里改字段要 `object.__setattr__`,是在用户背后动他给的值。报错消息直接告诉他去掉后缀即可 | -| B. 构造期规范化(`object.__setattr__` 剥后缀) | 与 `from_env` 行为完全对齐,调用方不用管;但 frozen 类在构造期悄悄改字段,后续 `replace`/相等性比较都会有惊喜 | +| A. 构造期校验,含 `+driver` 即报错 | 显式,库不碰用户给的值;但两条装配路对同一输入接受度不同 | +| B. 构造期静默剥后缀 | 两条路完全对齐;但 frozen 类在构造期悄悄改字段,调用方不知情 | +| **C. 构造期剥后缀 + `logger.warning`(用户 2026-07-30 拍板)** | 两条路行为对齐,同时不静默——调用方在日志里看得见库动了他的值,想根治就自己改 DSN | -推荐 A。代价是两条装配路对同一输入的**接受度**不同(`from_env` 接受带后缀的、直接构造不接受),但两者的**产出**一致——`GatewaySettings.telemetry_pg_dsn` 永远是干净 DSN。这个不对称是有意的:env 那条路要吃下三项目历史遗留的 SQLAlchemy DSN 写法(迁移兼容),代码构造那条路没有历史包袱。 +选 C。实现要点:`object.__setattr__` 改 frozen 字段(`SourceConfig` 无此先例,但 frozen 的约束是对**外部**不可变,构造期规范化是既有 dataclass 惯用法);warning 走 loguru(核心依赖,库内 `ocr.py:183`/`embedding.py:318` 同款用法)。 + +**warning 不会打扰 env 用户**:`_load_pg_dsn` 保留现有的剥离逻辑,`from_env` 传给构造函数时 DSN 已经干净,`__post_init__` 无事可做。只有手工构造传了带后缀的 DSN 才会触发。三项目 `.env` 里那些 SQLAlchemy 写法不会每次装配刷一条 warning。 + +代价是同一件事有两处剥离逻辑。用同一个模块级 helper `_strip_dsn_driver(dsn)` 供两处调用,避免实现分叉。 ## 6. 行为审计 @@ -112,7 +117,8 @@ assert settings.redis_url is not None # 内部不变量: config 已校验 | cache namespace 缺失 / ttl ≤ 0 | 抛 `ValueError` | | telemetry sqlite path / pg dsn 缺失 | 抛 `ValueError` | | `structured_max_retries=-1`、`scope=""` | 抛 `ValueError` | -| pg dsn 含 `+asyncpg` | 抛 `ValueError`,消息给出去后缀的写法 | +| pg dsn 含 `+asyncpg`(直接构造) | 后缀被剥,字段值为干净 DSN,且发出一条 warning(用 `caplog`/loguru sink 断言) | +| pg dsn 干净(直接构造)、或经 `from_env` 传入 | **不发** warning——env 路已在 `_load_pg_dsn` 剥过,不该刷噪音 | | 合法组合(每种 backend 组合各一) | 构造成功——收紧的是错的那些 | | **回归护栏**:`GatewayClient.from_settings` 走 redis 三后端的合法配置 | 装配成功,证明 assert 前提真的被保证了 | @@ -120,7 +126,11 @@ TDD:先跑出红,预计 ≥14 条失败。要求同第一轮——每条实现 版本:**1.0.2**(patch),CHANGELOG 同样单列"行为收紧"小节。 -## 9. 待人类拍板 +## 9. 人类拍板结论(2026-07-30) -1. §5 的 DSN 处置:选 A(校验拒绝,推荐)还是 B(构造期剥后缀)? -2. §4 的 assert 保留 + 改注释,是否认同? +| 问题 | 结论 | +|---|---| +| §5 DSN 处置 | **选 C**:构造期剥后缀 + `logger.warning`。不静默改用户的值,也不让两条装配路产出不一致 | +| §4 assert 处置 | **保留,只改注释**,点明由哪个方法保证前提 | +| 方案主体 | 沿用第一轮已批准的方案 A,无需重新论证 | +| 版本 | 1.0.2(patch) | diff --git a/src/polygateway/__init__.py b/src/polygateway/__init__.py index efb48f8..ea0b2bd 100644 --- a/src/polygateway/__init__.py +++ b/src/polygateway/__init__.py @@ -31,7 +31,7 @@ from polygateway.types import ( SourceConfig, ) -__version__ = "1.0.1" +__version__ = "1.0.2" __all__ = [ "DEFAULT_PROFILES", diff --git a/src/polygateway/client.py b/src/polygateway/client.py index 3205243..7ea7837 100644 --- a/src/polygateway/client.py +++ b/src/polygateway/client.py @@ -259,7 +259,7 @@ def _build_limiter(settings: GatewaySettings, sources: list[SourceConfig]) -> Ra if settings.limiter_backend == "redis": from polygateway.backends.redis.limiter import RedisLimiter - assert settings.redis_url is not None # 内部不变量: config 已校验 + assert settings.redis_url is not None # 内部不变量: _validate_backends 已保证 return RedisLimiter.from_url( settings.redis_url, scope=settings.scope, @@ -279,7 +279,7 @@ def _build_breaker(settings: GatewaySettings) -> ProviderGate: if settings.breaker_backend == "redis": from polygateway.backends.redis.breaker import RedisGate - assert settings.redis_url is not None # 内部不变量: config 已校验 + assert settings.redis_url is not None # 内部不变量: _validate_backends 已保证 return RedisGate.from_url(settings.redis_url, config=settings.breaker, scope=settings.scope) return InMemoryGate(config=settings.breaker) @@ -299,7 +299,7 @@ def _build_cache(settings: GatewaySettings) -> CacheBackend | None: return InMemoryCache() from polygateway.backends.redis_cache import RedisCache - assert settings.redis_url is not None # 内部不变量: config 已校验 + assert settings.redis_url is not None # 内部不变量: _validate_backends 已保证 return RedisCache.from_url(settings.redis_url) @@ -309,11 +309,11 @@ def _build_telemetry(settings: GatewaySettings) -> TelemetryRecorder | None: if settings.telemetry_backend == "postgres": from polygateway.telemetry.postgres import PostgresRecorder - assert settings.telemetry_pg_dsn is not None # 内部不变量: config 已校验 + assert settings.telemetry_pg_dsn is not None # 内部不变量: _validate_telemetry 已保证 return PostgresRecorder(settings.telemetry_pg_dsn) from polygateway.telemetry.sqlite import SQLiteRecorder - assert settings.telemetry_sqlite_path is not None # 内部不变量: config 已校验 + assert settings.telemetry_sqlite_path is not None # 内部不变量: _validate_telemetry 已保证 return SQLiteRecorder(settings.telemetry_sqlite_path) diff --git a/src/polygateway/config.py b/src/polygateway/config.py index 6483337..47fe0c1 100644 --- a/src/polygateway/config.py +++ b/src/polygateway/config.py @@ -16,6 +16,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING from dotenv import dotenv_values +from loguru import logger from polygateway.types import ( BackpressurePolicy, @@ -47,6 +48,12 @@ _SOURCE_FIELDS: dict[str, tuple[str, str]] = { _RESERVED_SEGMENTS = frozenset({"GLOBAL", "RETRY", "BREAKER", "BACKPRESSURE"}) _SELECTORS = frozenset({"round_robin", "least_inflight", "health_aware"}) _QUOTA_FULL = frozenset({"wait", "fail_fast"}) +# 后端合法域: env 解析与构造期校验共用一份定义,避免两处分叉 +_LIMITER_BACKENDS = frozenset({"memory", "redis"}) +_BREAKER_BACKENDS = frozenset({"memory", "redis"}) +_CACHE_BACKENDS = frozenset({"redis", "memory", "none"}) +_TELEMETRY_BACKENDS = frozenset({"sqlite", "postgres", "none"}) +_REDIS_DEPENDENT_BACKENDS = ("limiter_backend", "breaker_backend", "cache_backend") # 背压默认(M1 仅 poll 生效;CHS _BACKOFF_S=0.05 同源) _DEFAULT_STALL_WINDOW_S = 300.0 _DEFAULT_POLL_INTERVAL_S = 0.05 @@ -122,15 +129,72 @@ class GatewaySettings: lease_ttl_s: float def __post_init__(self) -> None: - self._validate_sources() + self._validate_identity() + self._validate_backends() + self._validate_cache() + self._validate_telemetry() self._validate_lease() self._validate_stall() self._validate_probe() - def _validate_sources(self) -> None: - """零源的配置装出来选源必然失败,构造期即拒。""" + def _validate_identity(self) -> None: + """本类自身字段的基本域: 空 scope 会污染遥测与缓存命名空间;零源必然选源失败。""" + if not self.scope.strip(): + raise ValueError("GatewaySettings.scope 不能为空") if not self.sources: raise ValueError("GatewaySettings.sources 不能为空: 至少一个源") + if self.structured_max_retries < 0: + raise ValueError(f"structured_max_retries 不能为负: {self.structured_max_retries}") + + def _validate_backends(self) -> None: + """后端选择必须落在合法域内,取 redis 的还必须有连接串。 + + 域外取值此前只有 `from_env` 拦得住,直接构造会一路走到 `client.py` 的 + `_build_*`,落进 else 分支静默不建后端,或撞上那里的断言。 + """ + for field, allowed in ( + ("limiter_backend", _LIMITER_BACKENDS), + ("breaker_backend", _BREAKER_BACKENDS), + ("cache_backend", _CACHE_BACKENDS), + ("telemetry_backend", _TELEMETRY_BACKENDS), + ("selector", _SELECTORS), + ("quota_full", _QUOTA_FULL), + ): + value = getattr(self, field) + if value not in allowed: + raise ValueError(f"{field} 非法值 {value!r};允许: {sorted(allowed)}") + on_redis = [f for f in _REDIS_DEPENDENT_BACKENDS if getattr(self, f) == "redis"] + if on_redis and self.redis_url is None: + raise ValueError(f"{'、'.join(on_redis)} 取 redis 时必须提供 redis_url") + + def _validate_cache(self) -> None: + """启用缓存必须有命名空间与正 TTL(缺命名空间即失去租户隔离,会毒化缓存)。""" + if self.cache_backend == "none": + return + if not self.cache_namespace: + raise ValueError("启用缓存时 cache_namespace 不能为空: 缓存 key 靠它做租户隔离") + if self.cache_ttl_s is None or self.cache_ttl_s <= 0: + raise ValueError(f"cache_ttl_s 必须 > 0(禁止永不过期): {self.cache_ttl_s}") + + def _validate_telemetry(self) -> None: + """遥测后端各自的落点必填;顺带剥掉 asyncpg 不认的 SQLAlchemy 驱动后缀。 + + 剥而不是拒: 两条装配路对同一 DSN 应产出同一结果。但不静默——`from_env` + 那条路在 `_load_pg_dsn` 就剥干净了,能走到这里的只有手工构造的调用方, + 他有权知道库动了他给的值。 + """ + if self.telemetry_backend == "sqlite" and not self.telemetry_sqlite_path: + raise ValueError("telemetry_backend=sqlite 时必须提供 telemetry_sqlite_path") + if self.telemetry_backend != "postgres": + return + if not self.telemetry_pg_dsn: + raise ValueError("telemetry_backend=postgres 时必须提供 telemetry_pg_dsn") + stripped = _strip_dsn_driver(self.telemetry_pg_dsn) + if stripped != self.telemetry_pg_dsn: + logger.warning( + "telemetry_pg_dsn 含 SQLAlchemy 驱动后缀(asyncpg 不认),已剥为 {}", stripped + ) + object.__setattr__(self, "telemetry_pg_dsn", stripped) def _validate_lease(self) -> None: """调用超时须 ≤ permit 租约 TTL,防租约先于请求过期使并发超出配额。""" @@ -318,17 +382,15 @@ def _load_choice(env: Mapping[str, str], key: str, allowed: frozenset[str], defa def _load_pgw(env: Mapping[str, str]) -> dict[str, object]: - limiter_backend = _load_choice( - env, "PGW_LIMITER_BACKEND", frozenset({"memory", "redis"}), "memory" - ) - breaker_backend = _load_choice( - env, "PGW_BREAKER_BACKEND", frozenset({"memory", "redis"}), "memory" - ) + # 合法域与构造期守卫共用常量;此处的检查保留是为了报错能点出 env 键名, + # 构造期那道点的是字段名(两类调用方各看得懂自己那套) + limiter_backend = _load_choice(env, "PGW_LIMITER_BACKEND", _LIMITER_BACKENDS, "memory") + breaker_backend = _load_choice(env, "PGW_BREAKER_BACKEND", _BREAKER_BACKENDS, "memory") _, cache_backend = _require(env, "PGW_CACHE_BACKEND") _, telemetry_backend = _require(env, "PGW_TELEMETRY_BACKEND") - if cache_backend not in ("redis", "memory", "none"): + if cache_backend not in _CACHE_BACKENDS: raise ValueError(f"PGW_CACHE_BACKEND 非法值 {cache_backend!r}") - if telemetry_backend not in ("sqlite", "postgres", "none"): + if telemetry_backend not in _TELEMETRY_BACKENDS: raise ValueError(f"PGW_TELEMETRY_BACKEND 非法值 {telemetry_backend!r}") redis_url = env.get("REDIS_URL") or None if "redis" in (limiter_backend, breaker_backend) and redis_url is None: @@ -350,13 +412,22 @@ def _load_pgw(env: Mapping[str, str]) -> dict[str, object]: } -def _load_pg_dsn(env: Mapping[str, str]) -> str: - """读取 Postgres DSN 并剥 SQLAlchemy 风格驱动后缀(asyncpg 不认 `+driver`)。""" - _, dsn = _require(env, "PGW_TELEMETRY_PG_DSN") +def _strip_dsn_driver(dsn: str) -> str: + """剥 SQLAlchemy 风格的 `+driver` 后缀(asyncpg 不认);已干净的原样返回。""" scheme, sep, rest = dsn.partition("://") return f"{scheme.partition('+')[0]}{sep}{rest}" +def _load_pg_dsn(env: Mapping[str, str]) -> str: + """读取 Postgres DSN 并剥驱动后缀。 + + env 路在此剥干净,构造期那道就无事可做——三项目 `.env` 里的 SQLAlchemy + 写法不会每次装配都刷一条 warning。 + """ + _, dsn = _require(env, "PGW_TELEMETRY_PG_DSN") + return _strip_dsn_driver(dsn) + + def _load_cache_keys( env: Mapping[str, str], cache_backend: str, redis_url: str | None ) -> dict[str, object]: diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index b7fd079..da21c4f 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1,8 +1,10 @@ """config.py 配置聚合测试(设计 §8): 多源命名、键优先级、缺失报错。""" +import contextlib import dataclasses import pytest +from loguru import logger from polygateway.client import GatewayClient from polygateway.config import GatewaySettings, OcrSettings @@ -22,6 +24,17 @@ _BASE_ENV = { } +@contextlib.contextmanager +def _captured_warnings(): + """捕获库发出的 WARNING;loguru 不经标准 logging,pytest 的 caplog 抓不到。""" + messages: list[str] = [] + sink_id = logger.add(messages.append, level="WARNING") + try: + yield messages + finally: + logger.remove(sink_id) + + def _env(**overrides): env = dict(_BASE_ENV) env.update({k: v for k, v in overrides.items() if v is not None}) @@ -413,3 +426,131 @@ class TestCrossFieldInvariants: base = self._base() with pytest.raises(ValueError, match="lease_ttl_s"): OcrSettings(gateway=dataclasses.replace(base, lease_ttl_s=1.0)) + + # —— 第二轮(设计 2026-07-30): 后端枚举合法域 —— + + @pytest.mark.parametrize( + ("field", "bad_value"), + [ + ("limiter_backend", "rediss"), + ("breaker_backend", "sqlite"), + ("cache_backend", "postgres"), + ("telemetry_backend", "redis"), + ("selector", "random"), + ("quota_full", "block"), + ], + ) + def test_enum_field_rejects_value_outside_domain(self, field, bad_value): + """域外取值此前只有 from_env 拦得住,直接构造会落进 _build_* 的 else 分支。""" + base = self._base() + with pytest.raises(ValueError, match=field): + dataclasses.replace(base, **{field: bad_value}) + + # —— 条件必填: 取 redis 的后端必须有 redis_url —— + + @pytest.mark.parametrize("field", ["limiter_backend", "breaker_backend"]) + def test_redis_backend_requires_redis_url(self, field): + """client.py 的 assert settings.redis_url is not None 依赖的正是这条。""" + base = self._base() # redis_url=None + with pytest.raises(ValueError, match="redis_url"): + dataclasses.replace(base, **{field: "redis"}) + + def test_redis_cache_requires_redis_url(self): + base = self._base() + with pytest.raises(ValueError, match="redis_url"): + dataclasses.replace(base, cache_backend="redis", cache_namespace="ns", cache_ttl_s=60) + + # —— 条件必填: 启用缓存必须有命名空间与正 TTL —— + + def test_cache_requires_namespace(self): + """缺命名空间即失去租户隔离,踩"无缓存毒化"铁律。""" + base = self._base() + with pytest.raises(ValueError, match="cache_namespace"): + dataclasses.replace(base, cache_backend="memory", cache_ttl_s=60) + + def test_cache_ttl_must_be_positive(self): + """from_env 明令禁止的"永不过期"不能从另一条路进来。""" + base = self._base() + with pytest.raises(ValueError, match="cache_ttl_s"): + dataclasses.replace(base, cache_backend="memory", cache_namespace="ns", cache_ttl_s=0) + + # —— 条件必填: 遥测后端各自的落点 —— + + def test_sqlite_telemetry_requires_path(self): + base = self._base() + with pytest.raises(ValueError, match="telemetry_sqlite_path"): + dataclasses.replace(base, telemetry_backend="sqlite") + + def test_postgres_telemetry_requires_dsn(self): + base = self._base() + with pytest.raises(ValueError, match="telemetry_pg_dsn"): + dataclasses.replace(base, telemetry_backend="postgres") + + # —— 标量域 —— + + def test_negative_structured_retries_rejected(self): + base = self._base() + with pytest.raises(ValueError, match="structured_max_retries"): + dataclasses.replace(base, structured_max_retries=-1) + + def test_blank_scope_rejected(self): + """空 scope 会污染遥测与缓存命名空间。""" + base = self._base() + with pytest.raises(ValueError, match="scope"): + dataclasses.replace(base, scope=" ") + + # —— 合法组合仍可构造(收紧的是错的那些)—— + + def test_full_redis_stack_constructible(self): + base = self._base() + settings = dataclasses.replace( + base, + limiter_backend="redis", + breaker_backend="redis", + cache_backend="redis", + cache_namespace="ns", + cache_ttl_s=60, + redis_url="redis://127.0.0.1:6379/3", + ) + assert settings.cache_ttl_s == 60 and settings.redis_url is not None + + # —— Postgres DSN: 剥 SQLAlchemy 驱动后缀并出声(设计 §5 方案 C)—— + + def test_sqlalchemy_dsn_suffix_stripped_with_warning(self): + """asyncpg 不认 `+driver`;库替调用方剥掉,但不静默——日志里看得见。""" + base = self._base() + with _captured_warnings() as warnings: + settings = dataclasses.replace( + base, + telemetry_backend="postgres", + telemetry_pg_dsn="postgresql+asyncpg://u@h/db", + ) + assert settings.telemetry_pg_dsn == "postgresql://u@h/db" + assert any("asyncpg" in m for m in warnings) + + def test_env_path_strips_dsn_without_warning(self): + """env 路已在 _load_pg_dsn 剥过,不该给三项目的历史 DSN 写法刷噪音。""" + with _captured_warnings() as warnings: + settings = GatewaySettings.from_env( + "LLM", + env=_env( + PGW_TELEMETRY_BACKEND="postgres", + PGW_TELEMETRY_PG_DSN="postgresql+asyncpg://u@h/db", + ), + ) + assert settings.telemetry_pg_dsn == "postgresql://u@h/db" + assert not warnings + + # —— 回归护栏: client.py 的 assert 前提确实被保证了 —— + + def test_factory_accepts_valid_redis_stack(self): + """补齐校验后,client.py:262/282/302 的 assert 退回成纯内部不变量声明。""" + base = self._base() + settings = dataclasses.replace( + base, + limiter_backend="redis", + breaker_backend="redis", + redis_url="redis://127.0.0.1:6379/3", + ) + client = GatewayClient.from_settings(settings) + assert client is not None