feat: make the telemetry pool declare what it costs
The pool was the only external resource in the library that pre-allocated: asyncpg's default min_size=10 turned pool creation into an all-or-nothing action, so on a shared instance running low on connection budget the first thing to fall over was the one component that must not fail silently (4 clients x 10 = 40 idle connections just to write telemetry). min_size=0 means "do not pre-connect" - asyncpg only builds holders - so pool creation becomes free and never touches the database; connection failures then land on acquire, the path that already drops one row and lets the pool recover. max_size and the write budget become the library's explicit statement about its own footprint, configurable through two new keys whose defaults live in config alone (the recorder parameters are required keyword-only, same discipline as auto_migrate). The whole write - prepare, acquire, execute - now runs inside one asyncio.timeout: acquire used to have no timeout at all, so a full pool would hang forever on the caller's path. Release is explicit rather than `async with`, because asyncpg shields release and reuses the acquire timeout, which would let a single telemetry write consume twice the budget.
This commit is contained in:
@@ -63,6 +63,13 @@ _SCHEMA_MODES = frozenset({"auto", "manual"})
|
||||
_SCHEMA_MODE_KEY = "PGW_TELEMETRY_SCHEMA_MODE"
|
||||
# 遥测正文字符上限(issue #12);二态键,未设 = 不截断
|
||||
_TEXT_CAP_KEY = "PGW_TELEMETRY_TEXT_CAP"
|
||||
# 遥测池的资源占用与写入预算(issue #15);缺省只写在这里,recorder 侧是必填参数
|
||||
_POOL_MAX_KEY = "PGW_TELEMETRY_PG_POOL_MAX"
|
||||
_WRITE_TIMEOUT_KEY = "PGW_TELEMETRY_PG_WRITE_TIMEOUT_S"
|
||||
# 4 条 ≈ 32 行/秒(实测跨内网 RTT 123ms),够单 client 数十并发;闲时占 0 条
|
||||
_DEFAULT_PG_POOL_MAX = 4
|
||||
# 实测稳态写入 123ms、首次含建连 513ms;5s 宽松且**有界**
|
||||
_DEFAULT_PG_WRITE_TIMEOUT_S = 5.0
|
||||
_REDIS_DEPENDENT_BACKENDS = ("limiter_backend", "breaker_backend", "cache_backend")
|
||||
# 背压默认(M1 仅 poll 生效;CHS _BACKOFF_S=0.05 同源)
|
||||
_DEFAULT_STALL_WINDOW_S = 300.0
|
||||
@@ -152,6 +159,13 @@ class GatewaySettings:
|
||||
# 既有下游正依赖这一行为。值域(> 0)由 `_validate_telemetry` 把关,直接构造、
|
||||
# `dataclasses.replace` 与 env 三条路一并覆盖
|
||||
telemetry_text_cap: int | None
|
||||
# 遥测池对外声明的资源占用上限与整次写入的硬预算(issue #15)。库内每一处外部
|
||||
# 资源都按需建连,唯独遥测池此前预占 10 条(asyncpg 默认 `min_size`),共享实例
|
||||
# 余量紧张时先倒下的必然是它。这两个字段是库对自己占用的**显式表态**:
|
||||
# 稳态并发上限 = `pool_max`,闲时 0 条;单次写入(准备+取连接+执行)≤ 预算。
|
||||
# 值域由 `_validate_telemetry` 把关,直接构造、`dataclasses.replace` 与 env 三条路一致
|
||||
telemetry_pg_pool_max: int
|
||||
telemetry_pg_write_timeout_s: float
|
||||
redis_url: str | None
|
||||
pricing_path: str | None
|
||||
structured_max_retries: int
|
||||
@@ -248,6 +262,7 @@ class GatewaySettings:
|
||||
f"telemetry_text_cap({_TEXT_CAP_KEY})必须 > 0: {self.telemetry_text_cap};"
|
||||
"不截断请不设该键(None),0 只会让每条正文退化成一个省略标记"
|
||||
)
|
||||
self._validate_telemetry_pool()
|
||||
if self.telemetry_backend == "none" and self.telemetry_auto_migrate:
|
||||
object.__setattr__(self, "telemetry_auto_migrate", False)
|
||||
if self.telemetry_backend == "sqlite" and not self.telemetry_sqlite_path:
|
||||
@@ -266,6 +281,24 @@ class GatewaySettings:
|
||||
)
|
||||
object.__setattr__(self, "telemetry_pg_dsn", stripped)
|
||||
|
||||
def _validate_telemetry_pool(self) -> None:
|
||||
"""遥测池两个标量的值域(issue #15);与 backend 无关,三条装配路一并覆盖。
|
||||
|
||||
不按 `telemetry_backend == "postgres"` 才校验: 值域错就是错,提前拦住
|
||||
比等到有人把 backend 切成 postgres 时才炸更接近"缺失关键配置直接报错"。
|
||||
报错文本同时点字段名与 env 键名(两类调用方各看得懂自己那套)。
|
||||
"""
|
||||
if self.telemetry_pg_pool_max < 1:
|
||||
raise ValueError(
|
||||
f"telemetry_pg_pool_max({_POOL_MAX_KEY})必须 >= 1: "
|
||||
f"{self.telemetry_pg_pool_max};0 条上限等于永远取不到连接,遥测会全灭"
|
||||
)
|
||||
if self.telemetry_pg_write_timeout_s <= 0:
|
||||
raise ValueError(
|
||||
f"telemetry_pg_write_timeout_s({_WRITE_TIMEOUT_KEY})必须 > 0: "
|
||||
f"{self.telemetry_pg_write_timeout_s};预算 0 会让每一行当场超预算被丢弃"
|
||||
)
|
||||
|
||||
def _validate_lease(self) -> None:
|
||||
"""调用超时须 ≤ permit 租约 TTL,防租约先于请求过期使并发超出配额。"""
|
||||
slowest = max(s.timeout_s for s in self.sources)
|
||||
@@ -486,6 +519,8 @@ def _load_pgw(env: Mapping[str, str]) -> dict[str, object]:
|
||||
"telemetry_pg_dsn": _load_pg_dsn(env) if telemetry_backend == "postgres" else None,
|
||||
"telemetry_auto_migrate": auto_migrate,
|
||||
"telemetry_text_cap": _load_text_cap(env),
|
||||
"telemetry_pg_pool_max": _load_pool_max(env),
|
||||
"telemetry_pg_write_timeout_s": _load_write_timeout(env),
|
||||
"redis_url": redis_url,
|
||||
"pricing_path": env.get("PGW_PRICING_PATH") or None,
|
||||
"structured_max_retries": _load_structured_retries(env),
|
||||
@@ -543,6 +578,43 @@ def _load_text_cap(env: Mapping[str, str]) -> int | None:
|
||||
return int(_cast(found[1], "int", found[0]))
|
||||
|
||||
|
||||
def _load_pool_max(env: Mapping[str, str]) -> int:
|
||||
"""读 `PGW_TELEMETRY_PG_POOL_MAX`(issue #15);未设即缺省 4。
|
||||
|
||||
与 `_load_text_cap` 同为二态键,只是"未设"落到一个具体缺省而非 None:
|
||||
池上限没有"不设上限"这一档——不表态就是继承第三方默认值,而那正是本 issue
|
||||
的病灶。值域(>= 1)留给构造期守卫,它同时覆盖直接构造与 `dataclasses.replace`。
|
||||
|
||||
Args:
|
||||
env: 已合并的环境映射。
|
||||
|
||||
Returns:
|
||||
遥测池允许的最大连接数。
|
||||
"""
|
||||
found = _first(env, _POOL_MAX_KEY)
|
||||
if found is None:
|
||||
return _DEFAULT_PG_POOL_MAX
|
||||
return int(_cast(found[1], "int", found[0]))
|
||||
|
||||
|
||||
def _load_write_timeout(env: Mapping[str, str]) -> float:
|
||||
"""读 `PGW_TELEMETRY_PG_WRITE_TIMEOUT_S`(issue #15);未设即缺省 5.0 秒。
|
||||
|
||||
这个值同时是 connect、acquire 与整次写入的上界: 遥测是业务路径上的内联
|
||||
await,"不设预算"不是一个允许存在的档位(铁律"丢一条 < 拖垮调用")。
|
||||
|
||||
Args:
|
||||
env: 已合并的环境映射。
|
||||
|
||||
Returns:
|
||||
单次遥测写入的硬预算(秒)。
|
||||
"""
|
||||
found = _first(env, _WRITE_TIMEOUT_KEY)
|
||||
if found is None:
|
||||
return _DEFAULT_PG_WRITE_TIMEOUT_S
|
||||
return float(_cast(found[1], "float", found[0]))
|
||||
|
||||
|
||||
def _strip_dsn_driver(dsn: str) -> str:
|
||||
"""剥 SQLAlchemy 风格的 `+driver` 后缀(asyncpg 不认);已干净的原样返回。"""
|
||||
scheme, sep, rest = dsn.partition("://")
|
||||
|
||||
Reference in New Issue
Block a user