feat: make the telemetry pool declare what it costs
The pool was the only external resource in the library that pre-allocated: asyncpg's default min_size=10 turned pool creation into an all-or-nothing action, so on a shared instance running low on connection budget the first thing to fall over was the one component that must not fail silently (4 clients x 10 = 40 idle connections just to write telemetry). min_size=0 means "do not pre-connect" - asyncpg only builds holders - so pool creation becomes free and never touches the database; connection failures then land on acquire, the path that already drops one row and lets the pool recover. max_size and the write budget become the library's explicit statement about its own footprint, configurable through two new keys whose defaults live in config alone (the recorder parameters are required keyword-only, same discipline as auto_migrate). The whole write - prepare, acquire, execute - now runs inside one asyncio.timeout: acquire used to have no timeout at all, so a full pool would hang forever on the caller's path. Release is explicit rather than `async with`, because asyncpg shields release and reuses the acquire timeout, which would let a single telemetry write consume twice the budget.
This commit is contained in:
@@ -425,6 +425,73 @@ class TestTelemetryTextCap:
|
||||
GatewaySettings.from_env("LLM", env=_env(PGW_TELEMETRY_TEXT_CAP="2k"))
|
||||
|
||||
|
||||
class TestTelemetryPoolKeys:
|
||||
"""`PGW_TELEMETRY_PG_POOL_MAX` / `PGW_TELEMETRY_PG_WRITE_TIMEOUT_S`(issue #15)。
|
||||
|
||||
两键都带 `PG` 前缀,与 `PGW_TELEMETRY_PG_DSN` 一致: SQLite 侧没有池、也没有
|
||||
等价的写入预算旋钮,这个不对称是已知且有理由的。缺省值(4 / 5.0)只写在
|
||||
config 一处——recorder 的两个同名参数是必填 keyword-only,不许各带一份缺省。
|
||||
"""
|
||||
|
||||
def _pg_env(self, **overrides):
|
||||
return _env(
|
||||
PGW_TELEMETRY_BACKEND="postgres",
|
||||
PGW_TELEMETRY_PG_DSN="postgresql://u:p@h:5432/polygateway",
|
||||
**overrides,
|
||||
)
|
||||
|
||||
def test_unset_keys_fall_back_to_the_documented_defaults(self):
|
||||
s = GatewaySettings.from_env("LLM", env=self._pg_env())
|
||||
assert s.telemetry_pg_pool_max == 4
|
||||
assert s.telemetry_pg_write_timeout_s == 5.0
|
||||
|
||||
def test_values_parsed_from_env(self):
|
||||
s = GatewaySettings.from_env(
|
||||
"LLM",
|
||||
env=self._pg_env(PGW_TELEMETRY_PG_POOL_MAX="8", PGW_TELEMETRY_PG_WRITE_TIMEOUT_S="1.5"),
|
||||
)
|
||||
assert s.telemetry_pg_pool_max == 8
|
||||
assert s.telemetry_pg_write_timeout_s == 1.5
|
||||
|
||||
@pytest.mark.parametrize("raw", ["0", "-1"])
|
||||
def test_non_positive_pool_max_rejected_naming_the_env_key(self, raw):
|
||||
"""池上限 0 = 永远拿不到连接(遥测全灭),负数无意义。"""
|
||||
with pytest.raises(ValueError, match="PGW_TELEMETRY_PG_POOL_MAX"):
|
||||
GatewaySettings.from_env("LLM", env=self._pg_env(PGW_TELEMETRY_PG_POOL_MAX=raw))
|
||||
|
||||
@pytest.mark.parametrize("raw", ["0", "-1"])
|
||||
def test_non_positive_write_timeout_rejected_naming_the_env_key(self, raw):
|
||||
"""预算 0 = 每一行都当场超预算;不设预算不是这个键的写法。"""
|
||||
with pytest.raises(ValueError, match="PGW_TELEMETRY_PG_WRITE_TIMEOUT_S"):
|
||||
GatewaySettings.from_env("LLM", env=self._pg_env(PGW_TELEMETRY_PG_WRITE_TIMEOUT_S=raw))
|
||||
|
||||
def test_non_numeric_rejected_naming_the_env_key(self):
|
||||
with pytest.raises(ValueError, match="PGW_TELEMETRY_PG_POOL_MAX"):
|
||||
GatewaySettings.from_env("LLM", env=self._pg_env(PGW_TELEMETRY_PG_POOL_MAX="many"))
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value"),
|
||||
[("telemetry_pg_pool_max", 0), ("telemetry_pg_write_timeout_s", 0.0)],
|
||||
)
|
||||
def test_direct_construction_and_replace_are_validated_too(self, field, value):
|
||||
"""env 路只覆盖 from_env;直接构造与 replace 是同等官方的装配路(与 text_cap 同款)。"""
|
||||
base = GatewaySettings.from_env("LLM", env=_env())
|
||||
with pytest.raises(ValueError, match=field):
|
||||
dataclasses.replace(base, **{field: value})
|
||||
|
||||
def test_values_reach_the_recorder(self):
|
||||
"""配置到 recorder 之间不得断链——两个键唯一的作用就是抵达那里。"""
|
||||
from polygateway.client import _build_telemetry
|
||||
|
||||
settings = GatewaySettings.from_env(
|
||||
"LLM",
|
||||
env=self._pg_env(PGW_TELEMETRY_PG_POOL_MAX="7", PGW_TELEMETRY_PG_WRITE_TIMEOUT_S="2.5"),
|
||||
)
|
||||
recorder = _build_telemetry(settings)
|
||||
assert recorder._pool_max == 7
|
||||
assert recorder._write_timeout_s == 2.5
|
||||
|
||||
|
||||
class TestOcrSettings:
|
||||
"""M3 OcrSettings(设计 §3.4): 复用 GatewaySettings,无 OCR 专用键。"""
|
||||
|
||||
|
||||
+160
-15
@@ -706,6 +706,13 @@ class TestSQLiteSchemaMode:
|
||||
SQLiteRecorder(tmp_path / "t.db") # type: ignore[call-arg]
|
||||
|
||||
|
||||
# 假池用例的池上限与写入预算: 两者都是必填 keyword-only(缺省只写在 config 一处),
|
||||
# 本文件统一取这一份,免得每个 helper 各写一个数字
|
||||
_TEST_POOL_MAX = 2
|
||||
_TEST_WRITE_TIMEOUT_S = 5.0
|
||||
_PG_DSN = "postgresql://u:p@h:5432/polygateway"
|
||||
|
||||
|
||||
class _FakePgConn:
|
||||
"""记录执行过的语句;可让 ALTER/CREATE/探测抛错以模拟权限不足与抖动。
|
||||
|
||||
@@ -720,15 +727,20 @@ class _FakePgConn:
|
||||
fail_alter: bool = False,
|
||||
fail_create: bool = False,
|
||||
probe_errors: int = 0,
|
||||
hang_insert: bool = False,
|
||||
):
|
||||
self.existing = existing
|
||||
self.fail_alter = fail_alter
|
||||
self.fail_create = fail_create
|
||||
self.probe_errors = probe_errors
|
||||
# 只挂 INSERT: 准备期照常完成,挂住的才是业务路径上那次内联 await
|
||||
self.hang_insert = hang_insert
|
||||
self.statements: list[str] = []
|
||||
|
||||
async def execute(self, sql, *args):
|
||||
self.statements.append(sql)
|
||||
if sql.startswith("INSERT INTO") and self.hang_insert:
|
||||
await asyncio.sleep(3600)
|
||||
if sql.startswith("ALTER TABLE") and self.fail_alter:
|
||||
raise RuntimeError("must be owner of table llm_calls")
|
||||
if sql.lstrip().startswith("CREATE TABLE"):
|
||||
@@ -749,20 +761,33 @@ class _FakePgConn:
|
||||
|
||||
|
||||
class _FakePgPool:
|
||||
def __init__(self, conn):
|
||||
"""假池: 记 acquire/release 的配对次数与实参 timeout(issue #15 T3)。
|
||||
|
||||
形状跟着被测代码走: recorder 改用**显式** `acquire(timeout=)` /
|
||||
`release(conn, timeout=)`,不再用 `async with pool.acquire()`(那条路
|
||||
的 shielded release 会把写入的真实上界撑成 ≈2× 预算,设计 §3.1),
|
||||
故这里也不再提供上下文管理器。
|
||||
"""
|
||||
|
||||
def __init__(self, conn, *, hang_acquire: bool = False):
|
||||
self._conn = conn
|
||||
self.hang_acquire = hang_acquire
|
||||
self.acquired = 0
|
||||
self.released = 0
|
||||
self.acquire_timeouts: list[object] = []
|
||||
self.release_timeouts: list[object] = []
|
||||
|
||||
def acquire(self):
|
||||
conn = self._conn
|
||||
async def acquire(self, *, timeout=None):
|
||||
self.acquire_timeouts.append(timeout)
|
||||
if self.hang_acquire:
|
||||
await asyncio.sleep(3600)
|
||||
self.acquired += 1
|
||||
return self._conn
|
||||
|
||||
class _Ctx:
|
||||
async def __aenter__(self):
|
||||
return conn
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
return False
|
||||
|
||||
return _Ctx()
|
||||
async def release(self, conn, *, timeout=None):
|
||||
assert conn is self._conn
|
||||
self.release_timeouts.append(timeout)
|
||||
self.released += 1
|
||||
|
||||
|
||||
class TestPostgresBackfillDiscipline:
|
||||
@@ -785,7 +810,11 @@ class TestPostgresBackfillDiscipline:
|
||||
from polygateway.telemetry.postgres import PostgresRecorder
|
||||
|
||||
return PostgresRecorder(
|
||||
"postgresql://u:p@h:5432/polygateway", pool=_FakePgPool(conn), auto_migrate=True
|
||||
"postgresql://u:p@h:5432/polygateway",
|
||||
pool=_FakePgPool(conn),
|
||||
auto_migrate=True,
|
||||
pool_max=_TEST_POOL_MAX,
|
||||
write_timeout_s=_TEST_WRITE_TIMEOUT_S,
|
||||
)
|
||||
|
||||
async def test_alter_failure_does_not_disable_the_recorder(self):
|
||||
@@ -841,7 +870,11 @@ class TestPostgresTableProbe:
|
||||
from polygateway.telemetry.postgres import PostgresRecorder
|
||||
|
||||
return PostgresRecorder(
|
||||
"postgresql://u:p@h:5432/polygateway", pool=_FakePgPool(conn), auto_migrate=True
|
||||
"postgresql://u:p@h:5432/polygateway",
|
||||
pool=_FakePgPool(conn),
|
||||
auto_migrate=True,
|
||||
pool_max=_TEST_POOL_MAX,
|
||||
write_timeout_s=_TEST_WRITE_TIMEOUT_S,
|
||||
)
|
||||
|
||||
def _created(self, conn):
|
||||
@@ -903,7 +936,11 @@ class TestPostgresSchemaMode:
|
||||
from polygateway.telemetry.postgres import PostgresRecorder
|
||||
|
||||
return PostgresRecorder(
|
||||
"postgresql://u:p@h:5432/polygateway", pool=_FakePgPool(conn), auto_migrate=auto_migrate
|
||||
"postgresql://u:p@h:5432/polygateway",
|
||||
pool=_FakePgPool(conn),
|
||||
auto_migrate=auto_migrate,
|
||||
pool_max=_TEST_POOL_MAX,
|
||||
write_timeout_s=_TEST_WRITE_TIMEOUT_S,
|
||||
)
|
||||
|
||||
async def test_manual_mode_trims_the_insert_instead_of_altering(self, captured_warnings):
|
||||
@@ -1852,7 +1889,11 @@ class TestPostgresStatusVisibility:
|
||||
from polygateway.telemetry.postgres import PostgresRecorder
|
||||
|
||||
return PostgresRecorder(
|
||||
"postgresql://u:p@h:5432/polygateway", pool=_FakePgPool(conn), auto_migrate=True
|
||||
"postgresql://u:p@h:5432/polygateway",
|
||||
pool=_FakePgPool(conn),
|
||||
auto_migrate=True,
|
||||
pool_max=_TEST_POOL_MAX,
|
||||
write_timeout_s=_TEST_WRITE_TIMEOUT_S,
|
||||
)
|
||||
|
||||
async def test_unusable_table_shows_up_in_the_status(self, captured_warnings):
|
||||
@@ -1869,3 +1910,107 @@ class TestPostgresStatusVisibility:
|
||||
await _record_minimal(recorder)
|
||||
status = recorder.telemetry_status
|
||||
assert status.degraded is False and status.dropped_rows == 0
|
||||
|
||||
|
||||
class TestPostgresPoolResourceSemantics:
|
||||
"""issue #15 A 组: 库必须自己声明池的资源占用,并给写入一个硬预算。
|
||||
|
||||
建池这条路在本 issue 之前**零测试覆盖**(全部 PG 用例都经 `pool=` 注入,
|
||||
走的是外部池分支),`min_size=10` 因此潜伏至今: 4 个 client × 10 = 40 条
|
||||
常驻连接专用于写遥测,共享实例余量不足时先倒下的必然是它。
|
||||
"""
|
||||
|
||||
def _recorder(self, pool, **overrides):
|
||||
from polygateway.telemetry.postgres import PostgresRecorder
|
||||
|
||||
kwargs: dict[str, object] = {
|
||||
"auto_migrate": True,
|
||||
"pool_max": _TEST_POOL_MAX,
|
||||
"write_timeout_s": _TEST_WRITE_TIMEOUT_S,
|
||||
}
|
||||
kwargs.update(overrides)
|
||||
return PostgresRecorder(_PG_DSN, pool=pool, **kwargs)
|
||||
|
||||
async def test_pool_is_created_without_preconnecting(self, monkeypatch):
|
||||
"""**主回归钉子**: `min_size=0` 且 `max_size` 取配置值。
|
||||
|
||||
`min_size` 的语义是"预连接"而非"下限"(asyncpg `pool.py:457` 为 0 时
|
||||
一条连接都不连),故它是"建池要么全有要么全无"这个脆点的唯一来源。
|
||||
继承第三方默认值等于库对自己的资源占用不表态(P4),本条防的就是回归。
|
||||
"""
|
||||
import asyncpg
|
||||
|
||||
from polygateway.telemetry.postgres import PostgresRecorder
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
pool = _FakePgPool(_FakePgConn(list(_EXPECTED_COLUMNS)))
|
||||
|
||||
async def fake_create_pool(dsn, **kwargs):
|
||||
captured["dsn"] = dsn
|
||||
captured.update(kwargs)
|
||||
return pool
|
||||
|
||||
monkeypatch.setattr(asyncpg, "create_pool", fake_create_pool)
|
||||
recorder = PostgresRecorder(_PG_DSN, auto_migrate=True, pool_max=3, write_timeout_s=2.5)
|
||||
await _record_minimal(recorder)
|
||||
|
||||
assert captured["min_size"] == 0
|
||||
assert captured["max_size"] == 3
|
||||
# connect 与单条语句都在同一份写入预算内,不留继承来的 10s 默认值
|
||||
assert captured["timeout"] == 2.5
|
||||
assert captured["command_timeout"] == 2.5
|
||||
|
||||
async def test_acquire_gets_an_explicit_timeout(self):
|
||||
"""`pool.acquire()` 无参 = 无限等(asyncpg 缺省 `timeout=None`)。
|
||||
|
||||
池满时那是挂在业务路径上的无限期 await,`max_size` 收到个位数后必现。
|
||||
"""
|
||||
pool = _FakePgPool(_FakePgConn(list(_EXPECTED_COLUMNS)))
|
||||
await _record_minimal(self._recorder(pool))
|
||||
assert pool.acquire_timeouts # 准备期与写入期各一次
|
||||
assert all(t == _TEST_WRITE_TIMEOUT_S for t in pool.acquire_timeouts)
|
||||
|
||||
async def test_write_budget_drops_the_row_instead_of_blocking_the_caller(
|
||||
self, captured_warnings
|
||||
):
|
||||
"""整次写入有硬预算: 后端挂住时丢一行,绝不把业务调用拖在那里。"""
|
||||
pool = _FakePgPool(_FakePgConn(list(_EXPECTED_COLUMNS), hang_insert=True))
|
||||
recorder = self._recorder(pool, write_timeout_s=0.05)
|
||||
loop = asyncio.get_running_loop()
|
||||
started = loop.time()
|
||||
# 挂死就当场红,而不是把整个套件拖到 CI 超时
|
||||
await asyncio.wait_for(_record_minimal(recorder), timeout=5)
|
||||
assert loop.time() - started < 1.0
|
||||
assert any("预算" in m for m in captured_warnings)
|
||||
assert recorder.telemetry_status.dropped_rows == 1
|
||||
|
||||
async def test_release_is_paired_even_when_the_budget_fires(self):
|
||||
"""预算取消发生在 execute 上,连接照样要还回去——否则池被慢查询吃干。"""
|
||||
pool = _FakePgPool(_FakePgConn(list(_EXPECTED_COLUMNS), hang_insert=True))
|
||||
recorder = self._recorder(pool, write_timeout_s=0.05)
|
||||
await asyncio.wait_for(_record_minimal(recorder), timeout=5)
|
||||
assert pool.acquired == 2 # 准备期一次 + 写入一次
|
||||
assert pool.released == pool.acquired
|
||||
# 归还有独立的小上限: 复用写入预算就等于允许再等一个预算(设计 §3.1)
|
||||
assert all(t is not None and t < _TEST_WRITE_TIMEOUT_S for t in pool.release_timeouts)
|
||||
|
||||
async def test_acquire_timeout_drops_the_row_without_leaking(self, captured_warnings):
|
||||
"""取连接本身挂住时同样丢行;没拿到的连接不许伪造一次 release。"""
|
||||
pool = _FakePgPool(_FakePgConn(list(_EXPECTED_COLUMNS)), hang_acquire=True)
|
||||
recorder = self._recorder(pool, write_timeout_s=0.05)
|
||||
await asyncio.wait_for(_record_minimal(recorder), timeout=5)
|
||||
assert pool.acquired == 0 and pool.released == 0
|
||||
assert captured_warnings
|
||||
|
||||
async def test_external_cancellation_is_not_swallowed_as_a_timeout(self):
|
||||
"""铁律"取消可穿透": `asyncio.timeout` 只把**自己**触发的 cancel 转成
|
||||
TimeoutError,外部取消必须照常以 CancelledError 冒出去。
|
||||
"""
|
||||
pool = _FakePgPool(_FakePgConn(list(_EXPECTED_COLUMNS), hang_insert=True))
|
||||
recorder = self._recorder(pool, write_timeout_s=30.0)
|
||||
task = asyncio.create_task(_record_minimal(recorder))
|
||||
await asyncio.sleep(0.05) # 让它跑到挂住的那次 INSERT
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert pool.released == pool.acquired # 取消路径上也不许泄漏连接
|
||||
|
||||
Reference in New Issue
Block a user