fix: make closing the telemetry pool bounded and final

Closing was the last unbounded wait on the shutdown path: asyncpg's
Pool.close() awaits wait_until_released() on every holder, so a single
in-flight connection parks the caller forever (60s only buys a warning).
It now runs under asyncio.wait_for and terminates the pool on timeout;
external cancellation still propagates untouched.

Closing is also final now. Clearing _pool used to leave the recorder free
to build a fresh pool on the next write - worse in the injected case,
where the owner believes it still holds every connection while the
recorder quietly opened its own. Recovery is a runtime concern (cooldown
retry), not a side effect of shutdown, so writes after aclose short out
and count the dropped row with a reason of their own.

Also covers the release/terminate fallback left untested by the pool
work: the fake pool needed for the close cases makes it nearly free.
This commit is contained in:
2026-08-24 09:53:23 -04:00
parent 84c2cc11a4
commit bc071c6f41
2 changed files with 217 additions and 14 deletions
+50 -13
View File
@@ -47,6 +47,13 @@ _TABLE_EXISTS = "SELECT to_regclass('llm_calls')"
# asyncpg 会在下次 acquire 时补一条新连接
_RELEASE_TIMEOUT_S = 1.0
# 关闭池的独立上限(issue #15)。**不**复用写入预算: 关闭跑在收尾路径而非业务
# 路径上,给它一个略宽的固定额度即可,但必须**有界**——asyncpg 的
# `Pool.close()` 会 await 每个 holder 的 `wait_until_released()`,in-flight
# 连接不归还就无限等(`pool.py:939-948, 961-972`,60s 只发一条 warning),
# 其 docstring 自己写着 "advisable to use asyncio.wait_for to set a timeout"
_CLOSE_TIMEOUT_S = 5.0
# 探测现有列;尊重 search_path(to_regclass 按当前 search_path 解析)
_EXISTING_COLUMNS = (
"SELECT attname FROM pg_attribute "
@@ -99,6 +106,7 @@ class PostgresRecorder:
self._insert = insert_sql("postgres", COLUMNS)
self._schema_ready = False
self._failed = False # 结构性降级标志: 置位后所有写入短路
self._closed = False # 关了就是关了: 置位后写入短路且**不重建池**
# 降级的可编程出口与节流日志;`_failed` 与它并存是 issue #15 的过渡态,
# 判据改造(冷却自愈)落地时状态收归 tracker 一处
self._status = TelemetryStatusTracker(backend="postgres", now=now)
@@ -110,13 +118,18 @@ class PostgresRecorder:
return self._status.snapshot()
async def _ensure_ready(self) -> asyncpg.Pool | None:
"""lazy 建池+备表;判死只认「确定写不进去」(issue #9),其余失败都留活路。"""
if self._failed:
"""lazy 建池+备表;判死只认「确定写不进去」(issue #9),其余失败都留活路。
`_closed` 在锁内**必须复查**: 等锁期间发生的 `aclose` 否则会被这次
等待"绕过",等到锁时照旧建出一个没人负责关的池(注入档更隐蔽——
注入方以为自己管着全部连接,实际早已不是)。
"""
if self._closed or self._failed:
return None
if self._schema_ready:
return self._pool
async with self._init_lock:
if self._failed:
if self._closed or self._failed:
return None
if self._schema_ready:
return self._pool
@@ -317,8 +330,10 @@ class PostgresRecorder:
"""
pool = await self._ensure_ready()
if pool is None:
# 降级期间静默 return 就是 issue #15 的破口: 丢行必须计数且节流出声
self._status.record_drop("遥测已降级")
# 降级期间静默 return 就是 issue #15 的破口: 丢行必须计数且节流出声
# 关闭后的丢行同样要计数,但原因不是降级——两者的处置完全不同
# (一个等自愈,一个是调用方自己关了却还在写)
self._status.record_drop("遥测已关闭" if self._closed else "遥测已降级")
return
row = tuple(fields[col] for col in self._columns)
conn = await pool.acquire(timeout=self._write_timeout_s)
@@ -346,21 +361,43 @@ class PostgresRecorder:
# 含 TimeoutError: 归还超时与归还出错的处置相同——断开而不是留一条
# 状态不明的连接在池里(asyncpg 的 reset 失败路径也是这么做的)
logger.warning("Postgres 遥测连接归还失败(强制断开): {}", exc)
self._terminate(conn)
self._terminate(conn, label="连接")
@staticmethod
def _terminate(conn: object) -> None:
"""强制断开一条连接;断开本身再失败也只记 warning(遥测绝不冒泡)。"""
def _terminate(target: object, *, label: str) -> None:
"""强制断开一条连接或整个池;断开本身再失败也只记 warning(遥测绝不冒泡)。
`label` 必填(不给默认值): 两个调用点的诊断价值全在"拆的是哪一层",
默认值只会让其中一处悄悄报错成另一处。
"""
try:
conn.terminate() # type: ignore[attr-defined]
target.terminate() # type: ignore[attr-defined]
except asyncio.CancelledError:
raise
except Exception as exc:
logger.warning("Postgres 遥测连接断开失败(交给自行回收): {}", exc)
logger.warning("Postgres 遥测{}断开失败(交给上层自行回收): {}", label, exc)
async def aclose(self) -> None:
"""幂等关闭自建池;注入的池归注入方管理。"""
"""幂等关闭自建池;**关了就是关了**,此后写入短路且不复活。注入的池归注入方管理。
取消"关完还能自己重建池"的灰色状态(设计 §3.2 第 4 点): 关闭是所有权的
终结,而恢复是运行时行为(冷却重试),不该是关闭动作的副作用。
**关闭动作本身也有界**: `Pool.close()` 会 await 每个 holder 的
`wait_until_released()`,in-flight 连接不归还就无限等——收尾路径上照样是
"遥测拖垮业务"。超时即 `terminate()` 强拆: 关闭已在进行,留着一个关不掉的池
既不会自愈也没人再来收。外部取消照常穿透(铁律),不当成一次关闭超时。
"""
self._closed = True
pool, self._pool = self._pool, None
self._schema_ready = False
if pool is not None and not self._external_pool:
await pool.close()
if pool is None or self._external_pool:
return
try:
await asyncio.wait_for(pool.close(), timeout=_CLOSE_TIMEOUT_S)
except asyncio.CancelledError:
raise
except Exception as exc:
# 含 TimeoutError: 关不掉与关出错的处置相同——强拆
logger.warning("Postgres 遥测池关闭失败(强制断开): {}", exc)
self._terminate(pool, label="")