refactor: make the telemetry schema a single source of truth

DDL, column order and backfill statements lived twice, once in each
recorder. A public telemetry_schema_sql() would have made three copies,
and the drift shows up downstream as "I ran the printed SQL and the
library still reports a missing column".

Move both DDLs, both backfill lists and the 24 INSERT fields into
telemetry/schema.py verbatim; the recorders now import them and build
_INSERT through insert_sql(backend, COLUMNS) at import time. The
generated statements are byte-identical to the previous constants, so
runtime behaviour is unchanged (the postgres conflict target stays
bound to call_id for now).

insert_sql() validates its columns against COLUMNS: from the next task
on those names come from database probing, not from a constant, so the
subset check is the gate on the only injection surface. The new
telemetry_schema_sql() prints a paste-ready migration script; its
postgres backfill deliberately uses ADD COLUMN IF NOT EXISTS while the
library's own statements do not, because that form takes an ACCESS
EXCLUSIVE lock even when the column exists. Both variants are derived
from one declaration list so their column sets cannot drift.
This commit is contained in:
2026-08-19 11:18:06 -04:00
parent e9adb36577
commit 1471e0a2c6
5 changed files with 367 additions and 182 deletions
+6 -82
View File
@@ -21,56 +21,11 @@ from typing import TYPE_CHECKING
from loguru import logger
from polygateway.telemetry.schema import COLUMNS, PG_BACKFILL, PG_DDL, insert_sql
if TYPE_CHECKING:
import asyncpg
_DDL = """
CREATE TABLE IF NOT EXISTS llm_calls (
call_id TEXT PRIMARY KEY,
parent_call_id TEXT,
session_id TEXT,
model TEXT NOT NULL,
provider TEXT NOT NULL,
source_name TEXT NOT NULL,
messages TEXT NOT NULL,
response TEXT NOT NULL,
thinking TEXT NOT NULL DEFAULT '',
prompt_tokens INTEGER NOT NULL,
completion_tokens INTEGER NOT NULL,
usage_source TEXT NOT NULL,
latency_ms INTEGER NOT NULL,
ttft_ms DOUBLE PRECISION,
max_inter_token_ms DOUBLE PRECISION,
cache_hit BOOLEAN NOT NULL DEFAULT FALSE,
error TEXT,
cost DOUBLE PRECISION,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
cached_prompt_tokens INTEGER,
model_reported TEXT,
sampling TEXT,
reasoning_tokens INTEGER,
tenant_id TEXT NOT NULL DEFAULT '',
meta JSONB NOT NULL DEFAULT '{}'::jsonb
);
"""
# 新列排在 created_at 之后: 与旧表 ALTER 追加的位置一致(见 sqlite.py 同款注释)
_BACKFILL = (
("cached_prompt_tokens", "ALTER TABLE llm_calls ADD COLUMN cached_prompt_tokens INTEGER"),
("model_reported", "ALTER TABLE llm_calls ADD COLUMN model_reported TEXT"),
("sampling", "ALTER TABLE llm_calls ADD COLUMN sampling TEXT"),
("reasoning_tokens", "ALTER TABLE llm_calls ADD COLUMN reasoning_tokens INTEGER"),
# 两个默认值都是非易失常量,PG 11+ 只改 catalog 不重写全表,故大表补列亦是秒级
(
"tenant_id",
"ALTER TABLE llm_calls ADD COLUMN tenant_id TEXT NOT NULL DEFAULT ''",
),
(
"meta",
"ALTER TABLE llm_calls ADD COLUMN meta JSONB NOT NULL DEFAULT '{}'::jsonb",
),
)
# 探测表是否存在;不需要任何权限,且与 INSERT 走同一套 search_path 解析
_TABLE_EXISTS = "SELECT to_regclass('llm_calls')"
@@ -80,38 +35,7 @@ _EXISTING_COLUMNS = (
"WHERE attrelid = to_regclass('llm_calls') AND attnum > 0 AND NOT attisdropped"
)
_COLUMNS = (
"call_id",
"parent_call_id",
"session_id",
"model",
"provider",
"source_name",
"messages",
"response",
"thinking",
"prompt_tokens",
"completion_tokens",
"usage_source",
"latency_ms",
"ttft_ms",
"max_inter_token_ms",
"cache_hit",
"error",
"cost",
"cached_prompt_tokens",
"model_reported",
"sampling",
"reasoning_tokens",
"tenant_id",
"meta",
)
_INSERT = (
f"INSERT INTO llm_calls ({', '.join(_COLUMNS)}) "
f"VALUES ({', '.join(f'${i + 1}' for i in range(len(_COLUMNS)))}) "
"ON CONFLICT (call_id) DO NOTHING"
)
_INSERT = insert_sql("postgres", COLUMNS)
class PostgresRecorder:
@@ -200,7 +124,7 @@ class PostgresRecorder:
await self._backfill_columns(conn) # 旧表可能缺列;失败只逐行降级
return True
try:
await conn.execute(_DDL) # type: ignore[attr-defined]
await conn.execute(PG_DDL) # type: ignore[attr-defined]
except asyncio.CancelledError:
raise
except Exception as exc:
@@ -223,7 +147,7 @@ class PostgresRecorder:
"""
try:
existing = {row["attname"] for row in await conn.fetch(_EXISTING_COLUMNS)} # type: ignore[attr-defined]
for column, statement in _BACKFILL:
for column, statement in PG_BACKFILL:
if column not in existing:
await conn.execute(statement) # type: ignore[attr-defined]
except asyncio.CancelledError:
@@ -236,7 +160,7 @@ class PostgresRecorder:
pool = await self._ensure_ready()
if pool is None:
return
row = tuple(fields[col] for col in _COLUMNS)
row = tuple(fields[col] for col in COLUMNS)
try:
async with pool.acquire() as conn:
await conn.execute(_INSERT, *row)
+228
View File
@@ -0,0 +1,228 @@
"""遥测表 `llm_calls` 的 schema 单一事实源: 列序、两端 DDL、补列语句与 INSERT 构造。
两个 recorder(`sqlite.py` / `postgres.py`)与公共函数 `telemetry_schema_sql` 共用本模块。
收敛的理由是**正确性**而非整洁: 打印给下游的 SQL 必须与库真正执行的 DDL 同源——常量在
多处各存一份必然漂移,而漂移的表现是"下游照打印的 SQL 建完表,库仍报缺列"
**`COLUMNS` 是 INSERT 字段序,不是物理列序**: 数据库自填的 `created_at` 不在其中(它带
`DEFAULT now()` / `datetime('now')`,库从不显式写它)。物理表列 = 24 个 INSERT 字段 +
`created_at` = 25;列数断言一律按物理列数写,两套口径混用是最易错处。
本模块只依赖标准库: `telemetry/` 与 `backends/`、`transports/`、`structured/` 同层且
互不依赖(import-linter 契约执法)。
"""
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Sequence
TABLE = "llm_calls"
# 支持的后端;`insert_sql` / `telemetry_schema_sql` 的取值域
_BACKENDS = ("sqlite", "postgres")
SQLITE_DDL = """
CREATE TABLE IF NOT EXISTS llm_calls (
call_id TEXT PRIMARY KEY,
parent_call_id TEXT,
session_id TEXT,
model TEXT NOT NULL,
provider TEXT NOT NULL,
source_name TEXT NOT NULL,
messages TEXT NOT NULL,
response TEXT NOT NULL,
thinking TEXT NOT NULL DEFAULT '',
prompt_tokens INTEGER NOT NULL,
completion_tokens INTEGER NOT NULL,
usage_source TEXT NOT NULL,
latency_ms INTEGER NOT NULL,
ttft_ms REAL,
max_inter_token_ms REAL,
cache_hit INTEGER NOT NULL DEFAULT 0,
error TEXT,
cost REAL,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
cached_prompt_tokens INTEGER,
model_reported TEXT,
sampling TEXT,
reasoning_tokens INTEGER,
tenant_id TEXT NOT NULL DEFAULT '',
meta TEXT NOT NULL DEFAULT '{}'
);
"""
PG_DDL = """
CREATE TABLE IF NOT EXISTS llm_calls (
call_id TEXT PRIMARY KEY,
parent_call_id TEXT,
session_id TEXT,
model TEXT NOT NULL,
provider TEXT NOT NULL,
source_name TEXT NOT NULL,
messages TEXT NOT NULL,
response TEXT NOT NULL,
thinking TEXT NOT NULL DEFAULT '',
prompt_tokens INTEGER NOT NULL,
completion_tokens INTEGER NOT NULL,
usage_source TEXT NOT NULL,
latency_ms INTEGER NOT NULL,
ttft_ms DOUBLE PRECISION,
max_inter_token_ms DOUBLE PRECISION,
cache_hit BOOLEAN NOT NULL DEFAULT FALSE,
error TEXT,
cost DOUBLE PRECISION,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
cached_prompt_tokens INTEGER,
model_reported TEXT,
sampling TEXT,
reasoning_tokens INTEGER,
tenant_id TEXT NOT NULL DEFAULT '',
meta JSONB NOT NULL DEFAULT '{}'::jsonb
);
"""
# 新列必须排在 created_at 之后: 旧表只能经 ALTER 追加到末尾,新建库若把它们
# 插在前面,两条路径的物理列序会分叉(列序断言测试无合规修法)。
SQLITE_BACKFILL = (
("cached_prompt_tokens", "INTEGER"),
("model_reported", "TEXT"),
("sampling", "TEXT"),
("reasoning_tokens", "INTEGER"),
# NOT NULL 补列必须带非 NULL 常量默认值,否则 SQLite 直接拒绝该 ALTER
# ("Cannot add a NOT NULL column with default value NULL"),补列全盘失败。
("tenant_id", "TEXT NOT NULL DEFAULT ''"),
("meta", "TEXT NOT NULL DEFAULT '{}'"),
)
# PG 补列的列定义。语句由此派生成两份文本(见下),使"库内执行的那份"与"打印给
# 下游的那份"的列集合与列定义**无法分叉**——本模块存在的全部理由就是不许漂移。
_PG_BACKFILL_DECLS = (
("cached_prompt_tokens", "INTEGER"),
("model_reported", "TEXT"),
("sampling", "TEXT"),
("reasoning_tokens", "INTEGER"),
# 两个默认值都是非易失常量,PG 11+ 只改 catalog 不重写全表,故大表补列亦是秒级
("tenant_id", "TEXT NOT NULL DEFAULT ''"),
("meta", "JSONB NOT NULL DEFAULT '{}'::jsonb"),
)
# 新列排在 created_at 之后: 与旧表 ALTER 追加的位置一致(见 SQLITE_BACKFILL 同款注释)。
# **库内执行的这份有意不带 `IF NOT EXISTS`**: PG 对它即便列已存在也会先取 ACCESS
# EXCLUSIVE 锁,而遥测是业务路径上的内联 await,故库侧一律"先探测后 ALTER"
# (postgres.py `_backfill_columns` 记有实测)。给人执行的那份见 `telemetry_schema_sql`。
PG_BACKFILL = tuple(
(column, f"ALTER TABLE {TABLE} ADD COLUMN {column} {decl}")
for column, decl in _PG_BACKFILL_DECLS
)
COLUMNS = (
"call_id",
"parent_call_id",
"session_id",
"model",
"provider",
"source_name",
"messages",
"response",
"thinking",
"prompt_tokens",
"completion_tokens",
"usage_source",
"latency_ms",
"ttft_ms",
"max_inter_token_ms",
"cache_hit",
"error",
"cost",
"cached_prompt_tokens",
"model_reported",
"sampling",
"reasoning_tokens",
"tenant_id",
"meta",
)
_COLUMN_SET = frozenset(COLUMNS)
def insert_sql(backend: str, columns: Sequence[str]) -> str:
"""按给定列构造 INSERT;列必须是 `COLUMNS` 的子集,否则 ValueError。
子集校验是**注入面的闸**: 列名来自数据库探测结果,不是常量,不校验就等于把外部
字符串拼进 SQL(占位符只保护值,保护不了列名)。sqlite 用 `?`、postgres 用 `$n`,
两端的重复键处理都不绑定具体约束名(`INSERT OR IGNORE` / `ON CONFLICT`)。
Args:
backend: `"sqlite"` 或 `"postgres"`。
columns: 要写入的列,顺序即占位符顺序(调用方须按同序取值)。
Returns:
完整的 INSERT 语句。
Raises:
ValueError: backend 不在取值域内,或 columns 含 `COLUMNS` 之外的列名。
"""
if backend not in _BACKENDS:
raise ValueError(f"未知遥测后端 {backend!r}: 只支持 {list(_BACKENDS)}")
selected = tuple(columns)
unknown = [column for column in selected if column not in _COLUMN_SET]
if unknown:
raise ValueError(f"列名不在遥测 schema 内(拒绝拼进 SQL): {unknown}")
names = ", ".join(selected)
if backend == "sqlite":
placeholders = ", ".join("?" for _ in selected)
return f"INSERT OR IGNORE INTO {TABLE} ({names}) VALUES ({placeholders})"
placeholders = ", ".join(f"${i + 1}" for i in range(len(selected)))
return f"INSERT INTO {TABLE} ({names}) VALUES ({placeholders}) ON CONFLICT (call_id) DO NOTHING"
def telemetry_schema_sql(backend: str) -> str:
"""返回可直接粘进迁移文件的完整脚本(建表 + 各补列语句 + 注释)。
给不愿意让库在自己的生产表上发 DDL 的下游用: 输出与库运行时执行的 DDL 同源,
照它建完表,库探测到的列就是齐的。
**补列语句与库内执行的那份是两套文本,不是一份**: 这份给人执行,必须可重复执行,
故 PG 变体带 `ADD COLUMN IF NOT EXISTS`(它会先取 ACCESS EXCLUSIVE 锁,但执行时机
由 DBA 自己挑,锁风险可控);库内那份不带,靠先探测后 ALTER 规避锁。SQLite 没有
`ADD COLUMN IF NOT EXISTS` 语法,只能以注释交代"仅当该列不存在时执行"
Args:
backend: `"sqlite"` 或 `"postgres"`。
Returns:
含注释的完整 SQL 脚本。
Raises:
ValueError: backend 不在取值域内。
"""
if backend not in _BACKENDS:
raise ValueError(f"未知遥测后端 {backend!r}: 只支持 {list(_BACKENDS)}")
if backend == "sqlite":
ddl = SQLITE_DDL
notes = (
f"-- 旧表补列(库升级后新增的列)。SQLite 无 ADD COLUMN IF NOT EXISTS 语法,\n"
f"-- 以下每条**仅当该列不存在时执行**(先 PRAGMA table_info({TABLE}) 对照)。"
)
alters = [
f"ALTER TABLE {TABLE} ADD COLUMN {column} {decl};" for column, decl in SQLITE_BACKFILL
]
else:
ddl = PG_DDL
notes = (
"-- 旧表补列(库升级后新增的列)。带 IF NOT EXISTS,整段可重复执行;\n"
"-- 注意它即便列已存在也会先取 ACCESS EXCLUSIVE 锁,请挑低峰执行。"
)
alters = [
f"ALTER TABLE {TABLE} ADD COLUMN IF NOT EXISTS {column} {decl};"
for column, decl in _PG_BACKFILL_DECLS
]
header = (
f"-- PolyGateway 遥测表 {TABLE}({backend})\n"
f'-- 由 polygateway.telemetry_schema_sql("{backend}") 生成,与库运行时执行的 DDL 同源。\n'
"-- 新建库执行整段;已有旧表则建表语句自动跳过,只需关注下方补列语句。"
)
return "\n".join([header, "", ddl.strip(), "", notes, *alters, ""])
+5 -76
View File
@@ -22,80 +22,9 @@ from pathlib import Path
from loguru import logger
_DDL = """
CREATE TABLE IF NOT EXISTS llm_calls (
call_id TEXT PRIMARY KEY,
parent_call_id TEXT,
session_id TEXT,
model TEXT NOT NULL,
provider TEXT NOT NULL,
source_name TEXT NOT NULL,
messages TEXT NOT NULL,
response TEXT NOT NULL,
thinking TEXT NOT NULL DEFAULT '',
prompt_tokens INTEGER NOT NULL,
completion_tokens INTEGER NOT NULL,
usage_source TEXT NOT NULL,
latency_ms INTEGER NOT NULL,
ttft_ms REAL,
max_inter_token_ms REAL,
cache_hit INTEGER NOT NULL DEFAULT 0,
error TEXT,
cost REAL,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
cached_prompt_tokens INTEGER,
model_reported TEXT,
sampling TEXT,
reasoning_tokens INTEGER,
tenant_id TEXT NOT NULL DEFAULT '',
meta TEXT NOT NULL DEFAULT '{}'
);
"""
from polygateway.telemetry.schema import COLUMNS, SQLITE_BACKFILL, SQLITE_DDL, insert_sql
# 新列必须排在 created_at 之后: 旧表只能经 ALTER 追加到末尾,新建库若把它们
# 插在前面,两条路径的物理列序会分叉(列序断言测试无合规修法)。
_BACKFILL_COLUMNS = (
("cached_prompt_tokens", "INTEGER"),
("model_reported", "TEXT"),
("sampling", "TEXT"),
("reasoning_tokens", "INTEGER"),
# NOT NULL 补列必须带非 NULL 常量默认值,否则 SQLite 直接拒绝该 ALTER
# ("Cannot add a NOT NULL column with default value NULL"),补列全盘失败。
("tenant_id", "TEXT NOT NULL DEFAULT ''"),
("meta", "TEXT NOT NULL DEFAULT '{}'"),
)
_COLUMNS = (
"call_id",
"parent_call_id",
"session_id",
"model",
"provider",
"source_name",
"messages",
"response",
"thinking",
"prompt_tokens",
"completion_tokens",
"usage_source",
"latency_ms",
"ttft_ms",
"max_inter_token_ms",
"cache_hit",
"error",
"cost",
"cached_prompt_tokens",
"model_reported",
"sampling",
"reasoning_tokens",
"tenant_id",
"meta",
)
_INSERT = (
f"INSERT OR IGNORE INTO llm_calls ({', '.join(_COLUMNS)}) "
f"VALUES ({', '.join('?' for _ in _COLUMNS)})"
)
_INSERT = insert_sql("sqlite", COLUMNS)
class SQLiteRecorder:
@@ -110,7 +39,7 @@ class SQLiteRecorder:
conn = sqlite3.connect(path, check_same_thread=False, timeout=10.0)
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA busy_timeout=5000")
conn.execute(_DDL)
conn.execute(SQLITE_DDL)
conn.commit()
self._conn = conn
except (OSError, sqlite3.Error) as exc:
@@ -132,7 +61,7 @@ class SQLiteRecorder:
except sqlite3.Error as exc:
logger.warning("SQLite 遥测列探测失败(写入将逐行降级): {}", exc)
return
for column, decl in _BACKFILL_COLUMNS:
for column, decl in SQLITE_BACKFILL:
if column in existing:
continue
# 逐列独立 try: 一列撞上 duplicate 不得让后面的列漏补
@@ -148,7 +77,7 @@ class SQLiteRecorder:
"""写一行遥测;字段集合即 24 字段冻结签名(ports.TelemetryRecorder)。"""
if self._conn is None:
return
row = tuple(fields[col] for col in _COLUMNS)
row = tuple(fields[col] for col in COLUMNS)
try:
await asyncio.to_thread(self._write, row)
except (OSError, sqlite3.Error) as exc: