refactor: make the telemetry schema a single source of truth

DDL, column order and backfill statements lived twice, once in each
recorder. A public telemetry_schema_sql() would have made three copies,
and the drift shows up downstream as "I ran the printed SQL and the
library still reports a missing column".

Move both DDLs, both backfill lists and the 24 INSERT fields into
telemetry/schema.py verbatim; the recorders now import them and build
_INSERT through insert_sql(backend, COLUMNS) at import time. The
generated statements are byte-identical to the previous constants, so
runtime behaviour is unchanged (the postgres conflict target stays
bound to call_id for now).

insert_sql() validates its columns against COLUMNS: from the next task
on those names come from database probing, not from a constant, so the
subset check is the gate on the only injection surface. The new
telemetry_schema_sql() prints a paste-ready migration script; its
postgres backfill deliberately uses ADD COLUMN IF NOT EXISTS while the
library's own statements do not, because that form takes an ACCESS
EXCLUSIVE lock even when the column exists. Both variants are derived
from one declaration list so their column sets cannot drift.
This commit is contained in:
2026-08-19 11:18:06 -04:00
parent e9adb36577
commit 1471e0a2c6
5 changed files with 367 additions and 182 deletions
+5 -76
View File
@@ -22,80 +22,9 @@ from pathlib import Path
from loguru import logger
_DDL = """
CREATE TABLE IF NOT EXISTS llm_calls (
call_id TEXT PRIMARY KEY,
parent_call_id TEXT,
session_id TEXT,
model TEXT NOT NULL,
provider TEXT NOT NULL,
source_name TEXT NOT NULL,
messages TEXT NOT NULL,
response TEXT NOT NULL,
thinking TEXT NOT NULL DEFAULT '',
prompt_tokens INTEGER NOT NULL,
completion_tokens INTEGER NOT NULL,
usage_source TEXT NOT NULL,
latency_ms INTEGER NOT NULL,
ttft_ms REAL,
max_inter_token_ms REAL,
cache_hit INTEGER NOT NULL DEFAULT 0,
error TEXT,
cost REAL,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
cached_prompt_tokens INTEGER,
model_reported TEXT,
sampling TEXT,
reasoning_tokens INTEGER,
tenant_id TEXT NOT NULL DEFAULT '',
meta TEXT NOT NULL DEFAULT '{}'
);
"""
from polygateway.telemetry.schema import COLUMNS, SQLITE_BACKFILL, SQLITE_DDL, insert_sql
# 新列必须排在 created_at 之后: 旧表只能经 ALTER 追加到末尾,新建库若把它们
# 插在前面,两条路径的物理列序会分叉(列序断言测试无合规修法)。
_BACKFILL_COLUMNS = (
("cached_prompt_tokens", "INTEGER"),
("model_reported", "TEXT"),
("sampling", "TEXT"),
("reasoning_tokens", "INTEGER"),
# NOT NULL 补列必须带非 NULL 常量默认值,否则 SQLite 直接拒绝该 ALTER
# ("Cannot add a NOT NULL column with default value NULL"),补列全盘失败。
("tenant_id", "TEXT NOT NULL DEFAULT ''"),
("meta", "TEXT NOT NULL DEFAULT '{}'"),
)
_COLUMNS = (
"call_id",
"parent_call_id",
"session_id",
"model",
"provider",
"source_name",
"messages",
"response",
"thinking",
"prompt_tokens",
"completion_tokens",
"usage_source",
"latency_ms",
"ttft_ms",
"max_inter_token_ms",
"cache_hit",
"error",
"cost",
"cached_prompt_tokens",
"model_reported",
"sampling",
"reasoning_tokens",
"tenant_id",
"meta",
)
_INSERT = (
f"INSERT OR IGNORE INTO llm_calls ({', '.join(_COLUMNS)}) "
f"VALUES ({', '.join('?' for _ in _COLUMNS)})"
)
_INSERT = insert_sql("sqlite", COLUMNS)
class SQLiteRecorder:
@@ -110,7 +39,7 @@ class SQLiteRecorder:
conn = sqlite3.connect(path, check_same_thread=False, timeout=10.0)
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA busy_timeout=5000")
conn.execute(_DDL)
conn.execute(SQLITE_DDL)
conn.commit()
self._conn = conn
except (OSError, sqlite3.Error) as exc:
@@ -132,7 +61,7 @@ class SQLiteRecorder:
except sqlite3.Error as exc:
logger.warning("SQLite 遥测列探测失败(写入将逐行降级): {}", exc)
return
for column, decl in _BACKFILL_COLUMNS:
for column, decl in SQLITE_BACKFILL:
if column in existing:
continue
# 逐列独立 try: 一列撞上 duplicate 不得让后面的列漏补
@@ -148,7 +77,7 @@ class SQLiteRecorder:
"""写一行遥测;字段集合即 24 字段冻结签名(ports.TelemetryRecorder)。"""
if self._conn is None:
return
row = tuple(fields[col] for col in _COLUMNS)
row = tuple(fields[col] for col in COLUMNS)
try:
await asyncio.to_thread(self._write, row)
except (OSError, sqlite3.Error) as exc: