refactor: make the telemetry schema a single source of truth
DDL, column order and backfill statements lived twice, once in each recorder. A public telemetry_schema_sql() would have made three copies, and the drift shows up downstream as "I ran the printed SQL and the library still reports a missing column". Move both DDLs, both backfill lists and the 24 INSERT fields into telemetry/schema.py verbatim; the recorders now import them and build _INSERT through insert_sql(backend, COLUMNS) at import time. The generated statements are byte-identical to the previous constants, so runtime behaviour is unchanged (the postgres conflict target stays bound to call_id for now). insert_sql() validates its columns against COLUMNS: from the next task on those names come from database probing, not from a constant, so the subset check is the gate on the only injection surface. The new telemetry_schema_sql() prints a paste-ready migration script; its postgres backfill deliberately uses ADD COLUMN IF NOT EXISTS while the library's own statements do not, because that form takes an ACCESS EXCLUSIVE lock even when the column exists. Both variants are derived from one declaration list so their column sets cannot drift.
This commit is contained in:
@@ -22,80 +22,9 @@ from pathlib import Path
|
||||
|
||||
from loguru import logger
|
||||
|
||||
_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS llm_calls (
|
||||
call_id TEXT PRIMARY KEY,
|
||||
parent_call_id TEXT,
|
||||
session_id TEXT,
|
||||
model TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
source_name TEXT NOT NULL,
|
||||
messages TEXT NOT NULL,
|
||||
response TEXT NOT NULL,
|
||||
thinking TEXT NOT NULL DEFAULT '',
|
||||
prompt_tokens INTEGER NOT NULL,
|
||||
completion_tokens INTEGER NOT NULL,
|
||||
usage_source TEXT NOT NULL,
|
||||
latency_ms INTEGER NOT NULL,
|
||||
ttft_ms REAL,
|
||||
max_inter_token_ms REAL,
|
||||
cache_hit INTEGER NOT NULL DEFAULT 0,
|
||||
error TEXT,
|
||||
cost REAL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
cached_prompt_tokens INTEGER,
|
||||
model_reported TEXT,
|
||||
sampling TEXT,
|
||||
reasoning_tokens INTEGER,
|
||||
tenant_id TEXT NOT NULL DEFAULT '',
|
||||
meta TEXT NOT NULL DEFAULT '{}'
|
||||
);
|
||||
"""
|
||||
from polygateway.telemetry.schema import COLUMNS, SQLITE_BACKFILL, SQLITE_DDL, insert_sql
|
||||
|
||||
# 新列必须排在 created_at 之后: 旧表只能经 ALTER 追加到末尾,新建库若把它们
|
||||
# 插在前面,两条路径的物理列序会分叉(列序断言测试无合规修法)。
|
||||
_BACKFILL_COLUMNS = (
|
||||
("cached_prompt_tokens", "INTEGER"),
|
||||
("model_reported", "TEXT"),
|
||||
("sampling", "TEXT"),
|
||||
("reasoning_tokens", "INTEGER"),
|
||||
# NOT NULL 补列必须带非 NULL 常量默认值,否则 SQLite 直接拒绝该 ALTER
|
||||
# ("Cannot add a NOT NULL column with default value NULL"),补列全盘失败。
|
||||
("tenant_id", "TEXT NOT NULL DEFAULT ''"),
|
||||
("meta", "TEXT NOT NULL DEFAULT '{}'"),
|
||||
)
|
||||
|
||||
_COLUMNS = (
|
||||
"call_id",
|
||||
"parent_call_id",
|
||||
"session_id",
|
||||
"model",
|
||||
"provider",
|
||||
"source_name",
|
||||
"messages",
|
||||
"response",
|
||||
"thinking",
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"usage_source",
|
||||
"latency_ms",
|
||||
"ttft_ms",
|
||||
"max_inter_token_ms",
|
||||
"cache_hit",
|
||||
"error",
|
||||
"cost",
|
||||
"cached_prompt_tokens",
|
||||
"model_reported",
|
||||
"sampling",
|
||||
"reasoning_tokens",
|
||||
"tenant_id",
|
||||
"meta",
|
||||
)
|
||||
|
||||
_INSERT = (
|
||||
f"INSERT OR IGNORE INTO llm_calls ({', '.join(_COLUMNS)}) "
|
||||
f"VALUES ({', '.join('?' for _ in _COLUMNS)})"
|
||||
)
|
||||
_INSERT = insert_sql("sqlite", COLUMNS)
|
||||
|
||||
|
||||
class SQLiteRecorder:
|
||||
@@ -110,7 +39,7 @@ class SQLiteRecorder:
|
||||
conn = sqlite3.connect(path, check_same_thread=False, timeout=10.0)
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
conn.execute(_DDL)
|
||||
conn.execute(SQLITE_DDL)
|
||||
conn.commit()
|
||||
self._conn = conn
|
||||
except (OSError, sqlite3.Error) as exc:
|
||||
@@ -132,7 +61,7 @@ class SQLiteRecorder:
|
||||
except sqlite3.Error as exc:
|
||||
logger.warning("SQLite 遥测列探测失败(写入将逐行降级): {}", exc)
|
||||
return
|
||||
for column, decl in _BACKFILL_COLUMNS:
|
||||
for column, decl in SQLITE_BACKFILL:
|
||||
if column in existing:
|
||||
continue
|
||||
# 逐列独立 try: 一列撞上 duplicate 不得让后面的列漏补
|
||||
@@ -148,7 +77,7 @@ class SQLiteRecorder:
|
||||
"""写一行遥测;字段集合即 24 字段冻结签名(ports.TelemetryRecorder)。"""
|
||||
if self._conn is None:
|
||||
return
|
||||
row = tuple(fields[col] for col in _COLUMNS)
|
||||
row = tuple(fields[col] for col in COLUMNS)
|
||||
try:
|
||||
await asyncio.to_thread(self._write, row)
|
||||
except (OSError, sqlite3.Error) as exc:
|
||||
|
||||
Reference in New Issue
Block a user