From e949edb62af31439c14fe6931c31d3c58a11f1a1 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Wed, 19 Aug 2026 11:57:41 -0400 Subject: [PATCH] feat: gate the automatic ALTER behind an explicit mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 两个 recorder 的 `__init__` 增 keyword-only 必填 `auto_migrate`(设计 D-c: 缺省规则只写在 config 一处,不与类签名漂移),并把写入语句从模块级常量改为 实例级: manual 档探测到旧表缺列时一条 ALTER 都不发,改按现有列裁剪 INSERT, 准备期发一次 warning(逐列点名 + "以下维度不会被记录" + 可直接执行的补列 SQL)。 裁剪是关掉 ALTER 的前提而非增强: 旧表缺列时若既不 ALTER 又不裁剪,每一行 INSERT 都撞 `no column named tenant_id` 被整行丢弃,比自动 ALTER 更严重地 违反"遥测必录"。auto 档行为逐字不变(先探测后 ALTER、duplicate column 视为 成功、失败只 warning 不判死、写入沿用全量列)。 探测失败、或探测结果与 COLUMNS 毫无交集,两档都保守回落全量列——空列集会让 `insert_sql` 产出 `INSERT INTO llm_calls () VALUES ()`(它不拒空列表,空集 技术上是子集)。PG 侧 `_columns`/`_insert` 与 `_schema_ready` 在同一处一起 赋值,不留"已就绪但语句还是旧的"窗口。 同批改 `GatewaySettings.telemetry_auto_migrate`(按后端派生: PG False、 SQLite True)与 `client._build_telemetry` 透传: 签名变更与其唯一调用点必须 落在同一次提交,否则该提交点整条装配路 TypeError。env 键留给下一步。 --- src/polygateway/client.py | 8 +- src/polygateway/config.py | 8 + src/polygateway/telemetry/postgres.py | 151 ++++++++++--- src/polygateway/telemetry/sqlite.py | 116 ++++++++-- tests/integration/test_governance_stack.py | 12 +- tests/integration/test_postgres_telemetry.py | 28 +-- tests/unit/test_telemetry.py | 214 +++++++++++++++++-- tools/soak/run_soak.py | 2 +- 8 files changed, 466 insertions(+), 73 deletions(-) diff --git a/src/polygateway/client.py b/src/polygateway/client.py index e1413b4..f8352a6 100644 --- a/src/polygateway/client.py +++ b/src/polygateway/client.py @@ -400,11 +400,15 @@ def _build_telemetry(settings: GatewaySettings) -> TelemetryRecorder | None: from polygateway.telemetry.postgres import PostgresRecorder assert settings.telemetry_pg_dsn is not None # 内部不变量: _validate_telemetry 已保证 - return PostgresRecorder(settings.telemetry_pg_dsn) + return PostgresRecorder( + settings.telemetry_pg_dsn, auto_migrate=settings.telemetry_auto_migrate + ) from polygateway.telemetry.sqlite import SQLiteRecorder assert settings.telemetry_sqlite_path is not None # 内部不变量: _validate_telemetry 已保证 - return SQLiteRecorder(settings.telemetry_sqlite_path) + return SQLiteRecorder( + settings.telemetry_sqlite_path, auto_migrate=settings.telemetry_auto_migrate + ) def _build_structured( diff --git a/src/polygateway/config.py b/src/polygateway/config.py index 3c1031d..66c776d 100644 --- a/src/polygateway/config.py +++ b/src/polygateway/config.py @@ -131,6 +131,9 @@ class GatewaySettings: telemetry_backend: str telemetry_sqlite_path: str | None telemetry_pg_dsn: str | None + # 是否允许 recorder 给已存在的旧表自动 ALTER 补列(issue #13); + # 派生规则只写在 `_load_pgw` 一处,不与 recorder 的类签名漂移 + telemetry_auto_migrate: bool redis_url: str | None pricing_path: str | None structured_max_retries: int @@ -444,6 +447,11 @@ def _load_pgw(env: Mapping[str, str]) -> dict[str, object]: if telemetry_backend == "sqlite" else None, "telemetry_pg_dsn": _load_pg_dsn(env) if telemetry_backend == "postgres" else None, + # 按后端不对称派生(issue #13): SQLite 是下游自己的本地文件(无 DBA、无迁移 + # 工具),补列是毫秒级元数据操作;PG 是共享生产表,ALTER 取 ACCESS EXCLUSIVE + # 锁会阻塞该表其后所有查询,而遥测是业务路径上的内联 await。 + # backend=none 时无 recorder 消费该值,派生结果恒 False。 + "telemetry_auto_migrate": telemetry_backend == "sqlite", "redis_url": redis_url, "pricing_path": env.get("PGW_PRICING_PATH") or None, "structured_max_retries": _load_structured_retries(env), diff --git a/src/polygateway/telemetry/postgres.py b/src/polygateway/telemetry/postgres.py index b0e819c..4e270df 100644 --- a/src/polygateway/telemetry/postgres.py +++ b/src/polygateway/telemetry/postgres.py @@ -35,13 +35,60 @@ _EXISTING_COLUMNS = ( "WHERE attrelid = to_regclass('llm_calls') AND attnum > 0 AND NOT attisdropped" ) -_INSERT = insert_sql("postgres", COLUMNS) +# 缺列 warning 要打印可直接执行的补列语句,与库内 ALTER 同源(不许两份) +_BACKFILL_STATEMENTS = dict(PG_BACKFILL) + + +def _missing_columns_message(missing: list[str], *, alien_table: bool) -> str: + """拼 manual 档的缺列告警: 逐列点名 + 讲清后果 + 给出可直接执行的 SQL。 + + 只说"缺列"是不够的: 静默丢维度的后果是多租户账目全归空串且无任何报错, + 看告警的人必须一眼看到丢的是哪几个维度、以及怎么补。 + + Args: + missing: 缺失的列名(按 `COLUMNS` 保序)。 + alien_table: 连主键列 `call_id` 都没有——该表多半不是本库的 `llm_calls`。 + + Returns: + 单条 warning 的完整文本(库只在准备期发一次,不逐行发)。 + """ + statements = [ + f"{_BACKFILL_STATEMENTS[column]};" for column in missing if column in _BACKFILL_STATEMENTS + ] + unknown = [column for column in missing if column not in _BACKFILL_STATEMENTS] + if unknown: + # 这些列本库从未经 ALTER 补过(建表即有),给不出单条 ALTER,指向完整脚本 + statements.append( + f"-- 另缺 {', '.join(unknown)};完整建表脚本见 " + 'polygateway.telemetry_schema_sql("postgres")' + ) + head = ( + "Postgres 遥测表 llm_calls 缺主键列 call_id,很可能不是本库的遥测表" + "(库不做二次判定,仍照常尝试写入)" + if alien_table + else "Postgres 遥测表 llm_calls 缺列,且 auto_migrate=False(库不发任何 DDL)" + ) + return ( + f"{head};以下维度不会被记录: {', '.join(missing)}。" + "补列请自行执行(建议挑低峰,ALTER 取 ACCESS EXCLUSIVE 锁):\n" + "\n".join(statements) + ) class PostgresRecorder: """TelemetryRecorder 端口的 Postgres 实现;asyncpg 原生异步,无线程桥接。""" - def __init__(self, dsn: str, *, pool: asyncpg.Pool | None = None) -> None: + def __init__(self, dsn: str, *, pool: asyncpg.Pool | None = None, auto_migrate: bool) -> None: + """记下装配参数(不连库);列与 INSERT 语句在首次准备期定型。 + + Args: + dsn: asyncpg 连接串(已剥驱动后缀)。 + pool: 外部注入的池;注入方自己负责关闭。 + auto_migrate: True 则给已存在的旧表自动补列;False(PG 侧的缺省档) + 则一条 ALTER 都不发——`ALTER TABLE ADD COLUMN` 取 ACCESS EXCLUSIVE + 锁,会排在长事务后阻塞该表其后所有查询,而遥测是业务路径上的内联 + await。keyword-only **必填**: 缺省规则只写在 config 一处,不与本类 + 签名漂移(设计 D-c)。 + """ try: import asyncpg # noqa: F401 - 仅探测 extra 是否安装 except ImportError as exc: @@ -51,6 +98,10 @@ class PostgresRecorder: self._dsn = dsn self._pool: asyncpg.Pool | None = pool self._external_pool = pool is not None + self._auto_migrate = auto_migrate + # 先按全量列定型: 准备期探测失败时保守沿用全量(今天的行为) + self._columns: tuple[str, ...] = COLUMNS + self._insert = insert_sql("postgres", COLUMNS) self._schema_ready = False self._failed = False # 结构性降级标志: 置位后所有写入短路 self._init_lock = asyncio.Lock() @@ -93,7 +144,7 @@ class PostgresRecorder: """备好表并交回可用的池;瞬时失败只跳过本次,确定写不进去才判死。""" try: async with pool.acquire() as conn: - writable = await self._prepare_table(conn) + columns = await self._prepare_table(conn) except asyncio.CancelledError: raise except Exception as exc: @@ -101,14 +152,20 @@ class PostgresRecorder: # 只跳过本次记录,下次调用重新准备 logger.warning("Postgres 遥测建表探测失败(跳过本条,下次重试): {}", exc) return None - if not writable: + if columns is None: self._failed = True return None + # 写入列、语句与就绪标志必须**一起**生效: `_ensure_ready` 只看 `_schema_ready` + # 就绕开 `_init_lock` 直接返回池,先置就绪会开出"已就绪但语句还是旧的"的窗口 + self._columns = columns + self._insert = insert_sql("postgres", columns) self._schema_ready = True return pool - async def _prepare_table(self, conn: object) -> bool: - """备好 `llm_calls`;**表存在就绝不发 DDL**。返回 False 仅表示表确定不存在。 + async def _prepare_table(self, conn: object) -> tuple[str, ...] | None: + """备好 `llm_calls` 并返回本实例要写的列;**表存在就绝不发 DDL**。 + + 返回 None 仅表示表确定不存在且建不出来(唯一允许判死的情形)。 `CREATE TABLE IF NOT EXISTS` 不能无条件发: PostgreSQL 对 schema 的 CREATE 权限检查**早于** `IF NOT EXISTS` 的存在性判断(PG 16.14 实测: @@ -121,32 +178,74 @@ class PostgresRecorder: """ exists = await conn.fetchval(_TABLE_EXISTS) is not None # type: ignore[attr-defined] if exists: - await self._backfill_columns(conn) # 旧表可能缺列;失败只逐行降级 - return True + return await self._resolve_columns(conn) # 旧表可能缺列 try: await conn.execute(PG_DDL) # type: ignore[attr-defined] except asyncio.CancelledError: raise except Exception as exc: logger.warning("Postgres 遥测建表失败(表不存在,记录无处可落): {}", exc) - return False - return True # 新建表列已齐全,无需再走补列 + return None + return COLUMNS # 新建表列已齐全,无需再走补列 - async def _backfill_columns(self, conn: object) -> None: - """给已存在的旧表补新列(issue #3);**先探测再 ALTER,失败绝不置 `_failed`**。 + async def _resolve_columns(self, conn: object) -> tuple[str, ...]: + """探测旧表现有列并定型写入列: auto 档先补齐,manual 档改为裁剪(issue #13)。 - 两条纪律各有实测理由: - ① 不置 `_failed`: 应用账号只有 INSERT 权限时,`ALTER TABLE` 的 ownership - 检查早于 `IF NOT EXISTS` 的存在性判断——列明明齐全也会失败。置位会让 - 整个 recorder 永久 no-op,与「补列失败只降级为逐行丢弃」的承诺相悖 - (SQLite 侧同款守卫,两侧必须对称)。 - ② 先探测: `ADD COLUMN IF NOT EXISTS` 即便列已存在,也会**先取 ACCESS - EXCLUSIVE 锁**再判存在性(实测会被一个开着的读事务阻塞)。遥测是内联 - await,让每个进程的首次写入都去抢共享审计表的排他锁,等于用记录基础设施 - 拖垮业务调用。探测走 ACCESS SHARE,稳态下一条 ALTER 都不会发。 + **先探测**的理由(两档共用): `ADD COLUMN IF NOT EXISTS` 即便列已存在,也会 + **先取 ACCESS EXCLUSIVE 锁**再判存在性(实测会被一个开着的读事务阻塞)。遥测是 + 内联 await,让每个进程的首次写入都去抢共享审计表的排他锁,等于用记录基础设施 + 拖垮业务调用。探测走 ACCESS SHARE,稳态下一条 ALTER 都不会发。 + + 探测失败保守沿用全量列(今天的行为): 猜不出真实列集合时,让写入照常尝试。 """ try: existing = {row["attname"] for row in await conn.fetch(_EXISTING_COLUMNS)} # type: ignore[attr-defined] + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning("Postgres 遥测列探测失败(沿用全量列,写入将逐行降级): {}", exc) + return COLUMNS + if self._auto_migrate: + await self._backfill_columns(conn, existing) + return COLUMNS + return self._trim_columns(existing) + + def _trim_columns(self, existing: set[str]) -> tuple[str, ...]: + """manual 档: 按现有列裁剪写入列,并把缺列一次讲清楚。 + + 裁剪是关掉 ALTER 的**前提**而非增强: 旧表缺列时仍发全量 INSERT,每一行 + 都会因未知列被拒 → 遥测彻底丢失,比自动 ALTER 更严重地违反"遥测必录"。 + 探测结果与 `COLUMNS` 毫无交集时视同探测异常保守回落全量: 空列集会构造出 + `INSERT INTO llm_calls () VALUES ()` 这种语法非法的语句(`insert_sql` 拦 + 不住——空集技术上是子集),必须在交给它之前拦下。 + """ + effective = tuple(column for column in COLUMNS if column in existing) + if not effective: + logger.warning( + "Postgres 遥测表 llm_calls 没有任何本库认识的列(沿用全量列,写入将逐行降级);" + "现有列: {}", + sorted(existing), + ) + return COLUMNS + missing = [column for column in COLUMNS if column not in existing] + if missing: + # 单参数传入: 补列 SQL 里带 `'{}'::jsonb` 字面量,拼进 format 模板会被当占位符 + logger.warning( + "{}", _missing_columns_message(missing, alien_table="call_id" not in existing) + ) + return effective + + async def _backfill_columns(self, conn: object, existing: set[str]) -> None: + """auto 档: 给已存在的旧表补新列(issue #3);**失败绝不置 `_failed`**。 + + 不置 `_failed` 的实测理由: 应用账号只有 INSERT 权限时,`ALTER TABLE` 的 + ownership 检查早于 `IF NOT EXISTS` 的存在性判断——列明明齐全也会失败。置位会让 + 整个 recorder 永久 no-op,与「补列失败只降级为逐行丢弃」的承诺相悖 + (SQLite 侧同款守卫,两侧必须对称)。补列失败后写入沿用全量列(今天的行为): + auto 档承诺的是"把列补上",补不上就让缺列以逐行 warning 暴露;要降级写入 + 请显式选 manual。 + """ + try: for column, statement in PG_BACKFILL: if column not in existing: await conn.execute(statement) # type: ignore[attr-defined] @@ -156,14 +255,18 @@ class PostgresRecorder: logger.warning("Postgres 遥测补列失败(写入将逐行降级): {}", exc) async def record_llm_call(self, **fields: object) -> None: - """写一行遥测;单条失败逐条 warning 丢弃(两级降级之二),绝不冒泡。""" + """写一行遥测;单条失败逐条 warning 丢弃(两级降级之二),绝不冒泡。 + + 取值按 `self._columns`(manual 档可能已被裁剪),与 `self._insert` 的 + 占位符同序——两者必须一起改,分开改就是把值写进错位的列。 + """ pool = await self._ensure_ready() if pool is None: return - row = tuple(fields[col] for col in COLUMNS) + row = tuple(fields[col] for col in self._columns) try: async with pool.acquire() as conn: - await conn.execute(_INSERT, *row) + await conn.execute(self._insert, *row) except asyncio.CancelledError: raise except Exception as exc: diff --git a/src/polygateway/telemetry/sqlite.py b/src/polygateway/telemetry/sqlite.py index abf3d1b..604a676 100644 --- a/src/polygateway/telemetry/sqlite.py +++ b/src/polygateway/telemetry/sqlite.py @@ -24,15 +24,65 @@ from loguru import logger from polygateway.telemetry.schema import COLUMNS, SQLITE_BACKFILL, SQLITE_DDL, insert_sql -_INSERT = insert_sql("sqlite", COLUMNS) +# 缺列 warning 要打印可直接执行的补列语句,列定义与库内 ALTER 同源(不许两份) +_BACKFILL_DECLS = dict(SQLITE_BACKFILL) + + +def _missing_columns_message(missing: list[str], *, alien_table: bool) -> str: + """拼 manual 档的缺列告警: 逐列点名 + 讲清后果 + 给出可直接执行的 SQL。 + + 只说"缺列"是不够的: 静默丢维度的后果是多租户账目全归空串且无任何报错, + 看告警的人必须一眼看到丢的是哪几个维度、以及怎么补。 + + Args: + missing: 缺失的列名(按 `COLUMNS` 保序)。 + alien_table: 连主键列 `call_id` 都没有——该表多半不是本库的 `llm_calls`。 + + Returns: + 单条 warning 的完整文本(库只在准备期发一次,不逐行发)。 + """ + statements = [ + f"ALTER TABLE llm_calls ADD COLUMN {column} {_BACKFILL_DECLS[column]};" + for column in missing + if column in _BACKFILL_DECLS + ] + unknown = [column for column in missing if column not in _BACKFILL_DECLS] + if unknown: + # 这些列本库从未经 ALTER 补过(建表即有),给不出单条 ALTER,指向完整脚本 + statements.append( + f"-- 另缺 {', '.join(unknown)};完整建表脚本见 " + 'polygateway.telemetry_schema_sql("sqlite")' + ) + head = ( + "SQLite 遥测表 llm_calls 缺主键列 call_id,很可能不是本库的遥测表" + "(库不做二次判定,仍照常尝试写入)" + if alien_table + else "SQLite 遥测表 llm_calls 缺列,且 auto_migrate=False(库不发任何 DDL)" + ) + return f"{head};以下维度不会被记录: {', '.join(missing)}。补列请自行执行:\n" + "\n".join( + statements + ) class SQLiteRecorder: """TelemetryRecorder 端口的 SQLite 实现;初始化/写入失败全降级 warning。""" - def __init__(self, db_path: Path | str) -> None: + def __init__(self, db_path: Path | str, *, auto_migrate: bool) -> None: + """建连接与表,并按探测到的列定型本实例的 INSERT 语句。 + + Args: + db_path: 库文件路径;父目录不存在会自动创建。 + auto_migrate: True 则给已存在的旧表自动补列(SQLite 侧的缺省档: + 下游本地文件,无 DBA 无迁移工具);False 则一条 ALTER 都不发, + 改为按现有列裁剪写入。keyword-only **必填**: 缺省规则只写在 + config 一处,不与本类签名漂移(设计 D-c)。 + """ + self._auto_migrate = auto_migrate self._lock = threading.Lock() self._conn: sqlite3.Connection | None = None + # 先按全量列定型: 连接失败/探测失败时保守沿用全量(今天的行为) + self._columns: tuple[str, ...] = COLUMNS + self._insert = insert_sql("sqlite", COLUMNS) try: path = Path(db_path) path.parent.mkdir(parents=True, exist_ok=True) @@ -44,23 +94,61 @@ class SQLiteRecorder: self._conn = conn except (OSError, sqlite3.Error) as exc: logger.warning("SQLite 遥测初始化失败,后续记录降级为 no-op: {}", exc) - self._backfill_columns() + self._prepare_columns() - def _backfill_columns(self) -> None: - """给已存在的旧表补新列(issue #3);独立 try,失败只降级为逐行丢弃。 + def _prepare_columns(self) -> None: + """探测现有列后定型写入: auto 档补齐缺列,manual 档改为裁剪写入(issue #13)。 必须放在 `self._conn` 赋值**之后**并先判空: 初始化失败时连接为 None, - 无守卫的补列会抛 AttributeError 逃出 `__init__`,把"静默降级"变成崩溃。 - 补列失败也绝不清空 `self._conn`——那会让整个 recorder 永久 no-op, - 比逐行丢弃严重得多。 + 无守卫的探测会抛 AttributeError 逃出 `__init__`,把"静默降级"变成崩溃。 + 探测失败保守沿用全量列(今天的行为): 猜不出真实列集合时,让写入照常尝试。 """ if self._conn is None: return try: existing = {row[1] for row in self._conn.execute("PRAGMA table_info(llm_calls)")} except sqlite3.Error as exc: - logger.warning("SQLite 遥测列探测失败(写入将逐行降级): {}", exc) + logger.warning("SQLite 遥测列探测失败(沿用全量列,写入将逐行降级): {}", exc) return + if self._auto_migrate: + self._backfill_columns(existing) + return + self._adopt_existing_columns(existing) + + def _adopt_existing_columns(self, existing: set[str]) -> None: + """manual 档: 不发任何 DDL,按现有列裁剪 INSERT,并把缺列一次讲清楚。 + + 裁剪是关掉 ALTER 的**前提**而非增强: 旧表缺列时仍发全量 INSERT,每一行 + 都会因未知列被拒 → 遥测彻底丢失,比自动 ALTER 更严重地违反"遥测必录"。 + 探测结果与 `COLUMNS` 毫无交集时视同探测异常保守回落全量: 空列集会构造出 + `INSERT INTO llm_calls () VALUES ()` 这种语法非法的语句(`insert_sql` 拦 + 不住——空集技术上是子集),必须在交给它之前拦下。 + """ + effective = tuple(column for column in COLUMNS if column in existing) + if not effective: + logger.warning( + "SQLite 遥测表 llm_calls 没有任何本库认识的列(沿用全量列,写入将逐行降级);" + "现有列: {}", + sorted(existing), + ) + return + self._columns = effective + self._insert = insert_sql("sqlite", effective) + missing = [column for column in COLUMNS if column not in existing] + if missing: + # 单参数传入: 补列 SQL 里带 `'{}'` 字面量,拼进 format 模板会被当占位符 + logger.warning( + "{}", _missing_columns_message(missing, alien_table="call_id" not in existing) + ) + + def _backfill_columns(self, existing: set[str]) -> None: + """auto 档: 给已存在的旧表补新列(issue #3);逐列独立 try,失败只降级为逐行丢弃。 + + 补列失败绝不清空 `self._conn`——那会让整个 recorder 永久 no-op, + 比逐行丢弃严重得多。失败后写入沿用全量列(今天的行为): auto 档承诺的是 + "把列补上",补不上就让缺列以逐行 warning 暴露;要降级写入请显式选 manual。 + """ + assert self._conn is not None # 内部不变量: 调用方已判空 for column, decl in SQLITE_BACKFILL: if column in existing: continue @@ -74,10 +162,14 @@ class SQLiteRecorder: logger.warning("SQLite 遥测补列失败(写入将逐行降级): {}", exc) async def record_llm_call(self, **fields: object) -> None: - """写一行遥测;字段集合即 24 字段冻结签名(ports.TelemetryRecorder)。""" + """写一行遥测;字段集合即 24 字段冻结签名(ports.TelemetryRecorder)。 + + 取值按 `self._columns`(manual 档可能已被裁剪),与 `self._insert` 的 + 占位符同序——两者必须一起改,分开改就是把值写进错位的列。 + """ if self._conn is None: return - row = tuple(fields[col] for col in COLUMNS) + row = tuple(fields[col] for col in self._columns) try: await asyncio.to_thread(self._write, row) except (OSError, sqlite3.Error) as exc: @@ -86,7 +178,7 @@ class SQLiteRecorder: def _write(self, row: tuple) -> None: assert self._conn is not None # 内部不变量: 调用方已判空 with self._lock: - self._conn.execute(_INSERT, row) + self._conn.execute(self._insert, row) self._conn.commit() def close(self) -> None: diff --git a/tests/integration/test_governance_stack.py b/tests/integration/test_governance_stack.py index acffbcb..2b140f7 100644 --- a/tests/integration/test_governance_stack.py +++ b/tests/integration/test_governance_stack.py @@ -119,7 +119,7 @@ class TestBreakerRecoveryFullChain: class TestCancellationThroughStack: async def test_cancel_mid_request_releases_and_records(self, tmp_path): - recorder = SQLiteRecorder(tmp_path / "t.db") + recorder = SQLiteRecorder(tmp_path / "t.db", auto_migrate=True) entered = asyncio.Event() async def hanging_handler(request): @@ -144,7 +144,7 @@ class TestCancellationThroughStack: class TestTelemetryAcrossPaths: async def test_success_cache_hit_and_failure_rows(self, tmp_path): - recorder = SQLiteRecorder(tmp_path / "t.db") + recorder = SQLiteRecorder(tmp_path / "t.db", auto_migrate=True) client = _full_client(lambda req: _sse(), telemetry=recorder, cache=InMemoryCache()) await client.chat([{"role": "user", "content": "hi"}]) # 成功(尝试行) await client.chat([{"role": "user", "content": "hi"}]) # 缓存命中行 @@ -155,7 +155,7 @@ class TestTelemetryAcrossPaths: assert hits == 1 and total == 2 async def test_transient_attempts_each_recorded(self, tmp_path): - recorder = SQLiteRecorder(tmp_path / "t.db") + recorder = SQLiteRecorder(tmp_path / "t.db", auto_migrate=True) calls = {"n": 0} def flaky(request): @@ -193,7 +193,7 @@ class TestRejectionReasonIsQueryable: ) async def test_rejected_call_leaves_the_reason_in_telemetry(self, tmp_path): - recorder = SQLiteRecorder(tmp_path / "t.db") + recorder = SQLiteRecorder(tmp_path / "t.db", auto_migrate=True) client = _full_client( lambda req: httpx.Response(400, content=self._BODY.encode()), telemetry=recorder ) @@ -257,7 +257,7 @@ class TestSamplingThroughStack: return _sse() db = tmp_path / "t.db" - recorder = SQLiteRecorder(db) + recorder = SQLiteRecorder(db, auto_migrate=True) client = _full_client(handler, telemetry=recorder) await client.chat([{"role": "user", "content": "hi"}], overlay={"seed": 42}) recorder.close() @@ -276,7 +276,7 @@ class TestSamplingThroughStack: src = dataclasses.replace(_source(), extra_body={"temperature": 0}) db = tmp_path / "t.db" - recorder = SQLiteRecorder(db) + recorder = SQLiteRecorder(db, auto_migrate=True) client = GatewayClient( scope="llm", sources=[src], diff --git a/tests/integration/test_postgres_telemetry.py b/tests/integration/test_postgres_telemetry.py index 48fd4c9..a801c7e 100644 --- a/tests/integration/test_postgres_telemetry.py +++ b/tests/integration/test_postgres_telemetry.py @@ -185,7 +185,7 @@ class TestObservabilityColumns: """issue #3: 两列写入可回读,且已存在的 18 列旧表会被自动补列。""" async def test_values_round_trip(self, dsn): - recorder = PostgresRecorder(dsn) + recorder = PostgresRecorder(dsn, auto_migrate=True) try: await _record_minimal(recorder, call_id=_cid("hit"), cached_prompt_tokens=64) await _record_minimal(recorder, call_id=_cid("zero"), cached_prompt_tokens=0) @@ -213,7 +213,7 @@ class TestObservabilityColumns: async def test_legacy_table_is_upgraded_in_place(self, legacy_schema): """18 列旧表不补列的话,每行写入都会被逐行 warning 丢弃(遥测静默全失)。""" schema_dsn, schema = legacy_schema - recorder = PostgresRecorder(schema_dsn) + recorder = PostgresRecorder(schema_dsn, auto_migrate=True) try: await _record_minimal( recorder, call_id=_cid("legacy"), cached_prompt_tokens=7, model_reported="m-real" @@ -238,7 +238,7 @@ class TestObservabilityColumns: class TestSchema: async def test_schema_has_frozen_columns_in_order(self, dsn): - recorder = PostgresRecorder(dsn) + recorder = PostgresRecorder(dsn, auto_migrate=True) try: await _record_minimal(recorder) rows = await _fetch( @@ -251,7 +251,7 @@ class TestSchema: await recorder.aclose() async def test_call_id_idempotent(self, dsn): - recorder = PostgresRecorder(dsn) + recorder = PostgresRecorder(dsn, auto_migrate=True) try: await _record_minimal(recorder, call_id=_cid("dup")) await _record_minimal(recorder, call_id=_cid("dup"), response="second") @@ -263,7 +263,7 @@ class TestSchema: await recorder.aclose() async def test_concurrent_writes_all_land(self, dsn): - recorder = PostgresRecorder(dsn) + recorder = PostgresRecorder(dsn, auto_migrate=True) try: await asyncio.gather( *(_record_minimal(recorder, call_id=_cid(f"c{i}")) for i in range(50)) @@ -281,14 +281,14 @@ class TestSchema: class TestDegradation: async def test_unreachable_server_degrades_silently(self): """结构性失败(建池不通)→ warning 一次后永久降级,业务零感知。""" - recorder = PostgresRecorder("postgresql://u:p@127.0.0.1:1/x") + recorder = PostgresRecorder("postgresql://u:p@127.0.0.1:1/x", auto_migrate=True) await _record_minimal(recorder) # 不抛 await _record_minimal(recorder, call_id=_cid("c2")) # 已降级短路,同样不抛 await recorder.aclose() async def test_row_failure_does_not_poison_later_rows(self, dsn): """运行时单条写失败(NUL 字节文本被 PG 拒)→ 丢该行,后续行照常落库。""" - recorder = PostgresRecorder(dsn) + recorder = PostgresRecorder(dsn, auto_migrate=True) try: await _record_minimal(recorder, call_id=_cid("bad"), response="nul\x00byte") await _record_minimal(recorder, call_id=_cid("good")) @@ -302,7 +302,7 @@ class TestDegradation: await recorder.aclose() async def test_aclose_idempotent(self, dsn): - recorder = PostgresRecorder(dsn) + recorder = PostgresRecorder(dsn, auto_migrate=True) await _record_minimal(recorder) await recorder.aclose() await recorder.aclose() @@ -374,7 +374,7 @@ class TestLeastPrivilegeDeployment: async def test_records_land_without_schema_create_privilege(self, least_privilege_dsn): """修复前: 建表被拒 → _failed → 整个进程一条不落(下游 150 次调用全丢)。""" low_dsn, schema = least_privilege_dsn - recorder = PostgresRecorder(low_dsn) + recorder = PostgresRecorder(low_dsn, auto_migrate=True) try: await _record_minimal(recorder, call_id=_cid("lp1")) await _record_minimal(recorder, call_id=_cid("lp2"), cost=1.5) @@ -534,7 +534,7 @@ class TestCallerDimensionsAcceptance: async def test_fresh_schema_round_trips_the_dimensions(self, fresh_schema): """新建库: 列齐全,且维度值原样读回——只验列存在会漏掉写错列位的错。""" fresh_dsn, schema = fresh_schema - recorder = PostgresRecorder(fresh_dsn) + recorder = PostgresRecorder(fresh_dsn, auto_migrate=True) try: await _record_minimal( recorder, call_id=_cid("dim"), tenant_id="tenant-a", meta='{"batch": "b7"}' @@ -568,7 +568,7 @@ class TestCallerDimensionsAcceptance: 审计出来,历史欠账是可见、可量化、可补录的。 """ schema_dsn, schema = pre_tenant_schema - recorder = PostgresRecorder(schema_dsn) + recorder = PostgresRecorder(schema_dsn, auto_migrate=True) try: await _record_minimal( recorder, call_id=_cid("new"), tenant_id="tenant-a", meta='{"k": 1}' @@ -620,7 +620,7 @@ class TestCallerDimensionsAcceptance: 置 `_failed` 会让整个进程从此一条遥测都不写(比逐行丢弃严重得多), 且一旦 DBA 补上列也不会自愈——必须等重启。 """ - recorder = PostgresRecorder(least_privilege_pre_tenant_dsn) + recorder = PostgresRecorder(least_privilege_pre_tenant_dsn, auto_migrate=True) try: await _record_minimal(recorder, call_id=_cid("lpp1")) # 不得抛 assert recorder._failed is False @@ -716,7 +716,7 @@ class TestConflictTargetFreeInsert: 断言"无写入失败 warning"是为了区分"冲突被忽略"与"整条被 PG 拒收"。 """ fresh_dsn, _ = fresh_schema - recorder = PostgresRecorder(fresh_dsn) + recorder = PostgresRecorder(fresh_dsn, auto_migrate=True) try: await _record_minimal(recorder, call_id=_cid("nodup")) await _record_minimal(recorder, call_id=_cid("nodup"), response="second") @@ -737,7 +737,7 @@ class TestConflictTargetFreeInsert: 遥测全线写不进去却一声不吭,只能靠"读不回来"暴露。 """ part_dsn, _ = partitioned_schema - recorder = PostgresRecorder(part_dsn) + recorder = PostgresRecorder(part_dsn, auto_migrate=True) try: await _record_minimal(recorder, call_id=_cid("part"), tenant_id="tenant-p") assert [m for m in captured_warnings if "写入失败" in m] == [] diff --git a/tests/unit/test_telemetry.py b/tests/unit/test_telemetry.py index c3a43b8..48e7a95 100644 --- a/tests/unit/test_telemetry.py +++ b/tests/unit/test_telemetry.py @@ -115,6 +115,21 @@ async def _record_minimal(recorder, call_id="c1", **overrides): await recorder.record_llm_call(**fields) +@pytest.fixture +def captured_warnings(): + """捕获库发出的 WARNING;loguru 不经标准 logging,pytest 的 caplog 抓不到。 + + 名字避开裸 `warnings`: 那会遮蔽标准库模块名,本文件将来任何一次 + `import warnings` 都会与它静默互相顶掉,而报错点离真因很远。 + """ + from loguru import logger + + messages: list[str] = [] + sink_id = logger.add(messages.append, level="WARNING") + yield messages + logger.remove(sink_id) + + # 搬迁前(1.2.1)两个 recorder 各自持有的 INSERT 常量原文,逐字冻结在此。 # 这两条字符串是"纯搬迁不改行为"的机械证据: 构造逻辑换了地方,产物必须一字不差。 _FROZEN_SQLITE_INSERT = ( @@ -249,7 +264,7 @@ class TestBackendColumnParity: class TestSQLiteRecorder: async def test_schema_has_frozen_columns(self, tmp_path): - recorder = SQLiteRecorder(tmp_path / "t.db") + recorder = SQLiteRecorder(tmp_path / "t.db", auto_migrate=True) await _record_minimal(recorder) recorder.close() cols = [ @@ -258,7 +273,7 @@ class TestSQLiteRecorder: assert cols == _EXPECTED_COLUMNS async def test_call_id_idempotent(self, tmp_path): - recorder = SQLiteRecorder(tmp_path / "t.db") + recorder = SQLiteRecorder(tmp_path / "t.db", auto_migrate=True) await _record_minimal(recorder, call_id="dup") await _record_minimal(recorder, call_id="dup", response="second") recorder.close() @@ -270,7 +285,7 @@ class TestSQLiteRecorder: assert rows == [("ok",)] # INSERT OR IGNORE: 第二次静默忽略 async def test_concurrent_writes_all_land(self, tmp_path): - recorder = SQLiteRecorder(tmp_path / "t.db") + recorder = SQLiteRecorder(tmp_path / "t.db", auto_migrate=True) await asyncio.gather(*(_record_minimal(recorder, call_id=f"c{i}") for i in range(50))) recorder.close() (count,) = ( @@ -279,12 +294,12 @@ class TestSQLiteRecorder: assert count == 50 async def test_unwritable_path_degrades_silently(self): - recorder = SQLiteRecorder(Path("/nonexistent-root/deep/t.db")) + recorder = SQLiteRecorder(Path("/nonexistent-root/deep/t.db"), auto_migrate=True) await _record_minimal(recorder) # 不抛 recorder.close() async def test_observability_columns_round_trip(self, tmp_path): - recorder = SQLiteRecorder(tmp_path / "t.db") + recorder = SQLiteRecorder(tmp_path / "t.db", auto_migrate=True) await _record_minimal(recorder, call_id="c-hit", cached_prompt_tokens=64) await _record_minimal(recorder, call_id="c-zero", cached_prompt_tokens=0) await _record_minimal(recorder, call_id="c-none", model_reported="MiniMax-Text-01") @@ -300,7 +315,7 @@ class TestSQLiteRecorder: async def test_reasoning_tokens_column_round_trip(self, tmp_path): """issue #6: 7 / 0 / None 三种值各自如实落库,0 与 NULL 不得混同。""" - recorder = SQLiteRecorder(tmp_path / "t.db") + recorder = SQLiteRecorder(tmp_path / "t.db", auto_migrate=True) await _record_minimal(recorder, call_id="r-some", reasoning_tokens=7) await _record_minimal(recorder, call_id="r-zero", reasoning_tokens=0) await _record_minimal(recorder, call_id="r-none", reasoning_tokens=None) @@ -316,7 +331,7 @@ class TestSQLiteRecorder: async def test_sampling_column_round_trips(self, tmp_path): """issue #4: 采样参数落库,否则事后无法证明某批数据跑在什么温度下。""" - recorder = SQLiteRecorder(tmp_path / "t.db") + recorder = SQLiteRecorder(tmp_path / "t.db", auto_migrate=True) await _record_minimal(recorder, call_id="c-s", sampling='{"seed": 42, "temperature": 0}') await _record_minimal(recorder, call_id="c-plain") recorder.close() @@ -363,7 +378,7 @@ class TestSQLiteColumnBackfill: legacy.commit() legacy.close() - recorder = SQLiteRecorder(db) + recorder = SQLiteRecorder(db, auto_migrate=True) await _record_minimal(recorder, cached_prompt_tokens=7, model_reported="m-real") recorder.close() @@ -388,7 +403,7 @@ class TestSQLiteColumnBackfill: conn.commit() conn.close() - recorder = SQLiteRecorder(db) # 不得抛 + recorder = SQLiteRecorder(db, auto_migrate=True) # 不得抛 assert recorder._conn is not None # 补列失败 ≠ recorder 失能(D1 纪律) await _record_minimal(recorder) # 不得抛 recorder.close() @@ -445,7 +460,7 @@ class TestSQLiteCallerDimensionsAcceptance: async def test_fresh_db_round_trips_the_dimensions(self, tmp_path): """新建库: 列齐全,且维度值原样读回——只验列存在会漏掉写错列位的错。""" db = tmp_path / "fresh.db" - recorder = SQLiteRecorder(db) + recorder = SQLiteRecorder(db, auto_migrate=True) await _record_minimal( recorder, call_id="c-dim", tenant_id="tenant-a", meta='{"batch": "b7"}' ) @@ -471,7 +486,7 @@ class TestSQLiteCallerDimensionsAcceptance: db = tmp_path / "pre_tenant.db" _make_pre_tenant_db(db) - recorder = SQLiteRecorder(db) + recorder = SQLiteRecorder(db, auto_migrate=True) await _record_minimal(recorder, call_id="new-row", tenant_id="tenant-a", meta='{"k": 1}') recorder.close() @@ -510,7 +525,7 @@ class TestSQLiteCallerDimensionsAcceptance: # finally 还原权限位: 任一断言先失败时,不还原会让 tmp_path 清理连带报错, # 把"某条断言失败"的真因盖成一个无关的 PermissionError try: - recorder = SQLiteRecorder(db) # 不得抛 + recorder = SQLiteRecorder(db, auto_migrate=True) # 不得抛 assert recorder._conn is not None # 补列失败 ≠ recorder 失能 await _record_minimal(recorder, call_id="doomed") # 只读库写不进,但不得抛 recorder.close() @@ -523,6 +538,119 @@ class TestSQLiteCallerDimensionsAcceptance: ) # 补列确实没成功,用例不是在只读库上空转 +class TestSQLiteSchemaMode: + """issue #13: `auto_migrate` 两档——auto 保持自动补列,manual 只裁剪写入不发 DDL。 + + 列数断言一律按**物理列数**写: 旧表 22 个 INSERT 字段 + `created_at` = 23, + 补齐后 24 + `created_at` = 25。混用 INSERT 字段数与物理列数是本处最易错的地方。 + """ + + def _physical_columns(self, db: Path) -> list[str]: + conn = sqlite3.connect(db) + try: + return [r[1] for r in conn.execute("PRAGMA table_info(llm_calls)")] + finally: + conn.close() + + async def test_manual_mode_trims_the_insert_instead_of_altering( + self, tmp_path, captured_warnings + ): + """manual + 22 字段旧表: 一条 ALTER 都不发,写入按现有列裁剪后照样落库。 + + 裁剪是关掉 ALTER 的前提: 不裁剪的话每行 INSERT 都撞 `no column named + tenant_id` 而被整行丢弃——那是把自动补列换成静默全失能。 + """ + db = tmp_path / "manual_legacy.db" + _make_pre_tenant_db(db) + + recorder = SQLiteRecorder(db, auto_migrate=False) + await _record_minimal(recorder, call_id="new-row", tenant_id="tenant-a", meta='{"k": 1}') + recorder.close() + + assert len(self._physical_columns(db)) == 23 # 未 ALTER: 物理列数原封不动 + conn = sqlite3.connect(db) + assert conn.execute( + "SELECT response, model FROM llm_calls WHERE call_id = 'new-row'" + ).fetchone() == ("ok", "m") # 裁剪后的列值仍对得上位 + conn.close() + + assert len(captured_warnings) == 1 # 缺列只讲一次,不逐行刷屏 + message = captured_warnings[0] + assert "tenant_id" in message and "meta" in message # 逐列点名 + assert "不会被记录" in message # 讲清后果 + assert "ALTER TABLE" in message # 给出可直接执行的补列 SQL + + async def test_auto_mode_still_upgrades_the_legacy_table(self, tmp_path): + """auto + 同款旧表: 现状回归,补列后物理列数 23 → 25。""" + db = tmp_path / "auto_legacy.db" + _make_pre_tenant_db(db) + + recorder = SQLiteRecorder(db, auto_migrate=True) + await _record_minimal(recorder, call_id="new-row", tenant_id="tenant-a") + recorder.close() + + assert self._physical_columns(db) == _EXPECTED_COLUMNS + assert len(self._physical_columns(db)) == 25 + + async def test_manual_mode_still_creates_a_fresh_table(self, tmp_path): + """manual 只管 ALTER,不管 CREATE: 全新库照建,25 个物理列齐全(设计 §4.2)。""" + db = tmp_path / "manual_fresh.db" + recorder = SQLiteRecorder(db, auto_migrate=False) + await _record_minimal(recorder, call_id="c-fresh", tenant_id="tenant-a") + recorder.close() + + assert self._physical_columns(db) == _EXPECTED_COLUMNS + conn = sqlite3.connect(db) + assert ( + conn.execute("SELECT tenant_id FROM llm_calls WHERE call_id = 'c-fresh'").fetchone()[0] + == "tenant-a" + ) + conn.close() + + async def test_table_without_call_id_escalates_the_wording(self, tmp_path, captured_warnings): + """缺主键列 call_id = 该表压根不是本库的 llm_calls: 措辞升级,但库不做二次判定。""" + db = tmp_path / "alien.db" + conn = sqlite3.connect(db) + conn.execute("CREATE TABLE llm_calls (model TEXT, provider TEXT)") + conn.commit() + conn.close() + + recorder = SQLiteRecorder(db, auto_migrate=False) # 不得抛 + await _record_minimal(recorder) # 照常尝试写入 + recorder.close() + + message = "\n".join(captured_warnings) + assert "call_id" in message + assert "不是本库" in message + + async def test_no_recognizable_column_falls_back_to_the_full_column_set( + self, tmp_path, captured_warnings + ): + """探测结果与 COLUMNS 毫无交集视同探测异常: 保守回落全量列。 + + 空列集会构造出 `INSERT INTO llm_calls () VALUES ()` 这种语法非法的语句 + (`insert_sql` 不拒空列表——空集技术上是子集),故必须在交给它之前拦住。 + """ + from polygateway.telemetry.schema import COLUMNS + + db = tmp_path / "foreign.db" + conn = sqlite3.connect(db) + conn.execute("CREATE TABLE llm_calls (foo TEXT, bar TEXT)") + conn.commit() + conn.close() + + recorder = SQLiteRecorder(db, auto_migrate=False) # 不得抛 + assert recorder._columns == COLUMNS + await _record_minimal(recorder) # 写不进去,但只逐行 warning,不抛 + recorder.close() + assert captured_warnings # 沉默地退化成空语句是最坏结果,必须有声 + + async def test_auto_migrate_is_required_keyword_only(self, tmp_path): + """关键行为参数不给默认值(P4): 缺省规则只写在 config 一处,不与类签名漂移。""" + with pytest.raises(TypeError): + SQLiteRecorder(tmp_path / "t.db") # type: ignore[call-arg] + + class _FakePgConn: """记录执行过的语句;可让 ALTER/CREATE/探测抛错以模拟权限不足与抖动。 @@ -601,7 +729,9 @@ class TestPostgresBackfillDiscipline: def _recorder(self, conn): from polygateway.telemetry.postgres import PostgresRecorder - return PostgresRecorder("postgresql://u:p@h:5432/polygateway", pool=_FakePgPool(conn)) + return PostgresRecorder( + "postgresql://u:p@h:5432/polygateway", pool=_FakePgPool(conn), auto_migrate=True + ) async def test_alter_failure_does_not_disable_the_recorder(self): """ALTER 失败(如账号只有 INSERT 权限)不得置 _failed —— 那会让遥测全灭。""" @@ -655,7 +785,9 @@ class TestPostgresTableProbe: def _recorder(self, conn): from polygateway.telemetry.postgres import PostgresRecorder - return PostgresRecorder("postgresql://u:p@h:5432/polygateway", pool=_FakePgPool(conn)) + return PostgresRecorder( + "postgresql://u:p@h:5432/polygateway", pool=_FakePgPool(conn), auto_migrate=True + ) def _created(self, conn): return [s for s in conn.statements if s.lstrip().startswith("CREATE TABLE")] @@ -703,6 +835,60 @@ class TestPostgresTableProbe: assert [s for s in conn.statements if s.startswith("INSERT INTO llm_calls")] +class TestPostgresSchemaMode: + """issue #13: PG 侧 manual 档一条 ALTER 都不发,改按现有列裁剪 INSERT。 + + 真实 PG 的验收在 `tests/integration/test_postgres_telemetry.py`;这里用 fake 连接 + 锁住"发了哪些语句",无 DSN 环境下集成用例被 skip 时仍有回归保护。 + """ + + _LEGACY = ["call_id", "cost", "created_at"] + + def _recorder(self, conn, *, auto_migrate): + from polygateway.telemetry.postgres import PostgresRecorder + + return PostgresRecorder( + "postgresql://u:p@h:5432/polygateway", pool=_FakePgPool(conn), auto_migrate=auto_migrate + ) + + async def test_manual_mode_trims_the_insert_instead_of_altering(self, captured_warnings): + conn = _FakePgConn(self._LEGACY) + recorder = self._recorder(conn, auto_migrate=False) + await _record_minimal(recorder) + + assert not [s for s in conn.statements if s.startswith("ALTER TABLE")] + assert "INSERT INTO llm_calls (call_id, cost) VALUES ($1, $2) ON CONFLICT DO NOTHING" in ( + conn.statements + ) + assert recorder._columns == ("call_id", "cost") + message = "\n".join(captured_warnings) + assert "tenant_id" in message and "meta" in message # 逐列点名 + assert "不会被记录" in message # 讲清后果 + assert "ALTER TABLE llm_calls ADD COLUMN tenant_id" in message # 可直接执行的 SQL + + async def test_manual_mode_still_creates_a_missing_table(self): + """manual 只管 ALTER 不管 CREATE: 新建表列已齐全,写入照发全量列。""" + from polygateway.telemetry.schema import COLUMNS + + conn = _FakePgConn([]) + recorder = self._recorder(conn, auto_migrate=False) + await _record_minimal(recorder) + + assert [s for s in conn.statements if s.lstrip().startswith("CREATE TABLE")] + assert recorder._columns == COLUMNS + + async def test_auto_mode_still_backfills(self): + """auto 档现状回归: 缺列照补,补完写全量列。""" + from polygateway.telemetry.schema import COLUMNS, PG_BACKFILL + + conn = _FakePgConn(self._LEGACY) + recorder = self._recorder(conn, auto_migrate=True) + await _record_minimal(recorder) + + assert len([s for s in conn.statements if s.startswith("ALTER TABLE")]) == len(PG_BACKFILL) + assert recorder._columns == COLUMNS + + class _MemoryRecorder: def __init__(self): self.rows = [] diff --git a/tools/soak/run_soak.py b/tools/soak/run_soak.py index 53018e1..a4358db 100644 --- a/tools/soak/run_soak.py +++ b/tools/soak/run_soak.py @@ -127,7 +127,7 @@ async def _worker_async(args: argparse.Namespace, worker_idx: int) -> None: env = _merged_env() run_id = args.run_id telemetry_path = _ROOT / f"data/soak/telemetry_{run_id}_{worker_idx}.db" - recorder = SQLiteRecorder(telemetry_path) + recorder = SQLiteRecorder(telemetry_path, auto_migrate=True) if args.scenario == "P7": from polygateway.ocr import OcrClient