feat: record the observability fields end to end through telemetry
This commit is contained in:
@@ -39,6 +39,8 @@ _EXPECTED_COLUMNS = [
|
||||
"error",
|
||||
"cost",
|
||||
"created_at",
|
||||
"cached_prompt_tokens",
|
||||
"model_reported",
|
||||
]
|
||||
|
||||
# run 级前缀: 同库并存的其他运行(迁移批跑/另一开发机)互不可见
|
||||
@@ -100,6 +102,8 @@ async def _record_minimal(
|
||||
"cache_hit": False,
|
||||
"error": None,
|
||||
"cost": None,
|
||||
"cached_prompt_tokens": None,
|
||||
"model_reported": None,
|
||||
}
|
||||
fields.update(overrides)
|
||||
await recorder.record_llm_call(**fields)
|
||||
|
||||
@@ -114,6 +114,8 @@ class _DummyRecorder:
|
||||
cache_hit,
|
||||
error,
|
||||
cost,
|
||||
cached_prompt_tokens,
|
||||
model_reported,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
|
||||
@@ -35,6 +35,8 @@ _EXPECTED_COLUMNS = [
|
||||
"error",
|
||||
"cost",
|
||||
"created_at",
|
||||
"cached_prompt_tokens",
|
||||
"model_reported",
|
||||
]
|
||||
|
||||
|
||||
@@ -93,6 +95,8 @@ async def _record_minimal(recorder, call_id="c1", **overrides):
|
||||
"cache_hit": False,
|
||||
"error": None,
|
||||
"cost": None,
|
||||
"cached_prompt_tokens": None,
|
||||
"model_reported": None,
|
||||
}
|
||||
fields.update(overrides)
|
||||
await recorder.record_llm_call(**fields)
|
||||
@@ -134,6 +138,84 @@ class TestSQLiteRecorder:
|
||||
await _record_minimal(recorder) # 不抛
|
||||
recorder.close()
|
||||
|
||||
async def test_observability_columns_round_trip(self, tmp_path):
|
||||
recorder = SQLiteRecorder(tmp_path / "t.db")
|
||||
await _record_minimal(recorder, call_id="c-hit", cached_prompt_tokens=64)
|
||||
await _record_minimal(recorder, call_id="c-zero", cached_prompt_tokens=0)
|
||||
await _record_minimal(recorder, call_id="c-none", model_reported="MiniMax-Text-01")
|
||||
recorder.close()
|
||||
rows = dict(
|
||||
sqlite3.connect(tmp_path / "t.db")
|
||||
.execute("SELECT call_id, cached_prompt_tokens FROM llm_calls")
|
||||
.fetchall()
|
||||
)
|
||||
assert rows["c-hit"] == 64
|
||||
assert rows["c-zero"] == 0 # 真实零命中,读回仍是 0 而非 NULL
|
||||
assert rows["c-none"] is None
|
||||
|
||||
|
||||
class TestSQLiteColumnBackfill:
|
||||
"""issue #3: 已存在的 18 列旧表必须自动补列,否则每行写入都被丢弃。"""
|
||||
|
||||
_LEGACY_DDL = """
|
||||
CREATE TABLE llm_calls (
|
||||
call_id TEXT PRIMARY KEY,
|
||||
parent_call_id TEXT,
|
||||
session_id TEXT,
|
||||
model TEXT NOT NULL,
|
||||
provider TEXT NOT NULL,
|
||||
source_name TEXT NOT NULL,
|
||||
messages TEXT NOT NULL,
|
||||
response TEXT NOT NULL,
|
||||
thinking TEXT NOT NULL DEFAULT '',
|
||||
prompt_tokens INTEGER NOT NULL,
|
||||
completion_tokens INTEGER NOT NULL,
|
||||
usage_source TEXT NOT NULL,
|
||||
latency_ms INTEGER NOT NULL,
|
||||
ttft_ms REAL,
|
||||
max_inter_token_ms REAL,
|
||||
cache_hit INTEGER NOT NULL DEFAULT 0,
|
||||
error TEXT,
|
||||
cost REAL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
"""
|
||||
|
||||
async def test_legacy_table_is_upgraded_in_place(self, tmp_path):
|
||||
db = tmp_path / "legacy.db"
|
||||
legacy = sqlite3.connect(db)
|
||||
legacy.execute(self._LEGACY_DDL)
|
||||
legacy.commit()
|
||||
legacy.close()
|
||||
|
||||
recorder = SQLiteRecorder(db)
|
||||
await _record_minimal(recorder, cached_prompt_tokens=7, model_reported="m-real")
|
||||
recorder.close()
|
||||
|
||||
conn = sqlite3.connect(db)
|
||||
cols = [r[1] for r in conn.execute("PRAGMA table_info(llm_calls)")]
|
||||
assert cols == _EXPECTED_COLUMNS # ALTER 追加到末尾,与新建库列序一致
|
||||
assert conn.execute(
|
||||
"SELECT cached_prompt_tokens, model_reported FROM llm_calls"
|
||||
).fetchone() == (7, "m-real")
|
||||
|
||||
async def test_backfill_failure_keeps_the_recorder_usable(self, tmp_path):
|
||||
"""补列失败只能逐行降级,绝不能把 recorder 整体变成 no-op(设计 D1 纪律)。
|
||||
|
||||
把 llm_calls 建成 view: 表不存在故 CREATE TABLE IF NOT EXISTS 会撞名失败,
|
||||
ALTER 也无从谈起——这是最坏路径。
|
||||
"""
|
||||
db = tmp_path / "view.db"
|
||||
conn = sqlite3.connect(db)
|
||||
conn.execute("CREATE TABLE real_rows (call_id TEXT)")
|
||||
conn.execute("CREATE VIEW llm_calls AS SELECT call_id FROM real_rows")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
recorder = SQLiteRecorder(db) # 不得抛
|
||||
await _record_minimal(recorder) # 不得抛
|
||||
recorder.close()
|
||||
|
||||
|
||||
class _MemoryRecorder:
|
||||
def __init__(self):
|
||||
@@ -143,6 +225,155 @@ class _MemoryRecorder:
|
||||
self.rows.append(fields)
|
||||
|
||||
|
||||
class TestEmitterRecorderContract:
|
||||
"""emitter 的实参键集合必须与两个后端的 _COLUMNS 完全一致(issue #3)。
|
||||
|
||||
两个后端的 `row = tuple(fields[col] for col in _COLUMNS)` 都在 try **之外**,
|
||||
emitter 漏传一个键就抛 KeyError,被 `_record` 的 except Exception 吞成 warning
|
||||
→ 遥测静默全丢。而 8 个 `**fields` 形态的 fake 一个都拦不住,故显式断言。
|
||||
"""
|
||||
|
||||
async def test_emitter_supplies_exactly_the_backend_columns(self):
|
||||
from polygateway.telemetry.postgres import _COLUMNS as PG_COLUMNS
|
||||
from polygateway.telemetry.sqlite import _COLUMNS as SQLITE_COLUMNS
|
||||
|
||||
rec = _MemoryRecorder()
|
||||
await TelemetryEmitter(rec).emit_attempt(
|
||||
request=_REQ,
|
||||
source=_source(),
|
||||
call_id="cid-1",
|
||||
latency_ms=42,
|
||||
response=_resp(),
|
||||
error=None,
|
||||
)
|
||||
assert set(rec.rows[0]) == set(SQLITE_COLUMNS) == set(PG_COLUMNS)
|
||||
|
||||
@pytest.mark.parametrize("emit", ["attempt", "cache_hit", "terminal_failure"])
|
||||
async def test_every_entry_point_supplies_the_same_keys(self, emit):
|
||||
from polygateway.telemetry.sqlite import _COLUMNS as SQLITE_COLUMNS
|
||||
|
||||
rec = _MemoryRecorder()
|
||||
emitter = TelemetryEmitter(rec)
|
||||
if emit == "attempt":
|
||||
await emitter.emit_attempt(
|
||||
request=_REQ,
|
||||
source=_source(),
|
||||
call_id="c",
|
||||
latency_ms=1,
|
||||
response=None,
|
||||
error="boom",
|
||||
)
|
||||
elif emit == "cache_hit":
|
||||
await emitter.emit_cache_hit(request=_REQ, response=_resp())
|
||||
else:
|
||||
await emitter.emit_terminal_failure(
|
||||
request=_REQ, call_id="c", latency_ms=1, error="dead"
|
||||
)
|
||||
assert set(rec.rows[0]) == set(SQLITE_COLUMNS)
|
||||
|
||||
|
||||
class TestEmitterObservabilityFields:
|
||||
"""issue #3: 三个入口各自的取值口径(设计 §5 表)。"""
|
||||
|
||||
async def test_attempt_carries_the_response_values(self):
|
||||
rec = _MemoryRecorder()
|
||||
await TelemetryEmitter(rec).emit_attempt(
|
||||
request=_REQ,
|
||||
source=_source(),
|
||||
call_id="cid-1",
|
||||
latency_ms=42,
|
||||
response=_resp(cached_prompt_tokens=64, model_reported="m-real"),
|
||||
error=None,
|
||||
)
|
||||
assert rec.rows[0]["cached_prompt_tokens"] == 64
|
||||
assert rec.rows[0]["model_reported"] == "m-real"
|
||||
|
||||
async def test_failed_attempt_has_no_provider_facts(self):
|
||||
rec = _MemoryRecorder()
|
||||
await TelemetryEmitter(rec).emit_attempt(
|
||||
request=_REQ,
|
||||
source=_source(),
|
||||
call_id="cid-2",
|
||||
latency_ms=7,
|
||||
response=None,
|
||||
error="boom",
|
||||
)
|
||||
assert rec.rows[0]["cached_prompt_tokens"] is None
|
||||
assert rec.rows[0]["model_reported"] is None
|
||||
|
||||
async def test_cache_hit_replays_the_recorded_values(self):
|
||||
"""决策 B1: 命中行原样回放,故命中率统计必须带 WHERE cache_hit = false。"""
|
||||
rec = _MemoryRecorder()
|
||||
await TelemetryEmitter(rec).emit_cache_hit(
|
||||
request=_REQ, response=_resp(cached_prompt_tokens=64, model_reported="m-real")
|
||||
)
|
||||
row = rec.rows[0]
|
||||
assert row["cache_hit"] is True
|
||||
assert row["cached_prompt_tokens"] == 64 and row["model_reported"] == "m-real"
|
||||
|
||||
async def test_terminal_failure_records_none(self):
|
||||
rec = _MemoryRecorder()
|
||||
await TelemetryEmitter(rec).emit_terminal_failure(
|
||||
request=_REQ, call_id="c", latency_ms=1, error="dead"
|
||||
)
|
||||
assert rec.rows[0]["cached_prompt_tokens"] is None
|
||||
assert rec.rows[0]["model_reported"] is None
|
||||
|
||||
|
||||
class TestCostWithCachedTier:
|
||||
"""issue #3: 命中部分按缓存单价计费,避免 cost 系统性高估。"""
|
||||
|
||||
_TABLE = PricingTable(
|
||||
{"m": ModelPrice(input_per_1m=10.0, output_per_1m=20.0, cached_input_per_1m=2.0)}
|
||||
)
|
||||
|
||||
async def test_cached_hit_lowers_the_recorded_cost(self):
|
||||
rec = _MemoryRecorder()
|
||||
emitter = TelemetryEmitter(rec, pricing=self._TABLE)
|
||||
full = _resp(prompt_tokens=1_000_000, completion_tokens=0)
|
||||
await emitter.emit_attempt(
|
||||
request=_REQ,
|
||||
source=_source(),
|
||||
call_id="c1",
|
||||
latency_ms=1,
|
||||
response=full,
|
||||
error=None,
|
||||
)
|
||||
await emitter.emit_attempt(
|
||||
request=_REQ,
|
||||
source=_source(),
|
||||
call_id="c2",
|
||||
latency_ms=1,
|
||||
response=_resp(
|
||||
prompt_tokens=1_000_000, completion_tokens=0, cached_prompt_tokens=600_000
|
||||
),
|
||||
error=None,
|
||||
)
|
||||
assert rec.rows[0]["cost"] == pytest.approx(10.0)
|
||||
assert rec.rows[1]["cost"] == pytest.approx(5.2) # 400k×10 + 600k×2
|
||||
|
||||
async def test_cache_hit_row_still_costs_zero(self):
|
||||
"""缓存命中未产生新调用 → cost 恒 0.0,该短路必须排在任何换算之前。"""
|
||||
rec = _MemoryRecorder()
|
||||
await TelemetryEmitter(rec, pricing=self._TABLE).emit_cache_hit(
|
||||
request=_REQ,
|
||||
response=_resp(prompt_tokens=1_000_000, cached_prompt_tokens=600_000),
|
||||
)
|
||||
assert rec.rows[0]["cost"] == 0.0
|
||||
|
||||
async def test_unavailable_usage_still_costs_none(self):
|
||||
rec = _MemoryRecorder()
|
||||
await TelemetryEmitter(rec, pricing=self._TABLE).emit_attempt(
|
||||
request=_REQ,
|
||||
source=_source(),
|
||||
call_id="c",
|
||||
latency_ms=1,
|
||||
response=_resp(usage_source="unavailable", cached_prompt_tokens=5),
|
||||
error=None,
|
||||
)
|
||||
assert rec.rows[0]["cost"] is None
|
||||
|
||||
|
||||
class TestEmitter:
|
||||
async def test_attempt_success_row(self):
|
||||
rec = _MemoryRecorder()
|
||||
|
||||
Reference in New Issue
Block a user