feat: record call observability columns and terminal failure rows

Grow the telemetry contract from 26 to 36 fields and give every logical
call a failure terminal row, so SQL can finally answer "how many calls
failed" and "why did the whole pool die".

Schema and port move together with the emitter writes in one commit:
splitting them would ship columns that nothing populates.

- schema: append 10 nullable columns (scope, operation, logical_call_id,
  event_kind, http_status_code, error_type, cause_type, error_body,
  attempts, total_latency_ms) to all five definition sites in one order
- ports: 10 keyword-only parameters without defaults; the protocol
  signature is now the single source the assembly gate derives from
- emitter: take domain exception objects instead of pre-flattened text
  and pin down the diagnostics in one helper; a relabelled 503 stays
  503 and success rows leave all five columns NULL
- emitter: reject recorders whose record_llm_call cannot accept the
  current field shape at assembly time, since _record would otherwise
  swallow the TypeError and drop every row while calls keep succeeding
- clients: write at most one terminal row per logical call through a
  single shared exit, deduplicated by the call context; TelemetryMW
  stops writing terminals so the two sites cannot double count
- clients: cancellation stays best effort and propagates, non-domain
  exceptions get no terminal row and keep their classification
- transports: give _status_to_error an explicit operation and fix the
  historically mislabelled embedding HTTP failures
- structured: promote the bounded error formatter so the reask feedback
  and the terminal explanation share one set of limits

Terminal rows carry no cost and no tokens, so cost aggregation is
unchanged; failure counts must now filter on event_kind.
This commit is contained in:
2026-09-09 11:27:52 -04:00
parent 87c261bf73
commit 393f2bf617
19 changed files with 1628 additions and 194 deletions
+147 -1
View File
@@ -13,6 +13,7 @@ from polygateway import (
GatewayClient,
GatewaySettings,
RequestRejectedError,
ResultInvalidError,
gather_bounded,
)
from polygateway.backends.memory.breaker import InMemoryGate
@@ -25,6 +26,7 @@ from polygateway.transports.openai_compat import OpenAICompatTransport
from polygateway.types import (
BackpressurePolicy,
BreakerConfig,
ChatRequest,
Effort,
GlobalLimits,
RetryPolicy,
@@ -846,11 +848,19 @@ _CACHE_ENV = dict(
class _Closable:
"""记 close 次数的假组件;所有权纪律的唯一观测点。"""
"""记 close 次数的假组件;所有权纪律的唯一观测点。
带 `record_llm_call(**fields)` 是因为它也被当作注入的 telemetry recorder 用:
1.3.5 的装配闸在构造期就会拒掉不满足 `TelemetryRecorder` 的对象
(否则下游升级后 100% 丢遥测而调用照常成功)。`**fields` 形态天然兼容。
"""
def __init__(self):
self.closed = 0
async def record_llm_call(self, **fields):
pass
async def aclose(self):
self.closed += 1
@@ -861,6 +871,9 @@ class _SyncClosable:
def __init__(self):
self.closed = 0
async def record_llm_call(self, **fields):
pass
def close(self):
self.closed += 1
@@ -1499,3 +1512,136 @@ class TestLogicalCallStats:
async with _client() as client:
with pytest.raises(ValueError, match="meta"):
await client.chat(self._MSG, meta={"BAD-KEY": 1})
class TestChatTerminalFailureRows:
"""chat 链路的**真实**终态行(1.3.5 设计 §6;补漏而非改口径)。
这些路径改前一条失败行都没有(结构化耗尽)或只有尝试行,
"这次调用到底失败了几次"因此 SQL 答不出来。
"""
_MSG = [{"role": "user", "content": "hi"}]
def _rows(self, recorder, kind):
return [r for r in recorder.rows if r["event_kind"] == kind]
async def test_structured_exhaustion_writes_the_only_failure_row(self):
"""结构化耗尽发生在 transport 成功之后: attempt 行全是成功行,终态是唯一记录。"""
from pydantic import BaseModel
class Answer(BaseModel):
answer: int
recorder = _MemoryRecorder()
async with _client(
handler=lambda request: _sse("not json at all"),
telemetry=recorder,
structured_max_retries=1,
) as client:
with pytest.raises(ResultInvalidError):
await client.chat(self._MSG, structured=Answer)
attempts = self._rows(recorder, "attempt")
terminals = self._rows(recorder, "terminal_failure")
assert [a["error"] for a in attempts] == [None, None] # 两次尝试都成功
assert len(terminals) == 1
row = terminals[0]
assert row["error_type"] == "ResultInvalidError"
# C2: 有界结构化说明并入 error,且不含 raw_text(正文预算已由 attempt 行承担)
assert "validation=" in row["error"] or "repair=" in row["error"]
assert len(row["error"]) < 1200
async def test_retry_exhaustion_writes_exactly_one_terminal_row(self):
"""重试耗尽: 逐次 attempt 行之外只能有**一条**终态行。
终态行的 `attempts` 是整次逻辑调用的真实尝试数——这正是改前 SQL
答不出的"这次调用到底重试了几次"
"""
recorder = _MemoryRecorder()
async with _client(
handler=lambda request: httpx.Response(503),
telemetry=recorder,
retry=RetryPolicy(3, 0.001, 0.01),
) as client:
with pytest.raises(AllSourcesExhausted):
await client.chat(self._MSG)
attempts = self._rows(recorder, "attempt")
terminals = self._rows(recorder, "terminal_failure")
assert len(attempts) == 3
assert len(terminals) == 1
row = terminals[0]
assert row["error_type"] == "AllSourcesExhausted"
assert row["attempts"] == 3 # 整次逻辑调用的尝试数
assert row["scope"] == "llm" and row["operation"] == "chat"
# 终态行的 latency_ms 与 total_latency_ms 同取一份冻结快照
assert row["latency_ms"] == row["total_latency_ms"]
async def test_request_rejected_now_has_both_an_attempt_and_a_terminal_row(self):
"""已批准的下游可见变化: 400 密集负载的错误行翻倍,失败计数只能取终态。"""
recorder = _MemoryRecorder()
async with _client(
handler=lambda request: httpx.Response(400, json={"error": {"message": "bad"}}),
telemetry=recorder,
) as client:
with pytest.raises(RequestRejectedError):
await client.chat(self._MSG)
attempts = self._rows(recorder, "attempt")
terminals = self._rows(recorder, "terminal_failure")
assert len(attempts) == 1 and attempts[0]["http_status_code"] == 400
assert len(terminals) == 1
# C1: 终态不搬运最后一次 attempt 的状态码与正文
assert terminals[0]["http_status_code"] is None
assert terminals[0]["error_body"] is None
# 两类行共享同一 logical_call_id,归因查询才连得起来
assert attempts[0]["logical_call_id"] == terminals[0]["logical_call_id"]
async def test_non_domain_exception_writes_no_terminal_row(self):
"""编程错原样传播,本版**不承诺**任何统计或终态行,也不偷偷改分类。"""
recorder = _MemoryRecorder()
async def boom(request):
raise KeyError("programming error")
async with _client(telemetry=recorder) as client:
client._handler = boom
with pytest.raises(KeyError):
await client.chat(self._MSG)
assert self._rows(recorder, "terminal_failure") == []
async def test_cancellation_writes_at_most_one_terminal_row(self):
"""取消尽力写一条(允许 0),且 `CancelledError` 类型与语义不变。"""
recorder = _MemoryRecorder()
async def hang(request):
raise asyncio.CancelledError
async with _client(telemetry=recorder) as client:
client._handler = hang
with pytest.raises(asyncio.CancelledError):
await client.chat(self._MSG)
terminals = self._rows(recorder, "terminal_failure")
assert len(terminals) == 1
assert terminals[0]["error"] == "cancelled"
# 字符串不解析猜诊断
assert terminals[0]["error_type"] is None
async def test_terminal_row_is_written_once_per_logical_call(self):
"""`claim_terminal` 去重: 同一次调用即便出口被多次触达也只有一条。"""
from polygateway.middleware.telemetry import emit_terminal_once
from polygateway.types import _CallContext
recorder = _MemoryRecorder()
async with _client(telemetry=recorder) as client:
context = _CallContext(now=client._now)
request = ChatRequest(messages=self._MSG, call_context=context)
for _ in range(3):
await emit_terminal_once(
client._emitter,
request=request,
context=context,
error=AllSourcesExhausted(
scope="llm", reason="retry_exhausted", retry_after_s=1.0
),
operation="chat",
)
assert len(self._rows(recorder, "terminal_failure")) == 1