feat: record call observability columns and terminal failure rows
Grow the telemetry contract from 26 to 36 fields and give every logical call a failure terminal row, so SQL can finally answer "how many calls failed" and "why did the whole pool die". Schema and port move together with the emitter writes in one commit: splitting them would ship columns that nothing populates. - schema: append 10 nullable columns (scope, operation, logical_call_id, event_kind, http_status_code, error_type, cause_type, error_body, attempts, total_latency_ms) to all five definition sites in one order - ports: 10 keyword-only parameters without defaults; the protocol signature is now the single source the assembly gate derives from - emitter: take domain exception objects instead of pre-flattened text and pin down the diagnostics in one helper; a relabelled 503 stays 503 and success rows leave all five columns NULL - emitter: reject recorders whose record_llm_call cannot accept the current field shape at assembly time, since _record would otherwise swallow the TypeError and drop every row while calls keep succeeding - clients: write at most one terminal row per logical call through a single shared exit, deduplicated by the call context; TelemetryMW stops writing terminals so the two sites cannot double count - clients: cancellation stays best effort and propagates, non-domain exceptions get no terminal row and keep their classification - transports: give _status_to_error an explicit operation and fix the historically mislabelled embedding HTTP failures - structured: promote the bounded error formatter so the reask feedback and the terminal explanation share one set of limits Terminal rows carry no cost and no tokens, so cost aggregation is unchanged; failure counts must now filter on event_kind.
This commit is contained in:
+147
-1
@@ -13,6 +13,7 @@ from polygateway import (
|
||||
GatewayClient,
|
||||
GatewaySettings,
|
||||
RequestRejectedError,
|
||||
ResultInvalidError,
|
||||
gather_bounded,
|
||||
)
|
||||
from polygateway.backends.memory.breaker import InMemoryGate
|
||||
@@ -25,6 +26,7 @@ from polygateway.transports.openai_compat import OpenAICompatTransport
|
||||
from polygateway.types import (
|
||||
BackpressurePolicy,
|
||||
BreakerConfig,
|
||||
ChatRequest,
|
||||
Effort,
|
||||
GlobalLimits,
|
||||
RetryPolicy,
|
||||
@@ -846,11 +848,19 @@ _CACHE_ENV = dict(
|
||||
|
||||
|
||||
class _Closable:
|
||||
"""记 close 次数的假组件;所有权纪律的唯一观测点。"""
|
||||
"""记 close 次数的假组件;所有权纪律的唯一观测点。
|
||||
|
||||
带 `record_llm_call(**fields)` 是因为它也被当作注入的 telemetry recorder 用:
|
||||
1.3.5 的装配闸在构造期就会拒掉不满足 `TelemetryRecorder` 的对象
|
||||
(否则下游升级后 100% 丢遥测而调用照常成功)。`**fields` 形态天然兼容。
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.closed = 0
|
||||
|
||||
async def record_llm_call(self, **fields):
|
||||
pass
|
||||
|
||||
async def aclose(self):
|
||||
self.closed += 1
|
||||
|
||||
@@ -861,6 +871,9 @@ class _SyncClosable:
|
||||
def __init__(self):
|
||||
self.closed = 0
|
||||
|
||||
async def record_llm_call(self, **fields):
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
self.closed += 1
|
||||
|
||||
@@ -1499,3 +1512,136 @@ class TestLogicalCallStats:
|
||||
async with _client() as client:
|
||||
with pytest.raises(ValueError, match="meta"):
|
||||
await client.chat(self._MSG, meta={"BAD-KEY": 1})
|
||||
|
||||
|
||||
class TestChatTerminalFailureRows:
|
||||
"""chat 链路的**真实**终态行(1.3.5 设计 §6;补漏而非改口径)。
|
||||
|
||||
这些路径改前一条失败行都没有(结构化耗尽)或只有尝试行,
|
||||
"这次调用到底失败了几次"因此 SQL 答不出来。
|
||||
"""
|
||||
|
||||
_MSG = [{"role": "user", "content": "hi"}]
|
||||
|
||||
def _rows(self, recorder, kind):
|
||||
return [r for r in recorder.rows if r["event_kind"] == kind]
|
||||
|
||||
async def test_structured_exhaustion_writes_the_only_failure_row(self):
|
||||
"""结构化耗尽发生在 transport 成功之后: attempt 行全是成功行,终态是唯一记录。"""
|
||||
from pydantic import BaseModel
|
||||
|
||||
class Answer(BaseModel):
|
||||
answer: int
|
||||
|
||||
recorder = _MemoryRecorder()
|
||||
async with _client(
|
||||
handler=lambda request: _sse("not json at all"),
|
||||
telemetry=recorder,
|
||||
structured_max_retries=1,
|
||||
) as client:
|
||||
with pytest.raises(ResultInvalidError):
|
||||
await client.chat(self._MSG, structured=Answer)
|
||||
attempts = self._rows(recorder, "attempt")
|
||||
terminals = self._rows(recorder, "terminal_failure")
|
||||
assert [a["error"] for a in attempts] == [None, None] # 两次尝试都成功
|
||||
assert len(terminals) == 1
|
||||
row = terminals[0]
|
||||
assert row["error_type"] == "ResultInvalidError"
|
||||
# C2: 有界结构化说明并入 error,且不含 raw_text(正文预算已由 attempt 行承担)
|
||||
assert "validation=" in row["error"] or "repair=" in row["error"]
|
||||
assert len(row["error"]) < 1200
|
||||
|
||||
async def test_retry_exhaustion_writes_exactly_one_terminal_row(self):
|
||||
"""重试耗尽: 逐次 attempt 行之外只能有**一条**终态行。
|
||||
|
||||
终态行的 `attempts` 是整次逻辑调用的真实尝试数——这正是改前 SQL
|
||||
答不出的"这次调用到底重试了几次"。
|
||||
"""
|
||||
recorder = _MemoryRecorder()
|
||||
async with _client(
|
||||
handler=lambda request: httpx.Response(503),
|
||||
telemetry=recorder,
|
||||
retry=RetryPolicy(3, 0.001, 0.01),
|
||||
) as client:
|
||||
with pytest.raises(AllSourcesExhausted):
|
||||
await client.chat(self._MSG)
|
||||
attempts = self._rows(recorder, "attempt")
|
||||
terminals = self._rows(recorder, "terminal_failure")
|
||||
assert len(attempts) == 3
|
||||
assert len(terminals) == 1
|
||||
row = terminals[0]
|
||||
assert row["error_type"] == "AllSourcesExhausted"
|
||||
assert row["attempts"] == 3 # 整次逻辑调用的尝试数
|
||||
assert row["scope"] == "llm" and row["operation"] == "chat"
|
||||
# 终态行的 latency_ms 与 total_latency_ms 同取一份冻结快照
|
||||
assert row["latency_ms"] == row["total_latency_ms"]
|
||||
|
||||
async def test_request_rejected_now_has_both_an_attempt_and_a_terminal_row(self):
|
||||
"""已批准的下游可见变化: 400 密集负载的错误行翻倍,失败计数只能取终态。"""
|
||||
recorder = _MemoryRecorder()
|
||||
async with _client(
|
||||
handler=lambda request: httpx.Response(400, json={"error": {"message": "bad"}}),
|
||||
telemetry=recorder,
|
||||
) as client:
|
||||
with pytest.raises(RequestRejectedError):
|
||||
await client.chat(self._MSG)
|
||||
attempts = self._rows(recorder, "attempt")
|
||||
terminals = self._rows(recorder, "terminal_failure")
|
||||
assert len(attempts) == 1 and attempts[0]["http_status_code"] == 400
|
||||
assert len(terminals) == 1
|
||||
# C1: 终态不搬运最后一次 attempt 的状态码与正文
|
||||
assert terminals[0]["http_status_code"] is None
|
||||
assert terminals[0]["error_body"] is None
|
||||
# 两类行共享同一 logical_call_id,归因查询才连得起来
|
||||
assert attempts[0]["logical_call_id"] == terminals[0]["logical_call_id"]
|
||||
|
||||
async def test_non_domain_exception_writes_no_terminal_row(self):
|
||||
"""编程错原样传播,本版**不承诺**任何统计或终态行,也不偷偷改分类。"""
|
||||
recorder = _MemoryRecorder()
|
||||
|
||||
async def boom(request):
|
||||
raise KeyError("programming error")
|
||||
|
||||
async with _client(telemetry=recorder) as client:
|
||||
client._handler = boom
|
||||
with pytest.raises(KeyError):
|
||||
await client.chat(self._MSG)
|
||||
assert self._rows(recorder, "terminal_failure") == []
|
||||
|
||||
async def test_cancellation_writes_at_most_one_terminal_row(self):
|
||||
"""取消尽力写一条(允许 0),且 `CancelledError` 类型与语义不变。"""
|
||||
recorder = _MemoryRecorder()
|
||||
|
||||
async def hang(request):
|
||||
raise asyncio.CancelledError
|
||||
|
||||
async with _client(telemetry=recorder) as client:
|
||||
client._handler = hang
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await client.chat(self._MSG)
|
||||
terminals = self._rows(recorder, "terminal_failure")
|
||||
assert len(terminals) == 1
|
||||
assert terminals[0]["error"] == "cancelled"
|
||||
# 字符串不解析猜诊断
|
||||
assert terminals[0]["error_type"] is None
|
||||
|
||||
async def test_terminal_row_is_written_once_per_logical_call(self):
|
||||
"""`claim_terminal` 去重: 同一次调用即便出口被多次触达也只有一条。"""
|
||||
from polygateway.middleware.telemetry import emit_terminal_once
|
||||
from polygateway.types import _CallContext
|
||||
|
||||
recorder = _MemoryRecorder()
|
||||
async with _client(telemetry=recorder) as client:
|
||||
context = _CallContext(now=client._now)
|
||||
request = ChatRequest(messages=self._MSG, call_context=context)
|
||||
for _ in range(3):
|
||||
await emit_terminal_once(
|
||||
client._emitter,
|
||||
request=request,
|
||||
context=context,
|
||||
error=AllSourcesExhausted(
|
||||
scope="llm", reason="retry_exhausted", retry_after_s=1.0
|
||||
),
|
||||
operation="chat",
|
||||
)
|
||||
assert len(self._rows(recorder, "terminal_failure")) == 1
|
||||
|
||||
Reference in New Issue
Block a user