iomgaa/PolyGateway
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 13 Wiki Activity

feat: record call observability columns and terminal failure rows

Browse Source 
Grow the telemetry contract from 26 to 36 fields and give every logical
call a failure terminal row, so SQL can finally answer "how many calls
failed" and "why did the whole pool die".

Schema and port move together with the emitter writes in one commit:
splitting them would ship columns that nothing populates.

- schema: append 10 nullable columns (scope, operation, logical_call_id,
  event_kind, http_status_code, error_type, cause_type, error_body,
  attempts, total_latency_ms) to all five definition sites in one order
- ports: 10 keyword-only parameters without defaults; the protocol
  signature is now the single source the assembly gate derives from
- emitter: take domain exception objects instead of pre-flattened text
  and pin down the diagnostics in one helper; a relabelled 503 stays
  503 and success rows leave all five columns NULL
- emitter: reject recorders whose record_llm_call cannot accept the
  current field shape at assembly time, since _record would otherwise
  swallow the TypeError and drop every row while calls keep succeeding
- clients: write at most one terminal row per logical call through a
  single shared exit, deduplicated by the call context; TelemetryMW
  stops writing terminals so the two sites cannot double count
- clients: cancellation stays best effort and propagates, non-domain
  exceptions get no terminal row and keep their classification
- transports: give _status_to_error an explicit operation and fix the
  historically mislabelled embedding HTTP failures
- structured: promote the bounded error formatter so the reask feedback
  and the terminal explanation share one set of limits

Terminal rows carry no cost and no tokens, so cost aggregation is
unchanged; failure counts must now filter on event_kind.
This commit is contained in:
iomgaa
2026-09-09 11:27:52 -04:00
parent 87c261bf73
commit 393f2bf617
19 changed files with 1628 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/polygateway/ocr.py
+130 -19
View File
@@ -37,7 +37,7 @@ from polygateway.middleware.admission import SourceAdmission, settle_and_release
from polygateway.middleware.breaker import BreakerGate
from polygateway.middleware.ratelimit import QuotaGate
from polygateway.middleware.retry import StallClock, _failure_reason, backoff_delay
from polygateway.middleware.telemetry import TelemetryEmitter
from polygateway.middleware.telemetry import TelemetryEmitter, emit_terminal_once
from polygateway.ports import OutcomeAwareSelector
from polygateway.types import (
CallStats,
@@ -67,6 +67,7 @@ if TYPE_CHECKING:
)
from polygateway.types import (
BackpressurePolicy,
CallOperation,
OcrLayoutTransportResult,
OcrTextTransportResult,
RetryPolicy,
@@ -128,7 +129,9 @@ class OcrClient:
self._breaker = BreakerGate(breaker, scope=self._scope)
self._transport = transport
self._retry = retry
self._emitter = TelemetryEmitter(telemetry, text_cap=text_cap) if telemetry else None
self._emitter = (
TelemetryEmitter(telemetry, scope=self._scope, text_cap=text_cap) if telemetry else None
)
self._telemetry = telemetry
# 限流/熔断后端在此之外只以 QuotaGate/BreakerGate 的形态存在,自持一份
# 引用才关得到自建的 redis 客户端(设计 §3.4)
@@ -179,7 +182,13 @@ class OcrClient:
tenant_id, meta, origin="recognize_text(tenant_id=..., meta=...)"
)
outcome, call_stats = await self._call(
"text", image, session_id, parent_call_id, dimension_tenant_id, dimensions
"text",
"recognize_text",
image,
session_id,
parent_call_id,
dimension_tenant_id,
dimensions,
)
result = outcome.result
return OcrTextResult(
@@ -210,7 +219,13 @@ class OcrClient:
tenant_id, meta, origin="parse_layout(tenant_id=..., meta=...)"
)
outcome, call_stats = await self._call(
"layout", image, session_id, parent_call_id, dimension_tenant_id, dimensions
"layout",
"parse_layout",
image,
session_id,
parent_call_id,
dimension_tenant_id,
dimensions,
)
result = outcome.result
return OcrLayoutResult(
@@ -241,6 +256,7 @@ class OcrClient:
async def _call(
self,
kind: _OcrKind,
operation: CallOperation,
image: bytes,
session_id: str | None,
parent_call_id: str | None,
@@ -254,6 +270,46 @@ class OcrClient:
# M1 例外: `image` 校验在 `_call` 内而非公开方法,故上下文在该校验
# **通过之后**创建——这样设计 §3 的"校验在统计边界外"对 OCR 才成立
context = _CallContext(now=self._now)
try:
return await self._run(
kind, operation, image, session_id, parent_call_id, tenant_id, meta, context
)
except PolyGatewayError as exc:
await self._emit_terminal(
kind, operation, image, session_id, parent_call_id, tenant_id, meta, context, exc
)
raise
except asyncio.CancelledError:
# 三条链路同一口径尽力写一条(允许 0 条);取消优先,不 shield
await self._emit_terminal(
kind,
operation,
image,
session_id,
parent_call_id,
tenant_id,
meta,
context,
"cancelled",
)
raise
async def _run(
self,
kind: _OcrKind,
operation: CallOperation,
image: bytes,
session_id: str | None,
parent_call_id: str | None,
tenant_id: str | None,
meta: dict[str, Any],
context: _CallContext,
) -> tuple[_AttemptOutcome, CallStats]:
"""选源与重试循环。
无源的 raise 必须在本方法内(而非循环之前的调用方): 它得被 `_call` 的
`try` 包住,否则无源终态行根本写不出来(设计 §3.5)。
"""
if not self._sources:
raise AllSourcesExhausted(scope=self._scope, reason="no_sources", retry_after_s=0.0)
fails = 0
@@ -268,6 +324,7 @@ class OcrClient:
async with clock.attempting():
outcome = await self._attempt(
kind,
operation,
image,
*picked,
reasons,
@@ -293,6 +350,7 @@ class OcrClient:
async def _attempt(
self,
kind: _OcrKind,
operation: CallOperation,
image: bytes,
source: SourceConfig,
permit: Permit,
@@ -318,6 +376,7 @@ class OcrClient:
latency_ms = int((self._now() - started) * 1000)
await self._emit(
kind,
operation,
image,
source,
call_id,
@@ -326,6 +385,7 @@ class OcrClient:
parent_call_id,
tenant_id,
meta,
context,
result,
)
return _AttemptOutcome(result, source, call_id, latency_ms)
@@ -333,6 +393,7 @@ class OcrClient:
await self._gate_on_terminal(exc, entry)
await self._emit(
kind,
operation,
image,
source,
call_id,
@@ -341,6 +402,7 @@ class OcrClient:
parent_call_id,
tenant_id,
meta,
context,
error=exc,
)
raise
@@ -349,6 +411,7 @@ class OcrClient:
await self._record_quietly(self._breaker.release_probe(entry))
await self._emit(
kind,
operation,
image,
source,
call_id,
@@ -357,6 +420,7 @@ class OcrClient:
parent_call_id,
tenant_id,
meta,
context,
error="cancelled",
)
raise
@@ -368,6 +432,7 @@ class OcrClient:
self._feed_outcome(source.name, ok=False)
await self._emit(
kind,
operation,
image,
source,
call_id,
@@ -376,6 +441,7 @@ class OcrClient:
parent_call_id,
tenant_id,
meta,
context,
error=exc,
)
return _FailedAttempt(exc, immediate=dead)
@@ -417,9 +483,60 @@ class OcrClient:
except (GovernanceBackendError, SourceNotConfiguredError) as exc:
logger.warning("OCR 治理记账写回降级(不冒泡): {}", exc)
def _request_for(
self,
kind: _OcrKind,
image: bytes,
session_id: str | None,
parent_call_id: str | None,
tenant_id: str | None,
meta: dict[str, Any],
context: _CallContext,
) -> ChatRequest:
"""OCR 行的现场 ChatRequest: 占位摘要,**图像 bytes 永不入库**。
尝试行与终态行共用同一个构造点: 占位字面量复制成两份就会漂移。
调用方维度与上下文必须显式填回(OCR 不走 chat 洋葱),否则 OCR 行的
维度恒为空、`logical_call_id` 恒为 NULL。
"""
return ChatRequest(
messages=[{"role": "user", "content": f"<ocr:{kind} image_bytes={len(image)}>"}],
session_id=session_id,
parent_call_id=parent_call_id,
tenant_id=tenant_id,
meta=meta,
call_context=context,
)
async def _emit_terminal(
self,
kind: _OcrKind,
operation: CallOperation,
image: bytes,
session_id: str | None,
parent_call_id: str | None,
tenant_id: str | None,
meta: dict[str, Any],
context: _CallContext,
error: PolyGatewayError | str,
) -> None:
"""终态行: 沿用 `<ocr:{kind} image_bytes=…>` 占位,错误文本保留类名前缀。"""
await emit_terminal_once(
self._emitter,
request=self._request_for(
kind, image, session_id, parent_call_id, tenant_id, meta, context
),
context=context,
error=error,
operation=operation,
# metric ocr-call-success 的注册口径是按类名归组,终态行同款保留
class_prefixed_error=True,
)
async def _emit(
self,
kind: _OcrKind,
operation: CallOperation,
image: bytes,
source: SourceConfig,
call_id: str,
@@ -428,20 +545,15 @@ class OcrClient:
parent_call_id: str | None,
tenant_id: str | None,
meta: dict[str, Any],
context: _CallContext,
result: OcrTextTransportResult | OcrLayoutTransportResult | None = None,
error: object | None = None,
error: PolyGatewayError | str | None = None,
) -> None:
"""逐尝试遥测(单一 Emitter): messages 占位摘要,图像 bytes 绝不入库。"""
if self._emitter is None:
return
# 这个 ChatRequest 只为复用同一个 Emitter 而现场构造(OCR 不走 chat 洋葱),
# 故调用方维度必须在这里显式填回,否则 OCR 行的维度恒为空
request = ChatRequest(
messages=[{"role": "user", "content": f"<ocr:{kind} image_bytes={len(image)}>"}],
session_id=session_id,
parent_call_id=parent_call_id,
tenant_id=tenant_id,
meta=meta,
request = self._request_for(
kind, image, session_id, parent_call_id, tenant_id, meta, context
)
latency_ms = int((self._now() - started) * 1000)
response = None
@@ -461,20 +573,19 @@ class OcrClient:
source_name=source.name,
usage_source="measured",
)
# 错误带异常类名前缀(metric ocr-call-success 注册口径: 按类名归组)
if error is None or isinstance(error, str):
error_text = error
else:
error_text = f"{type(error).__name__}: {error}"
# 错误文本的类名前缀现由出口的显式策略参数承担(设计 §6 I7):
# 三处各拼一遍才是下一次漂移的种子,而取消行传的是字符串,不受前缀影响
await self._emitter.emit_attempt(
request=request,
source=source,
call_id=call_id,
latency_ms=latency_ms,
response=response,
error=error_text,
error=error,
# OCR 走 MonkeyOCR 自有端点,没有推理参数可言(理由同 embedding)
reasoning_applies=False,
operation=operation,
class_prefixed_error=True,
)
@staticmethod