feat: record call observability columns and terminal failure rows
Grow the telemetry contract from 26 to 36 fields and give every logical call a failure terminal row, so SQL can finally answer "how many calls failed" and "why did the whole pool die". Schema and port move together with the emitter writes in one commit: splitting them would ship columns that nothing populates. - schema: append 10 nullable columns (scope, operation, logical_call_id, event_kind, http_status_code, error_type, cause_type, error_body, attempts, total_latency_ms) to all five definition sites in one order - ports: 10 keyword-only parameters without defaults; the protocol signature is now the single source the assembly gate derives from - emitter: take domain exception objects instead of pre-flattened text and pin down the diagnostics in one helper; a relabelled 503 stays 503 and success rows leave all five columns NULL - emitter: reject recorders whose record_llm_call cannot accept the current field shape at assembly time, since _record would otherwise swallow the TypeError and drop every row while calls keep succeeding - clients: write at most one terminal row per logical call through a single shared exit, deduplicated by the call context; TelemetryMW stops writing terminals so the two sites cannot double count - clients: cancellation stays best effort and propagates, non-domain exceptions get no terminal row and keep their classification - transports: give _status_to_error an explicit operation and fix the historically mislabelled embedding HTTP failures - structured: promote the bounded error formatter so the reask feedback and the terminal explanation share one set of limits Terminal rows carry no cost and no tokens, so cost aggregation is unchanged; failure counts must now filter on event_kind.
This commit is contained in:
@@ -20,11 +20,12 @@ from polygateway.backends.memory.breaker import InMemoryGate
|
||||
from polygateway.backends.memory.cache import InMemoryCache
|
||||
from polygateway.backends.memory.limiter import InMemoryLimiter
|
||||
from polygateway.config import GatewaySettings
|
||||
from polygateway.errors import PolyGatewayError
|
||||
from polygateway.middleware.base import compose
|
||||
from polygateway.middleware.cache import CacheMW
|
||||
from polygateway.middleware.retry import RetryMW
|
||||
from polygateway.middleware.structured import StructuredMW
|
||||
from polygateway.middleware.telemetry import TelemetryEmitter, TelemetryMW
|
||||
from polygateway.middleware.telemetry import TelemetryEmitter, TelemetryMW, emit_terminal_once
|
||||
from polygateway.ports import TelemetryStatusProvider
|
||||
from polygateway.pricing import PricingTable
|
||||
from polygateway.providers import get_provider
|
||||
@@ -239,7 +240,7 @@ class GatewayClient:
|
||||
rng: Any = random.random,
|
||||
) -> None:
|
||||
emitter = (
|
||||
TelemetryEmitter(telemetry, pricing=pricing, text_cap=text_cap)
|
||||
TelemetryEmitter(telemetry, scope=scope, pricing=pricing, text_cap=text_cap)
|
||||
if telemetry is not None
|
||||
else None
|
||||
)
|
||||
@@ -393,7 +394,27 @@ class GatewayClient:
|
||||
meta=dimensions,
|
||||
call_context=context,
|
||||
)
|
||||
response = await self._handler(request)
|
||||
try:
|
||||
response = await self._handler(request)
|
||||
except PolyGatewayError as exc:
|
||||
# 统计边界内的一切领域失败均尝试写一条终态行(1.3.5 设计 §6 I3),
|
||||
# 包括已有 attempt 错误行的 RequestRejected / ResultInvalid——两类行描述
|
||||
# 的不是同一件事(尝试 vs 逻辑终态),由 `event_kind` 区分
|
||||
await emit_terminal_once(
|
||||
self._emitter, request=request, context=context, error=exc, operation="chat"
|
||||
)
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
# 尽力而为且**取消优先**: 不 shield、不开后台任务;写入那一次 await 上
|
||||
# 再被取消则 `CancelledError` 照常传播(与 TelemetryMW 历史行为同款)
|
||||
await emit_terminal_once(
|
||||
self._emitter,
|
||||
request=request,
|
||||
context=context,
|
||||
error="cancelled",
|
||||
operation="chat",
|
||||
)
|
||||
raise
|
||||
# 快照在返回前冻结: 故它含缓存命中路径与已完成的内联遥测耗时
|
||||
return dataclasses.replace(response, call_stats=context.snapshot())
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ from polygateway.middleware.admission import SourceAdmission, settle_and_release
|
||||
from polygateway.middleware.breaker import BreakerGate
|
||||
from polygateway.middleware.ratelimit import QuotaGate
|
||||
from polygateway.middleware.retry import StallClock, _failure_reason, backoff_delay
|
||||
from polygateway.middleware.telemetry import TelemetryEmitter
|
||||
from polygateway.middleware.telemetry import TelemetryEmitter, emit_terminal_once
|
||||
from polygateway.types import (
|
||||
ChatRequest,
|
||||
EmbeddingResponse,
|
||||
@@ -129,7 +129,9 @@ class EmbeddingClient:
|
||||
self._transport = transport
|
||||
self._retry = retry
|
||||
self._emitter = (
|
||||
TelemetryEmitter(telemetry, pricing=pricing, text_cap=text_cap) if telemetry else None
|
||||
TelemetryEmitter(telemetry, scope=self._scope, pricing=pricing, text_cap=text_cap)
|
||||
if telemetry
|
||||
else None
|
||||
)
|
||||
self._telemetry = telemetry
|
||||
# 限流/熔断后端在此之外只以 QuotaGate/BreakerGate 的形态存在,自持一份
|
||||
@@ -203,6 +205,38 @@ class EmbeddingClient:
|
||||
source_name="",
|
||||
call_stats=context.snapshot(),
|
||||
)
|
||||
try:
|
||||
return await self._embed_all(
|
||||
texts, session_id, parent_call_id, dimension_tenant_id, dimensions, context
|
||||
)
|
||||
except PolyGatewayError as exc:
|
||||
await self._emit_terminal(
|
||||
texts, session_id, parent_call_id, dimension_tenant_id, dimensions, context, exc
|
||||
)
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
# 三条链路同一口径尽力写一条(允许 0 条);取消优先,不 shield
|
||||
await self._emit_terminal(
|
||||
texts,
|
||||
session_id,
|
||||
parent_call_id,
|
||||
dimension_tenant_id,
|
||||
dimensions,
|
||||
context,
|
||||
"cancelled",
|
||||
)
|
||||
raise
|
||||
|
||||
async def _embed_all(
|
||||
self,
|
||||
texts: list[str],
|
||||
session_id: str | None,
|
||||
parent_call_id: str | None,
|
||||
tenant_id: str | None,
|
||||
meta: dict[str, Any],
|
||||
context: _CallContext,
|
||||
) -> EmbeddingResponse:
|
||||
"""切批串行执行并合并;无源的 raise 必须在本方法内——否则无源终态行写不出。"""
|
||||
if not self._sources:
|
||||
raise AllSourcesExhausted(scope=self._scope, reason="no_sources", retry_after_s=0.0)
|
||||
outcomes = []
|
||||
@@ -212,14 +246,50 @@ class EmbeddingClient:
|
||||
texts[start : start + self._batch_size],
|
||||
session_id,
|
||||
parent_call_id,
|
||||
dimension_tenant_id,
|
||||
dimensions,
|
||||
tenant_id,
|
||||
meta,
|
||||
context,
|
||||
)
|
||||
)
|
||||
# 全批共享同一上下文,故分批是实现细节而非 N 次独立逻辑调用
|
||||
return dataclasses.replace(self._merge(outcomes), call_stats=context.snapshot())
|
||||
|
||||
async def _emit_terminal(
|
||||
self,
|
||||
texts: list[str],
|
||||
session_id: str | None,
|
||||
parent_call_id: str | None,
|
||||
tenant_id: str | None,
|
||||
meta: dict[str, Any],
|
||||
context: _CallContext,
|
||||
error: PolyGatewayError | str,
|
||||
) -> None:
|
||||
"""终态行的请求摘要(设计 §6 M4): 计数占位 + 第一批截断文本。
|
||||
|
||||
描述的是**本次调用的整体输入**但不扩大单行正文预算: 失败批的具体文本由同
|
||||
`logical_call_id` 的 attempt 行给出,终态行不保存全量原输入。
|
||||
"""
|
||||
batches = math.ceil(len(texts) / self._batch_size)
|
||||
messages = [{"role": "user", "content": f"<embed texts={len(texts)} batches={batches}>"}]
|
||||
# 与逐批行同款构造(至多 `batch_size` 条、每条 200 字符)
|
||||
messages += [
|
||||
{"role": "user", "content": t[:_TELEMETRY_TEXT_CAP]} for t in texts[: self._batch_size]
|
||||
]
|
||||
await emit_terminal_once(
|
||||
self._emitter,
|
||||
request=ChatRequest(
|
||||
messages=messages,
|
||||
session_id=session_id,
|
||||
parent_call_id=parent_call_id,
|
||||
tenant_id=tenant_id,
|
||||
meta=meta,
|
||||
call_context=context,
|
||||
),
|
||||
context=context,
|
||||
error=error,
|
||||
operation="embed",
|
||||
)
|
||||
|
||||
# —— 治理循环(与 RetryMW 同构;设计 §7.1 已声明的有限重复)——
|
||||
|
||||
async def _embed_batch(
|
||||
@@ -300,6 +370,7 @@ class EmbeddingClient:
|
||||
parent_call_id,
|
||||
tenant_id,
|
||||
meta,
|
||||
context,
|
||||
result,
|
||||
)
|
||||
return _BatchOutcome(result, source, call_id, latency_ms)
|
||||
@@ -314,6 +385,7 @@ class EmbeddingClient:
|
||||
parent_call_id,
|
||||
tenant_id,
|
||||
meta,
|
||||
context,
|
||||
error=exc,
|
||||
)
|
||||
raise
|
||||
@@ -329,6 +401,7 @@ class EmbeddingClient:
|
||||
parent_call_id,
|
||||
tenant_id,
|
||||
meta,
|
||||
context,
|
||||
error="cancelled",
|
||||
)
|
||||
raise
|
||||
@@ -349,6 +422,7 @@ class EmbeddingClient:
|
||||
parent_call_id,
|
||||
tenant_id,
|
||||
meta,
|
||||
context,
|
||||
error=exc,
|
||||
)
|
||||
return _FailedBatch(exc, immediate=dead)
|
||||
@@ -385,8 +459,9 @@ class EmbeddingClient:
|
||||
parent_call_id: str | None,
|
||||
tenant_id: str | None,
|
||||
meta: dict[str, Any],
|
||||
context: _CallContext,
|
||||
result: EmbeddingTransportResult | None = None,
|
||||
error: object | None = None,
|
||||
error: PolyGatewayError | str | None = None,
|
||||
) -> None:
|
||||
"""逐批遥测(经同一 Emitter): messages=截断 texts、向量绝不入库。"""
|
||||
if self._emitter is None:
|
||||
@@ -399,6 +474,7 @@ class EmbeddingClient:
|
||||
parent_call_id=parent_call_id,
|
||||
tenant_id=tenant_id,
|
||||
meta=meta,
|
||||
call_context=context,
|
||||
)
|
||||
response = None
|
||||
if result is not None:
|
||||
@@ -423,10 +499,12 @@ class EmbeddingClient:
|
||||
call_id=call_id,
|
||||
latency_ms=int((self._now() - started) * 1000),
|
||||
response=response,
|
||||
error=None if error is None else str(error),
|
||||
# 异常对象原样下传: 状态码/底层异常类型/网关正文在 Emitter 内定型
|
||||
error=error,
|
||||
# embedding payload 硬编码 {model, input},从不带推理参数;源上即便
|
||||
# 误配了 ENABLE_THINKING,记一个档也是替这次调用声称它没做过的事
|
||||
reasoning_applies=False,
|
||||
operation="embed",
|
||||
)
|
||||
|
||||
def _merge(self, outcomes: list[_BatchOutcome]) -> EmbeddingResponse:
|
||||
|
||||
@@ -42,6 +42,7 @@ from polygateway.types import LLMResponse
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable
|
||||
|
||||
from polygateway.middleware.telemetry import TelemetryEmitter
|
||||
from polygateway.ports import (
|
||||
GateDecision,
|
||||
Permit,
|
||||
@@ -181,7 +182,7 @@ class RetryMW:
|
||||
circuit_open: str = "fail_fast",
|
||||
cooldown_memo: SourceCooldownMemo | None = None,
|
||||
pacer: AdaptivePacer | None = None,
|
||||
emitter: object | None = None,
|
||||
emitter: TelemetryEmitter | None = None,
|
||||
now: Callable[[], float] = time.monotonic,
|
||||
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||
rng: Callable[[], float] = random.random,
|
||||
@@ -413,9 +414,13 @@ class RetryMW:
|
||||
started: float,
|
||||
*,
|
||||
response: LLMResponse | None = None,
|
||||
error: object | None = None,
|
||||
error: PolyGatewayError | str | None = None,
|
||||
) -> None:
|
||||
"""逐次遥测(经注入的单一 Emitter);遥测失败不得影响调用(铁律)。"""
|
||||
"""逐次遥测(经注入的单一 Emitter);遥测失败不得影响调用(铁律)。
|
||||
|
||||
异常**对象原样下传**而非先 `str()` 压平(1.3.5 设计 §5): 状态码、底层异常
|
||||
类型与网关响应体已经在异常上了,在这里压平就是把它们丢掉。
|
||||
"""
|
||||
if self._emitter is None:
|
||||
return
|
||||
try:
|
||||
@@ -425,9 +430,11 @@ class RetryMW:
|
||||
call_id=call_id,
|
||||
latency_ms=int((self._now() - started) * 1000),
|
||||
response=response,
|
||||
error=None if error is None else str(error),
|
||||
error=error,
|
||||
# chat 路径是唯一带推理参数的路径,故实发档由这里的响应说了算
|
||||
reasoning_applies=True,
|
||||
# 公开方法四值之一;本中间件只服务 chat 洋葱
|
||||
operation="chat",
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
|
||||
@@ -14,6 +14,8 @@ from typing import TYPE_CHECKING
|
||||
from polygateway.errors import ResultInvalidError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
from polygateway.ports import CallNext, StructuredOutputStrategy
|
||||
from polygateway.types import ChatRequest, LLMResponse
|
||||
|
||||
@@ -22,12 +24,18 @@ _FEEDBACK_TEMPLATE = (
|
||||
"Your previous reply was not valid JSON matching the required schema. "
|
||||
"Errors: {errors}. Reply with ONLY the corrected JSON object."
|
||||
)
|
||||
_MAX_FEEDBACK_ERRORS = 3
|
||||
_MAX_ERROR_CHARS = 200
|
||||
MAX_FEEDBACK_ERRORS = 3
|
||||
MAX_ERROR_CHARS = 200
|
||||
|
||||
|
||||
def _format_errors(errors: list[str]) -> str:
|
||||
clipped = [e[:_MAX_ERROR_CHARS] for e in errors[:_MAX_FEEDBACK_ERRORS]]
|
||||
def format_bounded_errors(errors: Sequence[str]) -> str:
|
||||
"""校验错误的有界拼装: 至多 3 条 × 每条 200 字符。
|
||||
|
||||
**本模块是这条规则的所有者**: 重问反馈文案与 1.3.5 终态行的结构化说明
|
||||
两个消费者共用同一份实现与同一组数值——数值复制成两份必然漂移,而漂移后
|
||||
"模型看到的错误"与"台账里记的错误"就不再是同一件事。行为与重命名前逐字相同。
|
||||
"""
|
||||
clipped = [e[:MAX_ERROR_CHARS] for e in list(errors)[:MAX_FEEDBACK_ERRORS]]
|
||||
return "; ".join(clipped) if clipped else "output could not be parsed"
|
||||
|
||||
|
||||
@@ -104,7 +112,10 @@ class StructuredMW:
|
||||
messages = [
|
||||
*current.messages,
|
||||
{"role": "assistant", "content": bad_content},
|
||||
{"role": "user", "content": _FEEDBACK_TEMPLATE.format(errors=_format_errors(errors))},
|
||||
{
|
||||
"role": "user",
|
||||
"content": _FEEDBACK_TEMPLATE.format(errors=format_bounded_errors(errors)),
|
||||
},
|
||||
]
|
||||
reask = dataclasses.replace(current, messages=messages)
|
||||
if self._escalation is not None:
|
||||
|
||||
@@ -2,13 +2,19 @@
|
||||
|
||||
Emitter 是全库**唯一**调用 `record_llm_call` 的地方(三项目 4 处逐字复制
|
||||
15 参调用的教训)。分工: RetryMW 经 Emitter 逐次记录每次尝试;TelemetryMW
|
||||
(最外层)只记尝试层看不见的事件——缓存命中、scope 级失败、取消;
|
||||
RequestRejected/ResultInvalid 已被尝试层记录,最外层放行不重复记。
|
||||
(最外层)只记尝试层看不见的缓存命中;而**终态失败行**由三个 client 的公开
|
||||
边界经 `emit_terminal_once` 统一写出(1.3.5)——两处同时写就会双计。
|
||||
|
||||
一行遥测属于三类事件之一(`event_kind`): `attempt`(一次尝试)、`cache_hit`
|
||||
(未产生网关调用)、`terminal_failure`(一次**逻辑调用**的失败终态)。后两者与
|
||||
前者**不是重复事实**,故统计失败调用次数只能取 `terminal_failure`,
|
||||
不得按 `error IS NOT NULL` 跨两类直接计数(设计 §6/§8)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
@@ -17,12 +23,10 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from polygateway.errors import (
|
||||
GatewayUnavailableError,
|
||||
GovernanceBackendError,
|
||||
SourceNotConfiguredError,
|
||||
)
|
||||
from polygateway.errors import PolyGatewayError, ResultInvalidError
|
||||
from polygateway.middleware.cache import digest_messages
|
||||
from polygateway.middleware.structured import MAX_ERROR_CHARS, format_bounded_errors
|
||||
from polygateway.ports import TelemetryRecorder
|
||||
from polygateway.thinking import effective_effort
|
||||
from polygateway.types import Effort, ThinkingObservation, canonical_sampling_json, merge_sampling
|
||||
|
||||
@@ -30,9 +34,17 @@ if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Any
|
||||
|
||||
from polygateway.ports import CallNext, TelemetryRecorder
|
||||
from polygateway.ports import CallNext
|
||||
from polygateway.pricing import PricingTable
|
||||
from polygateway.types import ChatRequest, LLMResponse, SourceConfig
|
||||
from polygateway.types import (
|
||||
CallOperation,
|
||||
CallStats,
|
||||
ChatRequest,
|
||||
EventKind,
|
||||
LLMResponse,
|
||||
SourceConfig,
|
||||
_CallContext,
|
||||
)
|
||||
|
||||
|
||||
def _canonical_meta_json(meta: Mapping[str, Any]) -> str:
|
||||
@@ -225,26 +237,159 @@ class _AttemptUsage:
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ErrorFields:
|
||||
"""一行遥测的错误列;未知一律 `None`。
|
||||
|
||||
存在的理由是把"三种入参形态 × 两类行"的定型规则收敛到**一处**:
|
||||
改前调用方先 `str(exc)` 压平,状态码、底层异常类型与网关正文全部丢失。
|
||||
"""
|
||||
|
||||
error: str | None = None
|
||||
error_type: str | None = None
|
||||
cause_type: str | None = None
|
||||
http_status_code: int | None = None
|
||||
error_body: str | None = None
|
||||
|
||||
|
||||
def _structured_detail(exc: ResultInvalidError) -> str:
|
||||
"""结构化阶梯耗尽的**有界**说明(设计 §5 C2)。
|
||||
|
||||
`ResultInvalidError("结构化输出阶梯耗尽")` 的 message 不含校验与修复错误,而该
|
||||
失败发生在 StructuredMW 之上——RetryMW 侧的 attempt 行全是**成功行**,终态行是
|
||||
唯一记录。故把说明并入现有 `error` 串。
|
||||
|
||||
**不含 `raw_text`**: 它是模型正文,attempt 行的 `response` 列已按 `text_cap` 记过
|
||||
一份;再存一份等于绕过既有的正文预算。条数与限长复用 `structured.py` 的同一
|
||||
套常量(重问反馈与本说明同一口径),数值只有一份。
|
||||
"""
|
||||
parts: list[str] = []
|
||||
if exc.repair_error:
|
||||
parts.append(f"repair={exc.repair_error[:MAX_ERROR_CHARS]}")
|
||||
if exc.validation_errors:
|
||||
parts.append(f"validation={format_bounded_errors(exc.validation_errors)}")
|
||||
return " | ".join(parts)
|
||||
|
||||
|
||||
def _error_fields(
|
||||
error: PolyGatewayError | str | None,
|
||||
*,
|
||||
event_kind: EventKind,
|
||||
class_prefixed: bool,
|
||||
) -> _ErrorFields:
|
||||
"""三种入参形态的唯一定型点(设计 §5/§6)。
|
||||
|
||||
- `None` → 全 None(成功行不统一填 200: 那会让"有状态码"不再等价于"失败了")。
|
||||
- `str`(取消路径的 `"cancelled"`)→ 原样落 `error`,**不解析字符串猜诊断**。
|
||||
- 领域异常 → 只读它既有的属性,不遍历任意对象、不猜正文。
|
||||
|
||||
**终态行的三列恒为 NULL(C1 红线)**: `GatewayUnavailableError` 家族从不携带
|
||||
状态码与响应体,NULL 正是它自身的真实状态——把最后一次 attempt 的状态码与正文
|
||||
搬上来,就是拿最后一个源冒充整池归因。逐源现场由同一 `logical_call_id` 的
|
||||
attempt 行给出。
|
||||
"""
|
||||
if error is None:
|
||||
return _ErrorFields()
|
||||
if isinstance(error, str):
|
||||
return _ErrorFields(error=error)
|
||||
name = type(error).__name__
|
||||
# 空 `str()` 退回类名(httpx 的 Connect/Read/Write/PoolTimeout 文案就是空的);
|
||||
# `class_prefixed` 是 OCR 的既有口径(按类名归组的 metric),故逐字保留它的拼法
|
||||
text = f"{name}: {error}" if class_prefixed else (str(error) or name)
|
||||
if event_kind == "terminal_failure":
|
||||
if isinstance(error, ResultInvalidError):
|
||||
detail = _structured_detail(error)
|
||||
if detail:
|
||||
text = f"{text} | {detail}"
|
||||
return _ErrorFields(error=text, error_type=name)
|
||||
cause = error.__cause__
|
||||
return _ErrorFields(
|
||||
error=text,
|
||||
error_type=name,
|
||||
cause_type=type(cause).__name__ if cause is not None else None,
|
||||
# getattr 而非直读: 本函数在 `_record` 的降级 try **之外**求值,
|
||||
# 一个非领域异常误传进来不得把一次真实失败换成 AttributeError
|
||||
http_status_code=getattr(error, "status_code", None),
|
||||
# 空串归 None: 既有 `body_text` 的缺省就是空串,而本列的语义是"未知"
|
||||
error_body=getattr(error, "body_text", "") or None,
|
||||
)
|
||||
|
||||
|
||||
def _assert_recorder_shape(recorder: TelemetryRecorder) -> None:
|
||||
"""装配期一次 `signature.bind` 形状校验: 不执行写入,只证明该形状能被接受。
|
||||
|
||||
`_record` 的 `except Exception` 会把旧 recorder 的 `TypeError` 吞成 warning,
|
||||
后果是自定义 recorder 在下游升级后**100% 丢遥测且调用照常成功**——正是
|
||||
"遥测必录"要防的形态,而文档级迁移清单挡不住它。故在装配期当场报错
|
||||
(不是 warning: 降级方向的铁律管的是**运行期写失败**,不是装配错误)。
|
||||
|
||||
参数名从 `TelemetryRecorder.record_llm_call` 的协议签名**派生**(不手抄第四份
|
||||
字段清单),绑定用哨兵 `None`,不读任何真实请求数据;`**kwargs`
|
||||
(VAR_KEYWORD)自动通过。不可 inspect(C 实现等)同样按配置错误报错——宁可
|
||||
装配不起来,不进入"运行期静默丢行"。
|
||||
|
||||
边界诚实声明: 它只证明该形状能被接受,**不能证明函数体真的落这些列**。
|
||||
|
||||
Raises:
|
||||
ValueError: 签名不符、不可 inspect,或协议本身不可 inspect。
|
||||
"""
|
||||
try:
|
||||
# 模块全局查找而非常量快照: 协议改了,闸就跟着改(测试可据此机械验证)
|
||||
protocol = inspect.signature(TelemetryRecorder.record_llm_call).parameters
|
||||
except (TypeError, ValueError) as exc: # pragma: no cover - 协议一向可 inspect
|
||||
raise ValueError(f"TelemetryRecorder.record_llm_call 签名不可读取: {exc}") from exc
|
||||
sentinels = {name: None for name in protocol if name != "self"}
|
||||
label = type(recorder).__name__
|
||||
method = getattr(recorder, "record_llm_call", None)
|
||||
if method is None:
|
||||
# 连方法都没有: 比旧签名更明确的配置错误。不让它以裸 AttributeError
|
||||
# 逆流而上——那不属错误四分类,且现场离"注错了东西"这个真因很远
|
||||
raise ValueError(
|
||||
f"注入的遥测 recorder {label} 没有 record_llm_call 方法,不满足 TelemetryRecorder 端口"
|
||||
)
|
||||
try:
|
||||
signature = inspect.signature(method)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(
|
||||
f"遥测 recorder {label} 的 record_llm_call 不可 inspect(如 C 实现),"
|
||||
"无法在装配期确认它接受当前字段形状;请换成 Python 实现或包一层"
|
||||
) from exc
|
||||
try:
|
||||
signature.bind(**sentinels)
|
||||
except TypeError as exc:
|
||||
raise ValueError(
|
||||
f"遥测 recorder {label} 的 record_llm_call 签名与 TelemetryRecorder 不符"
|
||||
f"(当前 {len(sentinels)} 个字段): {exc}。"
|
||||
"这一条故意在装配期报错——放行的后果是每行遥测都被降级成 warning 后丢弃"
|
||||
) from exc
|
||||
|
||||
|
||||
class TelemetryEmitter:
|
||||
"""从请求与结果组装 26 字段并写入 recorder;一切写失败降级 warning。"""
|
||||
"""从请求与结果组装 36 字段并写入 recorder;一切写失败降级 warning。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
recorder: TelemetryRecorder,
|
||||
*,
|
||||
scope: str,
|
||||
pricing: PricingTable | None = None,
|
||||
text_cap: int | None,
|
||||
) -> None:
|
||||
"""`text_cap` 无默认值是有意的: 它是关键行为参数,漏传即静默改变落库正文。
|
||||
"""`text_cap` 与 `scope` 无默认值是有意的: 两者都是关键行为参数。
|
||||
|
||||
`text_cap` 漏传即静默改变落库正文;`scope` 漏传则三类行都失去池名
|
||||
——终态失败可能根本没选出源,但 scope 始终已知,不拿 `source_name` 顶替。
|
||||
本类是库内部类,唯一构造者是三个公共 Client,必填能保证没有一处漏传。
|
||||
同理,值域校验也放在这一处: 三个 Client 的 `text_cap` 全部汇流到这里,
|
||||
|
||||
同理,值域校验与**装配闸**都放在这一处: 三个 Client 全部汇流到这里,
|
||||
`GatewaySettings` 那道只管 env 一条路,而直接构造 Client 是库承诺的另一
|
||||
条公共装配路——`text_cap=0` 会让每条正文只剩一个省略标记(P5 不得静默)。
|
||||
条公共装配路(`text_cap=0` 会让每条正文只剩一个省略标记;P5 不得静默)。
|
||||
"""
|
||||
if text_cap is not None and text_cap <= 0:
|
||||
raise ValueError(f"text_cap 必须 > 0(不截断请传 None): {text_cap}")
|
||||
_assert_recorder_shape(recorder)
|
||||
self._recorder = recorder
|
||||
self._scope = scope
|
||||
self._pricing = pricing
|
||||
self._text_cap = text_cap
|
||||
|
||||
@@ -256,8 +401,10 @@ class TelemetryEmitter:
|
||||
call_id: str,
|
||||
latency_ms: int,
|
||||
response: LLMResponse | None,
|
||||
error: str | None,
|
||||
error: PolyGatewayError | str | None,
|
||||
reasoning_applies: bool,
|
||||
operation: CallOperation,
|
||||
class_prefixed_error: bool = False,
|
||||
) -> None:
|
||||
"""逐次尝试记录(三个 Client 的重试层调用);失败尝试无用量可言,记 0 并标 unavailable。
|
||||
|
||||
@@ -266,7 +413,11 @@ class TelemetryEmitter:
|
||||
共用同一个 `SourceConfig` 类型,一个误配了 `ENABLE_THINKING` 的 embedding 源
|
||||
会让下面的回落算出 `auto`,给一次从来不带推理参数的调用挂上一个从未发出过的
|
||||
档。**不设默认值**: 与 `TelemetryRecorder` 同一约定,库外无第三方调用者,漏传
|
||||
当场 TypeError,好过被静默当成"没表态"。
|
||||
当场 TypeError,好过被静默当成"没表态"。`operation` 同理且另有一层:
|
||||
它只能由调用点给定,**绝不读 `exc.operation`**(后者是 HTTP 子操作)。
|
||||
|
||||
`error` 收**领域异常对象**而非预先 `str()` 压平的文本: 状态码/底层异常类型/
|
||||
网关正文在此提取成四列(设计 §5)。取消路径仍传既有字符串 `"cancelled"`。
|
||||
"""
|
||||
usage = _AttemptUsage.of(response)
|
||||
await self._record(
|
||||
@@ -284,7 +435,7 @@ class TelemetryEmitter:
|
||||
ttft_ms=usage.ttft_ms,
|
||||
max_inter_token_ms=usage.max_inter_token_ms,
|
||||
cache_hit=False,
|
||||
error=error,
|
||||
errors=_error_fields(error, event_kind="attempt", class_prefixed=class_prefixed_error),
|
||||
cached_prompt_tokens=usage.cached_prompt_tokens,
|
||||
model_reported=usage.model_reported,
|
||||
reasoning_tokens=usage.reasoning_tokens,
|
||||
@@ -296,10 +447,19 @@ class TelemetryEmitter:
|
||||
reasoning_effort=_attempt_effort(
|
||||
request=request, source=source, response=response, applies=reasoning_applies
|
||||
),
|
||||
operation=operation,
|
||||
event_kind="attempt",
|
||||
attempts=None,
|
||||
total_latency_ms=None,
|
||||
)
|
||||
|
||||
async def emit_cache_hit(self, *, request: ChatRequest, response: LLMResponse) -> None:
|
||||
"""缓存命中记录: cache_hit=True、latency_ms=0(VT 同款)。"""
|
||||
async def emit_cache_hit(
|
||||
self, *, request: ChatRequest, response: LLMResponse, operation: CallOperation
|
||||
) -> None:
|
||||
"""缓存命中记录: cache_hit=True、latency_ms=0(VT 同款)。
|
||||
|
||||
逻辑计数两列恒 NULL: 本行描述的是"一次命中",不是一次逻辑调用的终态。
|
||||
"""
|
||||
await self._record(
|
||||
request=request,
|
||||
call_id=response.call_id,
|
||||
@@ -315,7 +475,7 @@ class TelemetryEmitter:
|
||||
ttft_ms=None,
|
||||
max_inter_token_ms=None,
|
||||
cache_hit=True,
|
||||
error=None,
|
||||
errors=_ErrorFields(),
|
||||
# 决策 B1: 与 model/prompt_tokens 同一口径,原样回放历史值。
|
||||
# 统计供应商缓存命中率必须带 WHERE cache_hit = false,否则重复计数。
|
||||
cached_prompt_tokens=response.cached_prompt_tokens,
|
||||
@@ -334,12 +494,28 @@ class TelemetryEmitter:
|
||||
# 与 sampling 同一口径: 命中行没有选中源,源级档位与 `nearest` 映射
|
||||
# 都无从谈起,只记调用方这次要的档(response 里那个是历史那次实发的)
|
||||
reasoning_effort=_normalize_effort(request.reasoning_effort),
|
||||
operation=operation,
|
||||
event_kind="cache_hit",
|
||||
attempts=None,
|
||||
total_latency_ms=None,
|
||||
)
|
||||
|
||||
async def emit_terminal_failure(
|
||||
self, *, request: ChatRequest, call_id: str, latency_ms: int, error: str
|
||||
self,
|
||||
*,
|
||||
request: ChatRequest,
|
||||
call_id: str,
|
||||
error: PolyGatewayError | str,
|
||||
operation: CallOperation,
|
||||
stats: CallStats,
|
||||
class_prefixed_error: bool = False,
|
||||
) -> None:
|
||||
"""scope 级失败/取消记录: 无具体源,溯源字段置空标记。"""
|
||||
"""一次**逻辑调用**的失败终态: 无具体源,溯源字段置空标记。
|
||||
|
||||
`latency_ms` 与 `total_latency_ms` 同取**同一份冻结快照**,避免双时钟微差;
|
||||
故本方法不再收 `latency_ms`。token 与 cost 一律不从 attempt 行复制
|
||||
(费用聚合仍只由 attempt / cache_hit 行决定,口径不变)。
|
||||
"""
|
||||
await self._record(
|
||||
request=request,
|
||||
call_id=call_id,
|
||||
@@ -351,11 +527,13 @@ class TelemetryEmitter:
|
||||
prompt_tokens=0,
|
||||
completion_tokens=0,
|
||||
usage_source="unavailable",
|
||||
latency_ms=latency_ms,
|
||||
latency_ms=stats.total_latency_ms,
|
||||
ttft_ms=None,
|
||||
max_inter_token_ms=None,
|
||||
cache_hit=False,
|
||||
error=error,
|
||||
errors=_error_fields(
|
||||
error, event_kind="terminal_failure", class_prefixed=class_prefixed_error
|
||||
),
|
||||
cached_prompt_tokens=None,
|
||||
model_reported=None,
|
||||
reasoning_tokens=None,
|
||||
@@ -368,6 +546,10 @@ class TelemetryEmitter:
|
||||
meta=request.meta,
|
||||
# 可能根本没选出源,故与 sampling 同样只取请求档
|
||||
reasoning_effort=_normalize_effort(request.reasoning_effort),
|
||||
operation=operation,
|
||||
event_kind="terminal_failure",
|
||||
attempts=stats.attempts,
|
||||
total_latency_ms=stats.total_latency_ms,
|
||||
)
|
||||
|
||||
async def _record(
|
||||
@@ -387,7 +569,9 @@ class TelemetryEmitter:
|
||||
ttft_ms: float | None,
|
||||
max_inter_token_ms: float | None,
|
||||
cache_hit: bool,
|
||||
error: str | None,
|
||||
# 1.3.5: 错误四列已由 `_error_fields` 定型(三种入参形态 × 两类行的唯一规则所有者),
|
||||
# 本方法只搬运——拆成五个平铺参数就是把"一处定型"换回"三处各自拼"
|
||||
errors: _ErrorFields,
|
||||
cached_prompt_tokens: int | None,
|
||||
model_reported: str | None,
|
||||
sampling: str | None,
|
||||
@@ -403,6 +587,11 @@ class TelemetryEmitter:
|
||||
# 注释),本方法只搬运——把定型放这里就得再传一遍 response/source,等于把
|
||||
# "唯一 record_llm_call 调用点"换成"两处口径判断",那正是要避免的复制
|
||||
reasoning_effort: str | None,
|
||||
# —— 1.3.5: 行形态与逻辑调用快照 ——
|
||||
operation: CallOperation,
|
||||
event_kind: EventKind,
|
||||
attempts: int | None,
|
||||
total_latency_ms: int | None,
|
||||
) -> None:
|
||||
try:
|
||||
# 成本换算(M2 §6): 成功行按单价换算;缓存命中 0.0(未产生新调用);
|
||||
@@ -413,7 +602,7 @@ class TelemetryEmitter:
|
||||
# 用量不可得: 宁可算不出成本,也不算错成本(解耦设计 §3.1 不变式)。
|
||||
# 必须排在 cache_hit 之后——缓存命中未产生新调用,0.0 是事实而非未知
|
||||
cost = None
|
||||
elif error is None and model and self._pricing is not None:
|
||||
elif errors.error is None and model and self._pricing is not None:
|
||||
cost = self._pricing.cost(
|
||||
model, prompt_tokens, completion_tokens, cached_prompt_tokens
|
||||
)
|
||||
@@ -425,6 +614,7 @@ class TelemetryEmitter:
|
||||
_cap_messages(digest_messages(request.messages), self._text_cap),
|
||||
ensure_ascii=False,
|
||||
)
|
||||
context = request.call_context
|
||||
await self._recorder.record_llm_call(
|
||||
call_id=call_id,
|
||||
parent_call_id=request.parent_call_id,
|
||||
@@ -442,7 +632,7 @@ class TelemetryEmitter:
|
||||
ttft_ms=ttft_ms,
|
||||
max_inter_token_ms=max_inter_token_ms,
|
||||
cache_hit=cache_hit,
|
||||
error=error,
|
||||
error=errors.error,
|
||||
cost=cost,
|
||||
cached_prompt_tokens=cached_prompt_tokens,
|
||||
model_reported=model_reported,
|
||||
@@ -457,6 +647,19 @@ class TelemetryEmitter:
|
||||
# Postgres 那一路悄悄少一列数据
|
||||
thinking_observation=_normalize_observation(thinking_observation),
|
||||
reasoning_effort=reasoning_effort,
|
||||
# —— 1.3.5 十列 ——
|
||||
scope=self._scope,
|
||||
# 调用点给定的公开方法四值,**绝不读 `exc.operation`**(设计 §5 I1/I2)
|
||||
operation=operation,
|
||||
# 上下文缺席(库内现场构造的 ChatRequest)→ NULL,**不造 ID**(I5)
|
||||
logical_call_id=None if context is None else context.logical_call_id,
|
||||
event_kind=event_kind,
|
||||
http_status_code=errors.http_status_code,
|
||||
error_type=errors.error_type,
|
||||
cause_type=errors.cause_type,
|
||||
error_body=errors.error_body,
|
||||
attempts=attempts,
|
||||
total_latency_ms=total_latency_ms,
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
@@ -464,8 +667,45 @@ class TelemetryEmitter:
|
||||
logger.warning("遥测记录失败(降级不冒泡): {}", exc)
|
||||
|
||||
|
||||
async def emit_terminal_once(
|
||||
emitter: TelemetryEmitter | None,
|
||||
*,
|
||||
request: ChatRequest,
|
||||
context: _CallContext,
|
||||
error: PolyGatewayError | str,
|
||||
operation: CallOperation,
|
||||
class_prefixed_error: bool = False,
|
||||
) -> None:
|
||||
"""三个 client 共用的**终态唯一出口**: 去重 + 同步冻结快照 + best effort 写入。
|
||||
|
||||
去重由 `claim_terminal()` 承担(每逻辑调用至多一条终态行);`emitter is None`
|
||||
或已写过 → 直接返回。写入侧异常按既有降级只落 warning(在 `_record` 内)。
|
||||
|
||||
**`CancelledError` 原样传播**(取消优先,不 shield、不开后台任务): 这一次
|
||||
`await` 本身就是新的取消点,外部取消落在它上时调用方会看到 `CancelledError`
|
||||
而非领域错误——与 TelemetryMW 的历史行为同款,已经人类批准(设计 §6/§10)。
|
||||
快照冻结是**同步**动作,故终态行不含它自身的写入耗时。
|
||||
"""
|
||||
if emitter is None or not context.claim_terminal():
|
||||
return
|
||||
stats = context.snapshot()
|
||||
await emitter.emit_terminal_failure(
|
||||
request=request,
|
||||
call_id=str(uuid.uuid4()),
|
||||
error=error,
|
||||
operation=operation,
|
||||
stats=stats,
|
||||
class_prefixed_error=class_prefixed_error,
|
||||
)
|
||||
|
||||
|
||||
class TelemetryMW:
|
||||
"""洋葱最外层: 观测尝试层看不见的路径,任何路径都留痕(遥测必录)。"""
|
||||
"""洋葱最外层: 只观测尝试层看不见的**缓存命中**。
|
||||
|
||||
1.3.5 起不再在此写终态失败行: 终态由 `GatewayClient.chat` 的公开边界经
|
||||
`emit_terminal_once` 统一写出。两处同时写会让同一次失败出两条终态行,
|
||||
而下游正是按 `event_kind = 'terminal_failure'` 计失败调用次数的。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, emitter: TelemetryEmitter, now: Callable[[], float] = time.monotonic
|
||||
@@ -474,26 +714,7 @@ class TelemetryMW:
|
||||
self._now = now
|
||||
|
||||
async def __call__(self, request: ChatRequest, call_next: CallNext) -> LLMResponse:
|
||||
started = self._now()
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except (GatewayUnavailableError, GovernanceBackendError, SourceNotConfiguredError) as exc:
|
||||
await self._emitter.emit_terminal_failure(
|
||||
request=request,
|
||||
call_id=str(uuid.uuid4()),
|
||||
latency_ms=int((self._now() - started) * 1000),
|
||||
error=str(exc),
|
||||
)
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
# 尽力而为: 取消也留痕(§5.1 约定④);随后立即重抛
|
||||
await self._emitter.emit_terminal_failure(
|
||||
request=request,
|
||||
call_id=str(uuid.uuid4()),
|
||||
latency_ms=int((self._now() - started) * 1000),
|
||||
error="cancelled",
|
||||
)
|
||||
raise
|
||||
response = await call_next(request)
|
||||
if response.cache_hit:
|
||||
await self._emitter.emit_cache_hit(request=request, response=response)
|
||||
await self._emitter.emit_cache_hit(request=request, response=response, operation="chat")
|
||||
return response
|
||||
|
||||
+130
-19
@@ -37,7 +37,7 @@ from polygateway.middleware.admission import SourceAdmission, settle_and_release
|
||||
from polygateway.middleware.breaker import BreakerGate
|
||||
from polygateway.middleware.ratelimit import QuotaGate
|
||||
from polygateway.middleware.retry import StallClock, _failure_reason, backoff_delay
|
||||
from polygateway.middleware.telemetry import TelemetryEmitter
|
||||
from polygateway.middleware.telemetry import TelemetryEmitter, emit_terminal_once
|
||||
from polygateway.ports import OutcomeAwareSelector
|
||||
from polygateway.types import (
|
||||
CallStats,
|
||||
@@ -67,6 +67,7 @@ if TYPE_CHECKING:
|
||||
)
|
||||
from polygateway.types import (
|
||||
BackpressurePolicy,
|
||||
CallOperation,
|
||||
OcrLayoutTransportResult,
|
||||
OcrTextTransportResult,
|
||||
RetryPolicy,
|
||||
@@ -128,7 +129,9 @@ class OcrClient:
|
||||
self._breaker = BreakerGate(breaker, scope=self._scope)
|
||||
self._transport = transport
|
||||
self._retry = retry
|
||||
self._emitter = TelemetryEmitter(telemetry, text_cap=text_cap) if telemetry else None
|
||||
self._emitter = (
|
||||
TelemetryEmitter(telemetry, scope=self._scope, text_cap=text_cap) if telemetry else None
|
||||
)
|
||||
self._telemetry = telemetry
|
||||
# 限流/熔断后端在此之外只以 QuotaGate/BreakerGate 的形态存在,自持一份
|
||||
# 引用才关得到自建的 redis 客户端(设计 §3.4)
|
||||
@@ -179,7 +182,13 @@ class OcrClient:
|
||||
tenant_id, meta, origin="recognize_text(tenant_id=..., meta=...)"
|
||||
)
|
||||
outcome, call_stats = await self._call(
|
||||
"text", image, session_id, parent_call_id, dimension_tenant_id, dimensions
|
||||
"text",
|
||||
"recognize_text",
|
||||
image,
|
||||
session_id,
|
||||
parent_call_id,
|
||||
dimension_tenant_id,
|
||||
dimensions,
|
||||
)
|
||||
result = outcome.result
|
||||
return OcrTextResult(
|
||||
@@ -210,7 +219,13 @@ class OcrClient:
|
||||
tenant_id, meta, origin="parse_layout(tenant_id=..., meta=...)"
|
||||
)
|
||||
outcome, call_stats = await self._call(
|
||||
"layout", image, session_id, parent_call_id, dimension_tenant_id, dimensions
|
||||
"layout",
|
||||
"parse_layout",
|
||||
image,
|
||||
session_id,
|
||||
parent_call_id,
|
||||
dimension_tenant_id,
|
||||
dimensions,
|
||||
)
|
||||
result = outcome.result
|
||||
return OcrLayoutResult(
|
||||
@@ -241,6 +256,7 @@ class OcrClient:
|
||||
async def _call(
|
||||
self,
|
||||
kind: _OcrKind,
|
||||
operation: CallOperation,
|
||||
image: bytes,
|
||||
session_id: str | None,
|
||||
parent_call_id: str | None,
|
||||
@@ -254,6 +270,46 @@ class OcrClient:
|
||||
# M1 例外: `image` 校验在 `_call` 内而非公开方法,故上下文在该校验
|
||||
# **通过之后**创建——这样设计 §3 的"校验在统计边界外"对 OCR 才成立
|
||||
context = _CallContext(now=self._now)
|
||||
try:
|
||||
return await self._run(
|
||||
kind, operation, image, session_id, parent_call_id, tenant_id, meta, context
|
||||
)
|
||||
except PolyGatewayError as exc:
|
||||
await self._emit_terminal(
|
||||
kind, operation, image, session_id, parent_call_id, tenant_id, meta, context, exc
|
||||
)
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
# 三条链路同一口径尽力写一条(允许 0 条);取消优先,不 shield
|
||||
await self._emit_terminal(
|
||||
kind,
|
||||
operation,
|
||||
image,
|
||||
session_id,
|
||||
parent_call_id,
|
||||
tenant_id,
|
||||
meta,
|
||||
context,
|
||||
"cancelled",
|
||||
)
|
||||
raise
|
||||
|
||||
async def _run(
|
||||
self,
|
||||
kind: _OcrKind,
|
||||
operation: CallOperation,
|
||||
image: bytes,
|
||||
session_id: str | None,
|
||||
parent_call_id: str | None,
|
||||
tenant_id: str | None,
|
||||
meta: dict[str, Any],
|
||||
context: _CallContext,
|
||||
) -> tuple[_AttemptOutcome, CallStats]:
|
||||
"""选源与重试循环。
|
||||
|
||||
无源的 raise 必须在本方法内(而非循环之前的调用方): 它得被 `_call` 的
|
||||
`try` 包住,否则无源终态行根本写不出来(设计 §3.5)。
|
||||
"""
|
||||
if not self._sources:
|
||||
raise AllSourcesExhausted(scope=self._scope, reason="no_sources", retry_after_s=0.0)
|
||||
fails = 0
|
||||
@@ -268,6 +324,7 @@ class OcrClient:
|
||||
async with clock.attempting():
|
||||
outcome = await self._attempt(
|
||||
kind,
|
||||
operation,
|
||||
image,
|
||||
*picked,
|
||||
reasons,
|
||||
@@ -293,6 +350,7 @@ class OcrClient:
|
||||
async def _attempt(
|
||||
self,
|
||||
kind: _OcrKind,
|
||||
operation: CallOperation,
|
||||
image: bytes,
|
||||
source: SourceConfig,
|
||||
permit: Permit,
|
||||
@@ -318,6 +376,7 @@ class OcrClient:
|
||||
latency_ms = int((self._now() - started) * 1000)
|
||||
await self._emit(
|
||||
kind,
|
||||
operation,
|
||||
image,
|
||||
source,
|
||||
call_id,
|
||||
@@ -326,6 +385,7 @@ class OcrClient:
|
||||
parent_call_id,
|
||||
tenant_id,
|
||||
meta,
|
||||
context,
|
||||
result,
|
||||
)
|
||||
return _AttemptOutcome(result, source, call_id, latency_ms)
|
||||
@@ -333,6 +393,7 @@ class OcrClient:
|
||||
await self._gate_on_terminal(exc, entry)
|
||||
await self._emit(
|
||||
kind,
|
||||
operation,
|
||||
image,
|
||||
source,
|
||||
call_id,
|
||||
@@ -341,6 +402,7 @@ class OcrClient:
|
||||
parent_call_id,
|
||||
tenant_id,
|
||||
meta,
|
||||
context,
|
||||
error=exc,
|
||||
)
|
||||
raise
|
||||
@@ -349,6 +411,7 @@ class OcrClient:
|
||||
await self._record_quietly(self._breaker.release_probe(entry))
|
||||
await self._emit(
|
||||
kind,
|
||||
operation,
|
||||
image,
|
||||
source,
|
||||
call_id,
|
||||
@@ -357,6 +420,7 @@ class OcrClient:
|
||||
parent_call_id,
|
||||
tenant_id,
|
||||
meta,
|
||||
context,
|
||||
error="cancelled",
|
||||
)
|
||||
raise
|
||||
@@ -368,6 +432,7 @@ class OcrClient:
|
||||
self._feed_outcome(source.name, ok=False)
|
||||
await self._emit(
|
||||
kind,
|
||||
operation,
|
||||
image,
|
||||
source,
|
||||
call_id,
|
||||
@@ -376,6 +441,7 @@ class OcrClient:
|
||||
parent_call_id,
|
||||
tenant_id,
|
||||
meta,
|
||||
context,
|
||||
error=exc,
|
||||
)
|
||||
return _FailedAttempt(exc, immediate=dead)
|
||||
@@ -417,9 +483,60 @@ class OcrClient:
|
||||
except (GovernanceBackendError, SourceNotConfiguredError) as exc:
|
||||
logger.warning("OCR 治理记账写回降级(不冒泡): {}", exc)
|
||||
|
||||
def _request_for(
|
||||
self,
|
||||
kind: _OcrKind,
|
||||
image: bytes,
|
||||
session_id: str | None,
|
||||
parent_call_id: str | None,
|
||||
tenant_id: str | None,
|
||||
meta: dict[str, Any],
|
||||
context: _CallContext,
|
||||
) -> ChatRequest:
|
||||
"""OCR 行的现场 ChatRequest: 占位摘要,**图像 bytes 永不入库**。
|
||||
|
||||
尝试行与终态行共用同一个构造点: 占位字面量复制成两份就会漂移。
|
||||
调用方维度与上下文必须显式填回(OCR 不走 chat 洋葱),否则 OCR 行的
|
||||
维度恒为空、`logical_call_id` 恒为 NULL。
|
||||
"""
|
||||
return ChatRequest(
|
||||
messages=[{"role": "user", "content": f"<ocr:{kind} image_bytes={len(image)}>"}],
|
||||
session_id=session_id,
|
||||
parent_call_id=parent_call_id,
|
||||
tenant_id=tenant_id,
|
||||
meta=meta,
|
||||
call_context=context,
|
||||
)
|
||||
|
||||
async def _emit_terminal(
|
||||
self,
|
||||
kind: _OcrKind,
|
||||
operation: CallOperation,
|
||||
image: bytes,
|
||||
session_id: str | None,
|
||||
parent_call_id: str | None,
|
||||
tenant_id: str | None,
|
||||
meta: dict[str, Any],
|
||||
context: _CallContext,
|
||||
error: PolyGatewayError | str,
|
||||
) -> None:
|
||||
"""终态行: 沿用 `<ocr:{kind} image_bytes=…>` 占位,错误文本保留类名前缀。"""
|
||||
await emit_terminal_once(
|
||||
self._emitter,
|
||||
request=self._request_for(
|
||||
kind, image, session_id, parent_call_id, tenant_id, meta, context
|
||||
),
|
||||
context=context,
|
||||
error=error,
|
||||
operation=operation,
|
||||
# metric ocr-call-success 的注册口径是按类名归组,终态行同款保留
|
||||
class_prefixed_error=True,
|
||||
)
|
||||
|
||||
async def _emit(
|
||||
self,
|
||||
kind: _OcrKind,
|
||||
operation: CallOperation,
|
||||
image: bytes,
|
||||
source: SourceConfig,
|
||||
call_id: str,
|
||||
@@ -428,20 +545,15 @@ class OcrClient:
|
||||
parent_call_id: str | None,
|
||||
tenant_id: str | None,
|
||||
meta: dict[str, Any],
|
||||
context: _CallContext,
|
||||
result: OcrTextTransportResult | OcrLayoutTransportResult | None = None,
|
||||
error: object | None = None,
|
||||
error: PolyGatewayError | str | None = None,
|
||||
) -> None:
|
||||
"""逐尝试遥测(单一 Emitter): messages 占位摘要,图像 bytes 绝不入库。"""
|
||||
if self._emitter is None:
|
||||
return
|
||||
# 这个 ChatRequest 只为复用同一个 Emitter 而现场构造(OCR 不走 chat 洋葱),
|
||||
# 故调用方维度必须在这里显式填回,否则 OCR 行的维度恒为空
|
||||
request = ChatRequest(
|
||||
messages=[{"role": "user", "content": f"<ocr:{kind} image_bytes={len(image)}>"}],
|
||||
session_id=session_id,
|
||||
parent_call_id=parent_call_id,
|
||||
tenant_id=tenant_id,
|
||||
meta=meta,
|
||||
request = self._request_for(
|
||||
kind, image, session_id, parent_call_id, tenant_id, meta, context
|
||||
)
|
||||
latency_ms = int((self._now() - started) * 1000)
|
||||
response = None
|
||||
@@ -461,20 +573,19 @@ class OcrClient:
|
||||
source_name=source.name,
|
||||
usage_source="measured",
|
||||
)
|
||||
# 错误带异常类名前缀(metric ocr-call-success 注册口径: 按类名归组)
|
||||
if error is None or isinstance(error, str):
|
||||
error_text = error
|
||||
else:
|
||||
error_text = f"{type(error).__name__}: {error}"
|
||||
# 错误文本的类名前缀现由出口的显式策略参数承担(设计 §6 I7):
|
||||
# 三处各拼一遍才是下一次漂移的种子,而取消行传的是字符串,不受前缀影响
|
||||
await self._emitter.emit_attempt(
|
||||
request=request,
|
||||
source=source,
|
||||
call_id=call_id,
|
||||
latency_ms=latency_ms,
|
||||
response=response,
|
||||
error=error_text,
|
||||
error=error,
|
||||
# OCR 走 MonkeyOCR 自有端点,没有推理参数可言(理由同 embedding)
|
||||
reasoning_applies=False,
|
||||
operation=operation,
|
||||
class_prefixed_error=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -271,7 +271,7 @@ class TelemetryStatusProvider(Protocol):
|
||||
|
||||
@runtime_checkable
|
||||
class TelemetryRecorder(Protocol):
|
||||
"""遥测后端;26 字段冻结(M1 设计 §4.4 + issue #3/#4/#11/#16/#20),唯一调用点是 TelemetryEmitter。
|
||||
"""遥测后端;36 字段冻结(M1 设计 §4.4 + issue #3/#4/#11/#16/#20 + 1.3.5),唯一调用点是 TelemetryEmitter。
|
||||
|
||||
新增参数不设默认值: 库外无第三方实现者(三项目迁移时删除了各自的同名
|
||||
Protocol),完整签名的成本为零,而少写一列会被 emitter 的降级吞成 warning。
|
||||
@@ -286,6 +286,15 @@ class TelemetryRecorder(Protocol):
|
||||
|
||||
recorder 只负责落库,不做任何语义判断,与 `sampling` 列由
|
||||
`canonical_sampling_json()` 在 emitter 侧定型是同一先例。
|
||||
|
||||
1.3.5 新增十列同理已在 emitter 侧定型: `operation` 是**公开方法**四值之一
|
||||
(与 `PolyGatewayError.operation` 这个 HTTP 子操作是两个语义);`event_kind` 区分
|
||||
attempt / cache_hit / terminal_failure 三类行;`attempts` 与 `total_latency_ms`
|
||||
只在终态行非空;`error_body` 沿用 transport 侧 `summarize_body` 的上限,
|
||||
**不进 `PGW_TELEMETRY_TEXT_CAP` 的覆盖面**。
|
||||
|
||||
**本签名是装配闸的唯一事实源**: `TelemetryEmitter.__init__` 按它派生参数名做
|
||||
一次 `signature.bind` 形状校验(设计 §7),改本签名即改闸的判据。
|
||||
"""
|
||||
|
||||
async def record_llm_call(
|
||||
@@ -317,4 +326,14 @@ class TelemetryRecorder(Protocol):
|
||||
meta: str,
|
||||
thinking_observation: str,
|
||||
reasoning_effort: str | None,
|
||||
scope: str,
|
||||
operation: str,
|
||||
logical_call_id: str | None,
|
||||
event_kind: str,
|
||||
http_status_code: int | None,
|
||||
error_type: str | None,
|
||||
cause_type: str | None,
|
||||
error_body: str | None,
|
||||
attempts: int | None,
|
||||
total_latency_ms: int | None,
|
||||
) -> None: ...
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
多处各存一份必然漂移,而漂移的表现是"下游照打印的 SQL 建完表,库仍报缺列"。
|
||||
|
||||
**`COLUMNS` 是 INSERT 字段序,不是物理列序**: 数据库自填的 `created_at` 不在其中(它带
|
||||
`DEFAULT now()` / `datetime('now')`,库从不显式写它)。物理表列 = 26 个 INSERT 字段 +
|
||||
`created_at` = 27;列数断言一律按物理列数写,两套口径混用是最易错处。
|
||||
`DEFAULT now()` / `datetime('now')`,库从不显式写它)。物理表列 = 36 个 INSERT 字段 +
|
||||
`created_at` = 37;列数断言一律按物理列数写,两套口径混用是最易错处。
|
||||
|
||||
本模块只依赖标准库: `telemetry/` 与 `backends/`、`transports/`、`structured/` 同层且
|
||||
互不依赖(import-linter 契约执法)。
|
||||
@@ -52,7 +52,17 @@ CREATE TABLE IF NOT EXISTS llm_calls (
|
||||
tenant_id TEXT NOT NULL DEFAULT '',
|
||||
meta TEXT NOT NULL DEFAULT '{}',
|
||||
thinking_observation TEXT,
|
||||
reasoning_effort TEXT
|
||||
reasoning_effort TEXT,
|
||||
scope TEXT,
|
||||
operation TEXT,
|
||||
logical_call_id TEXT,
|
||||
event_kind TEXT,
|
||||
http_status_code INTEGER,
|
||||
error_type TEXT,
|
||||
cause_type TEXT,
|
||||
error_body TEXT,
|
||||
attempts INTEGER,
|
||||
total_latency_ms INTEGER
|
||||
);
|
||||
"""
|
||||
|
||||
@@ -84,7 +94,17 @@ CREATE TABLE IF NOT EXISTS llm_calls (
|
||||
tenant_id TEXT NOT NULL DEFAULT '',
|
||||
meta JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
thinking_observation TEXT,
|
||||
reasoning_effort TEXT
|
||||
reasoning_effort TEXT,
|
||||
scope TEXT,
|
||||
operation TEXT,
|
||||
logical_call_id TEXT,
|
||||
event_kind TEXT,
|
||||
http_status_code INTEGER,
|
||||
error_type TEXT,
|
||||
cause_type TEXT,
|
||||
error_body TEXT,
|
||||
attempts INTEGER,
|
||||
total_latency_ms INTEGER
|
||||
);
|
||||
"""
|
||||
|
||||
@@ -105,6 +125,18 @@ SQLITE_BACKFILL = (
|
||||
# 同样可空,但这里 NULL 表达的是"调用方没表态"(issue #20): 它与 'none'
|
||||
# (明确要求不推理)是两回事,折叠成任一档都等于替上游声称了它没说过的事
|
||||
("reasoning_effort", "TEXT"),
|
||||
# 1.3.5 十列: 全部可空且无默认值——旧行的 NULL 表达的是"补列之前根本没记过
|
||||
# 这件事",与任何哨兵值都不是一回事,故不回填(设计 §5)
|
||||
("scope", "TEXT"),
|
||||
("operation", "TEXT"),
|
||||
("logical_call_id", "TEXT"),
|
||||
("event_kind", "TEXT"),
|
||||
("http_status_code", "INTEGER"),
|
||||
("error_type", "TEXT"),
|
||||
("cause_type", "TEXT"),
|
||||
("error_body", "TEXT"),
|
||||
("attempts", "INTEGER"),
|
||||
("total_latency_ms", "INTEGER"),
|
||||
)
|
||||
|
||||
# PG 补列的列定义。语句由此派生成两份文本(见下),使"库内执行的那份"与"打印给
|
||||
@@ -120,6 +152,17 @@ _PG_BACKFILL_DECLS = (
|
||||
# 可空,理由同 SQLITE_BACKFILL 同名项
|
||||
("thinking_observation", "TEXT"),
|
||||
("reasoning_effort", "TEXT"),
|
||||
# 1.3.5 十列,列序与 SQLITE_BACKFILL 逐项对齐(两条路径的物理列序不许分叉)
|
||||
("scope", "TEXT"),
|
||||
("operation", "TEXT"),
|
||||
("logical_call_id", "TEXT"),
|
||||
("event_kind", "TEXT"),
|
||||
("http_status_code", "INTEGER"),
|
||||
("error_type", "TEXT"),
|
||||
("cause_type", "TEXT"),
|
||||
("error_body", "TEXT"),
|
||||
("attempts", "INTEGER"),
|
||||
("total_latency_ms", "INTEGER"),
|
||||
)
|
||||
|
||||
# 新列排在 created_at 之后: 与旧表 ALTER 追加的位置一致(见 SQLITE_BACKFILL 同款注释)。
|
||||
@@ -158,6 +201,17 @@ COLUMNS = (
|
||||
"meta",
|
||||
"thinking_observation",
|
||||
"reasoning_effort",
|
||||
# —— 1.3.5 逻辑调用统计与结构化失败诊断(issue #19/#23)——
|
||||
"scope",
|
||||
"operation",
|
||||
"logical_call_id",
|
||||
"event_kind",
|
||||
"http_status_code",
|
||||
"error_type",
|
||||
"cause_type",
|
||||
"error_body",
|
||||
"attempts",
|
||||
"total_latency_ms",
|
||||
)
|
||||
|
||||
_COLUMN_SET = frozenset(COLUMNS)
|
||||
|
||||
@@ -154,18 +154,28 @@ def _classify(status: int) -> tuple[type[PolyGatewayError], str]:
|
||||
|
||||
|
||||
def _status_to_error(
|
||||
source: SourceConfig, status: int, body_text: str, headers: Mapping[str, str]
|
||||
source: SourceConfig,
|
||||
status: int,
|
||||
body_text: str,
|
||||
headers: Mapping[str, str],
|
||||
*,
|
||||
operation: str,
|
||||
) -> Exception:
|
||||
"""非 2xx → 领域错误,**全部分支**携带响应体摘要(issue #10)。
|
||||
|
||||
摘要只算一次,message 与 `body_text` 共用同一份串: 两份不同长度会让"遥测里
|
||||
看到的"与"下游 catch 到的"对不上,排查时反而多一层困惑。
|
||||
|
||||
`operation` 是 **HTTP 子操作**词表(`chat` / `embedding` / `ocr_text` / ...),由
|
||||
调用点显式给定。它曾被硬编码成 `"chat"`,而 `embed()` 的非 200 分支也走它
|
||||
——于是现存所有 embedding HTTP 失败的 `exc.operation` 都是错的(1.3.5 设计 §5 I1)。
|
||||
注意它与遥测新列 `operation`(公开方法四值)是**两个语义**,不做自动转换。
|
||||
"""
|
||||
summary = summarize_body(body_text)
|
||||
ctx: dict[str, Any] = {
|
||||
"source_name": source.name,
|
||||
"status_code": status,
|
||||
"operation": "chat",
|
||||
"operation": operation,
|
||||
"body_text": summary,
|
||||
}
|
||||
if status == 429:
|
||||
@@ -509,7 +519,10 @@ class OpenAICompatTransport:
|
||||
except httpx.TransportError as exc:
|
||||
raise TransientError(f"{source.name} 网络错误: {exc}", **ctx) from exc
|
||||
if resp.status_code != 200:
|
||||
raise _status_to_error(source, resp.status_code, resp.text, resp.headers)
|
||||
# 历史误标修正: 本分支属 `embed()`,与上方 ctx 同为 `"embedding"`
|
||||
raise _status_to_error(
|
||||
source, resp.status_code, resp.text, resp.headers, operation="embedding"
|
||||
)
|
||||
return _parse_embedding_payload(resp, source, len(texts))
|
||||
|
||||
async def _complete_stream(
|
||||
@@ -524,7 +537,9 @@ class OpenAICompatTransport:
|
||||
async with client.stream("POST", url, json=payload) as resp:
|
||||
if resp.status_code != 200:
|
||||
body = (await resp.aread()).decode("utf-8", errors="replace")
|
||||
raise _status_to_error(source, resp.status_code, body, resp.headers)
|
||||
raise _status_to_error(
|
||||
source, resp.status_code, body, resp.headers, operation="chat"
|
||||
)
|
||||
sink: dict[str, Any] = {}
|
||||
guarded = stream_with_liveness_timeouts(
|
||||
_iter_sse_deltas(resp.aiter_lines(), sink),
|
||||
@@ -622,7 +637,9 @@ class OpenAICompatTransport:
|
||||
"""非流式快路径(三项目均无,库新增): 单 JSON 响应,仅 total 超时。"""
|
||||
resp = await client.post(url, json=payload)
|
||||
if resp.status_code != 200:
|
||||
raise _status_to_error(source, resp.status_code, resp.text, resp.headers)
|
||||
raise _status_to_error(
|
||||
source, resp.status_code, resp.text, resp.headers, operation="chat"
|
||||
)
|
||||
try:
|
||||
body = resp.json()
|
||||
except json.JSONDecodeError as exc:
|
||||
|
||||
Reference in New Issue
Block a user