feat: make telemetry degradation a first-class state
Telemetry degradation used to be a single warning and a private boolean. In a long-running process that is indistinguishable from telemetry working: issue #15 was only found by hand-reconciling milestone log lines against llm_calls rows, after 19 calls had silently gone unrecorded. The SQLite side was worse — once init failed, every write returned without even a log line. Degradation now has one shared owner. TelemetryStatusTracker holds the state machine (enter/recover/drop/should-retry), announces entry and recovery once each, and repeats the drop count under a row-and-time double threshold so a degraded backend neither floods the log nor goes quiet. Both recorders hold one; both count the rows they drop. For programmatic consumers, TelemetryStatus is a frozen snapshot exposed as telemetry_status on all three clients, resolved through a single isinstance check. It is a separate optional port rather than a member of TelemetryRecorder: that protocol is @runtime_checkable, so adding an attribute would make every implementation that only defines record_llm_call stop satisfying it — downstream isinstance assertions would break on upgrade. The existing assertion in test_ports.py is what keeps that decision honest. Failure criteria are deliberately untouched here: Postgres still treats a pool failure as permanent, only now visibly. `_failed` and the tracker therefore both carry the verdict for the span of this one change; the cooldown rework collapses them into the tracker alone.
This commit is contained in:
@@ -24,6 +24,7 @@ from polygateway.middleware.cache import CacheMW
|
||||
from polygateway.middleware.retry import RetryMW
|
||||
from polygateway.middleware.structured import StructuredMW
|
||||
from polygateway.middleware.telemetry import TelemetryEmitter, TelemetryMW
|
||||
from polygateway.ports import TelemetryStatusProvider
|
||||
from polygateway.pricing import PricingTable
|
||||
from polygateway.providers import get_capability, get_provider, resolve_thinking
|
||||
from polygateway.sources import (
|
||||
@@ -37,6 +38,7 @@ from polygateway.transports.openai_compat import OpenAICompatTransport
|
||||
from polygateway.types import (
|
||||
ChatRequest,
|
||||
LLMResponse,
|
||||
TelemetryStatus,
|
||||
validate_caller_dimensions,
|
||||
validate_request_overlay,
|
||||
)
|
||||
@@ -135,6 +137,19 @@ async def _aclose_component(component: object | None) -> None:
|
||||
close()
|
||||
|
||||
|
||||
def _telemetry_status_of(telemetry: TelemetryRecorder | None) -> TelemetryStatus | None:
|
||||
"""三个 client 共用的状态取值点: 不提供状态的 recorder 一律返回 None。
|
||||
|
||||
判定写成 `isinstance(可选端口)` 而不是裸 `getattr`: 两者运行时都是结构检查
|
||||
(`@runtime_checkable` 按属性存在性判定),差别在**契约有没有名字**——端口是
|
||||
写进 `ports.py` 的公开承诺,下游可以照着实现;散落的 `getattr` 不是,而
|
||||
`aclose` 当年正是被复制成三份鸭子类型探测才漂移出越权关闭(设计 §3.3/§3.4)。
|
||||
"""
|
||||
if isinstance(telemetry, TelemetryStatusProvider):
|
||||
return telemetry.telemetry_status
|
||||
return None
|
||||
|
||||
|
||||
def _mark_owned_components(
|
||||
client: Any,
|
||||
*,
|
||||
@@ -252,6 +267,15 @@ class GatewayClient:
|
||||
self._owns_breaker = False
|
||||
self._closed = False
|
||||
|
||||
@property
|
||||
def telemetry_status(self) -> TelemetryStatus | None:
|
||||
"""遥测后端的可写状态;无遥测或注入的 recorder 不提供状态时为 None。
|
||||
|
||||
判定收敛在 `_telemetry_status_of` 一处(不是三处各自探测): 三个 client
|
||||
的 `aclose` 曾各持一份逐字复制,漂移的结果就是越权关闭(设计 §3.3/§3.4)。
|
||||
"""
|
||||
return _telemetry_status_of(self._telemetry)
|
||||
|
||||
async def chat(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
|
||||
@@ -25,7 +25,7 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from polygateway.client import _aclose_component
|
||||
from polygateway.client import _aclose_component, _telemetry_status_of
|
||||
from polygateway.config import EmbeddingSettings
|
||||
from polygateway.errors import (
|
||||
AllSourcesExhausted,
|
||||
@@ -46,6 +46,7 @@ from polygateway.types import (
|
||||
ChatRequest,
|
||||
EmbeddingResponse,
|
||||
LLMResponse,
|
||||
TelemetryStatus,
|
||||
strip_unsupported_extra_body,
|
||||
validate_caller_dimensions,
|
||||
)
|
||||
@@ -454,6 +455,15 @@ class EmbeddingClient:
|
||||
known = [c for c in costs if c is not None]
|
||||
return sum(known) if known else None
|
||||
|
||||
@property
|
||||
def telemetry_status(self) -> TelemetryStatus | None:
|
||||
"""遥测后端的可写状态;无遥测或注入的 recorder 不提供状态时为 None。
|
||||
|
||||
判定收敛在 `_telemetry_status_of` 一处(不是三处各自探测): 三个 client
|
||||
的 `aclose` 曾各持一份逐字复制,漂移的结果就是越权关闭(设计 §3.3/§3.4)。
|
||||
"""
|
||||
return _telemetry_status_of(self._telemetry)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""幂等释放**自建**资源(与 GatewayClient 对称);注入的组件一律不碰。"""
|
||||
if self._closed:
|
||||
|
||||
+11
-1
@@ -22,7 +22,7 @@ from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from polygateway.client import _aclose_component
|
||||
from polygateway.client import _aclose_component, _telemetry_status_of
|
||||
from polygateway.errors import (
|
||||
AllSourcesExhausted,
|
||||
GovernanceBackendError,
|
||||
@@ -44,6 +44,7 @@ from polygateway.types import (
|
||||
LLMResponse,
|
||||
OcrLayoutResult,
|
||||
OcrTextResult,
|
||||
TelemetryStatus,
|
||||
Usage,
|
||||
strip_unsupported_extra_body,
|
||||
validate_caller_dimensions,
|
||||
@@ -464,6 +465,15 @@ class OcrClient:
|
||||
|
||||
# —— 生命周期 ——
|
||||
|
||||
@property
|
||||
def telemetry_status(self) -> TelemetryStatus | None:
|
||||
"""遥测后端的可写状态;无遥测或注入的 recorder 不提供状态时为 None。
|
||||
|
||||
判定收敛在 `_telemetry_status_of` 一处(不是三处各自探测): 三个 client
|
||||
的 `aclose` 曾各持一份逐字复制,漂移的结果就是越权关闭(设计 §3.3/§3.4)。
|
||||
"""
|
||||
return _telemetry_status_of(self._telemetry)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""幂等释放**自建**资源(与 EmbeddingClient 对称);注入的组件一律不碰。"""
|
||||
if self._closed:
|
||||
|
||||
@@ -20,6 +20,7 @@ from .types import (
|
||||
OcrTextTransportResult,
|
||||
SourceConfig,
|
||||
SourceStats,
|
||||
TelemetryStatus,
|
||||
TransportResult,
|
||||
)
|
||||
|
||||
@@ -243,6 +244,20 @@ class StructuredOutputStrategy(Protocol):
|
||||
def parse(self, text: str) -> Any: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class TelemetryStatusProvider(Protocol):
|
||||
"""可自述可写状态的遥测后端;`TelemetryRecorder` 的**可选**伴生端口(issue #15)。
|
||||
|
||||
与 `TelemetryRecorder` 分开而不是给它加成员,是因为后者是 `@runtime_checkable`
|
||||
而运行时检查按属性存在性做: 加一个属性会让所有只实现 `record_llm_call` 的
|
||||
实现**当场不再是** `TelemetryRecorder`,下游若有同款 isinstance 断言,升级即断
|
||||
(设计 §3.3)。消费方一律先 isinstance 再取值,取不到就当没有状态可报。
|
||||
"""
|
||||
|
||||
@property
|
||||
def telemetry_status(self) -> TelemetryStatus: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class TelemetryRecorder(Protocol):
|
||||
"""遥测后端;24 字段冻结(M1 设计 §4.4 + issue #3/#4/#11),唯一调用点是 TelemetryEmitter。
|
||||
|
||||
@@ -28,10 +28,13 @@ from polygateway.telemetry.schema import (
|
||||
insert_sql,
|
||||
missing_columns_warning,
|
||||
)
|
||||
from polygateway.telemetry.status import TelemetryStatusTracker
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import asyncpg
|
||||
|
||||
from polygateway.types import TelemetryStatus
|
||||
|
||||
# 探测表是否存在;不需要任何权限,且与 INSERT 走同一套 search_path 解析
|
||||
_TABLE_EXISTS = "SELECT to_regclass('llm_calls')"
|
||||
|
||||
@@ -72,8 +75,16 @@ class PostgresRecorder:
|
||||
self._insert = insert_sql("postgres", COLUMNS)
|
||||
self._schema_ready = False
|
||||
self._failed = False # 结构性降级标志: 置位后所有写入短路
|
||||
# 降级的可编程出口与节流日志;`_failed` 与它并存是 issue #15 的过渡态,
|
||||
# 判据改造(冷却自愈)落地时状态收归 tracker 一处
|
||||
self._status = TelemetryStatusTracker(backend="postgres")
|
||||
self._init_lock = asyncio.Lock()
|
||||
|
||||
@property
|
||||
def telemetry_status(self) -> TelemetryStatus:
|
||||
"""当前可写状态快照(ports.TelemetryStatusProvider)。"""
|
||||
return self._status.snapshot()
|
||||
|
||||
async def _ensure_ready(self) -> asyncpg.Pool | None:
|
||||
"""lazy 建池+备表;判死只认「确定写不进去」(issue #9),其余失败都留活路。"""
|
||||
if self._failed:
|
||||
@@ -104,7 +115,7 @@ class PostgresRecorder:
|
||||
# 池建不出来 = 确定写不进去;且每次调用重试都要内联吞掉 connect
|
||||
# 超时,而遥测是业务路径上的 await —— 此处必须永久降级
|
||||
self._failed = True
|
||||
logger.warning("Postgres 遥测建池失败,后续记录降级为 no-op: {}", exc)
|
||||
self._status.enter_degraded(f"建池失败: {exc}", fatal=True, cooldown_s=None)
|
||||
return None
|
||||
return self._pool
|
||||
|
||||
@@ -122,6 +133,9 @@ class PostgresRecorder:
|
||||
return None
|
||||
if columns is None:
|
||||
self._failed = True
|
||||
self._status.enter_degraded(
|
||||
"表 llm_calls 不存在且建不出来(记录无处可落)", fatal=True, cooldown_s=None
|
||||
)
|
||||
return None
|
||||
# 写入列、语句与就绪标志必须**一起**生效: `_ensure_ready` 只看 `_schema_ready`
|
||||
# 就绕开 `_init_lock` 直接返回池,先置就绪会开出"已就绪但语句还是旧的"的窗口
|
||||
@@ -232,6 +246,8 @@ class PostgresRecorder:
|
||||
"""
|
||||
pool = await self._ensure_ready()
|
||||
if pool is None:
|
||||
# 降级期间静默 return 就是 issue #15 的破口: 丢行必须计数且节流出声
|
||||
self._status.record_drop("遥测已降级")
|
||||
return
|
||||
row = tuple(fields[col] for col in self._columns)
|
||||
try:
|
||||
|
||||
@@ -19,6 +19,7 @@ import asyncio
|
||||
import sqlite3
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from loguru import logger
|
||||
|
||||
@@ -29,6 +30,10 @@ from polygateway.telemetry.schema import (
|
||||
insert_sql,
|
||||
missing_columns_warning,
|
||||
)
|
||||
from polygateway.telemetry.status import TelemetryStatusTracker
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from polygateway.types import TelemetryStatus
|
||||
|
||||
|
||||
class SQLiteRecorder:
|
||||
@@ -45,6 +50,9 @@ class SQLiteRecorder:
|
||||
config 一处,不与本类签名漂移(设计 D-c)。
|
||||
"""
|
||||
self._auto_migrate = auto_migrate
|
||||
# SQLite 侧本次只做可见性: 它的失败模式(目录不可写、文件损坏)在装配期
|
||||
# 就暴露给下游,不是"跑到一半悄悄断",故降级恒为 fatal,不做冷却重连
|
||||
self._status = TelemetryStatusTracker(backend="sqlite")
|
||||
self._lock = threading.Lock()
|
||||
self._conn: sqlite3.Connection | None = None
|
||||
# 先按全量列定型: 连接失败/探测失败时保守沿用全量(今天的行为)
|
||||
@@ -60,9 +68,14 @@ class SQLiteRecorder:
|
||||
conn.commit()
|
||||
self._conn = conn
|
||||
except (OSError, sqlite3.Error) as exc:
|
||||
logger.warning("SQLite 遥测初始化失败,后续记录降级为 no-op: {}", exc)
|
||||
self._status.enter_degraded(f"初始化失败: {exc}", fatal=True, cooldown_s=None)
|
||||
self._prepare_columns()
|
||||
|
||||
@property
|
||||
def telemetry_status(self) -> TelemetryStatus:
|
||||
"""当前可写状态快照(ports.TelemetryStatusProvider)。"""
|
||||
return self._status.snapshot()
|
||||
|
||||
def _prepare_columns(self) -> None:
|
||||
"""探测现有列后定型写入: auto 档补齐缺列,manual 档改为裁剪写入(issue #13)。
|
||||
|
||||
@@ -136,6 +149,9 @@ class SQLiteRecorder:
|
||||
占位符同序——两者必须一起改,分开改就是把值写进错位的列。
|
||||
"""
|
||||
if self._conn is None:
|
||||
# 改前这里是**裸 return**: 初始化失败后每一行都无声消失,长跑进程里
|
||||
# 与"遥测正常"外观上完全一致(设计 §1.4 的直接钉子)
|
||||
self._status.record_drop("遥测已降级")
|
||||
return
|
||||
row = tuple(fields[col] for col in self._columns)
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
"""遥测降级状态机(issue #15 C 组): 两个 recorder 共用的降级事实源。
|
||||
|
||||
存在的理由(设计 §1.4): 遥测降级过去只有**一条** warning,长跑进程里等同于
|
||||
静默——issue 是手工对账(日志里的完成里程碑条数 vs `llm_calls` 行数)才发现的,
|
||||
期间 19 次调用一行未落。"遥测必录"铁律的实质要求是: 库做不到必录时,必须
|
||||
**持续、可编程地**让下游知道。故降级升格为一等对象,两条出路各走一边:
|
||||
|
||||
- 人看: 进入/恢复各一条日志,降级期间按行数与时间**双阈值节流复述**(不刷屏,
|
||||
也不静默);
|
||||
- 程序看: `snapshot()` 给只读 `TelemetryStatus`,下游可据此对账或告警。
|
||||
|
||||
本模块**不含任何后端知识**(不 import asyncpg/sqlite3,也不判失败性质): 失败
|
||||
分类是各 recorder 的事,tracker 只接受"降级了/恢复了/丢了一行"三个事实。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from polygateway.types import TelemetryStatus
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
_DROP_REPEAT_EVERY_ROWS = 100
|
||||
"""降级期间每丢这么多行复述一次;首行必报。"""
|
||||
|
||||
_DROP_REPEAT_EVERY_S = 300.0
|
||||
"""降级期间距上次复述超过这么久就再报一次——低频调用的进程不能因行数不够而静默。"""
|
||||
|
||||
|
||||
class TelemetryStatusTracker:
|
||||
"""单个 recorder 的降级状态;非线程安全,由持有它的 recorder 在自己的时序内使用。
|
||||
|
||||
时钟经构造参数注入(与 `GatewayClient(now=...)` 同款): 冷却窗口与节流窗口
|
||||
都必须能用假时钟测,否则这些行为只能靠真睡验证,而真睡的用例是间歇红的源头。
|
||||
"""
|
||||
|
||||
def __init__(self, *, backend: str, now: Callable[[], float] = time.monotonic) -> None:
|
||||
"""记下后端名(只用于日志前缀)与时钟;构造后即"未降级"。
|
||||
|
||||
Args:
|
||||
backend: 后端名(如 `postgres`/`sqlite`),仅进日志文案。
|
||||
now: 单调时钟;测试可注入假时钟推进冷却与节流窗口。
|
||||
"""
|
||||
self._backend = backend
|
||||
self._now = now
|
||||
self._degraded_since: float | None = None
|
||||
self._fatal = False
|
||||
self._reason: str | None = None
|
||||
self._retry_at: float | None = None
|
||||
self._dropped_rows = 0
|
||||
# 节流窗口: 本段降级里"自上次复述以来"丢了多少行、上次复述在什么时候
|
||||
self._dropped_since_report = 0
|
||||
self._last_report_at: float | None = None
|
||||
self._dropped_at_entry = 0
|
||||
|
||||
def enter_degraded(self, reason: str, *, fatal: bool, cooldown_s: float | None) -> None:
|
||||
"""进入(或续期)降级;同一原因只讲一次,只刷新冷却窗口。
|
||||
|
||||
不重复打日志是刚需而非优化: 冷却到期重试再失败会反复走到这里,每次都讲
|
||||
就把"降级中"刷成噪音。原因变了才算新事实,值得再讲一遍。
|
||||
|
||||
Args:
|
||||
reason: 降级原因(已含具体异常文本);同值视为同一次降级的续期。
|
||||
fatal: True = 本进程内不可恢复,此后 `should_retry()` 恒 False。
|
||||
cooldown_s: 距下次允许重新准备的秒数;None 表示不自动重试。
|
||||
"""
|
||||
if self._fatal:
|
||||
return # 永久档不可被后来的失败覆盖,也不再刷屏
|
||||
now = self._now()
|
||||
first_of_this_episode = self._degraded_since is None
|
||||
announce = first_of_this_episode or reason != self._reason
|
||||
if first_of_this_episode:
|
||||
self._degraded_since = now
|
||||
self._dropped_at_entry = self._dropped_rows
|
||||
self._dropped_since_report = 0
|
||||
self._last_report_at = None
|
||||
self._reason = reason
|
||||
self._fatal = fatal
|
||||
self._retry_at = None if fatal or cooldown_s is None else now + cooldown_s
|
||||
if announce:
|
||||
logger.warning(
|
||||
"{} 遥测降级(后续记录将被丢弃): {};恢复条件: {}",
|
||||
self._backend,
|
||||
reason,
|
||||
self._recovery_hint(cooldown_s, fatal=fatal),
|
||||
)
|
||||
|
||||
def recover(self) -> None:
|
||||
"""退出降级并报告本段期间丢了多少行;未降级时是 no-op。
|
||||
|
||||
`dropped_rows` **不清零**: 它是进程生命周期内的累计量,下游靠它对账。
|
||||
"""
|
||||
if self._degraded_since is None:
|
||||
return
|
||||
dropped = self._dropped_rows - self._dropped_at_entry
|
||||
logger.info(
|
||||
"{} 遥测已恢复(降级持续 {:.1f}s,期间丢弃 {} 行)",
|
||||
self._backend,
|
||||
self._now() - self._degraded_since,
|
||||
dropped,
|
||||
)
|
||||
self._degraded_since = None
|
||||
self._fatal = False
|
||||
self._reason = None
|
||||
self._retry_at = None
|
||||
self._dropped_since_report = 0
|
||||
self._last_report_at = None
|
||||
|
||||
def record_drop(self, reason: str) -> None:
|
||||
"""记一行被丢弃;按行数与时间双阈值节流复述。
|
||||
|
||||
双阈值缺一不可: 只按行数,低频调用的进程会长时间完全静默;只按时间,
|
||||
高频进程在窗口内丢几万行也只有一条日志,看不出量级。
|
||||
"""
|
||||
self._dropped_rows += 1
|
||||
self._dropped_since_report += 1
|
||||
if not self._should_report():
|
||||
return
|
||||
logger.warning(
|
||||
"{} 遥测丢弃记录(累计 {} 行): {}",
|
||||
self._backend,
|
||||
self._dropped_rows,
|
||||
reason,
|
||||
)
|
||||
self._dropped_since_report = 0
|
||||
self._last_report_at = self._now()
|
||||
|
||||
def should_retry(self) -> bool:
|
||||
"""现在是否允许(重新)准备后端: 纯查询,不触库也不改状态。
|
||||
|
||||
未降级 → True(本就该正常走准备路径);fatal → False;冷却未到 → False;
|
||||
非 fatal 但没给冷却 → False(调用方没安排自动重试,tracker 不替它决定)。
|
||||
"""
|
||||
if self._fatal:
|
||||
return False
|
||||
if self._degraded_since is None:
|
||||
return True
|
||||
if self._retry_at is None:
|
||||
return False
|
||||
return self._now() >= self._retry_at
|
||||
|
||||
def snapshot(self) -> TelemetryStatus:
|
||||
"""当前状态的只读快照(公共出口 `client.telemetry_status` 的取值点)。"""
|
||||
now = self._now()
|
||||
since = self._degraded_since
|
||||
retry_after_s: float | None = None
|
||||
if since is not None and self._retry_at is not None:
|
||||
retry_after_s = max(0.0, self._retry_at - now) # 到期后钳到 0,不给负数
|
||||
return TelemetryStatus(
|
||||
degraded=since is not None,
|
||||
fatal=self._fatal,
|
||||
reason=self._reason,
|
||||
degraded_for_s=None if since is None else now - since,
|
||||
dropped_rows=self._dropped_rows,
|
||||
retry_after_s=retry_after_s,
|
||||
)
|
||||
|
||||
def _should_report(self) -> bool:
|
||||
"""本次丢弃是否该出声: 本段降级的第一行、满行数阈值、或超时间阈值。"""
|
||||
if self._last_report_at is None:
|
||||
return True
|
||||
if self._dropped_since_report >= _DROP_REPEAT_EVERY_ROWS:
|
||||
return True
|
||||
return self._now() - self._last_report_at >= _DROP_REPEAT_EVERY_S
|
||||
|
||||
@staticmethod
|
||||
def _recovery_hint(cooldown_s: float | None, *, fatal: bool) -> str:
|
||||
"""把恢复条件写进日志: 运维看到降级后第一个问题就是"它自己会好吗"。"""
|
||||
if fatal:
|
||||
return "需修正配置后重启进程(本进程内不会自愈)"
|
||||
if cooldown_s is None:
|
||||
return "下次调用时重试"
|
||||
return f"约 {cooldown_s:.0f}s 后自动重试"
|
||||
@@ -258,6 +258,31 @@ class SourceStats:
|
||||
tpm_used: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TelemetryStatus:
|
||||
"""遥测后端的可写状态快照;degraded 期间下游可据此对账(issue #15)。
|
||||
|
||||
不叫 `health`: 库内 `health` 一律指**源的健康度**(`OcrTransport.check_health`
|
||||
探活、`SourceSelector.health` 成功率 EWMA),而这里描述的是"这个 recorder
|
||||
现在能不能写、为什么不能、丢了多少",是状态不是评分(设计 §3.3)。
|
||||
|
||||
时长一律给**相对秒数**而非绝对时间戳: 库内的时钟是 monotonic,把它的读数
|
||||
交给下游会与 wall clock 混淆成两个不可比的时间轴。
|
||||
"""
|
||||
|
||||
degraded: bool
|
||||
fatal: bool
|
||||
"""True = 本进程内不可恢复(仅 DSN 不可解析一类配置级失败),需改配置并重启。"""
|
||||
reason: str | None
|
||||
"""降级原因;未降级为 None。"""
|
||||
degraded_for_s: float | None
|
||||
"""已降级时长;未降级为 None。"""
|
||||
dropped_rows: int
|
||||
"""累计丢弃行数;**进程生命周期内单调不减**——恢复不等于没丢过。"""
|
||||
retry_after_s: float | None
|
||||
"""距下次重新准备的秒数;fatal 或未降级为 None,冷却已到期为 0.0。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TransportResult:
|
||||
"""transport 单次原始调用的产物;治理字段由 RetryMW 补齐为 LLMResponse。"""
|
||||
|
||||
@@ -835,3 +835,40 @@ class TestRedisCacheOwnership:
|
||||
await cache.aclose()
|
||||
await cache.aclose() # 幂等: 不重复关
|
||||
assert built.closed == 1
|
||||
|
||||
|
||||
class TestTelemetryStatusExposure:
|
||||
"""降级状态的只读出口: 一处 isinstance 判定,三个 client 各钉一次(设计 §3.3)。"""
|
||||
|
||||
def _recorder(self, tmp_path):
|
||||
from polygateway.telemetry.sqlite import SQLiteRecorder
|
||||
|
||||
return SQLiteRecorder(tmp_path / "telemetry.db", auto_migrate=True)
|
||||
|
||||
def _assert_snapshot(self, status):
|
||||
from polygateway.types import TelemetryStatus
|
||||
|
||||
assert isinstance(status, TelemetryStatus)
|
||||
assert status.degraded is False
|
||||
|
||||
def test_gateway_client_without_telemetry_reports_none(self):
|
||||
assert _client().telemetry_status is None
|
||||
|
||||
def test_gateway_client_with_foreign_recorder_reports_none(self):
|
||||
"""注入的第三方 recorder 不提供状态 → None,绝不得抛 AttributeError。"""
|
||||
assert _client(telemetry=_Closable()).telemetry_status is None
|
||||
|
||||
def test_gateway_client_with_builtin_recorder_reports_snapshot(self, tmp_path):
|
||||
self._assert_snapshot(_client(telemetry=self._recorder(tmp_path)).telemetry_status)
|
||||
|
||||
def test_embedding_client_exposes_the_same_outlet(self, tmp_path):
|
||||
assert _embedding_client().telemetry_status is None
|
||||
assert _embedding_client(telemetry=_Closable()).telemetry_status is None
|
||||
self._assert_snapshot(
|
||||
_embedding_client(telemetry=self._recorder(tmp_path)).telemetry_status
|
||||
)
|
||||
|
||||
def test_ocr_client_exposes_the_same_outlet(self, tmp_path):
|
||||
assert _ocr_client().telemetry_status is None
|
||||
assert _ocr_client(telemetry=_Closable()).telemetry_status is None
|
||||
self._assert_snapshot(_ocr_client(telemetry=self._recorder(tmp_path)).telemetry_status)
|
||||
|
||||
@@ -16,9 +16,10 @@ from polygateway.ports import (
|
||||
SourceSelector,
|
||||
StructuredOutputStrategy,
|
||||
TelemetryRecorder,
|
||||
TelemetryStatusProvider,
|
||||
Transport,
|
||||
)
|
||||
from polygateway.types import LLMResponse, SourceStats
|
||||
from polygateway.types import LLMResponse, SourceStats, TelemetryStatus
|
||||
|
||||
|
||||
def _resp() -> LLMResponse:
|
||||
@@ -141,6 +142,28 @@ def test_protocols_are_runtime_checkable(impl, protocol):
|
||||
assert isinstance(impl, protocol)
|
||||
|
||||
|
||||
class _DummyStatusProvider(_DummyRecorder):
|
||||
@property
|
||||
def telemetry_status(self) -> TelemetryStatus:
|
||||
return TelemetryStatus(
|
||||
degraded=False,
|
||||
fatal=False,
|
||||
reason=None,
|
||||
degraded_for_s=None,
|
||||
dropped_rows=0,
|
||||
retry_after_s=None,
|
||||
)
|
||||
|
||||
|
||||
def test_status_provider_is_a_separate_optional_port():
|
||||
"""状态**不得**并进 TelemetryRecorder: 那会让只实现 record_llm_call 的对象
|
||||
当场不再满足 @runtime_checkable 的结构检查(设计 §3.3,Codex 审查)。"""
|
||||
assert isinstance(_DummyStatusProvider(), TelemetryStatusProvider)
|
||||
assert isinstance(_DummyStatusProvider(), TelemetryRecorder)
|
||||
assert not isinstance(_DummyRecorder(), TelemetryStatusProvider)
|
||||
assert isinstance(_DummyRecorder(), TelemetryRecorder) # 这条断言是那条决策的执法点
|
||||
|
||||
|
||||
def _decision(**overrides) -> GateDecision:
|
||||
base = {
|
||||
"source_name": "qwen_1",
|
||||
|
||||
@@ -1686,3 +1686,186 @@ class TestTextCapCoversEmbedAndOcrChains:
|
||||
# 占位串 `<ocr:text image_bytes=3>` 共 24 字
|
||||
assert json.loads(row["messages"])[0]["content"] == "<ocr:tex…(略 16 字)"
|
||||
assert row["response"] == "识别结果识别结果…(略 32 字)" # 先经 OCR 自有的 200 字上限
|
||||
|
||||
|
||||
# —— 遥测降级状态(issue #15 C 组): 共用 tracker + 只读快照 + 节流日志 ——
|
||||
|
||||
|
||||
class _FakeClock:
|
||||
"""可手动推进的单调时钟: 冷却与节流都靠它测,用例里绝不真睡。"""
|
||||
|
||||
def __init__(self, start: float = 1_000.0) -> None:
|
||||
self.t = start
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.t
|
||||
|
||||
def advance(self, seconds: float) -> None:
|
||||
self.t += seconds
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def captured_infos():
|
||||
"""捕获 INFO 及以上;恢复那条 info 是"降级已结束"的唯一外部信号。"""
|
||||
from loguru import logger
|
||||
|
||||
messages: list[str] = []
|
||||
sink_id = logger.add(messages.append, level="INFO")
|
||||
yield messages
|
||||
logger.remove(sink_id)
|
||||
|
||||
|
||||
class TestTelemetryStatusTracker:
|
||||
"""状态机六字段逐个钉;它是两个 recorder 共用的降级事实源(设计 §3.3)。"""
|
||||
|
||||
def _tracker(self, clock):
|
||||
from polygateway.telemetry.status import TelemetryStatusTracker
|
||||
|
||||
return TelemetryStatusTracker(backend="postgres", now=clock)
|
||||
|
||||
def test_fresh_tracker_reports_no_degradation(self):
|
||||
tracker = self._tracker(_FakeClock())
|
||||
status = tracker.snapshot()
|
||||
assert (status.degraded, status.fatal, status.reason) == (False, False, None)
|
||||
assert status.degraded_for_s is None and status.retry_after_s is None
|
||||
assert status.dropped_rows == 0
|
||||
assert tracker.should_retry() is True # 未降级本就该正常走准备路径
|
||||
|
||||
def test_entering_degraded_reports_reason_and_cooldown(self, captured_warnings):
|
||||
tracker = self._tracker(_FakeClock())
|
||||
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
|
||||
status = tracker.snapshot()
|
||||
assert status.degraded is True and status.fatal is False
|
||||
assert status.reason == "连接被拒"
|
||||
assert status.degraded_for_s == pytest.approx(0.0)
|
||||
assert status.retry_after_s == pytest.approx(60.0)
|
||||
assert len(captured_warnings) == 1
|
||||
assert "连接被拒" in captured_warnings[0] and "60" in captured_warnings[0]
|
||||
|
||||
def test_fake_clock_drains_the_cooldown(self):
|
||||
clock = _FakeClock()
|
||||
tracker = self._tracker(clock)
|
||||
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
|
||||
clock.advance(25.0)
|
||||
status = tracker.snapshot()
|
||||
assert status.degraded_for_s == pytest.approx(25.0)
|
||||
assert status.retry_after_s == pytest.approx(35.0)
|
||||
assert tracker.should_retry() is False
|
||||
clock.advance(40.0) # 越过冷却窗口
|
||||
assert tracker.snapshot().retry_after_s == pytest.approx(0.0) # 不得为负
|
||||
assert tracker.should_retry() is True
|
||||
|
||||
def test_recover_clears_degradation_but_keeps_dropped_rows(self, captured_infos):
|
||||
tracker = self._tracker(_FakeClock())
|
||||
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
|
||||
for _ in range(3):
|
||||
tracker.record_drop("遥测已降级")
|
||||
tracker.recover()
|
||||
status = tracker.snapshot()
|
||||
assert (status.degraded, status.fatal, status.reason) == (False, False, None)
|
||||
assert status.degraded_for_s is None and status.retry_after_s is None
|
||||
# 进程生命周期内单调不减: 恢复不是"没丢过",下游要靠它对账
|
||||
assert status.dropped_rows == 3
|
||||
assert any("3" in message and "恢复" in message for message in captured_infos)
|
||||
|
||||
def test_drop_warnings_are_throttled_by_the_row_constant(self, captured_warnings):
|
||||
from polygateway.telemetry.status import _DROP_REPEAT_EVERY_ROWS
|
||||
|
||||
tracker = self._tracker(_FakeClock()) # 时钟不动: 只有行数阈值能触发复述
|
||||
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
|
||||
captured_warnings.clear() # 只数丢弃复述,不数进入降级那条
|
||||
total = _DROP_REPEAT_EVERY_ROWS * 2 + 1
|
||||
for _ in range(total):
|
||||
tracker.record_drop("遥测已降级")
|
||||
# 首条必报,其后每满一个阈值报一次 —— 关系由常量决定,不写死数字
|
||||
assert len(captured_warnings) == 1 + (total - 1) // _DROP_REPEAT_EVERY_ROWS
|
||||
assert tracker.snapshot().dropped_rows == total
|
||||
|
||||
def test_drop_warnings_are_also_throttled_by_time(self, captured_warnings):
|
||||
from polygateway.telemetry.status import _DROP_REPEAT_EVERY_S
|
||||
|
||||
clock = _FakeClock()
|
||||
tracker = self._tracker(clock)
|
||||
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
|
||||
captured_warnings.clear()
|
||||
tracker.record_drop("遥测已降级")
|
||||
assert len(captured_warnings) == 1
|
||||
tracker.record_drop("遥测已降级")
|
||||
assert len(captured_warnings) == 1 # 同一窗口内不刷屏
|
||||
clock.advance(_DROP_REPEAT_EVERY_S)
|
||||
tracker.record_drop("遥测已降级")
|
||||
assert len(captured_warnings) == 2 # 长跑进程里也不会静默
|
||||
|
||||
def test_fatal_degradation_never_retries(self, captured_warnings):
|
||||
clock = _FakeClock()
|
||||
tracker = self._tracker(clock)
|
||||
tracker.enter_degraded("DSN 不可解析", fatal=True, cooldown_s=None)
|
||||
clock.advance(1_000_000.0)
|
||||
status = tracker.snapshot()
|
||||
assert status.fatal is True and status.retry_after_s is None
|
||||
assert tracker.should_retry() is False
|
||||
assert "重启" in captured_warnings[0] # 恢复条件必须写在日志里
|
||||
|
||||
def test_repeating_the_same_reason_does_not_spam(self, captured_warnings):
|
||||
"""冷却到期重试再失败会反复进入降级: 同一原因只讲一次,只刷新窗口。"""
|
||||
clock = _FakeClock()
|
||||
tracker = self._tracker(clock)
|
||||
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
|
||||
clock.advance(60.0)
|
||||
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
|
||||
assert len(captured_warnings) == 1
|
||||
assert tracker.snapshot().retry_after_s == pytest.approx(60.0) # 窗口已刷新
|
||||
assert tracker.snapshot().degraded_for_s == pytest.approx(60.0) # 但仍是同一段降级
|
||||
|
||||
|
||||
class TestSQLiteStatusVisibility:
|
||||
"""SQLite 侧今天初始化失败后写入静默 return,连一条日志都没有(设计 §1.4)。"""
|
||||
|
||||
def _broken(self, tmp_path):
|
||||
blocker = tmp_path / "blocker"
|
||||
blocker.write_text("父目录是个文件,mkdir 必然失败")
|
||||
return SQLiteRecorder(blocker / "telemetry.db", auto_migrate=True)
|
||||
|
||||
def test_init_failure_is_degraded_and_fatal(self, tmp_path, captured_warnings):
|
||||
recorder = self._broken(tmp_path)
|
||||
status = recorder.telemetry_status
|
||||
assert status.degraded is True and status.fatal is True
|
||||
assert captured_warnings # 静默降级 ≠ 静默
|
||||
|
||||
async def test_dropped_rows_are_counted_and_visible(self, tmp_path, captured_warnings):
|
||||
recorder = self._broken(tmp_path)
|
||||
captured_warnings.clear()
|
||||
await _record_minimal(recorder) # 不得抛: 遥测绝不冒泡
|
||||
assert recorder.telemetry_status.dropped_rows == 1
|
||||
assert captured_warnings # 丢的第一行必须出声
|
||||
|
||||
def test_healthy_recorder_is_not_degraded(self, tmp_path):
|
||||
recorder = SQLiteRecorder(tmp_path / "ok.db", auto_migrate=True)
|
||||
assert recorder.telemetry_status.degraded is False
|
||||
recorder.close()
|
||||
|
||||
|
||||
class TestPostgresStatusVisibility:
|
||||
"""PG 侧的判死本任务不改判据,只让它经 tracker 变得可见(计划 T2)。"""
|
||||
|
||||
def _recorder(self, conn):
|
||||
from polygateway.telemetry.postgres import PostgresRecorder
|
||||
|
||||
return PostgresRecorder(
|
||||
"postgresql://u:p@h:5432/polygateway", pool=_FakePgPool(conn), auto_migrate=True
|
||||
)
|
||||
|
||||
async def test_unusable_table_shows_up_in_the_status(self, captured_warnings):
|
||||
"""表确定不存在且建不出来 = 既有的判死档;现在它要能被下游查到。"""
|
||||
recorder = self._recorder(_FakePgConn([], fail_create=True))
|
||||
await _record_minimal(recorder)
|
||||
status = recorder.telemetry_status
|
||||
assert status.degraded is True and status.fatal is True
|
||||
assert status.dropped_rows == 1 # 判死那一次调用本身也丢了一行
|
||||
assert recorder._failed is True # 过渡期两份状态并存(T5 收掉 `_failed`)
|
||||
|
||||
async def test_healthy_recorder_is_not_degraded(self):
|
||||
recorder = self._recorder(_FakePgConn(list(_EXPECTED_COLUMNS)))
|
||||
await _record_minimal(recorder)
|
||||
status = recorder.telemetry_status
|
||||
assert status.degraded is False and status.dropped_rows == 0
|
||||
|
||||
Reference in New Issue
Block a user