feat: make telemetry degradation a first-class state

Telemetry degradation used to be a single warning and a private boolean.
In a long-running process that is indistinguishable from telemetry working:
issue #15 was only found by hand-reconciling milestone log lines against
llm_calls rows, after 19 calls had silently gone unrecorded. The SQLite
side was worse — once init failed, every write returned without even a
log line.

Degradation now has one shared owner. TelemetryStatusTracker holds the
state machine (enter/recover/drop/should-retry), announces entry and
recovery once each, and repeats the drop count under a row-and-time
double threshold so a degraded backend neither floods the log nor goes
quiet. Both recorders hold one; both count the rows they drop.

For programmatic consumers, TelemetryStatus is a frozen snapshot exposed
as telemetry_status on all three clients, resolved through a single
isinstance check. It is a separate optional port rather than a member of
TelemetryRecorder: that protocol is @runtime_checkable, so adding an
attribute would make every implementation that only defines
record_llm_call stop satisfying it — downstream isinstance assertions
would break on upgrade. The existing assertion in test_ports.py is what
keeps that decision honest.

Failure criteria are deliberately untouched here: Postgres still treats a
pool failure as permanent, only now visibly. `_failed` and the tracker
therefore both carry the verdict for the span of this one change; the
cooldown rework collapses them into the tracker alone.
This commit is contained in:
2026-08-24 08:57:23 -04:00
parent e69ca4c82c
commit f958138e83
11 changed files with 542 additions and 5 deletions
+24
View File
@@ -24,6 +24,7 @@ from polygateway.middleware.cache import CacheMW
from polygateway.middleware.retry import RetryMW from polygateway.middleware.retry import RetryMW
from polygateway.middleware.structured import StructuredMW from polygateway.middleware.structured import StructuredMW
from polygateway.middleware.telemetry import TelemetryEmitter, TelemetryMW from polygateway.middleware.telemetry import TelemetryEmitter, TelemetryMW
from polygateway.ports import TelemetryStatusProvider
from polygateway.pricing import PricingTable from polygateway.pricing import PricingTable
from polygateway.providers import get_capability, get_provider, resolve_thinking from polygateway.providers import get_capability, get_provider, resolve_thinking
from polygateway.sources import ( from polygateway.sources import (
@@ -37,6 +38,7 @@ from polygateway.transports.openai_compat import OpenAICompatTransport
from polygateway.types import ( from polygateway.types import (
ChatRequest, ChatRequest,
LLMResponse, LLMResponse,
TelemetryStatus,
validate_caller_dimensions, validate_caller_dimensions,
validate_request_overlay, validate_request_overlay,
) )
@@ -135,6 +137,19 @@ async def _aclose_component(component: object | None) -> None:
close() close()
def _telemetry_status_of(telemetry: TelemetryRecorder | None) -> TelemetryStatus | None:
"""三个 client 共用的状态取值点: 不提供状态的 recorder 一律返回 None。
判定写成 `isinstance(可选端口)` 而不是裸 `getattr`: 两者运行时都是结构检查
(`@runtime_checkable` 按属性存在性判定),差别在**契约有没有名字**——端口是
写进 `ports.py` 的公开承诺,下游可以照着实现;散落的 `getattr` 不是,而
`aclose` 当年正是被复制成三份鸭子类型探测才漂移出越权关闭(设计 §3.3/§3.4)。
"""
if isinstance(telemetry, TelemetryStatusProvider):
return telemetry.telemetry_status
return None
def _mark_owned_components( def _mark_owned_components(
client: Any, client: Any,
*, *,
@@ -252,6 +267,15 @@ class GatewayClient:
self._owns_breaker = False self._owns_breaker = False
self._closed = False self._closed = False
@property
def telemetry_status(self) -> TelemetryStatus | None:
"""遥测后端的可写状态;无遥测或注入的 recorder 不提供状态时为 None。
判定收敛在 `_telemetry_status_of` 一处(不是三处各自探测): 三个 client
的 `aclose` 曾各持一份逐字复制,漂移的结果就是越权关闭(设计 §3.3/§3.4)。
"""
return _telemetry_status_of(self._telemetry)
async def chat( async def chat(
self, self,
messages: list[dict[str, Any]], messages: list[dict[str, Any]],
+11 -1
View File
@@ -25,7 +25,7 @@ from typing import TYPE_CHECKING, Any
from loguru import logger from loguru import logger
from polygateway.client import _aclose_component from polygateway.client import _aclose_component, _telemetry_status_of
from polygateway.config import EmbeddingSettings from polygateway.config import EmbeddingSettings
from polygateway.errors import ( from polygateway.errors import (
AllSourcesExhausted, AllSourcesExhausted,
@@ -46,6 +46,7 @@ from polygateway.types import (
ChatRequest, ChatRequest,
EmbeddingResponse, EmbeddingResponse,
LLMResponse, LLMResponse,
TelemetryStatus,
strip_unsupported_extra_body, strip_unsupported_extra_body,
validate_caller_dimensions, validate_caller_dimensions,
) )
@@ -454,6 +455,15 @@ class EmbeddingClient:
known = [c for c in costs if c is not None] known = [c for c in costs if c is not None]
return sum(known) if known else None return sum(known) if known else None
@property
def telemetry_status(self) -> TelemetryStatus | None:
"""遥测后端的可写状态;无遥测或注入的 recorder 不提供状态时为 None。
判定收敛在 `_telemetry_status_of` 一处(不是三处各自探测): 三个 client
的 `aclose` 曾各持一份逐字复制,漂移的结果就是越权关闭(设计 §3.3/§3.4)。
"""
return _telemetry_status_of(self._telemetry)
async def aclose(self) -> None: async def aclose(self) -> None:
"""幂等释放**自建**资源(与 GatewayClient 对称);注入的组件一律不碰。""" """幂等释放**自建**资源(与 GatewayClient 对称);注入的组件一律不碰。"""
if self._closed: if self._closed:
+11 -1
View File
@@ -22,7 +22,7 @@ from typing import TYPE_CHECKING, Any, Literal
from loguru import logger from loguru import logger
from polygateway.client import _aclose_component from polygateway.client import _aclose_component, _telemetry_status_of
from polygateway.errors import ( from polygateway.errors import (
AllSourcesExhausted, AllSourcesExhausted,
GovernanceBackendError, GovernanceBackendError,
@@ -44,6 +44,7 @@ from polygateway.types import (
LLMResponse, LLMResponse,
OcrLayoutResult, OcrLayoutResult,
OcrTextResult, OcrTextResult,
TelemetryStatus,
Usage, Usage,
strip_unsupported_extra_body, strip_unsupported_extra_body,
validate_caller_dimensions, validate_caller_dimensions,
@@ -464,6 +465,15 @@ class OcrClient:
# —— 生命周期 —— # —— 生命周期 ——
@property
def telemetry_status(self) -> TelemetryStatus | None:
"""遥测后端的可写状态;无遥测或注入的 recorder 不提供状态时为 None。
判定收敛在 `_telemetry_status_of` 一处(不是三处各自探测): 三个 client
的 `aclose` 曾各持一份逐字复制,漂移的结果就是越权关闭(设计 §3.3/§3.4)。
"""
return _telemetry_status_of(self._telemetry)
async def aclose(self) -> None: async def aclose(self) -> None:
"""幂等释放**自建**资源(与 EmbeddingClient 对称);注入的组件一律不碰。""" """幂等释放**自建**资源(与 EmbeddingClient 对称);注入的组件一律不碰。"""
if self._closed: if self._closed:
+15
View File
@@ -20,6 +20,7 @@ from .types import (
OcrTextTransportResult, OcrTextTransportResult,
SourceConfig, SourceConfig,
SourceStats, SourceStats,
TelemetryStatus,
TransportResult, TransportResult,
) )
@@ -243,6 +244,20 @@ class StructuredOutputStrategy(Protocol):
def parse(self, text: str) -> Any: ... def parse(self, text: str) -> Any: ...
@runtime_checkable
class TelemetryStatusProvider(Protocol):
"""可自述可写状态的遥测后端;`TelemetryRecorder` 的**可选**伴生端口(issue #15)。
与 `TelemetryRecorder` 分开而不是给它加成员,是因为后者是 `@runtime_checkable`
而运行时检查按属性存在性做: 加一个属性会让所有只实现 `record_llm_call` 的
实现**当场不再是** `TelemetryRecorder`,下游若有同款 isinstance 断言,升级即断
(设计 §3.3)。消费方一律先 isinstance 再取值,取不到就当没有状态可报。
"""
@property
def telemetry_status(self) -> TelemetryStatus: ...
@runtime_checkable @runtime_checkable
class TelemetryRecorder(Protocol): class TelemetryRecorder(Protocol):
"""遥测后端;24 字段冻结(M1 设计 §4.4 + issue #3/#4/#11),唯一调用点是 TelemetryEmitter。 """遥测后端;24 字段冻结(M1 设计 §4.4 + issue #3/#4/#11),唯一调用点是 TelemetryEmitter。
+17 -1
View File
@@ -28,10 +28,13 @@ from polygateway.telemetry.schema import (
insert_sql, insert_sql,
missing_columns_warning, missing_columns_warning,
) )
from polygateway.telemetry.status import TelemetryStatusTracker
if TYPE_CHECKING: if TYPE_CHECKING:
import asyncpg import asyncpg
from polygateway.types import TelemetryStatus
# 探测表是否存在;不需要任何权限,且与 INSERT 走同一套 search_path 解析 # 探测表是否存在;不需要任何权限,且与 INSERT 走同一套 search_path 解析
_TABLE_EXISTS = "SELECT to_regclass('llm_calls')" _TABLE_EXISTS = "SELECT to_regclass('llm_calls')"
@@ -72,8 +75,16 @@ class PostgresRecorder:
self._insert = insert_sql("postgres", COLUMNS) self._insert = insert_sql("postgres", COLUMNS)
self._schema_ready = False self._schema_ready = False
self._failed = False # 结构性降级标志: 置位后所有写入短路 self._failed = False # 结构性降级标志: 置位后所有写入短路
# 降级的可编程出口与节流日志;`_failed` 与它并存是 issue #15 的过渡态,
# 判据改造(冷却自愈)落地时状态收归 tracker 一处
self._status = TelemetryStatusTracker(backend="postgres")
self._init_lock = asyncio.Lock() self._init_lock = asyncio.Lock()
@property
def telemetry_status(self) -> TelemetryStatus:
"""当前可写状态快照(ports.TelemetryStatusProvider)。"""
return self._status.snapshot()
async def _ensure_ready(self) -> asyncpg.Pool | None: async def _ensure_ready(self) -> asyncpg.Pool | None:
"""lazy 建池+备表;判死只认「确定写不进去」(issue #9),其余失败都留活路。""" """lazy 建池+备表;判死只认「确定写不进去」(issue #9),其余失败都留活路。"""
if self._failed: if self._failed:
@@ -104,7 +115,7 @@ class PostgresRecorder:
# 池建不出来 = 确定写不进去;且每次调用重试都要内联吞掉 connect # 池建不出来 = 确定写不进去;且每次调用重试都要内联吞掉 connect
# 超时,而遥测是业务路径上的 await —— 此处必须永久降级 # 超时,而遥测是业务路径上的 await —— 此处必须永久降级
self._failed = True self._failed = True
logger.warning("Postgres 遥测建池失败,后续记录降级为 no-op: {}", exc) self._status.enter_degraded(f"建池失败: {exc}", fatal=True, cooldown_s=None)
return None return None
return self._pool return self._pool
@@ -122,6 +133,9 @@ class PostgresRecorder:
return None return None
if columns is None: if columns is None:
self._failed = True self._failed = True
self._status.enter_degraded(
"表 llm_calls 不存在且建不出来(记录无处可落)", fatal=True, cooldown_s=None
)
return None return None
# 写入列、语句与就绪标志必须**一起**生效: `_ensure_ready` 只看 `_schema_ready` # 写入列、语句与就绪标志必须**一起**生效: `_ensure_ready` 只看 `_schema_ready`
# 就绕开 `_init_lock` 直接返回池,先置就绪会开出"已就绪但语句还是旧的"的窗口 # 就绕开 `_init_lock` 直接返回池,先置就绪会开出"已就绪但语句还是旧的"的窗口
@@ -232,6 +246,8 @@ class PostgresRecorder:
""" """
pool = await self._ensure_ready() pool = await self._ensure_ready()
if pool is None: if pool is None:
# 降级期间静默 return 就是 issue #15 的破口: 丢行必须计数且节流出声
self._status.record_drop("遥测已降级")
return return
row = tuple(fields[col] for col in self._columns) row = tuple(fields[col] for col in self._columns)
try: try:
+17 -1
View File
@@ -19,6 +19,7 @@ import asyncio
import sqlite3 import sqlite3
import threading import threading
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING
from loguru import logger from loguru import logger
@@ -29,6 +30,10 @@ from polygateway.telemetry.schema import (
insert_sql, insert_sql,
missing_columns_warning, missing_columns_warning,
) )
from polygateway.telemetry.status import TelemetryStatusTracker
if TYPE_CHECKING:
from polygateway.types import TelemetryStatus
class SQLiteRecorder: class SQLiteRecorder:
@@ -45,6 +50,9 @@ class SQLiteRecorder:
config 一处,不与本类签名漂移(设计 D-c)。 config 一处,不与本类签名漂移(设计 D-c)。
""" """
self._auto_migrate = auto_migrate self._auto_migrate = auto_migrate
# SQLite 侧本次只做可见性: 它的失败模式(目录不可写、文件损坏)在装配期
# 就暴露给下游,不是"跑到一半悄悄断",故降级恒为 fatal,不做冷却重连
self._status = TelemetryStatusTracker(backend="sqlite")
self._lock = threading.Lock() self._lock = threading.Lock()
self._conn: sqlite3.Connection | None = None self._conn: sqlite3.Connection | None = None
# 先按全量列定型: 连接失败/探测失败时保守沿用全量(今天的行为) # 先按全量列定型: 连接失败/探测失败时保守沿用全量(今天的行为)
@@ -60,9 +68,14 @@ class SQLiteRecorder:
conn.commit() conn.commit()
self._conn = conn self._conn = conn
except (OSError, sqlite3.Error) as exc: except (OSError, sqlite3.Error) as exc:
logger.warning("SQLite 遥测初始化失败,后续记录降级为 no-op: {}", exc) self._status.enter_degraded(f"初始化失败: {exc}", fatal=True, cooldown_s=None)
self._prepare_columns() self._prepare_columns()
@property
def telemetry_status(self) -> TelemetryStatus:
"""当前可写状态快照(ports.TelemetryStatusProvider)。"""
return self._status.snapshot()
def _prepare_columns(self) -> None: def _prepare_columns(self) -> None:
"""探测现有列后定型写入: auto 档补齐缺列,manual 档改为裁剪写入(issue #13)。 """探测现有列后定型写入: auto 档补齐缺列,manual 档改为裁剪写入(issue #13)。
@@ -136,6 +149,9 @@ class SQLiteRecorder:
占位符同序——两者必须一起改,分开改就是把值写进错位的列。 占位符同序——两者必须一起改,分开改就是把值写进错位的列。
""" """
if self._conn is None: if self._conn is None:
# 改前这里是**裸 return**: 初始化失败后每一行都无声消失,长跑进程里
# 与"遥测正常"外观上完全一致(设计 §1.4 的直接钉子)
self._status.record_drop("遥测已降级")
return return
row = tuple(fields[col] for col in self._columns) row = tuple(fields[col] for col in self._columns)
try: try:
+178
View File
@@ -0,0 +1,178 @@
"""遥测降级状态机(issue #15 C 组): 两个 recorder 共用的降级事实源。
存在的理由(设计 §1.4): 遥测降级过去只有**一条** warning,长跑进程里等同于
静默——issue 是手工对账(日志里的完成里程碑条数 vs `llm_calls` 行数)才发现的,
期间 19 次调用一行未落。"遥测必录"铁律的实质要求是: 库做不到必录时,必须
**持续、可编程地**让下游知道。故降级升格为一等对象,两条出路各走一边:
- 人看: 进入/恢复各一条日志,降级期间按行数与时间**双阈值节流复述**(不刷屏,
也不静默);
- 程序看: `snapshot()` 给只读 `TelemetryStatus`,下游可据此对账或告警。
本模块**不含任何后端知识**(不 import asyncpg/sqlite3,也不判失败性质): 失败
分类是各 recorder 的事,tracker 只接受"降级了/恢复了/丢了一行"三个事实。
"""
from __future__ import annotations
import time
from typing import TYPE_CHECKING
from loguru import logger
from polygateway.types import TelemetryStatus
if TYPE_CHECKING:
from collections.abc import Callable
_DROP_REPEAT_EVERY_ROWS = 100
"""降级期间每丢这么多行复述一次;首行必报。"""
_DROP_REPEAT_EVERY_S = 300.0
"""降级期间距上次复述超过这么久就再报一次——低频调用的进程不能因行数不够而静默。"""
class TelemetryStatusTracker:
"""单个 recorder 的降级状态;非线程安全,由持有它的 recorder 在自己的时序内使用。
时钟经构造参数注入(与 `GatewayClient(now=...)` 同款): 冷却窗口与节流窗口
都必须能用假时钟测,否则这些行为只能靠真睡验证,而真睡的用例是间歇红的源头。
"""
def __init__(self, *, backend: str, now: Callable[[], float] = time.monotonic) -> None:
"""记下后端名(只用于日志前缀)与时钟;构造后即"未降级"
Args:
backend: 后端名(如 `postgres`/`sqlite`),仅进日志文案。
now: 单调时钟;测试可注入假时钟推进冷却与节流窗口。
"""
self._backend = backend
self._now = now
self._degraded_since: float | None = None
self._fatal = False
self._reason: str | None = None
self._retry_at: float | None = None
self._dropped_rows = 0
# 节流窗口: 本段降级里"自上次复述以来"丢了多少行、上次复述在什么时候
self._dropped_since_report = 0
self._last_report_at: float | None = None
self._dropped_at_entry = 0
def enter_degraded(self, reason: str, *, fatal: bool, cooldown_s: float | None) -> None:
"""进入(或续期)降级;同一原因只讲一次,只刷新冷却窗口。
不重复打日志是刚需而非优化: 冷却到期重试再失败会反复走到这里,每次都讲
就把"降级中"刷成噪音。原因变了才算新事实,值得再讲一遍。
Args:
reason: 降级原因(已含具体异常文本);同值视为同一次降级的续期。
fatal: True = 本进程内不可恢复,此后 `should_retry()` 恒 False。
cooldown_s: 距下次允许重新准备的秒数;None 表示不自动重试。
"""
if self._fatal:
return # 永久档不可被后来的失败覆盖,也不再刷屏
now = self._now()
first_of_this_episode = self._degraded_since is None
announce = first_of_this_episode or reason != self._reason
if first_of_this_episode:
self._degraded_since = now
self._dropped_at_entry = self._dropped_rows
self._dropped_since_report = 0
self._last_report_at = None
self._reason = reason
self._fatal = fatal
self._retry_at = None if fatal or cooldown_s is None else now + cooldown_s
if announce:
logger.warning(
"{} 遥测降级(后续记录将被丢弃): {};恢复条件: {}",
self._backend,
reason,
self._recovery_hint(cooldown_s, fatal=fatal),
)
def recover(self) -> None:
"""退出降级并报告本段期间丢了多少行;未降级时是 no-op。
`dropped_rows` **不清零**: 它是进程生命周期内的累计量,下游靠它对账。
"""
if self._degraded_since is None:
return
dropped = self._dropped_rows - self._dropped_at_entry
logger.info(
"{} 遥测已恢复(降级持续 {:.1f}s,期间丢弃 {} 行)",
self._backend,
self._now() - self._degraded_since,
dropped,
)
self._degraded_since = None
self._fatal = False
self._reason = None
self._retry_at = None
self._dropped_since_report = 0
self._last_report_at = None
def record_drop(self, reason: str) -> None:
"""记一行被丢弃;按行数与时间双阈值节流复述。
双阈值缺一不可: 只按行数,低频调用的进程会长时间完全静默;只按时间,
高频进程在窗口内丢几万行也只有一条日志,看不出量级。
"""
self._dropped_rows += 1
self._dropped_since_report += 1
if not self._should_report():
return
logger.warning(
"{} 遥测丢弃记录(累计 {} 行): {}",
self._backend,
self._dropped_rows,
reason,
)
self._dropped_since_report = 0
self._last_report_at = self._now()
def should_retry(self) -> bool:
"""现在是否允许(重新)准备后端: 纯查询,不触库也不改状态。
未降级 → True(本就该正常走准备路径);fatal → False;冷却未到 → False;
非 fatal 但没给冷却 → False(调用方没安排自动重试,tracker 不替它决定)。
"""
if self._fatal:
return False
if self._degraded_since is None:
return True
if self._retry_at is None:
return False
return self._now() >= self._retry_at
def snapshot(self) -> TelemetryStatus:
"""当前状态的只读快照(公共出口 `client.telemetry_status` 的取值点)。"""
now = self._now()
since = self._degraded_since
retry_after_s: float | None = None
if since is not None and self._retry_at is not None:
retry_after_s = max(0.0, self._retry_at - now) # 到期后钳到 0,不给负数
return TelemetryStatus(
degraded=since is not None,
fatal=self._fatal,
reason=self._reason,
degraded_for_s=None if since is None else now - since,
dropped_rows=self._dropped_rows,
retry_after_s=retry_after_s,
)
def _should_report(self) -> bool:
"""本次丢弃是否该出声: 本段降级的第一行、满行数阈值、或超时间阈值。"""
if self._last_report_at is None:
return True
if self._dropped_since_report >= _DROP_REPEAT_EVERY_ROWS:
return True
return self._now() - self._last_report_at >= _DROP_REPEAT_EVERY_S
@staticmethod
def _recovery_hint(cooldown_s: float | None, *, fatal: bool) -> str:
"""把恢复条件写进日志: 运维看到降级后第一个问题就是"它自己会好吗""""
if fatal:
return "需修正配置后重启进程(本进程内不会自愈)"
if cooldown_s is None:
return "下次调用时重试"
return f"{cooldown_s:.0f}s 后自动重试"
+25
View File
@@ -258,6 +258,31 @@ class SourceStats:
tpm_used: int tpm_used: int
@dataclass(frozen=True)
class TelemetryStatus:
"""遥测后端的可写状态快照;degraded 期间下游可据此对账(issue #15)。
不叫 `health`: 库内 `health` 一律指**源的健康度**(`OcrTransport.check_health`
探活、`SourceSelector.health` 成功率 EWMA),而这里描述的是"这个 recorder
现在能不能写、为什么不能、丢了多少",是状态不是评分(设计 §3.3)。
时长一律给**相对秒数**而非绝对时间戳: 库内的时钟是 monotonic,把它的读数
交给下游会与 wall clock 混淆成两个不可比的时间轴。
"""
degraded: bool
fatal: bool
"""True = 本进程内不可恢复(仅 DSN 不可解析一类配置级失败),需改配置并重启。"""
reason: str | None
"""降级原因;未降级为 None。"""
degraded_for_s: float | None
"""已降级时长;未降级为 None。"""
dropped_rows: int
"""累计丢弃行数;**进程生命周期内单调不减**——恢复不等于没丢过。"""
retry_after_s: float | None
"""距下次重新准备的秒数;fatal 或未降级为 None,冷却已到期为 0.0。"""
@dataclass(frozen=True) @dataclass(frozen=True)
class TransportResult: class TransportResult:
"""transport 单次原始调用的产物;治理字段由 RetryMW 补齐为 LLMResponse。""" """transport 单次原始调用的产物;治理字段由 RetryMW 补齐为 LLMResponse。"""
+37
View File
@@ -835,3 +835,40 @@ class TestRedisCacheOwnership:
await cache.aclose() await cache.aclose()
await cache.aclose() # 幂等: 不重复关 await cache.aclose() # 幂等: 不重复关
assert built.closed == 1 assert built.closed == 1
class TestTelemetryStatusExposure:
"""降级状态的只读出口: 一处 isinstance 判定,三个 client 各钉一次(设计 §3.3)。"""
def _recorder(self, tmp_path):
from polygateway.telemetry.sqlite import SQLiteRecorder
return SQLiteRecorder(tmp_path / "telemetry.db", auto_migrate=True)
def _assert_snapshot(self, status):
from polygateway.types import TelemetryStatus
assert isinstance(status, TelemetryStatus)
assert status.degraded is False
def test_gateway_client_without_telemetry_reports_none(self):
assert _client().telemetry_status is None
def test_gateway_client_with_foreign_recorder_reports_none(self):
"""注入的第三方 recorder 不提供状态 → None,绝不得抛 AttributeError。"""
assert _client(telemetry=_Closable()).telemetry_status is None
def test_gateway_client_with_builtin_recorder_reports_snapshot(self, tmp_path):
self._assert_snapshot(_client(telemetry=self._recorder(tmp_path)).telemetry_status)
def test_embedding_client_exposes_the_same_outlet(self, tmp_path):
assert _embedding_client().telemetry_status is None
assert _embedding_client(telemetry=_Closable()).telemetry_status is None
self._assert_snapshot(
_embedding_client(telemetry=self._recorder(tmp_path)).telemetry_status
)
def test_ocr_client_exposes_the_same_outlet(self, tmp_path):
assert _ocr_client().telemetry_status is None
assert _ocr_client(telemetry=_Closable()).telemetry_status is None
self._assert_snapshot(_ocr_client(telemetry=self._recorder(tmp_path)).telemetry_status)
+24 -1
View File
@@ -16,9 +16,10 @@ from polygateway.ports import (
SourceSelector, SourceSelector,
StructuredOutputStrategy, StructuredOutputStrategy,
TelemetryRecorder, TelemetryRecorder,
TelemetryStatusProvider,
Transport, Transport,
) )
from polygateway.types import LLMResponse, SourceStats from polygateway.types import LLMResponse, SourceStats, TelemetryStatus
def _resp() -> LLMResponse: def _resp() -> LLMResponse:
@@ -141,6 +142,28 @@ def test_protocols_are_runtime_checkable(impl, protocol):
assert isinstance(impl, protocol) assert isinstance(impl, protocol)
class _DummyStatusProvider(_DummyRecorder):
@property
def telemetry_status(self) -> TelemetryStatus:
return TelemetryStatus(
degraded=False,
fatal=False,
reason=None,
degraded_for_s=None,
dropped_rows=0,
retry_after_s=None,
)
def test_status_provider_is_a_separate_optional_port():
"""状态**不得**并进 TelemetryRecorder: 那会让只实现 record_llm_call 的对象
当场不再满足 @runtime_checkable 的结构检查(设计 §3.3,Codex 审查)。"""
assert isinstance(_DummyStatusProvider(), TelemetryStatusProvider)
assert isinstance(_DummyStatusProvider(), TelemetryRecorder)
assert not isinstance(_DummyRecorder(), TelemetryStatusProvider)
assert isinstance(_DummyRecorder(), TelemetryRecorder) # 这条断言是那条决策的执法点
def _decision(**overrides) -> GateDecision: def _decision(**overrides) -> GateDecision:
base = { base = {
"source_name": "qwen_1", "source_name": "qwen_1",
+183
View File
@@ -1686,3 +1686,186 @@ class TestTextCapCoversEmbedAndOcrChains:
# 占位串 `<ocr:text image_bytes=3>` 共 24 字 # 占位串 `<ocr:text image_bytes=3>` 共 24 字
assert json.loads(row["messages"])[0]["content"] == "<ocr:tex…(略 16 字)" assert json.loads(row["messages"])[0]["content"] == "<ocr:tex…(略 16 字)"
assert row["response"] == "识别结果识别结果…(略 32 字)" # 先经 OCR 自有的 200 字上限 assert row["response"] == "识别结果识别结果…(略 32 字)" # 先经 OCR 自有的 200 字上限
# —— 遥测降级状态(issue #15 C 组): 共用 tracker + 只读快照 + 节流日志 ——
class _FakeClock:
"""可手动推进的单调时钟: 冷却与节流都靠它测,用例里绝不真睡。"""
def __init__(self, start: float = 1_000.0) -> None:
self.t = start
def __call__(self) -> float:
return self.t
def advance(self, seconds: float) -> None:
self.t += seconds
@pytest.fixture
def captured_infos():
"""捕获 INFO 及以上;恢复那条 info 是"降级已结束"的唯一外部信号。"""
from loguru import logger
messages: list[str] = []
sink_id = logger.add(messages.append, level="INFO")
yield messages
logger.remove(sink_id)
class TestTelemetryStatusTracker:
"""状态机六字段逐个钉;它是两个 recorder 共用的降级事实源(设计 §3.3)。"""
def _tracker(self, clock):
from polygateway.telemetry.status import TelemetryStatusTracker
return TelemetryStatusTracker(backend="postgres", now=clock)
def test_fresh_tracker_reports_no_degradation(self):
tracker = self._tracker(_FakeClock())
status = tracker.snapshot()
assert (status.degraded, status.fatal, status.reason) == (False, False, None)
assert status.degraded_for_s is None and status.retry_after_s is None
assert status.dropped_rows == 0
assert tracker.should_retry() is True # 未降级本就该正常走准备路径
def test_entering_degraded_reports_reason_and_cooldown(self, captured_warnings):
tracker = self._tracker(_FakeClock())
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
status = tracker.snapshot()
assert status.degraded is True and status.fatal is False
assert status.reason == "连接被拒"
assert status.degraded_for_s == pytest.approx(0.0)
assert status.retry_after_s == pytest.approx(60.0)
assert len(captured_warnings) == 1
assert "连接被拒" in captured_warnings[0] and "60" in captured_warnings[0]
def test_fake_clock_drains_the_cooldown(self):
clock = _FakeClock()
tracker = self._tracker(clock)
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
clock.advance(25.0)
status = tracker.snapshot()
assert status.degraded_for_s == pytest.approx(25.0)
assert status.retry_after_s == pytest.approx(35.0)
assert tracker.should_retry() is False
clock.advance(40.0) # 越过冷却窗口
assert tracker.snapshot().retry_after_s == pytest.approx(0.0) # 不得为负
assert tracker.should_retry() is True
def test_recover_clears_degradation_but_keeps_dropped_rows(self, captured_infos):
tracker = self._tracker(_FakeClock())
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
for _ in range(3):
tracker.record_drop("遥测已降级")
tracker.recover()
status = tracker.snapshot()
assert (status.degraded, status.fatal, status.reason) == (False, False, None)
assert status.degraded_for_s is None and status.retry_after_s is None
# 进程生命周期内单调不减: 恢复不是"没丢过",下游要靠它对账
assert status.dropped_rows == 3
assert any("3" in message and "恢复" in message for message in captured_infos)
def test_drop_warnings_are_throttled_by_the_row_constant(self, captured_warnings):
from polygateway.telemetry.status import _DROP_REPEAT_EVERY_ROWS
tracker = self._tracker(_FakeClock()) # 时钟不动: 只有行数阈值能触发复述
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
captured_warnings.clear() # 只数丢弃复述,不数进入降级那条
total = _DROP_REPEAT_EVERY_ROWS * 2 + 1
for _ in range(total):
tracker.record_drop("遥测已降级")
# 首条必报,其后每满一个阈值报一次 —— 关系由常量决定,不写死数字
assert len(captured_warnings) == 1 + (total - 1) // _DROP_REPEAT_EVERY_ROWS
assert tracker.snapshot().dropped_rows == total
def test_drop_warnings_are_also_throttled_by_time(self, captured_warnings):
from polygateway.telemetry.status import _DROP_REPEAT_EVERY_S
clock = _FakeClock()
tracker = self._tracker(clock)
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
captured_warnings.clear()
tracker.record_drop("遥测已降级")
assert len(captured_warnings) == 1
tracker.record_drop("遥测已降级")
assert len(captured_warnings) == 1 # 同一窗口内不刷屏
clock.advance(_DROP_REPEAT_EVERY_S)
tracker.record_drop("遥测已降级")
assert len(captured_warnings) == 2 # 长跑进程里也不会静默
def test_fatal_degradation_never_retries(self, captured_warnings):
clock = _FakeClock()
tracker = self._tracker(clock)
tracker.enter_degraded("DSN 不可解析", fatal=True, cooldown_s=None)
clock.advance(1_000_000.0)
status = tracker.snapshot()
assert status.fatal is True and status.retry_after_s is None
assert tracker.should_retry() is False
assert "重启" in captured_warnings[0] # 恢复条件必须写在日志里
def test_repeating_the_same_reason_does_not_spam(self, captured_warnings):
"""冷却到期重试再失败会反复进入降级: 同一原因只讲一次,只刷新窗口。"""
clock = _FakeClock()
tracker = self._tracker(clock)
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
clock.advance(60.0)
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
assert len(captured_warnings) == 1
assert tracker.snapshot().retry_after_s == pytest.approx(60.0) # 窗口已刷新
assert tracker.snapshot().degraded_for_s == pytest.approx(60.0) # 但仍是同一段降级
class TestSQLiteStatusVisibility:
"""SQLite 侧今天初始化失败后写入静默 return,连一条日志都没有(设计 §1.4)。"""
def _broken(self, tmp_path):
blocker = tmp_path / "blocker"
blocker.write_text("父目录是个文件,mkdir 必然失败")
return SQLiteRecorder(blocker / "telemetry.db", auto_migrate=True)
def test_init_failure_is_degraded_and_fatal(self, tmp_path, captured_warnings):
recorder = self._broken(tmp_path)
status = recorder.telemetry_status
assert status.degraded is True and status.fatal is True
assert captured_warnings # 静默降级 ≠ 静默
async def test_dropped_rows_are_counted_and_visible(self, tmp_path, captured_warnings):
recorder = self._broken(tmp_path)
captured_warnings.clear()
await _record_minimal(recorder) # 不得抛: 遥测绝不冒泡
assert recorder.telemetry_status.dropped_rows == 1
assert captured_warnings # 丢的第一行必须出声
def test_healthy_recorder_is_not_degraded(self, tmp_path):
recorder = SQLiteRecorder(tmp_path / "ok.db", auto_migrate=True)
assert recorder.telemetry_status.degraded is False
recorder.close()
class TestPostgresStatusVisibility:
"""PG 侧的判死本任务不改判据,只让它经 tracker 变得可见(计划 T2)。"""
def _recorder(self, conn):
from polygateway.telemetry.postgres import PostgresRecorder
return PostgresRecorder(
"postgresql://u:p@h:5432/polygateway", pool=_FakePgPool(conn), auto_migrate=True
)
async def test_unusable_table_shows_up_in_the_status(self, captured_warnings):
"""表确定不存在且建不出来 = 既有的判死档;现在它要能被下游查到。"""
recorder = self._recorder(_FakePgConn([], fail_create=True))
await _record_minimal(recorder)
status = recorder.telemetry_status
assert status.degraded is True and status.fatal is True
assert status.dropped_rows == 1 # 判死那一次调用本身也丢了一行
assert recorder._failed is True # 过渡期两份状态并存(T5 收掉 `_failed`)
async def test_healthy_recorder_is_not_degraded(self):
recorder = self._recorder(_FakePgConn(list(_EXPECTED_COLUMNS)))
await _record_minimal(recorder)
status = recorder.telemetry_status
assert status.degraded is False and status.dropped_rows == 0