feat: make telemetry degradation a first-class state

Telemetry degradation used to be a single warning and a private boolean.
In a long-running process that is indistinguishable from telemetry working:
issue #15 was only found by hand-reconciling milestone log lines against
llm_calls rows, after 19 calls had silently gone unrecorded. The SQLite
side was worse — once init failed, every write returned without even a
log line.

Degradation now has one shared owner. TelemetryStatusTracker holds the
state machine (enter/recover/drop/should-retry), announces entry and
recovery once each, and repeats the drop count under a row-and-time
double threshold so a degraded backend neither floods the log nor goes
quiet. Both recorders hold one; both count the rows they drop.

For programmatic consumers, TelemetryStatus is a frozen snapshot exposed
as telemetry_status on all three clients, resolved through a single
isinstance check. It is a separate optional port rather than a member of
TelemetryRecorder: that protocol is @runtime_checkable, so adding an
attribute would make every implementation that only defines
record_llm_call stop satisfying it — downstream isinstance assertions
would break on upgrade. The existing assertion in test_ports.py is what
keeps that decision honest.

Failure criteria are deliberately untouched here: Postgres still treats a
pool failure as permanent, only now visibly. `_failed` and the tracker
therefore both carry the verdict for the span of this one change; the
cooldown rework collapses them into the tracker alone.
This commit is contained in:
2026-08-24 08:57:23 -04:00
parent e69ca4c82c
commit f958138e83
11 changed files with 542 additions and 5 deletions
+24
View File
@@ -24,6 +24,7 @@ from polygateway.middleware.cache import CacheMW
from polygateway.middleware.retry import RetryMW
from polygateway.middleware.structured import StructuredMW
from polygateway.middleware.telemetry import TelemetryEmitter, TelemetryMW
from polygateway.ports import TelemetryStatusProvider
from polygateway.pricing import PricingTable
from polygateway.providers import get_capability, get_provider, resolve_thinking
from polygateway.sources import (
@@ -37,6 +38,7 @@ from polygateway.transports.openai_compat import OpenAICompatTransport
from polygateway.types import (
ChatRequest,
LLMResponse,
TelemetryStatus,
validate_caller_dimensions,
validate_request_overlay,
)
@@ -135,6 +137,19 @@ async def _aclose_component(component: object | None) -> None:
close()
def _telemetry_status_of(telemetry: TelemetryRecorder | None) -> TelemetryStatus | None:
"""三个 client 共用的状态取值点: 不提供状态的 recorder 一律返回 None。
判定写成 `isinstance(可选端口)` 而不是裸 `getattr`: 两者运行时都是结构检查
(`@runtime_checkable` 按属性存在性判定),差别在**契约有没有名字**——端口是
写进 `ports.py` 的公开承诺,下游可以照着实现;散落的 `getattr` 不是,而
`aclose` 当年正是被复制成三份鸭子类型探测才漂移出越权关闭(设计 §3.3/§3.4)。
"""
if isinstance(telemetry, TelemetryStatusProvider):
return telemetry.telemetry_status
return None
def _mark_owned_components(
client: Any,
*,
@@ -252,6 +267,15 @@ class GatewayClient:
self._owns_breaker = False
self._closed = False
@property
def telemetry_status(self) -> TelemetryStatus | None:
"""遥测后端的可写状态;无遥测或注入的 recorder 不提供状态时为 None。
判定收敛在 `_telemetry_status_of` 一处(不是三处各自探测): 三个 client
的 `aclose` 曾各持一份逐字复制,漂移的结果就是越权关闭(设计 §3.3/§3.4)。
"""
return _telemetry_status_of(self._telemetry)
async def chat(
self,
messages: list[dict[str, Any]],
+11 -1
View File
@@ -25,7 +25,7 @@ from typing import TYPE_CHECKING, Any
from loguru import logger
from polygateway.client import _aclose_component
from polygateway.client import _aclose_component, _telemetry_status_of
from polygateway.config import EmbeddingSettings
from polygateway.errors import (
AllSourcesExhausted,
@@ -46,6 +46,7 @@ from polygateway.types import (
ChatRequest,
EmbeddingResponse,
LLMResponse,
TelemetryStatus,
strip_unsupported_extra_body,
validate_caller_dimensions,
)
@@ -454,6 +455,15 @@ class EmbeddingClient:
known = [c for c in costs if c is not None]
return sum(known) if known else None
@property
def telemetry_status(self) -> TelemetryStatus | None:
"""遥测后端的可写状态;无遥测或注入的 recorder 不提供状态时为 None。
判定收敛在 `_telemetry_status_of` 一处(不是三处各自探测): 三个 client
的 `aclose` 曾各持一份逐字复制,漂移的结果就是越权关闭(设计 §3.3/§3.4)。
"""
return _telemetry_status_of(self._telemetry)
async def aclose(self) -> None:
"""幂等释放**自建**资源(与 GatewayClient 对称);注入的组件一律不碰。"""
if self._closed:
+11 -1
View File
@@ -22,7 +22,7 @@ from typing import TYPE_CHECKING, Any, Literal
from loguru import logger
from polygateway.client import _aclose_component
from polygateway.client import _aclose_component, _telemetry_status_of
from polygateway.errors import (
AllSourcesExhausted,
GovernanceBackendError,
@@ -44,6 +44,7 @@ from polygateway.types import (
LLMResponse,
OcrLayoutResult,
OcrTextResult,
TelemetryStatus,
Usage,
strip_unsupported_extra_body,
validate_caller_dimensions,
@@ -464,6 +465,15 @@ class OcrClient:
# —— 生命周期 ——
@property
def telemetry_status(self) -> TelemetryStatus | None:
"""遥测后端的可写状态;无遥测或注入的 recorder 不提供状态时为 None。
判定收敛在 `_telemetry_status_of` 一处(不是三处各自探测): 三个 client
的 `aclose` 曾各持一份逐字复制,漂移的结果就是越权关闭(设计 §3.3/§3.4)。
"""
return _telemetry_status_of(self._telemetry)
async def aclose(self) -> None:
"""幂等释放**自建**资源(与 EmbeddingClient 对称);注入的组件一律不碰。"""
if self._closed:
+15
View File
@@ -20,6 +20,7 @@ from .types import (
OcrTextTransportResult,
SourceConfig,
SourceStats,
TelemetryStatus,
TransportResult,
)
@@ -243,6 +244,20 @@ class StructuredOutputStrategy(Protocol):
def parse(self, text: str) -> Any: ...
@runtime_checkable
class TelemetryStatusProvider(Protocol):
"""可自述可写状态的遥测后端;`TelemetryRecorder` 的**可选**伴生端口(issue #15)。
与 `TelemetryRecorder` 分开而不是给它加成员,是因为后者是 `@runtime_checkable`
而运行时检查按属性存在性做: 加一个属性会让所有只实现 `record_llm_call` 的
实现**当场不再是** `TelemetryRecorder`,下游若有同款 isinstance 断言,升级即断
(设计 §3.3)。消费方一律先 isinstance 再取值,取不到就当没有状态可报。
"""
@property
def telemetry_status(self) -> TelemetryStatus: ...
@runtime_checkable
class TelemetryRecorder(Protocol):
"""遥测后端;24 字段冻结(M1 设计 §4.4 + issue #3/#4/#11),唯一调用点是 TelemetryEmitter。
+17 -1
View File
@@ -28,10 +28,13 @@ from polygateway.telemetry.schema import (
insert_sql,
missing_columns_warning,
)
from polygateway.telemetry.status import TelemetryStatusTracker
if TYPE_CHECKING:
import asyncpg
from polygateway.types import TelemetryStatus
# 探测表是否存在;不需要任何权限,且与 INSERT 走同一套 search_path 解析
_TABLE_EXISTS = "SELECT to_regclass('llm_calls')"
@@ -72,8 +75,16 @@ class PostgresRecorder:
self._insert = insert_sql("postgres", COLUMNS)
self._schema_ready = False
self._failed = False # 结构性降级标志: 置位后所有写入短路
# 降级的可编程出口与节流日志;`_failed` 与它并存是 issue #15 的过渡态,
# 判据改造(冷却自愈)落地时状态收归 tracker 一处
self._status = TelemetryStatusTracker(backend="postgres")
self._init_lock = asyncio.Lock()
@property
def telemetry_status(self) -> TelemetryStatus:
"""当前可写状态快照(ports.TelemetryStatusProvider)。"""
return self._status.snapshot()
async def _ensure_ready(self) -> asyncpg.Pool | None:
"""lazy 建池+备表;判死只认「确定写不进去」(issue #9),其余失败都留活路。"""
if self._failed:
@@ -104,7 +115,7 @@ class PostgresRecorder:
# 池建不出来 = 确定写不进去;且每次调用重试都要内联吞掉 connect
# 超时,而遥测是业务路径上的 await —— 此处必须永久降级
self._failed = True
logger.warning("Postgres 遥测建池失败,后续记录降级为 no-op: {}", exc)
self._status.enter_degraded(f"建池失败: {exc}", fatal=True, cooldown_s=None)
return None
return self._pool
@@ -122,6 +133,9 @@ class PostgresRecorder:
return None
if columns is None:
self._failed = True
self._status.enter_degraded(
"表 llm_calls 不存在且建不出来(记录无处可落)", fatal=True, cooldown_s=None
)
return None
# 写入列、语句与就绪标志必须**一起**生效: `_ensure_ready` 只看 `_schema_ready`
# 就绕开 `_init_lock` 直接返回池,先置就绪会开出"已就绪但语句还是旧的"的窗口
@@ -232,6 +246,8 @@ class PostgresRecorder:
"""
pool = await self._ensure_ready()
if pool is None:
# 降级期间静默 return 就是 issue #15 的破口: 丢行必须计数且节流出声
self._status.record_drop("遥测已降级")
return
row = tuple(fields[col] for col in self._columns)
try:
+17 -1
View File
@@ -19,6 +19,7 @@ import asyncio
import sqlite3
import threading
from pathlib import Path
from typing import TYPE_CHECKING
from loguru import logger
@@ -29,6 +30,10 @@ from polygateway.telemetry.schema import (
insert_sql,
missing_columns_warning,
)
from polygateway.telemetry.status import TelemetryStatusTracker
if TYPE_CHECKING:
from polygateway.types import TelemetryStatus
class SQLiteRecorder:
@@ -45,6 +50,9 @@ class SQLiteRecorder:
config 一处,不与本类签名漂移(设计 D-c)。
"""
self._auto_migrate = auto_migrate
# SQLite 侧本次只做可见性: 它的失败模式(目录不可写、文件损坏)在装配期
# 就暴露给下游,不是"跑到一半悄悄断",故降级恒为 fatal,不做冷却重连
self._status = TelemetryStatusTracker(backend="sqlite")
self._lock = threading.Lock()
self._conn: sqlite3.Connection | None = None
# 先按全量列定型: 连接失败/探测失败时保守沿用全量(今天的行为)
@@ -60,9 +68,14 @@ class SQLiteRecorder:
conn.commit()
self._conn = conn
except (OSError, sqlite3.Error) as exc:
logger.warning("SQLite 遥测初始化失败,后续记录降级为 no-op: {}", exc)
self._status.enter_degraded(f"初始化失败: {exc}", fatal=True, cooldown_s=None)
self._prepare_columns()
@property
def telemetry_status(self) -> TelemetryStatus:
"""当前可写状态快照(ports.TelemetryStatusProvider)。"""
return self._status.snapshot()
def _prepare_columns(self) -> None:
"""探测现有列后定型写入: auto 档补齐缺列,manual 档改为裁剪写入(issue #13)。
@@ -136,6 +149,9 @@ class SQLiteRecorder:
占位符同序——两者必须一起改,分开改就是把值写进错位的列。
"""
if self._conn is None:
# 改前这里是**裸 return**: 初始化失败后每一行都无声消失,长跑进程里
# 与"遥测正常"外观上完全一致(设计 §1.4 的直接钉子)
self._status.record_drop("遥测已降级")
return
row = tuple(fields[col] for col in self._columns)
try:
+178
View File
@@ -0,0 +1,178 @@
"""遥测降级状态机(issue #15 C 组): 两个 recorder 共用的降级事实源。
存在的理由(设计 §1.4): 遥测降级过去只有**一条** warning,长跑进程里等同于
静默——issue 是手工对账(日志里的完成里程碑条数 vs `llm_calls` 行数)才发现的,
期间 19 次调用一行未落。"遥测必录"铁律的实质要求是: 库做不到必录时,必须
**持续、可编程地**让下游知道。故降级升格为一等对象,两条出路各走一边:
- 人看: 进入/恢复各一条日志,降级期间按行数与时间**双阈值节流复述**(不刷屏,
也不静默);
- 程序看: `snapshot()` 给只读 `TelemetryStatus`,下游可据此对账或告警。
本模块**不含任何后端知识**(不 import asyncpg/sqlite3,也不判失败性质): 失败
分类是各 recorder 的事,tracker 只接受"降级了/恢复了/丢了一行"三个事实。
"""
from __future__ import annotations
import time
from typing import TYPE_CHECKING
from loguru import logger
from polygateway.types import TelemetryStatus
if TYPE_CHECKING:
from collections.abc import Callable
_DROP_REPEAT_EVERY_ROWS = 100
"""降级期间每丢这么多行复述一次;首行必报。"""
_DROP_REPEAT_EVERY_S = 300.0
"""降级期间距上次复述超过这么久就再报一次——低频调用的进程不能因行数不够而静默。"""
class TelemetryStatusTracker:
"""单个 recorder 的降级状态;非线程安全,由持有它的 recorder 在自己的时序内使用。
时钟经构造参数注入(与 `GatewayClient(now=...)` 同款): 冷却窗口与节流窗口
都必须能用假时钟测,否则这些行为只能靠真睡验证,而真睡的用例是间歇红的源头。
"""
def __init__(self, *, backend: str, now: Callable[[], float] = time.monotonic) -> None:
"""记下后端名(只用于日志前缀)与时钟;构造后即"未降级"
Args:
backend: 后端名(如 `postgres`/`sqlite`),仅进日志文案。
now: 单调时钟;测试可注入假时钟推进冷却与节流窗口。
"""
self._backend = backend
self._now = now
self._degraded_since: float | None = None
self._fatal = False
self._reason: str | None = None
self._retry_at: float | None = None
self._dropped_rows = 0
# 节流窗口: 本段降级里"自上次复述以来"丢了多少行、上次复述在什么时候
self._dropped_since_report = 0
self._last_report_at: float | None = None
self._dropped_at_entry = 0
def enter_degraded(self, reason: str, *, fatal: bool, cooldown_s: float | None) -> None:
"""进入(或续期)降级;同一原因只讲一次,只刷新冷却窗口。
不重复打日志是刚需而非优化: 冷却到期重试再失败会反复走到这里,每次都讲
就把"降级中"刷成噪音。原因变了才算新事实,值得再讲一遍。
Args:
reason: 降级原因(已含具体异常文本);同值视为同一次降级的续期。
fatal: True = 本进程内不可恢复,此后 `should_retry()` 恒 False。
cooldown_s: 距下次允许重新准备的秒数;None 表示不自动重试。
"""
if self._fatal:
return # 永久档不可被后来的失败覆盖,也不再刷屏
now = self._now()
first_of_this_episode = self._degraded_since is None
announce = first_of_this_episode or reason != self._reason
if first_of_this_episode:
self._degraded_since = now
self._dropped_at_entry = self._dropped_rows
self._dropped_since_report = 0
self._last_report_at = None
self._reason = reason
self._fatal = fatal
self._retry_at = None if fatal or cooldown_s is None else now + cooldown_s
if announce:
logger.warning(
"{} 遥测降级(后续记录将被丢弃): {};恢复条件: {}",
self._backend,
reason,
self._recovery_hint(cooldown_s, fatal=fatal),
)
def recover(self) -> None:
"""退出降级并报告本段期间丢了多少行;未降级时是 no-op。
`dropped_rows` **不清零**: 它是进程生命周期内的累计量,下游靠它对账。
"""
if self._degraded_since is None:
return
dropped = self._dropped_rows - self._dropped_at_entry
logger.info(
"{} 遥测已恢复(降级持续 {:.1f}s,期间丢弃 {} 行)",
self._backend,
self._now() - self._degraded_since,
dropped,
)
self._degraded_since = None
self._fatal = False
self._reason = None
self._retry_at = None
self._dropped_since_report = 0
self._last_report_at = None
def record_drop(self, reason: str) -> None:
"""记一行被丢弃;按行数与时间双阈值节流复述。
双阈值缺一不可: 只按行数,低频调用的进程会长时间完全静默;只按时间,
高频进程在窗口内丢几万行也只有一条日志,看不出量级。
"""
self._dropped_rows += 1
self._dropped_since_report += 1
if not self._should_report():
return
logger.warning(
"{} 遥测丢弃记录(累计 {} 行): {}",
self._backend,
self._dropped_rows,
reason,
)
self._dropped_since_report = 0
self._last_report_at = self._now()
def should_retry(self) -> bool:
"""现在是否允许(重新)准备后端: 纯查询,不触库也不改状态。
未降级 → True(本就该正常走准备路径);fatal → False;冷却未到 → False;
非 fatal 但没给冷却 → False(调用方没安排自动重试,tracker 不替它决定)。
"""
if self._fatal:
return False
if self._degraded_since is None:
return True
if self._retry_at is None:
return False
return self._now() >= self._retry_at
def snapshot(self) -> TelemetryStatus:
"""当前状态的只读快照(公共出口 `client.telemetry_status` 的取值点)。"""
now = self._now()
since = self._degraded_since
retry_after_s: float | None = None
if since is not None and self._retry_at is not None:
retry_after_s = max(0.0, self._retry_at - now) # 到期后钳到 0,不给负数
return TelemetryStatus(
degraded=since is not None,
fatal=self._fatal,
reason=self._reason,
degraded_for_s=None if since is None else now - since,
dropped_rows=self._dropped_rows,
retry_after_s=retry_after_s,
)
def _should_report(self) -> bool:
"""本次丢弃是否该出声: 本段降级的第一行、满行数阈值、或超时间阈值。"""
if self._last_report_at is None:
return True
if self._dropped_since_report >= _DROP_REPEAT_EVERY_ROWS:
return True
return self._now() - self._last_report_at >= _DROP_REPEAT_EVERY_S
@staticmethod
def _recovery_hint(cooldown_s: float | None, *, fatal: bool) -> str:
"""把恢复条件写进日志: 运维看到降级后第一个问题就是"它自己会好吗""""
if fatal:
return "需修正配置后重启进程(本进程内不会自愈)"
if cooldown_s is None:
return "下次调用时重试"
return f"{cooldown_s:.0f}s 后自动重试"
+25
View File
@@ -258,6 +258,31 @@ class SourceStats:
tpm_used: int
@dataclass(frozen=True)
class TelemetryStatus:
"""遥测后端的可写状态快照;degraded 期间下游可据此对账(issue #15)。
不叫 `health`: 库内 `health` 一律指**源的健康度**(`OcrTransport.check_health`
探活、`SourceSelector.health` 成功率 EWMA),而这里描述的是"这个 recorder
现在能不能写、为什么不能、丢了多少",是状态不是评分(设计 §3.3)。
时长一律给**相对秒数**而非绝对时间戳: 库内的时钟是 monotonic,把它的读数
交给下游会与 wall clock 混淆成两个不可比的时间轴。
"""
degraded: bool
fatal: bool
"""True = 本进程内不可恢复(仅 DSN 不可解析一类配置级失败),需改配置并重启。"""
reason: str | None
"""降级原因;未降级为 None。"""
degraded_for_s: float | None
"""已降级时长;未降级为 None。"""
dropped_rows: int
"""累计丢弃行数;**进程生命周期内单调不减**——恢复不等于没丢过。"""
retry_after_s: float | None
"""距下次重新准备的秒数;fatal 或未降级为 None,冷却已到期为 0.0。"""
@dataclass(frozen=True)
class TransportResult:
"""transport 单次原始调用的产物;治理字段由 RetryMW 补齐为 LLMResponse。"""
+37
View File
@@ -835,3 +835,40 @@ class TestRedisCacheOwnership:
await cache.aclose()
await cache.aclose() # 幂等: 不重复关
assert built.closed == 1
class TestTelemetryStatusExposure:
"""降级状态的只读出口: 一处 isinstance 判定,三个 client 各钉一次(设计 §3.3)。"""
def _recorder(self, tmp_path):
from polygateway.telemetry.sqlite import SQLiteRecorder
return SQLiteRecorder(tmp_path / "telemetry.db", auto_migrate=True)
def _assert_snapshot(self, status):
from polygateway.types import TelemetryStatus
assert isinstance(status, TelemetryStatus)
assert status.degraded is False
def test_gateway_client_without_telemetry_reports_none(self):
assert _client().telemetry_status is None
def test_gateway_client_with_foreign_recorder_reports_none(self):
"""注入的第三方 recorder 不提供状态 → None,绝不得抛 AttributeError。"""
assert _client(telemetry=_Closable()).telemetry_status is None
def test_gateway_client_with_builtin_recorder_reports_snapshot(self, tmp_path):
self._assert_snapshot(_client(telemetry=self._recorder(tmp_path)).telemetry_status)
def test_embedding_client_exposes_the_same_outlet(self, tmp_path):
assert _embedding_client().telemetry_status is None
assert _embedding_client(telemetry=_Closable()).telemetry_status is None
self._assert_snapshot(
_embedding_client(telemetry=self._recorder(tmp_path)).telemetry_status
)
def test_ocr_client_exposes_the_same_outlet(self, tmp_path):
assert _ocr_client().telemetry_status is None
assert _ocr_client(telemetry=_Closable()).telemetry_status is None
self._assert_snapshot(_ocr_client(telemetry=self._recorder(tmp_path)).telemetry_status)
+24 -1
View File
@@ -16,9 +16,10 @@ from polygateway.ports import (
SourceSelector,
StructuredOutputStrategy,
TelemetryRecorder,
TelemetryStatusProvider,
Transport,
)
from polygateway.types import LLMResponse, SourceStats
from polygateway.types import LLMResponse, SourceStats, TelemetryStatus
def _resp() -> LLMResponse:
@@ -141,6 +142,28 @@ def test_protocols_are_runtime_checkable(impl, protocol):
assert isinstance(impl, protocol)
class _DummyStatusProvider(_DummyRecorder):
@property
def telemetry_status(self) -> TelemetryStatus:
return TelemetryStatus(
degraded=False,
fatal=False,
reason=None,
degraded_for_s=None,
dropped_rows=0,
retry_after_s=None,
)
def test_status_provider_is_a_separate_optional_port():
"""状态**不得**并进 TelemetryRecorder: 那会让只实现 record_llm_call 的对象
当场不再满足 @runtime_checkable 的结构检查(设计 §3.3,Codex 审查)。"""
assert isinstance(_DummyStatusProvider(), TelemetryStatusProvider)
assert isinstance(_DummyStatusProvider(), TelemetryRecorder)
assert not isinstance(_DummyRecorder(), TelemetryStatusProvider)
assert isinstance(_DummyRecorder(), TelemetryRecorder) # 这条断言是那条决策的执法点
def _decision(**overrides) -> GateDecision:
base = {
"source_name": "qwen_1",
+183
View File
@@ -1686,3 +1686,186 @@ class TestTextCapCoversEmbedAndOcrChains:
# 占位串 `<ocr:text image_bytes=3>` 共 24 字
assert json.loads(row["messages"])[0]["content"] == "<ocr:tex…(略 16 字)"
assert row["response"] == "识别结果识别结果…(略 32 字)" # 先经 OCR 自有的 200 字上限
# —— 遥测降级状态(issue #15 C 组): 共用 tracker + 只读快照 + 节流日志 ——
class _FakeClock:
"""可手动推进的单调时钟: 冷却与节流都靠它测,用例里绝不真睡。"""
def __init__(self, start: float = 1_000.0) -> None:
self.t = start
def __call__(self) -> float:
return self.t
def advance(self, seconds: float) -> None:
self.t += seconds
@pytest.fixture
def captured_infos():
"""捕获 INFO 及以上;恢复那条 info 是"降级已结束"的唯一外部信号。"""
from loguru import logger
messages: list[str] = []
sink_id = logger.add(messages.append, level="INFO")
yield messages
logger.remove(sink_id)
class TestTelemetryStatusTracker:
"""状态机六字段逐个钉;它是两个 recorder 共用的降级事实源(设计 §3.3)。"""
def _tracker(self, clock):
from polygateway.telemetry.status import TelemetryStatusTracker
return TelemetryStatusTracker(backend="postgres", now=clock)
def test_fresh_tracker_reports_no_degradation(self):
tracker = self._tracker(_FakeClock())
status = tracker.snapshot()
assert (status.degraded, status.fatal, status.reason) == (False, False, None)
assert status.degraded_for_s is None and status.retry_after_s is None
assert status.dropped_rows == 0
assert tracker.should_retry() is True # 未降级本就该正常走准备路径
def test_entering_degraded_reports_reason_and_cooldown(self, captured_warnings):
tracker = self._tracker(_FakeClock())
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
status = tracker.snapshot()
assert status.degraded is True and status.fatal is False
assert status.reason == "连接被拒"
assert status.degraded_for_s == pytest.approx(0.0)
assert status.retry_after_s == pytest.approx(60.0)
assert len(captured_warnings) == 1
assert "连接被拒" in captured_warnings[0] and "60" in captured_warnings[0]
def test_fake_clock_drains_the_cooldown(self):
clock = _FakeClock()
tracker = self._tracker(clock)
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
clock.advance(25.0)
status = tracker.snapshot()
assert status.degraded_for_s == pytest.approx(25.0)
assert status.retry_after_s == pytest.approx(35.0)
assert tracker.should_retry() is False
clock.advance(40.0) # 越过冷却窗口
assert tracker.snapshot().retry_after_s == pytest.approx(0.0) # 不得为负
assert tracker.should_retry() is True
def test_recover_clears_degradation_but_keeps_dropped_rows(self, captured_infos):
tracker = self._tracker(_FakeClock())
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
for _ in range(3):
tracker.record_drop("遥测已降级")
tracker.recover()
status = tracker.snapshot()
assert (status.degraded, status.fatal, status.reason) == (False, False, None)
assert status.degraded_for_s is None and status.retry_after_s is None
# 进程生命周期内单调不减: 恢复不是"没丢过",下游要靠它对账
assert status.dropped_rows == 3
assert any("3" in message and "恢复" in message for message in captured_infos)
def test_drop_warnings_are_throttled_by_the_row_constant(self, captured_warnings):
from polygateway.telemetry.status import _DROP_REPEAT_EVERY_ROWS
tracker = self._tracker(_FakeClock()) # 时钟不动: 只有行数阈值能触发复述
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
captured_warnings.clear() # 只数丢弃复述,不数进入降级那条
total = _DROP_REPEAT_EVERY_ROWS * 2 + 1
for _ in range(total):
tracker.record_drop("遥测已降级")
# 首条必报,其后每满一个阈值报一次 —— 关系由常量决定,不写死数字
assert len(captured_warnings) == 1 + (total - 1) // _DROP_REPEAT_EVERY_ROWS
assert tracker.snapshot().dropped_rows == total
def test_drop_warnings_are_also_throttled_by_time(self, captured_warnings):
from polygateway.telemetry.status import _DROP_REPEAT_EVERY_S
clock = _FakeClock()
tracker = self._tracker(clock)
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
captured_warnings.clear()
tracker.record_drop("遥测已降级")
assert len(captured_warnings) == 1
tracker.record_drop("遥测已降级")
assert len(captured_warnings) == 1 # 同一窗口内不刷屏
clock.advance(_DROP_REPEAT_EVERY_S)
tracker.record_drop("遥测已降级")
assert len(captured_warnings) == 2 # 长跑进程里也不会静默
def test_fatal_degradation_never_retries(self, captured_warnings):
clock = _FakeClock()
tracker = self._tracker(clock)
tracker.enter_degraded("DSN 不可解析", fatal=True, cooldown_s=None)
clock.advance(1_000_000.0)
status = tracker.snapshot()
assert status.fatal is True and status.retry_after_s is None
assert tracker.should_retry() is False
assert "重启" in captured_warnings[0] # 恢复条件必须写在日志里
def test_repeating_the_same_reason_does_not_spam(self, captured_warnings):
"""冷却到期重试再失败会反复进入降级: 同一原因只讲一次,只刷新窗口。"""
clock = _FakeClock()
tracker = self._tracker(clock)
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
clock.advance(60.0)
tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
assert len(captured_warnings) == 1
assert tracker.snapshot().retry_after_s == pytest.approx(60.0) # 窗口已刷新
assert tracker.snapshot().degraded_for_s == pytest.approx(60.0) # 但仍是同一段降级
class TestSQLiteStatusVisibility:
"""SQLite 侧今天初始化失败后写入静默 return,连一条日志都没有(设计 §1.4)。"""
def _broken(self, tmp_path):
blocker = tmp_path / "blocker"
blocker.write_text("父目录是个文件,mkdir 必然失败")
return SQLiteRecorder(blocker / "telemetry.db", auto_migrate=True)
def test_init_failure_is_degraded_and_fatal(self, tmp_path, captured_warnings):
recorder = self._broken(tmp_path)
status = recorder.telemetry_status
assert status.degraded is True and status.fatal is True
assert captured_warnings # 静默降级 ≠ 静默
async def test_dropped_rows_are_counted_and_visible(self, tmp_path, captured_warnings):
recorder = self._broken(tmp_path)
captured_warnings.clear()
await _record_minimal(recorder) # 不得抛: 遥测绝不冒泡
assert recorder.telemetry_status.dropped_rows == 1
assert captured_warnings # 丢的第一行必须出声
def test_healthy_recorder_is_not_degraded(self, tmp_path):
recorder = SQLiteRecorder(tmp_path / "ok.db", auto_migrate=True)
assert recorder.telemetry_status.degraded is False
recorder.close()
class TestPostgresStatusVisibility:
"""PG 侧的判死本任务不改判据,只让它经 tracker 变得可见(计划 T2)。"""
def _recorder(self, conn):
from polygateway.telemetry.postgres import PostgresRecorder
return PostgresRecorder(
"postgresql://u:p@h:5432/polygateway", pool=_FakePgPool(conn), auto_migrate=True
)
async def test_unusable_table_shows_up_in_the_status(self, captured_warnings):
"""表确定不存在且建不出来 = 既有的判死档;现在它要能被下游查到。"""
recorder = self._recorder(_FakePgConn([], fail_create=True))
await _record_minimal(recorder)
status = recorder.telemetry_status
assert status.degraded is True and status.fatal is True
assert status.dropped_rows == 1 # 判死那一次调用本身也丢了一行
assert recorder._failed is True # 过渡期两份状态并存(T5 收掉 `_failed`)
async def test_healthy_recorder_is_not_degraded(self):
recorder = self._recorder(_FakePgConn(list(_EXPECTED_COLUMNS)))
await _record_minimal(recorder)
status = recorder.telemetry_status
assert status.degraded is False and status.dropped_rows == 0