From f958138e83bf4be9b52148043daeb27ea0f8df62 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Mon, 24 Aug 2026 08:57:23 -0400 Subject: [PATCH] feat: make telemetry degradation a first-class state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Telemetry degradation used to be a single warning and a private boolean. In a long-running process that is indistinguishable from telemetry working: issue #15 was only found by hand-reconciling milestone log lines against llm_calls rows, after 19 calls had silently gone unrecorded. The SQLite side was worse — once init failed, every write returned without even a log line. Degradation now has one shared owner. TelemetryStatusTracker holds the state machine (enter/recover/drop/should-retry), announces entry and recovery once each, and repeats the drop count under a row-and-time double threshold so a degraded backend neither floods the log nor goes quiet. Both recorders hold one; both count the rows they drop. For programmatic consumers, TelemetryStatus is a frozen snapshot exposed as telemetry_status on all three clients, resolved through a single isinstance check. It is a separate optional port rather than a member of TelemetryRecorder: that protocol is @runtime_checkable, so adding an attribute would make every implementation that only defines record_llm_call stop satisfying it — downstream isinstance assertions would break on upgrade. The existing assertion in test_ports.py is what keeps that decision honest. Failure criteria are deliberately untouched here: Postgres still treats a pool failure as permanent, only now visibly. `_failed` and the tracker therefore both carry the verdict for the span of this one change; the cooldown rework collapses them into the tracker alone. --- src/polygateway/client.py | 24 ++++ src/polygateway/embedding.py | 12 +- src/polygateway/ocr.py | 12 +- src/polygateway/ports.py | 15 +++ src/polygateway/telemetry/postgres.py | 18 ++- src/polygateway/telemetry/sqlite.py | 18 ++- src/polygateway/telemetry/status.py | 178 +++++++++++++++++++++++++ src/polygateway/types.py | 25 ++++ tests/unit/test_client.py | 37 ++++++ tests/unit/test_ports.py | 25 +++- tests/unit/test_telemetry.py | 183 ++++++++++++++++++++++++++ 11 files changed, 542 insertions(+), 5 deletions(-) create mode 100644 src/polygateway/telemetry/status.py diff --git a/src/polygateway/client.py b/src/polygateway/client.py index d23defd..d980274 100644 --- a/src/polygateway/client.py +++ b/src/polygateway/client.py @@ -24,6 +24,7 @@ from polygateway.middleware.cache import CacheMW from polygateway.middleware.retry import RetryMW from polygateway.middleware.structured import StructuredMW from polygateway.middleware.telemetry import TelemetryEmitter, TelemetryMW +from polygateway.ports import TelemetryStatusProvider from polygateway.pricing import PricingTable from polygateway.providers import get_capability, get_provider, resolve_thinking from polygateway.sources import ( @@ -37,6 +38,7 @@ from polygateway.transports.openai_compat import OpenAICompatTransport from polygateway.types import ( ChatRequest, LLMResponse, + TelemetryStatus, validate_caller_dimensions, validate_request_overlay, ) @@ -135,6 +137,19 @@ async def _aclose_component(component: object | None) -> None: close() +def _telemetry_status_of(telemetry: TelemetryRecorder | None) -> TelemetryStatus | None: + """三个 client 共用的状态取值点: 不提供状态的 recorder 一律返回 None。 + + 判定写成 `isinstance(可选端口)` 而不是裸 `getattr`: 两者运行时都是结构检查 + (`@runtime_checkable` 按属性存在性判定),差别在**契约有没有名字**——端口是 + 写进 `ports.py` 的公开承诺,下游可以照着实现;散落的 `getattr` 不是,而 + `aclose` 当年正是被复制成三份鸭子类型探测才漂移出越权关闭(设计 §3.3/§3.4)。 + """ + if isinstance(telemetry, TelemetryStatusProvider): + return telemetry.telemetry_status + return None + + def _mark_owned_components( client: Any, *, @@ -252,6 +267,15 @@ class GatewayClient: self._owns_breaker = False self._closed = False + @property + def telemetry_status(self) -> TelemetryStatus | None: + """遥测后端的可写状态;无遥测或注入的 recorder 不提供状态时为 None。 + + 判定收敛在 `_telemetry_status_of` 一处(不是三处各自探测): 三个 client + 的 `aclose` 曾各持一份逐字复制,漂移的结果就是越权关闭(设计 §3.3/§3.4)。 + """ + return _telemetry_status_of(self._telemetry) + async def chat( self, messages: list[dict[str, Any]], diff --git a/src/polygateway/embedding.py b/src/polygateway/embedding.py index 4a7ba3e..068d547 100644 --- a/src/polygateway/embedding.py +++ b/src/polygateway/embedding.py @@ -25,7 +25,7 @@ from typing import TYPE_CHECKING, Any from loguru import logger -from polygateway.client import _aclose_component +from polygateway.client import _aclose_component, _telemetry_status_of from polygateway.config import EmbeddingSettings from polygateway.errors import ( AllSourcesExhausted, @@ -46,6 +46,7 @@ from polygateway.types import ( ChatRequest, EmbeddingResponse, LLMResponse, + TelemetryStatus, strip_unsupported_extra_body, validate_caller_dimensions, ) @@ -454,6 +455,15 @@ class EmbeddingClient: known = [c for c in costs if c is not None] return sum(known) if known else None + @property + def telemetry_status(self) -> TelemetryStatus | None: + """遥测后端的可写状态;无遥测或注入的 recorder 不提供状态时为 None。 + + 判定收敛在 `_telemetry_status_of` 一处(不是三处各自探测): 三个 client + 的 `aclose` 曾各持一份逐字复制,漂移的结果就是越权关闭(设计 §3.3/§3.4)。 + """ + return _telemetry_status_of(self._telemetry) + async def aclose(self) -> None: """幂等释放**自建**资源(与 GatewayClient 对称);注入的组件一律不碰。""" if self._closed: diff --git a/src/polygateway/ocr.py b/src/polygateway/ocr.py index a39c98f..d2b486b 100644 --- a/src/polygateway/ocr.py +++ b/src/polygateway/ocr.py @@ -22,7 +22,7 @@ from typing import TYPE_CHECKING, Any, Literal from loguru import logger -from polygateway.client import _aclose_component +from polygateway.client import _aclose_component, _telemetry_status_of from polygateway.errors import ( AllSourcesExhausted, GovernanceBackendError, @@ -44,6 +44,7 @@ from polygateway.types import ( LLMResponse, OcrLayoutResult, OcrTextResult, + TelemetryStatus, Usage, strip_unsupported_extra_body, validate_caller_dimensions, @@ -464,6 +465,15 @@ class OcrClient: # —— 生命周期 —— + @property + def telemetry_status(self) -> TelemetryStatus | None: + """遥测后端的可写状态;无遥测或注入的 recorder 不提供状态时为 None。 + + 判定收敛在 `_telemetry_status_of` 一处(不是三处各自探测): 三个 client + 的 `aclose` 曾各持一份逐字复制,漂移的结果就是越权关闭(设计 §3.3/§3.4)。 + """ + return _telemetry_status_of(self._telemetry) + async def aclose(self) -> None: """幂等释放**自建**资源(与 EmbeddingClient 对称);注入的组件一律不碰。""" if self._closed: diff --git a/src/polygateway/ports.py b/src/polygateway/ports.py index 66c80c2..9429e4c 100644 --- a/src/polygateway/ports.py +++ b/src/polygateway/ports.py @@ -20,6 +20,7 @@ from .types import ( OcrTextTransportResult, SourceConfig, SourceStats, + TelemetryStatus, TransportResult, ) @@ -243,6 +244,20 @@ class StructuredOutputStrategy(Protocol): def parse(self, text: str) -> Any: ... +@runtime_checkable +class TelemetryStatusProvider(Protocol): + """可自述可写状态的遥测后端;`TelemetryRecorder` 的**可选**伴生端口(issue #15)。 + + 与 `TelemetryRecorder` 分开而不是给它加成员,是因为后者是 `@runtime_checkable` + 而运行时检查按属性存在性做: 加一个属性会让所有只实现 `record_llm_call` 的 + 实现**当场不再是** `TelemetryRecorder`,下游若有同款 isinstance 断言,升级即断 + (设计 §3.3)。消费方一律先 isinstance 再取值,取不到就当没有状态可报。 + """ + + @property + def telemetry_status(self) -> TelemetryStatus: ... + + @runtime_checkable class TelemetryRecorder(Protocol): """遥测后端;24 字段冻结(M1 设计 §4.4 + issue #3/#4/#11),唯一调用点是 TelemetryEmitter。 diff --git a/src/polygateway/telemetry/postgres.py b/src/polygateway/telemetry/postgres.py index 101f240..a94b1ac 100644 --- a/src/polygateway/telemetry/postgres.py +++ b/src/polygateway/telemetry/postgres.py @@ -28,10 +28,13 @@ from polygateway.telemetry.schema import ( insert_sql, missing_columns_warning, ) +from polygateway.telemetry.status import TelemetryStatusTracker if TYPE_CHECKING: import asyncpg + from polygateway.types import TelemetryStatus + # 探测表是否存在;不需要任何权限,且与 INSERT 走同一套 search_path 解析 _TABLE_EXISTS = "SELECT to_regclass('llm_calls')" @@ -72,8 +75,16 @@ class PostgresRecorder: self._insert = insert_sql("postgres", COLUMNS) self._schema_ready = False self._failed = False # 结构性降级标志: 置位后所有写入短路 + # 降级的可编程出口与节流日志;`_failed` 与它并存是 issue #15 的过渡态, + # 判据改造(冷却自愈)落地时状态收归 tracker 一处 + self._status = TelemetryStatusTracker(backend="postgres") self._init_lock = asyncio.Lock() + @property + def telemetry_status(self) -> TelemetryStatus: + """当前可写状态快照(ports.TelemetryStatusProvider)。""" + return self._status.snapshot() + async def _ensure_ready(self) -> asyncpg.Pool | None: """lazy 建池+备表;判死只认「确定写不进去」(issue #9),其余失败都留活路。""" if self._failed: @@ -104,7 +115,7 @@ class PostgresRecorder: # 池建不出来 = 确定写不进去;且每次调用重试都要内联吞掉 connect # 超时,而遥测是业务路径上的 await —— 此处必须永久降级 self._failed = True - logger.warning("Postgres 遥测建池失败,后续记录降级为 no-op: {}", exc) + self._status.enter_degraded(f"建池失败: {exc}", fatal=True, cooldown_s=None) return None return self._pool @@ -122,6 +133,9 @@ class PostgresRecorder: return None if columns is None: self._failed = True + self._status.enter_degraded( + "表 llm_calls 不存在且建不出来(记录无处可落)", fatal=True, cooldown_s=None + ) return None # 写入列、语句与就绪标志必须**一起**生效: `_ensure_ready` 只看 `_schema_ready` # 就绕开 `_init_lock` 直接返回池,先置就绪会开出"已就绪但语句还是旧的"的窗口 @@ -232,6 +246,8 @@ class PostgresRecorder: """ pool = await self._ensure_ready() if pool is None: + # 降级期间静默 return 就是 issue #15 的破口: 丢行必须计数且节流出声 + self._status.record_drop("遥测已降级") return row = tuple(fields[col] for col in self._columns) try: diff --git a/src/polygateway/telemetry/sqlite.py b/src/polygateway/telemetry/sqlite.py index 9b68ab8..cc75ede 100644 --- a/src/polygateway/telemetry/sqlite.py +++ b/src/polygateway/telemetry/sqlite.py @@ -19,6 +19,7 @@ import asyncio import sqlite3 import threading from pathlib import Path +from typing import TYPE_CHECKING from loguru import logger @@ -29,6 +30,10 @@ from polygateway.telemetry.schema import ( insert_sql, missing_columns_warning, ) +from polygateway.telemetry.status import TelemetryStatusTracker + +if TYPE_CHECKING: + from polygateway.types import TelemetryStatus class SQLiteRecorder: @@ -45,6 +50,9 @@ class SQLiteRecorder: config 一处,不与本类签名漂移(设计 D-c)。 """ self._auto_migrate = auto_migrate + # SQLite 侧本次只做可见性: 它的失败模式(目录不可写、文件损坏)在装配期 + # 就暴露给下游,不是"跑到一半悄悄断",故降级恒为 fatal,不做冷却重连 + self._status = TelemetryStatusTracker(backend="sqlite") self._lock = threading.Lock() self._conn: sqlite3.Connection | None = None # 先按全量列定型: 连接失败/探测失败时保守沿用全量(今天的行为) @@ -60,9 +68,14 @@ class SQLiteRecorder: conn.commit() self._conn = conn except (OSError, sqlite3.Error) as exc: - logger.warning("SQLite 遥测初始化失败,后续记录降级为 no-op: {}", exc) + self._status.enter_degraded(f"初始化失败: {exc}", fatal=True, cooldown_s=None) self._prepare_columns() + @property + def telemetry_status(self) -> TelemetryStatus: + """当前可写状态快照(ports.TelemetryStatusProvider)。""" + return self._status.snapshot() + def _prepare_columns(self) -> None: """探测现有列后定型写入: auto 档补齐缺列,manual 档改为裁剪写入(issue #13)。 @@ -136,6 +149,9 @@ class SQLiteRecorder: 占位符同序——两者必须一起改,分开改就是把值写进错位的列。 """ if self._conn is None: + # 改前这里是**裸 return**: 初始化失败后每一行都无声消失,长跑进程里 + # 与"遥测正常"外观上完全一致(设计 §1.4 的直接钉子) + self._status.record_drop("遥测已降级") return row = tuple(fields[col] for col in self._columns) try: diff --git a/src/polygateway/telemetry/status.py b/src/polygateway/telemetry/status.py new file mode 100644 index 0000000..17cbe07 --- /dev/null +++ b/src/polygateway/telemetry/status.py @@ -0,0 +1,178 @@ +"""遥测降级状态机(issue #15 C 组): 两个 recorder 共用的降级事实源。 + +存在的理由(设计 §1.4): 遥测降级过去只有**一条** warning,长跑进程里等同于 +静默——issue 是手工对账(日志里的完成里程碑条数 vs `llm_calls` 行数)才发现的, +期间 19 次调用一行未落。"遥测必录"铁律的实质要求是: 库做不到必录时,必须 +**持续、可编程地**让下游知道。故降级升格为一等对象,两条出路各走一边: + +- 人看: 进入/恢复各一条日志,降级期间按行数与时间**双阈值节流复述**(不刷屏, + 也不静默); +- 程序看: `snapshot()` 给只读 `TelemetryStatus`,下游可据此对账或告警。 + +本模块**不含任何后端知识**(不 import asyncpg/sqlite3,也不判失败性质): 失败 +分类是各 recorder 的事,tracker 只接受"降级了/恢复了/丢了一行"三个事实。 +""" + +from __future__ import annotations + +import time +from typing import TYPE_CHECKING + +from loguru import logger + +from polygateway.types import TelemetryStatus + +if TYPE_CHECKING: + from collections.abc import Callable + +_DROP_REPEAT_EVERY_ROWS = 100 +"""降级期间每丢这么多行复述一次;首行必报。""" + +_DROP_REPEAT_EVERY_S = 300.0 +"""降级期间距上次复述超过这么久就再报一次——低频调用的进程不能因行数不够而静默。""" + + +class TelemetryStatusTracker: + """单个 recorder 的降级状态;非线程安全,由持有它的 recorder 在自己的时序内使用。 + + 时钟经构造参数注入(与 `GatewayClient(now=...)` 同款): 冷却窗口与节流窗口 + 都必须能用假时钟测,否则这些行为只能靠真睡验证,而真睡的用例是间歇红的源头。 + """ + + def __init__(self, *, backend: str, now: Callable[[], float] = time.monotonic) -> None: + """记下后端名(只用于日志前缀)与时钟;构造后即"未降级"。 + + Args: + backend: 后端名(如 `postgres`/`sqlite`),仅进日志文案。 + now: 单调时钟;测试可注入假时钟推进冷却与节流窗口。 + """ + self._backend = backend + self._now = now + self._degraded_since: float | None = None + self._fatal = False + self._reason: str | None = None + self._retry_at: float | None = None + self._dropped_rows = 0 + # 节流窗口: 本段降级里"自上次复述以来"丢了多少行、上次复述在什么时候 + self._dropped_since_report = 0 + self._last_report_at: float | None = None + self._dropped_at_entry = 0 + + def enter_degraded(self, reason: str, *, fatal: bool, cooldown_s: float | None) -> None: + """进入(或续期)降级;同一原因只讲一次,只刷新冷却窗口。 + + 不重复打日志是刚需而非优化: 冷却到期重试再失败会反复走到这里,每次都讲 + 就把"降级中"刷成噪音。原因变了才算新事实,值得再讲一遍。 + + Args: + reason: 降级原因(已含具体异常文本);同值视为同一次降级的续期。 + fatal: True = 本进程内不可恢复,此后 `should_retry()` 恒 False。 + cooldown_s: 距下次允许重新准备的秒数;None 表示不自动重试。 + """ + if self._fatal: + return # 永久档不可被后来的失败覆盖,也不再刷屏 + now = self._now() + first_of_this_episode = self._degraded_since is None + announce = first_of_this_episode or reason != self._reason + if first_of_this_episode: + self._degraded_since = now + self._dropped_at_entry = self._dropped_rows + self._dropped_since_report = 0 + self._last_report_at = None + self._reason = reason + self._fatal = fatal + self._retry_at = None if fatal or cooldown_s is None else now + cooldown_s + if announce: + logger.warning( + "{} 遥测降级(后续记录将被丢弃): {};恢复条件: {}", + self._backend, + reason, + self._recovery_hint(cooldown_s, fatal=fatal), + ) + + def recover(self) -> None: + """退出降级并报告本段期间丢了多少行;未降级时是 no-op。 + + `dropped_rows` **不清零**: 它是进程生命周期内的累计量,下游靠它对账。 + """ + if self._degraded_since is None: + return + dropped = self._dropped_rows - self._dropped_at_entry + logger.info( + "{} 遥测已恢复(降级持续 {:.1f}s,期间丢弃 {} 行)", + self._backend, + self._now() - self._degraded_since, + dropped, + ) + self._degraded_since = None + self._fatal = False + self._reason = None + self._retry_at = None + self._dropped_since_report = 0 + self._last_report_at = None + + def record_drop(self, reason: str) -> None: + """记一行被丢弃;按行数与时间双阈值节流复述。 + + 双阈值缺一不可: 只按行数,低频调用的进程会长时间完全静默;只按时间, + 高频进程在窗口内丢几万行也只有一条日志,看不出量级。 + """ + self._dropped_rows += 1 + self._dropped_since_report += 1 + if not self._should_report(): + return + logger.warning( + "{} 遥测丢弃记录(累计 {} 行): {}", + self._backend, + self._dropped_rows, + reason, + ) + self._dropped_since_report = 0 + self._last_report_at = self._now() + + def should_retry(self) -> bool: + """现在是否允许(重新)准备后端: 纯查询,不触库也不改状态。 + + 未降级 → True(本就该正常走准备路径);fatal → False;冷却未到 → False; + 非 fatal 但没给冷却 → False(调用方没安排自动重试,tracker 不替它决定)。 + """ + if self._fatal: + return False + if self._degraded_since is None: + return True + if self._retry_at is None: + return False + return self._now() >= self._retry_at + + def snapshot(self) -> TelemetryStatus: + """当前状态的只读快照(公共出口 `client.telemetry_status` 的取值点)。""" + now = self._now() + since = self._degraded_since + retry_after_s: float | None = None + if since is not None and self._retry_at is not None: + retry_after_s = max(0.0, self._retry_at - now) # 到期后钳到 0,不给负数 + return TelemetryStatus( + degraded=since is not None, + fatal=self._fatal, + reason=self._reason, + degraded_for_s=None if since is None else now - since, + dropped_rows=self._dropped_rows, + retry_after_s=retry_after_s, + ) + + def _should_report(self) -> bool: + """本次丢弃是否该出声: 本段降级的第一行、满行数阈值、或超时间阈值。""" + if self._last_report_at is None: + return True + if self._dropped_since_report >= _DROP_REPEAT_EVERY_ROWS: + return True + return self._now() - self._last_report_at >= _DROP_REPEAT_EVERY_S + + @staticmethod + def _recovery_hint(cooldown_s: float | None, *, fatal: bool) -> str: + """把恢复条件写进日志: 运维看到降级后第一个问题就是"它自己会好吗"。""" + if fatal: + return "需修正配置后重启进程(本进程内不会自愈)" + if cooldown_s is None: + return "下次调用时重试" + return f"约 {cooldown_s:.0f}s 后自动重试" diff --git a/src/polygateway/types.py b/src/polygateway/types.py index bcb5180..7226f9b 100644 --- a/src/polygateway/types.py +++ b/src/polygateway/types.py @@ -258,6 +258,31 @@ class SourceStats: tpm_used: int +@dataclass(frozen=True) +class TelemetryStatus: + """遥测后端的可写状态快照;degraded 期间下游可据此对账(issue #15)。 + + 不叫 `health`: 库内 `health` 一律指**源的健康度**(`OcrTransport.check_health` + 探活、`SourceSelector.health` 成功率 EWMA),而这里描述的是"这个 recorder + 现在能不能写、为什么不能、丢了多少",是状态不是评分(设计 §3.3)。 + + 时长一律给**相对秒数**而非绝对时间戳: 库内的时钟是 monotonic,把它的读数 + 交给下游会与 wall clock 混淆成两个不可比的时间轴。 + """ + + degraded: bool + fatal: bool + """True = 本进程内不可恢复(仅 DSN 不可解析一类配置级失败),需改配置并重启。""" + reason: str | None + """降级原因;未降级为 None。""" + degraded_for_s: float | None + """已降级时长;未降级为 None。""" + dropped_rows: int + """累计丢弃行数;**进程生命周期内单调不减**——恢复不等于没丢过。""" + retry_after_s: float | None + """距下次重新准备的秒数;fatal 或未降级为 None,冷却已到期为 0.0。""" + + @dataclass(frozen=True) class TransportResult: """transport 单次原始调用的产物;治理字段由 RetryMW 补齐为 LLMResponse。""" diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index b49fb1b..98b5d14 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -835,3 +835,40 @@ class TestRedisCacheOwnership: await cache.aclose() await cache.aclose() # 幂等: 不重复关 assert built.closed == 1 + + +class TestTelemetryStatusExposure: + """降级状态的只读出口: 一处 isinstance 判定,三个 client 各钉一次(设计 §3.3)。""" + + def _recorder(self, tmp_path): + from polygateway.telemetry.sqlite import SQLiteRecorder + + return SQLiteRecorder(tmp_path / "telemetry.db", auto_migrate=True) + + def _assert_snapshot(self, status): + from polygateway.types import TelemetryStatus + + assert isinstance(status, TelemetryStatus) + assert status.degraded is False + + def test_gateway_client_without_telemetry_reports_none(self): + assert _client().telemetry_status is None + + def test_gateway_client_with_foreign_recorder_reports_none(self): + """注入的第三方 recorder 不提供状态 → None,绝不得抛 AttributeError。""" + assert _client(telemetry=_Closable()).telemetry_status is None + + def test_gateway_client_with_builtin_recorder_reports_snapshot(self, tmp_path): + self._assert_snapshot(_client(telemetry=self._recorder(tmp_path)).telemetry_status) + + def test_embedding_client_exposes_the_same_outlet(self, tmp_path): + assert _embedding_client().telemetry_status is None + assert _embedding_client(telemetry=_Closable()).telemetry_status is None + self._assert_snapshot( + _embedding_client(telemetry=self._recorder(tmp_path)).telemetry_status + ) + + def test_ocr_client_exposes_the_same_outlet(self, tmp_path): + assert _ocr_client().telemetry_status is None + assert _ocr_client(telemetry=_Closable()).telemetry_status is None + self._assert_snapshot(_ocr_client(telemetry=self._recorder(tmp_path)).telemetry_status) diff --git a/tests/unit/test_ports.py b/tests/unit/test_ports.py index c5009ee..abcdd75 100644 --- a/tests/unit/test_ports.py +++ b/tests/unit/test_ports.py @@ -16,9 +16,10 @@ from polygateway.ports import ( SourceSelector, StructuredOutputStrategy, TelemetryRecorder, + TelemetryStatusProvider, Transport, ) -from polygateway.types import LLMResponse, SourceStats +from polygateway.types import LLMResponse, SourceStats, TelemetryStatus def _resp() -> LLMResponse: @@ -141,6 +142,28 @@ def test_protocols_are_runtime_checkable(impl, protocol): assert isinstance(impl, protocol) +class _DummyStatusProvider(_DummyRecorder): + @property + def telemetry_status(self) -> TelemetryStatus: + return TelemetryStatus( + degraded=False, + fatal=False, + reason=None, + degraded_for_s=None, + dropped_rows=0, + retry_after_s=None, + ) + + +def test_status_provider_is_a_separate_optional_port(): + """状态**不得**并进 TelemetryRecorder: 那会让只实现 record_llm_call 的对象 + 当场不再满足 @runtime_checkable 的结构检查(设计 §3.3,Codex 审查)。""" + assert isinstance(_DummyStatusProvider(), TelemetryStatusProvider) + assert isinstance(_DummyStatusProvider(), TelemetryRecorder) + assert not isinstance(_DummyRecorder(), TelemetryStatusProvider) + assert isinstance(_DummyRecorder(), TelemetryRecorder) # 这条断言是那条决策的执法点 + + def _decision(**overrides) -> GateDecision: base = { "source_name": "qwen_1", diff --git a/tests/unit/test_telemetry.py b/tests/unit/test_telemetry.py index 30d938b..38a687d 100644 --- a/tests/unit/test_telemetry.py +++ b/tests/unit/test_telemetry.py @@ -1686,3 +1686,186 @@ class TestTextCapCoversEmbedAndOcrChains: # 占位串 `` 共 24 字 assert json.loads(row["messages"])[0]["content"] == " None: + self.t = start + + def __call__(self) -> float: + return self.t + + def advance(self, seconds: float) -> None: + self.t += seconds + + +@pytest.fixture +def captured_infos(): + """捕获 INFO 及以上;恢复那条 info 是"降级已结束"的唯一外部信号。""" + from loguru import logger + + messages: list[str] = [] + sink_id = logger.add(messages.append, level="INFO") + yield messages + logger.remove(sink_id) + + +class TestTelemetryStatusTracker: + """状态机六字段逐个钉;它是两个 recorder 共用的降级事实源(设计 §3.3)。""" + + def _tracker(self, clock): + from polygateway.telemetry.status import TelemetryStatusTracker + + return TelemetryStatusTracker(backend="postgres", now=clock) + + def test_fresh_tracker_reports_no_degradation(self): + tracker = self._tracker(_FakeClock()) + status = tracker.snapshot() + assert (status.degraded, status.fatal, status.reason) == (False, False, None) + assert status.degraded_for_s is None and status.retry_after_s is None + assert status.dropped_rows == 0 + assert tracker.should_retry() is True # 未降级本就该正常走准备路径 + + def test_entering_degraded_reports_reason_and_cooldown(self, captured_warnings): + tracker = self._tracker(_FakeClock()) + tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0) + status = tracker.snapshot() + assert status.degraded is True and status.fatal is False + assert status.reason == "连接被拒" + assert status.degraded_for_s == pytest.approx(0.0) + assert status.retry_after_s == pytest.approx(60.0) + assert len(captured_warnings) == 1 + assert "连接被拒" in captured_warnings[0] and "60" in captured_warnings[0] + + def test_fake_clock_drains_the_cooldown(self): + clock = _FakeClock() + tracker = self._tracker(clock) + tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0) + clock.advance(25.0) + status = tracker.snapshot() + assert status.degraded_for_s == pytest.approx(25.0) + assert status.retry_after_s == pytest.approx(35.0) + assert tracker.should_retry() is False + clock.advance(40.0) # 越过冷却窗口 + assert tracker.snapshot().retry_after_s == pytest.approx(0.0) # 不得为负 + assert tracker.should_retry() is True + + def test_recover_clears_degradation_but_keeps_dropped_rows(self, captured_infos): + tracker = self._tracker(_FakeClock()) + tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0) + for _ in range(3): + tracker.record_drop("遥测已降级") + tracker.recover() + status = tracker.snapshot() + assert (status.degraded, status.fatal, status.reason) == (False, False, None) + assert status.degraded_for_s is None and status.retry_after_s is None + # 进程生命周期内单调不减: 恢复不是"没丢过",下游要靠它对账 + assert status.dropped_rows == 3 + assert any("3" in message and "恢复" in message for message in captured_infos) + + def test_drop_warnings_are_throttled_by_the_row_constant(self, captured_warnings): + from polygateway.telemetry.status import _DROP_REPEAT_EVERY_ROWS + + tracker = self._tracker(_FakeClock()) # 时钟不动: 只有行数阈值能触发复述 + tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0) + captured_warnings.clear() # 只数丢弃复述,不数进入降级那条 + total = _DROP_REPEAT_EVERY_ROWS * 2 + 1 + for _ in range(total): + tracker.record_drop("遥测已降级") + # 首条必报,其后每满一个阈值报一次 —— 关系由常量决定,不写死数字 + assert len(captured_warnings) == 1 + (total - 1) // _DROP_REPEAT_EVERY_ROWS + assert tracker.snapshot().dropped_rows == total + + def test_drop_warnings_are_also_throttled_by_time(self, captured_warnings): + from polygateway.telemetry.status import _DROP_REPEAT_EVERY_S + + clock = _FakeClock() + tracker = self._tracker(clock) + tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0) + captured_warnings.clear() + tracker.record_drop("遥测已降级") + assert len(captured_warnings) == 1 + tracker.record_drop("遥测已降级") + assert len(captured_warnings) == 1 # 同一窗口内不刷屏 + clock.advance(_DROP_REPEAT_EVERY_S) + tracker.record_drop("遥测已降级") + assert len(captured_warnings) == 2 # 长跑进程里也不会静默 + + def test_fatal_degradation_never_retries(self, captured_warnings): + clock = _FakeClock() + tracker = self._tracker(clock) + tracker.enter_degraded("DSN 不可解析", fatal=True, cooldown_s=None) + clock.advance(1_000_000.0) + status = tracker.snapshot() + assert status.fatal is True and status.retry_after_s is None + assert tracker.should_retry() is False + assert "重启" in captured_warnings[0] # 恢复条件必须写在日志里 + + def test_repeating_the_same_reason_does_not_spam(self, captured_warnings): + """冷却到期重试再失败会反复进入降级: 同一原因只讲一次,只刷新窗口。""" + clock = _FakeClock() + tracker = self._tracker(clock) + tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0) + clock.advance(60.0) + tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0) + assert len(captured_warnings) == 1 + assert tracker.snapshot().retry_after_s == pytest.approx(60.0) # 窗口已刷新 + assert tracker.snapshot().degraded_for_s == pytest.approx(60.0) # 但仍是同一段降级 + + +class TestSQLiteStatusVisibility: + """SQLite 侧今天初始化失败后写入静默 return,连一条日志都没有(设计 §1.4)。""" + + def _broken(self, tmp_path): + blocker = tmp_path / "blocker" + blocker.write_text("父目录是个文件,mkdir 必然失败") + return SQLiteRecorder(blocker / "telemetry.db", auto_migrate=True) + + def test_init_failure_is_degraded_and_fatal(self, tmp_path, captured_warnings): + recorder = self._broken(tmp_path) + status = recorder.telemetry_status + assert status.degraded is True and status.fatal is True + assert captured_warnings # 静默降级 ≠ 静默 + + async def test_dropped_rows_are_counted_and_visible(self, tmp_path, captured_warnings): + recorder = self._broken(tmp_path) + captured_warnings.clear() + await _record_minimal(recorder) # 不得抛: 遥测绝不冒泡 + assert recorder.telemetry_status.dropped_rows == 1 + assert captured_warnings # 丢的第一行必须出声 + + def test_healthy_recorder_is_not_degraded(self, tmp_path): + recorder = SQLiteRecorder(tmp_path / "ok.db", auto_migrate=True) + assert recorder.telemetry_status.degraded is False + recorder.close() + + +class TestPostgresStatusVisibility: + """PG 侧的判死本任务不改判据,只让它经 tracker 变得可见(计划 T2)。""" + + def _recorder(self, conn): + from polygateway.telemetry.postgres import PostgresRecorder + + return PostgresRecorder( + "postgresql://u:p@h:5432/polygateway", pool=_FakePgPool(conn), auto_migrate=True + ) + + async def test_unusable_table_shows_up_in_the_status(self, captured_warnings): + """表确定不存在且建不出来 = 既有的判死档;现在它要能被下游查到。""" + recorder = self._recorder(_FakePgConn([], fail_create=True)) + await _record_minimal(recorder) + status = recorder.telemetry_status + assert status.degraded is True and status.fatal is True + assert status.dropped_rows == 1 # 判死那一次调用本身也丢了一行 + assert recorder._failed is True # 过渡期两份状态并存(T5 收掉 `_failed`) + + async def test_healthy_recorder_is_not_degraded(self): + recorder = self._recorder(_FakePgConn(list(_EXPECTED_COLUMNS))) + await _record_minimal(recorder) + status = recorder.telemetry_status + assert status.degraded is False and status.dropped_rows == 0