feat: make telemetry degradation a first-class state

Telemetry degradation used to be a single warning and a private boolean.
In a long-running process that is indistinguishable from telemetry working:
issue #15 was only found by hand-reconciling milestone log lines against
llm_calls rows, after 19 calls had silently gone unrecorded. The SQLite
side was worse — once init failed, every write returned without even a
log line.

Degradation now has one shared owner. TelemetryStatusTracker holds the
state machine (enter/recover/drop/should-retry), announces entry and
recovery once each, and repeats the drop count under a row-and-time
double threshold so a degraded backend neither floods the log nor goes
quiet. Both recorders hold one; both count the rows they drop.

For programmatic consumers, TelemetryStatus is a frozen snapshot exposed
as telemetry_status on all three clients, resolved through a single
isinstance check. It is a separate optional port rather than a member of
TelemetryRecorder: that protocol is @runtime_checkable, so adding an
attribute would make every implementation that only defines
record_llm_call stop satisfying it — downstream isinstance assertions
would break on upgrade. The existing assertion in test_ports.py is what
keeps that decision honest.

Failure criteria are deliberately untouched here: Postgres still treats a
pool failure as permanent, only now visibly. `_failed` and the tracker
therefore both carry the verdict for the span of this one change; the
cooldown rework collapses them into the tracker alone.
This commit is contained in:
2026-08-24 08:57:23 -04:00
parent e69ca4c82c
commit f958138e83
11 changed files with 542 additions and 5 deletions
+178
View File
@@ -0,0 +1,178 @@
"""遥测降级状态机(issue #15 C 组): 两个 recorder 共用的降级事实源。
存在的理由(设计 §1.4): 遥测降级过去只有**一条** warning,长跑进程里等同于
静默——issue 是手工对账(日志里的完成里程碑条数 vs `llm_calls` 行数)才发现的,
期间 19 次调用一行未落。"遥测必录"铁律的实质要求是: 库做不到必录时,必须
**持续、可编程地**让下游知道。故降级升格为一等对象,两条出路各走一边:
- 人看: 进入/恢复各一条日志,降级期间按行数与时间**双阈值节流复述**(不刷屏,
也不静默);
- 程序看: `snapshot()` 给只读 `TelemetryStatus`,下游可据此对账或告警。
本模块**不含任何后端知识**(不 import asyncpg/sqlite3,也不判失败性质): 失败
分类是各 recorder 的事,tracker 只接受"降级了/恢复了/丢了一行"三个事实。
"""
from __future__ import annotations
import time
from typing import TYPE_CHECKING
from loguru import logger
from polygateway.types import TelemetryStatus
if TYPE_CHECKING:
from collections.abc import Callable
_DROP_REPEAT_EVERY_ROWS = 100
"""降级期间每丢这么多行复述一次;首行必报。"""
_DROP_REPEAT_EVERY_S = 300.0
"""降级期间距上次复述超过这么久就再报一次——低频调用的进程不能因行数不够而静默。"""
class TelemetryStatusTracker:
"""单个 recorder 的降级状态;非线程安全,由持有它的 recorder 在自己的时序内使用。
时钟经构造参数注入(与 `GatewayClient(now=...)` 同款): 冷却窗口与节流窗口
都必须能用假时钟测,否则这些行为只能靠真睡验证,而真睡的用例是间歇红的源头。
"""
def __init__(self, *, backend: str, now: Callable[[], float] = time.monotonic) -> None:
"""记下后端名(只用于日志前缀)与时钟;构造后即"未降级"
Args:
backend: 后端名(如 `postgres`/`sqlite`),仅进日志文案。
now: 单调时钟;测试可注入假时钟推进冷却与节流窗口。
"""
self._backend = backend
self._now = now
self._degraded_since: float | None = None
self._fatal = False
self._reason: str | None = None
self._retry_at: float | None = None
self._dropped_rows = 0
# 节流窗口: 本段降级里"自上次复述以来"丢了多少行、上次复述在什么时候
self._dropped_since_report = 0
self._last_report_at: float | None = None
self._dropped_at_entry = 0
def enter_degraded(self, reason: str, *, fatal: bool, cooldown_s: float | None) -> None:
"""进入(或续期)降级;同一原因只讲一次,只刷新冷却窗口。
不重复打日志是刚需而非优化: 冷却到期重试再失败会反复走到这里,每次都讲
就把"降级中"刷成噪音。原因变了才算新事实,值得再讲一遍。
Args:
reason: 降级原因(已含具体异常文本);同值视为同一次降级的续期。
fatal: True = 本进程内不可恢复,此后 `should_retry()` 恒 False。
cooldown_s: 距下次允许重新准备的秒数;None 表示不自动重试。
"""
if self._fatal:
return # 永久档不可被后来的失败覆盖,也不再刷屏
now = self._now()
first_of_this_episode = self._degraded_since is None
announce = first_of_this_episode or reason != self._reason
if first_of_this_episode:
self._degraded_since = now
self._dropped_at_entry = self._dropped_rows
self._dropped_since_report = 0
self._last_report_at = None
self._reason = reason
self._fatal = fatal
self._retry_at = None if fatal or cooldown_s is None else now + cooldown_s
if announce:
logger.warning(
"{} 遥测降级(后续记录将被丢弃): {};恢复条件: {}",
self._backend,
reason,
self._recovery_hint(cooldown_s, fatal=fatal),
)
def recover(self) -> None:
"""退出降级并报告本段期间丢了多少行;未降级时是 no-op。
`dropped_rows` **不清零**: 它是进程生命周期内的累计量,下游靠它对账。
"""
if self._degraded_since is None:
return
dropped = self._dropped_rows - self._dropped_at_entry
logger.info(
"{} 遥测已恢复(降级持续 {:.1f}s,期间丢弃 {} 行)",
self._backend,
self._now() - self._degraded_since,
dropped,
)
self._degraded_since = None
self._fatal = False
self._reason = None
self._retry_at = None
self._dropped_since_report = 0
self._last_report_at = None
def record_drop(self, reason: str) -> None:
"""记一行被丢弃;按行数与时间双阈值节流复述。
双阈值缺一不可: 只按行数,低频调用的进程会长时间完全静默;只按时间,
高频进程在窗口内丢几万行也只有一条日志,看不出量级。
"""
self._dropped_rows += 1
self._dropped_since_report += 1
if not self._should_report():
return
logger.warning(
"{} 遥测丢弃记录(累计 {} 行): {}",
self._backend,
self._dropped_rows,
reason,
)
self._dropped_since_report = 0
self._last_report_at = self._now()
def should_retry(self) -> bool:
"""现在是否允许(重新)准备后端: 纯查询,不触库也不改状态。
未降级 → True(本就该正常走准备路径);fatal → False;冷却未到 → False;
非 fatal 但没给冷却 → False(调用方没安排自动重试,tracker 不替它决定)。
"""
if self._fatal:
return False
if self._degraded_since is None:
return True
if self._retry_at is None:
return False
return self._now() >= self._retry_at
def snapshot(self) -> TelemetryStatus:
"""当前状态的只读快照(公共出口 `client.telemetry_status` 的取值点)。"""
now = self._now()
since = self._degraded_since
retry_after_s: float | None = None
if since is not None and self._retry_at is not None:
retry_after_s = max(0.0, self._retry_at - now) # 到期后钳到 0,不给负数
return TelemetryStatus(
degraded=since is not None,
fatal=self._fatal,
reason=self._reason,
degraded_for_s=None if since is None else now - since,
dropped_rows=self._dropped_rows,
retry_after_s=retry_after_s,
)
def _should_report(self) -> bool:
"""本次丢弃是否该出声: 本段降级的第一行、满行数阈值、或超时间阈值。"""
if self._last_report_at is None:
return True
if self._dropped_since_report >= _DROP_REPEAT_EVERY_ROWS:
return True
return self._now() - self._last_report_at >= _DROP_REPEAT_EVERY_S
@staticmethod
def _recovery_hint(cooldown_s: float | None, *, fatal: bool) -> str:
"""把恢复条件写进日志: 运维看到降级后第一个问题就是"它自己会好吗""""
if fatal:
return "需修正配置后重启进程(本进程内不会自愈)"
if cooldown_s is None:
return "下次调用时重试"
return f"{cooldown_s:.0f}s 后自动重试"