feat: make telemetry degradation a first-class state

Telemetry degradation used to be a single warning and a private boolean. In a long-running process that is indistinguishable from telemetry working: issue #15 was only found by hand-reconciling milestone log lines against llm_calls rows, after 19 calls had silently gone unrecorded. The SQLite side was worse — once init failed, every write returned without even a log line. Degradation now has one shared owner. TelemetryStatusTracker holds the state machine (enter/recover/drop/should-retry), announces entry and recovery once each, and repeats the drop count under a row-and-time double threshold so a degraded backend neither floods the log nor goes quiet. Both recorders hold one; both count the rows they drop. For programmatic consumers, TelemetryStatus is a frozen snapshot exposed as telemetry_status on all three clients, resolved through a single isinstance check. It is a separate optional port rather than a member of TelemetryRecorder: that protocol is @runtime_checkable, so adding an attribute would make every implementation that only defines record_llm_call stop satisfying it — downstream isinstance assertions would break on upgrade. The existing assertion in test_ports.py is what keeps that decision honest. Failure criteria are deliberately untouched here: Postgres still treats a pool failure as permanent, only now visibly. `_failed` and the tracker therefore both carry the verdict for the span of this one change; the cooldown rework collapses them into the tracker alone.
2026-08-24 08:57:23 -04:00
parent e69ca4c82c
commit f958138e83
11 changed files with 542 additions and 5 deletions
@@ -1686,3 +1686,186 @@ class TestTextCapCoversEmbedAndOcrChains:
        # 占位串 `<ocr:text image_bytes=3>` 共 24 字
        assert json.loads(row["messages"])[0]["content"] == "<ocr:tex…(略 16 字)"
        assert row["response"] == "识别结果识别结果…(略 32 字)"  # 先经 OCR 自有的 200 字上限
+
+
+# —— 遥测降级状态(issue #15 C 组): 共用 tracker + 只读快照 + 节流日志 ——
+
+
+class _FakeClock:
+    """可手动推进的单调时钟: 冷却与节流都靠它测,用例里绝不真睡。"""
+
+    def __init__(self, start: float = 1_000.0) -> None:
+        self.t = start
+
+    def __call__(self) -> float:
+        return self.t
+
+    def advance(self, seconds: float) -> None:
+        self.t += seconds
+
+
+@pytest.fixture
+def captured_infos():
+    """捕获 INFO 及以上;恢复那条 info 是"降级已结束"的唯一外部信号。"""
+    from loguru import logger
+
+    messages: list[str] = []
+    sink_id = logger.add(messages.append, level="INFO")
+    yield messages
+    logger.remove(sink_id)
+
+
+class TestTelemetryStatusTracker:
+    """状态机六字段逐个钉;它是两个 recorder 共用的降级事实源(设计 §3.3)。"""
+
+    def _tracker(self, clock):
+        from polygateway.telemetry.status import TelemetryStatusTracker
+
+        return TelemetryStatusTracker(backend="postgres", now=clock)
+
+    def test_fresh_tracker_reports_no_degradation(self):
+        tracker = self._tracker(_FakeClock())
+        status = tracker.snapshot()
+        assert (status.degraded, status.fatal, status.reason) == (False, False, None)
+        assert status.degraded_for_s is None and status.retry_after_s is None
+        assert status.dropped_rows == 0
+        assert tracker.should_retry() is True  # 未降级本就该正常走准备路径
+
+    def test_entering_degraded_reports_reason_and_cooldown(self, captured_warnings):
+        tracker = self._tracker(_FakeClock())
+        tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
+        status = tracker.snapshot()
+        assert status.degraded is True and status.fatal is False
+        assert status.reason == "连接被拒"
+        assert status.degraded_for_s == pytest.approx(0.0)
+        assert status.retry_after_s == pytest.approx(60.0)
+        assert len(captured_warnings) == 1
+        assert "连接被拒" in captured_warnings[0] and "60" in captured_warnings[0]
+
+    def test_fake_clock_drains_the_cooldown(self):
+        clock = _FakeClock()
+        tracker = self._tracker(clock)
+        tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
+        clock.advance(25.0)
+        status = tracker.snapshot()
+        assert status.degraded_for_s == pytest.approx(25.0)
+        assert status.retry_after_s == pytest.approx(35.0)
+        assert tracker.should_retry() is False
+        clock.advance(40.0)  # 越过冷却窗口
+        assert tracker.snapshot().retry_after_s == pytest.approx(0.0)  # 不得为负
+        assert tracker.should_retry() is True
+
+    def test_recover_clears_degradation_but_keeps_dropped_rows(self, captured_infos):
+        tracker = self._tracker(_FakeClock())
+        tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
+        for _ in range(3):
+            tracker.record_drop("遥测已降级")
+        tracker.recover()
+        status = tracker.snapshot()
+        assert (status.degraded, status.fatal, status.reason) == (False, False, None)
+        assert status.degraded_for_s is None and status.retry_after_s is None
+        # 进程生命周期内单调不减: 恢复不是"没丢过",下游要靠它对账
+        assert status.dropped_rows == 3
+        assert any("3" in message and "恢复" in message for message in captured_infos)
+
+    def test_drop_warnings_are_throttled_by_the_row_constant(self, captured_warnings):
+        from polygateway.telemetry.status import _DROP_REPEAT_EVERY_ROWS
+
+        tracker = self._tracker(_FakeClock())  # 时钟不动: 只有行数阈值能触发复述
+        tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
+        captured_warnings.clear()  # 只数丢弃复述,不数进入降级那条
+        total = _DROP_REPEAT_EVERY_ROWS * 2 + 1
+        for _ in range(total):
+            tracker.record_drop("遥测已降级")
+        # 首条必报,其后每满一个阈值报一次 —— 关系由常量决定,不写死数字
+        assert len(captured_warnings) == 1 + (total - 1) // _DROP_REPEAT_EVERY_ROWS
+        assert tracker.snapshot().dropped_rows == total
+
+    def test_drop_warnings_are_also_throttled_by_time(self, captured_warnings):
+        from polygateway.telemetry.status import _DROP_REPEAT_EVERY_S
+
+        clock = _FakeClock()
+        tracker = self._tracker(clock)
+        tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
+        captured_warnings.clear()
+        tracker.record_drop("遥测已降级")
+        assert len(captured_warnings) == 1
+        tracker.record_drop("遥测已降级")
+        assert len(captured_warnings) == 1  # 同一窗口内不刷屏
+        clock.advance(_DROP_REPEAT_EVERY_S)
+        tracker.record_drop("遥测已降级")
+        assert len(captured_warnings) == 2  # 长跑进程里也不会静默
+
+    def test_fatal_degradation_never_retries(self, captured_warnings):
+        clock = _FakeClock()
+        tracker = self._tracker(clock)
+        tracker.enter_degraded("DSN 不可解析", fatal=True, cooldown_s=None)
+        clock.advance(1_000_000.0)
+        status = tracker.snapshot()
+        assert status.fatal is True and status.retry_after_s is None
+        assert tracker.should_retry() is False
+        assert "重启" in captured_warnings[0]  # 恢复条件必须写在日志里
+
+    def test_repeating_the_same_reason_does_not_spam(self, captured_warnings):
+        """冷却到期重试再失败会反复进入降级: 同一原因只讲一次,只刷新窗口。"""
+        clock = _FakeClock()
+        tracker = self._tracker(clock)
+        tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
+        clock.advance(60.0)
+        tracker.enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
+        assert len(captured_warnings) == 1
+        assert tracker.snapshot().retry_after_s == pytest.approx(60.0)  # 窗口已刷新
+        assert tracker.snapshot().degraded_for_s == pytest.approx(60.0)  # 但仍是同一段降级
+
+
+class TestSQLiteStatusVisibility:
+    """SQLite 侧今天初始化失败后写入静默 return,连一条日志都没有(设计 §1.4)。"""
+
+    def _broken(self, tmp_path):
+        blocker = tmp_path / "blocker"
+        blocker.write_text("父目录是个文件,mkdir 必然失败")
+        return SQLiteRecorder(blocker / "telemetry.db", auto_migrate=True)
+
+    def test_init_failure_is_degraded_and_fatal(self, tmp_path, captured_warnings):
+        recorder = self._broken(tmp_path)
+        status = recorder.telemetry_status
+        assert status.degraded is True and status.fatal is True
+        assert captured_warnings  # 静默降级 ≠ 静默
+
+    async def test_dropped_rows_are_counted_and_visible(self, tmp_path, captured_warnings):
+        recorder = self._broken(tmp_path)
+        captured_warnings.clear()
+        await _record_minimal(recorder)  # 不得抛: 遥测绝不冒泡
+        assert recorder.telemetry_status.dropped_rows == 1
+        assert captured_warnings  # 丢的第一行必须出声
+
+    def test_healthy_recorder_is_not_degraded(self, tmp_path):
+        recorder = SQLiteRecorder(tmp_path / "ok.db", auto_migrate=True)
+        assert recorder.telemetry_status.degraded is False
+        recorder.close()
+
+
+class TestPostgresStatusVisibility:
+    """PG 侧的判死本任务不改判据,只让它经 tracker 变得可见(计划 T2)。"""
+
+    def _recorder(self, conn):
+        from polygateway.telemetry.postgres import PostgresRecorder
+
+        return PostgresRecorder(
+            "postgresql://u:p@h:5432/polygateway", pool=_FakePgPool(conn), auto_migrate=True
+        )
+
+    async def test_unusable_table_shows_up_in_the_status(self, captured_warnings):
+        """表确定不存在且建不出来 = 既有的判死档;现在它要能被下游查到。"""
+        recorder = self._recorder(_FakePgConn([], fail_create=True))
+        await _record_minimal(recorder)
+        status = recorder.telemetry_status
+        assert status.degraded is True and status.fatal is True
+        assert status.dropped_rows == 1  # 判死那一次调用本身也丢了一行
+        assert recorder._failed is True  # 过渡期两份状态并存(T5 收掉 `_failed`)
+
+    async def test_healthy_recorder_is_not_degraded(self):
+        recorder = self._recorder(_FakePgConn(list(_EXPECTED_COLUMNS)))
+        await _record_minimal(recorder)
+        status = recorder.telemetry_status
+        assert status.degraded is False and status.dropped_rows == 0