test: give the log level and ownership rules real enforcement

两条"确证的假绿"(独立验证发现):

① 设计 §3.2 的"配置级致命发 error 而非 warning"没有执法点:
   `captured_warnings` fixture 挂在 level="WARNING",ERROR 与 WARNING
   同池,且 tracker 自己那条 WARNING 文案就含"重启"——把 recorder 的
   `logger.error` 整块删掉,原用例照样绿。新增 `captured_logs` fixture
   连级别一起捕获,三处补上级别断言。

   顺带消掉实现与设计的偏离: 原实现同时发 1 条 ERROR(recorder)+ 1 条
   语义重复的 WARNING(tracker)。级别决策收敛到 tracker 一处(fatal →
   error,其余 → warning),recorder 侧不再另发,SQLite 侧同时受益。

② 所有权判定的 `is None` / `is not None` 纪律(设计 §3.4)零覆盖:
   所有假件都是 truthy,把工厂改回 `limiter or _build_limiter(...)`
   全套件照样绿。补 `_FalsyClosable`(`__bool__` 返 False)与三个工厂
   各一条用例: 注入 falsy 后端时工厂不得自建、`_owns_*` 为 False、
   `aclose` 不得关它。
This commit is contained in:
2026-08-24 11:45:32 -04:00
parent 9026acd7dc
commit f90f7b036c
4 changed files with 168 additions and 13 deletions
+3 -6
View File
@@ -376,12 +376,9 @@ class PostgresRecorder:
""" """
verdict = _classify_failure(exc) verdict = _classify_failure(exc)
if verdict == _FATAL: if verdict == _FATAL:
# error 而非 warning: 这是人配错了,且本进程内不会自愈,运维要看见 # 这里**不再**另发一条 error: 级别由 tracker 按 `fatal` 决定(致命档发
logger.error( # error——人配错了,本进程内不会自愈)。此处复制一条只会让同一个事实出
"Postgres 遥测{}失败: 配置有误,本进程内不会自愈(请修正 DSN 后重启): {}", # 两条语义重复的日志,并给"级别"这个决策造出第二个源头
stage,
exc,
)
self._status.enter_degraded( self._status.enter_degraded(
f"{stage}失败(配置有误): {exc}", fatal=True, cooldown_s=None f"{stage}失败(配置有误): {exc}", fatal=True, cooldown_s=None
) )
+9 -2
View File
@@ -66,7 +66,8 @@ class TelemetryStatusTracker:
Args: Args:
reason: 降级原因(已含具体异常文本);同值视为同一次降级的续期。 reason: 降级原因(已含具体异常文本);同值视为同一次降级的续期。
fatal: True = 本进程内不可恢复,此后 `should_retry()` 恒 False fatal: True = 本进程内不可恢复,此后 `should_retry()` 恒 False;
**同时决定日志级别**(见下方发日志处)。
cooldown_s: 距下次允许重新准备的秒数;None 表示不自动重试。 cooldown_s: 距下次允许重新准备的秒数;None 表示不自动重试。
""" """
if self._fatal: if self._fatal:
@@ -83,7 +84,13 @@ class TelemetryStatusTracker:
self._fatal = fatal self._fatal = fatal
self._retry_at = None if fatal or cooldown_s is None else now + cooldown_s self._retry_at = None if fatal or cooldown_s is None else now + cooldown_s
if announce: if announce:
logger.warning( # 级别由 `fatal` 决定,且**只在这一处**决定(设计 §3.2): 致命档是"人把
# 配置写错了、本进程内不会自愈",运维必须看见 → error;其余都是外部
# 状态、会自愈 → warning。recorder 侧一度各自再发一条 error,同一个
# 事实因此出两条语义重复的日志,"级别"这个决策也就有了两个源头——两个
# 源头必然漂移,正是本 issue 反复踩的那类错
emit = logger.error if fatal else logger.warning
emit(
"{} 遥测降级(后续记录将被丢弃): {};恢复条件: {}", "{} 遥测降级(后续记录将被丢弃): {};恢复条件: {}",
self._backend, self._backend,
reason, reason,
+111
View File
@@ -585,10 +585,27 @@ class _SyncClosable:
self.closed += 1 self.closed += 1
class _FalsyClosable(_Closable):
"""`bool()` 为假的组件(空容器形态的后端就长这样)。
所有权判定必须写 `is None` / `is not None`,不得写 `or`(设计 §3.4,ARCH §4.5
细则 2): 写 `or` 时注入这样一个后端会**悄悄走自建分支**,而所有权标志按
`is None` 判成 False——于是既没用上注入的那个,自建的那个又没人关,正是本
issue 要修的泄漏原地复活。判定与标志一漂移,两个 bug 一起回来。
"""
def __bool__(self):
return False
def _parts(*names): def _parts(*names):
return {name: _Closable() for name in names} return {name: _Closable() for name in names}
def _falsy_parts(*names):
return {name: _FalsyClosable() for name in names}
def _patch_builders(monkeypatch, built, *, transport_path): def _patch_builders(monkeypatch, built, *, transport_path):
"""把工厂的自建点换成可计数假件;transport 无注入入口,故恒自建。""" """把工厂的自建点换成可计数假件;transport 无注入入口,故恒自建。"""
monkeypatch.setattr(transport_path, lambda **kwargs: built["transport"]) monkeypatch.setattr(transport_path, lambda **kwargs: built["transport"])
@@ -654,6 +671,38 @@ class TestGatewayClientOwnership:
assert all(part.closed == 0 for part in injected.values()) assert all(part.closed == 0 for part in injected.values())
assert built["transport"].closed == 1 # 工厂恒自建 transport,归 client assert built["transport"].closed == 1 # 工厂恒自建 transport,归 client
async def test_falsy_injected_components_are_still_injected(self, monkeypatch):
"""`is not None` 是所有权判定成立的**必要条件**,不是风格偏好(设计 §3.4)。
改回 `or` 时: 工厂拿自建件顶掉注入件(下游以为在共享,其实各跑各的),
且自建件的 `_owns_*` 仍是 False —— redis 客户端就地泄漏。
"""
built = _parts("transport", "telemetry", "cache", "limiter", "breaker")
_patch_builders(monkeypatch, built, transport_path=self._GATEWAY_TRANSPORT)
injected = _falsy_parts("telemetry", "cache", "limiter", "breaker")
client = GatewayClient.from_settings(
GatewaySettings.from_env("LLM", env=_CACHE_ENV),
limiter=injected["limiter"],
breaker=injected["breaker"],
cache=injected["cache"],
telemetry=injected["telemetry"],
)
assert client._limiter_backend is injected["limiter"]
assert client._breaker_backend is injected["breaker"]
assert client._cache is injected["cache"]
assert client._telemetry is injected["telemetry"]
owns = (client._owns_limiter, client._owns_breaker, client._owns_cache)
assert owns == (False, False, False) and client._owns_telemetry is False
await client.aclose()
assert all(part.closed == 0 for part in injected.values())
# 自建件根本不该被造出来更不该被关;只有恒自建的 transport 归 client
assert [built[name].closed for name in ("telemetry", "cache", "limiter", "breaker")] == [
0,
0,
0,
0,
]
async def test_aclose_is_idempotent(self, monkeypatch): async def test_aclose_is_idempotent(self, monkeypatch):
built = _parts("transport", "telemetry", "cache", "limiter", "breaker") built = _parts("transport", "telemetry", "cache", "limiter", "breaker")
_patch_builders(monkeypatch, built, transport_path=self._GATEWAY_TRANSPORT) _patch_builders(monkeypatch, built, transport_path=self._GATEWAY_TRANSPORT)
@@ -746,6 +795,39 @@ class TestEmbeddingClientOwnership:
assert all(part.closed == 0 for part in injected.values()) assert all(part.closed == 0 for part in injected.values())
assert built["transport"].closed == 1 assert built["transport"].closed == 1
async def test_falsy_injected_components_are_still_injected(self, monkeypatch):
"""三处工厂各写一遍 `is not None`,就是三处各有一次漂移回 `or` 的机会。"""
from polygateway.config import EmbeddingSettings
from polygateway.embedding import EmbeddingClient
built = _parts("transport", "telemetry", "limiter", "breaker")
_patch_builders(
monkeypatch,
built,
transport_path="polygateway.transports.openai_compat.OpenAICompatTransport",
)
injected = _falsy_parts("telemetry", "limiter", "breaker")
settings = EmbeddingSettings(
gateway=GatewaySettings.from_env("LLM", env=_ENV), batch_size=2
)
client = EmbeddingClient.from_settings(
settings,
limiter=injected["limiter"],
breaker=injected["breaker"],
telemetry=injected["telemetry"],
)
assert client._limiter_backend is injected["limiter"]
assert client._breaker_backend is injected["breaker"]
assert client._telemetry is injected["telemetry"]
assert (client._owns_limiter, client._owns_breaker, client._owns_telemetry) == (
False,
False,
False,
)
await client.aclose()
assert all(part.closed == 0 for part in injected.values())
assert [built[name].closed for name in ("telemetry", "limiter", "breaker")] == [0, 0, 0]
def _ocr_client(**overrides): def _ocr_client(**overrides):
from polygateway.ocr import OcrClient from polygateway.ocr import OcrClient
@@ -811,6 +893,35 @@ class TestOcrClientOwnership:
assert all(part.closed == 0 for part in injected.values()) assert all(part.closed == 0 for part in injected.values())
assert built["transport"].closed == 1 assert built["transport"].closed == 1
async def test_falsy_injected_components_are_still_injected(self, monkeypatch):
from polygateway.config import OcrSettings
from polygateway.ocr import OcrClient
built = _parts("transport", "telemetry", "limiter", "breaker")
_patch_builders(
monkeypatch,
built,
transport_path="polygateway.transports.monkey_ocr.MonkeyOcrTransport",
)
injected = _falsy_parts("telemetry", "limiter", "breaker")
client = OcrClient.from_settings(
OcrSettings.from_env("OCR", env=dict(_OCR_ENV)),
limiter=injected["limiter"],
breaker=injected["breaker"],
telemetry=injected["telemetry"],
)
assert client._limiter_backend is injected["limiter"]
assert client._breaker_backend is injected["breaker"]
assert client._telemetry is injected["telemetry"]
assert (client._owns_limiter, client._owns_breaker, client._owns_telemetry) == (
False,
False,
False,
)
await client.aclose()
assert all(part.closed == 0 for part in injected.values())
assert [built[name].closed for name in ("telemetry", "limiter", "breaker")] == [0, 0, 0]
class TestRedisCacheOwnership: class TestRedisCacheOwnership:
"""组件内部自建的连接归组件自己;照抄 RedisLimiter._owns_client 的正确先例。""" """组件内部自建的连接归组件自己;照抄 RedisLimiter._owns_client 的正确先例。"""
+45 -5
View File
@@ -147,6 +147,25 @@ def captured_warnings():
logger.remove(sink_id) logger.remove(sink_id)
@pytest.fixture
def captured_logs():
"""捕获 WARNING 及以上的日志**连同级别**,产出 `(级别名, 文案)` 列表。
与 `captured_warnings` 分开存在是有理由的: 后者只留文案,而 ERROR 与 WARNING
在 `level="WARNING"` 的 sink 里同池——于是"配置级致命发 error 而非 warning"
这条设计决策(§3.2)长期**没有执法点**,把发 error 的那几行整块删掉,原有用例
照样全绿。级别是决策的一部分,就必须能被断言。
"""
from loguru import logger
records: list[tuple[str, str]] = []
sink_id = logger.add(
lambda m: records.append((m.record["level"].name, m.record["message"])), level="WARNING"
)
yield records
logger.remove(sink_id)
# 搬迁前(1.2.1)两个 recorder 各自持有的 INSERT 常量原文,逐字冻结在此。 # 搬迁前(1.2.1)两个 recorder 各自持有的 INSERT 常量原文,逐字冻结在此。
# 这两条字符串是"纯搬迁不改行为"的机械证据: 构造逻辑换了地方,产物必须一字不差。 # 这两条字符串是"纯搬迁不改行为"的机械证据: 构造逻辑换了地方,产物必须一字不差。
_FROZEN_SQLITE_INSERT = ( _FROZEN_SQLITE_INSERT = (
@@ -1888,6 +1907,17 @@ class TestTelemetryStatusTracker:
assert tracker.should_retry() is False assert tracker.should_retry() is False
assert "重启" in captured_warnings[0] # 恢复条件必须写在日志里 assert "重启" in captured_warnings[0] # 恢复条件必须写在日志里
def test_log_level_is_decided_by_fatal_and_only_here(self, captured_logs):
"""级别决策**收敛在 tracker 一处**: fatal → ERROR,其余 → WARNING(设计 §3.2)。
这是那条决策在全库唯一的执法点。它同时钉两个方向: 把 fatal 那档改回
warning 会红,把两档都提成 error 也会红——"人配错了""外部挂了"必须
在日志级别上分得开,运维的告警规则就架在这个区分上。
"""
self._tracker(_FakeClock()).enter_degraded("DSN 不可解析", fatal=True, cooldown_s=None)
self._tracker(_FakeClock()).enter_degraded("连接被拒", fatal=False, cooldown_s=60.0)
assert [level for level, _ in captured_logs] == ["ERROR", "WARNING"]
def test_repeating_the_same_reason_does_not_spam(self, captured_warnings): def test_repeating_the_same_reason_does_not_spam(self, captured_warnings):
"""冷却到期重试再失败会反复进入降级: 同一原因只讲一次,只刷新窗口。""" """冷却到期重试再失败会反复进入降级: 同一原因只讲一次,只刷新窗口。"""
clock = _FakeClock() clock = _FakeClock()
@@ -1908,11 +1938,13 @@ class TestSQLiteStatusVisibility:
blocker.write_text("父目录是个文件,mkdir 必然失败") blocker.write_text("父目录是个文件,mkdir 必然失败")
return SQLiteRecorder(blocker / "telemetry.db", auto_migrate=True) return SQLiteRecorder(blocker / "telemetry.db", auto_migrate=True)
def test_init_failure_is_degraded_and_fatal(self, tmp_path, captured_warnings): def test_init_failure_is_degraded_and_fatal(self, tmp_path, captured_logs):
recorder = self._broken(tmp_path) recorder = self._broken(tmp_path)
status = recorder.telemetry_status status = recorder.telemetry_status
assert status.degraded is True and status.fatal is True assert status.degraded is True and status.fatal is True
assert captured_warnings # 静默降级 ≠ 静默 # 静默降级 ≠ 静默;且致命档两侧同级别——tracker 是级别的唯一决策点,
# SQLite 侧不该因为没人复制那条 error 就降一级
assert [(level, "重启" in m) for level, m in captured_logs] == [("ERROR", True)]
async def test_dropped_rows_are_counted_and_visible(self, tmp_path, captured_warnings): async def test_dropped_rows_are_counted_and_visible(self, tmp_path, captured_warnings):
recorder = self._broken(tmp_path) recorder = self._broken(tmp_path)
@@ -2291,8 +2323,13 @@ class TestPostgresFailureClassification:
assert recorder.telemetry_status.degraded is False # 自愈,无需重启进程 assert recorder.telemetry_status.degraded is False # 自愈,无需重启进程
assert recorder.telemetry_status.dropped_rows == 2 # 降级期间那两行确实丢了 assert recorder.telemetry_status.dropped_rows == 2 # 降级期间那两行确实丢了
async def test_unparseable_dsn_is_fatal_and_costs_nothing_afterwards(self, captured_warnings): async def test_unparseable_dsn_is_fatal_and_costs_nothing_afterwards(self, captured_logs):
"""DSN 是构造期定死的字符串: 唯一"进程内不可能变好"的东西,故唯一的致命档。""" """DSN 是构造期定死的字符串: 唯一"进程内不可能变好"的东西,故唯一的致命档。
级别与条数一起断言: 同一个事实只该出一条 **ERROR**。recorder 侧曾在 tracker
之外另发一条,于是同一次配置错误刷出 error + warning 两条语义重复的日志,
"级别"这个决策也就有了两个源头(设计 §3.2)。
"""
import asyncpg import asyncpg
clock = _FakeClock() clock = _FakeClock()
@@ -2306,7 +2343,10 @@ class TestPostgresFailureClassification:
status = recorder.telemetry_status status = recorder.telemetry_status
assert status.degraded is True and status.fatal is True assert status.degraded is True and status.fatal is True
assert status.retry_after_s is None # 本进程内不会自愈 assert status.retry_after_s is None # 本进程内不会自愈
assert any("重启" in m for m in captured_warnings) # 恢复条件必须写在日志里 # 按"恢复条件"这个标记捞降级播报本身(丢行复述是另一个事实,不在此列):
# 同一次配置错误只该播报**一条**,且级别是 error
announced = [(level, m) for level, m in captured_logs if "重启" in m]
assert [level for level, _ in announced] == ["ERROR"]
clock.advance(1_000_000.0) clock.advance(1_000_000.0)
attempts = len(pool.acquire_timeouts) attempts = len(pool.acquire_timeouts)