fix: close the failure modes review found in the new code

Three of them were the same shape as the bug this branch exists to fix:
something goes wrong, the library swallows it, and the caller is left
with a number that means the opposite of what happened.

The throttle key had no source in it. Five sources on one model is the
normal case here, so the first one to break would warn once and silence
the other four for the life of the process, and the message never said
which gateway to look at.

An unknown verdict in a cached entry threw away the whole response. The
rehydrator tolerates unknown fields but not unknown values of a known
field, so two library versions sharing a Redis would each invalidate
the other's entries: halved hit rate, and the only log line says the
cache rebuild failed. A purely observational field should not be able
to void a response whose content is intact.

Normalising for telemetry now degrades instead of raising, both for a
bare string and for a value outside the domain. Either one used to
reach the same except and cost the whole row, which is exactly how
1.3.0 lost nineteen calls without anyone noticing.
This commit is contained in:
2026-08-26 02:37:24 -04:00
parent c0b544d233
commit 1307a02b92
9 changed files with 270 additions and 30 deletions
+25
View File
@@ -413,6 +413,31 @@ class TestOtherProviders:
)
assert len(offs) == len(obs), f"{provider} 关闭方向未满足: {obs}"
async def test_qwen_enabled_is_observed(self):
"""设计 §14 验收: qwen 开启档必须裁定为 `OBSERVED`,不是 `UNKNOWN`。
本条是三态裁定的**跨供应商对照组**: MiniMax 这一路两个信号都可能缺失
(非流式档整片 `UNKNOWN`),若只按它调判据,很容易把"观测不到"当成常态;
qwen 在同一网关同一 key 上照常返回推理信号(findings 2026-08-25 §2),
故这里能且必须要求正面结论——它一旦掉成 `UNKNOWN`,说明的是库的组装路径
丢了信号,而不是上游行为变了。
"""
matrix, provider, model = "L6b", "qwen", "qwen3.7-plus"
desc = f"{provider} enable_thinking=True"
try:
obs = await _run_rounds(_ROUNDS, provider=provider, model=model, enable_thinking=True)
except (AllSourcesExhausted, SourceDeadError, TransientError) as exc:
_skip_if_unreachable(exc, matrix, desc)
ons = [o for o in obs if _reasoning_on(o)]
_record(
matrix,
desc,
"PASS" if len(ons) * 2 > len(obs) else "FAIL",
f"{len(ons)}/{len(obs)} 轮观测到推理(OBSERVED)",
obs,
)
assert len(ons) * 2 > len(obs), f"{provider} 开启方向要求多数轮 OBSERVED: {obs}"
class TestCapabilityDrift:
"""L8 漂移哨兵: 能力表过期是必然事件,这里是它的过期告警。"""
+34 -5
View File
@@ -5,6 +5,7 @@ import hashlib
import json
import pytest
from loguru import logger
from polygateway.backends.memory.cache import InMemoryCache
from polygateway.errors import ResultInvalidError, TransientError
@@ -266,19 +267,47 @@ class TestThinkingObservationRehydration:
assert isinstance(hit.thinking_observation, ThinkingObservation)
assert hit.thinking_observation is ThinkingObservation.OBSERVED
async def test_illegal_value_falls_back_to_source(self):
"""污染值(旧版本写入或人为篡改)按未命中回源,不得复活出域外取值。"""
async def test_unknown_value_degrades_to_unknown_and_still_hits(self):
"""域外取值降级为 UNKNOWN,内容照常复活——不得因此作废整条缓存。
真实场景: 三项目共用一个 Redis,先升级的项目写入了本版没有的第四态,
未升级的两个项目若把它判成未命中,就会在这些 key 上每次真打网关、随后
覆写回旧值,两个版本互相打对方的缓存(表现是命中率莫名腰斩)。一个纯
可观测性字段不该有能力废掉内容完好的缓存响应。
"""
backend = InMemoryCache()
mw = _mw(backend)
key = build_cache_key("m", _MSGS, "proj", None)
poisoned = dataclasses.asdict(_resp(content="poisoned"))
poisoned["thinking_observation"] = "bogus"
poisoned = dataclasses.asdict(_resp(content="from-a-newer-version"))
poisoned["thinking_observation"] = "partially_observed"
poisoned.pop("structured_data", None)
await backend.set(key, json.dumps(poisoned), 3600)
terminal = _Terminal(_resp())
messages: list[str] = []
sink_id = logger.add(messages.append, level="WARNING")
try:
resp = await mw(ChatRequest(messages=_MSGS), terminal)
finally:
logger.remove(sink_id)
assert terminal.calls == 0 and resp.cache_hit is True
assert resp.content == "from-a-newer-version" # 内容完好,照常复活
assert resp.thinking_observation is ThinkingObservation.UNKNOWN
# 单独一条讲清原因的 warning: 通用的"重建失败"没有任何线索指向真因
hits = [m for m in messages if "partially_observed" in m]
assert len(hits) == 1, f"域外取值必须单独告警一次,实得 {len(hits)} 条: {messages}"
assert "thinking_observation" in hits[0]
assert [m for m in messages if "重建失败" in m] == []
async def test_a_broken_payload_still_falls_back_to_source(self):
"""对照组: 内容完整性真被破坏时,仍必须按未命中回源(降级方向不变)。"""
backend = InMemoryCache()
mw = _mw(backend)
key = build_cache_key("m", _MSGS, "proj", None)
await backend.set(key, "{not json at all", 3600)
terminal = _Terminal(_resp())
resp = await mw(ChatRequest(messages=_MSGS), terminal)
assert terminal.calls == 1 and resp.cache_hit is False
assert resp.content == "cached" # 回源结果,不是被污染的那条
assert resp.content == "cached"
async def test_legacy_entry_without_key_rehydrates_to_default(self):
"""升级前写入的条目没有该键,必须照常复活并落到默认 UNKNOWN。"""
+36 -5
View File
@@ -547,10 +547,11 @@ class TestThinkingObservationVerdict:
class TestThinkingReconciliation:
"""对账告警按 (model, direction) 节流(设计 §5)。
"""对账告警按 (source, model, direction) 节流(设计 §5)。
节流键必须含方向: 同一模型的开、关两档是两个独立的矛盾,合并键会让先出现
的那一档把另一档永久静音。
键的三段缺一不可,理由同源: 合并任意一段,都会让先出现的那一组把另一组
永久静音——同一模型的开/关两档是两个独立的矛盾,同一模型的两个源背后是
两个独立的账号/网关。
"""
def _handler(self, request):
@@ -563,9 +564,9 @@ class TestThinkingReconciliation:
# 开启档却零信号 → UNKNOWN,无法确认是否生效(M3 实测形态)
return _sse_stream(_chunk(content="ok"), _chunk(usage=_USAGE))
def _minimax(self, enable_thinking):
def _minimax(self, enable_thinking, name="mm"):
return _source(
name="mm", provider="minimax", model="MiniMax-M3", enable_thinking=enable_thinking
name=name, provider="minimax", model="MiniMax-M3", enable_thinking=enable_thinking
)
async def test_same_model_and_direction_warns_only_once(self):
@@ -581,6 +582,36 @@ class TestThinkingReconciliation:
hits = [m for m in messages if "MiniMax-M3" in m]
assert len(hits) == 1, f"同一 (model, direction) 应只告警一次,实得 {len(hits)}"
async def test_each_source_gets_its_own_warning(self):
"""多源多账号是本库的核心场景: 同一 model 跨 N 个源不得只喊第一个。
节流键漏掉源标识时,5 个共用同一模型的源里第一个出问题的喊完一次,其余
四个**永久静音**——而每个源背后是独立的账号/网关,它们的行为互不代表。
"""
transport = _transport_for(self._handler)
messages: list[str] = []
sink_id = logger.add(messages.append, level="WARNING")
try:
await _complete(transport, self._minimax(False, name="gw-a"))
await _complete(transport, self._minimax(False, name="gw-b"))
finally:
logger.remove(sink_id)
hits = [m for m in messages if "MiniMax-M3" in m]
assert len(hits) == 2, f"两个源各应告警一次,实得 {len(hits)}"
async def test_the_warning_names_the_source(self):
"""拿到告警的人得知道该查哪个网关: 只报模型名定位不到源。"""
transport = _transport_for(self._handler)
messages: list[str] = []
sink_id = logger.add(messages.append, level="WARNING")
try:
await _complete(transport, self._minimax(False, name="gw-a"))
finally:
logger.remove(sink_id)
hits = [m for m in messages if "MiniMax-M3" in m]
assert len(hits) == 1
assert "gw-a" in hits[0], f"告警未点名出问题的源: {hits[0]}"
async def test_switching_direction_earns_a_second_warning(self):
transport = _transport_for(self._handler)
messages: list[str] = []
+52
View File
@@ -9,6 +9,7 @@ import subprocess
from pathlib import Path
import pytest
from loguru import logger
from polygateway.backends.memory.breaker import InMemoryGate
from polygateway.backends.memory.limiter import InMemoryLimiter
@@ -1169,6 +1170,57 @@ class TestEmitterThinkingObservation:
assert value == "observed"
assert type(value) is str # 不是 ThinkingObservation: 子类实例不得下沉到 recorder
async def test_a_bare_string_verdict_still_lands(self):
"""下游填裸 str 时**整行**不得丢失(遥测必录)。
`LLMResponse` 是无运行时校验的 frozen dataclass,写
`LLMResponse(..., thinking_observation="observed")` 完全自然且 `==` 比较
照常成立;若 emitter 直接取 `.value`,这里会抛 `AttributeError` 并被
`_record` 的 `except Exception` 吞成一条泛化 warning——丢的不是这一列,
是整行,正是 1.3.0 那次"19 次调用一行未落"的同款形态。
"""
rec = _MemoryRecorder()
await TelemetryEmitter(rec, text_cap=None).emit_attempt(
request=_REQ,
source=_source(),
call_id="c",
latency_ms=1,
response=_resp(thinking_observation="observed"),
error=None,
)
assert len(rec.rows) == 1, "整行被吞了"
value = rec.rows[0]["thinking_observation"]
assert value == "observed"
assert type(value) is str
async def test_an_out_of_domain_verdict_degrades_but_keeps_the_row(self):
"""域外取值挡在落库前,但**降级不丢行**: 列的取值域由库守,代价不是整行。
直接 `ThinkingObservation(x).value` 会在这里抛 `ValueError`,同样被
`_record` 的 `except Exception` 吞成丢整行——那只修好了裸 str 一半,
口误值(大小写不符、拼错)对测试替身同样自然。故降级为 `unknown`
(对库而言本次确实判不出来)并单独告警,与缓存回放的方向选择一致。
"""
rec = _MemoryRecorder()
messages: list[str] = []
sink_id = logger.add(messages.append, level="WARNING")
try:
await TelemetryEmitter(rec, text_cap=None).emit_attempt(
request=_REQ,
source=_source(),
call_id="c",
latency_ms=1,
response=_resp(thinking_observation="OBSERVED"), # 大小写不符即域外
error=None,
)
finally:
logger.remove(sink_id)
assert len(rec.rows) == 1, "整行被吞了"
assert rec.rows[0]["thinking_observation"] == "unknown"
hits = [m for m in messages if "OBSERVED" in m]
assert len(hits) == 1, f"域外取值必须单独告警: {messages}"
assert [m for m in messages if "遥测记录失败" in m] == []
async def test_cache_hit_replays_the_recorded_verdict(self):
"""缓存命中回放历史那次的裁定: 与 model/prompt_tokens 同一口径。"""
rec = _MemoryRecorder()
+35 -1
View File
@@ -66,6 +66,18 @@ class TestObserveThinking:
observe_thinking(thinking="想了想", reasoning_tokens=0) is ThinkingObservation.OBSERVED
)
@pytest.mark.parametrize("negative", [-1, -205])
def test_negative_token_count_is_not_evidence_of_absence(self, negative):
"""负数是坏数据,不是"上游明确上报未推理"这个最强的正面结论。
当前 transport 已在边界把负数归 `None`,所以这条走不通;但本函数的
docstring 自称"外部输入校验后使用",第二个 transport 直接填该值时,
`> 0 else ABSENT` 会给出一个方向相反的强结论。函数自身必须闭合(P5)。
"""
assert observe_thinking(thinking="", reasoning_tokens=negative) is (
ThinkingObservation.UNKNOWN
)
class TestThinkingObservationEnum:
def test_values_are_stable_strings(self):
@@ -85,7 +97,12 @@ class TestThinkingObservationEnum:
@pytest.mark.parametrize("bogus", ["", "OBSERVED", "yes", "none"])
def test_unknown_strings_are_rejected(bogus):
"""非法值必须抛 ValueError: 缓存回放靠它把污染数据挡成"未命中"(设计 §6)。"""
"""非法值必须抛 ValueError: 缓存回放与遥测归一化都靠它识别域外取值(设计 §6)。
两处接住这个 ValueError 后**降级而非作废**(缓存复活内容 + 记 UNKNOWN、遥测
照常落行),但降级的前提是构造器真的会拒绝——它一旦放行,域外取值就会一路
进到 `LLMResponse` 与遥测列里。
"""
with pytest.raises(ValueError):
ThinkingObservation(bogus)
@@ -239,6 +256,23 @@ class TestReconcileThinking:
assert msg is not None
assert "MiniMax-M3" in msg
def test_off_and_absent_stays_silent(self):
"""要求关闭 + 上游明确上报未推理 = 要求被满足,没有可报的矛盾。
这一格与 `test_off_and_unknown_stays_silent` 的沉默理由**不同**: 那里是
"没有证伪力",这里是"正面证实要求已满足"。两者都必须沉默,漏测哪一格,
把 Phase 2 的判据写成 `is ABSENT` 之类的反向条件都不会被抓住。
"""
assert (
reconcile_thinking(
enable_thinking=False,
observation=ThinkingObservation.ABSENT,
capability=self._CAP,
model="qwen3.7-plus",
)
is None
)
def test_off_and_unknown_stays_silent(self):
"""UNKNOWN 没有证伪力: 拿它报警等于每次关闭调用都喊(M3 关闭档恒落此档)。"""
assert (