From 33c8e8274b8d0ff447e82ca6e2d969c8894a8b83 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Sat, 5 Sep 2026 02:39:38 -0400 Subject: [PATCH] fix: keep a low-tier answer out of the cache slot a max-tier one filled The per-call reasoning tier never reached the cache key, and the model fingerprint could not stand in for it: the fingerprint is computed once at assembly time, so two calls on the same client asking for low and max looked identical to it. Same messages, different tiers, one shared entry -- the verbatim replay of issue #4's five seeds all hitting the same response. Source-level tiers join the fingerprint under the same rule enable_thinking already follows (appended only when the source takes a position), and the filter that decides which sources enter the mark set is widened to match -- without that, a source configured with nothing but REASONING_EFFORT would never reach _fingerprint_mark at all. None (no opinion) and Effort.NONE (asked not to reason) stay distinct keys. Sources that opine on neither keep byte-identical keys and fingerprints, so nothing existing cold-starts. --- src/polygateway/client.py | 32 ++++++++++++---- src/polygateway/middleware/cache.py | 16 +++++++- tests/unit/test_cache.py | 57 ++++++++++++++++++++++++++++- tests/unit/test_client.py | 42 +++++++++++++++++++++ 4 files changed, 137 insertions(+), 10 deletions(-) diff --git a/src/polygateway/client.py b/src/polygateway/client.py index 3b98800..31f77d0 100644 --- a/src/polygateway/client.py +++ b/src/polygateway/client.py @@ -99,14 +99,21 @@ def _guard_thinking( def _fingerprint_mark(source: SourceConfig) -> str: - """单源的指纹标记;`enable_thinking` 仅在**表态时**追加。 + """单源的指纹标记;`enable_thinking` 与 `reasoning_effort` 仅在**表态时**追加。 只在表态时追加不是省事: 这样只配了 `extra_body` 的存量源字面量与 issue #4 时期逐字相同,升级本版本不会给它们平白来一次全量缓存冷启动。 + + `reasoning_effort`(issue #20)与 `enable_thinking` 同规则、同理由: 它一旦真正 + 改变请求体,"把源级档位从 low 改成 max 后重启"就会读到 low 档时缓存的旧响应。 + 两者取值域不相交(`"none"`/`"low"`… vs `true`/`false`),故追加进同一个列表也 + 不会把两种写法摘要成同一身份。 """ parts: list[Any] = [source.model, dict(source.extra_body)] if source.enable_thinking is not None: parts.append(source.enable_thinking) + if source.reasoning_effort is not None: + parts.append(source.reasoning_effort) return json.dumps(parts, sort_keys=True, ensure_ascii=False) @@ -114,16 +121,25 @@ def build_model_fingerprint(sources: Iterable[SourceConfig]) -> str: """缓存 key 的模型身份: 多源 scope = 排序去重的 model 合集。 配置级采样参数(`extra_body`)必须参与,否则把 temperature 从 0 改成 1 - 后重启仍会读到旧缓存(issue #4 设计决策 C)。`enable_thinking` 同理 - (issue #5): 它一旦真正改变请求体,"关掉推理后重启"就会读到开着推理时 - 缓存的旧响应。全源两者皆未表态时字面量与历史实现逐字相同,不触发存量 - 缓存冷启动。 + 后重启仍会读到旧缓存(issue #4 设计决策 C)。`enable_thinking`(issue #5)与 + 源级 `reasoning_effort`(issue #20)同理: 它们一旦真正改变请求体,"关掉推理后 + 重启"就会读到开着推理时缓存的旧响应。全源三者皆未表态时字面量与历史实现逐字 + 相同,不触发存量缓存冷启动。 + + 注意本指纹是**装配期**算出的**集合级**身份,覆盖不到逐次调用变化的请求级档位 + ——后者由 `build_cache_key` 的 `reasoning_effort` 参数单独承担(ARCH §7.5)。 """ fingerprint = ",".join(sorted({s.model for s in sources})) - # 按 (model, extra_body[, enable_thinking]) 而非源名摘要: 语义是"本 scope - # 会用哪些(模型, 请求形态)组合",改源名不该误触全量冷启动 + # 按 (model, extra_body[, enable_thinking][, reasoning_effort]) 而非源名摘要: + # 语义是"本 scope 会用哪些(模型, 请求形态)组合",改源名不该误触全量冷启动。 + # 过滤条件必须与 `_fingerprint_mark` 追加的字段逐项对齐: 漏掉一项,只配了该项 + # 的源根本进不了 marks,`_fingerprint_mark` 改了也白改 marks = sorted( - {_fingerprint_mark(s) for s in sources if s.extra_body or s.enable_thinking is not None} + { + _fingerprint_mark(s) + for s in sources + if s.extra_body or s.enable_thinking is not None or s.reasoning_effort is not None + } ) if marks: digest = hashlib.sha256("".join(marks).encode("utf-8")).hexdigest() diff --git a/src/polygateway/middleware/cache.py b/src/polygateway/middleware/cache.py index 2e386bf..bc9cb3e 100644 --- a/src/polygateway/middleware/cache.py +++ b/src/polygateway/middleware/cache.py @@ -17,7 +17,7 @@ from typing import TYPE_CHECKING, Any from loguru import logger -from polygateway.types import ChatRequest, LLMResponse, ThinkingObservation +from polygateway.types import ChatRequest, Effort, LLMResponse, ThinkingObservation if TYPE_CHECKING: from collections.abc import Mapping @@ -83,12 +83,21 @@ def build_cache_key( salt: str | None, *, sampling: Mapping[str, Any] | None = None, + reasoning_effort: Effort | None = None, ) -> str: """缓存 key 公式;salt 仅非 None 时参与(VT 旧键语义: 不传 salt 键形不变)。 `sampling` 仅**非空**时参与(与 salt 的"仅非 None"不同——空串是有意义的 salt,而空采样参数与不传无语义差别)。它必须进 key: 否则同 messages 跑 5 个 seed 会全部命中第一次的响应,标准差恒为 0 且不报错(issue #4 决策 C)。 + + `reasoning_effort` 是**请求级**档位(issue #20),仅非 `None` 时参与。它不能靠 + `model_fingerprint` 代劳: 后者是**装配期**算出的集合级指纹,一次调用改档位不会 + 让它变一个字节;不进 key 则同 messages 跑 low 与 max 互相命中,是 issue #4 + 「5 个 seed 全命中同一响应」的逐字翻版。 + + 判据用 `is not None` 而非真值: `Effort.NONE`(明确要求不推理)与 `None` + (不表态)语义不同——前者拿到的是没有推理过程的响应,合并即毒化。 """ key_obj: dict[str, Any] = { "model": model_fingerprint, @@ -99,6 +108,8 @@ def build_cache_key( key_obj["salt"] = salt if sampling: key_obj["sampling"] = dict(sampling) + if reasoning_effort is not None: + key_obj["reasoning_effort"] = str(reasoning_effort) payload = json.dumps(key_obj, sort_keys=True, ensure_ascii=False) return _KEY_PREFIX + hashlib.sha256(payload.encode("utf-8")).hexdigest() @@ -139,6 +150,9 @@ class CacheMW: namespace, request.cache_salt, sampling=request.sampling, + # 请求级档位必须逐次进 key: `self._fingerprint` 是装配期的集合级指纹, + # 同一个 client 上 low 与 max 两次调用在它眼里毫无分别(issue #20) + reasoning_effort=request.reasoning_effort, ) cached = await self._safe_get(key) if cached is not None: diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py index 290ba64..6f4ea19 100644 --- a/tests/unit/test_cache.py +++ b/tests/unit/test_cache.py @@ -11,7 +11,13 @@ from polygateway.backends.memory.cache import InMemoryCache from polygateway.errors import ResultInvalidError, TransientError from polygateway.middleware.cache import CacheMW, build_cache_key, digest_messages from polygateway.middleware.telemetry import TelemetryEmitter -from polygateway.types import ChatRequest, LLMResponse, SourceConfig, ThinkingObservation +from polygateway.types import ( + ChatRequest, + Effort, + LLMResponse, + SourceConfig, + ThinkingObservation, +) _MSGS = [{"role": "user", "content": "hi"}] @@ -114,6 +120,40 @@ class TestKeyFormula: "m", messages2, "p", None ) + def test_request_tier_changes_key(self): + """同 messages 跑 low 与 max 不得互相命中(issue #20;issue #4 的逐字翻版)。 + + 请求级档位必须**独立于** `model_fingerprint` 进 key: 后者是装配期算出的 + 集合级指纹,一次调用改档位不会让它变一个字节。 + """ + k_low = build_cache_key("m", _MSGS, "proj", None, reasoning_effort=Effort.LOW) + k_max = build_cache_key("m", _MSGS, "proj", None, reasoning_effort=Effort.MAX) + assert k_low != k_max + + def test_explicit_none_tier_is_not_the_absent_tier(self): + """`None`(不表态)与 `Effort.NONE`(要求不推理)是两个 key。 + + 二者合并即毒化: "没写档位"的调用会读到"明确关掉推理"那次的响应, + 而后者的内容恰恰是缺推理过程的。 + """ + assert build_cache_key("m", _MSGS, "proj", None) != build_cache_key( + "m", _MSGS, "proj", None, reasoning_effort=Effort.NONE + ) + + def test_absent_tier_keeps_legacy_key(self): + """不表态档位时键形逐字不变,存量缓存不被本次升级全量作废。 + + golden 值与 `test_empty_sampling_keeps_legacy_key` 同源,取自加 + `reasoning_effort` 维度之前的实现,不得随实现漂移。 + """ + assert build_cache_key( + "qwen-max", + [{"role": "user", "content": "hi"}], + "proj", + None, + reasoning_effort=None, + ) == ("pgw:cache:c54544e8672f4c91373b4a72716a88497445b440b89445aa5379b356b228f58b") + class _Terminal: def __init__(self, response): @@ -163,6 +203,21 @@ class TestCacheFlow: third = await mw(ChatRequest(messages=_MSGS, sampling={"seed": 1}), terminal) assert third.cache_hit is True and terminal.calls == 2 + async def test_differing_reasoning_effort_does_not_hit(self): + """接线门: `CacheMW` 必须把 `request.reasoning_effort` 传进 key 公式。 + + 只测 `build_cache_key` 不够——参数加了却没人传是本改动最可能的落地方式, + 那种缺口在公式层的用例里完全看不见。 + """ + backend = InMemoryCache() + mw = _mw(backend) + terminal = _Terminal(_resp()) + await mw(ChatRequest(messages=_MSGS, reasoning_effort=Effort.LOW), terminal) + await mw(ChatRequest(messages=_MSGS, reasoning_effort=Effort.MAX), terminal) + assert terminal.calls == 2 # 两档各自回源 + third = await mw(ChatRequest(messages=_MSGS, reasoning_effort=Effort.LOW), terminal) + assert third.cache_hit is True and terminal.calls == 2 # 同档才命中 + async def test_structured_injection_does_not_pollute_key(self): """CacheMW 读 sampling 而非 overlay: 结构化注入不该改变缓存身份。""" backend = InMemoryCache() diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 4b3482e..451e594 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -401,6 +401,48 @@ class TestModelFingerprint: b = build_model_fingerprint([_source(extra_body={"temperature": 1})]) assert a != b + def test_source_tier_enters_fingerprint(self): + """源级 `reasoning_effort` 改变请求体,就必须改变缓存身份(与 issue #5 同理)。 + + 本用例同时守着一个易漏点: 只配 `REASONING_EFFORT`、既无 `extra_body` 也无 + `ENABLE_THINKING` 的源,必须能进入指纹的 marks 集合——否则 `_fingerprint_mark` + 改了也白改,四个指纹会全部相等。 + """ + from polygateway.client import build_model_fingerprint + + plain = build_model_fingerprint([_source()]) + low = build_model_fingerprint([_source(reasoning_effort=Effort.LOW)]) + max_ = build_model_fingerprint([_source(reasoning_effort=Effort.MAX)]) + off = build_model_fingerprint([_source(reasoning_effort=Effort.NONE)]) + assert len({plain, low, max_, off}) == 4 + + def test_source_tier_is_distinguished_from_the_thinking_sugar(self): + """`reasoning_effort=NONE` 与 `enable_thinking=False` 不得摘要成同一个指纹。 + + 两者语义等价但取值不同(`"none"` vs `false`),让它们撞车会把"两种写法" + 变成"一种缓存身份",日后任一侧语义微调都会静默复用另一侧的响应。 + """ + from polygateway.client import build_model_fingerprint + + by_tier = build_model_fingerprint([_source(reasoning_effort=Effort.NONE)]) + by_sugar = build_model_fingerprint([_source(enable_thinking=False)]) + assert by_tier != by_sugar + + def test_absent_tier_fingerprint_is_byte_identical_to_before(self): + """不表态档位的存量源不得因本次升级平白冷启动: 字面量逐字相同。 + + 两条: 纯净源仍是裸 model 合集;只配 extra_body 的源仍是升级前那个摘要。 + """ + import hashlib + import json + + from polygateway.client import build_model_fingerprint + + assert build_model_fingerprint([_source()]) == "qwen-max" + mark = json.dumps(["qwen-max", {"temperature": 0}], sort_keys=True, ensure_ascii=False) + expected = "qwen-max|" + hashlib.sha256(mark.encode("utf-8")).hexdigest() + assert build_model_fingerprint([_source(extra_body={"temperature": 0})]) == expected + class TestFactories: def test_from_env_assembles(self):