From c31cc1adad909581517ac738302b75c2d7cae1f7 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 31 Jul 2026 21:20:56 -0400 Subject: [PATCH] feat: fold sampling parameters into the cache key Without this, five seeds over identical messages all hit the first cached response and the reported standard deviation is silently always zero. --- src/polygateway/middleware/cache.py | 28 +++++++++++++-- tests/unit/test_cache.py | 55 +++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/src/polygateway/middleware/cache.py b/src/polygateway/middleware/cache.py index 252474c..88a63a7 100644 --- a/src/polygateway/middleware/cache.py +++ b/src/polygateway/middleware/cache.py @@ -20,6 +20,8 @@ from loguru import logger from polygateway.types import ChatRequest, LLMResponse if TYPE_CHECKING: + from collections.abc import Mapping + from polygateway.ports import CacheBackend, CallNext, StructuredOutputStrategy _KEY_PREFIX = "pgw:cache:" @@ -50,9 +52,19 @@ def _digest_part(part: Any) -> Any: def build_cache_key( - model_fingerprint: str, messages: list[dict[str, Any]], namespace: str, salt: str | None + model_fingerprint: str, + messages: list[dict[str, Any]], + namespace: str, + salt: str | None, + *, + sampling: Mapping[str, Any] | None = None, ) -> str: - """缓存 key 公式;salt 仅非 None 时参与(VT 旧键语义: 不传 salt 键形不变)。""" + """缓存 key 公式;salt 仅非 None 时参与(VT 旧键语义: 不传 salt 键形不变)。 + + `sampling` 仅**非空**时参与(与 salt 的"仅非 None"不同——空串是有意义的 + salt,而空采样参数与不传无语义差别)。它必须进 key: 否则同 messages 跑 5 个 + seed 会全部命中第一次的响应,标准差恒为 0 且不报错(issue #4 决策 C)。 + """ key_obj: dict[str, Any] = { "model": model_fingerprint, "messages": digest_messages(messages), @@ -60,6 +72,8 @@ def build_cache_key( } if salt is not None: key_obj["salt"] = salt + if sampling: + key_obj["sampling"] = dict(sampling) payload = json.dumps(key_obj, sort_keys=True, ensure_ascii=False) return _KEY_PREFIX + hashlib.sha256(payload.encode("utf-8")).hexdigest() @@ -92,7 +106,15 @@ class CacheMW: async def __call__(self, request: ChatRequest, call_next: CallNext) -> LLMResponse: namespace = request.cache_namespace or self._namespace - key = build_cache_key(self._fingerprint, request.messages, namespace, request.cache_salt) + # 读 sampling 而非 overlay: 语义明确,且不依赖"CacheMW 恰在 StructuredMW + # 外侧"这一层序巧合——结构化注入不该改变缓存身份(设计决策 C) + key = build_cache_key( + self._fingerprint, + request.messages, + namespace, + request.cache_salt, + sampling=request.sampling, + ) cached = await self._safe_get(key) if cached is not None: hit = self._rehydrate(cached, request) diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py index 59d7c8a..c23340c 100644 --- a/tests/unit/test_cache.py +++ b/tests/unit/test_cache.py @@ -53,6 +53,35 @@ class TestKeyFormula: def test_any_dimension_change_changes_key(self, a, b): assert build_cache_key(*a) != build_cache_key(*b) + def test_empty_sampling_keeps_legacy_key(self): + """空采样参数时键形逐字不变,存量缓存不被全量作废(issue #4 决策 C)。 + + golden 值取自加 sampling 维度之前的实现,不得随实现漂移。 + """ + assert build_cache_key("qwen-max", [{"role": "user", "content": "hi"}], "proj", None) == ( + "pgw:cache:c54544e8672f4c91373b4a72716a88497445b440b89445aa5379b356b228f58b" + ) + assert build_cache_key("qwen-max", [{"role": "user", "content": "hi"}], "proj", "s1") == ( + "pgw:cache:eed9cd9cc06acc0dedf4f337b74e06ed3482afdc30fa2acedd194f6cc1df33bf" + ) + + def test_differing_seed_changes_key(self): + """issue #4 的直接回归: 5 个 seed 若共用一个 key,标准差会恒为 0。""" + k1 = build_cache_key("m", _MSGS, "proj", None, sampling={"seed": 1}) + k2 = build_cache_key("m", _MSGS, "proj", None, sampling={"seed": 2}) + assert k1 != k2 + + def test_sampling_key_order_irrelevant(self): + k1 = build_cache_key("m", _MSGS, "proj", None, sampling={"seed": 1, "temperature": 0}) + k2 = build_cache_key("m", _MSGS, "proj", None, sampling={"temperature": 0, "seed": 1}) + assert k1 == k2 + + def test_empty_sampling_equals_omitted(self): + """空 dict 与不传须同键,否则升级后存量缓存全部 miss。""" + assert build_cache_key("m", _MSGS, "proj", None, sampling={}) == build_cache_key( + "m", _MSGS, "proj", None + ) + def test_multimodal_part_digested_not_inlined(self): big_b64 = "data:image/png;base64," + "A" * 1_000_000 messages = [ @@ -120,6 +149,32 @@ class TestCacheFlow: assert second.call_id != first.call_id # 命中生成独立 cache_call_id assert terminal.calls == 1 # 未再触达内层 + async def test_differing_sampling_does_not_hit(self): + """issue #4 的中间件层回归: 逐 rollout 变 seed 必须回源,不得复用响应。""" + backend = InMemoryCache() + mw = _mw(backend) + terminal = _Terminal(_resp()) + await mw(ChatRequest(messages=_MSGS, sampling={"seed": 1}), terminal) + await mw(ChatRequest(messages=_MSGS, sampling={"seed": 2}), terminal) + assert terminal.calls == 2 # 两次都回源 + # 同 seed 才命中 + third = await mw(ChatRequest(messages=_MSGS, sampling={"seed": 1}), terminal) + assert third.cache_hit is True and terminal.calls == 2 + + async def test_structured_injection_does_not_pollute_key(self): + """CacheMW 读 sampling 而非 overlay: 结构化注入不该改变缓存身份。""" + backend = InMemoryCache() + mw = _mw(backend) + terminal = _Terminal(_resp()) + await mw(ChatRequest(messages=_MSGS, sampling={"seed": 1}), terminal) + polluted = ChatRequest( + messages=_MSGS, + sampling={"seed": 1}, + overlay={"seed": 1, "response_format": {"type": "json_object"}}, + ) + assert (await mw(polluted, terminal)).cache_hit is True + assert terminal.calls == 1 + async def test_per_call_namespace_overrides_default(self): backend = InMemoryCache() mw = _mw(backend)