fix: gate cache writes and reads on a shared cacheability verdict

Cacheable is now a strict subset of deliverable: salvaged streams,
content-less results and length/filter/unknown finishes are delivered
but never written, and the read path applies the same verdict to the
stored values before hit overrides (invalid stream_end or non-string
finish read as a miss; 1.3.9 fidelity entries fall back to the
completion's finish). Hits reset stream_end. The legacy
_has_completion_output helper is folded into _is_cacheable.

Red: 13 tests in tests/unit/test_cache.py failed before this change
(uncacheable results were written and replayed; new fields not reset).
This commit is contained in:
2026-10-14 23:39:50 -04:00
parent 39a319e34e
commit 48ba36dd94
4 changed files with 203 additions and 36 deletions
+58 -20
View File
@@ -24,7 +24,8 @@ from polygateway.types import (
OpenAIChatCompletion,
ResponseMode,
ThinkingObservation,
_has_completion_output,
_effective_finish,
_is_cacheable,
)
if TYPE_CHECKING:
@@ -148,6 +149,54 @@ def build_cache_key(
return _KEY_PREFIX + hashlib.sha256(payload.encode("utf-8")).hexdigest()
def _response_cacheable(mode: ResponseMode, response: LLMResponse) -> bool:
"""写路径判据;与读路径共用 `_is_cacheable` 与 `_effective_finish`,口径不分叉。"""
return _is_cacheable(
mode=mode,
content=response.content,
finish_reason=_effective_finish(response.finish_reason, response.completion),
stream_end=response.stream_end,
completion=response.completion,
)
def _restore_fields(raw: str, mode: ResponseMode) -> dict[str, Any]:
"""缓存原文 → LLMResponse 字段;不可缓存或形态坏抛 ValueError(调用方按 miss)。
可缓存判定对**存储原值**进行、先于命中覆盖: 覆盖后 `stream_end` 恒 None,
在那之后判会把 salvaged 条目放行(设计 §6)。
"""
data = json.loads(raw)
fields = {k: v for k, v in data.items() if k in _RESPONSE_FIELDS}
completion = fields.get("completion")
if completion is not None:
if not isinstance(completion, dict):
raise ValueError("缓存 completion 必须为对象")
fields["completion"] = OpenAIChatCompletion(**completion)
finish = _effective_finish(fields.get("finish_reason"), fields.get("completion"))
if finish is not None and not isinstance(finish, str):
raise ValueError("缓存 finish_reason 必须为字符串或 null")
if not _is_cacheable(
mode=mode,
content=fields.get("content", ""),
finish_reason=finish,
stream_end=fields.get("stream_end"),
completion=fields.get("completion"),
):
# 非法 stream_end 也落这里: 它关乎内容完整性,不像 thinking_observation
# 那样可以宽容降级
raise ValueError("缓存条目不可缓存(截断/空正文/非自然结束/非法 stream_end)")
fields["finish_reason"] = finish
# JSON 里存的是 StrEnum 的字符串值,不转就复活成裸 str,与字段注解分叉
# (下游 `is ThinkingObservation.OBSERVED` 会在命中路径上静默为 False);
# 键缺失即升级前写入的旧条目,交给 dataclass 默认值
if "thinking_observation" in fields:
fields["thinking_observation"] = _coerce_observation(fields["thinking_observation"])
if "applied_effort" in fields:
fields["applied_effort"] = _coerce_applied_effort(fields["applied_effort"])
return fields
class CacheMW:
"""洋葱第二层(遥测内、结构化外)。
@@ -195,6 +244,11 @@ class CacheMW:
if hit is not None:
return hit
response = await call_next(request)
if not _response_cacheable(request.response_mode, response):
# 交付照常,只是不进 TTL: 截断/空正文/非自然结束固化后会把一次偶然
# 变成必然(设计 §6)。debug 级: 这是常规分流,不是故障
logger.debug("结果不可缓存(截断/空正文/非自然结束),跳过写入")
return response
try:
serialized = self._serialize(response)
except Exception as exc:
@@ -208,31 +262,15 @@ class CacheMW:
def _rehydrate(self, raw: str, request: ChatRequest) -> LLMResponse | None:
"""反序列化 + 按本次调用的 structured 档零网络重建;失败按未命中。"""
try:
data = json.loads(raw)
fields = {k: v for k, v in data.items() if k in _RESPONSE_FIELDS}
completion = fields.get("completion")
if completion is not None:
if not isinstance(completion, dict):
raise ValueError("缓存 completion 必须为对象")
fields["completion"] = OpenAIChatCompletion(**completion)
if request.response_mode == "openai":
if fields.get("completion") is None:
raise ValueError("保真缓存缺 completion")
if not _has_completion_output(fields.get("content", ""), fields["completion"]):
raise ValueError("保真缓存无有效正文或工具产物")
fields = _restore_fields(raw, request.response_mode)
structured_data = self._rebuild_structured(fields.get("content", ""), request)
# JSON 里存的是 StrEnum 的字符串值,不转就复活成裸 str,与字段注解分叉
# (下游 `is ThinkingObservation.OBSERVED` 会在命中路径上静默为 False);
# 键缺失即升级前写入的旧条目,交给 dataclass 默认值
if "thinking_observation" in fields:
fields["thinking_observation"] = _coerce_observation(fields["thinking_observation"])
if "applied_effort" in fields:
fields["applied_effort"] = _coerce_applied_effort(fields["applied_effort"])
fields.update(
cache_hit=True,
latency_ms=0,
ttft_ms=None,
max_inter_token_ms=None,
# 结束方式描述的是"那次流",命中没有流可言(与 ttft_ms 同口径)
stream_end=None,
call_id=str(uuid.uuid4()),
structured_data=structured_data,
# 显式覆盖: 历史条目里的 `call_stats` 是个 dict,而 `_RESPONSE_FIELDS`
+7 -16
View File
@@ -663,23 +663,14 @@ def _is_cacheable(
if _classify_finish(finish_reason) not in ("natural", "absent"):
return False
if mode == "openai":
return completion is not None and _has_completion_output(content, completion)
return content.strip() != ""
def _has_completion_output(content: str, completion: OpenAIChatCompletion | None) -> bool:
"""成功须有投影正文或完整工具;协议原文为空时不信任缓存旧投影。
只判定产物存在性,不替代协议结构校验,也不收紧快照类型的构造用途。
transport 与缓存共用此规则,避免空响应从缓存旁路成功。
"""
if completion is not None:
if completion.message.get("tool_calls"):
return True
raw_content = completion.message.get("content")
if not isinstance(raw_content, str) or not raw_content.strip():
# 判原文而非投影: 协议原文为空时不信任缓存里的旧投影(1.3.9 同口径)
if completion is None:
return False
return isinstance(content, str) and bool(content.strip())
raw = completion.message.get("content")
return _summarize_message(
completion.message, raw if isinstance(raw, str) else ""
).body_output
return content.strip() != ""
@dataclass(frozen=True)