fix: gate cache writes and reads on a shared cacheability verdict
Cacheable is now a strict subset of deliverable: salvaged streams, content-less results and length/filter/unknown finishes are delivered but never written, and the read path applies the same verdict to the stored values before hit overrides (invalid stream_end or non-string finish read as a miss; 1.3.9 fidelity entries fall back to the completion's finish). Hits reset stream_end. The legacy _has_completion_output helper is folded into _is_cacheable. Red: 13 tests in tests/unit/test_cache.py failed before this change (uncacheable results were written and replayed; new fields not reset).
This commit is contained in:
@@ -24,7 +24,8 @@ from polygateway.types import (
|
||||
OpenAIChatCompletion,
|
||||
ResponseMode,
|
||||
ThinkingObservation,
|
||||
_has_completion_output,
|
||||
_effective_finish,
|
||||
_is_cacheable,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -148,6 +149,54 @@ def build_cache_key(
|
||||
return _KEY_PREFIX + hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _response_cacheable(mode: ResponseMode, response: LLMResponse) -> bool:
|
||||
"""写路径判据;与读路径共用 `_is_cacheable` 与 `_effective_finish`,口径不分叉。"""
|
||||
return _is_cacheable(
|
||||
mode=mode,
|
||||
content=response.content,
|
||||
finish_reason=_effective_finish(response.finish_reason, response.completion),
|
||||
stream_end=response.stream_end,
|
||||
completion=response.completion,
|
||||
)
|
||||
|
||||
|
||||
def _restore_fields(raw: str, mode: ResponseMode) -> dict[str, Any]:
|
||||
"""缓存原文 → LLMResponse 字段;不可缓存或形态坏抛 ValueError(调用方按 miss)。
|
||||
|
||||
可缓存判定对**存储原值**进行、先于命中覆盖: 覆盖后 `stream_end` 恒 None,
|
||||
在那之后判会把 salvaged 条目放行(设计 §6)。
|
||||
"""
|
||||
data = json.loads(raw)
|
||||
fields = {k: v for k, v in data.items() if k in _RESPONSE_FIELDS}
|
||||
completion = fields.get("completion")
|
||||
if completion is not None:
|
||||
if not isinstance(completion, dict):
|
||||
raise ValueError("缓存 completion 必须为对象")
|
||||
fields["completion"] = OpenAIChatCompletion(**completion)
|
||||
finish = _effective_finish(fields.get("finish_reason"), fields.get("completion"))
|
||||
if finish is not None and not isinstance(finish, str):
|
||||
raise ValueError("缓存 finish_reason 必须为字符串或 null")
|
||||
if not _is_cacheable(
|
||||
mode=mode,
|
||||
content=fields.get("content", ""),
|
||||
finish_reason=finish,
|
||||
stream_end=fields.get("stream_end"),
|
||||
completion=fields.get("completion"),
|
||||
):
|
||||
# 非法 stream_end 也落这里: 它关乎内容完整性,不像 thinking_observation
|
||||
# 那样可以宽容降级
|
||||
raise ValueError("缓存条目不可缓存(截断/空正文/非自然结束/非法 stream_end)")
|
||||
fields["finish_reason"] = finish
|
||||
# JSON 里存的是 StrEnum 的字符串值,不转就复活成裸 str,与字段注解分叉
|
||||
# (下游 `is ThinkingObservation.OBSERVED` 会在命中路径上静默为 False);
|
||||
# 键缺失即升级前写入的旧条目,交给 dataclass 默认值
|
||||
if "thinking_observation" in fields:
|
||||
fields["thinking_observation"] = _coerce_observation(fields["thinking_observation"])
|
||||
if "applied_effort" in fields:
|
||||
fields["applied_effort"] = _coerce_applied_effort(fields["applied_effort"])
|
||||
return fields
|
||||
|
||||
|
||||
class CacheMW:
|
||||
"""洋葱第二层(遥测内、结构化外)。
|
||||
|
||||
@@ -195,6 +244,11 @@ class CacheMW:
|
||||
if hit is not None:
|
||||
return hit
|
||||
response = await call_next(request)
|
||||
if not _response_cacheable(request.response_mode, response):
|
||||
# 交付照常,只是不进 TTL: 截断/空正文/非自然结束固化后会把一次偶然
|
||||
# 变成必然(设计 §6)。debug 级: 这是常规分流,不是故障
|
||||
logger.debug("结果不可缓存(截断/空正文/非自然结束),跳过写入")
|
||||
return response
|
||||
try:
|
||||
serialized = self._serialize(response)
|
||||
except Exception as exc:
|
||||
@@ -208,31 +262,15 @@ class CacheMW:
|
||||
def _rehydrate(self, raw: str, request: ChatRequest) -> LLMResponse | None:
|
||||
"""反序列化 + 按本次调用的 structured 档零网络重建;失败按未命中。"""
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
fields = {k: v for k, v in data.items() if k in _RESPONSE_FIELDS}
|
||||
completion = fields.get("completion")
|
||||
if completion is not None:
|
||||
if not isinstance(completion, dict):
|
||||
raise ValueError("缓存 completion 必须为对象")
|
||||
fields["completion"] = OpenAIChatCompletion(**completion)
|
||||
if request.response_mode == "openai":
|
||||
if fields.get("completion") is None:
|
||||
raise ValueError("保真缓存缺 completion")
|
||||
if not _has_completion_output(fields.get("content", ""), fields["completion"]):
|
||||
raise ValueError("保真缓存无有效正文或工具产物")
|
||||
fields = _restore_fields(raw, request.response_mode)
|
||||
structured_data = self._rebuild_structured(fields.get("content", ""), request)
|
||||
# JSON 里存的是 StrEnum 的字符串值,不转就复活成裸 str,与字段注解分叉
|
||||
# (下游 `is ThinkingObservation.OBSERVED` 会在命中路径上静默为 False);
|
||||
# 键缺失即升级前写入的旧条目,交给 dataclass 默认值
|
||||
if "thinking_observation" in fields:
|
||||
fields["thinking_observation"] = _coerce_observation(fields["thinking_observation"])
|
||||
if "applied_effort" in fields:
|
||||
fields["applied_effort"] = _coerce_applied_effort(fields["applied_effort"])
|
||||
fields.update(
|
||||
cache_hit=True,
|
||||
latency_ms=0,
|
||||
ttft_ms=None,
|
||||
max_inter_token_ms=None,
|
||||
# 结束方式描述的是"那次流",命中没有流可言(与 ttft_ms 同口径)
|
||||
stream_end=None,
|
||||
call_id=str(uuid.uuid4()),
|
||||
structured_data=structured_data,
|
||||
# 显式覆盖: 历史条目里的 `call_stats` 是个 dict,而 `_RESPONSE_FIELDS`
|
||||
|
||||
@@ -663,23 +663,14 @@ def _is_cacheable(
|
||||
if _classify_finish(finish_reason) not in ("natural", "absent"):
|
||||
return False
|
||||
if mode == "openai":
|
||||
return completion is not None and _has_completion_output(content, completion)
|
||||
return content.strip() != ""
|
||||
|
||||
|
||||
def _has_completion_output(content: str, completion: OpenAIChatCompletion | None) -> bool:
|
||||
"""成功须有投影正文或完整工具;协议原文为空时不信任缓存旧投影。
|
||||
|
||||
只判定产物存在性,不替代协议结构校验,也不收紧快照类型的构造用途。
|
||||
transport 与缓存共用此规则,避免空响应从缓存旁路成功。
|
||||
"""
|
||||
if completion is not None:
|
||||
if completion.message.get("tool_calls"):
|
||||
return True
|
||||
raw_content = completion.message.get("content")
|
||||
if not isinstance(raw_content, str) or not raw_content.strip():
|
||||
# 判原文而非投影: 协议原文为空时不信任缓存里的旧投影(1.3.9 同口径)
|
||||
if completion is None:
|
||||
return False
|
||||
return isinstance(content, str) and bool(content.strip())
|
||||
raw = completion.message.get("content")
|
||||
return _summarize_message(
|
||||
completion.message, raw if isinstance(raw, str) else ""
|
||||
).body_output
|
||||
return content.strip() != ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
Reference in New Issue
Block a user