feat: judge whether reasoning actually happened from multiple signals

reasoning_tokens=None has been carrying two meanings at once, no
reasoning and no report, and the library resolved the ambiguity by
quietly claiming the first. ThinkingObservation splits them: UNKNOWN
says the call left no signal, ABSENT says the provider reported zero.

The verdict ranks evidence by hardness. Reasoning prose is the fact
itself; reasoning_tokens is a report about the fact, so a missing
report cannot overrule prose that is right there. The prose check
strips first, since a gateway that returns whitespace is not evidence.

The enum lives in types.py, not in the new thinking.py, because
LLMResponse is typed on it and the innermost layer must not import a
decision module.
This commit is contained in:
2026-08-25 23:40:39 -04:00
parent 85bcc23a6b
commit e90bb3d6a4
5 changed files with 141 additions and 0 deletions
+1
View File
@@ -81,6 +81,7 @@ layers = [
"polygateway.config", "polygateway.config",
"polygateway.middleware", "polygateway.middleware",
"polygateway.transports | polygateway.backends | polygateway.telemetry | polygateway.structured", "polygateway.transports | polygateway.backends | polygateway.telemetry | polygateway.structured",
"polygateway.thinking",
"polygateway.providers : polygateway.sources", "polygateway.providers : polygateway.sources",
"polygateway.ports : polygateway.types : polygateway.errors : polygateway.streaming", "polygateway.ports : polygateway.types : polygateway.errors : polygateway.streaming",
] ]
+29
View File
@@ -0,0 +1,29 @@
"""推理这件事的全部**决策**: 请求侧注入形态、响应侧结果裁定、二者的对账。
与 `providers.py` 的分工: 那里是**注册表**(provider 长什么样,静态声明的存放
与查找),这里是**决策**(拿声明和响应做判断)。P7"决策逻辑与状态存储分离"
本模块**不定义** `ThinkingObservation` —— 它是 `LLMResponse` 的字段类型,归最
内层 `types.py`;定义在这里会让 `types.py` 反向 import 决策模块(依赖铁律)。
"""
from polygateway.types import ThinkingObservation
def observe_thinking(*, thinking: str, reasoning_tokens: int | None) -> ThinkingObservation:
"""由多信号裁定推理是否发生;判据按**证据硬度**排序(issue #16/#17)。
推理正文是事实本身,`reasoning_tokens` 是对事实的转述——转述缺失时事实仍然
作数。2026-08-25 实测: MiniMax 这一路已不再返回
`usage.completion_tokens_details`,而同一次调用里库拿得到 185 字符推理正文;
只认 token 数的判据会把这种情形误判成"没推理"
正文判据取 `strip()` 而非 truthy: 网关响应是外部输入,纯空白串不是证据(P5)。
判不出来时返回 `UNKNOWN` 而非 `ABSENT`——**不许把"没看见"说成"没发生"**。
"""
if thinking.strip():
return ThinkingObservation.OBSERVED
if reasoning_tokens is None:
return ThinkingObservation.UNKNOWN
return ThinkingObservation.OBSERVED if reasoning_tokens > 0 else ThinkingObservation.ABSENT
+22
View File
@@ -10,6 +10,7 @@ import math
import re import re
from collections.abc import Mapping from collections.abc import Mapping
from dataclasses import dataclass, field from dataclasses import dataclass, field
from enum import StrEnum
from types import MappingProxyType from types import MappingProxyType
from typing import Any from typing import Any
@@ -166,6 +167,27 @@ def canonical_sampling_json(merged: Mapping[str, Any]) -> str | None:
return json.dumps(dict(merged), sort_keys=True, ensure_ascii=False) return json.dumps(dict(merged), sort_keys=True, ensure_ascii=False)
class ThinkingObservation(StrEnum):
"""一次调用中"推理是否真的发生"的裁定结果(issue #16/#17)。
三态**不可折叠为布尔**: `UNKNOWN` 是"本次无任何信号,判不出来",与
`ABSENT`("上游明确上报了未推理")语义不同。把前者折叠进后者,正是
`reasoning_tokens=None` 制造的那个歧义——库据此静默宣称"没推理",而实际
可能推理了且已计费(MiniMax-M3 非流式实测: completion 53 vs 关闭档 3,
推理正文与 usage 明细双双不回传)。
裁定由 `thinking.observe_thinking` 做,本类只是取值域。**枚举定义在最内层
而非决策层**: 它是 `LLMResponse` 的字段类型,放进 `thinking.py` 会让
`types.py` 反向 import 决策模块(P7 依赖铁律)。
取值进遥测落库,改名即造成历史数据断层。
"""
OBSERVED = "observed"
ABSENT = "absent"
UNKNOWN = "unknown"
@dataclass(frozen=True) @dataclass(frozen=True)
class LLMResponse: class LLMResponse:
"""一次治理调用的统一响应(与三项目超集兼容,ARCH §5.1)。""" """一次治理调用的统一响应(与三项目超集兼容,ARCH §5.1)。"""
+76
View File
@@ -0,0 +1,76 @@
"""推理裁定与对账的行为测试(issue #16/#17 设计 §4-§5)。
判据来自 2026-08-25 实测(findings): MiniMax-M3 在开启档流式路径下返回 185 字符
推理正文却不上报 `completion_tokens_details`,而 qwen/deepseek 两者都报。库因此
不能把任何单一信号当权威——本组用例逐条钉死"哪个信号该赢"
"""
import pytest
from polygateway.thinking import observe_thinking
from polygateway.types import ThinkingObservation
class TestObserveThinking:
"""三态裁定: 证据硬度决定优先级,无信号一律 UNKNOWN。"""
def test_reasoning_text_alone_proves_it_happened(self):
"""推理正文是事实本身: 上游不报 token 数也照样成立(M3 流式实测形态)。"""
assert (
observe_thinking(thinking="先解方程 x+y=35", reasoning_tokens=None)
is ThinkingObservation.OBSERVED
)
def test_blank_text_is_not_evidence(self):
"""纯空白正文不算证据: 网关响应是外部输入,truthy 判据会把空格计成推理(P5)。"""
assert (
observe_thinking(thinking=" \n\t ", reasoning_tokens=None)
is ThinkingObservation.UNKNOWN
)
def test_positive_token_count_proves_it_happened(self):
"""无正文但上游报了推理用量(qwen 非流式形态)。"""
assert (
observe_thinking(thinking="", reasoning_tokens=205) is ThinkingObservation.OBSERVED
)
def test_zero_token_count_is_positive_evidence_of_absence(self):
"""`0` 是"上报了且为零",与"没上报"语义不同,故是 ABSENT 而非 UNKNOWN。"""
assert observe_thinking(thinking="", reasoning_tokens=0) is ThinkingObservation.ABSENT
def test_no_signal_at_all_stays_unknown(self):
"""M3 非流式开启档的真实形态: 推理已计费却既无正文也无 token 数。
判成 ABSENT 就是伪装成"没推理"——正是 issue #16/#17 的病根。
"""
assert observe_thinking(thinking="", reasoning_tokens=None) is ThinkingObservation.UNKNOWN
def test_text_outranks_a_zero_count(self):
"""转述与事实冲突时事实赢: 正文在,`reasoning_tokens=0` 不能翻案。"""
assert (
observe_thinking(thinking="想了想", reasoning_tokens=0)
is ThinkingObservation.OBSERVED
)
class TestThinkingObservationEnum:
def test_values_are_stable_strings(self):
"""取值进遥测落库,改名即历史数据断层。"""
assert ThinkingObservation.OBSERVED == "observed"
assert ThinkingObservation.ABSENT == "absent"
assert ThinkingObservation.UNKNOWN == "unknown"
def test_enum_lives_in_the_innermost_layer(self):
"""枚举必须定义在 `types.py`(最内层)。
它是 `LLMResponse` 的字段类型;定义在决策层 `thinking.py` 会让 `types.py`
反向 import 决策模块,违反 P7 依赖铁律(import-linter 契约执法)。
"""
assert ThinkingObservation.__module__ == "polygateway.types"
@pytest.mark.parametrize("bogus", ["", "OBSERVED", "yes", "none"])
def test_unknown_strings_are_rejected(bogus):
"""非法值必须抛 ValueError: 缓存回放靠它把污染数据挡成"未命中"(设计 §6)。"""
with pytest.raises(ValueError):
ThinkingObservation(bogus)
+13
View File
@@ -14,6 +14,7 @@ from polygateway.types import (
LLMResponse, LLMResponse,
RetryPolicy, RetryPolicy,
SourceConfig, SourceConfig,
ThinkingObservation,
TransportResult, TransportResult,
Usage, Usage,
) )
@@ -33,6 +34,18 @@ def _make_source(**overrides):
return SourceConfig(**base) return SourceConfig(**base)
class TestThinkingObservationLayering:
"""枚举必须留在最内层,别被后来的重构挪进决策模块。"""
def test_defined_in_types_not_in_thinking(self):
"""`LLMResponse` 拿它当字段类型,定义在 `thinking.py` 会让最内层反向依赖决策层。
这条不是风格洁癖: import-linter 会判红,但那要等代码写完才发现;本用例
把约束前移到类型层面。
"""
assert ThinkingObservation.__module__ == "polygateway.types"
class TestLLMResponse: class TestLLMResponse:
def test_eleven_legacy_fields_positional(self): def test_eleven_legacy_fields_positional(self):
"""三项目 fake 的 11 参位置构造必须零改动成立(迁移兼容硬约束)。""" """三项目 fake 的 11 参位置构造必须零改动成立(迁移兼容硬约束)。"""