iomgaa
f5e6fafe8d
T10 经 new-api 中转对 26 个可调用模型逐个实测(约 500 次真实调用),把 DEFAULT_CAPABILITIES 里的文档推定换成实测结论,并把实测本身固化成可复跑、 可只跑单模型的 slow 用例。 三处与文档推定不符: - kimi-k3 实测可关(关闭档 completion 恒 9 且与 max 档锚点可分),T1 的保守 登记被推翻,OpenRouter 的 mandatory:false 胜出; - MiniMax-M3 的 auto 实测不推理(minimax 的"开"是不注入任何参数,而 M3 默认档 本就不推理),六个强度值反而全部生效且彼此等价,故清单由 (none, auto) 换成 none + 六个强度值; - glm-5.3 关不掉推理坐实(请求 none 后 4/5 轮仍观测到推理),issue #20 的核心 争议就此了结——当时短提示词下的 rt≈1.2 是采样噪声。 新增 kimi-for-coding(现有该型号自己的实测证据,不再走 Phase 3)。 用例带三条判据: 关闭方向要求每轮未观测到推理;短提示词的"关掉了"必须过长 上下文复核(glm-5.3-flash 正是短 5/5 未观测、长上下文 2/3 露馅);上游整片 不回传推理信号时另取无魔数锚点,不许把"没看见"写成"没发生"。另加一道模型 身份核对——该渠道把 glm-5/5.1/5.2 全部路由到 glm-5.3,那三组数据一律作废。 未覆盖 9 个型号(claude 三个 7 天限额用尽、gemini 两个上游报错、gpt-5.4 限流、 glm 三个被路由),evidence 逐条写明原因,不得被后续文档写成"已实测"。 单元测试里三条以 MiniMax-M3 当"纯开关型"样本的用例改用 glm-4.6v——实测后 M3 不再是那个形状,而 glm-4.6v 是全表证据最硬的 (none, auto)。
823 lines
37 KiB
Python
823 lines
37 KiB
Python
"""推理裁定与对账的行为测试(issue #16/#17 设计 §4-§5)。
|
||
|
||
判据来自 2026-08-25 实测(findings): MiniMax-M3 在开启档流式路径下返回 185 字符
|
||
推理正文却不上报 `completion_tokens_details`,而 qwen/deepseek 两者都报。库因此
|
||
不能把任何单一信号当权威——本组用例逐条钉死"哪个信号该赢"。
|
||
"""
|
||
|
||
import pytest
|
||
from loguru import logger
|
||
|
||
from polygateway.providers import ProviderProfile, ThinkingWire, get_provider
|
||
from polygateway.thinking import (
|
||
DEFAULT_CAPABILITIES,
|
||
ThinkingCapability,
|
||
ThinkingUnsupportedError,
|
||
effective_effort,
|
||
get_capability,
|
||
observe_thinking,
|
||
reconcile_thinking,
|
||
register_capability,
|
||
resolve_thinking,
|
||
)
|
||
from polygateway.types import Effort, ThinkingObservation
|
||
|
||
_MYSTERY = ProviderProfile(
|
||
name="mystery",
|
||
thinking=ThinkingWire(off=None, on_base=None, effort_key=None),
|
||
strip_think_tags=False,
|
||
)
|
||
"""形态完全未知的 provider(issue #5 的守卫对象)。
|
||
|
||
2026-09-04 起默认表 8 段全部有形态,故未知样本改为显式构造——测的是**机制**
|
||
(不知道怎么表达就报错并指路),不是某个段当时的配置。"""
|
||
|
||
|
||
def _warnings():
|
||
"""捕获库发出的 WARNING;loguru 不经标准 logging,pytest 的 caplog 抓不到。"""
|
||
messages: list[str] = []
|
||
sink_id = logger.add(messages.append, level="WARNING")
|
||
return messages, sink_id
|
||
|
||
|
||
class TestObserveThinking:
|
||
"""三态裁定: 证据硬度决定优先级,无信号一律 UNKNOWN。"""
|
||
|
||
def test_reasoning_text_alone_proves_it_happened(self):
|
||
"""推理正文是事实本身: 上游不报 token 数也照样成立(M3 流式实测形态)。"""
|
||
assert (
|
||
observe_thinking(thinking="先解方程 x+y=35", reasoning_tokens=None)
|
||
is ThinkingObservation.OBSERVED
|
||
)
|
||
|
||
def test_blank_text_is_not_evidence(self):
|
||
"""纯空白正文不算证据: 网关响应是外部输入,truthy 判据会把空格计成推理(P5)。"""
|
||
assert (
|
||
observe_thinking(thinking=" \n\t ", reasoning_tokens=None)
|
||
is ThinkingObservation.UNKNOWN
|
||
)
|
||
|
||
def test_positive_token_count_proves_it_happened(self):
|
||
"""无正文但上游报了推理用量(qwen 非流式形态)。"""
|
||
assert observe_thinking(thinking="", reasoning_tokens=205) is ThinkingObservation.OBSERVED
|
||
|
||
def test_zero_token_count_is_positive_evidence_of_absence(self):
|
||
"""`0` 是"上报了且为零",与"没上报"语义不同,故是 ABSENT 而非 UNKNOWN。"""
|
||
assert observe_thinking(thinking="", reasoning_tokens=0) is ThinkingObservation.ABSENT
|
||
|
||
def test_no_signal_at_all_stays_unknown(self):
|
||
"""M3 非流式开启档的真实形态: 推理已计费却既无正文也无 token 数。
|
||
|
||
判成 ABSENT 就是伪装成"没推理"——正是 issue #16/#17 的病根。
|
||
"""
|
||
assert observe_thinking(thinking="", reasoning_tokens=None) is ThinkingObservation.UNKNOWN
|
||
|
||
def test_text_outranks_a_zero_count(self):
|
||
"""转述与事实冲突时事实赢: 正文在,`reasoning_tokens=0` 不能翻案。"""
|
||
assert (
|
||
observe_thinking(thinking="想了想", reasoning_tokens=0) is ThinkingObservation.OBSERVED
|
||
)
|
||
|
||
@pytest.mark.parametrize("negative", [-1, -205])
|
||
def test_negative_token_count_is_not_evidence_of_absence(self, negative):
|
||
"""负数是坏数据,不是"上游明确上报未推理"这个最强的正面结论。
|
||
|
||
当前 transport 已在边界把负数归 `None`,所以这条走不通;但本函数的
|
||
docstring 自称"外部输入校验后使用",第二个 transport 直接填该值时,
|
||
`> 0 else ABSENT` 会给出一个方向相反的强结论。函数自身必须闭合(P5)。
|
||
"""
|
||
assert observe_thinking(thinking="", reasoning_tokens=negative) is (
|
||
ThinkingObservation.UNKNOWN
|
||
)
|
||
|
||
|
||
class TestThinkingObservationEnum:
|
||
def test_values_are_stable_strings(self):
|
||
"""取值进遥测落库,改名即历史数据断层。"""
|
||
assert ThinkingObservation.OBSERVED == "observed"
|
||
assert ThinkingObservation.ABSENT == "absent"
|
||
assert ThinkingObservation.UNKNOWN == "unknown"
|
||
|
||
def test_enum_lives_in_the_innermost_layer(self):
|
||
"""枚举必须定义在 `types.py`(最内层)。
|
||
|
||
它是 `LLMResponse` 的字段类型;定义在决策层 `thinking.py` 会让 `types.py`
|
||
反向 import 决策模块,违反 P7 依赖铁律(import-linter 契约执法)。
|
||
"""
|
||
assert ThinkingObservation.__module__ == "polygateway.types"
|
||
|
||
|
||
@pytest.mark.parametrize("bogus", ["", "OBSERVED", "yes", "none"])
|
||
def test_unknown_strings_are_rejected(bogus):
|
||
"""非法值必须抛 ValueError: 缓存回放与遥测归一化都靠它识别域外取值(设计 §6)。
|
||
|
||
两处接住这个 ValueError 后**降级而非作废**(缓存复活内容 + 记 UNKNOWN、遥测
|
||
照常落行),但降级的前提是构造器真的会拒绝——它一旦放行,域外取值就会一路
|
||
进到 `LLMResponse` 与遥测列里。
|
||
"""
|
||
with pytest.raises(ValueError):
|
||
ThinkingObservation(bogus)
|
||
|
||
|
||
class TestThinkingCapability:
|
||
"""issue #5: 能力按 model 登记——同一 provider 内部代际差异是决定性的。"""
|
||
|
||
def test_registered_models_carry_evidence(self):
|
||
"""登记必须附实测证据: 表会过期,没有出处就无从判断该不该信。"""
|
||
for model in ("MiniMax-M3", "MiniMax-M2.7", "MiniMax-M2.5"):
|
||
cap = get_capability(model)
|
||
assert cap is not None and cap.evidence.strip()
|
||
|
||
def test_m3_can_disable_but_m2x_cannot(self):
|
||
assert get_capability("MiniMax-M3").can_disable is True
|
||
assert get_capability("MiniMax-M2.7").can_disable is False
|
||
assert get_capability("MiniMax-M2.5").can_disable is False
|
||
|
||
def test_unregistered_model_is_unknown(self):
|
||
assert get_capability("some-brand-new-model") is None
|
||
|
||
def test_register_capability_is_pure(self):
|
||
table = register_capability("x-1", ThinkingCapability((Effort.NONE, Effort.AUTO), "实测"))
|
||
assert get_capability("x-1", table=table) is not None
|
||
assert get_capability("x-1") is None # 默认表未被污染
|
||
|
||
def test_default_capabilities_mapping_is_read_only(self):
|
||
with pytest.raises(TypeError):
|
||
DEFAULT_CAPABILITIES["hack"] = None # type: ignore[index]
|
||
|
||
|
||
class TestResolveThinking:
|
||
"""五道关卡(顺序即语义)与 nearest 映射;设计 §4.1。
|
||
|
||
每一关都有独立的失败模式,漏测哪一关,判定顺序被调换都不会被抓住——而顺序
|
||
在本函数里**就是**语义(Phase 4 落进 Phase 5 就丢掉"这个模型根本关不掉")。
|
||
"""
|
||
|
||
# —— Phase 1: 不表态 ——
|
||
|
||
def test_phase1_absent_effort_injects_nothing(self):
|
||
"""没表态就什么都不注入,用模型自己的默认档(与 `none` 严格区分)。"""
|
||
got = resolve_thinking(get_provider("minimax"), None, None, model="MiniMax-M3")
|
||
assert got.payload == {}
|
||
assert got.applied_effort is None
|
||
|
||
# —— Phase 2: 形态未知 ——
|
||
|
||
@pytest.mark.parametrize("effort", [Effort.NONE, Effort.AUTO, Effort.HIGH])
|
||
def test_phase2_unknown_wire_points_to_register(self, effort):
|
||
"""不知道怎么发就报错并指路;静默放行是 issue #5 修掉的那种欺骗。
|
||
|
||
文案必须报出**请求的档位**而非"开/关"方向: `Effort` 是非空字符串,拿它
|
||
的真值判方向会把 `none` 说成"开启形态未知",指错了排查方向。
|
||
"""
|
||
with pytest.raises(ThinkingUnsupportedError, match="register_provider") as exc:
|
||
resolve_thinking(_MYSTERY, None, effort, model="kimi-k3")
|
||
msg = str(exc.value)
|
||
assert "extra_body" in msg
|
||
assert "kimi-k3" in msg
|
||
assert effort.value in msg
|
||
|
||
def test_phase2_reads_the_form_the_asked_for_tier_needs(self):
|
||
"""请求 `none` 只需要**关闭**形态: 开启形态未知与这次请求无关。
|
||
|
||
旧版 `slot = thinking_on if enable_thinking else thinking_off` 即按请求方向
|
||
取字段;档位化后一度写成"只看 `on_base`",于是一个已注册了关闭形态的自定义
|
||
provider 在请求 `none` 时被误拒,还被指向它已经做过的 `register_provider`
|
||
——指错方向比不指更糟(设计 §2 处置表第 2 条,2026-09-05 独立验证查出)。
|
||
"""
|
||
profile = ProviderProfile(
|
||
name="off_only",
|
||
thinking=ThinkingWire(
|
||
off={"thinking": {"type": "disabled"}}, on_base=None, effort_key=None
|
||
),
|
||
strip_think_tags=False,
|
||
)
|
||
cap = ThinkingCapability((Effort.NONE, Effort.AUTO), "构造: 关得掉,开启形态却未登记")
|
||
got = resolve_thinking(profile, cap, Effort.NONE, model="x-1")
|
||
assert got.payload == {"thinking": {"type": "disabled"}}
|
||
assert got.applied_effort is Effort.NONE
|
||
|
||
@pytest.mark.parametrize("effort", [Effort.AUTO, Effort.HIGH])
|
||
def test_phase2_still_fires_when_the_on_form_is_the_missing_half(self, effort):
|
||
"""反方向不得被一并放过: 要开推理而开启形态未知,仍须报错并指路注册。"""
|
||
profile = ProviderProfile(
|
||
name="off_only",
|
||
thinking=ThinkingWire(
|
||
off={"thinking": {"type": "disabled"}}, on_base=None, effort_key=None
|
||
),
|
||
strip_think_tags=False,
|
||
)
|
||
cap = ThinkingCapability((Effort.NONE, Effort.AUTO, Effort.HIGH), "构造")
|
||
with pytest.raises(ThinkingUnsupportedError, match="register_provider") as exc:
|
||
resolve_thinking(profile, cap, effort, model="x-1")
|
||
assert effort.value in str(exc.value)
|
||
|
||
def test_phase2_beats_the_capability_checks(self):
|
||
"""形态未知时无从注入,能力如何无关紧要——Phase 2 必须先于 4/5。"""
|
||
cap = ThinkingCapability((Effort.AUTO,), "构造")
|
||
with pytest.raises(ThinkingUnsupportedError, match="register_provider"):
|
||
resolve_thinking(_MYSTERY, cap, Effort.NONE, model="whatever")
|
||
|
||
# —— Phase 3: 能力未登记 ——
|
||
|
||
def test_phase3_unregistered_warns_then_injects(self):
|
||
"""新模型上线不该被库挡住,但也不该假装成功: 喊一声再尽力注入。"""
|
||
messages, sink_id = _warnings()
|
||
try:
|
||
got = resolve_thinking(get_provider("minimax"), None, Effort.NONE, model="MiniMax-M9")
|
||
finally:
|
||
logger.remove(sink_id)
|
||
assert got.payload == {"reasoning_effort": "none"}
|
||
assert got.applied_effort is Effort.NONE
|
||
assert any("MiniMax-M9" in m for m in messages)
|
||
|
||
def test_phase3_can_be_silenced_on_the_hot_path(self):
|
||
"""装配期已经喊过一次,逐次调用再喊只会刷屏;判定结果不受影响。"""
|
||
messages, sink_id = _warnings()
|
||
try:
|
||
got = resolve_thinking(
|
||
get_provider("minimax"),
|
||
None,
|
||
Effort.NONE,
|
||
model="MiniMax-M9",
|
||
warn_unregistered=False,
|
||
)
|
||
finally:
|
||
logger.remove(sink_id)
|
||
assert got.payload == {"reasoning_effort": "none"}
|
||
assert not [m for m in messages if "MiniMax-M9" in m]
|
||
|
||
def test_phase3_does_not_validate_tiers(self):
|
||
"""能力未知就没有清单可比对,拿空清单去拒绝档位等于凭空报错。"""
|
||
got = resolve_thinking(
|
||
get_provider("zhipu"),
|
||
None,
|
||
Effort.XHIGH,
|
||
model="glm-9-not-registered",
|
||
warn_unregistered=False,
|
||
)
|
||
assert got.payload == {"thinking": {"type": "enabled"}, "reasoning_effort": "xhigh"}
|
||
assert got.applied_effort is Effort.XHIGH
|
||
|
||
# —— Phase 4: 关不掉 ——
|
||
|
||
def test_phase4_before_phase5(self):
|
||
"""请求 `none` 而模型关不掉: 文案必须给出可执行替代与 env 键名。
|
||
|
||
若落进 Phase 5 的通用分支,报错会退化成"不支持 none,可选 low/high/max",
|
||
丢掉"这个模型根本关不掉"这个关键信息——下游随后就会去找 extra_body 那条
|
||
绕过的路,而那正是 issue #20 的成因。
|
||
"""
|
||
cap = get_capability("glm-5.3")
|
||
with pytest.raises(ThinkingUnsupportedError) as exc:
|
||
resolve_thinking(get_provider("zhipu"), cap, Effort.NONE, model="glm-5.3")
|
||
msg = str(exc.value)
|
||
assert "glm-5.3" in msg
|
||
assert "'low'" in msg, "必须给出 cheapest_effort 的值"
|
||
assert "REASONING_EFFORT" in msg, "必须给出 env 键名"
|
||
assert "可选档位" not in msg, "退化成 Phase 5 的通用文案即失去可执行替代"
|
||
|
||
def test_phase4_never_maps_even_with_nearest(self):
|
||
"""`none` 不走映射: 把"关不掉"映射成"开着最低档"就是又一次静默降级。"""
|
||
cap = get_capability("glm-5.3")
|
||
with pytest.raises(ThinkingUnsupportedError, match="REASONING_EFFORT"):
|
||
resolve_thinking(
|
||
get_provider("zhipu"), cap, Effort.NONE, model="glm-5.3", fallback="nearest"
|
||
)
|
||
|
||
def test_phase4_only_blocks_the_off_direction(self):
|
||
"""关不掉 ≠ 开不了: M2.x 默认就在推理,开的方向不该被拦。"""
|
||
cap = get_capability("MiniMax-M2.7")
|
||
got = resolve_thinking(get_provider("minimax"), cap, Effort.AUTO, model="MiniMax-M2.7")
|
||
assert got.payload == {}
|
||
assert got.applied_effort is Effort.AUTO
|
||
|
||
def test_phase4_passes_when_none_is_registered(self):
|
||
cap = get_capability("MiniMax-M3")
|
||
got = resolve_thinking(get_provider("minimax"), cap, Effort.NONE, model="MiniMax-M3")
|
||
assert got.payload == {"reasoning_effort": "none"}
|
||
assert got.applied_effort is Effort.NONE
|
||
|
||
# —— Phase 5: 档位打空 ——
|
||
|
||
def test_phase5_lists_tiers_for_tiered_model(self):
|
||
"""档位型模型: 文案必须列出它真有的档,否则下游只能猜。"""
|
||
cap = get_capability("glm-5.3")
|
||
with pytest.raises(ThinkingUnsupportedError) as exc:
|
||
resolve_thinking(get_provider("zhipu"), cap, Effort.MEDIUM, model="glm-5.3")
|
||
msg = str(exc.value)
|
||
assert "medium" in msg and "glm-5.3" in msg
|
||
assert "可选档位" in msg
|
||
assert "low" in msg and "high" in msg and "max" in msg
|
||
|
||
def test_phase5_says_toggle_only_for_switch_model(self):
|
||
"""纯开关型模型没有档位,对它说"可选档位"是错的(设计 §3.2 第三个派生量)。
|
||
|
||
样本 2026-09-05 由 MiniMax-M3 换成 glm-4.6v: T10 实测 M3 的六个强度值全部生效,
|
||
它不再是纯开关型;glm-4.6v 是实测证据最硬的 (none, auto) 模型,且 zhipu 的 wire
|
||
有 effort_key——这两点缺一不可,否则命中的是"该 provider 没有档位键"那条分支。
|
||
"""
|
||
cap = get_capability("glm-4.6v") # (none, auto): 能开能关,但没有强度档
|
||
with pytest.raises(ThinkingUnsupportedError) as exc:
|
||
resolve_thinking(get_provider("zhipu"), cap, Effort.HIGH, model="glm-4.6v")
|
||
msg = str(exc.value)
|
||
assert "可选档位" not in msg
|
||
assert "该模型只有开关" in msg
|
||
assert "auto" in msg and "none" in msg
|
||
|
||
def test_phase5_wording_forks_on_is_tiered(self):
|
||
"""两条分叉必须真的不同——同一句话套两种模型等于没分叉。"""
|
||
with pytest.raises(ThinkingUnsupportedError) as tiered:
|
||
resolve_thinking(
|
||
get_provider("zhipu"), get_capability("glm-5.3"), Effort.MEDIUM, model="glm-5.3"
|
||
)
|
||
with pytest.raises(ThinkingUnsupportedError) as switch:
|
||
resolve_thinking(
|
||
get_provider("zhipu"),
|
||
get_capability("glm-4.6v"),
|
||
Effort.MEDIUM,
|
||
model="glm-4.6v",
|
||
)
|
||
assert str(tiered.value) != str(switch.value)
|
||
|
||
def test_phase5_passes_a_supported_tier(self):
|
||
cap = get_capability("glm-5.3")
|
||
got = resolve_thinking(get_provider("zhipu"), cap, Effort.MAX, model="glm-5.3")
|
||
assert got.payload == {"thinking": {"type": "enabled"}, "reasoning_effort": "max"}
|
||
assert got.applied_effort is Effort.MAX
|
||
|
||
def test_auto_never_trips_phase5(self):
|
||
"""`auto` = 不指定档位,可满足性只取决于 wire 有没有 on_base。
|
||
|
||
它不是写进 `effort_key` 的取值,故不受档位清单约束。反过来判会让存量的
|
||
`ENABLE_THINKING=true`(T5 起等价于 auto)在 deepseek/glm-5.3 这类清单里
|
||
没有 auto 的模型上当场报错——设计 §12 明确承诺存量配置继续可跑。
|
||
"""
|
||
cap = get_capability("deepseek-v4-pro") # (none, high, max),清单里没有 auto
|
||
got = resolve_thinking(get_provider("deepseek"), cap, Effort.AUTO, model="deepseek-v4-pro")
|
||
assert got.payload == {"thinking": {"type": "enabled"}}
|
||
assert got.applied_effort is Effort.AUTO
|
||
|
||
# —— nearest 映射(fallback 的逃生口)——
|
||
|
||
def test_nearest_ties_go_cheaper(self):
|
||
"""等距取弱: 省钱优先,库不替下游涨价(一次 medium→max 是数倍账单)。"""
|
||
cap = get_capability("glm-5.3") # (low, high, max)
|
||
messages, sink_id = _warnings()
|
||
try:
|
||
got = resolve_thinking(
|
||
get_provider("zhipu"), cap, Effort.MEDIUM, model="glm-5.3", fallback="nearest"
|
||
)
|
||
finally:
|
||
logger.remove(sink_id)
|
||
assert got.payload == {"thinking": {"type": "enabled"}, "reasoning_effort": "low"}
|
||
assert any("glm-5.3" in m and "medium" in m and "low" in m for m in messages)
|
||
|
||
def test_nearest_ties_go_cheaper_on_the_strong_side_too(self):
|
||
"""xhigh 与 high/max 位序各差 1,同样取弱侧——规则不因方向而变。"""
|
||
cap = get_capability("glm-5.3")
|
||
got = resolve_thinking(
|
||
get_provider("zhipu"), cap, Effort.XHIGH, model="glm-5.3", fallback="nearest"
|
||
)
|
||
assert got.applied_effort is Effort.HIGH
|
||
|
||
def test_nearest_goes_up_when_the_only_neighbour_is_stronger(self):
|
||
"""minimal 之下无档可选,映射必须上行到 low,而不是无解报错。"""
|
||
cap = get_capability("glm-5.3")
|
||
got = resolve_thinking(
|
||
get_provider("zhipu"), cap, Effort.MINIMAL, model="glm-5.3", fallback="nearest"
|
||
)
|
||
assert got.applied_effort is Effort.LOW
|
||
|
||
def test_nearest_never_turns_reasoning_off(self):
|
||
"""请求"想得浅一点"绝不能被映射成"别想了": 那是方向反转,不是省钱。"""
|
||
cap = get_capability("glm-4.6v") # (none, auto)
|
||
got = resolve_thinking(
|
||
get_provider("zhipu"), cap, Effort.HIGH, model="glm-4.6v", fallback="nearest"
|
||
)
|
||
assert got.applied_effort is Effort.AUTO
|
||
assert got.payload == {"thinking": {"type": "enabled"}}
|
||
|
||
def test_nearest_still_errors_when_no_on_tier_exists(self):
|
||
"""只能关不能开的模型,映射无解——报错而非挑一个反向的档。"""
|
||
cap = ThinkingCapability((Effort.NONE,), "构造: 只登记了关闭档")
|
||
with pytest.raises(ThinkingUnsupportedError, match="only-off"):
|
||
resolve_thinking(
|
||
get_provider("minimax"), cap, Effort.HIGH, model="only-off", fallback="nearest"
|
||
)
|
||
|
||
def test_error_fallback_is_the_default(self):
|
||
"""默认关闭映射的理由是钱: 静默的 medium→max 在 GLM-5.3 上是数倍账单。"""
|
||
cap = get_capability("glm-5.3")
|
||
with pytest.raises(ThinkingUnsupportedError):
|
||
resolve_thinking(get_provider("zhipu"), cap, Effort.MEDIUM, model="glm-5.3")
|
||
|
||
def test_resolution_reports_applied_effort_after_mapping(self):
|
||
"""遥测记的必须是**实际**发出去的档,否则压测按档分组时挂在从未发出的档下。"""
|
||
cap = get_capability("glm-5.3")
|
||
got = resolve_thinking(
|
||
get_provider("zhipu"), cap, Effort.MEDIUM, model="glm-5.3", fallback="nearest"
|
||
)
|
||
assert got.applied_effort is Effort.LOW
|
||
assert got.applied_effort is not Effort.MEDIUM
|
||
|
||
# —— 注入形态 ——
|
||
|
||
def test_auto_injects_on_base_only(self):
|
||
"""`auto` 逐字节等于旧的 `thinking_on`: 开启,但不附任何档位。"""
|
||
got = resolve_thinking(
|
||
get_provider("qwen"), get_capability("qwen3.7-plus"), Effort.AUTO, model="qwen3.7-plus"
|
||
)
|
||
assert got.payload == {"enable_thinking": True}
|
||
|
||
def test_effort_key_none_rejects_a_tier(self):
|
||
"""qwen 系只有开关没有档位键: 硬塞一个档位只会发出一个厂商不认的字段。"""
|
||
cap = ThinkingCapability((Effort.NONE, Effort.LOW), "构造: 假设它有档位")
|
||
with pytest.raises(ThinkingUnsupportedError, match="没有档位键"):
|
||
resolve_thinking(get_provider("qwen"), cap, Effort.LOW, model="qwen-hypothetical")
|
||
|
||
def test_provider_without_an_off_form_says_which_half_is_missing(self):
|
||
"""`off is None` ≠ `on_base is None`: 前者是"关不了",后者是"不知道怎么发"。"""
|
||
profile = ProviderProfile(
|
||
name="no_off",
|
||
thinking=ThinkingWire(off=None, on_base={}, effort_key="reasoning_effort"),
|
||
strip_think_tags=False,
|
||
)
|
||
cap = ThinkingCapability((Effort.NONE, Effort.LOW), "构造: 能力表说能关,形态却没有")
|
||
with pytest.raises(ThinkingUnsupportedError, match="没有关闭形态") as exc:
|
||
resolve_thinking(profile, cap, Effort.NONE, model="x-1")
|
||
assert "register_provider" not in str(exc.value), "形态已知,不该指向注册"
|
||
|
||
# —— 归一化: 本函数是档位进入库内的第四条入口(设计 §4.4) ——
|
||
|
||
def test_a_bare_string_tier_is_normalised_at_the_door(self):
|
||
"""`resolve_thinking` 在 `__all__` 里,下游直调时传的天然是裸串。
|
||
|
||
第三参数本次由 `bool` 换成 `Effort`,而下游最自然的写法是从 JSON/配置读出来
|
||
的 `"low"`。不在入口归一,`_inject` 撞 `.value` 抛的是 `AttributeError`——
|
||
一个未文档化、也不属错误四分类的异常(2026-09-05 独立验证查出)。
|
||
"""
|
||
got = resolve_thinking(
|
||
get_provider("zhipu"), get_capability("glm-5.3"), "low", model="glm-5.3"
|
||
)
|
||
assert got.payload == {"thinking": {"type": "enabled"}, "reasoning_effort": "low"}
|
||
assert got.applied_effort is Effort.LOW
|
||
|
||
def test_a_bare_none_string_still_means_the_off_tier(self):
|
||
"""裸 `"none"` 必须走到关闭形态,而不是被当成某个开启档。
|
||
|
||
身份比较 `"none" is Effort.NONE` 恒假,漏归一的后果是**静默判否**:
|
||
`_wire_unknown_for` 的 `effort is not Effort.NONE` 恒真,于是关闭请求会去看
|
||
`on_base`——正是设计 §2 处置表第 2 条点名要避免的误判方向。
|
||
"""
|
||
got = resolve_thinking(
|
||
get_provider("zhipu"), get_capability("glm-5.2"), "none", model="glm-5.2"
|
||
)
|
||
assert got.payload == {"thinking": {"type": "disabled"}}
|
||
assert got.applied_effort is Effort.NONE
|
||
|
||
def test_a_bare_none_string_reaches_phase4_on_a_model_that_cannot_disable(self):
|
||
"""漏归一时 Phase 4 整条被绕过: 关不掉的模型会被静默放行成"开启"。"""
|
||
with pytest.raises(ThinkingUnsupportedError, match="无法关闭推理") as exc:
|
||
resolve_thinking(
|
||
get_provider("zhipu"), get_capability("glm-5.3"), "none", model="glm-5.3"
|
||
)
|
||
assert "'low'" in str(exc.value), "Phase 4 的可执行替代不能丢"
|
||
|
||
def test_an_illegal_tier_string_names_this_function_as_the_origin(self):
|
||
"""非法档位报 `ValueError` 并指回**是哪一处**填错——档位有四条入口,不说清
|
||
就得让人自己去翻。"""
|
||
with pytest.raises(ValueError, match="resolve_thinking") as exc:
|
||
resolve_thinking(
|
||
get_provider("zhipu"), get_capability("glm-5.3"), "lowest", model="glm-5.3"
|
||
)
|
||
assert "非法推理档位" in str(exc.value)
|
||
|
||
|
||
class TestReconcileThinking:
|
||
"""声明 × 观测对账(设计 §4.3): 矛盾出文案,不表态出 None。
|
||
|
||
文案本身是被断言对象——判定与日志分离正是为此: 告警内容可直接比对,不必
|
||
去解析日志格式。
|
||
|
||
判据自 2026-09-05 起是**档位**而非布尔(设计 §4.3): `Effort.NONE` 走"要求
|
||
关闭"一支,其余档走"要求开启"一支。档位化不是换个参数名——文案里写的是本次
|
||
真正发出去的那一档,而 transport 的节流键正按它分离,两者必须同源。
|
||
"""
|
||
|
||
_CAP = ThinkingCapability(
|
||
(Effort.NONE, Effort.AUTO), "2026-08-02 实测 reasoning_effort=none 可关闭"
|
||
)
|
||
|
||
def test_off_but_observed_with_a_registered_capability_blames_the_table(self):
|
||
"""已登记却实测推理了 = 能力表漂移: 必须附 evidence 与更新指路。"""
|
||
msg = reconcile_thinking(
|
||
effort=Effort.NONE,
|
||
observation=ThinkingObservation.OBSERVED,
|
||
capability=self._CAP,
|
||
model="MiniMax-M3",
|
||
)
|
||
assert msg is not None
|
||
assert "MiniMax-M3" in msg
|
||
assert "2026-08-02 实测 reasoning_effort=none 可关闭" in msg
|
||
assert "register_capability" in msg
|
||
|
||
def test_off_but_observed_unregistered_never_claims_a_table_entry(self):
|
||
"""未登记模型没有"能力表声称"这回事——说它就是撒谎。"""
|
||
msg = reconcile_thinking(
|
||
effort=Effort.NONE,
|
||
observation=ThinkingObservation.OBSERVED,
|
||
capability=None,
|
||
model="MiniMax-M9",
|
||
)
|
||
assert msg is not None
|
||
assert "MiniMax-M9" in msg
|
||
assert "能力表" not in msg
|
||
assert "register_capability" in msg
|
||
|
||
def test_registered_and_unregistered_wordings_differ(self):
|
||
registered = reconcile_thinking(
|
||
effort=Effort.NONE,
|
||
observation=ThinkingObservation.OBSERVED,
|
||
capability=self._CAP,
|
||
model="MiniMax-M3",
|
||
)
|
||
unregistered = reconcile_thinking(
|
||
effort=Effort.NONE,
|
||
observation=ThinkingObservation.OBSERVED,
|
||
capability=None,
|
||
model="MiniMax-M3",
|
||
)
|
||
assert registered != unregistered
|
||
|
||
@pytest.mark.parametrize("capability", [None, _CAP])
|
||
def test_on_but_absent_is_a_contradiction(self, capability):
|
||
"""上游明确上报未推理: 这是唯一的正面证伪,与能力表登记与否无关。"""
|
||
msg = reconcile_thinking(
|
||
effort=Effort.AUTO,
|
||
observation=ThinkingObservation.ABSENT,
|
||
capability=capability,
|
||
model="qwen3.7-plus",
|
||
)
|
||
assert msg is not None
|
||
assert "qwen3.7-plus" in msg
|
||
|
||
@pytest.mark.parametrize("capability", [None, _CAP])
|
||
def test_on_but_unknown_admits_it_cannot_confirm(self, capability):
|
||
"""issue #17 的诚实版本: 明说"我注入了,但我看不见结果"。"""
|
||
msg = reconcile_thinking(
|
||
effort=Effort.AUTO,
|
||
observation=ThinkingObservation.UNKNOWN,
|
||
capability=capability,
|
||
model="MiniMax-M3",
|
||
)
|
||
assert msg is not None
|
||
assert "MiniMax-M3" in msg
|
||
|
||
def test_off_and_absent_stays_silent(self):
|
||
"""要求关闭 + 上游明确上报未推理 = 要求被满足,没有可报的矛盾。
|
||
|
||
这一格与 `test_off_and_unknown_stays_silent` 的沉默理由**不同**: 那里是
|
||
"没有证伪力",这里是"正面证实要求已满足"。两者都必须沉默,漏测哪一格,
|
||
把 Phase 2 的判据写成 `is ABSENT` 之类的反向条件都不会被抓住。
|
||
"""
|
||
assert (
|
||
reconcile_thinking(
|
||
effort=Effort.NONE,
|
||
observation=ThinkingObservation.ABSENT,
|
||
capability=self._CAP,
|
||
model="qwen3.7-plus",
|
||
)
|
||
is None
|
||
)
|
||
|
||
def test_off_and_unknown_stays_silent(self):
|
||
"""UNKNOWN 没有证伪力: 拿它报警等于每次关闭调用都喊(M3 关闭档恒落此档)。"""
|
||
assert (
|
||
reconcile_thinking(
|
||
effort=Effort.NONE,
|
||
observation=ThinkingObservation.UNKNOWN,
|
||
capability=self._CAP,
|
||
model="MiniMax-M3",
|
||
)
|
||
is None
|
||
)
|
||
|
||
@pytest.mark.parametrize(
|
||
"observation",
|
||
[ThinkingObservation.OBSERVED, ThinkingObservation.ABSENT, ThinkingObservation.UNKNOWN],
|
||
)
|
||
def test_no_request_no_grievance(self, observation):
|
||
"""调用方不表态,就无从谈"违背"。"""
|
||
assert (
|
||
reconcile_thinking(
|
||
effort=None,
|
||
observation=observation,
|
||
capability=self._CAP,
|
||
model="MiniMax-M3",
|
||
)
|
||
is None
|
||
)
|
||
|
||
def test_on_and_observed_is_exactly_what_was_asked_for(self):
|
||
assert (
|
||
reconcile_thinking(
|
||
effort=Effort.AUTO,
|
||
observation=ThinkingObservation.OBSERVED,
|
||
capability=self._CAP,
|
||
model="MiniMax-M3",
|
||
)
|
||
is None
|
||
)
|
||
|
||
@pytest.mark.parametrize("effort", [Effort.LOW, Effort.HIGH, Effort.MAX])
|
||
def test_a_strength_tier_is_an_on_request_not_an_off_one(self, effort):
|
||
"""强度档必须走"要求开启"一支: 观测到推理正是它要的结果,不得报警。
|
||
|
||
判据写成真值性(`if not effort`)会在这里翻车——`Effort.NONE` 的取值是
|
||
非空串 `"none"`,恒为真;那种写法会把每一个强度档都送进"要求关闭"分支,
|
||
于是"想了"被当成矛盾,而"没想"反倒沉默,告警方向整个颠倒。
|
||
"""
|
||
assert (
|
||
reconcile_thinking(
|
||
effort=effort,
|
||
observation=ThinkingObservation.OBSERVED,
|
||
capability=ThinkingCapability((Effort.LOW, Effort.HIGH, Effort.MAX), "构造"),
|
||
model="glm-5.3",
|
||
)
|
||
is None
|
||
)
|
||
|
||
def test_the_wording_names_the_tier_that_was_asked_for(self):
|
||
"""文案要写出**本次这一档**: 节流键按档分离,文案不分档就看不出是哪一档。"""
|
||
low = reconcile_thinking(
|
||
effort=Effort.LOW,
|
||
observation=ThinkingObservation.ABSENT,
|
||
capability=None,
|
||
model="glm-5.3",
|
||
)
|
||
max_ = reconcile_thinking(
|
||
effort=Effort.MAX,
|
||
observation=ThinkingObservation.ABSENT,
|
||
capability=None,
|
||
model="glm-5.3",
|
||
)
|
||
assert low is not None and max_ is not None
|
||
assert "low" in low and "max" in max_
|
||
assert low != max_
|
||
|
||
def test_none_and_observed_is_the_issue_20_contradiction(self):
|
||
"""请求 `none` 却观测到推理 —— issue #20 要恢复的那条报警,判据是**档位相等**。
|
||
|
||
与上一条互为对照: 同样是 OBSERVED,`none` 必须喊、强度档必须沉默。把分支
|
||
条件写反(`is not Effort.NONE`)会让这两条同时红,单有一条则抓不住。
|
||
"""
|
||
msg = reconcile_thinking(
|
||
effort=Effort.NONE,
|
||
observation=ThinkingObservation.OBSERVED,
|
||
capability=None,
|
||
model="glm-5.3",
|
||
)
|
||
assert msg is not None and "none" in msg
|
||
|
||
|
||
class TestEffortVocabulary:
|
||
"""八档封闭词汇(设计 §3.1);`auto` 不可省——9 个纯开关型模型无强度档可填。"""
|
||
|
||
def test_none_and_auto_are_distinct_members(self):
|
||
assert Effort.NONE != Effort.AUTO
|
||
assert Effort("none") is Effort.NONE
|
||
assert Effort("auto") is Effort.AUTO
|
||
|
||
def test_vocabulary_is_exactly_eight(self):
|
||
assert len(list(Effort)) == 8
|
||
|
||
def test_values_are_wire_literals(self):
|
||
# 档位值直接写进请求体,改名即改变发出去的字节
|
||
assert [e.value for e in Effort] == [
|
||
"none",
|
||
"auto",
|
||
"minimal",
|
||
"low",
|
||
"medium",
|
||
"high",
|
||
"xhigh",
|
||
"max",
|
||
]
|
||
|
||
|
||
class TestCapabilityTierList:
|
||
"""能力表从 bool 变成档位清单(设计 §3.2);三个派生量不存字段,存了必漂移。"""
|
||
|
||
def test_capability_derives_can_disable(self):
|
||
assert ThinkingCapability((Effort.NONE, Effort.AUTO), "实测").can_disable is True
|
||
assert ThinkingCapability((Effort.LOW, Effort.MAX), "实测").can_disable is False
|
||
|
||
def test_cheapest_effort_skips_none(self):
|
||
# 「关不掉时的可执行替代」取的是除 none 外最弱的一档
|
||
assert (
|
||
ThinkingCapability((Effort.LOW, Effort.HIGH, Effort.MAX), "实测").cheapest_effort
|
||
is Effort.LOW
|
||
)
|
||
assert (
|
||
ThinkingCapability((Effort.NONE, Effort.HIGH, Effort.MAX), "实测").cheapest_effort
|
||
is Effort.HIGH
|
||
)
|
||
assert ThinkingCapability((Effort.NONE, Effort.AUTO), "实测").cheapest_effort is Effort.AUTO
|
||
assert ThinkingCapability((Effort.AUTO,), "实测").cheapest_effort is Effort.AUTO
|
||
|
||
def test_cheapest_effort_is_none_when_only_none(self):
|
||
# 只能关不能开: 没有可推荐的「最省的开启档」
|
||
assert ThinkingCapability((Effort.NONE,), "实测").cheapest_effort is None
|
||
|
||
def test_is_tiered_excludes_none_and_auto(self):
|
||
# 纯开关型模型不该被告知「可选档位」——它没有档位
|
||
assert ThinkingCapability((Effort.NONE, Effort.AUTO), "实测").is_tiered is False
|
||
assert ThinkingCapability((Effort.AUTO,), "实测").is_tiered is False
|
||
assert ThinkingCapability((Effort.LOW, Effort.MAX), "实测").is_tiered is True
|
||
|
||
def test_empty_efforts_rejected(self):
|
||
with pytest.raises(ValueError, match="至少"):
|
||
ThinkingCapability((), "实测")
|
||
|
||
def test_duplicate_efforts_rejected(self):
|
||
with pytest.raises(ValueError, match="重复"):
|
||
ThinkingCapability((Effort.LOW, Effort.LOW), "实测")
|
||
|
||
def test_glm53_cannot_be_disabled(self):
|
||
# 三源一致(智谱官方文档/cherry-studio/OpenRouter): thinking.type 只接受 enabled
|
||
cap = get_capability("glm-5.3")
|
||
assert cap is not None
|
||
assert cap.can_disable is False
|
||
assert cap.cheapest_effort is Effort.LOW
|
||
|
||
def test_m2_series_still_cannot_be_disabled(self):
|
||
# 迁移回归: 旧表用 can_disable=False 表达的事实,新表用「none 不在清单里」表达
|
||
assert get_capability("MiniMax-M2.7").can_disable is False
|
||
assert get_capability("MiniMax-M2.5").can_disable is False
|
||
assert get_capability("MiniMax-M3").can_disable is True
|
||
|
||
|
||
class TestEffectiveEffort:
|
||
"""三层优先级的**唯一**判定处(设计 §4.2): 请求级 > 源级 > 语法糖 > 不表态。
|
||
|
||
收口成一个纯函数,是因为它此前在装配守卫与 transport 里各写了一份就地转换:
|
||
两份各自演化的判定,迟早会在"装配期放行、运行期报错"这种最难查的形态上分叉。
|
||
"""
|
||
|
||
def test_request_beats_source(self):
|
||
assert (
|
||
effective_effort(
|
||
request_effort=Effort.MAX, source_effort=Effort.LOW, enable_thinking=None
|
||
)
|
||
is Effort.MAX
|
||
)
|
||
|
||
def test_source_beats_sugar(self):
|
||
assert (
|
||
effective_effort(request_effort=None, source_effort=Effort.HIGH, enable_thinking=None)
|
||
is Effort.HIGH
|
||
)
|
||
|
||
def test_none_request_does_not_clear_source(self):
|
||
"""请求级"没表态"绝不能被读成"要求关闭"——那会静默改掉源级的默认档。"""
|
||
assert (
|
||
effective_effort(request_effort=None, source_effort=Effort.LOW, enable_thinking=None)
|
||
is Effort.LOW
|
||
)
|
||
|
||
def test_request_none_tier_is_an_opinion(self):
|
||
"""`Effort.NONE` 是一次明确的表态,必须压过源级档位而不是被当成缺省。"""
|
||
assert (
|
||
effective_effort(
|
||
request_effort=Effort.NONE, source_effort=Effort.MAX, enable_thinking=None
|
||
)
|
||
is Effort.NONE
|
||
)
|
||
|
||
def test_enable_thinking_true_is_auto(self):
|
||
"""`True` → `auto`(开启但不指定强度),而**不是**旧版硬编码的 medium。"""
|
||
assert (
|
||
effective_effort(request_effort=None, source_effort=None, enable_thinking=True)
|
||
is Effort.AUTO
|
||
)
|
||
|
||
def test_enable_thinking_false_is_the_none_tier(self):
|
||
assert (
|
||
effective_effort(request_effort=None, source_effort=None, enable_thinking=False)
|
||
is Effort.NONE
|
||
)
|
||
|
||
def test_sugar_is_the_last_word_only(self):
|
||
"""语法糖排在最末: 显式配了档位就以档位为准(矛盾组合已被构造期挡下)。"""
|
||
assert (
|
||
effective_effort(request_effort=None, source_effort=Effort.LOW, enable_thinking=True)
|
||
is Effort.LOW
|
||
)
|
||
|
||
def test_all_absent_is_no_opinion(self):
|
||
"""三层都不表态 → None(随模型默认),与 `Effort.NONE` 严格区分。"""
|
||
assert (
|
||
effective_effort(request_effort=None, source_effort=None, enable_thinking=None) is None
|
||
)
|