"""真实 API 验证推理开关与推理可观测性(issue #5 + #6;判据于 #16/#17 重建)。 本组用例**必须真跑**: 改动的正确性与具体模型强相关,mock 只能验证代码路径, 验证不了"这个参数在这个模型上到底关没关掉推理"。 三条判据纪律(第 1、2 条来自 findings §4c,第 1 条的推翻与第 3 条来自 `findings/2026-08-25-thinking-observability-regression.md`): 1. **判别量是库裁定的三态 `thinking_observation`,既不是 `reasoning_tokens` 也不是 `completion_tokens`。** 长度判据早已排除: 两档的输出长度分布**是 重叠的**(实测关闭档最高 46 token、开启档最低 13 token),按阈值判两边都会 误判。而 `reasoning_tokens` 这个曾经"干净分开"的判据也已失效——MiniMax 这一路上游不再返回 `usage.completion_tokens_details`,该字段恒 `None`;同一 次调用里库明明拿得到 185 字符推理正文,单看 token 计数却把"推理正常"读成 "没推理"(2026-08-25 findings §3.4/结论③,四条用例因此假红)。三态裁定同时 看正文与计数: **正文是事实本身,token 计数只是对事实的转述**。 2. **另配一个不含魔数的确定性锚点**(见 L2b、L5): 同一模型上,关闭档的 `prompt_tokens` 严格小于开启档——供应商在开启时注入了推理指令,输入侧 token 数随之变大。这是相对比较,不硬编码任何具体数值;且它不依赖上游是否 回传推理正文,所以在"观测不到推理"的非流式路径上依然作数。 3. **`UNKNOWN` 不等于"没推理",不能拿它判红。** 关闭方向要求每轮"未观测到 推理"(`UNKNOWN` 计入满足——它没有证伪力),其证伪力来自: 模型若偷偷推理了, 可观测路径会翻成 `OBSERVED`。开启方向只要求多数轮 `OBSERVED`;M3 非流式 路径整片观测不到,该档由 L5 用另一套断言覆盖。 源不可用一律 `skip` 并在报告中记为「未覆盖」,**绝不静默计入通过**。 """ import dataclasses import json import os from collections import Counter from datetime import datetime from pathlib import Path import pytest from dotenv import dotenv_values from polygateway import GatewayClient, GatewaySettings, ThinkingObservation from polygateway.errors import ( AllSourcesExhausted, RequestRejectedError, SourceDeadError, TransientError, ) from polygateway.thinking import DEFAULT_CAPABILITIES, get_capability _ENV = {k: v for k, v in {**dotenv_values(".env"), **os.environ}.items() if v is not None} _HAS_SOURCE = any(k.split("__")[0] == "LLM" and k.endswith("__API_KEY") for k in _ENV) # slow: 本组 92 次真实调用、约 4 分半(2026-08-26 判据换三态后实测;此前记的 # "137 次、约 7 分钟"已被证伪,别照旧值估 CI 预算),且判据是统计性的——网络抖动 # 会让它偶发失败(实测有一次 network_error 连续三次耗尽源)。让它阻断 `make ci` # 会把测试变成噪声源,故沿用项目既有的 slow 标记默认排除,合并前用 `-m slow` # 显式真跑并存档报告。"不自动门控"不等于"可跳过"。 pytestmark = [ pytest.mark.slow, pytest.mark.skipif( not _HAS_SOURCE, reason="需真实网关凭据: 在 .env 配置 LLM__{PROVIDER}__1__*(本组必须真跑)" ), ] _OUT_DIR = Path("tests/outputs/e2e") _ROUNDS = int(os.environ.get("PGW_E2E_THINKING_ROUNDS", "10")) # 需要一点推理才能答对,但答案极短: 关掉推理时 completion 稳定在个位数, # 开着时则是几百——两档之间隔着一个数量级,判据不必卡在噪声里 _PROMPT = "一个笼子里有若干鸡和兔,共 35 个头、94 只脚。鸡和兔各有多少只?只输出两个数字。" _ROWS: list[dict] = [] # 显式映射,不按模型名猜 provider —— 那正是 D11 要消灭的东西(providers.py 开篇)。 # 漏登记会被 test_every_capability_has_a_provider_mapping 当场抓住,而不是 # 在 L8 里被"源不可用"这个假理由吞掉 _MODEL_PROVIDER = { "MiniMax-M3": "minimax", "MiniMax-M2.7": "minimax", "MiniMax-M2.5": "minimax", "qwen3.7-plus": "qwen", "deepseek-v4-pro": "deepseek", } def _base_settings() -> GatewaySettings: # 强制关缓存: 多轮测量要求每一轮都真的打到供应商,命中缓存会把后续轮次 # 变成对第一轮的回放,整组判据随之失效 return GatewaySettings.from_env("LLM", env={**_ENV, "PGW_CACHE_BACKEND": "none"}) def _settings(**source_overrides) -> GatewaySettings: base = _base_settings() source = dataclasses.replace(base.sources[0], **source_overrides) return dataclasses.replace(base, sources=(source,)) async def _run_rounds(rounds: int, *, stream: bool = True, **source_overrides) -> list[dict]: """跑 N 轮真实调用,返回逐轮观测;任一轮抛错即向上冒泡由用例决定处置。""" client = GatewayClient.from_settings(_settings(**source_overrides)) observations = [] try: for i in range(rounds): resp = await client.chat( [{"role": "user", "content": _PROMPT}], stream=stream, # 每轮独立 salt: 即便某层缓存意外开着也不会回放 cache_salt=f"thinking-live-{i}", ) observations.append( { "round": i + 1, "prompt_tokens": resp.prompt_tokens, "completion_tokens": resp.completion_tokens, "reasoning_tokens": resp.reasoning_tokens, # 结论与证据一起入报告: 只记 observation 会让"为什么这么判" # 不可复核,而 thinking_chars 正是本次改判的直接证据 "thinking_observation": resp.thinking_observation, "thinking_chars": len(resp.thinking), "content": resp.content[:60], } ) finally: await client.aclose() return observations def _record(matrix_id: str, desc: str, status: str, detail, observations=None) -> None: _ROWS.append( { "matrix": matrix_id, "desc": desc, "status": status, "detail": detail, "observations": observations or [], } ) def _reasoning_off(obs: dict) -> bool: """关闭方向: 只要没观测到推理即算满足。 `UNKNOWN` 计入满足是有意的: 它没有证伪力(本次无任何信号,判不出来),拿它 判红等于每次关闭调用都喊一遍。本判据真正的证伪力在于——模型若偷偷推理了, 可观测路径会把裁定翻成 `OBSERVED`。 **刻意不设 completion_tokens 上限**: 实测关闭档偶尔会到 46 token(模型没照做 "只输出两个数字",把解题过程写进了正文),而那是正文不是推理。加长度门只会 把这种正常波动误判成"没关掉"。 """ return obs["thinking_observation"] != ThinkingObservation.OBSERVED def _reasoning_on(obs: dict) -> bool: """开启方向: 观测到推理即为真。 判据从 `reasoning_tokens` 换成库的三态裁定,因为 MiniMax 这一路已不再上报 `completion_tokens_details`(2026-08-25 findings 结论②),该字段恒 `None`; 而库在同一次调用里拿得到 185 字符推理正文(findings §3.4)——旧判据看不见 它,L2/L3b/L4/L5 四条因此假红。 也不能退回 completion_tokens 当判据: medium 档的推理量方差极大(实测 15 轮 跨 7-170 token),两档分布还与关闭档重叠,按长度阈值判会把"推理了但想得少" 误判成没推理。 """ return obs["thinking_observation"] == ThinkingObservation.OBSERVED def _skip_if_unreachable(exc: Exception, matrix_id: str, desc: str): """源不可用(渠道下线/模型未开通)→ 跳过并记为未覆盖,不伪装成通过。""" _record(matrix_id, desc, "SKIP(源不可用)", str(exc)[:200]) pytest.skip(f"{matrix_id} 源不可用,已记为未覆盖: {str(exc)[:120]}") @pytest.fixture(scope="module", autouse=True) def _write_report(): yield _OUT_DIR.mkdir(parents=True, exist_ok=True) ts = datetime.now().strftime("%Y%m%d_%H%M%S") path = _OUT_DIR / f"test_thinking_live_{ts}.md" lines = [ "# 推理开关与推理可观测性真实 API 验证", "", f"- 时间: {ts}", f"- 每档轮数: {_ROUNDS}", "- 判别量: 库裁定的三态 `thinking_observation`(OBSERVED/ABSENT/UNKNOWN)," "由推理正文与 reasoning_tokens 共同裁定 —— 正文是事实,token 计数只是转述", "- 关闭判据: **每轮** observation != OBSERVED(UNKNOWN 计入满足,它没有证伪力);" "刻意不设输出长度上限(两档的 completion 分布重叠: 实测关闭档最高 46、开启档最低 13)", "- 开启判据: **多数轮** observation == OBSERVED", "- 确定性锚点(L2b、L5): 关闭档 prompt_tokens 最大值 < 开启档最小值,相对比较无魔数", "- L5(非流式): M3 该路径推理已计费却不回传正文,故不断言「观测到推理」," "改断锚点可分 + 开启档不被误判为 ABSENT", "", "## 矩阵结论", "", "| 矩阵 | 场景 | 结论 | 说明 |", "|---|---|---|---|", ] total_calls = 0 for row in _ROWS: detail = str(row["detail"]).replace("|", "\\|").replace("\n", " ")[:160] lines.append(f"| {row['matrix']} | {row['desc']} | {row['status']} | {detail} |") total_calls += len(row["observations"]) lines += ["", f"**总真实调用次数: {total_calls}**", "", "## 逐轮原始观测", ""] for row in _ROWS: if not row["observations"]: continue lines += [f"### {row['matrix']} — {row['desc']}", "", "```json"] lines.append(json.dumps(row["observations"], ensure_ascii=False, indent=2)) lines += ["```", ""] uncovered = [r["matrix"] for r in _ROWS if r["status"].startswith("SKIP")] if uncovered: lines += ["## 未覆盖", "", f"以下矩阵行未跑到: {', '.join(uncovered)}", ""] path.write_text("\n".join(lines), encoding="utf-8") print(f"\n[e2e 报告] {path}") class TestMiniMaxM3: """M3 是唯一实测可关闭推理的 MiniMax 模型,修复的地基压在它身上。""" async def test_l1_disable_actually_disables(self): obs = await _run_rounds(_ROUNDS, model="MiniMax-M3", enable_thinking=False) offs = [o for o in obs if _reasoning_off(o)] _record( "L1", "enable_thinking=False(流式)", "PASS" if len(offs) == len(obs) else "FAIL", f"{len(offs)}/{len(obs)} 轮未观测到推理", obs, ) assert len(offs) == len(obs), f"关闭方向要求每轮满足: {obs}" async def test_l2_enable_actually_enables(self): obs = await _run_rounds(_ROUNDS, model="MiniMax-M3", enable_thinking=True) ons = [o for o in obs if _reasoning_on(o)] _record( "L2", "enable_thinking=True(流式,注入 medium)", "PASS" if len(ons) * 2 > len(obs) else "FAIL", f"{len(ons)}/{len(obs)} 轮观察到推理", obs, ) assert len(ons) * 2 > len(obs), f"开启方向要求多数轮满足: {obs}" async def test_l2b_off_and_on_are_distinguishable_without_magic_numbers(self): """确定性锚点: 开启档的 prompt_tokens 严格大于关闭档。 供应商在开启推理时会向模板注入推理指令,输入侧 token 数随之变大。这是 本组唯一不依赖输出侧噪声的证据,且是相对比较——不硬编码任何具体数值, 供应商改模板也不会让它假红。 """ rounds = max(3, _ROUNDS // 3) off = await _run_rounds(rounds, model="MiniMax-M3", enable_thinking=False) on = await _run_rounds(rounds, model="MiniMax-M3", enable_thinking=True) off_max = max(o["prompt_tokens"] for o in off) on_min = min(o["prompt_tokens"] for o in on) _record( "L2b", "关闭/开启的 prompt_tokens 可分", "PASS" if off_max < on_min else "FAIL", f"关闭档最大 {off_max} < 开启档最小 {on_min}", off + on, ) assert off_max < on_min, ( f"两档的 prompt_tokens 未分开(关闭最大 {off_max},开启最小 {on_min}): 注入可能没到达模型" ) async def test_l3_no_opinion_is_the_model_default(self): obs = await _run_rounds(_ROUNDS, model="MiniMax-M3", enable_thinking=None) # M3 的默认档实测就是不推理(findings §2.1),所以不干预时也应观测不到推理。 # 注意这**不能**反过来证明关闭方向生效 —— L1 与本行同分布,区分二者的是 # L2b 的 prompt_tokens 与 L3b 的乱码值反证 quiet = [o for o in obs if _reasoning_off(o)] _record( "L3", "enable_thinking=None(不干预,基线)", "PASS" if len(quiet) == len(obs) else "FAIL", f"{len(quiet)}/{len(obs)} 轮未观测到推理(M3 默认档本就不推理)", obs, ) assert len(quiet) == len(obs), f"M3 默认档不应推理: {obs}" async def test_l3b_none_is_recognised_not_silently_dropped(self): """反证: 关闭方向的观测必须排除"参数被静默丢弃"这一伪解释。 L1(关闭)与 L3(不干预)在 M3 上**同分布**——因为 M3 默认档本就不推理。 所以 L1 单独看不能区分"`none` 真的被消费"与"`none` 被中转吞了",而后者 正是 issue #5 的原始故障形态(`enable_thinking` 就是这么被吞的)。 判别方法: 发一个**非法值**。若未知值会被静默丢弃,它的表现应与"不注入" 一致(不推理);实测它反而开启了推理,说明网关认这个键、只是不认这个值。 既然非法值与 `none` 的表现不同,`none` 就必然是被识别的枚举值。 **该手法不可移植,只对"认这个键但不校验值"的 provider 成立**: minimax 对 非法 `reasoning_effort` 返回 200 且照常推理(2026-08-25 findings §5: prompt 207,介于基线 194 与 medium 216 之间,走了第三条模板路径);而 qwen 对同样的值直接返回 **HTTP 400**。把本用例套到 qwen 那类会校验值的 provider 上,拿到的会是异常而非"不推理",是假红。 """ rounds = max(3, _ROUNDS // 3) bogus = await _run_rounds( rounds, model="MiniMax-M3", enable_thinking=None, extra_body={"reasoning_effort": "definitely-not-a-real-level"}, ) off = await _run_rounds(rounds, model="MiniMax-M3", enable_thinking=False) bogus_on = [o for o in bogus if _reasoning_on(o)] off_quiet = [o for o in off if _reasoning_off(o)] ok = len(bogus_on) * 2 > len(bogus) and len(off_quiet) == len(off) _record( "L3b", "非法值反证 none 被识别", "PASS" if ok else "FAIL", f"非法值 {len(bogus_on)}/{len(bogus)} 轮推理,none {len(off_quiet)}/{len(off)} 轮不推理" "(两者表现不同 ⇒ none 非被丢弃)", bogus + off, ) assert len(bogus_on) * 2 > len(bogus), ( f"非法值未开启推理,无法排除'未知值被静默丢弃'这一伪解释: {bogus}" ) assert len(off_quiet) == len(off), f"none 未关闭推理: {off}" async def test_l4_extra_body_overrides_the_profile(self): """profile 注入 none,extra_body 要求 high —— 后者必须赢(优先级不可调换)。 判据是行为而非报文: 若 extra_body 没赢,拿到的就是 none 的结果(不推理)。 """ rounds = max(3, _ROUNDS // 2) obs = await _run_rounds( rounds, model="MiniMax-M3", enable_thinking=False, extra_body={"reasoning_effort": "high"}, ) ons = [o for o in obs if _reasoning_on(o)] _record( "L4", "extra_body 覆盖 profile 注入", "PASS" if len(ons) * 2 > len(obs) else "FAIL", f"{len(ons)}/{len(obs)} 轮观察到推理(证明 high 生效而非 none)", obs, ) assert len(ons) * 2 > len(obs), f"extra_body 未能覆盖 profile: {obs}" async def test_l5_non_stream_path_is_distinguishable_and_honestly_unknown(self): """非流式快路径: 参数确实到达了模型,而推理信号被如实标成"观测不到"。 **本用例不能断言"非流式开启档观测到推理"——那永远不成立**: M3 在非流式 路径下推理段确实产生并计费(2026-08-25 findings §3.4: 开启档 completion 53 vs 关闭档 3),但 `message` 里没有 `reasoning_content`、`usage` 里也没有 `completion_tokens_details`,推理内容整体不回传。**这是上游行为,库修不了; 库能做也必须做的是让它可见**——下游在为看不见的东西付费,不该由库替它 沉默。 故改断两件在非流式下真实成立的事: 其一 `prompt_tokens` 锚点仍把两档分开(判据形态照抄 L2b,证明注入到达了模型, 排除"非流式路径把参数弄丢了"这一伪解释); 其二开启档的裁定**不是 `ABSENT`**——`ABSENT` 的语义是"上游明确上报未推理", 而实情是"判不出来"(`UNKNOWN`),库若把后者伪装成前者,正是 issue #16/#17 里 那个静默错觉。这里断 `!= ABSENT` 而非 `== UNKNOWN`,是为了留出上游哪天开始 回传正文的余地: 那时裁定会翻成 `OBSERVED`,是好事,不该让它把测试判红。 """ rounds = max(3, _ROUNDS // 2) off = await _run_rounds(rounds, stream=False, model="MiniMax-M3", enable_thinking=False) on = await _run_rounds(rounds, stream=False, model="MiniMax-M3", enable_thinking=True) offs = [o for o in off if _reasoning_off(o)] off_max = max(o["prompt_tokens"] for o in off) on_min = min(o["prompt_tokens"] for o in on) not_absent = [o for o in on if o["thinking_observation"] != ThinkingObservation.ABSENT] on_states = Counter(str(o["thinking_observation"]) for o in on) ok = len(offs) == len(off) and off_max < on_min and len(not_absent) == len(on) _record( "L5", "非流式: prompt 锚点可分 + 开启档如实标 UNKNOWN 而非 ABSENT", "PASS" if ok else "FAIL", f"关闭 {len(offs)}/{len(off)} 轮未观测到推理;" f"关闭档 prompt 最大 {off_max} < 开启档最小 {on_min};" f"开启档裁定分布 {dict(on_states)}", off + on, ) assert len(offs) == len(off), f"非流式关闭方向未满足: {off}" assert off_max < on_min, ( f"非流式两档 prompt_tokens 未分开(关闭最大 {off_max},开启最小 {on_min}): " f"开启参数可能没到达模型" ) assert len(not_absent) == len(on), ( f"非流式开启档被裁成 ABSENT(声称上游明确上报未推理),而实情是观测不到: {on}" ) class TestOtherProviders: """qwen / deepseek 的 profile 是既有实现,本组防的是"改 minimax 时误伤它们"。""" @pytest.mark.parametrize( ("matrix", "provider", "model"), [("L6", "qwen", "qwen3.7-plus"), ("L7", "deepseek", "deepseek-v4-pro")], ) async def test_existing_profiles_still_disable(self, matrix, provider, model): desc = f"{provider} enable_thinking=False" try: obs = await _run_rounds(_ROUNDS, provider=provider, model=model, enable_thinking=False) except (AllSourcesExhausted, SourceDeadError, TransientError) as exc: # 只吞网关/网络类失败。**不吞 ValueError / RequestRejected** —— # 那两类正是本次改动最可能的误伤方向,吞掉就成了纪律(c)要防的静默 _skip_if_unreachable(exc, matrix, desc) offs = [o for o in obs if _reasoning_off(o)] _record( matrix, desc, "PASS" if len(offs) == len(obs) else "FAIL", f"{len(offs)}/{len(obs)} 轮未观测到推理", obs, ) assert len(offs) == len(obs), f"{provider} 关闭方向未满足: {obs}" async def test_qwen_enabled_is_observed(self): """设计 §14 验收: qwen 开启档必须裁定为 `OBSERVED`,不是 `UNKNOWN`。 本条是三态裁定的**跨供应商对照组**: MiniMax 这一路两个信号都可能缺失 (非流式档整片 `UNKNOWN`),若只按它调判据,很容易把"观测不到"当成常态; qwen 在同一网关同一 key 上照常返回推理信号(findings 2026-08-25 §2), 故这里能且必须要求正面结论——它一旦掉成 `UNKNOWN`,说明的是库的组装路径 丢了信号,而不是上游行为变了。 """ matrix, provider, model = "L6b", "qwen", "qwen3.7-plus" desc = f"{provider} enable_thinking=True" try: obs = await _run_rounds(_ROUNDS, provider=provider, model=model, enable_thinking=True) except (AllSourcesExhausted, SourceDeadError, TransientError) as exc: _skip_if_unreachable(exc, matrix, desc) ons = [o for o in obs if _reasoning_on(o)] _record( matrix, desc, "PASS" if len(ons) * 2 > len(obs) else "FAIL", f"{len(ons)}/{len(obs)} 轮观测到推理(OBSERVED)", obs, ) assert len(ons) * 2 > len(obs), f"{provider} 开启方向要求多数轮 OBSERVED: {obs}" class TestCapabilityDrift: """L8 漂移哨兵: 能力表过期是必然事件,这里是它的过期告警。""" def test_every_capability_has_a_provider_mapping(self): """能力表新增条目必须同步本测试的映射,否则该行会被静默跳过。""" missing = sorted(set(DEFAULT_CAPABILITIES) - set(_MODEL_PROVIDER)) assert not missing, f"这些模型缺 provider 映射,L8 会漏测: {missing}" @pytest.mark.parametrize("model", sorted(DEFAULT_CAPABILITIES)) async def test_declared_capability_matches_reality(self, model): cap = get_capability(model) provider = _MODEL_PROVIDER[model] rounds = max(3, _ROUNDS // 2) desc = f"{model} 声明 can_disable={cap.can_disable}" if not cap.can_disable: # 声明关不掉: 装配期就该炸,炸了即与声明一致(不必真调用) with pytest.raises(ValueError, match=model): GatewayClient.from_settings( _settings(provider=provider, model=model, enable_thinking=False) ) _record("L8", desc, "PASS", "装配期按声明拒绝,与实测一致") return try: obs = await _run_rounds(rounds, provider=provider, model=model, enable_thinking=False) except (AllSourcesExhausted, SourceDeadError, TransientError) as exc: _skip_if_unreachable(exc, "L8", desc) offs = [o for o in obs if _reasoning_off(o)] verdict = Counter(_reasoning_off(o) for o in obs) _record( "L8", desc, "PASS" if len(offs) == len(obs) else "FAIL(能力表已漂移)", f"实测未观测到推理 {dict(verdict)}(True=满足);声明 can_disable=True 要求每轮满足", obs, ) assert len(offs) == len(obs), ( f"能力表漂移: {model} 声明可关闭推理,实测未关掉 —— 请复测后更新 DEFAULT_CAPABILITIES" ) class TestAssemblyGuardAgainstRealConfig: """L9: 纯本地,但用的是 .env 里的真实配置形态,防"守卫只在合成配置上生效"。""" def test_l9_m27_rejected_at_assembly(self): with pytest.raises(ValueError, match="MiniMax-M2.7"): GatewayClient.from_settings( _settings(provider="minimax", model="MiniMax-M2.7", enable_thinking=False) ) _record("L9", "M2.7 + enable_thinking=False", "PASS", "装配期报错,未发出任何请求") def test_l9_unknown_shape_rejected_at_assembly(self): with pytest.raises(ValueError, match="register_provider"): GatewayClient.from_settings( _settings(provider="openai", model="kimi-k3", enable_thinking=False) ) _record("L9", "provider=openai 形态未知", "PASS", "装配期报错并指路") async def test_transport_layer_rejects_when_guard_is_bypassed(self): """构造函数全量注入这条路绕过装配守卫,transport 必须兜住并归四分类。""" settings = _settings(provider="minimax", model="MiniMax-M2.7", enable_thinking=False) client = GatewayClient.from_settings( dataclasses.replace( settings, sources=(dataclasses.replace(settings.sources[0], enable_thinking=None),) ) ) try: # 装配用 None 绕过守卫,再把源换成 False 直接喂给 transport bad = dataclasses.replace(settings.sources[0], enable_thinking=False) with pytest.raises(RequestRejectedError, match="MiniMax-M2.7"): await client._terminal._transport.complete( messages=[{"role": "user", "content": _PROMPT}], source=bad, stream=True, overlay={}, call_id="e2e-guard", ) finally: await client.aclose() _record("L9", "绕过装配守卫时 transport 兜底", "PASS", "RequestRejectedError,属四分类")