iomgaa
80a8013642
The Transport port took the request apart into five arguments, so a tier placed on ChatRequest could never reach _build_payload: the field was set, read by nobody, and silently ignored - the exact shape of failure that sent downstream to extra_body in the first place. complete() now takes reasoning_effort with no default, matching the TelemetryRecorder convention: a default would turn a missing hand-off into a silent 'no opinion'. All four fakes move with it, since @runtime_checkable checks method names and not signatures. EmbeddingTransport and OcrTransport are deliberately left alone - they have no reasoning semantics - and a test now holds that line. _build_payload drops its inline sugar conversion for effective_effort(), so the guard and the hot path share one judgement, and passes the source's effort_fallback for the same reason.
522 lines
26 KiB
Python
522 lines
26 KiB
Python
"""真实 API 验证推理开关与推理可观测性(issue #5 + #6;判据于 #16/#17 重建)。
|
|
|
|
本组用例**必须真跑**: 改动的正确性与具体模型强相关,mock 只能验证代码路径,
|
|
验证不了"这个参数在这个模型上到底关没关掉推理"。
|
|
|
|
三条判据纪律(第 1、2 条来自 findings §4c,第 1 条的推翻与第 3 条来自
|
|
`findings/2026-08-25-thinking-observability-regression.md`):
|
|
|
|
1. **判别量是库裁定的三态 `thinking_observation`,既不是 `reasoning_tokens`
|
|
也不是 `completion_tokens`。** 长度判据早已排除: 两档的输出长度分布**是
|
|
重叠的**(实测关闭档最高 46 token、开启档最低 13 token),按阈值判两边都会
|
|
误判。而 `reasoning_tokens` 这个曾经"干净分开"的判据也已失效——MiniMax
|
|
这一路上游不再返回 `usage.completion_tokens_details`,该字段恒 `None`;同一
|
|
次调用里库明明拿得到 185 字符推理正文,单看 token 计数却把"推理正常"读成
|
|
"没推理"(2026-08-25 findings §3.4/结论③,四条用例因此假红)。三态裁定同时
|
|
看正文与计数: **正文是事实本身,token 计数只是对事实的转述**。
|
|
2. **另配一个不含魔数的确定性锚点**(见 L2b、L5): 同一模型上,关闭档的
|
|
`prompt_tokens` 严格小于开启档——供应商在开启时注入了推理指令,输入侧
|
|
token 数随之变大。这是相对比较,不硬编码任何具体数值;且它不依赖上游是否
|
|
回传推理正文,所以在"观测不到推理"的非流式路径上依然作数。
|
|
3. **`UNKNOWN` 不等于"没推理",不能拿它判红。** 关闭方向要求每轮"未观测到
|
|
推理"(`UNKNOWN` 计入满足——它没有证伪力),其证伪力来自: 模型若偷偷推理了,
|
|
可观测路径会翻成 `OBSERVED`。开启方向只要求多数轮 `OBSERVED`;M3 非流式
|
|
路径整片观测不到,该档由 L5 用另一套断言覆盖。
|
|
|
|
源不可用一律 `skip` 并在报告中记为「未覆盖」,**绝不静默计入通过**。
|
|
"""
|
|
|
|
import dataclasses
|
|
import json
|
|
import os
|
|
from collections import Counter
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from dotenv import dotenv_values
|
|
|
|
from polygateway import GatewayClient, GatewaySettings, ThinkingObservation
|
|
from polygateway.errors import (
|
|
AllSourcesExhausted,
|
|
RequestRejectedError,
|
|
SourceDeadError,
|
|
TransientError,
|
|
)
|
|
from polygateway.thinking import DEFAULT_CAPABILITIES, get_capability
|
|
|
|
_ENV = {k: v for k, v in {**dotenv_values(".env"), **os.environ}.items() if v is not None}
|
|
_HAS_SOURCE = any(k.split("__")[0] == "LLM" and k.endswith("__API_KEY") for k in _ENV)
|
|
|
|
# slow: 本组 92 次真实调用、约 4 分半(2026-08-26 判据换三态后实测;此前记的
|
|
# "137 次、约 7 分钟"已被证伪,别照旧值估 CI 预算),且判据是统计性的——网络抖动
|
|
# 会让它偶发失败(实测有一次 network_error 连续三次耗尽源)。让它阻断 `make ci`
|
|
# 会把测试变成噪声源,故沿用项目既有的 slow 标记默认排除,合并前用 `-m slow`
|
|
# 显式真跑并存档报告。"不自动门控"不等于"可跳过"。
|
|
pytestmark = [
|
|
pytest.mark.slow,
|
|
pytest.mark.skipif(
|
|
not _HAS_SOURCE, reason="需真实网关凭据: 在 .env 配置 LLM__{PROVIDER}__1__*(本组必须真跑)"
|
|
),
|
|
]
|
|
|
|
_OUT_DIR = Path("tests/outputs/e2e")
|
|
_ROUNDS = int(os.environ.get("PGW_E2E_THINKING_ROUNDS", "10"))
|
|
|
|
# 需要一点推理才能答对,但答案极短: 关掉推理时 completion 稳定在个位数,
|
|
# 开着时则是几百——两档之间隔着一个数量级,判据不必卡在噪声里
|
|
_PROMPT = "一个笼子里有若干鸡和兔,共 35 个头、94 只脚。鸡和兔各有多少只?只输出两个数字。"
|
|
|
|
_ROWS: list[dict] = []
|
|
|
|
# 显式映射,不按模型名猜 provider —— 那正是 D11 要消灭的东西(providers.py 开篇)。
|
|
# 漏登记会被 test_every_capability_has_a_provider_mapping 当场抓住,而不是
|
|
# 在 L8 里被"源不可用"这个假理由吞掉
|
|
_MODEL_PROVIDER = {
|
|
"MiniMax-M3": "minimax",
|
|
"MiniMax-M2.7": "minimax",
|
|
"MiniMax-M2.5": "minimax",
|
|
"qwen3.7-plus": "qwen",
|
|
"deepseek-v4-pro": "deepseek",
|
|
}
|
|
|
|
|
|
def _base_settings() -> GatewaySettings:
|
|
# 强制关缓存: 多轮测量要求每一轮都真的打到供应商,命中缓存会把后续轮次
|
|
# 变成对第一轮的回放,整组判据随之失效
|
|
return GatewaySettings.from_env("LLM", env={**_ENV, "PGW_CACHE_BACKEND": "none"})
|
|
|
|
|
|
def _settings(**source_overrides) -> GatewaySettings:
|
|
base = _base_settings()
|
|
source = dataclasses.replace(base.sources[0], **source_overrides)
|
|
return dataclasses.replace(base, sources=(source,))
|
|
|
|
|
|
async def _run_rounds(rounds: int, *, stream: bool = True, **source_overrides) -> list[dict]:
|
|
"""跑 N 轮真实调用,返回逐轮观测;任一轮抛错即向上冒泡由用例决定处置。"""
|
|
client = GatewayClient.from_settings(_settings(**source_overrides))
|
|
observations = []
|
|
try:
|
|
for i in range(rounds):
|
|
resp = await client.chat(
|
|
[{"role": "user", "content": _PROMPT}],
|
|
stream=stream,
|
|
# 每轮独立 salt: 即便某层缓存意外开着也不会回放
|
|
cache_salt=f"thinking-live-{i}",
|
|
)
|
|
observations.append(
|
|
{
|
|
"round": i + 1,
|
|
"prompt_tokens": resp.prompt_tokens,
|
|
"completion_tokens": resp.completion_tokens,
|
|
"reasoning_tokens": resp.reasoning_tokens,
|
|
# 结论与证据一起入报告: 只记 observation 会让"为什么这么判"
|
|
# 不可复核,而 thinking_chars 正是本次改判的直接证据
|
|
"thinking_observation": resp.thinking_observation,
|
|
"thinking_chars": len(resp.thinking),
|
|
"content": resp.content[:60],
|
|
}
|
|
)
|
|
finally:
|
|
await client.aclose()
|
|
return observations
|
|
|
|
|
|
def _record(matrix_id: str, desc: str, status: str, detail, observations=None) -> None:
|
|
_ROWS.append(
|
|
{
|
|
"matrix": matrix_id,
|
|
"desc": desc,
|
|
"status": status,
|
|
"detail": detail,
|
|
"observations": observations or [],
|
|
}
|
|
)
|
|
|
|
|
|
def _reasoning_off(obs: dict) -> bool:
|
|
"""关闭方向: 只要没观测到推理即算满足。
|
|
|
|
`UNKNOWN` 计入满足是有意的: 它没有证伪力(本次无任何信号,判不出来),拿它
|
|
判红等于每次关闭调用都喊一遍。本判据真正的证伪力在于——模型若偷偷推理了,
|
|
可观测路径会把裁定翻成 `OBSERVED`。
|
|
|
|
**刻意不设 completion_tokens 上限**: 实测关闭档偶尔会到 46 token(模型没照做
|
|
"只输出两个数字",把解题过程写进了正文),而那是正文不是推理。加长度门只会
|
|
把这种正常波动误判成"没关掉"。
|
|
"""
|
|
return obs["thinking_observation"] != ThinkingObservation.OBSERVED
|
|
|
|
|
|
def _reasoning_on(obs: dict) -> bool:
|
|
"""开启方向: 观测到推理即为真。
|
|
|
|
判据从 `reasoning_tokens` 换成库的三态裁定,因为 MiniMax 这一路已不再上报
|
|
`completion_tokens_details`(2026-08-25 findings 结论②),该字段恒 `None`;
|
|
而库在同一次调用里拿得到 185 字符推理正文(findings §3.4)——旧判据看不见
|
|
它,L2/L3b/L4/L5 四条因此假红。
|
|
|
|
也不能退回 completion_tokens 当判据: medium 档的推理量方差极大(实测 15 轮
|
|
跨 7-170 token),两档分布还与关闭档重叠,按长度阈值判会把"推理了但想得少"
|
|
误判成没推理。
|
|
"""
|
|
return obs["thinking_observation"] == ThinkingObservation.OBSERVED
|
|
|
|
|
|
def _skip_if_unreachable(exc: Exception, matrix_id: str, desc: str):
|
|
"""源不可用(渠道下线/模型未开通)→ 跳过并记为未覆盖,不伪装成通过。"""
|
|
_record(matrix_id, desc, "SKIP(源不可用)", str(exc)[:200])
|
|
pytest.skip(f"{matrix_id} 源不可用,已记为未覆盖: {str(exc)[:120]}")
|
|
|
|
|
|
@pytest.fixture(scope="module", autouse=True)
|
|
def _write_report():
|
|
yield
|
|
_OUT_DIR.mkdir(parents=True, exist_ok=True)
|
|
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
path = _OUT_DIR / f"test_thinking_live_{ts}.md"
|
|
lines = [
|
|
"# 推理开关与推理可观测性真实 API 验证",
|
|
"",
|
|
f"- 时间: {ts}",
|
|
f"- 每档轮数: {_ROUNDS}",
|
|
"- 判别量: 库裁定的三态 `thinking_observation`(OBSERVED/ABSENT/UNKNOWN),"
|
|
"由推理正文与 reasoning_tokens 共同裁定 —— 正文是事实,token 计数只是转述",
|
|
"- 关闭判据: **每轮** observation != OBSERVED(UNKNOWN 计入满足,它没有证伪力);"
|
|
"刻意不设输出长度上限(两档的 completion 分布重叠: 实测关闭档最高 46、开启档最低 13)",
|
|
"- 开启判据: **多数轮** observation == OBSERVED",
|
|
"- 确定性锚点(L2b、L5): 关闭档 prompt_tokens 最大值 < 开启档最小值,相对比较无魔数",
|
|
"- L5(非流式): M3 该路径推理已计费却不回传正文,故不断言「观测到推理」,"
|
|
"改断锚点可分 + 开启档不被误判为 ABSENT",
|
|
"",
|
|
"## 矩阵结论",
|
|
"",
|
|
"| 矩阵 | 场景 | 结论 | 说明 |",
|
|
"|---|---|---|---|",
|
|
]
|
|
total_calls = 0
|
|
for row in _ROWS:
|
|
detail = str(row["detail"]).replace("|", "\\|").replace("\n", " ")[:160]
|
|
lines.append(f"| {row['matrix']} | {row['desc']} | {row['status']} | {detail} |")
|
|
total_calls += len(row["observations"])
|
|
lines += ["", f"**总真实调用次数: {total_calls}**", "", "## 逐轮原始观测", ""]
|
|
for row in _ROWS:
|
|
if not row["observations"]:
|
|
continue
|
|
lines += [f"### {row['matrix']} — {row['desc']}", "", "```json"]
|
|
lines.append(json.dumps(row["observations"], ensure_ascii=False, indent=2))
|
|
lines += ["```", ""]
|
|
uncovered = [r["matrix"] for r in _ROWS if r["status"].startswith("SKIP")]
|
|
if uncovered:
|
|
lines += ["## 未覆盖", "", f"以下矩阵行未跑到: {', '.join(uncovered)}", ""]
|
|
path.write_text("\n".join(lines), encoding="utf-8")
|
|
print(f"\n[e2e 报告] {path}")
|
|
|
|
|
|
class TestMiniMaxM3:
|
|
"""M3 是唯一实测可关闭推理的 MiniMax 模型,修复的地基压在它身上。"""
|
|
|
|
async def test_l1_disable_actually_disables(self):
|
|
obs = await _run_rounds(_ROUNDS, model="MiniMax-M3", enable_thinking=False)
|
|
offs = [o for o in obs if _reasoning_off(o)]
|
|
_record(
|
|
"L1",
|
|
"enable_thinking=False(流式)",
|
|
"PASS" if len(offs) == len(obs) else "FAIL",
|
|
f"{len(offs)}/{len(obs)} 轮未观测到推理",
|
|
obs,
|
|
)
|
|
assert len(offs) == len(obs), f"关闭方向要求每轮满足: {obs}"
|
|
|
|
async def test_l2_enable_actually_enables(self):
|
|
obs = await _run_rounds(_ROUNDS, model="MiniMax-M3", enable_thinking=True)
|
|
ons = [o for o in obs if _reasoning_on(o)]
|
|
_record(
|
|
"L2",
|
|
"enable_thinking=True(流式,注入 medium)",
|
|
"PASS" if len(ons) * 2 > len(obs) else "FAIL",
|
|
f"{len(ons)}/{len(obs)} 轮观察到推理",
|
|
obs,
|
|
)
|
|
assert len(ons) * 2 > len(obs), f"开启方向要求多数轮满足: {obs}"
|
|
|
|
async def test_l2b_off_and_on_are_distinguishable_without_magic_numbers(self):
|
|
"""确定性锚点: 开启档的 prompt_tokens 严格大于关闭档。
|
|
|
|
供应商在开启推理时会向模板注入推理指令,输入侧 token 数随之变大。这是
|
|
本组唯一不依赖输出侧噪声的证据,且是相对比较——不硬编码任何具体数值,
|
|
供应商改模板也不会让它假红。
|
|
"""
|
|
rounds = max(3, _ROUNDS // 3)
|
|
off = await _run_rounds(rounds, model="MiniMax-M3", enable_thinking=False)
|
|
on = await _run_rounds(rounds, model="MiniMax-M3", enable_thinking=True)
|
|
off_max = max(o["prompt_tokens"] for o in off)
|
|
on_min = min(o["prompt_tokens"] for o in on)
|
|
_record(
|
|
"L2b",
|
|
"关闭/开启的 prompt_tokens 可分",
|
|
"PASS" if off_max < on_min else "FAIL",
|
|
f"关闭档最大 {off_max} < 开启档最小 {on_min}",
|
|
off + on,
|
|
)
|
|
assert off_max < on_min, (
|
|
f"两档的 prompt_tokens 未分开(关闭最大 {off_max},开启最小 {on_min}): 注入可能没到达模型"
|
|
)
|
|
|
|
async def test_l3_no_opinion_is_the_model_default(self):
|
|
obs = await _run_rounds(_ROUNDS, model="MiniMax-M3", enable_thinking=None)
|
|
# M3 的默认档实测就是不推理(findings §2.1),所以不干预时也应观测不到推理。
|
|
# 注意这**不能**反过来证明关闭方向生效 —— L1 与本行同分布,区分二者的是
|
|
# L2b 的 prompt_tokens 与 L3b 的乱码值反证
|
|
quiet = [o for o in obs if _reasoning_off(o)]
|
|
_record(
|
|
"L3",
|
|
"enable_thinking=None(不干预,基线)",
|
|
"PASS" if len(quiet) == len(obs) else "FAIL",
|
|
f"{len(quiet)}/{len(obs)} 轮未观测到推理(M3 默认档本就不推理)",
|
|
obs,
|
|
)
|
|
assert len(quiet) == len(obs), f"M3 默认档不应推理: {obs}"
|
|
|
|
async def test_l3b_none_is_recognised_not_silently_dropped(self):
|
|
"""反证: 关闭方向的观测必须排除"参数被静默丢弃"这一伪解释。
|
|
|
|
L1(关闭)与 L3(不干预)在 M3 上**同分布**——因为 M3 默认档本就不推理。
|
|
所以 L1 单独看不能区分"`none` 真的被消费"与"`none` 被中转吞了",而后者
|
|
正是 issue #5 的原始故障形态(`enable_thinking` 就是这么被吞的)。
|
|
|
|
判别方法: 发一个**非法值**。若未知值会被静默丢弃,它的表现应与"不注入"
|
|
一致(不推理);实测它反而开启了推理,说明网关认这个键、只是不认这个值。
|
|
既然非法值与 `none` 的表现不同,`none` 就必然是被识别的枚举值。
|
|
|
|
**该手法不可移植,只对"认这个键但不校验值"的 provider 成立**: minimax 对
|
|
非法 `reasoning_effort` 返回 200 且照常推理(2026-08-25 findings §5:
|
|
prompt 207,介于基线 194 与 medium 216 之间,走了第三条模板路径);而 qwen
|
|
对同样的值直接返回 **HTTP 400**。把本用例套到 qwen 那类会校验值的 provider
|
|
上,拿到的会是异常而非"不推理",是假红。
|
|
"""
|
|
rounds = max(3, _ROUNDS // 3)
|
|
bogus = await _run_rounds(
|
|
rounds,
|
|
model="MiniMax-M3",
|
|
enable_thinking=None,
|
|
extra_body={"reasoning_effort": "definitely-not-a-real-level"},
|
|
)
|
|
off = await _run_rounds(rounds, model="MiniMax-M3", enable_thinking=False)
|
|
bogus_on = [o for o in bogus if _reasoning_on(o)]
|
|
off_quiet = [o for o in off if _reasoning_off(o)]
|
|
ok = len(bogus_on) * 2 > len(bogus) and len(off_quiet) == len(off)
|
|
_record(
|
|
"L3b",
|
|
"非法值反证 none 被识别",
|
|
"PASS" if ok else "FAIL",
|
|
f"非法值 {len(bogus_on)}/{len(bogus)} 轮推理,none {len(off_quiet)}/{len(off)} 轮不推理"
|
|
"(两者表现不同 ⇒ none 非被丢弃)",
|
|
bogus + off,
|
|
)
|
|
assert len(bogus_on) * 2 > len(bogus), (
|
|
f"非法值未开启推理,无法排除'未知值被静默丢弃'这一伪解释: {bogus}"
|
|
)
|
|
assert len(off_quiet) == len(off), f"none 未关闭推理: {off}"
|
|
|
|
async def test_l4_extra_body_overrides_the_profile(self):
|
|
"""profile 注入 none,extra_body 要求 high —— 后者必须赢(优先级不可调换)。
|
|
|
|
判据是行为而非报文: 若 extra_body 没赢,拿到的就是 none 的结果(不推理)。
|
|
"""
|
|
rounds = max(3, _ROUNDS // 2)
|
|
obs = await _run_rounds(
|
|
rounds,
|
|
model="MiniMax-M3",
|
|
enable_thinking=False,
|
|
extra_body={"reasoning_effort": "high"},
|
|
)
|
|
ons = [o for o in obs if _reasoning_on(o)]
|
|
_record(
|
|
"L4",
|
|
"extra_body 覆盖 profile 注入",
|
|
"PASS" if len(ons) * 2 > len(obs) else "FAIL",
|
|
f"{len(ons)}/{len(obs)} 轮观察到推理(证明 high 生效而非 none)",
|
|
obs,
|
|
)
|
|
assert len(ons) * 2 > len(obs), f"extra_body 未能覆盖 profile: {obs}"
|
|
|
|
async def test_l5_non_stream_path_is_distinguishable_and_honestly_unknown(self):
|
|
"""非流式快路径: 参数确实到达了模型,而推理信号被如实标成"观测不到"。
|
|
|
|
**本用例不能断言"非流式开启档观测到推理"——那永远不成立**: M3 在非流式
|
|
路径下推理段确实产生并计费(2026-08-25 findings §3.4: 开启档 completion 53
|
|
vs 关闭档 3),但 `message` 里没有 `reasoning_content`、`usage` 里也没有
|
|
`completion_tokens_details`,推理内容整体不回传。**这是上游行为,库修不了;
|
|
库能做也必须做的是让它可见**——下游在为看不见的东西付费,不该由库替它
|
|
沉默。
|
|
|
|
故改断两件在非流式下真实成立的事:
|
|
其一 `prompt_tokens` 锚点仍把两档分开(判据形态照抄 L2b,证明注入到达了模型,
|
|
排除"非流式路径把参数弄丢了"这一伪解释);
|
|
其二开启档的裁定**不是 `ABSENT`**——`ABSENT` 的语义是"上游明确上报未推理",
|
|
而实情是"判不出来"(`UNKNOWN`),库若把后者伪装成前者,正是 issue #16/#17 里
|
|
那个静默错觉。这里断 `!= ABSENT` 而非 `== UNKNOWN`,是为了留出上游哪天开始
|
|
回传正文的余地: 那时裁定会翻成 `OBSERVED`,是好事,不该让它把测试判红。
|
|
"""
|
|
rounds = max(3, _ROUNDS // 2)
|
|
off = await _run_rounds(rounds, stream=False, model="MiniMax-M3", enable_thinking=False)
|
|
on = await _run_rounds(rounds, stream=False, model="MiniMax-M3", enable_thinking=True)
|
|
offs = [o for o in off if _reasoning_off(o)]
|
|
off_max = max(o["prompt_tokens"] for o in off)
|
|
on_min = min(o["prompt_tokens"] for o in on)
|
|
not_absent = [o for o in on if o["thinking_observation"] != ThinkingObservation.ABSENT]
|
|
on_states = Counter(str(o["thinking_observation"]) for o in on)
|
|
ok = len(offs) == len(off) and off_max < on_min and len(not_absent) == len(on)
|
|
_record(
|
|
"L5",
|
|
"非流式: prompt 锚点可分 + 开启档如实标 UNKNOWN 而非 ABSENT",
|
|
"PASS" if ok else "FAIL",
|
|
f"关闭 {len(offs)}/{len(off)} 轮未观测到推理;"
|
|
f"关闭档 prompt 最大 {off_max} < 开启档最小 {on_min};"
|
|
f"开启档裁定分布 {dict(on_states)}",
|
|
off + on,
|
|
)
|
|
assert len(offs) == len(off), f"非流式关闭方向未满足: {off}"
|
|
assert off_max < on_min, (
|
|
f"非流式两档 prompt_tokens 未分开(关闭最大 {off_max},开启最小 {on_min}): "
|
|
f"开启参数可能没到达模型"
|
|
)
|
|
assert len(not_absent) == len(on), (
|
|
f"非流式开启档被裁成 ABSENT(声称上游明确上报未推理),而实情是观测不到: {on}"
|
|
)
|
|
|
|
|
|
class TestOtherProviders:
|
|
"""qwen / deepseek 的 profile 是既有实现,本组防的是"改 minimax 时误伤它们"。"""
|
|
|
|
@pytest.mark.parametrize(
|
|
("matrix", "provider", "model"),
|
|
[("L6", "qwen", "qwen3.7-plus"), ("L7", "deepseek", "deepseek-v4-pro")],
|
|
)
|
|
async def test_existing_profiles_still_disable(self, matrix, provider, model):
|
|
desc = f"{provider} enable_thinking=False"
|
|
try:
|
|
obs = await _run_rounds(_ROUNDS, provider=provider, model=model, enable_thinking=False)
|
|
except (AllSourcesExhausted, SourceDeadError, TransientError) as exc:
|
|
# 只吞网关/网络类失败。**不吞 ValueError / RequestRejected** ——
|
|
# 那两类正是本次改动最可能的误伤方向,吞掉就成了纪律(c)要防的静默
|
|
_skip_if_unreachable(exc, matrix, desc)
|
|
offs = [o for o in obs if _reasoning_off(o)]
|
|
_record(
|
|
matrix,
|
|
desc,
|
|
"PASS" if len(offs) == len(obs) else "FAIL",
|
|
f"{len(offs)}/{len(obs)} 轮未观测到推理",
|
|
obs,
|
|
)
|
|
assert len(offs) == len(obs), f"{provider} 关闭方向未满足: {obs}"
|
|
|
|
async def test_qwen_enabled_is_observed(self):
|
|
"""设计 §14 验收: qwen 开启档必须裁定为 `OBSERVED`,不是 `UNKNOWN`。
|
|
|
|
本条是三态裁定的**跨供应商对照组**: MiniMax 这一路两个信号都可能缺失
|
|
(非流式档整片 `UNKNOWN`),若只按它调判据,很容易把"观测不到"当成常态;
|
|
qwen 在同一网关同一 key 上照常返回推理信号(findings 2026-08-25 §2),
|
|
故这里能且必须要求正面结论——它一旦掉成 `UNKNOWN`,说明的是库的组装路径
|
|
丢了信号,而不是上游行为变了。
|
|
"""
|
|
matrix, provider, model = "L6b", "qwen", "qwen3.7-plus"
|
|
desc = f"{provider} enable_thinking=True"
|
|
try:
|
|
obs = await _run_rounds(_ROUNDS, provider=provider, model=model, enable_thinking=True)
|
|
except (AllSourcesExhausted, SourceDeadError, TransientError) as exc:
|
|
_skip_if_unreachable(exc, matrix, desc)
|
|
ons = [o for o in obs if _reasoning_on(o)]
|
|
_record(
|
|
matrix,
|
|
desc,
|
|
"PASS" if len(ons) * 2 > len(obs) else "FAIL",
|
|
f"{len(ons)}/{len(obs)} 轮观测到推理(OBSERVED)",
|
|
obs,
|
|
)
|
|
assert len(ons) * 2 > len(obs), f"{provider} 开启方向要求多数轮 OBSERVED: {obs}"
|
|
|
|
|
|
class TestCapabilityDrift:
|
|
"""L8 漂移哨兵: 能力表过期是必然事件,这里是它的过期告警。"""
|
|
|
|
def test_every_capability_has_a_provider_mapping(self):
|
|
"""能力表新增条目必须同步本测试的映射,否则该行会被静默跳过。"""
|
|
missing = sorted(set(DEFAULT_CAPABILITIES) - set(_MODEL_PROVIDER))
|
|
assert not missing, f"这些模型缺 provider 映射,L8 会漏测: {missing}"
|
|
|
|
@pytest.mark.parametrize("model", sorted(DEFAULT_CAPABILITIES))
|
|
async def test_declared_capability_matches_reality(self, model):
|
|
cap = get_capability(model)
|
|
provider = _MODEL_PROVIDER[model]
|
|
rounds = max(3, _ROUNDS // 2)
|
|
desc = f"{model} 声明 can_disable={cap.can_disable}"
|
|
if not cap.can_disable:
|
|
# 声明关不掉: 装配期就该炸,炸了即与声明一致(不必真调用)
|
|
with pytest.raises(ValueError, match=model):
|
|
GatewayClient.from_settings(
|
|
_settings(provider=provider, model=model, enable_thinking=False)
|
|
)
|
|
_record("L8", desc, "PASS", "装配期按声明拒绝,与实测一致")
|
|
return
|
|
try:
|
|
obs = await _run_rounds(rounds, provider=provider, model=model, enable_thinking=False)
|
|
except (AllSourcesExhausted, SourceDeadError, TransientError) as exc:
|
|
_skip_if_unreachable(exc, "L8", desc)
|
|
offs = [o for o in obs if _reasoning_off(o)]
|
|
verdict = Counter(_reasoning_off(o) for o in obs)
|
|
_record(
|
|
"L8",
|
|
desc,
|
|
"PASS" if len(offs) == len(obs) else "FAIL(能力表已漂移)",
|
|
f"实测未观测到推理 {dict(verdict)}(True=满足);声明 can_disable=True 要求每轮满足",
|
|
obs,
|
|
)
|
|
assert len(offs) == len(obs), (
|
|
f"能力表漂移: {model} 声明可关闭推理,实测未关掉 —— 请复测后更新 DEFAULT_CAPABILITIES"
|
|
)
|
|
|
|
|
|
class TestAssemblyGuardAgainstRealConfig:
|
|
"""L9: 纯本地,但用的是 .env 里的真实配置形态,防"守卫只在合成配置上生效"。"""
|
|
|
|
def test_l9_m27_rejected_at_assembly(self):
|
|
with pytest.raises(ValueError, match="MiniMax-M2.7"):
|
|
GatewayClient.from_settings(
|
|
_settings(provider="minimax", model="MiniMax-M2.7", enable_thinking=False)
|
|
)
|
|
_record("L9", "M2.7 + enable_thinking=False", "PASS", "装配期报错,未发出任何请求")
|
|
|
|
def test_l9_unknown_shape_rejected_at_assembly(self):
|
|
with pytest.raises(ValueError, match="register_provider"):
|
|
GatewayClient.from_settings(
|
|
_settings(provider="openai", model="kimi-k3", enable_thinking=False)
|
|
)
|
|
_record("L9", "provider=openai 形态未知", "PASS", "装配期报错并指路")
|
|
|
|
async def test_transport_layer_rejects_when_guard_is_bypassed(self):
|
|
"""构造函数全量注入这条路绕过装配守卫,transport 必须兜住并归四分类。"""
|
|
settings = _settings(provider="minimax", model="MiniMax-M2.7", enable_thinking=False)
|
|
client = GatewayClient.from_settings(
|
|
dataclasses.replace(
|
|
settings, sources=(dataclasses.replace(settings.sources[0], enable_thinking=None),)
|
|
)
|
|
)
|
|
try:
|
|
# 装配用 None 绕过守卫,再把源换成 False 直接喂给 transport
|
|
bad = dataclasses.replace(settings.sources[0], enable_thinking=False)
|
|
with pytest.raises(RequestRejectedError, match="MiniMax-M2.7"):
|
|
await client._terminal._transport.complete(
|
|
messages=[{"role": "user", "content": _PROMPT}],
|
|
source=bad,
|
|
stream=True,
|
|
overlay={},
|
|
call_id="e2e-guard",
|
|
reasoning_effort=None,
|
|
)
|
|
finally:
|
|
await client.aclose()
|
|
_record("L9", "绕过装配守卫时 transport 兜底", "PASS", "RequestRejectedError,属四分类")
|