fix: address the independent verification findings (issue #5, #6)

The verifier caught that the disable-direction evidence only proved "no
regression", not "actually took effect": on M3 the disabled runs and the
no-opinion baseline are identically distributed, because that model does
not reason by default anyway. So the disable runs alone cannot rule out
the very failure mode issue #5 is about -- the parameter being silently
dropped upstream. The bogus-value experiment that does rule it out was
sitting in the findings document instead of the test suite; it is now
case L3b, and the L3 assertion that could never fail is gone.

Also from the review: the e2e helper caught bare Exception, which would
have disguised a library bug as an unavailable source, exactly the
silence the reporting discipline exists to prevent; the unregistered
model warning fired on every request instead of once per source; and the
transport caught ValueError broadly enough to mislabel unrelated errors,
now narrowed to a dedicated ThinkingUnsupportedError.

The design and plan still described the original judgement criteria,
which the measurements had already overturned. Both now match what the
tests actually do, and the design no longer claims the only new failure
surface is the openai one -- dissect configures MiniMax-M2.7 with
ENABLE_THINKING=false and will fail at assembly, which has to be
coordinated before this merges.
This commit is contained in:
2026-08-02 07:40:06 -04:00
parent 4c135075b3
commit 48805cb9fb
7 changed files with 155 additions and 26 deletions
+57 -5
View File
@@ -31,7 +31,12 @@ import pytest
from dotenv import dotenv_values
from polygateway import GatewayClient, GatewaySettings
from polygateway.errors import RequestRejectedError
from polygateway.errors import (
AllSourcesExhausted,
RequestRejectedError,
SourceDeadError,
TransientError,
)
from polygateway.providers import DEFAULT_CAPABILITIES, get_capability
_ENV = {k: v for k, v in {**dotenv_values(".env"), **os.environ}.items() if v is not None}
@@ -236,8 +241,53 @@ class TestMiniMaxM3:
async def test_l3_no_opinion_is_the_model_default(self):
obs = await _run_rounds(_ROUNDS, model="MiniMax-M3", enable_thinking=None)
_record("L3", "enable_thinking=None(不干预,基线)", "PASS", "仅记录基线,不断言方向", obs)
assert len(obs) == _ROUNDS
# M3 的默认档实测就是不推理(findings §2.1),所以不干预时也应观测不到推理。
# 注意这**不能**反过来证明关闭方向生效 —— L1 与本行同分布,区分二者的是
# L2b 的 prompt_tokens 与 L3b 的乱码值反证
quiet = [o for o in obs if _reasoning_off(o)]
_record(
"L3",
"enable_thinking=None(不干预,基线)",
"PASS" if len(quiet) == len(obs) else "FAIL",
f"{len(quiet)}/{len(obs)} 轮未推理(M3 默认档本就不推理)",
obs,
)
assert len(quiet) == len(obs), f"M3 默认档不应推理: {obs}"
async def test_l3b_none_is_recognised_not_silently_dropped(self):
"""反证: 关闭方向的观测必须排除"参数被静默丢弃"这一伪解释。
L1(关闭)与 L3(不干预)在 M3 上**同分布**——因为 M3 默认档本就不推理。
所以 L1 单独看不能区分"`none` 真的被消费""`none` 被中转吞了",而后者
正是 issue #5 的原始故障形态(`enable_thinking` 就是这么被吞的)。
判别方法: 发一个**非法值**。若未知值会被静默丢弃,它的表现应与"不注入"
一致(不推理);实测它反而开启了推理,说明网关认这个键、只是不认这个值。
既然非法值与 `none` 的表现不同,`none` 就必然是被识别的枚举值。
"""
rounds = max(3, _ROUNDS // 3)
bogus = await _run_rounds(
rounds,
model="MiniMax-M3",
enable_thinking=None,
extra_body={"reasoning_effort": "definitely-not-a-real-level"},
)
off = await _run_rounds(rounds, model="MiniMax-M3", enable_thinking=False)
bogus_on = [o for o in bogus if _reasoning_on(o)]
off_quiet = [o for o in off if _reasoning_off(o)]
ok = len(bogus_on) * 2 > len(bogus) and len(off_quiet) == len(off)
_record(
"L3b",
"非法值反证 none 被识别",
"PASS" if ok else "FAIL",
f"非法值 {len(bogus_on)}/{len(bogus)} 轮推理,none {len(off_quiet)}/{len(off)} 轮不推理"
"(两者表现不同 ⇒ none 非被丢弃)",
bogus + off,
)
assert len(bogus_on) * 2 > len(bogus), (
f"非法值未开启推理,无法排除'未知值被静默丢弃'这一伪解释: {bogus}"
)
assert len(off_quiet) == len(off), f"none 未关闭推理: {off}"
async def test_l4_extra_body_overrides_the_profile(self):
"""profile 注入 none,extra_body 要求 high —— 后者必须赢(优先级不可调换)。
@@ -291,7 +341,9 @@ class TestOtherProviders:
desc = f"{provider} enable_thinking=False"
try:
obs = await _run_rounds(_ROUNDS, provider=provider, model=model, enable_thinking=False)
except Exception as exc: # 渠道未开通/下线: 记为未覆盖
except (AllSourcesExhausted, SourceDeadError, TransientError) as exc:
# 只吞网关/网络类失败。**不吞 ValueError / RequestRejected** ——
# 那两类正是本次改动最可能的误伤方向,吞掉就成了纪律(c)要防的静默
_skip_if_unreachable(exc, matrix, desc)
offs = [o for o in obs if _reasoning_off(o)]
_record(
@@ -328,7 +380,7 @@ class TestCapabilityDrift:
return
try:
obs = await _run_rounds(rounds, provider=provider, model=model, enable_thinking=False)
except Exception as exc:
except (AllSourcesExhausted, SourceDeadError, TransientError) as exc:
_skip_if_unreachable(exc, "L8", desc)
offs = [o for o in obs if _reasoning_off(o)]
verdict = Counter(_reasoning_off(o) for o in obs)
+40
View File
@@ -7,6 +7,7 @@ import json
import httpx
import pytest
from loguru import logger
from polygateway.errors import (
RequestRejectedError,
@@ -557,6 +558,45 @@ class TestRequestShaping:
with pytest.raises(RequestRejectedError, match="MiniMax-M2.7"):
await _complete(_transport_for(handler), source)
async def test_unregistered_model_warns_only_once_per_source(self):
"""未登记模型的告警不能打在请求热路径上: 装配期已喊过,逐次再喊是刷屏。"""
def handler(request):
return _sse_stream(_chunk(content="x"), _chunk(usage=_USAGE))
source = _source(name="mm", provider="minimax", model="MiniMax-M99", enable_thinking=False)
transport = _transport_for(handler)
messages: list[str] = []
sink_id = logger.add(messages.append, level="WARNING")
try:
await _complete(transport, source)
await _complete(transport, source)
await _complete(transport, source)
finally:
logger.remove(sink_id)
hits = [m for m in messages if "MiniMax-M99" in m]
assert len(hits) == 1, f"三次调用应只告警一次,实得 {len(hits)}"
async def test_unrelated_value_error_is_not_mislabelled(self, monkeypatch):
"""只捕 ThinkingUnsupportedError: 无关的 ValueError 不该被贴成推理开关的错。
今天 `_build_payload` 里只有 resolve_thinking 会抛 ValueError,所以这条
是防御未来 —— 但正因如此才要钉住: 将来谁在那里加一处校验,宽 catch 会
把它的错误信息盖掉,而这个用例会先红。
"""
def handler(request): # pragma: no cover - 不该走到发请求
raise AssertionError("请求不该发出")
def _boom(*args, **kwargs):
raise ValueError("故意的无关错误")
monkeypatch.setattr("polygateway.transports.openai_compat.resolve_thinking", _boom)
with pytest.raises(ValueError, match="故意的无关错误") as exc:
await _complete(_transport_for(handler), _source(enable_thinking=False))
assert "推理开关" not in str(exc.value)
assert not isinstance(exc.value, RequestRejectedError)
async def test_unknown_shape_is_rejected(self):
def handler(request): # pragma: no cover - 不该走到发请求
raise AssertionError("请求不该发出")