The verifier caught that the disable-direction evidence only proved "no regression", not "actually took effect": on M3 the disabled runs and the no-opinion baseline are identically distributed, because that model does not reason by default anyway. So the disable runs alone cannot rule out the very failure mode issue #5 is about -- the parameter being silently dropped upstream. The bogus-value experiment that does rule it out was sitting in the findings document instead of the test suite; it is now case L3b, and the L3 assertion that could never fail is gone. Also from the review: the e2e helper caught bare Exception, which would have disguised a library bug as an unavailable source, exactly the silence the reporting discipline exists to prevent; the unregistered model warning fired on every request instead of once per source; and the transport caught ValueError broadly enough to mislabel unrelated errors, now narrowed to a dedicated ThinkingUnsupportedError. The design and plan still described the original judgement criteria, which the measurements had already overturned. Both now match what the tests actually do, and the design no longer claims the only new failure surface is the openai one -- dissect configures MiniMax-M2.7 with ENABLE_THINKING=false and will fail at assembly, which has to be coordinated before this merges.
This commit is contained in:
@@ -31,7 +31,12 @@ import pytest
|
||||
from dotenv import dotenv_values
|
||||
|
||||
from polygateway import GatewayClient, GatewaySettings
|
||||
from polygateway.errors import RequestRejectedError
|
||||
from polygateway.errors import (
|
||||
AllSourcesExhausted,
|
||||
RequestRejectedError,
|
||||
SourceDeadError,
|
||||
TransientError,
|
||||
)
|
||||
from polygateway.providers import DEFAULT_CAPABILITIES, get_capability
|
||||
|
||||
_ENV = {k: v for k, v in {**dotenv_values(".env"), **os.environ}.items() if v is not None}
|
||||
@@ -236,8 +241,53 @@ class TestMiniMaxM3:
|
||||
|
||||
async def test_l3_no_opinion_is_the_model_default(self):
|
||||
obs = await _run_rounds(_ROUNDS, model="MiniMax-M3", enable_thinking=None)
|
||||
_record("L3", "enable_thinking=None(不干预,基线)", "PASS", "仅记录基线,不断言方向", obs)
|
||||
assert len(obs) == _ROUNDS
|
||||
# M3 的默认档实测就是不推理(findings §2.1),所以不干预时也应观测不到推理。
|
||||
# 注意这**不能**反过来证明关闭方向生效 —— L1 与本行同分布,区分二者的是
|
||||
# L2b 的 prompt_tokens 与 L3b 的乱码值反证
|
||||
quiet = [o for o in obs if _reasoning_off(o)]
|
||||
_record(
|
||||
"L3",
|
||||
"enable_thinking=None(不干预,基线)",
|
||||
"PASS" if len(quiet) == len(obs) else "FAIL",
|
||||
f"{len(quiet)}/{len(obs)} 轮未推理(M3 默认档本就不推理)",
|
||||
obs,
|
||||
)
|
||||
assert len(quiet) == len(obs), f"M3 默认档不应推理: {obs}"
|
||||
|
||||
async def test_l3b_none_is_recognised_not_silently_dropped(self):
|
||||
"""反证: 关闭方向的观测必须排除"参数被静默丢弃"这一伪解释。
|
||||
|
||||
L1(关闭)与 L3(不干预)在 M3 上**同分布**——因为 M3 默认档本就不推理。
|
||||
所以 L1 单独看不能区分"`none` 真的被消费"与"`none` 被中转吞了",而后者
|
||||
正是 issue #5 的原始故障形态(`enable_thinking` 就是这么被吞的)。
|
||||
|
||||
判别方法: 发一个**非法值**。若未知值会被静默丢弃,它的表现应与"不注入"
|
||||
一致(不推理);实测它反而开启了推理,说明网关认这个键、只是不认这个值。
|
||||
既然非法值与 `none` 的表现不同,`none` 就必然是被识别的枚举值。
|
||||
"""
|
||||
rounds = max(3, _ROUNDS // 3)
|
||||
bogus = await _run_rounds(
|
||||
rounds,
|
||||
model="MiniMax-M3",
|
||||
enable_thinking=None,
|
||||
extra_body={"reasoning_effort": "definitely-not-a-real-level"},
|
||||
)
|
||||
off = await _run_rounds(rounds, model="MiniMax-M3", enable_thinking=False)
|
||||
bogus_on = [o for o in bogus if _reasoning_on(o)]
|
||||
off_quiet = [o for o in off if _reasoning_off(o)]
|
||||
ok = len(bogus_on) * 2 > len(bogus) and len(off_quiet) == len(off)
|
||||
_record(
|
||||
"L3b",
|
||||
"非法值反证 none 被识别",
|
||||
"PASS" if ok else "FAIL",
|
||||
f"非法值 {len(bogus_on)}/{len(bogus)} 轮推理,none {len(off_quiet)}/{len(off)} 轮不推理"
|
||||
"(两者表现不同 ⇒ none 非被丢弃)",
|
||||
bogus + off,
|
||||
)
|
||||
assert len(bogus_on) * 2 > len(bogus), (
|
||||
f"非法值未开启推理,无法排除'未知值被静默丢弃'这一伪解释: {bogus}"
|
||||
)
|
||||
assert len(off_quiet) == len(off), f"none 未关闭推理: {off}"
|
||||
|
||||
async def test_l4_extra_body_overrides_the_profile(self):
|
||||
"""profile 注入 none,extra_body 要求 high —— 后者必须赢(优先级不可调换)。
|
||||
@@ -291,7 +341,9 @@ class TestOtherProviders:
|
||||
desc = f"{provider} enable_thinking=False"
|
||||
try:
|
||||
obs = await _run_rounds(_ROUNDS, provider=provider, model=model, enable_thinking=False)
|
||||
except Exception as exc: # 渠道未开通/下线: 记为未覆盖
|
||||
except (AllSourcesExhausted, SourceDeadError, TransientError) as exc:
|
||||
# 只吞网关/网络类失败。**不吞 ValueError / RequestRejected** ——
|
||||
# 那两类正是本次改动最可能的误伤方向,吞掉就成了纪律(c)要防的静默
|
||||
_skip_if_unreachable(exc, matrix, desc)
|
||||
offs = [o for o in obs if _reasoning_off(o)]
|
||||
_record(
|
||||
@@ -328,7 +380,7 @@ class TestCapabilityDrift:
|
||||
return
|
||||
try:
|
||||
obs = await _run_rounds(rounds, provider=provider, model=model, enable_thinking=False)
|
||||
except Exception as exc:
|
||||
except (AllSourcesExhausted, SourceDeadError, TransientError) as exc:
|
||||
_skip_if_unreachable(exc, "L8", desc)
|
||||
offs = [o for o in obs if _reasoning_off(o)]
|
||||
verdict = Counter(_reasoning_off(o) for o in obs)
|
||||
|
||||
Reference in New Issue
Block a user