526 lines
19 KiB
Python
526 lines
19 KiB
Python
"""真实推理矩阵：独立请求／身份资格，逐轮留证，UNKNOWN 不证明关闭。
|
||
|
||
不新增成功 SSE 捕获器，不把整类异常跳过；历史长度／prompt 锚点仅作
|
||
指定样本的形态回归，不提升能力覆盖。T10 不可关闭与预期拒绝是独立命题。
|
||
"""
|
||
|
||
import asyncio
|
||
import dataclasses
|
||
import os
|
||
from collections.abc import Mapping
|
||
from pathlib import Path
|
||
from types import MappingProxyType
|
||
from uuid import uuid4
|
||
|
||
import pytest
|
||
from dotenv import dotenv_values
|
||
|
||
from polygateway import GatewaySettings, ThinkingObservation
|
||
from polygateway.thinking import DEFAULT_CAPABILITIES, ThinkingCapability
|
||
from polygateway.types import EFFORT_ORDER, Effort
|
||
from tests.e2e.conftest import (
|
||
LiveCapture,
|
||
captured_chat_round,
|
||
chat_expectations,
|
||
declared_control,
|
||
enforce_verdict,
|
||
observed_client,
|
||
source_controls,
|
||
)
|
||
from tests.live_evidence import (
|
||
LiveVerdict,
|
||
assess_thinking_coverage,
|
||
combine_live_verdicts,
|
||
qualify_live_rounds,
|
||
summarize_verdicts,
|
||
write_live_round,
|
||
)
|
||
|
||
_ENV = {k: v for k, v in {**dotenv_values(".env"), **os.environ}.items() if v is not None}
|
||
_HAS_SOURCE = any(k.startswith("LLM__") and k.endswith("__API_KEY") for k in _ENV)
|
||
pytestmark = [
|
||
pytest.mark.slow,
|
||
pytest.mark.skipif(not _HAS_SOURCE, reason="缺少矩阵必需凭据，未覆盖"),
|
||
]
|
||
_OUT_DIR = Path("tests/outputs/134/live")
|
||
_ROUNDS = int(os.environ.get("PGW_E2E_THINKING_ROUNDS", "10"))
|
||
_TIER_ROUNDS = int(os.environ.get("PGW_E2E_TIER_ROUNDS", "5"))
|
||
_TIER_LONG_ROUNDS = int(os.environ.get("PGW_E2E_TIER_LONG_ROUNDS", "3"))
|
||
_TIER_CONCURRENCY = int(os.environ.get("PGW_E2E_TIER_CONCURRENCY", "3"))
|
||
_PROMPT = "一个笼子里有若干鸡和兔,共 35 个头、94 只脚。鸡和兔各有多少只?只输出两个数字。"
|
||
_TIER_PROMPT = "23 乘以 47 等于多少?只回答一个数字,不要解释。"
|
||
_TIER_LONG_PROMPT = (
|
||
"\n".join(
|
||
f"{i:04d}. 这是一段与题目无关的填充文字,仅用于把上下文撑到数千 token,"
|
||
"以复核短提示词下得到的关闭结论在长上下文下是否依然成立。"
|
||
for i in range(120)
|
||
)
|
||
+ "\n\n"
|
||
+ _TIER_PROMPT
|
||
)
|
||
_ALL_EFFORTS = (*EFFORT_ORDER, Effort.AUTO)
|
||
|
||
_MODEL_PROVIDER = {
|
||
"MiniMax-M3": "minimax",
|
||
"MiniMax-M2.7": "minimax",
|
||
"MiniMax-M2.5": "minimax",
|
||
"qwen3.7-plus": "qwen",
|
||
"qwen3.7-max": "qwen",
|
||
"qwen3.6-plus": "qwen",
|
||
"qwen3.5-flash": "qwen",
|
||
"qwen-plus-latest": "qwen",
|
||
"deepseek-v4-pro": "deepseek",
|
||
"deepseek-v4-flash": "deepseek",
|
||
"deepseek-v4-flash-vision-exp": "deepseek",
|
||
"glm-5.3": "zhipu",
|
||
"glm-5.3-flash": "zhipu",
|
||
"glm-5.2": "zhipu",
|
||
"glm-5.1": "zhipu",
|
||
"glm-5": "zhipu",
|
||
"glm-4.6v": "zhipu",
|
||
"kimi-k3": "moonshot",
|
||
"kimi-for-coding": "moonshot",
|
||
"gpt-5.4": "openai",
|
||
"gpt-5.5": "openai",
|
||
"claude-opus-5": "anthropic",
|
||
"claude-sonnet-5": "anthropic",
|
||
"claude-haiku-5": "anthropic",
|
||
"gemini-3.1-pro": "google",
|
||
"gemini-3-flash": "google",
|
||
}
|
||
|
||
_MODEL_REPORTED_ALIASES: Mapping[str, frozenset[str]] = MappingProxyType(
|
||
{
|
||
"kimi-k3": frozenset({"k3"}),
|
||
"kimi-for-coding": frozenset({"k3"}),
|
||
"gemini-3-flash": frozenset({"gemini-3-flash-preview"}),
|
||
"gemini-3.1-pro": frozenset({"gemini-3.1-pro-preview"}),
|
||
}
|
||
)
|
||
|
||
|
||
def _settings(**source_overrides):
|
||
"""强制关闭缓存，清除 inherited 受管意图后应用本矩阵配置。"""
|
||
base = GatewaySettings.from_env("LLM", env={**_ENV, "PGW_CACHE_BACKEND": "none"})
|
||
source = dataclasses.replace(
|
||
base.sources[0], enable_thinking=None, reasoning_effort=None, extra_body={}
|
||
)
|
||
source = dataclasses.replace(source, **source_overrides)
|
||
return dataclasses.replace(base, sources=(source,))
|
||
|
||
|
||
def _tier_settings(model):
|
||
"""保留既有一次 retry 探测预算；不压缩生产 stall 或单次 timeout。"""
|
||
base = GatewaySettings.from_env(
|
||
"LLM", env={**_ENV, "PGW_CACHE_BACKEND": "none", "LLM_MAX_RETRIES": "1"}
|
||
)
|
||
source = dataclasses.replace(
|
||
base.sources[0],
|
||
model=model,
|
||
provider=_MODEL_PROVIDER[model],
|
||
enable_thinking=None,
|
||
reasoning_effort=None,
|
||
extra_body={},
|
||
)
|
||
return dataclasses.replace(base, sources=(source,))
|
||
|
||
|
||
async def _collect_rounds(
|
||
settings, *, rounds, stream, prompt, matrix_id, effort=None, capabilities=None, concurrency=1
|
||
):
|
||
"""保留所有失败轮，不把可用轮集合偷偷当新分母。"""
|
||
if rounds < 1 or concurrency < 1:
|
||
raise ValueError("轮次／并发必须为正数")
|
||
messages = [{"role": "user", "content": prompt}]
|
||
controls = (
|
||
source_controls(settings)
|
||
if effort is None
|
||
else {s.name: declared_control(s.provider, effort) for s in settings.sources}
|
||
)
|
||
capture = LiveCapture(
|
||
expectations=chat_expectations(
|
||
settings, messages=messages, stream=stream, controls=controls
|
||
)
|
||
)
|
||
run_id = uuid4().hex
|
||
semaphore = asyncio.Semaphore(concurrency)
|
||
async with observed_client(settings, capture, capabilities=capabilities) as client:
|
||
|
||
async def one(index):
|
||
"""每轮已落盘后才回到汇总，断言失败也有记录。"""
|
||
|
||
def validate(response):
|
||
assert response.content.strip()
|
||
if effort is not None:
|
||
assert response.applied_effort is effort
|
||
|
||
async with semaphore:
|
||
response, verdict = await captured_chat_round(
|
||
client,
|
||
capture,
|
||
run_id=run_id,
|
||
matrix_id=matrix_id,
|
||
round_index=index + 1,
|
||
output_dir=_OUT_DIR,
|
||
messages=messages,
|
||
models={s.name: s.model for s in settings.sources},
|
||
providers={s.name: s.provider for s in settings.sources},
|
||
source_efforts={
|
||
s.name: s.reasoning_effort
|
||
if s.reasoning_effort is not None
|
||
else (Effort.AUTO if s.enable_thinking else Effort.NONE)
|
||
if s.enable_thinking is not None
|
||
else None
|
||
for s in settings.sources
|
||
},
|
||
aliases=_MODEL_REPORTED_ALIASES,
|
||
stream=stream,
|
||
reasoning_effort=effort,
|
||
cache_salt=f"{run_id}-{index}",
|
||
validate=validate,
|
||
)
|
||
return {"round": index + 1, "verdict": verdict, "response": response}
|
||
|
||
# return_exceptions 保证一个取证写失败不使其他任务越过资源关闭边界。
|
||
results = await asyncio.gather(*(one(i) for i in range(rounds)), return_exceptions=True)
|
||
values = []
|
||
for value in results:
|
||
if isinstance(value, BaseException):
|
||
raise value
|
||
values.append(value)
|
||
counts = summarize_verdicts([value["verdict"] for value in values], planned_rounds=rounds)
|
||
write_live_round(
|
||
_OUT_DIR,
|
||
run_id=run_id,
|
||
matrix_id=matrix_id + "-rounds",
|
||
round_index=0,
|
||
safe_fields={"counts": counts, "completed_rounds": len(values), "planned_rounds": rounds},
|
||
)
|
||
return values
|
||
|
||
|
||
async def _run_rounds(rounds, *, stream=True, matrix_id="thinking", **source_overrides):
|
||
"""L1–L8 的资格证据出口，不作整类 skip。"""
|
||
return await _collect_rounds(
|
||
_settings(**source_overrides),
|
||
rounds=rounds,
|
||
stream=stream,
|
||
prompt=_PROMPT,
|
||
matrix_id=matrix_id,
|
||
)
|
||
|
||
|
||
def _qualified(rows, *, planned_rounds):
|
||
"""汇总资格先失败后未覆盖；部分失败不能被成功轮掩盖。"""
|
||
return qualify_live_rounds([row["verdict"] for row in rows], planned_rounds=planned_rounds)
|
||
|
||
|
||
def _coverage(rows, *, planned_rounds, proposition):
|
||
"""只有全轮资格通过才进入推理观测命题。"""
|
||
verdict = _qualified(rows, planned_rounds=planned_rounds)
|
||
if verdict.status == "PASS":
|
||
verdict = assess_thinking_coverage(
|
||
[row["response"].thinking_observation for row in rows],
|
||
planned_rounds=planned_rounds,
|
||
proposition=proposition,
|
||
)
|
||
return verdict
|
||
|
||
|
||
def _conclude(matrix, verdict, *, proposition=None):
|
||
"""命题汇总先落盘再交给 pytest，不覆盖逐轮原件。"""
|
||
write_live_round(
|
||
_OUT_DIR,
|
||
run_id=uuid4().hex,
|
||
matrix_id=matrix,
|
||
round_index=0,
|
||
safe_fields={
|
||
"status": verdict.status,
|
||
"reason": verdict.reason,
|
||
"proposition": proposition,
|
||
},
|
||
)
|
||
enforce_verdict(verdict)
|
||
|
||
|
||
class TestMiniMaxM3:
|
||
"""AUTO 拒绝已移至离线契约；真实开启明确请求 medium。"""
|
||
|
||
async def test_l1_disable_actually_disables(self):
|
||
rows = await _run_rounds(
|
||
_ROUNDS, matrix_id="L1", provider="minimax", model="MiniMax-M3", enable_thinking=False
|
||
)
|
||
_conclude(
|
||
"L1",
|
||
_coverage(rows, planned_rounds=_ROUNDS, proposition="disabled"),
|
||
proposition="disabled",
|
||
)
|
||
|
||
async def test_l2_enable_actually_enables(self):
|
||
rows = await _run_rounds(
|
||
_ROUNDS,
|
||
matrix_id="L2",
|
||
provider="minimax",
|
||
model="MiniMax-M3",
|
||
reasoning_effort=Effort.MEDIUM,
|
||
)
|
||
_conclude(
|
||
"L2",
|
||
_coverage(rows, planned_rounds=_ROUNDS, proposition="enabled"),
|
||
proposition="enabled",
|
||
)
|
||
|
||
async def test_l2b_off_and_on_are_distinguishable_without_magic_numbers(self):
|
||
"""指定历史 prompt 锚点回归，不宣称关闭能力已覆盖。"""
|
||
rounds = max(3, _ROUNDS // 3)
|
||
off = await _run_rounds(
|
||
rounds,
|
||
matrix_id="L2b-off",
|
||
provider="minimax",
|
||
model="MiniMax-M3",
|
||
enable_thinking=False,
|
||
)
|
||
on = await _run_rounds(
|
||
rounds,
|
||
matrix_id="L2b-on",
|
||
provider="minimax",
|
||
model="MiniMax-M3",
|
||
reasoning_effort=Effort.MEDIUM,
|
||
)
|
||
verdict = _qualified(off + on, planned_rounds=rounds * 2)
|
||
if verdict.status == "PASS":
|
||
distinct = max(r["response"].prompt_tokens for r in off) < min(
|
||
r["response"].prompt_tokens for r in on
|
||
)
|
||
verdict = LiveVerdict(
|
||
"PASS" if distinct else "FAIL", "指定历史 prompt 锚点比较；不是关闭证明"
|
||
)
|
||
_conclude("L2b", verdict, proposition="historical-prompt-anchor")
|
||
|
||
async def test_l3_no_opinion_is_the_model_default(self):
|
||
rows = await _run_rounds(_ROUNDS, matrix_id="L3", provider="minimax", model="MiniMax-M3")
|
||
verdict = _qualified(rows, planned_rounds=_ROUNDS)
|
||
if verdict.status == "PASS" and any(
|
||
row["response"].applied_effort is not None for row in rows
|
||
):
|
||
verdict = LiveVerdict("FAIL", "不表态路径擅自记录档位")
|
||
_conclude("L3", verdict, proposition="no-opinion-not-capability")
|
||
|
||
async def test_l3b_none_is_recognised_not_silently_dropped(self):
|
||
"""保留原非法 raw 值对照预算，但不提升 UNKNOWN。"""
|
||
rounds = max(3, _ROUNDS // 3)
|
||
bogus = await _run_rounds(
|
||
rounds,
|
||
matrix_id="L3b-bogus",
|
||
provider="minimax",
|
||
model="MiniMax-M3",
|
||
extra_body={"reasoning_effort": "definitely-not-a-real-level"},
|
||
)
|
||
off = await _run_rounds(
|
||
rounds,
|
||
matrix_id="L3b-off",
|
||
provider="minimax",
|
||
model="MiniMax-M3",
|
||
enable_thinking=False,
|
||
)
|
||
verdicts = [
|
||
_coverage(bogus, planned_rounds=rounds, proposition="enabled"),
|
||
_coverage(off, planned_rounds=rounds, proposition="disabled"),
|
||
]
|
||
_conclude("L3b", _combine(verdicts), proposition="raw-counterexample")
|
||
|
||
async def test_l4_raw_only_explicit_high(self):
|
||
"""退出受管意图后才保留 raw high；双来源拒绝在 unit 守卫。"""
|
||
rounds = max(3, _ROUNDS // 2)
|
||
rows = await _run_rounds(
|
||
rounds,
|
||
matrix_id="L4",
|
||
provider="minimax",
|
||
model="MiniMax-M3",
|
||
extra_body={"reasoning_effort": "high"},
|
||
)
|
||
_conclude(
|
||
"L4",
|
||
_coverage(rows, planned_rounds=rounds, proposition="enabled"),
|
||
proposition="enabled",
|
||
)
|
||
|
||
async def test_l5_non_stream_path_is_distinguishable_and_honestly_unknown(self):
|
||
"""保留流／非流预算；UNKNOWN 是明确未覆盖而非长度锚点成功。"""
|
||
rounds = max(3, _ROUNDS // 2)
|
||
off = await _run_rounds(
|
||
rounds,
|
||
matrix_id="L5-off",
|
||
stream=False,
|
||
provider="minimax",
|
||
model="MiniMax-M3",
|
||
enable_thinking=False,
|
||
)
|
||
on = await _run_rounds(
|
||
rounds,
|
||
matrix_id="L5-on",
|
||
stream=False,
|
||
provider="minimax",
|
||
model="MiniMax-M3",
|
||
reasoning_effort=Effort.MEDIUM,
|
||
)
|
||
_conclude(
|
||
"L5",
|
||
_combine(
|
||
[
|
||
_coverage(off, planned_rounds=rounds, proposition="disabled"),
|
||
_coverage(on, planned_rounds=rounds, proposition="enabled"),
|
||
]
|
||
),
|
||
proposition="nonstream-enabled-disabled",
|
||
)
|
||
|
||
|
||
def _combine(verdicts):
|
||
"""任一失败优先，部分未覆盖不得汇总全 PASS。"""
|
||
return combine_live_verdicts(verdicts)
|
||
|
||
|
||
class TestOtherProviders:
|
||
"""既有供应商开启／关闭真实矩阵。"""
|
||
|
||
@pytest.mark.parametrize(
|
||
("matrix", "provider", "model"),
|
||
[("L6", "qwen", "qwen3.7-plus"), ("L7", "deepseek", "deepseek-v4-pro")],
|
||
)
|
||
async def test_existing_profiles_still_disable(self, matrix, provider, model):
|
||
rows = await _run_rounds(
|
||
_ROUNDS, matrix_id=matrix, provider=provider, model=model, enable_thinking=False
|
||
)
|
||
_conclude(
|
||
matrix,
|
||
_coverage(rows, planned_rounds=_ROUNDS, proposition="disabled"),
|
||
proposition="disabled",
|
||
)
|
||
|
||
async def test_qwen_enabled_is_observed(self):
|
||
rows = await _run_rounds(
|
||
_ROUNDS, matrix_id="L6b", provider="qwen", model="qwen3.7-plus", enable_thinking=True
|
||
)
|
||
_conclude(
|
||
"L6b",
|
||
_coverage(rows, planned_rounds=_ROUNDS, proposition="enabled"),
|
||
proposition="enabled",
|
||
)
|
||
|
||
|
||
class TestCapabilityDrift:
|
||
"""只运行可关闭声明的真实验证；不可关闭装配拒绝另在 unit。"""
|
||
|
||
@pytest.mark.parametrize(
|
||
"model",
|
||
sorted(
|
||
model for model, capability in DEFAULT_CAPABILITIES.items() if capability.can_disable
|
||
),
|
||
)
|
||
async def test_declared_capability_matches_reality(self, model):
|
||
rounds = max(3, _ROUNDS // 2)
|
||
rows = await _run_rounds(
|
||
rounds,
|
||
matrix_id="L8",
|
||
provider=_MODEL_PROVIDER[model],
|
||
model=model,
|
||
enable_thinking=False,
|
||
)
|
||
_conclude(
|
||
"L8",
|
||
_coverage(rows, planned_rounds=rounds, proposition="disabled"),
|
||
proposition="disabled",
|
||
)
|
||
|
||
|
||
async def _probe_effort(model, effort, *, rounds, prompt, prompt_kind):
|
||
"""临时全档表仅用于 T10 探测，不写回 DEFAULT，也不生成预期 wire。"""
|
||
return await _collect_rounds(
|
||
_tier_settings(model),
|
||
rounds=rounds,
|
||
stream=True,
|
||
prompt=prompt,
|
||
matrix_id="T10-" + prompt_kind,
|
||
effort=effort,
|
||
capabilities={model: ThinkingCapability(_ALL_EFFORTS, evidence="T10 临时探测声明")},
|
||
concurrency=_TIER_CONCURRENCY,
|
||
)
|
||
|
||
|
||
class TestTierProbe:
|
||
"""逐型号能力命题，不把拒绝、不可关闭和 UNKNOWN 混在一起。"""
|
||
|
||
@pytest.mark.parametrize("model", sorted(_MODEL_PROVIDER))
|
||
async def test_t10_none_direction_matches_declaration(self, model):
|
||
capability = DEFAULT_CAPABILITIES.get(model)
|
||
proposition = "disabled" if capability and capability.can_disable else "cannot_disable"
|
||
short = await _probe_effort(
|
||
model, Effort.NONE, rounds=_TIER_ROUNDS, prompt=_TIER_PROMPT, prompt_kind="none-short"
|
||
)
|
||
verdict = _coverage(short, planned_rounds=_TIER_ROUNDS, proposition=proposition)
|
||
# 沿既有矩阵：短档没有 OBSERVED 才做长上下文复核；不新增锚点调用。
|
||
if _qualified(short, planned_rounds=_TIER_ROUNDS).status == "PASS" and not any(
|
||
r["response"].thinking_observation is ThinkingObservation.OBSERVED for r in short
|
||
):
|
||
long_rows = await _probe_effort(
|
||
model,
|
||
Effort.NONE,
|
||
rounds=_TIER_LONG_ROUNDS,
|
||
prompt=_TIER_LONG_PROMPT,
|
||
prompt_kind="none-long",
|
||
)
|
||
verdict = _coverage(
|
||
short + long_rows,
|
||
planned_rounds=_TIER_ROUNDS + _TIER_LONG_ROUNDS,
|
||
proposition=proposition,
|
||
)
|
||
if capability is None and verdict.status != "FAIL":
|
||
verdict = LiveVerdict("UNCOVERED", "未登记候选只保留观测，不自动登记能力")
|
||
_conclude("T10-none", verdict, proposition=proposition)
|
||
|
||
@pytest.mark.parametrize("model", sorted(DEFAULT_CAPABILITIES))
|
||
async def test_t10_declared_tiers_actually_reason(self, model):
|
||
tiers = [e for e in DEFAULT_CAPABILITIES[model].supported_efforts if e is not Effort.NONE]
|
||
verdicts = []
|
||
for tier in tiers:
|
||
rows = await _probe_effort(
|
||
model,
|
||
tier,
|
||
rounds=_TIER_ROUNDS,
|
||
prompt=_TIER_PROMPT,
|
||
prompt_kind="tier-" + tier.value,
|
||
)
|
||
verdict = _coverage(rows, planned_rounds=_TIER_ROUNDS, proposition="enabled")
|
||
verdicts.append(verdict)
|
||
write_live_round(
|
||
_OUT_DIR,
|
||
run_id=uuid4().hex,
|
||
matrix_id="T10-tier",
|
||
round_index=0,
|
||
safe_fields={
|
||
"requested_model": model,
|
||
"requested_effort": tier.value,
|
||
"status": verdict.status,
|
||
"reason": verdict.reason,
|
||
},
|
||
)
|
||
_conclude("T10-tiers", _combine(verdicts), proposition="enabled-all-declared-tiers")
|
||
|
||
@pytest.mark.parametrize("model", ["gemini-3.1-pro", "gpt-5.5", "glm-5.3"])
|
||
async def test_t10_no_opinion_stays_no_opinion(self, model):
|
||
rows = await _collect_rounds(
|
||
_tier_settings(model),
|
||
rounds=_TIER_ROUNDS,
|
||
stream=True,
|
||
prompt=_TIER_PROMPT,
|
||
matrix_id="T10-default",
|
||
capabilities={model: ThinkingCapability(_ALL_EFFORTS, evidence="T10 临时探测声明")},
|
||
)
|
||
verdict = _qualified(rows, planned_rounds=_TIER_ROUNDS)
|
||
if verdict.status == "PASS" and any(
|
||
row["response"].applied_effort is not None for row in rows
|
||
):
|
||
verdict = LiveVerdict("FAIL", "默认基线擅自推定档位")
|
||
_conclude("T10-default", verdict, proposition="no-opinion-not-capability")
|