From 73008ad7d55fbcbd4b53027530fda06eae629d05 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Wed, 9 Sep 2026 02:40:13 -0400 Subject: [PATCH] test: apply evidence-based live checks without hiding regressions --- tests/e2e/conftest.py | 496 +++++++++ tests/e2e/test_compat_projects.py | 155 +-- tests/e2e/test_embed_probe.py | 142 +-- tests/e2e/test_smoke_gateway.py | 173 ++-- tests/e2e/test_thinking_live.py | 1553 ++++++++--------------------- tests/live_evidence.py | 308 ++++++ tests/unit/test_client.py | 22 + tests/unit/test_config.py | 44 + tests/unit/test_live_evidence.py | 816 +++++++++++++++ 9 files changed, 2350 insertions(+), 1359 deletions(-) create mode 100644 tests/e2e/conftest.py create mode 100644 tests/live_evidence.py create mode 100644 tests/unit/test_live_evidence.py diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 0000000..eef1582 --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,496 @@ +"""测试侧独立 HTTP 取证装配；无环境自读取或成功 SSE 预读。""" + +from collections.abc import AsyncIterator, Iterator, Mapping +from contextlib import AsyncExitStack, asynccontextmanager, contextmanager +from contextvars import ContextVar +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import httpx +import pytest + +from polygateway import GatewayClient, GatewaySettings +from polygateway.client import ( + _aclose_component, + _build_breaker, + _build_limiter, + _build_selector, + _build_structured, +) +from polygateway.providers import get_provider +from polygateway.transports.openai_compat import OpenAICompatTransport +from polygateway.types import Effort, EmbeddingTransportResult, SourceConfig, TransportResult +from tests.live_evidence import ( + AttemptEvidence, + HttpEvidence, + LiveVerdict, + assess_model_identity, + classify_live_failure, + messages_digest, + request_is_valid, + safe_attempts, + strict_json, + write_live_round, +) + + +@dataclass +class _Exchange: + """仅在 attempt 生命周期持有原始响应引用。""" + + request: httpx.Request + checks: tuple[tuple[str, bool], ...] + response: httpx.Response | None = None + + +@dataclass +class _Attempt: + """任务内可变收集器，结束时转换为冻结快照。""" + + call_id: str + exchanges: list[_Exchange] = field(default_factory=list) + + +class LiveCapture: + """矩阵显式预期与按逻辑轮次关联的独立证据。""" + + def __init__(self, *, expectations: Mapping[str, Mapping[str, Any]]) -> None: + """预期缺项即配置错误，不从实发 payload 补齐。""" + for expected in expectations.values(): + required = {"model", "origin", "path", "control", "messages_digest"} + if not required <= expected.keys() or ("stream" in expected) == ( + "input_shape" in expected + ): + raise ValueError("取证矩阵缺少必需预期或混用 chat/embed") + if not isinstance(expected["control"], dict): + raise ValueError("control 必须是显式对象") + self._expectations = {name: dict(value) for name, value in expectations.items()} + self._round: ContextVar[tuple[str, str]] = ContextVar("live_round") + self._attempt: ContextVar[_Attempt] = ContextVar("live_attempt") + self._records: dict[tuple[str, str], list[AttemptEvidence]] = {} + self._owners: dict[str, tuple[str, str]] = {} + self._notes: dict[tuple[str, str], list[str]] = {} + + @contextmanager + def round_context(self, *, session_id: str, parent_call_id: str) -> Iterator[None]: + """外围逻辑轮次绑定，异常和取消均复位。""" + key = session_id, parent_call_id + token = self._round.set(key) + self._records.setdefault(key, []) + self._notes.setdefault(key, []) + try: + yield + finally: + self._round.reset(token) + + @contextmanager + def attempt_context(self, call_id: str) -> Iterator[None]: + """零 HTTP 尝试也有快照；重复／跨轮 UUID 是契约错误。""" + key = self._round.get() + if call_id in self._owners: + raise ValueError("重复或跨轮 call_id") + self._owners[call_id] = key + attempt = _Attempt(call_id) + token = self._attempt.set(attempt) + error = None + try: + yield + except Exception as exc: + error = exc + raise + finally: + try: + events = tuple( + self._snapshot(exchange, call_id, key) for exchange in attempt.exchanges + ) + self._records[key].append(AttemptEvidence(call_id, events, error)) + finally: + self._attempt.reset(token) + + def _snapshot(self, exchange: _Exchange, call_id: str, key: tuple[str, str]) -> HttpEvidence: + """complete 结束后只读已缓冲内容，拒绝截断和歧义身份。""" + response = exchange.response + identity: tuple[bool, str | None] = (False, None) + body = None + status = response.status_code if response is not None else 0 + if response is None: + self._notes[key].append("无可配对响应") + else: + try: + content = response.content + except httpx.ResponseNotRead: + self._notes[key].append("响应未缓冲，证据不足") + else: + if status >= 400: + if len(content) <= 65536: + body = content + else: + self._notes[key].append("错误体超过 64 KiB，不接受截断证据") + elif 200 <= status < 300 and dict(exchange.checks).get("stream") is not None: + try: + payload = strict_json(exchange.request.content) + except (ValueError, UnicodeError): + payload = {} + if isinstance(payload, dict) and payload.get("stream") is False: + try: + data = strict_json(content) + if not isinstance(data, dict): + raise ValueError("原始 JSON 非对象") + model = data.get("model") + if model is not None and not isinstance(model, str): + raise ValueError("原始 model 类型非法") + identity = (True, model) + except (ValueError, UnicodeError): + self._notes[key].append("原始 JSON 身份无法独立解析") + return HttpEvidence(call_id, exchange.checks, status, body, identity) + + def client_factory(self, source: SourceConfig) -> httpx.AsyncClient: + """鉴权仅内存比较；沿已校验源 timeout/trust_env。""" + expected = self._expectations[source.name] + + async def request_hook(request: httpx.Request) -> None: + """校验实发请求而不修正它。""" + attempt = self._attempt.get() + try: + payload = strict_json(request.content) + except (ValueError, UnicodeError): + payload = {} + if not isinstance(payload, dict): + payload = {} + url = request.url + origin = str( + url.copy_with(path="", query=None, fragment=None, username=None, password=None) + ).rstrip("/") + # control 是本轮完整附加字段；基础键以外均比较，漏／多键都失败。 + basic = {"model", "messages", "stream", "stream_options", "input"} + control = {k: v for k, v in payload.items() if k not in basic} + checks = { + "method": request.method == "POST", + "origin": origin == expected["origin"] + and not url.username + and not url.password + and not url.query, + "path": url.path == expected["path"], + "model": payload.get("model") == expected["model"], + "authorization": request.headers.get("Authorization") == f"Bearer {source.api_key}", + "control": control == expected["control"], + "messages_digest": messages_digest(payload.get("messages", payload.get("input"))) + == expected["messages_digest"], + } + if "stream" in expected: + checks["stream"] = payload.get("stream") is expected["stream"] and ( + payload.get("stream_options") == {"include_usage": True} + if expected["stream"] + else "stream_options" not in payload + ) + else: + texts = payload.get("input") + checks["input_shape"] = ( + isinstance(texts, list) + and all(isinstance(text, str) for text in texts) + and len(texts) == expected["input_shape"] + ) + attempt.exchanges.append(_Exchange(request, tuple(checks.items()))) + + async def response_hook(response: httpx.Response) -> None: + """只持有引用，绝不提前读取成功 SSE。""" + attempt = self._attempt.get() + matching = [event for event in attempt.exchanges if event.request is response.request] + if len(matching) != 1 or matching[0].response is not None: + raise ValueError("响应无法唯一配对") + matching[0].response = response + + return httpx.AsyncClient( + headers={"Authorization": f"Bearer {source.api_key}"}, + timeout=source.timeout_s, + trust_env=source.trust_env, + event_hooks={"request": [request_hook], "response": [response_hook]}, + ) + + def attempts(self, *, session_id: str, parent_call_id: str) -> tuple[AttemptEvidence, ...]: + """按逻辑轮次返回不可变快照。""" + return tuple(self._records.get((session_id, parent_call_id), ())) + + def notes(self, *, session_id: str, parent_call_id: str) -> tuple[str, ...]: + """只含固定安全原因，不包含响应正文。""" + return tuple(self._notes.get((session_id, parent_call_id), ())) + + def raw_identity( + self, *, session_id: str, parent_call_id: str, call_id: str + ) -> tuple[bool, str | None]: + """精确取最终成功 attempt，不猜本轮最后一条响应。""" + key = session_id, parent_call_id + if call_id in self._owners and self._owners[call_id] != key: + raise ValueError("跨轮 call_id 身份查询") + matches = [attempt for attempt in self._records.get(key, []) if attempt.call_id == call_id] + if len(matches) > 1: + raise ValueError("重复 call_id 身份查询") + if not matches: + return False, None + events = [event for event in matches[0].http if 200 <= event.status_code < 300] + if len(events) > 1: + raise ValueError("多个成功 HTTP 身份候选") + return events[0].raw_identity if events else (False, None) + + +class ObservedTransport: + """原样委托同一个真实 transport，无重试、payload 修正或异常翻译。""" + + def __init__(self, transport: OpenAICompatTransport, capture: LiveCapture) -> None: + """资源所有权留给装配者。""" + self._transport = transport + self._capture = capture + + async def complete( + self, + *, + messages: list[dict[str, Any]], + source: SourceConfig, + stream: bool, + overlay: dict[str, Any], + call_id: str, + reasoning_effort: Effort | None, + ) -> TransportResult: + """与生产端口逐参数同签名。""" + with self._capture.attempt_context(call_id): + return await self._transport.complete( + messages=messages, + source=source, + stream=stream, + overlay=overlay, + call_id=call_id, + reasoning_effort=reasoning_effort, + ) + + async def embed( + self, *, texts: list[str], source: SourceConfig, call_id: str + ) -> EmbeddingTransportResult: + """embedding 使用同一取证关联，不套 chat 推理判据。""" + with self._capture.attempt_context(call_id): + return await self._transport.embed(texts=texts, source=source, call_id=call_id) + + +@asynccontextmanager +async def observed_client( + settings: GatewaySettings, capture: LiveCapture, *, capabilities=None +) -> AsyncIterator[GatewayClient]: + """全量注入复用生产装配函数；自建组件显式关闭，不启用响应缓存。""" + async with AsyncExitStack() as stack: + sources = list(settings.sources) + limiter = _build_limiter(settings, sources) + stack.push_async_callback(_aclose_component, limiter) + breaker = _build_breaker(settings) + stack.push_async_callback(_aclose_component, breaker) + real = OpenAICompatTransport( + client_factory=capture.client_factory, capabilities=capabilities + ) + stack.push_async_callback(real.aclose) + strategy, escalation = _build_structured( + [get_provider(source.provider) for source in sources] + ) + client = GatewayClient( + scope=settings.scope, + sources=sources, + selector=_build_selector(settings.selector), + limiter=limiter, + breaker=breaker, + transport=ObservedTransport(real, capture), + retry=settings.retry, + backpressure=settings.backpressure, + quota_full=settings.quota_full, + circuit_open=settings.circuit_open, + structured_strategy=strategy, + structured_escalation=escalation, + structured_max_retries=settings.structured_max_retries, + ) + stack.push_async_callback(client.aclose) + yield client + + +def chat_expectations( + settings: GatewaySettings, + *, + messages: list[dict[str, Any]], + stream: bool, + controls: Mapping[str, dict[str, Any]], +) -> dict[str, dict[str, Any]]: + """URL 从源配置声明，控制片段必须由矩阵独立给出。""" + result = {} + for source in settings.sources: + url = httpx.URL(source.base_url) + result[source.name] = { + "model": source.model, + "origin": str( + url.copy_with(path="", query=None, fragment=None, username=None, password=None) + ).rstrip("/"), + "path": url.path.rstrip("/") + "/chat/completions", + "stream": stream, + "control": controls[source.name], + "messages_digest": messages_digest(messages), + } + return result + + +def enforce_verdict(verdict: LiveVerdict) -> None: + """仅在报告已写入后调用；默认失败，不打印上游异常正文。""" + if verdict.status == "UNCOVERED": + pytest.skip(verdict.reason) + assert verdict.status == "PASS", verdict.reason + + +async def captured_chat_round( + client: GatewayClient, + capture: LiveCapture, + *, + run_id: str, + matrix_id: str, + round_index: int, + output_dir: Path, + messages: list[dict[str, Any]], + models: Mapping[str, str], + aliases: Mapping[str, frozenset[str]], + validate=None, + providers: Mapping[str, str] | None = None, + source_efforts: Mapping[str, Effort | None] | None = None, + **kwargs: Any, +) -> tuple[Any, LiveVerdict]: + """请求、身份与行为断言均先记逐轮证据；异常不漏轮。""" + parent = uuid4().hex + response = None + error = None + verdict = LiveVerdict("FAIL", "轮次未完成") + with capture.round_context(session_id=run_id, parent_call_id=parent): + try: + response = await client.chat( + messages, session_id=run_id, parent_call_id=parent, **kwargs + ) + attempts = capture.attempts(session_id=run_id, parent_call_id=parent) + events = [event for attempt in attempts for event in attempt.http] + successful = [attempt for attempt in attempts if attempt.call_id == response.call_id] + success_paired = ( + len(successful) == 1 + and successful[0].error is None + and len(successful[0].http) == 1 + and 200 <= successful[0].http[0].status_code < 300 + ) + verdict = assess_model_identity( + requested=models[response.source_name], + aliases=aliases.get(models[response.source_name], frozenset()), + reported=response.model_reported, + raw_identity=capture.raw_identity( + session_id=run_id, parent_call_id=parent, call_id=response.call_id + ), + request_valid=success_paired + and bool(events) + and all(request_is_valid(event) for event in events), + ) + if verdict.status == "PASS" and validate is not None: + validate(response) + except Exception as exc: + error = exc + verdict = classify_live_failure( + exc, capture.attempts(session_id=run_id, parent_call_id=parent) + ) + finally: + attempts = capture.attempts(session_id=run_id, parent_call_id=parent) + # 上游 model 可能回显提示词；仅输出允许集合内的名字，其他统一省略。 + allowed = set(models.values()) | { + alias for values in aliases.values() for alias in values + } + write_live_round( + output_dir, + run_id=run_id, + matrix_id=matrix_id, + round_index=round_index, + safe_fields={ + "session_id": run_id, + "parent_call_id": parent, + "requested_model": list(models.values()), + "provider": list(providers.values()) if providers is not None else None, + "attempts": safe_attempts(attempts), + "status": verdict.status, + "reason": verdict.reason, + "stream": kwargs.get("stream", True), + "requested_effort": kwargs.get("reasoning_effort") + or (list(source_efforts.values()) if source_efforts is not None else None), + "completed_rounds": 1, + "prompt_tokens": response.prompt_tokens if response else None, + "completion_tokens": response.completion_tokens if response else None, + "reasoning_tokens": response.reasoning_tokens if response else None, + "thinking_chars": len(response.thinking) if response else None, + "evidence_notes": ( + "原始异常／正文摘要省略以避免回显泄露", + *capture.notes(session_id=run_id, parent_call_id=parent), + ), + "reported_model": response.model_reported + if response and response.model_reported in allowed + else None, + "applied_effort": response.applied_effort if response else None, + "thinking_observation": response.thinking_observation if response else None, + "error_type": type(error).__name__ if error else None, + "error_status": getattr(error, "status_code", None), + }, + ) + return response, verdict + + +def declared_control(provider: str, effort: Effort | None) -> dict[str, Any]: + """测试矩阵的独立 wire 声明；不调用 resolver 或生产 payload 构造器。""" + if effort is None: + return {} + if provider == "qwen": + if effort not in (Effort.AUTO, Effort.NONE): + raise ValueError("测试矩阵未声明 qwen 强度映射") + return {"enable_thinking": effort is not Effort.NONE} + if provider in {"deepseek", "zhipu", "moonshot"}: + result: dict[str, Any] = { + "thinking": {"type": "disabled" if effort is Effort.NONE else "enabled"} + } + if effort not in (Effort.AUTO, Effort.NONE): + result["reasoning_effort"] = effort.value + return result + if provider not in {"minimax", "openai", "anthropic", "google"}: + raise ValueError("测试矩阵没有该 provider 的控制声明") + return {} if effort is Effort.AUTO else {"reasoning_effort": effort.value} + + +def source_controls(settings: GatewaySettings) -> dict[str, dict[str, Any]]: + """源级矩阵预期独立表达；受管 raw 冲突由生产路径拒绝。""" + result = {} + for source in settings.sources: + effort = source.reasoning_effort + if effort is None and source.enable_thinking is not None: + effort = Effort.AUTO if source.enable_thinking else Effort.NONE + control = declared_control(source.provider, effort) + if effort is None: + control.update(source.extra_body) + else: + # 不用 update 覆盖控制声明，否则会掩盖所有权回归。 + for key, value in source.extra_body.items(): + if key in control: + raise ValueError("取证矩阵有双来源控制") + control[key] = value + result[source.name] = control + return result + + +# pytest 用例终态补充网络前缺配置、装配失败及未完成轮次；不代替逐轮报告。 +@pytest.hookimpl(wrapper=True) +def pytest_runtest_makereport(item, call): + """仅记录安全矩阵标识和阶段结果，不序列化 pytest 异常长文本。""" + report = yield + if report.skipped or report.failed: + write_live_round( + Path("tests/outputs/134/live"), + run_id=uuid4().hex, + matrix_id="pytest-" + messages_digest(item.nodeid)[:16], + round_index=0, + safe_fields={ + "status": "UNCOVERED" if report.skipped else "FAIL", + "reason": "用例阶段未覆盖或失败；详情按逐轮安全证据核验，不能视为能力通过", + "evidence_notes": [report.when], + }, + ) + return report diff --git a/tests/e2e/test_compat_projects.py b/tests/e2e/test_compat_projects.py index 234d6e6..af31bc7 100644 --- a/tests/e2e/test_compat_projects.py +++ b/tests/e2e/test_compat_projects.py @@ -1,98 +1,119 @@ -"""GovDoc 与 Video-Tree 最小接入冒烟(2026-07-20 拍板: 两个项目都做)。 - -复刻两项目的真实调用点形态,对真实网关跑一次治理调用,证明"调用点零改动 -迁移"成立;并验证 VT 现有平铺键名(LLM_TIMEOUT 等)可直接装配。 -reference/ 只读——本文件只 import 其 Protocol,绝不修改。 -""" +"""历史接入调用形态的真实冒烟；不能替代缺失下游的现行配置验收。""" import os import sys from pathlib import Path +from uuid import uuid4 import pytest from dotenv import dotenv_values -from polygateway import GatewayClient +from polygateway import Effort, GatewayClient, GatewaySettings +from tests.e2e.conftest import ( + LiveCapture, + captured_chat_round, + chat_expectations, + enforce_verdict, + observed_client, + source_controls, +) +from tests.live_evidence import write_live_round _REPO = Path(__file__).resolve().parents[2] _ENV = {k: v for k, v in {**dotenv_values(".env"), **os.environ}.items() if v is not None} -_HAS_SOURCE = any(k.split("__")[0] == "LLM" and k.endswith("__API_KEY") for k in _ENV) - -# 真实网关调用: 与 test_thinking_live.py 同待遇标 slow(pytest addopts 默认排除, -# 显式 `pytest -m slow` 运行)。理由是这些用例的成败取决于网关此刻快不快,而 -# pre-commit 关卡跑全套件——网关一抖就挡住与之无关的提交,久了会把"测试红了 -# 先怀疑网关"变成惯性,真 bug 也会被当成抖动重试掉。发版清单负责让它们真跑。 +_HAS_SOURCE = any(k.startswith("LLM__") and k.endswith("__API_KEY") for k in _ENV) pytestmark = [ pytest.mark.slow, - pytest.mark.skipif( - not _HAS_SOURCE, reason="需真实网关凭据: 在 .env 配置 LLM__{PROVIDER}__1__*" - ), + pytest.mark.skipif(not _HAS_SOURCE, reason="缺少矩阵必需凭据，未覆盖"), ] +_OUT = Path("tests/outputs/134/live") -@pytest.fixture -async def client(): - c = GatewayClient.from_env("LLM", env=_ENV) - yield c - await c.aclose() +async def _call_shape(matrix, **kwargs): + """session/parent 总由逐轮 UUID 传入；保留 cache_salt 调用形态。""" + settings = GatewaySettings.from_env("LLM", env=_ENV) + messages = [{"role": "user", "content": "Reply with exactly: compatibility-ok"}] + capture = LiveCapture( + expectations=chat_expectations( + settings, messages=messages, stream=True, controls=source_controls(settings) + ) + ) + + def validate(response): + assert response.content.strip() and response.call_id + + async with observed_client(settings, capture) as client: + _, verdict = await captured_chat_round( + client, + capture, + run_id=uuid4().hex, + matrix_id=matrix, + round_index=1, + output_dir=_OUT, + messages=messages, + models={s.name: s.model for s in settings.sources}, + providers={s.name: s.provider for s in settings.sources}, + source_efforts={ + s.name: s.reasoning_effort + if s.reasoning_effort is not None + else (Effort.AUTO if s.enable_thinking else Effort.NONE) + if s.enable_thinking is not None + else None + for s in settings.sources + }, + aliases={}, + validate=validate, + **kwargs, + ) + enforce_verdict(verdict) class TestGovDocOnboarding: - """GovDoc agent/loop.py:377 调用形态: session_id + parent_call_id。""" + """历史 session_id＋parent_call_id 调用点契约。""" - async def test_call_site_shape_runs_governed(self, client): - response = await client.chat( - [{"role": "user", "content": "Reply with exactly: govdoc-ok"}], - session_id="govdoc-e2e", - parent_call_id="step-1", - ) - assert response.content.strip() - assert response.call_id # GovernedLLMClient 契约字段全在 + async def test_call_site_shape_runs_governed(self): + await _call_shape("compat-parent") - async def test_structural_protocol_match(self, client): + async def test_structural_protocol_match(self): + """外部 Protocol 缺包单列未覆盖；合成契约另在 unit 跑。""" + run_id = uuid4().hex sys.path.insert(0, str(_REPO / "reference/GovDoc-SaaS/packages/docagent-core/src")) try: from docagent_core.protocols import LLMProvider except ImportError: - pytest.skip("GovDoc protocols 依赖不可导入(结构断言已由单测兜底覆盖)") + write_live_round( + _OUT, + run_id=run_id, + matrix_id="external-protocol", + round_index=0, + safe_fields={ + "status": "UNCOVERED", + "reason": "外部 Protocol 包缺失；未验证真实下游", + }, + ) + pytest.skip("外部 Protocol 包缺失，未覆盖") finally: sys.path.pop(0) - assert isinstance(client, LLMProvider) + status = "FAIL" + client = None + try: + client = GatewayClient.from_env("LLM", env=_ENV) + assert isinstance(client, LLMProvider) + status = "PASS" + finally: + if client is not None: + await client.aclose() + write_live_round( + _OUT, + run_id=run_id, + matrix_id="external-protocol", + round_index=0, + safe_fields={"status": status, "reason": "外部 Protocol 结构契约，不是模型能力"}, + ) class TestVideoTreeOnboarding: - """VT loop.py:336 调用形态: session_id + cache_salt(跨 epoch 重采样)。""" + """历史 cache_salt 调用点契约，平铺键装配已移至 unit。""" - async def test_call_site_shape_with_cache_salt(self, client): - response = await client.chat( - [{"role": "user", "content": "Reply with exactly: vt-ok"}], - session_id="vt-e2e", - cache_salt="epoch-1", - ) - assert response.content.strip() - - async def test_flat_legacy_keys_assemble(self): - """VT 现有键名(LLM_TIMEOUT/LLM_MAX_RETRIES 等)零改名装配成功。""" - source_keys = {k: v for k, v in _ENV.items() if k.split("__")[0] == "LLM" and "__" in k} - flat_env = { - **source_keys, - # 与 .env 的 LLM__MINIMAX__1__TIMEOUT_S 同值。取 120(VT 旧值)会让本用例的 - # 超时比生产配置还紧一半,在慢网关上必然间歇红——而本用例断言的是平铺 - # 键名能否解析成 SourceConfig.timeout_s,超时取值本身不是被测对象 - "LLM_TIMEOUT": "300", - "LLM_MAX_RETRIES": "3", - "LLM_RETRY_BASE_DELAY": "2.0", - "LLM_RETRY_MAX_DELAY": "30.0", - "LLM_CIRCUIT_BREAKER_THRESHOLD": "5", - "LLM_CIRCUIT_BREAKER_COOLDOWN": "60", - "LLM_TTFT_TIMEOUT": "30", - "LLM_INTER_TOKEN_TIMEOUT": "15", - "PGW_CACHE_BACKEND": "none", - "PGW_TELEMETRY_BACKEND": "none", - } - client = GatewayClient.from_env("LLM", env=flat_env) - try: - resp = await client.chat([{"role": "user", "content": "Reply: flat-ok"}]) - assert resp.content.strip() - finally: - await client.aclose() + async def test_call_site_shape_with_cache_salt(self): + await _call_shape("compat-salt", cache_salt="epoch-1") diff --git a/tests/e2e/test_embed_probe.py b/tests/e2e/test_embed_probe.py index 90d018c..7f8662b 100644 --- a/tests/e2e/test_embed_probe.py +++ b/tests/e2e/test_embed_probe.py @@ -1,89 +1,91 @@ -"""真实网关 /embeddings 端点探测(M2 设计 §11.6;人类默认口径: 实现时探测)。 - -对 .env 的 LLM 源网关发一次真实 embeddings 请求: 支持则记录向量证据, -不支持(404/翻译为领域错误)则 skip 并把响应记录进 tests/outputs/ -(降级证据)。无 EMBED scope 配置时复用 LLM 源的 base_url/api_key。 -""" - -from __future__ import annotations +"""真实 embedding 探测；404 仅证明请求型号不可用，不外推端点能力。""" import dataclasses import os -from datetime import datetime from pathlib import Path +from uuid import uuid4 +import httpx import pytest from dotenv import dotenv_values -from polygateway.errors import PolyGatewayError +from polygateway import GatewaySettings from polygateway.transports.openai_compat import OpenAICompatTransport -from polygateway.types import SourceConfig +from tests.e2e.conftest import LiveCapture, ObservedTransport, enforce_verdict +from tests.live_evidence import ( + LiveVerdict, + classify_live_failure, + messages_digest, + request_is_valid, + safe_attempts, + write_live_round, +) _ENV = {k: v for k, v in {**dotenv_values(".env"), **os.environ}.items() if v is not None} - -# 真实网关调用: 与 test_thinking_live.py 同待遇标 slow(pytest addopts 默认排除, -# 显式 `pytest -m slow` 运行)。理由见 test_compat_projects.py 同处注释。 pytestmark = [ pytest.mark.slow, - pytest.mark.skipif( - "LLM__MINIMAX__1__BASE_URL" not in _ENV, - reason="缺真实网关配置(.env)", - ), + pytest.mark.skipif("LLM__MINIMAX__1__BASE_URL" not in _ENV, reason="缺少矩阵必需配置，未覆盖"), ] -_OUT = Path("tests/outputs/embedding") - - -def _record(name: str, lines: list[str]) -> Path: - _OUT.mkdir(parents=True, exist_ok=True) - path = _OUT / f"{name}_{datetime.now():%Y%m%d_%H%M%S}.md" - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - return path - async def test_probe_real_gateway_embeddings(): - source = SourceConfig( - name="probe_1", - provider="minimax", - base_url=_ENV["LLM__MINIMAX__1__BASE_URL"], - api_key=_ENV["LLM__MINIMAX__1__API_KEY"], - model=_ENV.get("PGW_EMBED_PROBE_MODEL", "text-embedding-v1"), - timeout_s=30.0, - est_tokens=8, + """沿已校验源的 timeout/trust_env，所有路径 finally 关闭。""" + settings = GatewaySettings.from_env("LLM", env=_ENV) + configured = next(s for s in settings.sources if s.name == "minimax_1") + source = dataclasses.replace( + configured, model=_ENV.get("PGW_EMBED_PROBE_MODEL", "text-embedding-v1") ) - transport = OpenAICompatTransport() + texts = ["polygateway embedding probe"] + url = httpx.URL(source.base_url) + capture = LiveCapture( + expectations={ + source.name: { + "model": source.model, + "origin": str(url.copy_with(path="", query=None)).rstrip("/"), + "path": url.path.rstrip("/") + "/embeddings", + "input_shape": 1, + "control": {}, + "messages_digest": messages_digest(texts), + } + } + ) + real = OpenAICompatTransport(client_factory=capture.client_factory) + transport = ObservedTransport(real, capture) + run_id, parent, call_id = uuid4().hex, uuid4().hex, uuid4().hex + verdict = LiveVerdict("FAIL", "轮次未完成") try: - result = await transport.embed( - texts=["polygateway embedding probe"], source=source, call_id="probe" - ) - except PolyGatewayError as exc: - path = _record( - "probe_unsupported", - [ - "# Embedding 端点探测: 网关不支持", - f"- base_url: {source.base_url}", - f"- model: {source.model}", - f"- 错误分类: {type(exc).__name__}", - f"- status_code: {exc.status_code}", - f"- 详情: {exc}", - "", - "结论: e2e 按设计 §11.6 降级,embedding 行为由 unit 全覆盖。", - ], - ) - await transport.aclose() - pytest.skip(f"网关不支持 embeddings({type(exc).__name__}),证据: {path}") - else: - await transport.aclose() - assert result.dim > 0 and len(result.vectors) == 1 - _record( - "probe_supported", - [ - "# Embedding 端点探测: 网关支持", - f"- base_url: {source.base_url}", - f"- model: {source.model}", - f"- dim: {result.dim}", - f"- usage: {result.prompt_tokens}({result.usage_source})", - f"- 向量前 5 维: {result.vectors[0][:5]}", - f"- raw: {dataclasses.asdict(result)['raw']}", - ], - ) + with capture.round_context(session_id=run_id, parent_call_id=parent): + try: + result = await transport.embed(texts=texts, source=source, call_id=call_id) + events = [ + e + for a in capture.attempts(session_id=run_id, parent_call_id=parent) + for e in a.http + ] + assert len(events) == 1 and request_is_valid(events[0]) + assert result.dim > 0 and len(result.vectors) == 1 + verdict = LiveVerdict("PASS", "向量形状与实发请求合格") + except Exception as error: + verdict = classify_live_failure( + error, capture.attempts(session_id=run_id, parent_call_id=parent) + ) + finally: + write_live_round( + Path("tests/outputs/134/live"), + run_id=run_id, + matrix_id="embedding", + round_index=1, + safe_fields={ + "status": verdict.status, + "reason": verdict.reason, + "session_id": run_id, + "parent_call_id": parent, + "attempts": safe_attempts( + capture.attempts(session_id=run_id, parent_call_id=parent) + ), + "evidence_notes": capture.notes(session_id=run_id, parent_call_id=parent), + }, + ) + finally: + await real.aclose() + enforce_verdict(verdict) diff --git a/tests/e2e/test_smoke_gateway.py b/tests/e2e/test_smoke_gateway.py index e318c85..208ad20 100644 --- a/tests/e2e/test_smoke_gateway.py +++ b/tests/e2e/test_smoke_gateway.py @@ -1,123 +1,110 @@ -"""真实网关端到端冒烟(M1 验收第 7 步)。 +"""真实网关冒烟：逐轮独立取证，行为断言失败也必须留档。""" -前置: `.env` 配置至少一个 `LLM__{PROVIDER}__1__*` 真实源 + 韧性键。 -缺配置时 skip(验收前必须真跑)。输出结构化 Markdown 落 -`tests/outputs/e2e/`(CLAUDE.md §4.6,不提交 git)。 -""" - -import json import os -from datetime import datetime from pathlib import Path +from uuid import uuid4 import pytest from dotenv import dotenv_values from pydantic import BaseModel -from polygateway import GatewayClient +from polygateway import Effort, GatewaySettings +from tests.e2e.conftest import ( + LiveCapture, + captured_chat_round, + chat_expectations, + enforce_verdict, + observed_client, + source_controls, +) _ENV = {k: v for k, v in {**dotenv_values(".env"), **os.environ}.items() if v is not None} -_HAS_SOURCE = any(k.split("__")[0] == "LLM" and k.endswith("__API_KEY") for k in _ENV) - -# 真实网关调用: 与 test_thinking_live.py 同待遇标 slow(pytest addopts 默认排除, -# 显式 `pytest -m slow` 运行)。理由见 test_compat_projects.py 同处注释。 +_HAS_SOURCE = any(k.startswith("LLM__") and k.endswith("__API_KEY") for k in _ENV) pytestmark = [ pytest.mark.slow, - pytest.mark.skipif( - not _HAS_SOURCE, - reason="需真实网关凭据: 在 .env 配置 LLM__{PROVIDER}__1__*(M1 验收前必须真跑)", - ), + pytest.mark.skipif(not _HAS_SOURCE, reason="缺少矩阵必需凭据，未覆盖"), ] -_OUT_DIR = Path("tests/outputs/e2e") - class MiniAnswer(BaseModel): + """最小结构化响应契约。""" + answer: int reason: str -def _report(name: str, sections: list[tuple[str, str]]) -> Path: - _OUT_DIR.mkdir(parents=True, exist_ok=True) - ts = datetime.now().strftime("%Y%m%d_%H%M%S") - path = _OUT_DIR / f"{name}_{ts}.md" - body = [f"# e2e 冒烟: {name}", ""] - for title, content in sections: - body += [f"## {title}", "", "```", content, "```", ""] - path.write_text("\n".join(body), encoding="utf-8") - return path - - -@pytest.fixture -async def client(): - c = GatewayClient.from_env("LLM", env=_ENV) - yield c - await c.aclose() +async def _smoke(matrix, prompt, validate, *, stream=True, structured=None): + """全量注入仅替换取证装配，仍调用生产结构化策略。""" + settings = GatewaySettings.from_env("LLM", env=_ENV) + messages = [{"role": "user", "content": prompt}] + controls = source_controls(settings) + capture = LiveCapture( + expectations=chat_expectations( + settings, messages=messages, stream=stream, controls=controls + ) + ) + async with observed_client(settings, capture) as client: + _, verdict = await captured_chat_round( + client, + capture, + run_id=uuid4().hex, + matrix_id=matrix, + round_index=1, + output_dir=Path("tests/outputs/134/live"), + messages=messages, + models={s.name: s.model for s in settings.sources}, + providers={s.name: s.provider for s in settings.sources}, + source_efforts={ + s.name: s.reasoning_effort + if s.reasoning_effort is not None + else (Effort.AUTO if s.enable_thinking else Effort.NONE) + if s.enable_thinking is not None + else None + for s in settings.sources + }, + aliases={}, + validate=validate, + stream=stream, + structured=structured, + ) + enforce_verdict(verdict) class TestRealGatewaySmoke: - async def test_stream_chat(self, client): - resp = await client.chat( - [{"role": "user", "content": "Reply with exactly: pong"}], session_id="e2e-smoke" - ) - path = _report( - "stream_chat", - [ - ("响应", resp.content), - ( - "元数据", - json.dumps( - { - "model": resp.model, - "source": resp.source_name, - "usage_source": resp.usage_source, - "prompt_tokens": resp.prompt_tokens, - "completion_tokens": resp.completion_tokens, - "latency_ms": resp.latency_ms, - "ttft_ms": resp.ttft_ms, - }, - ensure_ascii=False, - indent=2, - ), - ), - ], - ) - assert resp.content.strip() - assert resp.ttft_ms is not None and resp.latency_ms > 0 - print(f"输出: {path}") + """保留流／非流和结构化真实行为断言。""" - async def test_non_stream_fast_path(self, client): - resp = await client.chat( - [{"role": "user", "content": "Reply with exactly: pong"}], stream=False - ) - _report("non_stream", [("响应", resp.content)]) - assert resp.content.strip() and resp.ttft_ms is None + async def test_stream_chat(self): + def validate(response): + assert response.content.strip() + assert response.ttft_ms is not None and response.latency_ms > 0 - async def test_structured_json_tier(self, client): - resp = await client.chat( - [{"role": "user", "content": 'Reply ONLY with JSON: {"ok": true}'}], + await _smoke("smoke-stream", "Reply with exactly: pong", validate) + + async def test_non_stream_fast_path(self): + def validate(response): + assert response.content.strip() and response.ttft_ms is None + + await _smoke("smoke-json", "Reply with exactly: pong", validate, stream=False) + + async def test_structured_json_tier(self): + def validate(response): + assert isinstance(response.structured_data, dict | list) + + await _smoke( + "smoke-structured-json", + 'Reply ONLY with JSON: {"ok": true}', + validate, structured="json", ) - _report("structured_json", [("解析产物", repr(resp.structured_data))]) - assert isinstance(resp.structured_data, dict | list) - async def test_structured_model_ladder(self, client): - resp = await client.chat( - [ - { - "role": "user", - "content": "What is 2+3? Reply ONLY with JSON matching " - '{"answer": , "reason": }', - } - ], + async def test_structured_model_ladder(self): + def validate(response): + assert isinstance(response.structured_data, MiniAnswer) + assert response.structured_data.answer == 5 + + await _smoke( + "smoke-structured-model", + 'What is 2+3? Reply ONLY with JSON matching {"answer": , "reason": }', + validate, structured=MiniAnswer, ) - _report( - "structured_model", - [ - ("原始响应", resp.content), - ("校验产物", resp.structured_data.model_dump_json()), - ], - ) - assert isinstance(resp.structured_data, MiniAnswer) - assert resp.structured_data.answer == 5 diff --git a/tests/e2e/test_thinking_live.py b/tests/e2e/test_thinking_live.py index 054e7a6..6a24c42 100644 --- a/tests/e2e/test_thinking_live.py +++ b/tests/e2e/test_thinking_live.py @@ -1,83 +1,65 @@ -"""真实 API 验证推理开关与推理可观测性(issue #5 + #6;判据于 #16/#17 重建)。 +"""真实推理矩阵：独立请求／身份资格，逐轮留证，UNKNOWN 不证明关闭。 -本组用例**必须真跑**: 改动的正确性与具体模型强相关,mock 只能验证代码路径, -验证不了"这个参数在这个模型上到底关没关掉推理"。 - -三条判据纪律(第 1、2 条来自 findings §4c,第 1 条的推翻与第 3 条来自 -`findings/2026-08-25-thinking-observability-regression.md`): - -1. **判别量是库裁定的三态 `thinking_observation`,既不是 `reasoning_tokens` - 也不是 `completion_tokens`。** 长度判据早已排除: 两档的输出长度分布**是 - 重叠的**(实测关闭档最高 46 token、开启档最低 13 token),按阈值判两边都会 - 误判。而 `reasoning_tokens` 这个曾经"干净分开"的判据也已失效——MiniMax - 这一路上游不再返回 `usage.completion_tokens_details`,该字段恒 `None`;同一 - 次调用里库明明拿得到 185 字符推理正文,单看 token 计数却把"推理正常"读成 - "没推理"(2026-08-25 findings §3.4/结论③,四条用例因此假红)。三态裁定同时 - 看正文与计数: **正文是事实本身,token 计数只是对事实的转述**。 -2. **另配一个不含魔数的确定性锚点**(见 L2b、L5): 同一模型上,关闭档的 - `prompt_tokens` 严格小于开启档——供应商在开启时注入了推理指令,输入侧 - token 数随之变大。这是相对比较,不硬编码任何具体数值;且它不依赖上游是否 - 回传推理正文,所以在"观测不到推理"的非流式路径上依然作数。 -3. **`UNKNOWN` 不等于"没推理",不能拿它判红。** 关闭方向要求每轮"未观测到 - 推理"(`UNKNOWN` 计入满足——它没有证伪力),其证伪力来自: 模型若偷偷推理了, - 可观测路径会翻成 `OBSERVED`。开启方向只要求多数轮 `OBSERVED`;M3 非流式 - 路径整片观测不到,该档由 L5 用另一套断言覆盖。 - -源不可用一律 `skip` 并在报告中记为「未覆盖」,**绝不静默计入通过**。 +不新增成功 SSE 捕获器，不把整类异常跳过；历史长度／prompt 锚点仅作 +指定样本的形态回归，不提升能力覆盖。T10 不可关闭与预期拒绝是独立命题。 """ import asyncio import dataclasses -import json import os -from collections import Counter from collections.abc import Mapping -from datetime import datetime from pathlib import Path from types import MappingProxyType +from uuid import uuid4 import pytest from dotenv import dotenv_values -from polygateway import GatewayClient, GatewaySettings, ThinkingObservation -from polygateway.errors import ( - AllSourcesExhausted, - GatewayUnavailableError, - RequestRejectedError, - SourceDeadError, - TransientError, -) -from polygateway.providers import ProviderProfile, ThinkingWire, register_provider -from polygateway.thinking import DEFAULT_CAPABILITIES, ThinkingCapability, get_capability +from polygateway import GatewaySettings, ThinkingObservation +from polygateway.thinking import DEFAULT_CAPABILITIES, ThinkingCapability from polygateway.types import EFFORT_ORDER, Effort +from tests.e2e.conftest import ( + LiveCapture, + captured_chat_round, + chat_expectations, + declared_control, + enforce_verdict, + observed_client, + source_controls, +) +from tests.live_evidence import ( + LiveVerdict, + assess_thinking_coverage, + combine_live_verdicts, + qualify_live_rounds, + summarize_verdicts, + write_live_round, +) _ENV = {k: v for k, v in {**dotenv_values(".env"), **os.environ}.items() if v is not None} -_HAS_SOURCE = any(k.split("__")[0] == "LLM" and k.endswith("__API_KEY") for k in _ENV) - -# slow: 本组 92 次真实调用、约 4 分半(2026-08-26 判据换三态后实测;此前记的 -# "137 次、约 7 分钟"已被证伪,别照旧值估 CI 预算),且判据是统计性的——网络抖动 -# 会让它偶发失败(实测有一次 network_error 连续三次耗尽源)。让它阻断 `make ci` -# 会把测试变成噪声源,故沿用项目既有的 slow 标记默认排除,合并前用 `-m slow` -# 显式真跑并存档报告。"不自动门控"不等于"可跳过"。 +_HAS_SOURCE = any(k.startswith("LLM__") and k.endswith("__API_KEY") for k in _ENV) pytestmark = [ pytest.mark.slow, - pytest.mark.skipif( - not _HAS_SOURCE, reason="需真实网关凭据: 在 .env 配置 LLM__{PROVIDER}__1__*(本组必须真跑)" - ), + pytest.mark.skipif(not _HAS_SOURCE, reason="缺少矩阵必需凭据，未覆盖"), ] - -_OUT_DIR = Path("tests/outputs/e2e") +_OUT_DIR = Path("tests/outputs/134/live") _ROUNDS = int(os.environ.get("PGW_E2E_THINKING_ROUNDS", "10")) - -# 需要一点推理才能答对,但答案极短: 关掉推理时 completion 稳定在个位数, -# 开着时则是几百——两档之间隔着一个数量级,判据不必卡在噪声里 +_TIER_ROUNDS = int(os.environ.get("PGW_E2E_TIER_ROUNDS", "5")) +_TIER_LONG_ROUNDS = int(os.environ.get("PGW_E2E_TIER_LONG_ROUNDS", "3")) +_TIER_CONCURRENCY = int(os.environ.get("PGW_E2E_TIER_CONCURRENCY", "3")) _PROMPT = "一个笼子里有若干鸡和兔,共 35 个头、94 只脚。鸡和兔各有多少只?只输出两个数字。" +_TIER_PROMPT = "23 乘以 47 等于多少?只回答一个数字,不要解释。" +_TIER_LONG_PROMPT = ( + "\n".join( + f"{i:04d}. 这是一段与题目无关的填充文字,仅用于把上下文撑到数千 token," + "以复核短提示词下得到的关闭结论在长上下文下是否依然成立。" + for i in range(120) + ) + + "\n\n" + + _TIER_PROMPT +) +_ALL_EFFORTS = (*EFFORT_ORDER, Effort.AUTO) -_ROWS: list[dict] = [] - -# 显式映射,不按模型名猜 provider —— 那正是 D11 要消灭的东西(providers.py 开篇)。 -# 漏登记会被 test_every_capability_has_a_provider_mapping 当场抓住,而不是 -# 在 L8 里被"源不可用"这个假理由吞掉 _MODEL_PROVIDER = { "MiniMax-M3": "minimax", "MiniMax-M2.7": "minimax", @@ -107,768 +89,6 @@ _MODEL_PROVIDER = { "gemini-3-flash": "google", } - -def _base_settings() -> GatewaySettings: - # 强制关缓存: 多轮测量要求每一轮都真的打到供应商,命中缓存会把后续轮次 - # 变成对第一轮的回放,整组判据随之失效 - return GatewaySettings.from_env("LLM", env={**_ENV, "PGW_CACHE_BACKEND": "none"}) - - -def _settings(**source_overrides) -> GatewaySettings: - base = _base_settings() - source = dataclasses.replace(base.sources[0], **source_overrides) - return dataclasses.replace(base, sources=(source,)) - - -async def _run_rounds(rounds: int, *, stream: bool = True, **source_overrides) -> list[dict]: - """跑 N 轮真实调用,返回逐轮观测;任一轮抛错即向上冒泡由用例决定处置。""" - client = GatewayClient.from_settings(_settings(**source_overrides)) - observations = [] - try: - for i in range(rounds): - resp = await client.chat( - [{"role": "user", "content": _PROMPT}], - stream=stream, - # 每轮独立 salt: 即便某层缓存意外开着也不会回放 - cache_salt=f"thinking-live-{i}", - ) - observations.append( - { - "round": i + 1, - "prompt_tokens": resp.prompt_tokens, - "completion_tokens": resp.completion_tokens, - "reasoning_tokens": resp.reasoning_tokens, - # 结论与证据一起入报告: 只记 observation 会让"为什么这么判" - # 不可复核,而 thinking_chars 正是本次改判的直接证据 - "thinking_observation": resp.thinking_observation, - "thinking_chars": len(resp.thinking), - # 核对模型身份: 结论依赖"这组数说的是哪个模型"时(L8 的能力表 - # 对账),渠道串台会把渠道的路由问题记成库的漂移(issue #20) - "model_reported": resp.model_reported, - "content": resp.content[:60], - } - ) - finally: - await client.aclose() - return observations - - -def _record(matrix_id: str, desc: str, status: str, detail, observations=None) -> None: - _ROWS.append( - { - "matrix": matrix_id, - "desc": desc, - "status": status, - "detail": detail, - "observations": observations or [], - } - ) - - -def _reasoning_off(obs: dict) -> bool: - """关闭方向: 只要没观测到推理即算满足。 - - `UNKNOWN` 计入满足是有意的: 它没有证伪力(本次无任何信号,判不出来),拿它 - 判红等于每次关闭调用都喊一遍。本判据真正的证伪力在于——模型若偷偷推理了, - 可观测路径会把裁定翻成 `OBSERVED`。 - - **刻意不设 completion_tokens 上限**: 实测关闭档偶尔会到 46 token(模型没照做 - "只输出两个数字",把解题过程写进了正文),而那是正文不是推理。加长度门只会 - 把这种正常波动误判成"没关掉"。 - """ - return obs["thinking_observation"] != ThinkingObservation.OBSERVED - - -def _reasoning_on(obs: dict) -> bool: - """开启方向: 观测到推理即为真。 - - 判据从 `reasoning_tokens` 换成库的三态裁定,因为 MiniMax 这一路已不再上报 - `completion_tokens_details`(2026-08-25 findings 结论②),该字段恒 `None`; - 而库在同一次调用里拿得到 185 字符推理正文(findings §3.4)——旧判据看不见 - 它,L2/L3b/L4/L5 四条因此假红。 - - 也不能退回 completion_tokens 当判据: medium 档的推理量方差极大(实测 15 轮 - 跨 7-170 token),两档分布还与关闭档重叠,按长度阈值判会把"推理了但想得少" - 误判成没推理。 - """ - return obs["thinking_observation"] == ThinkingObservation.OBSERVED - - -def _skip_if_unreachable(exc: Exception, matrix_id: str, desc: str): - """源不可用(渠道下线/模型未开通)→ 跳过并记为未覆盖,不伪装成通过。""" - _record(matrix_id, desc, "SKIP(源不可用)", str(exc)[:200]) - pytest.skip(f"{matrix_id} 源不可用,已记为未覆盖: {str(exc)[:120]}") - - -def _is_model_not_found(status_code: int | None, body_text: str) -> bool: - """上游说的是**该渠道根本没有这个型号**(404 `model_not_found`),不是"这次请求有问题"。 - - 2026-09-05 实测: kimi-for-coding 上午 09:44 四项全 PASS 且 `model_reported` 正确, - 15:00 起变成 `404 | {"error":{...,"type":"model_not_found"}}` —— 渠道把它从账号组里 - 摘掉了。它与"上游拒绝这一档"(400 invalid tier)完全不是一件事: 后者是**关于档位/ - 能力的结论**,前者对它们一无所知,只说明源当下不可用。混为一谈会让一次渠道调整变成 - "能力表漂移"的假红,严重时反过来把能力表改错。 - - 判据取 `status_code` 与响应体里的 `type` 字段(机器可判的那格),不做整条 message - 的模糊匹配 —— message 里还拼着源名与库自己的话,匹配它等于赌文案不变。 - """ - return status_code == 404 and "model_not_found" in (body_text or "") - - -async def _rounds_or_skip(matrix_id: str, desc: str, rounds: int, **source_overrides) -> list[dict]: - """`_run_rounds` 加上"源不可用即记为未覆盖"的兜底(本模块 docstring 的纪律)。 - - 直接调 `_run_rounds` 的代价有两层,2026-09-05 那次 `-m slow` 两样都踩到了: - 其一外部抖动会以 FAIL 的形态冒出来,与"库真的坏了"无法区分;其二异常发生在 - `_record()` **之前**,报告里连一行「未覆盖」都不会留下——事后翻报告只看到该 - 矩阵行凭空消失,判断不出当时到底发生了什么。 - - 只吞网关/网络三类,外加"该渠道没有这个型号"这**一种**被拒(见 - `_is_model_not_found`)。**其余 `ValueError` / `RequestRejectedError` 照旧冒泡**: - 前者是装配守卫,后者是"请求本身被拒",两者都是本组要抓的真失败,吞掉即成静默。 - """ - try: - return await _run_rounds(rounds, **source_overrides) - except (AllSourcesExhausted, SourceDeadError, TransientError) as exc: - # `_skip_if_unreachable` 内部 `pytest.skip` 必抛,此处不会落到函数末尾 - _skip_if_unreachable(exc, matrix_id, desc) - raise # pragma: no cover —— 只为让静态读者看清控制流不会往下走 - except RequestRejectedError as exc: - if not _is_model_not_found(exc.status_code, exc.body_text): - raise - _skip_if_unreachable(exc, matrix_id, desc) - raise # pragma: no cover - - -@pytest.fixture(scope="module", autouse=True) -def _write_report(): - yield - _OUT_DIR.mkdir(parents=True, exist_ok=True) - ts = datetime.now().strftime("%Y%m%d_%H%M%S") - path = _OUT_DIR / f"test_thinking_live_{ts}.md" - lines = [ - "# 推理开关与推理可观测性真实 API 验证", - "", - f"- 时间: {ts}", - f"- 每档轮数: {_ROUNDS}", - "- 判别量: 库裁定的三态 `thinking_observation`(OBSERVED/ABSENT/UNKNOWN)," - "由推理正文与 reasoning_tokens 共同裁定 —— 正文是事实,token 计数只是转述", - "- 关闭判据: **每轮** observation != OBSERVED(UNKNOWN 计入满足,它没有证伪力);" - "刻意不设输出长度上限(两档的 completion 分布重叠: 实测关闭档最高 46、开启档最低 13)", - "- 开启判据: **多数轮** observation == OBSERVED", - "- 确定性锚点(L2b、L5): 关闭档 prompt_tokens 最大值 < 开启档最小值,相对比较无魔数", - "- L5(非流式): M3 该路径推理已计费却不回传正文,故不断言「观测到推理」," - "改断锚点可分 + 开启档不被误判为 ABSENT", - "", - "## 矩阵结论", - "", - "| 矩阵 | 场景 | 结论 | 说明 |", - "|---|---|---|---|", - ] - total_calls = 0 - for row in _ROWS: - detail = str(row["detail"]).replace("|", "\\|").replace("\n", " ")[:160] - lines.append(f"| {row['matrix']} | {row['desc']} | {row['status']} | {detail} |") - total_calls += len(row["observations"]) - lines += ["", f"**总真实调用次数: {total_calls}**", "", "## 逐轮原始观测", ""] - for row in _ROWS: - if not row["observations"]: - continue - lines += [f"### {row['matrix']} — {row['desc']}", "", "```json"] - lines.append(json.dumps(row["observations"], ensure_ascii=False, indent=2)) - lines += ["```", ""] - uncovered = [r["matrix"] for r in _ROWS if r["status"].startswith("SKIP")] - if uncovered: - lines += ["## 未覆盖", "", f"以下矩阵行未跑到: {', '.join(uncovered)}", ""] - path.write_text("\n".join(lines), encoding="utf-8") - print(f"\n[e2e 报告] {path}") - - -class TestMiniMaxM3: - """M3 是唯一实测可关闭推理的 MiniMax 模型,修复的地基压在它身上。""" - - async def test_l1_disable_actually_disables(self): - desc = "enable_thinking=False(流式)" - obs = await _rounds_or_skip("L1", desc, _ROUNDS, model="MiniMax-M3", enable_thinking=False) - offs = [o for o in obs if _reasoning_off(o)] - _record( - "L1", - desc, - "PASS" if len(offs) == len(obs) else "FAIL", - f"{len(offs)}/{len(obs)} 轮未观测到推理", - obs, - ) - assert len(offs) == len(obs), f"关闭方向要求每轮满足: {obs}" - - async def test_l2_enable_actually_enables(self): - desc = "enable_thinking=True(流式,注入 medium)" - obs = await _rounds_or_skip("L2", desc, _ROUNDS, model="MiniMax-M3", enable_thinking=True) - ons = [o for o in obs if _reasoning_on(o)] - _record( - "L2", - desc, - "PASS" if len(ons) * 2 > len(obs) else "FAIL", - f"{len(ons)}/{len(obs)} 轮观察到推理", - obs, - ) - assert len(ons) * 2 > len(obs), f"开启方向要求多数轮满足: {obs}" - - async def test_l2b_off_and_on_are_distinguishable_without_magic_numbers(self): - """确定性锚点: 开启档的 prompt_tokens 严格大于关闭档。 - - 供应商在开启推理时会向模板注入推理指令,输入侧 token 数随之变大。这是 - 本组唯一不依赖输出侧噪声的证据,且是相对比较——不硬编码任何具体数值, - 供应商改模板也不会让它假红。 - """ - rounds = max(3, _ROUNDS // 3) - desc = "关闭/开启的 prompt_tokens 可分" - off = await _rounds_or_skip("L2b", desc, rounds, model="MiniMax-M3", enable_thinking=False) - on = await _rounds_or_skip("L2b", desc, rounds, model="MiniMax-M3", enable_thinking=True) - off_max = max(o["prompt_tokens"] for o in off) - on_min = min(o["prompt_tokens"] for o in on) - _record( - "L2b", - desc, - "PASS" if off_max < on_min else "FAIL", - f"关闭档最大 {off_max} < 开启档最小 {on_min}", - off + on, - ) - assert off_max < on_min, ( - f"两档的 prompt_tokens 未分开(关闭最大 {off_max},开启最小 {on_min}): 注入可能没到达模型" - ) - - async def test_l3_no_opinion_is_the_model_default(self): - desc = "enable_thinking=None(不干预,基线)" - obs = await _rounds_or_skip("L3", desc, _ROUNDS, model="MiniMax-M3", enable_thinking=None) - # M3 的默认档实测就是不推理(findings §2.1),所以不干预时也应观测不到推理。 - # 注意这**不能**反过来证明关闭方向生效 —— L1 与本行同分布,区分二者的是 - # L2b 的 prompt_tokens 与 L3b 的乱码值反证 - quiet = [o for o in obs if _reasoning_off(o)] - _record( - "L3", - desc, - "PASS" if len(quiet) == len(obs) else "FAIL", - f"{len(quiet)}/{len(obs)} 轮未观测到推理(M3 默认档本就不推理)", - obs, - ) - assert len(quiet) == len(obs), f"M3 默认档不应推理: {obs}" - - async def test_l3b_none_is_recognised_not_silently_dropped(self): - """反证: 关闭方向的观测必须排除"参数被静默丢弃"这一伪解释。 - - L1(关闭)与 L3(不干预)在 M3 上**同分布**——因为 M3 默认档本就不推理。 - 所以 L1 单独看不能区分"`none` 真的被消费"与"`none` 被中转吞了",而后者 - 正是 issue #5 的原始故障形态(`enable_thinking` 就是这么被吞的)。 - - 判别方法: 发一个**非法值**。若未知值会被静默丢弃,它的表现应与"不注入" - 一致(不推理);实测它反而开启了推理,说明网关认这个键、只是不认这个值。 - 既然非法值与 `none` 的表现不同,`none` 就必然是被识别的枚举值。 - - **该手法不可移植,只对"认这个键但不校验值"的 provider 成立**: minimax 对 - 非法 `reasoning_effort` 返回 200 且照常推理(2026-08-25 findings §5: - prompt 207,介于基线 194 与 medium 216 之间,走了第三条模板路径);而 qwen - 对同样的值直接返回 **HTTP 400**。把本用例套到 qwen 那类会校验值的 provider - 上,拿到的会是异常而非"不推理",是假红。 - """ - rounds = max(3, _ROUNDS // 3) - desc = "非法值反证 none 被识别" - bogus = await _rounds_or_skip( - "L3b", - desc, - rounds, - model="MiniMax-M3", - enable_thinking=None, - extra_body={"reasoning_effort": "definitely-not-a-real-level"}, - ) - off = await _rounds_or_skip("L3b", desc, rounds, model="MiniMax-M3", enable_thinking=False) - bogus_on = [o for o in bogus if _reasoning_on(o)] - off_quiet = [o for o in off if _reasoning_off(o)] - ok = len(bogus_on) * 2 > len(bogus) and len(off_quiet) == len(off) - _record( - "L3b", - desc, - "PASS" if ok else "FAIL", - f"非法值 {len(bogus_on)}/{len(bogus)} 轮推理,none {len(off_quiet)}/{len(off)} 轮不推理" - "(两者表现不同 ⇒ none 非被丢弃)", - bogus + off, - ) - assert len(bogus_on) * 2 > len(bogus), ( - f"非法值未开启推理,无法排除'未知值被静默丢弃'这一伪解释: {bogus}" - ) - assert len(off_quiet) == len(off), f"none 未关闭推理: {off}" - - async def test_l4_extra_body_overrides_the_profile(self): - """profile 注入 none,extra_body 要求 high —— 后者必须赢(优先级不可调换)。 - - 判据是行为而非报文: 若 extra_body 没赢,拿到的就是 none 的结果(不推理)。 - """ - rounds = max(3, _ROUNDS // 2) - desc = "extra_body 覆盖 profile 注入" - obs = await _rounds_or_skip( - "L4", - desc, - rounds, - model="MiniMax-M3", - enable_thinking=False, - extra_body={"reasoning_effort": "high"}, - ) - ons = [o for o in obs if _reasoning_on(o)] - _record( - "L4", - desc, - "PASS" if len(ons) * 2 > len(obs) else "FAIL", - f"{len(ons)}/{len(obs)} 轮观察到推理(证明 high 生效而非 none)", - obs, - ) - assert len(ons) * 2 > len(obs), f"extra_body 未能覆盖 profile: {obs}" - - async def test_l5_non_stream_path_is_distinguishable_and_honestly_unknown(self): - """非流式快路径: 参数确实到达了模型,而推理信号被如实标成"观测不到"。 - - **本用例不能断言"非流式开启档观测到推理"——那永远不成立**: M3 在非流式 - 路径下推理段确实产生并计费(2026-08-25 findings §3.4: 开启档 completion 53 - vs 关闭档 3),但 `message` 里没有 `reasoning_content`、`usage` 里也没有 - `completion_tokens_details`,推理内容整体不回传。**这是上游行为,库修不了; - 库能做也必须做的是让它可见**——下游在为看不见的东西付费,不该由库替它 - 沉默。 - - 故改断两件在非流式下真实成立的事: - 其一 `prompt_tokens` 锚点仍把两档分开(判据形态照抄 L2b,证明注入到达了模型, - 排除"非流式路径把参数弄丢了"这一伪解释); - 其二开启档的裁定**不是 `ABSENT`**——`ABSENT` 的语义是"上游明确上报未推理", - 而实情是"判不出来"(`UNKNOWN`),库若把后者伪装成前者,正是 issue #16/#17 里 - 那个静默错觉。这里断 `!= ABSENT` 而非 `== UNKNOWN`,是为了留出上游哪天开始 - 回传正文的余地: 那时裁定会翻成 `OBSERVED`,是好事,不该让它把测试判红。 - """ - rounds = max(3, _ROUNDS // 2) - desc = "非流式: prompt 锚点可分 + 开启档如实标 UNKNOWN 而非 ABSENT" - off = await _rounds_or_skip( - "L5", desc, rounds, stream=False, model="MiniMax-M3", enable_thinking=False - ) - on = await _rounds_or_skip( - "L5", desc, rounds, stream=False, model="MiniMax-M3", enable_thinking=True - ) - offs = [o for o in off if _reasoning_off(o)] - off_max = max(o["prompt_tokens"] for o in off) - on_min = min(o["prompt_tokens"] for o in on) - not_absent = [o for o in on if o["thinking_observation"] != ThinkingObservation.ABSENT] - on_states = Counter(str(o["thinking_observation"]) for o in on) - ok = len(offs) == len(off) and off_max < on_min and len(not_absent) == len(on) - _record( - "L5", - desc, - "PASS" if ok else "FAIL", - f"关闭 {len(offs)}/{len(off)} 轮未观测到推理;" - f"关闭档 prompt 最大 {off_max} < 开启档最小 {on_min};" - f"开启档裁定分布 {dict(on_states)}", - off + on, - ) - assert len(offs) == len(off), f"非流式关闭方向未满足: {off}" - assert off_max < on_min, ( - f"非流式两档 prompt_tokens 未分开(关闭最大 {off_max},开启最小 {on_min}): " - f"开启参数可能没到达模型" - ) - assert len(not_absent) == len(on), ( - f"非流式开启档被裁成 ABSENT(声称上游明确上报未推理),而实情是观测不到: {on}" - ) - - -class TestOtherProviders: - """qwen / deepseek 的 profile 是既有实现,本组防的是"改 minimax 时误伤它们"。""" - - @pytest.mark.parametrize( - ("matrix", "provider", "model"), - [("L6", "qwen", "qwen3.7-plus"), ("L7", "deepseek", "deepseek-v4-pro")], - ) - async def test_existing_profiles_still_disable(self, matrix, provider, model): - desc = f"{provider} enable_thinking=False" - try: - obs = await _run_rounds(_ROUNDS, provider=provider, model=model, enable_thinking=False) - except (AllSourcesExhausted, SourceDeadError, TransientError) as exc: - # 只吞网关/网络类失败。**不吞 ValueError / RequestRejected** —— - # 那两类正是本次改动最可能的误伤方向,吞掉就成了纪律(c)要防的静默 - _skip_if_unreachable(exc, matrix, desc) - offs = [o for o in obs if _reasoning_off(o)] - _record( - matrix, - desc, - "PASS" if len(offs) == len(obs) else "FAIL", - f"{len(offs)}/{len(obs)} 轮未观测到推理", - obs, - ) - assert len(offs) == len(obs), f"{provider} 关闭方向未满足: {obs}" - - async def test_qwen_enabled_is_observed(self): - """设计 §14 验收: qwen 开启档必须裁定为 `OBSERVED`,不是 `UNKNOWN`。 - - 本条是三态裁定的**跨供应商对照组**: MiniMax 这一路两个信号都可能缺失 - (非流式档整片 `UNKNOWN`),若只按它调判据,很容易把"观测不到"当成常态; - qwen 在同一网关同一 key 上照常返回推理信号(findings 2026-08-25 §2), - 故这里能且必须要求正面结论——它一旦掉成 `UNKNOWN`,说明的是库的组装路径 - 丢了信号,而不是上游行为变了。 - """ - matrix, provider, model = "L6b", "qwen", "qwen3.7-plus" - desc = f"{provider} enable_thinking=True" - try: - obs = await _run_rounds(_ROUNDS, provider=provider, model=model, enable_thinking=True) - except (AllSourcesExhausted, SourceDeadError, TransientError) as exc: - _skip_if_unreachable(exc, matrix, desc) - ons = [o for o in obs if _reasoning_on(o)] - _record( - matrix, - desc, - "PASS" if len(ons) * 2 > len(obs) else "FAIL", - f"{len(ons)}/{len(obs)} 轮观测到推理(OBSERVED)", - obs, - ) - assert len(ons) * 2 > len(obs), f"{provider} 开启方向要求多数轮 OBSERVED: {obs}" - - -class TestCapabilityDrift: - """L8 漂移哨兵: 能力表过期是必然事件,这里是它的过期告警。""" - - def test_every_capability_has_a_provider_mapping(self): - """能力表新增条目必须同步本测试的映射,否则该行会被静默跳过。""" - missing = sorted(set(DEFAULT_CAPABILITIES) - set(_MODEL_PROVIDER)) - assert not missing, f"这些模型缺 provider 映射,L8 会漏测: {missing}" - - @pytest.mark.parametrize("model", sorted(DEFAULT_CAPABILITIES)) - async def test_declared_capability_matches_reality(self, model): - """声明 can_disable 的模型必须真的关得掉,否则能力表已漂移。 - - **结论依赖模型身份,故先过身份关**: 2026-09-05 实测该渠道对 glm-5 / glm-5.1 / - glm-5.2 三个型号的请求全部回报 `model=glm-5.3`(issue #20 的路由问题仍在)。 - 照单全收的话,glm-5.3 那一轮碰巧推理了就会被记成"glm-5 的能力表漂移"——把 - 渠道串台记成库的缺陷,而库这边已经喊了对账告警,行为是对的。 - 身份不符一律 SKIP 记为未覆盖: 那是外部渠道问题,不是能力表的证据。 - **三个型号一视同仁**,不能只挡报错的那两个: glm-5.2 这次侥幸 PASS(被路由到的 - glm-5.3 那几轮恰好没推理),而侥幸绿的数据与红的数据一样不可信。 - - 另一种同类外部状况走 `_rounds_or_skip`: 渠道把型号从账号组里摘了(404 - `model_not_found`),同样是源不可用而非能力表漂移(2026-09-05 kimi-for-coding - 当天从 PASS 变 404,旧版把它记成了一条 FAIL)。 - """ - cap = get_capability(model) - provider = _MODEL_PROVIDER[model] - rounds = max(3, _ROUNDS // 2) - desc = f"{model} 声明 can_disable={cap.can_disable}" - if not cap.can_disable: - # 声明关不掉: 装配期就该炸,炸了即与声明一致(不必真调用) - with pytest.raises(ValueError, match=model): - GatewayClient.from_settings( - _settings(provider=provider, model=model, enable_thinking=False) - ) - _record("L8", desc, "PASS", "装配期按声明拒绝,与实测一致") - return - obs = await _rounds_or_skip( - "L8", desc, rounds, provider=provider, model=model, enable_thinking=False - ) - strangers = _identity_mismatch(model, obs) - if strangers: - _record( - "L8", - desc, - "SKIP(身份不符,数据不可信)", - f"该渠道把请求回报成 {strangers},本次观测说的不是这个模型", - obs, - ) - pytest.skip(f"{model} 被该渠道路由到 {strangers},本次观测说的不是这个模型") - offs = [o for o in obs if _reasoning_off(o)] - verdict = Counter(_reasoning_off(o) for o in obs) - _record( - "L8", - desc, - "PASS" if len(offs) == len(obs) else "FAIL(能力表已漂移)", - f"实测未观测到推理 {dict(verdict)}(True=满足);声明 can_disable=True 要求每轮满足", - obs, - ) - assert len(offs) == len(obs), ( - f"能力表漂移: {model} 声明可关闭推理,实测未关掉 —— 请复测后更新 DEFAULT_CAPABILITIES" - ) - - -_MYSTERY_PROFILE = ProviderProfile( - name="mystery", - thinking=ThinkingWire(off=None, on_base=None, effort_key=None), - strip_think_tags=False, -) -"""形态完全未知的 provider(issue #5 守卫的对象),与单元测试 `_MYSTERY` 同款。 - -**为什么不再借用默认表里的某一段**: L9 原先拿 `openai` 段当"形态未知"的样本,而 -1.3.3 起该段已按 OpenAI 标准形态登记(`off={"reasoning_effort":"none"}`、 -`on_base={}`、`effort_key="reasoning_effort"`),前提消失,用例随之 DID NOT RAISE。 -守的不变量一天没变,变的只是"哪个段当时恰好没形态"——所以样本改为显式构造, -让本条测的是**机制**而不是默认表某一格的当下取值。""" - - -class TestAssemblyGuardAgainstRealConfig: - """L9: 纯本地,但用的是 .env 里的真实配置形态,防"守卫只在合成配置上生效"。""" - - def test_l9_m27_rejected_at_assembly(self): - with pytest.raises(ValueError, match="MiniMax-M2.7"): - GatewayClient.from_settings( - _settings(provider="minimax", model="MiniMax-M2.7", enable_thinking=False) - ) - _record("L9", "M2.7 + enable_thinking=False", "PASS", "装配期报错,未发出任何请求") - - def test_l9_unknown_shape_rejected_at_assembly(self): - """形态未知的 provider 配了推理开关 → 装配期报错并指路 `register_provider`。 - - 样本经 `register_provider` 挂进注册表再用,而不是拿默认表里"当时恰好没形态" - 的那一段——后者的前提会随默认表增补而失效(见 `_MYSTERY_PROFILE`)。 - """ - registry = register_provider(_MYSTERY_PROFILE) - with pytest.raises(ValueError, match="register_provider"): - GatewayClient.from_settings( - _settings(provider="mystery", model="kimi-k3", enable_thinking=False), - registry=registry, - ) - _record("L9", "形态未知的 provider(构造)", "PASS", "装配期报错并指路") - - async def test_transport_layer_rejects_when_guard_is_bypassed(self): - """构造函数全量注入这条路绕过装配守卫,transport 必须兜住并归四分类。""" - settings = _settings(provider="minimax", model="MiniMax-M2.7", enable_thinking=False) - client = GatewayClient.from_settings( - dataclasses.replace( - settings, sources=(dataclasses.replace(settings.sources[0], enable_thinking=None),) - ) - ) - try: - # 装配用 None 绕过守卫,再把源换成 False 直接喂给 transport - bad = dataclasses.replace(settings.sources[0], enable_thinking=False) - with pytest.raises(RequestRejectedError, match="MiniMax-M2.7"): - await client._terminal._transport.complete( - messages=[{"role": "user", "content": _PROMPT}], - source=bad, - stream=True, - overlay={}, - call_id="e2e-guard", - reasoning_effort=None, - ) - finally: - await client.aclose() - _record("L9", "绕过装配守卫时 transport 兜底", "PASS", "RequestRejectedError,属四分类") - - -# ══════════════════════════════════════════════════════════════════════════════ -# T10: 逐模型档位实测(方法论沿用 issue #20) -# -# 本节与上面的 L1-L9 分工不同: 上面验的是**库的行为**(注入到没到、观测准不准), -# 这里验的是**能力表的内容**(`DEFAULT_CAPABILITIES` 里那 20 多条声明是不是真的)。 -# 二者判据可以共用,数据源却必须分开——能力表实测要**绕过能力表**才有意义, -# 否则拿待验证的声明去挡请求,等于用结论证明前提。 -# -# 判据(三条,均沿用已有纪律): -# ① 关闭方向: 每轮 `thinking_observation != OBSERVED` 才算真关掉;任一轮 -# OBSERVED 即证伪(推理正文是事实本身,不需要多数票)。 -# ② **短提示词的"关掉了"必须经长上下文复核**: issue #20 实测 GLM 系在短提示词 -# 下 reasoning_tokens≈1.2 像是关了,5552 token 长上下文下跳到 0/54/167 即露馅。 -# 短提示词下推理量本就趋近于 0,分不出"关了"与"没什么可想的"。 -# ③ 开启方向: 多数轮 OBSERVED(单轮抖动不判红,与 L2 同口径)。 -# ④ 关闭结论**不许只靠 `UNKNOWN`**: 上游整片不回传推理信号时(kimi、MiniMax 两路 -# 都是),"没看见"不是"没发生"。此时补一个不含魔数的锚点——关闭档的 -# `completion_tokens` 必须严格小于 `max` 档,否则结论记为「判不出来」。 -# ══════════════════════════════════════════════════════════════════════════════ - -_TIER_OUT_DIR = Path("tests/outputs/thinking") -_TIER_ROUNDS = int(os.environ.get("PGW_E2E_TIER_ROUNDS", "5")) -_TIER_LONG_ROUNDS = int(os.environ.get("PGW_E2E_TIER_LONG_ROUNDS", "3")) -# 共用生产网关,宁慢勿冲(人类 2026-09-05 指令): 默认 3,可下调不建议上调 -_TIER_CONCURRENCY = int(os.environ.get("PGW_E2E_TIER_CONCURRENCY", "3")) - -# 固定短提示词: 答案本身约 4 token,推理 token 的信噪比高(issue #20 同款) -_TIER_PROMPT = "23 乘以 47 等于多少?只回答一个数字,不要解释。" - -# 长上下文对照组(判据②)。填充文本与题目无关且不含任何业务领域词汇(零业务假设 -# 铁律),只为把输入撑到数千 token;题目放在最后,避免被当成"读完就忘"的前缀 -_TIER_LONG_PROMPT = ( - "\n".join( - f"{i:04d}. 这是一段与题目无关的填充文字,仅用于把上下文撑到数千 token," - "以复核短提示词下得到的关闭结论在长上下文下是否依然成立。" - for i in range(120) - ) - + "\n\n" - + _TIER_PROMPT -) - -_ALL_EFFORTS: tuple[Effort, ...] = (*EFFORT_ORDER, Effort.AUTO) - -_PROBE_ROWS: list[dict] = [] - - -def _tier_settings(model: str) -> GatewaySettings: - """探测用配置: 生产口径的超时,但**重试预算压到 1 次**。 - - 压重试是因为探测里"这一轮失败"本身就是数据(逐轮进报告),库替它重试只会 - 把"渠道当下不可用"变成三倍等待——2026-09-05 实测 claude 系 7 天限额用尽时 - 每轮 429,三次重试让单个模型阻塞三分钟以上,26 个模型跑不完。 - - **单次请求的超时不动**(仍是 .env 的生产值 300s): §4.6 那条"测试超时不得紧于 - 生产配置"防的是把慢而正常的模型误判成不可用,那个风险在这里照旧存在。重试次数 - 与背压窗口不属于同一类——它们决定"失败之后还等多久",而不是"多慢算失败"; - 一个真在出字的模型永远碰不到这两者。 - """ - base = GatewaySettings.from_env( - "LLM", - env={ - **_ENV, - "PGW_CACHE_BACKEND": "none", - "LLM_MAX_RETRIES": "1", - # 探测是**单源**的,没有别的源可换。生产值 1200s 的 stall window 在这里 - # 只会把"这个模型当下不可用"拖成 20 分钟一轮: 2026-09-05 实测 claude 系 - # 7 天限额用尽返回 429 且不带 Retry-After,库据此判"无可运行源"并按背压 - # 语义等到窗口耗尽(实测把窗口调到 45s 即在 46.7s 报 stalled)。多源生产 - # 场景下这段等待是有意义的(等别的源恢复),探测场景下等不到任何东西 - "LLM__BACKPRESSURE__STALL_WINDOW_S": "60", - }, - ) - source = dataclasses.replace( - base.sources[0], - provider=_MODEL_PROVIDER[model], - model=model, - enable_thinking=None, - reasoning_effort=None, - ) - return dataclasses.replace(base, sources=(source,)) - - -def _probe_capabilities(model: str) -> dict[str, ThinkingCapability]: - """临时全档能力表: **实测的对象正是能力表本身**,不能拿它当前提去挡请求。 - - 不传 `capabilities={}`(即"未登记")的理由是噪声: 那条路会走 Phase 3,每轮都 - warning 一句"能力未登记",几百轮下来把真正的告警淹没。全档表让五关全部放行, - 请求原样发出去,由上游而不是由库来回答"这一档到底行不行"。 - """ - return {model: ThinkingCapability(_ALL_EFFORTS, evidence="T10 实测临时表(不进 DEFAULT)")} - - -async def _probe_effort( - model: str, effort: Effort, *, rounds: int, prompt: str, prompt_kind: str -) -> list[dict]: - """对一个 (模型, 档位) 打 N 轮真实请求,逐轮记录;失败轮记 `error` 而不冒泡。 - - 失败不冒泡是本函数与 `_run_rounds` 的唯一区别: 这里"上游拒绝这一档"本身就是 - **实测结论**(HTTP 400 = 该档不被接受),把它抛出去会让数据采集半途而废。 - 只吞四分类与 `AllSourcesExhausted`——库自身的 `ValueError` 等仍然冒泡,那是 - bug 不是数据。 - """ - client = GatewayClient.from_settings( - _tier_settings(model), capabilities=_probe_capabilities(model) - ) - semaphore = asyncio.Semaphore(_TIER_CONCURRENCY) - - async def _one(index: int) -> dict: - base = {"round": index + 1, "effort": effort.value, "prompt_kind": prompt_kind} - async with semaphore: - try: - resp = await client.chat( - [{"role": "user", "content": prompt}], - stream=True, - reasoning_effort=effort, - cache_salt=f"tier-probe-{model}-{effort.value}-{prompt_kind}-{index}", - ) - except ( - RequestRejectedError, - GatewayUnavailableError, - SourceDeadError, - TransientError, - ) as exc: - # 捕 `GatewayUnavailableError` 而不是只捕 `AllSourcesExhausted`: - # 某个模型在网关上不通时,连续失败会把熔断门打开,后续轮次抛的是 - # `CircuitOpenError`(同一父类的兄弟)。只捕子类会让"源不可用"这 - # 件事在第 N 轮换个类型冒出去,把数据采集打断成一次红测 - return { - **base, - "error": f"{type(exc).__name__}: {str(exc)[:160]}", - # 另存机器可判的两格: 「上游拒绝这一档」与「该渠道没有这个型号」 - # 都是 `RequestRejectedError`,`_probe_rejected` 要靠状态码与 - # 响应体里的 `type` 把它们分开,而不是去模糊匹配整条 message - "error_status": exc.status_code, - "error_body": exc.body_text, - } - return { - **base, - "error": None, - "prompt_tokens": resp.prompt_tokens, - "completion_tokens": resp.completion_tokens, - "reasoning_tokens": resp.reasoning_tokens, - "thinking_chars": len(resp.thinking), - "thinking_observation": resp.thinking_observation, - "applied_effort": resp.applied_effort, - # 核对模型身份: issue #20 记录本渠道对 glm-5.2 的请求 6/6 回报 - # model=glm-5.3。凡结论依赖模型身份的,对不上即数据不可信 - "model_reported": resp.model_reported, - "content": resp.content[:40], - } - - try: - return list(await asyncio.gather(*(_one(i) for i in range(rounds)))) - finally: - await client.aclose() - - -def _probe_ok(obs: dict) -> bool: - """这一轮拿到了真实观测。 - - 用 `.get` 而非下标: `_identity_mismatch` 被 L8 复用,而 `_run_rounds` 产出的 - 逐轮字典里根本没有 `error` 键(那条路径上失败是冒泡的,不会留下失败轮)。 - """ - return obs.get("error") is None - - -def _model_missing(obs: dict) -> bool: - """`_is_model_not_found` 的逐轮字典适配:这一轮是"该渠道没有这个型号"而失败的。 - - 判据本身与 L1-L9 共用一份(理由见 `_is_model_not_found`),此处只负责从失败轮 - 里取出那两格 —— 两处各写一份迟早只改一处。 - """ - return _is_model_not_found(obs.get("error_status"), obs.get("error_body") or "") - - -def _probe_rejected(obs: dict) -> bool: - """上游明确拒绝**这一档**(400 / Unsupported value)⇒ 结论: 该档不受支持。 - - 显式排除 `_model_missing`: 型号不存在时上游没有对档位表过任何态。 - """ - return (obs.get("error") or "").startswith("RequestRejected") and not _model_missing(obs) - - -def _unreachable_verdict(observations: list[dict]) -> str: - """源不可用的两种成因在报告里必须分得开: 渠道摘了型号 vs 渠道当下抖动。""" - broken = [o for o in observations if o.get("error")] - if broken and all(_model_missing(o) for o in broken): - return "SKIP(源不可用: 该渠道未提供此型号)" - return "SKIP(源不可用)" - - -def _probe_quiet(obs: dict) -> bool: - """成功且未观测到推理(判据①的满足条件);失败轮不算"安静",它没有观测。""" - return _probe_ok(obs) and obs["thinking_observation"] != ThinkingObservation.OBSERVED - - -def _probe_observed(obs: dict) -> bool: - return _probe_ok(obs) and obs["thinking_observation"] == ThinkingObservation.OBSERVED - - -def _rt_summary(observations: list[dict]) -> str: - """报告里的一行摘要: rt 观测值序列 + 裁定分布 + 身份核对,三样缺一不可复核。""" - ok = [o for o in observations if _probe_ok(o)] - if not ok: - return f"全部 {len(observations)} 轮失败: {observations[0]['error']}" - rts = [o["reasoning_tokens"] for o in ok] - verdicts = Counter(str(o["thinking_observation"]) for o in ok) - reported = sorted({str(o["model_reported"]) for o in ok}) - failed = len(observations) - len(ok) - tail = f";{failed} 轮失败" if failed else "" - return ( - f"rt={rts};裁定 {dict(verdicts)};thinking_chars=" - f"{[o['thinking_chars'] for o in ok]};model_reported={reported}{tail}" - ) - - -# 已知的合法别名: 供应商回报的名字与配置里的别名本就可以不同(月之暗面回 -# `k3`、Google 回 `-preview` 后缀)。**显式登记而不是按前缀猜**——猜的话 -# `glm-5.2 → glm-5.3` 这种真·串台也会被当成"同族别名"放过,而那正是本表要抓的 _MODEL_REPORTED_ALIASES: Mapping[str, frozenset[str]] = MappingProxyType( { "kimi-k3": frozenset({"k3"}), @@ -879,352 +99,427 @@ _MODEL_REPORTED_ALIASES: Mapping[str, frozenset[str]] = MappingProxyType( ) -def _identity_mismatch(model: str, observations: list[dict]) -> list[str]: - """响应体里的 `model` 与请求的模型对不上 → 本次数据说的不是这个模型。 +def _settings(**source_overrides): + """强制关闭缓存，清除 inherited 受管意图后应用本矩阵配置。""" + base = GatewaySettings.from_env("LLM", env={**_ENV, "PGW_CACHE_BACKEND": "none"}) + source = dataclasses.replace( + base.sources[0], enable_thinking=None, reasoning_effort=None, extra_body={} + ) + source = dataclasses.replace(source, **source_overrides) + return dataclasses.replace(base, sources=(source,)) - issue #20 就栽在这里: 该渠道对 `glm-5.2` 的请求 6/6 回报 `model=glm-5.3`, - 照单全收的话,能力表里 glm-5.2 那一行记的其实是 glm-5.3 的行为。凡结论依赖 - 模型身份的,对不上就必须当场作废,而不是打个折扣继续用。 - `None`(上游未上报)不算不符: 那是"没说",不是"说了别的"。 +def _tier_settings(model): + """保留既有一次 retry 探测预算；不压缩生产 stall 或单次 timeout。""" + base = GatewaySettings.from_env( + "LLM", env={**_ENV, "PGW_CACHE_BACKEND": "none", "LLM_MAX_RETRIES": "1"} + ) + source = dataclasses.replace( + base.sources[0], + model=model, + provider=_MODEL_PROVIDER[model], + enable_thinking=None, + reasoning_effort=None, + extra_body={}, + ) + return dataclasses.replace(base, sources=(source,)) - 定义在 T10 段内但**不专属于它**: L8 的能力表对账同样以模型身份为前提, - 2026-09-05 那次假红就是它缺了这道关(见该用例 docstring)。 - """ - allowed = {model, *_MODEL_REPORTED_ALIASES.get(model, frozenset())} - return sorted( - { - o["model_reported"] - for o in observations - if _probe_ok(o) - and o["model_reported"] is not None - and o["model_reported"] not in allowed - } + +async def _collect_rounds( + settings, *, rounds, stream, prompt, matrix_id, effort=None, capabilities=None, concurrency=1 +): + """保留所有失败轮，不把可用轮集合偷偷当新分母。""" + if rounds < 1 or concurrency < 1: + raise ValueError("轮次／并发必须为正数") + messages = [{"role": "user", "content": prompt}] + controls = ( + source_controls(settings) + if effort is None + else {s.name: declared_control(s.provider, effort) for s in settings.sources} + ) + capture = LiveCapture( + expectations=chat_expectations( + settings, messages=messages, stream=stream, controls=controls + ) + ) + run_id = uuid4().hex + semaphore = asyncio.Semaphore(concurrency) + async with observed_client(settings, capture, capabilities=capabilities) as client: + + async def one(index): + """每轮已落盘后才回到汇总，断言失败也有记录。""" + + def validate(response): + assert response.content.strip() + if effort is not None: + assert response.applied_effort is effort + + async with semaphore: + response, verdict = await captured_chat_round( + client, + capture, + run_id=run_id, + matrix_id=matrix_id, + round_index=index + 1, + output_dir=_OUT_DIR, + messages=messages, + models={s.name: s.model for s in settings.sources}, + providers={s.name: s.provider for s in settings.sources}, + source_efforts={ + s.name: s.reasoning_effort + if s.reasoning_effort is not None + else (Effort.AUTO if s.enable_thinking else Effort.NONE) + if s.enable_thinking is not None + else None + for s in settings.sources + }, + aliases=_MODEL_REPORTED_ALIASES, + stream=stream, + reasoning_effort=effort, + cache_salt=f"{run_id}-{index}", + validate=validate, + ) + return {"round": index + 1, "verdict": verdict, "response": response} + + # return_exceptions 保证一个取证写失败不使其他任务越过资源关闭边界。 + results = await asyncio.gather(*(one(i) for i in range(rounds)), return_exceptions=True) + values = [] + for value in results: + if isinstance(value, BaseException): + raise value + values.append(value) + counts = summarize_verdicts([value["verdict"] for value in values], planned_rounds=rounds) + write_live_round( + _OUT_DIR, + run_id=run_id, + matrix_id=matrix_id + "-rounds", + round_index=0, + safe_fields={"counts": counts, "completed_rounds": len(values), "planned_rounds": rounds}, + ) + return values + + +async def _run_rounds(rounds, *, stream=True, matrix_id="thinking", **source_overrides): + """L1–L8 的资格证据出口，不作整类 skip。""" + return await _collect_rounds( + _settings(**source_overrides), + rounds=rounds, + stream=stream, + prompt=_PROMPT, + matrix_id=matrix_id, ) -async def _anchor_off_against_on( - model: str, off_observations: list[dict] -) -> tuple[list[dict], Effort | None, bool]: - """判据④: 拿"开启档的 completion 明显更大"给关闭结论补一个正面证据。 +def _qualified(rows, *, planned_rounds): + """汇总资格先失败后未覆盖；部分失败不能被成功轮掩盖。""" + return qualify_live_rounds([row["verdict"] for row in rows], planned_rounds=planned_rounds) - 需要它是因为 `UNKNOWN` 的语义: 它是"本次没有任何信号,判不出来",不是"没推理" - (`observe_thinking` 的 docstring 把这条写死了)。kimi 与 MiniMax 这两路上游都 - 不回传 `completion_tokens_details`,关闭档整片 `UNKNOWN`——此时若直接把"没看见" - 读成"关掉了",库就会登记一个自己从未验证过的 `none`,而下游据此以为省了钱。 - 锚点取 `completion_tokens` 的相对比较(关闭档最大值 < 开启档最小值),**不含 - 任何魔数**: 推理段计在 completion 里,真开着时两档差一个数量级(实测 kimi-k3 - 关闭档恒 9 token)。取 `max` 档而非 `auto`: 后者在 `on_base={}` 的 provider 上 - 等于"什么都不注入",那是模型默认档而不是"开",拿它当对照组会把 M3 这种默认不推理的 - 模型判成"分不开"(minimax 段已按 issue #21 改回带 medium,openai/anthropic/google - 三段仍是空片段,故该风险仍在)。`max` 打不通时才退到 `auto`。 - """ - off_usable = [o for o in off_observations if _probe_ok(o)] - for tier in (Effort.MAX, Effort.AUTO): - anchor = await _probe_effort( - model, - tier, - rounds=_TIER_LONG_ROUNDS, - prompt=_TIER_PROMPT, - prompt_kind=f"anchor({tier.value})", +def _coverage(rows, *, planned_rounds, proposition): + """只有全轮资格通过才进入推理观测命题。""" + verdict = _qualified(rows, planned_rounds=planned_rounds) + if verdict.status == "PASS": + verdict = assess_thinking_coverage( + [row["response"].thinking_observation for row in rows], + planned_rounds=planned_rounds, + proposition=proposition, ) - on_usable = [o for o in anchor if _probe_ok(o)] - if not on_usable: - continue - off_max = max(o["completion_tokens"] for o in off_usable) - on_min = min(o["completion_tokens"] for o in on_usable) - return anchor, tier, off_max < on_min - return [], None, False + return verdict -def _probe_record(model: str, phase: str, verdict: str, detail: str, observations: list[dict]): - _PROBE_ROWS.append( - { - "model": model, - "provider": _MODEL_PROVIDER[model], - "phase": phase, - "verdict": verdict, - "detail": detail, - "observations": observations, - } +def _conclude(matrix, verdict, *, proposition=None): + """命题汇总先落盘再交给 pytest，不覆盖逐轮原件。""" + write_live_round( + _OUT_DIR, + run_id=uuid4().hex, + matrix_id=matrix, + round_index=0, + safe_fields={ + "status": verdict.status, + "reason": verdict.reason, + "proposition": proposition, + }, ) + enforce_verdict(verdict) -@pytest.fixture(scope="module", autouse=True) -def _write_tier_report(): - """T10 报告独立成文件: 它的读者是"能力表该怎么改",与 L1-L9 的"库对不对"不同。""" - yield - if not _PROBE_ROWS: - return - _TIER_OUT_DIR.mkdir(parents=True, exist_ok=True) - ts = datetime.now().strftime("%Y%m%d_%H%M%S") - path = _TIER_OUT_DIR / f"tier_probe_{ts}.md" - lines = [ - "# 推理档位能力表实测(T10,经 new-api 中转)", - "", - f"- 时间: {ts}", - f"- 短提示词轮数: {_TIER_ROUNDS};长上下文复核轮数: {_TIER_LONG_ROUNDS};" - f"并发: {_TIER_CONCURRENCY}(共用生产网关,宁慢勿冲)", - f"- 短提示词: `{_TIER_PROMPT}`", - f"- 长上下文: 同题 + {len(_TIER_LONG_PROMPT)} 字符无关填充(判据②)", - "- 判据: 关闭方向要求**每轮**未观测到推理,且短提示词的「关掉了」必须经长上下文复核;" - "开启方向要求多数轮 OBSERVED", - "- 能力表在探测时被临时替换为全档表: 实测的对象正是它,不能拿它挡请求", - "", - "## 逐模型结论", - "", - "| 模型 | provider | 阶段 | 结论 | 观测 |", - "|---|---|---|---|---|", - ] - total = 0 - for row in _PROBE_ROWS: - detail = str(row["detail"]).replace("|", "\\|").replace("\n", " ")[:220] - lines.append( - f"| {row['model']} | {row['provider']} | {row['phase']} | {row['verdict']} | {detail} |" +class TestMiniMaxM3: + """AUTO 拒绝已移至离线契约；真实开启明确请求 medium。""" + + async def test_l1_disable_actually_disables(self): + rows = await _run_rounds( + _ROUNDS, matrix_id="L1", provider="minimax", model="MiniMax-M3", enable_thinking=False ) - total += len(row["observations"]) - lines += ["", f"**总真实调用次数: {total}**", "", "## 逐轮原始观测", ""] - for row in _PROBE_ROWS: - if not row["observations"]: - continue - lines += [f"### {row['model']} — {row['phase']}", "", "```json"] - lines.append(json.dumps(row["observations"], ensure_ascii=False, indent=2, default=str)) - lines += ["```", ""] - path.write_text("\n".join(lines), encoding="utf-8") - print(f"\n[T10 报告] {path}") + _conclude( + "L1", + _coverage(rows, planned_rounds=_ROUNDS, proposition="disabled"), + proposition="disabled", + ) + + async def test_l2_enable_actually_enables(self): + rows = await _run_rounds( + _ROUNDS, + matrix_id="L2", + provider="minimax", + model="MiniMax-M3", + reasoning_effort=Effort.MEDIUM, + ) + _conclude( + "L2", + _coverage(rows, planned_rounds=_ROUNDS, proposition="enabled"), + proposition="enabled", + ) + + async def test_l2b_off_and_on_are_distinguishable_without_magic_numbers(self): + """指定历史 prompt 锚点回归，不宣称关闭能力已覆盖。""" + rounds = max(3, _ROUNDS // 3) + off = await _run_rounds( + rounds, + matrix_id="L2b-off", + provider="minimax", + model="MiniMax-M3", + enable_thinking=False, + ) + on = await _run_rounds( + rounds, + matrix_id="L2b-on", + provider="minimax", + model="MiniMax-M3", + reasoning_effort=Effort.MEDIUM, + ) + verdict = _qualified(off + on, planned_rounds=rounds * 2) + if verdict.status == "PASS": + distinct = max(r["response"].prompt_tokens for r in off) < min( + r["response"].prompt_tokens for r in on + ) + verdict = LiveVerdict( + "PASS" if distinct else "FAIL", "指定历史 prompt 锚点比较；不是关闭证明" + ) + _conclude("L2b", verdict, proposition="historical-prompt-anchor") + + async def test_l3_no_opinion_is_the_model_default(self): + rows = await _run_rounds(_ROUNDS, matrix_id="L3", provider="minimax", model="MiniMax-M3") + verdict = _qualified(rows, planned_rounds=_ROUNDS) + if verdict.status == "PASS" and any( + row["response"].applied_effort is not None for row in rows + ): + verdict = LiveVerdict("FAIL", "不表态路径擅自记录档位") + _conclude("L3", verdict, proposition="no-opinion-not-capability") + + async def test_l3b_none_is_recognised_not_silently_dropped(self): + """保留原非法 raw 值对照预算，但不提升 UNKNOWN。""" + rounds = max(3, _ROUNDS // 3) + bogus = await _run_rounds( + rounds, + matrix_id="L3b-bogus", + provider="minimax", + model="MiniMax-M3", + extra_body={"reasoning_effort": "definitely-not-a-real-level"}, + ) + off = await _run_rounds( + rounds, + matrix_id="L3b-off", + provider="minimax", + model="MiniMax-M3", + enable_thinking=False, + ) + verdicts = [ + _coverage(bogus, planned_rounds=rounds, proposition="enabled"), + _coverage(off, planned_rounds=rounds, proposition="disabled"), + ] + _conclude("L3b", _combine(verdicts), proposition="raw-counterexample") + + async def test_l4_raw_only_explicit_high(self): + """退出受管意图后才保留 raw high；双来源拒绝在 unit 守卫。""" + rounds = max(3, _ROUNDS // 2) + rows = await _run_rounds( + rounds, + matrix_id="L4", + provider="minimax", + model="MiniMax-M3", + extra_body={"reasoning_effort": "high"}, + ) + _conclude( + "L4", + _coverage(rows, planned_rounds=rounds, proposition="enabled"), + proposition="enabled", + ) + + async def test_l5_non_stream_path_is_distinguishable_and_honestly_unknown(self): + """保留流／非流预算；UNKNOWN 是明确未覆盖而非长度锚点成功。""" + rounds = max(3, _ROUNDS // 2) + off = await _run_rounds( + rounds, + matrix_id="L5-off", + stream=False, + provider="minimax", + model="MiniMax-M3", + enable_thinking=False, + ) + on = await _run_rounds( + rounds, + matrix_id="L5-on", + stream=False, + provider="minimax", + model="MiniMax-M3", + reasoning_effort=Effort.MEDIUM, + ) + _conclude( + "L5", + _combine( + [ + _coverage(off, planned_rounds=rounds, proposition="disabled"), + _coverage(on, planned_rounds=rounds, proposition="enabled"), + ] + ), + proposition="nonstream-enabled-disabled", + ) + + +def _combine(verdicts): + """任一失败优先，部分未覆盖不得汇总全 PASS。""" + return combine_live_verdicts(verdicts) + + +class TestOtherProviders: + """既有供应商开启／关闭真实矩阵。""" + + @pytest.mark.parametrize( + ("matrix", "provider", "model"), + [("L6", "qwen", "qwen3.7-plus"), ("L7", "deepseek", "deepseek-v4-pro")], + ) + async def test_existing_profiles_still_disable(self, matrix, provider, model): + rows = await _run_rounds( + _ROUNDS, matrix_id=matrix, provider=provider, model=model, enable_thinking=False + ) + _conclude( + matrix, + _coverage(rows, planned_rounds=_ROUNDS, proposition="disabled"), + proposition="disabled", + ) + + async def test_qwen_enabled_is_observed(self): + rows = await _run_rounds( + _ROUNDS, matrix_id="L6b", provider="qwen", model="qwen3.7-plus", enable_thinking=True + ) + _conclude( + "L6b", + _coverage(rows, planned_rounds=_ROUNDS, proposition="enabled"), + proposition="enabled", + ) + + +class TestCapabilityDrift: + """只运行可关闭声明的真实验证；不可关闭装配拒绝另在 unit。""" + + @pytest.mark.parametrize( + "model", + sorted( + model for model, capability in DEFAULT_CAPABILITIES.items() if capability.can_disable + ), + ) + async def test_declared_capability_matches_reality(self, model): + rounds = max(3, _ROUNDS // 2) + rows = await _run_rounds( + rounds, + matrix_id="L8", + provider=_MODEL_PROVIDER[model], + model=model, + enable_thinking=False, + ) + _conclude( + "L8", + _coverage(rows, planned_rounds=rounds, proposition="disabled"), + proposition="disabled", + ) + + +async def _probe_effort(model, effort, *, rounds, prompt, prompt_kind): + """临时全档表仅用于 T10 探测，不写回 DEFAULT，也不生成预期 wire。""" + return await _collect_rounds( + _tier_settings(model), + rounds=rounds, + stream=True, + prompt=prompt, + matrix_id="T10-" + prompt_kind, + effort=effort, + capabilities={model: ThinkingCapability(_ALL_EFFORTS, evidence="T10 临时探测声明")}, + concurrency=_TIER_CONCURRENCY, + ) class TestTierProbe: - """能力表实测。可只跑单个模型: `-k "test_t10 and glm-5.3"`。""" + """逐型号能力命题，不把拒绝、不可关闭和 UNKNOWN 混在一起。""" @pytest.mark.parametrize("model", sorted(_MODEL_PROVIDER)) async def test_t10_none_direction_matches_declaration(self, model): - """「这个模型到底关不关得掉」——能力表里唯一会**报错**的那条声明。 - - 它是本节最要紧的一条: `Effort.NONE` 在不在清单里,决定 Phase 4 是放行还是 - 当场报错。声明错了,两个方向的代价都很实在——多写了 `none` 会让下游以为 - 关掉了(issue #20 的静默失效),漏写了会把一条本来可用的路堵死。 - """ + capability = DEFAULT_CAPABILITIES.get(model) + proposition = "disabled" if capability and capability.can_disable else "cannot_disable" short = await _probe_effort( - model, Effort.NONE, rounds=_TIER_ROUNDS, prompt=_TIER_PROMPT, prompt_kind="short" + model, Effort.NONE, rounds=_TIER_ROUNDS, prompt=_TIER_PROMPT, prompt_kind="none-short" ) - # 上游拒绝这一档(400)是**结论**而非故障: 它等价于"关不掉"; - # 其余失败(渠道下线/型号被摘/超时)才是源不可用,按既有纪律记为未覆盖。 - # 404 `model_not_found` 因此不算 rejected —— 它会自然落进下面那条源不可用分支 - rejected = [o for o in short if _probe_rejected(o)] - usable = [o for o in short if _probe_ok(o)] - # **按可用轮判,而不是一有失败就整条跳过**: 共用网关上偶发 429/503 是常态, - # 一票否决会让整张表因为一次抖动而没有数据。样本低于 3 轮才是真的没结论 - if not rejected and len(usable) < min(3, _TIER_ROUNDS): - broken = [o for o in short if o["error"]] - _probe_record( - model, "none 方向", _unreachable_verdict(short), _rt_summary(short), short - ) - pytest.skip(f"{model} 源不可用,已记为未覆盖: {broken[0]['error'][:120]}") - - strangers = _identity_mismatch(model, short) - if strangers: - _probe_record( - model, - "none 方向", - "SKIP(身份不符,数据不可信)", - f"该渠道把请求回报成 {strangers};{_rt_summary(short)}", - short, - ) - pytest.skip(f"{model} 被该渠道路由到 {strangers},本次观测说的不是这个模型") - - observations = list(short) - measured_can_disable = not rejected and all(_probe_quiet(o) for o in usable) - note = "" - if measured_can_disable: - # 判据②: 短提示词下"看起来关了"必须过长上下文这一关 - long_ctx = await _probe_effort( + verdict = _coverage(short, planned_rounds=_TIER_ROUNDS, proposition=proposition) + # 沿既有矩阵：短档没有 OBSERVED 才做长上下文复核；不新增锚点调用。 + if _qualified(short, planned_rounds=_TIER_ROUNDS).status == "PASS" and not any( + r["response"].thinking_observation is ThinkingObservation.OBSERVED for r in short + ): + long_rows = await _probe_effort( model, Effort.NONE, rounds=_TIER_LONG_ROUNDS, prompt=_TIER_LONG_PROMPT, - prompt_kind="long", + prompt_kind="none-long", ) - observations += long_ctx - usable = [o for o in long_ctx if _probe_ok(o)] - if not usable: - note = ";长上下文复核未跑通,结论只在短提示词下成立" - else: - measured_can_disable = all(_probe_quiet(o) for o in usable) - note = ";长上下文复核" + ("同样未观测到推理" if measured_can_disable else "露馅") - - if measured_can_disable and not any( - o["thinking_observation"] is ThinkingObservation.ABSENT for o in observations - ): - # 判据④: 全程 `UNKNOWN` 时,"关掉了"是一句没有正面证据的话 - anchor, anchor_tier, separable = await _anchor_off_against_on(model, observations) - observations += anchor - if anchor_tier is None: - note += ";锚点未跑通,关闭结论缺正面证据" - elif separable: - note += f";锚点可分(关闭档 completion 严格小于 {anchor_tier.value} 档)" - else: - measured_can_disable = None - note += f";**锚点不可分**(与 {anchor_tier.value} 档的 completion 分不开),判不出来" - - detail = f"实测 can_disable={measured_can_disable}{note}。短: {_rt_summary(short)}" + ( - f" ‖ 后续: {_rt_summary(observations[len(short) :])}" - if len(observations) > len(short) - else "" - ) - if measured_can_disable is None: - _probe_record(model, "none 方向", "INCONCLUSIVE(无正面证据)", detail, observations) - pytest.skip(f"{model} 判不出来,已记为未覆盖: {detail[:160]}") - capability = get_capability(model) - if capability is None: - _probe_record(model, "none 方向", "DATA(未登记)", detail, observations) - pytest.skip(f"{model} 未登记(设计 §8 第三档),本条只采数据: {detail[:120]}") - agrees = measured_can_disable == capability.can_disable - _probe_record( - model, - "none 方向", - "PASS" if agrees else "FAIL(能力表已漂移)", - f"声明 can_disable={capability.can_disable};{detail}", - observations, - ) - assert agrees, ( - f"{model} 的能力表与实测不符: 声明 can_disable={capability.can_disable}," - f"实测 {measured_can_disable}。{detail}" - ) + verdict = _coverage( + short + long_rows, + planned_rounds=_TIER_ROUNDS + _TIER_LONG_ROUNDS, + proposition=proposition, + ) + if capability is None and verdict.status != "FAIL": + verdict = LiveVerdict("UNCOVERED", "未登记候选只保留观测，不自动登记能力") + _conclude("T10-none", verdict, proposition=proposition) @pytest.mark.parametrize("model", sorted(DEFAULT_CAPABILITIES)) async def test_t10_declared_tiers_actually_reason(self, model): - """已登记的每个**开启档**都必须被上游接受,且真的推理。 - - 证伪力只在"被拒"与"没推理"两件事上——**不断言档位之间的 rt 高低**: - 设计 §4.3 已定,同一档 rt 实测在 8~56 之间跳,拿它比大小必然是噪声。 - 故本条能证伪的是"登记了一个上游根本不认的档",不是"档位排序对不对"。 - """ - capability = get_capability(model) - tiers = [e for e in capability.supported_efforts if e is not Effort.NONE] - if not tiers: - pytest.skip(f"{model} 只登记了 none,没有开启档可验") - failures = [] + tiers = [e for e in DEFAULT_CAPABILITIES[model].supported_efforts if e is not Effort.NONE] verdicts = [] for tier in tiers: - observations = await _probe_effort( - model, tier, rounds=_TIER_ROUNDS, prompt=_TIER_PROMPT, prompt_kind="short" + rows = await _probe_effort( + model, + tier, + rounds=_TIER_ROUNDS, + prompt=_TIER_PROMPT, + prompt_kind="tier-" + tier.value, ) - # 同 none 方向: 只有"拒绝这一档"才是关于档位的结论,404 型号不存在 - # 说明的是源不可用,记成 FAIL 会把渠道摘型号读成"登记了个上游不认的档" - rejected = [o for o in observations if _probe_rejected(o)] - usable = [o for o in observations if _probe_ok(o)] - observed = [o for o in observations if _probe_observed(o)] - strangers = _identity_mismatch(model, observations) - if strangers: - # 与 none 方向同一条纪律: 回报的不是这个模型,这组数就不是它的 - _probe_record( - model, - f"档位 {tier.value}", - "SKIP(身份不符,数据不可信)", - f"该渠道把请求回报成 {strangers};{_rt_summary(observations)}", - observations, - ) - pytest.skip(f"{model} 被该渠道路由到 {strangers},本次观测说的不是这个模型") - if rejected: - verdict, problem = "FAIL(上游拒绝该档)", f"{tier.value}: 上游拒绝" - elif not usable: - verdict, problem = _unreachable_verdict(observations), None - elif len(observed) * 2 > len(usable): - verdict, problem = "PASS", None - else: - verdict, problem = "FAIL(该档未推理)", f"{tier.value}: 多数轮未观测到推理" - if problem: - failures.append(problem) + verdict = _coverage(rows, planned_rounds=_TIER_ROUNDS, proposition="enabled") verdicts.append(verdict) - _probe_record( - model, f"档位 {tier.value}", verdict, _rt_summary(observations), observations + write_live_round( + _OUT_DIR, + run_id=uuid4().hex, + matrix_id="T10-tier", + round_index=0, + safe_fields={ + "requested_model": model, + "requested_effort": tier.value, + "status": verdict.status, + "reason": verdict.reason, + }, ) - assert not failures, f"{model} 登记的档位与实测不符: {failures}" - if all(v.startswith("SKIP") for v in verdicts): - # 一档都没跑通却判绿,就是本模块 docstring 明令禁止的"静默计入通过": - # 绿色在这里会被读成"登记的档位都验过了",而实情是一条都没验 - pytest.skip(f"{model} 各档均源不可用,已记为未覆盖: {verdicts}") + _conclude("T10-tiers", _combine(verdicts), proposition="enabled-all-declared-tiers") @pytest.mark.parametrize("model", ["gemini-3.1-pro", "gpt-5.5", "glm-5.3"]) async def test_t10_no_opinion_stays_no_opinion(self, model): - """不表态时库**不推定**模型自己的默认档(Phase 1),顺带采下默认档的 rt 基线。 - - 为什么给这三个模型单列一条: 它们的「厂商默认档」是 evidence 里写着、却最容易 - 写错的一格(Gemini 3.1 Pro 官方文档说 HIGH、OpenRouter 说 medium,两源打架), - 而默认档写错会误导下游估成本。库本身不依赖这个值——**它不表态就什么都不注入**, - 这正是本条断言的东西;默认档的 rt 观测只作报告里的旁证,**不作断言**: 单一模型上 - rt 与档位没有可判定的函数关系(设计 §4.3),拿它反推默认档只能存疑,不能定论。 - - 2026-09-05: gemini 一路当下在本渠道上游报错,claude 一路 7 天限额用尽,故把 - 另两格换成当下可测的 gpt-5.5 与 glm-5.3;gemini 留着,渠道恢复即有数。 - """ - client = GatewayClient.from_settings( - _tier_settings(model), capabilities=_probe_capabilities(model) + rows = await _collect_rounds( + _tier_settings(model), + rounds=_TIER_ROUNDS, + stream=True, + prompt=_TIER_PROMPT, + matrix_id="T10-default", + capabilities={model: ThinkingCapability(_ALL_EFFORTS, evidence="T10 临时探测声明")}, ) - observations = [] - try: - for i in range(_TIER_ROUNDS): - try: - resp = await client.chat( - [{"role": "user", "content": _TIER_PROMPT}], - stream=True, - cache_salt=f"tier-default-{model}-{i}", - ) - except ( - RequestRejectedError, - GatewayUnavailableError, - SourceDeadError, - TransientError, - ) as exc: - observations.append( - { - "round": i + 1, - "effort": "(不表态)", - "prompt_kind": "short", - "error": f"{type(exc).__name__}: {str(exc)[:160]}", - # 与 `_probe_effort` 的失败轮同形: 少这两格, - # `_unreachable_verdict` 会把"型号被摘"读成普通抖动 - "error_status": exc.status_code, - "error_body": exc.body_text, - } - ) - continue - observations.append( - { - "round": i + 1, - "effort": "(不表态)", - "prompt_kind": "short", - "error": None, - "prompt_tokens": resp.prompt_tokens, - "completion_tokens": resp.completion_tokens, - "reasoning_tokens": resp.reasoning_tokens, - "thinking_chars": len(resp.thinking), - "thinking_observation": resp.thinking_observation, - "applied_effort": resp.applied_effort, - "model_reported": resp.model_reported, - "content": resp.content[:40], - } - ) - finally: - await client.aclose() - usable = [o for o in observations if _probe_ok(o)] - if not usable: - _probe_record( - model, - "默认档基线(不表态)", - _unreachable_verdict(observations), - _rt_summary(observations), - observations, - ) - pytest.skip(f"{model} 源不可用,已记为未覆盖: {observations[0]['error'][:120]}") - leaked = [o for o in usable if o["applied_effort"] is not None] - _probe_record( - model, - "默认档基线(不表态)", - "PASS" if not leaked else "FAIL(库替模型推定了默认档)", - _rt_summary(observations), - observations, - ) - assert not leaked, f"{model}: 不表态时 applied_effort 应为 None,实测 {leaked}" + verdict = _qualified(rows, planned_rounds=_TIER_ROUNDS) + if verdict.status == "PASS" and any( + row["response"].applied_effort is not None for row in rows + ): + verdict = LiveVerdict("FAIL", "默认基线擅自推定档位") + _conclude("T10-default", verdict, proposition="no-opinion-not-capability") diff --git a/tests/live_evidence.py b/tests/live_evidence.py new file mode 100644 index 0000000..e870d7c --- /dev/null +++ b/tests/live_evidence.py @@ -0,0 +1,308 @@ +"""真实测试的有限证据判定；不读取环境、不请求网络、不记录原始正文。""" + +import hashlib +import json +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal +from uuid import uuid4 + +from polygateway.errors import RequestRejectedError +from polygateway.types import ThinkingObservation + + +@dataclass(frozen=True) +class HttpEvidence: + """一次 HTTP 的内存证据，禁止直接序列化。""" + + call_id: str + request_checks: tuple[tuple[str, bool], ...] + status_code: int + error_body: bytes | None + raw_identity: tuple[bool, str | None] + + +@dataclass(frozen=True) +class AttemptEvidence: + """一次 transport 尝试，可以没有 HTTP。""" + + call_id: str + http: tuple[HttpEvidence, ...] + error: Exception | None + + +@dataclass(frozen=True) +class LiveVerdict: + """测试命题的结论，不等同于 pytest 退出码。""" + + status: Literal["PASS", "FAIL", "UNCOVERED"] + reason: str + + +def strict_json(body: bytes) -> Any: + """独立解析完整 UTF-8 JSON，拒绝重复键及非标准常量。""" + + def pairs(items: list[tuple[str, Any]]) -> dict[str, Any]: + """重复键不允许被后值掩盖。""" + result = {} + for key, value in items: + if key in result: + raise ValueError("重复 JSON 键") + result[key] = value + return result + + def invalid(value: str) -> None: + """拒绝非标准数值常量。""" + raise ValueError("非法 JSON 常量") + + return json.loads(body.decode("utf-8"), object_pairs_hook=pairs, parse_constant=invalid) + + +def messages_digest(messages: Any) -> str: + """只在内存比较提示词摘要，不写提示词。""" + return hashlib.sha256( + json.dumps(messages, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +def request_is_valid(event: HttpEvidence) -> bool: + """完整且无重复的显式检查才能作为请求资格。""" + checks = dict(event.request_checks) + common = {"method", "origin", "path", "model", "authorization", "control", "messages_digest"} + return ( + len(checks) == len(event.request_checks) + and set(checks) in (common | {"stream"}, common | {"input_shape"}) + and all(value is True for value in checks.values()) + ) + + +def _single_error(error: Exception, attempts: Sequence[AttemptEvidence]) -> HttpEvidence | None: + """仅接受可与最终异常精确配对的独立单次错误。""" + if not isinstance(error, RequestRejectedError) or len(attempts) != 1: + return None + attempt = attempts[0] + if attempt.error is not error or len(attempt.http) != 1: + return None + event = attempt.http[0] + if event.call_id != attempt.call_id or not request_is_valid(event): + return None + if event.status_code != error.status_code: + return None + return event + + +def error_machine_type(event: HttpEvidence) -> str | None: + """完整错误体中的唯一机器字段；不检查正文子串。""" + body = event.error_body + if body is None or not 0 < len(body) <= 65536: + return None + try: + data = strict_json(body) + except (ValueError, UnicodeError): + return None + if not isinstance(data, dict) or not isinstance(data.get("error"), dict): + return None + value = data["error"].get("type") + return value if isinstance(value, str) else None + + +def classify_live_failure(error: Exception, attempts: Sequence[AttemptEvidence]) -> LiveVerdict: + """默认失败；唯一自动外因是完整证据支持的 404 model_not_found。""" + event = _single_error(error, attempts) + if ( + event is not None + and event.status_code == 404 + and error_machine_type(event) == "model_not_found" + ): + return LiveVerdict("UNCOVERED", "端点回报该请求型号不可用") + return LiveVerdict("FAIL", "无满足窄外因契约的完整独立证据") + + +def assess_expected_rejection( + error: Exception, attempts: Sequence[AttemptEvidence], *, status_code: int, machine_type: str +) -> LiveVerdict: + """预先声明的拒绝命题，不把一般 400 当不支持档位。""" + event = _single_error(error, attempts) + if ( + event is not None + and event.status_code == status_code + and error_machine_type(event) == machine_type + ): + return LiveVerdict("PASS", "符合预声明的拒绝类型、状态和机器字段") + return LiveVerdict("FAIL", "不符合预声明拒绝证据") + + +def assess_model_identity( + *, + requested: str, + aliases: frozenset[str], + reported: str | None, + raw_identity: tuple[bool, str | None], + request_valid: bool, +) -> LiveVerdict: + """公共身份异常只有原始独立证据才能归因上游。""" + if not request_valid: + return LiveVerdict("FAIL", "实发请求校验不完整或不符") + allowed = {requested, *aliases} + captured, raw = raw_identity + if captured and raw != reported: + return LiveVerdict("FAIL", "公共身份与独立原始身份不一致") + if reported in allowed: + return LiveVerdict("PASS", "身份合格") + if not captured: + return LiveVerdict("FAIL", "身份来源无法区分") + return LiveVerdict("UNCOVERED", "独立原始响应身份缺失或不属于显式别名") + + +def assess_thinking_coverage( + observations: Sequence[ThinkingObservation], + *, + planned_rounds: int, + proposition: Literal["enabled", "disabled", "cannot_disable"], +) -> LiveVerdict: + """按独立命题裁定；缺轮不减分母，UNKNOWN 不证明关闭。""" + if planned_rounds < 1 or len(observations) != planned_rounds: + return LiveVerdict("FAIL", "计划轮次不完整") + if any(not isinstance(item, ThinkingObservation) for item in observations): + return LiveVerdict("FAIL", "观测类型不符") + observed = observations.count(ThinkingObservation.OBSERVED) + unknown = observations.count(ThinkingObservation.UNKNOWN) + if proposition == "enabled": + if observed > planned_rounds / 2: + return LiveVerdict("PASS", "完整轮次多数观测到推理") + return LiveVerdict("UNCOVERED" if unknown else "FAIL", "开启证据未达多数") + if proposition == "disabled": + if observed: + return LiveVerdict("FAIL", "观测到推理，证伪关闭声明") + return LiveVerdict( + "UNCOVERED" if unknown else "PASS", + "UNKNOWN 不证明关闭" if unknown else "每轮明确 ABSENT", + ) + if proposition == "cannot_disable": + if observed: + return LiveVerdict("PASS", "本条件下仍推理；不外推所有私有参数") + return LiveVerdict("UNCOVERED" if unknown else "FAIL", "缺少仍推理的证据") + raise ValueError("未知测试命题") + + +def summarize_verdicts(verdicts: Sequence[LiveVerdict], *, planned_rounds: int) -> dict[str, int]: + """保留失败、未覆盖与缺轮的独立计数。""" + if planned_rounds < len(verdicts): + raise ValueError("实际轮次超出计划") + return { + **{ + status: sum(v.status == status for v in verdicts) + for status in ("PASS", "FAIL", "UNCOVERED") + }, + "missing": planned_rounds - len(verdicts), + } + + +_SAFE_FIELDS = frozenset( + { + "provider", + "requested_model", + "reported_model", + "stream", + "requested_effort", + "applied_effort", + "session_id", + "parent_call_id", + "attempts", + "status", + "reason", + "completed_rounds", + "planned_rounds", + "thinking_observation", + "prompt_tokens", + "completion_tokens", + "reasoning_tokens", + "thinking_chars", + "counts", + "proposition", + "error_type", + "error_status", + "evidence_notes", + } +) +_ATTEMPT_FIELDS = frozenset({"call_id", "error_type", "http"}) +_HTTP_FIELDS = frozenset( + {"status_code", "request_checks", "identity_captured", "error_body_complete"} +) + + +def _validate_safe(fields: Mapping[str, Any]) -> None: + """拒绝原始异常和证据对象，嵌套字段也有白名单。""" + if set(fields) - _SAFE_FIELDS: + raise ValueError("报告含非白名单字段") + for attempt in fields.get("attempts", []): + if not isinstance(attempt, dict) or set(attempt) != _ATTEMPT_FIELDS: + raise ValueError("非法 attempt 报告") + for event in attempt["http"]: + if not isinstance(event, dict) or set(event) != _HTTP_FIELDS: + raise ValueError("非法 HTTP 报告") + if not isinstance(event["request_checks"], dict) or any( + type(v) is not bool for v in event["request_checks"].values() + ): + raise ValueError("请求校验报告只允许布尔值") + # 不提供 default=str：原始异常、bytes、dataclass 均必须失败。 + json.dumps(fields, ensure_ascii=False, allow_nan=False) + + +def write_live_round( + output_dir: Path, + *, + run_id: str, + matrix_id: str, + round_index: int, + safe_fields: Mapping[str, Any], +) -> Path: + """仅接收已脱敏字段；独占文件写入失败必须冒泡。""" + _validate_safe(safe_fields) + if round_index < 0 or not re.fullmatch(r"[a-zA-Z0-9_-]+", run_id + matrix_id): + raise ValueError("报告路径标识非法") + directory = output_dir / run_id + directory.mkdir(parents=True, exist_ok=True) + path = directory / f"{matrix_id}-{round_index}-{uuid4().hex}.md" + text = json.dumps(dict(safe_fields), ensure_ascii=False, indent=2, allow_nan=False) + with path.open("x", encoding="utf-8") as handle: + handle.write(f"# {matrix_id} · 轮次 {round_index}\n\n```json\n{text}\n```\n") + return path + + +def safe_attempts(attempts: Sequence[AttemptEvidence]) -> list[dict[str, Any]]: + """只导出事实布尔值、异常类和状态；不落盘任何上游正文。""" + return [ + { + "call_id": attempt.call_id, + "error_type": type(attempt.error).__name__ if attempt.error else None, + "http": [ + { + "status_code": event.status_code, + "request_checks": dict(event.request_checks), + "identity_captured": event.raw_identity[0], + "error_body_complete": event.error_body is not None, + } + for event in attempt.http + ], + } + for attempt in attempts + ] + + +def combine_live_verdicts(verdicts: Sequence[LiveVerdict]) -> LiveVerdict: + """部分失败优先于未覆盖；部分未覆盖不得汇总全 PASS。""" + if any(verdict.status == "FAIL" for verdict in verdicts): + return LiveVerdict("FAIL", "至少一个必需命题失败") + if not verdicts or any(verdict.status == "UNCOVERED" for verdict in verdicts): + return LiveVerdict("UNCOVERED", "至少一个必需命题未覆盖") + return LiveVerdict("PASS", "所有必需命题通过") + + +def qualify_live_rounds(verdicts: Sequence[LiveVerdict], *, planned_rounds: int) -> LiveVerdict: + """汇总请求／身份资格，缺轮绝不缩小分母。""" + if planned_rounds < 1 or len(verdicts) != planned_rounds: + return LiveVerdict("FAIL", "计划轮次缺失") + return combine_live_verdicts(verdicts) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index e08e8b8..2f4d8e7 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1340,3 +1340,25 @@ async def test_managed_conflict_returns_half_open_probe_and_permit(): await gate.release_probe(next_entry) finally: await client._transport.aclose() + + +async def test_synthetic_runtime_protocol_and_legacy_call_signatures(): + """合成 Protocol 仅证明本库兼容契约，不冒充缺失下游实际验收。""" + import inspect + from typing import Protocol, runtime_checkable + + @runtime_checkable + class Caller(Protocol): + """旧调用点只依赖 chat 协议。""" + + async def chat(self, messages, **kwargs): ... + + client = GatewayClient.from_env("LLM", env=_ENV) + try: + assert isinstance(client, Caller) + signature = inspect.signature(client.chat) + signature.bind([], session_id="session", parent_call_id="step") + signature.bind([], session_id="session", cache_salt="epoch-1") + assert client._transport._clients == {} + finally: + await client.aclose() diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 61af01c..e853783 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -997,3 +997,47 @@ class TestCrossFieldInvariants: ) client = GatewayClient.from_settings(settings) assert client is not None + + +async def test_flat_legacy_keys_assemble_without_source_timeout_or_network(): + """完整合成 env 验证平铺回落，不借真实配置或 Redis/PG。""" + env = dict(_BASE_ENV) + del env["LLM__QWEN__1__TIMEOUT_S"] + env.update({"LLM_TIMEOUT": "317", "LLM_TTFT_TIMEOUT": "41", "LLM_INTER_TOKEN_TIMEOUT": "19"}) + settings = GatewaySettings.from_env("LLM", env=env) + assert settings.sources[0].timeout_s == 317 + assert settings.sources[0].ttft_timeout_s == 41 + assert settings.sources[0].inter_token_timeout_s == 19 + assert settings.retry.max_attempts == 3 + client = GatewayClient.from_settings(settings) + try: + assert client._transport._clients == {} + finally: + await client.aclose() + + +@pytest.mark.parametrize("model", ["MiniMax-M2.7", "MiniMax-M3"]) +def test_live_assembly_rejection_is_local_only(model): + """M2.7 NONE 与 M3 AUTO 的旧 live 装配断言离线执行。""" + env = _env(**{"LLM__QWEN__1__MODEL": model}) + settings = GatewaySettings.from_env("LLM", env=env) + source = dataclasses.replace( + settings.sources[0], provider="minimax", enable_thinking=model == "MiniMax-M3" + ) + with pytest.raises(ValueError, match=model): + GatewayClient.from_settings(dataclasses.replace(settings, sources=(source,))) + + +def test_live_unknown_wire_assembly_is_local_only(): + """L9 明确全 None profile，不从当前默认注册表猜未知形态。""" + mystery = ProviderProfile( + name="mystery", + thinking=ThinkingWire(off=None, on_base=None, effort_key=None), + strip_think_tags=False, + ) + settings = GatewaySettings.from_env("LLM", env=_BASE_ENV) + source = dataclasses.replace(settings.sources[0], provider="mystery", enable_thinking=False) + with pytest.raises(ValueError, match="register_provider"): + GatewayClient.from_settings( + dataclasses.replace(settings, sources=(source,)), registry=register_provider(mystery) + ) diff --git a/tests/unit/test_live_evidence.py b/tests/unit/test_live_evidence.py new file mode 100644 index 0000000..7a4d86c --- /dev/null +++ b/tests/unit/test_live_evidence.py @@ -0,0 +1,816 @@ +"""live_evidence 与测试侧取证装配的日常离线反例。""" + +import asyncio +import inspect +import json +from dataclasses import replace + +import httpx +import pytest + +from polygateway.errors import RequestRejectedError, SourceDeadError, TransientError +from polygateway.ports import EmbeddingTransport, Transport +from polygateway.transports.openai_compat import OpenAICompatTransport +from polygateway.types import SourceConfig +from polygateway.types import ThinkingObservation as O +from tests.e2e.conftest import LiveCapture, ObservedTransport +from tests.live_evidence import ( + AttemptEvidence, + HttpEvidence, + LiveVerdict, + assess_expected_rejection, + assess_model_identity, + assess_thinking_coverage, + classify_live_failure, + messages_digest, + request_is_valid, + safe_attempts, + summarize_verdicts, + write_live_round, +) + +_CHECKS = tuple( + (key, True) + for key in ( + "method", + "origin", + "path", + "model", + "stream", + "authorization", + "control", + "messages_digest", + ) +) +_BODY = b'{"error":{"type":"model_not_found"}}' +_SECRET = "fake-unique-credential-sentinel" +_PROMPT = "fake-private-prompt-sentinel" +_MESSAGES = [{"role": "user", "content": _PROMPT}] + + +def _failure(*, body=_BODY, status=404, checks=_CHECKS): + """完整的唯一错误证据，供逐维破坏。""" + error = RequestRejectedError("safe", status_code=status) + event = HttpEvidence("a", checks, status, body, (False, None)) + return error, (AttemptEvidence("a", (event,), error),) + + +def test_exact_model_not_found_is_uncovered(): + error, attempts = _failure() + assert classify_live_failure(error, attempts).status == "UNCOVERED" + + +@pytest.mark.parametrize( + "body", + [ + None, + b"", + b"{", + b"[]", + b'{"error":[]}', + b'{"error":{"type":7}}', + b'{"error":{"message":"model_not_found"}}', + b'{"error":{"type":"MODEL_NOT_FOUND"}}', + b'{"error":{"type":"wrong","type":"model_not_found"}}', + b'{"error":{},"error":{"type":"model_not_found"}}', + b"\xff", + _BODY + b" " * 65536, + ], +) +def test_incomplete_or_ambiguous_error_body_fails(body): + assert classify_live_failure(*_failure(body=body)).status == "FAIL" + + +@pytest.mark.parametrize("status", [400, 401, 403, 429, 500, 503]) +def test_status_classes_never_automatically_skip(status): + assert classify_live_failure(*_failure(status=status)).status == "FAIL" + + +@pytest.mark.parametrize( + "error", + [TransientError("safe"), SourceDeadError("safe"), ValueError("safe"), AssertionError("safe")], +) +def test_whole_exception_class_skip_is_forbidden(error): + assert classify_live_failure(error, ()).status == "FAIL" + + +@pytest.mark.parametrize("key", [key for key, _ in _CHECKS]) +@pytest.mark.parametrize("mode", ["missing", "false", "duplicate"]) +def test_all_request_checks_are_required(key, mode): + checks = tuple( + (k, False if k == key and mode == "false" else v) + for k, v in _CHECKS + if mode != "missing" or k != key + ) + if mode == "duplicate": + checks += ((key, True),) + assert classify_live_failure(*_failure(checks=checks)).status == "FAIL" + + +@pytest.mark.parametrize( + "mode", + ["empty", "second_attempt", "second_http", "wrong_id", "different_error", "status_mismatch"], +) +def test_attempt_pairing_cannot_be_guessed(mode): + error, attempts = _failure() + first = attempts[0] + if mode == "empty": + attempts = (replace(first, http=()),) + elif mode == "second_attempt": + attempts += (AttemptEvidence("b", (), TransientError("safe")),) + elif mode == "second_http": + attempts = (replace(first, http=first.http * 2),) + elif mode == "wrong_id": + attempts = (replace(first, call_id="other"),) + elif mode == "different_error": + attempts = (replace(first, error=RequestRejectedError("safe", status_code=404)),) + else: + attempts = (replace(first, http=(replace(first.http[0], status_code=400),)),) + assert classify_live_failure(error, attempts).status == "FAIL" + + +@pytest.mark.parametrize( + ("reported", "raw", "expected"), + [ + ("m", (False, None), "PASS"), + ("alias", (True, "alias"), "PASS"), + (None, (False, None), "FAIL"), + ("other", (False, None), "FAIL"), + (None, (True, None), "UNCOVERED"), + ("other", (True, "other"), "UNCOVERED"), + (None, (True, "m"), "FAIL"), + ("wrong", (True, "m"), "FAIL"), + ("m", (True, None), "FAIL"), + ], +) +def test_identity_requires_independent_raw_evidence(reported, raw, expected): + assert ( + assess_model_identity( + requested="m", + aliases=frozenset({"alias"}), + reported=reported, + raw_identity=raw, + request_valid=True, + ).status + == expected + ) + assert ( + assess_model_identity( + requested="m", + aliases=frozenset(), + reported=reported, + raw_identity=raw, + request_valid=False, + ).status + == "FAIL" + ) + + +@pytest.mark.parametrize( + ("proposition", "observations", "expected"), + [ + ("enabled", [O.OBSERVED, O.OBSERVED, O.UNKNOWN], "PASS"), + ("enabled", [O.UNKNOWN] * 3, "UNCOVERED"), + ("enabled", [O.ABSENT] * 3, "FAIL"), + ("disabled", [O.ABSENT] * 3, "PASS"), + ("disabled", [O.ABSENT, O.UNKNOWN, O.ABSENT], "UNCOVERED"), + ("disabled", [O.OBSERVED, O.UNKNOWN, O.ABSENT], "FAIL"), + ("cannot_disable", [O.OBSERVED, O.UNKNOWN, O.ABSENT], "PASS"), + ("cannot_disable", [O.ABSENT] * 3, "FAIL"), + ("cannot_disable", [O.UNKNOWN] * 3, "UNCOVERED"), + ], +) +def test_coverage_is_proposition_specific(proposition, observations, expected): + assert ( + assess_thinking_coverage(observations, planned_rounds=3, proposition=proposition).status + == expected + ) + + +def test_missing_round_never_reduces_denominator(): + assert ( + assess_thinking_coverage([O.OBSERVED] * 2, planned_rounds=3, proposition="enabled").status + == "FAIL" + ) + assert summarize_verdicts( + [LiveVerdict("PASS", "ok"), LiveVerdict("FAIL", "bad")], planned_rounds=3 + ) == {"PASS": 1, "FAIL": 1, "UNCOVERED": 0, "missing": 1} + + +def test_expected_400_is_a_separate_negative_proposition(): + error, attempts = _failure(status=400, body=b'{"error":{"type":"unsupported_value"}}') + assert classify_live_failure(error, attempts).status == "FAIL" + assert ( + assess_expected_rejection( + error, attempts, status_code=400, machine_type="unsupported_value" + ).status + == "PASS" + ) + assert ( + assess_expected_rejection(error, attempts, status_code=400, machine_type="other").status + == "FAIL" + ) + + +def _source(): + """假凭据与非默认超时，不接真实网络。""" + return SourceConfig( + name="source", + provider="openai", + model="gpt-5.5", + base_url="https://example.test/v1", + api_key=_SECRET, + timeout_s=137, + trust_env=False, + ) + + +def _capture(*, stream=False, embedding=False, **changes): + """预期独立于生产 payload。""" + expected = { + "model": "gpt-5.5", + "origin": "https://example.test", + "path": "/v1/embeddings" if embedding else "/v1/chat/completions", + "control": {}, + "messages_digest": messages_digest([_PROMPT] if embedding else _MESSAGES), + } + expected.update({"input_shape": 1} if embedding else {"stream": stream}) + expected.update(changes) + return LiveCapture(expectations={"source": expected}) + + +def _response(model="gpt-5.5"): + """OpenAI 非流式完整响应形态。""" + result = { + "id": "test", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + if model != "missing": + result["model"] = model + return result + + +def _real_transport(capture, handler, *, tamper=None): + """仅替换 HTTP 边界，保留真实 hooks 与 production transport。""" + + def factory(source): + """在网络边界换成 MockTransport。""" + client = capture.client_factory(source) + client._transport = httpx.MockTransport(handler) + if tamper is not None: + client.event_hooks["request"].insert(0, tamper) + return client + + return OpenAICompatTransport(client_factory=factory) + + +async def _complete(observed, *, call_id="a", stream=False): + """沿原参数调用薄委托器。""" + return await observed.complete( + messages=_MESSAGES, + source=_source(), + stream=stream, + overlay={}, + call_id=call_id, + reasoning_effort=None, + ) + + +@pytest.mark.parametrize("model", ["gpt-5.5", "missing", None]) +async def test_raw_identity_snapshot_reaches_round_consumer(model): + capture = _capture() + real = _real_transport(capture, lambda request: httpx.Response(200, json=_response(model))) + try: + with capture.round_context(session_id="s", parent_call_id="p"): + result = await _complete(ObservedTransport(real, capture)) + raw = capture.raw_identity(session_id="s", parent_call_id="p", call_id="a") + assert raw == (True, None if model == "missing" else model) + assert request_is_valid(capture.attempts(session_id="s", parent_call_id="p")[0].http[0]) + verdict = assess_model_identity( + requested="gpt-5.5", + aliases=frozenset(), + reported=result.model_reported, + raw_identity=raw, + request_valid=True, + ) + assert verdict.status == ("PASS" if model == "gpt-5.5" else "UNCOVERED") + if model == "gpt-5.5": + assert ( + assess_model_identity( + requested="gpt-5.5", + aliases=frozenset(), + reported=None, + raw_identity=raw, + request_valid=True, + ).status + == "FAIL" + ) + finally: + await real.aclose() + + +@pytest.mark.parametrize( + "body", [b"[]", b"{", b'{"model":"a","model":"b"}', b'{"model":7}', b"\xff"] +) +async def test_invalid_raw_json_is_not_missing_identity(body): + capture = _capture() + real = _real_transport(capture, lambda request: httpx.Response(200, content=body)) + try: + with ( + capture.round_context(session_id="s", parent_call_id="p"), + pytest.raises((ValueError, AttributeError, KeyError, TransientError)), + ): + await _complete(ObservedTransport(real, capture)) + assert capture.raw_identity(session_id="s", parent_call_id="p", call_id="a") == ( + False, + None, + ) + assert capture.notes(session_id="s", parent_call_id="p") + finally: + await real.aclose() + + +@pytest.mark.parametrize( + "field", ["model", "authorization", "path", "stream", "control", "messages_digest"] +) +async def test_hooks_detect_corrupted_request_without_repair(field): + capture = _capture() + + async def tamper(request): + """故意改错实发请求，而不是改预期。""" + if field == "authorization": + request.headers["Authorization"] = "Bearer wrong" + elif field == "path": + request.url = request.url.copy_with(path="/wrong") + else: + body = json.loads(request.content) + key, value = { + "model": ("model", "wrong"), + "stream": ("stream", True), + "control": ("reasoning_effort", "high"), + "messages_digest": ("messages", []), + }[field] + body[key] = value + request._content = json.dumps(body).encode() + + real = _real_transport( + capture, lambda request: httpx.Response(404, content=_BODY), tamper=tamper + ) + try: + with ( + capture.round_context(session_id="s", parent_call_id="p"), + pytest.raises(RequestRejectedError) as caught, + ): + await _complete(ObservedTransport(real, capture)) + attempts = capture.attempts(session_id="s", parent_call_id="p") + assert dict(attempts[0].http[0].request_checks)[field] is False + assert classify_live_failure(caught.value, attempts).status == "FAIL" + finally: + await real.aclose() + + +async def test_embed_uses_same_capture_and_source_factory_contract(): + capture = _capture(embedding=True) + real = _real_transport( + capture, + lambda request: httpx.Response( + 200, + json={"data": [{"index": 0, "embedding": [0.1, 0.2]}], "usage": {"prompt_tokens": 1}}, + ), + ) + try: + with capture.round_context(session_id="s", parent_call_id="p"): + result = await ObservedTransport(real, capture).embed( + texts=[_PROMPT], source=_source(), call_id="e" + ) + assert result.dim == 2 + assert request_is_valid(capture.attempts(session_id="s", parent_call_id="p")[0].http[0]) + client = real._clients["source"] + assert client.timeout.read == 137 and client.trust_env is False + assert client.headers["Authorization"] == f"Bearer {_SECRET}" + finally: + await real.aclose() + assert real._clients == {} + + +class _SSE(httpx.AsyncByteStream): + """可检查是否被 hook 提前消费的 SSE。""" + + def __init__(self): + self.reads = 0 + self.closed = False + + async def __aiter__(self): + self.reads += 1 + yield b'data: {"model":"gpt-5.5","choices":[{"delta":{"content":"ok"},"finish_reason":null}]}\n\n' + yield b'data: {"choices":[],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}\n\ndata: [DONE]\n\n' + + async def aclose(self): + self.closed = True + + +async def test_success_sse_is_not_preconsumed_and_has_no_raw_snapshot(): + capture = _capture(stream=True) + stream = _SSE() + + def handler(request): + return httpx.Response(200, stream=stream) + + def factory(source): + client = capture.client_factory(source) + + async def after_hook(response): + assert stream.reads == 0 + + client.event_hooks["response"].append(after_hook) + client._transport = httpx.MockTransport(handler) + return client + + real = OpenAICompatTransport(client_factory=factory) + try: + with capture.round_context(session_id="s", parent_call_id="p"): + result = await _complete(ObservedTransport(real, capture), stream=True) + assert result.content == "ok" and stream.reads == 1 and stream.closed + raw = capture.raw_identity(session_id="s", parent_call_id="p", call_id="a") + assert raw == (False, None) + assert ( + assess_model_identity( + requested="gpt-5.5", + aliases=frozenset(), + reported=None, + raw_identity=raw, + request_valid=True, + ).status + == "FAIL" + ) + finally: + await real.aclose() + + +async def test_concurrent_rounds_retry_and_cancellation_reset_context(): + capture = _capture() + reached = asyncio.Event() + + async def handler(request): + await asyncio.sleep(0) + return httpx.Response(200, json=_response()) + + real = _real_transport(capture, handler) + observed = ObservedTransport(real, capture) + + async def one(parent): + with capture.round_context(session_id="s", parent_call_id=parent): + # 零 HTTP 的前次失败与成功 call_id 不得混淆。 + with pytest.raises(ValueError), capture.attempt_context(parent + "-failed"): + raise ValueError("safe") + await _complete(observed, call_id=parent + "-ok") + return capture.raw_identity(session_id="s", parent_call_id=parent, call_id=parent + "-ok") + + async def cancelled(): + with ( + capture.round_context(session_id="s", parent_call_id="cancel"), + capture.attempt_context("cancelled"), + ): + reached.set() + await asyncio.Future() + + try: + assert await asyncio.gather(one("p1"), one("p2")) == [(True, "gpt-5.5")] * 2 + for parent in ("p1", "p2"): + attempts = capture.attempts(session_id="s", parent_call_id=parent) + assert len(attempts) == 2 and len(attempts[0].http) == 0 and len(attempts[1].http) == 1 + with pytest.raises(ValueError, match="跨轮"): + capture.raw_identity(session_id="s", parent_call_id="p1", call_id="p2-ok") + task = asyncio.create_task(cancelled()) + await reached.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert len(capture.attempts(session_id="s", parent_call_id="cancel")) == 1 + with pytest.raises(LookupError): + capture._round.get() + with pytest.raises(LookupError): + capture._attempt.get() + with ( + capture.round_context(session_id="s", parent_call_id="new"), + pytest.raises(ValueError, match="重复"), + capture.attempt_context("p1-ok"), + ): + pytest.fail("不应进入") + finally: + await real.aclose() + + +def test_delegator_signatures_match_ports(): + assert inspect.signature(ObservedTransport.complete) == inspect.signature(Transport.complete) + assert inspect.signature(ObservedTransport.embed) == inspect.signature(EmbeddingTransport.embed) + + +def test_missing_expectations_fail_before_network(): + with pytest.raises(ValueError): + LiveCapture(expectations={"s": {"model": "m"}}) + + +def test_round_reports_preserve_prior_round_and_redact_sentinels(tmp_path): + error, attempts = _failure( + body=(f'{{"error":{{"type":"model_not_found","message":"{_SECRET} {_PROMPT}"}}}}').encode() + ) + error.args = (f"{_SECRET} {_PROMPT}",) + first = write_live_round( + tmp_path, + run_id="run", + matrix_id="matrix", + round_index=1, + safe_fields={"status": "PASS", "attempts": safe_attempts(attempts)}, + ) + second = write_live_round( + tmp_path, + run_id="run", + matrix_id="matrix", + round_index=2, + safe_fields={"status": "FAIL", "attempts": safe_attempts(attempts)}, + ) + assert first != second and first.exists() + for path in tmp_path.rglob("*.md"): + text = path.read_text() + assert _SECRET not in text and _PROMPT not in text + assert '"status": "PASS"' in first.read_text() + assert '"status": "FAIL"' in second.read_text() + + +@pytest.mark.parametrize( + "fields", + [ + {"raw": _BODY}, + {"error_type": ValueError("secret")}, + {"attempts": [AttemptEvidence("a", (), None)]}, + {"attempts": [{"call_id": "a", "error_type": None, "http": [{"Authorization": "secret"}]}]}, + ], +) +def test_report_refuses_raw_objects_and_unknown_fields(tmp_path, fields): + with pytest.raises((ValueError, TypeError)): + write_live_round( + tmp_path, run_id="run", matrix_id="matrix", round_index=1, safe_fields=fields + ) + + +def test_report_write_failure_is_not_skip(tmp_path): + blocked = tmp_path / "file" + blocked.write_text("not a directory") + with pytest.raises(OSError): + write_live_round( + blocked, + run_id="run", + matrix_id="matrix", + round_index=1, + safe_fields={"status": "UNCOVERED"}, + ) + + +@pytest.mark.parametrize( + "body", [b"", _BODY, _BODY + b" " * (65536 - len(_BODY)), _BODY + b" " * 65536] +) +async def test_error_snapshot_keeps_only_complete_bounded_body(body): + """真实 hook 的边界，不把生产摘要当完整错误 JSON。""" + capture = _capture() + real = _real_transport(capture, lambda request: httpx.Response(404, content=body)) + try: + with ( + capture.round_context(session_id="s", parent_call_id="p"), + pytest.raises(RequestRejectedError) as caught, + ): + await _complete(ObservedTransport(real, capture)) + attempts = capture.attempts(session_id="s", parent_call_id="p") + assert attempts[0].http[0].error_body == (body if len(body) <= 65536 else None) + expected = "UNCOVERED" if 0 < len(body) <= 65536 else "FAIL" + assert classify_live_failure(caught.value, attempts).status == expected + finally: + await real.aclose() + + +async def test_multiple_success_http_candidates_fail_identity_accessor(): + """一个 attempt 多 HTTP 不能猜最后一条。""" + capture = _capture() + client = capture.client_factory(_source()) + client._transport = httpx.MockTransport(lambda request: httpx.Response(200, json=_response())) + try: + with ( + capture.round_context(session_id="s", parent_call_id="p"), + capture.attempt_context("a"), + ): + for _ in range(2): + await client.post( + "https://example.test/v1/chat/completions", + json={"model": "gpt-5.5", "stream": False, "messages": _MESSAGES}, + ) + with pytest.raises(ValueError, match="多个成功"): + capture.raw_identity(session_id="s", parent_call_id="p", call_id="a") + finally: + await client.aclose() + + +@pytest.mark.parametrize( + "body", [b"data: not-json\n\n", b'data: {"choices":[]}\n\ndata: [DONE]\n\n'] +) +async def test_sse_parser_failure_is_not_external_uncovered(body): + capture = _capture(stream=True) + real = _real_transport(capture, lambda request: httpx.Response(200, content=body)) + try: + with ( + capture.round_context(session_id="s", parent_call_id="p"), + pytest.raises(TransientError) as caught, + ): + await _complete(ObservedTransport(real, capture), stream=True) + assert ( + classify_live_failure( + caught.value, capture.attempts(session_id="s", parent_call_id="p") + ).status + == "FAIL" + ) + finally: + await real.aclose() + + +async def test_round_consumer_keeps_first_success_when_second_assertion_fails(tmp_path): + """真实 GatewayClient 与报告出口连通；断言异常不得漏报告。""" + from polygateway import GatewaySettings + from tests.e2e.conftest import captured_chat_round, observed_client + from tests.unit.test_config import _BASE_ENV + + settings = GatewaySettings.from_env("LLM", env=_BASE_ENV) + settings = replace(settings, sources=(_source(),)) + capture = _capture() + original_factory = capture.client_factory + clients = [] + + def factory(source): + client = original_factory(source) + client._transport = httpx.MockTransport( + lambda request: httpx.Response(200, json=_response()) + ) + clients.append(client) + return client + + capture.client_factory = factory + + def fail(response): + raise AssertionError(_SECRET + _PROMPT) + + async with observed_client(settings, capture) as client: + for index, validate in ((1, None), (2, fail)): + _, verdict = await captured_chat_round( + client, + capture, + run_id="run", + matrix_id="matrix", + round_index=index, + output_dir=tmp_path, + messages=_MESSAGES, + models={"source": "gpt-5.5"}, + aliases={}, + stream=False, + validate=validate, + ) + assert verdict.status == ("PASS" if index == 1 else "FAIL") + paths = list(tmp_path.rglob("*.md")) + assert len(paths) == 2 + text = "".join(path.read_text() for path in paths) + assert '"status": "PASS"' in text and '"status": "FAIL"' in text + assert _SECRET not in text and _PROMPT not in text + assert clients and all(client.is_closed for client in clients) + + +def test_partial_uncovered_and_failed_rounds_never_become_model_pass(): + from tests.live_evidence import combine_live_verdicts, qualify_live_rounds + + passed = LiveVerdict("PASS", "safe") + uncovered = LiveVerdict("UNCOVERED", "safe") + failed = LiveVerdict("FAIL", "safe") + assert combine_live_verdicts([passed, uncovered]).status == "UNCOVERED" + assert combine_live_verdicts([passed, uncovered, failed]).status == "FAIL" + assert qualify_live_rounds([passed, passed], planned_rounds=3).status == "FAIL" + assert qualify_live_rounds([passed, failed, passed], planned_rounds=3).status == "FAIL" + + +async def test_real_retry_success_call_id_selects_exact_raw_identity(tmp_path): + """真实 RetryMW 先503再成功，两个并发 parent 的身份不串线。""" + from polygateway import GatewaySettings + from tests.e2e.conftest import captured_chat_round, observed_client + from tests.unit.test_config import _BASE_ENV + + settings = GatewaySettings.from_env("LLM", env=_BASE_ENV) + settings = replace( + settings, + sources=(_source(),), + retry=replace(settings.retry, backoff_base_s=0.001, backoff_max_s=0.002), + ) + capture = _capture() + original_factory = capture.client_factory + counts = {} + + async def handler(request): + key = capture._round.get() + counts[key] = counts.get(key, 0) + 1 + await asyncio.sleep(0) + if counts[key] == 1: + return httpx.Response(503, json={"error": {"type": "temporary"}, "model": "wrong"}) + return httpx.Response(200, json=_response()) + + def factory(source): + client = original_factory(source) + client._transport = httpx.MockTransport(handler) + return client + + capture.client_factory = factory + async with observed_client(settings, capture) as client: + + async def one(index): + return await captured_chat_round( + client, + capture, + run_id="retry", + matrix_id="retry", + round_index=index, + output_dir=tmp_path, + messages=_MESSAGES, + models={"source": "gpt-5.5"}, + aliases={}, + stream=False, + ) + + results = await asyncio.gather(one(1), one(2)) + assert all(verdict.status == "PASS" for _, verdict in results) + assert len(counts) == 2 and set(counts.values()) == {2} + for key in counts: + attempts = capture.attempts(session_id=key[0], parent_call_id=key[1]) + assert len(attempts) == 2 + assert attempts[0].http[0].raw_identity == (False, None) + assert capture.raw_identity( + session_id=key[0], parent_call_id=key[1], call_id=attempts[1].call_id + ) == (True, "gpt-5.5") + + +def test_live_model_provider_mapping_covers_default_capabilities_without_env_import(): + """L8 映射装配守卫离线化，只读取显式字面矩阵，不执行 .env 模块。""" + import ast + from pathlib import Path + + from polygateway.thinking import DEFAULT_CAPABILITIES + + tree = ast.parse((Path(__file__).parents[1] / "e2e/test_thinking_live.py").read_text()) + mapping = next( + ast.literal_eval(node.value) + for node in tree.body + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "_MODEL_PROVIDER" + for target in node.targets + ) + ) + assert not set(DEFAULT_CAPABILITIES) - mapping.keys() + + +async def test_unbuffered_error_is_insufficient_evidence(): + """response hook 不预读错误流；未完成缓冲不能借片段归因。""" + capture = _capture() + stream = _SSE() + client = capture.client_factory(_source()) + client._transport = httpx.MockTransport(lambda request: httpx.Response(404, stream=stream)) + error = RequestRejectedError("safe", status_code=404) + try: + with ( + pytest.raises(RequestRejectedError), + capture.round_context(session_id="s", parent_call_id="p"), + capture.attempt_context("a"), + ): + async with client.stream( + "POST", + "https://example.test/v1/chat/completions", + json={"model": "gpt-5.5", "messages": _MESSAGES, "stream": False}, + ): + raise error + attempts = capture.attempts(session_id="s", parent_call_id="p") + assert attempts[0].http[0].error_body is None and stream.reads == 0 + assert classify_live_failure(error, attempts).status == "FAIL" + finally: + await client.aclose() + + +def test_pytest_report_hook_preserves_report_and_uses_safe_fallback(tmp_path, monkeypatch): + """pytest wrapper generator 的 return 值是协议必需，不可按错误 LSP 建议删除。""" + from types import SimpleNamespace + + from tests.e2e.conftest import pytest_runtest_makereport + + monkeypatch.chdir(tmp_path) + report = SimpleNamespace(skipped=False, failed=True, when="call") + hook = pytest_runtest_makereport(SimpleNamespace(nodeid="safe-node"), None) + assert next(hook) is None + with pytest.raises(StopIteration) as finished: + hook.send(report) + assert finished.value.value is report + paths = list(tmp_path.rglob("*.md")) + assert len(paths) == 1 and '"status": "FAIL"' in paths[0].read_text()