test: apply evidence-based live checks without hiding regressions
This commit is contained in:
@@ -0,0 +1,496 @@
|
||||
"""测试侧独立 HTTP 取证装配；无环境自读取或成功 SSE 预读。"""
|
||||
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from contextlib import AsyncExitStack, asynccontextmanager, contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from polygateway import GatewayClient, GatewaySettings
|
||||
from polygateway.client import (
|
||||
_aclose_component,
|
||||
_build_breaker,
|
||||
_build_limiter,
|
||||
_build_selector,
|
||||
_build_structured,
|
||||
)
|
||||
from polygateway.providers import get_provider
|
||||
from polygateway.transports.openai_compat import OpenAICompatTransport
|
||||
from polygateway.types import Effort, EmbeddingTransportResult, SourceConfig, TransportResult
|
||||
from tests.live_evidence import (
|
||||
AttemptEvidence,
|
||||
HttpEvidence,
|
||||
LiveVerdict,
|
||||
assess_model_identity,
|
||||
classify_live_failure,
|
||||
messages_digest,
|
||||
request_is_valid,
|
||||
safe_attempts,
|
||||
strict_json,
|
||||
write_live_round,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Exchange:
|
||||
"""仅在 attempt 生命周期持有原始响应引用。"""
|
||||
|
||||
request: httpx.Request
|
||||
checks: tuple[tuple[str, bool], ...]
|
||||
response: httpx.Response | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Attempt:
|
||||
"""任务内可变收集器，结束时转换为冻结快照。"""
|
||||
|
||||
call_id: str
|
||||
exchanges: list[_Exchange] = field(default_factory=list)
|
||||
|
||||
|
||||
class LiveCapture:
|
||||
"""矩阵显式预期与按逻辑轮次关联的独立证据。"""
|
||||
|
||||
def __init__(self, *, expectations: Mapping[str, Mapping[str, Any]]) -> None:
|
||||
"""预期缺项即配置错误，不从实发 payload 补齐。"""
|
||||
for expected in expectations.values():
|
||||
required = {"model", "origin", "path", "control", "messages_digest"}
|
||||
if not required <= expected.keys() or ("stream" in expected) == (
|
||||
"input_shape" in expected
|
||||
):
|
||||
raise ValueError("取证矩阵缺少必需预期或混用 chat/embed")
|
||||
if not isinstance(expected["control"], dict):
|
||||
raise ValueError("control 必须是显式对象")
|
||||
self._expectations = {name: dict(value) for name, value in expectations.items()}
|
||||
self._round: ContextVar[tuple[str, str]] = ContextVar("live_round")
|
||||
self._attempt: ContextVar[_Attempt] = ContextVar("live_attempt")
|
||||
self._records: dict[tuple[str, str], list[AttemptEvidence]] = {}
|
||||
self._owners: dict[str, tuple[str, str]] = {}
|
||||
self._notes: dict[tuple[str, str], list[str]] = {}
|
||||
|
||||
@contextmanager
|
||||
def round_context(self, *, session_id: str, parent_call_id: str) -> Iterator[None]:
|
||||
"""外围逻辑轮次绑定，异常和取消均复位。"""
|
||||
key = session_id, parent_call_id
|
||||
token = self._round.set(key)
|
||||
self._records.setdefault(key, [])
|
||||
self._notes.setdefault(key, [])
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._round.reset(token)
|
||||
|
||||
@contextmanager
|
||||
def attempt_context(self, call_id: str) -> Iterator[None]:
|
||||
"""零 HTTP 尝试也有快照；重复／跨轮 UUID 是契约错误。"""
|
||||
key = self._round.get()
|
||||
if call_id in self._owners:
|
||||
raise ValueError("重复或跨轮 call_id")
|
||||
self._owners[call_id] = key
|
||||
attempt = _Attempt(call_id)
|
||||
token = self._attempt.set(attempt)
|
||||
error = None
|
||||
try:
|
||||
yield
|
||||
except Exception as exc:
|
||||
error = exc
|
||||
raise
|
||||
finally:
|
||||
try:
|
||||
events = tuple(
|
||||
self._snapshot(exchange, call_id, key) for exchange in attempt.exchanges
|
||||
)
|
||||
self._records[key].append(AttemptEvidence(call_id, events, error))
|
||||
finally:
|
||||
self._attempt.reset(token)
|
||||
|
||||
def _snapshot(self, exchange: _Exchange, call_id: str, key: tuple[str, str]) -> HttpEvidence:
|
||||
"""complete 结束后只读已缓冲内容，拒绝截断和歧义身份。"""
|
||||
response = exchange.response
|
||||
identity: tuple[bool, str | None] = (False, None)
|
||||
body = None
|
||||
status = response.status_code if response is not None else 0
|
||||
if response is None:
|
||||
self._notes[key].append("无可配对响应")
|
||||
else:
|
||||
try:
|
||||
content = response.content
|
||||
except httpx.ResponseNotRead:
|
||||
self._notes[key].append("响应未缓冲，证据不足")
|
||||
else:
|
||||
if status >= 400:
|
||||
if len(content) <= 65536:
|
||||
body = content
|
||||
else:
|
||||
self._notes[key].append("错误体超过 64 KiB，不接受截断证据")
|
||||
elif 200 <= status < 300 and dict(exchange.checks).get("stream") is not None:
|
||||
try:
|
||||
payload = strict_json(exchange.request.content)
|
||||
except (ValueError, UnicodeError):
|
||||
payload = {}
|
||||
if isinstance(payload, dict) and payload.get("stream") is False:
|
||||
try:
|
||||
data = strict_json(content)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("原始 JSON 非对象")
|
||||
model = data.get("model")
|
||||
if model is not None and not isinstance(model, str):
|
||||
raise ValueError("原始 model 类型非法")
|
||||
identity = (True, model)
|
||||
except (ValueError, UnicodeError):
|
||||
self._notes[key].append("原始 JSON 身份无法独立解析")
|
||||
return HttpEvidence(call_id, exchange.checks, status, body, identity)
|
||||
|
||||
def client_factory(self, source: SourceConfig) -> httpx.AsyncClient:
|
||||
"""鉴权仅内存比较；沿已校验源 timeout/trust_env。"""
|
||||
expected = self._expectations[source.name]
|
||||
|
||||
async def request_hook(request: httpx.Request) -> None:
|
||||
"""校验实发请求而不修正它。"""
|
||||
attempt = self._attempt.get()
|
||||
try:
|
||||
payload = strict_json(request.content)
|
||||
except (ValueError, UnicodeError):
|
||||
payload = {}
|
||||
if not isinstance(payload, dict):
|
||||
payload = {}
|
||||
url = request.url
|
||||
origin = str(
|
||||
url.copy_with(path="", query=None, fragment=None, username=None, password=None)
|
||||
).rstrip("/")
|
||||
# control 是本轮完整附加字段；基础键以外均比较，漏／多键都失败。
|
||||
basic = {"model", "messages", "stream", "stream_options", "input"}
|
||||
control = {k: v for k, v in payload.items() if k not in basic}
|
||||
checks = {
|
||||
"method": request.method == "POST",
|
||||
"origin": origin == expected["origin"]
|
||||
and not url.username
|
||||
and not url.password
|
||||
and not url.query,
|
||||
"path": url.path == expected["path"],
|
||||
"model": payload.get("model") == expected["model"],
|
||||
"authorization": request.headers.get("Authorization") == f"Bearer {source.api_key}",
|
||||
"control": control == expected["control"],
|
||||
"messages_digest": messages_digest(payload.get("messages", payload.get("input")))
|
||||
== expected["messages_digest"],
|
||||
}
|
||||
if "stream" in expected:
|
||||
checks["stream"] = payload.get("stream") is expected["stream"] and (
|
||||
payload.get("stream_options") == {"include_usage": True}
|
||||
if expected["stream"]
|
||||
else "stream_options" not in payload
|
||||
)
|
||||
else:
|
||||
texts = payload.get("input")
|
||||
checks["input_shape"] = (
|
||||
isinstance(texts, list)
|
||||
and all(isinstance(text, str) for text in texts)
|
||||
and len(texts) == expected["input_shape"]
|
||||
)
|
||||
attempt.exchanges.append(_Exchange(request, tuple(checks.items())))
|
||||
|
||||
async def response_hook(response: httpx.Response) -> None:
|
||||
"""只持有引用，绝不提前读取成功 SSE。"""
|
||||
attempt = self._attempt.get()
|
||||
matching = [event for event in attempt.exchanges if event.request is response.request]
|
||||
if len(matching) != 1 or matching[0].response is not None:
|
||||
raise ValueError("响应无法唯一配对")
|
||||
matching[0].response = response
|
||||
|
||||
return httpx.AsyncClient(
|
||||
headers={"Authorization": f"Bearer {source.api_key}"},
|
||||
timeout=source.timeout_s,
|
||||
trust_env=source.trust_env,
|
||||
event_hooks={"request": [request_hook], "response": [response_hook]},
|
||||
)
|
||||
|
||||
def attempts(self, *, session_id: str, parent_call_id: str) -> tuple[AttemptEvidence, ...]:
|
||||
"""按逻辑轮次返回不可变快照。"""
|
||||
return tuple(self._records.get((session_id, parent_call_id), ()))
|
||||
|
||||
def notes(self, *, session_id: str, parent_call_id: str) -> tuple[str, ...]:
|
||||
"""只含固定安全原因，不包含响应正文。"""
|
||||
return tuple(self._notes.get((session_id, parent_call_id), ()))
|
||||
|
||||
def raw_identity(
|
||||
self, *, session_id: str, parent_call_id: str, call_id: str
|
||||
) -> tuple[bool, str | None]:
|
||||
"""精确取最终成功 attempt，不猜本轮最后一条响应。"""
|
||||
key = session_id, parent_call_id
|
||||
if call_id in self._owners and self._owners[call_id] != key:
|
||||
raise ValueError("跨轮 call_id 身份查询")
|
||||
matches = [attempt for attempt in self._records.get(key, []) if attempt.call_id == call_id]
|
||||
if len(matches) > 1:
|
||||
raise ValueError("重复 call_id 身份查询")
|
||||
if not matches:
|
||||
return False, None
|
||||
events = [event for event in matches[0].http if 200 <= event.status_code < 300]
|
||||
if len(events) > 1:
|
||||
raise ValueError("多个成功 HTTP 身份候选")
|
||||
return events[0].raw_identity if events else (False, None)
|
||||
|
||||
|
||||
class ObservedTransport:
|
||||
"""原样委托同一个真实 transport，无重试、payload 修正或异常翻译。"""
|
||||
|
||||
def __init__(self, transport: OpenAICompatTransport, capture: LiveCapture) -> None:
|
||||
"""资源所有权留给装配者。"""
|
||||
self._transport = transport
|
||||
self._capture = capture
|
||||
|
||||
async def complete(
|
||||
self,
|
||||
*,
|
||||
messages: list[dict[str, Any]],
|
||||
source: SourceConfig,
|
||||
stream: bool,
|
||||
overlay: dict[str, Any],
|
||||
call_id: str,
|
||||
reasoning_effort: Effort | None,
|
||||
) -> TransportResult:
|
||||
"""与生产端口逐参数同签名。"""
|
||||
with self._capture.attempt_context(call_id):
|
||||
return await self._transport.complete(
|
||||
messages=messages,
|
||||
source=source,
|
||||
stream=stream,
|
||||
overlay=overlay,
|
||||
call_id=call_id,
|
||||
reasoning_effort=reasoning_effort,
|
||||
)
|
||||
|
||||
async def embed(
|
||||
self, *, texts: list[str], source: SourceConfig, call_id: str
|
||||
) -> EmbeddingTransportResult:
|
||||
"""embedding 使用同一取证关联，不套 chat 推理判据。"""
|
||||
with self._capture.attempt_context(call_id):
|
||||
return await self._transport.embed(texts=texts, source=source, call_id=call_id)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def observed_client(
|
||||
settings: GatewaySettings, capture: LiveCapture, *, capabilities=None
|
||||
) -> AsyncIterator[GatewayClient]:
|
||||
"""全量注入复用生产装配函数；自建组件显式关闭，不启用响应缓存。"""
|
||||
async with AsyncExitStack() as stack:
|
||||
sources = list(settings.sources)
|
||||
limiter = _build_limiter(settings, sources)
|
||||
stack.push_async_callback(_aclose_component, limiter)
|
||||
breaker = _build_breaker(settings)
|
||||
stack.push_async_callback(_aclose_component, breaker)
|
||||
real = OpenAICompatTransport(
|
||||
client_factory=capture.client_factory, capabilities=capabilities
|
||||
)
|
||||
stack.push_async_callback(real.aclose)
|
||||
strategy, escalation = _build_structured(
|
||||
[get_provider(source.provider) for source in sources]
|
||||
)
|
||||
client = GatewayClient(
|
||||
scope=settings.scope,
|
||||
sources=sources,
|
||||
selector=_build_selector(settings.selector),
|
||||
limiter=limiter,
|
||||
breaker=breaker,
|
||||
transport=ObservedTransport(real, capture),
|
||||
retry=settings.retry,
|
||||
backpressure=settings.backpressure,
|
||||
quota_full=settings.quota_full,
|
||||
circuit_open=settings.circuit_open,
|
||||
structured_strategy=strategy,
|
||||
structured_escalation=escalation,
|
||||
structured_max_retries=settings.structured_max_retries,
|
||||
)
|
||||
stack.push_async_callback(client.aclose)
|
||||
yield client
|
||||
|
||||
|
||||
def chat_expectations(
|
||||
settings: GatewaySettings,
|
||||
*,
|
||||
messages: list[dict[str, Any]],
|
||||
stream: bool,
|
||||
controls: Mapping[str, dict[str, Any]],
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""URL 从源配置声明，控制片段必须由矩阵独立给出。"""
|
||||
result = {}
|
||||
for source in settings.sources:
|
||||
url = httpx.URL(source.base_url)
|
||||
result[source.name] = {
|
||||
"model": source.model,
|
||||
"origin": str(
|
||||
url.copy_with(path="", query=None, fragment=None, username=None, password=None)
|
||||
).rstrip("/"),
|
||||
"path": url.path.rstrip("/") + "/chat/completions",
|
||||
"stream": stream,
|
||||
"control": controls[source.name],
|
||||
"messages_digest": messages_digest(messages),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def enforce_verdict(verdict: LiveVerdict) -> None:
|
||||
"""仅在报告已写入后调用；默认失败，不打印上游异常正文。"""
|
||||
if verdict.status == "UNCOVERED":
|
||||
pytest.skip(verdict.reason)
|
||||
assert verdict.status == "PASS", verdict.reason
|
||||
|
||||
|
||||
async def captured_chat_round(
|
||||
client: GatewayClient,
|
||||
capture: LiveCapture,
|
||||
*,
|
||||
run_id: str,
|
||||
matrix_id: str,
|
||||
round_index: int,
|
||||
output_dir: Path,
|
||||
messages: list[dict[str, Any]],
|
||||
models: Mapping[str, str],
|
||||
aliases: Mapping[str, frozenset[str]],
|
||||
validate=None,
|
||||
providers: Mapping[str, str] | None = None,
|
||||
source_efforts: Mapping[str, Effort | None] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> tuple[Any, LiveVerdict]:
|
||||
"""请求、身份与行为断言均先记逐轮证据；异常不漏轮。"""
|
||||
parent = uuid4().hex
|
||||
response = None
|
||||
error = None
|
||||
verdict = LiveVerdict("FAIL", "轮次未完成")
|
||||
with capture.round_context(session_id=run_id, parent_call_id=parent):
|
||||
try:
|
||||
response = await client.chat(
|
||||
messages, session_id=run_id, parent_call_id=parent, **kwargs
|
||||
)
|
||||
attempts = capture.attempts(session_id=run_id, parent_call_id=parent)
|
||||
events = [event for attempt in attempts for event in attempt.http]
|
||||
successful = [attempt for attempt in attempts if attempt.call_id == response.call_id]
|
||||
success_paired = (
|
||||
len(successful) == 1
|
||||
and successful[0].error is None
|
||||
and len(successful[0].http) == 1
|
||||
and 200 <= successful[0].http[0].status_code < 300
|
||||
)
|
||||
verdict = assess_model_identity(
|
||||
requested=models[response.source_name],
|
||||
aliases=aliases.get(models[response.source_name], frozenset()),
|
||||
reported=response.model_reported,
|
||||
raw_identity=capture.raw_identity(
|
||||
session_id=run_id, parent_call_id=parent, call_id=response.call_id
|
||||
),
|
||||
request_valid=success_paired
|
||||
and bool(events)
|
||||
and all(request_is_valid(event) for event in events),
|
||||
)
|
||||
if verdict.status == "PASS" and validate is not None:
|
||||
validate(response)
|
||||
except Exception as exc:
|
||||
error = exc
|
||||
verdict = classify_live_failure(
|
||||
exc, capture.attempts(session_id=run_id, parent_call_id=parent)
|
||||
)
|
||||
finally:
|
||||
attempts = capture.attempts(session_id=run_id, parent_call_id=parent)
|
||||
# 上游 model 可能回显提示词；仅输出允许集合内的名字，其他统一省略。
|
||||
allowed = set(models.values()) | {
|
||||
alias for values in aliases.values() for alias in values
|
||||
}
|
||||
write_live_round(
|
||||
output_dir,
|
||||
run_id=run_id,
|
||||
matrix_id=matrix_id,
|
||||
round_index=round_index,
|
||||
safe_fields={
|
||||
"session_id": run_id,
|
||||
"parent_call_id": parent,
|
||||
"requested_model": list(models.values()),
|
||||
"provider": list(providers.values()) if providers is not None else None,
|
||||
"attempts": safe_attempts(attempts),
|
||||
"status": verdict.status,
|
||||
"reason": verdict.reason,
|
||||
"stream": kwargs.get("stream", True),
|
||||
"requested_effort": kwargs.get("reasoning_effort")
|
||||
or (list(source_efforts.values()) if source_efforts is not None else None),
|
||||
"completed_rounds": 1,
|
||||
"prompt_tokens": response.prompt_tokens if response else None,
|
||||
"completion_tokens": response.completion_tokens if response else None,
|
||||
"reasoning_tokens": response.reasoning_tokens if response else None,
|
||||
"thinking_chars": len(response.thinking) if response else None,
|
||||
"evidence_notes": (
|
||||
"原始异常／正文摘要省略以避免回显泄露",
|
||||
*capture.notes(session_id=run_id, parent_call_id=parent),
|
||||
),
|
||||
"reported_model": response.model_reported
|
||||
if response and response.model_reported in allowed
|
||||
else None,
|
||||
"applied_effort": response.applied_effort if response else None,
|
||||
"thinking_observation": response.thinking_observation if response else None,
|
||||
"error_type": type(error).__name__ if error else None,
|
||||
"error_status": getattr(error, "status_code", None),
|
||||
},
|
||||
)
|
||||
return response, verdict
|
||||
|
||||
|
||||
def declared_control(provider: str, effort: Effort | None) -> dict[str, Any]:
|
||||
"""测试矩阵的独立 wire 声明；不调用 resolver 或生产 payload 构造器。"""
|
||||
if effort is None:
|
||||
return {}
|
||||
if provider == "qwen":
|
||||
if effort not in (Effort.AUTO, Effort.NONE):
|
||||
raise ValueError("测试矩阵未声明 qwen 强度映射")
|
||||
return {"enable_thinking": effort is not Effort.NONE}
|
||||
if provider in {"deepseek", "zhipu", "moonshot"}:
|
||||
result: dict[str, Any] = {
|
||||
"thinking": {"type": "disabled" if effort is Effort.NONE else "enabled"}
|
||||
}
|
||||
if effort not in (Effort.AUTO, Effort.NONE):
|
||||
result["reasoning_effort"] = effort.value
|
||||
return result
|
||||
if provider not in {"minimax", "openai", "anthropic", "google"}:
|
||||
raise ValueError("测试矩阵没有该 provider 的控制声明")
|
||||
return {} if effort is Effort.AUTO else {"reasoning_effort": effort.value}
|
||||
|
||||
|
||||
def source_controls(settings: GatewaySettings) -> dict[str, dict[str, Any]]:
|
||||
"""源级矩阵预期独立表达；受管 raw 冲突由生产路径拒绝。"""
|
||||
result = {}
|
||||
for source in settings.sources:
|
||||
effort = source.reasoning_effort
|
||||
if effort is None and source.enable_thinking is not None:
|
||||
effort = Effort.AUTO if source.enable_thinking else Effort.NONE
|
||||
control = declared_control(source.provider, effort)
|
||||
if effort is None:
|
||||
control.update(source.extra_body)
|
||||
else:
|
||||
# 不用 update 覆盖控制声明，否则会掩盖所有权回归。
|
||||
for key, value in source.extra_body.items():
|
||||
if key in control:
|
||||
raise ValueError("取证矩阵有双来源控制")
|
||||
control[key] = value
|
||||
result[source.name] = control
|
||||
return result
|
||||
|
||||
|
||||
# pytest 用例终态补充网络前缺配置、装配失败及未完成轮次；不代替逐轮报告。
|
||||
@pytest.hookimpl(wrapper=True)
|
||||
def pytest_runtest_makereport(item, call):
|
||||
"""仅记录安全矩阵标识和阶段结果，不序列化 pytest 异常长文本。"""
|
||||
report = yield
|
||||
if report.skipped or report.failed:
|
||||
write_live_round(
|
||||
Path("tests/outputs/134/live"),
|
||||
run_id=uuid4().hex,
|
||||
matrix_id="pytest-" + messages_digest(item.nodeid)[:16],
|
||||
round_index=0,
|
||||
safe_fields={
|
||||
"status": "UNCOVERED" if report.skipped else "FAIL",
|
||||
"reason": "用例阶段未覆盖或失败；详情按逐轮安全证据核验，不能视为能力通过",
|
||||
"evidence_notes": [report.when],
|
||||
},
|
||||
)
|
||||
return report
|
||||
@@ -1,98 +1,119 @@
|
||||
"""GovDoc 与 Video-Tree 最小接入冒烟(2026-07-20 拍板: 两个项目都做)。
|
||||
|
||||
复刻两项目的真实调用点形态,对真实网关跑一次治理调用,证明"调用点零改动
|
||||
迁移"成立;并验证 VT 现有平铺键名(LLM_TIMEOUT 等)可直接装配。
|
||||
reference/ 只读——本文件只 import 其 Protocol,绝不修改。
|
||||
"""
|
||||
"""历史接入调用形态的真实冒烟；不能替代缺失下游的现行配置验收。"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from dotenv import dotenv_values
|
||||
|
||||
from polygateway import GatewayClient
|
||||
from polygateway import Effort, GatewayClient, GatewaySettings
|
||||
from tests.e2e.conftest import (
|
||||
LiveCapture,
|
||||
captured_chat_round,
|
||||
chat_expectations,
|
||||
enforce_verdict,
|
||||
observed_client,
|
||||
source_controls,
|
||||
)
|
||||
from tests.live_evidence import write_live_round
|
||||
|
||||
_REPO = Path(__file__).resolve().parents[2]
|
||||
_ENV = {k: v for k, v in {**dotenv_values(".env"), **os.environ}.items() if v is not None}
|
||||
_HAS_SOURCE = any(k.split("__")[0] == "LLM" and k.endswith("__API_KEY") for k in _ENV)
|
||||
|
||||
# 真实网关调用: 与 test_thinking_live.py 同待遇标 slow(pytest addopts 默认排除,
|
||||
# 显式 `pytest -m slow` 运行)。理由是这些用例的成败取决于网关此刻快不快,而
|
||||
# pre-commit 关卡跑全套件——网关一抖就挡住与之无关的提交,久了会把"测试红了
|
||||
# 先怀疑网关"变成惯性,真 bug 也会被当成抖动重试掉。发版清单负责让它们真跑。
|
||||
_HAS_SOURCE = any(k.startswith("LLM__") and k.endswith("__API_KEY") for k in _ENV)
|
||||
pytestmark = [
|
||||
pytest.mark.slow,
|
||||
pytest.mark.skipif(
|
||||
not _HAS_SOURCE, reason="需真实网关凭据: 在 .env 配置 LLM__{PROVIDER}__1__*"
|
||||
),
|
||||
pytest.mark.skipif(not _HAS_SOURCE, reason="缺少矩阵必需凭据，未覆盖"),
|
||||
]
|
||||
_OUT = Path("tests/outputs/134/live")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
c = GatewayClient.from_env("LLM", env=_ENV)
|
||||
yield c
|
||||
await c.aclose()
|
||||
async def _call_shape(matrix, **kwargs):
|
||||
"""session/parent 总由逐轮 UUID 传入；保留 cache_salt 调用形态。"""
|
||||
settings = GatewaySettings.from_env("LLM", env=_ENV)
|
||||
messages = [{"role": "user", "content": "Reply with exactly: compatibility-ok"}]
|
||||
capture = LiveCapture(
|
||||
expectations=chat_expectations(
|
||||
settings, messages=messages, stream=True, controls=source_controls(settings)
|
||||
)
|
||||
)
|
||||
|
||||
def validate(response):
|
||||
assert response.content.strip() and response.call_id
|
||||
|
||||
async with observed_client(settings, capture) as client:
|
||||
_, verdict = await captured_chat_round(
|
||||
client,
|
||||
capture,
|
||||
run_id=uuid4().hex,
|
||||
matrix_id=matrix,
|
||||
round_index=1,
|
||||
output_dir=_OUT,
|
||||
messages=messages,
|
||||
models={s.name: s.model for s in settings.sources},
|
||||
providers={s.name: s.provider for s in settings.sources},
|
||||
source_efforts={
|
||||
s.name: s.reasoning_effort
|
||||
if s.reasoning_effort is not None
|
||||
else (Effort.AUTO if s.enable_thinking else Effort.NONE)
|
||||
if s.enable_thinking is not None
|
||||
else None
|
||||
for s in settings.sources
|
||||
},
|
||||
aliases={},
|
||||
validate=validate,
|
||||
**kwargs,
|
||||
)
|
||||
enforce_verdict(verdict)
|
||||
|
||||
|
||||
class TestGovDocOnboarding:
|
||||
"""GovDoc agent/loop.py:377 调用形态: session_id + parent_call_id。"""
|
||||
"""历史 session_id＋parent_call_id 调用点契约。"""
|
||||
|
||||
async def test_call_site_shape_runs_governed(self, client):
|
||||
response = await client.chat(
|
||||
[{"role": "user", "content": "Reply with exactly: govdoc-ok"}],
|
||||
session_id="govdoc-e2e",
|
||||
parent_call_id="step-1",
|
||||
)
|
||||
assert response.content.strip()
|
||||
assert response.call_id # GovernedLLMClient 契约字段全在
|
||||
async def test_call_site_shape_runs_governed(self):
|
||||
await _call_shape("compat-parent")
|
||||
|
||||
async def test_structural_protocol_match(self, client):
|
||||
async def test_structural_protocol_match(self):
|
||||
"""外部 Protocol 缺包单列未覆盖；合成契约另在 unit 跑。"""
|
||||
run_id = uuid4().hex
|
||||
sys.path.insert(0, str(_REPO / "reference/GovDoc-SaaS/packages/docagent-core/src"))
|
||||
try:
|
||||
from docagent_core.protocols import LLMProvider
|
||||
except ImportError:
|
||||
pytest.skip("GovDoc protocols 依赖不可导入(结构断言已由单测兜底覆盖)")
|
||||
write_live_round(
|
||||
_OUT,
|
||||
run_id=run_id,
|
||||
matrix_id="external-protocol",
|
||||
round_index=0,
|
||||
safe_fields={
|
||||
"status": "UNCOVERED",
|
||||
"reason": "外部 Protocol 包缺失；未验证真实下游",
|
||||
},
|
||||
)
|
||||
pytest.skip("外部 Protocol 包缺失，未覆盖")
|
||||
finally:
|
||||
sys.path.pop(0)
|
||||
status = "FAIL"
|
||||
client = None
|
||||
try:
|
||||
client = GatewayClient.from_env("LLM", env=_ENV)
|
||||
assert isinstance(client, LLMProvider)
|
||||
status = "PASS"
|
||||
finally:
|
||||
if client is not None:
|
||||
await client.aclose()
|
||||
write_live_round(
|
||||
_OUT,
|
||||
run_id=run_id,
|
||||
matrix_id="external-protocol",
|
||||
round_index=0,
|
||||
safe_fields={"status": status, "reason": "外部 Protocol 结构契约，不是模型能力"},
|
||||
)
|
||||
|
||||
|
||||
class TestVideoTreeOnboarding:
|
||||
"""VT loop.py:336 调用形态: session_id + cache_salt(跨 epoch 重采样)。"""
|
||||
"""历史 cache_salt 调用点契约，平铺键装配已移至 unit。"""
|
||||
|
||||
async def test_call_site_shape_with_cache_salt(self, client):
|
||||
response = await client.chat(
|
||||
[{"role": "user", "content": "Reply with exactly: vt-ok"}],
|
||||
session_id="vt-e2e",
|
||||
cache_salt="epoch-1",
|
||||
)
|
||||
assert response.content.strip()
|
||||
|
||||
async def test_flat_legacy_keys_assemble(self):
|
||||
"""VT 现有键名(LLM_TIMEOUT/LLM_MAX_RETRIES 等)零改名装配成功。"""
|
||||
source_keys = {k: v for k, v in _ENV.items() if k.split("__")[0] == "LLM" and "__" in k}
|
||||
flat_env = {
|
||||
**source_keys,
|
||||
# 与 .env 的 LLM__MINIMAX__1__TIMEOUT_S 同值。取 120(VT 旧值)会让本用例的
|
||||
# 超时比生产配置还紧一半,在慢网关上必然间歇红——而本用例断言的是平铺
|
||||
# 键名能否解析成 SourceConfig.timeout_s,超时取值本身不是被测对象
|
||||
"LLM_TIMEOUT": "300",
|
||||
"LLM_MAX_RETRIES": "3",
|
||||
"LLM_RETRY_BASE_DELAY": "2.0",
|
||||
"LLM_RETRY_MAX_DELAY": "30.0",
|
||||
"LLM_CIRCUIT_BREAKER_THRESHOLD": "5",
|
||||
"LLM_CIRCUIT_BREAKER_COOLDOWN": "60",
|
||||
"LLM_TTFT_TIMEOUT": "30",
|
||||
"LLM_INTER_TOKEN_TIMEOUT": "15",
|
||||
"PGW_CACHE_BACKEND": "none",
|
||||
"PGW_TELEMETRY_BACKEND": "none",
|
||||
}
|
||||
client = GatewayClient.from_env("LLM", env=flat_env)
|
||||
try:
|
||||
resp = await client.chat([{"role": "user", "content": "Reply: flat-ok"}])
|
||||
assert resp.content.strip()
|
||||
finally:
|
||||
await client.aclose()
|
||||
async def test_call_site_shape_with_cache_salt(self):
|
||||
await _call_shape("compat-salt", cache_salt="epoch-1")
|
||||
|
||||
@@ -1,89 +1,91 @@
|
||||
"""真实网关 /embeddings 端点探测(M2 设计 §11.6;人类默认口径: 实现时探测)。
|
||||
|
||||
对 .env 的 LLM 源网关发一次真实 embeddings 请求: 支持则记录向量证据,
|
||||
不支持(404/翻译为领域错误)则 skip 并把响应记录进 tests/outputs/
|
||||
(降级证据)。无 EMBED scope 配置时复用 LLM 源的 base_url/api_key。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
"""真实 embedding 探测；404 仅证明请求型号不可用，不外推端点能力。"""
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from dotenv import dotenv_values
|
||||
|
||||
from polygateway.errors import PolyGatewayError
|
||||
from polygateway import GatewaySettings
|
||||
from polygateway.transports.openai_compat import OpenAICompatTransport
|
||||
from polygateway.types import SourceConfig
|
||||
from tests.e2e.conftest import LiveCapture, ObservedTransport, enforce_verdict
|
||||
from tests.live_evidence import (
|
||||
LiveVerdict,
|
||||
classify_live_failure,
|
||||
messages_digest,
|
||||
request_is_valid,
|
||||
safe_attempts,
|
||||
write_live_round,
|
||||
)
|
||||
|
||||
_ENV = {k: v for k, v in {**dotenv_values(".env"), **os.environ}.items() if v is not None}
|
||||
|
||||
# 真实网关调用: 与 test_thinking_live.py 同待遇标 slow(pytest addopts 默认排除,
|
||||
# 显式 `pytest -m slow` 运行)。理由见 test_compat_projects.py 同处注释。
|
||||
pytestmark = [
|
||||
pytest.mark.slow,
|
||||
pytest.mark.skipif(
|
||||
"LLM__MINIMAX__1__BASE_URL" not in _ENV,
|
||||
reason="缺真实网关配置(.env)",
|
||||
),
|
||||
pytest.mark.skipif("LLM__MINIMAX__1__BASE_URL" not in _ENV, reason="缺少矩阵必需配置，未覆盖"),
|
||||
]
|
||||
|
||||
_OUT = Path("tests/outputs/embedding")
|
||||
|
||||
|
||||
def _record(name: str, lines: list[str]) -> Path:
|
||||
_OUT.mkdir(parents=True, exist_ok=True)
|
||||
path = _OUT / f"{name}_{datetime.now():%Y%m%d_%H%M%S}.md"
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
async def test_probe_real_gateway_embeddings():
|
||||
source = SourceConfig(
|
||||
name="probe_1",
|
||||
provider="minimax",
|
||||
base_url=_ENV["LLM__MINIMAX__1__BASE_URL"],
|
||||
api_key=_ENV["LLM__MINIMAX__1__API_KEY"],
|
||||
model=_ENV.get("PGW_EMBED_PROBE_MODEL", "text-embedding-v1"),
|
||||
timeout_s=30.0,
|
||||
est_tokens=8,
|
||||
"""沿已校验源的 timeout/trust_env，所有路径 finally 关闭。"""
|
||||
settings = GatewaySettings.from_env("LLM", env=_ENV)
|
||||
configured = next(s for s in settings.sources if s.name == "minimax_1")
|
||||
source = dataclasses.replace(
|
||||
configured, model=_ENV.get("PGW_EMBED_PROBE_MODEL", "text-embedding-v1")
|
||||
)
|
||||
transport = OpenAICompatTransport()
|
||||
texts = ["polygateway embedding probe"]
|
||||
url = httpx.URL(source.base_url)
|
||||
capture = LiveCapture(
|
||||
expectations={
|
||||
source.name: {
|
||||
"model": source.model,
|
||||
"origin": str(url.copy_with(path="", query=None)).rstrip("/"),
|
||||
"path": url.path.rstrip("/") + "/embeddings",
|
||||
"input_shape": 1,
|
||||
"control": {},
|
||||
"messages_digest": messages_digest(texts),
|
||||
}
|
||||
}
|
||||
)
|
||||
real = OpenAICompatTransport(client_factory=capture.client_factory)
|
||||
transport = ObservedTransport(real, capture)
|
||||
run_id, parent, call_id = uuid4().hex, uuid4().hex, uuid4().hex
|
||||
verdict = LiveVerdict("FAIL", "轮次未完成")
|
||||
try:
|
||||
result = await transport.embed(
|
||||
texts=["polygateway embedding probe"], source=source, call_id="probe"
|
||||
)
|
||||
except PolyGatewayError as exc:
|
||||
path = _record(
|
||||
"probe_unsupported",
|
||||
[
|
||||
"# Embedding 端点探测: 网关不支持",
|
||||
f"- base_url: {source.base_url}",
|
||||
f"- model: {source.model}",
|
||||
f"- 错误分类: {type(exc).__name__}",
|
||||
f"- status_code: {exc.status_code}",
|
||||
f"- 详情: {exc}",
|
||||
"",
|
||||
"结论: e2e 按设计 §11.6 降级,embedding 行为由 unit 全覆盖。",
|
||||
],
|
||||
)
|
||||
await transport.aclose()
|
||||
pytest.skip(f"网关不支持 embeddings({type(exc).__name__}),证据: {path}")
|
||||
else:
|
||||
await transport.aclose()
|
||||
with capture.round_context(session_id=run_id, parent_call_id=parent):
|
||||
try:
|
||||
result = await transport.embed(texts=texts, source=source, call_id=call_id)
|
||||
events = [
|
||||
e
|
||||
for a in capture.attempts(session_id=run_id, parent_call_id=parent)
|
||||
for e in a.http
|
||||
]
|
||||
assert len(events) == 1 and request_is_valid(events[0])
|
||||
assert result.dim > 0 and len(result.vectors) == 1
|
||||
_record(
|
||||
"probe_supported",
|
||||
[
|
||||
"# Embedding 端点探测: 网关支持",
|
||||
f"- base_url: {source.base_url}",
|
||||
f"- model: {source.model}",
|
||||
f"- dim: {result.dim}",
|
||||
f"- usage: {result.prompt_tokens}({result.usage_source})",
|
||||
f"- 向量前 5 维: {result.vectors[0][:5]}",
|
||||
f"- raw: {dataclasses.asdict(result)['raw']}",
|
||||
],
|
||||
verdict = LiveVerdict("PASS", "向量形状与实发请求合格")
|
||||
except Exception as error:
|
||||
verdict = classify_live_failure(
|
||||
error, capture.attempts(session_id=run_id, parent_call_id=parent)
|
||||
)
|
||||
finally:
|
||||
write_live_round(
|
||||
Path("tests/outputs/134/live"),
|
||||
run_id=run_id,
|
||||
matrix_id="embedding",
|
||||
round_index=1,
|
||||
safe_fields={
|
||||
"status": verdict.status,
|
||||
"reason": verdict.reason,
|
||||
"session_id": run_id,
|
||||
"parent_call_id": parent,
|
||||
"attempts": safe_attempts(
|
||||
capture.attempts(session_id=run_id, parent_call_id=parent)
|
||||
),
|
||||
"evidence_notes": capture.notes(session_id=run_id, parent_call_id=parent),
|
||||
},
|
||||
)
|
||||
finally:
|
||||
await real.aclose()
|
||||
enforce_verdict(verdict)
|
||||
|
||||
@@ -1,123 +1,110 @@
|
||||
"""真实网关端到端冒烟(M1 验收第 7 步)。
|
||||
"""真实网关冒烟：逐轮独立取证，行为断言失败也必须留档。"""
|
||||
|
||||
前置: `.env` 配置至少一个 `LLM__{PROVIDER}__1__*` 真实源 + 韧性键。
|
||||
缺配置时 skip(验收前必须真跑)。输出结构化 Markdown 落
|
||||
`tests/outputs/e2e/`(CLAUDE.md §4.6,不提交 git)。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from dotenv import dotenv_values
|
||||
from pydantic import BaseModel
|
||||
|
||||
from polygateway import GatewayClient
|
||||
from polygateway import Effort, GatewaySettings
|
||||
from tests.e2e.conftest import (
|
||||
LiveCapture,
|
||||
captured_chat_round,
|
||||
chat_expectations,
|
||||
enforce_verdict,
|
||||
observed_client,
|
||||
source_controls,
|
||||
)
|
||||
|
||||
_ENV = {k: v for k, v in {**dotenv_values(".env"), **os.environ}.items() if v is not None}
|
||||
_HAS_SOURCE = any(k.split("__")[0] == "LLM" and k.endswith("__API_KEY") for k in _ENV)
|
||||
|
||||
# 真实网关调用: 与 test_thinking_live.py 同待遇标 slow(pytest addopts 默认排除,
|
||||
# 显式 `pytest -m slow` 运行)。理由见 test_compat_projects.py 同处注释。
|
||||
_HAS_SOURCE = any(k.startswith("LLM__") and k.endswith("__API_KEY") for k in _ENV)
|
||||
pytestmark = [
|
||||
pytest.mark.slow,
|
||||
pytest.mark.skipif(
|
||||
not _HAS_SOURCE,
|
||||
reason="需真实网关凭据: 在 .env 配置 LLM__{PROVIDER}__1__*(M1 验收前必须真跑)",
|
||||
),
|
||||
pytest.mark.skipif(not _HAS_SOURCE, reason="缺少矩阵必需凭据，未覆盖"),
|
||||
]
|
||||
|
||||
_OUT_DIR = Path("tests/outputs/e2e")
|
||||
|
||||
|
||||
class MiniAnswer(BaseModel):
|
||||
"""最小结构化响应契约。"""
|
||||
|
||||
answer: int
|
||||
reason: str
|
||||
|
||||
|
||||
def _report(name: str, sections: list[tuple[str, str]]) -> Path:
|
||||
_OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
path = _OUT_DIR / f"{name}_{ts}.md"
|
||||
body = [f"# e2e 冒烟: {name}", ""]
|
||||
for title, content in sections:
|
||||
body += [f"## {title}", "", "```", content, "```", ""]
|
||||
path.write_text("\n".join(body), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
c = GatewayClient.from_env("LLM", env=_ENV)
|
||||
yield c
|
||||
await c.aclose()
|
||||
async def _smoke(matrix, prompt, validate, *, stream=True, structured=None):
|
||||
"""全量注入仅替换取证装配，仍调用生产结构化策略。"""
|
||||
settings = GatewaySettings.from_env("LLM", env=_ENV)
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
controls = source_controls(settings)
|
||||
capture = LiveCapture(
|
||||
expectations=chat_expectations(
|
||||
settings, messages=messages, stream=stream, controls=controls
|
||||
)
|
||||
)
|
||||
async with observed_client(settings, capture) as client:
|
||||
_, verdict = await captured_chat_round(
|
||||
client,
|
||||
capture,
|
||||
run_id=uuid4().hex,
|
||||
matrix_id=matrix,
|
||||
round_index=1,
|
||||
output_dir=Path("tests/outputs/134/live"),
|
||||
messages=messages,
|
||||
models={s.name: s.model for s in settings.sources},
|
||||
providers={s.name: s.provider for s in settings.sources},
|
||||
source_efforts={
|
||||
s.name: s.reasoning_effort
|
||||
if s.reasoning_effort is not None
|
||||
else (Effort.AUTO if s.enable_thinking else Effort.NONE)
|
||||
if s.enable_thinking is not None
|
||||
else None
|
||||
for s in settings.sources
|
||||
},
|
||||
aliases={},
|
||||
validate=validate,
|
||||
stream=stream,
|
||||
structured=structured,
|
||||
)
|
||||
enforce_verdict(verdict)
|
||||
|
||||
|
||||
class TestRealGatewaySmoke:
|
||||
async def test_stream_chat(self, client):
|
||||
resp = await client.chat(
|
||||
[{"role": "user", "content": "Reply with exactly: pong"}], session_id="e2e-smoke"
|
||||
)
|
||||
path = _report(
|
||||
"stream_chat",
|
||||
[
|
||||
("响应", resp.content),
|
||||
(
|
||||
"元数据",
|
||||
json.dumps(
|
||||
{
|
||||
"model": resp.model,
|
||||
"source": resp.source_name,
|
||||
"usage_source": resp.usage_source,
|
||||
"prompt_tokens": resp.prompt_tokens,
|
||||
"completion_tokens": resp.completion_tokens,
|
||||
"latency_ms": resp.latency_ms,
|
||||
"ttft_ms": resp.ttft_ms,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
assert resp.content.strip()
|
||||
assert resp.ttft_ms is not None and resp.latency_ms > 0
|
||||
print(f"输出: {path}")
|
||||
"""保留流／非流和结构化真实行为断言。"""
|
||||
|
||||
async def test_non_stream_fast_path(self, client):
|
||||
resp = await client.chat(
|
||||
[{"role": "user", "content": "Reply with exactly: pong"}], stream=False
|
||||
)
|
||||
_report("non_stream", [("响应", resp.content)])
|
||||
assert resp.content.strip() and resp.ttft_ms is None
|
||||
async def test_stream_chat(self):
|
||||
def validate(response):
|
||||
assert response.content.strip()
|
||||
assert response.ttft_ms is not None and response.latency_ms > 0
|
||||
|
||||
async def test_structured_json_tier(self, client):
|
||||
resp = await client.chat(
|
||||
[{"role": "user", "content": 'Reply ONLY with JSON: {"ok": true}'}],
|
||||
await _smoke("smoke-stream", "Reply with exactly: pong", validate)
|
||||
|
||||
async def test_non_stream_fast_path(self):
|
||||
def validate(response):
|
||||
assert response.content.strip() and response.ttft_ms is None
|
||||
|
||||
await _smoke("smoke-json", "Reply with exactly: pong", validate, stream=False)
|
||||
|
||||
async def test_structured_json_tier(self):
|
||||
def validate(response):
|
||||
assert isinstance(response.structured_data, dict | list)
|
||||
|
||||
await _smoke(
|
||||
"smoke-structured-json",
|
||||
'Reply ONLY with JSON: {"ok": true}',
|
||||
validate,
|
||||
structured="json",
|
||||
)
|
||||
_report("structured_json", [("解析产物", repr(resp.structured_data))])
|
||||
assert isinstance(resp.structured_data, dict | list)
|
||||
|
||||
async def test_structured_model_ladder(self, client):
|
||||
resp = await client.chat(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is 2+3? Reply ONLY with JSON matching "
|
||||
'{"answer": <int>, "reason": <short string>}',
|
||||
}
|
||||
],
|
||||
async def test_structured_model_ladder(self):
|
||||
def validate(response):
|
||||
assert isinstance(response.structured_data, MiniAnswer)
|
||||
assert response.structured_data.answer == 5
|
||||
|
||||
await _smoke(
|
||||
"smoke-structured-model",
|
||||
'What is 2+3? Reply ONLY with JSON matching {"answer": <int>, "reason": <short string>}',
|
||||
validate,
|
||||
structured=MiniAnswer,
|
||||
)
|
||||
_report(
|
||||
"structured_model",
|
||||
[
|
||||
("原始响应", resp.content),
|
||||
("校验产物", resp.structured_data.model_dump_json()),
|
||||
],
|
||||
)
|
||||
assert isinstance(resp.structured_data, MiniAnswer)
|
||||
assert resp.structured_data.answer == 5
|
||||
|
||||
+420
-1125
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,308 @@
|
||||
"""真实测试的有限证据判定；不读取环境、不请求网络、不记录原始正文。"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
from uuid import uuid4
|
||||
|
||||
from polygateway.errors import RequestRejectedError
|
||||
from polygateway.types import ThinkingObservation
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HttpEvidence:
|
||||
"""一次 HTTP 的内存证据，禁止直接序列化。"""
|
||||
|
||||
call_id: str
|
||||
request_checks: tuple[tuple[str, bool], ...]
|
||||
status_code: int
|
||||
error_body: bytes | None
|
||||
raw_identity: tuple[bool, str | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AttemptEvidence:
|
||||
"""一次 transport 尝试，可以没有 HTTP。"""
|
||||
|
||||
call_id: str
|
||||
http: tuple[HttpEvidence, ...]
|
||||
error: Exception | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LiveVerdict:
|
||||
"""测试命题的结论，不等同于 pytest 退出码。"""
|
||||
|
||||
status: Literal["PASS", "FAIL", "UNCOVERED"]
|
||||
reason: str
|
||||
|
||||
|
||||
def strict_json(body: bytes) -> Any:
|
||||
"""独立解析完整 UTF-8 JSON，拒绝重复键及非标准常量。"""
|
||||
|
||||
def pairs(items: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
"""重复键不允许被后值掩盖。"""
|
||||
result = {}
|
||||
for key, value in items:
|
||||
if key in result:
|
||||
raise ValueError("重复 JSON 键")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
def invalid(value: str) -> None:
|
||||
"""拒绝非标准数值常量。"""
|
||||
raise ValueError("非法 JSON 常量")
|
||||
|
||||
return json.loads(body.decode("utf-8"), object_pairs_hook=pairs, parse_constant=invalid)
|
||||
|
||||
|
||||
def messages_digest(messages: Any) -> str:
|
||||
"""只在内存比较提示词摘要，不写提示词。"""
|
||||
return hashlib.sha256(
|
||||
json.dumps(messages, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def request_is_valid(event: HttpEvidence) -> bool:
|
||||
"""完整且无重复的显式检查才能作为请求资格。"""
|
||||
checks = dict(event.request_checks)
|
||||
common = {"method", "origin", "path", "model", "authorization", "control", "messages_digest"}
|
||||
return (
|
||||
len(checks) == len(event.request_checks)
|
||||
and set(checks) in (common | {"stream"}, common | {"input_shape"})
|
||||
and all(value is True for value in checks.values())
|
||||
)
|
||||
|
||||
|
||||
def _single_error(error: Exception, attempts: Sequence[AttemptEvidence]) -> HttpEvidence | None:
|
||||
"""仅接受可与最终异常精确配对的独立单次错误。"""
|
||||
if not isinstance(error, RequestRejectedError) or len(attempts) != 1:
|
||||
return None
|
||||
attempt = attempts[0]
|
||||
if attempt.error is not error or len(attempt.http) != 1:
|
||||
return None
|
||||
event = attempt.http[0]
|
||||
if event.call_id != attempt.call_id or not request_is_valid(event):
|
||||
return None
|
||||
if event.status_code != error.status_code:
|
||||
return None
|
||||
return event
|
||||
|
||||
|
||||
def error_machine_type(event: HttpEvidence) -> str | None:
|
||||
"""完整错误体中的唯一机器字段；不检查正文子串。"""
|
||||
body = event.error_body
|
||||
if body is None or not 0 < len(body) <= 65536:
|
||||
return None
|
||||
try:
|
||||
data = strict_json(body)
|
||||
except (ValueError, UnicodeError):
|
||||
return None
|
||||
if not isinstance(data, dict) or not isinstance(data.get("error"), dict):
|
||||
return None
|
||||
value = data["error"].get("type")
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def classify_live_failure(error: Exception, attempts: Sequence[AttemptEvidence]) -> LiveVerdict:
|
||||
"""默认失败；唯一自动外因是完整证据支持的 404 model_not_found。"""
|
||||
event = _single_error(error, attempts)
|
||||
if (
|
||||
event is not None
|
||||
and event.status_code == 404
|
||||
and error_machine_type(event) == "model_not_found"
|
||||
):
|
||||
return LiveVerdict("UNCOVERED", "端点回报该请求型号不可用")
|
||||
return LiveVerdict("FAIL", "无满足窄外因契约的完整独立证据")
|
||||
|
||||
|
||||
def assess_expected_rejection(
|
||||
error: Exception, attempts: Sequence[AttemptEvidence], *, status_code: int, machine_type: str
|
||||
) -> LiveVerdict:
|
||||
"""预先声明的拒绝命题，不把一般 400 当不支持档位。"""
|
||||
event = _single_error(error, attempts)
|
||||
if (
|
||||
event is not None
|
||||
and event.status_code == status_code
|
||||
and error_machine_type(event) == machine_type
|
||||
):
|
||||
return LiveVerdict("PASS", "符合预声明的拒绝类型、状态和机器字段")
|
||||
return LiveVerdict("FAIL", "不符合预声明拒绝证据")
|
||||
|
||||
|
||||
def assess_model_identity(
|
||||
*,
|
||||
requested: str,
|
||||
aliases: frozenset[str],
|
||||
reported: str | None,
|
||||
raw_identity: tuple[bool, str | None],
|
||||
request_valid: bool,
|
||||
) -> LiveVerdict:
|
||||
"""公共身份异常只有原始独立证据才能归因上游。"""
|
||||
if not request_valid:
|
||||
return LiveVerdict("FAIL", "实发请求校验不完整或不符")
|
||||
allowed = {requested, *aliases}
|
||||
captured, raw = raw_identity
|
||||
if captured and raw != reported:
|
||||
return LiveVerdict("FAIL", "公共身份与独立原始身份不一致")
|
||||
if reported in allowed:
|
||||
return LiveVerdict("PASS", "身份合格")
|
||||
if not captured:
|
||||
return LiveVerdict("FAIL", "身份来源无法区分")
|
||||
return LiveVerdict("UNCOVERED", "独立原始响应身份缺失或不属于显式别名")
|
||||
|
||||
|
||||
def assess_thinking_coverage(
|
||||
observations: Sequence[ThinkingObservation],
|
||||
*,
|
||||
planned_rounds: int,
|
||||
proposition: Literal["enabled", "disabled", "cannot_disable"],
|
||||
) -> LiveVerdict:
|
||||
"""按独立命题裁定；缺轮不减分母，UNKNOWN 不证明关闭。"""
|
||||
if planned_rounds < 1 or len(observations) != planned_rounds:
|
||||
return LiveVerdict("FAIL", "计划轮次不完整")
|
||||
if any(not isinstance(item, ThinkingObservation) for item in observations):
|
||||
return LiveVerdict("FAIL", "观测类型不符")
|
||||
observed = observations.count(ThinkingObservation.OBSERVED)
|
||||
unknown = observations.count(ThinkingObservation.UNKNOWN)
|
||||
if proposition == "enabled":
|
||||
if observed > planned_rounds / 2:
|
||||
return LiveVerdict("PASS", "完整轮次多数观测到推理")
|
||||
return LiveVerdict("UNCOVERED" if unknown else "FAIL", "开启证据未达多数")
|
||||
if proposition == "disabled":
|
||||
if observed:
|
||||
return LiveVerdict("FAIL", "观测到推理，证伪关闭声明")
|
||||
return LiveVerdict(
|
||||
"UNCOVERED" if unknown else "PASS",
|
||||
"UNKNOWN 不证明关闭" if unknown else "每轮明确 ABSENT",
|
||||
)
|
||||
if proposition == "cannot_disable":
|
||||
if observed:
|
||||
return LiveVerdict("PASS", "本条件下仍推理；不外推所有私有参数")
|
||||
return LiveVerdict("UNCOVERED" if unknown else "FAIL", "缺少仍推理的证据")
|
||||
raise ValueError("未知测试命题")
|
||||
|
||||
|
||||
def summarize_verdicts(verdicts: Sequence[LiveVerdict], *, planned_rounds: int) -> dict[str, int]:
|
||||
"""保留失败、未覆盖与缺轮的独立计数。"""
|
||||
if planned_rounds < len(verdicts):
|
||||
raise ValueError("实际轮次超出计划")
|
||||
return {
|
||||
**{
|
||||
status: sum(v.status == status for v in verdicts)
|
||||
for status in ("PASS", "FAIL", "UNCOVERED")
|
||||
},
|
||||
"missing": planned_rounds - len(verdicts),
|
||||
}
|
||||
|
||||
|
||||
_SAFE_FIELDS = frozenset(
|
||||
{
|
||||
"provider",
|
||||
"requested_model",
|
||||
"reported_model",
|
||||
"stream",
|
||||
"requested_effort",
|
||||
"applied_effort",
|
||||
"session_id",
|
||||
"parent_call_id",
|
||||
"attempts",
|
||||
"status",
|
||||
"reason",
|
||||
"completed_rounds",
|
||||
"planned_rounds",
|
||||
"thinking_observation",
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"reasoning_tokens",
|
||||
"thinking_chars",
|
||||
"counts",
|
||||
"proposition",
|
||||
"error_type",
|
||||
"error_status",
|
||||
"evidence_notes",
|
||||
}
|
||||
)
|
||||
_ATTEMPT_FIELDS = frozenset({"call_id", "error_type", "http"})
|
||||
_HTTP_FIELDS = frozenset(
|
||||
{"status_code", "request_checks", "identity_captured", "error_body_complete"}
|
||||
)
|
||||
|
||||
|
||||
def _validate_safe(fields: Mapping[str, Any]) -> None:
|
||||
"""拒绝原始异常和证据对象，嵌套字段也有白名单。"""
|
||||
if set(fields) - _SAFE_FIELDS:
|
||||
raise ValueError("报告含非白名单字段")
|
||||
for attempt in fields.get("attempts", []):
|
||||
if not isinstance(attempt, dict) or set(attempt) != _ATTEMPT_FIELDS:
|
||||
raise ValueError("非法 attempt 报告")
|
||||
for event in attempt["http"]:
|
||||
if not isinstance(event, dict) or set(event) != _HTTP_FIELDS:
|
||||
raise ValueError("非法 HTTP 报告")
|
||||
if not isinstance(event["request_checks"], dict) or any(
|
||||
type(v) is not bool for v in event["request_checks"].values()
|
||||
):
|
||||
raise ValueError("请求校验报告只允许布尔值")
|
||||
# 不提供 default=str：原始异常、bytes、dataclass 均必须失败。
|
||||
json.dumps(fields, ensure_ascii=False, allow_nan=False)
|
||||
|
||||
|
||||
def write_live_round(
|
||||
output_dir: Path,
|
||||
*,
|
||||
run_id: str,
|
||||
matrix_id: str,
|
||||
round_index: int,
|
||||
safe_fields: Mapping[str, Any],
|
||||
) -> Path:
|
||||
"""仅接收已脱敏字段；独占文件写入失败必须冒泡。"""
|
||||
_validate_safe(safe_fields)
|
||||
if round_index < 0 or not re.fullmatch(r"[a-zA-Z0-9_-]+", run_id + matrix_id):
|
||||
raise ValueError("报告路径标识非法")
|
||||
directory = output_dir / run_id
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
path = directory / f"{matrix_id}-{round_index}-{uuid4().hex}.md"
|
||||
text = json.dumps(dict(safe_fields), ensure_ascii=False, indent=2, allow_nan=False)
|
||||
with path.open("x", encoding="utf-8") as handle:
|
||||
handle.write(f"# {matrix_id} · 轮次 {round_index}\n\n```json\n{text}\n```\n")
|
||||
return path
|
||||
|
||||
|
||||
def safe_attempts(attempts: Sequence[AttemptEvidence]) -> list[dict[str, Any]]:
|
||||
"""只导出事实布尔值、异常类和状态；不落盘任何上游正文。"""
|
||||
return [
|
||||
{
|
||||
"call_id": attempt.call_id,
|
||||
"error_type": type(attempt.error).__name__ if attempt.error else None,
|
||||
"http": [
|
||||
{
|
||||
"status_code": event.status_code,
|
||||
"request_checks": dict(event.request_checks),
|
||||
"identity_captured": event.raw_identity[0],
|
||||
"error_body_complete": event.error_body is not None,
|
||||
}
|
||||
for event in attempt.http
|
||||
],
|
||||
}
|
||||
for attempt in attempts
|
||||
]
|
||||
|
||||
|
||||
def combine_live_verdicts(verdicts: Sequence[LiveVerdict]) -> LiveVerdict:
|
||||
"""部分失败优先于未覆盖；部分未覆盖不得汇总全 PASS。"""
|
||||
if any(verdict.status == "FAIL" for verdict in verdicts):
|
||||
return LiveVerdict("FAIL", "至少一个必需命题失败")
|
||||
if not verdicts or any(verdict.status == "UNCOVERED" for verdict in verdicts):
|
||||
return LiveVerdict("UNCOVERED", "至少一个必需命题未覆盖")
|
||||
return LiveVerdict("PASS", "所有必需命题通过")
|
||||
|
||||
|
||||
def qualify_live_rounds(verdicts: Sequence[LiveVerdict], *, planned_rounds: int) -> LiveVerdict:
|
||||
"""汇总请求／身份资格，缺轮绝不缩小分母。"""
|
||||
if planned_rounds < 1 or len(verdicts) != planned_rounds:
|
||||
return LiveVerdict("FAIL", "计划轮次缺失")
|
||||
return combine_live_verdicts(verdicts)
|
||||
@@ -1340,3 +1340,25 @@ async def test_managed_conflict_returns_half_open_probe_and_permit():
|
||||
await gate.release_probe(next_entry)
|
||||
finally:
|
||||
await client._transport.aclose()
|
||||
|
||||
|
||||
async def test_synthetic_runtime_protocol_and_legacy_call_signatures():
|
||||
"""合成 Protocol 仅证明本库兼容契约，不冒充缺失下游实际验收。"""
|
||||
import inspect
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
@runtime_checkable
|
||||
class Caller(Protocol):
|
||||
"""旧调用点只依赖 chat 协议。"""
|
||||
|
||||
async def chat(self, messages, **kwargs): ...
|
||||
|
||||
client = GatewayClient.from_env("LLM", env=_ENV)
|
||||
try:
|
||||
assert isinstance(client, Caller)
|
||||
signature = inspect.signature(client.chat)
|
||||
signature.bind([], session_id="session", parent_call_id="step")
|
||||
signature.bind([], session_id="session", cache_salt="epoch-1")
|
||||
assert client._transport._clients == {}
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
@@ -997,3 +997,47 @@ class TestCrossFieldInvariants:
|
||||
)
|
||||
client = GatewayClient.from_settings(settings)
|
||||
assert client is not None
|
||||
|
||||
|
||||
async def test_flat_legacy_keys_assemble_without_source_timeout_or_network():
|
||||
"""完整合成 env 验证平铺回落，不借真实配置或 Redis/PG。"""
|
||||
env = dict(_BASE_ENV)
|
||||
del env["LLM__QWEN__1__TIMEOUT_S"]
|
||||
env.update({"LLM_TIMEOUT": "317", "LLM_TTFT_TIMEOUT": "41", "LLM_INTER_TOKEN_TIMEOUT": "19"})
|
||||
settings = GatewaySettings.from_env("LLM", env=env)
|
||||
assert settings.sources[0].timeout_s == 317
|
||||
assert settings.sources[0].ttft_timeout_s == 41
|
||||
assert settings.sources[0].inter_token_timeout_s == 19
|
||||
assert settings.retry.max_attempts == 3
|
||||
client = GatewayClient.from_settings(settings)
|
||||
try:
|
||||
assert client._transport._clients == {}
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["MiniMax-M2.7", "MiniMax-M3"])
|
||||
def test_live_assembly_rejection_is_local_only(model):
|
||||
"""M2.7 NONE 与 M3 AUTO 的旧 live 装配断言离线执行。"""
|
||||
env = _env(**{"LLM__QWEN__1__MODEL": model})
|
||||
settings = GatewaySettings.from_env("LLM", env=env)
|
||||
source = dataclasses.replace(
|
||||
settings.sources[0], provider="minimax", enable_thinking=model == "MiniMax-M3"
|
||||
)
|
||||
with pytest.raises(ValueError, match=model):
|
||||
GatewayClient.from_settings(dataclasses.replace(settings, sources=(source,)))
|
||||
|
||||
|
||||
def test_live_unknown_wire_assembly_is_local_only():
|
||||
"""L9 明确全 None profile，不从当前默认注册表猜未知形态。"""
|
||||
mystery = ProviderProfile(
|
||||
name="mystery",
|
||||
thinking=ThinkingWire(off=None, on_base=None, effort_key=None),
|
||||
strip_think_tags=False,
|
||||
)
|
||||
settings = GatewaySettings.from_env("LLM", env=_BASE_ENV)
|
||||
source = dataclasses.replace(settings.sources[0], provider="mystery", enable_thinking=False)
|
||||
with pytest.raises(ValueError, match="register_provider"):
|
||||
GatewayClient.from_settings(
|
||||
dataclasses.replace(settings, sources=(source,)), registry=register_provider(mystery)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,816 @@
|
||||
"""live_evidence 与测试侧取证装配的日常离线反例。"""
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import json
|
||||
from dataclasses import replace
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from polygateway.errors import RequestRejectedError, SourceDeadError, TransientError
|
||||
from polygateway.ports import EmbeddingTransport, Transport
|
||||
from polygateway.transports.openai_compat import OpenAICompatTransport
|
||||
from polygateway.types import SourceConfig
|
||||
from polygateway.types import ThinkingObservation as O
|
||||
from tests.e2e.conftest import LiveCapture, ObservedTransport
|
||||
from tests.live_evidence import (
|
||||
AttemptEvidence,
|
||||
HttpEvidence,
|
||||
LiveVerdict,
|
||||
assess_expected_rejection,
|
||||
assess_model_identity,
|
||||
assess_thinking_coverage,
|
||||
classify_live_failure,
|
||||
messages_digest,
|
||||
request_is_valid,
|
||||
safe_attempts,
|
||||
summarize_verdicts,
|
||||
write_live_round,
|
||||
)
|
||||
|
||||
_CHECKS = tuple(
|
||||
(key, True)
|
||||
for key in (
|
||||
"method",
|
||||
"origin",
|
||||
"path",
|
||||
"model",
|
||||
"stream",
|
||||
"authorization",
|
||||
"control",
|
||||
"messages_digest",
|
||||
)
|
||||
)
|
||||
_BODY = b'{"error":{"type":"model_not_found"}}'
|
||||
_SECRET = "fake-unique-credential-sentinel"
|
||||
_PROMPT = "fake-private-prompt-sentinel"
|
||||
_MESSAGES = [{"role": "user", "content": _PROMPT}]
|
||||
|
||||
|
||||
def _failure(*, body=_BODY, status=404, checks=_CHECKS):
|
||||
"""完整的唯一错误证据，供逐维破坏。"""
|
||||
error = RequestRejectedError("safe", status_code=status)
|
||||
event = HttpEvidence("a", checks, status, body, (False, None))
|
||||
return error, (AttemptEvidence("a", (event,), error),)
|
||||
|
||||
|
||||
def test_exact_model_not_found_is_uncovered():
|
||||
error, attempts = _failure()
|
||||
assert classify_live_failure(error, attempts).status == "UNCOVERED"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
[
|
||||
None,
|
||||
b"",
|
||||
b"{",
|
||||
b"[]",
|
||||
b'{"error":[]}',
|
||||
b'{"error":{"type":7}}',
|
||||
b'{"error":{"message":"model_not_found"}}',
|
||||
b'{"error":{"type":"MODEL_NOT_FOUND"}}',
|
||||
b'{"error":{"type":"wrong","type":"model_not_found"}}',
|
||||
b'{"error":{},"error":{"type":"model_not_found"}}',
|
||||
b"\xff",
|
||||
_BODY + b" " * 65536,
|
||||
],
|
||||
)
|
||||
def test_incomplete_or_ambiguous_error_body_fails(body):
|
||||
assert classify_live_failure(*_failure(body=body)).status == "FAIL"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", [400, 401, 403, 429, 500, 503])
|
||||
def test_status_classes_never_automatically_skip(status):
|
||||
assert classify_live_failure(*_failure(status=status)).status == "FAIL"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[TransientError("safe"), SourceDeadError("safe"), ValueError("safe"), AssertionError("safe")],
|
||||
)
|
||||
def test_whole_exception_class_skip_is_forbidden(error):
|
||||
assert classify_live_failure(error, ()).status == "FAIL"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", [key for key, _ in _CHECKS])
|
||||
@pytest.mark.parametrize("mode", ["missing", "false", "duplicate"])
|
||||
def test_all_request_checks_are_required(key, mode):
|
||||
checks = tuple(
|
||||
(k, False if k == key and mode == "false" else v)
|
||||
for k, v in _CHECKS
|
||||
if mode != "missing" or k != key
|
||||
)
|
||||
if mode == "duplicate":
|
||||
checks += ((key, True),)
|
||||
assert classify_live_failure(*_failure(checks=checks)).status == "FAIL"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mode",
|
||||
["empty", "second_attempt", "second_http", "wrong_id", "different_error", "status_mismatch"],
|
||||
)
|
||||
def test_attempt_pairing_cannot_be_guessed(mode):
|
||||
error, attempts = _failure()
|
||||
first = attempts[0]
|
||||
if mode == "empty":
|
||||
attempts = (replace(first, http=()),)
|
||||
elif mode == "second_attempt":
|
||||
attempts += (AttemptEvidence("b", (), TransientError("safe")),)
|
||||
elif mode == "second_http":
|
||||
attempts = (replace(first, http=first.http * 2),)
|
||||
elif mode == "wrong_id":
|
||||
attempts = (replace(first, call_id="other"),)
|
||||
elif mode == "different_error":
|
||||
attempts = (replace(first, error=RequestRejectedError("safe", status_code=404)),)
|
||||
else:
|
||||
attempts = (replace(first, http=(replace(first.http[0], status_code=400),)),)
|
||||
assert classify_live_failure(error, attempts).status == "FAIL"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("reported", "raw", "expected"),
|
||||
[
|
||||
("m", (False, None), "PASS"),
|
||||
("alias", (True, "alias"), "PASS"),
|
||||
(None, (False, None), "FAIL"),
|
||||
("other", (False, None), "FAIL"),
|
||||
(None, (True, None), "UNCOVERED"),
|
||||
("other", (True, "other"), "UNCOVERED"),
|
||||
(None, (True, "m"), "FAIL"),
|
||||
("wrong", (True, "m"), "FAIL"),
|
||||
("m", (True, None), "FAIL"),
|
||||
],
|
||||
)
|
||||
def test_identity_requires_independent_raw_evidence(reported, raw, expected):
|
||||
assert (
|
||||
assess_model_identity(
|
||||
requested="m",
|
||||
aliases=frozenset({"alias"}),
|
||||
reported=reported,
|
||||
raw_identity=raw,
|
||||
request_valid=True,
|
||||
).status
|
||||
== expected
|
||||
)
|
||||
assert (
|
||||
assess_model_identity(
|
||||
requested="m",
|
||||
aliases=frozenset(),
|
||||
reported=reported,
|
||||
raw_identity=raw,
|
||||
request_valid=False,
|
||||
).status
|
||||
== "FAIL"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("proposition", "observations", "expected"),
|
||||
[
|
||||
("enabled", [O.OBSERVED, O.OBSERVED, O.UNKNOWN], "PASS"),
|
||||
("enabled", [O.UNKNOWN] * 3, "UNCOVERED"),
|
||||
("enabled", [O.ABSENT] * 3, "FAIL"),
|
||||
("disabled", [O.ABSENT] * 3, "PASS"),
|
||||
("disabled", [O.ABSENT, O.UNKNOWN, O.ABSENT], "UNCOVERED"),
|
||||
("disabled", [O.OBSERVED, O.UNKNOWN, O.ABSENT], "FAIL"),
|
||||
("cannot_disable", [O.OBSERVED, O.UNKNOWN, O.ABSENT], "PASS"),
|
||||
("cannot_disable", [O.ABSENT] * 3, "FAIL"),
|
||||
("cannot_disable", [O.UNKNOWN] * 3, "UNCOVERED"),
|
||||
],
|
||||
)
|
||||
def test_coverage_is_proposition_specific(proposition, observations, expected):
|
||||
assert (
|
||||
assess_thinking_coverage(observations, planned_rounds=3, proposition=proposition).status
|
||||
== expected
|
||||
)
|
||||
|
||||
|
||||
def test_missing_round_never_reduces_denominator():
|
||||
assert (
|
||||
assess_thinking_coverage([O.OBSERVED] * 2, planned_rounds=3, proposition="enabled").status
|
||||
== "FAIL"
|
||||
)
|
||||
assert summarize_verdicts(
|
||||
[LiveVerdict("PASS", "ok"), LiveVerdict("FAIL", "bad")], planned_rounds=3
|
||||
) == {"PASS": 1, "FAIL": 1, "UNCOVERED": 0, "missing": 1}
|
||||
|
||||
|
||||
def test_expected_400_is_a_separate_negative_proposition():
|
||||
error, attempts = _failure(status=400, body=b'{"error":{"type":"unsupported_value"}}')
|
||||
assert classify_live_failure(error, attempts).status == "FAIL"
|
||||
assert (
|
||||
assess_expected_rejection(
|
||||
error, attempts, status_code=400, machine_type="unsupported_value"
|
||||
).status
|
||||
== "PASS"
|
||||
)
|
||||
assert (
|
||||
assess_expected_rejection(error, attempts, status_code=400, machine_type="other").status
|
||||
== "FAIL"
|
||||
)
|
||||
|
||||
|
||||
def _source():
|
||||
"""假凭据与非默认超时，不接真实网络。"""
|
||||
return SourceConfig(
|
||||
name="source",
|
||||
provider="openai",
|
||||
model="gpt-5.5",
|
||||
base_url="https://example.test/v1",
|
||||
api_key=_SECRET,
|
||||
timeout_s=137,
|
||||
trust_env=False,
|
||||
)
|
||||
|
||||
|
||||
def _capture(*, stream=False, embedding=False, **changes):
|
||||
"""预期独立于生产 payload。"""
|
||||
expected = {
|
||||
"model": "gpt-5.5",
|
||||
"origin": "https://example.test",
|
||||
"path": "/v1/embeddings" if embedding else "/v1/chat/completions",
|
||||
"control": {},
|
||||
"messages_digest": messages_digest([_PROMPT] if embedding else _MESSAGES),
|
||||
}
|
||||
expected.update({"input_shape": 1} if embedding else {"stream": stream})
|
||||
expected.update(changes)
|
||||
return LiveCapture(expectations={"source": expected})
|
||||
|
||||
|
||||
def _response(model="gpt-5.5"):
|
||||
"""OpenAI 非流式完整响应形态。"""
|
||||
result = {
|
||||
"id": "test",
|
||||
"choices": [
|
||||
{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}
|
||||
],
|
||||
"usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
|
||||
}
|
||||
if model != "missing":
|
||||
result["model"] = model
|
||||
return result
|
||||
|
||||
|
||||
def _real_transport(capture, handler, *, tamper=None):
|
||||
"""仅替换 HTTP 边界，保留真实 hooks 与 production transport。"""
|
||||
|
||||
def factory(source):
|
||||
"""在网络边界换成 MockTransport。"""
|
||||
client = capture.client_factory(source)
|
||||
client._transport = httpx.MockTransport(handler)
|
||||
if tamper is not None:
|
||||
client.event_hooks["request"].insert(0, tamper)
|
||||
return client
|
||||
|
||||
return OpenAICompatTransport(client_factory=factory)
|
||||
|
||||
|
||||
async def _complete(observed, *, call_id="a", stream=False):
|
||||
"""沿原参数调用薄委托器。"""
|
||||
return await observed.complete(
|
||||
messages=_MESSAGES,
|
||||
source=_source(),
|
||||
stream=stream,
|
||||
overlay={},
|
||||
call_id=call_id,
|
||||
reasoning_effort=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-5.5", "missing", None])
|
||||
async def test_raw_identity_snapshot_reaches_round_consumer(model):
|
||||
capture = _capture()
|
||||
real = _real_transport(capture, lambda request: httpx.Response(200, json=_response(model)))
|
||||
try:
|
||||
with capture.round_context(session_id="s", parent_call_id="p"):
|
||||
result = await _complete(ObservedTransport(real, capture))
|
||||
raw = capture.raw_identity(session_id="s", parent_call_id="p", call_id="a")
|
||||
assert raw == (True, None if model == "missing" else model)
|
||||
assert request_is_valid(capture.attempts(session_id="s", parent_call_id="p")[0].http[0])
|
||||
verdict = assess_model_identity(
|
||||
requested="gpt-5.5",
|
||||
aliases=frozenset(),
|
||||
reported=result.model_reported,
|
||||
raw_identity=raw,
|
||||
request_valid=True,
|
||||
)
|
||||
assert verdict.status == ("PASS" if model == "gpt-5.5" else "UNCOVERED")
|
||||
if model == "gpt-5.5":
|
||||
assert (
|
||||
assess_model_identity(
|
||||
requested="gpt-5.5",
|
||||
aliases=frozenset(),
|
||||
reported=None,
|
||||
raw_identity=raw,
|
||||
request_valid=True,
|
||||
).status
|
||||
== "FAIL"
|
||||
)
|
||||
finally:
|
||||
await real.aclose()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body", [b"[]", b"{", b'{"model":"a","model":"b"}', b'{"model":7}', b"\xff"]
|
||||
)
|
||||
async def test_invalid_raw_json_is_not_missing_identity(body):
|
||||
capture = _capture()
|
||||
real = _real_transport(capture, lambda request: httpx.Response(200, content=body))
|
||||
try:
|
||||
with (
|
||||
capture.round_context(session_id="s", parent_call_id="p"),
|
||||
pytest.raises((ValueError, AttributeError, KeyError, TransientError)),
|
||||
):
|
||||
await _complete(ObservedTransport(real, capture))
|
||||
assert capture.raw_identity(session_id="s", parent_call_id="p", call_id="a") == (
|
||||
False,
|
||||
None,
|
||||
)
|
||||
assert capture.notes(session_id="s", parent_call_id="p")
|
||||
finally:
|
||||
await real.aclose()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field", ["model", "authorization", "path", "stream", "control", "messages_digest"]
|
||||
)
|
||||
async def test_hooks_detect_corrupted_request_without_repair(field):
|
||||
capture = _capture()
|
||||
|
||||
async def tamper(request):
|
||||
"""故意改错实发请求，而不是改预期。"""
|
||||
if field == "authorization":
|
||||
request.headers["Authorization"] = "Bearer wrong"
|
||||
elif field == "path":
|
||||
request.url = request.url.copy_with(path="/wrong")
|
||||
else:
|
||||
body = json.loads(request.content)
|
||||
key, value = {
|
||||
"model": ("model", "wrong"),
|
||||
"stream": ("stream", True),
|
||||
"control": ("reasoning_effort", "high"),
|
||||
"messages_digest": ("messages", []),
|
||||
}[field]
|
||||
body[key] = value
|
||||
request._content = json.dumps(body).encode()
|
||||
|
||||
real = _real_transport(
|
||||
capture, lambda request: httpx.Response(404, content=_BODY), tamper=tamper
|
||||
)
|
||||
try:
|
||||
with (
|
||||
capture.round_context(session_id="s", parent_call_id="p"),
|
||||
pytest.raises(RequestRejectedError) as caught,
|
||||
):
|
||||
await _complete(ObservedTransport(real, capture))
|
||||
attempts = capture.attempts(session_id="s", parent_call_id="p")
|
||||
assert dict(attempts[0].http[0].request_checks)[field] is False
|
||||
assert classify_live_failure(caught.value, attempts).status == "FAIL"
|
||||
finally:
|
||||
await real.aclose()
|
||||
|
||||
|
||||
async def test_embed_uses_same_capture_and_source_factory_contract():
|
||||
capture = _capture(embedding=True)
|
||||
real = _real_transport(
|
||||
capture,
|
||||
lambda request: httpx.Response(
|
||||
200,
|
||||
json={"data": [{"index": 0, "embedding": [0.1, 0.2]}], "usage": {"prompt_tokens": 1}},
|
||||
),
|
||||
)
|
||||
try:
|
||||
with capture.round_context(session_id="s", parent_call_id="p"):
|
||||
result = await ObservedTransport(real, capture).embed(
|
||||
texts=[_PROMPT], source=_source(), call_id="e"
|
||||
)
|
||||
assert result.dim == 2
|
||||
assert request_is_valid(capture.attempts(session_id="s", parent_call_id="p")[0].http[0])
|
||||
client = real._clients["source"]
|
||||
assert client.timeout.read == 137 and client.trust_env is False
|
||||
assert client.headers["Authorization"] == f"Bearer {_SECRET}"
|
||||
finally:
|
||||
await real.aclose()
|
||||
assert real._clients == {}
|
||||
|
||||
|
||||
class _SSE(httpx.AsyncByteStream):
|
||||
"""可检查是否被 hook 提前消费的 SSE。"""
|
||||
|
||||
def __init__(self):
|
||||
self.reads = 0
|
||||
self.closed = False
|
||||
|
||||
async def __aiter__(self):
|
||||
self.reads += 1
|
||||
yield b'data: {"model":"gpt-5.5","choices":[{"delta":{"content":"ok"},"finish_reason":null}]}\n\n'
|
||||
yield b'data: {"choices":[],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}\n\ndata: [DONE]\n\n'
|
||||
|
||||
async def aclose(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
async def test_success_sse_is_not_preconsumed_and_has_no_raw_snapshot():
|
||||
capture = _capture(stream=True)
|
||||
stream = _SSE()
|
||||
|
||||
def handler(request):
|
||||
return httpx.Response(200, stream=stream)
|
||||
|
||||
def factory(source):
|
||||
client = capture.client_factory(source)
|
||||
|
||||
async def after_hook(response):
|
||||
assert stream.reads == 0
|
||||
|
||||
client.event_hooks["response"].append(after_hook)
|
||||
client._transport = httpx.MockTransport(handler)
|
||||
return client
|
||||
|
||||
real = OpenAICompatTransport(client_factory=factory)
|
||||
try:
|
||||
with capture.round_context(session_id="s", parent_call_id="p"):
|
||||
result = await _complete(ObservedTransport(real, capture), stream=True)
|
||||
assert result.content == "ok" and stream.reads == 1 and stream.closed
|
||||
raw = capture.raw_identity(session_id="s", parent_call_id="p", call_id="a")
|
||||
assert raw == (False, None)
|
||||
assert (
|
||||
assess_model_identity(
|
||||
requested="gpt-5.5",
|
||||
aliases=frozenset(),
|
||||
reported=None,
|
||||
raw_identity=raw,
|
||||
request_valid=True,
|
||||
).status
|
||||
== "FAIL"
|
||||
)
|
||||
finally:
|
||||
await real.aclose()
|
||||
|
||||
|
||||
async def test_concurrent_rounds_retry_and_cancellation_reset_context():
|
||||
capture = _capture()
|
||||
reached = asyncio.Event()
|
||||
|
||||
async def handler(request):
|
||||
await asyncio.sleep(0)
|
||||
return httpx.Response(200, json=_response())
|
||||
|
||||
real = _real_transport(capture, handler)
|
||||
observed = ObservedTransport(real, capture)
|
||||
|
||||
async def one(parent):
|
||||
with capture.round_context(session_id="s", parent_call_id=parent):
|
||||
# 零 HTTP 的前次失败与成功 call_id 不得混淆。
|
||||
with pytest.raises(ValueError), capture.attempt_context(parent + "-failed"):
|
||||
raise ValueError("safe")
|
||||
await _complete(observed, call_id=parent + "-ok")
|
||||
return capture.raw_identity(session_id="s", parent_call_id=parent, call_id=parent + "-ok")
|
||||
|
||||
async def cancelled():
|
||||
with (
|
||||
capture.round_context(session_id="s", parent_call_id="cancel"),
|
||||
capture.attempt_context("cancelled"),
|
||||
):
|
||||
reached.set()
|
||||
await asyncio.Future()
|
||||
|
||||
try:
|
||||
assert await asyncio.gather(one("p1"), one("p2")) == [(True, "gpt-5.5")] * 2
|
||||
for parent in ("p1", "p2"):
|
||||
attempts = capture.attempts(session_id="s", parent_call_id=parent)
|
||||
assert len(attempts) == 2 and len(attempts[0].http) == 0 and len(attempts[1].http) == 1
|
||||
with pytest.raises(ValueError, match="跨轮"):
|
||||
capture.raw_identity(session_id="s", parent_call_id="p1", call_id="p2-ok")
|
||||
task = asyncio.create_task(cancelled())
|
||||
await reached.wait()
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert len(capture.attempts(session_id="s", parent_call_id="cancel")) == 1
|
||||
with pytest.raises(LookupError):
|
||||
capture._round.get()
|
||||
with pytest.raises(LookupError):
|
||||
capture._attempt.get()
|
||||
with (
|
||||
capture.round_context(session_id="s", parent_call_id="new"),
|
||||
pytest.raises(ValueError, match="重复"),
|
||||
capture.attempt_context("p1-ok"),
|
||||
):
|
||||
pytest.fail("不应进入")
|
||||
finally:
|
||||
await real.aclose()
|
||||
|
||||
|
||||
def test_delegator_signatures_match_ports():
|
||||
assert inspect.signature(ObservedTransport.complete) == inspect.signature(Transport.complete)
|
||||
assert inspect.signature(ObservedTransport.embed) == inspect.signature(EmbeddingTransport.embed)
|
||||
|
||||
|
||||
def test_missing_expectations_fail_before_network():
|
||||
with pytest.raises(ValueError):
|
||||
LiveCapture(expectations={"s": {"model": "m"}})
|
||||
|
||||
|
||||
def test_round_reports_preserve_prior_round_and_redact_sentinels(tmp_path):
|
||||
error, attempts = _failure(
|
||||
body=(f'{{"error":{{"type":"model_not_found","message":"{_SECRET} {_PROMPT}"}}}}').encode()
|
||||
)
|
||||
error.args = (f"{_SECRET} {_PROMPT}",)
|
||||
first = write_live_round(
|
||||
tmp_path,
|
||||
run_id="run",
|
||||
matrix_id="matrix",
|
||||
round_index=1,
|
||||
safe_fields={"status": "PASS", "attempts": safe_attempts(attempts)},
|
||||
)
|
||||
second = write_live_round(
|
||||
tmp_path,
|
||||
run_id="run",
|
||||
matrix_id="matrix",
|
||||
round_index=2,
|
||||
safe_fields={"status": "FAIL", "attempts": safe_attempts(attempts)},
|
||||
)
|
||||
assert first != second and first.exists()
|
||||
for path in tmp_path.rglob("*.md"):
|
||||
text = path.read_text()
|
||||
assert _SECRET not in text and _PROMPT not in text
|
||||
assert '"status": "PASS"' in first.read_text()
|
||||
assert '"status": "FAIL"' in second.read_text()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"fields",
|
||||
[
|
||||
{"raw": _BODY},
|
||||
{"error_type": ValueError("secret")},
|
||||
{"attempts": [AttemptEvidence("a", (), None)]},
|
||||
{"attempts": [{"call_id": "a", "error_type": None, "http": [{"Authorization": "secret"}]}]},
|
||||
],
|
||||
)
|
||||
def test_report_refuses_raw_objects_and_unknown_fields(tmp_path, fields):
|
||||
with pytest.raises((ValueError, TypeError)):
|
||||
write_live_round(
|
||||
tmp_path, run_id="run", matrix_id="matrix", round_index=1, safe_fields=fields
|
||||
)
|
||||
|
||||
|
||||
def test_report_write_failure_is_not_skip(tmp_path):
|
||||
blocked = tmp_path / "file"
|
||||
blocked.write_text("not a directory")
|
||||
with pytest.raises(OSError):
|
||||
write_live_round(
|
||||
blocked,
|
||||
run_id="run",
|
||||
matrix_id="matrix",
|
||||
round_index=1,
|
||||
safe_fields={"status": "UNCOVERED"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body", [b"", _BODY, _BODY + b" " * (65536 - len(_BODY)), _BODY + b" " * 65536]
|
||||
)
|
||||
async def test_error_snapshot_keeps_only_complete_bounded_body(body):
|
||||
"""真实 hook 的边界，不把生产摘要当完整错误 JSON。"""
|
||||
capture = _capture()
|
||||
real = _real_transport(capture, lambda request: httpx.Response(404, content=body))
|
||||
try:
|
||||
with (
|
||||
capture.round_context(session_id="s", parent_call_id="p"),
|
||||
pytest.raises(RequestRejectedError) as caught,
|
||||
):
|
||||
await _complete(ObservedTransport(real, capture))
|
||||
attempts = capture.attempts(session_id="s", parent_call_id="p")
|
||||
assert attempts[0].http[0].error_body == (body if len(body) <= 65536 else None)
|
||||
expected = "UNCOVERED" if 0 < len(body) <= 65536 else "FAIL"
|
||||
assert classify_live_failure(caught.value, attempts).status == expected
|
||||
finally:
|
||||
await real.aclose()
|
||||
|
||||
|
||||
async def test_multiple_success_http_candidates_fail_identity_accessor():
|
||||
"""一个 attempt 多 HTTP 不能猜最后一条。"""
|
||||
capture = _capture()
|
||||
client = capture.client_factory(_source())
|
||||
client._transport = httpx.MockTransport(lambda request: httpx.Response(200, json=_response()))
|
||||
try:
|
||||
with (
|
||||
capture.round_context(session_id="s", parent_call_id="p"),
|
||||
capture.attempt_context("a"),
|
||||
):
|
||||
for _ in range(2):
|
||||
await client.post(
|
||||
"https://example.test/v1/chat/completions",
|
||||
json={"model": "gpt-5.5", "stream": False, "messages": _MESSAGES},
|
||||
)
|
||||
with pytest.raises(ValueError, match="多个成功"):
|
||||
capture.raw_identity(session_id="s", parent_call_id="p", call_id="a")
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body", [b"data: not-json\n\n", b'data: {"choices":[]}\n\ndata: [DONE]\n\n']
|
||||
)
|
||||
async def test_sse_parser_failure_is_not_external_uncovered(body):
|
||||
capture = _capture(stream=True)
|
||||
real = _real_transport(capture, lambda request: httpx.Response(200, content=body))
|
||||
try:
|
||||
with (
|
||||
capture.round_context(session_id="s", parent_call_id="p"),
|
||||
pytest.raises(TransientError) as caught,
|
||||
):
|
||||
await _complete(ObservedTransport(real, capture), stream=True)
|
||||
assert (
|
||||
classify_live_failure(
|
||||
caught.value, capture.attempts(session_id="s", parent_call_id="p")
|
||||
).status
|
||||
== "FAIL"
|
||||
)
|
||||
finally:
|
||||
await real.aclose()
|
||||
|
||||
|
||||
async def test_round_consumer_keeps_first_success_when_second_assertion_fails(tmp_path):
|
||||
"""真实 GatewayClient 与报告出口连通；断言异常不得漏报告。"""
|
||||
from polygateway import GatewaySettings
|
||||
from tests.e2e.conftest import captured_chat_round, observed_client
|
||||
from tests.unit.test_config import _BASE_ENV
|
||||
|
||||
settings = GatewaySettings.from_env("LLM", env=_BASE_ENV)
|
||||
settings = replace(settings, sources=(_source(),))
|
||||
capture = _capture()
|
||||
original_factory = capture.client_factory
|
||||
clients = []
|
||||
|
||||
def factory(source):
|
||||
client = original_factory(source)
|
||||
client._transport = httpx.MockTransport(
|
||||
lambda request: httpx.Response(200, json=_response())
|
||||
)
|
||||
clients.append(client)
|
||||
return client
|
||||
|
||||
capture.client_factory = factory
|
||||
|
||||
def fail(response):
|
||||
raise AssertionError(_SECRET + _PROMPT)
|
||||
|
||||
async with observed_client(settings, capture) as client:
|
||||
for index, validate in ((1, None), (2, fail)):
|
||||
_, verdict = await captured_chat_round(
|
||||
client,
|
||||
capture,
|
||||
run_id="run",
|
||||
matrix_id="matrix",
|
||||
round_index=index,
|
||||
output_dir=tmp_path,
|
||||
messages=_MESSAGES,
|
||||
models={"source": "gpt-5.5"},
|
||||
aliases={},
|
||||
stream=False,
|
||||
validate=validate,
|
||||
)
|
||||
assert verdict.status == ("PASS" if index == 1 else "FAIL")
|
||||
paths = list(tmp_path.rglob("*.md"))
|
||||
assert len(paths) == 2
|
||||
text = "".join(path.read_text() for path in paths)
|
||||
assert '"status": "PASS"' in text and '"status": "FAIL"' in text
|
||||
assert _SECRET not in text and _PROMPT not in text
|
||||
assert clients and all(client.is_closed for client in clients)
|
||||
|
||||
|
||||
def test_partial_uncovered_and_failed_rounds_never_become_model_pass():
|
||||
from tests.live_evidence import combine_live_verdicts, qualify_live_rounds
|
||||
|
||||
passed = LiveVerdict("PASS", "safe")
|
||||
uncovered = LiveVerdict("UNCOVERED", "safe")
|
||||
failed = LiveVerdict("FAIL", "safe")
|
||||
assert combine_live_verdicts([passed, uncovered]).status == "UNCOVERED"
|
||||
assert combine_live_verdicts([passed, uncovered, failed]).status == "FAIL"
|
||||
assert qualify_live_rounds([passed, passed], planned_rounds=3).status == "FAIL"
|
||||
assert qualify_live_rounds([passed, failed, passed], planned_rounds=3).status == "FAIL"
|
||||
|
||||
|
||||
async def test_real_retry_success_call_id_selects_exact_raw_identity(tmp_path):
|
||||
"""真实 RetryMW 先503再成功，两个并发 parent 的身份不串线。"""
|
||||
from polygateway import GatewaySettings
|
||||
from tests.e2e.conftest import captured_chat_round, observed_client
|
||||
from tests.unit.test_config import _BASE_ENV
|
||||
|
||||
settings = GatewaySettings.from_env("LLM", env=_BASE_ENV)
|
||||
settings = replace(
|
||||
settings,
|
||||
sources=(_source(),),
|
||||
retry=replace(settings.retry, backoff_base_s=0.001, backoff_max_s=0.002),
|
||||
)
|
||||
capture = _capture()
|
||||
original_factory = capture.client_factory
|
||||
counts = {}
|
||||
|
||||
async def handler(request):
|
||||
key = capture._round.get()
|
||||
counts[key] = counts.get(key, 0) + 1
|
||||
await asyncio.sleep(0)
|
||||
if counts[key] == 1:
|
||||
return httpx.Response(503, json={"error": {"type": "temporary"}, "model": "wrong"})
|
||||
return httpx.Response(200, json=_response())
|
||||
|
||||
def factory(source):
|
||||
client = original_factory(source)
|
||||
client._transport = httpx.MockTransport(handler)
|
||||
return client
|
||||
|
||||
capture.client_factory = factory
|
||||
async with observed_client(settings, capture) as client:
|
||||
|
||||
async def one(index):
|
||||
return await captured_chat_round(
|
||||
client,
|
||||
capture,
|
||||
run_id="retry",
|
||||
matrix_id="retry",
|
||||
round_index=index,
|
||||
output_dir=tmp_path,
|
||||
messages=_MESSAGES,
|
||||
models={"source": "gpt-5.5"},
|
||||
aliases={},
|
||||
stream=False,
|
||||
)
|
||||
|
||||
results = await asyncio.gather(one(1), one(2))
|
||||
assert all(verdict.status == "PASS" for _, verdict in results)
|
||||
assert len(counts) == 2 and set(counts.values()) == {2}
|
||||
for key in counts:
|
||||
attempts = capture.attempts(session_id=key[0], parent_call_id=key[1])
|
||||
assert len(attempts) == 2
|
||||
assert attempts[0].http[0].raw_identity == (False, None)
|
||||
assert capture.raw_identity(
|
||||
session_id=key[0], parent_call_id=key[1], call_id=attempts[1].call_id
|
||||
) == (True, "gpt-5.5")
|
||||
|
||||
|
||||
def test_live_model_provider_mapping_covers_default_capabilities_without_env_import():
|
||||
"""L8 映射装配守卫离线化，只读取显式字面矩阵，不执行 .env 模块。"""
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
from polygateway.thinking import DEFAULT_CAPABILITIES
|
||||
|
||||
tree = ast.parse((Path(__file__).parents[1] / "e2e/test_thinking_live.py").read_text())
|
||||
mapping = next(
|
||||
ast.literal_eval(node.value)
|
||||
for node in tree.body
|
||||
if isinstance(node, ast.Assign)
|
||||
and any(
|
||||
isinstance(target, ast.Name) and target.id == "_MODEL_PROVIDER"
|
||||
for target in node.targets
|
||||
)
|
||||
)
|
||||
assert not set(DEFAULT_CAPABILITIES) - mapping.keys()
|
||||
|
||||
|
||||
async def test_unbuffered_error_is_insufficient_evidence():
|
||||
"""response hook 不预读错误流；未完成缓冲不能借片段归因。"""
|
||||
capture = _capture()
|
||||
stream = _SSE()
|
||||
client = capture.client_factory(_source())
|
||||
client._transport = httpx.MockTransport(lambda request: httpx.Response(404, stream=stream))
|
||||
error = RequestRejectedError("safe", status_code=404)
|
||||
try:
|
||||
with (
|
||||
pytest.raises(RequestRejectedError),
|
||||
capture.round_context(session_id="s", parent_call_id="p"),
|
||||
capture.attempt_context("a"),
|
||||
):
|
||||
async with client.stream(
|
||||
"POST",
|
||||
"https://example.test/v1/chat/completions",
|
||||
json={"model": "gpt-5.5", "messages": _MESSAGES, "stream": False},
|
||||
):
|
||||
raise error
|
||||
attempts = capture.attempts(session_id="s", parent_call_id="p")
|
||||
assert attempts[0].http[0].error_body is None and stream.reads == 0
|
||||
assert classify_live_failure(error, attempts).status == "FAIL"
|
||||
finally:
|
||||
await client.aclose()
|
||||
|
||||
|
||||
def test_pytest_report_hook_preserves_report_and_uses_safe_fallback(tmp_path, monkeypatch):
|
||||
"""pytest wrapper generator 的 return 值是协议必需，不可按错误 LSP 建议删除。"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from tests.e2e.conftest import pytest_runtest_makereport
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
report = SimpleNamespace(skipped=False, failed=True, when="call")
|
||||
hook = pytest_runtest_makereport(SimpleNamespace(nodeid="safe-node"), None)
|
||||
assert next(hook) is None
|
||||
with pytest.raises(StopIteration) as finished:
|
||||
hook.send(report)
|
||||
assert finished.value.value is report
|
||||
paths = list(tmp_path.rglob("*.md"))
|
||||
assert len(paths) == 1 and '"status": "FAIL"' in paths[0].read_text()
|
||||
Reference in New Issue
Block a user