541 lines
23 KiB
Python
541 lines
23 KiB
Python
"""测试侧独立 HTTP 取证装配；无环境自读取或成功 SSE 预读。"""
|
|
|
|
from collections.abc import AsyncIterator, Iterator, Mapping
|
|
from contextlib import AsyncExitStack, asynccontextmanager, contextmanager
|
|
from contextvars import ContextVar
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from polygateway import GatewayClient, GatewaySettings
|
|
from polygateway.client import (
|
|
_aclose_component,
|
|
_build_breaker,
|
|
_build_limiter,
|
|
_build_selector,
|
|
_build_structured,
|
|
)
|
|
from polygateway.providers import get_provider
|
|
from polygateway.transports.openai_compat import OpenAICompatTransport
|
|
from polygateway.types import Effort, EmbeddingTransportResult, SourceConfig, TransportResult
|
|
from tests.live_evidence import (
|
|
AttemptEvidence,
|
|
HttpEvidence,
|
|
LiveVerdict,
|
|
assess_model_identity,
|
|
classify_live_failure,
|
|
messages_digest,
|
|
request_is_valid,
|
|
safe_attempts,
|
|
strict_json,
|
|
write_live_round,
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class _Exchange:
|
|
"""仅在 attempt 生命周期持有原始响应引用。"""
|
|
|
|
request: httpx.Request
|
|
checks: tuple[tuple[str, bool], ...]
|
|
response: httpx.Response | None = None
|
|
|
|
|
|
@dataclass
|
|
class _Attempt:
|
|
"""任务内可变收集器，结束时转换为冻结快照。"""
|
|
|
|
call_id: str
|
|
exchanges: list[_Exchange] = field(default_factory=list)
|
|
messages_digest: str | None = None
|
|
messages_valid: bool = False
|
|
|
|
|
|
class LiveCapture:
|
|
"""矩阵显式预期与按逻辑轮次关联的独立证据。"""
|
|
|
|
def __init__(self, *, expectations: Mapping[str, Mapping[str, Any]]) -> None:
|
|
"""预期缺项即配置错误，不从实发 payload 补齐。"""
|
|
for expected in expectations.values():
|
|
required = {"model", "origin", "path", "control", "messages_digest"}
|
|
if not required <= expected.keys() or ("stream" in expected) == (
|
|
"input_shape" in expected
|
|
):
|
|
raise ValueError("取证矩阵缺少必需预期或混用 chat/embed")
|
|
if not isinstance(expected["control"], dict):
|
|
raise ValueError("control 必须是显式对象")
|
|
if "structured_max_retries" in expected and (
|
|
"stream" not in expected
|
|
or type(expected["structured_max_retries"]) is not int
|
|
or expected["structured_max_retries"] < 0
|
|
or type(expected.get("messages_prefix_length")) is not int
|
|
or expected["messages_prefix_length"] < 1
|
|
):
|
|
raise ValueError("结构化预期缺少合法前缀长度或重问预算")
|
|
self._expectations = {name: dict(value) for name, value in expectations.items()}
|
|
self._round: ContextVar[tuple[str, str]] = ContextVar("live_round")
|
|
self._attempt: ContextVar[_Attempt] = ContextVar("live_attempt")
|
|
self._records: dict[tuple[str, str], list[AttemptEvidence]] = {}
|
|
self._owners: dict[str, tuple[str, str]] = {}
|
|
self._notes: dict[tuple[str, str], list[str]] = {}
|
|
|
|
@contextmanager
|
|
def round_context(self, *, session_id: str, parent_call_id: str) -> Iterator[None]:
|
|
"""外围逻辑轮次绑定，异常和取消均复位。"""
|
|
key = session_id, parent_call_id
|
|
token = self._round.set(key)
|
|
self._records.setdefault(key, [])
|
|
self._notes.setdefault(key, [])
|
|
try:
|
|
yield
|
|
finally:
|
|
self._round.reset(token)
|
|
|
|
@contextmanager
|
|
def attempt_context(self, call_id: str) -> Iterator[None]:
|
|
"""零 HTTP 尝试也有快照；重复／跨轮 UUID 是契约错误。"""
|
|
key = self._round.get()
|
|
if call_id in self._owners:
|
|
raise ValueError("重复或跨轮 call_id")
|
|
self._owners[call_id] = key
|
|
attempt = _Attempt(call_id)
|
|
token = self._attempt.set(attempt)
|
|
error = None
|
|
try:
|
|
yield
|
|
except Exception as exc:
|
|
error = exc
|
|
raise
|
|
finally:
|
|
try:
|
|
events = tuple(
|
|
self._snapshot(exchange, call_id, key) for exchange in attempt.exchanges
|
|
)
|
|
self._records[key].append(AttemptEvidence(call_id, events, error))
|
|
finally:
|
|
self._attempt.reset(token)
|
|
|
|
def _snapshot(self, exchange: _Exchange, call_id: str, key: tuple[str, str]) -> HttpEvidence:
|
|
"""complete 结束后只读已缓冲内容，拒绝截断和歧义身份。"""
|
|
response = exchange.response
|
|
identity: tuple[bool, str | None] = (False, None)
|
|
body = None
|
|
status = response.status_code if response is not None else 0
|
|
if response is None:
|
|
self._notes[key].append("无可配对响应")
|
|
else:
|
|
try:
|
|
content = response.content
|
|
except httpx.ResponseNotRead:
|
|
self._notes[key].append("响应未缓冲，证据不足")
|
|
else:
|
|
if status >= 400:
|
|
if len(content) <= 65536:
|
|
body = content
|
|
else:
|
|
self._notes[key].append("错误体超过 64 KiB，不接受截断证据")
|
|
elif 200 <= status < 300 and dict(exchange.checks).get("stream") is not None:
|
|
try:
|
|
payload = strict_json(exchange.request.content)
|
|
except (ValueError, UnicodeError):
|
|
payload = {}
|
|
if isinstance(payload, dict) and payload.get("stream") is False:
|
|
try:
|
|
data = strict_json(content)
|
|
if not isinstance(data, dict):
|
|
raise ValueError("原始 JSON 非对象")
|
|
model = data.get("model")
|
|
if model is not None and not isinstance(model, str):
|
|
raise ValueError("原始 model 类型非法")
|
|
identity = (True, model)
|
|
except (ValueError, UnicodeError):
|
|
self._notes[key].append("原始 JSON 身份无法独立解析")
|
|
return HttpEvidence(call_id, exchange.checks, status, body, identity)
|
|
|
|
def observe_messages(self, source: SourceConfig, messages: list[dict[str, Any]]) -> None:
|
|
"""先验前缀／反馈契约与委托摘要分开；摘要仅验证 HTTP 序列化保真。"""
|
|
expected = self._expectations[source.name]
|
|
if "structured_max_retries" not in expected:
|
|
return
|
|
attempt = self._attempt.get()
|
|
prefix_length = expected["messages_prefix_length"]
|
|
feedback = messages[prefix_length:]
|
|
attempt.messages_digest = messages_digest(messages)
|
|
attempt.messages_valid = (
|
|
(not feedback or bool(self._records[self._round.get()]))
|
|
and messages_digest(messages[:prefix_length]) == expected["messages_digest"]
|
|
and len(feedback) % 2 == 0
|
|
and len(feedback) <= 2 * expected["structured_max_retries"]
|
|
and all(
|
|
isinstance(message, dict)
|
|
and set(message) == {"role", "content"}
|
|
and message["role"] == ("assistant" if index % 2 == 0 else "user")
|
|
and isinstance(message["content"], str)
|
|
for index, message in enumerate(feedback)
|
|
)
|
|
)
|
|
|
|
def client_factory(self, source: SourceConfig) -> httpx.AsyncClient:
|
|
"""鉴权仅内存比较；沿已校验源 timeout/trust_env。"""
|
|
expected = self._expectations[source.name]
|
|
|
|
async def request_hook(request: httpx.Request) -> None:
|
|
"""校验实发请求而不修正它。"""
|
|
attempt = self._attempt.get()
|
|
try:
|
|
payload = strict_json(request.content)
|
|
except (ValueError, UnicodeError):
|
|
payload = {}
|
|
if not isinstance(payload, dict):
|
|
payload = {}
|
|
url = request.url
|
|
origin = str(
|
|
url.copy_with(path="", query=None, fragment=None, username=None, password=None)
|
|
).rstrip("/")
|
|
# control 是本轮完整附加字段；基础键以外均比较，漏／多键都失败。
|
|
basic = {"model", "messages", "stream", "stream_options", "input"}
|
|
control = {k: v for k, v in payload.items() if k not in basic}
|
|
checks = {
|
|
"method": request.method == "POST",
|
|
"origin": origin == expected["origin"]
|
|
and not url.username
|
|
and not url.password
|
|
and not url.query,
|
|
"path": url.path == expected["path"],
|
|
"model": payload.get("model") == expected["model"],
|
|
"authorization": request.headers.get("Authorization") == f"Bearer {source.api_key}",
|
|
"control": control == expected["control"],
|
|
"messages_digest": (
|
|
attempt.messages_valid
|
|
and messages_digest(payload.get("messages")) == attempt.messages_digest
|
|
if "structured_max_retries" in expected
|
|
else messages_digest(payload.get("messages", payload.get("input")))
|
|
== expected["messages_digest"]
|
|
),
|
|
}
|
|
if "stream" in expected:
|
|
checks["stream"] = payload.get("stream") is expected["stream"] and (
|
|
payload.get("stream_options") == {"include_usage": True}
|
|
if expected["stream"]
|
|
else "stream_options" not in payload
|
|
)
|
|
else:
|
|
texts = payload.get("input")
|
|
checks["input_shape"] = (
|
|
isinstance(texts, list)
|
|
and all(isinstance(text, str) for text in texts)
|
|
and len(texts) == expected["input_shape"]
|
|
)
|
|
attempt.exchanges.append(_Exchange(request, tuple(checks.items())))
|
|
|
|
async def response_hook(response: httpx.Response) -> None:
|
|
"""只持有引用，绝不提前读取成功 SSE。"""
|
|
attempt = self._attempt.get()
|
|
matching = [event for event in attempt.exchanges if event.request is response.request]
|
|
if len(matching) != 1 or matching[0].response is not None:
|
|
raise ValueError("响应无法唯一配对")
|
|
matching[0].response = response
|
|
|
|
return httpx.AsyncClient(
|
|
headers={"Authorization": f"Bearer {source.api_key}"},
|
|
timeout=source.timeout_s,
|
|
trust_env=source.trust_env,
|
|
event_hooks={"request": [request_hook], "response": [response_hook]},
|
|
)
|
|
|
|
def attempts(self, *, session_id: str, parent_call_id: str) -> tuple[AttemptEvidence, ...]:
|
|
"""按逻辑轮次返回不可变快照。"""
|
|
return tuple(self._records.get((session_id, parent_call_id), ()))
|
|
|
|
def notes(self, *, session_id: str, parent_call_id: str) -> tuple[str, ...]:
|
|
"""只含固定安全原因，不包含响应正文。"""
|
|
return tuple(self._notes.get((session_id, parent_call_id), ()))
|
|
|
|
def raw_identity(
|
|
self, *, session_id: str, parent_call_id: str, call_id: str
|
|
) -> tuple[bool, str | None]:
|
|
"""精确取最终成功 attempt，不猜本轮最后一条响应。"""
|
|
key = session_id, parent_call_id
|
|
if call_id in self._owners and self._owners[call_id] != key:
|
|
raise ValueError("跨轮 call_id 身份查询")
|
|
matches = [attempt for attempt in self._records.get(key, []) if attempt.call_id == call_id]
|
|
if len(matches) > 1:
|
|
raise ValueError("重复 call_id 身份查询")
|
|
if not matches:
|
|
return False, None
|
|
events = [event for event in matches[0].http if 200 <= event.status_code < 300]
|
|
if len(events) > 1:
|
|
raise ValueError("多个成功 HTTP 身份候选")
|
|
return events[0].raw_identity if events else (False, None)
|
|
|
|
|
|
class ObservedTransport:
|
|
"""原样委托同一个真实 transport，无重试、payload 修正或异常翻译。"""
|
|
|
|
def __init__(self, transport: OpenAICompatTransport, capture: LiveCapture) -> None:
|
|
"""资源所有权留给装配者。"""
|
|
self._transport = transport
|
|
self._capture = capture
|
|
|
|
async def complete(
|
|
self,
|
|
*,
|
|
messages: list[dict[str, Any]],
|
|
source: SourceConfig,
|
|
stream: bool,
|
|
overlay: dict[str, Any],
|
|
call_id: str,
|
|
reasoning_effort: Effort | None,
|
|
) -> TransportResult:
|
|
"""与生产端口逐参数同签名。"""
|
|
with self._capture.attempt_context(call_id):
|
|
self._capture.observe_messages(source, messages)
|
|
return await self._transport.complete(
|
|
messages=messages,
|
|
source=source,
|
|
stream=stream,
|
|
overlay=overlay,
|
|
call_id=call_id,
|
|
reasoning_effort=reasoning_effort,
|
|
)
|
|
|
|
async def embed(
|
|
self, *, texts: list[str], source: SourceConfig, call_id: str
|
|
) -> EmbeddingTransportResult:
|
|
"""embedding 使用同一取证关联，不套 chat 推理判据。"""
|
|
with self._capture.attempt_context(call_id):
|
|
return await self._transport.embed(texts=texts, source=source, call_id=call_id)
|
|
|
|
|
|
@asynccontextmanager
|
|
async def observed_client(
|
|
settings: GatewaySettings, capture: LiveCapture, *, capabilities=None
|
|
) -> AsyncIterator[GatewayClient]:
|
|
"""全量注入复用生产装配函数；自建组件显式关闭，不启用响应缓存。"""
|
|
async with AsyncExitStack() as stack:
|
|
sources = list(settings.sources)
|
|
limiter = _build_limiter(settings, sources)
|
|
stack.push_async_callback(_aclose_component, limiter)
|
|
breaker = _build_breaker(settings)
|
|
stack.push_async_callback(_aclose_component, breaker)
|
|
real = OpenAICompatTransport(
|
|
client_factory=capture.client_factory, capabilities=capabilities
|
|
)
|
|
stack.push_async_callback(real.aclose)
|
|
strategy, escalation = _build_structured(
|
|
[get_provider(source.provider) for source in sources]
|
|
)
|
|
client = GatewayClient(
|
|
scope=settings.scope,
|
|
sources=sources,
|
|
selector=_build_selector(settings.selector),
|
|
limiter=limiter,
|
|
breaker=breaker,
|
|
transport=ObservedTransport(real, capture),
|
|
retry=settings.retry,
|
|
backpressure=settings.backpressure,
|
|
quota_full=settings.quota_full,
|
|
circuit_open=settings.circuit_open,
|
|
structured_strategy=strategy,
|
|
structured_escalation=escalation,
|
|
structured_max_retries=settings.structured_max_retries,
|
|
)
|
|
stack.push_async_callback(client.aclose)
|
|
yield client
|
|
|
|
|
|
def chat_expectations(
|
|
settings: GatewaySettings,
|
|
*,
|
|
messages: list[dict[str, Any]],
|
|
stream: bool,
|
|
controls: Mapping[str, dict[str, Any]],
|
|
structured_max_retries: int | None = None,
|
|
) -> dict[str, dict[str, Any]]:
|
|
"""URL 从源配置声明，控制片段必须由矩阵独立给出。"""
|
|
result = {}
|
|
for source in settings.sources:
|
|
url = httpx.URL(source.base_url)
|
|
result[source.name] = {
|
|
"model": source.model,
|
|
"origin": str(
|
|
url.copy_with(path="", query=None, fragment=None, username=None, password=None)
|
|
).rstrip("/"),
|
|
"path": url.path.rstrip("/") + "/chat/completions",
|
|
"stream": stream,
|
|
"control": controls[source.name],
|
|
"messages_digest": messages_digest(messages),
|
|
}
|
|
if structured_max_retries is not None:
|
|
result[source.name].update(
|
|
messages_prefix_length=len(messages), structured_max_retries=structured_max_retries
|
|
)
|
|
return result
|
|
|
|
|
|
def enforce_verdict(verdict: LiveVerdict) -> None:
|
|
"""仅在报告已写入后调用；默认失败，不打印上游异常正文。"""
|
|
if verdict.status == "UNCOVERED":
|
|
pytest.skip(verdict.reason)
|
|
assert verdict.status == "PASS", verdict.reason
|
|
|
|
|
|
async def captured_chat_round(
|
|
client: GatewayClient,
|
|
capture: LiveCapture,
|
|
*,
|
|
run_id: str,
|
|
matrix_id: str,
|
|
round_index: int,
|
|
output_dir: Path,
|
|
messages: list[dict[str, Any]],
|
|
models: Mapping[str, str],
|
|
aliases: Mapping[str, frozenset[str]],
|
|
validate=None,
|
|
providers: Mapping[str, str] | None = None,
|
|
source_efforts: Mapping[str, Effort | None] | None = None,
|
|
**kwargs: Any,
|
|
) -> tuple[Any, LiveVerdict]:
|
|
"""请求、身份与行为断言均先记逐轮证据；异常不漏轮。"""
|
|
parent = uuid4().hex
|
|
response = None
|
|
error = None
|
|
verdict = LiveVerdict("FAIL", "轮次未完成")
|
|
with capture.round_context(session_id=run_id, parent_call_id=parent):
|
|
try:
|
|
response = await client.chat(
|
|
messages, session_id=run_id, parent_call_id=parent, **kwargs
|
|
)
|
|
attempts = capture.attempts(session_id=run_id, parent_call_id=parent)
|
|
events = [event for attempt in attempts for event in attempt.http]
|
|
successful = [attempt for attempt in attempts if attempt.call_id == response.call_id]
|
|
success_paired = (
|
|
len(successful) == 1
|
|
and successful[0].error is None
|
|
and len(successful[0].http) == 1
|
|
and 200 <= successful[0].http[0].status_code < 300
|
|
)
|
|
verdict = assess_model_identity(
|
|
requested=models[response.source_name],
|
|
aliases=aliases.get(models[response.source_name], frozenset()),
|
|
reported=response.model_reported,
|
|
raw_identity=capture.raw_identity(
|
|
session_id=run_id, parent_call_id=parent, call_id=response.call_id
|
|
),
|
|
request_valid=success_paired
|
|
and bool(events)
|
|
and all(request_is_valid(event) for event in events),
|
|
)
|
|
if verdict.status == "PASS" and validate is not None:
|
|
validate(response)
|
|
except Exception as exc:
|
|
error = exc
|
|
verdict = classify_live_failure(
|
|
exc, capture.attempts(session_id=run_id, parent_call_id=parent)
|
|
)
|
|
finally:
|
|
attempts = capture.attempts(session_id=run_id, parent_call_id=parent)
|
|
# 上游 model 可能回显提示词；仅输出允许集合内的名字，其他统一省略。
|
|
allowed = set(models.values()) | {
|
|
alias for values in aliases.values() for alias in values
|
|
}
|
|
write_live_round(
|
|
output_dir,
|
|
run_id=run_id,
|
|
matrix_id=matrix_id,
|
|
round_index=round_index,
|
|
safe_fields={
|
|
"session_id": run_id,
|
|
"parent_call_id": parent,
|
|
"requested_model": list(models.values()),
|
|
"provider": list(providers.values()) if providers is not None else None,
|
|
"attempts": safe_attempts(attempts),
|
|
"status": verdict.status,
|
|
"reason": verdict.reason,
|
|
"stream": kwargs.get("stream", True),
|
|
"requested_effort": kwargs.get("reasoning_effort")
|
|
or (list(source_efforts.values()) if source_efforts is not None else None),
|
|
"completed_rounds": 1,
|
|
"prompt_tokens": response.prompt_tokens if response else None,
|
|
"completion_tokens": response.completion_tokens if response else None,
|
|
"reasoning_tokens": response.reasoning_tokens if response else None,
|
|
"thinking_chars": len(response.thinking) if response else None,
|
|
"evidence_notes": (
|
|
"原始异常／正文摘要省略以避免回显泄露",
|
|
*capture.notes(session_id=run_id, parent_call_id=parent),
|
|
),
|
|
"reported_model": response.model_reported
|
|
if response and response.model_reported in allowed
|
|
else None,
|
|
"applied_effort": response.applied_effort if response else None,
|
|
"thinking_observation": response.thinking_observation if response else None,
|
|
"error_type": type(error).__name__ if error else None,
|
|
"error_status": getattr(error, "status_code", None),
|
|
},
|
|
)
|
|
return response, verdict
|
|
|
|
|
|
def declared_control(provider: str, effort: Effort | None) -> dict[str, Any]:
|
|
"""测试矩阵的独立 wire 声明；不调用 resolver 或生产 payload 构造器。"""
|
|
if effort is None:
|
|
return {}
|
|
if provider == "qwen":
|
|
if effort not in (Effort.AUTO, Effort.NONE):
|
|
raise ValueError("测试矩阵未声明 qwen 强度映射")
|
|
return {"enable_thinking": effort is not Effort.NONE}
|
|
if provider in {"deepseek", "zhipu", "moonshot"}:
|
|
result: dict[str, Any] = {
|
|
"thinking": {"type": "disabled" if effort is Effort.NONE else "enabled"}
|
|
}
|
|
if effort not in (Effort.AUTO, Effort.NONE):
|
|
result["reasoning_effort"] = effort.value
|
|
return result
|
|
if provider not in {"minimax", "openai", "anthropic", "google"}:
|
|
raise ValueError("测试矩阵没有该 provider 的控制声明")
|
|
return {} if effort is Effort.AUTO else {"reasoning_effort": effort.value}
|
|
|
|
|
|
def source_controls(settings: GatewaySettings) -> dict[str, dict[str, Any]]:
|
|
"""源级矩阵预期独立表达；受管 raw 冲突由生产路径拒绝。"""
|
|
result = {}
|
|
for source in settings.sources:
|
|
effort = source.reasoning_effort
|
|
if effort is None and source.enable_thinking is not None:
|
|
effort = Effort.AUTO if source.enable_thinking else Effort.NONE
|
|
control = declared_control(source.provider, effort)
|
|
if effort is None:
|
|
control.update(source.extra_body)
|
|
else:
|
|
# 不用 update 覆盖控制声明，否则会掩盖所有权回归。
|
|
for key, value in source.extra_body.items():
|
|
if key in control:
|
|
raise ValueError("取证矩阵有双来源控制")
|
|
control[key] = value
|
|
result[source.name] = control
|
|
return result
|
|
|
|
|
|
# pytest 用例终态补充网络前缺配置、装配失败及未完成轮次；不代替逐轮报告。
|
|
@pytest.hookimpl(wrapper=True)
|
|
def pytest_runtest_makereport(item, call):
|
|
"""仅记录安全矩阵标识和阶段结果，不序列化 pytest 异常长文本。"""
|
|
report = yield
|
|
if report.skipped or report.failed:
|
|
write_live_round(
|
|
Path("tests/outputs/134/live"),
|
|
run_id=uuid4().hex,
|
|
matrix_id="pytest-" + messages_digest(item.nodeid)[:16],
|
|
round_index=0,
|
|
safe_fields={
|
|
"status": "UNCOVERED" if report.skipped else "FAIL",
|
|
"reason": "用例阶段未覆盖或失败；详情按逐轮安全证据核验，不能视为能力通过",
|
|
"evidence_notes": [report.when],
|
|
},
|
|
)
|
|
return report
|