test: apply evidence-based live checks without hiding regressions
This commit is contained in:
@@ -0,0 +1,496 @@
|
||||
"""测试侧独立 HTTP 取证装配；无环境自读取或成功 SSE 预读。"""
|
||||
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from contextlib import AsyncExitStack, asynccontextmanager, contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from polygateway import GatewayClient, GatewaySettings
|
||||
from polygateway.client import (
|
||||
_aclose_component,
|
||||
_build_breaker,
|
||||
_build_limiter,
|
||||
_build_selector,
|
||||
_build_structured,
|
||||
)
|
||||
from polygateway.providers import get_provider
|
||||
from polygateway.transports.openai_compat import OpenAICompatTransport
|
||||
from polygateway.types import Effort, EmbeddingTransportResult, SourceConfig, TransportResult
|
||||
from tests.live_evidence import (
|
||||
AttemptEvidence,
|
||||
HttpEvidence,
|
||||
LiveVerdict,
|
||||
assess_model_identity,
|
||||
classify_live_failure,
|
||||
messages_digest,
|
||||
request_is_valid,
|
||||
safe_attempts,
|
||||
strict_json,
|
||||
write_live_round,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Exchange:
|
||||
"""仅在 attempt 生命周期持有原始响应引用。"""
|
||||
|
||||
request: httpx.Request
|
||||
checks: tuple[tuple[str, bool], ...]
|
||||
response: httpx.Response | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Attempt:
|
||||
"""任务内可变收集器，结束时转换为冻结快照。"""
|
||||
|
||||
call_id: str
|
||||
exchanges: list[_Exchange] = field(default_factory=list)
|
||||
|
||||
|
||||
class LiveCapture:
|
||||
"""矩阵显式预期与按逻辑轮次关联的独立证据。"""
|
||||
|
||||
def __init__(self, *, expectations: Mapping[str, Mapping[str, Any]]) -> None:
|
||||
"""预期缺项即配置错误，不从实发 payload 补齐。"""
|
||||
for expected in expectations.values():
|
||||
required = {"model", "origin", "path", "control", "messages_digest"}
|
||||
if not required <= expected.keys() or ("stream" in expected) == (
|
||||
"input_shape" in expected
|
||||
):
|
||||
raise ValueError("取证矩阵缺少必需预期或混用 chat/embed")
|
||||
if not isinstance(expected["control"], dict):
|
||||
raise ValueError("control 必须是显式对象")
|
||||
self._expectations = {name: dict(value) for name, value in expectations.items()}
|
||||
self._round: ContextVar[tuple[str, str]] = ContextVar("live_round")
|
||||
self._attempt: ContextVar[_Attempt] = ContextVar("live_attempt")
|
||||
self._records: dict[tuple[str, str], list[AttemptEvidence]] = {}
|
||||
self._owners: dict[str, tuple[str, str]] = {}
|
||||
self._notes: dict[tuple[str, str], list[str]] = {}
|
||||
|
||||
@contextmanager
|
||||
def round_context(self, *, session_id: str, parent_call_id: str) -> Iterator[None]:
|
||||
"""外围逻辑轮次绑定，异常和取消均复位。"""
|
||||
key = session_id, parent_call_id
|
||||
token = self._round.set(key)
|
||||
self._records.setdefault(key, [])
|
||||
self._notes.setdefault(key, [])
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._round.reset(token)
|
||||
|
||||
@contextmanager
|
||||
def attempt_context(self, call_id: str) -> Iterator[None]:
|
||||
"""零 HTTP 尝试也有快照；重复／跨轮 UUID 是契约错误。"""
|
||||
key = self._round.get()
|
||||
if call_id in self._owners:
|
||||
raise ValueError("重复或跨轮 call_id")
|
||||
self._owners[call_id] = key
|
||||
attempt = _Attempt(call_id)
|
||||
token = self._attempt.set(attempt)
|
||||
error = None
|
||||
try:
|
||||
yield
|
||||
except Exception as exc:
|
||||
error = exc
|
||||
raise
|
||||
finally:
|
||||
try:
|
||||
events = tuple(
|
||||
self._snapshot(exchange, call_id, key) for exchange in attempt.exchanges
|
||||
)
|
||||
self._records[key].append(AttemptEvidence(call_id, events, error))
|
||||
finally:
|
||||
self._attempt.reset(token)
|
||||
|
||||
def _snapshot(self, exchange: _Exchange, call_id: str, key: tuple[str, str]) -> HttpEvidence:
|
||||
"""complete 结束后只读已缓冲内容，拒绝截断和歧义身份。"""
|
||||
response = exchange.response
|
||||
identity: tuple[bool, str | None] = (False, None)
|
||||
body = None
|
||||
status = response.status_code if response is not None else 0
|
||||
if response is None:
|
||||
self._notes[key].append("无可配对响应")
|
||||
else:
|
||||
try:
|
||||
content = response.content
|
||||
except httpx.ResponseNotRead:
|
||||
self._notes[key].append("响应未缓冲，证据不足")
|
||||
else:
|
||||
if status >= 400:
|
||||
if len(content) <= 65536:
|
||||
body = content
|
||||
else:
|
||||
self._notes[key].append("错误体超过 64 KiB，不接受截断证据")
|
||||
elif 200 <= status < 300 and dict(exchange.checks).get("stream") is not None:
|
||||
try:
|
||||
payload = strict_json(exchange.request.content)
|
||||
except (ValueError, UnicodeError):
|
||||
payload = {}
|
||||
if isinstance(payload, dict) and payload.get("stream") is False:
|
||||
try:
|
||||
data = strict_json(content)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("原始 JSON 非对象")
|
||||
model = data.get("model")
|
||||
if model is not None and not isinstance(model, str):
|
||||
raise ValueError("原始 model 类型非法")
|
||||
identity = (True, model)
|
||||
except (ValueError, UnicodeError):
|
||||
self._notes[key].append("原始 JSON 身份无法独立解析")
|
||||
return HttpEvidence(call_id, exchange.checks, status, body, identity)
|
||||
|
||||
def client_factory(self, source: SourceConfig) -> httpx.AsyncClient:
|
||||
"""鉴权仅内存比较；沿已校验源 timeout/trust_env。"""
|
||||
expected = self._expectations[source.name]
|
||||
|
||||
async def request_hook(request: httpx.Request) -> None:
|
||||
"""校验实发请求而不修正它。"""
|
||||
attempt = self._attempt.get()
|
||||
try:
|
||||
payload = strict_json(request.content)
|
||||
except (ValueError, UnicodeError):
|
||||
payload = {}
|
||||
if not isinstance(payload, dict):
|
||||
payload = {}
|
||||
url = request.url
|
||||
origin = str(
|
||||
url.copy_with(path="", query=None, fragment=None, username=None, password=None)
|
||||
).rstrip("/")
|
||||
# control 是本轮完整附加字段；基础键以外均比较，漏／多键都失败。
|
||||
basic = {"model", "messages", "stream", "stream_options", "input"}
|
||||
control = {k: v for k, v in payload.items() if k not in basic}
|
||||
checks = {
|
||||
"method": request.method == "POST",
|
||||
"origin": origin == expected["origin"]
|
||||
and not url.username
|
||||
and not url.password
|
||||
and not url.query,
|
||||
"path": url.path == expected["path"],
|
||||
"model": payload.get("model") == expected["model"],
|
||||
"authorization": request.headers.get("Authorization") == f"Bearer {source.api_key}",
|
||||
"control": control == expected["control"],
|
||||
"messages_digest": messages_digest(payload.get("messages", payload.get("input")))
|
||||
== expected["messages_digest"],
|
||||
}
|
||||
if "stream" in expected:
|
||||
checks["stream"] = payload.get("stream") is expected["stream"] and (
|
||||
payload.get("stream_options") == {"include_usage": True}
|
||||
if expected["stream"]
|
||||
else "stream_options" not in payload
|
||||
)
|
||||
else:
|
||||
texts = payload.get("input")
|
||||
checks["input_shape"] = (
|
||||
isinstance(texts, list)
|
||||
and all(isinstance(text, str) for text in texts)
|
||||
and len(texts) == expected["input_shape"]
|
||||
)
|
||||
attempt.exchanges.append(_Exchange(request, tuple(checks.items())))
|
||||
|
||||
async def response_hook(response: httpx.Response) -> None:
|
||||
"""只持有引用，绝不提前读取成功 SSE。"""
|
||||
attempt = self._attempt.get()
|
||||
matching = [event for event in attempt.exchanges if event.request is response.request]
|
||||
if len(matching) != 1 or matching[0].response is not None:
|
||||
raise ValueError("响应无法唯一配对")
|
||||
matching[0].response = response
|
||||
|
||||
return httpx.AsyncClient(
|
||||
headers={"Authorization": f"Bearer {source.api_key}"},
|
||||
timeout=source.timeout_s,
|
||||
trust_env=source.trust_env,
|
||||
event_hooks={"request": [request_hook], "response": [response_hook]},
|
||||
)
|
||||
|
||||
def attempts(self, *, session_id: str, parent_call_id: str) -> tuple[AttemptEvidence, ...]:
|
||||
"""按逻辑轮次返回不可变快照。"""
|
||||
return tuple(self._records.get((session_id, parent_call_id), ()))
|
||||
|
||||
def notes(self, *, session_id: str, parent_call_id: str) -> tuple[str, ...]:
|
||||
"""只含固定安全原因，不包含响应正文。"""
|
||||
return tuple(self._notes.get((session_id, parent_call_id), ()))
|
||||
|
||||
def raw_identity(
|
||||
self, *, session_id: str, parent_call_id: str, call_id: str
|
||||
) -> tuple[bool, str | None]:
|
||||
"""精确取最终成功 attempt，不猜本轮最后一条响应。"""
|
||||
key = session_id, parent_call_id
|
||||
if call_id in self._owners and self._owners[call_id] != key:
|
||||
raise ValueError("跨轮 call_id 身份查询")
|
||||
matches = [attempt for attempt in self._records.get(key, []) if attempt.call_id == call_id]
|
||||
if len(matches) > 1:
|
||||
raise ValueError("重复 call_id 身份查询")
|
||||
if not matches:
|
||||
return False, None
|
||||
events = [event for event in matches[0].http if 200 <= event.status_code < 300]
|
||||
if len(events) > 1:
|
||||
raise ValueError("多个成功 HTTP 身份候选")
|
||||
return events[0].raw_identity if events else (False, None)
|
||||
|
||||
|
||||
class ObservedTransport:
|
||||
"""原样委托同一个真实 transport，无重试、payload 修正或异常翻译。"""
|
||||
|
||||
def __init__(self, transport: OpenAICompatTransport, capture: LiveCapture) -> None:
|
||||
"""资源所有权留给装配者。"""
|
||||
self._transport = transport
|
||||
self._capture = capture
|
||||
|
||||
async def complete(
|
||||
self,
|
||||
*,
|
||||
messages: list[dict[str, Any]],
|
||||
source: SourceConfig,
|
||||
stream: bool,
|
||||
overlay: dict[str, Any],
|
||||
call_id: str,
|
||||
reasoning_effort: Effort | None,
|
||||
) -> TransportResult:
|
||||
"""与生产端口逐参数同签名。"""
|
||||
with self._capture.attempt_context(call_id):
|
||||
return await self._transport.complete(
|
||||
messages=messages,
|
||||
source=source,
|
||||
stream=stream,
|
||||
overlay=overlay,
|
||||
call_id=call_id,
|
||||
reasoning_effort=reasoning_effort,
|
||||
)
|
||||
|
||||
async def embed(
|
||||
self, *, texts: list[str], source: SourceConfig, call_id: str
|
||||
) -> EmbeddingTransportResult:
|
||||
"""embedding 使用同一取证关联，不套 chat 推理判据。"""
|
||||
with self._capture.attempt_context(call_id):
|
||||
return await self._transport.embed(texts=texts, source=source, call_id=call_id)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def observed_client(
|
||||
settings: GatewaySettings, capture: LiveCapture, *, capabilities=None
|
||||
) -> AsyncIterator[GatewayClient]:
|
||||
"""全量注入复用生产装配函数；自建组件显式关闭，不启用响应缓存。"""
|
||||
async with AsyncExitStack() as stack:
|
||||
sources = list(settings.sources)
|
||||
limiter = _build_limiter(settings, sources)
|
||||
stack.push_async_callback(_aclose_component, limiter)
|
||||
breaker = _build_breaker(settings)
|
||||
stack.push_async_callback(_aclose_component, breaker)
|
||||
real = OpenAICompatTransport(
|
||||
client_factory=capture.client_factory, capabilities=capabilities
|
||||
)
|
||||
stack.push_async_callback(real.aclose)
|
||||
strategy, escalation = _build_structured(
|
||||
[get_provider(source.provider) for source in sources]
|
||||
)
|
||||
client = GatewayClient(
|
||||
scope=settings.scope,
|
||||
sources=sources,
|
||||
selector=_build_selector(settings.selector),
|
||||
limiter=limiter,
|
||||
breaker=breaker,
|
||||
transport=ObservedTransport(real, capture),
|
||||
retry=settings.retry,
|
||||
backpressure=settings.backpressure,
|
||||
quota_full=settings.quota_full,
|
||||
circuit_open=settings.circuit_open,
|
||||
structured_strategy=strategy,
|
||||
structured_escalation=escalation,
|
||||
structured_max_retries=settings.structured_max_retries,
|
||||
)
|
||||
stack.push_async_callback(client.aclose)
|
||||
yield client
|
||||
|
||||
|
||||
def chat_expectations(
|
||||
settings: GatewaySettings,
|
||||
*,
|
||||
messages: list[dict[str, Any]],
|
||||
stream: bool,
|
||||
controls: Mapping[str, dict[str, Any]],
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""URL 从源配置声明，控制片段必须由矩阵独立给出。"""
|
||||
result = {}
|
||||
for source in settings.sources:
|
||||
url = httpx.URL(source.base_url)
|
||||
result[source.name] = {
|
||||
"model": source.model,
|
||||
"origin": str(
|
||||
url.copy_with(path="", query=None, fragment=None, username=None, password=None)
|
||||
).rstrip("/"),
|
||||
"path": url.path.rstrip("/") + "/chat/completions",
|
||||
"stream": stream,
|
||||
"control": controls[source.name],
|
||||
"messages_digest": messages_digest(messages),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def enforce_verdict(verdict: LiveVerdict) -> None:
|
||||
"""仅在报告已写入后调用；默认失败，不打印上游异常正文。"""
|
||||
if verdict.status == "UNCOVERED":
|
||||
pytest.skip(verdict.reason)
|
||||
assert verdict.status == "PASS", verdict.reason
|
||||
|
||||
|
||||
async def captured_chat_round(
|
||||
client: GatewayClient,
|
||||
capture: LiveCapture,
|
||||
*,
|
||||
run_id: str,
|
||||
matrix_id: str,
|
||||
round_index: int,
|
||||
output_dir: Path,
|
||||
messages: list[dict[str, Any]],
|
||||
models: Mapping[str, str],
|
||||
aliases: Mapping[str, frozenset[str]],
|
||||
validate=None,
|
||||
providers: Mapping[str, str] | None = None,
|
||||
source_efforts: Mapping[str, Effort | None] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> tuple[Any, LiveVerdict]:
|
||||
"""请求、身份与行为断言均先记逐轮证据；异常不漏轮。"""
|
||||
parent = uuid4().hex
|
||||
response = None
|
||||
error = None
|
||||
verdict = LiveVerdict("FAIL", "轮次未完成")
|
||||
with capture.round_context(session_id=run_id, parent_call_id=parent):
|
||||
try:
|
||||
response = await client.chat(
|
||||
messages, session_id=run_id, parent_call_id=parent, **kwargs
|
||||
)
|
||||
attempts = capture.attempts(session_id=run_id, parent_call_id=parent)
|
||||
events = [event for attempt in attempts for event in attempt.http]
|
||||
successful = [attempt for attempt in attempts if attempt.call_id == response.call_id]
|
||||
success_paired = (
|
||||
len(successful) == 1
|
||||
and successful[0].error is None
|
||||
and len(successful[0].http) == 1
|
||||
and 200 <= successful[0].http[0].status_code < 300
|
||||
)
|
||||
verdict = assess_model_identity(
|
||||
requested=models[response.source_name],
|
||||
aliases=aliases.get(models[response.source_name], frozenset()),
|
||||
reported=response.model_reported,
|
||||
raw_identity=capture.raw_identity(
|
||||
session_id=run_id, parent_call_id=parent, call_id=response.call_id
|
||||
),
|
||||
request_valid=success_paired
|
||||
and bool(events)
|
||||
and all(request_is_valid(event) for event in events),
|
||||
)
|
||||
if verdict.status == "PASS" and validate is not None:
|
||||
validate(response)
|
||||
except Exception as exc:
|
||||
error = exc
|
||||
verdict = classify_live_failure(
|
||||
exc, capture.attempts(session_id=run_id, parent_call_id=parent)
|
||||
)
|
||||
finally:
|
||||
attempts = capture.attempts(session_id=run_id, parent_call_id=parent)
|
||||
# 上游 model 可能回显提示词；仅输出允许集合内的名字，其他统一省略。
|
||||
allowed = set(models.values()) | {
|
||||
alias for values in aliases.values() for alias in values
|
||||
}
|
||||
write_live_round(
|
||||
output_dir,
|
||||
run_id=run_id,
|
||||
matrix_id=matrix_id,
|
||||
round_index=round_index,
|
||||
safe_fields={
|
||||
"session_id": run_id,
|
||||
"parent_call_id": parent,
|
||||
"requested_model": list(models.values()),
|
||||
"provider": list(providers.values()) if providers is not None else None,
|
||||
"attempts": safe_attempts(attempts),
|
||||
"status": verdict.status,
|
||||
"reason": verdict.reason,
|
||||
"stream": kwargs.get("stream", True),
|
||||
"requested_effort": kwargs.get("reasoning_effort")
|
||||
or (list(source_efforts.values()) if source_efforts is not None else None),
|
||||
"completed_rounds": 1,
|
||||
"prompt_tokens": response.prompt_tokens if response else None,
|
||||
"completion_tokens": response.completion_tokens if response else None,
|
||||
"reasoning_tokens": response.reasoning_tokens if response else None,
|
||||
"thinking_chars": len(response.thinking) if response else None,
|
||||
"evidence_notes": (
|
||||
"原始异常／正文摘要省略以避免回显泄露",
|
||||
*capture.notes(session_id=run_id, parent_call_id=parent),
|
||||
),
|
||||
"reported_model": response.model_reported
|
||||
if response and response.model_reported in allowed
|
||||
else None,
|
||||
"applied_effort": response.applied_effort if response else None,
|
||||
"thinking_observation": response.thinking_observation if response else None,
|
||||
"error_type": type(error).__name__ if error else None,
|
||||
"error_status": getattr(error, "status_code", None),
|
||||
},
|
||||
)
|
||||
return response, verdict
|
||||
|
||||
|
||||
def declared_control(provider: str, effort: Effort | None) -> dict[str, Any]:
|
||||
"""测试矩阵的独立 wire 声明；不调用 resolver 或生产 payload 构造器。"""
|
||||
if effort is None:
|
||||
return {}
|
||||
if provider == "qwen":
|
||||
if effort not in (Effort.AUTO, Effort.NONE):
|
||||
raise ValueError("测试矩阵未声明 qwen 强度映射")
|
||||
return {"enable_thinking": effort is not Effort.NONE}
|
||||
if provider in {"deepseek", "zhipu", "moonshot"}:
|
||||
result: dict[str, Any] = {
|
||||
"thinking": {"type": "disabled" if effort is Effort.NONE else "enabled"}
|
||||
}
|
||||
if effort not in (Effort.AUTO, Effort.NONE):
|
||||
result["reasoning_effort"] = effort.value
|
||||
return result
|
||||
if provider not in {"minimax", "openai", "anthropic", "google"}:
|
||||
raise ValueError("测试矩阵没有该 provider 的控制声明")
|
||||
return {} if effort is Effort.AUTO else {"reasoning_effort": effort.value}
|
||||
|
||||
|
||||
def source_controls(settings: GatewaySettings) -> dict[str, dict[str, Any]]:
|
||||
"""源级矩阵预期独立表达；受管 raw 冲突由生产路径拒绝。"""
|
||||
result = {}
|
||||
for source in settings.sources:
|
||||
effort = source.reasoning_effort
|
||||
if effort is None and source.enable_thinking is not None:
|
||||
effort = Effort.AUTO if source.enable_thinking else Effort.NONE
|
||||
control = declared_control(source.provider, effort)
|
||||
if effort is None:
|
||||
control.update(source.extra_body)
|
||||
else:
|
||||
# 不用 update 覆盖控制声明，否则会掩盖所有权回归。
|
||||
for key, value in source.extra_body.items():
|
||||
if key in control:
|
||||
raise ValueError("取证矩阵有双来源控制")
|
||||
control[key] = value
|
||||
result[source.name] = control
|
||||
return result
|
||||
|
||||
|
||||
# pytest 用例终态补充网络前缺配置、装配失败及未完成轮次；不代替逐轮报告。
|
||||
@pytest.hookimpl(wrapper=True)
|
||||
def pytest_runtest_makereport(item, call):
|
||||
"""仅记录安全矩阵标识和阶段结果，不序列化 pytest 异常长文本。"""
|
||||
report = yield
|
||||
if report.skipped or report.failed:
|
||||
write_live_round(
|
||||
Path("tests/outputs/134/live"),
|
||||
run_id=uuid4().hex,
|
||||
matrix_id="pytest-" + messages_digest(item.nodeid)[:16],
|
||||
round_index=0,
|
||||
safe_fields={
|
||||
"status": "UNCOVERED" if report.skipped else "FAIL",
|
||||
"reason": "用例阶段未覆盖或失败；详情按逐轮安全证据核验，不能视为能力通过",
|
||||
"evidence_notes": [report.when],
|
||||
},
|
||||
)
|
||||
return report
|
||||
Reference in New Issue
Block a user