117 lines
3.7 KiB
Python
117 lines
3.7 KiB
Python
"""真实网关冒烟：逐轮独立取证，行为断言失败也必须留档。"""
|
|
|
|
import os
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from dotenv import dotenv_values
|
|
from pydantic import BaseModel
|
|
|
|
from polygateway import Effort, GatewaySettings
|
|
from tests.e2e.conftest import (
|
|
LiveCapture,
|
|
captured_chat_round,
|
|
chat_expectations,
|
|
enforce_verdict,
|
|
observed_client,
|
|
source_controls,
|
|
)
|
|
|
|
_ENV = {k: v for k, v in {**dotenv_values(".env"), **os.environ}.items() if v is not None}
|
|
_HAS_SOURCE = any(k.startswith("LLM__") and k.endswith("__API_KEY") for k in _ENV)
|
|
pytestmark = [
|
|
pytest.mark.slow,
|
|
pytest.mark.skipif(not _HAS_SOURCE, reason="缺少矩阵必需凭据，未覆盖"),
|
|
]
|
|
|
|
|
|
class MiniAnswer(BaseModel):
|
|
"""最小结构化响应契约。"""
|
|
|
|
answer: int
|
|
reason: str
|
|
|
|
|
|
async def _smoke(matrix, prompt, validate, *, stream=True, structured=None):
|
|
"""全量注入仅替换取证装配，仍调用生产结构化策略。"""
|
|
settings = GatewaySettings.from_env("LLM", env=_ENV)
|
|
messages = [{"role": "user", "content": prompt}]
|
|
controls = source_controls(settings)
|
|
capture = LiveCapture(
|
|
expectations=chat_expectations(
|
|
settings,
|
|
messages=messages,
|
|
stream=stream,
|
|
controls=controls,
|
|
structured_max_retries=(
|
|
settings.structured_max_retries if isinstance(structured, type) else None
|
|
),
|
|
)
|
|
)
|
|
async with observed_client(settings, capture) as client:
|
|
_, verdict = await captured_chat_round(
|
|
client,
|
|
capture,
|
|
run_id=uuid4().hex,
|
|
matrix_id=matrix,
|
|
round_index=1,
|
|
output_dir=Path("tests/outputs/134/live"),
|
|
messages=messages,
|
|
models={s.name: s.model for s in settings.sources},
|
|
providers={s.name: s.provider for s in settings.sources},
|
|
source_efforts={
|
|
s.name: s.reasoning_effort
|
|
if s.reasoning_effort is not None
|
|
else (Effort.AUTO if s.enable_thinking else Effort.NONE)
|
|
if s.enable_thinking is not None
|
|
else None
|
|
for s in settings.sources
|
|
},
|
|
aliases={},
|
|
validate=validate,
|
|
stream=stream,
|
|
structured=structured,
|
|
)
|
|
enforce_verdict(verdict)
|
|
|
|
|
|
class TestRealGatewaySmoke:
|
|
"""保留流／非流和结构化真实行为断言。"""
|
|
|
|
async def test_stream_chat(self):
|
|
def validate(response):
|
|
assert response.content.strip()
|
|
assert response.ttft_ms is not None and response.latency_ms > 0
|
|
|
|
await _smoke("smoke-stream", "Reply with exactly: pong", validate)
|
|
|
|
async def test_non_stream_fast_path(self):
|
|
def validate(response):
|
|
assert response.content.strip() and response.ttft_ms is None
|
|
|
|
await _smoke("smoke-json", "Reply with exactly: pong", validate, stream=False)
|
|
|
|
async def test_structured_json_tier(self):
|
|
def validate(response):
|
|
assert isinstance(response.structured_data, dict | list)
|
|
|
|
await _smoke(
|
|
"smoke-structured-json",
|
|
'Reply ONLY with JSON: {"ok": true}',
|
|
validate,
|
|
structured="json",
|
|
)
|
|
|
|
async def test_structured_model_ladder(self):
|
|
def validate(response):
|
|
assert isinstance(response.structured_data, MiniAnswer)
|
|
assert response.structured_data.answer == 5
|
|
|
|
await _smoke(
|
|
"smoke-structured-model",
|
|
'What is 2+3? Reply ONLY with JSON matching {"answer": <int>, "reason": <short string>}',
|
|
validate,
|
|
structured=MiniAnswer,
|
|
)
|