test: apply evidence-based live checks without hiding regressions
This commit is contained in:
@@ -1,89 +1,91 @@
|
||||
"""真实网关 /embeddings 端点探测(M2 设计 §11.6;人类默认口径: 实现时探测)。
|
||||
|
||||
对 .env 的 LLM 源网关发一次真实 embeddings 请求: 支持则记录向量证据,
|
||||
不支持(404/翻译为领域错误)则 skip 并把响应记录进 tests/outputs/
|
||||
(降级证据)。无 EMBED scope 配置时复用 LLM 源的 base_url/api_key。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
"""真实 embedding 探测；404 仅证明请求型号不可用，不外推端点能力。"""
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from dotenv import dotenv_values
|
||||
|
||||
from polygateway.errors import PolyGatewayError
|
||||
from polygateway import GatewaySettings
|
||||
from polygateway.transports.openai_compat import OpenAICompatTransport
|
||||
from polygateway.types import SourceConfig
|
||||
from tests.e2e.conftest import LiveCapture, ObservedTransport, enforce_verdict
|
||||
from tests.live_evidence import (
|
||||
LiveVerdict,
|
||||
classify_live_failure,
|
||||
messages_digest,
|
||||
request_is_valid,
|
||||
safe_attempts,
|
||||
write_live_round,
|
||||
)
|
||||
|
||||
_ENV = {k: v for k, v in {**dotenv_values(".env"), **os.environ}.items() if v is not None}
|
||||
|
||||
# 真实网关调用: 与 test_thinking_live.py 同待遇标 slow(pytest addopts 默认排除,
|
||||
# 显式 `pytest -m slow` 运行)。理由见 test_compat_projects.py 同处注释。
|
||||
pytestmark = [
|
||||
pytest.mark.slow,
|
||||
pytest.mark.skipif(
|
||||
"LLM__MINIMAX__1__BASE_URL" not in _ENV,
|
||||
reason="缺真实网关配置(.env)",
|
||||
),
|
||||
pytest.mark.skipif("LLM__MINIMAX__1__BASE_URL" not in _ENV, reason="缺少矩阵必需配置，未覆盖"),
|
||||
]
|
||||
|
||||
_OUT = Path("tests/outputs/embedding")
|
||||
|
||||
|
||||
def _record(name: str, lines: list[str]) -> Path:
|
||||
_OUT.mkdir(parents=True, exist_ok=True)
|
||||
path = _OUT / f"{name}_{datetime.now():%Y%m%d_%H%M%S}.md"
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
async def test_probe_real_gateway_embeddings():
|
||||
source = SourceConfig(
|
||||
name="probe_1",
|
||||
provider="minimax",
|
||||
base_url=_ENV["LLM__MINIMAX__1__BASE_URL"],
|
||||
api_key=_ENV["LLM__MINIMAX__1__API_KEY"],
|
||||
model=_ENV.get("PGW_EMBED_PROBE_MODEL", "text-embedding-v1"),
|
||||
timeout_s=30.0,
|
||||
est_tokens=8,
|
||||
"""沿已校验源的 timeout/trust_env，所有路径 finally 关闭。"""
|
||||
settings = GatewaySettings.from_env("LLM", env=_ENV)
|
||||
configured = next(s for s in settings.sources if s.name == "minimax_1")
|
||||
source = dataclasses.replace(
|
||||
configured, model=_ENV.get("PGW_EMBED_PROBE_MODEL", "text-embedding-v1")
|
||||
)
|
||||
transport = OpenAICompatTransport()
|
||||
texts = ["polygateway embedding probe"]
|
||||
url = httpx.URL(source.base_url)
|
||||
capture = LiveCapture(
|
||||
expectations={
|
||||
source.name: {
|
||||
"model": source.model,
|
||||
"origin": str(url.copy_with(path="", query=None)).rstrip("/"),
|
||||
"path": url.path.rstrip("/") + "/embeddings",
|
||||
"input_shape": 1,
|
||||
"control": {},
|
||||
"messages_digest": messages_digest(texts),
|
||||
}
|
||||
}
|
||||
)
|
||||
real = OpenAICompatTransport(client_factory=capture.client_factory)
|
||||
transport = ObservedTransport(real, capture)
|
||||
run_id, parent, call_id = uuid4().hex, uuid4().hex, uuid4().hex
|
||||
verdict = LiveVerdict("FAIL", "轮次未完成")
|
||||
try:
|
||||
result = await transport.embed(
|
||||
texts=["polygateway embedding probe"], source=source, call_id="probe"
|
||||
)
|
||||
except PolyGatewayError as exc:
|
||||
path = _record(
|
||||
"probe_unsupported",
|
||||
[
|
||||
"# Embedding 端点探测: 网关不支持",
|
||||
f"- base_url: {source.base_url}",
|
||||
f"- model: {source.model}",
|
||||
f"- 错误分类: {type(exc).__name__}",
|
||||
f"- status_code: {exc.status_code}",
|
||||
f"- 详情: {exc}",
|
||||
"",
|
||||
"结论: e2e 按设计 §11.6 降级,embedding 行为由 unit 全覆盖。",
|
||||
],
|
||||
)
|
||||
await transport.aclose()
|
||||
pytest.skip(f"网关不支持 embeddings({type(exc).__name__}),证据: {path}")
|
||||
else:
|
||||
await transport.aclose()
|
||||
assert result.dim > 0 and len(result.vectors) == 1
|
||||
_record(
|
||||
"probe_supported",
|
||||
[
|
||||
"# Embedding 端点探测: 网关支持",
|
||||
f"- base_url: {source.base_url}",
|
||||
f"- model: {source.model}",
|
||||
f"- dim: {result.dim}",
|
||||
f"- usage: {result.prompt_tokens}({result.usage_source})",
|
||||
f"- 向量前 5 维: {result.vectors[0][:5]}",
|
||||
f"- raw: {dataclasses.asdict(result)['raw']}",
|
||||
],
|
||||
)
|
||||
with capture.round_context(session_id=run_id, parent_call_id=parent):
|
||||
try:
|
||||
result = await transport.embed(texts=texts, source=source, call_id=call_id)
|
||||
events = [
|
||||
e
|
||||
for a in capture.attempts(session_id=run_id, parent_call_id=parent)
|
||||
for e in a.http
|
||||
]
|
||||
assert len(events) == 1 and request_is_valid(events[0])
|
||||
assert result.dim > 0 and len(result.vectors) == 1
|
||||
verdict = LiveVerdict("PASS", "向量形状与实发请求合格")
|
||||
except Exception as error:
|
||||
verdict = classify_live_failure(
|
||||
error, capture.attempts(session_id=run_id, parent_call_id=parent)
|
||||
)
|
||||
finally:
|
||||
write_live_round(
|
||||
Path("tests/outputs/134/live"),
|
||||
run_id=run_id,
|
||||
matrix_id="embedding",
|
||||
round_index=1,
|
||||
safe_fields={
|
||||
"status": verdict.status,
|
||||
"reason": verdict.reason,
|
||||
"session_id": run_id,
|
||||
"parent_call_id": parent,
|
||||
"attempts": safe_attempts(
|
||||
capture.attempts(session_id=run_id, parent_call_id=parent)
|
||||
),
|
||||
"evidence_notes": capture.notes(session_id=run_id, parent_call_id=parent),
|
||||
},
|
||||
)
|
||||
finally:
|
||||
await real.aclose()
|
||||
enforce_verdict(verdict)
|
||||
|
||||
Reference in New Issue
Block a user