test: complete embedding probe report identity and round counts

This commit is contained in:
2026-09-09 05:06:39 -04:00
parent 3eb22d2a55
commit 7f6a824e79
3 changed files with 160 additions and 0 deletions
+4
View File
@@ -76,6 +76,10 @@ async def test_probe_real_gateway_embeddings():
matrix_id="embedding",
round_index=1,
safe_fields={
"requested_model": source.model,
"provider": source.provider,
"planned_rounds": 1,
"completed_rounds": 1,
"status": verdict.status,
"reason": verdict.reason,
"session_id": run_id,
+129
View File
@@ -684,6 +684,135 @@ async def test_round_consumer_keeps_first_success_when_second_assertion_fails(tm
assert clients and all(client.is_closed for client in clients)
@pytest.mark.parametrize("model", ["embedding-override-a", "embedding-override-b"])
@pytest.mark.parametrize(
("outcome", "status", "error_type"),
[
("success", "PASS", None),
("503", "FAIL", "TransientError"),
("404", "UNCOVERED", "RequestRejectedError"),
("request_error", "FAIL", "TransientError"),
],
)
async def test_embed_probe_report_keeps_actual_source_and_round_identity(
tmp_path, monkeypatch, model, outcome, status, error_type
):
"""真实探测消费者在成功与失败均留完整关联;只替换外部 HTTP。"""
import ast
from pathlib import Path
from tests.unit.test_config import _BASE_ENV
path = Path(__file__).parents[1] / "e2e/test_embed_probe.py"
tree = ast.parse(path.read_text())
tree.body = [
node
for node in tree.body
if not (
isinstance(node, ast.Assign)
and any(
isinstance(target, ast.Name) and target.id in {"_ENV", "pytestmark"}
for target in node.targets
)
)
]
env = {
**_BASE_ENV,
"LLM__MINIMAX__1__BASE_URL": "https://example.test/v1",
"LLM__MINIMAX__1__API_KEY": _SECRET,
"LLM__MINIMAX__1__MODEL": "configured-chat-model",
"LLM__MINIMAX__1__TIMEOUT_S": "137",
"LLM__MINIMAX__1__TRUST_ENV": "false",
"PGW_EMBED_PROBE_MODEL": model,
}
namespace: dict[str, Any] = {"_ENV": env}
exec(compile(tree, str(path), "exec"), namespace)
monkeypatch.chdir(tmp_path)
original_factory = LiveCapture.client_factory
clients = []
calls = []
def factory(capture, source):
"""保留真实取证 hooks、源与逻辑 ID,只隔离网络出口。"""
client = original_factory(capture, source)
def handler(request):
"""提供完整成功/错误样本,敏感回显不得进入报告。"""
payload = json.loads(request.content)
assert payload["model"] == model == source.model
assert source.model != env["LLM__MINIMAX__1__MODEL"]
assert request.headers["Authorization"] == f"Bearer {_SECRET}"
calls.append((source, capture._round.get(), capture._attempt.get().call_id))
if outcome == "request_error":
raise httpx.ConnectError(_SECRET + _PROMPT, request=request)
if outcome == "success":
return httpx.Response(
200,
json={
"data": [{"index": 0, "embedding": [0.1, 0.2]}],
"usage": {"prompt_tokens": 1},
"model": _SECRET + _PROMPT,
},
)
return httpx.Response(
int(outcome),
json={"error": {"type": "model_not_found", "message": _SECRET + _PROMPT}},
)
client._transport = httpx.MockTransport(handler)
clients.append(client)
return client
monkeypatch.setattr(LiveCapture, "client_factory", factory)
probe = namespace["test_probe_real_gateway_embeddings"]
if status == "PASS":
await probe()
elif status == "UNCOVERED":
with pytest.raises(pytest.skip.Exception):
await probe()
else:
with pytest.raises(AssertionError):
await probe()
assert len(calls) == len(clients) == 1
assert all(client.is_closed for client in clients)
source, (session_id, parent_call_id), call_id = calls[0]
paths = list((tmp_path / "tests/outputs/134/live").rglob("*.md"))
assert len(paths) == 1
text = paths[0].read_text()
assert _SECRET not in text and _PROMPT not in text
row = json.loads(text.split("```json\n")[1].split("\n```")[0])
assert row["status"] == status
assert row["requested_model"] == source.model == model
assert row["provider"] == source.provider == "minimax"
assert row["planned_rounds"] == row["completed_rounds"] == 1
assert row["session_id"] == paths[0].parent.name == session_id
assert row["parent_call_id"] == parent_call_id
assert len({session_id, parent_call_id, call_id}) == 3
assert paths[0].name.startswith("embedding-1-")
assert text.startswith("# embedding · 轮次 1\n")
assert len(row["attempts"]) == 1
attempt = row["attempts"][0]
assert attempt["call_id"] == call_id and attempt["error_type"] == error_type
assert len(attempt["http"]) == 1
event = attempt["http"][0]
assert all(event["request_checks"].values())
assert (
event["status_code"]
== {
"success": 200,
"503": 503,
"404": 404,
"request_error": 0,
}[outcome]
)
if outcome in {"503", "404"}:
assert event["error_body_complete"] is True
assert event["machine_type"] == "model_not_found"
elif outcome == "request_error":
assert "无可配对响应" in row["evidence_notes"]
def test_partial_uncovered_and_failed_rounds_never_become_model_pass():
from tests.live_evidence import combine_live_verdicts, qualify_live_rounds