test: complete embedding probe report identity and round counts
This commit is contained in:
@@ -76,6 +76,10 @@ async def test_probe_real_gateway_embeddings():
|
||||
matrix_id="embedding",
|
||||
round_index=1,
|
||||
safe_fields={
|
||||
"requested_model": source.model,
|
||||
"provider": source.provider,
|
||||
"planned_rounds": 1,
|
||||
"completed_rounds": 1,
|
||||
"status": verdict.status,
|
||||
"reason": verdict.reason,
|
||||
"session_id": run_id,
|
||||
|
||||
@@ -684,6 +684,135 @@ async def test_round_consumer_keeps_first_success_when_second_assertion_fails(tm
|
||||
assert clients and all(client.is_closed for client in clients)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["embedding-override-a", "embedding-override-b"])
|
||||
@pytest.mark.parametrize(
|
||||
("outcome", "status", "error_type"),
|
||||
[
|
||||
("success", "PASS", None),
|
||||
("503", "FAIL", "TransientError"),
|
||||
("404", "UNCOVERED", "RequestRejectedError"),
|
||||
("request_error", "FAIL", "TransientError"),
|
||||
],
|
||||
)
|
||||
async def test_embed_probe_report_keeps_actual_source_and_round_identity(
|
||||
tmp_path, monkeypatch, model, outcome, status, error_type
|
||||
):
|
||||
"""真实探测消费者在成功与失败均留完整关联;只替换外部 HTTP。"""
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
from tests.unit.test_config import _BASE_ENV
|
||||
|
||||
path = Path(__file__).parents[1] / "e2e/test_embed_probe.py"
|
||||
tree = ast.parse(path.read_text())
|
||||
tree.body = [
|
||||
node
|
||||
for node in tree.body
|
||||
if not (
|
||||
isinstance(node, ast.Assign)
|
||||
and any(
|
||||
isinstance(target, ast.Name) and target.id in {"_ENV", "pytestmark"}
|
||||
for target in node.targets
|
||||
)
|
||||
)
|
||||
]
|
||||
env = {
|
||||
**_BASE_ENV,
|
||||
"LLM__MINIMAX__1__BASE_URL": "https://example.test/v1",
|
||||
"LLM__MINIMAX__1__API_KEY": _SECRET,
|
||||
"LLM__MINIMAX__1__MODEL": "configured-chat-model",
|
||||
"LLM__MINIMAX__1__TIMEOUT_S": "137",
|
||||
"LLM__MINIMAX__1__TRUST_ENV": "false",
|
||||
"PGW_EMBED_PROBE_MODEL": model,
|
||||
}
|
||||
namespace: dict[str, Any] = {"_ENV": env}
|
||||
exec(compile(tree, str(path), "exec"), namespace)
|
||||
monkeypatch.chdir(tmp_path)
|
||||
original_factory = LiveCapture.client_factory
|
||||
clients = []
|
||||
calls = []
|
||||
|
||||
def factory(capture, source):
|
||||
"""保留真实取证 hooks、源与逻辑 ID,只隔离网络出口。"""
|
||||
client = original_factory(capture, source)
|
||||
|
||||
def handler(request):
|
||||
"""提供完整成功/错误样本,敏感回显不得进入报告。"""
|
||||
payload = json.loads(request.content)
|
||||
assert payload["model"] == model == source.model
|
||||
assert source.model != env["LLM__MINIMAX__1__MODEL"]
|
||||
assert request.headers["Authorization"] == f"Bearer {_SECRET}"
|
||||
calls.append((source, capture._round.get(), capture._attempt.get().call_id))
|
||||
if outcome == "request_error":
|
||||
raise httpx.ConnectError(_SECRET + _PROMPT, request=request)
|
||||
if outcome == "success":
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"data": [{"index": 0, "embedding": [0.1, 0.2]}],
|
||||
"usage": {"prompt_tokens": 1},
|
||||
"model": _SECRET + _PROMPT,
|
||||
},
|
||||
)
|
||||
return httpx.Response(
|
||||
int(outcome),
|
||||
json={"error": {"type": "model_not_found", "message": _SECRET + _PROMPT}},
|
||||
)
|
||||
|
||||
client._transport = httpx.MockTransport(handler)
|
||||
clients.append(client)
|
||||
return client
|
||||
|
||||
monkeypatch.setattr(LiveCapture, "client_factory", factory)
|
||||
probe = namespace["test_probe_real_gateway_embeddings"]
|
||||
if status == "PASS":
|
||||
await probe()
|
||||
elif status == "UNCOVERED":
|
||||
with pytest.raises(pytest.skip.Exception):
|
||||
await probe()
|
||||
else:
|
||||
with pytest.raises(AssertionError):
|
||||
await probe()
|
||||
|
||||
assert len(calls) == len(clients) == 1
|
||||
assert all(client.is_closed for client in clients)
|
||||
source, (session_id, parent_call_id), call_id = calls[0]
|
||||
paths = list((tmp_path / "tests/outputs/134/live").rglob("*.md"))
|
||||
assert len(paths) == 1
|
||||
text = paths[0].read_text()
|
||||
assert _SECRET not in text and _PROMPT not in text
|
||||
row = json.loads(text.split("```json\n")[1].split("\n```")[0])
|
||||
assert row["status"] == status
|
||||
assert row["requested_model"] == source.model == model
|
||||
assert row["provider"] == source.provider == "minimax"
|
||||
assert row["planned_rounds"] == row["completed_rounds"] == 1
|
||||
assert row["session_id"] == paths[0].parent.name == session_id
|
||||
assert row["parent_call_id"] == parent_call_id
|
||||
assert len({session_id, parent_call_id, call_id}) == 3
|
||||
assert paths[0].name.startswith("embedding-1-")
|
||||
assert text.startswith("# embedding · 轮次 1\n")
|
||||
assert len(row["attempts"]) == 1
|
||||
attempt = row["attempts"][0]
|
||||
assert attempt["call_id"] == call_id and attempt["error_type"] == error_type
|
||||
assert len(attempt["http"]) == 1
|
||||
event = attempt["http"][0]
|
||||
assert all(event["request_checks"].values())
|
||||
assert (
|
||||
event["status_code"]
|
||||
== {
|
||||
"success": 200,
|
||||
"503": 503,
|
||||
"404": 404,
|
||||
"request_error": 0,
|
||||
}[outcome]
|
||||
)
|
||||
if outcome in {"503", "404"}:
|
||||
assert event["error_body_complete"] is True
|
||||
assert event["machine_type"] == "model_not_found"
|
||||
elif outcome == "request_error":
|
||||
assert "无可配对响应" in row["evidence_notes"]
|
||||
|
||||
|
||||
def test_partial_uncovered_and_failed_rounds_never_become_model_pass():
|
||||
from tests.live_evidence import combine_live_verdicts, qualify_live_rounds
|
||||
|
||||
|
||||
Reference in New Issue
Block a user