test: complete embedding probe report identity and round counts

This commit is contained in:
2026-09-09 05:06:39 -04:00
parent 3eb22d2a55
commit 7f6a824e79
3 changed files with 160 additions and 0 deletions
@@ -142,3 +142,30 @@ date: 2026-09-09
上表节点命令也均加 `conda run --no-capture-output -n PolyGateway`;未使用缺 import 的伪红。最终受影响文件129单测通过,相比111新增18节点。pi-lens仍提示非conda缺httpx/pytest/pydantic及StrEnum等旧噪音,按已批准方向记录后继续conda门;新增测试命名空间显式 Any 类型,不抑制真实错误。
**尚未完成**:修复后独立复审、集成/make test覆盖率、真实能力取证、下游迁移和发布;T8/T9保持未勾选。M2空wire、M3非流UNKNOWN、真实机器拒绝白名单及外部服务状态不因离线绿变成已覆盖。
## 重启恢复与 embedding 报告补漏(起点 3eb22d2
恢复时实际分支为 `feature/1.3.4-thinking-contracts`HEAD=`3eb22d2`,已跟踪工作区无差异,仅既有 `.pi/` 未跟踪;前轮代码提交仍在,重启未丢代码。按用户限定只修测试与本 finding,不动生产 API/版本/计划,不联网、不付费、不运行 slow、不派子代理。本节为实现者验证,不冒充独立复审或版本验收。
`tests/outputs/134/slow-gate.log` 保留,大小 1625 字节;`slow-gate.exit` 不存在,属于重启中断、未取得终态,不能宣称 slow 通过。修复前后 SHA-256 均为 `05967dcf749b13db815dd80449a0db5b3e7ad2144aa2ae216f8c9ab5ecb21bf5`。历史日志不覆盖、不续写,也不把旧 full-gate 作为本轮测试证据。
| 根因/范围 | 本轮修复与证据 |
| --- | --- |
| embedding 消费者遗漏四个字段 | 仅在原 finally 报告出口增加 `requested_model=source.model``provider=source.provider``planned_rounds=1``completed_rounds=1`;完成计数指已收尾轮次,FAIL/UNCOVERED 也计入,不代表成功。沿用原 matrixroundsessionparentattempt 关联,不新建报告框架 |
| 环境可覆盖请求型号 | 既有 `test_live_evidence.py` 增加真实 probe 消费者回归:AST 仅剔除 `.env`pytestmark 顶层读取,合成环境实际经过 GatewaySettings;两种 probe 型号覆盖都与原 chat 型号不同,HTTP 与报告必须等于本次 source,不能拿默认型号占位 |
| 成功与所有目标错误路径 | 真实 OpenAICompatTransportLiveCaptureObservedTransport+报告写入;仅 HTTP 边界 MockTransport。两型号×成功/503/严格404ConnectError 共8节点,分别 PASSFAILUNCOVEREDFAIL;同时断言唯一报告、逻辑 UUID/attempt 配对、请求校验、状态/错误类型与客户端关闭 |
| 安全边界 | 假凭据及私有提示词 sentinel 放入成功 model 回显、错误正文和请求异常;逐份 Markdown 断言不泄漏,仍只写安全机器枚举/固定原因,不复制原始正文与异常 |
日志均在 `tests/outputs/134/`,命令不接管道,先保存真实退出码到同名 `.exit` 再展示输出。
| 命令(pytest 前缀均为 `conda run --no-capture-output -n PolyGateway` | 实际结果/日志 |
| --- | --- |
| `pytest tests/unit/test_live_evidence.py -k embed_probe_report -q`(修复前) | `embed-report-red.log/.exit`8 failed129 deselectedexit 1;八例均在真实报告消费处 `KeyError: requested_model`,不是 importmock 签名失败 |
| 同命令(四字段补齐后) | `embed-report-green.log/.exit`8 passed129 deselectedexit 0 |
| `pytest tests/unit/test_live_evidence.py tests/unit/test_embedding.py tests/unit/test_openai_compat.py::TestDefaultClientFactory -q` | `embed-report-affected.log/.exit`188 passedexit 0 |
| `pytest tests/unit/ -q` | `embed-report-unit.log/.exit`1383 passedexit 0;格式化后 `embed-report-final-unit.log/.exit`1383 passed4.45秒,exit 0 |
| `make check` | 首次 `embed-report-check.log/.exit` 为新断言排版失败(exit 2),不是行为红;仅对该测试文件运行 conda ruff format。`embed-report-check-green.log/.exit`:94文件格式合格、ruff通过、import-linter 1 kept0 brokenexit 0 |
pi-lens 仍报非 conda 解释器缺 httpxpytestdotenvpydantic 及旧 StrEnum 噪音;按任务授权记录,不添加 ignore、不改枚举、不扩环境修复范围。实际 conda 解释器为 `/home/iomgaa/miniconda3/envs/PolyGateway/bin/python`,本会话导入四依赖成功(httpx 0.28.1、pytest 9.1.1、python-dotenv 1.2.3、pydantic 2.13.4)。conda 启动器自身另有 base Python 3.13 的 RequestsDependencyWarning;未静音,不宣称输出零告警,测试进程与静态门实际退出0。
本修复不补写真正缺失的历史报告、不改变 embedding 能力判据;真实服务、slow、下游与发布证据仍由后续验收负责。
+4
View File
@@ -76,6 +76,10 @@ async def test_probe_real_gateway_embeddings():
matrix_id="embedding",
round_index=1,
safe_fields={
"requested_model": source.model,
"provider": source.provider,
"planned_rounds": 1,
"completed_rounds": 1,
"status": verdict.status,
"reason": verdict.reason,
"session_id": run_id,
+129
View File
@@ -684,6 +684,135 @@ async def test_round_consumer_keeps_first_success_when_second_assertion_fails(tm
assert clients and all(client.is_closed for client in clients)
@pytest.mark.parametrize("model", ["embedding-override-a", "embedding-override-b"])
@pytest.mark.parametrize(
("outcome", "status", "error_type"),
[
("success", "PASS", None),
("503", "FAIL", "TransientError"),
("404", "UNCOVERED", "RequestRejectedError"),
("request_error", "FAIL", "TransientError"),
],
)
async def test_embed_probe_report_keeps_actual_source_and_round_identity(
tmp_path, monkeypatch, model, outcome, status, error_type
):
"""真实探测消费者在成功与失败均留完整关联;只替换外部 HTTP。"""
import ast
from pathlib import Path
from tests.unit.test_config import _BASE_ENV
path = Path(__file__).parents[1] / "e2e/test_embed_probe.py"
tree = ast.parse(path.read_text())
tree.body = [
node
for node in tree.body
if not (
isinstance(node, ast.Assign)
and any(
isinstance(target, ast.Name) and target.id in {"_ENV", "pytestmark"}
for target in node.targets
)
)
]
env = {
**_BASE_ENV,
"LLM__MINIMAX__1__BASE_URL": "https://example.test/v1",
"LLM__MINIMAX__1__API_KEY": _SECRET,
"LLM__MINIMAX__1__MODEL": "configured-chat-model",
"LLM__MINIMAX__1__TIMEOUT_S": "137",
"LLM__MINIMAX__1__TRUST_ENV": "false",
"PGW_EMBED_PROBE_MODEL": model,
}
namespace: dict[str, Any] = {"_ENV": env}
exec(compile(tree, str(path), "exec"), namespace)
monkeypatch.chdir(tmp_path)
original_factory = LiveCapture.client_factory
clients = []
calls = []
def factory(capture, source):
"""保留真实取证 hooks、源与逻辑 ID,只隔离网络出口。"""
client = original_factory(capture, source)
def handler(request):
"""提供完整成功/错误样本,敏感回显不得进入报告。"""
payload = json.loads(request.content)
assert payload["model"] == model == source.model
assert source.model != env["LLM__MINIMAX__1__MODEL"]
assert request.headers["Authorization"] == f"Bearer {_SECRET}"
calls.append((source, capture._round.get(), capture._attempt.get().call_id))
if outcome == "request_error":
raise httpx.ConnectError(_SECRET + _PROMPT, request=request)
if outcome == "success":
return httpx.Response(
200,
json={
"data": [{"index": 0, "embedding": [0.1, 0.2]}],
"usage": {"prompt_tokens": 1},
"model": _SECRET + _PROMPT,
},
)
return httpx.Response(
int(outcome),
json={"error": {"type": "model_not_found", "message": _SECRET + _PROMPT}},
)
client._transport = httpx.MockTransport(handler)
clients.append(client)
return client
monkeypatch.setattr(LiveCapture, "client_factory", factory)
probe = namespace["test_probe_real_gateway_embeddings"]
if status == "PASS":
await probe()
elif status == "UNCOVERED":
with pytest.raises(pytest.skip.Exception):
await probe()
else:
with pytest.raises(AssertionError):
await probe()
assert len(calls) == len(clients) == 1
assert all(client.is_closed for client in clients)
source, (session_id, parent_call_id), call_id = calls[0]
paths = list((tmp_path / "tests/outputs/134/live").rglob("*.md"))
assert len(paths) == 1
text = paths[0].read_text()
assert _SECRET not in text and _PROMPT not in text
row = json.loads(text.split("```json\n")[1].split("\n```")[0])
assert row["status"] == status
assert row["requested_model"] == source.model == model
assert row["provider"] == source.provider == "minimax"
assert row["planned_rounds"] == row["completed_rounds"] == 1
assert row["session_id"] == paths[0].parent.name == session_id
assert row["parent_call_id"] == parent_call_id
assert len({session_id, parent_call_id, call_id}) == 3
assert paths[0].name.startswith("embedding-1-")
assert text.startswith("# embedding · 轮次 1\n")
assert len(row["attempts"]) == 1
attempt = row["attempts"][0]
assert attempt["call_id"] == call_id and attempt["error_type"] == error_type
assert len(attempt["http"]) == 1
event = attempt["http"][0]
assert all(event["request_checks"].values())
assert (
event["status_code"]
== {
"success": 200,
"503": 503,
"404": 404,
"request_error": 0,
}[outcome]
)
if outcome in {"503", "404"}:
assert event["error_body_complete"] is True
assert event["machine_type"] == "model_not_found"
elif outcome == "request_error":
assert "无可配对响应" in row["evidence_notes"]
def test_partial_uncovered_and_failed_rounds_never_become_model_pass():
from tests.live_evidence import combine_live_verdicts, qualify_live_rounds