test: complete embedding probe report identity and round counts
This commit is contained in:
@@ -142,3 +142,30 @@ date: 2026-09-09
|
|||||||
上表节点命令也均加 `conda run --no-capture-output -n PolyGateway`;未使用缺 import 的伪红。最终受影响文件129单测通过,相比111新增18节点。pi-lens仍提示非conda缺httpx/pytest/pydantic及StrEnum等旧噪音,按已批准方向记录后继续conda门;新增测试命名空间显式 Any 类型,不抑制真实错误。
|
上表节点命令也均加 `conda run --no-capture-output -n PolyGateway`;未使用缺 import 的伪红。最终受影响文件129单测通过,相比111新增18节点。pi-lens仍提示非conda缺httpx/pytest/pydantic及StrEnum等旧噪音,按已批准方向记录后继续conda门;新增测试命名空间显式 Any 类型,不抑制真实错误。
|
||||||
|
|
||||||
**尚未完成**:修复后独立复审、集成/make test覆盖率、真实能力取证、下游迁移和发布;T8/T9保持未勾选。M2空wire、M3非流UNKNOWN、真实机器拒绝白名单及外部服务状态不因离线绿变成已覆盖。
|
**尚未完成**:修复后独立复审、集成/make test覆盖率、真实能力取证、下游迁移和发布;T8/T9保持未勾选。M2空wire、M3非流UNKNOWN、真实机器拒绝白名单及外部服务状态不因离线绿变成已覆盖。
|
||||||
|
|
||||||
|
## 重启恢复与 embedding 报告补漏(起点 3eb22d2)
|
||||||
|
|
||||||
|
恢复时实际分支为 `feature/1.3.4-thinking-contracts`,HEAD=`3eb22d2`,已跟踪工作区无差异,仅既有 `.pi/` 未跟踪;前轮代码提交仍在,重启未丢代码。按用户限定只修测试与本 finding,不动生产 API/版本/计划,不联网、不付费、不运行 slow、不派子代理。本节为实现者验证,不冒充独立复审或版本验收。
|
||||||
|
|
||||||
|
原 `tests/outputs/134/slow-gate.log` 保留,大小 1625 字节;`slow-gate.exit` 不存在,属于重启中断、未取得终态,不能宣称 slow 通过。修复前后 SHA-256 均为 `05967dcf749b13db815dd80449a0db5b3e7ad2144aa2ae216f8c9ab5ecb21bf5`。历史日志不覆盖、不续写,也不把旧 full-gate 作为本轮测试证据。
|
||||||
|
|
||||||
|
| 根因/范围 | 本轮修复与证据 |
|
||||||
|
| --- | --- |
|
||||||
|
| embedding 消费者遗漏四个字段 | 仅在原 finally 报告出口增加 `requested_model=source.model`、`provider=source.provider`、`planned_rounds=1`、`completed_rounds=1`;完成计数指已收尾轮次,FAIL/UNCOVERED 也计入,不代表成功。沿用原 matrix/round/session/parent/attempt 关联,不新建报告框架 |
|
||||||
|
| 环境可覆盖请求型号 | 既有 `test_live_evidence.py` 增加真实 probe 消费者回归:AST 仅剔除 `.env`/pytestmark 顶层读取,合成环境实际经过 GatewaySettings;两种 probe 型号覆盖都与原 chat 型号不同,HTTP 与报告必须等于本次 source,不能拿默认型号占位 |
|
||||||
|
| 成功与所有目标错误路径 | 真实 OpenAICompatTransport+LiveCapture+ObservedTransport+报告写入;仅 HTTP 边界 MockTransport。两型号×成功/503/严格404/ConnectError 共8节点,分别 PASS/FAIL/UNCOVERED/FAIL;同时断言唯一报告、逻辑 UUID/attempt 配对、请求校验、状态/错误类型与客户端关闭 |
|
||||||
|
| 安全边界 | 假凭据及私有提示词 sentinel 放入成功 model 回显、错误正文和请求异常;逐份 Markdown 断言不泄漏,仍只写安全机器枚举/固定原因,不复制原始正文与异常 |
|
||||||
|
|
||||||
|
日志均在 `tests/outputs/134/`,命令不接管道,先保存真实退出码到同名 `.exit` 再展示输出。
|
||||||
|
|
||||||
|
| 命令(pytest 前缀均为 `conda run --no-capture-output -n PolyGateway`) | 实际结果/日志 |
|
||||||
|
| --- | --- |
|
||||||
|
| `pytest tests/unit/test_live_evidence.py -k embed_probe_report -q`(修复前) | `embed-report-red.log/.exit`:8 failed,129 deselected,exit 1;八例均在真实报告消费处 `KeyError: requested_model`,不是 import/mock 签名失败 |
|
||||||
|
| 同命令(四字段补齐后) | `embed-report-green.log/.exit`:8 passed,129 deselected,exit 0 |
|
||||||
|
| `pytest tests/unit/test_live_evidence.py tests/unit/test_embedding.py tests/unit/test_openai_compat.py::TestDefaultClientFactory -q` | `embed-report-affected.log/.exit`:188 passed,exit 0 |
|
||||||
|
| `pytest tests/unit/ -q` | `embed-report-unit.log/.exit`:1383 passed,exit 0;格式化后 `embed-report-final-unit.log/.exit`:1383 passed,4.45秒,exit 0 |
|
||||||
|
| `make check` | 首次 `embed-report-check.log/.exit` 为新断言排版失败(exit 2),不是行为红;仅对该测试文件运行 conda ruff format。`embed-report-check-green.log/.exit`:94文件格式合格、ruff通过、import-linter 1 kept/0 broken,exit 0 |
|
||||||
|
|
||||||
|
pi-lens 仍报非 conda 解释器缺 httpx/pytest/dotenv/pydantic 及旧 StrEnum 噪音;按任务授权记录,不添加 ignore、不改枚举、不扩环境修复范围。实际 conda 解释器为 `/home/iomgaa/miniconda3/envs/PolyGateway/bin/python`,本会话导入四依赖成功(httpx 0.28.1、pytest 9.1.1、python-dotenv 1.2.3、pydantic 2.13.4)。conda 启动器自身另有 base Python 3.13 的 RequestsDependencyWarning;未静音,不宣称输出零告警,测试进程与静态门实际退出0。
|
||||||
|
|
||||||
|
本修复不补写真正缺失的历史报告、不改变 embedding 能力判据;真实服务、slow、下游与发布证据仍由后续验收负责。
|
||||||
|
|||||||
@@ -76,6 +76,10 @@ async def test_probe_real_gateway_embeddings():
|
|||||||
matrix_id="embedding",
|
matrix_id="embedding",
|
||||||
round_index=1,
|
round_index=1,
|
||||||
safe_fields={
|
safe_fields={
|
||||||
|
"requested_model": source.model,
|
||||||
|
"provider": source.provider,
|
||||||
|
"planned_rounds": 1,
|
||||||
|
"completed_rounds": 1,
|
||||||
"status": verdict.status,
|
"status": verdict.status,
|
||||||
"reason": verdict.reason,
|
"reason": verdict.reason,
|
||||||
"session_id": run_id,
|
"session_id": run_id,
|
||||||
|
|||||||
@@ -684,6 +684,135 @@ async def test_round_consumer_keeps_first_success_when_second_assertion_fails(tm
|
|||||||
assert clients and all(client.is_closed for client in clients)
|
assert clients and all(client.is_closed for client in clients)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("model", ["embedding-override-a", "embedding-override-b"])
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("outcome", "status", "error_type"),
|
||||||
|
[
|
||||||
|
("success", "PASS", None),
|
||||||
|
("503", "FAIL", "TransientError"),
|
||||||
|
("404", "UNCOVERED", "RequestRejectedError"),
|
||||||
|
("request_error", "FAIL", "TransientError"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
async def test_embed_probe_report_keeps_actual_source_and_round_identity(
|
||||||
|
tmp_path, monkeypatch, model, outcome, status, error_type
|
||||||
|
):
|
||||||
|
"""真实探测消费者在成功与失败均留完整关联;只替换外部 HTTP。"""
|
||||||
|
import ast
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from tests.unit.test_config import _BASE_ENV
|
||||||
|
|
||||||
|
path = Path(__file__).parents[1] / "e2e/test_embed_probe.py"
|
||||||
|
tree = ast.parse(path.read_text())
|
||||||
|
tree.body = [
|
||||||
|
node
|
||||||
|
for node in tree.body
|
||||||
|
if not (
|
||||||
|
isinstance(node, ast.Assign)
|
||||||
|
and any(
|
||||||
|
isinstance(target, ast.Name) and target.id in {"_ENV", "pytestmark"}
|
||||||
|
for target in node.targets
|
||||||
|
)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
env = {
|
||||||
|
**_BASE_ENV,
|
||||||
|
"LLM__MINIMAX__1__BASE_URL": "https://example.test/v1",
|
||||||
|
"LLM__MINIMAX__1__API_KEY": _SECRET,
|
||||||
|
"LLM__MINIMAX__1__MODEL": "configured-chat-model",
|
||||||
|
"LLM__MINIMAX__1__TIMEOUT_S": "137",
|
||||||
|
"LLM__MINIMAX__1__TRUST_ENV": "false",
|
||||||
|
"PGW_EMBED_PROBE_MODEL": model,
|
||||||
|
}
|
||||||
|
namespace: dict[str, Any] = {"_ENV": env}
|
||||||
|
exec(compile(tree, str(path), "exec"), namespace)
|
||||||
|
monkeypatch.chdir(tmp_path)
|
||||||
|
original_factory = LiveCapture.client_factory
|
||||||
|
clients = []
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def factory(capture, source):
|
||||||
|
"""保留真实取证 hooks、源与逻辑 ID,只隔离网络出口。"""
|
||||||
|
client = original_factory(capture, source)
|
||||||
|
|
||||||
|
def handler(request):
|
||||||
|
"""提供完整成功/错误样本,敏感回显不得进入报告。"""
|
||||||
|
payload = json.loads(request.content)
|
||||||
|
assert payload["model"] == model == source.model
|
||||||
|
assert source.model != env["LLM__MINIMAX__1__MODEL"]
|
||||||
|
assert request.headers["Authorization"] == f"Bearer {_SECRET}"
|
||||||
|
calls.append((source, capture._round.get(), capture._attempt.get().call_id))
|
||||||
|
if outcome == "request_error":
|
||||||
|
raise httpx.ConnectError(_SECRET + _PROMPT, request=request)
|
||||||
|
if outcome == "success":
|
||||||
|
return httpx.Response(
|
||||||
|
200,
|
||||||
|
json={
|
||||||
|
"data": [{"index": 0, "embedding": [0.1, 0.2]}],
|
||||||
|
"usage": {"prompt_tokens": 1},
|
||||||
|
"model": _SECRET + _PROMPT,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return httpx.Response(
|
||||||
|
int(outcome),
|
||||||
|
json={"error": {"type": "model_not_found", "message": _SECRET + _PROMPT}},
|
||||||
|
)
|
||||||
|
|
||||||
|
client._transport = httpx.MockTransport(handler)
|
||||||
|
clients.append(client)
|
||||||
|
return client
|
||||||
|
|
||||||
|
monkeypatch.setattr(LiveCapture, "client_factory", factory)
|
||||||
|
probe = namespace["test_probe_real_gateway_embeddings"]
|
||||||
|
if status == "PASS":
|
||||||
|
await probe()
|
||||||
|
elif status == "UNCOVERED":
|
||||||
|
with pytest.raises(pytest.skip.Exception):
|
||||||
|
await probe()
|
||||||
|
else:
|
||||||
|
with pytest.raises(AssertionError):
|
||||||
|
await probe()
|
||||||
|
|
||||||
|
assert len(calls) == len(clients) == 1
|
||||||
|
assert all(client.is_closed for client in clients)
|
||||||
|
source, (session_id, parent_call_id), call_id = calls[0]
|
||||||
|
paths = list((tmp_path / "tests/outputs/134/live").rglob("*.md"))
|
||||||
|
assert len(paths) == 1
|
||||||
|
text = paths[0].read_text()
|
||||||
|
assert _SECRET not in text and _PROMPT not in text
|
||||||
|
row = json.loads(text.split("```json\n")[1].split("\n```")[0])
|
||||||
|
assert row["status"] == status
|
||||||
|
assert row["requested_model"] == source.model == model
|
||||||
|
assert row["provider"] == source.provider == "minimax"
|
||||||
|
assert row["planned_rounds"] == row["completed_rounds"] == 1
|
||||||
|
assert row["session_id"] == paths[0].parent.name == session_id
|
||||||
|
assert row["parent_call_id"] == parent_call_id
|
||||||
|
assert len({session_id, parent_call_id, call_id}) == 3
|
||||||
|
assert paths[0].name.startswith("embedding-1-")
|
||||||
|
assert text.startswith("# embedding · 轮次 1\n")
|
||||||
|
assert len(row["attempts"]) == 1
|
||||||
|
attempt = row["attempts"][0]
|
||||||
|
assert attempt["call_id"] == call_id and attempt["error_type"] == error_type
|
||||||
|
assert len(attempt["http"]) == 1
|
||||||
|
event = attempt["http"][0]
|
||||||
|
assert all(event["request_checks"].values())
|
||||||
|
assert (
|
||||||
|
event["status_code"]
|
||||||
|
== {
|
||||||
|
"success": 200,
|
||||||
|
"503": 503,
|
||||||
|
"404": 404,
|
||||||
|
"request_error": 0,
|
||||||
|
}[outcome]
|
||||||
|
)
|
||||||
|
if outcome in {"503", "404"}:
|
||||||
|
assert event["error_body_complete"] is True
|
||||||
|
assert event["machine_type"] == "model_not_found"
|
||||||
|
elif outcome == "request_error":
|
||||||
|
assert "无可配对响应" in row["evidence_notes"]
|
||||||
|
|
||||||
|
|
||||||
def test_partial_uncovered_and_failed_rounds_never_become_model_pass():
|
def test_partial_uncovered_and_failed_rounds_never_become_model_pass():
|
||||||
from tests.live_evidence import combine_live_verdicts, qualify_live_rounds
|
from tests.live_evidence import combine_live_verdicts, qualify_live_rounds
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user