diff --git a/research-wiki/findings/2026-09-09-134-thinking-contracts-validation.md b/research-wiki/findings/2026-09-09-134-thinking-contracts-validation.md index 5f9b1a5..79c97b4 100644 --- a/research-wiki/findings/2026-09-09-134-thinking-contracts-validation.md +++ b/research-wiki/findings/2026-09-09-134-thinking-contracts-validation.md @@ -142,3 +142,30 @@ date: 2026-09-09 上表节点命令也均加 `conda run --no-capture-output -n PolyGateway`;未使用缺 import 的伪红。最终受影响文件129单测通过,相比111新增18节点。pi-lens仍提示非conda缺httpx/pytest/pydantic及StrEnum等旧噪音,按已批准方向记录后继续conda门;新增测试命名空间显式 Any 类型,不抑制真实错误。 **尚未完成**:修复后独立复审、集成/make test覆盖率、真实能力取证、下游迁移和发布;T8/T9保持未勾选。M2空wire、M3非流UNKNOWN、真实机器拒绝白名单及外部服务状态不因离线绿变成已覆盖。 + +## 重启恢复与 embedding 报告补漏(起点 3eb22d2) + +恢复时实际分支为 `feature/1.3.4-thinking-contracts`,HEAD=`3eb22d2`,已跟踪工作区无差异,仅既有 `.pi/` 未跟踪;前轮代码提交仍在,重启未丢代码。按用户限定只修测试与本 finding,不动生产 API/版本/计划,不联网、不付费、不运行 slow、不派子代理。本节为实现者验证,不冒充独立复审或版本验收。 + +原 `tests/outputs/134/slow-gate.log` 保留,大小 1625 字节;`slow-gate.exit` 不存在,属于重启中断、未取得终态,不能宣称 slow 通过。修复前后 SHA-256 均为 `05967dcf749b13db815dd80449a0db5b3e7ad2144aa2ae216f8c9ab5ecb21bf5`。历史日志不覆盖、不续写,也不把旧 full-gate 作为本轮测试证据。 + +| 根因/范围 | 本轮修复与证据 | +| --- | --- | +| embedding 消费者遗漏四个字段 | 仅在原 finally 报告出口增加 `requested_model=source.model`、`provider=source.provider`、`planned_rounds=1`、`completed_rounds=1`;完成计数指已收尾轮次,FAIL/UNCOVERED 也计入,不代表成功。沿用原 matrix/round/session/parent/attempt 关联,不新建报告框架 | +| 环境可覆盖请求型号 | 既有 `test_live_evidence.py` 增加真实 probe 消费者回归:AST 仅剔除 `.env`/pytestmark 顶层读取,合成环境实际经过 GatewaySettings;两种 probe 型号覆盖都与原 chat 型号不同,HTTP 与报告必须等于本次 source,不能拿默认型号占位 | +| 成功与所有目标错误路径 | 真实 OpenAICompatTransport+LiveCapture+ObservedTransport+报告写入;仅 HTTP 边界 MockTransport。两型号×成功/503/严格404/ConnectError 共8节点,分别 PASS/FAIL/UNCOVERED/FAIL;同时断言唯一报告、逻辑 UUID/attempt 配对、请求校验、状态/错误类型与客户端关闭 | +| 安全边界 | 假凭据及私有提示词 sentinel 放入成功 model 回显、错误正文和请求异常;逐份 Markdown 断言不泄漏,仍只写安全机器枚举/固定原因,不复制原始正文与异常 | + +日志均在 `tests/outputs/134/`,命令不接管道,先保存真实退出码到同名 `.exit` 再展示输出。 + +| 命令(pytest 前缀均为 `conda run --no-capture-output -n PolyGateway`) | 实际结果/日志 | +| --- | --- | +| `pytest tests/unit/test_live_evidence.py -k embed_probe_report -q`(修复前) | `embed-report-red.log/.exit`:8 failed,129 deselected,exit 1;八例均在真实报告消费处 `KeyError: requested_model`,不是 import/mock 签名失败 | +| 同命令(四字段补齐后) | `embed-report-green.log/.exit`:8 passed,129 deselected,exit 0 | +| `pytest tests/unit/test_live_evidence.py tests/unit/test_embedding.py tests/unit/test_openai_compat.py::TestDefaultClientFactory -q` | `embed-report-affected.log/.exit`:188 passed,exit 0 | +| `pytest tests/unit/ -q` | `embed-report-unit.log/.exit`:1383 passed,exit 0;格式化后 `embed-report-final-unit.log/.exit`:1383 passed,4.45秒,exit 0 | +| `make check` | 首次 `embed-report-check.log/.exit` 为新断言排版失败(exit 2),不是行为红;仅对该测试文件运行 conda ruff format。`embed-report-check-green.log/.exit`:94文件格式合格、ruff通过、import-linter 1 kept/0 broken,exit 0 | + +pi-lens 仍报非 conda 解释器缺 httpx/pytest/dotenv/pydantic 及旧 StrEnum 噪音;按任务授权记录,不添加 ignore、不改枚举、不扩环境修复范围。实际 conda 解释器为 `/home/iomgaa/miniconda3/envs/PolyGateway/bin/python`,本会话导入四依赖成功(httpx 0.28.1、pytest 9.1.1、python-dotenv 1.2.3、pydantic 2.13.4)。conda 启动器自身另有 base Python 3.13 的 RequestsDependencyWarning;未静音,不宣称输出零告警,测试进程与静态门实际退出0。 + +本修复不补写真正缺失的历史报告、不改变 embedding 能力判据;真实服务、slow、下游与发布证据仍由后续验收负责。 diff --git a/tests/e2e/test_embed_probe.py b/tests/e2e/test_embed_probe.py index 7f8662b..44bef9b 100644 --- a/tests/e2e/test_embed_probe.py +++ b/tests/e2e/test_embed_probe.py @@ -76,6 +76,10 @@ async def test_probe_real_gateway_embeddings(): matrix_id="embedding", round_index=1, safe_fields={ + "requested_model": source.model, + "provider": source.provider, + "planned_rounds": 1, + "completed_rounds": 1, "status": verdict.status, "reason": verdict.reason, "session_id": run_id, diff --git a/tests/unit/test_live_evidence.py b/tests/unit/test_live_evidence.py index 3b9c28c..edb13f5 100644 --- a/tests/unit/test_live_evidence.py +++ b/tests/unit/test_live_evidence.py @@ -684,6 +684,135 @@ async def test_round_consumer_keeps_first_success_when_second_assertion_fails(tm assert clients and all(client.is_closed for client in clients) +@pytest.mark.parametrize("model", ["embedding-override-a", "embedding-override-b"]) +@pytest.mark.parametrize( + ("outcome", "status", "error_type"), + [ + ("success", "PASS", None), + ("503", "FAIL", "TransientError"), + ("404", "UNCOVERED", "RequestRejectedError"), + ("request_error", "FAIL", "TransientError"), + ], +) +async def test_embed_probe_report_keeps_actual_source_and_round_identity( + tmp_path, monkeypatch, model, outcome, status, error_type +): + """真实探测消费者在成功与失败均留完整关联;只替换外部 HTTP。""" + import ast + from pathlib import Path + + from tests.unit.test_config import _BASE_ENV + + path = Path(__file__).parents[1] / "e2e/test_embed_probe.py" + tree = ast.parse(path.read_text()) + tree.body = [ + node + for node in tree.body + if not ( + isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id in {"_ENV", "pytestmark"} + for target in node.targets + ) + ) + ] + env = { + **_BASE_ENV, + "LLM__MINIMAX__1__BASE_URL": "https://example.test/v1", + "LLM__MINIMAX__1__API_KEY": _SECRET, + "LLM__MINIMAX__1__MODEL": "configured-chat-model", + "LLM__MINIMAX__1__TIMEOUT_S": "137", + "LLM__MINIMAX__1__TRUST_ENV": "false", + "PGW_EMBED_PROBE_MODEL": model, + } + namespace: dict[str, Any] = {"_ENV": env} + exec(compile(tree, str(path), "exec"), namespace) + monkeypatch.chdir(tmp_path) + original_factory = LiveCapture.client_factory + clients = [] + calls = [] + + def factory(capture, source): + """保留真实取证 hooks、源与逻辑 ID,只隔离网络出口。""" + client = original_factory(capture, source) + + def handler(request): + """提供完整成功/错误样本,敏感回显不得进入报告。""" + payload = json.loads(request.content) + assert payload["model"] == model == source.model + assert source.model != env["LLM__MINIMAX__1__MODEL"] + assert request.headers["Authorization"] == f"Bearer {_SECRET}" + calls.append((source, capture._round.get(), capture._attempt.get().call_id)) + if outcome == "request_error": + raise httpx.ConnectError(_SECRET + _PROMPT, request=request) + if outcome == "success": + return httpx.Response( + 200, + json={ + "data": [{"index": 0, "embedding": [0.1, 0.2]}], + "usage": {"prompt_tokens": 1}, + "model": _SECRET + _PROMPT, + }, + ) + return httpx.Response( + int(outcome), + json={"error": {"type": "model_not_found", "message": _SECRET + _PROMPT}}, + ) + + client._transport = httpx.MockTransport(handler) + clients.append(client) + return client + + monkeypatch.setattr(LiveCapture, "client_factory", factory) + probe = namespace["test_probe_real_gateway_embeddings"] + if status == "PASS": + await probe() + elif status == "UNCOVERED": + with pytest.raises(pytest.skip.Exception): + await probe() + else: + with pytest.raises(AssertionError): + await probe() + + assert len(calls) == len(clients) == 1 + assert all(client.is_closed for client in clients) + source, (session_id, parent_call_id), call_id = calls[0] + paths = list((tmp_path / "tests/outputs/134/live").rglob("*.md")) + assert len(paths) == 1 + text = paths[0].read_text() + assert _SECRET not in text and _PROMPT not in text + row = json.loads(text.split("```json\n")[1].split("\n```")[0]) + assert row["status"] == status + assert row["requested_model"] == source.model == model + assert row["provider"] == source.provider == "minimax" + assert row["planned_rounds"] == row["completed_rounds"] == 1 + assert row["session_id"] == paths[0].parent.name == session_id + assert row["parent_call_id"] == parent_call_id + assert len({session_id, parent_call_id, call_id}) == 3 + assert paths[0].name.startswith("embedding-1-") + assert text.startswith("# embedding · 轮次 1\n") + assert len(row["attempts"]) == 1 + attempt = row["attempts"][0] + assert attempt["call_id"] == call_id and attempt["error_type"] == error_type + assert len(attempt["http"]) == 1 + event = attempt["http"][0] + assert all(event["request_checks"].values()) + assert ( + event["status_code"] + == { + "success": 200, + "503": 503, + "404": 404, + "request_error": 0, + }[outcome] + ) + if outcome in {"503", "404"}: + assert event["error_body_complete"] is True + assert event["machine_type"] == "model_not_found" + elif outcome == "request_error": + assert "无可配对响应" in row["evidence_notes"] + + def test_partial_uncovered_and_failed_rounds_never_become_model_pass(): from tests.live_evidence import combine_live_verdicts, qualify_live_rounds