test: fix structured reask evidence and live coverage conclusions

This commit is contained in:
2026-09-09 03:34:48 -04:00
parent d332287b28
commit 3eb22d2a55
7 changed files with 540 additions and 28 deletions
+46 -2
View File
@@ -51,6 +51,8 @@ class _Attempt:
call_id: str
exchanges: list[_Exchange] = field(default_factory=list)
messages_digest: str | None = None
messages_valid: bool = False
class LiveCapture:
@@ -66,6 +68,14 @@ class LiveCapture:
raise ValueError("取证矩阵缺少必需预期或混用 chat/embed")
if not isinstance(expected["control"], dict):
raise ValueError("control 必须是显式对象")
if "structured_max_retries" in expected and (
"stream" not in expected
or type(expected["structured_max_retries"]) is not int
or expected["structured_max_retries"] < 0
or type(expected.get("messages_prefix_length")) is not int
or expected["messages_prefix_length"] < 1
):
raise ValueError("结构化预期缺少合法前缀长度或重问预算")
self._expectations = {name: dict(value) for name, value in expectations.items()}
self._round: ContextVar[tuple[str, str]] = ContextVar("live_round")
self._attempt: ContextVar[_Attempt] = ContextVar("live_attempt")
@@ -146,6 +156,29 @@ class LiveCapture:
self._notes[key].append("原始 JSON 身份无法独立解析")
return HttpEvidence(call_id, exchange.checks, status, body, identity)
def observe_messages(self, source: SourceConfig, messages: list[dict[str, Any]]) -> None:
"""先验前缀/反馈契约与委托摘要分开;摘要仅验证 HTTP 序列化保真。"""
expected = self._expectations[source.name]
if "structured_max_retries" not in expected:
return
attempt = self._attempt.get()
prefix_length = expected["messages_prefix_length"]
feedback = messages[prefix_length:]
attempt.messages_digest = messages_digest(messages)
attempt.messages_valid = (
(not feedback or bool(self._records[self._round.get()]))
and messages_digest(messages[:prefix_length]) == expected["messages_digest"]
and len(feedback) % 2 == 0
and len(feedback) <= 2 * expected["structured_max_retries"]
and all(
isinstance(message, dict)
and set(message) == {"role", "content"}
and message["role"] == ("assistant" if index % 2 == 0 else "user")
and isinstance(message["content"], str)
for index, message in enumerate(feedback)
)
)
def client_factory(self, source: SourceConfig) -> httpx.AsyncClient:
"""鉴权仅内存比较;沿已校验源 timeout/trust_env。"""
expected = self._expectations[source.name]
@@ -176,8 +209,13 @@ class LiveCapture:
"model": payload.get("model") == expected["model"],
"authorization": request.headers.get("Authorization") == f"Bearer {source.api_key}",
"control": control == expected["control"],
"messages_digest": messages_digest(payload.get("messages", payload.get("input")))
== expected["messages_digest"],
"messages_digest": (
attempt.messages_valid
and messages_digest(payload.get("messages")) == attempt.messages_digest
if "structured_max_retries" in expected
else messages_digest(payload.get("messages", payload.get("input")))
== expected["messages_digest"]
),
}
if "stream" in expected:
checks["stream"] = payload.get("stream") is expected["stream"] and (
@@ -255,6 +293,7 @@ class ObservedTransport:
) -> TransportResult:
"""与生产端口逐参数同签名。"""
with self._capture.attempt_context(call_id):
self._capture.observe_messages(source, messages)
return await self._transport.complete(
messages=messages,
source=source,
@@ -315,6 +354,7 @@ def chat_expectations(
messages: list[dict[str, Any]],
stream: bool,
controls: Mapping[str, dict[str, Any]],
structured_max_retries: int | None = None,
) -> dict[str, dict[str, Any]]:
"""URL 从源配置声明,控制片段必须由矩阵独立给出。"""
result = {}
@@ -330,6 +370,10 @@ def chat_expectations(
"control": controls[source.name],
"messages_digest": messages_digest(messages),
}
if structured_max_retries is not None:
result[source.name].update(
messages_prefix_length=len(messages), structured_max_retries=structured_max_retries
)
return result