test: fix structured reask evidence and live coverage conclusions
This commit is contained in:
+46
-2
@@ -51,6 +51,8 @@ class _Attempt:
|
||||
|
||||
call_id: str
|
||||
exchanges: list[_Exchange] = field(default_factory=list)
|
||||
messages_digest: str | None = None
|
||||
messages_valid: bool = False
|
||||
|
||||
|
||||
class LiveCapture:
|
||||
@@ -66,6 +68,14 @@ class LiveCapture:
|
||||
raise ValueError("取证矩阵缺少必需预期或混用 chat/embed")
|
||||
if not isinstance(expected["control"], dict):
|
||||
raise ValueError("control 必须是显式对象")
|
||||
if "structured_max_retries" in expected and (
|
||||
"stream" not in expected
|
||||
or type(expected["structured_max_retries"]) is not int
|
||||
or expected["structured_max_retries"] < 0
|
||||
or type(expected.get("messages_prefix_length")) is not int
|
||||
or expected["messages_prefix_length"] < 1
|
||||
):
|
||||
raise ValueError("结构化预期缺少合法前缀长度或重问预算")
|
||||
self._expectations = {name: dict(value) for name, value in expectations.items()}
|
||||
self._round: ContextVar[tuple[str, str]] = ContextVar("live_round")
|
||||
self._attempt: ContextVar[_Attempt] = ContextVar("live_attempt")
|
||||
@@ -146,6 +156,29 @@ class LiveCapture:
|
||||
self._notes[key].append("原始 JSON 身份无法独立解析")
|
||||
return HttpEvidence(call_id, exchange.checks, status, body, identity)
|
||||
|
||||
def observe_messages(self, source: SourceConfig, messages: list[dict[str, Any]]) -> None:
|
||||
"""先验前缀/反馈契约与委托摘要分开;摘要仅验证 HTTP 序列化保真。"""
|
||||
expected = self._expectations[source.name]
|
||||
if "structured_max_retries" not in expected:
|
||||
return
|
||||
attempt = self._attempt.get()
|
||||
prefix_length = expected["messages_prefix_length"]
|
||||
feedback = messages[prefix_length:]
|
||||
attempt.messages_digest = messages_digest(messages)
|
||||
attempt.messages_valid = (
|
||||
(not feedback or bool(self._records[self._round.get()]))
|
||||
and messages_digest(messages[:prefix_length]) == expected["messages_digest"]
|
||||
and len(feedback) % 2 == 0
|
||||
and len(feedback) <= 2 * expected["structured_max_retries"]
|
||||
and all(
|
||||
isinstance(message, dict)
|
||||
and set(message) == {"role", "content"}
|
||||
and message["role"] == ("assistant" if index % 2 == 0 else "user")
|
||||
and isinstance(message["content"], str)
|
||||
for index, message in enumerate(feedback)
|
||||
)
|
||||
)
|
||||
|
||||
def client_factory(self, source: SourceConfig) -> httpx.AsyncClient:
|
||||
"""鉴权仅内存比较;沿已校验源 timeout/trust_env。"""
|
||||
expected = self._expectations[source.name]
|
||||
@@ -176,8 +209,13 @@ class LiveCapture:
|
||||
"model": payload.get("model") == expected["model"],
|
||||
"authorization": request.headers.get("Authorization") == f"Bearer {source.api_key}",
|
||||
"control": control == expected["control"],
|
||||
"messages_digest": messages_digest(payload.get("messages", payload.get("input")))
|
||||
== expected["messages_digest"],
|
||||
"messages_digest": (
|
||||
attempt.messages_valid
|
||||
and messages_digest(payload.get("messages")) == attempt.messages_digest
|
||||
if "structured_max_retries" in expected
|
||||
else messages_digest(payload.get("messages", payload.get("input")))
|
||||
== expected["messages_digest"]
|
||||
),
|
||||
}
|
||||
if "stream" in expected:
|
||||
checks["stream"] = payload.get("stream") is expected["stream"] and (
|
||||
@@ -255,6 +293,7 @@ class ObservedTransport:
|
||||
) -> TransportResult:
|
||||
"""与生产端口逐参数同签名。"""
|
||||
with self._capture.attempt_context(call_id):
|
||||
self._capture.observe_messages(source, messages)
|
||||
return await self._transport.complete(
|
||||
messages=messages,
|
||||
source=source,
|
||||
@@ -315,6 +354,7 @@ def chat_expectations(
|
||||
messages: list[dict[str, Any]],
|
||||
stream: bool,
|
||||
controls: Mapping[str, dict[str, Any]],
|
||||
structured_max_retries: int | None = None,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""URL 从源配置声明,控制片段必须由矩阵独立给出。"""
|
||||
result = {}
|
||||
@@ -330,6 +370,10 @@ def chat_expectations(
|
||||
"control": controls[source.name],
|
||||
"messages_digest": messages_digest(messages),
|
||||
}
|
||||
if structured_max_retries is not None:
|
||||
result[source.name].update(
|
||||
messages_prefix_length=len(messages), structured_max_retries=structured_max_retries
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user