test: apply evidence-based live checks without hiding regressions
This commit is contained in:
@@ -0,0 +1,308 @@
|
||||
"""真实测试的有限证据判定；不读取环境、不请求网络、不记录原始正文。"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
from uuid import uuid4
|
||||
|
||||
from polygateway.errors import RequestRejectedError
|
||||
from polygateway.types import ThinkingObservation
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HttpEvidence:
|
||||
"""一次 HTTP 的内存证据，禁止直接序列化。"""
|
||||
|
||||
call_id: str
|
||||
request_checks: tuple[tuple[str, bool], ...]
|
||||
status_code: int
|
||||
error_body: bytes | None
|
||||
raw_identity: tuple[bool, str | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AttemptEvidence:
|
||||
"""一次 transport 尝试，可以没有 HTTP。"""
|
||||
|
||||
call_id: str
|
||||
http: tuple[HttpEvidence, ...]
|
||||
error: Exception | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LiveVerdict:
|
||||
"""测试命题的结论，不等同于 pytest 退出码。"""
|
||||
|
||||
status: Literal["PASS", "FAIL", "UNCOVERED"]
|
||||
reason: str
|
||||
|
||||
|
||||
def strict_json(body: bytes) -> Any:
|
||||
"""独立解析完整 UTF-8 JSON，拒绝重复键及非标准常量。"""
|
||||
|
||||
def pairs(items: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
"""重复键不允许被后值掩盖。"""
|
||||
result = {}
|
||||
for key, value in items:
|
||||
if key in result:
|
||||
raise ValueError("重复 JSON 键")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
def invalid(value: str) -> None:
|
||||
"""拒绝非标准数值常量。"""
|
||||
raise ValueError("非法 JSON 常量")
|
||||
|
||||
return json.loads(body.decode("utf-8"), object_pairs_hook=pairs, parse_constant=invalid)
|
||||
|
||||
|
||||
def messages_digest(messages: Any) -> str:
|
||||
"""只在内存比较提示词摘要，不写提示词。"""
|
||||
return hashlib.sha256(
|
||||
json.dumps(messages, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def request_is_valid(event: HttpEvidence) -> bool:
|
||||
"""完整且无重复的显式检查才能作为请求资格。"""
|
||||
checks = dict(event.request_checks)
|
||||
common = {"method", "origin", "path", "model", "authorization", "control", "messages_digest"}
|
||||
return (
|
||||
len(checks) == len(event.request_checks)
|
||||
and set(checks) in (common | {"stream"}, common | {"input_shape"})
|
||||
and all(value is True for value in checks.values())
|
||||
)
|
||||
|
||||
|
||||
def _single_error(error: Exception, attempts: Sequence[AttemptEvidence]) -> HttpEvidence | None:
|
||||
"""仅接受可与最终异常精确配对的独立单次错误。"""
|
||||
if not isinstance(error, RequestRejectedError) or len(attempts) != 1:
|
||||
return None
|
||||
attempt = attempts[0]
|
||||
if attempt.error is not error or len(attempt.http) != 1:
|
||||
return None
|
||||
event = attempt.http[0]
|
||||
if event.call_id != attempt.call_id or not request_is_valid(event):
|
||||
return None
|
||||
if event.status_code != error.status_code:
|
||||
return None
|
||||
return event
|
||||
|
||||
|
||||
def error_machine_type(event: HttpEvidence) -> str | None:
|
||||
"""完整错误体中的唯一机器字段；不检查正文子串。"""
|
||||
body = event.error_body
|
||||
if body is None or not 0 < len(body) <= 65536:
|
||||
return None
|
||||
try:
|
||||
data = strict_json(body)
|
||||
except (ValueError, UnicodeError):
|
||||
return None
|
||||
if not isinstance(data, dict) or not isinstance(data.get("error"), dict):
|
||||
return None
|
||||
value = data["error"].get("type")
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def classify_live_failure(error: Exception, attempts: Sequence[AttemptEvidence]) -> LiveVerdict:
|
||||
"""默认失败；唯一自动外因是完整证据支持的 404 model_not_found。"""
|
||||
event = _single_error(error, attempts)
|
||||
if (
|
||||
event is not None
|
||||
and event.status_code == 404
|
||||
and error_machine_type(event) == "model_not_found"
|
||||
):
|
||||
return LiveVerdict("UNCOVERED", "端点回报该请求型号不可用")
|
||||
return LiveVerdict("FAIL", "无满足窄外因契约的完整独立证据")
|
||||
|
||||
|
||||
def assess_expected_rejection(
|
||||
error: Exception, attempts: Sequence[AttemptEvidence], *, status_code: int, machine_type: str
|
||||
) -> LiveVerdict:
|
||||
"""预先声明的拒绝命题，不把一般 400 当不支持档位。"""
|
||||
event = _single_error(error, attempts)
|
||||
if (
|
||||
event is not None
|
||||
and event.status_code == status_code
|
||||
and error_machine_type(event) == machine_type
|
||||
):
|
||||
return LiveVerdict("PASS", "符合预声明的拒绝类型、状态和机器字段")
|
||||
return LiveVerdict("FAIL", "不符合预声明拒绝证据")
|
||||
|
||||
|
||||
def assess_model_identity(
|
||||
*,
|
||||
requested: str,
|
||||
aliases: frozenset[str],
|
||||
reported: str | None,
|
||||
raw_identity: tuple[bool, str | None],
|
||||
request_valid: bool,
|
||||
) -> LiveVerdict:
|
||||
"""公共身份异常只有原始独立证据才能归因上游。"""
|
||||
if not request_valid:
|
||||
return LiveVerdict("FAIL", "实发请求校验不完整或不符")
|
||||
allowed = {requested, *aliases}
|
||||
captured, raw = raw_identity
|
||||
if captured and raw != reported:
|
||||
return LiveVerdict("FAIL", "公共身份与独立原始身份不一致")
|
||||
if reported in allowed:
|
||||
return LiveVerdict("PASS", "身份合格")
|
||||
if not captured:
|
||||
return LiveVerdict("FAIL", "身份来源无法区分")
|
||||
return LiveVerdict("UNCOVERED", "独立原始响应身份缺失或不属于显式别名")
|
||||
|
||||
|
||||
def assess_thinking_coverage(
|
||||
observations: Sequence[ThinkingObservation],
|
||||
*,
|
||||
planned_rounds: int,
|
||||
proposition: Literal["enabled", "disabled", "cannot_disable"],
|
||||
) -> LiveVerdict:
|
||||
"""按独立命题裁定；缺轮不减分母，UNKNOWN 不证明关闭。"""
|
||||
if planned_rounds < 1 or len(observations) != planned_rounds:
|
||||
return LiveVerdict("FAIL", "计划轮次不完整")
|
||||
if any(not isinstance(item, ThinkingObservation) for item in observations):
|
||||
return LiveVerdict("FAIL", "观测类型不符")
|
||||
observed = observations.count(ThinkingObservation.OBSERVED)
|
||||
unknown = observations.count(ThinkingObservation.UNKNOWN)
|
||||
if proposition == "enabled":
|
||||
if observed > planned_rounds / 2:
|
||||
return LiveVerdict("PASS", "完整轮次多数观测到推理")
|
||||
return LiveVerdict("UNCOVERED" if unknown else "FAIL", "开启证据未达多数")
|
||||
if proposition == "disabled":
|
||||
if observed:
|
||||
return LiveVerdict("FAIL", "观测到推理，证伪关闭声明")
|
||||
return LiveVerdict(
|
||||
"UNCOVERED" if unknown else "PASS",
|
||||
"UNKNOWN 不证明关闭" if unknown else "每轮明确 ABSENT",
|
||||
)
|
||||
if proposition == "cannot_disable":
|
||||
if observed:
|
||||
return LiveVerdict("PASS", "本条件下仍推理；不外推所有私有参数")
|
||||
return LiveVerdict("UNCOVERED" if unknown else "FAIL", "缺少仍推理的证据")
|
||||
raise ValueError("未知测试命题")
|
||||
|
||||
|
||||
def summarize_verdicts(verdicts: Sequence[LiveVerdict], *, planned_rounds: int) -> dict[str, int]:
|
||||
"""保留失败、未覆盖与缺轮的独立计数。"""
|
||||
if planned_rounds < len(verdicts):
|
||||
raise ValueError("实际轮次超出计划")
|
||||
return {
|
||||
**{
|
||||
status: sum(v.status == status for v in verdicts)
|
||||
for status in ("PASS", "FAIL", "UNCOVERED")
|
||||
},
|
||||
"missing": planned_rounds - len(verdicts),
|
||||
}
|
||||
|
||||
|
||||
_SAFE_FIELDS = frozenset(
|
||||
{
|
||||
"provider",
|
||||
"requested_model",
|
||||
"reported_model",
|
||||
"stream",
|
||||
"requested_effort",
|
||||
"applied_effort",
|
||||
"session_id",
|
||||
"parent_call_id",
|
||||
"attempts",
|
||||
"status",
|
||||
"reason",
|
||||
"completed_rounds",
|
||||
"planned_rounds",
|
||||
"thinking_observation",
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"reasoning_tokens",
|
||||
"thinking_chars",
|
||||
"counts",
|
||||
"proposition",
|
||||
"error_type",
|
||||
"error_status",
|
||||
"evidence_notes",
|
||||
}
|
||||
)
|
||||
_ATTEMPT_FIELDS = frozenset({"call_id", "error_type", "http"})
|
||||
_HTTP_FIELDS = frozenset(
|
||||
{"status_code", "request_checks", "identity_captured", "error_body_complete"}
|
||||
)
|
||||
|
||||
|
||||
def _validate_safe(fields: Mapping[str, Any]) -> None:
|
||||
"""拒绝原始异常和证据对象，嵌套字段也有白名单。"""
|
||||
if set(fields) - _SAFE_FIELDS:
|
||||
raise ValueError("报告含非白名单字段")
|
||||
for attempt in fields.get("attempts", []):
|
||||
if not isinstance(attempt, dict) or set(attempt) != _ATTEMPT_FIELDS:
|
||||
raise ValueError("非法 attempt 报告")
|
||||
for event in attempt["http"]:
|
||||
if not isinstance(event, dict) or set(event) != _HTTP_FIELDS:
|
||||
raise ValueError("非法 HTTP 报告")
|
||||
if not isinstance(event["request_checks"], dict) or any(
|
||||
type(v) is not bool for v in event["request_checks"].values()
|
||||
):
|
||||
raise ValueError("请求校验报告只允许布尔值")
|
||||
# 不提供 default=str：原始异常、bytes、dataclass 均必须失败。
|
||||
json.dumps(fields, ensure_ascii=False, allow_nan=False)
|
||||
|
||||
|
||||
def write_live_round(
|
||||
output_dir: Path,
|
||||
*,
|
||||
run_id: str,
|
||||
matrix_id: str,
|
||||
round_index: int,
|
||||
safe_fields: Mapping[str, Any],
|
||||
) -> Path:
|
||||
"""仅接收已脱敏字段；独占文件写入失败必须冒泡。"""
|
||||
_validate_safe(safe_fields)
|
||||
if round_index < 0 or not re.fullmatch(r"[a-zA-Z0-9_-]+", run_id + matrix_id):
|
||||
raise ValueError("报告路径标识非法")
|
||||
directory = output_dir / run_id
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
path = directory / f"{matrix_id}-{round_index}-{uuid4().hex}.md"
|
||||
text = json.dumps(dict(safe_fields), ensure_ascii=False, indent=2, allow_nan=False)
|
||||
with path.open("x", encoding="utf-8") as handle:
|
||||
handle.write(f"# {matrix_id} · 轮次 {round_index}\n\n```json\n{text}\n```\n")
|
||||
return path
|
||||
|
||||
|
||||
def safe_attempts(attempts: Sequence[AttemptEvidence]) -> list[dict[str, Any]]:
|
||||
"""只导出事实布尔值、异常类和状态；不落盘任何上游正文。"""
|
||||
return [
|
||||
{
|
||||
"call_id": attempt.call_id,
|
||||
"error_type": type(attempt.error).__name__ if attempt.error else None,
|
||||
"http": [
|
||||
{
|
||||
"status_code": event.status_code,
|
||||
"request_checks": dict(event.request_checks),
|
||||
"identity_captured": event.raw_identity[0],
|
||||
"error_body_complete": event.error_body is not None,
|
||||
}
|
||||
for event in attempt.http
|
||||
],
|
||||
}
|
||||
for attempt in attempts
|
||||
]
|
||||
|
||||
|
||||
def combine_live_verdicts(verdicts: Sequence[LiveVerdict]) -> LiveVerdict:
|
||||
"""部分失败优先于未覆盖；部分未覆盖不得汇总全 PASS。"""
|
||||
if any(verdict.status == "FAIL" for verdict in verdicts):
|
||||
return LiveVerdict("FAIL", "至少一个必需命题失败")
|
||||
if not verdicts or any(verdict.status == "UNCOVERED" for verdict in verdicts):
|
||||
return LiveVerdict("UNCOVERED", "至少一个必需命题未覆盖")
|
||||
return LiveVerdict("PASS", "所有必需命题通过")
|
||||
|
||||
|
||||
def qualify_live_rounds(verdicts: Sequence[LiveVerdict], *, planned_rounds: int) -> LiveVerdict:
|
||||
"""汇总请求／身份资格，缺轮绝不缩小分母。"""
|
||||
if planned_rounds < 1 or len(verdicts) != planned_rounds:
|
||||
return LiveVerdict("FAIL", "计划轮次缺失")
|
||||
return combine_live_verdicts(verdicts)
|
||||
Reference in New Issue
Block a user