iomgaa/PolyGateway
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases 10 Wiki Activity

test: fix structured reask evidence and live coverage conclusions

Browse Source
This commit is contained in:
iomgaa
2026-09-09 03:34:48 -04:00
parent d332287b28
commit 3eb22d2a55
7 changed files with 540 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e/conftest.py
+46 -2
View File
@@ -51,6 +51,8 @@ class _Attempt:
call_id: str
exchanges: list[_Exchange] = field(default_factory=list)
messages_digest: str | None = None
messages_valid: bool = False
class LiveCapture:
@@ -66,6 +68,14 @@ class LiveCapture:
raise ValueError("取证矩阵缺少必需预期或混用 chat/embed")
if not isinstance(expected["control"], dict):
raise ValueError("control 必须是显式对象")
if "structured_max_retries" in expected and (
"stream" not in expected
or type(expected["structured_max_retries"]) is not int
or expected["structured_max_retries"] < 0
or type(expected.get("messages_prefix_length")) is not int
or expected["messages_prefix_length"] < 1
):
raise ValueError("结构化预期缺少合法前缀长度或重问预算")
self._expectations = {name: dict(value) for name, value in expectations.items()}
self._round: ContextVar[tuple[str, str]] = ContextVar("live_round")
self._attempt: ContextVar[_Attempt] = ContextVar("live_attempt")
@@ -146,6 +156,29 @@ class LiveCapture:
self._notes[key].append("原始 JSON 身份无法独立解析")
return HttpEvidence(call_id, exchange.checks, status, body, identity)
def observe_messages(self, source: SourceConfig, messages: list[dict[str, Any]]) -> None:
"""先验前缀／反馈契约与委托摘要分开；摘要仅验证 HTTP 序列化保真。"""
expected = self._expectations[source.name]
if "structured_max_retries" not in expected:
return
attempt = self._attempt.get()
prefix_length = expected["messages_prefix_length"]
feedback = messages[prefix_length:]
attempt.messages_digest = messages_digest(messages)
attempt.messages_valid = (
(not feedback or bool(self._records[self._round.get()]))
and messages_digest(messages[:prefix_length]) == expected["messages_digest"]
and len(feedback) % 2 == 0
and len(feedback) <= 2 * expected["structured_max_retries"]
and all(
isinstance(message, dict)
and set(message) == {"role", "content"}
and message["role"] == ("assistant" if index % 2 == 0 else "user")
and isinstance(message["content"], str)
for index, message in enumerate(feedback)
)
)
def client_factory(self, source: SourceConfig) -> httpx.AsyncClient:
"""鉴权仅内存比较；沿已校验源 timeout/trust_env。"""
expected = self._expectations[source.name]
@@ -176,8 +209,13 @@ class LiveCapture:
"model": payload.get("model") == expected["model"],
"authorization": request.headers.get("Authorization") == f"Bearer {source.api_key}",
"control": control == expected["control"],
"messages_digest": messages_digest(payload.get("messages", payload.get("input")))
== expected["messages_digest"],
"messages_digest": (
attempt.messages_valid
and messages_digest(payload.get("messages")) == attempt.messages_digest
if "structured_max_retries" in expected
else messages_digest(payload.get("messages", payload.get("input")))
== expected["messages_digest"]
),
}
if "stream" in expected:
checks["stream"] = payload.get("stream") is expected["stream"] and (
@@ -255,6 +293,7 @@ class ObservedTransport:
) -> TransportResult:
"""与生产端口逐参数同签名。"""
with self._capture.attempt_context(call_id):
self._capture.observe_messages(source, messages)
return await self._transport.complete(
messages=messages,
source=source,
@@ -315,6 +354,7 @@ def chat_expectations(
messages: list[dict[str, Any]],
stream: bool,
controls: Mapping[str, dict[str, Any]],
structured_max_retries: int | None = None,
) -> dict[str, dict[str, Any]]:
"""URL 从源配置声明，控制片段必须由矩阵独立给出。"""
result = {}
@@ -330,6 +370,10 @@ def chat_expectations(
"control": controls[source.name],
"messages_digest": messages_digest(messages),
}
if structured_max_retries is not None:
result[source.name].update(
messages_prefix_length=len(messages), structured_max_retries=structured_max_retries
)
return result
tests/e2e/test_smoke_gateway.py
+7 -1
View File
@@ -40,7 +40,13 @@ async def _smoke(matrix, prompt, validate, *, stream=True, structured=None):
controls = source_controls(settings)
capture = LiveCapture(
expectations=chat_expectations(
settings, messages=messages, stream=stream, controls=controls
settings,
messages=messages,
stream=stream,
controls=controls,
structured_max_retries=(
settings.structured_max_retries if isinstance(structured, type) else None
),
)
)
async with observed_client(settings, capture) as client:
tests/e2e/test_thinking_live.py
+133 -22
View File
@@ -125,8 +125,36 @@ def _tier_settings(model):
return dataclasses.replace(base, sources=(source,))
@dataclasses.dataclass
class _CaseRun:
"""单型号用例关联；子运行以同一 run_id 下的 matrix_id 唯一定位原件。"""
run_id: str
model: str
subruns: list[dict] = dataclasses.field(default_factory=list)
def report_fields(self):
"""计划分母在收集前登记，完成数量只按实际回收轮次填写。"""
return {
"session_id": self.run_id,
"requested_model": self.model,
"subruns": self.subruns,
"planned_rounds": sum(group["planned_rounds"] for group in self.subruns),
"completed_rounds": sum(group["completed_rounds"] for group in self.subruns),
}
async def _collect_rounds(
settings, *, rounds, stream, prompt, matrix_id, effort=None, capabilities=None, concurrency=1
settings,
*,
run,
rounds,
stream,
prompt,
matrix_id,
effort=None,
capabilities=None,
concurrency=1,
):
"""保留所有失败轮，不把可用轮集合偷偷当新分母。"""
if rounds < 1 or concurrency < 1:
@@ -142,7 +170,13 @@ async def _collect_rounds(
settings, messages=messages, stream=stream, controls=controls
)
)
run_id = uuid4().hex
if any(source.model != run.model for source in settings.sources):
raise ValueError("用例型号与收集源不一致")
if any(group["matrix_id"] == matrix_id for group in run.subruns):
raise ValueError("用例子运行标识重复")
group = {"matrix_id": matrix_id, "planned_rounds": rounds, "completed_rounds": 0}
run.subruns.append(group)
run_id = run.run_id
semaphore = asyncio.Semaphore(concurrency)
async with observed_client(settings, capture, capabilities=capabilities) as client:
@@ -188,21 +222,29 @@ async def _collect_rounds(
if isinstance(value, BaseException):
raise value
values.append(value)
group["completed_rounds"] = len(values)
counts = summarize_verdicts([value["verdict"] for value in values], planned_rounds=rounds)
write_live_round(
_OUT_DIR,
run_id=run_id,
matrix_id=matrix_id + "-rounds",
round_index=0,
safe_fields={"counts": counts, "completed_rounds": len(values), "planned_rounds": rounds},
safe_fields={
"session_id": run_id,
"requested_model": run.model,
"counts": counts,
"completed_rounds": len(values),
"planned_rounds": rounds,
},
)
return values
async def _run_rounds(rounds, *, stream=True, matrix_id="thinking", **source_overrides):
async def _run_rounds(rounds, *, run, stream=True, matrix_id="thinking", **source_overrides):
"""L1–L8 的资格证据出口，不作整类 skip。"""
return await _collect_rounds(
_settings(**source_overrides),
run=run,
rounds=rounds,
stream=stream,
prompt=_PROMPT,
@@ -218,6 +260,8 @@ def _qualified(rows, *, planned_rounds):
def _coverage(rows, *, planned_rounds, proposition):
"""只有全轮资格通过才进入推理观测命题。"""
verdict = _qualified(rows, planned_rounds=planned_rounds)
if verdict.status == "PASS" and proposition == "observation-only":
return LiveVerdict("UNCOVERED", "未登记候选只保留观测，不自动登记能力")
if verdict.status == "PASS":
verdict = assess_thinking_coverage(
[row["response"].thinking_observation for row in rows],
@@ -227,17 +271,18 @@ def _coverage(rows, *, planned_rounds, proposition):
return verdict
def _conclude(matrix, verdict, *, proposition=None):
def _conclude(matrix, verdict, *, run, proposition=None):
"""命题汇总先落盘再交给 pytest，不覆盖逐轮原件。"""
write_live_round(
_OUT_DIR,
run_id=uuid4().hex,
run_id=run.run_id,
matrix_id=matrix,
round_index=0,
safe_fields={
"status": verdict.status,
"reason": verdict.reason,
"proposition": proposition,
**run.report_fields(),
},
)
enforce_verdict(verdict)
@@ -247,18 +292,27 @@ class TestMiniMaxM3:
"""AUTO 拒绝已移至离线契约；真实开启明确请求 medium。"""
async def test_l1_disable_actually_disables(self):
run = _CaseRun(uuid4().hex, "MiniMax-M3")
rows = await _run_rounds(
_ROUNDS, matrix_id="L1", provider="minimax", model="MiniMax-M3", enable_thinking=False
_ROUNDS,
run=run,
matrix_id="L1",
provider="minimax",
model="MiniMax-M3",
enable_thinking=False,
)
_conclude(
"L1",
_coverage(rows, planned_rounds=_ROUNDS, proposition="disabled"),
run=run,
proposition="disabled",
)
async def test_l2_enable_actually_enables(self):
run = _CaseRun(uuid4().hex, "MiniMax-M3")
rows = await _run_rounds(
_ROUNDS,
run=run,
matrix_id="L2",
provider="minimax",
model="MiniMax-M3",
@@ -267,14 +321,17 @@ class TestMiniMaxM3:
_conclude(
"L2",
_coverage(rows, planned_rounds=_ROUNDS, proposition="enabled"),
run=run,
proposition="enabled",
)
async def test_l2b_off_and_on_are_distinguishable_without_magic_numbers(self):
"""指定历史 prompt 锚点回归，不宣称关闭能力已覆盖。"""
run = _CaseRun(uuid4().hex, "MiniMax-M3")
rounds = max(3, _ROUNDS // 3)
off = await _run_rounds(
rounds,
run=run,
matrix_id="L2b-off",
provider="minimax",
model="MiniMax-M3",
@@ -282,6 +339,7 @@ class TestMiniMaxM3:
)
on = await _run_rounds(
rounds,
run=run,
matrix_id="L2b-on",
provider="minimax",
model="MiniMax-M3",
@@ -295,22 +353,27 @@ class TestMiniMaxM3:
verdict = LiveVerdict(
"PASS" if distinct else "FAIL", "指定历史 prompt 锚点比较；不是关闭证明"
)
_conclude("L2b", verdict, proposition="historical-prompt-anchor")
_conclude("L2b", verdict, run=run, proposition="historical-prompt-anchor")
async def test_l3_no_opinion_is_the_model_default(self):
rows = await _run_rounds(_ROUNDS, matrix_id="L3", provider="minimax", model="MiniMax-M3")
run = _CaseRun(uuid4().hex, "MiniMax-M3")
rows = await _run_rounds(
_ROUNDS, run=run, matrix_id="L3", provider="minimax", model="MiniMax-M3"
)
verdict = _qualified(rows, planned_rounds=_ROUNDS)
if verdict.status == "PASS" and any(
row["response"].applied_effort is not None for row in rows
):
verdict = LiveVerdict("FAIL", "不表态路径擅自记录档位")
_conclude("L3", verdict, proposition="no-opinion-not-capability")
_conclude("L3", verdict, run=run, proposition="no-opinion-not-capability")
async def test_l3b_none_is_recognised_not_silently_dropped(self):
"""保留原非法 raw 值对照预算，但不提升 UNKNOWN。"""
run = _CaseRun(uuid4().hex, "MiniMax-M3")
rounds = max(3, _ROUNDS // 3)
bogus = await _run_rounds(
rounds,
run=run,
matrix_id="L3b-bogus",
provider="minimax",
model="MiniMax-M3",
@@ -318,6 +381,7 @@ class TestMiniMaxM3:
)
off = await _run_rounds(
rounds,
run=run,
matrix_id="L3b-off",
provider="minimax",
model="MiniMax-M3",
@@ -327,13 +391,15 @@ class TestMiniMaxM3:
_coverage(bogus, planned_rounds=rounds, proposition="enabled"),
_coverage(off, planned_rounds=rounds, proposition="disabled"),
]
_conclude("L3b", _combine(verdicts), proposition="raw-counterexample")
_conclude("L3b", _combine(verdicts), run=run, proposition="raw-counterexample")
async def test_l4_raw_only_explicit_high(self):
"""退出受管意图后才保留 raw high；双来源拒绝在 unit 守卫。"""
run = _CaseRun(uuid4().hex, "MiniMax-M3")
rounds = max(3, _ROUNDS // 2)
rows = await _run_rounds(
rounds,
run=run,
matrix_id="L4",
provider="minimax",
model="MiniMax-M3",
@@ -342,14 +408,17 @@ class TestMiniMaxM3:
_conclude(
"L4",
_coverage(rows, planned_rounds=rounds, proposition="enabled"),
run=run,
proposition="enabled",
)
async def test_l5_non_stream_path_is_distinguishable_and_honestly_unknown(self):
"""保留流／非流预算；UNKNOWN 是明确未覆盖而非长度锚点成功。"""
run = _CaseRun(uuid4().hex, "MiniMax-M3")
rounds = max(3, _ROUNDS // 2)
off = await _run_rounds(
rounds,
run=run,
matrix_id="L5-off",
stream=False,
provider="minimax",
@@ -358,6 +427,7 @@ class TestMiniMaxM3:
)
on = await _run_rounds(
rounds,
run=run,
matrix_id="L5-on",
stream=False,
provider="minimax",
@@ -372,6 +442,7 @@ class TestMiniMaxM3:
_coverage(on, planned_rounds=rounds, proposition="enabled"),
]
),
run=run,
proposition="nonstream-enabled-disabled",
)
@@ -389,22 +460,36 @@ class TestOtherProviders:
[("L6", "qwen", "qwen3.7-plus"), ("L7", "deepseek", "deepseek-v4-pro")],
)
async def test_existing_profiles_still_disable(self, matrix, provider, model):
run = _CaseRun(uuid4().hex, model)
rows = await _run_rounds(
_ROUNDS, matrix_id=matrix, provider=provider, model=model, enable_thinking=False
_ROUNDS,
run=run,
matrix_id=matrix,
provider=provider,
model=model,
enable_thinking=False,
)
_conclude(
matrix,
_coverage(rows, planned_rounds=_ROUNDS, proposition="disabled"),
run=run,
proposition="disabled",
)
async def test_qwen_enabled_is_observed(self):
run = _CaseRun(uuid4().hex, "qwen3.7-plus")
rows = await _run_rounds(
_ROUNDS, matrix_id="L6b", provider="qwen", model="qwen3.7-plus", enable_thinking=True
_ROUNDS,
run=run,
matrix_id="L6b",
provider="qwen",
model="qwen3.7-plus",
enable_thinking=True,
)
_conclude(
"L6b",
_coverage(rows, planned_rounds=_ROUNDS, proposition="enabled"),
run=run,
proposition="enabled",
)
@@ -419,9 +504,11 @@ class TestCapabilityDrift:
),
)
async def test_declared_capability_matches_reality(self, model):
run = _CaseRun(uuid4().hex, model)
rounds = max(3, _ROUNDS // 2)
rows = await _run_rounds(
rounds,
run=run,
matrix_id="L8",
provider=_MODEL_PROVIDER[model],
model=model,
@@ -430,14 +517,16 @@ class TestCapabilityDrift:
_conclude(
"L8",
_coverage(rows, planned_rounds=rounds, proposition="disabled"),
run=run,
proposition="disabled",
)
async def _probe_effort(model, effort, *, rounds, prompt, prompt_kind):
async def _probe_effort(model, effort, *, run, rounds, prompt, prompt_kind):
"""临时全档表仅用于 T10 探测，不写回 DEFAULT，也不生成预期 wire。"""
return await _collect_rounds(
_tier_settings(model),
run=run,
rounds=rounds,
stream=True,
prompt=prompt,
@@ -453,10 +542,22 @@ class TestTierProbe:
@pytest.mark.parametrize("model", sorted(_MODEL_PROVIDER))
async def test_t10_none_direction_matches_declaration(self, model):
run = _CaseRun(uuid4().hex, model)
capability = DEFAULT_CAPABILITIES.get(model)
proposition = "disabled" if capability and capability.can_disable else "cannot_disable"
proposition = (
"observation-only"
if capability is None
else "disabled"
if capability.can_disable
else "cannot_disable"
)
short = await _probe_effort(
model, Effort.NONE, rounds=_TIER_ROUNDS, prompt=_TIER_PROMPT, prompt_kind="none-short"
model,
Effort.NONE,
run=run,
rounds=_TIER_ROUNDS,
prompt=_TIER_PROMPT,
prompt_kind="none-short",
)
verdict = _coverage(short, planned_rounds=_TIER_ROUNDS, proposition=proposition)
# 沿既有矩阵：短档没有 OBSERVED 才做长上下文复核；不新增锚点调用。
@@ -466,6 +567,7 @@ class TestTierProbe:
long_rows = await _probe_effort(
model,
Effort.NONE,
run=run,
rounds=_TIER_LONG_ROUNDS,
prompt=_TIER_LONG_PROMPT,
prompt_kind="none-long",
@@ -475,18 +577,18 @@ class TestTierProbe:
planned_rounds=_TIER_ROUNDS + _TIER_LONG_ROUNDS,
proposition=proposition,
)
if capability is None and verdict.status != "FAIL":
verdict = LiveVerdict("UNCOVERED", "未登记候选只保留观测，不自动登记能力")
_conclude("T10-none", verdict, proposition=proposition)
_conclude("T10-none", verdict, run=run, proposition=proposition)
@pytest.mark.parametrize("model", sorted(DEFAULT_CAPABILITIES))
async def test_t10_declared_tiers_actually_reason(self, model):
run = _CaseRun(uuid4().hex, model)
tiers = [e for e in DEFAULT_CAPABILITIES[model].supported_efforts if e is not Effort.NONE]
verdicts = []
for tier in tiers:
rows = await _probe_effort(
model,
tier,
run=run,
rounds=_TIER_ROUNDS,
prompt=_TIER_PROMPT,
prompt_kind="tier-" + tier.value,
@@ -495,22 +597,31 @@ class TestTierProbe:
verdicts.append(verdict)
write_live_round(
_OUT_DIR,
run_id=uuid4().hex,
run_id=run.run_id,
matrix_id="T10-tier",
round_index=0,
safe_fields={
"requested_model": model,
"session_id": run.run_id,
"proposition": "enabled",
"subruns": [run.subruns[-1]],
"planned_rounds": _TIER_ROUNDS,
"completed_rounds": len(rows),
"requested_effort": tier.value,
"status": verdict.status,
"reason": verdict.reason,
},
)
_conclude("T10-tiers", _combine(verdicts), proposition="enabled-all-declared-tiers")
_conclude(
"T10-tiers", _combine(verdicts), run=run, proposition="enabled-all-declared-tiers"
)
@pytest.mark.parametrize("model", ["gemini-3.1-pro", "gpt-5.5", "glm-5.3"])
async def test_t10_no_opinion_stays_no_opinion(self, model):
run = _CaseRun(uuid4().hex, model)
rows = await _collect_rounds(
_tier_settings(model),
run=run,
rounds=_TIER_ROUNDS,
stream=True,
prompt=_TIER_PROMPT,
@@ -522,4 +633,4 @@ class TestTierProbe:
row["response"].applied_effort is not None for row in rows
):
verdict = LiveVerdict("FAIL", "默认基线擅自推定档位")
_conclude("T10-default", verdict, proposition="no-opinion-not-capability")
_conclude("T10-default", verdict, run=run, proposition="no-opinion-not-capability")