test: fix structured reask evidence and live coverage conclusions
This commit is contained in:
@@ -4,6 +4,7 @@ import asyncio
|
||||
import inspect
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
@@ -814,3 +815,309 @@ def test_pytest_report_hook_preserves_report_and_uses_safe_fallback(tmp_path, mo
|
||||
assert finished.value.value is report
|
||||
paths = list(tmp_path.rglob("*.md"))
|
||||
assert len(paths) == 1 and '"status": "FAIL"' in paths[0].read_text()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"damage", [None, "prefix", "role", "content", "unpaired", "budget", "wire"]
|
||||
)
|
||||
async def test_structured_reask_preserves_message_contract(tmp_path, damage, monkeypatch):
|
||||
"""真实 StructuredMW 缺字段后重问成功；前缀、反馈结构及 wire 破坏均失败。"""
|
||||
from pydantic import BaseModel
|
||||
|
||||
from polygateway import GatewaySettings
|
||||
from polygateway.middleware.structured import StructuredMW
|
||||
from tests.e2e.conftest import captured_chat_round, observed_client
|
||||
from tests.unit.test_config import _BASE_ENV
|
||||
|
||||
class Answer(BaseModel):
|
||||
"""离线最小结构化契约。"""
|
||||
|
||||
answer: int
|
||||
reason: str
|
||||
|
||||
settings = replace(
|
||||
GatewaySettings.from_env("LLM", env=_BASE_ENV),
|
||||
sources=(_source(),),
|
||||
structured_max_retries=1,
|
||||
)
|
||||
capture = _capture(messages_prefix_length=1, structured_max_retries=1)
|
||||
original_feedback = StructuredMW._with_feedback
|
||||
|
||||
def feedback(self, *args):
|
||||
"""只破坏重问产物，不替代生产阶梯或解析。"""
|
||||
request = original_feedback(self, *args)
|
||||
messages = [dict(message) for message in request.messages]
|
||||
if damage == "prefix":
|
||||
messages[0]["content"] = "changed"
|
||||
elif damage == "role":
|
||||
messages[-1]["role"] = "assistant"
|
||||
elif damage == "content":
|
||||
messages[-1]["content"] = ["wrong-type"]
|
||||
elif damage == "unpaired":
|
||||
messages.pop()
|
||||
elif damage == "budget":
|
||||
messages.extend(messages[-2:])
|
||||
return replace(request, messages=messages)
|
||||
|
||||
monkeypatch.setattr(StructuredMW, "_with_feedback", feedback)
|
||||
requests = []
|
||||
|
||||
def handler(request):
|
||||
requests.append(json.loads(request.content))
|
||||
body = _response()
|
||||
body["choices"][0]["message"]["content"] = (
|
||||
'{"answer":5}' if len(requests) == 1 else '{"answer":5,"reason":"sum"}'
|
||||
)
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
original_factory = capture.client_factory
|
||||
|
||||
def factory(source):
|
||||
client = original_factory(source)
|
||||
client._transport = httpx.MockTransport(handler)
|
||||
if damage == "wire":
|
||||
|
||||
async def corrupt(request):
|
||||
payload = json.loads(request.content)
|
||||
if len(payload["messages"]) > 1:
|
||||
payload["messages"][-1]["content"] = "well-shaped-but-corrupted"
|
||||
request._content = json.dumps(payload).encode()
|
||||
|
||||
client.event_hooks["request"].insert(0, corrupt)
|
||||
return client
|
||||
|
||||
capture.client_factory = factory
|
||||
async with observed_client(settings, capture) as client:
|
||||
response, verdict = await captured_chat_round(
|
||||
client,
|
||||
capture,
|
||||
run_id="structured",
|
||||
matrix_id="reask",
|
||||
round_index=1,
|
||||
output_dir=tmp_path,
|
||||
messages=_MESSAGES,
|
||||
models={"source": "gpt-5.5"},
|
||||
aliases={},
|
||||
stream=False,
|
||||
structured=Answer,
|
||||
)
|
||||
assert response.structured_data.answer == 5
|
||||
assert len(requests) == 2
|
||||
assert verdict.status == ("PASS" if damage is None else "FAIL")
|
||||
text = "".join(path.read_text() for path in tmp_path.rglob("*.md"))
|
||||
assert _SECRET not in text and _PROMPT not in text
|
||||
|
||||
|
||||
def _thinking_consumer():
|
||||
"""仅加载 live 消费者定义与字面矩阵，跳过所有环境读取语句。"""
|
||||
import ast
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
path = Path(__file__).parents[1] / "e2e/test_thinking_live.py"
|
||||
tree = ast.parse(path.read_text())
|
||||
excluded = {
|
||||
"_ENV",
|
||||
"_HAS_SOURCE",
|
||||
"pytestmark",
|
||||
"_ROUNDS",
|
||||
"_TIER_ROUNDS",
|
||||
"_TIER_LONG_ROUNDS",
|
||||
"_TIER_CONCURRENCY",
|
||||
}
|
||||
tree.body = [
|
||||
node
|
||||
for node in tree.body
|
||||
if not (
|
||||
isinstance(node, ast.Assign)
|
||||
and any(
|
||||
isinstance(target, ast.Name) and target.id in excluded for target in node.targets
|
||||
)
|
||||
)
|
||||
]
|
||||
namespace: dict[str, Any] = {
|
||||
"_ROUNDS": 3,
|
||||
"_TIER_ROUNDS": 2,
|
||||
"_TIER_LONG_ROUNDS": 1,
|
||||
"_TIER_CONCURRENCY": 1,
|
||||
}
|
||||
exec(compile(tree, str(path), "exec"), namespace)
|
||||
return SimpleNamespace(**namespace), namespace
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("observation", "qualification", "expected"),
|
||||
[
|
||||
(O.ABSENT, "PASS", "UNCOVERED"),
|
||||
(O.OBSERVED, "PASS", "UNCOVERED"),
|
||||
(O.UNKNOWN, "PASS", "UNCOVERED"),
|
||||
(O.ABSENT, "FAIL", "FAIL"),
|
||||
],
|
||||
)
|
||||
async def test_unregistered_candidate_has_no_disable_declaration(
|
||||
observation, qualification, expected
|
||||
):
|
||||
"""执行真实 T10 消费者，未登记不等于不可关闭，资格失败仍红。"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
live, namespace = _thinking_consumer()
|
||||
conclusions = []
|
||||
calls = []
|
||||
|
||||
async def probe(model, effort, *, rounds, **kwargs):
|
||||
calls.append(rounds)
|
||||
return [
|
||||
{
|
||||
"verdict": LiveVerdict(qualification, "safe"),
|
||||
"response": SimpleNamespace(thinking_observation=observation),
|
||||
}
|
||||
for _ in range(rounds)
|
||||
]
|
||||
|
||||
namespace["_probe_effort"] = probe
|
||||
namespace["_conclude"] = lambda matrix, verdict, **kwargs: conclusions.append(verdict)
|
||||
await live.TestTierProbe().test_t10_none_direction_matches_declaration("claude-haiku-5")
|
||||
assert conclusions[0].status == expected
|
||||
assert calls == ([2, 1] if observation is not O.OBSERVED and qualification == "PASS" else [2])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["none", "tiers", "L8"])
|
||||
async def test_capability_conclusions_link_models_and_all_subruns(tmp_path, case):
|
||||
"""两个型号的 PASS／UNCOVERED 结论必须关联原件和完整短长／档位轮数。"""
|
||||
from contextlib import asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
from polygateway import GatewaySettings
|
||||
from tests.unit.test_config import _BASE_ENV
|
||||
|
||||
live, namespace = _thinking_consumer()
|
||||
settings = GatewaySettings.from_env("LLM", env=_BASE_ENV)
|
||||
namespace["_OUT_DIR"] = tmp_path
|
||||
namespace["_tier_settings"] = lambda model: replace(
|
||||
settings, sources=(replace(_source(), model=model),)
|
||||
)
|
||||
namespace["_settings"] = lambda **kwargs: replace(
|
||||
settings, sources=(replace(_source(), **kwargs),)
|
||||
)
|
||||
namespace["enforce_verdict"] = lambda verdict: None
|
||||
|
||||
@asynccontextmanager
|
||||
async def client(*args, **kwargs):
|
||||
yield None
|
||||
|
||||
async def round_call(client, capture, **kwargs):
|
||||
model = kwargs["models"]["source"]
|
||||
observation = (
|
||||
O.UNKNOWN if model == "gpt-5.4" else O.OBSERVED if case == "tiers" else O.ABSENT
|
||||
)
|
||||
write_live_round(
|
||||
kwargs["output_dir"],
|
||||
run_id=kwargs["run_id"],
|
||||
matrix_id=kwargs["matrix_id"],
|
||||
round_index=kwargs["round_index"],
|
||||
safe_fields={
|
||||
"requested_model": model,
|
||||
"session_id": kwargs["run_id"],
|
||||
"status": "PASS",
|
||||
"thinking_observation": observation,
|
||||
},
|
||||
)
|
||||
return SimpleNamespace(thinking_observation=observation), LiveVerdict("PASS", "safe")
|
||||
|
||||
namespace["observed_client"] = client
|
||||
namespace["captured_chat_round"] = round_call
|
||||
for model in ("gpt-5.5", "gpt-5.4"):
|
||||
if case == "none":
|
||||
await live.TestTierProbe().test_t10_none_direction_matches_declaration(model)
|
||||
elif case == "tiers":
|
||||
await live.TestTierProbe().test_t10_declared_tiers_actually_reason(model)
|
||||
else:
|
||||
await live.TestCapabilityDrift().test_declared_capability_matches_reality(model)
|
||||
matrix = {"none": "T10-none", "tiers": "T10-tiers", "L8": "L8"}[case]
|
||||
finals = list(tmp_path.rglob(f"{matrix}-0-*.md"))
|
||||
assert len(finals) == 2
|
||||
seen = set()
|
||||
for path in finals:
|
||||
row = json.loads(path.read_text().split("```json\n")[1].split("\n```")[0])
|
||||
assert "requested_model" in row, "结论缺型号，无法关联逐轮原件"
|
||||
model = row["requested_model"]
|
||||
seen.add(model)
|
||||
assert row["session_id"] == path.parent.name
|
||||
assert row["status"] == ("PASS" if model == "gpt-5.5" else "UNCOVERED")
|
||||
assert row["proposition"]
|
||||
subruns = row["subruns"]
|
||||
expected_groups = (
|
||||
2
|
||||
if case == "none"
|
||||
else len(
|
||||
[
|
||||
effort
|
||||
for effort in live.DEFAULT_CAPABILITIES[model].supported_efforts
|
||||
if effort is not live.Effort.NONE
|
||||
]
|
||||
)
|
||||
if case == "tiers"
|
||||
else 1
|
||||
)
|
||||
assert len(subruns) == expected_groups
|
||||
assert row["planned_rounds"] == sum(subrun["planned_rounds"] for subrun in subruns)
|
||||
assert row["completed_rounds"] == row["planned_rounds"]
|
||||
for subrun in subruns:
|
||||
originals = [
|
||||
p
|
||||
for p in path.parent.glob(f"{subrun['matrix_id']}-*.md")
|
||||
if p.name[len(subrun["matrix_id"]) + 1 :].split("-", 1)[0].isdigit()
|
||||
and not p.name.startswith(subrun["matrix_id"] + "-0-")
|
||||
]
|
||||
assert len(originals) == subrun["planned_rounds"] == subrun["completed_rounds"]
|
||||
for original in originals:
|
||||
data = json.loads(original.read_text().split("```json\n")[1].split("\n```")[0])
|
||||
assert data["requested_model"] == model
|
||||
assert data["session_id"] == row["session_id"]
|
||||
assert seen == {"gpt-5.5", "gpt-5.4"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("machine_type", ["model_not_found", _SECRET, "arbitrary-upstream-text"])
|
||||
def test_safe_machine_type_retains_only_known_enum(tmp_path, machine_type):
|
||||
"""认可机器枚举可复核；任意机器正文与 sentinel 不得落盘。"""
|
||||
_, attempts = _failure(body=json.dumps({"error": {"type": machine_type}}).encode())
|
||||
fields = {"attempts": safe_attempts(attempts)}
|
||||
event = fields["attempts"][0]["http"][0]
|
||||
assert event.get("machine_type") == (
|
||||
"model_not_found" if machine_type == "model_not_found" else "omitted"
|
||||
)
|
||||
path = write_live_round(
|
||||
tmp_path, run_id="safe", matrix_id="machine", round_index=1, safe_fields=fields
|
||||
)
|
||||
text = path.read_text()
|
||||
assert _SECRET not in text and "arbitrary-upstream-text" not in text
|
||||
if machine_type != "model_not_found":
|
||||
event["machine_type"] = machine_type
|
||||
with pytest.raises(ValueError, match="机器"):
|
||||
write_live_round(
|
||||
tmp_path, run_id="unsafe", matrix_id="machine", round_index=1, safe_fields=fields
|
||||
)
|
||||
|
||||
|
||||
async def test_structured_first_attempt_requires_exact_initial_messages():
|
||||
"""结构化窄规则不能允许首轮凭空带入一对反馈。"""
|
||||
capture = _capture(messages_prefix_length=1, structured_max_retries=1)
|
||||
real = _real_transport(capture, lambda request: httpx.Response(200, json=_response()))
|
||||
try:
|
||||
with capture.round_context(session_id="first", parent_call_id="parent"):
|
||||
await ObservedTransport(real, capture).complete(
|
||||
messages=[
|
||||
*_MESSAGES,
|
||||
{"role": "assistant", "content": "old"},
|
||||
{"role": "user", "content": "retry"},
|
||||
],
|
||||
source=_source(),
|
||||
stream=False,
|
||||
overlay={},
|
||||
call_id="first",
|
||||
reasoning_effort=None,
|
||||
)
|
||||
event = capture.attempts(session_id="first", parent_call_id="parent")[0].http[0]
|
||||
assert not request_is_valid(event)
|
||||
finally:
|
||||
await real.aclose()
|
||||
|
||||
Reference in New Issue
Block a user