test: verify sampling parameters through the full governance stack

This commit is contained in:
2026-07-31 21:46:05 -04:00
parent 2958dc8231
commit cce7562d07
5 changed files with 78 additions and 16 deletions
@@ -4,6 +4,7 @@
"""
import asyncio
import dataclasses
import json
import sqlite3
@@ -204,3 +205,74 @@ class TestTransientErrorExport:
assert isinstance(ei.value, polygateway.AllSourcesExhausted)
assert isinstance(ei.value.__cause__, TransientError)
class TestSamplingThroughStack:
"""issue #4: 采样参数经完整洋葱到达请求体,且缓存/遥测口径一致。"""
async def test_reaches_wire_and_lands_in_telemetry(self, tmp_path):
seen = []
def handler(request):
seen.append(json.loads(request.content))
return _sse()
db = tmp_path / "t.db"
recorder = SQLiteRecorder(db)
client = _full_client(handler, telemetry=recorder)
await client.chat([{"role": "user", "content": "hi"}], overlay={"seed": 42})
recorder.close()
assert seen[0]["seed"] == 42 # 穿过全栈到达线上
rows = sqlite3.connect(db).execute("SELECT sampling FROM llm_calls").fetchall()
assert json.loads(rows[0][0]) == {"seed": 42}
async def test_config_level_merges_and_records(self, tmp_path):
"""源级 extra_body 只有 emit_attempt 记得到(唯一有生效源的入口)。"""
seen = []
def handler(request):
seen.append(json.loads(request.content))
return _sse()
src = dataclasses.replace(_source(), extra_body={"temperature": 0})
db = tmp_path / "t.db"
recorder = SQLiteRecorder(db)
client = GatewayClient(
scope="llm",
sources=[src],
selector=RoundRobinSelector(),
limiter=InMemoryLimiter(
scope="llm", sources={src.name: src}, global_limits=GlobalLimits(0, 0, 0)
),
breaker=InMemoryGate(config=_BREAKER),
transport=OpenAICompatTransport(
client_factory=lambda s: httpx.AsyncClient(transport=httpx.MockTransport(handler))
),
retry=RetryPolicy(2, 2.0, 30.0),
backpressure=BackpressurePolicy(300.0, 0.01),
telemetry=recorder,
structured_strategy=JsonRepairStrategy(),
sleep=_noop_sleep,
)
await client.chat([{"role": "user", "content": "hi"}], overlay={"seed": 1})
recorder.close()
assert seen[0]["temperature"] == 0 and seen[0]["seed"] == 1
rows = sqlite3.connect(db).execute("SELECT sampling FROM llm_calls").fetchall()
assert json.loads(rows[0][0]) == {"seed": 1, "temperature": 0}
async def test_differing_seed_bypasses_cache_end_to_end(self):
"""issue 场景全栈回归: 逐 rollout 变 seed 必须真的回源。"""
calls = []
def handler(request):
calls.append(json.loads(request.content)["seed"])
return _sse()
client = _full_client(handler, cache=InMemoryCache())
await client.chat([{"role": "user", "content": "hi"}], overlay={"seed": 1})
await client.chat([{"role": "user", "content": "hi"}], overlay={"seed": 2})
second_same = await client.chat([{"role": "user", "content": "hi"}], overlay={"seed": 1})
assert calls == [1, 2] # 两个不同 seed 各自回源
assert second_same.cache_hit is True # 同 seed 才命中
+1 -3
View File
@@ -381,9 +381,7 @@ class TestExtraBodyStripped:
会显示这次调用带了 temperature=0——那是数据造假,比参数失效更坏。
"""
rec = _MemoryRecorder()
client, _ = _embed_client(
[_src(extra_body={"temperature": 0})], ["ok"], telemetry=rec
)
client, _ = _embed_client([_src(extra_body={"temperature": 0})], ["ok"], telemetry=rec)
await client.embed(["hi"])
assert rec.rows[0]["sampling"] is None
+1 -3
View File
@@ -394,9 +394,7 @@ class TestExtraBodyStripped:
async def test_telemetry_never_records_a_parameter_that_was_not_sent(self):
"""不剥离则审计表会显示这次 OCR 带了 temperature=0——数据造假。"""
recorder = _MemoryRecorder()
client, _, _ = _client(
[_src(extra_body={"temperature": 0})], ["text"], telemetry=recorder
)
client, _, _ = _client([_src(extra_body={"temperature": 0})], ["text"], telemetry=recorder)
await client.recognize_text(b"IMG")
assert recorder.rows[0]["sampling"] is None