feat: track logical call statistics across governed calls

This commit is contained in:
2026-09-09 10:04:39 -04:00
parent 300ced5dbd
commit 87c261bf73
13 changed files with 650 additions and 10 deletions
+72
View File
@@ -818,3 +818,75 @@ class TestExplicitCacheMigration:
finally:
for transport in transports:
await transport.aclose()
class TestCallStatsNotPoisoned:
"""缓存不得回放历史统计(1.3.5 设计 §3)。
统计描述**本次**调用;把上次那条存进去再放出来,等于对调用方谎称这次
重试了 N 次、耗了 M 毫秒。
"""
async def test_serialized_payload_carries_no_call_stats_key(self):
from polygateway.types import CallStats
backend = InMemoryCache()
mw = _mw(backend)
stats = CallStats(logical_call_id="lc-1", attempts=3, total_latency_ms=900)
terminal = _Terminal(_resp(call_stats=stats))
await mw(ChatRequest(messages=_MSGS), terminal)
key = build_cache_key("m", _MSGS, "proj", None)
stored = json.loads(await backend.get(key))
assert "call_stats" not in stored # asdict 会把它摊成 dict,必须显式剔除
async def test_historic_dict_never_impersonates_call_stats(self):
"""旧条目里的 `call_stats` dict 会被 `_RESPONSE_FIELDS` 放行,必须显式覆盖。
不覆盖就会有一个 dict 冒充 `CallStats` 从公共 API 漏给调用方,
`resp.call_stats.attempts` 当场 `AttributeError`。
"""
backend = InMemoryCache()
mw = _mw(backend)
key = build_cache_key("m", _MSGS, "proj", None)
poisoned = {
"content": "legacy",
"thinking": "",
"model": "m",
"provider": "p",
"prompt_tokens": 1,
"completion_tokens": 2,
"latency_ms": 30,
"ttft_ms": 5.0,
"max_inter_token_ms": 2.0,
"cache_hit": False,
"call_id": "orig",
"source_name": "s1",
"usage_source": "measured",
"call_stats": {
"logical_call_id": "stale-lc",
"attempts": 7,
"total_latency_ms": 9999,
},
}
await backend.set(key, json.dumps(poisoned), ttl_s=100)
terminal = _Terminal(_resp())
hit = await mw(ChatRequest(messages=_MSGS), terminal)
assert hit.content == "legacy" and terminal.calls == 0 # 真的走了缓存
assert hit.call_stats is None # dict 不得冒充 CallStats
async def test_cache_key_is_unchanged_by_the_new_field(self):
"""新增内部字段不得扰动 key 公式,否则存量缓存全量冷启动(黄金值)。"""
from polygateway.types import _CallContext
class _Clock:
def __call__(self):
return 1000.0
ctx = _CallContext(now=_Clock())
bare = build_cache_key("m", _MSGS, "proj", None)
assert bare == build_cache_key("m", _MSGS, "proj", None)
# 带上下文的请求与不带的请求必须落在同一个 key 上
with_ctx = ChatRequest(messages=_MSGS, call_context=ctx)
without = ChatRequest(messages=_MSGS)
assert with_ctx.cache_namespace == without.cache_namespace
assert digest_messages(with_ctx.messages) == digest_messages(without.messages)
+137
View File
@@ -1362,3 +1362,140 @@ async def test_synthetic_runtime_protocol_and_legacy_call_signatures():
assert client._transport._clients == {}
finally:
await client.aclose()
class _StatsClock:
"""确定性单调钟;测试主动推进以断言"哪些区段计入了总耗时""""
def __init__(self, start=1000.0):
self.t = start
def __call__(self):
return self.t
def advance(self, seconds):
self.t += seconds
class _TickingCache:
"""假缓存后端: 每次 IO 推进注入钟。
不推进时钟的替身会让"缓存 IO 计入总耗时"的断言退化成恒等于 0 的空转绿
(计划 §T1 替身构造要求)。
"""
def __init__(self, clock, tick=0.25):
self._clock = clock
self._tick = tick
self._data = {}
async def get(self, key):
self._clock.advance(self._tick)
return self._data.get(key)
async def set(self, key, value, ttl_s):
self._clock.advance(self._tick)
self._data[key] = value
class _TickingRecorder:
"""假 recorder: 写入时推进注入钟,用于断言内联遥测收尾计入总耗时。"""
def __init__(self, clock, tick=0.5):
self._clock = clock
self._tick = tick
self.rows = []
async def record_llm_call(self, **fields):
self._clock.advance(self._tick)
self.rows.append(fields)
class TestLogicalCallStats:
"""一次公开 chat 调用的统计(1.3.5 设计 §3)。"""
_MSG = [{"role": "user", "content": "hi"}]
async def test_success_reports_one_attempt(self):
async with _client() as client:
resp = await client.chat(self._MSG)
assert resp.call_stats is not None
assert resp.call_stats.attempts == 1
assert resp.call_stats.logical_call_id
async def test_concurrent_calls_do_not_share_counters_or_ids(self):
"""同一 client 并发两路必须各自计数与各自 ID(库铁律「纯 asyncio 中立」)。
上下文若被提升成 client 实例属性,这条就会红——那正是 VT
`evolve_llm = llm` 教训的同一形态。
"""
async with _client() as client:
a, b = await asyncio.gather(client.chat(self._MSG), client.chat(self._MSG))
assert a.call_stats.logical_call_id != b.call_stats.logical_call_id
assert a.call_stats.attempts == b.call_stats.attempts == 1
async def test_cache_hit_is_zero_attempts_with_a_fresh_logical_id(self):
"""命中不产生网关调用 → 0 尝试;且是**新**逻辑调用,不回放历史统计。"""
clock = _StatsClock()
cache = _TickingCache(clock)
async with _client(
cache=cache, cache_namespace="proj", cache_ttl_s=600, now=clock
) as client:
first = await client.chat(self._MSG)
second = await client.chat(self._MSG)
assert first.cache_hit is False and first.call_stats.attempts == 1
assert second.cache_hit is True
assert second.call_stats.attempts == 0
assert second.call_stats.logical_call_id != first.call_stats.logical_call_id
async def test_cache_io_counts_into_total_latency(self):
"""缓存读写是本次调用真实花掉的时间,必须进总耗时(设计 §3)。"""
clock = _StatsClock()
cache = _TickingCache(clock, tick=0.25)
async with _client(
cache=cache, cache_namespace="proj", cache_ttl_s=600, now=clock
) as client:
hit = (await client.chat(self._MSG), await client.chat(self._MSG))[1]
# 命中路径只有一次 get(0.25s),无网关调用
assert hit.call_stats.attempts == 0
assert hit.call_stats.total_latency_ms == 250
async def test_inline_telemetry_teardown_counts_into_total_latency(self):
"""成功响应的快照含返回前已完成的内联遥测耗时(设计 §6)。"""
clock = _StatsClock()
recorder = _TickingRecorder(clock, tick=0.5)
async with _client(telemetry=recorder, now=clock) as client:
resp = await client.chat(self._MSG)
assert recorder.rows # 确实写了行,否则本断言空转
assert resp.call_stats.total_latency_ms == 500
async def test_milliseconds_not_seconds(self):
"""毫秒/秒不混用: 1.5s 必须是 1500 而不是 1 或 1.5。"""
clock = _StatsClock()
recorder = _TickingRecorder(clock, tick=1.5)
async with _client(telemetry=recorder, now=clock) as client:
resp = await client.chat(self._MSG)
assert resp.call_stats.total_latency_ms == 1500
async def test_stats_work_without_any_telemetry(self):
"""统计生效与否**不由 telemetry 是否启用决定**(设计 §3.5)。"""
async with _client(telemetry=None) as client:
resp = await client.chat(self._MSG)
assert resp.call_stats is not None and resp.call_stats.attempts == 1
async def test_failure_exception_carries_no_stats_attribute(self):
"""本版**不向异常对象附加统计**(设计 §3.1): 第三方可能复用同一异常实例。"""
def reject(request):
return httpx.Response(400, json={"error": {"message": "bad"}})
async with _client(handler=reject) as client:
with pytest.raises(RequestRejectedError) as exc:
await client.chat(self._MSG)
assert hasattr(exc.value, "call_stats") is False
async def test_input_validation_stays_outside_the_stats_boundary(self):
"""校验异常保持原行为,发生在统计边界之外(设计 §3)。"""
async with _client() as client:
with pytest.raises(ValueError, match="meta"):
await client.chat(self._MSG, meta={"BAD-KEY": 1})
+40
View File
@@ -600,3 +600,43 @@ class TestReasonlessTelemetryContract:
assert seen == [{"model": "embed-1", "input": ["text"]}]
finally:
await transport.aclose()
class TestEmbedLogicalCallStats:
"""分批共享同一逻辑调用(1.3.5 设计 §3/§3.5)。"""
async def test_three_batches_count_three_attempts(self):
"""分批是库的实现细节,但每批都真打了一次网关,故计 3 次尝试。"""
client, _ = _embed_client([_src()], ["ok", "ok", "ok"], batch_size=2)
resp = await client.embed(["a", "bb", "ccc", "dddd", "eeeee"])
assert resp.call_stats is not None
assert resp.call_stats.attempts == 3
async def test_separate_calls_get_distinct_logical_ids(self):
"""一次公开调用一个 ID: 两次 `embed` 不得共用同一个。
共用就意味着上下文被提升成了 client 实例属性(库铁律禁止的形态)。
"""
client, _ = _embed_client([_src()], ["ok"] * 5, batch_size=2)
first = await client.embed(["a", "bb", "ccc", "dddd", "eeeee"]) # 3 批
second = await client.embed(["x", "y"]) # 1 批
assert first.call_stats.attempts == 3 and second.call_stats.attempts == 1
assert first.call_stats.logical_call_id != second.call_stats.logical_call_id
async def test_retry_within_a_batch_is_counted(self):
client, _ = _embed_client([_src(), _src(name="e2")], [TransientError("t1"), "ok"])
resp = await client.embed(["a"])
assert resp.call_stats.attempts == 2
async def test_empty_input_is_zero_attempts_and_writes_no_telemetry_row(self):
"""合法零尝试: 返回真实统计,且**不写任何遥测行**(设计 §3 M2)。
与 cache_hit 不同——不要按"遥测必录"推断空输入也有台账行。
"""
rec = _MemoryRecorder()
client, _ = _embed_client([_src()], [], telemetry=rec)
resp = await client.embed([])
assert resp.call_stats is not None
assert resp.call_stats.attempts == 0
assert resp.call_stats.logical_call_id # 真实 ID,不是空串
assert rec.rows == [] # 零遥测行
+31
View File
@@ -615,3 +615,34 @@ class TestReasonlessTelemetryContract:
await getattr(client, method)(b"image")
assert len(recorder.rows) == len(script)
assert all(r["error"] and r["reasoning_effort"] is None for r in recorder.rows)
class TestOcrLogicalCallStats:
"""OCR 两个公开方法各自拥有一次逻辑调用(1.3.5 设计 §3/§3.5)。"""
async def test_text_success_counts_one_attempt(self):
client, _, _ = _client([_src()], ["text"])
r = await client.recognize_text(b"jpg")
assert r.call_stats is not None and r.call_stats.attempts == 1
async def test_layout_two_http_calls_count_as_one_attempt(self):
"""POST + ZIP GET 在同一次 transport 调用内,计 **1** 次尝试而非 2。
`attempts` 的语义是"调用 transport 端口的次数",不是 HTTP 请求条数。
"""
client, _, _ = _client([_src()], ["layout"])
r = await client.parse_layout(b"jpg")
assert r.call_stats is not None and r.call_stats.attempts == 1
async def test_retry_counts_every_attempt(self):
client, _, _ = _client([_src(), _src(name="m2")], [TransientError("t1"), "text"])
r = await client.recognize_text(b"jpg")
assert r.call_stats.attempts == 2
async def test_input_validation_stays_outside_the_stats_boundary(self):
"""`image` 类型/空校验先于上下文创建(M1 例外),保持原异常行为。"""
client, _, _ = _client([_src()], [])
with pytest.raises(TypeError):
await client.recognize_text("not-bytes")
with pytest.raises(ValueError):
await client.recognize_text(b"")
+58
View File
@@ -5,6 +5,7 @@
"""
import asyncio
import dataclasses
import pytest
@@ -776,3 +777,60 @@ class TestRateLimitPushback:
await mw(_REQ)
assert ei.value.reason == "retry_exhausted"
assert len(transport.calls) == 3
class TestLogicalAttemptCounting:
"""尝试登记在 transport 调用**之前**(1.3.5 设计 §4)。
登记点若挪到成功之后,失败与取消的尝试就会从计数里消失——而那正是
诊断时最需要看见的几次。
"""
def _ctx(self, clock):
from polygateway.types import _CallContext
return _CallContext(now=clock)
async def test_single_success_counts_one(self):
mw, _, _, _, _, clock = _harness([_src("a")], [_ok()])
ctx = self._ctx(clock)
await mw(dataclasses.replace(_REQ, call_context=ctx))
assert ctx.snapshot().attempts == 1
async def test_failed_retries_are_counted(self):
"""两次可重试失败 + 一次成功 = 3 次尝试,不是 1 次。"""
mw, _, _, transport, _, clock = _harness(
[_src("a")], [TransientError("t1"), TransientError("t2"), _ok()]
)
ctx = self._ctx(clock)
await mw(dataclasses.replace(_REQ, call_context=ctx))
assert ctx.snapshot().attempts == 3 == len(transport.calls)
async def test_budget_free_429_still_counts_as_an_attempt(self):
"""429 免的是重试预算,不是"没发生过"——它确实打到了网关。"""
mw, _, _, transport, _, clock = _harness(
[_src("a")],
[
TransientError("t1", status_code=429, retry_after_s=1.0),
TransientError("t2", status_code=429, retry_after_s=1.0),
_ok(),
],
)
ctx = self._ctx(clock)
await mw(dataclasses.replace(_REQ, call_context=ctx))
assert ctx.snapshot().attempts == 3 == len(transport.calls)
async def test_retry_exhausted_counts_every_attempt(self):
mw, _, _, transport, _, clock = _harness(
[_src("a")], [TransientError(str(i)) for i in range(5)], max_attempts=3
)
ctx = self._ctx(clock)
with pytest.raises(AllSourcesExhausted):
await mw(dataclasses.replace(_REQ, call_context=ctx))
assert ctx.snapshot().attempts == 3 == len(transport.calls)
async def test_absent_context_does_not_break_the_call(self):
"""库内现场构造的 `ChatRequest` 没有上下文,不得因此报错(设计 §3.5)。"""
mw, _, _, _, _, _ = _harness([_src("a")], [_ok()])
resp = await mw(_REQ)
assert resp.content == "ok" and _REQ.call_context is None
+141
View File
@@ -610,3 +610,144 @@ class TestSourceConfigEffortNormalization:
"""非字符串同样只能是 `ValueError`: 公共入口不许把类型错误漏成 `AttributeError`。"""
with pytest.raises(ValueError, match="推理档位"):
_make_source(reasoning_effort=3)
class TestCallStatsAndContext:
"""逻辑调用统计内核(1.3.5 设计 §3/§4)。"""
def test_call_stats_is_frozen_snapshot(self):
from polygateway.types import CallStats
stats = CallStats(logical_call_id="lc-1", attempts=2, total_latency_ms=15)
with pytest.raises(dataclasses.FrozenInstanceError):
stats.attempts = 3
def test_context_counts_attempts_and_freezes_elapsed(self):
"""快照是同步冻结的时间切片: 登记两次尝试后耗时按注入钟折算成毫秒。"""
from polygateway.types import _CallContext
clock = _FakeMonotonic()
ctx = _CallContext(now=clock)
clock.advance(1.5)
ctx.register_attempt()
ctx.register_attempt()
stats = ctx.snapshot()
assert stats.attempts == 2
assert stats.total_latency_ms == 1500 # 秒→毫秒,不混用单位
def test_snapshot_is_repeatable_and_tracks_later_time(self):
from polygateway.types import _CallContext
clock = _FakeMonotonic()
ctx = _CallContext(now=clock)
first = ctx.snapshot()
clock.advance(2.0)
second = ctx.snapshot()
assert first.total_latency_ms == 0 and second.total_latency_ms == 2000
assert first.logical_call_id == second.logical_call_id
def test_each_context_gets_its_own_logical_id(self):
from polygateway.types import _CallContext
clock = _FakeMonotonic()
assert _CallContext(now=clock).logical_call_id != _CallContext(now=clock).logical_call_id
def test_claim_terminal_is_true_once(self):
"""终态去重位: 保证每逻辑调用至多写一条终态行(设计 §6 不变量 I3)。"""
from polygateway.types import _CallContext
ctx = _CallContext(now=_FakeMonotonic())
assert ctx.claim_terminal() is True
assert ctx.claim_terminal() is False
def test_chat_request_context_does_not_affect_equality_or_repr(self):
"""上下文是库内部件: 进 `compare`/`repr` 会污染既有请求语义与日志。"""
from polygateway.types import _CallContext
ctx = _CallContext(now=_FakeMonotonic())
bare = ChatRequest(messages=[{"role": "user", "content": "hi"}])
with_ctx = dataclasses.replace(bare, call_context=ctx)
assert with_ctx.call_context is ctx
assert with_ctx == bare
assert "call_context" not in repr(with_ctx)
def test_replace_preserves_the_same_context_reference(self):
"""洋葱各层经 `replace` 派生请求,上下文必须是同一实例而非拷贝。"""
from polygateway.types import _CallContext
ctx = _CallContext(now=_FakeMonotonic())
req = ChatRequest(messages=[{"role": "user", "content": "hi"}], call_context=ctx)
derived = dataclasses.replace(req, stream=False)
assert derived.call_context is ctx
def test_four_responses_default_call_stats_to_none(self):
"""第三方合成响应的 `None` 表示未知,不得伪造 0(设计 §3)。"""
from polygateway.types import (
EmbeddingResponse,
OcrLayoutResult,
OcrTextResult,
)
llm = LLMResponse(
content="c",
thinking="",
model="m",
provider="p",
prompt_tokens=1,
completion_tokens=1,
latency_ms=1,
ttft_ms=None,
max_inter_token_ms=None,
cache_hit=False,
call_id="c1",
)
emb = EmbeddingResponse(
vectors=[],
dim=0,
model="m",
provider="p",
prompt_tokens=0,
usage_source="measured",
latency_ms=0,
call_id="c2",
source_name="s",
)
text = OcrTextResult(
text="", source_name="s", usage=Usage(0, 0), latency_ms=0, call_id="c3", raw={}
)
layout = OcrLayoutResult(
elements=[],
page_sizes=[],
source_name="s",
usage=Usage(0, 0),
latency_ms=0,
call_id="c4",
raw={},
)
assert (llm.call_stats, emb.call_stats, text.call_stats, layout.call_stats) == (
None,
None,
None,
None,
)
def test_call_stats_is_exported_from_package_root(self):
"""四份平铺字段会漂移,故统计以单一对象出现在公共 API(设计 §3)。"""
import polygateway
from polygateway.types import CallStats
assert polygateway.CallStats is CallStats
assert "CallStats" in polygateway.__all__
class _FakeMonotonic:
"""确定性单调钟;不复用 contracts 的 FakeClock 以免 unit 反向依赖契约包。"""
def __init__(self, start: float = 1000.0) -> None:
self.t = start
def __call__(self) -> float:
return self.t
def advance(self, seconds: float) -> None:
self.t += seconds