feat: collect reasoning_tokens from the provider usage payload (issue #6)
Reasoning tokens are already counted inside completion_tokens, so the cost total was never wrong -- what was missing is the attribution: how much of a call was spent thinking rather than answering. LLMResponse and TransportResult each gain a trailing reasoning_tokens field, and the telemetry port grows from 21 to 22 columns with the new column appended in both backends so fresh and migrated schemas keep the same physical order. None means this particular call did not report the field, not that the source never reports it: a relay that falls back to a local tokenizer replaces the whole usage object and drops completion_tokens_details. Downstream checks must therefore read "in (None, 0)"; no provider was observed reporting a literal zero.
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
"""遥测子系统测试: SQLiteRecorder(21 列)+ TelemetryEmitter 单一 helper + TelemetryMW。"""
|
||||
"""遥测子系统测试: SQLiteRecorder(22 列)+ TelemetryEmitter 单一 helper + TelemetryMW。"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
@@ -39,6 +39,7 @@ _EXPECTED_COLUMNS = [
|
||||
"cached_prompt_tokens",
|
||||
"model_reported",
|
||||
"sampling",
|
||||
"reasoning_tokens",
|
||||
]
|
||||
|
||||
|
||||
@@ -102,6 +103,7 @@ async def _record_minimal(recorder, call_id="c1", **overrides):
|
||||
"cached_prompt_tokens": None,
|
||||
"model_reported": None,
|
||||
"sampling": None,
|
||||
"reasoning_tokens": None,
|
||||
}
|
||||
fields.update(overrides)
|
||||
await recorder.record_llm_call(**fields)
|
||||
@@ -158,6 +160,22 @@ class TestSQLiteRecorder:
|
||||
assert rows["c-zero"] == 0 # 真实零命中,读回仍是 0 而非 NULL
|
||||
assert rows["c-none"] is None
|
||||
|
||||
async def test_reasoning_tokens_column_round_trip(self, tmp_path):
|
||||
"""issue #6: 7 / 0 / None 三种值各自如实落库,0 与 NULL 不得混同。"""
|
||||
recorder = SQLiteRecorder(tmp_path / "t.db")
|
||||
await _record_minimal(recorder, call_id="r-some", reasoning_tokens=7)
|
||||
await _record_minimal(recorder, call_id="r-zero", reasoning_tokens=0)
|
||||
await _record_minimal(recorder, call_id="r-none", reasoning_tokens=None)
|
||||
recorder.close()
|
||||
rows = dict(
|
||||
sqlite3.connect(tmp_path / "t.db")
|
||||
.execute("SELECT call_id, reasoning_tokens FROM llm_calls")
|
||||
.fetchall()
|
||||
)
|
||||
assert rows["r-some"] == 7
|
||||
assert rows["r-zero"] == 0 # 上报了且确实没推理
|
||||
assert rows["r-none"] is None # 本次调用未上报
|
||||
|
||||
async def test_sampling_column_round_trips(self, tmp_path):
|
||||
"""issue #4: 采样参数落库,否则事后无法证明某批数据跑在什么温度下。"""
|
||||
recorder = SQLiteRecorder(tmp_path / "t.db")
|
||||
@@ -284,6 +302,7 @@ class TestPostgresBackfillDiscipline:
|
||||
"cached_prompt_tokens",
|
||||
"model_reported",
|
||||
"sampling",
|
||||
"reasoning_tokens",
|
||||
]
|
||||
|
||||
def _recorder(self, conn):
|
||||
@@ -384,11 +403,12 @@ class TestEmitterObservabilityFields:
|
||||
source=_source(),
|
||||
call_id="cid-1",
|
||||
latency_ms=42,
|
||||
response=_resp(cached_prompt_tokens=64, model_reported="m-real"),
|
||||
response=_resp(cached_prompt_tokens=64, model_reported="m-real", reasoning_tokens=7),
|
||||
error=None,
|
||||
)
|
||||
assert rec.rows[0]["cached_prompt_tokens"] == 64
|
||||
assert rec.rows[0]["model_reported"] == "m-real"
|
||||
assert rec.rows[0]["reasoning_tokens"] == 7
|
||||
|
||||
async def test_failed_attempt_has_no_provider_facts(self):
|
||||
rec = _MemoryRecorder()
|
||||
@@ -402,16 +422,19 @@ class TestEmitterObservabilityFields:
|
||||
)
|
||||
assert rec.rows[0]["cached_prompt_tokens"] is None
|
||||
assert rec.rows[0]["model_reported"] is None
|
||||
assert rec.rows[0]["reasoning_tokens"] is None
|
||||
|
||||
async def test_cache_hit_replays_the_recorded_values(self):
|
||||
"""决策 B1: 命中行原样回放,故命中率统计必须带 WHERE cache_hit = false。"""
|
||||
rec = _MemoryRecorder()
|
||||
await TelemetryEmitter(rec).emit_cache_hit(
|
||||
request=_REQ, response=_resp(cached_prompt_tokens=64, model_reported="m-real")
|
||||
request=_REQ,
|
||||
response=_resp(cached_prompt_tokens=64, model_reported="m-real", reasoning_tokens=7),
|
||||
)
|
||||
row = rec.rows[0]
|
||||
assert row["cache_hit"] is True
|
||||
assert row["cached_prompt_tokens"] == 64 and row["model_reported"] == "m-real"
|
||||
assert row["reasoning_tokens"] == 7 # 与 cached 同口径原样回放
|
||||
|
||||
async def test_terminal_failure_records_none(self):
|
||||
rec = _MemoryRecorder()
|
||||
@@ -420,6 +443,7 @@ class TestEmitterObservabilityFields:
|
||||
)
|
||||
assert rec.rows[0]["cached_prompt_tokens"] is None
|
||||
assert rec.rows[0]["model_reported"] is None
|
||||
assert rec.rows[0]["reasoning_tokens"] is None
|
||||
|
||||
|
||||
class TestEmitterSamplingColumn:
|
||||
|
||||
Reference in New Issue
Block a user