From 15b9b02e967f83e380fb2df2882ac0dcc5546f76 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Fri, 31 Jul 2026 22:01:51 -0400 Subject: [PATCH] fix: make the sampling invariant test actually enforce the constraint The test passed overlay and sampling as separate objects while production aliases them, so an in-place mutation slipped through it. Also syncs the telemetry schema page and adds the missing postgres round-trip assertion. --- research-wiki/index.md | 4 +-- research-wiki/log.md | 1 + research-wiki/schemas/llm-calls.md | 28 ++++++++++++++++++-- tests/integration/test_postgres_telemetry.py | 9 ++++++- tests/unit/test_ports.py | 1 + tests/unit/test_structured.py | 22 ++++++++++++--- 6 files changed, 57 insertions(+), 8 deletions(-) diff --git a/research-wiki/index.md b/research-wiki/index.md index 87592ab..7b60264 100644 --- a/research-wiki/index.md +++ b/research-wiki/index.md @@ -1,6 +1,6 @@ # Research Wiki 索引 -> 自动生成，更新时间：2026-07-31 17:01 UTC +> 自动生成，更新时间：2026-08-01 01:58 UTC ## design (20) - [2026-07-20-m1-core-design](designs/2026-07-20-m1-core-design.md) `design:2026-07-20-m1-core-design` @@ -56,7 +56,7 @@ - [采样参数透传实现计划(issue #4)](plans/sampling-params-plan.md) `plan:sampling-params-plan` ## schema (1) -- [表结构: llm_calls(遥测 20 字段)](schemas/llm-calls.md) `schema:llm-calls` +- [表结构: llm_calls(遥测 21 字段)](schemas/llm-calls.md) `schema:llm-calls` ## metric (2) - [OCR 治理调用成功率与错误分类分布](metrics/ocr-call-success.md) `metric:ocr-call-success` diff --git a/research-wiki/log.md b/research-wiki/log.md index 1a27b91..c71e234 100644 --- a/research-wiki/log.md +++ b/research-wiki/log.md @@ -72,3 +72,4 @@ - [2026-07-31 16:59 UTC] 新增边: plan:sampling-params-plan --implements--> design:sampling-params - [2026-07-31 16:59 UTC] 重建索引: 50 篇页面 - [2026-07-31 17:01 UTC] 重建索引: 50 篇页面 +- [2026-08-01 01:58 UTC] 重建索引: 50 篇页面 diff --git a/research-wiki/schemas/llm-calls.md b/research-wiki/schemas/llm-calls.md index d27b372..236478e 100644 --- a/research-wiki/schemas/llm-calls.md +++ b/research-wiki/schemas/llm-calls.md @@ -1,11 +1,11 @@ --- type: schema node_id: schema:llm-calls -title: "表结构: llm_calls(遥测 20 字段)" +title: "表结构: llm_calls(遥测 21 字段)" date: 2026-07-20 --- -# 表结构: llm_calls(遥测 20 字段) +# 表结构: llm_calls(遥测 21 字段) ## 列定义(冻结,M1 设计 §4.4 / ARCH §7.8) @@ -26,6 +26,7 @@ date: 2026-07-20 | created_at | TEXT NOT NULL DEFAULT (datetime('now')) | 落库时刻 | | cached_prompt_tokens | INTEGER | 供应商 prompt cache 命中的输入 token(2026-07-31,issue #3);NULL = 该源未上报,`0` = 上报了真实零命中,两者不可混同 | | model_reported | TEXT | API 响应体实际返回的 model;NULL = 未上报。与 `model`(配置别名)可能分叉 | +| sampling | TEXT | 本次调用的采样参数 canonical JSON(2026-07-31,issue #4);NULL = 未传。见下方口径 | ## usage/成本口径(2026-07-30,est_tokens 解耦) @@ -50,6 +51,29 @@ FROM llm_calls WHERE cache_hit = false AND cached_prompt_tokens IS NOT NULL; `WHERE cache_hit = false` 不可省,理由与上面 cost 缺口口径同源:回放行计入即重复计数。 +## 采样参数口径(2026-07-31,issue #4) + +`sampling` 列 = 「调用方采样意图 ⊎ 生效源 `extra_body`」的 canonical JSON,空则 NULL。**不含**结构化输出注入的 `response_format`——列名是采样参数,schema 不是,且数 KB schema 逐行落库会让审计表无谓膨胀。补列纪律与 issue #3 两列逐字相同(排在末尾、先探测再 ALTER、失败只逐行降级)。 + +三个 emit 入口的取值必须各自定死,否则同一列在不同行含义不同: + +| 入口 | 调用者 | 有生效源? | 记什么 | +|---|---|---|---| +| `emit_attempt` | RetryMW(最内) | 有 | `merge(source.extra_body, request.sampling)` | +| `emit_cache_hit` | TelemetryMW(最外) | 无 | 仅 `request.sampling` | +| `emit_terminal_failure` | TelemetryMW | 无 | 仅 `request.sampling` | + +后两行缺 `extra_body` 是客观事实而非口径瑕疵——它们没有"生效源"可言,与 `model`/`source_name` 在终态行置空是同一先例;缓存命中行亦无损:`sampling` 已进缓存 key,能命中即意味调用级参数与历史那次逐字相同。三者统一读 `request.sampling` 而非 `request.overlay`(后者在 RetryMW 处已被结构化注入污染、在 TelemetryMW 处未被污染,直接用必然三行分叉)。 + +OCR / embedding 路径的该列**恒为 NULL**:两条路径的 transport 不发 `extra_body`(embed payload 硬编码 `{model, input}`、MonkeyOCR 只发 multipart),故其源在构造期就被剥离——不剥离则该列会记录一个从未发出的参数,那是数据造假而非参数失效。 + +复现某批实验的解码条件: + +```sql +SELECT DISTINCT sampling FROM llm_calls +WHERE session_id = $1 AND cache_hit = false AND error IS NULL; +``` + ## 埋点位置(单一 helper 铁律) - `middleware/telemetry.py::TelemetryEmitter` 是全库**唯一** `record_llm_call` 调用点; diff --git a/tests/integration/test_postgres_telemetry.py b/tests/integration/test_postgres_telemetry.py index 50286c5..3b1203c 100644 --- a/tests/integration/test_postgres_telemetry.py +++ b/tests/integration/test_postgres_telemetry.py @@ -11,6 +11,7 @@ DSN 走 .env `PGW_TELEMETRY_PG_DSN`,缺则 skip。该实例上有 app/chs_prod from __future__ import annotations import asyncio +import json import os from uuid import uuid4 @@ -180,9 +181,12 @@ class TestObservabilityColumns: await _record_minimal(recorder, call_id=_cid("hit"), cached_prompt_tokens=64) await _record_minimal(recorder, call_id=_cid("zero"), cached_prompt_tokens=0) await _record_minimal(recorder, call_id=_cid("model"), model_reported="MiniMax-01") + await _record_minimal( + recorder, call_id=_cid("samp"), sampling='{"seed": 42, "temperature": 0}' + ) rows = await _fetch( dsn, - "SELECT call_id, cached_prompt_tokens, model_reported FROM llm_calls " + "SELECT call_id, cached_prompt_tokens, model_reported, sampling FROM llm_calls " "WHERE call_id LIKE $1", f"{_RUN_PREFIX}-%", ) @@ -191,6 +195,9 @@ class TestObservabilityColumns: assert by_id[_cid("zero")]["cached_prompt_tokens"] == 0 # 真实零命中 ≠ NULL assert by_id[_cid("model")]["cached_prompt_tokens"] is None assert by_id[_cid("model")]["model_reported"] == "MiniMax-01" + # issue #4: PG 侧也须验非空 sampling 能读回原值(不只是列存在) + assert json.loads(by_id[_cid("samp")]["sampling"]) == {"seed": 42, "temperature": 0} + assert by_id[_cid("hit")]["sampling"] is None finally: await recorder.aclose() diff --git a/tests/unit/test_ports.py b/tests/unit/test_ports.py index cb60efd..74bb5f9 100644 --- a/tests/unit/test_ports.py +++ b/tests/unit/test_ports.py @@ -116,6 +116,7 @@ class _DummyRecorder: cost, cached_prompt_tokens, model_reported, + sampling, ) -> None: ... diff --git a/tests/unit/test_structured.py b/tests/unit/test_structured.py index 2b4df28..0400ee0 100644 --- a/tests/unit/test_structured.py +++ b/tests/unit/test_structured.py @@ -193,7 +193,14 @@ class TestSamplingSnapshotInvariant: terminal = ScriptedTerminal(["not json at all", '{"answer": 1, "reason": "r"}']) mw = _mw(strategy=NativeSchemaStrategy(), max_retries=1) await mw( - ChatRequest(messages=_MSGS, structured=Verdict, sampling=caller_sampling), + # overlay 与 sampling 传**同一个对象**,复现 client.py 的别名关系 + # ——否则中间件就地改写 overlay 时不会波及 sampling,这条执法就是空的 + ChatRequest( + messages=_MSGS, + structured=Verdict, + overlay=caller_sampling, + sampling=caller_sampling, + ), terminal, ) assert len(terminal.requests) == 2 # 确实重问过 @@ -205,11 +212,20 @@ class TestSamplingSnapshotInvariant: assert "response_format" not in seen.sampling async def test_middleware_does_not_mutate_caller_mapping(self): - """决策 E 的第二条约束: 中间件只能 replace 派生,不得就地改这两个 dict。""" + """决策 E 的第二条约束: 中间件只能 replace 派生,不得就地改这两个 dict。 + + 同样传同一对象: 生产中 overlay 与 sampling 是别名,任何对 overlay 的 + 就地改写都会同步毒化缓存 key 与遥测列。 + """ caller_sampling = {"seed": 7} terminal = ScriptedTerminal(['{"answer": 1, "reason": "r"}']) await _mw(strategy=NativeSchemaStrategy())( - ChatRequest(messages=_MSGS, structured=Verdict, sampling=caller_sampling), + ChatRequest( + messages=_MSGS, + structured=Verdict, + overlay=caller_sampling, + sampling=caller_sampling, + ), terminal, ) assert caller_sampling == {"seed": 7} # 调用方的对象未被污染