135 lines
5.3 KiB
Python
135 lines
5.3 KiB
Python
"""压测记分板: 六条硬不变量断言 + 报告产出(findings §4;吃自己遥测的狗粮)。
|
|
|
|
输入口径三类(M2 计划 T11): 不变量 2/3/5/6 = 纯函数(遥测行迭代器);
|
|
不变量 1(记账归零)= 活 Redis/后端检查(async);不变量 4(RSS)= 纯函数
|
|
(run_soak 周期采样 json)。断言函数抛 AssertionError 即不变量被击穿。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
from collections import Counter
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
Row = dict[str, Any]
|
|
|
|
|
|
def load_rows(*db_paths: Path | str) -> list[Row]:
|
|
"""合并全部 worker 遥测库的 llm_calls 行(dict 形态)。"""
|
|
rows: list[Row] = []
|
|
for path in db_paths:
|
|
conn = sqlite3.connect(path)
|
|
conn.row_factory = sqlite3.Row
|
|
try:
|
|
rows.extend(dict(r) for r in conn.execute("SELECT * FROM llm_calls"))
|
|
finally:
|
|
conn.close()
|
|
return rows
|
|
|
|
|
|
# —— 硬不变量(纯函数) ——
|
|
|
|
|
|
def inv_rows_match_calls(rows: list[Row], *, expected_calls: int, tolerance: int = 0) -> None:
|
|
"""不变量 2a: 遥测行数 == 发出的请求数(±取消双记的已知语义容差)。"""
|
|
assert abs(len(rows) - expected_calls) <= tolerance, (
|
|
f"遥测行数 {len(rows)} ≠ 请求数 {expected_calls}(容差 {tolerance})"
|
|
)
|
|
|
|
|
|
def inv_call_ids_unique(rows: list[Row]) -> None:
|
|
"""不变量 2b: call_id 无重复。"""
|
|
dupes = [cid for cid, n in Counter(r["call_id"] for r in rows).items() if n > 1]
|
|
assert not dupes, f"call_id 重复: {dupes[:5]}"
|
|
|
|
|
|
def _minute_bucket(created_at: str) -> int:
|
|
return int(datetime.fromisoformat(str(created_at)).timestamp()) // 60
|
|
|
|
def inv_rpm_never_exceeded(rows: list[Row], per_source_rpm: dict[str, int]) -> None:
|
|
"""不变量 3: 按遥测时间戳重算,任一分钟桶内单源请求数 ≤ RPM 配置。
|
|
|
|
口径 = 固定分钟窗口(与限流器 `int(sec/60)` 同源);滑动 60s 窗会对
|
|
"窗尾+窗头"的合法背靠背流量误报,不采用(findings §4 条 3 的执行口径)。
|
|
"""
|
|
buckets: Counter[tuple[str, int]] = Counter(
|
|
(r["source_name"], _minute_bucket(r["created_at"]))
|
|
for r in rows
|
|
if per_source_rpm.get(r["source_name"], 0) > 0
|
|
)
|
|
breaches = {
|
|
key: n for key, n in buckets.items() if n > per_source_rpm[key[0]]
|
|
}
|
|
assert not breaches, f"RPM 击穿: {dict(list(breaches.items())[:5])}"
|
|
|
|
|
|
def inv_rss_stable(samples_mb: list[float], *, max_growth_mb: float) -> None:
|
|
"""不变量 4: 进程 RSS 首末差 < 阈值(窗口计数器/租约表无无界增长)。"""
|
|
assert len(samples_mb) >= 2, "RSS 采样不足"
|
|
growth = samples_mb[-1] - samples_mb[0]
|
|
assert growth < max_growth_mb, f"RSS 增长 {growth:.1f}MB ≥ 阈值 {max_growth_mb}MB"
|
|
|
|
|
|
def structured_success_rate(rows: list[Row], *, session_suffix: str) -> float:
|
|
"""不变量 5 的量: 指定场景行的最终成功率(error 为空即成功)。"""
|
|
scoped = [r for r in rows if str(r.get("session_id") or "").endswith(session_suffix)]
|
|
if not scoped:
|
|
return 0.0
|
|
ok = sum(1 for r in scoped if not r.get("error"))
|
|
return ok / len(scoped)
|
|
|
|
|
|
async def inv_accounting_zeroed(limiter, sources: list[str]) -> None:
|
|
"""不变量 1: 结束后全部源 inflight == 0(无泄漏租约)。活后端检查。"""
|
|
for name in sources:
|
|
stats = await limiter.source_stats(name)
|
|
assert stats.inflight == 0, f"源 {name} 结束后 inflight={stats.inflight}(租约泄漏)"
|
|
|
|
|
|
# —— 报告 ——
|
|
|
|
|
|
def _percentile(values: list[float], q: float) -> float:
|
|
if not values:
|
|
return 0.0
|
|
ordered = sorted(values)
|
|
return ordered[min(int(len(ordered) * q), len(ordered) - 1)]
|
|
|
|
|
|
def render_report(run_id: str, rows: list[Row], verdicts: list[tuple[str, str]]) -> str:
|
|
"""结构化 Markdown 报告(不变量 6: 成本/延迟基线产出)。"""
|
|
errors = Counter(str(r["error"]).split(":")[0] for r in rows if r.get("error"))
|
|
latencies = [float(r["latency_ms"]) for r in rows if not r.get("error")]
|
|
ttfts = [float(r["ttft_ms"]) for r in rows if r.get("ttft_ms") is not None]
|
|
costs = [float(r["cost"]) for r in rows if r.get("cost") is not None]
|
|
tokens = sum(int(r["prompt_tokens"]) + int(r["completion_tokens"]) for r in rows)
|
|
cache_hits = sum(1 for r in rows if r.get("cache_hit"))
|
|
lines = [
|
|
f"# Soak 报告: {run_id}",
|
|
"",
|
|
"## 不变量裁决",
|
|
*(f"- {name}: {verdict}" for name, verdict in verdicts),
|
|
"",
|
|
"## 规模",
|
|
f"- 遥测行数: {len(rows)};缓存命中: {cache_hits};总 token: {tokens}",
|
|
f"- 错误分布: {dict(errors) or '无'}",
|
|
"",
|
|
"## 延迟基线(成功行)",
|
|
f"- 总时长 p50/p95: {_percentile(latencies, 0.5):.0f} / {_percentile(latencies, 0.95):.0f} ms",
|
|
f"- TTFT p50/p95: {_percentile(ttfts, 0.5):.0f} / {_percentile(ttfts, 0.95):.0f} ms",
|
|
"",
|
|
"## 成本",
|
|
f"- 累计 cost: {sum(costs):.4f}(有单价行 {len(costs)}/{len(rows)})",
|
|
]
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def write_report(run_id: str, content: str, out_dir: Path | str = "tests/outputs/soak") -> Path:
|
|
out = Path(out_dir)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
path = out / f"{run_id}.md"
|
|
path.write_text(content, encoding="utf-8")
|
|
return path
|