fix: align soak scoreboard metrics with limiter semantics

RPM invariant now buckets by admit time on the Redis server clock,
exempts +/-2s boundary jitter, and excludes cache-hit rows that never
consumed a limiter slot. RSS sampling reports current ps RSS instead of
the monotonic ru_maxrss peak. Dispatch is paced by the concurrency
semaphore so --max-hours stays live and memory stays bounded. Add
--rescore RUN_ID to re-judge a finished run offline.
This commit is contained in:
2026-07-21 07:22:34 -04:00
parent ab9e3d10b2
commit 56defc88fb
3 changed files with 263 additions and 49 deletions
+49 -14
View File
@@ -45,23 +45,58 @@ def inv_call_ids_unique(rows: list[Row]) -> None:
assert not dupes, f"call_id 重复: {dupes[:5]}"
def _minute_bucket(created_at: str) -> int:
return int(datetime.fromisoformat(str(created_at)).timestamp()) // 60
def _admit_second(row: Row, clock_offset_s: float) -> float:
"""还原准入时刻(服务器钟): created_at 是完成落库时刻,减去调用延迟。"""
done = datetime.fromisoformat(str(row["created_at"])).timestamp()
latency_ms = row.get("latency_ms") or 0
return done - float(latency_ms) / 1000.0 + clock_offset_s
def inv_rpm_never_exceeded(rows: list[Row], per_source_rpm: dict[str, int]) -> None:
"""不变量 3: 按遥测时间戳重算,任一分钟桶内单源请求数 ≤ RPM 配置。
口径 = 固定分钟窗口(与限流器 `int(sec/60)` 同源);滑动 60s 窗会对
"窗尾+窗头"的合法背靠背流量误报,不采用(findings §4 条 3 的执行口径)。
"""
buckets: Counter[tuple[str, int]] = Counter(
(r["source_name"], _minute_bucket(r["created_at"]))
for r in rows
if per_source_rpm.get(r["source_name"], 0) > 0
def _movable_edge_rows(
admits: list[float], buckets: Counter[int], minute: int, limit: int, slack_s: float
) -> int:
"""超限窗口内可归邻窗的贴边行数(受邻窗余量约束)。"""
near_edge = sum(
1
for a in admits
if int(a // 60) == minute and min(a - minute * 60, (minute + 1) * 60 - a) <= slack_s
)
breaches = {key: n for key, n in buckets.items() if n > per_source_rpm[key[0]]}
assert not breaches, f"RPM 击穿: {dict(list(breaches.items())[:5])}"
room = max(0, limit - buckets.get(minute - 1, 0)) + max(0, limit - buckets.get(minute + 1, 0))
return min(near_edge, room)
def inv_rpm_never_exceeded(
rows: list[Row],
per_source_rpm: dict[str, int],
*,
clock_offset_s: float = 0.0,
boundary_slack_s: float = 2.0,
) -> None:
"""不变量 3: 任一限流器分钟窗口内单源准入数 ≤ RPM 配置。
口径与限流器同源(`backends/redis/limiter.py` `_window_id`): 窗口 =
**Redis 服务器钟**的固定分钟;准入时刻 = created_at(完成落库) latency。
滑动 60s 窗会对"窗尾+窗头"的合法背靠背流量误报,不采用(findings §4 条 3)。
created_at 秒级截断给准入时刻 ±秒级噪声,距窗口边界 ≤ boundary_slack_s 的
行允许归入有余量的邻窗。缓存命中行不计: 缓存在限流闸之前返回,未耗名额
也未打网关(遥测必录使其带原源名落库)。2026-07-21 P6 教训: 本机钟聚桶 +
完成时刻口径 + 计入缓存行,三重口径偏差曾把合规流量误判为击穿。
"""
per_source: dict[str, list[float]] = {}
for r in rows:
if per_source_rpm.get(r["source_name"], 0) > 0 and not r.get("cache_hit"):
per_source.setdefault(r["source_name"], []).append(_admit_second(r, clock_offset_s))
breaches: dict[tuple[str, int], int] = {}
for source, admits in per_source.items():
limit = per_source_rpm[source]
buckets = Counter(int(a // 60) for a in admits)
for minute, n in buckets.items():
if n <= limit:
continue
movable = _movable_edge_rows(admits, buckets, minute, limit, boundary_slack_s)
if n - movable > limit:
breaches[(source, minute)] = n
assert not breaches, f"RPM 击穿(准入时刻+服务器钟口径): {dict(list(breaches.items())[:5])}"
def inv_rss_stable(samples_mb: list[float], *, max_growth_mb: float) -> None: