fix: align soak scoreboard metrics with limiter semantics
RPM invariant now buckets by admit time on the Redis server clock, exempts +/-2s boundary jitter, and excludes cache-hit rows that never consumed a limiter slot. RSS sampling reports current ps RSS instead of the monotonic ru_maxrss peak. Dispatch is paced by the concurrency semaphore so --max-hours stays live and memory stays bounded. Add --rescore RUN_ID to re-judge a finished run offline.
This commit is contained in:
+49
-14
@@ -45,23 +45,58 @@ def inv_call_ids_unique(rows: list[Row]) -> None:
|
||||
assert not dupes, f"call_id 重复: {dupes[:5]}"
|
||||
|
||||
|
||||
def _minute_bucket(created_at: str) -> int:
|
||||
return int(datetime.fromisoformat(str(created_at)).timestamp()) // 60
|
||||
def _admit_second(row: Row, clock_offset_s: float) -> float:
|
||||
"""还原准入时刻(服务器钟): created_at 是完成落库时刻,减去调用延迟。"""
|
||||
done = datetime.fromisoformat(str(row["created_at"])).timestamp()
|
||||
latency_ms = row.get("latency_ms") or 0
|
||||
return done - float(latency_ms) / 1000.0 + clock_offset_s
|
||||
|
||||
|
||||
def inv_rpm_never_exceeded(rows: list[Row], per_source_rpm: dict[str, int]) -> None:
|
||||
"""不变量 3: 按遥测时间戳重算,任一分钟桶内单源请求数 ≤ RPM 配置。
|
||||
|
||||
口径 = 固定分钟窗口(与限流器 `int(sec/60)` 同源);滑动 60s 窗会对
|
||||
"窗尾+窗头"的合法背靠背流量误报,不采用(findings §4 条 3 的执行口径)。
|
||||
"""
|
||||
buckets: Counter[tuple[str, int]] = Counter(
|
||||
(r["source_name"], _minute_bucket(r["created_at"]))
|
||||
for r in rows
|
||||
if per_source_rpm.get(r["source_name"], 0) > 0
|
||||
def _movable_edge_rows(
|
||||
admits: list[float], buckets: Counter[int], minute: int, limit: int, slack_s: float
|
||||
) -> int:
|
||||
"""超限窗口内可归邻窗的贴边行数(受邻窗余量约束)。"""
|
||||
near_edge = sum(
|
||||
1
|
||||
for a in admits
|
||||
if int(a // 60) == minute and min(a - minute * 60, (minute + 1) * 60 - a) <= slack_s
|
||||
)
|
||||
breaches = {key: n for key, n in buckets.items() if n > per_source_rpm[key[0]]}
|
||||
assert not breaches, f"RPM 击穿: {dict(list(breaches.items())[:5])}"
|
||||
room = max(0, limit - buckets.get(minute - 1, 0)) + max(0, limit - buckets.get(minute + 1, 0))
|
||||
return min(near_edge, room)
|
||||
|
||||
|
||||
def inv_rpm_never_exceeded(
|
||||
rows: list[Row],
|
||||
per_source_rpm: dict[str, int],
|
||||
*,
|
||||
clock_offset_s: float = 0.0,
|
||||
boundary_slack_s: float = 2.0,
|
||||
) -> None:
|
||||
"""不变量 3: 任一限流器分钟窗口内单源准入数 ≤ RPM 配置。
|
||||
|
||||
口径与限流器同源(`backends/redis/limiter.py` `_window_id`): 窗口 =
|
||||
**Redis 服务器钟**的固定分钟;准入时刻 = created_at(完成落库)− latency。
|
||||
滑动 60s 窗会对"窗尾+窗头"的合法背靠背流量误报,不采用(findings §4 条 3)。
|
||||
created_at 秒级截断给准入时刻 ±秒级噪声,距窗口边界 ≤ boundary_slack_s 的
|
||||
行允许归入有余量的邻窗。缓存命中行不计: 缓存在限流闸之前返回,未耗名额
|
||||
也未打网关(遥测必录使其带原源名落库)。2026-07-21 P6 教训: 本机钟聚桶 +
|
||||
完成时刻口径 + 计入缓存行,三重口径偏差曾把合规流量误判为击穿。
|
||||
"""
|
||||
per_source: dict[str, list[float]] = {}
|
||||
for r in rows:
|
||||
if per_source_rpm.get(r["source_name"], 0) > 0 and not r.get("cache_hit"):
|
||||
per_source.setdefault(r["source_name"], []).append(_admit_second(r, clock_offset_s))
|
||||
breaches: dict[tuple[str, int], int] = {}
|
||||
for source, admits in per_source.items():
|
||||
limit = per_source_rpm[source]
|
||||
buckets = Counter(int(a // 60) for a in admits)
|
||||
for minute, n in buckets.items():
|
||||
if n <= limit:
|
||||
continue
|
||||
movable = _movable_edge_rows(admits, buckets, minute, limit, boundary_slack_s)
|
||||
if n - movable > limit:
|
||||
breaches[(source, minute)] = n
|
||||
assert not breaches, f"RPM 击穿(准入时刻+服务器钟口径): {dict(list(breaches.items())[:5])}"
|
||||
|
||||
|
||||
def inv_rss_stable(samples_mb: list[float], *, max_growth_mb: float) -> None:
|
||||
|
||||
Reference in New Issue
Block a user