fix: align soak scoreboard metrics with limiter semantics

RPM invariant now buckets by admit time on the Redis server clock,
exempts +/-2s boundary jitter, and excludes cache-hit rows that never
consumed a limiter slot. RSS sampling reports current ps RSS instead of
the monotonic ru_maxrss peak. Dispatch is paced by the concurrency
semaphore so --max-hours stays live and memory stays bounded. Add
--rescore RUN_ID to re-judge a finished run offline.
This commit is contained in:
2026-07-21 07:22:34 -04:00
parent ab9e3d10b2
commit 56defc88fb
3 changed files with 263 additions and 49 deletions
+107 -5
View File
@@ -145,13 +145,24 @@ from tools.soak.scoreboard import ( # noqa: E402
)
def _row(call_id, *, source="s1", created_at="2026-07-20T10:00:00", error=None, session="r-p3"):
def _row(
call_id,
*,
source="s1",
created_at="2026-07-20T10:00:00",
error=None,
session="r-p3",
latency_ms=0,
cache_hit=0,
):
return {
"call_id": call_id,
"source_name": source,
"created_at": created_at,
"error": error,
"session_id": session,
"latency_ms": latency_ms,
"cache_hit": cache_hit,
}
@@ -169,11 +180,11 @@ class TestInvariants:
def test_rpm_minute_bucket(self):
# 固定分钟窗口口径(与限流器 int(sec/60) 同源;滑动窗会对合法的
# 跨窗背靠背流量误报,不采用)
# 跨窗背靠背流量误报,不采用)。击穿用例取窗口中部,避免贴边歧义。
rows = [
_row("a", created_at="2026-07-20T10:00:01"),
_row("b", created_at="2026-07-20T10:00:59"),
_row("c", created_at="2026-07-20T10:01:01"),
_row("a", created_at="2026-07-20T10:00:10"),
_row("b", created_at="2026-07-20T10:00:50"),
_row("c", created_at="2026-07-20T10:01:10"),
]
inv_rpm_never_exceeded(rows, {"s1": 2})
rows.append(_row("d", created_at="2026-07-20T10:00:30"))
@@ -183,6 +194,49 @@ class TestInvariants:
def test_rpm_ignores_unlimited_sources(self):
inv_rpm_never_exceeded([_row(str(i)) for i in range(100)], {"s1": 0})
def test_rpm_buckets_by_admit_time_not_completion(self):
# 准入相隔 100s(各窗 1 次),长延迟让完成时刻挤进同一分钟——
# 完成时刻聚桶会误报(P6 伪击穿教训之一)
rows = [
_row("a", created_at="2026-07-20T10:02:30", latency_ms=120_000), # 准入 10:00:30
_row("b", created_at="2026-07-20T10:02:35", latency_ms=25_000), # 准入 10:02:10
]
inv_rpm_never_exceeded(rows, {"s1": 1})
def test_rpm_server_clock_offset_dedistorts(self):
# P6 实跑复现: Redis 服务器钟快 31s,本机分钟内 10 次准入
# 实为两个服务器窗口各 5 次(+31 后: 37..59 前窗 | 63..90 后窗)
secs = [6, 19, 21, 26, 28, 32, 42, 47, 52, 59]
rows = [_row(str(i), created_at=f"2026-07-20T10:00:{s:02d}") for i, s in enumerate(secs)]
inv_rpm_never_exceeded(rows, {"s1": 5}, clock_offset_s=31.0)
def test_rpm_true_breach_still_caught(self):
# 6 次准入全落窗口中部(无贴边歧义),限额 5 → 必须击穿
rows = [
_row(str(i), created_at=f"2026-07-20T10:00:{s:02d}")
for i, s in enumerate([10, 20, 25, 30, 40, 50])
]
with pytest.raises(AssertionError):
inv_rpm_never_exceeded(rows, {"s1": 5})
def test_rpm_excludes_cache_hits(self):
# P6 实跑教训之二: 缓存命中在限流闸之前返回,未耗 RPM 名额也未打
# 网关,但遥测按"遥测必录"记行且署原源名——不得计入 RPM 口径
rows = [
_row(str(i), created_at=f"2026-07-20T10:00:{10 + i * 5:02d}", cache_hit=1)
for i in range(8)
]
rows.append(_row("real", created_at="2026-07-20T10:00:30"))
inv_rpm_never_exceeded(rows, {"s1": 1})
def test_rpm_boundary_jitter_exempted(self):
# 贴边行(±2s 采样噪声)在邻窗有余量时可归邻窗,不算击穿
rows = [
_row(str(i), created_at=f"2026-07-20T10:00:{s:02d}")
for i, s in enumerate([59, 10, 20, 30, 40, 50])
]
inv_rpm_never_exceeded(rows, {"s1": 5})
def test_rss_stable(self):
inv_rss_stable([100.0, 105.0, 110.0], max_growth_mb=50.0)
with pytest.raises(AssertionError):
@@ -197,6 +251,54 @@ class TestInvariants:
assert structured_success_rate(rows, session_suffix="-p3") == pytest.approx(0.5)
class TestHarnessMeasurement:
"""P6 伪击穿修复(2026-07-21): RSS 采当前值、分发有界。"""
def test_rss_mb_reports_current_not_peak(self):
import resource
import sys as _sys
from tools.soak.run_soak import _rss_mb
ballast = bytearray(300 * 1024 * 1024)
ballast[::4096] = b"x" * len(ballast[::4096]) # 触页,确保计入 RSS
divisor = 1e6 if _sys.platform == "darwin" else 1024
peak_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / divisor
del ballast
# ru_maxrss 含 300MB 压舱物峰值;当前 RSS 应已显著回落
assert _rss_mb() < peak_mb - 100
async def test_paced_dispatch_bounds_inflight_and_honors_stop(self):
import asyncio
from tools.soak.run_soak import _paced_dispatch
gauge = {"now": 0, "max": 0, "done": 0}
dispatched: list[int] = []
async def _work():
gauge["now"] += 1
gauge["max"] = max(gauge["max"], gauge["now"])
await asyncio.sleep(0.02)
gauge["now"] -= 1
gauge["done"] += 1
async def _gen():
for i in range(10):
yield "chat", {"i": i}
inflight = await _paced_dispatch(
_gen(),
sem=asyncio.Semaphore(2),
spawn=lambda kind, kwargs: _work(),
should_stop=lambda: len(dispatched) >= 6,
on_dispatched=lambda: dispatched.append(1),
)
await asyncio.gather(*inflight, return_exceptions=True)
assert gauge["max"] <= 2 # 并发名额先占后建任务
assert gauge["done"] == 6 # should_stop 在预算命中处截停分发
class TestLiveInvariantsAndCaps:
def test_capped_budget_clamps_to_signed_values(self):
from tools.soak.scoreboard import capped_budget