fix: align soak scoreboard metrics with limiter semantics
RPM invariant now buckets by admit time on the Redis server clock, exempts +/-2s boundary jitter, and excludes cache-hit rows that never consumed a limiter slot. RSS sampling reports current ps RSS instead of the monotonic ru_maxrss peak. Dispatch is paced by the concurrency semaphore so --max-hours stays live and memory stays bounded. Add --rescore RUN_ID to re-judge a finished run offline.
This commit is contained in:
@@ -145,13 +145,24 @@ from tools.soak.scoreboard import ( # noqa: E402
|
||||
)
|
||||
|
||||
|
||||
def _row(call_id, *, source="s1", created_at="2026-07-20T10:00:00", error=None, session="r-p3"):
|
||||
def _row(
|
||||
call_id,
|
||||
*,
|
||||
source="s1",
|
||||
created_at="2026-07-20T10:00:00",
|
||||
error=None,
|
||||
session="r-p3",
|
||||
latency_ms=0,
|
||||
cache_hit=0,
|
||||
):
|
||||
return {
|
||||
"call_id": call_id,
|
||||
"source_name": source,
|
||||
"created_at": created_at,
|
||||
"error": error,
|
||||
"session_id": session,
|
||||
"latency_ms": latency_ms,
|
||||
"cache_hit": cache_hit,
|
||||
}
|
||||
|
||||
|
||||
@@ -169,11 +180,11 @@ class TestInvariants:
|
||||
|
||||
def test_rpm_minute_bucket(self):
|
||||
# 固定分钟窗口口径(与限流器 int(sec/60) 同源;滑动窗会对合法的
|
||||
# 跨窗背靠背流量误报,不采用)
|
||||
# 跨窗背靠背流量误报,不采用)。击穿用例取窗口中部,避免贴边歧义。
|
||||
rows = [
|
||||
_row("a", created_at="2026-07-20T10:00:01"),
|
||||
_row("b", created_at="2026-07-20T10:00:59"),
|
||||
_row("c", created_at="2026-07-20T10:01:01"),
|
||||
_row("a", created_at="2026-07-20T10:00:10"),
|
||||
_row("b", created_at="2026-07-20T10:00:50"),
|
||||
_row("c", created_at="2026-07-20T10:01:10"),
|
||||
]
|
||||
inv_rpm_never_exceeded(rows, {"s1": 2})
|
||||
rows.append(_row("d", created_at="2026-07-20T10:00:30"))
|
||||
@@ -183,6 +194,49 @@ class TestInvariants:
|
||||
def test_rpm_ignores_unlimited_sources(self):
|
||||
inv_rpm_never_exceeded([_row(str(i)) for i in range(100)], {"s1": 0})
|
||||
|
||||
def test_rpm_buckets_by_admit_time_not_completion(self):
|
||||
# 准入相隔 100s(各窗 1 次),长延迟让完成时刻挤进同一分钟——
|
||||
# 完成时刻聚桶会误报(P6 伪击穿教训之一)
|
||||
rows = [
|
||||
_row("a", created_at="2026-07-20T10:02:30", latency_ms=120_000), # 准入 10:00:30
|
||||
_row("b", created_at="2026-07-20T10:02:35", latency_ms=25_000), # 准入 10:02:10
|
||||
]
|
||||
inv_rpm_never_exceeded(rows, {"s1": 1})
|
||||
|
||||
def test_rpm_server_clock_offset_dedistorts(self):
|
||||
# P6 实跑复现: Redis 服务器钟快 31s,本机分钟内 10 次准入
|
||||
# 实为两个服务器窗口各 5 次(+31 后: 37..59 前窗 | 63..90 后窗)
|
||||
secs = [6, 19, 21, 26, 28, 32, 42, 47, 52, 59]
|
||||
rows = [_row(str(i), created_at=f"2026-07-20T10:00:{s:02d}") for i, s in enumerate(secs)]
|
||||
inv_rpm_never_exceeded(rows, {"s1": 5}, clock_offset_s=31.0)
|
||||
|
||||
def test_rpm_true_breach_still_caught(self):
|
||||
# 6 次准入全落窗口中部(无贴边歧义),限额 5 → 必须击穿
|
||||
rows = [
|
||||
_row(str(i), created_at=f"2026-07-20T10:00:{s:02d}")
|
||||
for i, s in enumerate([10, 20, 25, 30, 40, 50])
|
||||
]
|
||||
with pytest.raises(AssertionError):
|
||||
inv_rpm_never_exceeded(rows, {"s1": 5})
|
||||
|
||||
def test_rpm_excludes_cache_hits(self):
|
||||
# P6 实跑教训之二: 缓存命中在限流闸之前返回,未耗 RPM 名额也未打
|
||||
# 网关,但遥测按"遥测必录"记行且署原源名——不得计入 RPM 口径
|
||||
rows = [
|
||||
_row(str(i), created_at=f"2026-07-20T10:00:{10 + i * 5:02d}", cache_hit=1)
|
||||
for i in range(8)
|
||||
]
|
||||
rows.append(_row("real", created_at="2026-07-20T10:00:30"))
|
||||
inv_rpm_never_exceeded(rows, {"s1": 1})
|
||||
|
||||
def test_rpm_boundary_jitter_exempted(self):
|
||||
# 贴边行(±2s 采样噪声)在邻窗有余量时可归邻窗,不算击穿
|
||||
rows = [
|
||||
_row(str(i), created_at=f"2026-07-20T10:00:{s:02d}")
|
||||
for i, s in enumerate([59, 10, 20, 30, 40, 50])
|
||||
]
|
||||
inv_rpm_never_exceeded(rows, {"s1": 5})
|
||||
|
||||
def test_rss_stable(self):
|
||||
inv_rss_stable([100.0, 105.0, 110.0], max_growth_mb=50.0)
|
||||
with pytest.raises(AssertionError):
|
||||
@@ -197,6 +251,54 @@ class TestInvariants:
|
||||
assert structured_success_rate(rows, session_suffix="-p3") == pytest.approx(0.5)
|
||||
|
||||
|
||||
class TestHarnessMeasurement:
|
||||
"""P6 伪击穿修复(2026-07-21): RSS 采当前值、分发有界。"""
|
||||
|
||||
def test_rss_mb_reports_current_not_peak(self):
|
||||
import resource
|
||||
import sys as _sys
|
||||
|
||||
from tools.soak.run_soak import _rss_mb
|
||||
|
||||
ballast = bytearray(300 * 1024 * 1024)
|
||||
ballast[::4096] = b"x" * len(ballast[::4096]) # 触页,确保计入 RSS
|
||||
divisor = 1e6 if _sys.platform == "darwin" else 1024
|
||||
peak_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / divisor
|
||||
del ballast
|
||||
# ru_maxrss 含 300MB 压舱物峰值;当前 RSS 应已显著回落
|
||||
assert _rss_mb() < peak_mb - 100
|
||||
|
||||
async def test_paced_dispatch_bounds_inflight_and_honors_stop(self):
|
||||
import asyncio
|
||||
|
||||
from tools.soak.run_soak import _paced_dispatch
|
||||
|
||||
gauge = {"now": 0, "max": 0, "done": 0}
|
||||
dispatched: list[int] = []
|
||||
|
||||
async def _work():
|
||||
gauge["now"] += 1
|
||||
gauge["max"] = max(gauge["max"], gauge["now"])
|
||||
await asyncio.sleep(0.02)
|
||||
gauge["now"] -= 1
|
||||
gauge["done"] += 1
|
||||
|
||||
async def _gen():
|
||||
for i in range(10):
|
||||
yield "chat", {"i": i}
|
||||
|
||||
inflight = await _paced_dispatch(
|
||||
_gen(),
|
||||
sem=asyncio.Semaphore(2),
|
||||
spawn=lambda kind, kwargs: _work(),
|
||||
should_stop=lambda: len(dispatched) >= 6,
|
||||
on_dispatched=lambda: dispatched.append(1),
|
||||
)
|
||||
await asyncio.gather(*inflight, return_exceptions=True)
|
||||
assert gauge["max"] <= 2 # 并发名额先占后建任务
|
||||
assert gauge["done"] == 6 # should_stop 在预算命中处截停分发
|
||||
|
||||
|
||||
class TestLiveInvariantsAndCaps:
|
||||
def test_capped_budget_clamps_to_signed_values(self):
|
||||
from tools.soak.scoreboard import capped_budget
|
||||
|
||||
Reference in New Issue
Block a user