feat: add P7 OCR soak scenario and scoreboard

This commit is contained in:
2026-07-21 23:22:12 -04:00
parent d2138e535f
commit bb1600c0f1
6 changed files with 211 additions and 20 deletions
+32 -3
View File
@@ -9,7 +9,7 @@ from __future__ import annotations
import sqlite3
from collections import Counter
from datetime import datetime, timezone
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
@@ -50,7 +50,7 @@ def _admit_second(row: Row, clock_offset_s: float) -> float:
dt = datetime.fromisoformat(str(row["created_at"]))
if dt.tzinfo is None:
# SQLite datetime('now') 落库为 UTC naive;按本机时区解析会错位整时
dt = dt.replace(tzinfo=timezone.utc)
dt = dt.replace(tzinfo=UTC)
latency_ms = row.get("latency_ms") or 0
return dt.timestamp() - float(latency_ms) / 1000.0 + clock_offset_s
@@ -193,13 +193,14 @@ def write_report(run_id: str, content: str, out_dir: Path | str = "tests/outputs
# —— 活后端检查与预算帽(2026-07-21 verifier I1/M6/M7 补齐)——
SCENARIO_CALL_CAPS = { # 设计 §8.1 签字的单场景请求数上限
SCENARIO_CALL_CAPS = { # 设计 §8.1 签字的单场景请求数上限(P7: M3 计划 T8)
"P1": 500,
"P2": 450,
"P3": 2500,
"P4": 2500,
"P5": 1000,
"P6": 8000,
"P7": 1500,
}
@@ -239,6 +240,34 @@ def inv_fault_errors_present(rows: list[Row], *, fault_source_names: list[str])
assert fault_errors, f"故障源 {fault_source_names} 零错误行——故障混编未生效"
def inv_success_rate(ok: int, total: int, *, min_rate: float) -> None:
"""P7 不变量①: 调用级成功率(worker stats 口径,与 M2.5 验收同源)。"""
assert total > 0, "零调用无法评估成功率"
rate = ok / total
assert rate >= min_rate, f"成功率 {rate:.4f}({ok}/{total})< 阈值 {min_rate}"
def inv_fault_share(rows: list[Row], fault_source_names: list[str], *, max_share: float) -> None:
"""P7 不变量②: 坏源尝试占比受健康选源压制(排除缓存行)。"""
attempts = [r for r in rows if not r.get("cache_hit")]
assert attempts, "无尝试行"
fault = sum(1 for r in attempts if r["source_name"] in fault_source_names)
share = fault / len(attempts)
assert share <= max_share, (
f"坏源尝试占比 {share:.3f}({fault}/{len(attempts)})> 阈值 {max_share}"
)
def inv_errors_classified(rows: list[Row], known_prefixes: tuple[str, ...]) -> None:
"""P7 不变量⑦/⑧: 一切错误行可归入已知异常类名前缀(零未分类异常)。"""
unknown = Counter(
str(r["error"]).split(":")[0]
for r in rows
if r.get("error") and not str(r["error"]).startswith(known_prefixes)
)
assert not unknown, f"未分类错误前缀: {dict(unknown)}"
def inv_any_errors(rows: list[Row]) -> None:
"""不变量 2c 的兜底形态(P5/P6): 故障源混编池下错误行必然存在。