feat: add P7 OCR soak scenario and scoreboard

This commit is contained in:
2026-07-21 23:22:12 -04:00
parent d2138e535f
commit bb1600c0f1
6 changed files with 211 additions and 20 deletions
+1 -1
View File
@@ -350,7 +350,7 @@ class TestTelemetry:
for row in recorder.rows:
assert "<ocr:text image_bytes=15>" in row["messages"]
assert "RAW-IMAGE-BYTES" not in row["messages"] # 图像字节绝不入库
assert recorder.rows[0]["error"] is not None
assert recorder.rows[0]["error"].startswith("TransientError:") # 类名前缀口径
assert recorder.rows[1]["error"] is None
assert recorder.rows[1]["prompt_tokens"] == 0
+54
View File
@@ -0,0 +1,54 @@
"""P7 记分板新增纯函数测试(M3 计划 T8): 成功率/坏源占比/错误分类。"""
import pytest
from tools.soak.scoreboard import (
inv_errors_classified,
inv_fault_share,
inv_success_rate,
)
_KNOWN = ("TransientError", "AllSourcesExhausted", "cancelled")
def _row(source="monkey_1", error=None, cache_hit=0):
return {"source_name": source, "error": error, "cache_hit": cache_hit}
class TestSuccessRate:
def test_pass_at_threshold(self):
inv_success_rate(98, 100, min_rate=0.98)
def test_fail_below(self):
with pytest.raises(AssertionError, match="成功率"):
inv_success_rate(97, 100, min_rate=0.98)
def test_zero_total_rejected(self):
with pytest.raises(AssertionError):
inv_success_rate(0, 0, min_rate=0.98)
class TestFaultShare:
def test_share_within_threshold(self):
rows = [_row()] * 9 + [_row("monkey_3")]
inv_fault_share(rows, ["monkey_3", "monkey_4"], max_share=0.15)
def test_share_exceeds(self):
rows = [_row()] * 4 + [_row("monkey_3")] * 2
with pytest.raises(AssertionError, match="坏源尝试占比"):
inv_fault_share(rows, ["monkey_3"], max_share=0.15)
def test_cache_rows_excluded(self):
rows = [_row(cache_hit=1)] * 100 + [_row("monkey_3")] + [_row()] * 9
inv_fault_share(rows, ["monkey_3"], max_share=0.15) # 缓存行不入分母
class TestErrorsClassified:
def test_known_prefixes_pass(self):
rows = [_row(error="TransientError: m1 超时"), _row(error="cancelled"), _row()]
inv_errors_classified(rows, _KNOWN)
def test_unknown_prefix_fails(self):
rows = [_row(error="KeyError: 'oops'")]
with pytest.raises(AssertionError, match="未分类"):
inv_errors_classified(rows, _KNOWN)