feat: add P7 OCR soak scenario and scoreboard

This commit is contained in:
2026-07-21 23:22:12 -04:00
parent d2138e535f
commit bb1600c0f1
6 changed files with 211 additions and 20 deletions
+23 -5
View File
@@ -1,8 +1,9 @@
"""P1-P6 场景请求生成器(findings §2 矩阵;async 生成器,产出调用参数)。
"""P1-P7 场景请求生成器(findings §2 矩阵;async 生成器,产出调用参数)。
每项产出 `(kind, kwargs)`: kind ∈ {"chat"};kwargs 直接喂
`GatewayClient.chat(**kwargs)`。回放/组装场景一律掺 `cache_salt=run_id`
破缓存(缓存行为归 P4);P4 子流量特意重复 messages 且不掺 salt。
每项产出 `(kind, kwargs)`: kind ∈ {"chat", "ocr_text", "ocr_layout"};
chat 喂 `GatewayClient.chat(**kwargs)`,ocr_* 喂 `OcrClient` 对应方法
(M3 计划 T8)。回放/组装场景一律掺 `cache_salt=run_id` 破缓存(缓存
行为归 P4);P4 子流量特意重复 messages 且不掺 salt。
"""
from __future__ import annotations
@@ -10,10 +11,13 @@ from __future__ import annotations
import random
from dataclasses import dataclass
from pathlib import Path
from typing import Any, AsyncIterator
from typing import TYPE_CHECKING, Any
from pydantic import BaseModel
if TYPE_CHECKING:
from collections.abc import AsyncIterator
from tools.soak.corpus import (
assemble_frame_messages,
chs_image_messages,
@@ -182,6 +186,19 @@ async def p6_mixed_soak(
}[key](corpus, f"{run_id}-r{rng()}", rng)
async def p7_ocr(
corpus: SoakCorpus, run_id: str, rng=random.random, *, layout_ratio: float = 0.2
) -> AsyncIterator[Item]:
"""P7 OCR 双端点混合(M3 计划 T8): 真实图像语料循环,text/layout 8:2。
故障性来自 SOAK_OCR 源池配置(黑洞/坏端口),生成器本身不造故障。
"""
while True:
image = corpus.images[int(rng() * len(corpus.images)) % len(corpus.images)]
kind = "ocr_layout" if rng() < layout_ratio else "ocr_text"
yield (kind, {"image": image.read_bytes(), "session_id": f"{run_id}-p7"})
SCENARIOS = {
"P1": p1_trace_chains,
"P2": p2_multimodal_replay,
@@ -189,4 +206,5 @@ SCENARIOS = {
"P4": p4_cache_bidirectional,
"P5": p5_fault_mixed,
"P6": p6_mixed_soak,
"P7": p7_ocr,
}