feat: add OcrClient.parse_document with full governance

Reuses the existing retry, breaker, limiter and telemetry loop; the transport capability is checked once before the statistics boundary. Legacy entrances keep their error text. Tests cover the document path through the real transport so parser exceptions, deadlines and probe bookkeeping are proven end to end.
This commit is contained in:
2026-10-22 00:01:45 -04:00
parent 61cace71c4
commit 80bb8aba71
5 changed files with 524 additions and 50 deletions
+121 -36
View File
@@ -39,13 +39,17 @@ from polygateway.middleware.breaker import BreakerGate
from polygateway.middleware.ratelimit import QuotaGate
from polygateway.middleware.retry import StallClock, _failure_reason, backoff_delay
from polygateway.middleware.telemetry import TelemetryEmitter, emit_terminal_once
from polygateway.ports import OutcomeAwareSelector
from polygateway.ports import OcrDocumentTransport, OutcomeAwareSelector
from polygateway.types import (
CallStats,
ChatRequest,
LLMResponse,
OcrDocumentResult,
OcrDocumentTransportResult,
OcrLayoutResult,
OcrLayoutTransportResult,
OcrTextResult,
OcrTextTransportResult,
TelemetryStatus,
Usage,
_CallContext,
@@ -69,15 +73,18 @@ if TYPE_CHECKING:
from polygateway.types import (
BackpressurePolicy,
CallOperation,
OcrLayoutTransportResult,
OcrTextTransportResult,
RetryPolicy,
SourceConfig,
)
_RESPONSE_TEXT_CAP = 200 # 遥测行 text 截断长度(与 embedding 口径一致)
_OcrKind = Literal["text", "layout"]
_OcrKind = Literal["text", "layout", "document"]
# 输入校验文案里的名字。旧两个入口(text/layout)的报错文本保持不变,文档入口自称 document
_PAYLOAD_NAME: dict[_OcrKind, str] = {"text": "image", "layout": "image", "document": "document"}
_TransportResult = OcrTextTransportResult | OcrLayoutTransportResult | OcrDocumentTransportResult
@dataclass(frozen=True)
@@ -88,7 +95,7 @@ class _FailedAttempt:
@dataclass(frozen=True)
class _AttemptOutcome:
result: OcrTextTransportResult | OcrLayoutTransportResult
result: _TransportResult
source: SourceConfig
call_id: str
latency_ms: int
@@ -254,6 +261,53 @@ class OcrClient:
call_stats=call_stats,
)
async def parse_document(
self,
document: bytes,
*,
session_id: str | None = None,
parent_call_id: str | None = None,
tenant_id: str | None = None,
meta: Mapping[str, Any] | None = None,
call_deadline_s: float | None = None,
) -> OcrDocumentResult:
"""一次治理整份文档解析(/parse → ZIP);PDF 或图片,逐块带类型/页码/bbox/文字。
上传类型由 transport 按内容判定,调用方无需也无法声明。结果的 `markdown` 是服务
生成的整篇原文,含悬空的图片引用;服务端会丢弃脚注、页眉页脚与页码。
`tenant_id` 与 `meta` 是调用方自定义维度,只进遥测(issue #11)。
`call_deadline_s` 同 `recognize_text`(issue #22): `None` = 继承装配值。长文档的
硬上限靠它——源的 `timeout_s` 是逐次网络操作的超时,不是总耗时上界。
"""
# 校验早于链路,理由同 recognize_text;transport 能力校验也在统计边界之外
dimension_tenant_id, dimensions = validate_caller_dimensions(
tenant_id, meta, origin="parse_document(tenant_id=..., meta=...)"
)
self._document_transport()
outcome, call_stats = await self._call(
"document",
"parse_document",
document,
session_id,
parent_call_id,
dimension_tenant_id,
dimensions,
call_deadline_s,
)
result = outcome.result
return OcrDocumentResult(
elements=result.elements,
page_sizes=result.page_sizes,
markdown=result.markdown,
source_name=outcome.source.name,
usage=Usage(0, 0),
latency_ms=outcome.latency_ms,
call_id=outcome.call_id,
raw=result.raw,
call_stats=call_stats,
)
async def check_health(self) -> dict[str, bool]:
"""逐源并发健康预检(R10);探测失败=False 不上抛,取消穿透。"""
names = [s.name for s in self._sources]
@@ -272,37 +326,38 @@ class OcrClient:
self,
kind: _OcrKind,
operation: CallOperation,
image: bytes,
payload: bytes,
session_id: str | None,
parent_call_id: str | None,
tenant_id: str | None,
meta: dict[str, Any],
call_deadline_s: float | None = None,
) -> tuple[_AttemptOutcome, CallStats]:
if not isinstance(image, bytes):
raise TypeError("image 必须是 bytes(路径读取/批量拼帧留业务侧,D9)")
if not image:
raise ValueError("image 不能为空")
# 期限校验与 `image` 校验同列(仍在 `_CallContext` 之前、创建 awaitable 之前)
name = _PAYLOAD_NAME[kind]
if not isinstance(payload, bytes):
raise TypeError(f"{name} 必须是 bytes(路径读取/批量拼帧留业务侧,D9)")
if not payload:
raise ValueError(f"{name} 不能为空")
# 期限校验与 `payload` 校验同列(仍在 `_CallContext` 之前、创建 awaitable 之前)
deadline = (
self._call_deadline_s
if call_deadline_s is None
else ensure_call_deadline(call_deadline_s, f"{operation}(call_deadline_s=...)")
)
# M1 例外: `image` 校验在 `_call` 内而非公开方法,故上下文在该校验
# M1 例外: `payload` 校验在 `_call` 内而非公开方法,故上下文在该校验
# **通过之后**创建——这样设计 §3 的"校验在统计边界外"对 OCR 才成立
context = _CallContext(now=self._now)
try:
return await with_call_deadline(
self._run(
kind, operation, image, session_id, parent_call_id, tenant_id, meta, context
kind, operation, payload, session_id, parent_call_id, tenant_id, meta, context
),
deadline_s=deadline,
scope=self._scope,
)
except PolyGatewayError as exc:
await self._emit_terminal(
kind, operation, image, session_id, parent_call_id, tenant_id, meta, context, exc
kind, operation, payload, session_id, parent_call_id, tenant_id, meta, context, exc
)
raise
except asyncio.CancelledError:
@@ -310,7 +365,7 @@ class OcrClient:
await self._emit_terminal(
kind,
operation,
image,
payload,
session_id,
parent_call_id,
tenant_id,
@@ -324,7 +379,7 @@ class OcrClient:
self,
kind: _OcrKind,
operation: CallOperation,
image: bytes,
payload: bytes,
session_id: str | None,
parent_call_id: str | None,
tenant_id: str | None,
@@ -351,7 +406,7 @@ class OcrClient:
outcome = await self._attempt(
kind,
operation,
image,
payload,
*picked,
reasons,
session_id,
@@ -377,7 +432,7 @@ class OcrClient:
self,
kind: _OcrKind,
operation: CallOperation,
image: bytes,
payload: bytes,
source: SourceConfig,
permit: Permit,
entry: GateDecision,
@@ -398,7 +453,7 @@ class OcrClient:
# 裸生成时间(1.3.7 H8): 计时只包 transport 调用本身(同 RetryMW 口径);
# layout 的 POST + ZIP GET 在同一次 `_invoke` 内,属同一段生成耗时
gen_started = self._now()
result = await self._invoke(kind, image, source, call_id)
result = await self._invoke(kind, payload, source, call_id)
gen_ms = int((self._now() - gen_started) * 1000)
await self._record_quietly(self._breaker.record_success(entry))
await self._record_quietly(self._quota.mark_progress())
@@ -408,7 +463,7 @@ class OcrClient:
await self._emit(
kind,
operation,
image,
payload,
source,
call_id,
started,
@@ -425,7 +480,7 @@ class OcrClient:
await self._emit(
kind,
operation,
image,
payload,
source,
call_id,
started,
@@ -443,7 +498,7 @@ class OcrClient:
await self._emit(
kind,
operation,
image,
payload,
source,
call_id,
started,
@@ -464,7 +519,7 @@ class OcrClient:
await self._emit(
kind,
operation,
image,
payload,
source,
call_id,
started,
@@ -482,11 +537,31 @@ class OcrClient:
await settle_and_release(permit, 0)
async def _invoke(
self, kind: _OcrKind, image: bytes, source: SourceConfig, call_id: str
) -> OcrTextTransportResult | OcrLayoutTransportResult:
self, kind: _OcrKind, payload: bytes, source: SourceConfig, call_id: str
) -> _TransportResult:
if kind == "text":
return await self._transport.recognize_text(image=image, source=source, call_id=call_id)
return await self._transport.parse_layout(image=image, source=source, call_id=call_id)
return await self._transport.recognize_text(
image=payload, source=source, call_id=call_id
)
if kind == "document":
return await self._document_transport().parse_document(
document=payload, source=source, call_id=call_id
)
return await self._transport.parse_layout(image=payload, source=source, call_id=call_id)
def _document_transport(self) -> OcrDocumentTransport:
"""文档入口所需的 transport 能力校验(唯一的 isinstance 点)。
`OcrDocumentTransport` 独立于 `OcrTransport`: 往旧 Protocol 加成员会让所有现有
实现(测试替身、下游自带 fake)一夜之间不合格。代价是这里必须在运行时校验——
公共入口在进入统计边界前调用一次,让装配错误当场报。
"""
if not isinstance(self._transport, OcrDocumentTransport):
raise TypeError(
"parse_document 需要实现 OcrDocumentTransport.parse_document 的 transport: "
f"{type(self._transport).__name__} 不满足"
)
return self._transport
# —— 辅助(与 embedding 同口径)——
@@ -519,7 +594,7 @@ class OcrClient:
def _request_for(
self,
kind: _OcrKind,
image: bytes,
payload: bytes,
session_id: str | None,
parent_call_id: str | None,
tenant_id: str | None,
@@ -533,7 +608,7 @@ class OcrClient:
维度恒为空、`logical_call_id` 恒为 NULL。
"""
return ChatRequest(
messages=[{"role": "user", "content": f"<ocr:{kind} image_bytes={len(image)}>"}],
messages=[{"role": "user", "content": f"<ocr:{kind} image_bytes={len(payload)}>"}],
session_id=session_id,
parent_call_id=parent_call_id,
tenant_id=tenant_id,
@@ -545,7 +620,7 @@ class OcrClient:
self,
kind: _OcrKind,
operation: CallOperation,
image: bytes,
payload: bytes,
session_id: str | None,
parent_call_id: str | None,
tenant_id: str | None,
@@ -557,7 +632,7 @@ class OcrClient:
await emit_terminal_once(
self._emitter,
request=self._request_for(
kind, image, session_id, parent_call_id, tenant_id, meta, context
kind, payload, session_id, parent_call_id, tenant_id, meta, context
),
context=context,
error=error,
@@ -570,7 +645,7 @@ class OcrClient:
self,
kind: _OcrKind,
operation: CallOperation,
image: bytes,
payload: bytes,
source: SourceConfig,
call_id: str,
started: float,
@@ -579,14 +654,14 @@ class OcrClient:
tenant_id: str | None,
meta: dict[str, Any],
context: _CallContext,
result: OcrTextTransportResult | OcrLayoutTransportResult | None = None,
result: _TransportResult | None = None,
error: PolyGatewayError | str | None = None,
) -> None:
"""逐尝试遥测(单一 Emitter): messages 占位摘要,图像 bytes 绝不入库。"""
if self._emitter is None:
return
request = self._request_for(
kind, image, session_id, parent_call_id, tenant_id, meta, context
kind, payload, session_id, parent_call_id, tenant_id, meta, context
)
latency_ms = int((self._now() - started) * 1000)
response = None
@@ -622,8 +697,18 @@ class OcrClient:
)
@staticmethod
def _summarize(result: OcrTextTransportResult | OcrLayoutTransportResult) -> str:
if hasattr(result, "text"):
def _summarize(result: _TransportResult) -> str:
"""遥测行 response 摘要。**永不写入文档正文**:文档只记规模(元素数/页数/md 字数)。
用 isinstance 显式区分三类结果:旧的 `hasattr(result, "text")` 对没有顶层 `text` 的
新结果会静默套成 layout 摘要,漏记 `md_chars`。
"""
if isinstance(result, OcrDocumentTransportResult):
return (
f"<document elements={len(result.elements)} pages={len(result.page_sizes)} "
f"md_chars={len(result.markdown)}>"
)
if isinstance(result, OcrTextTransportResult):
return result.text[:_RESPONSE_TEXT_CAP]
return f"<elements n={len(result.elements)} pages={len(result.page_sizes)}>"