feat: add OcrClient.parse_document with full governance
Reuses the existing retry, breaker, limiter and telemetry loop; the transport capability is checked once before the statistics boundary. Legacy entrances keep their error text. Tests cover the document path through the real transport so parser exceptions, deadlines and probe bookkeeping are proven end to end.
This commit is contained in:
+121
-36
@@ -39,13 +39,17 @@ from polygateway.middleware.breaker import BreakerGate
|
||||
from polygateway.middleware.ratelimit import QuotaGate
|
||||
from polygateway.middleware.retry import StallClock, _failure_reason, backoff_delay
|
||||
from polygateway.middleware.telemetry import TelemetryEmitter, emit_terminal_once
|
||||
from polygateway.ports import OutcomeAwareSelector
|
||||
from polygateway.ports import OcrDocumentTransport, OutcomeAwareSelector
|
||||
from polygateway.types import (
|
||||
CallStats,
|
||||
ChatRequest,
|
||||
LLMResponse,
|
||||
OcrDocumentResult,
|
||||
OcrDocumentTransportResult,
|
||||
OcrLayoutResult,
|
||||
OcrLayoutTransportResult,
|
||||
OcrTextResult,
|
||||
OcrTextTransportResult,
|
||||
TelemetryStatus,
|
||||
Usage,
|
||||
_CallContext,
|
||||
@@ -69,15 +73,18 @@ if TYPE_CHECKING:
|
||||
from polygateway.types import (
|
||||
BackpressurePolicy,
|
||||
CallOperation,
|
||||
OcrLayoutTransportResult,
|
||||
OcrTextTransportResult,
|
||||
RetryPolicy,
|
||||
SourceConfig,
|
||||
)
|
||||
|
||||
_RESPONSE_TEXT_CAP = 200 # 遥测行 text 截断长度(与 embedding 口径一致)
|
||||
|
||||
_OcrKind = Literal["text", "layout"]
|
||||
_OcrKind = Literal["text", "layout", "document"]
|
||||
|
||||
# 输入校验文案里的名字。旧两个入口(text/layout)的报错文本保持不变,文档入口自称 document
|
||||
_PAYLOAD_NAME: dict[_OcrKind, str] = {"text": "image", "layout": "image", "document": "document"}
|
||||
|
||||
_TransportResult = OcrTextTransportResult | OcrLayoutTransportResult | OcrDocumentTransportResult
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -88,7 +95,7 @@ class _FailedAttempt:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _AttemptOutcome:
|
||||
result: OcrTextTransportResult | OcrLayoutTransportResult
|
||||
result: _TransportResult
|
||||
source: SourceConfig
|
||||
call_id: str
|
||||
latency_ms: int
|
||||
@@ -254,6 +261,53 @@ class OcrClient:
|
||||
call_stats=call_stats,
|
||||
)
|
||||
|
||||
async def parse_document(
|
||||
self,
|
||||
document: bytes,
|
||||
*,
|
||||
session_id: str | None = None,
|
||||
parent_call_id: str | None = None,
|
||||
tenant_id: str | None = None,
|
||||
meta: Mapping[str, Any] | None = None,
|
||||
call_deadline_s: float | None = None,
|
||||
) -> OcrDocumentResult:
|
||||
"""一次治理整份文档解析(/parse → ZIP);PDF 或图片,逐块带类型/页码/bbox/文字。
|
||||
|
||||
上传类型由 transport 按内容判定,调用方无需也无法声明。结果的 `markdown` 是服务
|
||||
生成的整篇原文,含悬空的图片引用;服务端会丢弃脚注、页眉页脚与页码。
|
||||
|
||||
`tenant_id` 与 `meta` 是调用方自定义维度,只进遥测(issue #11)。
|
||||
`call_deadline_s` 同 `recognize_text`(issue #22): `None` = 继承装配值。长文档的
|
||||
硬上限靠它——源的 `timeout_s` 是逐次网络操作的超时,不是总耗时上界。
|
||||
"""
|
||||
# 校验早于链路,理由同 recognize_text;transport 能力校验也在统计边界之外
|
||||
dimension_tenant_id, dimensions = validate_caller_dimensions(
|
||||
tenant_id, meta, origin="parse_document(tenant_id=..., meta=...)"
|
||||
)
|
||||
self._document_transport()
|
||||
outcome, call_stats = await self._call(
|
||||
"document",
|
||||
"parse_document",
|
||||
document,
|
||||
session_id,
|
||||
parent_call_id,
|
||||
dimension_tenant_id,
|
||||
dimensions,
|
||||
call_deadline_s,
|
||||
)
|
||||
result = outcome.result
|
||||
return OcrDocumentResult(
|
||||
elements=result.elements,
|
||||
page_sizes=result.page_sizes,
|
||||
markdown=result.markdown,
|
||||
source_name=outcome.source.name,
|
||||
usage=Usage(0, 0),
|
||||
latency_ms=outcome.latency_ms,
|
||||
call_id=outcome.call_id,
|
||||
raw=result.raw,
|
||||
call_stats=call_stats,
|
||||
)
|
||||
|
||||
async def check_health(self) -> dict[str, bool]:
|
||||
"""逐源并发健康预检(R10);探测失败=False 不上抛,取消穿透。"""
|
||||
names = [s.name for s in self._sources]
|
||||
@@ -272,37 +326,38 @@ class OcrClient:
|
||||
self,
|
||||
kind: _OcrKind,
|
||||
operation: CallOperation,
|
||||
image: bytes,
|
||||
payload: bytes,
|
||||
session_id: str | None,
|
||||
parent_call_id: str | None,
|
||||
tenant_id: str | None,
|
||||
meta: dict[str, Any],
|
||||
call_deadline_s: float | None = None,
|
||||
) -> tuple[_AttemptOutcome, CallStats]:
|
||||
if not isinstance(image, bytes):
|
||||
raise TypeError("image 必须是 bytes(路径读取/批量拼帧留业务侧,D9)")
|
||||
if not image:
|
||||
raise ValueError("image 不能为空")
|
||||
# 期限校验与 `image` 校验同列(仍在 `_CallContext` 之前、创建 awaitable 之前)
|
||||
name = _PAYLOAD_NAME[kind]
|
||||
if not isinstance(payload, bytes):
|
||||
raise TypeError(f"{name} 必须是 bytes(路径读取/批量拼帧留业务侧,D9)")
|
||||
if not payload:
|
||||
raise ValueError(f"{name} 不能为空")
|
||||
# 期限校验与 `payload` 校验同列(仍在 `_CallContext` 之前、创建 awaitable 之前)
|
||||
deadline = (
|
||||
self._call_deadline_s
|
||||
if call_deadline_s is None
|
||||
else ensure_call_deadline(call_deadline_s, f"{operation}(call_deadline_s=...)")
|
||||
)
|
||||
# M1 例外: `image` 校验在 `_call` 内而非公开方法,故上下文在该校验
|
||||
# M1 例外: `payload` 校验在 `_call` 内而非公开方法,故上下文在该校验
|
||||
# **通过之后**创建——这样设计 §3 的"校验在统计边界外"对 OCR 才成立
|
||||
context = _CallContext(now=self._now)
|
||||
try:
|
||||
return await with_call_deadline(
|
||||
self._run(
|
||||
kind, operation, image, session_id, parent_call_id, tenant_id, meta, context
|
||||
kind, operation, payload, session_id, parent_call_id, tenant_id, meta, context
|
||||
),
|
||||
deadline_s=deadline,
|
||||
scope=self._scope,
|
||||
)
|
||||
except PolyGatewayError as exc:
|
||||
await self._emit_terminal(
|
||||
kind, operation, image, session_id, parent_call_id, tenant_id, meta, context, exc
|
||||
kind, operation, payload, session_id, parent_call_id, tenant_id, meta, context, exc
|
||||
)
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
@@ -310,7 +365,7 @@ class OcrClient:
|
||||
await self._emit_terminal(
|
||||
kind,
|
||||
operation,
|
||||
image,
|
||||
payload,
|
||||
session_id,
|
||||
parent_call_id,
|
||||
tenant_id,
|
||||
@@ -324,7 +379,7 @@ class OcrClient:
|
||||
self,
|
||||
kind: _OcrKind,
|
||||
operation: CallOperation,
|
||||
image: bytes,
|
||||
payload: bytes,
|
||||
session_id: str | None,
|
||||
parent_call_id: str | None,
|
||||
tenant_id: str | None,
|
||||
@@ -351,7 +406,7 @@ class OcrClient:
|
||||
outcome = await self._attempt(
|
||||
kind,
|
||||
operation,
|
||||
image,
|
||||
payload,
|
||||
*picked,
|
||||
reasons,
|
||||
session_id,
|
||||
@@ -377,7 +432,7 @@ class OcrClient:
|
||||
self,
|
||||
kind: _OcrKind,
|
||||
operation: CallOperation,
|
||||
image: bytes,
|
||||
payload: bytes,
|
||||
source: SourceConfig,
|
||||
permit: Permit,
|
||||
entry: GateDecision,
|
||||
@@ -398,7 +453,7 @@ class OcrClient:
|
||||
# 裸生成时间(1.3.7 H8): 计时只包 transport 调用本身(同 RetryMW 口径);
|
||||
# layout 的 POST + ZIP GET 在同一次 `_invoke` 内,属同一段生成耗时
|
||||
gen_started = self._now()
|
||||
result = await self._invoke(kind, image, source, call_id)
|
||||
result = await self._invoke(kind, payload, source, call_id)
|
||||
gen_ms = int((self._now() - gen_started) * 1000)
|
||||
await self._record_quietly(self._breaker.record_success(entry))
|
||||
await self._record_quietly(self._quota.mark_progress())
|
||||
@@ -408,7 +463,7 @@ class OcrClient:
|
||||
await self._emit(
|
||||
kind,
|
||||
operation,
|
||||
image,
|
||||
payload,
|
||||
source,
|
||||
call_id,
|
||||
started,
|
||||
@@ -425,7 +480,7 @@ class OcrClient:
|
||||
await self._emit(
|
||||
kind,
|
||||
operation,
|
||||
image,
|
||||
payload,
|
||||
source,
|
||||
call_id,
|
||||
started,
|
||||
@@ -443,7 +498,7 @@ class OcrClient:
|
||||
await self._emit(
|
||||
kind,
|
||||
operation,
|
||||
image,
|
||||
payload,
|
||||
source,
|
||||
call_id,
|
||||
started,
|
||||
@@ -464,7 +519,7 @@ class OcrClient:
|
||||
await self._emit(
|
||||
kind,
|
||||
operation,
|
||||
image,
|
||||
payload,
|
||||
source,
|
||||
call_id,
|
||||
started,
|
||||
@@ -482,11 +537,31 @@ class OcrClient:
|
||||
await settle_and_release(permit, 0)
|
||||
|
||||
async def _invoke(
|
||||
self, kind: _OcrKind, image: bytes, source: SourceConfig, call_id: str
|
||||
) -> OcrTextTransportResult | OcrLayoutTransportResult:
|
||||
self, kind: _OcrKind, payload: bytes, source: SourceConfig, call_id: str
|
||||
) -> _TransportResult:
|
||||
if kind == "text":
|
||||
return await self._transport.recognize_text(image=image, source=source, call_id=call_id)
|
||||
return await self._transport.parse_layout(image=image, source=source, call_id=call_id)
|
||||
return await self._transport.recognize_text(
|
||||
image=payload, source=source, call_id=call_id
|
||||
)
|
||||
if kind == "document":
|
||||
return await self._document_transport().parse_document(
|
||||
document=payload, source=source, call_id=call_id
|
||||
)
|
||||
return await self._transport.parse_layout(image=payload, source=source, call_id=call_id)
|
||||
|
||||
def _document_transport(self) -> OcrDocumentTransport:
|
||||
"""文档入口所需的 transport 能力校验(唯一的 isinstance 点)。
|
||||
|
||||
`OcrDocumentTransport` 独立于 `OcrTransport`: 往旧 Protocol 加成员会让所有现有
|
||||
实现(测试替身、下游自带 fake)一夜之间不合格。代价是这里必须在运行时校验——
|
||||
公共入口在进入统计边界前调用一次,让装配错误当场报。
|
||||
"""
|
||||
if not isinstance(self._transport, OcrDocumentTransport):
|
||||
raise TypeError(
|
||||
"parse_document 需要实现 OcrDocumentTransport.parse_document 的 transport: "
|
||||
f"{type(self._transport).__name__} 不满足"
|
||||
)
|
||||
return self._transport
|
||||
|
||||
# —— 辅助(与 embedding 同口径)——
|
||||
|
||||
@@ -519,7 +594,7 @@ class OcrClient:
|
||||
def _request_for(
|
||||
self,
|
||||
kind: _OcrKind,
|
||||
image: bytes,
|
||||
payload: bytes,
|
||||
session_id: str | None,
|
||||
parent_call_id: str | None,
|
||||
tenant_id: str | None,
|
||||
@@ -533,7 +608,7 @@ class OcrClient:
|
||||
维度恒为空、`logical_call_id` 恒为 NULL。
|
||||
"""
|
||||
return ChatRequest(
|
||||
messages=[{"role": "user", "content": f"<ocr:{kind} image_bytes={len(image)}>"}],
|
||||
messages=[{"role": "user", "content": f"<ocr:{kind} image_bytes={len(payload)}>"}],
|
||||
session_id=session_id,
|
||||
parent_call_id=parent_call_id,
|
||||
tenant_id=tenant_id,
|
||||
@@ -545,7 +620,7 @@ class OcrClient:
|
||||
self,
|
||||
kind: _OcrKind,
|
||||
operation: CallOperation,
|
||||
image: bytes,
|
||||
payload: bytes,
|
||||
session_id: str | None,
|
||||
parent_call_id: str | None,
|
||||
tenant_id: str | None,
|
||||
@@ -557,7 +632,7 @@ class OcrClient:
|
||||
await emit_terminal_once(
|
||||
self._emitter,
|
||||
request=self._request_for(
|
||||
kind, image, session_id, parent_call_id, tenant_id, meta, context
|
||||
kind, payload, session_id, parent_call_id, tenant_id, meta, context
|
||||
),
|
||||
context=context,
|
||||
error=error,
|
||||
@@ -570,7 +645,7 @@ class OcrClient:
|
||||
self,
|
||||
kind: _OcrKind,
|
||||
operation: CallOperation,
|
||||
image: bytes,
|
||||
payload: bytes,
|
||||
source: SourceConfig,
|
||||
call_id: str,
|
||||
started: float,
|
||||
@@ -579,14 +654,14 @@ class OcrClient:
|
||||
tenant_id: str | None,
|
||||
meta: dict[str, Any],
|
||||
context: _CallContext,
|
||||
result: OcrTextTransportResult | OcrLayoutTransportResult | None = None,
|
||||
result: _TransportResult | None = None,
|
||||
error: PolyGatewayError | str | None = None,
|
||||
) -> None:
|
||||
"""逐尝试遥测(单一 Emitter): messages 占位摘要,图像 bytes 绝不入库。"""
|
||||
if self._emitter is None:
|
||||
return
|
||||
request = self._request_for(
|
||||
kind, image, session_id, parent_call_id, tenant_id, meta, context
|
||||
kind, payload, session_id, parent_call_id, tenant_id, meta, context
|
||||
)
|
||||
latency_ms = int((self._now() - started) * 1000)
|
||||
response = None
|
||||
@@ -622,8 +697,18 @@ class OcrClient:
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _summarize(result: OcrTextTransportResult | OcrLayoutTransportResult) -> str:
|
||||
if hasattr(result, "text"):
|
||||
def _summarize(result: _TransportResult) -> str:
|
||||
"""遥测行 response 摘要。**永不写入文档正文**:文档只记规模(元素数/页数/md 字数)。
|
||||
|
||||
用 isinstance 显式区分三类结果:旧的 `hasattr(result, "text")` 对没有顶层 `text` 的
|
||||
新结果会静默套成 layout 摘要,漏记 `md_chars`。
|
||||
"""
|
||||
if isinstance(result, OcrDocumentTransportResult):
|
||||
return (
|
||||
f"<document elements={len(result.elements)} pages={len(result.page_sizes)} "
|
||||
f"md_chars={len(result.markdown)}>"
|
||||
)
|
||||
if isinstance(result, OcrTextTransportResult):
|
||||
return result.text[:_RESPONSE_TEXT_CAP]
|
||||
return f"<elements n={len(result.elements)} pages={len(result.page_sizes)}>"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user