feat: parse document result packages from middle json only
Content comes from the middle json and its paired markdown; the content list is deliberately ignored because it escapes markdown characters. The external-data exception boundary covers both reading and structure parsing so no native exception escapes the OCR client. The layout parser is unchanged.
This commit is contained in:
@@ -0,0 +1,293 @@
|
||||
"""MonkeyOCR 结果包解析: bytes → 文档元素的纯函数(#34 设计 §5/§5.1)。
|
||||
|
||||
本模块**无 HTTP、无治理、无全局状态**,所以能在 `asyncio.to_thread` 里运行——
|
||||
同步解析大对象会占住事件循环,使取消与 `call_deadline_s`(基于 `asyncio.timeout`)
|
||||
无法及时生效。
|
||||
|
||||
内容只取自 `_middle.json`(bbox/类型/页码/阅读序的权威来源,spans 里直接带
|
||||
`content`/`html`)与配对的 `.md`。**有意不读 `_content_list.json`**: 它是展示格式,
|
||||
文本里 `* \\` ~ $` 会被转义成 `\\$` 之类(实测 134 个 text/title 块中 23 个含 `\\$`,
|
||||
取它会污染 LaTeX),且无 bbox、类型词汇与 middle 不同,逐块对位还会引入版本漂移
|
||||
时整份失败的新故障面。
|
||||
|
||||
与 layout 路径(`monkey_ocr._parse_middle_json`,医院线在用,保持原样)的三处差异:
|
||||
零页拒绝(成功解析的文档至少 1 页)、`page_idx` 必须等于页序(layout 路径回退枚举
|
||||
下标,会让 page_sizes 按 page_index 索引的契约静默错位)、bbox 只要求有限且有序
|
||||
(整数化退化校验专为 CHS shim 的 `int()` 裁剪,文档消费者不裁剪,一个退化框不该
|
||||
让数百个块的整份文档失败)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import zipfile
|
||||
import zlib
|
||||
from io import BytesIO
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from polygateway.errors import ResultInvalidError
|
||||
from polygateway.types import OcrDocumentElement
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
MAX_MIDDLE_BYTES = 64 * 1024 * 1024
|
||||
"""`_middle.json` 声明大小上限。样本约 72KB/页,64MiB 足够数百页;超限即拒,
|
||||
防止一个畸形/恶意包把解析线程的内存吃满。"""
|
||||
|
||||
MAX_MARKDOWN_BYTES = 16 * 1024 * 1024
|
||||
"""`.md` 声明大小上限(样本约 3KB/页)。"""
|
||||
|
||||
_MIDDLE_SUFFIX = "_middle.json"
|
||||
|
||||
|
||||
def _finite_number(value: object) -> float:
|
||||
"""JSON 数值 → 有限浮点;布尔与非有限数拒绝(CHS invokers.py:427 逐字)。"""
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ValueError("字段不是数值")
|
||||
number = float(value)
|
||||
if not math.isfinite(number):
|
||||
raise ValueError("字段不是有限数值")
|
||||
return number
|
||||
|
||||
|
||||
def _parse_page_size(page: dict) -> tuple[float, float]:
|
||||
page_size = page.get("page_size")
|
||||
if not isinstance(page_size, list) or len(page_size) != 2:
|
||||
raise ValueError("page_size 无效")
|
||||
width, height = (_finite_number(item) for item in page_size)
|
||||
if width <= 0 or height <= 0:
|
||||
raise ValueError("page_size 必须为正数")
|
||||
return (width, height)
|
||||
|
||||
|
||||
def parse_document_package(
|
||||
zip_bytes: bytes,
|
||||
) -> tuple[list[OcrDocumentElement], list[tuple[float, float]], str]:
|
||||
"""ZIP → (elements, page_sizes, markdown);一切形态异常归 `ResultInvalidError`。
|
||||
|
||||
外层异常边界**同时盖住「读取」与「全部结构解析」**: `_finite_number` 里
|
||||
`float(10**400)` 会抛 `OverflowError`,它发生在读取之后。放错位置的后果是原生
|
||||
异常漏出——`OcrClient._call` 只接 `PolyGatewayError`,于是既不写终态遥测行,又让
|
||||
持有的半开探针不被归还。`CancelledError` 是 `BaseException`,天然不在其内。
|
||||
"""
|
||||
try:
|
||||
payload, markdown = _read_members(zip_bytes)
|
||||
elements, page_sizes = _parse_pages(payload)
|
||||
except ResultInvalidError:
|
||||
raise
|
||||
except (
|
||||
ValueError, # 含 JSONDecodeError / UnicodeDecodeError
|
||||
TypeError,
|
||||
KeyError,
|
||||
OverflowError,
|
||||
RecursionError,
|
||||
zipfile.BadZipFile,
|
||||
zipfile.LargeZipFile,
|
||||
) as exc:
|
||||
raise ResultInvalidError(f"OCR 文档结果包内容无效: {exc}") from exc
|
||||
return elements, page_sizes, markdown
|
||||
|
||||
|
||||
def _read_members(zip_bytes: bytes) -> tuple[object, str]:
|
||||
"""取首个 `*_middle.json` 与按名配对的 `.md`,返回 (middle 解码对象, md 文本)。"""
|
||||
with zipfile.ZipFile(BytesIO(zip_bytes)) as archive:
|
||||
names = archive.namelist()
|
||||
middle_names = [n for n in names if n.endswith(_MIDDLE_SUFFIX)]
|
||||
if not middle_names:
|
||||
raise ResultInvalidError("OCR 文档结果包缺少 _middle.json")
|
||||
middle_name = middle_names[0]
|
||||
# 精确配对而非"取第一个 .md": 包里可能有多个 .md,取错就是把别的文档当本文档
|
||||
markdown_name = middle_name[: -len(_MIDDLE_SUFFIX)] + ".md"
|
||||
if markdown_name not in names:
|
||||
raise ResultInvalidError(f"OCR 文档结果包缺少与 {middle_name} 配对的 {markdown_name}")
|
||||
middle_raw = _read_member(archive, middle_name, MAX_MIDDLE_BYTES)
|
||||
markdown_raw = _read_member(archive, markdown_name, MAX_MARKDOWN_BYTES)
|
||||
return json.loads(middle_raw), markdown_raw.decode("utf-8-sig")
|
||||
|
||||
|
||||
def _read_member(archive: zipfile.ZipFile, name: str, cap: int) -> bytes:
|
||||
"""读一个成员:先查**声明大小**再解压。
|
||||
|
||||
`zipfile` 读取不会超过声明大小(超出即 CRC 失败),所以声明大小检查足以界定内存。
|
||||
窄 `try` 只包解压调用本身: 加密成员抛 `RuntimeError`、未知压缩方法抛
|
||||
`NotImplementedError`,这两个是标准库对"外部数据不可读"的约定用法,宽泛捕获
|
||||
`RuntimeError` 会掩盖编程错误,故不放到外层边界。
|
||||
"""
|
||||
info = archive.getinfo(name)
|
||||
if info.file_size > cap:
|
||||
raise ResultInvalidError(
|
||||
f"OCR 文档结果包成员 {name} 声明大小 {info.file_size} 超过上限 {cap}"
|
||||
)
|
||||
try:
|
||||
with archive.open(info) as stream:
|
||||
return stream.read()
|
||||
except (RuntimeError, NotImplementedError, zlib.error, EOFError) as exc:
|
||||
raise ResultInvalidError(f"OCR 文档结果包成员 {name} 无法解压: {exc}") from exc
|
||||
|
||||
|
||||
def _parse_pages(
|
||||
payload: object,
|
||||
) -> tuple[list[OcrDocumentElement], list[tuple[float, float]]]:
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("pdf_info"), list):
|
||||
raise ValueError("_middle.json 缺少 pdf_info 列表")
|
||||
pages = payload["pdf_info"]
|
||||
if not pages:
|
||||
raise ResultInvalidError("OCR 文档结果包 pdf_info 为空: 成功解析的文档至少应有 1 页")
|
||||
elements: list[OcrDocumentElement] = []
|
||||
page_sizes: list[tuple[float, float]] = []
|
||||
for position, page in enumerate(pages):
|
||||
page_elements, page_size = _parse_page(page, position)
|
||||
elements.extend(page_elements)
|
||||
page_sizes.append(page_size)
|
||||
return elements, page_sizes
|
||||
|
||||
|
||||
def _parse_page(
|
||||
page: object, position: int
|
||||
) -> tuple[list[OcrDocumentElement], tuple[float, float]]:
|
||||
if not isinstance(page, dict):
|
||||
raise ValueError("页面结构无效")
|
||||
page_size = _parse_page_size(page)
|
||||
page_index = _page_index(page, position)
|
||||
blocks = page.get("para_blocks", []) # 缺失 = 空页(空白页/无可识别块)是合法"无内容"
|
||||
if not isinstance(blocks, list):
|
||||
raise ValueError("para_blocks 无效")
|
||||
return [e for block in blocks for e in _block_elements(block, page_index)], page_size
|
||||
|
||||
|
||||
def _page_index(page: dict, position: int) -> int:
|
||||
"""`page_idx` 必须是等于页序的整数(非 bool)。
|
||||
|
||||
不沿用 layout 路径的"非法则回退枚举下标": 回退会掩盖服务端跳页/乱序,而
|
||||
`page_sizes` 是按 page_index 索引的,静默错位比响亮失败更糟。
|
||||
"""
|
||||
raw = page.get("page_idx")
|
||||
if isinstance(raw, bool) or not isinstance(raw, int) or raw != position:
|
||||
raise ValueError(f"page_idx 无效或与页序不符: {raw!r} != {position}")
|
||||
return raw
|
||||
|
||||
|
||||
def _block_type(block: object) -> str:
|
||||
if not isinstance(block, dict):
|
||||
raise ValueError("块结构无效")
|
||||
block_type = block.get("type")
|
||||
if not isinstance(block_type, str) or not block_type.strip():
|
||||
raise ValueError("块缺少 type")
|
||||
return block_type
|
||||
|
||||
|
||||
def _loose_bbox(raw: object) -> tuple[float, float, float, float]:
|
||||
"""长度 4 的有限数值且有序(允许零宽/零高);差异理由见模块 docstring。"""
|
||||
if not isinstance(raw, list) or len(raw) != 4:
|
||||
raise ValueError("bbox 无效")
|
||||
x1, y1, x2, y2 = (_finite_number(item) for item in raw)
|
||||
if x2 < x1 or y2 < y1:
|
||||
raise ValueError("bbox 边界顺序无效")
|
||||
return (x1, y1, x2, y2)
|
||||
|
||||
|
||||
def _sub_blocks(block: dict) -> list[dict]:
|
||||
subs = block.get("blocks", [])
|
||||
if not isinstance(subs, list):
|
||||
raise ValueError("blocks 无效")
|
||||
for sub in subs:
|
||||
_block_type(sub) # 顺带校验子块是 dict 且 type 合法
|
||||
return subs
|
||||
|
||||
|
||||
def _is_body(sub: dict) -> bool:
|
||||
return sub["type"].endswith("_body")
|
||||
|
||||
|
||||
def _block_elements(block: object, page_index: int) -> list[OcrDocumentElement]:
|
||||
"""父块 → 元素,外加其非 `*_body` 的嵌套子块(紧随父块、保持服务顺序)。"""
|
||||
block_type = _block_type(block)
|
||||
assert isinstance(block, dict) # _block_type 已校验;仅为类型收窄(内部不变量)
|
||||
parent = OcrDocumentElement(
|
||||
type=block_type,
|
||||
bbox=_loose_bbox(block.get("bbox")),
|
||||
page_index=page_index,
|
||||
text=_block_text(block_type, block),
|
||||
)
|
||||
return [parent, *_extra_elements(block, page_index)]
|
||||
|
||||
|
||||
def _extra_elements(block: dict, page_index: int) -> list[OcrDocumentElement]:
|
||||
"""caption/footnote 一类子块:默认部署不产生,出现时有文字就独立成元素,不静默丢弃。"""
|
||||
extras: list[OcrDocumentElement] = []
|
||||
for sub in _sub_blocks(block):
|
||||
if _is_body(sub):
|
||||
continue
|
||||
text = _lines_text(sub.get("lines", []))
|
||||
if text:
|
||||
extras.append(
|
||||
OcrDocumentElement(
|
||||
type=sub["type"],
|
||||
bbox=_loose_bbox(sub.get("bbox")),
|
||||
page_index=page_index,
|
||||
text=text,
|
||||
)
|
||||
)
|
||||
return extras
|
||||
|
||||
|
||||
def _block_text(block_type: str, block: dict) -> str:
|
||||
if block_type == "table":
|
||||
return _table_text(_sub_blocks(block))
|
||||
if block_type == "image":
|
||||
return "" # 图内文字服务不识别;span 只有 image_path
|
||||
if "lines" in block:
|
||||
return _lines_text(block["lines"])
|
||||
# 未知容器类型: 其 `*_body` 子块的文字并入父元素(非 body 子块已由 _extra_elements 输出)
|
||||
return "\n".join(
|
||||
_lines_text(sub.get("lines", [])) for sub in _sub_blocks(block) if _is_body(sub)
|
||||
)
|
||||
|
||||
|
||||
def _table_text(subs: list[dict]) -> str:
|
||||
bodies = [sub for sub in subs if sub["type"] == "table_body"]
|
||||
if not bodies:
|
||||
raise ValueError("table 块缺少 table_body 子块")
|
||||
return "\n".join(_lines_text(body.get("lines"), _table_span_text) for body in bodies)
|
||||
|
||||
|
||||
def _table_span_text(span: dict) -> str:
|
||||
"""表体 span 取 html;上游另有 latex 分支。皆无则响亮失败——表格"无内容"与
|
||||
合法空串在结果里无法区分,静默补空会让下游把缺失当成空表。"""
|
||||
for key in ("html", "latex"):
|
||||
value = span.get(key)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
raise ValueError("table span 缺少 html/latex")
|
||||
|
||||
|
||||
def _lines_text(lines: object, span_text: Callable[[dict], str] | None = None) -> str:
|
||||
"""行内 span 按 "" 拼、行间按 "\\n" 拼。
|
||||
|
||||
样本全是 1 行 1 span;多行/多 span 路径无真实样本,拼接规则由构造 fixture 钉住,
|
||||
文档注明"未经实测"。
|
||||
"""
|
||||
if not isinstance(lines, list):
|
||||
raise ValueError("lines 无效")
|
||||
return "\n".join(_line_text(line, span_text or _span_text) for line in lines)
|
||||
|
||||
|
||||
def _line_text(line: object, span_text: Callable[[dict], str]) -> str:
|
||||
if not isinstance(line, dict) or not isinstance(line.get("spans"), list):
|
||||
raise ValueError("行结构无效")
|
||||
parts = []
|
||||
for span in line["spans"]:
|
||||
if not isinstance(span, dict):
|
||||
raise ValueError("span 结构无效")
|
||||
parts.append(span_text(span))
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def _span_text(span: dict) -> str:
|
||||
"""文本 span 取 `content`(必须是 str);`inline_equation` 包 `$…$`(上游惯例)。"""
|
||||
content = span.get("content")
|
||||
if not isinstance(content, str):
|
||||
raise ValueError("span 缺少字符串 content")
|
||||
return f"${content}$" if span.get("type") == "inline_equation" else content
|
||||
@@ -10,7 +10,6 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
import zipfile
|
||||
from contextlib import contextmanager
|
||||
from io import BytesIO
|
||||
@@ -29,6 +28,7 @@ from polygateway.transports._http_errors import (
|
||||
response_body,
|
||||
summarize_body,
|
||||
)
|
||||
from polygateway.transports._monkey_document import _finite_number, _parse_page_size
|
||||
from polygateway.types import (
|
||||
OcrLayoutElement,
|
||||
OcrLayoutTransportResult,
|
||||
@@ -144,26 +144,6 @@ def _classify_status(
|
||||
return RequestRejectedError(message, **ctx)
|
||||
|
||||
|
||||
def _finite_number(value: object) -> float:
|
||||
"""JSON 数值 → 有限浮点;布尔与非有限数拒绝(CHS invokers.py:427 逐字)。"""
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ValueError("字段不是数值")
|
||||
number = float(value)
|
||||
if not math.isfinite(number):
|
||||
raise ValueError("字段不是有限数值")
|
||||
return number
|
||||
|
||||
|
||||
def _parse_page_size(page: dict) -> tuple[float, float]:
|
||||
page_size = page.get("page_size")
|
||||
if not isinstance(page_size, list) or len(page_size) != 2:
|
||||
raise ValueError("page_size 无效")
|
||||
width, height = (_finite_number(item) for item in page_size)
|
||||
if width <= 0 or height <= 0:
|
||||
raise ValueError("page_size 必须为正数")
|
||||
return (width, height)
|
||||
|
||||
|
||||
def _parse_bbox(bbox: object) -> tuple[float, float, float, float]:
|
||||
if not isinstance(bbox, list) or len(bbox) != 4:
|
||||
raise ValueError("bbox 无效")
|
||||
|
||||
Reference in New Issue
Block a user