fix: close independent verification findings on document parsing
Zip construction and member reads now share one narrow boundary so corrupt BZIP2, LZMA and version fields cannot leak native exceptions. Non-success statuses are read before classification, text-bearing blocks require lines, the size cap and reasoning contract are pinned by tests that fail under mutation, and the per-page timing claim is corrected to the observed range.
This commit is contained in:
@@ -10,7 +10,7 @@
|
||||
取它会污染 LaTeX),且无 bbox、类型词汇与 middle 不同,逐块对位还会引入版本漂移
|
||||
时整份失败的新故障面。
|
||||
|
||||
与 layout 路径(`monkey_ocr._parse_middle_json`,医院线在用,保持原样)的三处差异:
|
||||
与 layout 路径(`monkey_ocr._parse_middle_json`,既有调用方在用,保持原样)的三处差异:
|
||||
零页拒绝(成功解析的文档至少 1 页)、`page_idx` 必须等于页序(layout 路径回退枚举
|
||||
下标,会让 page_sizes 按 page_index 索引的契约静默错位)、bbox 只要求有限且有序
|
||||
(整数化退化校验专为 CHS shim 的 `int()` 裁剪,文档消费者不裁剪,一个退化框不该
|
||||
@@ -20,6 +20,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import lzma
|
||||
import math
|
||||
import zipfile
|
||||
import zlib
|
||||
@@ -41,8 +42,14 @@ MAX_MARKDOWN_BYTES = 16 * 1024 * 1024
|
||||
|
||||
_MIDDLE_SUFFIX = "_middle.json"
|
||||
|
||||
# ZIP 层「外部数据不可读」的标准库约定用法。只在 ZIP I/O 区域(构造 + 读取)内捕获,
|
||||
# 不放到外层边界——宽泛捕获 RuntimeError/OSError 会掩盖编程错误:
|
||||
# 加密成员 → RuntimeError;未知压缩方法/版本字段损坏 → NotImplementedError;
|
||||
# DEFLATE 流损坏 → zlib.error / EOFError;BZIP2 流损坏 → OSError;LZMA 流损坏 → LZMAError
|
||||
_ZIP_IO_ERRORS = (RuntimeError, NotImplementedError, zlib.error, EOFError, OSError, lzma.LZMAError)
|
||||
|
||||
def _finite_number(value: object) -> float:
|
||||
|
||||
def finite_number(value: object) -> float:
|
||||
"""JSON 数值 → 有限浮点;布尔与非有限数拒绝(CHS invokers.py:427 逐字)。"""
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ValueError("字段不是数值")
|
||||
@@ -52,11 +59,12 @@ def _finite_number(value: object) -> float:
|
||||
return number
|
||||
|
||||
|
||||
def _parse_page_size(page: dict) -> tuple[float, float]:
|
||||
def parse_page_size(page: dict) -> tuple[float, float]:
|
||||
"""`page_size` → (宽, 高);须为 2 个正有限数(CHS invokers.py 数值防御,原样迁入)。"""
|
||||
page_size = page.get("page_size")
|
||||
if not isinstance(page_size, list) or len(page_size) != 2:
|
||||
raise ValueError("page_size 无效")
|
||||
width, height = (_finite_number(item) for item in page_size)
|
||||
width, height = (finite_number(item) for item in page_size)
|
||||
if width <= 0 or height <= 0:
|
||||
raise ValueError("page_size 必须为正数")
|
||||
return (width, height)
|
||||
@@ -67,7 +75,10 @@ def parse_document_package(
|
||||
) -> tuple[list[OcrDocumentElement], list[tuple[float, float]], str]:
|
||||
"""ZIP → (elements, page_sizes, markdown);一切形态异常归 `ResultInvalidError`。
|
||||
|
||||
外层异常边界**同时盖住「读取」与「全部结构解析」**: `_finite_number` 里
|
||||
与 layout 路径的三处差异(理由见模块 docstring): ① 零页拒绝(成功解析的文档至少 1 页);
|
||||
② `page_idx` 必须等于页序,不回退枚举下标;③ bbox 只要求有限且有序,不做整数化退化校验。
|
||||
|
||||
外层异常边界**同时盖住「读取」与「全部结构解析」**: `finite_number` 里
|
||||
`float(10**400)` 会抛 `OverflowError`,它发生在读取之后。放错位置的后果是原生
|
||||
异常漏出——`OcrClient._call` 只接 `PolyGatewayError`,于是既不写终态遥测行,又让
|
||||
持有的半开探针不被归还。`CancelledError` 是 `BaseException`,天然不在其内。
|
||||
@@ -91,45 +102,55 @@ def parse_document_package(
|
||||
|
||||
|
||||
def _read_members(zip_bytes: bytes) -> tuple[object, str]:
|
||||
"""取首个 `*_middle.json` 与按名配对的 `.md`,返回 (middle 解码对象, md 文本)。"""
|
||||
with zipfile.ZipFile(BytesIO(zip_bytes)) as archive:
|
||||
names = archive.namelist()
|
||||
middle_names = [n for n in names if n.endswith(_MIDDLE_SUFFIX)]
|
||||
if not middle_names:
|
||||
raise ResultInvalidError("OCR 文档结果包缺少 _middle.json")
|
||||
middle_name = middle_names[0]
|
||||
# 精确配对而非"取第一个 .md": 包里可能有多个 .md,取错就是把别的文档当本文档
|
||||
markdown_name = middle_name[: -len(_MIDDLE_SUFFIX)] + ".md"
|
||||
if markdown_name not in names:
|
||||
raise ResultInvalidError(f"OCR 文档结果包缺少与 {middle_name} 配对的 {markdown_name}")
|
||||
middle_raw = _read_member(archive, middle_name, MAX_MIDDLE_BYTES)
|
||||
markdown_raw = _read_member(archive, markdown_name, MAX_MARKDOWN_BYTES)
|
||||
"""取首个 `*_middle.json` 与按名配对的 `.md`,返回 (middle 解码对象, md 文本)。
|
||||
|
||||
`ZipFile(...)` 的构造与成员读取在**同一个**窄 `try` 里: 独立验证发现中央目录版本字段
|
||||
损坏会在构造阶段抛 `NotImplementedError`,而旧写法只包了成员读取。
|
||||
"""
|
||||
try:
|
||||
with zipfile.ZipFile(BytesIO(zip_bytes)) as archive:
|
||||
middle_raw, markdown_raw = _read_pair(archive)
|
||||
except _ZIP_IO_ERRORS as exc:
|
||||
raise ResultInvalidError(f"OCR 文档结果包无法读取: {exc}") from exc
|
||||
return json.loads(middle_raw), markdown_raw.decode("utf-8-sig")
|
||||
|
||||
|
||||
def _read_pair(archive: zipfile.ZipFile) -> tuple[bytes, bytes]:
|
||||
"""定位并读取 middle 与配对的 md;缺失或超限抛 `ResultInvalidError`。"""
|
||||
names = archive.namelist()
|
||||
middle_names = [n for n in names if n.endswith(_MIDDLE_SUFFIX)]
|
||||
if not middle_names:
|
||||
raise ResultInvalidError("OCR 文档结果包缺少 _middle.json")
|
||||
middle_name = middle_names[0]
|
||||
# 精确配对而非"取第一个 .md": 包里可能有多个 .md,取错就是把别的文档当本文档
|
||||
markdown_name = middle_name[: -len(_MIDDLE_SUFFIX)] + ".md"
|
||||
if markdown_name not in names:
|
||||
raise ResultInvalidError(f"OCR 文档结果包缺少与 {middle_name} 配对的 {markdown_name}")
|
||||
return (
|
||||
_read_member(archive, middle_name, MAX_MIDDLE_BYTES),
|
||||
_read_member(archive, markdown_name, MAX_MARKDOWN_BYTES),
|
||||
)
|
||||
|
||||
|
||||
def _read_member(archive: zipfile.ZipFile, name: str, cap: int) -> bytes:
|
||||
"""读一个成员:先查**声明大小**再解压。
|
||||
|
||||
`zipfile` 读取不会超过声明大小(超出即 CRC 失败),所以声明大小检查足以界定内存。
|
||||
窄 `try` 只包解压调用本身: 加密成员抛 `RuntimeError`、未知压缩方法抛
|
||||
`NotImplementedError`,这两个是标准库对"外部数据不可读"的约定用法,宽泛捕获
|
||||
`RuntimeError` 会掩盖编程错误,故不放到外层边界。
|
||||
解压异常由调用方 `_read_members` 的窄 `try` 统一转换。
|
||||
"""
|
||||
info = archive.getinfo(name)
|
||||
if info.file_size > cap:
|
||||
raise ResultInvalidError(
|
||||
f"OCR 文档结果包成员 {name} 声明大小 {info.file_size} 超过上限 {cap}"
|
||||
)
|
||||
try:
|
||||
with archive.open(info) as stream:
|
||||
return stream.read()
|
||||
except (RuntimeError, NotImplementedError, zlib.error, EOFError) as exc:
|
||||
raise ResultInvalidError(f"OCR 文档结果包成员 {name} 无法解压: {exc}") from exc
|
||||
with archive.open(info) as stream:
|
||||
return stream.read()
|
||||
|
||||
|
||||
def _parse_pages(
|
||||
payload: object,
|
||||
) -> tuple[list[OcrDocumentElement], list[tuple[float, float]]]:
|
||||
"""`pdf_info` → 全部页的元素与页尺寸;零页拒绝。"""
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("pdf_info"), list):
|
||||
raise ValueError("_middle.json 缺少 pdf_info 列表")
|
||||
pages = payload["pdf_info"]
|
||||
@@ -147,9 +168,10 @@ def _parse_pages(
|
||||
def _parse_page(
|
||||
page: object, position: int
|
||||
) -> tuple[list[OcrDocumentElement], tuple[float, float]]:
|
||||
"""单页 → (该页元素, 页尺寸);`para_blocks` 缺失按空页处理。"""
|
||||
if not isinstance(page, dict):
|
||||
raise ValueError("页面结构无效")
|
||||
page_size = _parse_page_size(page)
|
||||
page_size = parse_page_size(page)
|
||||
page_index = _page_index(page, position)
|
||||
blocks = page.get("para_blocks", []) # 缺失 = 空页(空白页/无可识别块)是合法"无内容"
|
||||
if not isinstance(blocks, list):
|
||||
@@ -170,6 +192,7 @@ def _page_index(page: dict, position: int) -> int:
|
||||
|
||||
|
||||
def _block_type(block: object) -> str:
|
||||
"""校验块是 dict 且 `type` 为非空白字符串,返回 type。"""
|
||||
if not isinstance(block, dict):
|
||||
raise ValueError("块结构无效")
|
||||
block_type = block.get("type")
|
||||
@@ -182,13 +205,14 @@ def _loose_bbox(raw: object) -> tuple[float, float, float, float]:
|
||||
"""长度 4 的有限数值且有序(允许零宽/零高);差异理由见模块 docstring。"""
|
||||
if not isinstance(raw, list) or len(raw) != 4:
|
||||
raise ValueError("bbox 无效")
|
||||
x1, y1, x2, y2 = (_finite_number(item) for item in raw)
|
||||
x1, y1, x2, y2 = (finite_number(item) for item in raw)
|
||||
if x2 < x1 or y2 < y1:
|
||||
raise ValueError("bbox 边界顺序无效")
|
||||
return (x1, y1, x2, y2)
|
||||
|
||||
|
||||
def _sub_blocks(block: dict) -> list[dict]:
|
||||
"""嵌套子块列表(缺失为空);子块须是 dict 且 type 合法。"""
|
||||
subs = block.get("blocks", [])
|
||||
if not isinstance(subs, list):
|
||||
raise ValueError("blocks 无效")
|
||||
@@ -198,6 +222,7 @@ def _sub_blocks(block: dict) -> list[dict]:
|
||||
|
||||
|
||||
def _is_body(sub: dict) -> bool:
|
||||
"""子块是否正文部分(type 以 `_body` 结尾,如 table_body / image_body)。"""
|
||||
return sub["type"].endswith("_body")
|
||||
|
||||
|
||||
@@ -233,11 +258,19 @@ def _extra_elements(block: dict, page_index: int) -> list[OcrDocumentElement]:
|
||||
return extras
|
||||
|
||||
|
||||
_TEXT_BEARING_TYPES = frozenset({"text", "title", "interline_equation"})
|
||||
"""必带 `lines` 的块类型。缺 `lines` 不能当作"空文本"——与 table 缺内容就报错同口径,
|
||||
否则下游分不清"服务识别出空块"和"服务给了坏结构"。其余未知类型无 lines 仍是合法空文本。"""
|
||||
|
||||
|
||||
def _block_text(block_type: str, block: dict) -> str:
|
||||
"""块 → 文字;随 type 分派(table 取 html/latex,image 为空串,其余取 lines)。"""
|
||||
if block_type == "table":
|
||||
return _table_text(_sub_blocks(block))
|
||||
if block_type == "image":
|
||||
return "" # 图内文字服务不识别;span 只有 image_path
|
||||
if block_type in _TEXT_BEARING_TYPES and "lines" not in block:
|
||||
raise ValueError(f"{block_type} 块缺少 lines")
|
||||
if "lines" in block:
|
||||
return _lines_text(block["lines"])
|
||||
# 未知容器类型: 其 `*_body` 子块的文字并入父元素(非 body 子块已由 _extra_elements 输出)
|
||||
@@ -247,6 +280,7 @@ def _block_text(block_type: str, block: dict) -> str:
|
||||
|
||||
|
||||
def _table_text(subs: list[dict]) -> str:
|
||||
"""table 块 → 其 `table_body` 子块的 HTML(或 latex);没有表体即畸形。"""
|
||||
bodies = [sub for sub in subs if sub["type"] == "table_body"]
|
||||
if not bodies:
|
||||
raise ValueError("table 块缺少 table_body 子块")
|
||||
@@ -275,6 +309,7 @@ def _lines_text(lines: object, span_text: Callable[[dict], str] | None = None) -
|
||||
|
||||
|
||||
def _line_text(line: object, span_text: Callable[[dict], str]) -> str:
|
||||
"""一行 → 文字:该行全部 span 的文字直接相接。"""
|
||||
if not isinstance(line, dict) or not isinstance(line.get("spans"), list):
|
||||
raise ValueError("行结构无效")
|
||||
parts = []
|
||||
|
||||
@@ -29,9 +29,9 @@ from polygateway.transports._http_errors import (
|
||||
summarize_body,
|
||||
)
|
||||
from polygateway.transports._monkey_document import (
|
||||
_finite_number,
|
||||
_parse_page_size,
|
||||
finite_number,
|
||||
parse_document_package,
|
||||
parse_page_size,
|
||||
)
|
||||
from polygateway.types import (
|
||||
OcrDocumentTransportResult,
|
||||
@@ -166,7 +166,7 @@ def _classify_status(
|
||||
|
||||
唯一的可选窄化: `rejected_markers` 非空且为 500 且命中确定性坏输入标记时,归
|
||||
`RequestRejectedError`(**保留 `status_code=500`**: 服务有响应,熔断按「响应即健康」
|
||||
记成功且不计窗口)。默认空元组 = 既有行为字节级不变——医院线对坏图的处置属公共行为,
|
||||
记成功且不计窗口)。默认空元组 = 既有行为字节级不变——既有调用方对坏图的处置属公共行为,
|
||||
推广须另行设计(issue #35)。
|
||||
"""
|
||||
status = exc.response.status_code
|
||||
@@ -191,7 +191,7 @@ def _classify_status(
|
||||
def _parse_bbox(bbox: object) -> tuple[float, float, float, float]:
|
||||
if not isinstance(bbox, list) or len(bbox) != 4:
|
||||
raise ValueError("bbox 无效")
|
||||
x1, y1, x2, y2 = (_finite_number(item) for item in bbox)
|
||||
x1, y1, x2, y2 = (finite_number(item) for item in bbox)
|
||||
if x2 <= x1 or y2 <= y1:
|
||||
raise ValueError("bbox 边界顺序无效")
|
||||
# 库返回 float 原生 bbox;整数化退化校验专为 CHS shim 的 int() 裁剪路径
|
||||
@@ -220,9 +220,9 @@ def _parse_pages(payload: object) -> tuple[list[OcrLayoutElement], list[tuple[fl
|
||||
for fallback_index, page in enumerate(payload["pdf_info"]):
|
||||
if not isinstance(page, dict):
|
||||
raise ValueError("页面结构无效")
|
||||
page_sizes.append(_parse_page_size(page))
|
||||
page_sizes.append(parse_page_size(page))
|
||||
raw_index = page.get("page_idx", fallback_index)
|
||||
# bool 是 int 子类、负页号无意义——与 _finite_number 拒 bool 的口径一致
|
||||
# bool 是 int 子类、负页号无意义——与 finite_number 拒 bool 的口径一致
|
||||
valid_idx = (
|
||||
isinstance(raw_index, int) and not isinstance(raw_index, bool) and raw_index >= 0
|
||||
)
|
||||
@@ -429,8 +429,10 @@ class MonkeyOcrTransport:
|
||||
total = 0
|
||||
with _translate_http_errors(source.name, "download_result"):
|
||||
async with client.stream("GET", download_url) as resp:
|
||||
if resp.is_error:
|
||||
await resp.aread() # 先落缓冲:摘要口径只认已缓冲的响应体
|
||||
if not resp.is_success:
|
||||
# 先落缓冲:摘要口径只认已缓冲的响应体。条件是「非 2xx」而不是 `is_error`:
|
||||
# `raise_for_status()` 对 1xx/3xx 同样会抛,漏掉它们会让摘要丢失
|
||||
await resp.aread()
|
||||
resp.raise_for_status()
|
||||
_check_declared_length(resp, source)
|
||||
async for chunk in resp.aiter_bytes():
|
||||
|
||||
Reference in New Issue
Block a user