fix: close adversarial review findings on document parsing
Malformed download URLs are rejected before the second request, error bodies are read only up to a bound, the telemetry summary keeps structural dispatch for duck-typed results, and the BMP magic now requires a valid DIB header. Docs and design are aligned with the implementation, and new test wording is neutral.
This commit is contained in:
@@ -102,7 +102,7 @@ class _AttemptOutcome:
|
||||
|
||||
|
||||
class OcrClient:
|
||||
"""治理化 OCR 入口: 同时实现 OcrTextPort 与 OcrLayoutPort(D9 端口族)。
|
||||
"""治理化 OCR 入口: 同时实现 OcrTextPort、OcrLayoutPort 与 OcrDocumentPort(D9 端口族;文档入口见 #34)。
|
||||
|
||||
两端点打同一服务实例池,共享同一 scope 的限流/熔断/选源状态;
|
||||
与其他 scope 共享后端实例即共享全局闸(显式传入,禁止隐式全局)。
|
||||
@@ -700,15 +700,17 @@ class OcrClient:
|
||||
def _summarize(result: _TransportResult) -> str:
|
||||
"""遥测行 response 摘要。**永不写入文档正文**:文档只记规模(元素数/页数/md 字数)。
|
||||
|
||||
用 isinstance 显式区分三类结果:旧的 `hasattr(result, "text")` 对没有顶层 `text` 的
|
||||
新结果会静默套成 layout 摘要,漏记 `md_chars`。
|
||||
**按结构分派而不是 isinstance**: 旧实现就是 `hasattr(result, "text")`,下游测试替身
|
||||
可能返回非库类型的鸭子对象,换成 isinstance 会让它们在开启遥测后以原生
|
||||
AttributeError 漏出。顺序要紧:文档结果有 `markdown`,先于只看 `text` 的判定,否则
|
||||
会被静默套成 layout 摘要而漏记 `md_chars`。
|
||||
"""
|
||||
if isinstance(result, OcrDocumentTransportResult):
|
||||
if hasattr(result, "markdown"):
|
||||
return (
|
||||
f"<document elements={len(result.elements)} pages={len(result.page_sizes)} "
|
||||
f"md_chars={len(result.markdown)}>"
|
||||
)
|
||||
if isinstance(result, OcrTextTransportResult):
|
||||
if hasattr(result, "text"):
|
||||
return result.text[:_RESPONSE_TEXT_CAP]
|
||||
return f"<elements n={len(result.elements)} pages={len(result.page_sizes)}>"
|
||||
|
||||
|
||||
@@ -56,11 +56,12 @@ _IMAGE_MAGICS: tuple[bytes, ...] = (
|
||||
b"\x89PNG\r\n\x1a\n", # PNG
|
||||
b"II*\x00", # TIFF 小端
|
||||
b"MM\x00*", # TIFF 大端
|
||||
b"BM", # BMP
|
||||
b"GIF87a",
|
||||
b"GIF89a",
|
||||
)
|
||||
_PDF_HEADER = b"%PDF-"
|
||||
# BMP 的 "BM" 只有两字节,任何以 BM 开头的文本都会撞上。再要求偏移 14 处的 DIB 头大小字段合法
|
||||
_BMP_DIB_HEADER_SIZES = frozenset({12, 40, 52, 56, 64, 108, 124})
|
||||
|
||||
# 文档路径把这些「确定性坏输入」的 500 改归 RequestRejected(#34 设计 §6)。小写子串匹配,
|
||||
# 取 2026-10-21 实测(截断 PDF / 伪 PDF 头 / 乱码 / 空文件 / 扩展名与内容不符)。
|
||||
@@ -76,6 +77,10 @@ _DEFAULT_REJECTED_MARKERS: tuple[str, ...] = (
|
||||
# 所以前缀只是必要条件,必须同时命中具体标记
|
||||
_PARSING_FAILED_PREFIX = "parsing failed:"
|
||||
|
||||
_ERROR_BODY_READ_CAP = 64 * 1024
|
||||
"""文档路径读取错误响应体的字节上限。摘要只取几 KiB(`summarize_body`),服务或中间人回一个
|
||||
巨大的错误体不该被整个读进内存(独立审查实测 600 MiB 的 500 让 RSS 涨到 1.8 GiB)。"""
|
||||
|
||||
_MAX_PACKAGE_BYTES = 256 * 1024 * 1024
|
||||
"""整包(含 `images/*.jpg`)体积上限,仅文档路径。4 页含 4 图约 598KB,数百页含图的文档
|
||||
也远低于此;超限即拒,防止畸形/恶意响应把内存吃满。httpx 非流式 `get` 会先整包 `aread()`,
|
||||
@@ -85,6 +90,20 @@ _MAX_PACKAGE_BYTES = 256 * 1024 * 1024
|
||||
_PDF_HEADER_WINDOW = 1024
|
||||
|
||||
|
||||
def _is_webp(data: bytes) -> bool:
|
||||
"""RIFF 容器且类型码为 WEBP。"""
|
||||
return data[:4] == b"RIFF" and data[8:12] == b"WEBP"
|
||||
|
||||
|
||||
def _is_bmp(data: bytes) -> bool:
|
||||
"""`BM` 开头且 DIB 头大小字段合法(单靠两字节魔数会误判以 BM 开头的文本)。"""
|
||||
return (
|
||||
data.startswith(b"BM")
|
||||
and len(data) >= 18
|
||||
and int.from_bytes(data[14:18], "little") in _BMP_DIB_HEADER_SIZES
|
||||
)
|
||||
|
||||
|
||||
def _sniff_media(data: bytes) -> _MediaKind:
|
||||
"""按内容判定上传类型: 图片魔数 → 前 1KB 内 `%PDF-` → 未知。
|
||||
|
||||
@@ -92,7 +111,7 @@ def _sniff_media(data: bytes) -> _MediaKind:
|
||||
若与内容不符会得到 HTTP 500(`is no PDF`)或——在 `/ocr/text` 上——`success:true`
|
||||
的幻觉文本(#34 设计 E2/E4)。所以标签只能由库按内容定,不开放给调用方声明。
|
||||
"""
|
||||
if data.startswith(_IMAGE_MAGICS) or (data[:4] == b"RIFF" and data[8:12] == b"WEBP"):
|
||||
if data.startswith(_IMAGE_MAGICS) or _is_webp(data) or _is_bmp(data):
|
||||
return "image"
|
||||
if _PDF_HEADER in data[:_PDF_HEADER_WINDOW]:
|
||||
return "pdf"
|
||||
@@ -254,6 +273,30 @@ def _parse_middle_json(
|
||||
raise ResultInvalidError(f"OCR 结果包内容无效: {exc}") from exc
|
||||
|
||||
|
||||
def _check_download_url(download_url: str, source: SourceConfig) -> None:
|
||||
"""`download_url` 来自服务端响应,属外部输入:畸形值先于第二段请求拒绝。
|
||||
|
||||
控制字符、坏方括号会让 httpx 抛 `InvalidURL`(不属 `HTTPError` 体系),越界端口会在连接阶段
|
||||
抛 ExceptionGroup;二者都会从 `_translate_http_errors` 下漏出四分类,`OcrClient._call` 只接
|
||||
`PolyGatewayError`,于是不写终态遥测行、持有的半开探针也不归还。服务返回畸形 URL 是协议故障,
|
||||
与"响应缺少 download_url"同归瞬时错误。**不限制主机**(基线接受绝对地址,测试钉住)。
|
||||
"""
|
||||
try:
|
||||
url = httpx.URL(download_url)
|
||||
except httpx.InvalidURL as exc:
|
||||
raise TransientError(
|
||||
f"{source.name} /parse 响应的 download_url 非法: {exc}",
|
||||
source_name=source.name,
|
||||
operation="parse",
|
||||
) from exc
|
||||
if url.port is not None and not 0 < url.port <= 65535:
|
||||
raise TransientError(
|
||||
f"{source.name} /parse 响应的 download_url 端口越界: {url.port}",
|
||||
source_name=source.name,
|
||||
operation="parse",
|
||||
)
|
||||
|
||||
|
||||
def _package_too_large(source: SourceConfig, size: int) -> ResultInvalidError:
|
||||
return ResultInvalidError(
|
||||
f"{source.name} OCR 结果包体积超过上限 {_MAX_PACKAGE_BYTES}(已读 {size})",
|
||||
@@ -262,6 +305,22 @@ def _package_too_large(source: SourceConfig, size: int) -> ResultInvalidError:
|
||||
)
|
||||
|
||||
|
||||
async def _bounded_error_response(resp: httpx.Response) -> httpx.Response:
|
||||
"""读错误响应体的前 `_ERROR_BODY_READ_CAP` 字节,复制成已缓冲的响应。"""
|
||||
data = bytearray()
|
||||
async for chunk in resp.aiter_bytes():
|
||||
data += chunk
|
||||
if len(data) >= _ERROR_BODY_READ_CAP:
|
||||
break
|
||||
headers = {k: v for k, v in resp.headers.items() if k.lower() == "content-type"}
|
||||
return httpx.Response(
|
||||
resp.status_code,
|
||||
headers=headers,
|
||||
content=bytes(data[:_ERROR_BODY_READ_CAP]),
|
||||
request=resp.request,
|
||||
)
|
||||
|
||||
|
||||
def _check_declared_length(resp: httpx.Response, source: SourceConfig) -> None:
|
||||
"""声明长度超限就不必读了;缺失/非法的头部不作判据(靠累计字节兜底)。"""
|
||||
try:
|
||||
@@ -371,6 +430,7 @@ class MonkeyOcrTransport:
|
||||
source_name=source.name,
|
||||
operation="parse",
|
||||
)
|
||||
_check_download_url(download_url, source)
|
||||
return download_url
|
||||
|
||||
async def parse_layout(
|
||||
@@ -430,10 +490,13 @@ class MonkeyOcrTransport:
|
||||
with _translate_http_errors(source.name, "download_result"):
|
||||
async with client.stream("GET", download_url) as resp:
|
||||
if not resp.is_success:
|
||||
# 先落缓冲:摘要口径只认已缓冲的响应体。条件是「非 2xx」而不是 `is_error`:
|
||||
# `raise_for_status()` 对 1xx/3xx 同样会抛,漏掉它们会让摘要丢失
|
||||
await resp.aread()
|
||||
resp.raise_for_status()
|
||||
# 条件是「非 2xx」而不是 `is_error`:1xx/3xx 同样需要保留摘要。错误体只读
|
||||
# 有界的前缀并复制成已缓冲的响应(摘要口径只认已缓冲的响应体)
|
||||
raise httpx.HTTPStatusError(
|
||||
f"HTTP {resp.status_code}",
|
||||
request=resp.request,
|
||||
response=await _bounded_error_response(resp),
|
||||
)
|
||||
_check_declared_length(resp, source)
|
||||
async for chunk in resp.aiter_bytes():
|
||||
total += len(chunk)
|
||||
|
||||
Reference in New Issue
Block a user