fix: close adversarial review findings on document parsing
Malformed download URLs are rejected before the second request, error bodies are read only up to a bound, the telemetry summary keeps structural dispatch for duck-typed results, and the BMP magic now requires a valid DIB header. Docs and design are aligned with the implementation, and new test wording is neutral.
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
|
||||
**成败取决于外部服务当下状态**,按 CLAUDE.md §4.6 一律标 `slow`: 默认套件不收集,发布清单
|
||||
第 4 步按 diff 交集选跑(`pytest tests/integration/test_monkey_ocr_document_live.py -m slow`)。
|
||||
服务与医院线共用——所以每条用例只发**单份极小文档**、串行、不压测。
|
||||
服务与其它业务线共用——所以每条用例只发**单份极小文档**、串行、不压测。
|
||||
|
||||
文档字节由本文件手写构造(最小合法 PDF,无第三方内容、无新依赖)。
|
||||
不依赖 `data/soak` 图片语料,故**不**并入带语料 `skipif` 的 `test_monkey_ocr_live.py`。
|
||||
@@ -65,7 +65,7 @@ def _env(**extra: str) -> dict[str, str]:
|
||||
"OCR__MONKEY__1__API_KEY": "none",
|
||||
"OCR__MONKEY__1__MODEL": "monkey-ocr",
|
||||
"OCR__MONKEY__1__TIMEOUT_S": "300",
|
||||
"OCR__MONKEY__1__MAX_CONCURRENCY": "1", # 与医院线共用,串行
|
||||
"OCR__MONKEY__1__MAX_CONCURRENCY": "1", # 与其它业务线共用,串行
|
||||
# 服务在 LAN,开发机的系统代理会对内网地址回 403
|
||||
"OCR__MONKEY__1__TRUST_ENV": "false",
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
"""OCR 文档结果包的合成 fixtures(#34)。
|
||||
|
||||
全部**合成**: 保留 2026-10-21 实测的结构骨架(para_blocks 块类型、table/image 的嵌套
|
||||
`blocks`、span 字段),一切识别文本替换为 `SYNTH-*` 占位——第三方论文内容许可不明、
|
||||
医疗数据不入库(零业务假设 + P5),真实 ZIP 不提交。
|
||||
`blocks`、span 字段),一切识别文本替换为 `SYNTH-*` 占位——第三方文档内容许可不明、
|
||||
敏感数据不入库(零业务假设 + P5),真实 ZIP 不提交。
|
||||
"""
|
||||
|
||||
import io
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
fixtures 全部**合成**: 保留 2026-10-21 实测的结构骨架(para_blocks 的块类型、
|
||||
table/image 的嵌套 `blocks`、span 字段),一切识别文本替换为 `SYNTH-*` 占位——
|
||||
第三方论文内容许可不明、医疗数据不入库(零业务假设 + P5),真实 ZIP 不提交。
|
||||
第三方文档内容许可不明、敏感数据不入库(零业务假设 + P5),真实 ZIP 不提交。
|
||||
|
||||
契约表逐行一条,参数化列明各分支。解析函数是 bytes → 结果的纯函数,失败一律
|
||||
`ResultInvalidError`,**不得漏出原生异常**(漏出会让 `OcrClient._call` 既不写终态遥测
|
||||
@@ -498,7 +498,7 @@ class TestExternalDataExceptionBoundary:
|
||||
|
||||
|
||||
class TestLayoutPathUntouched:
|
||||
"""文档路径更严/更宽的口径不得外溢到医院线在用的 layout 解析。"""
|
||||
"""文档路径更严/更宽的口径不得外溢到既有调用方在用的 layout 解析。"""
|
||||
|
||||
def test_layout_parser_still_rejects_degenerate_int_boxes(self):
|
||||
from polygateway.transports.monkey_ocr import _parse_middle_json
|
||||
@@ -510,7 +510,7 @@ class TestLayoutPathUntouched:
|
||||
|
||||
def test_layout_parser_keeps_its_lenient_page_idx_handling(self):
|
||||
"""layout 路径: 合法的非负整数 `page_idx` 原样接受(哪怕与页序不符),非法值才回退
|
||||
枚举下标——这正是文档路径改为严格的原因,但医院线的既有行为不得被连带改动。"""
|
||||
枚举下标——这正是文档路径改为严格的原因,但既有调用方的行为不得被连带改动。"""
|
||||
from polygateway.transports.monkey_ocr import _parse_middle_json
|
||||
|
||||
accepted, _ = _parse_middle_json(_package([_page([_text_block()], page_idx=7)]))
|
||||
|
||||
@@ -177,6 +177,8 @@ class TestRecognizeText:
|
||||
|
||||
|
||||
_PDF_BODY = b"%PDF-1.4\n%synthetic body"
|
||||
# BMP: "BM" + 文件大小(4) + 保留(4) + 像素偏移(4) + DIB 头大小(4,常见 12/40/52/56/64/108/124)
|
||||
_BMP_HEADER = b"BM" + b"\x00" * 12 + (40).to_bytes(4, "little") + b"\x00" * 20
|
||||
_JUNK = b"x" * 200
|
||||
|
||||
|
||||
@@ -192,7 +194,11 @@ class TestSniffMedia:
|
||||
pytest.param(b"\x89PNG\r\n\x1a\n" + b"0" * 20, "image", id="png"),
|
||||
pytest.param(b"II*\x00" + b"0" * 20, "image", id="tiff-le"),
|
||||
pytest.param(b"MM\x00*" + b"0" * 20, "image", id="tiff-be"),
|
||||
pytest.param(b"BM" + b"0" * 20, "image", id="bmp"),
|
||||
pytest.param(_BMP_HEADER, "image", id="bmp"),
|
||||
pytest.param(
|
||||
b"BMW is a car maker, not a bitmap", "unknown", id="text-starting-with-BM"
|
||||
),
|
||||
pytest.param(b"BM" + b"\x00" * 40, "unknown", id="BM-with-bad-dib-header"),
|
||||
pytest.param(b"GIF89a" + b"0" * 20, "image", id="gif89a"),
|
||||
pytest.param(b"GIF87a" + b"0" * 20, "image", id="gif87a"),
|
||||
pytest.param(b"RIFF\x00\x00\x00\x00WEBPVP8 ", "image", id="webp"),
|
||||
@@ -479,7 +485,7 @@ def _parsing_failed(reason):
|
||||
class TestRejectedMarkers:
|
||||
"""#34 设计 §6: 确定性坏输入的 500 在文档路径归「请求被拒」,其余分类一概不动。
|
||||
|
||||
共享的 `_classify_status` 默认行为字节级不变——医院线对坏图的重试/熔断行为属公共行为,
|
||||
共享的 `_classify_status` 默认行为字节级不变——既有调用方对坏图的重试/熔断行为属公共行为,
|
||||
推广须另行设计(issue #35)。
|
||||
"""
|
||||
|
||||
@@ -555,7 +561,7 @@ class TestRejectedMarkers:
|
||||
assert MonkeyOcrTransport(deterministic_failure_markers=custom)._rejected_markers == custom
|
||||
|
||||
async def test_layout_and_text_paths_do_not_apply_the_markers(self):
|
||||
"""传输实例持有标记,但只有文档路径使用;医院线的两个方法行为不变。"""
|
||||
"""传输实例持有标记,但只有文档路径使用;既有的两个方法行为不变。"""
|
||||
body = _parsing_failed(_DEFAULT_REJECTED_MARKERS[0]).encode()
|
||||
t = _transport_for(
|
||||
_routes(
|
||||
@@ -569,6 +575,49 @@ class TestRejectedMarkers:
|
||||
await t.parse_layout(image=b"jpg", source=_source(), call_id="c1")
|
||||
|
||||
|
||||
class TestMalformedDownloadUrl:
|
||||
"""`download_url` 来自服务端响应,属外部输入。畸形值(控制字符、坏方括号、越界端口)会让 httpx
|
||||
抛 `InvalidURL`(不属 `HTTPError` 体系)或在连接阶段抛 ExceptionGroup,从 `_translate_http_errors`
|
||||
下漏出四分类——`OcrClient._call` 只接 `PolyGatewayError`,于是不写终态遥测行、半开探针不归还。
|
||||
(独立审查发现;`parse_layout` 的基线同样如此,共用的 `_post_parse` 一处修。)"""
|
||||
|
||||
_BAD_URLS = [
|
||||
pytest.param("http://[::1", id="broken-bracket"),
|
||||
pytest.param("/static/a\r\nb.zip", id="crlf"),
|
||||
pytest.param("/static/a\x00.zip", id="nul"),
|
||||
pytest.param("/static/a\tb.zip", id="tab"),
|
||||
pytest.param("http://127.0.0.1:99999/x", id="port-out-of-range"),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _transport_with(download_url, seen):
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append((request.method, request.url.path))
|
||||
if request.url.path == "/parse":
|
||||
return httpx.Response(200, json=_parse_body(download_url))
|
||||
return httpx.Response(200, content=b"never fetched")
|
||||
|
||||
return _transport_for(handler)
|
||||
|
||||
@pytest.mark.parametrize("download_url", _BAD_URLS)
|
||||
@pytest.mark.parametrize("method", ["parse_layout", "parse_document"])
|
||||
async def test_rejected_as_transient_without_fetching(self, download_url, method):
|
||||
seen = []
|
||||
t = self._transport_with(download_url, seen)
|
||||
kwargs = {"image": b"jpg"} if method == "parse_layout" else {"document": _PDF_BODY}
|
||||
with pytest.raises(TransientError, match="download_url"):
|
||||
await getattr(t, method)(**kwargs, source=_source(), call_id="c1")
|
||||
assert seen == [("POST", "/parse")] # 从未发出第二段请求
|
||||
|
||||
async def test_well_formed_urls_are_untouched(self):
|
||||
for url in ("/static/ok.zip", "http://10.77.0.20:7866/static/abs.zip"):
|
||||
seen = []
|
||||
t = self._transport_with(url, seen)
|
||||
with pytest.raises(ResultInvalidError): # 响应体不是 ZIP: 说明确实走到了第二段
|
||||
await t.parse_layout(image=b"jpg", source=_source(), call_id="c1")
|
||||
assert [m for m, _ in seen] == ["POST", "GET"]
|
||||
|
||||
|
||||
class TestParseDocument:
|
||||
"""#34 设计 §5: 本地预检 → POST /parse → 流式 GET(体积上限)→ 线程内解析。"""
|
||||
|
||||
@@ -693,6 +742,26 @@ class TestParseDocument:
|
||||
await t.parse_document(document=_PDF_BODY, source=_source(), call_id="c1")
|
||||
assert ei.value.status_code == status and ei.value.body_text == body
|
||||
|
||||
async def test_huge_error_body_is_read_only_up_to_a_bound(self):
|
||||
"""摘要只需要几 KiB;服务(或中间人)回一个巨大的错误体不该被整个读进内存。
|
||||
|
||||
(独立审查实测:600 MiB 的 500 响应让进程 RSS 涨到 1.8 GiB——新代码声称"流式 + 体积上限",
|
||||
错误分支却是例外。)
|
||||
"""
|
||||
consumed = []
|
||||
|
||||
async def endless_error_body():
|
||||
for _ in range(100_000):
|
||||
consumed.append(1)
|
||||
yield b"e" * 1024
|
||||
|
||||
t = self._transport(zip_resp=httpx.Response(500, content=endless_error_body()))
|
||||
with pytest.raises(TransientError) as ei:
|
||||
await t.parse_document(document=_PDF_BODY, source=_source(), call_id="c1")
|
||||
assert ei.value.status_code == 500
|
||||
assert len(consumed) <= monkey_ocr._ERROR_BODY_READ_CAP // 1024 + 2 # 只读了上限那么多
|
||||
assert ei.value.body_text.startswith("e" * 100) # 摘要仍在
|
||||
|
||||
async def test_error_status_with_oversized_content_length_is_still_classified_by_http(
|
||||
self, monkeypatch
|
||||
):
|
||||
|
||||
@@ -5,6 +5,7 @@ retry_exhausted/circuit_open/stalled 三组断言即设计 §6 ③ 的 G1 契约
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
@@ -826,6 +827,21 @@ class TestDocumentTelemetry:
|
||||
assert OcrClient._summarize(_LAYOUT_OK) == "<elements n=1 pages=1>"
|
||||
assert OcrClient._summarize(_TEXT_OK) == "LINE-1"
|
||||
|
||||
def test_summary_keeps_structural_dispatch_for_duck_typed_results(self):
|
||||
"""旧实现按结构分派(`hasattr(result, "text")`)。下游测试替身可能返回非库类型的鸭子对象,
|
||||
改成 isinstance 会让它们在开启遥测后以原生 AttributeError 漏出——无登记的回归。"""
|
||||
assert OcrClient._summarize(SimpleNamespace(text="hello", raw={})) == "hello"
|
||||
assert (
|
||||
OcrClient._summarize(SimpleNamespace(elements=[1, 2], page_sizes=[(1.0, 1.0)]))
|
||||
== "<elements n=2 pages=1>"
|
||||
)
|
||||
assert (
|
||||
OcrClient._summarize(
|
||||
SimpleNamespace(elements=[1], page_sizes=[(1.0, 1.0)], markdown="abc")
|
||||
)
|
||||
== "<document elements=1 pages=1 md_chars=3>"
|
||||
)
|
||||
|
||||
async def test_failed_attempt_row_carries_the_class_prefix(self):
|
||||
recorder = _MemoryRecorder()
|
||||
client, _, _ = _client([_src()], [ResultInvalidError("bad package")], telemetry=recorder)
|
||||
|
||||
Reference in New Issue
Block a user