fix: close adversarial review findings on document parsing
Malformed download URLs are rejected before the second request, error bodies are read only up to a bound, the telemetry summary keeps structural dispatch for duck-typed results, and the BMP magic now requires a valid DIB header. Docs and design are aligned with the implementation, and new test wording is neutral.
This commit is contained in:
@@ -177,6 +177,8 @@ class TestRecognizeText:
|
||||
|
||||
|
||||
_PDF_BODY = b"%PDF-1.4\n%synthetic body"
|
||||
# BMP: "BM" + 文件大小(4) + 保留(4) + 像素偏移(4) + DIB 头大小(4,常见 12/40/52/56/64/108/124)
|
||||
_BMP_HEADER = b"BM" + b"\x00" * 12 + (40).to_bytes(4, "little") + b"\x00" * 20
|
||||
_JUNK = b"x" * 200
|
||||
|
||||
|
||||
@@ -192,7 +194,11 @@ class TestSniffMedia:
|
||||
pytest.param(b"\x89PNG\r\n\x1a\n" + b"0" * 20, "image", id="png"),
|
||||
pytest.param(b"II*\x00" + b"0" * 20, "image", id="tiff-le"),
|
||||
pytest.param(b"MM\x00*" + b"0" * 20, "image", id="tiff-be"),
|
||||
pytest.param(b"BM" + b"0" * 20, "image", id="bmp"),
|
||||
pytest.param(_BMP_HEADER, "image", id="bmp"),
|
||||
pytest.param(
|
||||
b"BMW is a car maker, not a bitmap", "unknown", id="text-starting-with-BM"
|
||||
),
|
||||
pytest.param(b"BM" + b"\x00" * 40, "unknown", id="BM-with-bad-dib-header"),
|
||||
pytest.param(b"GIF89a" + b"0" * 20, "image", id="gif89a"),
|
||||
pytest.param(b"GIF87a" + b"0" * 20, "image", id="gif87a"),
|
||||
pytest.param(b"RIFF\x00\x00\x00\x00WEBPVP8 ", "image", id="webp"),
|
||||
@@ -479,7 +485,7 @@ def _parsing_failed(reason):
|
||||
class TestRejectedMarkers:
|
||||
"""#34 设计 §6: 确定性坏输入的 500 在文档路径归「请求被拒」,其余分类一概不动。
|
||||
|
||||
共享的 `_classify_status` 默认行为字节级不变——医院线对坏图的重试/熔断行为属公共行为,
|
||||
共享的 `_classify_status` 默认行为字节级不变——既有调用方对坏图的重试/熔断行为属公共行为,
|
||||
推广须另行设计(issue #35)。
|
||||
"""
|
||||
|
||||
@@ -555,7 +561,7 @@ class TestRejectedMarkers:
|
||||
assert MonkeyOcrTransport(deterministic_failure_markers=custom)._rejected_markers == custom
|
||||
|
||||
async def test_layout_and_text_paths_do_not_apply_the_markers(self):
|
||||
"""传输实例持有标记,但只有文档路径使用;医院线的两个方法行为不变。"""
|
||||
"""传输实例持有标记,但只有文档路径使用;既有的两个方法行为不变。"""
|
||||
body = _parsing_failed(_DEFAULT_REJECTED_MARKERS[0]).encode()
|
||||
t = _transport_for(
|
||||
_routes(
|
||||
@@ -569,6 +575,49 @@ class TestRejectedMarkers:
|
||||
await t.parse_layout(image=b"jpg", source=_source(), call_id="c1")
|
||||
|
||||
|
||||
class TestMalformedDownloadUrl:
|
||||
"""`download_url` 来自服务端响应,属外部输入。畸形值(控制字符、坏方括号、越界端口)会让 httpx
|
||||
抛 `InvalidURL`(不属 `HTTPError` 体系)或在连接阶段抛 ExceptionGroup,从 `_translate_http_errors`
|
||||
下漏出四分类——`OcrClient._call` 只接 `PolyGatewayError`,于是不写终态遥测行、半开探针不归还。
|
||||
(独立审查发现;`parse_layout` 的基线同样如此,共用的 `_post_parse` 一处修。)"""
|
||||
|
||||
_BAD_URLS = [
|
||||
pytest.param("http://[::1", id="broken-bracket"),
|
||||
pytest.param("/static/a\r\nb.zip", id="crlf"),
|
||||
pytest.param("/static/a\x00.zip", id="nul"),
|
||||
pytest.param("/static/a\tb.zip", id="tab"),
|
||||
pytest.param("http://127.0.0.1:99999/x", id="port-out-of-range"),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _transport_with(download_url, seen):
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append((request.method, request.url.path))
|
||||
if request.url.path == "/parse":
|
||||
return httpx.Response(200, json=_parse_body(download_url))
|
||||
return httpx.Response(200, content=b"never fetched")
|
||||
|
||||
return _transport_for(handler)
|
||||
|
||||
@pytest.mark.parametrize("download_url", _BAD_URLS)
|
||||
@pytest.mark.parametrize("method", ["parse_layout", "parse_document"])
|
||||
async def test_rejected_as_transient_without_fetching(self, download_url, method):
|
||||
seen = []
|
||||
t = self._transport_with(download_url, seen)
|
||||
kwargs = {"image": b"jpg"} if method == "parse_layout" else {"document": _PDF_BODY}
|
||||
with pytest.raises(TransientError, match="download_url"):
|
||||
await getattr(t, method)(**kwargs, source=_source(), call_id="c1")
|
||||
assert seen == [("POST", "/parse")] # 从未发出第二段请求
|
||||
|
||||
async def test_well_formed_urls_are_untouched(self):
|
||||
for url in ("/static/ok.zip", "http://10.77.0.20:7866/static/abs.zip"):
|
||||
seen = []
|
||||
t = self._transport_with(url, seen)
|
||||
with pytest.raises(ResultInvalidError): # 响应体不是 ZIP: 说明确实走到了第二段
|
||||
await t.parse_layout(image=b"jpg", source=_source(), call_id="c1")
|
||||
assert [m for m, _ in seen] == ["POST", "GET"]
|
||||
|
||||
|
||||
class TestParseDocument:
|
||||
"""#34 设计 §5: 本地预检 → POST /parse → 流式 GET(体积上限)→ 线程内解析。"""
|
||||
|
||||
@@ -693,6 +742,26 @@ class TestParseDocument:
|
||||
await t.parse_document(document=_PDF_BODY, source=_source(), call_id="c1")
|
||||
assert ei.value.status_code == status and ei.value.body_text == body
|
||||
|
||||
async def test_huge_error_body_is_read_only_up_to_a_bound(self):
|
||||
"""摘要只需要几 KiB;服务(或中间人)回一个巨大的错误体不该被整个读进内存。
|
||||
|
||||
(独立审查实测:600 MiB 的 500 响应让进程 RSS 涨到 1.8 GiB——新代码声称"流式 + 体积上限",
|
||||
错误分支却是例外。)
|
||||
"""
|
||||
consumed = []
|
||||
|
||||
async def endless_error_body():
|
||||
for _ in range(100_000):
|
||||
consumed.append(1)
|
||||
yield b"e" * 1024
|
||||
|
||||
t = self._transport(zip_resp=httpx.Response(500, content=endless_error_body()))
|
||||
with pytest.raises(TransientError) as ei:
|
||||
await t.parse_document(document=_PDF_BODY, source=_source(), call_id="c1")
|
||||
assert ei.value.status_code == 500
|
||||
assert len(consumed) <= monkey_ocr._ERROR_BODY_READ_CAP // 1024 + 2 # 只读了上限那么多
|
||||
assert ei.value.body_text.startswith("e" * 100) # 摘要仍在
|
||||
|
||||
async def test_error_status_with_oversized_content_length_is_still_classified_by_http(
|
||||
self, monkeypatch
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user