fix: close independent verification findings on document parsing
Zip construction and member reads now share one narrow boundary so corrupt BZIP2, LZMA and version fields cannot leak native exceptions. Non-success statuses are read before classification, text-bearing blocks require lines, the size cap and reasoning contract are pinned by tests that fail under mutation, and the per-page timing claim is corrected to the observed range.
This commit is contained in:
@@ -11,6 +11,7 @@ table/image 的嵌套 `blocks`、span 字段),一切识别文本替换为 `SYNTH
|
||||
|
||||
import io
|
||||
import json
|
||||
import random
|
||||
import struct
|
||||
import zipfile
|
||||
|
||||
@@ -287,7 +288,7 @@ class TestPageStructure:
|
||||
pytest.param("missing", id="missing"),
|
||||
pytest.param("0", id="string"),
|
||||
pytest.param(0.0, id="float"),
|
||||
pytest.param(True, id="bool"),
|
||||
pytest.param(True, id="bool-true-off-position"),
|
||||
pytest.param(-1, id="negative"),
|
||||
pytest.param(3, id="out-of-position"),
|
||||
],
|
||||
@@ -301,6 +302,20 @@ class TestPageStructure:
|
||||
page["page_idx"] = page_idx
|
||||
_invalid([page])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pages",
|
||||
[
|
||||
pytest.param([_page([], page_idx=False)], id="false-equals-0-on-first-page"),
|
||||
pytest.param(
|
||||
[_page([], page_idx=0), _page([], page_idx=True)], id="true-equals-1-on-second-page"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_bool_page_idx_is_rejected_even_when_it_equals_the_position(self, pages):
|
||||
"""`False == 0`、`True == 1`: 没有显式的 bool 守卫时它们恰好等于页序而被放行。"""
|
||||
with pytest.raises(ResultInvalidError, match="page_idx"):
|
||||
_parse(pages)
|
||||
|
||||
def test_second_page_must_not_claim_the_first_index(self):
|
||||
_invalid([_page([], page_idx=0), _page([], page_idx=0)])
|
||||
|
||||
@@ -467,12 +482,15 @@ class TestExternalDataExceptionBoundary:
|
||||
deep = b'{"pdf_info": ' + b"[" * 100_000 + b"]" * 100_000 + b"}"
|
||||
_invalid(middle=deep)
|
||||
|
||||
@pytest.mark.parametrize("member", ["middle", "markdown"])
|
||||
def test_declared_size_over_cap_is_rejected(self, member, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
_monkey_document, "MAX_MIDDLE_BYTES" if member == "middle" else "MAX_MARKDOWN_BYTES", 8
|
||||
)
|
||||
_invalid([_page([_text_block("A")])], md="# a markdown longer than eight bytes")
|
||||
@pytest.mark.parametrize(
|
||||
("cap_name", "member"),
|
||||
[("MAX_MIDDLE_BYTES", "doc_middle.json"), ("MAX_MARKDOWN_BYTES", "doc.md")],
|
||||
)
|
||||
def test_declared_size_over_cap_names_the_offending_member(self, cap_name, member, monkeypatch):
|
||||
"""断言消息里的成员名: 两个上限互换时,只断言异常类型的旧用例仍会通过。"""
|
||||
monkeypatch.setattr(_monkey_document, cap_name, 8)
|
||||
with pytest.raises(ResultInvalidError, match=member):
|
||||
_parse([_page([_text_block("A")])], md="# a markdown longer than eight bytes")
|
||||
|
||||
def test_existing_result_invalid_messages_pass_through_unchanged(self):
|
||||
with pytest.raises(ResultInvalidError, match="至少应有 1 页"):
|
||||
@@ -499,3 +517,74 @@ class TestLayoutPathUntouched:
|
||||
assert accepted[0].page_index == 7
|
||||
fallback, _ = _parse_middle_json(_package([_page([_text_block()], page_idx="x")]))
|
||||
assert fallback[0].page_index == 0
|
||||
|
||||
|
||||
class TestZipLayerNeverLeaksNativeExceptions:
|
||||
"""独立验证发现: BZIP2/LZMA 成员数据损坏会漏出 `OSError`/`LZMAError`,中央目录版本字段损坏会在
|
||||
`ZipFile` 构造阶段漏出 `NotImplementedError`——它们都不在外层边界里。漏出的后果与设计 §5 一致:
|
||||
`OcrClient._call` 只接 `PolyGatewayError`,既不写终态遥测行,也不归还半开探针。"""
|
||||
|
||||
@staticmethod
|
||||
def _zip(method: int) -> bytes:
|
||||
payload = json.dumps({"pdf_info": [_page([_text_block("SYNTH")])]}).encode() * 20
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w", compression=method) as zf:
|
||||
zf.writestr("doc_middle.json", payload)
|
||||
zf.writestr("doc.md", "# synth " * 50)
|
||||
return buf.getvalue()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"method",
|
||||
[
|
||||
pytest.param(zipfile.ZIP_DEFLATED, id="deflated"),
|
||||
pytest.param(zipfile.ZIP_BZIP2, id="bzip2"),
|
||||
pytest.param(zipfile.ZIP_LZMA, id="lzma"),
|
||||
],
|
||||
)
|
||||
def test_single_bit_flips_never_escape_as_native_exceptions(self, method):
|
||||
"""确定性字节翻转(固定种子): 每个结果只能是成功或 `ResultInvalidError`。"""
|
||||
base = self._zip(method)
|
||||
rng = random.Random(34)
|
||||
for _ in range(600):
|
||||
data = bytearray(base)
|
||||
position = rng.randrange(len(data))
|
||||
data[position] ^= 1 << rng.randrange(8)
|
||||
try:
|
||||
parse_document_package(bytes(data))
|
||||
except ResultInvalidError:
|
||||
continue
|
||||
except Exception as exc: # noqa: BLE001 — 测试要抓的正是任何漏出的原生异常
|
||||
pytest.fail(
|
||||
f"字节 {position} 翻转后漏出 {type(exc).__module__}.{type(exc).__name__}: {exc}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("method", [zipfile.ZIP_BZIP2, zipfile.ZIP_LZMA])
|
||||
def test_corrupt_non_deflate_member_data(self, method):
|
||||
"""压缩流内部损坏(而非 CRC 失败): 标准库抛 OSError / LZMAError。"""
|
||||
data = bytearray(self._zip(method))
|
||||
local = data.index(b"PK\x03\x04")
|
||||
name_len, extra_len = struct.unpack_from("<HH", data, local + 26)
|
||||
body = local + 30 + name_len + extra_len
|
||||
for offset in range(body, body + 12):
|
||||
data[offset] ^= 0xFF
|
||||
with pytest.raises(ResultInvalidError):
|
||||
parse_document_package(bytes(data))
|
||||
|
||||
def test_zipfile_construction_stage_is_inside_the_boundary(self):
|
||||
"""中央目录的 extract_version 损坏: `ZipFile(...)` 构造阶段抛 NotImplementedError。"""
|
||||
data = bytearray(self._zip(zipfile.ZIP_STORED))
|
||||
central = data.index(b"PK\x01\x02")
|
||||
data[central + 6] = 99 # extract_version = 9.9 → "zip file version 9.9"
|
||||
with pytest.raises(ResultInvalidError):
|
||||
parse_document_package(bytes(data))
|
||||
|
||||
|
||||
class TestTextBearingBlocksRequireLines:
|
||||
"""text/title/interline_equation 缺 `lines` 不是「空文本」而是畸形: 与 table 缺内容就报错同口径,
|
||||
否则下游分不清「服务识别出空块」和「服务给了坏结构」。"""
|
||||
|
||||
@pytest.mark.parametrize("block_type", ["text", "title", "interline_equation"])
|
||||
def test_missing_lines_is_rejected_with_its_reason(self, block_type):
|
||||
block = {"type": block_type, "bbox": list(_BBOX)}
|
||||
with pytest.raises(ResultInvalidError, match="缺少 lines"):
|
||||
_parse([_page([block])])
|
||||
|
||||
@@ -704,27 +704,83 @@ class TestParseDocument:
|
||||
document=_PDF_BODY, source=_source(), call_id="c1"
|
||||
)
|
||||
|
||||
async def test_oversized_declared_content_length_is_rejected(self, monkeypatch):
|
||||
monkeypatch.setattr(monkey_ocr, "_MAX_PACKAGE_BYTES", 100)
|
||||
resp = httpx.Response(200, content=b"x" * 10, headers={"content-length": "101"})
|
||||
with pytest.raises(ResultInvalidError):
|
||||
@staticmethod
|
||||
def _oversized_valid_package():
|
||||
"""**合法**的 ZIP 但整体超过上限: 去掉上限检查时它能被正常解析,用例才会转红。
|
||||
|
||||
(独立验证发现: 旧用例用非 ZIP 垃圾字节,去掉上限后同样抛 ResultInvalidError——
|
||||
走的是 BadZipFile 路径,上限从未被测到。)
|
||||
"""
|
||||
return _doc_package(
|
||||
[_doc_page([_text_block("SYNTH-A")])], extra=[("images/big.jpg", b"x" * 5000)]
|
||||
)
|
||||
|
||||
async def _assert_cap_rejects_before_parsing(self, monkeypatch, resp):
|
||||
monkeypatch.setattr(monkey_ocr, "_MAX_PACKAGE_BYTES", 2000)
|
||||
parsed = []
|
||||
|
||||
def spy(package):
|
||||
parsed.append(len(package))
|
||||
return [], [(1.0, 1.0)], ""
|
||||
|
||||
monkeypatch.setattr(monkey_ocr, "parse_document_package", spy)
|
||||
with pytest.raises(ResultInvalidError, match="体积超过上限"):
|
||||
await self._transport(zip_resp=resp).parse_document(
|
||||
document=_PDF_BODY, source=_source(), call_id="c1"
|
||||
)
|
||||
assert parsed == [] # 解析函数根本没被调用
|
||||
|
||||
async def test_oversized_streamed_body_is_rejected_by_cumulative_count(self, monkeypatch):
|
||||
"""没有 Content-Length(分块传输)时靠累计字节兜底: 多个小块合起来超限。"""
|
||||
monkeypatch.setattr(monkey_ocr, "_MAX_PACKAGE_BYTES", 100)
|
||||
async def test_oversized_valid_package_is_rejected_by_declared_content_length(
|
||||
self, monkeypatch
|
||||
):
|
||||
resp = httpx.Response(200, content=self._oversized_valid_package()) # 自动带 Content-Length
|
||||
await self._assert_cap_rejects_before_parsing(monkeypatch, resp)
|
||||
|
||||
async def test_oversized_valid_package_is_rejected_by_cumulative_count(self, monkeypatch):
|
||||
"""没有 Content-Length(分块传输)时靠累计字节兜底。"""
|
||||
package = self._oversized_valid_package()
|
||||
|
||||
async def chunks():
|
||||
for _ in range(3):
|
||||
yield b"x" * 60
|
||||
for offset in range(0, len(package), 700):
|
||||
yield package[offset : offset + 700]
|
||||
|
||||
resp = httpx.Response(200, content=chunks())
|
||||
with pytest.raises(ResultInvalidError):
|
||||
await self._transport(zip_resp=resp).parse_document(
|
||||
document=_PDF_BODY, source=_source(), call_id="c1"
|
||||
)
|
||||
await self._assert_cap_rejects_before_parsing(
|
||||
monkeypatch, httpx.Response(200, content=chunks())
|
||||
)
|
||||
|
||||
async def test_package_under_the_cap_still_parses(self, monkeypatch):
|
||||
"""对照: 上限只拦超限的,不误伤正常包。"""
|
||||
monkeypatch.setattr(monkey_ocr, "_MAX_PACKAGE_BYTES", 10_000)
|
||||
result = await self._transport(
|
||||
zip_resp=httpx.Response(200, content=self._oversized_valid_package())
|
||||
).parse_document(document=_PDF_BODY, source=_source(), call_id="c1")
|
||||
assert [e.text for e in result.elements] == ["SYNTH-A"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("status", "exc_type"),
|
||||
[
|
||||
(401, SourceDeadError),
|
||||
(404, RequestRejectedError),
|
||||
(500, TransientError),
|
||||
(302, RequestRejectedError), # 3xx 不是 `is_error`,但 raise_for_status 会抛
|
||||
],
|
||||
)
|
||||
async def test_streamed_error_body_is_read_before_classification(self, status, exc_type):
|
||||
"""真正的流式响应(异步迭代器 body)不会被预缓冲: 不先 `aread()`,摘要读到的是空串。
|
||||
|
||||
(独立验证发现: `httpx.Response(content=bytes)` 已预缓冲,删掉 `aread()` 后旧用例仍绿;
|
||||
且旧条件 `is_error` 漏掉 3xx。)
|
||||
"""
|
||||
body = '{"detail": "synthetic upstream reason"}'
|
||||
|
||||
async def streamed():
|
||||
yield body.encode()
|
||||
|
||||
t = self._transport(zip_resp=httpx.Response(status, content=streamed()))
|
||||
with pytest.raises(exc_type) as ei:
|
||||
await t.parse_document(document=_PDF_BODY, source=_source(), call_id="c1")
|
||||
assert ei.value.status_code == status
|
||||
assert ei.value.body_text == body
|
||||
|
||||
async def test_bad_package_is_result_invalid(self):
|
||||
t = self._transport(zip_resp=httpx.Response(200, content=b"not a zip"))
|
||||
|
||||
@@ -670,6 +670,15 @@ class TestParseDocument:
|
||||
assert r.markdown == "# SYNTH-MD"
|
||||
assert client._selector.outcomes == [("m1", False), ("m1", True)]
|
||||
|
||||
async def test_source_dead_switches_to_the_second_source_immediately(self):
|
||||
s1, s2 = _src(name="m1"), _src(name="m2", base_url="http://10.77.0.20:7867")
|
||||
client, _, _ = _client(
|
||||
[s1, s2], [SourceDeadError("401", status_code=401), "document"], sleep=_no_sleep
|
||||
)
|
||||
r = await client.parse_document(b"%PDF-1.4")
|
||||
assert r.source_name == "m2"
|
||||
assert client._selector.outcomes == [("m1", False), ("m2", True)]
|
||||
|
||||
async def test_result_invalid_counts_no_attempt_and_no_feed(self):
|
||||
client, _, gate = _client([_src()], [ResultInvalidError("bad package")])
|
||||
with pytest.raises(ResultInvalidError):
|
||||
@@ -742,6 +751,10 @@ class TestParseDocument:
|
||||
assert (await limiter.source_stats("m1")).inflight == 0
|
||||
|
||||
|
||||
async def _no_sleep(_delay):
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
||||
class TestNoLatencyJudgement:
|
||||
"""长文档不会被判慢(设计 §7): 库内没有任何基于延迟的熔断/选源判定。
|
||||
|
||||
@@ -893,6 +906,52 @@ class TestDocumentThroughTheRealTransport:
|
||||
assert after.allowed and not after.is_probe # 探针已按"坏结果≠坏服务"结算,没有悬挂
|
||||
assert inner._gates["m1"].fails == 0
|
||||
|
||||
@staticmethod
|
||||
def _slow_download_handler():
|
||||
async def slow_body():
|
||||
for _ in range(200):
|
||||
await asyncio.sleep(0.01)
|
||||
yield b"x" * 8
|
||||
|
||||
return _doc_handler(download=lambda: httpx.Response(200, content=slow_body()))
|
||||
|
||||
async def test_per_call_deadline_overrides_the_assembly_value(self):
|
||||
client, _, _ = _client(
|
||||
[_src(timeout_s=120.0)], [], transport=_real_transport(self._slow_download_handler())
|
||||
) # 装配值未设期限
|
||||
with pytest.raises(CallDeadlineExceeded):
|
||||
await client.parse_document(b"%PDF-1.4 synthetic", call_deadline_s=0.1)
|
||||
|
||||
async def test_deadline_while_holding_the_half_open_probe_returns_it_and_writes_a_terminal_row(
|
||||
self,
|
||||
):
|
||||
"""期限到期取消在飞请求时: 探针必须归还,且终态遥测行必须写出(遥测必录)。"""
|
||||
clock = FakeClock()
|
||||
inner = InMemoryGate(
|
||||
config=BreakerConfig(fail_threshold=3, cooldown_s=60.0, probe_ttl_s=120.0), now=clock
|
||||
)
|
||||
entry = await inner.try_enter("m1", "setup")
|
||||
await inner.record_failure(entry, "source_dead", True)
|
||||
clock.advance(61.0) # 半开: 本次调用持有探针
|
||||
gate = RecordingGate(inner)
|
||||
recorder = _MemoryRecorder()
|
||||
client, _, _ = _client(
|
||||
[_src(timeout_s=120.0)],
|
||||
[],
|
||||
now=clock,
|
||||
breaker=gate,
|
||||
telemetry=recorder,
|
||||
transport=_real_transport(self._slow_download_handler()),
|
||||
call_deadline_s=0.1,
|
||||
)
|
||||
with pytest.raises(CallDeadlineExceeded):
|
||||
await client.parse_document(b"%PDF-1.4 synthetic")
|
||||
assert gate.probe_releases >= 1
|
||||
again = await inner.try_enter("m1", "check")
|
||||
assert again.allowed and again.is_probe # 探针没有悬挂,下一位调用方可以再探
|
||||
terminals = [r for r in recorder.rows if r["event_kind"] == "terminal_failure"]
|
||||
assert len(terminals) == 1 and terminals[0]["operation"] == "parse_document"
|
||||
|
||||
async def test_sustained_slow_download_hits_the_call_deadline_not_a_transient_error(self):
|
||||
"""每个分块都在 read 超时内、总时长超过 call_deadline_s: 是期限,不是瞬时故障。"""
|
||||
closed = []
|
||||
@@ -958,10 +1017,14 @@ class TestAssembly:
|
||||
|
||||
|
||||
class TestReasonlessTelemetryContract:
|
||||
"""text/layout 两个入口分别验证错误行不受源级推理配置污染。"""
|
||||
"""text/layout/document 三个入口分别验证错误行不受源级推理配置污染。
|
||||
|
||||
独立验证发现文档入口原先不在矩阵里: 让 OCR 一律按 `reasoning_applies=True` 记账,
|
||||
15 个旧用例转红,但没有一个是 `parse_document` 的。"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"method,action", [("recognize_text", "text"), ("parse_layout", "layout")]
|
||||
"method,action",
|
||||
[("recognize_text", "text"), ("parse_layout", "layout"), ("parse_document", "document")],
|
||||
)
|
||||
@pytest.mark.parametrize("config", [{"enable_thinking": True}, {"reasoning_effort": "high"}])
|
||||
@pytest.mark.parametrize("backend", ["memory", "sqlite"])
|
||||
@@ -994,7 +1057,7 @@ class TestReasonlessTelemetryContract:
|
||||
if backend == "sqlite":
|
||||
recorder.close()
|
||||
|
||||
@pytest.mark.parametrize("method", ["recognize_text", "parse_layout"])
|
||||
@pytest.mark.parametrize("method", ["recognize_text", "parse_layout", "parse_document"])
|
||||
@pytest.mark.parametrize("exhausted", [False, True])
|
||||
async def test_failed_attempts_still_have_null_effort(self, method, exhausted):
|
||||
script = (
|
||||
|
||||
Reference in New Issue
Block a user