feat: let document calls treat deterministic parse failures as rejected input
The service wraps every failure, including its own 400s, as a 500 with a parsing-failed detail. Only a fixed set of known bad-input markers is narrowed to a rejected request, and only when the caller passes markers; the shared default classification is unchanged.
This commit is contained in:
@@ -56,6 +56,20 @@ _IMAGE_MAGICS: tuple[bytes, ...] = (
|
|||||||
b"GIF89a",
|
b"GIF89a",
|
||||||
)
|
)
|
||||||
_PDF_HEADER = b"%PDF-"
|
_PDF_HEADER = b"%PDF-"
|
||||||
|
|
||||||
|
# 文档路径把这些「确定性坏输入」的 500 改归 RequestRejected(#34 设计 §6)。小写子串匹配,
|
||||||
|
# 取 2026-10-21 实测(截断 PDF / 伪 PDF 头 / 乱码 / 空文件 / 扩展名与内容不符)。
|
||||||
|
# **不含** `unsupported file type`: 嗅探后它只可能由服务契约变化或我方标签 bug 引起,
|
||||||
|
# 应响亮失败并计熔断。标记是 PyMuPDF 的界面文案,随服务版本漂移,故可经构造参数覆盖。
|
||||||
|
_DEFAULT_REJECTED_MARKERS: tuple[str, ...] = (
|
||||||
|
"failed to open stream",
|
||||||
|
"cannot open empty stream",
|
||||||
|
"cannot save with zero pages",
|
||||||
|
"is no pdf",
|
||||||
|
)
|
||||||
|
# 服务把一切异常(含它自己抛的 400、`Model not initialized` 这类真故障)都包成同一前缀,
|
||||||
|
# 所以前缀只是必要条件,必须同时命中具体标记
|
||||||
|
_PARSING_FAILED_PREFIX = "parsing failed:"
|
||||||
# Adobe 阅读器容忍头部前有垃圾字节(惯例而非 ISO 32000 正文);服务端实测可解析
|
# Adobe 阅读器容忍头部前有垃圾字节(惯例而非 ISO 32000 正文);服务端实测可解析
|
||||||
# 前缀 200 字节垃圾的 PDF(#34 设计 E3)
|
# 前缀 200 字节垃圾的 PDF(#34 设计 E3)
|
||||||
_PDF_HEADER_WINDOW = 1024
|
_PDF_HEADER_WINDOW = 1024
|
||||||
@@ -96,11 +110,14 @@ def _default_client_factory(source: SourceConfig) -> httpx.AsyncClient:
|
|||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def _translate_http_errors(source_name: str, operation: str) -> Iterator[None]:
|
def _translate_http_errors(
|
||||||
|
source_name: str, operation: str, *, rejected_markers: tuple[str, ...] = ()
|
||||||
|
) -> Iterator[None]:
|
||||||
"""httpx 异常 → 错误四分类(CHS `_translate_http_errors` 语义主体保真)。
|
"""httpx 异常 → 错误四分类(CHS `_translate_http_errors` 语义主体保真)。
|
||||||
|
|
||||||
429 细分(insufficient_quota/Retry-After)有意放弃——MonkeyOCR 无鉴权
|
429 细分(insufficient_quota/Retry-After)有意放弃——MonkeyOCR 无鉴权
|
||||||
无计费(设计 §10.2);`from exc` 保留 __cause__ 供 RetryMW 归类 timeout。
|
无计费(设计 §10.2);`from exc` 保留 __cause__ 供 RetryMW 归类 timeout。
|
||||||
|
`rejected_markers` 非空时启用确定性坏输入窄化(仅文档路径传入,见 `_classify_status`)。
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
yield
|
yield
|
||||||
@@ -111,7 +128,7 @@ def _translate_http_errors(source_name: str, operation: str) -> Iterator[None]:
|
|||||||
operation=operation,
|
operation=operation,
|
||||||
) from exc
|
) from exc
|
||||||
except httpx.HTTPStatusError as exc:
|
except httpx.HTTPStatusError as exc:
|
||||||
raise _classify_status(exc, source_name, operation) from exc
|
raise _classify_status(exc, source_name, operation, rejected_markers) from exc
|
||||||
except httpx.HTTPError as exc:
|
except httpx.HTTPError as exc:
|
||||||
raise TransientError(
|
raise TransientError(
|
||||||
f"{source_name} OCR {operation} 网络错误: {exc}",
|
f"{source_name} OCR {operation} 网络错误: {exc}",
|
||||||
@@ -120,16 +137,31 @@ def _translate_http_errors(source_name: str, operation: str) -> Iterator[None]:
|
|||||||
) from exc
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _is_deterministic_failure(body: str, markers: tuple[str, ...]) -> bool:
|
||||||
|
"""响应体同时含 `Parsing failed:` 前缀与任一已知确定性标记。"""
|
||||||
|
text = body.lower()
|
||||||
|
return _PARSING_FAILED_PREFIX in text and any(marker in text for marker in markers)
|
||||||
|
|
||||||
|
|
||||||
def _classify_status(
|
def _classify_status(
|
||||||
exc: httpx.HTTPStatusError, source_name: str, operation: str
|
exc: httpx.HTTPStatusError,
|
||||||
|
source_name: str,
|
||||||
|
operation: str,
|
||||||
|
rejected_markers: tuple[str, ...] = (),
|
||||||
) -> TransientError | SourceDeadError | RequestRejectedError:
|
) -> TransientError | SourceDeadError | RequestRejectedError:
|
||||||
"""HTTP 状态码 → 错误四分类,**全部分支**携带响应体摘要(issue #10)。
|
"""HTTP 状态码 → 错误四分类,**全部分支**携带响应体摘要(issue #10)。
|
||||||
|
|
||||||
分类映射本身零变更;摘要口径与 chat 侧共用同一实现,不得在此另起一份——
|
分类映射本身零变更;摘要口径与 chat 侧共用同一实现,不得在此另起一份——
|
||||||
"只有一个分支用了响应体"正是 issue #10 的成因。
|
"只有一个分支用了响应体"正是 issue #10 的成因。
|
||||||
|
|
||||||
|
唯一的可选窄化: `rejected_markers` 非空且为 500 且命中确定性坏输入标记时,归
|
||||||
|
`RequestRejectedError`(**保留 `status_code=500`**: 服务有响应,熔断按「响应即健康」
|
||||||
|
记成功且不计窗口)。默认空元组 = 既有行为字节级不变——医院线对坏图的处置属公共行为,
|
||||||
|
推广须另行设计(issue #35)。
|
||||||
"""
|
"""
|
||||||
status = exc.response.status_code
|
status = exc.response.status_code
|
||||||
summary = summarize_body(response_body(exc.response))
|
body = response_body(exc.response)
|
||||||
|
summary = summarize_body(body)
|
||||||
ctx: dict[str, Any] = {
|
ctx: dict[str, Any] = {
|
||||||
"source_name": source_name,
|
"source_name": source_name,
|
||||||
"status_code": status,
|
"status_code": status,
|
||||||
@@ -137,6 +169,8 @@ def _classify_status(
|
|||||||
"body_text": summary,
|
"body_text": summary,
|
||||||
}
|
}
|
||||||
message = compose_message(f"{source_name} OCR {operation} HTTP {status}", summary)
|
message = compose_message(f"{source_name} OCR {operation} HTTP {status}", summary)
|
||||||
|
if status == 500 and rejected_markers and _is_deterministic_failure(body, rejected_markers):
|
||||||
|
return RequestRejectedError(message, **ctx)
|
||||||
if status >= 500 or status == 429:
|
if status >= 500 or status == 429:
|
||||||
return TransientError(message, **ctx)
|
return TransientError(message, **ctx)
|
||||||
if status in (401, 403):
|
if status in (401, 403):
|
||||||
@@ -217,8 +251,11 @@ class MonkeyOcrTransport:
|
|||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
client_factory: Callable[[SourceConfig], httpx.AsyncClient] | None = None,
|
client_factory: Callable[[SourceConfig], httpx.AsyncClient] | None = None,
|
||||||
|
deterministic_failure_markers: tuple[str, ...] = _DEFAULT_REJECTED_MARKERS,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._client_factory = client_factory or _default_client_factory
|
self._client_factory = client_factory or _default_client_factory
|
||||||
|
# 仅文档路径使用(见 `_classify_status`);recognize_text / parse_layout 不传
|
||||||
|
self._rejected_markers = deterministic_failure_markers
|
||||||
self._clients: dict[str, httpx.AsyncClient] = {}
|
self._clients: dict[str, httpx.AsyncClient] = {}
|
||||||
|
|
||||||
def _client_for(self, source: SourceConfig) -> httpx.AsyncClient:
|
def _client_for(self, source: SourceConfig) -> httpx.AsyncClient:
|
||||||
|
|||||||
@@ -20,10 +20,12 @@ from polygateway.errors import (
|
|||||||
TransientError,
|
TransientError,
|
||||||
)
|
)
|
||||||
from polygateway.transports.monkey_ocr import (
|
from polygateway.transports.monkey_ocr import (
|
||||||
|
_DEFAULT_REJECTED_MARKERS,
|
||||||
MonkeyOcrTransport,
|
MonkeyOcrTransport,
|
||||||
_classify_status,
|
_classify_status,
|
||||||
_parse_middle_json,
|
_parse_middle_json,
|
||||||
_sniff_media,
|
_sniff_media,
|
||||||
|
_translate_http_errors,
|
||||||
_upload_part,
|
_upload_part,
|
||||||
)
|
)
|
||||||
from polygateway.types import SourceConfig
|
from polygateway.types import SourceConfig
|
||||||
@@ -455,6 +457,112 @@ class TestErrorTranslation:
|
|||||||
assert isinstance(ei.value.__cause__, httpx.TimeoutException)
|
assert isinstance(ei.value.__cause__, httpx.TimeoutException)
|
||||||
|
|
||||||
|
|
||||||
|
def _http_error(status, body):
|
||||||
|
request = httpx.Request("POST", "http://ocr.example/parse")
|
||||||
|
return httpx.HTTPStatusError(
|
||||||
|
str(status),
|
||||||
|
request=request,
|
||||||
|
response=httpx.Response(status, content=body.encode(), request=request),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parsing_failed(reason):
|
||||||
|
# 真实形态(2026-10-21 实测): 服务把一切异常包成 500 + {"detail": "Parsing failed: …"}
|
||||||
|
return json.dumps({"detail": f"Parsing failed: {reason}"})
|
||||||
|
|
||||||
|
|
||||||
|
class TestRejectedMarkers:
|
||||||
|
"""#34 设计 §6: 确定性坏输入的 500 在文档路径归「请求被拒」,其余分类一概不动。
|
||||||
|
|
||||||
|
共享的 `_classify_status` 默认行为字节级不变——医院线对坏图的重试/熔断行为属公共行为,
|
||||||
|
推广须另行设计(issue #35)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("marker", _DEFAULT_REJECTED_MARKERS)
|
||||||
|
def test_known_deterministic_failures_become_rejected_with_status_500(self, marker):
|
||||||
|
exc = _http_error(500, _parsing_failed(marker.capitalize()))
|
||||||
|
err = _classify_status(exc, "monkey_1", "parse", _DEFAULT_REJECTED_MARKERS)
|
||||||
|
assert isinstance(err, RequestRejectedError)
|
||||||
|
assert err.status_code == 500 # 服务有响应: 熔断按「响应即健康」记成功
|
||||||
|
assert marker.capitalize() in err.body_text # 响应体摘要仍随该分支
|
||||||
|
|
||||||
|
def test_marker_matching_is_case_insensitive(self):
|
||||||
|
exc = _http_error(500, '{"detail": "PARSING FAILED: FAILED TO OPEN STREAM"}')
|
||||||
|
err = _classify_status(exc, "monkey_1", "parse", _DEFAULT_REJECTED_MARKERS)
|
||||||
|
assert isinstance(err, RequestRejectedError)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"body",
|
||||||
|
[
|
||||||
|
pytest.param('{"detail": "Failed to open stream"}', id="no-parsing-failed-prefix"),
|
||||||
|
pytest.param(
|
||||||
|
_parsing_failed("500: Model not initialized"), id="real-fault-same-prefix"
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
_parsing_failed("400: Unsupported file type: .webp"), id="unsupported-type"
|
||||||
|
),
|
||||||
|
pytest.param(_parsing_failed("CUDA out of memory"), id="oom"),
|
||||||
|
pytest.param("", id="empty-body"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_everything_else_stays_transient(self, body):
|
||||||
|
"""`Parsing failed:` 前缀不能单独当判据: 服务把真故障也包成同一前缀。
|
||||||
|
`Unsupported file type` 有意不在白名单——嗅探后它只可能是契约变化或我方标签 bug。"""
|
||||||
|
err = _classify_status(
|
||||||
|
_http_error(500, body), "monkey_1", "parse", _DEFAULT_REJECTED_MARKERS
|
||||||
|
)
|
||||||
|
assert isinstance(err, TransientError)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("body", [_parsing_failed(m) for m in _DEFAULT_REJECTED_MARKERS])
|
||||||
|
def test_default_call_without_markers_is_unchanged(self, body):
|
||||||
|
err = _classify_status(_http_error(500, body), "monkey_1", "parse")
|
||||||
|
assert isinstance(err, TransientError)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("status", "exc_type"),
|
||||||
|
[
|
||||||
|
(429, TransientError),
|
||||||
|
(502, TransientError),
|
||||||
|
(503, TransientError),
|
||||||
|
(401, SourceDeadError),
|
||||||
|
(403, SourceDeadError),
|
||||||
|
(404, RequestRejectedError),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_other_statuses_are_never_narrowed(self, status, exc_type):
|
||||||
|
body = _parsing_failed(_DEFAULT_REJECTED_MARKERS[0])
|
||||||
|
err = _classify_status(_http_error(status, body), "m", "parse", _DEFAULT_REJECTED_MARKERS)
|
||||||
|
assert type(err) is exc_type
|
||||||
|
|
||||||
|
def test_translate_http_errors_passes_markers_through(self):
|
||||||
|
with pytest.raises(RequestRejectedError), _translate_http_errors(
|
||||||
|
"monkey_1", "parse", rejected_markers=_DEFAULT_REJECTED_MARKERS
|
||||||
|
):
|
||||||
|
raise _http_error(500, _parsing_failed(_DEFAULT_REJECTED_MARKERS[0]))
|
||||||
|
# 默认不传: 行为不变
|
||||||
|
with pytest.raises(TransientError), _translate_http_errors("monkey_1", "parse"):
|
||||||
|
raise _http_error(500, _parsing_failed(_DEFAULT_REJECTED_MARKERS[0]))
|
||||||
|
|
||||||
|
def test_markers_are_configurable_per_transport(self):
|
||||||
|
assert MonkeyOcrTransport()._rejected_markers == _DEFAULT_REJECTED_MARKERS
|
||||||
|
custom = ("synthetic marker",)
|
||||||
|
assert MonkeyOcrTransport(deterministic_failure_markers=custom)._rejected_markers == custom
|
||||||
|
|
||||||
|
async def test_layout_and_text_paths_do_not_apply_the_markers(self):
|
||||||
|
"""传输实例持有标记,但只有文档路径使用;医院线的两个方法行为不变。"""
|
||||||
|
body = _parsing_failed(_DEFAULT_REJECTED_MARKERS[0]).encode()
|
||||||
|
t = _transport_for(
|
||||||
|
_routes(
|
||||||
|
text_resp=httpx.Response(500, content=body),
|
||||||
|
parse_resp=httpx.Response(500, content=body),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
with pytest.raises(TransientError):
|
||||||
|
await t.recognize_text(image=b"jpg", source=_source(), call_id="c1")
|
||||||
|
with pytest.raises(TransientError):
|
||||||
|
await t.parse_layout(image=b"jpg", source=_source(), call_id="c1")
|
||||||
|
|
||||||
|
|
||||||
class TestCheckHealth:
|
class TestCheckHealth:
|
||||||
async def test_healthy(self):
|
async def test_healthy(self):
|
||||||
t = _transport_for(_routes(health_resp=httpx.Response(200, json={"status": "healthy"})))
|
t = _transport_for(_routes(health_resp=httpx.Response(200, json={"status": "healthy"})))
|
||||||
|
|||||||
Reference in New Issue
Block a user