实现 ClinDB 第一批清洗组件

This commit is contained in:
2026-08-23 20:56:36 +08:00
parent 6fcc7d5736
commit 80001a8ab9
28 changed files with 2538 additions and 67 deletions
+85
View File
@@ -0,0 +1,85 @@
from __future__ import annotations
import pytest
from mdpolish import Pipeline, RunStatus
from mdpolish.components import ManuscriptLineNumberComponent
def _document(
*,
count: int = 20,
heading_numbers: frozenset[int] = frozenset({5, 15}),
numbers: tuple[int, ...] | None = None,
line_ending: str = "\n",
) -> str:
values = numbers if numbers is not None else tuple(range(1, count + 1))
body = [
f"## {number} Section {number}" if number in heading_numbers else f"{number} body {number}"
for number in values
]
return line_ending.join(("1 Affiliation", "2 Institute", "## Abstract", *body))
def transform(markdown: str): # type: ignore[no-untyped-def]
return Pipeline([ManuscriptLineNumberComponent()]).transform(markdown)
def test_removes_long_monotonic_sequence_but_preserves_pre_abstract_affiliations() -> None:
result = transform(_document())
assert result.status is RunStatus.SUCCESS
assert result.output_markdown is not None
assert result.output_markdown.startswith("1 Affiliation\n2 Institute\n## Abstract\nbody 1")
assert "## Section 5" in result.output_markdown
assert len(result.changes) == 20
@pytest.mark.parametrize("line_ending", ["\n", "\r\n", "\r"])
def test_preserves_all_supported_line_endings(line_ending: str) -> None:
result = transform(_document(line_ending=line_ending))
assert result.output_markdown is not None
assert result.output_markdown.count(line_ending) == _document(line_ending=line_ending).count(line_ending)
def test_allows_skipped_numbers_when_sequence_is_strictly_increasing() -> None:
numbers = tuple(range(10, 30))
result = transform(_document(numbers=numbers, heading_numbers=frozenset({14, 24})))
assert result.status is RunStatus.SUCCESS
assert len(result.changes) == 20
@pytest.mark.parametrize(
"markdown",
[
_document(count=19, heading_numbers=frozenset({5, 15})),
_document(heading_numbers=frozenset({5})),
_document(numbers=(*tuple(range(1, 20)), 10), heading_numbers=frozenset({5, 15})),
_document().replace("## Abstract", "## ABSTRACT"),
_document() + "\n## Abstract",
],
)
def test_incomplete_or_ambiguous_evidence_preserves_the_document(markdown: str) -> None:
result = transform(markdown)
assert result.output_markdown == markdown
assert result.changes == ()
def test_lists_years_and_numbers_inside_body_are_not_candidates() -> None:
markdown = _document() + "\n1. list\n1) list\n2024 report\nThe panel included 35 experts"
result = transform(markdown)
assert result.output_markdown is not None
assert result.output_markdown.endswith("1. list\n1) list\n2024 report\nThe panel included 35 experts")
assert len(result.changes) == 20
def test_successful_output_is_idempotent() -> None:
pipeline = Pipeline([ManuscriptLineNumberComponent()])
first = pipeline.transform(_document())
assert first.output_markdown is not None
second = pipeline.transform(first.output_markdown)
assert second.status is RunStatus.SUCCESS
assert second.changes == ()