557 lines
18 KiB
Python
557 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
from importlib import import_module
|
|
from typing import TYPE_CHECKING, cast
|
|
|
|
import pytest
|
|
|
|
from mdpolish import DocumentSnapshot, ModifierContractError, Pipeline, RunStatus
|
|
from mdpolish.modifiers import mapped_line_join
|
|
from mdpolish.modifiers.mapped_line_join import (
|
|
ExactLineJoinRule,
|
|
KeepLineJoinRule,
|
|
LexicalAmbiguityPolicy,
|
|
LexicalCandidateForm,
|
|
LexicalLineJoinRule,
|
|
LexiconBackend,
|
|
LineBreakPolicy,
|
|
LineEndRegexRule,
|
|
LineJoinBlock,
|
|
LineJoinConflictPolicy,
|
|
LineJoinRule,
|
|
RegexLineJoinRule,
|
|
UnicodeNormalization,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from _pytest.monkeypatch import MonkeyPatch
|
|
|
|
mapped_module = import_module("mdpolish.modifiers.mapped_line_join")
|
|
|
|
LEGACY_MAPPINGS = (
|
|
("exam-", "ple", "example"),
|
|
("rule-", "based", "rule-based"),
|
|
("value.", "ues", "values"),
|
|
)
|
|
|
|
|
|
def transform(
|
|
markdown: str,
|
|
rules: tuple[tuple[str, str, str] | LineJoinRule, ...] = LEGACY_MAPPINGS,
|
|
**options: object,
|
|
) -> str:
|
|
modifier = mapped_line_join(rules, **options) # type: ignore[arg-type]
|
|
result = Pipeline((modifier,)).transform(markdown)
|
|
assert result.status is RunStatus.SUCCESS
|
|
assert result.output_markdown is not None
|
|
return result.output_markdown
|
|
|
|
|
|
def exact_rule(**overrides: object) -> ExactLineJoinRule:
|
|
fields: dict[str, object] = {
|
|
"rule_id": "example.join",
|
|
"left": "exam",
|
|
"right": "ple",
|
|
"replacement": "example",
|
|
"separator": "-",
|
|
}
|
|
fields.update(overrides)
|
|
return ExactLineJoinRule(**fields) # type: ignore[arg-type]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("markdown", "expected"),
|
|
[
|
|
("an exam-\nple here", "an example here"),
|
|
("an exam-\n\nple here", "an example here"),
|
|
("an exam-\r\n\r\nple here", "an example here"),
|
|
("an exam-\r\rple here", "an example here"),
|
|
("a rule-\nbased method", "a rule-based method"),
|
|
("the value.\nues differ", "the values differ"),
|
|
],
|
|
)
|
|
def test_legacy_exact_mapping_supports_lf_crlf_cr_and_one_blank_line(markdown: str, expected: str) -> None:
|
|
assert transform(markdown) == expected
|
|
|
|
|
|
def test_exact_rules_distinguish_explicit_and_empty_separator() -> None:
|
|
hyphenated = exact_rule()
|
|
unseparated = exact_rule(rule_id="example.unseparated", separator="")
|
|
|
|
assert transform("exam-\nple", (hyphenated,)) == "example"
|
|
assert transform("exam\nple", (unseparated,)) == "example"
|
|
assert transform("exam\nple", (hyphenated,)) == "exam\nple"
|
|
|
|
|
|
def test_named_regex_combines_backreferences_from_both_sides() -> None:
|
|
rule = RegexLineJoinRule(
|
|
rule_id="regex.join",
|
|
left_pattern=r"(?P<stem>[A-Za-z]+)",
|
|
right_pattern=r"(?P<suffix>[a-z]+)",
|
|
replacement=r"\g<stem>_\g<suffix>",
|
|
separator="-",
|
|
)
|
|
|
|
assert transform("prefix exam-\nple suffix", (rule,)) == "prefix exam_ple suffix"
|
|
|
|
|
|
def test_regex_keeps_its_inline_flags() -> None:
|
|
rule = RegexLineJoinRule(
|
|
rule_id="regex.flags",
|
|
left_pattern=r"(?i:(?P<stem>exam))",
|
|
right_pattern=r"(?i:(?P<suffix>ple))",
|
|
replacement=r"\g<stem>\g<suffix>",
|
|
separator="-",
|
|
)
|
|
|
|
assert transform("EXAM-\nPLE", (rule,)) == "EXAMPLE"
|
|
|
|
|
|
def test_regex_case_insensitive_option_applies_to_the_pattern() -> None:
|
|
rule = RegexLineJoinRule(
|
|
rule_id="regex.ignore-case",
|
|
left_pattern=r"(?P<stem>EXAM)",
|
|
right_pattern=r"(?P<suffix>PLE)",
|
|
replacement=r"\g<stem>\g<suffix>",
|
|
separator="-",
|
|
case_sensitive=False,
|
|
)
|
|
|
|
assert transform("exam-\nple", (rule,)) == "example"
|
|
|
|
|
|
def test_regex_suffix_matching_considers_overlapping_starts() -> None:
|
|
rule = RegexLineJoinRule(
|
|
rule_id="regex.overlap",
|
|
left_pattern=r"(?P<stem>aba)",
|
|
right_pattern=r"(?P<suffix>x)",
|
|
replacement=r"\g<stem>\g<suffix>",
|
|
separator="-",
|
|
)
|
|
|
|
assert transform("ababa-\nx", (rule,)) == "ababax"
|
|
|
|
|
|
def test_line_end_regex_uses_right_match_only_as_a_condition() -> None:
|
|
rule = LineEndRegexRule(
|
|
rule_id="line-end.space",
|
|
right_pattern=r"(?P<initial>[a-z])",
|
|
line_break=LineBreakPolicy.SPACE,
|
|
)
|
|
|
|
assert transform("foo\nbar", (rule,)) == "foo bar"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("policy", "expected"),
|
|
[
|
|
(LineBreakPolicy.PRESERVE, "example\n"),
|
|
(LineBreakPolicy.DELETE, "example"),
|
|
(LineBreakPolicy.SPACE, "example "),
|
|
(LineBreakPolicy.PARAGRAPH, "example\n\n"),
|
|
],
|
|
)
|
|
def test_all_line_break_policies_are_exact(policy: LineBreakPolicy, expected: str) -> None:
|
|
assert transform("exam-\nple", (exact_rule(line_break=policy),)) == expected
|
|
|
|
|
|
def test_paragraph_policy_restores_quote_and_list_prefixes() -> None:
|
|
rule = exact_rule(line_break=LineBreakPolicy.PARAGRAPH)
|
|
|
|
assert transform("> exam-\n> ple", (rule,)) == "> example\n>\n> "
|
|
assert transform("- exam-\n ple", (rule,)) == "- example\n\n "
|
|
assert transform("> - exam-\n> ple", (rule,)) == "> - example\n>\n> "
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("markdown", "expected"),
|
|
[
|
|
("plain exam-\nple", "plain example"),
|
|
("# exam-\nple", "# example"),
|
|
("- exam-\n ple", "- example"),
|
|
("> exam-\n> ple", "> example"),
|
|
("> > > exam-\n> > > ple", "> > > example"),
|
|
("1. outer\n 2. exam-\n ple", "1. outer\n 2. example"),
|
|
],
|
|
)
|
|
def test_supported_markdown_blocks_and_valid_nested_containers(markdown: str, expected: str) -> None:
|
|
assert transform(markdown, (exact_rule(),)) == expected
|
|
|
|
|
|
def test_block_scope_is_explicit() -> None:
|
|
heading_only = exact_rule(blocks=frozenset({LineJoinBlock.HEADING}))
|
|
|
|
assert transform("# exam-\nple", (heading_only,)) == "# example"
|
|
assert transform("exam-\nple", (heading_only,)) == "exam-\nple"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"markdown",
|
|
[
|
|
"> > > exam-\n> > ple",
|
|
"> #broken exam-\n> ple",
|
|
"1. exam-\n2. ple",
|
|
"1. outer\n 2. exam-\n ple",
|
|
"- exam-\n ple",
|
|
"#broken exam-\nple",
|
|
],
|
|
)
|
|
def test_unknown_or_mismatched_containers_fail_closed_instead_of_becoming_paragraphs(markdown: str) -> None:
|
|
assert transform(markdown, (exact_rule(),)) == markdown
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"markdown",
|
|
[
|
|
"```text\nexam-\nple\n```",
|
|
" exam-\n ple",
|
|
"> exam-\n> ple",
|
|
"- ```\n exam-\n ple\n ```",
|
|
"| word |\n| --- |\n| exam- |\n| ple |",
|
|
"<table>\nexam-\nple\n</table>",
|
|
"<table>\nexam-\nple",
|
|
],
|
|
)
|
|
def test_code_and_table_regions_are_always_excluded(markdown: str) -> None:
|
|
assert transform(markdown, (exact_rule(),)) == markdown
|
|
|
|
|
|
def test_three_line_chain_is_one_proposal_and_uses_virtual_output() -> None:
|
|
rules = (
|
|
exact_rule(rule_id="chain.first"),
|
|
ExactLineJoinRule(
|
|
rule_id="chain.second",
|
|
left="example",
|
|
right="based",
|
|
replacement="example-based",
|
|
separator="-",
|
|
),
|
|
)
|
|
modifier = mapped_line_join(rules)
|
|
proposals = modifier.propose(DocumentSnapshot("exam-\nple-\nbased"))
|
|
|
|
assert len(proposals) == 1
|
|
assert len(proposals[0].edits) == 1
|
|
assert "chain.first, chain.second" in proposals[0].reason
|
|
assert transform("exam-\nple-\nbased", rules) == "example-based"
|
|
|
|
|
|
def test_mixed_line_endings_fail_the_entire_connected_chain_closed() -> None:
|
|
rules = (
|
|
exact_rule(rule_id="chain.first"),
|
|
ExactLineJoinRule("chain.second", "example", "based", "example-based", separator="-"),
|
|
)
|
|
markdown = "exam-\r\nple-\nbased"
|
|
|
|
assert transform(markdown, rules) == markdown
|
|
|
|
|
|
def test_multiple_independent_chains_are_reported_in_source_order() -> None:
|
|
modifier = mapped_line_join((exact_rule(),))
|
|
snapshot = DocumentSnapshot("exam-\nple and\n\nexam-\nple")
|
|
proposals = modifier.propose(snapshot)
|
|
|
|
assert len(proposals) == 2
|
|
assert proposals[0].edits[0].span.start < proposals[1].edits[0].span.start
|
|
|
|
|
|
def test_keep_rule_can_veto_a_lower_priority_replacement() -> None:
|
|
keep = KeepLineJoinRule("keep.example", "exam", "ple", separator="-", priority=10)
|
|
replace = exact_rule(priority=0)
|
|
|
|
assert transform("exam-\nple", (replace, keep)) == "exam-\nple"
|
|
|
|
|
|
def test_conflict_first_and_priority_are_deterministic() -> None:
|
|
first = exact_rule(rule_id="first", replacement="first", priority=0)
|
|
second = exact_rule(rule_id="second", replacement="second", priority=10)
|
|
|
|
assert transform(
|
|
"exam-\nple",
|
|
(first, second),
|
|
conflict_policy=LineJoinConflictPolicy.FIRST,
|
|
) == "first"
|
|
assert transform(
|
|
"exam-\nple",
|
|
(first, second),
|
|
conflict_policy=LineJoinConflictPolicy.PRIORITY,
|
|
) == "second"
|
|
|
|
|
|
def test_priority_tie_uses_declaration_order() -> None:
|
|
first = exact_rule(rule_id="first", replacement="first")
|
|
second = exact_rule(rule_id="second", replacement="second")
|
|
|
|
assert transform("exam-\nple", (second, first)) == "second"
|
|
|
|
|
|
def test_conflict_error_names_rules_but_not_surrounding_text() -> None:
|
|
modifier = mapped_line_join(
|
|
(exact_rule(rule_id="first"), exact_rule(rule_id="second")),
|
|
conflict_policy=LineJoinConflictPolicy.ERROR,
|
|
)
|
|
|
|
with pytest.raises(ModifierContractError, match=r"first, second") as raised:
|
|
modifier.propose(DocumentSnapshot("secret exam-\nple material"))
|
|
assert "secret" not in str(raised.value)
|
|
|
|
|
|
def test_case_insensitive_matching_is_opt_in() -> None:
|
|
assert transform("EXAM-\nPLE", (exact_rule(),)) == "EXAM-\nPLE"
|
|
assert transform("EXAM-\nPLE", (exact_rule(case_sensitive=False),)) == "example"
|
|
|
|
|
|
def test_unicode_normalization_is_opt_in_and_preserves_source_index_mapping() -> None:
|
|
decomposed_left = "cafe\N{COMBINING ACUTE ACCENT}"
|
|
markdown = "CAFÉ-\nteria"
|
|
default = ExactLineJoinRule(
|
|
"unicode.default",
|
|
decomposed_left,
|
|
"teria",
|
|
"cafeteria",
|
|
separator="-",
|
|
case_sensitive=False,
|
|
)
|
|
normalized = ExactLineJoinRule(
|
|
"unicode.nfc",
|
|
decomposed_left,
|
|
"teria",
|
|
"cafeteria",
|
|
separator="-",
|
|
case_sensitive=False,
|
|
normalization=UnicodeNormalization.NFC,
|
|
)
|
|
|
|
assert transform(markdown, (default,)) == markdown
|
|
assert transform(markdown, (normalized,)) == "cafeteria"
|
|
|
|
|
|
def test_unicode_word_boundaries_and_explicit_override() -> None:
|
|
strict = exact_rule()
|
|
permissive = exact_rule(rule_id="permissive", require_word_boundaries=False)
|
|
markdown = "éexam-\nplemore"
|
|
|
|
assert transform(markdown, (strict,)) == markdown
|
|
assert transform(markdown, (permissive,)) == "éexamplemore"
|
|
|
|
|
|
@pytest.mark.parametrize("max_blank_lines", [0, 1])
|
|
def test_blank_line_limit_is_configurable(max_blank_lines: int) -> None:
|
|
expected = "exam-\n\nple" if max_blank_lines == 0 else "example"
|
|
assert transform(
|
|
"exam-\n\nple",
|
|
(exact_rule(),),
|
|
max_intervening_blank_lines=max_blank_lines,
|
|
) == expected
|
|
|
|
|
|
@pytest.mark.parametrize("invalid", [-1, True, 2])
|
|
def test_invalid_blank_line_limits_are_rejected(invalid: object) -> None:
|
|
with pytest.raises(ModifierContractError):
|
|
mapped_line_join((), max_intervening_blank_lines=invalid) # type: ignore[arg-type]
|
|
|
|
|
|
def test_empty_document_and_empty_rule_set_are_no_ops() -> None:
|
|
modifier = mapped_line_join(())
|
|
|
|
assert transform("", ()) == ""
|
|
assert transform("exam-\nple", ()) == "exam-\nple"
|
|
assert modifier.version == "2.0.0"
|
|
assert dict(modifier.parameters)["rules"] == ()
|
|
|
|
|
|
def test_parameters_preserve_rule_order_and_record_all_options() -> None:
|
|
first = mapped_line_join((exact_rule(rule_id="first"), exact_rule(rule_id="second")))
|
|
second = mapped_line_join((exact_rule(rule_id="second"), exact_rule(rule_id="first")))
|
|
|
|
assert first.parameters != second.parameters
|
|
records = dict(first.parameters)["rules"]
|
|
assert isinstance(records, tuple)
|
|
first_record = cast(tuple[tuple[str, object], ...], records[0])
|
|
assert ("rule_id", "first") in first_record
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"rules",
|
|
[
|
|
(("", "right", "word"),),
|
|
(("left", "right", "two words"),),
|
|
(("left", "right", "word"), ("left", "right", "other")),
|
|
(("left", "right"),),
|
|
(exact_rule(rule_id="same"), exact_rule(rule_id="same")),
|
|
(exact_rule(case_sensitive=1),),
|
|
(exact_rule(blocks={LineJoinBlock.PARAGRAPH}),),
|
|
(RegexLineJoinRule("regex", "", r"(?P<right>x)", "x"),),
|
|
(RegexLineJoinRule("regex", r"(x)", r"(?P<right>x)", "x"),),
|
|
(RegexLineJoinRule("regex", r"(?P<x>x)", r"(?P<x>x)", r"\g<x>"),),
|
|
(RegexLineJoinRule("regex", r"(?P<x>x)", r"(?P<y>x)", r"\g<missing>"),),
|
|
(LineEndRegexRule("line-end", r"(?P<x>x)", line_break=LineBreakPolicy.PRESERVE),),
|
|
],
|
|
)
|
|
def test_invalid_rules_raise_contract_error(rules: object) -> None:
|
|
with pytest.raises(ModifierContractError):
|
|
mapped_line_join(rules) # type: ignore[arg-type]
|
|
|
|
|
|
class FakeLexicon:
|
|
package_version = "test-lexicon"
|
|
|
|
def __init__(self, scores: dict[str, tuple[float, float | None]]) -> None:
|
|
self._scores = scores
|
|
|
|
def evidence(
|
|
self,
|
|
candidate: str,
|
|
form: LexicalCandidateForm,
|
|
left: str,
|
|
right: str,
|
|
) -> object:
|
|
del form, left, right
|
|
score = self._scores.get(candidate)
|
|
return None if score is None else mapped_module._LexicalEvidence(*score)
|
|
|
|
|
|
def lexical_rule(**overrides: object) -> LexicalLineJoinRule:
|
|
fields: dict[str, object] = {
|
|
"rule_id": "english.lexical",
|
|
"left_pattern": r"[A-Za-z]+",
|
|
"right_pattern": r"[a-z]+",
|
|
"separator": "-",
|
|
"backend": LexiconBackend.WORDFREQ,
|
|
"language": "en",
|
|
"candidate_forms": (LexicalCandidateForm.JOINED, LexicalCandidateForm.HYPHENATED),
|
|
"minimum_score": 1.0,
|
|
"minimum_score_margin": 0.5,
|
|
}
|
|
fields.update(overrides)
|
|
return LexicalLineJoinRule(**fields) # type: ignore[arg-type]
|
|
|
|
|
|
def install_fake_lexicon(monkeypatch: MonkeyPatch, scores: dict[str, tuple[float, float | None]]) -> None:
|
|
def build(rule: LexicalLineJoinRule) -> FakeLexicon:
|
|
del rule
|
|
return FakeLexicon(scores)
|
|
|
|
monkeypatch.setattr(mapped_module, "_build_lexicon", build)
|
|
|
|
|
|
def test_lexical_rule_joins_a_unique_dictionary_candidate_without_per_word_mapping(
|
|
monkeypatch: MonkeyPatch,
|
|
) -> None:
|
|
install_fake_lexicon(monkeypatch, {"example": (5.0, 5.0)})
|
|
|
|
assert transform("exam-\nple", (lexical_rule(),)) == "example"
|
|
|
|
|
|
def test_lexical_rule_can_select_the_natural_hyphenated_form_by_score(monkeypatch: MonkeyPatch) -> None:
|
|
install_fake_lexicon(monkeypatch, {"rulebased": (3.0, 3.0), "rule-based": (6.0, 6.0)})
|
|
|
|
assert transform("rule-\nbased", (lexical_rule(),)) == "rule-based"
|
|
|
|
|
|
def test_lexical_margin_and_keep_policy_preserve_an_ambiguous_boundary(monkeypatch: MonkeyPatch) -> None:
|
|
install_fake_lexicon(monkeypatch, {"rulebased": (5.0, 5.0), "rule-based": (4.8, 4.8)})
|
|
|
|
assert transform("rule-\nbased", (lexical_rule(minimum_score_margin=0.5),)) == "rule-\nbased"
|
|
|
|
|
|
def test_spellchecker_style_spaced_proxy_is_not_ranked_against_single_token(monkeypatch: MonkeyPatch) -> None:
|
|
install_fake_lexicon(monkeypatch, {"inside": (6.0, 6.0), "in side": (5.0, None)})
|
|
rule = lexical_rule(
|
|
separator="",
|
|
candidate_forms=(LexicalCandidateForm.JOINED, LexicalCandidateForm.SPACED),
|
|
)
|
|
|
|
assert transform("in\nside", (rule,)) == "in\nside"
|
|
|
|
|
|
def test_lexical_ambiguity_error_is_source_safe(monkeypatch: MonkeyPatch) -> None:
|
|
install_fake_lexicon(monkeypatch, {"inside": (6.0, 6.0), "in side": (5.0, None)})
|
|
rule = lexical_rule(
|
|
separator="",
|
|
candidate_forms=(LexicalCandidateForm.JOINED, LexicalCandidateForm.SPACED),
|
|
ambiguity=LexicalAmbiguityPolicy.ERROR,
|
|
)
|
|
modifier = mapped_line_join((rule,))
|
|
|
|
with pytest.raises(ModifierContractError, match=r"english\.lexical") as raised:
|
|
modifier.propose(DocumentSnapshot("secret in\nside"))
|
|
assert "secret" not in str(raised.value)
|
|
|
|
|
|
class FakeHyphenator:
|
|
def __init__(self, positions: tuple[int, ...]) -> None:
|
|
self._positions = positions
|
|
|
|
def positions(self, word: str) -> tuple[int, ...]:
|
|
del word
|
|
return self._positions
|
|
|
|
|
|
@pytest.mark.parametrize(("positions", "expected"), [((4,), "example"), ((), "exam-ple")])
|
|
def test_pyphen_gate_only_controls_the_joined_candidate(
|
|
monkeypatch: MonkeyPatch,
|
|
positions: tuple[int, ...],
|
|
expected: str,
|
|
) -> None:
|
|
install_fake_lexicon(monkeypatch, {"example": (6.0, 6.0), "exam-ple": (4.0, 4.0)})
|
|
monkeypatch.setattr(
|
|
mapped_module,
|
|
"_build_hyphenator",
|
|
lambda language: (FakeHyphenator(positions), f"test-{language}"),
|
|
)
|
|
rule = lexical_rule(hyphenation_language="en_US", minimum_score_margin=0.5)
|
|
|
|
assert transform("exam-\nple", (rule,)) == expected
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"overrides",
|
|
[
|
|
{"separator": "/"},
|
|
{"candidate_forms": (LexicalCandidateForm.JOINED,)},
|
|
{"candidate_forms": (LexicalCandidateForm.JOINED, LexicalCandidateForm.JOINED)},
|
|
{"minimum_score": -1.0},
|
|
{"minimum_score": float("nan")},
|
|
{"minimum_score_margin": True},
|
|
{"language": ""},
|
|
{"hyphenation_language": ""},
|
|
],
|
|
)
|
|
def test_invalid_lexical_rules_fail_before_loading_a_backend(overrides: dict[str, object]) -> None:
|
|
with pytest.raises(ModifierContractError):
|
|
mapped_line_join((lexical_rule(**overrides),))
|
|
|
|
|
|
def test_missing_requested_optional_backend_raises_without_fallback(monkeypatch: MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mapped_module, "_zipf_frequency", None)
|
|
|
|
with pytest.raises(ModifierContractError, match="frequency"):
|
|
mapped_line_join((lexical_rule(),))
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("backend", "available"),
|
|
[
|
|
(LexiconBackend.SPELLCHECKER, mapped_module._SpellChecker is not None),
|
|
(LexiconBackend.WORDFREQ, mapped_module._zipf_frequency is not None),
|
|
],
|
|
)
|
|
def test_installed_optional_backend_constructs_or_is_reported_as_skipped(
|
|
backend: LexiconBackend,
|
|
available: bool,
|
|
) -> None:
|
|
if not available:
|
|
pytest.skip(f"optional backend is not installed: {backend.value}")
|
|
mapped_line_join((lexical_rule(backend=backend),))
|
|
|
|
|
|
def test_successful_output_is_idempotent() -> None:
|
|
rules = (exact_rule(),)
|
|
pipeline = Pipeline((mapped_line_join(rules),))
|
|
first = pipeline.transform("an exam-\n\nple here")
|
|
|
|
assert first.status is RunStatus.SUCCESS
|
|
assert first.output_markdown is not None
|
|
assert pipeline.transform(first.output_markdown).changes == ()
|