from __future__ import annotations from importlib import import_module from typing import TYPE_CHECKING, cast import pytest from mdpolish import DocumentSnapshot, ModifierContractError, Pipeline, RunStatus from mdpolish.modifiers import mapped_line_join from mdpolish.modifiers.mapped_line_join import ( ExactLineJoinRule, KeepLineJoinRule, LexicalAmbiguityPolicy, LexicalCandidateForm, LexicalLineJoinRule, LexiconBackend, LineBreakPolicy, LineEndRegexRule, LineJoinBlock, LineJoinConflictPolicy, LineJoinRule, RegexLineJoinRule, UnicodeNormalization, ) if TYPE_CHECKING: from _pytest.monkeypatch import MonkeyPatch mapped_module = import_module("mdpolish.modifiers.mapped_line_join") LEGACY_MAPPINGS = ( ("exam-", "ple", "example"), ("rule-", "based", "rule-based"), ("value.", "ues", "values"), ) def transform( markdown: str, rules: tuple[tuple[str, str, str] | LineJoinRule, ...] = LEGACY_MAPPINGS, **options: object, ) -> str: modifier = mapped_line_join(rules, **options) # type: ignore[arg-type] result = Pipeline((modifier,)).transform(markdown) assert result.status is RunStatus.SUCCESS assert result.output_markdown is not None return result.output_markdown def exact_rule(**overrides: object) -> ExactLineJoinRule: fields: dict[str, object] = { "rule_id": "example.join", "left": "exam", "right": "ple", "replacement": "example", "separator": "-", } fields.update(overrides) return ExactLineJoinRule(**fields) # type: ignore[arg-type] @pytest.mark.parametrize( ("markdown", "expected"), [ ("an exam-\nple here", "an example here"), ("an exam-\n\nple here", "an example here"), ("an exam-\r\n\r\nple here", "an example here"), ("an exam-\r\rple here", "an example here"), ("a rule-\nbased method", "a rule-based method"), ("the value.\nues differ", "the values differ"), ], ) def test_legacy_exact_mapping_supports_lf_crlf_cr_and_one_blank_line(markdown: str, expected: str) -> None: assert transform(markdown) == expected def test_exact_rules_distinguish_explicit_and_empty_separator() -> None: hyphenated = exact_rule() unseparated = exact_rule(rule_id="example.unseparated", separator="") assert transform("exam-\nple", (hyphenated,)) == "example" assert transform("exam\nple", (unseparated,)) == "example" assert transform("exam\nple", (hyphenated,)) == "exam\nple" def test_named_regex_combines_backreferences_from_both_sides() -> None: rule = RegexLineJoinRule( rule_id="regex.join", left_pattern=r"(?P[A-Za-z]+)", right_pattern=r"(?P[a-z]+)", replacement=r"\g_\g", separator="-", ) assert transform("prefix exam-\nple suffix", (rule,)) == "prefix exam_ple suffix" def test_regex_keeps_its_inline_flags() -> None: rule = RegexLineJoinRule( rule_id="regex.flags", left_pattern=r"(?i:(?Pexam))", right_pattern=r"(?i:(?Pple))", replacement=r"\g\g", separator="-", ) assert transform("EXAM-\nPLE", (rule,)) == "EXAMPLE" def test_regex_case_insensitive_option_applies_to_the_pattern() -> None: rule = RegexLineJoinRule( rule_id="regex.ignore-case", left_pattern=r"(?PEXAM)", right_pattern=r"(?PPLE)", replacement=r"\g\g", separator="-", case_sensitive=False, ) assert transform("exam-\nple", (rule,)) == "example" def test_regex_suffix_matching_considers_overlapping_starts() -> None: rule = RegexLineJoinRule( rule_id="regex.overlap", left_pattern=r"(?Paba)", right_pattern=r"(?Px)", replacement=r"\g\g", separator="-", ) assert transform("ababa-\nx", (rule,)) == "ababax" def test_line_end_regex_uses_right_match_only_as_a_condition() -> None: rule = LineEndRegexRule( rule_id="line-end.space", right_pattern=r"(?P[a-z])", line_break=LineBreakPolicy.SPACE, ) assert transform("foo\nbar", (rule,)) == "foo bar" @pytest.mark.parametrize( ("policy", "expected"), [ (LineBreakPolicy.PRESERVE, "example\n"), (LineBreakPolicy.DELETE, "example"), (LineBreakPolicy.SPACE, "example "), (LineBreakPolicy.PARAGRAPH, "example\n\n"), ], ) def test_all_line_break_policies_are_exact(policy: LineBreakPolicy, expected: str) -> None: assert transform("exam-\nple", (exact_rule(line_break=policy),)) == expected def test_paragraph_policy_restores_quote_and_list_prefixes() -> None: rule = exact_rule(line_break=LineBreakPolicy.PARAGRAPH) assert transform("> exam-\n> ple", (rule,)) == "> example\n>\n> " assert transform("- exam-\n ple", (rule,)) == "- example\n\n " assert transform("> - exam-\n> ple", (rule,)) == "> - example\n>\n> " @pytest.mark.parametrize( ("markdown", "expected"), [ ("plain exam-\nple", "plain example"), ("# exam-\nple", "# example"), ("- exam-\n ple", "- example"), ("> exam-\n> ple", "> example"), ("> > > exam-\n> > > ple", "> > > example"), ("1. outer\n 2. exam-\n ple", "1. outer\n 2. example"), ], ) def test_supported_markdown_blocks_and_valid_nested_containers(markdown: str, expected: str) -> None: assert transform(markdown, (exact_rule(),)) == expected def test_block_scope_is_explicit() -> None: heading_only = exact_rule(blocks=frozenset({LineJoinBlock.HEADING})) assert transform("# exam-\nple", (heading_only,)) == "# example" assert transform("exam-\nple", (heading_only,)) == "exam-\nple" @pytest.mark.parametrize( "markdown", [ "> > > exam-\n> > ple", "> #broken exam-\n> ple", "1. exam-\n2. ple", "1. outer\n 2. exam-\n ple", "- exam-\n ple", "#broken exam-\nple", ], ) def test_unknown_or_mismatched_containers_fail_closed_instead_of_becoming_paragraphs(markdown: str) -> None: assert transform(markdown, (exact_rule(),)) == markdown @pytest.mark.parametrize( "markdown", [ "```text\nexam-\nple\n```", " exam-\n ple", "> exam-\n> ple", "- ```\n exam-\n ple\n ```", "| word |\n| --- |\n| exam- |\n| ple |", "\nexam-\nple\n
", "\nexam-\nple", ], ) def test_code_and_table_regions_are_always_excluded(markdown: str) -> None: assert transform(markdown, (exact_rule(),)) == markdown def test_three_line_chain_is_one_proposal_and_uses_virtual_output() -> None: rules = ( exact_rule(rule_id="chain.first"), ExactLineJoinRule( rule_id="chain.second", left="example", right="based", replacement="example-based", separator="-", ), ) modifier = mapped_line_join(rules) proposals = modifier.propose(DocumentSnapshot("exam-\nple-\nbased")) assert len(proposals) == 1 assert len(proposals[0].edits) == 1 assert "chain.first, chain.second" in proposals[0].reason assert transform("exam-\nple-\nbased", rules) == "example-based" def test_mixed_line_endings_fail_the_entire_connected_chain_closed() -> None: rules = ( exact_rule(rule_id="chain.first"), ExactLineJoinRule("chain.second", "example", "based", "example-based", separator="-"), ) markdown = "exam-\r\nple-\nbased" assert transform(markdown, rules) == markdown def test_multiple_independent_chains_are_reported_in_source_order() -> None: modifier = mapped_line_join((exact_rule(),)) snapshot = DocumentSnapshot("exam-\nple and\n\nexam-\nple") proposals = modifier.propose(snapshot) assert len(proposals) == 2 assert proposals[0].edits[0].span.start < proposals[1].edits[0].span.start def test_keep_rule_can_veto_a_lower_priority_replacement() -> None: keep = KeepLineJoinRule("keep.example", "exam", "ple", separator="-", priority=10) replace = exact_rule(priority=0) assert transform("exam-\nple", (replace, keep)) == "exam-\nple" def test_conflict_first_and_priority_are_deterministic() -> None: first = exact_rule(rule_id="first", replacement="first", priority=0) second = exact_rule(rule_id="second", replacement="second", priority=10) assert transform( "exam-\nple", (first, second), conflict_policy=LineJoinConflictPolicy.FIRST, ) == "first" assert transform( "exam-\nple", (first, second), conflict_policy=LineJoinConflictPolicy.PRIORITY, ) == "second" def test_priority_tie_uses_declaration_order() -> None: first = exact_rule(rule_id="first", replacement="first") second = exact_rule(rule_id="second", replacement="second") assert transform("exam-\nple", (second, first)) == "second" def test_conflict_error_names_rules_but_not_surrounding_text() -> None: modifier = mapped_line_join( (exact_rule(rule_id="first"), exact_rule(rule_id="second")), conflict_policy=LineJoinConflictPolicy.ERROR, ) with pytest.raises(ModifierContractError, match=r"first, second") as raised: modifier.propose(DocumentSnapshot("secret exam-\nple material")) assert "secret" not in str(raised.value) def test_case_insensitive_matching_is_opt_in() -> None: assert transform("EXAM-\nPLE", (exact_rule(),)) == "EXAM-\nPLE" assert transform("EXAM-\nPLE", (exact_rule(case_sensitive=False),)) == "example" def test_unicode_normalization_is_opt_in_and_preserves_source_index_mapping() -> None: decomposed_left = "cafe\N{COMBINING ACUTE ACCENT}" markdown = "CAFÉ-\nteria" default = ExactLineJoinRule( "unicode.default", decomposed_left, "teria", "cafeteria", separator="-", case_sensitive=False, ) normalized = ExactLineJoinRule( "unicode.nfc", decomposed_left, "teria", "cafeteria", separator="-", case_sensitive=False, normalization=UnicodeNormalization.NFC, ) assert transform(markdown, (default,)) == markdown assert transform(markdown, (normalized,)) == "cafeteria" def test_unicode_word_boundaries_and_explicit_override() -> None: strict = exact_rule() permissive = exact_rule(rule_id="permissive", require_word_boundaries=False) markdown = "éexam-\nplemore" assert transform(markdown, (strict,)) == markdown assert transform(markdown, (permissive,)) == "éexamplemore" @pytest.mark.parametrize("max_blank_lines", [0, 1]) def test_blank_line_limit_is_configurable(max_blank_lines: int) -> None: expected = "exam-\n\nple" if max_blank_lines == 0 else "example" assert transform( "exam-\n\nple", (exact_rule(),), max_intervening_blank_lines=max_blank_lines, ) == expected @pytest.mark.parametrize("invalid", [-1, True, 2]) def test_invalid_blank_line_limits_are_rejected(invalid: object) -> None: with pytest.raises(ModifierContractError): mapped_line_join((), max_intervening_blank_lines=invalid) # type: ignore[arg-type] def test_empty_document_and_empty_rule_set_are_no_ops() -> None: modifier = mapped_line_join(()) assert transform("", ()) == "" assert transform("exam-\nple", ()) == "exam-\nple" assert modifier.version == "2.0.0" assert dict(modifier.parameters)["rules"] == () def test_parameters_preserve_rule_order_and_record_all_options() -> None: first = mapped_line_join((exact_rule(rule_id="first"), exact_rule(rule_id="second"))) second = mapped_line_join((exact_rule(rule_id="second"), exact_rule(rule_id="first"))) assert first.parameters != second.parameters records = dict(first.parameters)["rules"] assert isinstance(records, tuple) first_record = cast(tuple[tuple[str, object], ...], records[0]) assert ("rule_id", "first") in first_record @pytest.mark.parametrize( "rules", [ (("", "right", "word"),), (("left", "right", "two words"),), (("left", "right", "word"), ("left", "right", "other")), (("left", "right"),), (exact_rule(rule_id="same"), exact_rule(rule_id="same")), (exact_rule(case_sensitive=1),), (exact_rule(blocks={LineJoinBlock.PARAGRAPH}),), (RegexLineJoinRule("regex", "", r"(?Px)", "x"),), (RegexLineJoinRule("regex", r"(x)", r"(?Px)", "x"),), (RegexLineJoinRule("regex", r"(?Px)", r"(?Px)", r"\g"),), (RegexLineJoinRule("regex", r"(?Px)", r"(?Px)", r"\g"),), (LineEndRegexRule("line-end", r"(?Px)", line_break=LineBreakPolicy.PRESERVE),), ], ) def test_invalid_rules_raise_contract_error(rules: object) -> None: with pytest.raises(ModifierContractError): mapped_line_join(rules) # type: ignore[arg-type] class FakeLexicon: package_version = "test-lexicon" def __init__(self, scores: dict[str, tuple[float, float | None]]) -> None: self._scores = scores def evidence( self, candidate: str, form: LexicalCandidateForm, left: str, right: str, ) -> object: del form, left, right score = self._scores.get(candidate) return None if score is None else mapped_module._LexicalEvidence(*score) def lexical_rule(**overrides: object) -> LexicalLineJoinRule: fields: dict[str, object] = { "rule_id": "english.lexical", "left_pattern": r"[A-Za-z]+", "right_pattern": r"[a-z]+", "separator": "-", "backend": LexiconBackend.WORDFREQ, "language": "en", "candidate_forms": (LexicalCandidateForm.JOINED, LexicalCandidateForm.HYPHENATED), "minimum_score": 1.0, "minimum_score_margin": 0.5, } fields.update(overrides) return LexicalLineJoinRule(**fields) # type: ignore[arg-type] def install_fake_lexicon(monkeypatch: MonkeyPatch, scores: dict[str, tuple[float, float | None]]) -> None: def build(rule: LexicalLineJoinRule) -> FakeLexicon: del rule return FakeLexicon(scores) monkeypatch.setattr(mapped_module, "_build_lexicon", build) def test_lexical_rule_joins_a_unique_dictionary_candidate_without_per_word_mapping( monkeypatch: MonkeyPatch, ) -> None: install_fake_lexicon(monkeypatch, {"example": (5.0, 5.0)}) assert transform("exam-\nple", (lexical_rule(),)) == "example" def test_lexical_rule_can_select_the_natural_hyphenated_form_by_score(monkeypatch: MonkeyPatch) -> None: install_fake_lexicon(monkeypatch, {"rulebased": (3.0, 3.0), "rule-based": (6.0, 6.0)}) assert transform("rule-\nbased", (lexical_rule(),)) == "rule-based" def test_lexical_margin_and_keep_policy_preserve_an_ambiguous_boundary(monkeypatch: MonkeyPatch) -> None: install_fake_lexicon(monkeypatch, {"rulebased": (5.0, 5.0), "rule-based": (4.8, 4.8)}) assert transform("rule-\nbased", (lexical_rule(minimum_score_margin=0.5),)) == "rule-\nbased" def test_spellchecker_style_spaced_proxy_is_not_ranked_against_single_token(monkeypatch: MonkeyPatch) -> None: install_fake_lexicon(monkeypatch, {"inside": (6.0, 6.0), "in side": (5.0, None)}) rule = lexical_rule( separator="", candidate_forms=(LexicalCandidateForm.JOINED, LexicalCandidateForm.SPACED), ) assert transform("in\nside", (rule,)) == "in\nside" def test_lexical_ambiguity_error_is_source_safe(monkeypatch: MonkeyPatch) -> None: install_fake_lexicon(monkeypatch, {"inside": (6.0, 6.0), "in side": (5.0, None)}) rule = lexical_rule( separator="", candidate_forms=(LexicalCandidateForm.JOINED, LexicalCandidateForm.SPACED), ambiguity=LexicalAmbiguityPolicy.ERROR, ) modifier = mapped_line_join((rule,)) with pytest.raises(ModifierContractError, match=r"english\.lexical") as raised: modifier.propose(DocumentSnapshot("secret in\nside")) assert "secret" not in str(raised.value) class FakeHyphenator: def __init__(self, positions: tuple[int, ...]) -> None: self._positions = positions def positions(self, word: str) -> tuple[int, ...]: del word return self._positions @pytest.mark.parametrize(("positions", "expected"), [((4,), "example"), ((), "exam-ple")]) def test_pyphen_gate_only_controls_the_joined_candidate( monkeypatch: MonkeyPatch, positions: tuple[int, ...], expected: str, ) -> None: install_fake_lexicon(monkeypatch, {"example": (6.0, 6.0), "exam-ple": (4.0, 4.0)}) monkeypatch.setattr( mapped_module, "_build_hyphenator", lambda language: (FakeHyphenator(positions), f"test-{language}"), ) rule = lexical_rule(hyphenation_language="en_US", minimum_score_margin=0.5) assert transform("exam-\nple", (rule,)) == expected @pytest.mark.parametrize( "overrides", [ {"separator": "/"}, {"candidate_forms": (LexicalCandidateForm.JOINED,)}, {"candidate_forms": (LexicalCandidateForm.JOINED, LexicalCandidateForm.JOINED)}, {"minimum_score": -1.0}, {"minimum_score": float("nan")}, {"minimum_score_margin": True}, {"language": ""}, {"hyphenation_language": ""}, ], ) def test_invalid_lexical_rules_fail_before_loading_a_backend(overrides: dict[str, object]) -> None: with pytest.raises(ModifierContractError): mapped_line_join((lexical_rule(**overrides),)) def test_missing_requested_optional_backend_raises_without_fallback(monkeypatch: MonkeyPatch) -> None: monkeypatch.setattr(mapped_module, "_zipf_frequency", None) with pytest.raises(ModifierContractError, match="frequency"): mapped_line_join((lexical_rule(),)) @pytest.mark.parametrize( ("backend", "available"), [ (LexiconBackend.SPELLCHECKER, mapped_module._SpellChecker is not None), (LexiconBackend.WORDFREQ, mapped_module._zipf_frequency is not None), ], ) def test_installed_optional_backend_constructs_or_is_reported_as_skipped( backend: LexiconBackend, available: bool, ) -> None: if not available: pytest.skip(f"optional backend is not installed: {backend.value}") mapped_line_join((lexical_rule(backend=backend),)) def test_successful_output_is_idempotent() -> None: rules = (exact_rule(),) pipeline = Pipeline((mapped_line_join(rules),)) first = pipeline.transform("an exam-\n\nple here") assert first.status is RunStatus.SUCCESS assert first.output_markdown is not None assert pipeline.transform(first.output_markdown).changes == ()