feat: generalize mapped line joining

2026-08-26 20:52:42 +08:00
parent bb0507db30
commit 11349e007a
5 changed files with 2647 additions and 143 deletions
@@ -1,19 +1,62 @@
 from __future__ import annotations

+from importlib import import_module
+from typing import TYPE_CHECKING, cast
+
 import pytest

-from mdpolish import ModifierContractError, Pipeline, RunStatus
+from mdpolish import DocumentSnapshot, ModifierContractError, Pipeline, RunStatus
 from mdpolish.modifiers import mapped_line_join
+from mdpolish.modifiers.mapped_line_join import (
+    ExactLineJoinRule,
+    KeepLineJoinRule,
+    LexicalAmbiguityPolicy,
+    LexicalCandidateForm,
+    LexicalLineJoinRule,
+    LexiconBackend,
+    LineBreakPolicy,
+    LineEndRegexRule,
+    LineJoinBlock,
+    LineJoinConflictPolicy,
+    LineJoinRule,
+    RegexLineJoinRule,
+    UnicodeNormalization,
+)

-MAPPINGS = (
+if TYPE_CHECKING:
+    from _pytest.monkeypatch import MonkeyPatch
+
+mapped_module = import_module("mdpolish.modifiers.mapped_line_join")
+
+LEGACY_MAPPINGS = (
    ("exam-", "ple", "example"),
    ("rule-", "based", "rule-based"),
    ("value.", "ues", "values"),
 )


-def transform(markdown: str):  # type: ignore[no-untyped-def]
-    return Pipeline([mapped_line_join(MAPPINGS)]).transform(markdown)
+def transform(
+    markdown: str,
+    rules: tuple[tuple[str, str, str] | LineJoinRule, ...] = LEGACY_MAPPINGS,
+    **options: object,
+) -> str:
+    modifier = mapped_line_join(rules, **options)  # type: ignore[arg-type]
+    result = Pipeline((modifier,)).transform(markdown)
+    assert result.status is RunStatus.SUCCESS
+    assert result.output_markdown is not None
+    return result.output_markdown
+
+
+def exact_rule(**overrides: object) -> ExactLineJoinRule:
+    fields: dict[str, object] = {
+        "rule_id": "example.join",
+        "left": "exam",
+        "right": "ple",
+        "replacement": "example",
+        "separator": "-",
+    }
+    fields.update(overrides)
+    return ExactLineJoinRule(**fields)  # type: ignore[arg-type]


@pytest.mark.parametrize(
@@ -21,70 +64,493 @@ def transform(markdown: str):  # type: ignore[no-untyped-def]
    [
        ("an exam-\nple here", "an example here"),
        ("an exam-\n\nple here", "an example here"),
-        ("a rule-\nbased method", "a rule-based method"),
-        ("the value.\n\nues differ", "the values differ"),
        ("an exam-\r\n\r\nple here", "an example here"),
        ("an exam-\r\rple here", "an example here"),
+        ("a rule-\nbased method", "a rule-based method"),
+        ("the value.\nues differ", "the values differ"),
    ],
 )
-def test_applies_exact_mapping_across_supported_line_shapes(markdown: str, expected: str) -> None:
-    result = transform(markdown)
-    assert result.status is RunStatus.SUCCESS
-    assert result.output_markdown == expected
-    assert len(result.changes) == 1
+def test_legacy_exact_mapping_supports_lf_crlf_cr_and_one_blank_line(markdown: str, expected: str) -> None:
+    assert transform(markdown) == expected
+
+
+def test_exact_rules_distinguish_explicit_and_empty_separator() -> None:
+    hyphenated = exact_rule()
+    unseparated = exact_rule(rule_id="example.unseparated", separator="")
+
+    assert transform("exam-\nple", (hyphenated,)) == "example"
+    assert transform("exam\nple", (unseparated,)) == "example"
+    assert transform("exam\nple", (hyphenated,)) == "exam\nple"
+
+
+def test_named_regex_combines_backreferences_from_both_sides() -> None:
+    rule = RegexLineJoinRule(
+        rule_id="regex.join",
+        left_pattern=r"(?P<stem>[A-Za-z]+)",
+        right_pattern=r"(?P<suffix>[a-z]+)",
+        replacement=r"\g<stem>_\g<suffix>",
+        separator="-",
+    )
+
+    assert transform("prefix exam-\nple suffix", (rule,)) == "prefix exam_ple suffix"
+
+
+def test_regex_keeps_its_inline_flags() -> None:
+    rule = RegexLineJoinRule(
+        rule_id="regex.flags",
+        left_pattern=r"(?i:(?P<stem>exam))",
+        right_pattern=r"(?i:(?P<suffix>ple))",
+        replacement=r"\g<stem>\g<suffix>",
+        separator="-",
+    )
+
+    assert transform("EXAM-\nPLE", (rule,)) == "EXAMPLE"
+
+
+def test_regex_case_insensitive_option_applies_to_the_pattern() -> None:
+    rule = RegexLineJoinRule(
+        rule_id="regex.ignore-case",
+        left_pattern=r"(?P<stem>EXAM)",
+        right_pattern=r"(?P<suffix>PLE)",
+        replacement=r"\g<stem>\g<suffix>",
+        separator="-",
+        case_sensitive=False,
+    )
+
+    assert transform("exam-\nple", (rule,)) == "example"
+
+
+def test_regex_suffix_matching_considers_overlapping_starts() -> None:
+    rule = RegexLineJoinRule(
+        rule_id="regex.overlap",
+        left_pattern=r"(?P<stem>aba)",
+        right_pattern=r"(?P<suffix>x)",
+        replacement=r"\g<stem>\g<suffix>",
+        separator="-",
+    )
+
+    assert transform("ababa-\nx", (rule,)) == "ababax"
+
+
+def test_line_end_regex_uses_right_match_only_as_a_condition() -> None:
+    rule = LineEndRegexRule(
+        rule_id="line-end.space",
+        right_pattern=r"(?P<initial>[a-z])",
+        line_break=LineBreakPolicy.SPACE,
+    )
+
+    assert transform("foo\nbar", (rule,)) == "foo bar"
+
+
+@pytest.mark.parametrize(
+    ("policy", "expected"),
+    [
+        (LineBreakPolicy.PRESERVE, "example\n"),
+        (LineBreakPolicy.DELETE, "example"),
+        (LineBreakPolicy.SPACE, "example "),
+        (LineBreakPolicy.PARAGRAPH, "example\n\n"),
+    ],
+)
+def test_all_line_break_policies_are_exact(policy: LineBreakPolicy, expected: str) -> None:
+    assert transform("exam-\nple", (exact_rule(line_break=policy),)) == expected
+
+
+def test_paragraph_policy_restores_quote_and_list_prefixes() -> None:
+    rule = exact_rule(line_break=LineBreakPolicy.PARAGRAPH)
+
+    assert transform("> exam-\n> ple", (rule,)) == "> example\n>\n> "
+    assert transform("- exam-\n  ple", (rule,)) == "- example\n\n  "
+    assert transform("> - exam-\n>   ple", (rule,)) == "> - example\n>\n>   "
+
+
+@pytest.mark.parametrize(
+    ("markdown", "expected"),
+    [
+        ("plain exam-\nple", "plain example"),
+        ("# exam-\nple", "# example"),
+        ("- exam-\n  ple", "- example"),
+        ("> exam-\n> ple", "> example"),
+        ("> > > exam-\n> > > ple", "> > > example"),
+        ("1. outer\n   2. exam-\n      ple", "1. outer\n   2. example"),
+    ],
+)
+def test_supported_markdown_blocks_and_valid_nested_containers(markdown: str, expected: str) -> None:
+    assert transform(markdown, (exact_rule(),)) == expected
+
+
+def test_block_scope_is_explicit() -> None:
+    heading_only = exact_rule(blocks=frozenset({LineJoinBlock.HEADING}))
+
+    assert transform("# exam-\nple", (heading_only,)) == "# example"
+    assert transform("exam-\nple", (heading_only,)) == "exam-\nple"


@pytest.mark.parametrize(
    "markdown",
    [
-        "an unknown-\nword here",
-        "an exam-\n\n\nple here",
-        "an exam-\r\n\nple here",
-        "an EXAM-\nple here",
-        "an exam-\nplemore here",
-        "an xrule-\nbased method",
+        "> > > exam-\n> > ple",
+        "> #broken exam-\n> ple",
+        "1. exam-\n2. ple",
+        "1. outer\n   2. exam-\n    ple",
+        "- exam-\n ple",
+        "#broken exam-\nple",
    ],
 )
-def test_unknown_or_unsafe_boundaries_are_preserved(markdown: str) -> None:
-    assert transform(markdown).output_markdown == markdown
-
-
-def test_parameters_and_results_are_independent_of_mapping_order() -> None:
-    first = mapped_line_join(MAPPINGS)
-    second = mapped_line_join(reversed(MAPPINGS))
-    markdown = "example becomes exam-\nple"
-
-    first_result = Pipeline([first]).transform(markdown)
-    second_result = Pipeline([second]).transform(markdown)
-
-    assert first_result.modifiers == second_result.modifiers
-    assert first_result.output_markdown == second_result.output_markdown
-
-
-def test_library_contains_no_default_mapping() -> None:
-    modifier = mapped_line_join(())
-    result = Pipeline([modifier]).transform("an exam-\nple here")
-
-    assert modifier.parameters == (("mappings", ()),)
-    assert result.output_markdown == "an exam-\nple here"
+def test_unknown_or_mismatched_containers_fail_closed_instead_of_becoming_paragraphs(markdown: str) -> None:
+    assert transform(markdown, (exact_rule(),)) == markdown


@pytest.mark.parametrize(
-    "mappings",
+    "markdown",
+    [
+        "```text\nexam-\nple\n```",
+        "    exam-\n    ple",
+        ">     exam-\n>     ple",
+        "- ```\n  exam-\n  ple\n  ```",
+        "| word |\n| --- |\n| exam- |\n| ple |",
+        "<table>\nexam-\nple\n</table>",
+        "<table>\nexam-\nple",
+    ],
+)
+def test_code_and_table_regions_are_always_excluded(markdown: str) -> None:
+    assert transform(markdown, (exact_rule(),)) == markdown
+
+
+def test_three_line_chain_is_one_proposal_and_uses_virtual_output() -> None:
+    rules = (
+        exact_rule(rule_id="chain.first"),
+        ExactLineJoinRule(
+            rule_id="chain.second",
+            left="example",
+            right="based",
+            replacement="example-based",
+            separator="-",
+        ),
+    )
+    modifier = mapped_line_join(rules)
+    proposals = modifier.propose(DocumentSnapshot("exam-\nple-\nbased"))
+
+    assert len(proposals) == 1
+    assert len(proposals[0].edits) == 1
+    assert "chain.first, chain.second" in proposals[0].reason
+    assert transform("exam-\nple-\nbased", rules) == "example-based"
+
+
+def test_mixed_line_endings_fail_the_entire_connected_chain_closed() -> None:
+    rules = (
+        exact_rule(rule_id="chain.first"),
+        ExactLineJoinRule("chain.second", "example", "based", "example-based", separator="-"),
+    )
+    markdown = "exam-\r\nple-\nbased"
+
+    assert transform(markdown, rules) == markdown
+
+
+def test_multiple_independent_chains_are_reported_in_source_order() -> None:
+    modifier = mapped_line_join((exact_rule(),))
+    snapshot = DocumentSnapshot("exam-\nple and\n\nexam-\nple")
+    proposals = modifier.propose(snapshot)
+
+    assert len(proposals) == 2
+    assert proposals[0].edits[0].span.start < proposals[1].edits[0].span.start
+
+
+def test_keep_rule_can_veto_a_lower_priority_replacement() -> None:
+    keep = KeepLineJoinRule("keep.example", "exam", "ple", separator="-", priority=10)
+    replace = exact_rule(priority=0)
+
+    assert transform("exam-\nple", (replace, keep)) == "exam-\nple"
+
+
+def test_conflict_first_and_priority_are_deterministic() -> None:
+    first = exact_rule(rule_id="first", replacement="first", priority=0)
+    second = exact_rule(rule_id="second", replacement="second", priority=10)
+
+    assert transform(
+        "exam-\nple",
+        (first, second),
+        conflict_policy=LineJoinConflictPolicy.FIRST,
+    ) == "first"
+    assert transform(
+        "exam-\nple",
+        (first, second),
+        conflict_policy=LineJoinConflictPolicy.PRIORITY,
+    ) == "second"
+
+
+def test_priority_tie_uses_declaration_order() -> None:
+    first = exact_rule(rule_id="first", replacement="first")
+    second = exact_rule(rule_id="second", replacement="second")
+
+    assert transform("exam-\nple", (second, first)) == "second"
+
+
+def test_conflict_error_names_rules_but_not_surrounding_text() -> None:
+    modifier = mapped_line_join(
+        (exact_rule(rule_id="first"), exact_rule(rule_id="second")),
+        conflict_policy=LineJoinConflictPolicy.ERROR,
+    )
+
+    with pytest.raises(ModifierContractError, match=r"first, second") as raised:
+        modifier.propose(DocumentSnapshot("secret exam-\nple material"))
+    assert "secret" not in str(raised.value)
+
+
+def test_case_insensitive_matching_is_opt_in() -> None:
+    assert transform("EXAM-\nPLE", (exact_rule(),)) == "EXAM-\nPLE"
+    assert transform("EXAM-\nPLE", (exact_rule(case_sensitive=False),)) == "example"
+
+
+def test_unicode_normalization_is_opt_in_and_preserves_source_index_mapping() -> None:
+    decomposed_left = "cafe\N{COMBINING ACUTE ACCENT}"
+    markdown = "CAFÉ-\nteria"
+    default = ExactLineJoinRule(
+        "unicode.default",
+        decomposed_left,
+        "teria",
+        "cafeteria",
+        separator="-",
+        case_sensitive=False,
+    )
+    normalized = ExactLineJoinRule(
+        "unicode.nfc",
+        decomposed_left,
+        "teria",
+        "cafeteria",
+        separator="-",
+        case_sensitive=False,
+        normalization=UnicodeNormalization.NFC,
+    )
+
+    assert transform(markdown, (default,)) == markdown
+    assert transform(markdown, (normalized,)) == "cafeteria"
+
+
+def test_unicode_word_boundaries_and_explicit_override() -> None:
+    strict = exact_rule()
+    permissive = exact_rule(rule_id="permissive", require_word_boundaries=False)
+    markdown = "éexam-\nplemore"
+
+    assert transform(markdown, (strict,)) == markdown
+    assert transform(markdown, (permissive,)) == "éexamplemore"
+
+
+@pytest.mark.parametrize("max_blank_lines", [0, 1])
+def test_blank_line_limit_is_configurable(max_blank_lines: int) -> None:
+    expected = "exam-\n\nple" if max_blank_lines == 0 else "example"
+    assert transform(
+        "exam-\n\nple",
+        (exact_rule(),),
+        max_intervening_blank_lines=max_blank_lines,
+    ) == expected
+
+
+@pytest.mark.parametrize("invalid", [-1, True, 2])
+def test_invalid_blank_line_limits_are_rejected(invalid: object) -> None:
+    with pytest.raises(ModifierContractError):
+        mapped_line_join((), max_intervening_blank_lines=invalid)  # type: ignore[arg-type]
+
+
+def test_empty_document_and_empty_rule_set_are_no_ops() -> None:
+    modifier = mapped_line_join(())
+
+    assert transform("", ()) == ""
+    assert transform("exam-\nple", ()) == "exam-\nple"
+    assert modifier.version == "2.0.0"
+    assert dict(modifier.parameters)["rules"] == ()
+
+
+def test_parameters_preserve_rule_order_and_record_all_options() -> None:
+    first = mapped_line_join((exact_rule(rule_id="first"), exact_rule(rule_id="second")))
+    second = mapped_line_join((exact_rule(rule_id="second"), exact_rule(rule_id="first")))
+
+    assert first.parameters != second.parameters
+    records = dict(first.parameters)["rules"]
+    assert isinstance(records, tuple)
+    first_record = cast(tuple[tuple[str, object], ...], records[0])
+    assert ("rule_id", "first") in first_record
+
+
+@pytest.mark.parametrize(
+    "rules",
    [
        (("", "right", "word"),),
        (("left", "right", "two words"),),
        (("left", "right", "word"), ("left", "right", "other")),
        (("left", "right"),),
+        (exact_rule(rule_id="same"), exact_rule(rule_id="same")),
+        (exact_rule(case_sensitive=1),),
+        (exact_rule(blocks={LineJoinBlock.PARAGRAPH}),),
+        (RegexLineJoinRule("regex", "", r"(?P<right>x)", "x"),),
+        (RegexLineJoinRule("regex", r"(x)", r"(?P<right>x)", "x"),),
+        (RegexLineJoinRule("regex", r"(?P<x>x)", r"(?P<x>x)", r"\g<x>"),),
+        (RegexLineJoinRule("regex", r"(?P<x>x)", r"(?P<y>x)", r"\g<missing>"),),
+        (LineEndRegexRule("line-end", r"(?P<x>x)", line_break=LineBreakPolicy.PRESERVE),),
    ],
 )
-def test_invalid_mappings_raise_contract_error(mappings: object) -> None:
+def test_invalid_rules_raise_contract_error(rules: object) -> None:
    with pytest.raises(ModifierContractError):
-        mapped_line_join(mappings)  # type: ignore[arg-type]
+        mapped_line_join(rules)  # type: ignore[arg-type]
+
+
+class FakeLexicon:
+    package_version = "test-lexicon"
+
+    def __init__(self, scores: dict[str, tuple[float, float | None]]) -> None:
+        self._scores = scores
+
+    def evidence(
+        self,
+        candidate: str,
+        form: LexicalCandidateForm,
+        left: str,
+        right: str,
+    ) -> object:
+        del form, left, right
+        score = self._scores.get(candidate)
+        return None if score is None else mapped_module._LexicalEvidence(*score)
+
+
+def lexical_rule(**overrides: object) -> LexicalLineJoinRule:
+    fields: dict[str, object] = {
+        "rule_id": "english.lexical",
+        "left_pattern": r"[A-Za-z]+",
+        "right_pattern": r"[a-z]+",
+        "separator": "-",
+        "backend": LexiconBackend.WORDFREQ,
+        "language": "en",
+        "candidate_forms": (LexicalCandidateForm.JOINED, LexicalCandidateForm.HYPHENATED),
+        "minimum_score": 1.0,
+        "minimum_score_margin": 0.5,
+    }
+    fields.update(overrides)
+    return LexicalLineJoinRule(**fields)  # type: ignore[arg-type]
+
+
+def install_fake_lexicon(monkeypatch: MonkeyPatch, scores: dict[str, tuple[float, float | None]]) -> None:
+    def build(rule: LexicalLineJoinRule) -> FakeLexicon:
+        del rule
+        return FakeLexicon(scores)
+
+    monkeypatch.setattr(mapped_module, "_build_lexicon", build)
+
+
+def test_lexical_rule_joins_a_unique_dictionary_candidate_without_per_word_mapping(
+    monkeypatch: MonkeyPatch,
+) -> None:
+    install_fake_lexicon(monkeypatch, {"example": (5.0, 5.0)})
+
+    assert transform("exam-\nple", (lexical_rule(),)) == "example"
+
+
+def test_lexical_rule_can_select_the_natural_hyphenated_form_by_score(monkeypatch: MonkeyPatch) -> None:
+    install_fake_lexicon(monkeypatch, {"rulebased": (3.0, 3.0), "rule-based": (6.0, 6.0)})
+
+    assert transform("rule-\nbased", (lexical_rule(),)) == "rule-based"
+
+
+def test_lexical_margin_and_keep_policy_preserve_an_ambiguous_boundary(monkeypatch: MonkeyPatch) -> None:
+    install_fake_lexicon(monkeypatch, {"rulebased": (5.0, 5.0), "rule-based": (4.8, 4.8)})
+
+    assert transform("rule-\nbased", (lexical_rule(minimum_score_margin=0.5),)) == "rule-\nbased"
+
+
+def test_spellchecker_style_spaced_proxy_is_not_ranked_against_single_token(monkeypatch: MonkeyPatch) -> None:
+    install_fake_lexicon(monkeypatch, {"inside": (6.0, 6.0), "in side": (5.0, None)})
+    rule = lexical_rule(
+        separator="",
+        candidate_forms=(LexicalCandidateForm.JOINED, LexicalCandidateForm.SPACED),
+    )
+
+    assert transform("in\nside", (rule,)) == "in\nside"
+
+
+def test_lexical_ambiguity_error_is_source_safe(monkeypatch: MonkeyPatch) -> None:
+    install_fake_lexicon(monkeypatch, {"inside": (6.0, 6.0), "in side": (5.0, None)})
+    rule = lexical_rule(
+        separator="",
+        candidate_forms=(LexicalCandidateForm.JOINED, LexicalCandidateForm.SPACED),
+        ambiguity=LexicalAmbiguityPolicy.ERROR,
+    )
+    modifier = mapped_line_join((rule,))
+
+    with pytest.raises(ModifierContractError, match=r"english\.lexical") as raised:
+        modifier.propose(DocumentSnapshot("secret in\nside"))
+    assert "secret" not in str(raised.value)
+
+
+class FakeHyphenator:
+    def __init__(self, positions: tuple[int, ...]) -> None:
+        self._positions = positions
+
+    def positions(self, word: str) -> tuple[int, ...]:
+        del word
+        return self._positions
+
+
+@pytest.mark.parametrize(("positions", "expected"), [((4,), "example"), ((), "exam-ple")])
+def test_pyphen_gate_only_controls_the_joined_candidate(
+    monkeypatch: MonkeyPatch,
+    positions: tuple[int, ...],
+    expected: str,
+) -> None:
+    install_fake_lexicon(monkeypatch, {"example": (6.0, 6.0), "exam-ple": (4.0, 4.0)})
+    monkeypatch.setattr(
+        mapped_module,
+        "_build_hyphenator",
+        lambda language: (FakeHyphenator(positions), f"test-{language}"),
+    )
+    rule = lexical_rule(hyphenation_language="en_US", minimum_score_margin=0.5)
+
+    assert transform("exam-\nple", (rule,)) == expected
+
+
+@pytest.mark.parametrize(
+    "overrides",
+    [
+        {"separator": "/"},
+        {"candidate_forms": (LexicalCandidateForm.JOINED,)},
+        {"candidate_forms": (LexicalCandidateForm.JOINED, LexicalCandidateForm.JOINED)},
+        {"minimum_score": -1.0},
+        {"minimum_score": float("nan")},
+        {"minimum_score_margin": True},
+        {"language": ""},
+        {"hyphenation_language": ""},
+    ],
+)
+def test_invalid_lexical_rules_fail_before_loading_a_backend(overrides: dict[str, object]) -> None:
+    with pytest.raises(ModifierContractError):
+        mapped_line_join((lexical_rule(**overrides),))
+
+
+def test_missing_requested_optional_backend_raises_without_fallback(monkeypatch: MonkeyPatch) -> None:
+    monkeypatch.setattr(mapped_module, "_zipf_frequency", None)
+
+    with pytest.raises(ModifierContractError, match="frequency"):
+        mapped_line_join((lexical_rule(),))
+
+
+@pytest.mark.parametrize(
+    ("backend", "available"),
+    [
+        (LexiconBackend.SPELLCHECKER, mapped_module._SpellChecker is not None),
+        (LexiconBackend.WORDFREQ, mapped_module._zipf_frequency is not None),
+    ],
+)
+def test_installed_optional_backend_constructs_or_is_reported_as_skipped(
+    backend: LexiconBackend,
+    available: bool,
+) -> None:
+    if not available:
+        pytest.skip(f"optional backend is not installed: {backend.value}")
+    mapped_line_join((lexical_rule(backend=backend),))


 def test_successful_output_is_idempotent() -> None:
-    pipeline = Pipeline([mapped_line_join(MAPPINGS)])
+    rules = (exact_rule(),)
+    pipeline = Pipeline((mapped_line_join(rules),))
    first = pipeline.transform("an exam-\n\nple here")
+
+    assert first.status is RunStatus.SUCCESS
    assert first.output_markdown is not None
    assert pipeline.transform(first.output_markdown).changes == ()