From 11349e007a2773960cadc2bd57908e11f4614c06 Mon Sep 17 00:00:00 2001 From: Bepr4 <63661977@qq.com> Date: Wed, 26 Aug 2026 20:52:42 +0800 Subject: [PATCH] feat: generalize mapped line joining --- README.md | 29 +- pyproject.toml | 7 + .../0009-generalized-mapped-line-join.md | 762 +++++++++ src/mdpolish/modifiers/mapped_line_join.py | 1440 +++++++++++++++-- tests/test_mapped_line_join.py | 552 ++++++- 5 files changed, 2647 insertions(+), 143 deletions(-) create mode 100644 research-wiki/design/0009-generalized-mapped-line-join.md diff --git a/README.md b/README.md index 40c82c2..aa67dc3 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ | 精确编辑执行器 | 校验快照、范围、原文、重复和冲突后原子应用一个批次 | 不判断项目业务语义 | | `Pipeline` | 按调用方顺序运行修改器，并对最终快照做只读稳定性复查 | 不自动选规则、不重排、不循环执行 | | `regex_replace()` | 把非空正则匹配转换为精确编辑 | 不提供规则注册表、配置加载或默认模式 | -| `mapped_line_join()` | 按调用方提供的映射合并跨行片段 | 库内没有默认词表，不猜测未知词 | +| `mapped_line_join()` | 用精确、正则或可选本地词典规则合并跨行片段 | 无默认规则；代码、表格、未知结构和歧义失败关闭 | | HTML 表格修改器 | 处理严格表格子集的实体和单行布局 | 不是完整 HTML parser，也不是 HTML→GFM 转换器 | 一次运行会返回 `success`、`failed` 或 `unstable`： @@ -40,7 +40,14 @@ python -m venv .venv .venv/bin/python -m pip install -e '.[dev]' ``` -运行时只依赖 Python 标准库，支持 Python 3.11 及以上版本。 +核心运行时只依赖 Python 标准库，支持 Python 3.11 及以上版本。自动词典规则需要调用方明确安装并选择对应 extra： + +```bash +python -m pip install '/path/to/mdpolish[lexical]' # pyspellchecker + Pyphen +python -m pip install '/path/to/mdpolish[frequency]' # wordfreq，体积和传递依赖更大 +``` + +安装 extra 不会自动启用规则，也不会触发在线下载或改变精确/正则规则行为。 ## 组装自己的流水线 @@ -142,8 +149,16 @@ remove_marker = Modifier( ## 通用修改器的严格边界 -`mapped_line_join()` 只使用调用方显式传入的三元组：左片段、右片段和最终文本。它只处理相邻物理行或中间恰好一个 -同风格空行的情况，并检查 ASCII 词边界。 +`mapped_line_join()` 保留 `(left, right, replacement)` 三元组，也接受模块 +`mdpolish.modifiers.mapped_line_join` 中的不可变规则值。规则可以使用精确片段、两侧命名正则、只约束右侧的行尾正则， +或从行尾与行首自动生成 `JOINED` / `HYPHENATED` / `SPACED` 候选并查询显式选择的本地词典。自动规则不要求逐词维护 +映射；精确规则和 `KeepLineJoinRule` 用于项目词、例外与否决。 + +调用方还要显式决定块范围、换行处理、冲突策略、大小写和 Unicode 规范化。默认区分大小写且不规范化；支持相邻行或 +中间最多一个同风格空行，并可在一次提议内完成多行链式合并。保守词法扫描只正向识别段落、ATX 标题 continuation、 +列表 continuation 和同深度引用；代码块、GFM pipe table、raw HTML table、混合候选行尾及无法确认的容器保持原文。 +完整公共模型、选择流程和限制见 +[`0009-generalized-mapped-line-join.md`](research-wiki/design/0009-generalized-mapped-line-join.md)。 `html_table_entity_unescape()` 只在严格完整的 `` / `` 文本中处理 `&lt;`、`&gt;` 和 `&amp;`。`html_table_layout()` 只调整严格单行表格的外层行布局，并保留标签、属性和单元格内容。 @@ -174,7 +189,7 @@ research-wiki/ - 文件适配器、公共 CLI、配置文件、profile 或批处理协议； - 自动规则发现、注册表或默认流水线； -- Markdown AST、完整 HTML parser 或新运行依赖； +- Markdown AST、完整 HTML parser 或必装的第三方运行依赖； - artifact、报告、Web/桌面评审器； - 任何业务项目的规则、固定参数、文档 ID、数据或验收统计。 @@ -202,7 +217,7 @@ git status --short ``` 上述检查已于 2026-08-26 在 Python 3.13.11 环境实际运行：Ruff 通过，mypy 检查 21 个源码和测试文件无问题， -pytest 共 104 项测试通过，`mdpolish-0.2.0-py3-none-any.whl` 构建成功。wheel 内容已单独检查，只包含通用 Python -包、类型标记和包元数据，不包含项目规则、实验脚本、评审器或 Node.js 文件。 +pytest 共 168 项通过、2 项因本环境未安装可选词典 backend 而跳过，`mdpolish-0.2.0-py3-none-any.whl` 构建成功。 +wheel 内容已单独检查，只包含通用 Python 包、类型标记和包元数据，不包含项目规则、实验脚本、评审器或 Node.js 文件。 `pyproject.toml` 声明的 Python 3.11 及以上为支持范围；本次结果不表示已经在每个受支持版本上完成兼容性验证。 diff --git a/pyproject.toml b/pyproject.toml index 1c978f5..f517f6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,13 @@ requires-python = ">=3.11" dependencies = [] [project.optional-dependencies] +frequency = [ + "wordfreq>=3.1.1,<4", +] +lexical = [ + "pyphen>=0.18.1,<1", + "pyspellchecker>=0.9,<1", +] dev = [ "mypy>=1.15,<2", "pytest>=8.3,<10", diff --git a/research-wiki/design/0009-generalized-mapped-line-join.md b/research-wiki/design/0009-generalized-mapped-line-join.md new file mode 100644 index 0000000..99deb51 --- /dev/null +++ b/research-wiki/design/0009-generalized-mapped-line-join.md @@ -0,0 +1,762 @@ +# 0009：通用映射驱动的跨行片段合并 + +## 状态 + +已批准（2026-08-26）。 + +`supersedes: 0008`（范围有限）：本文替代 `0008` 第 6.1 节中“保留当前左片段、右片段、结果文本算法” +的窄接口，并为调用方显式启用的词典规则放宽“运行时只依赖标准库”这一点。核心安装仍然零依赖；精确和正则规则 +仍然只处理内存字符串。词典规则只允许在构造修改器时读取已安装 optional package 自带的版本化词典资源， +不读取调用方文档、任意路径或网络。`propose()` 仍然是只读快照的纯函数。 + +`0008` 确定的函数式 `Modifier`、无默认启用行为、无项目固定词表、无网络和无模型推断等边界继续有效。 + +本文批准后才能实施。批准本文不等于批准提交、推送、创建 PR、发布、修改其他仓库或处理真实材料。 + +## 1. 问题与可观察现象 + +当前 `mapped_line_join()` 只接受 `(left, right, replacement)` 三元组。它能处理调用方已经确认的少量精确断词， +但遇到下列情况时，调用方只能复制整个修改器或在库外另写一套边界扫描逻辑： + +- 同一类断词需要用命名正则捕获不同词干； +- 只需要匹配下一行开头，左侧只要求位于当前行尾； +- OCR 把一个词连续拆到三行以上，需要在一次提议中完成链式合并； +- 同一位置有多条规则，需要显式选择第一条、最高优先级或报错； +- 规则只允许用于段落、标题、列表项或引用，不能进入表格和代码块； +- 合并后需要保留换行、删除换行、换成空格或形成段落分隔； +- 文本和规则可能使用不同 Unicode 规范形式，或调用方明确需要忽略大小写。 + +更根本的问题是：如果每个 `example`、`international`、`database` 都要先手写一条映射，调用方实际上要自己维护 +一份英语词典。成熟的通用方案应能从行尾和行首自动生成候选词，查询明确配置的词典或词频资源；精确映射只负责 +项目术语、误判否决和其他例外，不应成为处理普通英文断词的唯一入口。 + +当前实现还会在整个物理行内容上匹配。它不知道 Markdown 围栏、缩进代码、pipe table、列表和引用前缀， +因此无法可靠表达“只在某类块中生效”。 + +这些都属于清洗语义、规则格式和冲突策略变化，不能作为 `0008` 已批准实现的机械扩展。 + +## 2. 调研结论及其边界 + +本次调研只用于确定库边界，不把外部工具行为直接变成 `mdpolish` 的默认规则。 + +### 2.1 Pandoc：软换行语义与源码重排是两件事 + +Pandoc 默认把段落内普通换行当作空格。`hard_line_breaks` 会把段落内每个换行解释为硬换行， +`ignore_line_breaks` 和 `east_asian_line_breaks` 则提供另外两种读取语义。输出侧的 `--wrap=auto|none|preserve` +决定生成源码怎样折行，不负责判断 OCR 断词。 + +当前 Pandoc 手册没有 `reflowed_text` 扩展。与“reflowed text”最接近的当前公共能力是输出侧 `--wrap`， +不能把这个非现行名称设计成库兼容目标。 + +来源： + +- +- +- + +因此，本修改器不能把所有物理换行统一解释成一种语义。每条规则必须明确怎样处理命中的边界，代码块和表格等结构 +必须先排除。 + +### 2.2 OCR 去连字符：成熟方案仍然需要证据来源和取舍 + +常见 OCR 后处理会组合以下信号： + +| 信号 | 能解决的问题 | 不能单独保证的事情 | +| --- | --- | --- | +| 词典查表 | 判断拼接词是否为已知词 | 专名、新词、领域词和真正带连字符的词 | +| 词缀与形态分析 | 识别屈折、派生和复合词 | 依赖语言及词典质量 | +| 词频或统计语言模型 | 在“连写、保留连字符、加空格”之间排序 | 结果依赖训练语料和领域分布 | +| OCR 置信度与版面元数据 | 利用识别器对字符、词或断词的显式证据 | 普通 Markdown 通常已经丢失这些信息 | + +ALTO 可以用 `SUBS_TYPE=HypPart1/HypPart2` 和 `SUBS_CONTENT` 表达断词及完整词，并提供词置信度；hOCR 定义了 +`x_wconf`。这说明如果上游仍持有结构化 OCR 证据，优先在上游使用它比从 Markdown 猜测更可靠。 + +来源： + +- +- +- +- + +本仓库当前只有 Markdown 字符串，没有 OCR 置信度、坐标或上游候选。因此本轮只实现调用方显式配置的本地词典判断 +和统计打分，不实现依赖版面证据的判断或模型推断。词典结果是候选证据，不绕过歧义策略和精确覆盖规则。 + +### 2.3 Python 词典与断词库可以提供证据，但必须显式选择 + +| 工具 | 主要能力 | 本轮不直接集成的原因 | +| --- | --- | --- | +| `wordninja` | 按 unigram 概率拆分粘连英文词 | 目标是拆词，不是恢复 Markdown 跨行结构；默认模型有语言和语料偏置 | +| `pyspellchecker` | 基于词频和编辑距离给出拼写候选 | 候选不是唯一正确修改，默认大小写和词典行为也需项目决定 | +| `PyEnchant` | 通过 Enchant provider 检查和建议拼写 | 依赖系统 provider 与外部词典，安装结果不完全由 Python 包锁定 | +| `wordfreq` | 查询多语言词频 | 能排序候选，但不能确定是否应保留自然连字符 | +| `PyHyphen` / `Pyphen` | 使用 TeX/Hunspell 断词模式找合法断点 | 正向排版断词不等于逆向 OCR 去断词 | +| Hunspell | 拼写检查、词缀、复合词和形态分析 | 需要语言词典；不同词典的形态字段和能力不同 | + +来源： + +- +- +- +- +- +- +- + +调研后的采用边界如下： + +- `pyspellchecker` 具有随包分发的多语言词频词典，能够同时做精确 `known()` 查询和频率比较，适合作为第一种较轻的 + 自动词典 backend；不使用它的编辑距离纠错，因为本任务只在几个明确候选之间选择，不改写 OCR 字符； +- `wordfreq` 提供可比较的 Zipf 频率和更广的多语言数据，适合作为可选的频率 backend；它的数据主要截至 2021 年， + 且官方明确说明多 token 查询会高估罕见组合，因此不能把任意短语分数当作可靠词典命中； +- `Pyphen` 能列出一个词的合法断词位置，适合作为“这个行尾位置是否可能是排版断词”的附加门槛，但它不能单独证明 + 拼接词真实存在； +- `PyEnchant` / Hunspell 能处理词缀、复合词和自然连字符，但依赖系统 provider 和外部词典，环境可复现性较差； + 本轮记录边界但不实现该 backend； +- `wordninja` 的目标是把粘连字符串拆成多个词，方向与本任务相反，本轮不集成。 + +这些 backend 必须由 `LexicalLineJoinRule` 显式选择。缺少请求的 extra 时立即报错，不退回另一个词典，也不改成只靠 +正则猜测。 + +### 2.4 Markdown lint 工具负责语法和风格，不负责 OCR 语义修复 + +`markdownlint` 的规则集中有行长、尾随空格、空行、标题和列表等结构/风格检查；`remark-lint` 的规则检查 mdast， +并允许项目自行编写插件。两者都没有“判断行尾连字符是不是排版断词并自动拼回”的内置规则。 + +这不是遗漏：lint 工具可以判断 Markdown 是否符合某种书写约定，却没有词典、原始页面或 OCR 置信度来唯一决定正文 +应该怎样改。OCR 语义修复应由上游转换器、专用后处理器或项目明确规则负责。 + +来源： + +- +- + +## 3. 目标与非目标 + +### 3.1 目标 + +- 保留旧三元组精确映射的调用方式； +- 增加不可变、带类型的精确规则和正则规则； +- 增加自动词典规则：从行尾、separator 和行首生成候选，查询显式配置的本地词典或词频资源； +- 让一条语言级规则可以复用于任意文档，不要求调用方逐词枚举普通英文； +- 所有规则仍由调用方显式传入，不内置词表或模式； +- 在一次 `propose()` 中处理独立命中和跨多行链式命中； +- 显式配置块范围、边界替换、冲突策略、大小写和 Unicode 规范化； +- 对代码块和表格失败关闭； +- 最终只产生绑定当前快照的非重叠精确 `TextEdit`； +- 规则、选项和顺序完整记录在 `Modifier.parameters` 中。 + +### 3.2 非目标 + +- 不实现自动语言识别、在线词典发现或模型调用；语言、backend、候选形式和阈值必须显式配置； +- 不读取 ALTO、hOCR、PDF、DOCX、图片或外部文件； +- 核心安装不引入 Markdown parser 或 OCR 库；词典能力只通过明确安装的 optional extra 启用； +- 不声称这个词法块分类器覆盖全部 CommonMark、GFM、Pandoc Markdown 或任意嵌套 HTML； +- 不自动选择规则、重排规则或循环运行整个 `Pipeline`； +- 不修改 `_text_ranges.py`； +- 不提供配置文件、CLI、默认映射或默认流水线。 + +## 4. 公共规则模型 + +新值类型均放在 `mapped_line_join.py`，使用 `@dataclass(frozen=True, slots=True)`。为避免扩大原始任务的源码范围， +它们先通过 `mdpolish.modifiers.mapped_line_join` 模块路径提供；本轮不要求修改 `modifiers/__init__.py`。 + +### 4.1 枚举 + +```python +class LineJoinBlock(StrEnum): + PARAGRAPH = "paragraph" + HEADING = "heading" + LIST_ITEM = "list_item" + BLOCK_QUOTE = "block_quote" + + +class LineBreakPolicy(StrEnum): + PRESERVE = "preserve" + DELETE = "delete" + SPACE = "space" + PARAGRAPH = "paragraph" + + +class LineJoinConflictPolicy(StrEnum): + PRIORITY = "priority" + FIRST = "first" + ERROR = "error" + + +class UnicodeNormalization(StrEnum): + NFC = "NFC" + NFD = "NFD" + NFKC = "NFKC" + NFKD = "NFKD" + + +class LexiconBackend(StrEnum): + SPELLCHECKER = "pyspellchecker" + WORDFREQ = "wordfreq" + + +class LexicalCandidateForm(StrEnum): + JOINED = "joined" + HYPHENATED = "hyphenated" + SPACED = "spaced" + + +class LexicalAmbiguityPolicy(StrEnum): + KEEP = "keep" + ERROR = "error" +``` + +默认可匹配块为四种 `LineJoinBlock` 的不可变集合。代码块和表格不是可选块类型，因为它们始终排除，调用方不能通过 +普通规则意外打开。 + +### 4.2 精确规则 + +```python +@dataclass(frozen=True, slots=True) +class ExactLineJoinRule: + rule_id: str + left: str + right: str + replacement: str + separator: str = "" + blocks: frozenset[LineJoinBlock] = ALL_LINE_JOIN_BLOCKS + line_break: LineBreakPolicy = LineBreakPolicy.DELETE + priority: int = 0 + case_sensitive: bool = True + normalization: UnicodeNormalization | None = None + require_word_boundaries: bool = True +``` + +`separator` 是紧跟在 `left` 后、位于物理换行前的显式文本。它可以是 `"-"`，也可以是空字符串： + +```python +ExactLineJoinRule( + rule_id="example.dehyphenate", + left="exam", + separator="-", + right="ple", + replacement="example", +) + +ExactLineJoinRule( + rule_id="example.join_split_word", + left="exam", + separator="", + right="ple", + replacement="example", +) +``` + +这样不需要靠“`left` 是否碰巧以连字符结尾”推断规则类型。`replacement` 是完整的替换片段；库不会自动拼接 +`left`、`separator` 或 `right`。 + +自动词典不可避免会遇到项目专名和自然连字符例外。为让映射表能明确否决而不是只能增加修改，再提供同样不可变的 +精确保护规则： + +```python +@dataclass(frozen=True, slots=True) +class KeepLineJoinRule: + rule_id: str + left: str + right: str + separator: str = "" + blocks: frozenset[LineJoinBlock] = ALL_LINE_JOIN_BLOCKS + priority: int = 0 + case_sensitive: bool = True + normalization: UnicodeNormalization | None = None + require_word_boundaries: bool = True +``` + +它命中并赢得冲突选择后保留原文，不产生无效 `TextEdit`，同时阻止低优先级词典规则处理该边界。保护规则会进入 +`Modifier.parameters`；由于没有实际修改，不伪造 `Change`。 + +### 4.3 左右正则规则 + +```python +@dataclass(frozen=True, slots=True) +class RegexLineJoinRule: + rule_id: str + left_pattern: str + right_pattern: str + replacement: str + separator: str = "" + blocks: frozenset[LineJoinBlock] = ALL_LINE_JOIN_BLOCKS + line_break: LineBreakPolicy = LineBreakPolicy.DELETE + priority: int = 0 + case_sensitive: bool = True + normalization: UnicodeNormalization | None = None +``` + +- `left_pattern` 必须匹配左侧逻辑内容的非空后缀； +- `right_pattern` 必须匹配右侧逻辑内容的非空前缀； +- 捕获组必须命名，左右两侧的组名不能重复； +- `replacement` 只支持 `\g` 命名反向引用，不支持数字组和动态回调； +- 非捕获组、lookaround 和普通正则语法可以使用，但最终左右匹配本身必须消耗字符； +- 正则按 Python `re` 语义编译，不启用 `MULTILINE` 或 `DOTALL`，因为物理行边界由修改器负责。 + +示例： + +```python +RegexLineJoinRule( + rule_id="example.named_regex", + left_pattern=r"(?P[A-Za-z]+)", + separator="-", + right_pattern=r"(?P[a-z]+)", + replacement=r"\g\g", +) +``` + +### 4.4 只指定右侧的行尾正则规则 + +```python +@dataclass(frozen=True, slots=True) +class LineEndRegexRule: + rule_id: str + right_pattern: str + separator: str = "" + blocks: frozenset[LineJoinBlock] = ALL_LINE_JOIN_BLOCKS + line_break: LineBreakPolicy = LineBreakPolicy.DELETE + priority: int = 0 + case_sensitive: bool = True + normalization: UnicodeNormalization | None = None +``` + +这类规则把左侧定义为逻辑行尾的零宽锚点。`right_pattern` 必须匹配右侧逻辑内容的非空前缀，但只作为边界条件； +右侧匹配文本不进入编辑范围，也不会被重新输出。实际编辑只覆盖可选 `separator`、物理行尾、允许的空行和右侧重复 +块前缀： + +```python +LineEndRegexRule( + rule_id="example.line_end_and_right", + right_pattern=r"(?P[a-z])", + line_break=LineBreakPolicy.SPACE, +) +``` + +上例把 `foo\nbar` 变成 `foo bar`，而不是把空格放到 `b` 之后。规则不会猜测左侧单词，只明确表示 +“当前逻辑行尾 + 下一逻辑行首命中该正则时怎样改”。它仍然是映射表规则，不是对所有小写行首自动生效的隐藏默认值。 + +`LineEndRegexRule` 不允许 `PRESERVE`，因为它不会改变 separator、边界或右侧文本，只会产生无效编辑。 + +### 4.5 自动词典规则 + +普通英文断词不要求逐词映射。调用方配置一条语言级规则，修改器从每个合格边界提取左右单词片段并自动查询本地词典： + +```python +@dataclass(frozen=True, slots=True) +class LexicalLineJoinRule: + rule_id: str + left_pattern: str + right_pattern: str + separator: str + backend: LexiconBackend + language: str + candidate_forms: tuple[LexicalCandidateForm, ...] + minimum_score: float + minimum_score_margin: float = 0.0 + hyphenation_language: str | None = None + ambiguity: LexicalAmbiguityPolicy = LexicalAmbiguityPolicy.KEEP + blocks: frozenset[LineJoinBlock] = ALL_LINE_JOIN_BLOCKS + priority: int = 0 + case_sensitive: bool = True + normalization: UnicodeNormalization | None = None +``` + +示例配置一次后可以复用于任意英文文档： + +```python +LexicalLineJoinRule( + rule_id="english.dictionary_dehyphenation", + left_pattern=r"[A-Za-z]{2,}", + right_pattern=r"[a-z]{2,}", + separator="-", + backend=LexiconBackend.SPELLCHECKER, + language="en", + candidate_forms=( + LexicalCandidateForm.JOINED, + LexicalCandidateForm.HYPHENATED, + ), + minimum_score=1.0, + minimum_score_margin=0.5, + hyphenation_language="en_US", +) +``` + +这条规则会自动处理 `exam-\nple`、`inter-\nnational` 等候选，不需要为 `example`、`international` 分别写映射。 +规则表在这里表达的是“什么文本形态、用哪个词典、比较哪些候选、阈值多保守”，不是穷举英语单词。 + +左右 pattern 分别锚定逻辑行尾和行首，整个非空匹配就是候选片段。`separator` 本轮只允许 `"-"` 或 `""`。 +修改器生成调用方列出的候选： + +| 候选 | 输出 | +| --- | --- | +| `JOINED` | `left + right` | +| `HYPHENATED` | `left + "-" + right` | +| `SPACED` | `left + " " + right` | + +规则必须包含 `JOINED`，并包含与源边界对应的竞争形式：`separator="-"` 时包含 `HYPHENATED`，空 separator 时包含 +`SPACED`。这样“拼接词在词典中”不会自动压过同样合理的自然连字符或两个独立单词。 + +backend 行为： + +- `SPELLCHECKER` 用 `SpellChecker.known()` 判断单 token 是否存在，并把 `word_usage_frequency()` 换算到每十亿词的 + 对数尺度。`SPACED` 或需要拆成多个单 token 验证的 `HYPHENATED` 只在各组成 token 分别已知时成立，准入分数取 + 各 token 的最低值；这个值不是短语频率，也不能与单 token 分数按 `minimum_score_margin` 排序； +- `WORDFREQ` 使用 `zipf_frequency(candidate, language)`；它可以评价多 token 字符串，但调用方必须考虑官方说明的 + 罕见组合高估问题，并通过 `minimum_score_margin`、精确否决规则和合成反例收紧； +- `hyphenation_language` 非空时，再用 Pyphen 检查 `len(left)` 是否在 `positions(left + right)` 中。这个门槛只作用于 + `JOINED` 候选，不替代词典存在性和候选比较。 + +`minimum_score` 和 `minimum_score_margin` 是 backend 内部的阈值，不是跨 backend 的统一物理量。对 `WORDFREQ`，二者 +都是 Zipf 尺度；对 `SPELLCHECKER`，`minimum_score` 可以过滤单 token 或各组成 token 中明显低频的候选，但 +`minimum_score_margin` 只适用于两个都有可比单 token 分数的候选。调用方不能把同一个数值配置在两个 backend 间 +直接搬用并期待相同含义。 + +选择流程固定为： + +1. 为每个显式候选查询 backend，得到“是否存在”、准入分数和可选的可比分数； +2. 丢弃不存在、准入分数低于 `minimum_score` 或未通过可选 Pyphen 门槛的候选； +3. 没有候选时保留原文； +4. 只有一个候选时选择它； +5. 多个候选且所有候选都有同一尺度的可比分数时，只有存在唯一最高分，且它比第二名至少高 + `minimum_score_margin` 才选择； +6. 存在不可排序候选、仍然并列或差距不足时，按 `ambiguity` 处理：`KEEP` 保留原文，`ERROR` 抛出包含 + `rule_id` 和位置但不包含正文的错误； +7. 选中候选后，把左右片段和物理边界合成一个精确替换；若选中的是自然连字符或空格形式，也会删除物理换行， + 但保留 `-` 或一个空格。 + +因此，`SPELLCHECKER + SPACED` 的两个单 token 最低频率只用于准入，不冒充短语频率。它与 `JOINED` 同时存活时属于 +不可排序歧义，不能靠 margin 自动裁决。`SPELLCHECKER + HYPHENATED` 在 backend 只能按组成 token 验证时遵循同一规则。 + +因此，“拼起来查词典，命中就合并”是自动规则的基础，但不是唯一判断。至少还要让源形式参与竞争，并对多候选歧义 +失败关闭。项目专名、缩写和已知例外仍可用更高优先级的精确/正则规则覆盖。 + +### 4.6 旧三元组兼容 + +以下调用继续有效： + +```python +mapped_line_join((("exam-", "ple", "example"),)) +``` + +旧三元组等价于一条精确规则：`left` 保持原值、`separator=""`、删除换行、区分大小写、不做 Unicode 规范化、 +启用词边界并允许四种文本块。它不会被改写成词典或正则规则。 + +旧三元组没有 `rule_id`。工厂按声明位置为审计和冲突消息生成 `legacy.0000`、`legacy.0001` 等内部 ID； +因此旧映射的声明顺序也会进入参数记录。这里保留的是调用语法和精确映射能力，不承诺继续在代码块或表格中修改， +因为本文明确把这两类区域改为始终排除。 + +旧类型别名保留： + +```python +LineJoinMapping = tuple[str, str, str] +LineJoinRule = ExactLineJoinRule | KeepLineJoinRule | RegexLineJoinRule | LineEndRegexRule | LexicalLineJoinRule +``` + +工厂签名为： + +```python +def mapped_line_join( + mappings: Iterable[LineJoinMapping | LineJoinRule], + *, + conflict_policy: LineJoinConflictPolicy = LineJoinConflictPolicy.PRIORITY, + max_intervening_blank_lines: int = 1, +) -> Modifier: + ... +``` + +保留 `max_intervening_blank_lines=1` 是为了兼容当前“可以隔一个同风格空行”的已实现行为。调用方可以设为 `0`， +禁止跨空行。这里只接受类型严格为 `int` 的 `0` 或 `1`，`bool`、负数和 `2` 以上都拒绝。两个及以上连续空行更像 +结构边界；在没有 OCR provenance、版面置信度或真实样本证据时，本轮不允许用一个任意整数把搜索扩大到更远段落。 +如果后续证据表明某个上游稳定地产生多空行误断，应以有上限的新设计明确扩大范围，而不是静默放宽本参数。 + +## 5. 匹配与替换语义 + +### 5.1 逻辑内容和锚点 + +修改器仍使用 `_text_ranges.py` 的 `PhysicalLine` 和 `physical_lines()` 获取准确范围，不改变底层工具。 + +目标文件内新增保守的块扫描，把每条物理行分成： + +- Markdown 容器/块前缀，例如 `> `、列表 marker 后的缩进或 `# `； +- 可匹配逻辑内容； +- 精确物理行尾。 + +左规则只在逻辑内容末尾匹配，右规则只在逻辑内容开头匹配。删除或替换物理边界时，右行为了表达同一块而重复的 +引用前缀或 continuation 缩进一并按第 7 节处理，不会残留在拼接词中。 + +### 5.2 精确词边界 + +`require_word_boundaries=True` 时： + +- 左匹配前一个字符不能是 Unicode 字母、数字或下划线； +- 右匹配后一个字符不能是 Unicode 字母、数字或下划线。 + +这替代当前只检查 ASCII 字母的窄规则。调用方确实要匹配词内片段时必须显式设为 `False`。正则规则不另加自动词边界， +调用方用正则本身表达边界。 + +### 5.3 空规则集和无命中 + +空映射仍创建合法的 no-op `Modifier`。没有规则命中时返回空提议，不修改换行、Unicode 或任何其他内容。 + +## 6. Markdown 块范围 + +本轮不引入 parser。目标文件内使用失败关闭的词法扫描，只为跨行候选提供最小结构边界。 + +扫描器内部必须有独立的 `UNKNOWN` 分类；它不是公共 `LineJoinBlock`，任何规则都不能选择它。`PARAGRAPH` 必须由 +“普通文本行、容器前缀一致且两侧都没有未识别块 marker”正向识别，不能作为“不像其他块”的兜底。`EXCLUDED` +用于已确认的代码和表格范围，`UNKNOWN` 用于 marker 损坏、容器深度不一致、缩进关系不足或方言结构无法确认的范围； +两者都不产生候选。 + +### 6.1 始终排除 + +- backtick 或 tilde fenced code block，包括 fence 行； +- 保守识别的四空格或 tab 缩进代码； +- 有 delimiter row 的 GFM pipe table 整个连续区域； +- 从 `` 的 raw HTML table 区域；不完整 opening 从该处排除到文档末尾。 + +围栏识别支持至多三个前导空格、相同 fence 字符和不短于 opening 的 closing。它不解释 fence 内语言。 + +### 6.2 可选块 + +| `LineJoinBlock` | 词法范围 | +| --- | --- | +| `PARAGRAPH` | 不属于其他支持块的连续普通文本行 | +| `HEADING` | ATX heading 的内容及其紧邻、没有新块 marker 的 OCR continuation；Setext underline 本身不参与匹配 | +| `LIST_ITEM` | list marker 行与达到该 item 内容列的 continuation；遇到下一个同级 marker 或块边界结束 | +| `BLOCK_QUOTE` | 具有相同 quote depth 的普通引用内容；引用内的 heading/list 使用更具体的块类型 | + +块类型取最内层可识别类型。例如引用中的列表项是 `LIST_ITEM`，引用中的普通段落是 `BLOCK_QUOTE`。 + +Markdown 允许 lazy continuation，标题后也允许紧接普通段落；仅靠文本无法总是区分这些情况。因此： + +- 规则映射仍是决定修改正确性的主要证据； +- 块分类只缩小规则适用范围，不证明词语一定应该合并； +- 无法确认同一块、容器深度不一致或前缀损坏时归入 `UNKNOWN`，不回退成 `PARAGRAPH`，也不提出修改； +- 合法且前缀对称的多层引用或嵌套列表可以正向识别；深度、item content column 或 continuation 缩进不一致时 + 失败关闭。`> > >` 和多级有序列表本身不自动等于损坏语法，测试必须同时固定合法嵌套和损坏变体。 + +## 7. 换行策略 + +一个候选的“原始边界”包括左物理行尾、允许的空行，以及右侧重复的块前缀。精确和左右正则规则先把左右匹配片段 +变成 `replacement`，再把所选边界输出放在 replacement 之后、右侧未匹配内容之前。行尾正则规则不消费右侧匹配， +只按下表替换原始边界： + +| 策略 | 输出 | +| --- | --- | +| `PRESERVE` | 保留原始物理行尾、空行和右侧块前缀 | +| `DELETE` | 删除整个边界和右侧重复前缀 | +| `SPACE` | 用一个 ASCII 空格替换整个边界和右侧重复前缀 | +| `PARAGRAPH` | 用左物理行尾样式生成恰好一个空行，并恢复右侧所需块前缀 | + +`PARAGRAPH` 在普通段落中分别产生 `EOL + EOL`；在 block quote 中产生合法的空 quote 行和下一行 quote 前缀； +在 list item 中保留 continuation 所需缩进。具体字节由合成测试固定，不根据操作系统默认换行。 + +候选内部出现混合 CR、LF、CRLF 时失败关闭，不匹配该候选。文档其他位置可以使用不同换行风格，不做全文规范化。 + +## 8. 冲突与链式合并 + +### 8.1 同一边界多规则冲突 + +每个候选边界先按调用方给出的规则顺序收集全部命中，再执行： + +| 策略 | 行为 | +| --- | --- | +| `FIRST` | 选择声明顺序最早的规则 | +| `PRIORITY` | 选择最高 `priority`；最高值相同时选择声明顺序最早的规则 | +| `ERROR` | 两条及以上规则命中即抛出 `ModifierContractError`，消息只含位置和 `rule_id` | + +自动词典规则只有在结构 pattern 命中且第 4.5 节选出了一个候选形式后才进入冲突集合；无词典候选或 `ambiguity=KEEP` +不算命中。`ambiguity=ERROR` 的词典歧义直接报错，不先交给规则冲突策略掩盖。 + +规则顺序因此是参数和行为的一部分，不再为了得到顺序无关结果而排序。`PRIORITY` 的同优先级 tie-break 明确采用 +声明顺序，不把集合迭代顺序或 replacement 字典序当作隐藏规则。 + +`KeepLineJoinRule` 与其他规则参加同一次冲突选择；它只有在赢得选择后才阻止修改。这样精确项目例外可以用更高 +优先级覆盖语言级词典规则，而不会让一条低优先级保护规则意外屏蔽更明确的替换。 + +### 8.2 跨多行链式合并 + +修改器从前到后处理物理边界，并维护只存在于当前 `propose()` 调用内的不可变中间片段。选中一条规则后： + +1. 组合该边界的虚拟输出； +2. 标记该原始物理边界已经消费； +3. 如果输出与下一原始物理行形成新的候选，继续匹配下一边界； +4. 每个原始物理边界最多消费一次，因此最多执行 `physical_line_count - 1` 次，不存在无界循环； +5. 相连边界最终合成一个覆盖原始连续范围的 `TextEdit`。 + +例如两条显式规则可以把： + +```text +exam- +ple- +based +``` + +在一次提议中变成 `example-based`。修改器不会依赖 `Pipeline` 自动运行第二轮，也不会先提交两个重叠编辑再让执行器猜测。 + +不相连的链生成不同且不重叠的 `TextEdit`。最终按原文起点排序。若内部规划仍产生重叠、无实际变化或无法映射回原始 +范围，修改器抛出契约错误，不静默丢弃整批正确候选。 + +## 9. 大小写和 Unicode 规范化 + +- `case_sensitive=True` 是默认值； +- `normalization=None` 是默认值； +- 规范化只建立匹配视图，不规范化整篇文档； +- 固定 replacement 按调用方原样输出； +- 正则反向引用使用映射回原文的捕获文本，不借匹配视图静默改变大小写或规范形式； +- 如果规范化后的匹配边界不能唯一映射回 Python 原字符串索引，该候选失败关闭； +- 兼容等价匹配、NFKC/NFKD 兼容匹配和忽略大小写都必须由规则显式开启。 + +实现可以为一个逻辑内容建立规范化视图及源索引边界表，但不能把规范化后的全文作为 `expected_text`，也不能绕过 +`TextSpan` 的原始 Python 字符索引契约。 + +## 10. 验证、错误与审计 + +工厂构造时验证： + +- 输入可迭代且每项是合法三元组或规则 dataclass； +- `rule_id` 非空、稳定、唯一； +- 枚举、布尔值、整数和块集合类型准确，不接受用真值冒充布尔值； +- 左右精确片段、separator 和 replacement 不包含 CR/LF； +- 精确规则的 `left`、`right` 和 replacement 非空；separator 可以为空； +- 正则可以编译，左右正则规则的两侧及行尾正则规则的右侧不会只产生零长度匹配； +- 捕获组均有名称、左右名称不重复、replacement 引用存在； +- 行尾正则规则不接受 `PRESERVE`； +- 自动词典规则的 separator 只能是 `"-"` 或 `""`，pattern 必须消费非空片段； +- 自动词典规则必须包含 `JOINED` 和对应源形式，候选形式不能重复； +- backend、非空语言代码、有限且非负的 `minimum_score` / `minimum_score_margin` 和可选 Pyphen 语言合法； +- 请求的 backend 或 Pyphen 不可用时明确报错，不创建行为不完整的修改器； +- `max_intervening_blank_lines` 的类型严格为 `int` 且只能是 `0` 或 `1`； +- 旧三元组仍拒绝空字段、换行和重复的 `(left, right)`。 + +运行时冲突、无法映射的内部范围和不变量破坏抛出明确异常。普通无匹配、结构排除和不能唯一映射的 Unicode 候选是 +失败关闭条件，返回无提议，不是异常。 + +每个实际链生成一个 `ProposedChange` 和一个 `TextEdit`。`reason` 包含所用 `rule_id` 序列，不记录未命中规则， +也不输出候选周围正文。`Modifier.parameters` 记录： + +- 调用顺序中的完整规则； +- 规则类型、pattern/literal、replacement、separator、块范围、换行策略、优先级、大小写和规范化选项； +- 冲突策略和允许的空行数。 + +修改器版本从 `1.0.0` 更新为 `2.0.0`，表示规则语义和参数记录发生变化。 + +## 11. 依赖和函数式边界 + +- 核心安装仍只依赖 Python 标准库； +- `pyproject.toml` 新增 `lexical` optional extra，包含受限版本的 `pyspellchecker` 和 `pyphen`； +- `wordfreq` 体积和传递依赖更大，放入单独的 `frequency` optional extra，不随 `lexical` 或核心安装； +- optional import 使用模块级 `try/except ImportError` 保存“不可用”状态；只有规则明确请求对应 backend 时才报 + `ModifierContractError`，精确和正则规则不受未安装 extra 影响； +- backend 和 Pyphen 实例在 `mapped_line_join()` 构造修改器时建立一次，不在每个文档或每个候选上重复加载词典； +- 参数记录 backend、第三方包版本、语言、阈值、候选形式和 Pyphen 语言； +- 不在 import 时读取词典、环境变量、当前时间或网络； +- 规则 dataclass、编译后的内部规则和扫描结果不可变； +- 闭包只捕获构造时验证和冻结的数据； +- `propose()` 只读取 `DocumentSnapshot` 并返回 tuple，不应用编辑或写文件。 + +`pyspellchecker` 和 `wordfreq` 自带的数据版本随包版本冻结；本轮不下载更新。Pyphen 使用其已安装包内词典。 +调用方仅安装 extra 不会自动启用任何规则，因而不会出现“环境里碰巧多了一个包，清洗结果就改变”的隐式行为。 + +PyEnchant/Hunspell backend 若以后进入运行时，需要新的 design 解决系统 provider、词典版本和可复现记录，不能在本轮 +用宽泛 `try/except` 悄悄替代已选择的 backend。 + +## 12. 文档与兼容范围 + +实现后的模块顶层 docstring 记录第 2 节的简短结论、支持模式和限制，不新增同目录 `NOTES.md`，避免形成第二份接口事实。 +具体签名和运行行为以代码与测试为准。 + +根 `README.md` 当前明确写着 `mapped_line_join()` 只使用三元组并只处理当前窄边界。实现本文后该说明会过期。 +按照 README 的事实权威要求，实施范围必须允许同步更新 README 的当前能力和示例。若仍要求 Git diff 只能包含目标源码 +和 tests，则本文不能实施。 + +本轮不承诺新 dataclass 从 `mdpolish.modifiers` 聚合模块导出。受支持导入路径是： + +```python +from mdpolish.modifiers import mapped_line_join +from mdpolish.modifiers.mapped_line_join import ( + ExactLineJoinRule, + KeepLineJoinRule, + LexicalLineJoinRule, + LineEndRegexRule, + RegexLineJoinRule, +) +``` + +旧 `mapped_line_join()` 和 `LineJoinMapping` 聚合导入保持可用。 + +## 13. 测试与验收 + +测试只使用虚构 Markdown，不读取真实或外部数据。至少覆盖： + +- 旧三元组精确匹配和新 `ExactLineJoinRule`； +- `separator="-"` 与 `separator=""`； +- 左右命名正则和跨两侧 backreference； +- `LineEndRegexRule` 的行尾锚点 + 行首正则，并确认右侧条件文本不被消费； +- 小型合成词典 backend 自动把行尾和行首组成 `JOINED`、`HYPHENATED`、`SPACED` 候选； +- 拼接词唯一命中自动合并，不需要逐词精确映射； +- 拼接词与源形式都有可比分数时按分差选择，分差不足按 `KEEP`/`ERROR` 处理； +- Pyphen 合法断点门槛允许和拒绝 `JOINED` 候选； +- 高优先级 `KeepLineJoinRule` 能否决自动词典规则； +- 请求未安装 backend 明确失败，绝不退回另一 backend 或正则猜测； +- 已安装 optional extra 时分别做最小真实 adapter 测试；未安装时明确 skip 并报告，不把 skip 写成通过； +- 段落、ATX 标题 continuation、列表 continuation 和同深度引用的正向范围； +- 合法的三层引用和正确缩进的嵌套有序列表可以合并； +- 引用深度不一致、损坏 quote 前缀、相邻同级列表项、缩进不足的嵌套列表 continuation，以及未识别的 marker-like + 结构归入内部 `UNKNOWN`，不合并且绝不回退成段落； +- heading/list/quote 前缀在四种换行策略下的准确输出； +- fenced code、缩进代码、GFM pipe table 和 raw HTML table 不合并； +- 空文档、空规则、文末无换行； +- LF、CR、CRLF 以及候选内混合行尾失败关闭； +- 隔零个或一个空行的配置，以及 `-1`、`True`、`2` 被构造期拒绝； +- 三行及以上链式合并、同一合并后逻辑行继续匹配； +- 一篇文档中的多个独立链按原文顺序报告； +- `FIRST`、`PRIORITY`、同优先级 tie-break 和 `ERROR`； +- 默认区分大小写和显式忽略大小写； +- 默认不规范化、NFC/NFD 匹配、不能唯一回映时失败关闭； +- Unicode 词边界和显式关闭词边界； +- 无效 dataclass 字段、正则、捕获组、replacement 引用和重复 `rule_id` 明确失败； +- 无效 backend、语言、候选组合、score 阈值和 margin 明确失败； +- `WORDFREQ` 候选按 Zipf margin 比较；`SPELLCHECKER` 单 token 与 `SPACED`/拆分验证的 `HYPHENATED` + 同时存活时按不可排序歧义处理，不用 margin 强行选择； +- 参数记录完整且确定； +- 所有正向输出再次运行零修改； +- 精确/正则规则不读取文件、网络、词典或环境；词典规则只读取已安装 extra 自带资源，且不访问网络或任意路径。 + +实现后运行根 README 届时列出的全部基础检查： + +```bash +.venv/bin/ruff check . +.venv/bin/mypy src tests +.venv/bin/pytest +.venv/bin/python -m pip wheel . --no-deps --wheel-dir /tmp/mdpolish-wheel-check +git diff --check +git status --short +``` + +最终报告只贴本轮真实输出。失败必须修复或明确报告，不能放宽断言掩盖。 + +## 14. 风险与代价 + +- **词法块扫描不是完整 parser：** 它能保守排除常见代码和表格，并处理约定的四类块；复杂嵌套或方言结构可能漏匹配。 +- **显式正则仍可能写错：** 库保证锚点、冲突和精确编辑安全，不保证调用方规则符合具体业务语义。 +- **heading continuation 有歧义：** 紧随标题的普通行可能本来就是段落；只有映射本身已获项目确认时才应允许 heading 规则。 +- **兼容规范化可能多对一：** 无法唯一回映时选择漏处理，不猜测源范围。 +- **链式规划增加实现复杂度：** 用每个物理边界最多消费一次和单个连续编辑限制状态空间，换取一次提议内稳定完成。 +- **规则顺序成为契约：** 这是支持 `FIRST` 以及优先级 tie-break 的必要代价，调用方必须把顺序纳入版本和评审。 +- **词典仍会漏掉专名和新词：** 自动规则对无命中和歧义失败关闭；项目可用精确规则补充，不把低覆盖伪装成全文正确。 + +## 15. 批准后的实施边界 + +批准本文将授权： + +1. 重构 `src/mdpolish/modifiers/mapped_line_join.py`； +2. 新增或扩展 `tests/test_mapped_line_join.py`，必要时只在 `tests/` 增加同主题测试文件； +3. 更新 `pyproject.toml`，只增加第 11 节批准的 optional extras； +4. 更新根 `README.md` 中已经过期的当前能力边界、安装方式和调用示例； +5. 在不改变正文的前提下检查 `AGENTS.md` 与 `CLAUDE.md` 镜像； +6. 运行第 13 节检查并查看 Git diff、wheel 内容和工作区状态。 + +批准本文不授权修改 `_text_ranges.py`、其他 modifier、其他 Wiki 文档、真实数据、外部系统， +也不授权提交、推送、创建 PR 或发布。 diff --git a/src/mdpolish/modifiers/mapped_line_join.py b/src/mdpolish/modifiers/mapped_line_join.py index b74f542..8e8e75f 100644 --- a/src/mdpolish/modifiers/mapped_line_join.py +++ b/src/mdpolish/modifiers/mapped_line_join.py @@ -1,120 +1,1374 @@ -"""Join exact caller-mapped fragments across nearby physical lines.""" +"""Build explicit cross-line joins from literals, regexes, or local lexicons. + +Pandoc separates soft-break meaning from source reflow, while mature OCR +dehyphenation combines dictionary membership, morphology, frequency, and (when +available) OCR confidence or layout metadata. Markdown linters intentionally +stay on the syntax/style side of that boundary. This modifier therefore guesses +nothing: every executable pattern, block scope, candidate form, ambiguity +choice, and local lexicon backend is supplied by the caller. + +``pyspellchecker`` supplies reproducible packaged membership/frequency data; +``wordfreq`` supplies heavier Zipf-frequency evidence; and Pyphen can only +validate a possible hyphenation position. ``wordninja`` solves the opposite +word-splitting problem, while Enchant/Hunspell depend on system dictionaries, +so those libraries are not runtime backends here. No backend is treated as +ground truth or used for edit-distance correction. + +Public rules support exact fragments, named regular expressions, a right-side +condition at a logical line end, explicit keep/veto rules, and local dictionary +candidate selection. A join may preserve the physical boundary, delete it, +replace it with one space, or emit one paragraph break. Consecutive boundaries +are planned as one non-overlapping edit, allowing one join result to participate +in the next join during the same ``propose()`` call. + +Matching is case-sensitive and performs no Unicode normalization by default. +Code blocks, tables, damaged or unknown Markdown containers, mixed candidate +line endings, and ambiguous dictionary candidates fail closed. The block +scanner is a conservative Markdown subset, not a CommonMark parser. Optional +lexicons use only resources shipped with explicitly installed packages; this +module never downloads data, opens caller files, calls a model, or silently +switches backend. +""" from __future__ import annotations -from collections import Counter -from collections.abc import Iterable +import math +import re +import unicodedata +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from enum import StrEnum +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as package_version from itertools import pairwise +from typing import Protocol, TypeAlias -from mdpolish._text_ranges import physical_lines +from mdpolish._text_ranges import PhysicalLine, physical_lines from mdpolish.models import DocumentSnapshot, ProposedChange, TextEdit, TextSpan from mdpolish.modifier import Modifier, ModifierContractError -LineJoinMapping = tuple[str, str, str] +try: + from spellchecker import SpellChecker as _SpellChecker # type: ignore[import-not-found] +except ImportError: + _SpellChecker = None + +try: + from wordfreq import zipf_frequency as _zipf_frequency # type: ignore[import-not-found] +except ImportError: + _zipf_frequency = None + +try: + from pyphen import Pyphen as _Pyphen # type: ignore[import-not-found] +except ImportError: + _Pyphen = None + + +LineJoinMapping: TypeAlias = tuple[str, str, str] + + +class LineJoinBlock(StrEnum): + """Supported lexical Markdown block scopes.""" + + PARAGRAPH = "paragraph" + HEADING = "heading" + LIST_ITEM = "list_item" + BLOCK_QUOTE = "block_quote" + + +ALL_LINE_JOIN_BLOCKS = frozenset(LineJoinBlock) + + +class LineBreakPolicy(StrEnum): + """How a selected rule renders the consumed physical boundary.""" + + PRESERVE = "preserve" + DELETE = "delete" + SPACE = "space" + PARAGRAPH = "paragraph" + + +class LineJoinConflictPolicy(StrEnum): + """How rules competing at one physical boundary are selected.""" + + PRIORITY = "priority" + FIRST = "first" + ERROR = "error" + + +class UnicodeNormalization(StrEnum): + """Explicit Unicode matching-view normalization forms.""" + + NFC = "NFC" + NFD = "NFD" + NFKC = "NFKC" + NFKD = "NFKD" + + +class LexiconBackend(StrEnum): + """Supported installed-package lexicon adapters.""" + + SPELLCHECKER = "pyspellchecker" + WORDFREQ = "wordfreq" + + +class LexicalCandidateForm(StrEnum): + """Rendered candidates compared by a lexical rule.""" + + JOINED = "joined" + HYPHENATED = "hyphenated" + SPACED = "spaced" + + +class LexicalAmbiguityPolicy(StrEnum): + """How a lexical rule handles candidates it cannot rank uniquely.""" + + KEEP = "keep" + ERROR = "error" + + +@dataclass(frozen=True, slots=True) +class ExactLineJoinRule: + """Replace exact logical-line suffix and prefix fragments.""" + + rule_id: str + left: str + right: str + replacement: str + separator: str = "" + blocks: frozenset[LineJoinBlock] = ALL_LINE_JOIN_BLOCKS + line_break: LineBreakPolicy = LineBreakPolicy.DELETE + priority: int = 0 + case_sensitive: bool = True + normalization: UnicodeNormalization | None = None + require_word_boundaries: bool = True + + +@dataclass(frozen=True, slots=True) +class KeepLineJoinRule: + """Veto lower-ranked rules at one exact fragment boundary.""" + + rule_id: str + left: str + right: str + separator: str = "" + blocks: frozenset[LineJoinBlock] = ALL_LINE_JOIN_BLOCKS + priority: int = 0 + case_sensitive: bool = True + normalization: UnicodeNormalization | None = None + require_word_boundaries: bool = True + + +@dataclass(frozen=True, slots=True) +class RegexLineJoinRule: + """Join named-regex suffix and prefix matches.""" + + rule_id: str + left_pattern: str + right_pattern: str + replacement: str + separator: str = "" + blocks: frozenset[LineJoinBlock] = ALL_LINE_JOIN_BLOCKS + line_break: LineBreakPolicy = LineBreakPolicy.DELETE + priority: int = 0 + case_sensitive: bool = True + normalization: UnicodeNormalization | None = None + + +@dataclass(frozen=True, slots=True) +class LineEndRegexRule: + """Replace a line boundary when the next logical line matches a regex.""" + + rule_id: str + right_pattern: str + separator: str = "" + blocks: frozenset[LineJoinBlock] = ALL_LINE_JOIN_BLOCKS + line_break: LineBreakPolicy = LineBreakPolicy.DELETE + priority: int = 0 + case_sensitive: bool = True + normalization: UnicodeNormalization | None = None + + +@dataclass(frozen=True, slots=True) +class LexicalLineJoinRule: + """Select an explicitly listed join form using an installed local lexicon.""" + + rule_id: str + left_pattern: str + right_pattern: str + separator: str + backend: LexiconBackend + language: str + candidate_forms: tuple[LexicalCandidateForm, ...] + minimum_score: float + minimum_score_margin: float = 0.0 + hyphenation_language: str | None = None + ambiguity: LexicalAmbiguityPolicy = LexicalAmbiguityPolicy.KEEP + blocks: frozenset[LineJoinBlock] = ALL_LINE_JOIN_BLOCKS + priority: int = 0 + case_sensitive: bool = True + normalization: UnicodeNormalization | None = None + + +LineJoinRule: TypeAlias = ( + ExactLineJoinRule | KeepLineJoinRule | RegexLineJoinRule | LineEndRegexRule | LexicalLineJoinRule +) _APPLICABILITY = ( - "处理调用方显式提供的左右片段映射；片段必须位于相邻物理行或只隔一个空行，并满足 ASCII 词边界；" - "不猜测未配置词语或段落结构。" + "按调用方显式规则合并段落、标题、列表项或引用中的跨行片段；可选本地词典只比较调用方列出的候选；" + "代码、表格、未知结构、混合候选行尾和歧义默认保持原文。" +) +_NAMED_REFERENCE = re.compile(r"\\g<(?P[A-Za-z_][A-Za-z0-9_]*)>") +_ATX_HEADING = re.compile(r"^(?P {0,3}#{1,6}[ \t]+)(?P.*)$") +_LIST_ITEM = re.compile(r"^(?P {0,3})(?P(?:[-+*]|[0-9]{1,9}[.)]))(?P[ \t]{1,4})(?P.*)$") +_FENCE_OPEN = re.compile(r"^ {0,3}(?P`{3,}|~{3,})") +_HTML_TABLE_OPEN = re.compile(r")", re.IGNORECASE) +_HTML_TABLE_CLOSE = re.compile(r"", re.IGNORECASE) +_TABLE_DELIMITER_CELL = re.compile(r"^:?-{3,}:?$") +_MARKER_LIKE = re.compile( + r"^(?: {0,3}(?:#{1,6}(?:\S|$)|[-+*](?:\S|$)|[0-9]{1,9}[.)](?:\S|$)|`{3,}|~{3,}))" ) -def _is_ascii_letter(character: str) -> bool: - return character in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +@dataclass(frozen=True, slots=True) +class _TextView: + text: str + source: str + source_boundaries: tuple[int | None, ...] + + def source_span(self, start: int, end: int) -> tuple[int, int] | None: + if start < 0 or end < start or end >= len(self.source_boundaries): + return None + source_start = self.source_boundaries[start] + source_end = self.source_boundaries[end] + if source_start is None or source_end is None: + return None + return source_start, source_end -def _validated_mappings(mappings: Iterable[LineJoinMapping]) -> tuple[LineJoinMapping, ...]: - normalized: list[LineJoinMapping] = [] - seen_pairs: set[tuple[str, str]] = set() +@dataclass(frozen=True, slots=True) +class _ScannedLine: + physical_index: int + physical: PhysicalLine + logical_start: int + logical_end: int + prefix: str + block: LineJoinBlock | None + scope: tuple[str, int] | None + + def logical_content(self, markdown: str) -> str: + return markdown[self.logical_start : self.logical_end] + + +@dataclass(frozen=True, slots=True) +class _ListContext: + quote_depth: int + marker_indent: int + content_indent: int + scope_id: int + + +@dataclass(frozen=True, slots=True) +class _Boundary: + raw: str + line_ending: str + right_prefix: str + blank_lines: int + + +@dataclass(frozen=True, slots=True) +class _LexicalEvidence: + admission_score: float + comparable_score: float | None + + +class _LexiconAdapter(Protocol): + package_version: str + + def evidence( + self, + candidate: str, + form: LexicalCandidateForm, + left: str, + right: str, + ) -> _LexicalEvidence | None: ... + + +class _Hyphenator(Protocol): + def positions(self, word: str) -> Sequence[int]: ... + + +@dataclass(frozen=True, slots=True) +class _PreparedRule: + rule: LineJoinRule + order: int + left_regex: re.Pattern[str] | None = None + right_regex: re.Pattern[str] | None = None + lexicon: _LexiconAdapter | None = None + hyphenator: _Hyphenator | None = None + package_versions: tuple[tuple[str, str], ...] = () + + +@dataclass(frozen=True, slots=True) +class _RuleMatch: + rule_id: str + order: int + priority: int + left_start: int + right_end: int + replacement: str + line_break: LineBreakPolicy + keep: bool = False + + +@dataclass(slots=True) +class _ChainState: + start: int + end: int + rendered: str + tail: str + last_line: _ScannedLine + rule_ids: list[str] + changed: bool = False + line_ending: str | None = None + + +class _SpellcheckerAdapter: + def __init__(self, language: str, case_sensitive: bool) -> None: + if _SpellChecker is None: + raise ModifierContractError("pyspellchecker backend requires the 'lexical' extra") + self.package_version = _installed_version("pyspellchecker") + try: + self._checker = _SpellChecker(language=language, case_sensitive=case_sensitive) + except Exception as error: + raise ModifierContractError("pyspellchecker could not load the requested language") from error + + def _single_score(self, token: str) -> float | None: + if token not in self._checker.known([token]): + return None + frequency = float(self._checker.word_usage_frequency(token)) + if not math.isfinite(frequency) or frequency <= 0: + return None + return math.log10(frequency * 1_000_000_000) + + def evidence( + self, + candidate: str, + form: LexicalCandidateForm, + left: str, + right: str, + ) -> _LexicalEvidence | None: + if form is not LexicalCandidateForm.SPACED: + direct_score = self._single_score(candidate) + if direct_score is not None: + return _LexicalEvidence(direct_score, direct_score) + if form is LexicalCandidateForm.JOINED: + return None + separator = " " if form is LexicalCandidateForm.SPACED else "-" + components = tuple(part for part in candidate.split(separator) if part) + if len(components) < 2: + components = (left, right) + component_scores = tuple(self._single_score(component) for component in components) + if any(score is None for score in component_scores): + return None + admission_score = min(score for score in component_scores if score is not None) + return _LexicalEvidence(admission_score, None) + + +class _WordfreqAdapter: + def __init__(self, language: str) -> None: + if _zipf_frequency is None: + raise ModifierContractError("wordfreq backend requires the 'frequency' extra") + self.package_version = _installed_version("wordfreq") + self._language = language + + def evidence( + self, + candidate: str, + form: LexicalCandidateForm, + left: str, + right: str, + ) -> _LexicalEvidence | None: + del form, left, right + assert _zipf_frequency is not None + try: + score = float(_zipf_frequency(candidate, self._language)) + except Exception as error: + raise ModifierContractError("wordfreq could not score the requested language") from error + if not math.isfinite(score) or score <= 0: + return None + return _LexicalEvidence(score, score) + + +def _installed_version(distribution: str) -> str: try: - iterator = iter(mappings) + return package_version(distribution) + except PackageNotFoundError as error: + raise ModifierContractError(f"installed optional package has no version metadata: {distribution}") from error + + +def _transform_text(text: str, normalization: UnicodeNormalization | None, case_sensitive: bool) -> str: + transformed = unicodedata.normalize(normalization.value, text) if normalization is not None else text + return transformed if case_sensitive else transformed.casefold() + + +def _text_view(text: str, normalization: UnicodeNormalization | None, case_sensitive: bool) -> _TextView: + transformed = _transform_text(text, normalization, case_sensitive) + if normalization is None and case_sensitive: + return _TextView(transformed, text, tuple(range(len(text) + 1))) + boundary_candidates: list[list[int]] = [[] for _ in range(len(transformed) + 1)] + for source_index in range(len(text) + 1): + prefix = _transform_text(text[:source_index], normalization, case_sensitive) + if not transformed.startswith(prefix): + continue + boundary_candidates[len(prefix)].append(source_index) + boundary_map = tuple(candidates[0] if len(candidates) == 1 else None for candidates in boundary_candidates) + return _TextView(transformed, text, boundary_map) + + +def _is_word_character(character: str) -> bool: + return character == "_" or character.isalnum() + + +def _word_boundaries_hold(left_text: str, left_start: int, right_text: str, right_end: int) -> bool: + return not ( + (left_start > 0 and _is_word_character(left_text[left_start - 1])) + or (right_end < len(right_text) and _is_word_character(right_text[right_end])) + ) + + +def _quote_prefix(text: str) -> tuple[int, int]: + position = 0 + depth = 0 + while True: + marker_start = position + spaces = 0 + while position < len(text) and text[position] == " " and spaces < 3: + position += 1 + spaces += 1 + if position >= len(text) or text[position] != ">": + position = marker_start + break + position += 1 + depth += 1 + if position < len(text) and text[position] in " \t": + position += 1 + return position, depth + + +def _table_delimiter(text: str) -> bool: + stripped = text.strip() + if not stripped: + return False + cells = stripped.strip("|").split("|") + return bool(cells) and all(_TABLE_DELIMITER_CELL.fullmatch(cell.strip()) is not None for cell in cells) + + +def _outer_container_content(text: str) -> str: + quote_end, _ = _quote_prefix(text) + content = text[quote_end:] + list_match = _LIST_ITEM.match(content) + return list_match.group("content") if list_match is not None else content + + +def _structural_exclusions(lines: tuple[PhysicalLine, ...], markdown: str) -> frozenset[int]: + excluded: set[int] = set() + fence_character: str | None = None + fence_length = 0 + html_table = False + for index, line in enumerate(lines): + raw = line.content(markdown) + content = _outer_container_content(raw) + if fence_character is not None: + excluded.add(index) + closing = re.match(rf"^ {{0,3}}{re.escape(fence_character)}{{{fence_length},}}[ \t]*$", content) + if closing is not None: + fence_character = None + fence_length = 0 + continue + opening = _FENCE_OPEN.match(content) + if opening is not None: + fence = opening.group("fence") + fence_character = fence[0] + fence_length = len(fence) + excluded.add(index) + continue + if html_table: + excluded.add(index) + if _HTML_TABLE_CLOSE.search(content) is not None: + html_table = False + continue + if _HTML_TABLE_OPEN.search(content) is not None: + excluded.add(index) + if _HTML_TABLE_CLOSE.search(content) is None: + html_table = True + + for index, line in enumerate(lines): + if index in excluded: + continue + raw = line.content(markdown) + content = _outer_container_content(raw) + if not _table_delimiter(content) or index == 0: + continue + previous = lines[index - 1].content(markdown) + previous_content = _outer_container_content(previous) + if "|" not in content and "|" not in previous_content: + continue + excluded.update((index - 1, index)) + following = index + 1 + while following < len(lines): + following_text = lines[following].content(markdown) + if not following_text.strip() or "|" not in following_text: + break + excluded.add(following) + following += 1 + return frozenset(excluded) + + +def _unknown_line(index: int, line: PhysicalLine) -> _ScannedLine: + return _ScannedLine(index, line, line.content_start, line.content_end, "", None, None) + + +def _scan_lines(markdown: str) -> tuple[_ScannedLine, ...]: + lines = physical_lines(markdown) + excluded = _structural_exclusions(lines, markdown) + scanned: list[_ScannedLine] = [] + list_stack: list[_ListContext] = [] + active_heading: tuple[int, int] | None = None + next_scope = 0 + blank_run = 0 + for index, line in enumerate(lines): + raw = line.content(markdown) + if line.is_blank(markdown): + blank_run += 1 + active_heading = None + if blank_run > 1: + list_stack.clear() + continue + if index in excluded: + scanned.append(_unknown_line(index, line)) + list_stack.clear() + active_heading = None + blank_run = 0 + continue + + quote_end, quote_depth = _quote_prefix(raw) + quote_prefix = raw[:quote_end] + content = raw[quote_end:] + list_match = _LIST_ITEM.match(content) + if list_match is not None: + marker_indent = len(list_match.group("indent")) + content_indent = list_match.start("content") + while list_stack and ( + list_stack[-1].quote_depth != quote_depth or marker_indent <= list_stack[-1].marker_indent + ): + list_stack.pop() + if list_stack and marker_indent < list_stack[-1].content_indent: + scanned.append(_unknown_line(index, line)) + list_stack.clear() + active_heading = None + blank_run = 0 + continue + next_scope += 1 + context = _ListContext(quote_depth, marker_indent, content_indent, next_scope) + list_stack.append(context) + logical_start = line.content_start + quote_end + content_indent + scanned.append( + _ScannedLine( + index, + line, + logical_start, + line.content_end, + markdown[line.content_start:logical_start], + LineJoinBlock.LIST_ITEM, + (LineJoinBlock.LIST_ITEM.value, context.scope_id), + ) + ) + active_heading = None + blank_run = 0 + continue + + leading_spaces = len(content) - len(content.lstrip(" ")) + continuation = next( + ( + context + for context in reversed(list_stack) + if context.quote_depth == quote_depth and leading_spaces >= context.content_indent + ), + None, + ) + if continuation is not None: + while list_stack and list_stack[-1] != continuation: + list_stack.pop() + logical_start = line.content_start + quote_end + leading_spaces + scanned.append( + _ScannedLine( + index, + line, + logical_start, + line.content_end, + markdown[line.content_start:logical_start], + LineJoinBlock.LIST_ITEM, + (LineJoinBlock.LIST_ITEM.value, continuation.scope_id), + ) + ) + active_heading = None + blank_run = 0 + continue + if list_stack: + scanned.append(_unknown_line(index, line)) + list_stack.clear() + active_heading = None + blank_run = 0 + continue + + if content.startswith("\t") or content.startswith(" "): + scanned.append(_unknown_line(index, line)) + active_heading = None + blank_run = 0 + continue + + heading_match = _ATX_HEADING.match(content) + if heading_match is not None: + next_scope += 1 + active_heading = (quote_depth, next_scope) + logical_start = line.content_start + quote_end + heading_match.end("prefix") + scanned.append( + _ScannedLine( + index, + line, + logical_start, + line.content_end, + markdown[line.content_start:logical_start], + LineJoinBlock.HEADING, + (LineJoinBlock.HEADING.value, next_scope), + ) + ) + blank_run = 0 + continue + if ( + active_heading is not None + and active_heading[0] == quote_depth + and blank_run == 0 + and not _MARKER_LIKE.match(content) + ): + scope_id = active_heading[1] + logical_start = line.content_start + quote_end + scanned.append( + _ScannedLine( + index, + line, + logical_start, + line.content_end, + quote_prefix, + LineJoinBlock.HEADING, + (LineJoinBlock.HEADING.value, scope_id), + ) + ) + blank_run = 0 + continue + + active_heading = None + if _MARKER_LIKE.match(content): + scanned.append(_unknown_line(index, line)) + elif quote_depth > 0: + logical_start = line.content_start + quote_end + scanned.append( + _ScannedLine( + index, + line, + logical_start, + line.content_end, + quote_prefix, + LineJoinBlock.BLOCK_QUOTE, + (LineJoinBlock.BLOCK_QUOTE.value, quote_depth), + ) + ) + elif raw.startswith("\t") or raw.startswith(" ") or _MARKER_LIKE.match(raw): + scanned.append(_unknown_line(index, line)) + else: + scanned.append( + _ScannedLine( + index, + line, + line.content_start, + line.content_end, + "", + LineJoinBlock.PARAGRAPH, + (LineJoinBlock.PARAGRAPH.value, 0), + ) + ) + blank_run = 0 + return tuple(scanned) + + +def _validate_common_rule(rule: LineJoinRule) -> None: + if not isinstance(rule.rule_id, str) or not rule.rule_id.strip(): + raise ModifierContractError("line join rule_id must be a non-empty string") + if type(rule.priority) is not int: + raise ModifierContractError("line join rule priority must be an integer") + if type(rule.case_sensitive) is not bool: + raise ModifierContractError("line join case_sensitive must be a boolean") + if rule.normalization is not None and not isinstance(rule.normalization, UnicodeNormalization): + raise ModifierContractError("line join normalization must be a UnicodeNormalization value") + if not isinstance(rule.blocks, frozenset) or any(not isinstance(block, LineJoinBlock) for block in rule.blocks): + raise ModifierContractError("line join blocks must be a frozenset of LineJoinBlock values") + if not isinstance(rule.separator, str) or "\r" in rule.separator or "\n" in rule.separator: + raise ModifierContractError("line join separator must be a string without line endings") + + +def _validate_literal(value: object, field_name: str, *, allow_empty: bool = False) -> str: + if not isinstance(value, str) or (not allow_empty and not value): + qualifier = "a string" if allow_empty else "a non-empty string" + raise ModifierContractError(f"{field_name} must be {qualifier}") + if "\r" in value or "\n" in value: + raise ModifierContractError(f"{field_name} cannot contain line endings") + return value + + +def _compile_pattern(pattern: object, field_name: str, *, case_sensitive: bool = True) -> re.Pattern[str]: + validated = _validate_literal(pattern, field_name) + try: + compiled = re.compile(validated, 0 if case_sensitive else re.IGNORECASE) + except re.error as error: + raise ModifierContractError(f"{field_name} must be a valid regular expression") from error + empty_match = compiled.search("") + if empty_match is not None and empty_match.start() == empty_match.end(): + raise ModifierContractError(f"{field_name} must consume text") + return compiled + + +def _validate_replacement(replacement: object, group_names: frozenset[str]) -> str: + validated = _validate_literal(replacement, "line join regex replacement") + references = tuple(_NAMED_REFERENCE.finditer(validated)) + without_references = _NAMED_REFERENCE.sub("", validated) + if "\\" in without_references: + raise ModifierContractError("line join regex replacement supports only named backreferences") + if any(match.group("name") not in group_names for match in references): + raise ModifierContractError("line join regex replacement references an unknown named group") + return validated + + +def _validate_rule(rule: LineJoinRule) -> tuple[re.Pattern[str] | None, re.Pattern[str] | None]: + _validate_common_rule(rule) + if isinstance(rule, (ExactLineJoinRule, KeepLineJoinRule)): + _validate_literal(rule.left, "line join left fragment") + _validate_literal(rule.right, "line join right fragment") + if isinstance(rule, ExactLineJoinRule): + _validate_literal(rule.replacement, "line join replacement") + if not isinstance(rule.line_break, LineBreakPolicy): + raise ModifierContractError("line join line_break must be a LineBreakPolicy value") + if type(rule.require_word_boundaries) is not bool: + raise ModifierContractError("line join require_word_boundaries must be a boolean") + return None, None + if isinstance(rule, RegexLineJoinRule): + if not isinstance(rule.line_break, LineBreakPolicy): + raise ModifierContractError("line join line_break must be a LineBreakPolicy value") + left = _compile_pattern(rule.left_pattern, "line join left_pattern", case_sensitive=rule.case_sensitive) + right = _compile_pattern(rule.right_pattern, "line join right_pattern", case_sensitive=rule.case_sensitive) + if left.groups != len(left.groupindex) or right.groups != len(right.groupindex): + raise ModifierContractError("line join regex capture groups must be named") + left_names = frozenset(left.groupindex) + right_names = frozenset(right.groupindex) + if left_names & right_names: + raise ModifierContractError("line join regex group names must be unique across both sides") + _validate_replacement(rule.replacement, left_names | right_names) + return left, right + if isinstance(rule, LineEndRegexRule): + if not isinstance(rule.line_break, LineBreakPolicy) or rule.line_break is LineBreakPolicy.PRESERVE: + raise ModifierContractError("line-end regex line_break must be DELETE, SPACE, or PARAGRAPH") + right = _compile_pattern(rule.right_pattern, "line join right_pattern", case_sensitive=rule.case_sensitive) + if right.groups != len(right.groupindex): + raise ModifierContractError("line join regex capture groups must be named") + return None, right + + if not isinstance(rule.backend, LexiconBackend): + raise ModifierContractError("line join backend must be a LexiconBackend value") + if not isinstance(rule.language, str) or not rule.language.strip(): + raise ModifierContractError("line join lexical language must be a non-empty string") + if rule.separator not in {"", "-"}: + raise ModifierContractError("line join lexical separator must be '-' or empty") + if not isinstance(rule.candidate_forms, tuple) or any( + not isinstance(form, LexicalCandidateForm) for form in rule.candidate_forms + ): + raise ModifierContractError("line join candidate_forms must be a tuple of LexicalCandidateForm values") + if len(set(rule.candidate_forms)) != len(rule.candidate_forms): + raise ModifierContractError("line join candidate_forms cannot repeat") + required_source = LexicalCandidateForm.HYPHENATED if rule.separator == "-" else LexicalCandidateForm.SPACED + if LexicalCandidateForm.JOINED not in rule.candidate_forms or required_source not in rule.candidate_forms: + raise ModifierContractError("line join lexical candidates must include JOINED and the source form") + for value, field_name in ( + (rule.minimum_score, "minimum_score"), + (rule.minimum_score_margin, "minimum_score_margin"), + ): + if not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value) or value < 0: + raise ModifierContractError(f"line join {field_name} must be a finite non-negative number") + if rule.hyphenation_language is not None and ( + not isinstance(rule.hyphenation_language, str) or not rule.hyphenation_language.strip() + ): + raise ModifierContractError("line join hyphenation_language must be a non-empty string or None") + if not isinstance(rule.ambiguity, LexicalAmbiguityPolicy): + raise ModifierContractError("line join ambiguity must be a LexicalAmbiguityPolicy value") + return ( + _compile_pattern(rule.left_pattern, "line join left_pattern", case_sensitive=rule.case_sensitive), + _compile_pattern(rule.right_pattern, "line join right_pattern", case_sensitive=rule.case_sensitive), + ) + + +def _legacy_rule(mapping: object, order: int) -> ExactLineJoinRule: + if not isinstance(mapping, tuple) or len(mapping) != 3 or any(not isinstance(value, str) for value in mapping): + raise ModifierContractError("line join mappings must be three-string tuples or rule values") + left, right, replacement = mapping + if not left or not right or not replacement: + raise ModifierContractError("legacy line join mapping fields must be non-empty") + if any(character.isspace() for value in mapping for character in value): + raise ModifierContractError("legacy line join mapping fields cannot contain whitespace") + return ExactLineJoinRule(rule_id=f"legacy.{order:04d}", left=left, right=right, replacement=replacement) + + +def _build_lexicon(rule: LexicalLineJoinRule) -> _LexiconAdapter: + if rule.backend is LexiconBackend.SPELLCHECKER: + return _SpellcheckerAdapter(rule.language, rule.case_sensitive) + return _WordfreqAdapter(rule.language) + + +def _build_hyphenator(language: str) -> tuple[_Hyphenator, str]: + if _Pyphen is None: + raise ModifierContractError("Pyphen validation requires the 'lexical' extra") + pyphen_version = _installed_version("pyphen") + try: + return _Pyphen(lang=language), pyphen_version + except Exception as error: + raise ModifierContractError("Pyphen could not load the requested language") from error + + +def _prepared_rules(items: Iterable[LineJoinMapping | LineJoinRule]) -> tuple[_PreparedRule, ...]: + try: + iterator = iter(items) except TypeError as error: raise ModifierContractError("line join mappings must be iterable") from error - for mapping in iterator: - if not isinstance(mapping, tuple) or len(mapping) != 3 or any(not isinstance(value, str) for value in mapping): - raise ModifierContractError("line join mappings must be three-string tuples") - left, right, replacement = mapping - if not left or not right or not replacement: - raise ModifierContractError("line join mapping fields must be non-empty") - if any(character.isspace() for value in mapping for character in value): - raise ModifierContractError("line join mapping fields cannot contain whitespace") - pair = (left, right) - if pair in seen_pairs: - raise ModifierContractError("line join mappings cannot repeat a fragment pair") - seen_pairs.add(pair) - normalized.append(mapping) - return tuple(sorted(normalized)) - - -def mapped_line_join(mappings: Iterable[LineJoinMapping]) -> Modifier: - """Create a modifier from exact left-fragment, right-fragment, replacement mappings.""" - normalized_mappings = _validated_mappings(mappings) - - def propose(snapshot: DocumentSnapshot) -> tuple[ProposedChange, ...]: - lines = physical_lines(snapshot.markdown) - matches: list[tuple[int, int, str, str, str]] = [] - for index, left_line in enumerate(lines[:-1]): - left_text = left_line.content(snapshot.markdown) - left_ending = left_line.line_ending(snapshot.markdown) - if not left_ending: - continue - for left, right, replacement in normalized_mappings: - if not left_text.endswith(left): - continue - left_start = left_line.content_end - len(left) - if left_start > left_line.content_start and _is_ascii_letter(snapshot.markdown[left_start - 1]): - continue - - right_index = index + 1 - if lines[right_index].is_blank(snapshot.markdown): - blank = lines[right_index] - if blank.line_ending(snapshot.markdown) != left_ending or right_index + 1 >= len(lines): - continue - right_index += 1 - if lines[right_index].is_blank(snapshot.markdown): - continue - right_line = lines[right_index] - right_text = right_line.content(snapshot.markdown) - if not right_text.startswith(right): - continue - if len(right_text) > len(right) and _is_ascii_letter(right_text[len(right)]): - continue - end = right_line.content_start + len(right) - matches.append((left_start, end, replacement, left, right)) - - matches.sort(key=lambda item: (item[0], item[1], item[2])) - span_counts = Counter(match[:2] for match in matches) - unique_matches = [match for match in matches if span_counts[match[:2]] == 1] - if any(previous[1] > current[0] for previous, current in pairwise(unique_matches)): - return () - - return tuple( - ProposedChange( - snapshot_sha256=snapshot.sha256, - reason=f"按显式映射合并跨行片段：{left} + {right} → {replacement}", - edits=( - TextEdit( - snapshot_sha256=snapshot.sha256, - span=TextSpan(start, end), - expected_text=snapshot.markdown[start:end], - replacement=replacement, - ), - ), + prepared: list[_PreparedRule] = [] + seen_ids: set[str] = set() + seen_legacy_pairs: set[tuple[str, str]] = set() + for order, item in enumerate(iterator): + if isinstance( + item, + (ExactLineJoinRule, KeepLineJoinRule, RegexLineJoinRule, LineEndRegexRule, LexicalLineJoinRule), + ): + rule = item + is_public_rule = True + else: + rule = _legacy_rule(item, order) + is_public_rule = False + if rule.rule_id in seen_ids: + raise ModifierContractError("line join rule_id values must be unique") + seen_ids.add(rule.rule_id) + if not is_public_rule: + assert isinstance(rule, ExactLineJoinRule) + pair = (rule.left, rule.right) + if pair in seen_legacy_pairs: + raise ModifierContractError("legacy line join mappings cannot repeat a fragment pair") + seen_legacy_pairs.add(pair) + left_regex, right_regex = _validate_rule(rule) + lexicon: _LexiconAdapter | None = None + hyphenator: _Hyphenator | None = None + versions: list[tuple[str, str]] = [] + if isinstance(rule, LexicalLineJoinRule): + lexicon = _build_lexicon(rule) + versions.append((rule.backend.value, lexicon.package_version)) + if rule.hyphenation_language is not None: + hyphenator, pyphen_version = _build_hyphenator(rule.hyphenation_language) + versions.append(("pyphen", pyphen_version)) + prepared.append( + _PreparedRule( + rule=rule, + order=order, + left_regex=left_regex, + right_regex=right_regex, + lexicon=lexicon, + hyphenator=hyphenator, + package_versions=tuple(versions), ) - for start, end, replacement, left, right in unique_matches + ) + return tuple(prepared) + + +def _literal_span( + text: str, + literal: str, + *, + suffix: bool, + normalization: UnicodeNormalization | None, + case_sensitive: bool, +) -> tuple[int, int] | None: + view = _text_view(text, normalization, case_sensitive) + expected = _transform_text(literal, normalization, case_sensitive) + if suffix: + if not view.text.endswith(expected): + return None + view_start, view_end = len(view.text) - len(expected), len(view.text) + else: + if not view.text.startswith(expected): + return None + view_start, view_end = 0, len(expected) + return view.source_span(view_start, view_end) + + +def _regex_match( + compiled: re.Pattern[str], + text: str, + *, + suffix: bool, + normalization: UnicodeNormalization | None, +) -> tuple[tuple[int, int], dict[str, str]] | None: + view = _text_view(text, normalization, True) + if suffix: + match = next( + ( + candidate + for start in range(len(view.text)) + if (candidate := compiled.match(view.text, start)) is not None + and candidate.end() == len(view.text) + and candidate.start() != candidate.end() + ), + None, + ) + else: + candidate = compiled.match(view.text) + match = candidate if candidate is not None and candidate.start() != candidate.end() else None + if match is None or match.start() == match.end(): + return None + source_span = view.source_span(match.start(), match.end()) + if source_span is None: + return None + captures: dict[str, str] = {} + for name in compiled.groupindex: + capture_start, capture_end = match.span(name) + if capture_start < 0: + captures[name] = "" + continue + capture_span = view.source_span(capture_start, capture_end) + if capture_span is None: + return None + captures[name] = text[capture_span[0] : capture_span[1]] + return source_span, captures + + +def _expand_replacement(replacement: str, captures: dict[str, str]) -> str: + return _NAMED_REFERENCE.sub(lambda match: captures[match.group("name")], replacement) + + +def _lexical_candidate( + prepared: _PreparedRule, + rule: LexicalLineJoinRule, + left: str, + right: str, + position: int, +) -> str | None: + assert prepared.lexicon is not None + rendered = { + LexicalCandidateForm.JOINED: left + right, + LexicalCandidateForm.HYPHENATED: left + "-" + right, + LexicalCandidateForm.SPACED: left + " " + right, + } + survivors: list[tuple[int, str, _LexicalEvidence]] = [] + for order, form in enumerate(rule.candidate_forms): + candidate = rendered[form] + query = _transform_text(candidate, rule.normalization, rule.case_sensitive) + evidence = prepared.lexicon.evidence(query, form, left, right) + if evidence is None or evidence.admission_score < rule.minimum_score: + continue + if form is LexicalCandidateForm.JOINED and prepared.hyphenator is not None: + joined_query = _transform_text(left + right, rule.normalization, rule.case_sensitive) + left_query = _transform_text(left, rule.normalization, rule.case_sensitive) + if len(left_query) not in prepared.hyphenator.positions(joined_query): + continue + survivors.append((order, candidate, evidence)) + if not survivors: + return None + if len(survivors) == 1: + return survivors[0][1] + if all(candidate[2].comparable_score is not None for candidate in survivors): + def ranking_key(candidate: tuple[int, str, _LexicalEvidence]) -> tuple[float, int]: + score = candidate[2].comparable_score + assert score is not None + return -score, candidate[0] + + ranked = sorted(survivors, key=ranking_key) + top_score = ranked[0][2].comparable_score + second_score = ranked[1][2].comparable_score + assert top_score is not None and second_score is not None + if top_score > second_score and top_score - second_score >= rule.minimum_score_margin: + return ranked[0][1] + if rule.ambiguity is LexicalAmbiguityPolicy.ERROR: + raise ModifierContractError(f"ambiguous lexical line join at position {position}: {rule.rule_id}") + return None + + +def _match_rule( + prepared: _PreparedRule, + left_text: str, + right_text: str, + block: LineJoinBlock, + position: int, +) -> _RuleMatch | None: + rule = prepared.rule + if block not in rule.blocks or not left_text.endswith(rule.separator): + return None + content_without_separator = left_text[: len(left_text) - len(rule.separator)] if rule.separator else left_text + if isinstance(rule, (ExactLineJoinRule, KeepLineJoinRule)): + left_span = _literal_span( + content_without_separator, + rule.left, + suffix=True, + normalization=rule.normalization, + case_sensitive=rule.case_sensitive, + ) + right_span = _literal_span( + right_text, + rule.right, + suffix=False, + normalization=rule.normalization, + case_sensitive=rule.case_sensitive, + ) + if left_span is None or right_span is None: + return None + if rule.require_word_boundaries and not _word_boundaries_hold( + content_without_separator, left_span[0], right_text, right_span[1] + ): + return None + return _RuleMatch( + rule_id=rule.rule_id, + order=prepared.order, + priority=rule.priority, + left_start=left_span[0], + right_end=right_span[1], + replacement=rule.replacement if isinstance(rule, ExactLineJoinRule) else "", + line_break=rule.line_break if isinstance(rule, ExactLineJoinRule) else LineBreakPolicy.PRESERVE, + keep=isinstance(rule, KeepLineJoinRule), + ) + if isinstance(rule, RegexLineJoinRule): + assert prepared.left_regex is not None and prepared.right_regex is not None + left_match = _regex_match( + prepared.left_regex, + content_without_separator, + suffix=True, + normalization=rule.normalization, + ) + right_match = _regex_match( + prepared.right_regex, + right_text, + suffix=False, + normalization=rule.normalization, + ) + if left_match is None or right_match is None: + return None + captures = left_match[1] | right_match[1] + return _RuleMatch( + rule.rule_id, + prepared.order, + rule.priority, + left_match[0][0], + right_match[0][1], + _expand_replacement(rule.replacement, captures), + rule.line_break, + ) + if isinstance(rule, LineEndRegexRule): + assert prepared.right_regex is not None + right_match = _regex_match( + prepared.right_regex, + right_text, + suffix=False, + normalization=rule.normalization, + ) + if right_match is None: + return None + return _RuleMatch( + rule.rule_id, + prepared.order, + rule.priority, + len(content_without_separator), + 0, + "", + rule.line_break, ) - records = tuple( - { - "left_fragment": left, - "replacement": replacement, - "right_fragment": right, - } - for left, right, replacement in normalized_mappings + assert prepared.left_regex is not None and prepared.right_regex is not None + left_match = _regex_match( + prepared.left_regex, + content_without_separator, + suffix=True, + normalization=rule.normalization, ) + right_match = _regex_match( + prepared.right_regex, + right_text, + suffix=False, + normalization=rule.normalization, + ) + if left_match is None or right_match is None: + return None + left = content_without_separator[left_match[0][0] : left_match[0][1]] + right = right_text[right_match[0][0] : right_match[0][1]] + replacement = _lexical_candidate(prepared, rule, left, right, position) + if replacement is None: + return None + return _RuleMatch( + rule.rule_id, + prepared.order, + rule.priority, + left_match[0][0], + right_match[0][1], + replacement, + LineBreakPolicy.DELETE, + ) + + +def _boundary_between( + markdown: str, + lines: tuple[PhysicalLine, ...], + left: _ScannedLine, + right: _ScannedLine, + max_blank_lines: int, +) -> _Boundary | None: + if right.physical_index <= left.physical_index: + raise ModifierContractError("line join scanner produced non-monotonic physical lines") + intervening = lines[left.physical_index + 1 : right.physical_index] + if len(intervening) > max_blank_lines or any(not line.is_blank(markdown) for line in intervening): + return None + ending = left.physical.line_ending(markdown) + if not ending or any(line.line_ending(markdown) != ending for line in intervening): + return None + return _Boundary( + markdown[left.physical.content_end : right.logical_start], + ending, + right.prefix, + len(intervening), + ) + + +def _compatible_blocks(left: _ScannedLine, right: _ScannedLine) -> bool: + return left.block is not None and left.block is right.block and left.scope == right.scope + + +def _select_match( + prepared_rules: tuple[_PreparedRule, ...], + left_text: str, + right_text: str, + block: LineJoinBlock, + policy: LineJoinConflictPolicy, + position: int, +) -> _RuleMatch | None: + matches = tuple( + match + for prepared in prepared_rules + if (match := _match_rule(prepared, left_text, right_text, block, position)) is not None + ) + if not matches: + return None + if policy is LineJoinConflictPolicy.ERROR and len(matches) > 1: + rule_ids = ", ".join(match.rule_id for match in matches) + raise ModifierContractError(f"conflicting line join rules at position {position}: {rule_ids}") + if policy is LineJoinConflictPolicy.FIRST: + return matches[0] + return min(matches, key=lambda match: (-match.priority, match.order)) + + +def _paragraph_boundary(block: LineJoinBlock, ending: str, right_prefix: str) -> str: + quote_end, quote_depth = _quote_prefix(right_prefix) + if block is LineJoinBlock.BLOCK_QUOTE or quote_depth > 0: + return ending + right_prefix[:quote_end].rstrip() + ending + right_prefix + return ending + ending + right_prefix + + +def _render_boundary(policy: LineBreakPolicy, boundary: _Boundary, block: LineJoinBlock) -> str: + if policy is LineBreakPolicy.PRESERVE: + return boundary.raw + if policy is LineBreakPolicy.DELETE: + return "" + if policy is LineBreakPolicy.SPACE: + return " " + return _paragraph_boundary(block, boundary.line_ending, boundary.right_prefix) + + +def _apply_match( + state: _ChainState, + match: _RuleMatch, + boundary: _Boundary, + right: _ScannedLine, + right_text: str, + block: LineJoinBlock, +) -> None: + rendered_prefix = state.rendered[: -len(state.tail)] if state.tail else state.rendered + left_before = state.tail[: match.left_start] + right_after = right_text[match.right_end :] + rendered_boundary = _render_boundary(match.line_break, boundary, block) + state.rendered = rendered_prefix + left_before + match.replacement + rendered_boundary + right_after + if "\r" in rendered_boundary or "\n" in rendered_boundary: + state.tail = right_after + else: + state.tail = left_before + match.replacement + rendered_boundary + right_after + state.end = right.logical_end + state.last_line = right + state.rule_ids.append(match.rule_id) + state.changed = True + if state.line_ending is None: + state.line_ending = boundary.line_ending + + +def _proposal(snapshot: DocumentSnapshot, state: _ChainState) -> ProposedChange: + expected = snapshot.markdown[state.start : state.end] + if expected == state.rendered: + raise ModifierContractError("line join planning produced an unchanged edit") + return ProposedChange( + snapshot_sha256=snapshot.sha256, + reason=f"按显式跨行规则合并片段：{', '.join(state.rule_ids)}", + edits=( + TextEdit( + snapshot_sha256=snapshot.sha256, + span=TextSpan(state.start, state.end), + expected_text=expected, + replacement=state.rendered, + ), + ), + ) + + +def _rule_record(prepared: _PreparedRule) -> dict[str, object]: + rule = prepared.rule + record: dict[str, object] = { + "blocks": tuple(sorted(block.value for block in rule.blocks)), + "case_sensitive": rule.case_sensitive, + "normalization": rule.normalization.value if rule.normalization is not None else None, + "order": prepared.order, + "priority": rule.priority, + "rule_id": rule.rule_id, + "rule_type": type(rule).__name__, + "separator": rule.separator, + } + if isinstance(rule, (ExactLineJoinRule, KeepLineJoinRule)): + record.update( + { + "left": rule.left, + "require_word_boundaries": rule.require_word_boundaries, + "right": rule.right, + } + ) + if isinstance(rule, ExactLineJoinRule): + record.update({"line_break": rule.line_break.value, "replacement": rule.replacement}) + elif isinstance(rule, RegexLineJoinRule): + record.update( + { + "left_pattern": rule.left_pattern, + "line_break": rule.line_break.value, + "replacement": rule.replacement, + "right_pattern": rule.right_pattern, + } + ) + elif isinstance(rule, LineEndRegexRule): + record.update({"line_break": rule.line_break.value, "right_pattern": rule.right_pattern}) + else: + record.update( + { + "ambiguity": rule.ambiguity.value, + "backend": rule.backend.value, + "candidate_forms": tuple(form.value for form in rule.candidate_forms), + "hyphenation_language": rule.hyphenation_language, + "language": rule.language, + "left_pattern": rule.left_pattern, + "minimum_score": float(rule.minimum_score), + "minimum_score_margin": float(rule.minimum_score_margin), + "package_versions": prepared.package_versions, + "right_pattern": rule.right_pattern, + } + ) + return record + + +def mapped_line_join( + mappings: Iterable[LineJoinMapping | LineJoinRule], + *, + conflict_policy: LineJoinConflictPolicy = LineJoinConflictPolicy.PRIORITY, + max_intervening_blank_lines: int = 1, +) -> Modifier: + """Create a deterministic modifier from explicit cross-line join rules.""" + if not isinstance(conflict_policy, LineJoinConflictPolicy): + raise ModifierContractError("conflict_policy must be a LineJoinConflictPolicy value") + if type(max_intervening_blank_lines) is not int or max_intervening_blank_lines not in {0, 1}: + raise ModifierContractError("max_intervening_blank_lines must be integer 0 or 1") + rules = _prepared_rules(mappings) + + def propose(snapshot: DocumentSnapshot) -> tuple[ProposedChange, ...]: + markdown = snapshot.markdown + physical = physical_lines(markdown) + scanned = _scan_lines(markdown) + if len(scanned) < 2 or not rules: + return () + proposals: list[ProposedChange] = [] + first = scanned[0] + first_content = first.logical_content(markdown) + state = _ChainState(first.logical_start, first.logical_end, first_content, first_content, first, []) + for right in scanned[1:]: + boundary = _boundary_between( + markdown, + physical, + state.last_line, + right, + max_intervening_blank_lines, + ) + selected: _RuleMatch | None = None + if boundary is not None and _compatible_blocks(state.last_line, right): + assert right.block is not None + selected = _select_match( + rules, + state.tail, + right.logical_content(markdown), + right.block, + conflict_policy, + state.last_line.physical.content_end, + ) + if selected is not None and not selected.keep: + assert boundary is not None and right.block is not None + if state.line_ending is not None and state.line_ending != boundary.line_ending: + right_content = right.logical_content(markdown) + state = _ChainState( + right.logical_start, + right.logical_end, + right_content, + right_content, + right, + [], + ) + continue + _apply_match( + state, + selected, + boundary, + right, + right.logical_content(markdown), + right.block, + ) + continue + if state.changed: + proposals.append(_proposal(snapshot, state)) + right_content = right.logical_content(markdown) + state = _ChainState(right.logical_start, right.logical_end, right_content, right_content, right, []) + if state.changed: + proposals.append(_proposal(snapshot, state)) + if any( + previous.edits[0].span.end > current.edits[0].span.start + for previous, current in pairwise(proposals) + ): + raise ModifierContractError("line join planning produced overlapping edits") + return tuple(proposals) + return Modifier( modifier_id="markdown.mapped_line_join", - version="1.0.0", - parameters={"mappings": records}, + version="2.0.0", + parameters={ + "conflict_policy": conflict_policy.value, + "max_intervening_blank_lines": max_intervening_blank_lines, + "rules": tuple(_rule_record(rule) for rule in rules), + }, applicability=_APPLICABILITY, propose=propose, ) diff --git a/tests/test_mapped_line_join.py b/tests/test_mapped_line_join.py index 31b104e..24d916e 100644 --- a/tests/test_mapped_line_join.py +++ b/tests/test_mapped_line_join.py @@ -1,19 +1,62 @@ from __future__ import annotations +from importlib import import_module +from typing import TYPE_CHECKING, cast + import pytest -from mdpolish import ModifierContractError, Pipeline, RunStatus +from mdpolish import DocumentSnapshot, ModifierContractError, Pipeline, RunStatus from mdpolish.modifiers import mapped_line_join +from mdpolish.modifiers.mapped_line_join import ( + ExactLineJoinRule, + KeepLineJoinRule, + LexicalAmbiguityPolicy, + LexicalCandidateForm, + LexicalLineJoinRule, + LexiconBackend, + LineBreakPolicy, + LineEndRegexRule, + LineJoinBlock, + LineJoinConflictPolicy, + LineJoinRule, + RegexLineJoinRule, + UnicodeNormalization, +) -MAPPINGS = ( +if TYPE_CHECKING: + from _pytest.monkeypatch import MonkeyPatch + +mapped_module = import_module("mdpolish.modifiers.mapped_line_join") + +LEGACY_MAPPINGS = ( ("exam-", "ple", "example"), ("rule-", "based", "rule-based"), ("value.", "ues", "values"), ) -def transform(markdown: str): # type: ignore[no-untyped-def] - return Pipeline([mapped_line_join(MAPPINGS)]).transform(markdown) +def transform( + markdown: str, + rules: tuple[tuple[str, str, str] | LineJoinRule, ...] = LEGACY_MAPPINGS, + **options: object, +) -> str: + modifier = mapped_line_join(rules, **options) # type: ignore[arg-type] + result = Pipeline((modifier,)).transform(markdown) + assert result.status is RunStatus.SUCCESS + assert result.output_markdown is not None + return result.output_markdown + + +def exact_rule(**overrides: object) -> ExactLineJoinRule: + fields: dict[str, object] = { + "rule_id": "example.join", + "left": "exam", + "right": "ple", + "replacement": "example", + "separator": "-", + } + fields.update(overrides) + return ExactLineJoinRule(**fields) # type: ignore[arg-type] @pytest.mark.parametrize( @@ -21,70 +64,493 @@ def transform(markdown: str): # type: ignore[no-untyped-def] [ ("an exam-\nple here", "an example here"), ("an exam-\n\nple here", "an example here"), - ("a rule-\nbased method", "a rule-based method"), - ("the value.\n\nues differ", "the values differ"), ("an exam-\r\n\r\nple here", "an example here"), ("an exam-\r\rple here", "an example here"), + ("a rule-\nbased method", "a rule-based method"), + ("the value.\nues differ", "the values differ"), ], ) -def test_applies_exact_mapping_across_supported_line_shapes(markdown: str, expected: str) -> None: - result = transform(markdown) - assert result.status is RunStatus.SUCCESS - assert result.output_markdown == expected - assert len(result.changes) == 1 +def test_legacy_exact_mapping_supports_lf_crlf_cr_and_one_blank_line(markdown: str, expected: str) -> None: + assert transform(markdown) == expected + + +def test_exact_rules_distinguish_explicit_and_empty_separator() -> None: + hyphenated = exact_rule() + unseparated = exact_rule(rule_id="example.unseparated", separator="") + + assert transform("exam-\nple", (hyphenated,)) == "example" + assert transform("exam\nple", (unseparated,)) == "example" + assert transform("exam\nple", (hyphenated,)) == "exam\nple" + + +def test_named_regex_combines_backreferences_from_both_sides() -> None: + rule = RegexLineJoinRule( + rule_id="regex.join", + left_pattern=r"(?P[A-Za-z]+)", + right_pattern=r"(?P[a-z]+)", + replacement=r"\g_\g", + separator="-", + ) + + assert transform("prefix exam-\nple suffix", (rule,)) == "prefix exam_ple suffix" + + +def test_regex_keeps_its_inline_flags() -> None: + rule = RegexLineJoinRule( + rule_id="regex.flags", + left_pattern=r"(?i:(?Pexam))", + right_pattern=r"(?i:(?Pple))", + replacement=r"\g\g", + separator="-", + ) + + assert transform("EXAM-\nPLE", (rule,)) == "EXAMPLE" + + +def test_regex_case_insensitive_option_applies_to_the_pattern() -> None: + rule = RegexLineJoinRule( + rule_id="regex.ignore-case", + left_pattern=r"(?PEXAM)", + right_pattern=r"(?PPLE)", + replacement=r"\g\g", + separator="-", + case_sensitive=False, + ) + + assert transform("exam-\nple", (rule,)) == "example" + + +def test_regex_suffix_matching_considers_overlapping_starts() -> None: + rule = RegexLineJoinRule( + rule_id="regex.overlap", + left_pattern=r"(?Paba)", + right_pattern=r"(?Px)", + replacement=r"\g\g", + separator="-", + ) + + assert transform("ababa-\nx", (rule,)) == "ababax" + + +def test_line_end_regex_uses_right_match_only_as_a_condition() -> None: + rule = LineEndRegexRule( + rule_id="line-end.space", + right_pattern=r"(?P[a-z])", + line_break=LineBreakPolicy.SPACE, + ) + + assert transform("foo\nbar", (rule,)) == "foo bar" + + +@pytest.mark.parametrize( + ("policy", "expected"), + [ + (LineBreakPolicy.PRESERVE, "example\n"), + (LineBreakPolicy.DELETE, "example"), + (LineBreakPolicy.SPACE, "example "), + (LineBreakPolicy.PARAGRAPH, "example\n\n"), + ], +) +def test_all_line_break_policies_are_exact(policy: LineBreakPolicy, expected: str) -> None: + assert transform("exam-\nple", (exact_rule(line_break=policy),)) == expected + + +def test_paragraph_policy_restores_quote_and_list_prefixes() -> None: + rule = exact_rule(line_break=LineBreakPolicy.PARAGRAPH) + + assert transform("> exam-\n> ple", (rule,)) == "> example\n>\n> " + assert transform("- exam-\n ple", (rule,)) == "- example\n\n " + assert transform("> - exam-\n> ple", (rule,)) == "> - example\n>\n> " + + +@pytest.mark.parametrize( + ("markdown", "expected"), + [ + ("plain exam-\nple", "plain example"), + ("# exam-\nple", "# example"), + ("- exam-\n ple", "- example"), + ("> exam-\n> ple", "> example"), + ("> > > exam-\n> > > ple", "> > > example"), + ("1. outer\n 2. exam-\n ple", "1. outer\n 2. example"), + ], +) +def test_supported_markdown_blocks_and_valid_nested_containers(markdown: str, expected: str) -> None: + assert transform(markdown, (exact_rule(),)) == expected + + +def test_block_scope_is_explicit() -> None: + heading_only = exact_rule(blocks=frozenset({LineJoinBlock.HEADING})) + + assert transform("# exam-\nple", (heading_only,)) == "# example" + assert transform("exam-\nple", (heading_only,)) == "exam-\nple" @pytest.mark.parametrize( "markdown", [ - "an unknown-\nword here", - "an exam-\n\n\nple here", - "an exam-\r\n\nple here", - "an EXAM-\nple here", - "an exam-\nplemore here", - "an xrule-\nbased method", + "> > > exam-\n> > ple", + "> #broken exam-\n> ple", + "1. exam-\n2. ple", + "1. outer\n 2. exam-\n ple", + "- exam-\n ple", + "#broken exam-\nple", ], ) -def test_unknown_or_unsafe_boundaries_are_preserved(markdown: str) -> None: - assert transform(markdown).output_markdown == markdown - - -def test_parameters_and_results_are_independent_of_mapping_order() -> None: - first = mapped_line_join(MAPPINGS) - second = mapped_line_join(reversed(MAPPINGS)) - markdown = "example becomes exam-\nple" - - first_result = Pipeline([first]).transform(markdown) - second_result = Pipeline([second]).transform(markdown) - - assert first_result.modifiers == second_result.modifiers - assert first_result.output_markdown == second_result.output_markdown - - -def test_library_contains_no_default_mapping() -> None: - modifier = mapped_line_join(()) - result = Pipeline([modifier]).transform("an exam-\nple here") - - assert modifier.parameters == (("mappings", ()),) - assert result.output_markdown == "an exam-\nple here" +def test_unknown_or_mismatched_containers_fail_closed_instead_of_becoming_paragraphs(markdown: str) -> None: + assert transform(markdown, (exact_rule(),)) == markdown @pytest.mark.parametrize( - "mappings", + "markdown", + [ + "```text\nexam-\nple\n```", + " exam-\n ple", + "> exam-\n> ple", + "- ```\n exam-\n ple\n ```", + "| word |\n| --- |\n| exam- |\n| ple |", + "\nexam-\nple\n
", + "\nexam-\nple", + ], +) +def test_code_and_table_regions_are_always_excluded(markdown: str) -> None: + assert transform(markdown, (exact_rule(),)) == markdown + + +def test_three_line_chain_is_one_proposal_and_uses_virtual_output() -> None: + rules = ( + exact_rule(rule_id="chain.first"), + ExactLineJoinRule( + rule_id="chain.second", + left="example", + right="based", + replacement="example-based", + separator="-", + ), + ) + modifier = mapped_line_join(rules) + proposals = modifier.propose(DocumentSnapshot("exam-\nple-\nbased")) + + assert len(proposals) == 1 + assert len(proposals[0].edits) == 1 + assert "chain.first, chain.second" in proposals[0].reason + assert transform("exam-\nple-\nbased", rules) == "example-based" + + +def test_mixed_line_endings_fail_the_entire_connected_chain_closed() -> None: + rules = ( + exact_rule(rule_id="chain.first"), + ExactLineJoinRule("chain.second", "example", "based", "example-based", separator="-"), + ) + markdown = "exam-\r\nple-\nbased" + + assert transform(markdown, rules) == markdown + + +def test_multiple_independent_chains_are_reported_in_source_order() -> None: + modifier = mapped_line_join((exact_rule(),)) + snapshot = DocumentSnapshot("exam-\nple and\n\nexam-\nple") + proposals = modifier.propose(snapshot) + + assert len(proposals) == 2 + assert proposals[0].edits[0].span.start < proposals[1].edits[0].span.start + + +def test_keep_rule_can_veto_a_lower_priority_replacement() -> None: + keep = KeepLineJoinRule("keep.example", "exam", "ple", separator="-", priority=10) + replace = exact_rule(priority=0) + + assert transform("exam-\nple", (replace, keep)) == "exam-\nple" + + +def test_conflict_first_and_priority_are_deterministic() -> None: + first = exact_rule(rule_id="first", replacement="first", priority=0) + second = exact_rule(rule_id="second", replacement="second", priority=10) + + assert transform( + "exam-\nple", + (first, second), + conflict_policy=LineJoinConflictPolicy.FIRST, + ) == "first" + assert transform( + "exam-\nple", + (first, second), + conflict_policy=LineJoinConflictPolicy.PRIORITY, + ) == "second" + + +def test_priority_tie_uses_declaration_order() -> None: + first = exact_rule(rule_id="first", replacement="first") + second = exact_rule(rule_id="second", replacement="second") + + assert transform("exam-\nple", (second, first)) == "second" + + +def test_conflict_error_names_rules_but_not_surrounding_text() -> None: + modifier = mapped_line_join( + (exact_rule(rule_id="first"), exact_rule(rule_id="second")), + conflict_policy=LineJoinConflictPolicy.ERROR, + ) + + with pytest.raises(ModifierContractError, match=r"first, second") as raised: + modifier.propose(DocumentSnapshot("secret exam-\nple material")) + assert "secret" not in str(raised.value) + + +def test_case_insensitive_matching_is_opt_in() -> None: + assert transform("EXAM-\nPLE", (exact_rule(),)) == "EXAM-\nPLE" + assert transform("EXAM-\nPLE", (exact_rule(case_sensitive=False),)) == "example" + + +def test_unicode_normalization_is_opt_in_and_preserves_source_index_mapping() -> None: + decomposed_left = "cafe\N{COMBINING ACUTE ACCENT}" + markdown = "CAFÉ-\nteria" + default = ExactLineJoinRule( + "unicode.default", + decomposed_left, + "teria", + "cafeteria", + separator="-", + case_sensitive=False, + ) + normalized = ExactLineJoinRule( + "unicode.nfc", + decomposed_left, + "teria", + "cafeteria", + separator="-", + case_sensitive=False, + normalization=UnicodeNormalization.NFC, + ) + + assert transform(markdown, (default,)) == markdown + assert transform(markdown, (normalized,)) == "cafeteria" + + +def test_unicode_word_boundaries_and_explicit_override() -> None: + strict = exact_rule() + permissive = exact_rule(rule_id="permissive", require_word_boundaries=False) + markdown = "éexam-\nplemore" + + assert transform(markdown, (strict,)) == markdown + assert transform(markdown, (permissive,)) == "éexamplemore" + + +@pytest.mark.parametrize("max_blank_lines", [0, 1]) +def test_blank_line_limit_is_configurable(max_blank_lines: int) -> None: + expected = "exam-\n\nple" if max_blank_lines == 0 else "example" + assert transform( + "exam-\n\nple", + (exact_rule(),), + max_intervening_blank_lines=max_blank_lines, + ) == expected + + +@pytest.mark.parametrize("invalid", [-1, True, 2]) +def test_invalid_blank_line_limits_are_rejected(invalid: object) -> None: + with pytest.raises(ModifierContractError): + mapped_line_join((), max_intervening_blank_lines=invalid) # type: ignore[arg-type] + + +def test_empty_document_and_empty_rule_set_are_no_ops() -> None: + modifier = mapped_line_join(()) + + assert transform("", ()) == "" + assert transform("exam-\nple", ()) == "exam-\nple" + assert modifier.version == "2.0.0" + assert dict(modifier.parameters)["rules"] == () + + +def test_parameters_preserve_rule_order_and_record_all_options() -> None: + first = mapped_line_join((exact_rule(rule_id="first"), exact_rule(rule_id="second"))) + second = mapped_line_join((exact_rule(rule_id="second"), exact_rule(rule_id="first"))) + + assert first.parameters != second.parameters + records = dict(first.parameters)["rules"] + assert isinstance(records, tuple) + first_record = cast(tuple[tuple[str, object], ...], records[0]) + assert ("rule_id", "first") in first_record + + +@pytest.mark.parametrize( + "rules", [ (("", "right", "word"),), (("left", "right", "two words"),), (("left", "right", "word"), ("left", "right", "other")), (("left", "right"),), + (exact_rule(rule_id="same"), exact_rule(rule_id="same")), + (exact_rule(case_sensitive=1),), + (exact_rule(blocks={LineJoinBlock.PARAGRAPH}),), + (RegexLineJoinRule("regex", "", r"(?Px)", "x"),), + (RegexLineJoinRule("regex", r"(x)", r"(?Px)", "x"),), + (RegexLineJoinRule("regex", r"(?Px)", r"(?Px)", r"\g"),), + (RegexLineJoinRule("regex", r"(?Px)", r"(?Px)", r"\g"),), + (LineEndRegexRule("line-end", r"(?Px)", line_break=LineBreakPolicy.PRESERVE),), ], ) -def test_invalid_mappings_raise_contract_error(mappings: object) -> None: +def test_invalid_rules_raise_contract_error(rules: object) -> None: with pytest.raises(ModifierContractError): - mapped_line_join(mappings) # type: ignore[arg-type] + mapped_line_join(rules) # type: ignore[arg-type] + + +class FakeLexicon: + package_version = "test-lexicon" + + def __init__(self, scores: dict[str, tuple[float, float | None]]) -> None: + self._scores = scores + + def evidence( + self, + candidate: str, + form: LexicalCandidateForm, + left: str, + right: str, + ) -> object: + del form, left, right + score = self._scores.get(candidate) + return None if score is None else mapped_module._LexicalEvidence(*score) + + +def lexical_rule(**overrides: object) -> LexicalLineJoinRule: + fields: dict[str, object] = { + "rule_id": "english.lexical", + "left_pattern": r"[A-Za-z]+", + "right_pattern": r"[a-z]+", + "separator": "-", + "backend": LexiconBackend.WORDFREQ, + "language": "en", + "candidate_forms": (LexicalCandidateForm.JOINED, LexicalCandidateForm.HYPHENATED), + "minimum_score": 1.0, + "minimum_score_margin": 0.5, + } + fields.update(overrides) + return LexicalLineJoinRule(**fields) # type: ignore[arg-type] + + +def install_fake_lexicon(monkeypatch: MonkeyPatch, scores: dict[str, tuple[float, float | None]]) -> None: + def build(rule: LexicalLineJoinRule) -> FakeLexicon: + del rule + return FakeLexicon(scores) + + monkeypatch.setattr(mapped_module, "_build_lexicon", build) + + +def test_lexical_rule_joins_a_unique_dictionary_candidate_without_per_word_mapping( + monkeypatch: MonkeyPatch, +) -> None: + install_fake_lexicon(monkeypatch, {"example": (5.0, 5.0)}) + + assert transform("exam-\nple", (lexical_rule(),)) == "example" + + +def test_lexical_rule_can_select_the_natural_hyphenated_form_by_score(monkeypatch: MonkeyPatch) -> None: + install_fake_lexicon(monkeypatch, {"rulebased": (3.0, 3.0), "rule-based": (6.0, 6.0)}) + + assert transform("rule-\nbased", (lexical_rule(),)) == "rule-based" + + +def test_lexical_margin_and_keep_policy_preserve_an_ambiguous_boundary(monkeypatch: MonkeyPatch) -> None: + install_fake_lexicon(monkeypatch, {"rulebased": (5.0, 5.0), "rule-based": (4.8, 4.8)}) + + assert transform("rule-\nbased", (lexical_rule(minimum_score_margin=0.5),)) == "rule-\nbased" + + +def test_spellchecker_style_spaced_proxy_is_not_ranked_against_single_token(monkeypatch: MonkeyPatch) -> None: + install_fake_lexicon(monkeypatch, {"inside": (6.0, 6.0), "in side": (5.0, None)}) + rule = lexical_rule( + separator="", + candidate_forms=(LexicalCandidateForm.JOINED, LexicalCandidateForm.SPACED), + ) + + assert transform("in\nside", (rule,)) == "in\nside" + + +def test_lexical_ambiguity_error_is_source_safe(monkeypatch: MonkeyPatch) -> None: + install_fake_lexicon(monkeypatch, {"inside": (6.0, 6.0), "in side": (5.0, None)}) + rule = lexical_rule( + separator="", + candidate_forms=(LexicalCandidateForm.JOINED, LexicalCandidateForm.SPACED), + ambiguity=LexicalAmbiguityPolicy.ERROR, + ) + modifier = mapped_line_join((rule,)) + + with pytest.raises(ModifierContractError, match=r"english\.lexical") as raised: + modifier.propose(DocumentSnapshot("secret in\nside")) + assert "secret" not in str(raised.value) + + +class FakeHyphenator: + def __init__(self, positions: tuple[int, ...]) -> None: + self._positions = positions + + def positions(self, word: str) -> tuple[int, ...]: + del word + return self._positions + + +@pytest.mark.parametrize(("positions", "expected"), [((4,), "example"), ((), "exam-ple")]) +def test_pyphen_gate_only_controls_the_joined_candidate( + monkeypatch: MonkeyPatch, + positions: tuple[int, ...], + expected: str, +) -> None: + install_fake_lexicon(monkeypatch, {"example": (6.0, 6.0), "exam-ple": (4.0, 4.0)}) + monkeypatch.setattr( + mapped_module, + "_build_hyphenator", + lambda language: (FakeHyphenator(positions), f"test-{language}"), + ) + rule = lexical_rule(hyphenation_language="en_US", minimum_score_margin=0.5) + + assert transform("exam-\nple", (rule,)) == expected + + +@pytest.mark.parametrize( + "overrides", + [ + {"separator": "/"}, + {"candidate_forms": (LexicalCandidateForm.JOINED,)}, + {"candidate_forms": (LexicalCandidateForm.JOINED, LexicalCandidateForm.JOINED)}, + {"minimum_score": -1.0}, + {"minimum_score": float("nan")}, + {"minimum_score_margin": True}, + {"language": ""}, + {"hyphenation_language": ""}, + ], +) +def test_invalid_lexical_rules_fail_before_loading_a_backend(overrides: dict[str, object]) -> None: + with pytest.raises(ModifierContractError): + mapped_line_join((lexical_rule(**overrides),)) + + +def test_missing_requested_optional_backend_raises_without_fallback(monkeypatch: MonkeyPatch) -> None: + monkeypatch.setattr(mapped_module, "_zipf_frequency", None) + + with pytest.raises(ModifierContractError, match="frequency"): + mapped_line_join((lexical_rule(),)) + + +@pytest.mark.parametrize( + ("backend", "available"), + [ + (LexiconBackend.SPELLCHECKER, mapped_module._SpellChecker is not None), + (LexiconBackend.WORDFREQ, mapped_module._zipf_frequency is not None), + ], +) +def test_installed_optional_backend_constructs_or_is_reported_as_skipped( + backend: LexiconBackend, + available: bool, +) -> None: + if not available: + pytest.skip(f"optional backend is not installed: {backend.value}") + mapped_line_join((lexical_rule(backend=backend),)) def test_successful_output_is_idempotent() -> None: - pipeline = Pipeline([mapped_line_join(MAPPINGS)]) + rules = (exact_rule(),) + pipeline = Pipeline((mapped_line_join(rules),)) first = pipeline.transform("an exam-\n\nple here") + + assert first.status is RunStatus.SUCCESS assert first.output_markdown is not None assert pipeline.transform(first.output_markdown).changes == ()