Bepr4
c19e9fcebf
- 8 条 YAML 声明规则：页码/页眉页脚/目录点线/图片/HTML表格/散落标签/行尾空白/空行 - 防误伤设计：protect 正则 + 内容形态豁免 + OCR burst 检测 - md-clean single/batch CLI，JSON 清洗报告 - 18 个单元测试 Co-Authored-By: Claude <noreply@anthropic.com>
57 lines
1.9 KiB
Python
57 lines
1.9 KiB
Python
"""清洗引擎：按规则顺序应用，输出清洗后文本 + 统计报告。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import difflib
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional
|
|
|
|
from cleaner.rules import REGISTRY, Rule, load_rules
|
|
|
|
|
|
@dataclass
|
|
class CleanResult:
|
|
"""一次清洗的结果：产物 + 可审计的统计。"""
|
|
|
|
text: str
|
|
stats: Dict[str, int] = field(default_factory=dict)
|
|
rules_applied: List[str] = field(default_factory=list)
|
|
|
|
@property
|
|
def total_hits(self) -> int:
|
|
return sum(v for k, v in self.stats.items() if k != "collapsed_blanks")
|
|
|
|
|
|
class MarkdownCleaner:
|
|
def __init__(self, rules: Optional[List[Rule]] = None, rules_path: Optional[Path] = None):
|
|
self.rules = rules if rules is not None else load_rules(rules_path)
|
|
|
|
def clean_text(self, text: str) -> CleanResult:
|
|
stats: Dict[str, int] = {}
|
|
applied: List[str] = []
|
|
for rule in self.rules:
|
|
if not rule.enabled:
|
|
continue
|
|
func = REGISTRY[rule.name]
|
|
text = func(text, rule.params, stats)
|
|
applied.append(rule.name)
|
|
return CleanResult(text=text, stats=stats, rules_applied=applied)
|
|
|
|
def clean_file(self, src: Path, dst: Optional[Path] = None) -> CleanResult:
|
|
raw = Path(src).read_text(encoding="utf-8")
|
|
result = self.clean_text(raw)
|
|
if dst is not None:
|
|
Path(dst).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(dst).write_text(result.text, encoding="utf-8")
|
|
return result
|
|
|
|
@staticmethod
|
|
def diff(before: str, after: str, context: int = 1) -> str:
|
|
return "\n".join(
|
|
difflib.unified_diff(
|
|
before.splitlines(), after.splitlines(),
|
|
fromfile="before", tofile="after", lineterm="", n=context,
|
|
)
|
|
)
|