govdoc-md-cleaner v0.1.0: 政务文档 Markdown 清洗工具(规则引擎 + CLI + 测试)
- 8 条 YAML 声明规则:页码/页眉页脚/目录点线/图片/HTML表格/散落标签/行尾空白/空行 - 防误伤设计:protect 正则 + 内容形态豁免 + OCR burst 检测 - md-clean single/batch CLI,JSON 清洗报告 - 18 个单元测试 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
"""清洗引擎:按规则顺序应用,输出清洗后文本 + 统计报告。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import difflib
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from cleaner.rules import REGISTRY, Rule, load_rules
|
||||
|
||||
|
||||
@dataclass
|
||||
class CleanResult:
|
||||
"""一次清洗的结果:产物 + 可审计的统计。"""
|
||||
|
||||
text: str
|
||||
stats: Dict[str, int] = field(default_factory=dict)
|
||||
rules_applied: List[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def total_hits(self) -> int:
|
||||
return sum(v for k, v in self.stats.items() if k != "collapsed_blanks")
|
||||
|
||||
|
||||
class MarkdownCleaner:
|
||||
def __init__(self, rules: Optional[List[Rule]] = None, rules_path: Optional[Path] = None):
|
||||
self.rules = rules if rules is not None else load_rules(rules_path)
|
||||
|
||||
def clean_text(self, text: str) -> CleanResult:
|
||||
stats: Dict[str, int] = {}
|
||||
applied: List[str] = []
|
||||
for rule in self.rules:
|
||||
if not rule.enabled:
|
||||
continue
|
||||
func = REGISTRY[rule.name]
|
||||
text = func(text, rule.params, stats)
|
||||
applied.append(rule.name)
|
||||
return CleanResult(text=text, stats=stats, rules_applied=applied)
|
||||
|
||||
def clean_file(self, src: Path, dst: Optional[Path] = None) -> CleanResult:
|
||||
raw = Path(src).read_text(encoding="utf-8")
|
||||
result = self.clean_text(raw)
|
||||
if dst is not None:
|
||||
Path(dst).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(dst).write_text(result.text, encoding="utf-8")
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def diff(before: str, after: str, context: int = 1) -> str:
|
||||
return "\n".join(
|
||||
difflib.unified_diff(
|
||||
before.splitlines(), after.splitlines(),
|
||||
fromfile="before", tofile="after", lineterm="", n=context,
|
||||
)
|
||||
)
|
||||
Reference in New Issue
Block a user