govdoc-md-cleaner v0.1.0: 政务文档 Markdown 清洗工具（规则引擎 + CLI + 测试）
- 8 条 YAML 声明规则：页码/页眉页脚/目录点线/图片/HTML表格/散落标签/行尾空白/空行 - 防误伤设计：protect 正则 + 内容形态豁免 + OCR burst 检测 - md-clean single/batch CLI，JSON 清洗报告 - 18 个单元测试 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
"""govdoc-md-cleaner: 政务文档 Markdown 清洗工具包。
|
||||
|
||||
针对 PDF→Markdown 转换产物（MinerU / OCR 管线输出）的常见脏数据，
|
||||
提供基于 YAML 规则的、可复现、可审计的清洗能力。
|
||||
"""
|
||||
|
||||
from cleaner.cleaner import MarkdownCleaner, CleanResult, load_rules
|
||||
from cleaner.cli import main
|
||||
|
||||
__version__ = "0.1.0"
|
||||
__all__ = ["MarkdownCleaner", "CleanResult", "load_rules", "main"]
|
||||
@@ -0,0 +1,56 @@
|
||||
"""清洗引擎：按规则顺序应用，输出清洗后文本 + 统计报告。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import difflib
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from cleaner.rules import REGISTRY, Rule, load_rules
|
||||
|
||||
|
||||
@dataclass
|
||||
class CleanResult:
|
||||
"""一次清洗的结果：产物 + 可审计的统计。"""
|
||||
|
||||
text: str
|
||||
stats: Dict[str, int] = field(default_factory=dict)
|
||||
rules_applied: List[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def total_hits(self) -> int:
|
||||
return sum(v for k, v in self.stats.items() if k != "collapsed_blanks")
|
||||
|
||||
|
||||
class MarkdownCleaner:
|
||||
def __init__(self, rules: Optional[List[Rule]] = None, rules_path: Optional[Path] = None):
|
||||
self.rules = rules if rules is not None else load_rules(rules_path)
|
||||
|
||||
def clean_text(self, text: str) -> CleanResult:
|
||||
stats: Dict[str, int] = {}
|
||||
applied: List[str] = []
|
||||
for rule in self.rules:
|
||||
if not rule.enabled:
|
||||
continue
|
||||
func = REGISTRY[rule.name]
|
||||
text = func(text, rule.params, stats)
|
||||
applied.append(rule.name)
|
||||
return CleanResult(text=text, stats=stats, rules_applied=applied)
|
||||
|
||||
def clean_file(self, src: Path, dst: Optional[Path] = None) -> CleanResult:
|
||||
raw = Path(src).read_text(encoding="utf-8")
|
||||
result = self.clean_text(raw)
|
||||
if dst is not None:
|
||||
Path(dst).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(dst).write_text(result.text, encoding="utf-8")
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def diff(before: str, after: str, context: int = 1) -> str:
|
||||
return "\n".join(
|
||||
difflib.unified_diff(
|
||||
before.splitlines(), after.splitlines(),
|
||||
fromfile="before", tofile="after", lineterm="", n=context,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,89 @@
|
||||
"""命令行入口。
|
||||
|
||||
用法:
|
||||
md-clean single <input.md> [-o output.md] [--diff] [--rules rules.yaml]
|
||||
md-clean batch <dir> [-o outdir] [--pattern "*.md"] [--report report.json]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from cleaner.cleaner import MarkdownCleaner
|
||||
from cleaner.rules import load_rules
|
||||
|
||||
|
||||
def _build(path: Optional[Path]) -> MarkdownCleaner:
|
||||
return MarkdownCleaner(rules=load_rules(path))
|
||||
|
||||
|
||||
def cmd_single(args: argparse.Namespace) -> int:
|
||||
cleaner = _build(args.rules)
|
||||
src = Path(args.input)
|
||||
raw = src.read_text(encoding="utf-8")
|
||||
result = cleaner.clean_text(raw)
|
||||
if args.output:
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.output).write_text(result.text, encoding="utf-8")
|
||||
print(f"已写入 {args.output}")
|
||||
if args.diff:
|
||||
print(MarkdownCleaner.diff(raw, result.text))
|
||||
print(json.dumps(result.stats, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_batch(args: argparse.Namespace) -> int:
|
||||
cleaner = _build(args.rules)
|
||||
src_dir = Path(args.directory)
|
||||
files = sorted(p for p in src_dir.rglob(args.pattern) if p.is_file())
|
||||
if not files:
|
||||
print(f"在 {src_dir} 下未找到匹配 {args.pattern} 的文件", file=sys.stderr)
|
||||
return 1
|
||||
out_dir = Path(args.output) if args.output else src_dir.parent / (src_dir.name + "_cleaned")
|
||||
report = []
|
||||
for f in files:
|
||||
rel = f.relative_to(src_dir)
|
||||
dst = out_dir / rel
|
||||
result = cleaner.clean_file(f, dst)
|
||||
report.append({"file": str(rel), "stats": result.stats})
|
||||
hits = result.total_hits
|
||||
print(f"[ok] {rel} 命中 {hits} 处")
|
||||
if args.report:
|
||||
Path(args.report).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.report).write_text(
|
||||
json.dumps({"files": report}, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
print(f"报告已写入 {args.report}")
|
||||
print(f"共清洗 {len(files)} 个文件 → {out_dir}")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: Optional[List[str]] = None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="md-clean", description="政务文档 Markdown 清洗工具")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
p1 = sub.add_parser("single", help="清洗单个文件")
|
||||
p1.add_argument("input", help="输入 .md 文件")
|
||||
p1.add_argument("-o", "--output", help="输出路径（缺省打印统计不写文件）")
|
||||
p1.add_argument("--diff", action="store_true", help="打印 unified diff")
|
||||
p1.add_argument("--rules", type=Path, help="规则 YAML 路径")
|
||||
|
||||
p2 = sub.add_parser("batch", help="批量清洗目录（递归）")
|
||||
p2.add_argument("directory", help="输入目录")
|
||||
p2.add_argument("-o", "--output", help="输出目录（缺省 <dir>_cleaned）")
|
||||
p2.add_argument("--pattern", default="*.md", help="文件 glob（默认 *.md）")
|
||||
p2.add_argument("--report", help="清洗报告 JSON 输出路径")
|
||||
p2.add_argument("--rules", type=Path, help="规则 YAML 路径")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
if args.command == "single":
|
||||
return cmd_single(args)
|
||||
return cmd_batch(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,281 @@
|
||||
"""规则模型：一条清洗规则 = 名称 + 开关 + 参数 + 应用顺序。
|
||||
|
||||
规则用 YAML 声明（rules/*.yaml），引擎按 order 依次应用，
|
||||
这样清洗过程可复现、可 diff、可回滚。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
@dataclass
|
||||
class Rule:
|
||||
"""一条清洗规则。
|
||||
|
||||
name: 唯一标识（报告里引用）
|
||||
order: 应用顺序，小的先执行
|
||||
enabled: 开关，方便对某个用例单独关掉
|
||||
params: 传给处理函数的额外参数
|
||||
"""
|
||||
|
||||
name: str
|
||||
order: int = 100
|
||||
enabled: bool = True
|
||||
params: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@staticmethod
|
||||
def from_dict(d: Dict[str, Any]) -> "Rule":
|
||||
return Rule(
|
||||
name=d["name"],
|
||||
order=int(d.get("order", 100)),
|
||||
enabled=bool(d.get("enabled", True)),
|
||||
params=dict(d.get("params") or {}),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 每条规则的具体实现。函数签名统一为 (text, params, stats) -> text。
|
||||
# stats 是 {rule_name: 删改行数}，用于生成清洗报告。
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
RuleFunc = Callable[[str, Dict[str, Any], Dict[str, int]], str]
|
||||
|
||||
# 页码类：第X页 共Y页 / 第X页共Y页 / Page x of y / - 3 - 等
|
||||
_RE_PAGE_CN = re.compile(
|
||||
r"^[ \t]*第\s*[0-9０-９]+\s*页\s*(?:[,，/]?\s*共\s*[0-9０-９]+\s*页)?[ \t]*$"
|
||||
)
|
||||
_RE_PAGE_EN = re.compile(
|
||||
r"^[ \t]*(?:[-–—]?\s*Page\s+\d+(?:\s+of\s+\d+)?\s*[-–—]?"
|
||||
r"|[-–—]\s*\d{1,4}\s*[-–—]"
|
||||
r"|\d+\s*/\s*\d+)[ \t]*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
# 纯页码数字行（单独一行只有 1-4 位数字，且不是标题编号场景）
|
||||
_RE_PAGE_BARE = re.compile(r"^[ \t]*\d{1,4}[ \t]*$")
|
||||
|
||||
|
||||
def _strip_page_lines(text: str, params: Dict[str, Any], stats: Dict[str, int]) -> str:
|
||||
keep_bare_numbers = bool(params.get("keep_bare_numbers", True))
|
||||
out, n = [], 0
|
||||
for line in text.splitlines():
|
||||
if _RE_PAGE_CN.match(line) or _RE_PAGE_EN.match(line):
|
||||
n += 1
|
||||
continue
|
||||
if not keep_bare_numbers and _RE_PAGE_BARE.match(line):
|
||||
n += 1
|
||||
continue
|
||||
out.append(line)
|
||||
stats["page_lines"] = stats.get("page_lines", 0) + n
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
# 页眉/页脚：同一短行在全篇重复出现 >= N 次（默认 3），视为页眉页脚删除。
|
||||
# 两道保险避免误伤正文：
|
||||
# 1. protect 正则 —— 标书里逐章重复的模板行（签章/日期/声明结尾）
|
||||
# 2. 内容形态行直接豁免 —— 编号条款 "(1)…"/"1、…"/"一、…"/列表/表格行
|
||||
# 在平行结构的标书里天然重复，但它们是正文不是页眉。
|
||||
_RE_CONTENT_LIKE = re.compile(
|
||||
r"^[\d０-９(（\[【\-–—*•·①-⑳一二三四五六七八九十百第章节条款、,.。:：|]"
|
||||
r"|[一-鿿]{1,6}[：:]\s*\S" # "乙方：xxx" / "地址：xxx" 这类字段行
|
||||
r"|^[一-鿿A-Za-z]{1,6}[：:]\s*$" # "乙方：" / "注：" 字段标签行
|
||||
)
|
||||
|
||||
|
||||
def _strip_repeated_short_lines(
|
||||
text: str, params: Dict[str, Any], stats: Dict[str, int]
|
||||
) -> str:
|
||||
threshold = int(params.get("threshold", 3))
|
||||
max_len = int(params.get("max_len", 40))
|
||||
# 同一短行在文中连续出现 >= burst_limit 次（间隔 <= burst_gap 行）视为
|
||||
# OCR 重复崩坏（如 003-10 案例"审计程序"连续刷屏 1359 次），无论阈值直接删。
|
||||
burst_limit = int(params.get("burst_limit", 5))
|
||||
burst_gap = int(params.get("burst_gap", 2))
|
||||
protect = [re.compile(p) for p in params.get("protect", [])]
|
||||
lines = text.splitlines()
|
||||
counts: Dict[str, int] = {}
|
||||
positions: Dict[str, List[int]] = {}
|
||||
for i, line in enumerate(lines):
|
||||
s = line.strip()
|
||||
if (
|
||||
0 < len(s) <= max_len
|
||||
and not s.startswith("#")
|
||||
and not _RE_CONTENT_LIKE.match(s)
|
||||
and "![" not in s
|
||||
):
|
||||
counts[s] = counts.get(s, 0) + 1
|
||||
positions.setdefault(s, []).append(i)
|
||||
repeated = {
|
||||
s
|
||||
for s, c in counts.items()
|
||||
if c >= threshold and not any(rx.search(s) for rx in protect)
|
||||
}
|
||||
# OCR 崩坏连续段：即使该行被 protect/内容豁免，连续刷屏也删
|
||||
for s, pos in positions.items():
|
||||
best_run = run = 1
|
||||
for a, b in zip(pos, pos[1:]):
|
||||
run = run + 1 if b - a <= burst_gap else 1
|
||||
best_run = max(best_run, run)
|
||||
if best_run >= burst_limit:
|
||||
repeated.add(s)
|
||||
if not repeated:
|
||||
return text
|
||||
out, n = [], 0
|
||||
for line in lines:
|
||||
if line.strip() in repeated:
|
||||
n += 1
|
||||
continue
|
||||
out.append(line)
|
||||
stats["header_footer_lines"] = stats.get("header_footer_lines", 0) + n
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
# 目录点线：标题文字 ………… 12 / ······ 3 之类（… U+2026 也算；# 前缀可选）
|
||||
_RE_TOC_DOTS = re.compile(
|
||||
r"^(#{1,6}\s+)?.*?[ \t]*[\.。·•‧…]{6,}[ \t]*[\d０-９]*[ \t]*$"
|
||||
)
|
||||
|
||||
|
||||
def _strip_toc_dots(text: str, params: Dict[str, Any], stats: Dict[str, int]) -> str:
|
||||
out, n = [], 0
|
||||
for line in text.splitlines():
|
||||
m = _RE_TOC_DOTS.match(line)
|
||||
if m:
|
||||
# 去掉点线和页码，保留标题文字；纯点线+页码的目录行整行删
|
||||
title = re.sub(r"[ \t]*[\.。·•‧…]{6,}[ \t]*[\d０-９]*[ \t]*$", "", line).rstrip()
|
||||
if title.strip() and not re.fullmatch(r"[\.。·•‧…\d０-９\s]+", title):
|
||||
out.append(title)
|
||||
n += 1
|
||||
continue
|
||||
out.append(line)
|
||||
stats["toc_dot_lines"] = stats.get("toc_dot_lines", 0) + n
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
# 图片引用： —— 图片目录不在交付物里，引用是死链
|
||||
_RE_IMAGE = re.compile(r"[ \t]*!\[[^\]]*\]\([^)]*\)[ \t]*")
|
||||
|
||||
|
||||
def _drop_images(text: str, params: Dict[str, Any], stats: Dict[str, int]) -> str:
|
||||
placeholder = params.get("placeholder") # None=整行删除；否则替换为占位文本
|
||||
out, n = [], 0
|
||||
for line in text.splitlines():
|
||||
if _RE_IMAGE.fullmatch(line):
|
||||
n += 1
|
||||
if placeholder:
|
||||
out.append(str(placeholder))
|
||||
continue
|
||||
new = _RE_IMAGE.sub("", line)
|
||||
if new != line:
|
||||
n += 1
|
||||
line = new.rstrip()
|
||||
out.append(line)
|
||||
stats["image_refs"] = stats.get("image_refs", 0) + n
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
# HTML 表格规范化：<table ...><tr><td>…</td></tr></table> 压缩为合法 Markdown 管道表格
|
||||
_RE_TABLE = re.compile(r"<table[^>]*>(.*?)</table>", re.DOTALL | re.IGNORECASE)
|
||||
_RE_TR = re.compile(r"<tr[^>]*>(.*?)</tr>", re.DOTALL | re.IGNORECASE)
|
||||
_RE_TD = re.compile(r"<t[dh][^>]*>(.*?)</t[dh]>", re.DOTALL | re.IGNORECASE)
|
||||
|
||||
|
||||
def _cell_text(raw: str) -> str:
|
||||
cell = re.sub(r"<br\s*/?>", " ", raw, flags=re.IGNORECASE)
|
||||
cell = re.sub(r"<[^>]+>", "", cell)
|
||||
return " ".join(cell.split()).replace("|", "\\|")
|
||||
|
||||
|
||||
def _table_to_md(tbl_html: str) -> str:
|
||||
rows: List[List[str]] = []
|
||||
for tr in _RE_TR.findall(tbl_html):
|
||||
cells = [_cell_text(td) for td in _RE_TD.findall(tr)]
|
||||
if cells:
|
||||
rows.append(cells)
|
||||
if not rows:
|
||||
return ""
|
||||
width = max(len(r) for r in rows)
|
||||
rows = [r + [""] * (width - len(r)) for r in rows]
|
||||
lines = ["| " + " | ".join(rows[0]) + " |", "|" + "---|" * width]
|
||||
lines.extend("| " + " | ".join(r) + " |" for r in rows[1:])
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _normalize_tables(text: str, params: Dict[str, Any], stats: Dict[str, int]) -> str:
|
||||
def _sub(m: re.Match[str]) -> str:
|
||||
md = _table_to_md(m.group(1))
|
||||
return md if md else ""
|
||||
|
||||
new, n = _RE_TABLE.subn(_sub, text)
|
||||
# 残缺兜底：文档截断导致 <table> 未闭合时，把剩余 <tr> 行也转掉
|
||||
if "<table" in new:
|
||||
tail_i = new.rfind("<table")
|
||||
head, tail = new[:tail_i], new[tail_i:]
|
||||
tail = _RE_TR.sub(
|
||||
lambda m: "\n" + _table_to_md("<table>" + m.group(0) + "</table>"),
|
||||
tail,
|
||||
)
|
||||
tail = re.sub(r"</?table[^>]*>", "", tail)
|
||||
new = head + tail
|
||||
n += 1
|
||||
stats["html_tables"] = stats.get("html_tables", 0) + n
|
||||
return new
|
||||
|
||||
|
||||
# 表格内 <br/> 会被上面规则拍平；这里处理散落的 HTML 换行/空白标签
|
||||
def _strip_stray_html(text: str, params: Dict[str, Any], stats: Dict[str, int]) -> str:
|
||||
new = re.sub(r"<br\s*/?>", " ", text, flags=re.IGNORECASE)
|
||||
if new != text:
|
||||
stats["stray_html"] = stats.get("stray_html", 0) + text.count("<br")
|
||||
return new
|
||||
|
||||
|
||||
# 行尾双空格（MinerU 每行都带，语义是硬换行，清洗后统一去掉）
|
||||
def _rstrip_lines(text: str, params: Dict[str, Any], stats: Dict[str, int]) -> str:
|
||||
out, n = [], 0
|
||||
for line in text.splitlines():
|
||||
stripped = line.rstrip()
|
||||
if stripped != line:
|
||||
n += 1
|
||||
out.append(stripped)
|
||||
stats["trailing_ws_lines"] = stats.get("trailing_ws_lines", 0) + n
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
# 连续空行压成一行；文首文末空白裁掉
|
||||
def _collapse_blank_lines(
|
||||
text: str, params: Dict[str, Any], stats: Dict[str, int]
|
||||
) -> str:
|
||||
text = re.sub(r"[ \t]*\n(?:[ \t]*\n){2,}", "\n\n", text)
|
||||
text = text.strip("\n") + "\n" if text.strip() else ""
|
||||
stats["collapsed_blanks"] = stats.get("collapsed_blanks", 1)
|
||||
return text
|
||||
|
||||
|
||||
REGISTRY: Dict[str, RuleFunc] = {
|
||||
"strip_page_lines": _strip_page_lines,
|
||||
"strip_repeated_short_lines": _strip_repeated_short_lines,
|
||||
"strip_toc_dots": _strip_toc_dots,
|
||||
"drop_images": _drop_images,
|
||||
"normalize_tables": _normalize_tables,
|
||||
"strip_stray_html": _strip_stray_html,
|
||||
"rstrip_lines": _rstrip_lines,
|
||||
"collapse_blank_lines": _collapse_blank_lines,
|
||||
}
|
||||
|
||||
|
||||
def load_rules(path: Optional[Path] = None) -> List[Rule]:
|
||||
"""从 YAML 加载规则；未指定路径时用包内默认规则。"""
|
||||
if path is None:
|
||||
path = Path(__file__).resolve().parent.parent / "rules" / "default.yaml"
|
||||
data = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {}
|
||||
rules = [Rule.from_dict(d) for d in data.get("rules", [])]
|
||||
unknown = [r.name for r in rules if r.name not in REGISTRY]
|
||||
if unknown:
|
||||
raise ValueError(f"未知规则: {unknown}，可用规则: {sorted(REGISTRY)}")
|
||||
return sorted(rules, key=lambda r: r.order)
|
||||
Reference in New Issue
Block a user