重置项目并建立文档治理基础架构
- 移除旧清洗器、规则、测试和打包配置 - 增加 AGENTS.md 与 CLAUDE.md 同步协作规范 - 建立 research-wiki 文档生命周期和首个设计记录
This commit is contained in:
@@ -1,154 +0,0 @@
|
||||
"""清洗规则单元测试。
|
||||
|
||||
fixtures 里是各脏数据模式的最小样例，跑一遍断言规则命中且正文无损。
|
||||
"""
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from cleaner.cleaner import MarkdownCleaner
|
||||
from cleaner.rules import (
|
||||
_drop_images,
|
||||
_normalize_tables,
|
||||
_rstrip_lines,
|
||||
_strip_page_lines,
|
||||
_strip_repeated_short_lines,
|
||||
_strip_toc_dots,
|
||||
load_rules,
|
||||
)
|
||||
|
||||
S = {} # 每个用例独立 stats
|
||||
|
||||
|
||||
def st():
|
||||
return {}
|
||||
|
||||
|
||||
class TestPageLines(unittest.TestCase):
|
||||
def test_cn_page(self):
|
||||
text = "正文A\n第 3 页 共 53 页\n正文B\n第4页共4页\n结尾"
|
||||
out = _strip_page_lines(text, {}, st())
|
||||
self.assertEqual(out, "正文A\n正文B\n结尾")
|
||||
|
||||
def test_en_page(self):
|
||||
text = "foo\nPage 3 of 10\nbar\n- 4 -\nbaz"
|
||||
out = _strip_page_lines(text, {"keep_bare_numbers": False}, st())
|
||||
self.assertNotIn("Page 3", out)
|
||||
self.assertNotIn("- 4 -", out)
|
||||
|
||||
def test_bare_number_kept_by_default(self):
|
||||
text = "条款\n12\n下文"
|
||||
out = _strip_page_lines(text, {}, st())
|
||||
self.assertIn("12", out)
|
||||
|
||||
|
||||
class TestRepeatedLines(unittest.TestCase):
|
||||
def test_header_removed(self):
|
||||
lines = ["某某采购项目招标文件"] + ["内容%d" % i for i in range(5)]
|
||||
text = "\n".join(("某某采购项目招标文件 \n" + l) for l in lines)
|
||||
out = _strip_repeated_short_lines(text, {"threshold": 3, "max_len": 40}, st())
|
||||
self.assertNotIn("某某采购项目招标文件", out)
|
||||
self.assertIn("内容1", out)
|
||||
|
||||
def test_protected_signature_kept(self):
|
||||
text = "\n".join(["投标人：（公章）"] * 4 + ["正文"])
|
||||
out = _strip_repeated_short_lines(
|
||||
text, {"threshold": 3, "max_len": 40, "protect": ["公章"]}, st()
|
||||
)
|
||||
self.assertIn("投标人：（公章）", out)
|
||||
|
||||
def test_numbered_clause_kept(self):
|
||||
# 编号条款是正文不是页眉
|
||||
text = "\n".join(["（1）乙方须接受甲方监督。"] * 4 + ["正文"])
|
||||
out = _strip_repeated_short_lines(text, {"threshold": 3, "max_len": 40}, st())
|
||||
self.assertIn("（1）乙方须接受甲方监督。", out)
|
||||
|
||||
def test_ocr_burst_removed(self):
|
||||
text = "\n".join(["审计程序"] * 30)
|
||||
out = _strip_repeated_short_lines(
|
||||
text,
|
||||
{"threshold": 999, "max_len": 40, "burst_limit": 5, "burst_gap": 2},
|
||||
st(),
|
||||
)
|
||||
self.assertNotIn("审计程序", out)
|
||||
|
||||
|
||||
class TestTocDots(unittest.TestCase):
|
||||
def test_toc_line(self):
|
||||
text = "第一章 投标邀请函 ……………………………………… 2"
|
||||
out = _strip_toc_dots(text, {}, st())
|
||||
self.assertEqual(out, "第一章 投标邀请函")
|
||||
|
||||
def test_plain_toc_line_keeps_text(self):
|
||||
# 无标题目录行：去掉点线页码，保留"序号+标题"文字
|
||||
text = "21 迷交的投标文件 ………………………………………………… 25"
|
||||
out = _strip_toc_dots(text, {}, st())
|
||||
self.assertEqual(out, "21 迷交的投标文件")
|
||||
|
||||
def test_body_with_ellipsis_kept(self):
|
||||
text = "此处省略部分内容……后续"
|
||||
out = _strip_toc_dots(text, {}, st())
|
||||
self.assertIn("后续", out)
|
||||
|
||||
|
||||
class TestImages(unittest.TestCase):
|
||||
def test_image_line_dropped(self):
|
||||
text = " \n正文"
|
||||
out = _drop_images(text, {}, st())
|
||||
self.assertEqual(out, "正文")
|
||||
|
||||
def test_placeholder(self):
|
||||
text = "\n正文"
|
||||
out = _drop_images(text, {"placeholder": "[图]"}, st())
|
||||
self.assertIn("[图]", out)
|
||||
|
||||
def test_base64_image_dropped(self):
|
||||
text = "\n正文"
|
||||
out = _drop_images(text, {}, st())
|
||||
self.assertNotIn("base64", out)
|
||||
|
||||
|
||||
class TestTables(unittest.TestCase):
|
||||
def test_table_to_pipe(self):
|
||||
html = "<table><tr><td>序号</td><td>名称</td></tr><tr><td>1</td><td>保洁</td></tr></table>"
|
||||
out = _normalize_tables(html, {}, st())
|
||||
self.assertIn("| 序号 | 名称 |", out)
|
||||
self.assertIn("| 1 | 保洁 |", out)
|
||||
self.assertIn("|---|---|", out)
|
||||
|
||||
def test_pipe_escaped(self):
|
||||
html = "<table><tr><td>a|b</td></tr></table>"
|
||||
out = _normalize_tables(html, {}, st())
|
||||
self.assertIn("a\\|b", out)
|
||||
|
||||
def test_br_in_cell(self):
|
||||
html = "<table><tr><td>品<br/>目</td></tr></table>"
|
||||
out = _normalize_tables(html, {}, st())
|
||||
self.assertIn("品 目", out)
|
||||
|
||||
|
||||
class TestEngine(unittest.TestCase):
|
||||
def test_full_pipeline(self):
|
||||
cleaner = MarkdownCleaner(rules=load_rules())
|
||||
raw = (
|
||||
"# 标题\n \n第 1 页 共 2 页 \n"
|
||||
"正文一段。 \n<table><tr><td>a</td></tr></table> \n\n\n\n尾部\n"
|
||||
)
|
||||
result = cleaner.clean_text(raw)
|
||||
self.assertNotIn("images/", result.text)
|
||||
self.assertNotIn("第 1 页", result.text)
|
||||
self.assertNotIn("<table>", result.text)
|
||||
self.assertIn("| a |", result.text)
|
||||
self.assertNotIn("\n\n\n", result.text)
|
||||
|
||||
def test_default_rules_load(self):
|
||||
rules = load_rules()
|
||||
self.assertTrue(len(rules) >= 8)
|
||||
self.assertEqual(rules, sorted(rules, key=lambda r: r.order))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user