iomgaa
e06e5ed7a2
对应 docs/04 §4 E1。三处对参考实现的替代:edit 吃 str 内部按词切(不再比 token id,回归 tokenizer 无关)、rouge1 集合改多重集(ROUGE-1 标准定义, 数学文本重复词多)、去掉 1e-8 分母平滑(全同串精确得 1)。φ 默认 edit_distance 对齐论文 §5.1。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
143 lines
4.6 KiB
Python
143 lines
4.6 KiB
Python
"""similarity.py 单测——docs/04 §5.1:手构字符串钉死 φ 与 k_sem(式3)。
|
||
|
||
全部本地 CPU、纯标准库,不依赖 torch/transformers。
|
||
关键手算用例在各测试的注释里逐步展开,方便对着验算。
|
||
"""
|
||
|
||
import math
|
||
|
||
import pytest
|
||
|
||
from ars_opd.similarity import aggregate_similarity, edit_similarity, phi, rouge1
|
||
|
||
# ---------------------------------------------------------------- rouge1
|
||
|
||
|
||
def test_rouge1_identical_is_exact_one():
|
||
# 全同串必须精确 = 1.0(参考实现因分母 +1e-8 只能得 ≈0.99999998)
|
||
s = "so x = 5 and y = 12"
|
||
assert rouge1(s, s) == 1.0
|
||
|
||
|
||
def test_rouge1_disjoint_is_zero():
|
||
assert rouge1("a b c", "x y z") == 0.0
|
||
|
||
|
||
def test_rouge1_partial_overlap_hand_computed():
|
||
# hyp = {a, b, c}, ref = {a, b, d}:overlap = 2
|
||
# precision = 2/3, recall = 2/3, F1 = 2·(2/3)(2/3) / (4/3) = 2/3
|
||
assert math.isclose(rouge1("a b c", "a b d"), 2 / 3)
|
||
|
||
|
||
def test_rouge1_multiset_counts_repeats():
|
||
# 多重集语义:hyp = [x,x,x,x], ref = [x] → overlap = min(4,1) = 1
|
||
# precision = 1/4, recall = 1/1, F1 = 2·(1/4)/(5/4) = 0.4
|
||
# (参考实现的 set 版会给满分 1.0——数学文本重复词多,这是关键失真点)
|
||
assert math.isclose(rouge1("x x x x", "x"), 0.4)
|
||
|
||
|
||
def test_rouge1_is_bag_of_words_order_blind():
|
||
# 词袋:只看用了哪些词,不看顺序
|
||
assert rouge1("a b", "b a") == 1.0
|
||
|
||
|
||
def test_rouge1_empty_sides():
|
||
assert rouge1("", "a b") == 0.0
|
||
assert rouge1("a b", "") == 0.0
|
||
assert rouge1("", "") == 0.0
|
||
assert rouge1(" ", "a") == 0.0 # 纯空白 split 后无词
|
||
|
||
|
||
# ---------------------------------------------------------- edit_similarity
|
||
|
||
|
||
def test_edit_identical_is_one():
|
||
s = "so x = 5 and y = 12"
|
||
assert edit_similarity(s, s) == 1.0
|
||
|
||
|
||
def test_edit_totally_different_is_zero():
|
||
# ["a","b"] vs ["c","d"]:2 次替换,dist=2, max(m,n)=2 → 1 − 1 = 0
|
||
assert edit_similarity("a b", "c d") == 0.0
|
||
|
||
|
||
def test_edit_single_substitution_hand_computed():
|
||
# ["a","b","c"] vs ["a","x","c"]:1 次替换,dist=1, max=3 → 2/3
|
||
assert math.isclose(edit_similarity("a b c", "a x c"), 2 / 3)
|
||
|
||
|
||
def test_edit_insertion_hand_computed():
|
||
# ["a","b"] vs ["a","x","b"]:1 次插入,dist=1, max=3 → 2/3
|
||
assert math.isclose(edit_similarity("a b", "a x b"), 2 / 3)
|
||
|
||
|
||
def test_edit_is_order_sensitive():
|
||
# ["a","b"] vs ["b","a"]:两次替换 dist=2 → 0.0;与 rouge1 的 1.0 互补
|
||
assert edit_similarity("a b", "b a") == 0.0
|
||
assert rouge1("a b", "b a") == 1.0
|
||
|
||
|
||
def test_edit_empty_sides():
|
||
assert edit_similarity("", "") == 1.0 # 零距离
|
||
assert edit_similarity("a b", "") == 0.0 # 全删
|
||
assert edit_similarity("", "a b") == 0.0 # 全插
|
||
|
||
|
||
def test_edit_asymmetric_lengths():
|
||
# ["a"] vs ["a","b","c","d"]:3 次插入,dist=3, max=4 → 1/4
|
||
assert math.isclose(edit_similarity("a", "a b c d"), 1 / 4)
|
||
|
||
|
||
# --------------------------------------------------------------------- phi
|
||
|
||
|
||
def test_phi_default_is_edit_distance():
|
||
# 论文 §5.1 默认;"a b" vs "b a" 恰能区分两度量(edit=0, rouge1=1)
|
||
assert phi("a b", "b a") == edit_similarity("a b", "b a") == 0.0
|
||
|
||
|
||
def test_phi_dispatch():
|
||
h, r = "a b c", "a b d"
|
||
assert phi(h, r, metric="rouge1") == rouge1(h, r)
|
||
assert phi(h, r, metric="edit_distance") == edit_similarity(h, r)
|
||
|
||
|
||
def test_phi_unknown_metric_raises():
|
||
with pytest.raises(ValueError, match="bleu"):
|
||
phi("a", "a", metric="bleu")
|
||
|
||
|
||
# ----------------------------------------------------- aggregate_similarity
|
||
|
||
|
||
def test_aggregate_is_sum_of_phi():
|
||
# 式(3) 手算:rollouts 与 "a b c" 的 edit 相似度分别为 1.0, 2/3, 0.0
|
||
chunk = "a b c"
|
||
rollouts = ["a b c", "a x c", "x y z"]
|
||
expected = 1.0 + 2 / 3 + 0.0
|
||
assert math.isclose(aggregate_similarity(chunk, rollouts), expected)
|
||
|
||
|
||
def test_aggregate_bounds():
|
||
# k_sem ∈ [0, N]:全同 → N,全不同 → 0
|
||
n = 5
|
||
assert aggregate_similarity("a b", ["a b"] * n) == float(n)
|
||
assert aggregate_similarity("a b", ["x y"] * n) == 0.0
|
||
|
||
|
||
def test_aggregate_is_continuous_soft_count():
|
||
# φ 连续 ⇒ k_sem 非整数是常态(区别于 token 精确匹配的硬计数)
|
||
k = aggregate_similarity("a b c", ["a b c", "a x c"])
|
||
assert 1.0 < k < 2.0
|
||
|
||
|
||
def test_aggregate_empty_rollouts():
|
||
assert aggregate_similarity("a b", []) == 0.0
|
||
|
||
|
||
def test_aggregate_metric_passthrough():
|
||
# "a b" vs "b a":edit 全零,rouge1 全满——验证 metric 真的传下去了
|
||
chunk, rollouts = "a b", ["b a", "b a"]
|
||
assert aggregate_similarity(chunk, rollouts, metric="edit_distance") == 0.0
|
||
assert aggregate_similarity(chunk, rollouts, metric="rouge1") == 2.0
|