fix(question_gen): check_verbatim covers question_text + add missing blacklist patterns
- check_verbatim now computes n-gram overlap for BOTH question_text and correct_option vs source texts, returning max(question_ratio, option_ratio). Extracted _ngram_overlap_ratio helper for reuse. - Added 4 missing blacklist patterns: 'this segment', 'this frame', 'the current frame', 'frame summary'. - Added 5 new test cases covering the above changes. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -52,6 +52,10 @@ _BLACKLIST_PATTERNS: list[tuple[re.Pattern[str], str]] = [
|
||||
(re.compile(r"the\s+clip", re.IGNORECASE), "the clip"),
|
||||
(re.compile(r"this\s+footage", re.IGNORECASE), "this footage"),
|
||||
(re.compile(r"the\s+footage", re.IGNORECASE), "the footage"),
|
||||
(re.compile(r"this\s+segment", re.IGNORECASE), "this segment"),
|
||||
(re.compile(r"this\s+frame", re.IGNORECASE), "this frame"),
|
||||
(re.compile(r"the\s+current\s+frame", re.IGNORECASE), "the current frame"),
|
||||
(re.compile(r"frame\s+summary", re.IGNORECASE), "frame summary"),
|
||||
(re.compile(r"上面的片段", re.IGNORECASE), "上面的片段"),
|
||||
(re.compile(r"这段视频", re.IGNORECASE), "这段视频"),
|
||||
(re.compile(r"该视频", re.IGNORECASE), "该视频"),
|
||||
@@ -179,47 +183,56 @@ def check_referent_blacklist(question_text: str) -> list[str]:
|
||||
return violations
|
||||
|
||||
|
||||
def _ngram_overlap_ratio(
|
||||
text_words: list[str],
|
||||
source_ngrams: set[tuple[str, ...]],
|
||||
window: int,
|
||||
) -> float:
|
||||
"""计算单段文本与来源 n-gram 集合的重叠率。
|
||||
|
||||
参数:
|
||||
text_words: 待比较文本的分词列表(已小写化)。
|
||||
source_ngrams: 来源素材的 n-gram 集合。
|
||||
window: n-gram 窗口大小。
|
||||
|
||||
返回:
|
||||
重叠率 [0.0, 1.0],词数不足窗口大小时返回 0.0。
|
||||
"""
|
||||
if len(text_words) < window:
|
||||
return 0.0
|
||||
|
||||
text_ngrams: set[tuple[str, ...]] = {
|
||||
tuple(text_words[i : i + window]) for i in range(len(text_words) - window + 1)
|
||||
}
|
||||
|
||||
if not text_ngrams:
|
||||
return 0.0
|
||||
|
||||
overlap = text_ngrams & source_ngrams
|
||||
return len(overlap) / len(text_ngrams)
|
||||
|
||||
|
||||
def check_verbatim(
|
||||
question_text: str,
|
||||
correct_option: str,
|
||||
source_texts: list[str],
|
||||
window: int = 6,
|
||||
) -> float:
|
||||
"""计算正确选项与来源素材的逐字重复率。
|
||||
"""计算题目文本和正确选项与来源素材的最大逐字重复率。
|
||||
|
||||
使用滑动窗口 n-gram 集合交集方法:从选项文本提取所有 n-gram,
|
||||
与来源文本的 n-gram 集合求交集,计算重叠比例。
|
||||
使用滑动窗口 n-gram 集合交集方法:分别从题目文本和选项文本提取 n-gram,
|
||||
各自与来源文本的 n-gram 集合求交集,返回两者中较大的重叠比例。
|
||||
|
||||
参数:
|
||||
question_text: 题目文本(当前未使用,预留接口)。
|
||||
question_text: 题目文本。
|
||||
correct_option: 正确选项文本(含 "X. " 前缀)。
|
||||
source_texts: 来源素材文本列表。
|
||||
window: n-gram 窗口大小。
|
||||
|
||||
返回:
|
||||
重复率 [0.0, 1.0]。0.0 表示无重叠,1.0 表示完全复制。
|
||||
最大重复率 [0.0, 1.0]。0.0 表示无重叠,1.0 表示完全复制。
|
||||
"""
|
||||
# Phase 1: 提取选项纯文本(去掉可能的 "X. " 前缀)
|
||||
option_text = correct_option
|
||||
if len(option_text) >= 3 and option_text[1] == "." and option_text[2] == " ":
|
||||
option_text = option_text[3:]
|
||||
|
||||
# Phase 2: 分词(简单空格分词,转小写)
|
||||
option_words = option_text.lower().split()
|
||||
|
||||
# Phase 3: 选项词数不足窗口大小则无法构成 n-gram
|
||||
if len(option_words) < window:
|
||||
return 0.0
|
||||
|
||||
# Phase 4: 构造选项的 n-gram 集合
|
||||
option_ngrams: set[tuple[str, ...]] = set()
|
||||
for i in range(len(option_words) - window + 1):
|
||||
option_ngrams.add(tuple(option_words[i : i + window]))
|
||||
|
||||
if not option_ngrams:
|
||||
return 0.0
|
||||
|
||||
# Phase 5: 构造来源文本的 n-gram 集合
|
||||
# Phase 1: 构造来源文本的 n-gram 集合
|
||||
source_ngrams: set[tuple[str, ...]] = set()
|
||||
for source in source_texts:
|
||||
words = source.lower().split()
|
||||
@@ -229,9 +242,20 @@ def check_verbatim(
|
||||
if not source_ngrams:
|
||||
return 0.0
|
||||
|
||||
# Phase 6: 计算交集比例
|
||||
overlap = option_ngrams & source_ngrams
|
||||
return len(overlap) / len(option_ngrams)
|
||||
# Phase 2: 计算题目文本的重叠率
|
||||
question_words = question_text.lower().split()
|
||||
question_ratio = _ngram_overlap_ratio(question_words, source_ngrams, window)
|
||||
|
||||
# Phase 3: 提取选项纯文本(去掉可能的 "X. " 前缀)
|
||||
option_text = correct_option
|
||||
if len(option_text) >= 3 and option_text[1] == "." and option_text[2] == " ":
|
||||
option_text = option_text[3:]
|
||||
|
||||
option_words = option_text.lower().split()
|
||||
option_ratio = _ngram_overlap_ratio(option_words, source_ngrams, window)
|
||||
|
||||
# Phase 4: 返回两者中较大的重叠率
|
||||
return max(question_ratio, option_ratio)
|
||||
|
||||
|
||||
def has_time_anchor(question_text: str) -> bool:
|
||||
|
||||
@@ -122,6 +122,31 @@ class TestReferentBlacklist:
|
||||
violations = check_referent_blacklist(q)
|
||||
assert len(violations) >= 1
|
||||
|
||||
def test_this_segment_caught(self) -> None:
|
||||
"""'this segment' 被检测为违规。"""
|
||||
q = "What action is performed in this segment?"
|
||||
violations = check_referent_blacklist(q)
|
||||
assert len(violations) >= 1
|
||||
assert any("this segment" in v.lower() for v in violations)
|
||||
|
||||
def test_this_frame_caught(self) -> None:
|
||||
"""'this frame' 被检测为违规。"""
|
||||
q = "What object is visible in this frame?"
|
||||
violations = check_referent_blacklist(q)
|
||||
assert len(violations) >= 1
|
||||
|
||||
def test_current_frame_caught(self) -> None:
|
||||
"""'the current frame' 被检测为违规。"""
|
||||
q = "Based on the current frame, what is happening?"
|
||||
violations = check_referent_blacklist(q)
|
||||
assert len(violations) >= 1
|
||||
|
||||
def test_frame_summary_caught(self) -> None:
|
||||
"""'frame summary' 被检测为违规。"""
|
||||
q = "According to the frame summary, what is the main event?"
|
||||
violations = check_referent_blacklist(q)
|
||||
assert len(violations) >= 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestVerbatim
|
||||
@@ -186,6 +211,21 @@ class TestVerbatim:
|
||||
)
|
||||
assert ratio == 0.0
|
||||
|
||||
def test_question_text_overlap_detected(self) -> None:
|
||||
"""题目文本从来源复制但选项原创时,仍检测到高重叠率。"""
|
||||
source = "The man walked slowly through the crowded market buying fresh vegetables"
|
||||
# 题目直接复制来源的大部分内容
|
||||
question = "The man walked slowly through the crowded market buying what?"
|
||||
# 选项完全原创
|
||||
option = "A. Fresh organic produce from the local farm stand"
|
||||
ratio = check_verbatim(
|
||||
question_text=question,
|
||||
correct_option=option,
|
||||
source_texts=[source],
|
||||
window=6,
|
||||
)
|
||||
assert ratio > 0.0, "题目文本的逐字重叠应被检测到"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TestTimeAnchor
|
||||
|
||||
Reference in New Issue
Block a user