diff --git a/app/question_gen/postprocess.py b/app/question_gen/postprocess.py index 0c24384..f13eb7e 100644 --- a/app/question_gen/postprocess.py +++ b/app/question_gen/postprocess.py @@ -52,6 +52,10 @@ _BLACKLIST_PATTERNS: list[tuple[re.Pattern[str], str]] = [ (re.compile(r"the\s+clip", re.IGNORECASE), "the clip"), (re.compile(r"this\s+footage", re.IGNORECASE), "this footage"), (re.compile(r"the\s+footage", re.IGNORECASE), "the footage"), + (re.compile(r"this\s+segment", re.IGNORECASE), "this segment"), + (re.compile(r"this\s+frame", re.IGNORECASE), "this frame"), + (re.compile(r"the\s+current\s+frame", re.IGNORECASE), "the current frame"), + (re.compile(r"frame\s+summary", re.IGNORECASE), "frame summary"), (re.compile(r"上面的片段", re.IGNORECASE), "上面的片段"), (re.compile(r"这段视频", re.IGNORECASE), "这段视频"), (re.compile(r"该视频", re.IGNORECASE), "该视频"), @@ -179,47 +183,56 @@ def check_referent_blacklist(question_text: str) -> list[str]: return violations +def _ngram_overlap_ratio( + text_words: list[str], + source_ngrams: set[tuple[str, ...]], + window: int, +) -> float: + """计算单段文本与来源 n-gram 集合的重叠率。 + + 参数: + text_words: 待比较文本的分词列表（已小写化）。 + source_ngrams: 来源素材的 n-gram 集合。 + window: n-gram 窗口大小。 + + 返回: + 重叠率 [0.0, 1.0]，词数不足窗口大小时返回 0.0。 + """ + if len(text_words) < window: + return 0.0 + + text_ngrams: set[tuple[str, ...]] = { + tuple(text_words[i : i + window]) for i in range(len(text_words) - window + 1) + } + + if not text_ngrams: + return 0.0 + + overlap = text_ngrams & source_ngrams + return len(overlap) / len(text_ngrams) + + def check_verbatim( question_text: str, correct_option: str, source_texts: list[str], window: int = 6, ) -> float: - """计算正确选项与来源素材的逐字重复率。 + """计算题目文本和正确选项与来源素材的最大逐字重复率。 - 使用滑动窗口 n-gram 集合交集方法：从选项文本提取所有 n-gram， - 与来源文本的 n-gram 集合求交集，计算重叠比例。 + 使用滑动窗口 n-gram 集合交集方法：分别从题目文本和选项文本提取 n-gram， + 各自与来源文本的 n-gram 集合求交集，返回两者中较大的重叠比例。 参数: - question_text: 题目文本（当前未使用，预留接口）。 + question_text: 题目文本。 correct_option: 正确选项文本（含 "X. " 前缀）。 source_texts: 来源素材文本列表。 window: n-gram 窗口大小。 返回: - 重复率 [0.0, 1.0]。0.0 表示无重叠，1.0 表示完全复制。 + 最大重复率 [0.0, 1.0]。0.0 表示无重叠，1.0 表示完全复制。 """ - # Phase 1: 提取选项纯文本（去掉可能的 "X. " 前缀） - option_text = correct_option - if len(option_text) >= 3 and option_text[1] == "." and option_text[2] == " ": - option_text = option_text[3:] - - # Phase 2: 分词（简单空格分词，转小写） - option_words = option_text.lower().split() - - # Phase 3: 选项词数不足窗口大小则无法构成 n-gram - if len(option_words) < window: - return 0.0 - - # Phase 4: 构造选项的 n-gram 集合 - option_ngrams: set[tuple[str, ...]] = set() - for i in range(len(option_words) - window + 1): - option_ngrams.add(tuple(option_words[i : i + window])) - - if not option_ngrams: - return 0.0 - - # Phase 5: 构造来源文本的 n-gram 集合 + # Phase 1: 构造来源文本的 n-gram 集合 source_ngrams: set[tuple[str, ...]] = set() for source in source_texts: words = source.lower().split() @@ -229,9 +242,20 @@ def check_verbatim( if not source_ngrams: return 0.0 - # Phase 6: 计算交集比例 - overlap = option_ngrams & source_ngrams - return len(overlap) / len(option_ngrams) + # Phase 2: 计算题目文本的重叠率 + question_words = question_text.lower().split() + question_ratio = _ngram_overlap_ratio(question_words, source_ngrams, window) + + # Phase 3: 提取选项纯文本（去掉可能的 "X. " 前缀） + option_text = correct_option + if len(option_text) >= 3 and option_text[1] == "." and option_text[2] == " ": + option_text = option_text[3:] + + option_words = option_text.lower().split() + option_ratio = _ngram_overlap_ratio(option_words, source_ngrams, window) + + # Phase 4: 返回两者中较大的重叠率 + return max(question_ratio, option_ratio) def has_time_anchor(question_text: str) -> bool: diff --git a/tests/unit/test_postprocess.py b/tests/unit/test_postprocess.py index 60ac59d..0041481 100644 --- a/tests/unit/test_postprocess.py +++ b/tests/unit/test_postprocess.py @@ -122,6 +122,31 @@ class TestReferentBlacklist: violations = check_referent_blacklist(q) assert len(violations) >= 1 + def test_this_segment_caught(self) -> None: + """'this segment' 被检测为违规。""" + q = "What action is performed in this segment?" + violations = check_referent_blacklist(q) + assert len(violations) >= 1 + assert any("this segment" in v.lower() for v in violations) + + def test_this_frame_caught(self) -> None: + """'this frame' 被检测为违规。""" + q = "What object is visible in this frame?" + violations = check_referent_blacklist(q) + assert len(violations) >= 1 + + def test_current_frame_caught(self) -> None: + """'the current frame' 被检测为违规。""" + q = "Based on the current frame, what is happening?" + violations = check_referent_blacklist(q) + assert len(violations) >= 1 + + def test_frame_summary_caught(self) -> None: + """'frame summary' 被检测为违规。""" + q = "According to the frame summary, what is the main event?" + violations = check_referent_blacklist(q) + assert len(violations) >= 1 + # --------------------------------------------------------------------------- # TestVerbatim @@ -186,6 +211,21 @@ class TestVerbatim: ) assert ratio == 0.0 + def test_question_text_overlap_detected(self) -> None: + """题目文本从来源复制但选项原创时，仍检测到高重叠率。""" + source = "The man walked slowly through the crowded market buying fresh vegetables" + # 题目直接复制来源的大部分内容 + question = "The man walked slowly through the crowded market buying what?" + # 选项完全原创 + option = "A. Fresh organic produce from the local farm stand" + ratio = check_verbatim( + question_text=question, + correct_option=option, + source_texts=[source], + window=6, + ) + assert ratio > 0.0, "题目文本的逐字重叠应被检测到" + # --------------------------------------------------------------------------- # TestTimeAnchor