fix(question_gen): Information Synopsis 去重阈值放宽至 0.90

该题型天然高相似(benchmark 均值 0.66,中位数 0.67),
通用阈值 0.75/0.85 导致几乎所有新题被误判重复。
按题型覆盖为 0.90,只拦截真正的复制品。

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-10 08:55:47 -04:00
parent 25c8d5ec42
commit 735471df3c
+6 -1
View File
@@ -811,6 +811,11 @@ async def _run_generate(args: argparse.Namespace) -> None:
output_dir = Path(args.output_dir) output_dir = Path(args.output_dir)
per_type = args.per_type per_type = args.per_type
similarity_threshold = args.similarity_threshold similarity_threshold = args.similarity_threshold
# Information Synopsis 题目天然高相似(均值 0.66,中位数 0.67),
# 用通用阈值会导致几乎所有新题被误判重复。按题型放宽。
_PER_TYPE_THRESHOLD = {
"Information Synopsis": max(similarity_threshold, 0.90),
}
max_retries = args.max_retries max_retries = args.max_retries
concurrency = args.concurrency concurrency = args.concurrency
seed = args.seed seed = args.seed
@@ -951,7 +956,7 @@ async def _run_generate(args: argparse.Namespace) -> None:
if ( if (
pool.ndim == 2 pool.ndim == 2
and pool.shape[0] > 0 and pool.shape[0] > 0
and is_duplicate(candidate.question, pool, embed_fn, similarity_threshold) and is_duplicate(candidate.question, pool, embed_fn, _PER_TYPE_THRESHOLD.get(task_type, similarity_threshold))
): ):
logger.warning("去重: {} 与池中题目相似", candidate.question_id) logger.warning("去重: {} 与池中题目相似", candidate.question_id)
continue continue