diff --git a/tools/generate_questions.py b/tools/generate_questions.py index 317795f..a1b9ee0 100644 --- a/tools/generate_questions.py +++ b/tools/generate_questions.py @@ -811,6 +811,11 @@ async def _run_generate(args: argparse.Namespace) -> None: output_dir = Path(args.output_dir) per_type = args.per_type similarity_threshold = args.similarity_threshold + # Information Synopsis 题目天然高相似(均值 0.66,中位数 0.67), + # 用通用阈值会导致几乎所有新题被误判重复。按题型放宽。 + _PER_TYPE_THRESHOLD = { + "Information Synopsis": max(similarity_threshold, 0.90), + } max_retries = args.max_retries concurrency = args.concurrency seed = args.seed @@ -951,7 +956,7 @@ async def _run_generate(args: argparse.Namespace) -> None: if ( pool.ndim == 2 and pool.shape[0] > 0 - and is_duplicate(candidate.question, pool, embed_fn, similarity_threshold) + and is_duplicate(candidate.question, pool, embed_fn, _PER_TYPE_THRESHOLD.get(task_type, similarity_threshold)) ): logger.warning("去重: {} 与池中题目相似", candidate.question_id) continue