fix(question_gen): Information Synopsis 去重阈值放宽至 0.90
该题型天然高相似(benchmark 均值 0.66,中位数 0.67), 通用阈值 0.75/0.85 导致几乎所有新题被误判重复。 按题型覆盖为 0.90,只拦截真正的复制品。 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -811,6 +811,11 @@ async def _run_generate(args: argparse.Namespace) -> None:
|
||||
output_dir = Path(args.output_dir)
|
||||
per_type = args.per_type
|
||||
similarity_threshold = args.similarity_threshold
|
||||
# Information Synopsis 题目天然高相似(均值 0.66,中位数 0.67),
|
||||
# 用通用阈值会导致几乎所有新题被误判重复。按题型放宽。
|
||||
_PER_TYPE_THRESHOLD = {
|
||||
"Information Synopsis": max(similarity_threshold, 0.90),
|
||||
}
|
||||
max_retries = args.max_retries
|
||||
concurrency = args.concurrency
|
||||
seed = args.seed
|
||||
@@ -951,7 +956,7 @@ async def _run_generate(args: argparse.Namespace) -> None:
|
||||
if (
|
||||
pool.ndim == 2
|
||||
and pool.shape[0] > 0
|
||||
and is_duplicate(candidate.question, pool, embed_fn, similarity_threshold)
|
||||
and is_duplicate(candidate.question, pool, embed_fn, _PER_TYPE_THRESHOLD.get(task_type, similarity_threshold))
|
||||
):
|
||||
logger.warning("去重: {} 与池中题目相似", candidate.question_id)
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user