fix: make pools.json freeze atomic + add split manifest

This commit is contained in:
2026-07-15 12:28:48 -04:00
parent fd907aab46
commit aa10485b9f
3 changed files with 141 additions and 8 deletions
+19 -8
View File
@@ -10,6 +10,7 @@ from __future__ import annotations
import json
import math
import os
import random
from collections import defaultdict
from dataclasses import dataclass, field
@@ -490,6 +491,22 @@ def _dict_to_q(d: dict) -> GeneratedQuestion:
)
def _atomic_write_json(path: Path, obj: object) -> None:
"""原子写 JSON:先写 <path>.tmp 再 os.replace,避免半截文件。
崩溃或并发写入时,直接 write_text 可能留下被截断的 JSON;本助手先把完整
内容写入同目录临时文件,再用同一文件系统上的原子 rename 替换目标,
保证读者只会看到旧完整文件或新完整文件。
参数:
path: 目标 JSON 文件路径。
obj: 可 json 序列化对象。
"""
tmp = path.with_suffix(path.suffix + ".tmp")
tmp.write_text(json.dumps(obj, ensure_ascii=False, indent=2), encoding="utf-8")
os.replace(tmp, path)
def save_pools(
pools: Pools,
path: Path,
@@ -542,10 +559,7 @@ def save_pools(
data["train_ratio"] = config.train_ratio
data["test_source"] = str(config.test_questions_dir) if config.test_questions_dir else None
path.write_text(
json.dumps(data, ensure_ascii=False, indent=2),
encoding="utf-8",
)
_atomic_write_json(path, data)
def load_pools(path: Path) -> Pools:
@@ -767,10 +781,7 @@ def build_or_load_pools(
},
}
# 重新冻结
pools_path.write_text(
json.dumps(raw, ensure_ascii=False, indent=2),
encoding="utf-8",
)
_atomic_write_json(pools_path, raw)
logger.info(
"per_category 增量追加 {} 个新类别: {}",
len(new_types),