fix: make pools.json freeze atomic + add split manifest
This commit is contained in:
+19
-8
@@ -10,6 +10,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
@@ -490,6 +491,22 @@ def _dict_to_q(d: dict) -> GeneratedQuestion:
|
||||
)
|
||||
|
||||
|
||||
def _atomic_write_json(path: Path, obj: object) -> None:
|
||||
"""原子写 JSON:先写 <path>.tmp 再 os.replace,避免半截文件。
|
||||
|
||||
崩溃或并发写入时,直接 write_text 可能留下被截断的 JSON;本助手先把完整
|
||||
内容写入同目录临时文件,再用同一文件系统上的原子 rename 替换目标,
|
||||
保证读者只会看到旧完整文件或新完整文件。
|
||||
|
||||
参数:
|
||||
path: 目标 JSON 文件路径。
|
||||
obj: 可 json 序列化对象。
|
||||
"""
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(obj, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def save_pools(
|
||||
pools: Pools,
|
||||
path: Path,
|
||||
@@ -542,10 +559,7 @@ def save_pools(
|
||||
data["train_ratio"] = config.train_ratio
|
||||
data["test_source"] = str(config.test_questions_dir) if config.test_questions_dir else None
|
||||
|
||||
path.write_text(
|
||||
json.dumps(data, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
_atomic_write_json(path, data)
|
||||
|
||||
|
||||
def load_pools(path: Path) -> Pools:
|
||||
@@ -767,10 +781,7 @@ def build_or_load_pools(
|
||||
},
|
||||
}
|
||||
# 重新冻结
|
||||
pools_path.write_text(
|
||||
json.dumps(raw, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
_atomic_write_json(pools_path, raw)
|
||||
logger.info(
|
||||
"per_category 增量追加 {} 个新类别: {}",
|
||||
len(new_types),
|
||||
|
||||
Reference in New Issue
Block a user