feat: guard frozen split products against silent overwrite (--force)
This commit is contained in:
@@ -12,6 +12,7 @@ test,再以视频组为原子切出诊断 / 验证池,原子冻结 pools.jso
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import json
|
||||||
import sqlite3
|
import sqlite3
|
||||||
from collections import Counter, defaultdict
|
from collections import Counter, defaultdict
|
||||||
from dataclasses import asdict, dataclass, field
|
from dataclasses import asdict, dataclass, field
|
||||||
@@ -104,6 +105,38 @@ class SplitBuildResult:
|
|||||||
return getattr(self, key)
|
return getattr(self, key)
|
||||||
|
|
||||||
|
|
||||||
|
def _guard_frozen_products(out_path: Path, manifest_path: Path, *, force: bool) -> None:
|
||||||
|
"""冻结前的覆盖保护:产物已存在时按 force 决定报错或备份。
|
||||||
|
|
||||||
|
参数:
|
||||||
|
out_path: 目标 pools.json 路径。
|
||||||
|
manifest_path: 目标 split_manifest.json 路径。
|
||||||
|
force: False 时已存在即 FileExistsError;True 时把旧产物重命名为
|
||||||
|
.bak.<旧 pools_sha256 前 8 位或 'prev'> 再放行。
|
||||||
|
|
||||||
|
异常:
|
||||||
|
FileExistsError: force=False 且产物已存在(防静默覆盖冻结锚点)。
|
||||||
|
"""
|
||||||
|
if not out_path.exists() and not manifest_path.exists():
|
||||||
|
return
|
||||||
|
if not force:
|
||||||
|
raise FileExistsError(
|
||||||
|
f"已存在冻结产物 {out_path}(或其 manifest)。重跑切分会覆盖训练依赖的"
|
||||||
|
"冻结锚点——确认要替换请加 --force(旧产物将备份为 .bak.*)。"
|
||||||
|
)
|
||||||
|
# 备份后缀取旧 manifest 的 pools_sha256 前 8 位,无则用 'prev'
|
||||||
|
suffix = "prev"
|
||||||
|
if manifest_path.exists():
|
||||||
|
try:
|
||||||
|
old = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
suffix = str(old.get("pools_sha256", "prev"))[:8] or "prev"
|
||||||
|
except (json.JSONDecodeError, OSError):
|
||||||
|
suffix = "prev"
|
||||||
|
for p in (out_path, manifest_path):
|
||||||
|
if p.exists():
|
||||||
|
p.rename(p.with_name(f"{p.name}.bak.{suffix}"))
|
||||||
|
|
||||||
|
|
||||||
def build_split(
|
def build_split(
|
||||||
*,
|
*,
|
||||||
db_path: Path,
|
db_path: Path,
|
||||||
@@ -115,6 +148,7 @@ def build_split(
|
|||||||
out_path: Path,
|
out_path: Path,
|
||||||
manifest_path: Path,
|
manifest_path: Path,
|
||||||
generated_at: str,
|
generated_at: str,
|
||||||
|
force: bool = False,
|
||||||
) -> SplitBuildResult:
|
) -> SplitBuildResult:
|
||||||
"""顶层编排结果驱动视频级切分,冻结 pools.json + manifest 并跑防御断言。
|
"""顶层编排结果驱动视频级切分,冻结 pools.json + manifest 并跑防御断言。
|
||||||
|
|
||||||
@@ -137,6 +171,9 @@ def build_split(
|
|||||||
out_path: 冻结 pools.json 目标路径(原子写)。
|
out_path: 冻结 pools.json 目标路径(原子写)。
|
||||||
manifest_path: 溯源 manifest 目标路径(原子写)。
|
manifest_path: 溯源 manifest 目标路径(原子写)。
|
||||||
generated_at: 生成时间戳(ISO 字符串),由调用方传入以保证可复现。
|
generated_at: 生成时间戳(ISO 字符串),由调用方传入以保证可复现。
|
||||||
|
force: 覆盖保护开关。False(默认)时若 out_path/manifest_path 已存在即
|
||||||
|
FileExistsError(防静默覆盖训练依赖的冻结锚点);True 时先把旧产物备份为
|
||||||
|
.bak.* 再放行覆盖。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
SplitBuildResult,含 pools / manifest / assignment,支持字典式访问。
|
SplitBuildResult,含 pools / manifest / assignment,支持字典式访问。
|
||||||
@@ -199,6 +236,7 @@ def build_split(
|
|||||||
val_wrong_min=config.val_wrong_min,
|
val_wrong_min=config.val_wrong_min,
|
||||||
wrong_tier_by_video=dict(wrong_tier_by_video),
|
wrong_tier_by_video=dict(wrong_tier_by_video),
|
||||||
)
|
)
|
||||||
|
_guard_frozen_products(out_path, manifest_path, force=force)
|
||||||
save_pools(pools, out_path)
|
save_pools(pools, out_path)
|
||||||
|
|
||||||
# Phase 4: 溯源 manifest(pools_sha256 锚定冻结内容)。
|
# Phase 4: 溯源 manifest(pools_sha256 锚定冻结内容)。
|
||||||
|
|||||||
@@ -519,6 +519,7 @@ async def run_pipeline(
|
|||||||
questions_dir: Path,
|
questions_dir: Path,
|
||||||
out_dir: Path,
|
out_dir: Path,
|
||||||
generated_at: str,
|
generated_at: str,
|
||||||
|
force: bool = False,
|
||||||
) -> SplitBuildResult:
|
) -> SplitBuildResult:
|
||||||
"""内联三阶段:Phase 0 INFRA T0 补录 → Phase 1 诊断 → Phase 2 冻结切分 → McNemar 护栏。
|
"""内联三阶段:Phase 0 INFRA T0 补录 → Phase 1 诊断 → Phase 2 冻结切分 → McNemar 护栏。
|
||||||
|
|
||||||
@@ -534,6 +535,7 @@ async def run_pipeline(
|
|||||||
questions_dir: benchmark 题库目录(Phase 2 加载题库切池)。
|
questions_dir: benchmark 题库目录(Phase 2 加载题库切池)。
|
||||||
out_dir: 冻结产物目录(pools.json + split_manifest.json)。
|
out_dir: 冻结产物目录(pools.json + split_manifest.json)。
|
||||||
generated_at: 生成时间戳(ISO 字符串,由调用方传入;见模块 C-2 复现锚点约定)。
|
generated_at: 生成时间戳(ISO 字符串,由调用方传入;见模块 C-2 复现锚点约定)。
|
||||||
|
force: 覆盖已存在冻结产物开关,透传给 build_split(False 时已存在即报错)。
|
||||||
|
|
||||||
返回:
|
返回:
|
||||||
SplitBuildResult(冻结三池 + manifest + assignment)。
|
SplitBuildResult(冻结三池 + manifest + assignment)。
|
||||||
@@ -577,6 +579,7 @@ async def run_pipeline(
|
|||||||
out_path=out_dir / "pools.json",
|
out_path=out_dir / "pools.json",
|
||||||
manifest_path=out_dir / "split_manifest.json",
|
manifest_path=out_dir / "split_manifest.json",
|
||||||
generated_at=generated_at,
|
generated_at=generated_at,
|
||||||
|
force=force,
|
||||||
)
|
)
|
||||||
|
|
||||||
# McNemar 功效护栏(build_split 契约外的 capstone 层校验)。
|
# McNemar 功效护栏(build_split 契约外的 capstone 层校验)。
|
||||||
@@ -650,6 +653,7 @@ def _execute_real(config: VideoSplitConfig, fingerprint: str, args: argparse.Nam
|
|||||||
questions_dir=questions_dir,
|
questions_dir=questions_dir,
|
||||||
out_dir=out_dir,
|
out_dir=out_dir,
|
||||||
generated_at=generated_at,
|
generated_at=generated_at,
|
||||||
|
force=args.force,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
@@ -773,6 +777,11 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
|||||||
"对 manifest 做字节级复现比对。"
|
"对 manifest 做字节级复现比对。"
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--force",
|
||||||
|
action="store_true",
|
||||||
|
help="覆盖已存在的冻结 pools.json/manifest(旧产物备份为 .bak.*)",
|
||||||
|
)
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -303,3 +303,29 @@ def test_git_short_sha_nonempty():
|
|||||||
sha = cli.git_short_sha()
|
sha = cli.git_short_sha()
|
||||||
assert sha
|
assert sha
|
||||||
assert len(sha) >= 4
|
assert len(sha) >= 4
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_split_refuses_overwrite_without_force(tmp_path):
|
||||||
|
"""已存在指纹不同的 pools.json 时,force=False 必须报错不覆盖。"""
|
||||||
|
from app.harness.build_split import _guard_frozen_products
|
||||||
|
|
||||||
|
out_path = tmp_path / "pools.json"
|
||||||
|
out_path.write_text('{"split_mode":"global"}', encoding="utf-8")
|
||||||
|
manifest_path = tmp_path / "split_manifest.json"
|
||||||
|
|
||||||
|
with pytest.raises(FileExistsError, match="已存在冻结产物"):
|
||||||
|
_guard_frozen_products(out_path, manifest_path, force=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_split_force_backs_up_old(tmp_path):
|
||||||
|
"""force=True 时旧产物被备份为 .bak.* 再允许覆盖。"""
|
||||||
|
from app.harness.build_split import _guard_frozen_products
|
||||||
|
|
||||||
|
out_path = tmp_path / "pools.json"
|
||||||
|
out_path.write_text('{"old":1}', encoding="utf-8")
|
||||||
|
manifest_path = tmp_path / "split_manifest.json"
|
||||||
|
manifest_path.write_text('{"pools_sha256":"deadbeef00000000"}', encoding="utf-8")
|
||||||
|
|
||||||
|
_guard_frozen_products(out_path, manifest_path, force=True)
|
||||||
|
baks = list(tmp_path.glob("pools.json.bak.*"))
|
||||||
|
assert len(baks) == 1, f"未备份旧产物: {list(tmp_path.iterdir())}"
|
||||||
|
|||||||
Reference in New Issue
Block a user