实现本地清洗实验与产物保存

2026-08-22 17:25:44 +08:00
parent eed2119016
commit 6fcc7d5736
13 changed files with 2701 additions and 20 deletions
@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+import json
+import stat
+from hashlib import sha256
+from pathlib import Path
+
+import pytest
+
+import mdpolish.artifact_store as artifact_store
+from mdpolish.artifact_store import ArtifactStoreError, StoredDocument, publish_run
+
+
+def stored_document(document_id: str = "paper", output: bytes | None = b"cleaned\n") -> StoredDocument:
+    status = "success" if output is not None else "failed"
+    current_sha256 = sha256(output).hexdigest() if output is not None else "2" * 64
+    payload = {
+        "schema_version": 1,
+        "document": {"document_id": document_id, "source_label": f"inputs/{document_id}.md"},
+        "status": status,
+        "input_sha256": "1" * 64,
+        "current_sha256": current_sha256,
+        "changes": [],
+        "errors": [] if status == "success" else [{"error_type": "SyntheticError"}],
+        "residual_proposals": [],
+        "output": {
+            "cleaned_path": "cleaned.md" if output is not None else None,
+            "diff_path": "changes.diff" if output is not None else None,
+        },
+    }
+    return StoredDocument(
+        document_id=document_id,
+        result_json=(json.dumps(payload, indent=2) + "\n").encode(),
+        cleaned_markdown=output,
+        diff=b"" if output is not None else None,
+        output_sha256=sha256(output).hexdigest() if output is not None else None,
+    )
+
+
+def manifest_json(
+    run_date: str,
+    run_id: str,
+    documents: tuple[StoredDocument, ...],
+) -> bytes:
+    indexes: list[dict[str, object]] = []
+    statuses: list[str] = []
+    change_count = 0
+    for document in documents:
+        report = json.loads(document.result_json)
+        status = report["status"]
+        statuses.append(status)
+        change_count += len(report["changes"])
+        base = f"documents/{document.document_id}"
+        indexes.append(
+            {
+                "document_id": document.document_id,
+                "source_label": report["document"]["source_label"],
+                "status": status,
+                "input_sha256": report["input_sha256"],
+                "current_sha256": report["current_sha256"],
+                "change_count": len(report["changes"]),
+                "result_path": f"{base}/result.json",
+                "cleaned_path": f"{base}/cleaned.md" if status == "success" else None,
+                "diff_path": f"{base}/changes.diff" if status == "success" else None,
+            }
+        )
+    failed_count = statuses.count("failed")
+    unstable_count = statuses.count("unstable")
+    overall_status = "failed" if failed_count else "unstable" if unstable_count else "success"
+    payload = {
+        "schema_version": 1,
+        "run": {"run_id": run_id, "run_date": run_date, "status": overall_status},
+        "documents": indexes,
+        "summary": {
+            "document_count": len(documents),
+            "success_count": statuses.count("success"),
+            "failed_count": failed_count,
+            "unstable_count": unstable_count,
+            "change_count": change_count,
+        },
+    }
+    return (json.dumps(payload, indent=2) + "\n").encode()
+
+
+def test_publish_run_creates_private_date_layout_and_status_specific_files(tmp_path: Path) -> None:
+    artifacts_root = tmp_path / "artifacts"
+    documents = (stored_document("success"), stored_document("failed", None))
+
+    run_directory = publish_run(
+        artifacts_root=artifacts_root,
+        run_date="2026-08-22",
+        run_id="example-run",
+        manifest_json=manifest_json("2026-08-22", "example-run", documents),
+        documents=documents,
+    )
+
+    assert run_directory == artifacts_root / "2026-08-22" / "runs" / "example-run"
+    assert json.loads((run_directory / "manifest.json").read_bytes())["run"]["status"] == "failed"
+    assert (run_directory / "documents/success/result.json").is_file()
+    assert (run_directory / "documents/success/cleaned.md").read_bytes() == b"cleaned\n"
+    assert (run_directory / "documents/success/changes.diff").read_bytes() == b""
+    assert (run_directory / "documents/failed/result.json").is_file()
+    assert not (run_directory / "documents/failed/cleaned.md").exists()
+    assert not (run_directory / "documents/failed/changes.diff").exists()
+
+    for directory in (
+        artifacts_root,
+        artifacts_root / "2026-08-22",
+        artifacts_root / "2026-08-22/runs",
+        run_directory,
+        run_directory / "documents",
+        run_directory / "documents/success",
+    ):
+        assert stat.S_IMODE(directory.stat().st_mode) == 0o700
+    for artifact_file in run_directory.rglob("*"):
+        if artifact_file.is_file():
+            assert stat.S_IMODE(artifact_file.stat().st_mode) == 0o600
+
+
+def test_publish_run_rejects_existing_target_without_overwriting(tmp_path: Path) -> None:
+    artifacts_root = tmp_path / "artifacts"
+    documents = (stored_document(),)
+    first_manifest = manifest_json("2026-08-22", "same-run", documents)
+    run_directory = publish_run(
+        artifacts_root=artifacts_root,
+        run_date="2026-08-22",
+        run_id="same-run",
+        manifest_json=first_manifest,
+        documents=documents,
+    )
+
+    with pytest.raises(ArtifactStoreError, match="already exists"):
+        publish_run(
+            artifacts_root=artifacts_root,
+            run_date="2026-08-22",
+            run_id="same-run",
+            manifest_json=first_manifest,
+            documents=documents,
+        )
+
+    assert (run_directory / "manifest.json").read_bytes() == first_manifest
+
+
+@pytest.mark.parametrize(
+    ("run_date", "run_id"),
+    [
+        ("2026-8-22", "valid"),
+        ("2026-02-30", "valid"),
+        ("2026-08-22", "Uppercase"),
+        ("2026-08-22", "../escape"),
+        ("2026-08-22", "two..dots"),
+    ],
+)
+def test_publish_run_rejects_unsafe_date_and_run_id(tmp_path: Path, run_date: str, run_id: str) -> None:
+    documents = (stored_document(),)
+    with pytest.raises(ArtifactStoreError):
+        publish_run(
+            artifacts_root=tmp_path / "artifacts",
+            run_date=run_date,
+            run_id=run_id,
+            manifest_json=manifest_json(run_date, run_id, documents),
+            documents=documents,
+        )
+
+
+def test_publish_run_rejects_inconsistent_or_duplicate_document_artifacts(tmp_path: Path) -> None:
+    valid = stored_document()
+    bad_hash = StoredDocument("paper", valid.result_json, b"output", b"", "0" * 64)
+    with pytest.raises(ArtifactStoreError, match="output hash"):
+        publish_run(
+            artifacts_root=tmp_path / "artifacts-a",
+            run_date="2026-08-22",
+            run_id="bad-hash",
+            manifest_json=manifest_json("2026-08-22", "bad-hash", (bad_hash,)),
+            documents=(bad_hash,),
+        )
+
+    duplicates = (stored_document(), stored_document())
+    with pytest.raises(ArtifactStoreError, match="unique"):
+        publish_run(
+            artifacts_root=tmp_path / "artifacts-b",
+            run_date="2026-08-22",
+            run_id="duplicate",
+            manifest_json=manifest_json("2026-08-22", "duplicate", duplicates),
+            documents=duplicates,
+        )
+
+
+def test_publish_run_rejects_manifest_path_or_document_mismatch(tmp_path: Path) -> None:
+    documents = (stored_document(),)
+    wrong_date = manifest_json("2026-08-21", "review", documents)
+    with pytest.raises(ArtifactStoreError, match="identity"):
+        publish_run(
+            artifacts_root=tmp_path / "artifacts-a",
+            run_date="2026-08-22",
+            run_id="review",
+            manifest_json=wrong_date,
+            documents=documents,
+        )
+
+    payload = json.loads(manifest_json("2026-08-22", "review", documents))
+    payload["documents"][0]["change_count"] = 99
+    mismatched_index = (json.dumps(payload, indent=2) + "\n").encode()
+    with pytest.raises(ArtifactStoreError, match="index"):
+        publish_run(
+            artifacts_root=tmp_path / "artifacts-b",
+            run_date="2026-08-22",
+            run_id="review",
+            manifest_json=mismatched_index,
+            documents=documents,
+        )
+
+
+def test_publish_race_does_not_replace_a_new_target(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    documents = (stored_document(),)
+    original_rename = artifact_store._rename_no_replace
+
+    def create_competing_target(source: Path, target: Path) -> None:
+        target.mkdir(mode=0o700)
+        (target / "keep").write_bytes(b"existing")
+        original_rename(source, target)
+
+    monkeypatch.setattr(artifact_store, "_rename_no_replace", create_competing_target)
+    with pytest.raises(ArtifactStoreError, match="already exists"):
+        publish_run(
+            artifacts_root=tmp_path / "artifacts",
+            run_date="2026-08-22",
+            run_id="raced",
+            manifest_json=manifest_json("2026-08-22", "raced", documents),
+            documents=documents,
+        )
+
+    target = tmp_path / "artifacts/2026-08-22/runs/raced"
+    assert (target / "keep").read_bytes() == b"existing"
+    assert not any(path.name.startswith(".raced.") for path in target.parent.iterdir())
+
+
+def test_write_failure_cleans_temporary_directory_and_does_not_publish(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    original_write = artifact_store._write_private_file
+    call_count = 0
+
+    def fail_second_write(path: Path, content: bytes) -> None:
+        nonlocal call_count
+        call_count += 1
+        if call_count == 2:
+            raise OSError("synthetic write failure")
+        original_write(path, content)
+
+    monkeypatch.setattr(artifact_store, "_write_private_file", fail_second_write)
+    documents = (stored_document(),)
+
+    with pytest.raises(OSError, match="synthetic"):
+        publish_run(
+            artifacts_root=tmp_path / "artifacts",
+            run_date="2026-08-22",
+            run_id="broken",
+            manifest_json=manifest_json("2026-08-22", "broken", documents),
+            documents=documents,
+        )
+
+    runs_directory = tmp_path / "artifacts/2026-08-22/runs"
+    assert list(runs_directory.iterdir()) == []