feat: 装配两个库（不含 CEUS 结果）
按任务书 01 实现 delivery、library、answers、runs（快照部分）、build 五个模块， scripts/build.py、scripts/scan_library.py 与 config/build.yaml（decode_codes: false）。 交付包整表检查 T1–T4、映射表检查 T5、逐例检查 C01–C13，一例遇到第一项不过即整例跳过。 与任务书原文的出入： - C04 报告表名随视频名配对（负责人 2026-10-16 定）。 - 按两轮独立审阅加严 C11（更多换行符、比例号冒号）、C12（同义词与词根）、 C13（空格、周岁、中文数字、繁体）；泄露扫描做 NFKC；关键图只接受能保真重编码的 PNG； YAML 配置拒绝重复键和空路径。 - import-linter 第 1 条拆成互不依赖与不引用其他 cadx 模块两条。 文档：结构.md 第五、六、八部分按实际更新；新增 guides/装配两个库.md；WORK_LOG 更新。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
"""answers：标准答案库的目录布局、文件格式、写入和评价要用的读取。
|
||||
|
||||
拒绝读取名字以 `.partial` 结尾的目录；读时核对清单里的 SHA-256。
|
||||
"""
|
||||
|
||||
from cadx.answers._files import (
|
||||
AliasRow,
|
||||
DiagnosisRow,
|
||||
ReviewRow,
|
||||
SkippedRow,
|
||||
read_aliases,
|
||||
read_diagnoses,
|
||||
write_answers,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AliasRow",
|
||||
"DiagnosisRow",
|
||||
"ReviewRow",
|
||||
"SkippedRow",
|
||||
"read_aliases",
|
||||
"read_diagnoses",
|
||||
"write_answers",
|
||||
]
|
||||
@@ -0,0 +1,177 @@
|
||||
"""标准答案库的目录布局、写入和读取。
|
||||
|
||||
```
|
||||
<构建目录>/
|
||||
manifest.json 构建号、跳过清单的说明、本目录其余全部文件的 SHA-256
|
||||
snapshot/ 运行快照（由 runs 写，先于本模块写入）
|
||||
aliases.csv 化名、病例号、视频号
|
||||
diagnoses.csv 化名、病例号、三张表的诊断编码、诊断名
|
||||
skipped.csv 没进资料库的病例和第一项没过的检查
|
||||
review.csv 进库病例报告里含复核措辞的句子
|
||||
```
|
||||
|
||||
CSV 用带 BOM 的 UTF-8，医生用 Excel 直接打开不乱码。
|
||||
"""
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
MANIFEST = "manifest.json"
|
||||
_ENCODING = "utf-8-sig"
|
||||
_JOIN = "; "
|
||||
|
||||
ALIASES_HEADER = ("化名", "病例号", "视频号")
|
||||
DIAGNOSES_HEADER = ("化名", "病例号", "临床表诊断编码", "超声表诊断编码", "MR表诊断编码", "诊断名")
|
||||
SKIPPED_HEADER = ("病例", "检查编号", "检查说明", "文件", "列", "涉及值")
|
||||
REVIEW_HEADER = ("化名", "路", "命中的词", "所在句子")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AliasRow:
|
||||
"""化名与真实病例号、视频号的对照。"""
|
||||
|
||||
alias: str
|
||||
case_no: str
|
||||
video_id: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DiagnosisRow:
|
||||
"""一例的金标准：三张表各自的诊断编码，解码后的诊断名（不解码时为 None）。"""
|
||||
|
||||
alias: str
|
||||
case_no: str
|
||||
clinical_code: int
|
||||
ultrasound_code: int
|
||||
mr_code: int
|
||||
name: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkippedRow:
|
||||
"""跳过清单的一行。`case` 是病例号，或认不出编号时的「文件 第 N 行」。"""
|
||||
|
||||
case: str
|
||||
check: str
|
||||
description: str
|
||||
files: tuple[str, ...]
|
||||
columns: tuple[str, ...]
|
||||
value_json: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReviewRow:
|
||||
"""复核清单的一行：`route` 是「超声」或「MR」。"""
|
||||
|
||||
alias: str
|
||||
route: str
|
||||
words: str
|
||||
sentence: str
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _write_csv(path: Path, header: tuple[str, ...], rows: Sequence[Sequence[object]]) -> None:
|
||||
with path.open("w", encoding=_ENCODING, newline="") as handle:
|
||||
writer = csv.writer(handle)
|
||||
writer.writerow(header)
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def write_answers(
|
||||
answers_dir: Path,
|
||||
build_id: str,
|
||||
skipped_note: str,
|
||||
aliases: Sequence[AliasRow],
|
||||
diagnoses: Sequence[DiagnosisRow],
|
||||
skipped: Sequence[SkippedRow],
|
||||
reviews: Sequence[ReviewRow],
|
||||
) -> None:
|
||||
"""写四张表，再给目录里全部文件（含已写好的 snapshot/）算 SHA-256 写进清单。"""
|
||||
_write_csv(
|
||||
answers_dir / "aliases.csv",
|
||||
ALIASES_HEADER,
|
||||
[(r.alias, r.case_no, r.video_id) for r in aliases],
|
||||
)
|
||||
_write_csv(
|
||||
answers_dir / "diagnoses.csv",
|
||||
DIAGNOSES_HEADER,
|
||||
[
|
||||
(r.alias, r.case_no, r.clinical_code, r.ultrasound_code, r.mr_code, r.name or "")
|
||||
for r in diagnoses
|
||||
],
|
||||
)
|
||||
_write_csv(
|
||||
answers_dir / "skipped.csv",
|
||||
SKIPPED_HEADER,
|
||||
[
|
||||
(
|
||||
r.case,
|
||||
r.check,
|
||||
r.description,
|
||||
_JOIN.join(r.files),
|
||||
_JOIN.join(r.columns),
|
||||
r.value_json,
|
||||
)
|
||||
for r in skipped
|
||||
],
|
||||
)
|
||||
_write_csv(
|
||||
answers_dir / "review.csv",
|
||||
REVIEW_HEADER,
|
||||
[(r.alias, r.route, r.words, r.sentence) for r in reviews],
|
||||
)
|
||||
files = {
|
||||
p.relative_to(answers_dir).as_posix(): _sha256(p)
|
||||
for p in sorted(answers_dir.rglob("*"))
|
||||
if p.is_file()
|
||||
}
|
||||
manifest = {"build_id": build_id, "skipped_note": skipped_note, "files": files}
|
||||
(answers_dir / MANIFEST).write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def _read_csv(answers_dir: Path, name: str, header: tuple[str, ...]) -> list[list[str]]:
|
||||
if answers_dir.name.endswith(".partial"):
|
||||
raise ValueError(f"{answers_dir} 是没写完的构建（.partial），不能读")
|
||||
manifest = cast(
|
||||
dict[str, Any], json.loads((answers_dir / MANIFEST).read_text(encoding="utf-8"))
|
||||
)
|
||||
path = answers_dir / name
|
||||
if _sha256(path) != manifest["files"][name]:
|
||||
raise ValueError(f"{name} 的 SHA-256 与清单不一致，答案库被改动过")
|
||||
with path.open(encoding=_ENCODING, newline="") as handle:
|
||||
rows = list(csv.reader(handle))
|
||||
if tuple(rows[0]) != header:
|
||||
raise ValueError(f"{name} 的表头与约定不一致")
|
||||
return rows[1:]
|
||||
|
||||
|
||||
def read_aliases(answers_dir: Path) -> tuple[AliasRow, ...]:
|
||||
"""读化名对照。"""
|
||||
rows = _read_csv(answers_dir, "aliases.csv", ALIASES_HEADER)
|
||||
return tuple(AliasRow(alias=a, case_no=c, video_id=v) for a, c, v in rows)
|
||||
|
||||
|
||||
def read_diagnoses(answers_dir: Path) -> tuple[DiagnosisRow, ...]:
|
||||
"""读金标准。诊断名一栏为空表示构建时没有解码。"""
|
||||
rows = _read_csv(answers_dir, "diagnoses.csv", DIAGNOSES_HEADER)
|
||||
return tuple(
|
||||
DiagnosisRow(
|
||||
alias=alias,
|
||||
case_no=case_no,
|
||||
clinical_code=int(clinical),
|
||||
ultrasound_code=int(ultrasound),
|
||||
mr_code=int(mr),
|
||||
name=name or None,
|
||||
)
|
||||
for alias, case_no, clinical, ultrasound, mr, name in rows
|
||||
)
|
||||
@@ -0,0 +1,13 @@
|
||||
"""build：装配两个库。可见性（每一路放什么）、化名、清单内容只在这里决定。"""
|
||||
|
||||
from cadx.build._assemble import BuildSummary, make_aliases, run_build
|
||||
from cadx.build._config import BuildConfig, load_build_config, load_code_maps
|
||||
|
||||
__all__ = [
|
||||
"BuildConfig",
|
||||
"BuildSummary",
|
||||
"load_build_config",
|
||||
"load_code_maps",
|
||||
"make_aliases",
|
||||
"run_build",
|
||||
]
|
||||
@@ -0,0 +1,198 @@
|
||||
"""装配一次构建：决定每一路放什么、化名怎么取，写出资料库和答案库。
|
||||
|
||||
全项目只有这里同时碰得到原始资料和答案。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import random
|
||||
import string
|
||||
from collections import Counter
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
from cadx.answers import AliasRow, DiagnosisRow, ReviewRow, SkippedRow, write_answers
|
||||
from cadx.build._config import load_build_config, load_code_maps
|
||||
from cadx.delivery import (
|
||||
CODED_COLUMNS,
|
||||
SKIPPED_NOTE,
|
||||
CaseRecord,
|
||||
CodeMaps,
|
||||
read_delivery,
|
||||
review_sentences,
|
||||
)
|
||||
from cadx.library import LibraryCase, write_library
|
||||
from cadx.runs import make_build_id, read_git_state, write_snapshot
|
||||
|
||||
ALIAS_ALPHABET = string.ascii_lowercase + string.digits
|
||||
ALIAS_LENGTH = 4
|
||||
# 化名里出现亚型缩写会被泄露扫描（不分大小写）误报，抽到就重抽。
|
||||
_ALIAS_BANNED = ("ipmn", "spn", "mcn", "scn")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BuildSummary:
|
||||
"""一次构建的计数，供终端打印；不含病例号和资料内容。"""
|
||||
|
||||
build_id: str
|
||||
library_dir: Path
|
||||
answers_dir: Path
|
||||
total_cases: int
|
||||
passed_cases: int
|
||||
skipped_cases: int
|
||||
unrecognized_rows: int
|
||||
skipped_by_check: dict[str, int]
|
||||
review_rows: int
|
||||
skipped_note: str
|
||||
|
||||
|
||||
def make_aliases(case_nos: Sequence[str], seed: int) -> dict[str, str]:
|
||||
"""按病例号排序后依次抽化名：同一个种子、同一批病例得到同样的化名。"""
|
||||
rng = random.Random(seed) # noqa: S311 化名只需可复现，不需要密码学强度
|
||||
aliases: dict[str, str] = {}
|
||||
for case_no in sorted(case_nos):
|
||||
while True:
|
||||
alias = "".join(rng.choice(ALIAS_ALPHABET) for _ in range(ALIAS_LENGTH))
|
||||
if alias not in aliases.values() and not any(b in alias for b in _ALIAS_BANNED):
|
||||
break
|
||||
aliases[case_no] = alias
|
||||
return aliases
|
||||
|
||||
|
||||
def _clinical_fields(
|
||||
case: CaseRecord, code_maps: CodeMaps | None
|
||||
) -> dict[str, str | int | float | None]:
|
||||
"""临床一路的 22 列；解码时编码列写映射表里的含义，没测的化验保持 None。"""
|
||||
if code_maps is None:
|
||||
return dict(case.clinical)
|
||||
return {
|
||||
column: code_maps[column][cast(int, value)]
|
||||
if column in CODED_COLUMNS and value is not None
|
||||
else value
|
||||
for column, value in case.clinical.items()
|
||||
}
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _rename(partial: Path, final: Path) -> None:
|
||||
# POSIX 的 rename 会悄悄覆盖空目录，所以改名前再查一次。
|
||||
if final.exists():
|
||||
raise FileExistsError(f"{final} 已存在，不覆盖")
|
||||
partial.rename(final)
|
||||
|
||||
|
||||
def run_build(config_path: Path, repo_root: Path, now: datetime) -> BuildSummary:
|
||||
"""读配置和交付包，装配一对同号的资料库与答案库。`now` 是本地时间，定构建号。"""
|
||||
config = load_build_config(config_path, repo_root)
|
||||
code_maps = load_code_maps(config.code_maps) if config.code_maps is not None else None
|
||||
delivery = read_delivery(config.delivery_dir, code_maps)
|
||||
|
||||
git = read_git_state(repo_root)
|
||||
build_id = make_build_id(now, git)
|
||||
library_dir = config.library_root / build_id
|
||||
answers_dir = config.answers_root / build_id
|
||||
library_partial = config.library_root / f"{build_id}.partial"
|
||||
answers_partial = config.answers_root / f"{build_id}.partial"
|
||||
for final in (library_dir, answers_dir):
|
||||
if final.exists():
|
||||
raise FileExistsError(f"{final} 已存在，不覆盖")
|
||||
for partial in (library_partial, answers_partial):
|
||||
partial.mkdir(parents=True, exist_ok=False)
|
||||
|
||||
aliases = make_aliases([case.case_no for case in delivery.cases], config.alias_seed)
|
||||
by_alias = sorted(delivery.cases, key=lambda case: aliases[case.case_no])
|
||||
|
||||
configs = {"build.yaml": config_path}
|
||||
if config.code_maps is not None:
|
||||
configs["code_maps.yaml"] = config.code_maps
|
||||
write_library(
|
||||
build_dir=library_partial,
|
||||
build_id=build_id,
|
||||
git_commit=git.commit,
|
||||
git_clean=git.clean,
|
||||
config_sha256={name: _sha256(path) for name, path in configs.items()},
|
||||
codes_decoded=code_maps is not None,
|
||||
uncompared=delivery.uncompared,
|
||||
ceus_source=None,
|
||||
cases=[
|
||||
LibraryCase(
|
||||
alias=aliases[case.case_no],
|
||||
clinical_fields=_clinical_fields(case, code_maps),
|
||||
ultrasound_report=case.ultrasound_report,
|
||||
key_image_source=case.key_image,
|
||||
mr_report=case.mr_report,
|
||||
)
|
||||
for case in by_alias
|
||||
],
|
||||
)
|
||||
|
||||
reviews = [
|
||||
ReviewRow(alias=aliases[case.case_no], route=route, words=words, sentence=sentence)
|
||||
for case in by_alias
|
||||
for route, text in (("超声", case.ultrasound_report), ("MR", case.mr_report))
|
||||
for words, sentence in review_sentences(text)
|
||||
]
|
||||
write_snapshot(
|
||||
snapshot_dir=answers_partial / "snapshot",
|
||||
build_id=build_id,
|
||||
git=git,
|
||||
started_at=now,
|
||||
finished_at=datetime.now().astimezone(),
|
||||
uv_lock=repo_root / "uv.lock",
|
||||
configs=configs,
|
||||
)
|
||||
write_answers(
|
||||
answers_dir=answers_partial,
|
||||
build_id=build_id,
|
||||
skipped_note=SKIPPED_NOTE,
|
||||
aliases=[
|
||||
AliasRow(alias=aliases[c.case_no], case_no=c.case_no, video_id=c.video_id)
|
||||
for c in delivery.cases
|
||||
],
|
||||
diagnoses=[
|
||||
DiagnosisRow(
|
||||
alias=aliases[c.case_no],
|
||||
case_no=c.case_no,
|
||||
clinical_code=c.diagnosis.clinical,
|
||||
ultrasound_code=c.diagnosis.ultrasound,
|
||||
mr_code=c.diagnosis.mr,
|
||||
name=None if code_maps is None else code_maps["Diagnosis"][c.diagnosis.clinical],
|
||||
)
|
||||
for c in delivery.cases
|
||||
],
|
||||
skipped=[
|
||||
SkippedRow(
|
||||
case=s.case,
|
||||
check=s.check,
|
||||
description=s.description,
|
||||
files=s.files,
|
||||
columns=s.columns,
|
||||
value_json=s.value_json,
|
||||
)
|
||||
for s in delivery.skipped
|
||||
],
|
||||
reviews=reviews,
|
||||
)
|
||||
|
||||
# 先改答案库，最后改资料库：资料库的 <构建号>/ 出现才算构建完成。
|
||||
_rename(answers_partial, answers_dir)
|
||||
_rename(library_partial, library_dir)
|
||||
|
||||
by_check = Counter(s.check for s in delivery.skipped)
|
||||
return BuildSummary(
|
||||
build_id=build_id,
|
||||
library_dir=library_dir,
|
||||
answers_dir=answers_dir,
|
||||
total_cases=delivery.total_cases,
|
||||
passed_cases=len(delivery.cases),
|
||||
skipped_cases=len(delivery.skipped) - delivery.unrecognized_rows,
|
||||
unrecognized_rows=delivery.unrecognized_rows,
|
||||
skipped_by_check=dict(sorted(by_check.items())),
|
||||
review_rows=len(reviews),
|
||||
skipped_note=SKIPPED_NOTE,
|
||||
)
|
||||
@@ -0,0 +1,96 @@
|
||||
"""`config/build.yaml` 与编码映射表的字段定义和校验（含 T5）。"""
|
||||
|
||||
from collections.abc import Hashable
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any, Self
|
||||
|
||||
import yaml
|
||||
from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, TypeAdapter, model_validator
|
||||
|
||||
from cadx.delivery import CODED_COLUMNS, CodeMaps
|
||||
|
||||
|
||||
def _reject_empty_path(value: object) -> object:
|
||||
# Path("") 会变成 "."（当前目录），空串在转换前就拦下。
|
||||
if value == "":
|
||||
raise ValueError("路径不能是空串")
|
||||
return value
|
||||
|
||||
|
||||
# YAML 里的路径是字符串；路径字段是唯一允许从字符串转换类型的地方。
|
||||
type _PathField = Annotated[Path, Field(strict=False), BeforeValidator(_reject_empty_path)]
|
||||
|
||||
|
||||
class _UniqueKeyLoader(yaml.SafeLoader):
|
||||
"""同一层出现重复键就报错：PyYAML 默认让后一个悄悄覆盖前一个。"""
|
||||
|
||||
# 签名（含 deep 的默认值）照抄 PyYAML 的基类方法，覆盖时必须一致。
|
||||
def construct_mapping(self, node: yaml.MappingNode, deep: bool = False) -> dict[Hashable, Any]:
|
||||
seen: set[Hashable] = set()
|
||||
for key_node, _ in node.value:
|
||||
key = self.construct_object(key_node, deep=deep) # pyright: ignore[reportUnknownMemberType]
|
||||
if key in seen:
|
||||
raise ValueError(f"YAML 第 {key_node.start_mark.line + 1} 行重复的键：{key!r}")
|
||||
seen.add(key)
|
||||
return super().construct_mapping(node, deep=deep)
|
||||
|
||||
|
||||
def _load_yaml(path: Path) -> object:
|
||||
return yaml.load(path.read_text(encoding="utf-8"), Loader=_UniqueKeyLoader) # noqa: S506 _UniqueKeyLoader 继承 SafeLoader
|
||||
|
||||
|
||||
SUBTYPES = frozenset({"IPMN", "SPN", "MCN", "SCN"})
|
||||
SEXES = frozenset({"男", "女"})
|
||||
|
||||
|
||||
class BuildConfig(BaseModel):
|
||||
"""装配两个库的配置。每个字段必填；相对路径相对仓库根目录解析。"""
|
||||
|
||||
model_config = ConfigDict(strict=True, extra="forbid", frozen=True)
|
||||
|
||||
delivery_dir: _PathField
|
||||
library_root: _PathField
|
||||
answers_root: _PathField
|
||||
alias_seed: int
|
||||
decode_codes: bool
|
||||
code_maps: Annotated[Path | None, Field(strict=False), BeforeValidator(_reject_empty_path)]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _code_maps_match_switch(self) -> Self:
|
||||
if self.decode_codes and self.code_maps is None:
|
||||
raise ValueError("decode_codes 为 true 时 code_maps 必须是映射表路径")
|
||||
if not self.decode_codes and self.code_maps is not None:
|
||||
raise ValueError("decode_codes 为 false 时 code_maps 必须是 null")
|
||||
return self
|
||||
|
||||
|
||||
def load_build_config(path: Path, repo_root: Path) -> BuildConfig:
|
||||
"""读并校验 `build.yaml`，把相对路径换成 `repo_root` 下的绝对路径。"""
|
||||
config = BuildConfig.model_validate(_load_yaml(path))
|
||||
return config.model_copy(
|
||||
update={
|
||||
"delivery_dir": repo_root / config.delivery_dir,
|
||||
"library_root": repo_root / config.library_root,
|
||||
"answers_root": repo_root / config.answers_root,
|
||||
"code_maps": None if config.code_maps is None else repo_root / config.code_maps,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_CODE_MAPS = TypeAdapter(dict[str, dict[int, str]], config=ConfigDict(strict=True))
|
||||
|
||||
|
||||
def load_code_maps(path: Path) -> CodeMaps:
|
||||
"""读映射表并做 T5：恰好覆盖全部编码列，性别映射到「男」「女」，诊断映射到四个亚型缩写。"""
|
||||
maps = _CODE_MAPS.validate_python(_load_yaml(path))
|
||||
if set(maps) != set(CODED_COLUMNS):
|
||||
extra = sorted(set(maps) - set(CODED_COLUMNS))
|
||||
missing = sorted(set(CODED_COLUMNS) - set(maps))
|
||||
raise ValueError(
|
||||
f"T5 映射表应恰好覆盖 {len(CODED_COLUMNS)} 项编码列：多了 {extra}，缺 {missing}"
|
||||
)
|
||||
for column, expected in (("Sex", SEXES), ("Diagnosis", SUBTYPES)):
|
||||
values = list(maps[column].values())
|
||||
if len(values) != len(expected) or frozenset(values) != expected:
|
||||
raise ValueError(f"T5 映射表的 {column} 应恰好映射到 {sorted(expected)}")
|
||||
return maps
|
||||
@@ -0,0 +1,31 @@
|
||||
"""delivery：读 40 例交付包、逐例检查。只读，不决定谁能看见什么。
|
||||
|
||||
整张表的问题（T1–T4）抛 `DeliveryError`，整次构建停下；单个病例的问题（C01–C13）
|
||||
跳过整例，记一条 `SkipRecord`。
|
||||
"""
|
||||
|
||||
from cadx.delivery._cases import (
|
||||
SKIPPED_NOTE,
|
||||
CaseRecord,
|
||||
CodeMaps,
|
||||
DeliveryResult,
|
||||
DiagnosisCodes,
|
||||
SkipRecord,
|
||||
read_delivery,
|
||||
)
|
||||
from cadx.delivery._shape import CODED_COLUMNS
|
||||
from cadx.delivery._tables import DeliveryError
|
||||
from cadx.delivery._text import review_sentences
|
||||
|
||||
__all__ = [
|
||||
"CODED_COLUMNS",
|
||||
"SKIPPED_NOTE",
|
||||
"CaseRecord",
|
||||
"CodeMaps",
|
||||
"DeliveryError",
|
||||
"DeliveryResult",
|
||||
"DiagnosisCodes",
|
||||
"SkipRecord",
|
||||
"read_delivery",
|
||||
"review_sentences",
|
||||
]
|
||||
@@ -0,0 +1,412 @@
|
||||
"""逐例检查 C01–C13，交出通过的病例和跳过记录。
|
||||
|
||||
一例遇到第一项不过的检查就跳过整例，后面的检查不再做（`AGENTS.md`「错误怎么处理」）。
|
||||
类型按 openpyxl 读出的 Python 类型判断，不做转换：要求整数的格读出 `45.0` 也不过，
|
||||
字符串不去首尾空格。
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TypeGuard, cast
|
||||
|
||||
from cadx.delivery import _text
|
||||
from cadx.delivery._shape import (
|
||||
CASE_REPORT_SHEET,
|
||||
CLINICAL,
|
||||
CLINICAL_ROUTE_COLUMNS,
|
||||
CODED_COLUMNS,
|
||||
DIAGNOSIS_HEADER,
|
||||
HISTORY_COLUMNS,
|
||||
LAB_PAIRS,
|
||||
MR,
|
||||
SUMMARY_TABLES,
|
||||
SYMPTOM_COLUMNS,
|
||||
ULTRASOUND,
|
||||
TableShape,
|
||||
case_dir_path,
|
||||
video_id,
|
||||
)
|
||||
from cadx.delivery._tables import Cell, Row, open_workbook, read_tables
|
||||
|
||||
type CodeMaps = Mapping[str, Mapping[int, str]]
|
||||
|
||||
CHECK_DESCRIPTIONS = {
|
||||
"C01": "编号能规范化",
|
||||
"C02": "同一张表里这个编号只出现一次",
|
||||
"C03": "三张表和病例目录里都有这一例",
|
||||
"C04": "病例目录恰好有关键图、视频、与视频名配对的单例报告表三个文件",
|
||||
"C05": "超声表的 CEUS视频ID 与病例号一致",
|
||||
"C06": "单例报告表与超声汇总表对应行一致",
|
||||
"C07": "临床表取值合规",
|
||||
"C08": "超声表取值合规",
|
||||
"C09": "MR 表取值合规",
|
||||
"C10": "跨表一致（诊断、年龄、性别）",
|
||||
"C11": "报告文字里没有结论段标题",
|
||||
"C12": "报告文字里没有亚型名",
|
||||
"C13": "报告文字里没有人口学信息",
|
||||
}
|
||||
|
||||
SKIPPED_NOTE = "每例只记第一项没过的检查：遇到它就跳过整例，后面的检查没有做过，不代表没有问题。"
|
||||
|
||||
UNCOMPARED_CLINICAL_SEX = "临床性别未比对"
|
||||
|
||||
_CLINICAL_ID = re.compile(r"P([0-9]{3})")
|
||||
_REPORT_ID = re.compile(r"[0-9]+")
|
||||
_REPORT_AGE = re.compile(r"[0-9]+岁")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DiagnosisCodes:
|
||||
"""三张表各自的诊断编码（C10 保证三者相等）。"""
|
||||
|
||||
clinical: int
|
||||
ultrasound: int
|
||||
mr: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CaseRecord:
|
||||
"""一个检查全部通过的病例读到的内容。"""
|
||||
|
||||
case_no: str # 三位补零
|
||||
video_id: str
|
||||
clinical: Mapping[str, int | float | None] # 临床表除 Case_ID、Diagnosis 外的 22 列
|
||||
ultrasound_report: str
|
||||
mr_report: str
|
||||
key_image: Path
|
||||
diagnosis: DiagnosisCodes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkipRecord:
|
||||
"""一条跳过记录。`case` 是三位病例号，C01 认不出编号时是「文件 第 N 行」。"""
|
||||
|
||||
case: str
|
||||
check: str
|
||||
description: str
|
||||
files: tuple[str, ...]
|
||||
columns: tuple[str, ...]
|
||||
value_json: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DeliveryResult:
|
||||
"""读一次交付包的结果。"""
|
||||
|
||||
cases: tuple[CaseRecord, ...] # 按病例号排序
|
||||
skipped: tuple[SkipRecord, ...]
|
||||
total_cases: int # 三张表和病例目录里认得出的病例号的并集大小
|
||||
unrecognized_rows: int # C01 认不出编号的行数，不计入病例数
|
||||
uncompared: tuple[str, ...] # 因为没有映射表而没比对的项
|
||||
|
||||
|
||||
def _skip(
|
||||
case: str, check: str, files: tuple[str, ...], columns: tuple[str, ...], value: object
|
||||
) -> SkipRecord:
|
||||
# 涉及值可能是 openpyxl 读出的任何类型（如日期），JSON 表示不了的按 repr 写，供医生核对。
|
||||
return SkipRecord(
|
||||
case=case,
|
||||
check=check,
|
||||
description=CHECK_DESCRIPTIONS[check],
|
||||
files=files,
|
||||
columns=columns,
|
||||
value_json=json.dumps(value, ensure_ascii=False, default=repr),
|
||||
)
|
||||
|
||||
|
||||
def _is_int(value: Cell) -> TypeGuard[int]:
|
||||
return type(value) is int # bool 是 int 的子类，要排除
|
||||
|
||||
|
||||
def _is_number(value: Cell) -> TypeGuard[int | float]:
|
||||
return type(value) in (int, float)
|
||||
|
||||
|
||||
def _in_range(value: Cell, low: int, high: int) -> bool:
|
||||
return _is_int(value) and low <= value <= high
|
||||
|
||||
|
||||
def _normalize_id(table: TableShape, raw: Cell) -> str | None:
|
||||
"""C01：认得出就返回三位病例号，认不出返回 None。"""
|
||||
number: int | None = None
|
||||
if table is CLINICAL:
|
||||
if isinstance(raw, str) and (m := _CLINICAL_ID.fullmatch(raw)):
|
||||
number = int(m.group(1))
|
||||
elif isinstance(raw, str) and _REPORT_ID.fullmatch(raw):
|
||||
number = int(raw)
|
||||
elif _is_int(raw):
|
||||
number = raw
|
||||
if number is None or not 1 <= number <= 999:
|
||||
return None
|
||||
return f"{number:03d}"
|
||||
|
||||
|
||||
def _index_rows(
|
||||
rows: dict[TableShape, list[Row]], case_dirs: list[str]
|
||||
) -> tuple[dict[TableShape, dict[str, list[Row]]], set[str], list[SkipRecord]]:
|
||||
index: dict[TableShape, dict[str, list[Row]]] = {}
|
||||
unrecognized: list[SkipRecord] = []
|
||||
for table in SUMMARY_TABLES:
|
||||
index[table] = {}
|
||||
for row in rows[table]:
|
||||
raw = row.cells[table.id_column]
|
||||
case_no = _normalize_id(table, raw)
|
||||
if case_no is None:
|
||||
unrecognized.append(
|
||||
_skip(
|
||||
f"{table.path} 第 {row.number} 行",
|
||||
"C01",
|
||||
(table.path,),
|
||||
(table.id_column,),
|
||||
{"原始写法": raw, "行号": row.number},
|
||||
)
|
||||
)
|
||||
continue
|
||||
index[table].setdefault(case_no, []).append(row)
|
||||
dirs: set[str] = set()
|
||||
for name in case_dirs:
|
||||
if not 1 <= int(name) <= 999:
|
||||
unrecognized.append(
|
||||
_skip(f"病例目录 {name}", "C01", (case_dir_path(name),), (), {"目录名": name})
|
||||
)
|
||||
continue
|
||||
dirs.add(name)
|
||||
return index, dirs, unrecognized
|
||||
|
||||
|
||||
# 单例报告表的名字随视频名配对：带 roi2 标记的视频配带 roi2 的报告，反之亦然（负责人 2026-10-16）。
|
||||
_VIDEO_REPORT_PAIRS = (("_roi2.mp4", "_roi2_report.xlsx"), (".mp4", "_report.xlsx"))
|
||||
|
||||
|
||||
def _check_case_dir(case_no: str, folder: Path) -> str | SkipRecord:
|
||||
"""C04：通过时返回这一例单例报告表的文件名。"""
|
||||
u = video_id(case_no)
|
||||
names = sorted(p.name for p in folder.iterdir())
|
||||
for video, report in _VIDEO_REPORT_PAIRS:
|
||||
if names == sorted((f"{u}_key.png", f"{u}{video}", f"{u}{report}")):
|
||||
return f"{u}{report}"
|
||||
return _skip(case_no, "C04", (case_dir_path(case_no),), (), {"文件": names})
|
||||
|
||||
|
||||
def _check_case_report(case_no: str, path: Path, summary: Row, rel: str) -> SkipRecord | None:
|
||||
"""C06：单例报告表与超声汇总表那一行逐格相等（类型也要相同）。"""
|
||||
sheets = open_workbook(path)
|
||||
if sheets.names != (CASE_REPORT_SHEET,):
|
||||
return _skip(case_no, "C06", (rel,), (), {"工作表": list(sheets.names)})
|
||||
if not sheets.rows or sheets.rows[0] != ULTRASOUND.header:
|
||||
return _skip(case_no, "C06", (rel,), (), {"表头": "与超声汇总表不一致"})
|
||||
if len(sheets.rows) != 2:
|
||||
return _skip(case_no, "C06", (rel,), (), {"数据行数": len(sheets.rows) - 1})
|
||||
diffs = {
|
||||
column: {"单例报告表": mine, "超声汇总表": summary.cells[column]}
|
||||
for column, mine in zip(ULTRASOUND.header, sheets.rows[1], strict=True)
|
||||
if not (type(mine) is type(summary.cells[column]) and mine == summary.cells[column])
|
||||
}
|
||||
if diffs:
|
||||
return _skip(case_no, "C06", (rel, ULTRASOUND.path), tuple(diffs), diffs)
|
||||
return None
|
||||
|
||||
|
||||
def _clinical_violations(cells: dict[str, Cell], code_maps: CodeMaps | None) -> dict[str, Cell]:
|
||||
"""C07：返回不合规的列和值。"""
|
||||
bad: dict[str, Cell] = {}
|
||||
if not _in_range(cells["Sex"], 1, 2):
|
||||
bad["Sex"] = cells["Sex"]
|
||||
if not (_is_int(cells["Age"]) and cells["Age"] > 0):
|
||||
bad["Age"] = cells["Age"]
|
||||
if not _in_range(cells["Diagnosis"], 1, 4):
|
||||
bad["Diagnosis"] = cells["Diagnosis"]
|
||||
for column in (*SYMPTOM_COLUMNS, *HISTORY_COLUMNS):
|
||||
if not _in_range(cells[column], 0, 1):
|
||||
bad[column] = cells[column]
|
||||
for raw_column, level_column in LAB_PAIRS:
|
||||
raw, level = cells[raw_column], cells[level_column]
|
||||
if raw is None and level is None:
|
||||
continue
|
||||
if not (_is_number(raw) and _in_range(level, 0, 1)):
|
||||
bad[raw_column], bad[level_column] = raw, level
|
||||
if code_maps is not None:
|
||||
for column in CODED_COLUMNS:
|
||||
value = cells[column]
|
||||
if value is not None and not (_is_int(value) and value in code_maps[column]):
|
||||
bad[column] = value
|
||||
return bad
|
||||
|
||||
|
||||
def _report_violations(
|
||||
cells: dict[str, Cell], text_column: str, code_maps: CodeMaps | None
|
||||
) -> dict[str, Cell]:
|
||||
"""C08 / C09 共同的部分：性别、年龄、诊断、描述文字。"""
|
||||
bad: dict[str, Cell] = {}
|
||||
if cells["性别"] not in ("男", "女"):
|
||||
bad["性别"] = cells["性别"]
|
||||
age = cells["年龄"]
|
||||
if not (isinstance(age, str) and _REPORT_AGE.fullmatch(age)):
|
||||
bad["年龄"] = age
|
||||
diagnosis = cells[DIAGNOSIS_HEADER]
|
||||
if not _in_range(diagnosis, 1, 4) or (
|
||||
code_maps is not None and diagnosis not in code_maps["Diagnosis"]
|
||||
):
|
||||
bad[DIAGNOSIS_HEADER] = diagnosis
|
||||
text = cells[text_column]
|
||||
if not (isinstance(text, str) and text):
|
||||
bad[text_column] = text
|
||||
return bad
|
||||
|
||||
|
||||
def _report_age(cells: dict[str, Cell]) -> int:
|
||||
return int(cast(str, cells["年龄"]).removesuffix("岁"))
|
||||
|
||||
|
||||
def _cross_table(
|
||||
clinical: dict[str, Cell],
|
||||
ultrasound: dict[str, Cell],
|
||||
mr: dict[str, Cell],
|
||||
code_maps: CodeMaps | None,
|
||||
) -> dict[str, dict[str, Cell]]:
|
||||
"""C10：返回对不上的字段和各表的值。"""
|
||||
bad: dict[str, dict[str, Cell]] = {}
|
||||
diagnoses = (clinical["Diagnosis"], ultrasound[DIAGNOSIS_HEADER], mr[DIAGNOSIS_HEADER])
|
||||
if len(set(diagnoses)) != 1:
|
||||
bad["诊断"] = dict(zip(("临床表", "超声表", "MR 表"), diagnoses, strict=True))
|
||||
ages = (clinical["Age"], _report_age(ultrasound), _report_age(mr))
|
||||
if len(set(ages)) != 1:
|
||||
bad["年龄"] = {"临床表": clinical["Age"], "超声表": ultrasound["年龄"], "MR 表": mr["年龄"]}
|
||||
sexes = {"超声表": ultrasound["性别"], "MR 表": mr["性别"]}
|
||||
if code_maps is not None:
|
||||
sexes["临床表（解码后）"] = code_maps["Sex"][cast(int, clinical["Sex"])]
|
||||
if len(set(sexes.values())) != 1:
|
||||
bad["性别"] = sexes
|
||||
return bad
|
||||
|
||||
|
||||
def _text_hits(
|
||||
case_no: str, check: str, reports: dict[TableShape, str], find: Callable[[str], list[object]]
|
||||
) -> SkipRecord | None:
|
||||
"""C11–C13 共同的部分：两份报告文字各查一遍，命中的都记下。"""
|
||||
text_columns = {ULTRASOUND: "超声描述", MR: "MR描述"}
|
||||
hits = {table: found for table, text in reports.items() if (found := find(text))}
|
||||
if not hits:
|
||||
return None
|
||||
return _skip(
|
||||
case_no,
|
||||
check,
|
||||
tuple(table.path for table in hits),
|
||||
tuple(text_columns[table] for table in hits),
|
||||
{text_columns[table]: found for table, found in hits.items()},
|
||||
)
|
||||
|
||||
|
||||
def _headings(text: str) -> list[object]:
|
||||
return [{"关键词": k, "句子": s} for k, s in _text.conclusion_headings(text)]
|
||||
|
||||
|
||||
def _subtypes(text: str) -> list[object]:
|
||||
return list(_text.subtype_terms(text))
|
||||
|
||||
|
||||
def _demographics(text: str) -> list[object]:
|
||||
return list(_text.demographic_terms(text))
|
||||
|
||||
|
||||
def _check_case(
|
||||
case_no: str,
|
||||
delivery_dir: Path,
|
||||
index: dict[TableShape, dict[str, list[Row]]],
|
||||
dirs: set[str],
|
||||
code_maps: CodeMaps | None,
|
||||
) -> CaseRecord | SkipRecord:
|
||||
"""按 C02–C13 的顺序检查一例，遇到第一项不过就返回它的跳过记录。"""
|
||||
for table in SUMMARY_TABLES:
|
||||
found = index[table].get(case_no, [])
|
||||
if len(found) > 1:
|
||||
numbers = [row.number for row in found]
|
||||
return _skip(case_no, "C02", (table.path,), (table.id_column,), {"行号": numbers})
|
||||
|
||||
missing = [table.path for table in SUMMARY_TABLES if case_no not in index[table]]
|
||||
if case_no not in dirs:
|
||||
missing.append(case_dir_path(case_no) + "/")
|
||||
if missing:
|
||||
return _skip(case_no, "C03", tuple(missing), (), {"缺在": missing})
|
||||
|
||||
clinical, ultrasound, mr = (index[table][case_no][0].cells for table in SUMMARY_TABLES)
|
||||
folder = delivery_dir / case_dir_path(case_no)
|
||||
u = video_id(case_no)
|
||||
|
||||
report_name = _check_case_dir(case_no, folder)
|
||||
if isinstance(report_name, SkipRecord):
|
||||
return report_name
|
||||
|
||||
if ultrasound["CEUS视频ID"] != u:
|
||||
value = {"CEUS视频ID": ultrasound["CEUS视频ID"], "病例号": case_no}
|
||||
return _skip(case_no, "C05", (ULTRASOUND.path,), ("CEUS视频ID",), value)
|
||||
|
||||
report_rel = f"{case_dir_path(case_no)}/{report_name}"
|
||||
summary_row = index[ULTRASOUND][case_no][0]
|
||||
report_path = delivery_dir / report_rel
|
||||
if (skip := _check_case_report(case_no, report_path, summary_row, report_rel)) is not None:
|
||||
return skip
|
||||
|
||||
if bad := _clinical_violations(clinical, code_maps):
|
||||
return _skip(case_no, "C07", (CLINICAL.path,), tuple(bad), bad)
|
||||
bad = _report_violations(ultrasound, "超声描述", code_maps)
|
||||
bolus = ultrasound["推注开始时间s"]
|
||||
if not (_is_int(bolus) and bolus >= 0):
|
||||
bad["推注开始时间s"] = bolus
|
||||
if bad:
|
||||
return _skip(case_no, "C08", (ULTRASOUND.path,), tuple(bad), bad)
|
||||
if bad := _report_violations(mr, "MR描述", code_maps):
|
||||
return _skip(case_no, "C09", (MR.path,), tuple(bad), bad)
|
||||
if mismatched := _cross_table(clinical, ultrasound, mr, code_maps):
|
||||
files = tuple(table.path for table in SUMMARY_TABLES)
|
||||
return _skip(case_no, "C10", files, tuple(mismatched), mismatched)
|
||||
|
||||
# C08、C09 已保证两份描述是非空字符串。
|
||||
reports = {ULTRASOUND: cast(str, ultrasound["超声描述"]), MR: cast(str, mr["MR描述"])}
|
||||
for check, find in (("C11", _headings), ("C12", _subtypes), ("C13", _demographics)):
|
||||
if (skip := _text_hits(case_no, check, reports, find)) is not None:
|
||||
return skip
|
||||
|
||||
# C07 已保证临床各列是整数、浮点数或空；C07–C10 已保证三个诊断是相等的整数。
|
||||
return CaseRecord(
|
||||
case_no=case_no,
|
||||
video_id=u,
|
||||
clinical={c: cast(int | float | None, clinical[c]) for c in CLINICAL_ROUTE_COLUMNS},
|
||||
ultrasound_report=reports[ULTRASOUND],
|
||||
mr_report=reports[MR],
|
||||
key_image=folder / f"{u}_key.png",
|
||||
diagnosis=DiagnosisCodes(
|
||||
clinical=cast(int, clinical["Diagnosis"]),
|
||||
ultrasound=cast(int, ultrasound[DIAGNOSIS_HEADER]),
|
||||
mr=cast(int, mr[DIAGNOSIS_HEADER]),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def read_delivery(delivery_dir: Path, code_maps: CodeMaps | None) -> DeliveryResult:
|
||||
"""读交付包并逐例检查。
|
||||
|
||||
`code_maps` 为 None 表示不解码：编码列只查取值范围，不查是否在映射表里，
|
||||
临床表的性别不和报告表比对。整张表的问题抛 `DeliveryError`。
|
||||
"""
|
||||
rows, case_dirs = read_tables(delivery_dir)
|
||||
index, dirs, skipped = _index_rows(rows, case_dirs)
|
||||
unrecognized = len(skipped)
|
||||
all_cases = sorted(dirs.union(*(index[table].keys() for table in SUMMARY_TABLES)))
|
||||
cases: list[CaseRecord] = []
|
||||
for case_no in all_cases:
|
||||
outcome = _check_case(case_no, delivery_dir, index, dirs, code_maps)
|
||||
if isinstance(outcome, SkipRecord):
|
||||
skipped.append(outcome)
|
||||
else:
|
||||
cases.append(outcome)
|
||||
return DeliveryResult(
|
||||
cases=tuple(cases),
|
||||
skipped=tuple(skipped),
|
||||
total_cases=len(all_cases),
|
||||
unrecognized_rows=unrecognized,
|
||||
uncompared=() if code_maps is not None else (UNCOMPARED_CLINICAL_SEX,),
|
||||
)
|
||||
@@ -0,0 +1,104 @@
|
||||
"""交付包的形状：目录、文件名、工作表名、表头、编码列名单。
|
||||
|
||||
全部按 40 例交付包（`data/交付包_20260908_40例/`）写死；交付方式变了就改这里，不做兼容。
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TableShape:
|
||||
"""一张汇总表的形状。"""
|
||||
|
||||
path: str # 相对交付包根目录
|
||||
sheet: str
|
||||
header: tuple[str, ...]
|
||||
id_column: str
|
||||
|
||||
|
||||
DIAGNOSIS_HEADER = "病理类型（1 IPMN，2 SPN，3 MCN，4 SCN）"
|
||||
|
||||
SYMPTOM_COLUMNS = (
|
||||
"Symptom_Pain",
|
||||
"Symptom_Jaundice",
|
||||
"Symptom_WeightLoss",
|
||||
"Symptom_Fever",
|
||||
"Symptom_Asymptomatic",
|
||||
)
|
||||
HISTORY_COLUMNS = (
|
||||
"Hx_Pancreatitis",
|
||||
"Hx_Alcohol",
|
||||
"Hx_Gallstones",
|
||||
"Hx_Malignancy",
|
||||
"Hx_Family_PC",
|
||||
)
|
||||
# 化验原值列与档位列成对出现。
|
||||
LAB_PAIRS = (
|
||||
("CA199", "CA199_Level"),
|
||||
("CEA", "CEA_Level"),
|
||||
("Amylase", "Amylase_Level"),
|
||||
("Lipase", "Lipase_Level"),
|
||||
("TBil", "TBil_Level"),
|
||||
)
|
||||
|
||||
CLINICAL = TableShape(
|
||||
path="Clinical_agent/临床工作簿_40例.xlsx",
|
||||
sheet="Sheet",
|
||||
header=(
|
||||
"Case_ID",
|
||||
"Sex",
|
||||
"Age",
|
||||
"Diagnosis",
|
||||
*SYMPTOM_COLUMNS,
|
||||
*HISTORY_COLUMNS,
|
||||
*(column for pair in LAB_PAIRS for column in pair),
|
||||
),
|
||||
id_column="Case_ID",
|
||||
)
|
||||
ULTRASOUND = TableShape(
|
||||
path="CEUS_agent/超声报告文本_40例.xlsx",
|
||||
sheet="超声报告",
|
||||
header=(
|
||||
"患者ID",
|
||||
"CEUS视频ID",
|
||||
"性别",
|
||||
"年龄",
|
||||
DIAGNOSIS_HEADER,
|
||||
"推注开始时间s",
|
||||
"超声描述",
|
||||
),
|
||||
id_column="患者ID",
|
||||
)
|
||||
MR = TableShape(
|
||||
path="MR_agent/MR报告文本_40例.xlsx",
|
||||
sheet="MR报告",
|
||||
header=("患者ID", "性别", "年龄", DIAGNOSIS_HEADER, "MR描述"),
|
||||
id_column="患者ID",
|
||||
)
|
||||
SUMMARY_TABLES = (CLINICAL, ULTRASOUND, MR)
|
||||
|
||||
# 病例目录里的单例报告表：工作表名固定，表头与超声汇总表相同。
|
||||
CASE_REPORT_SHEET = "Sheet1"
|
||||
|
||||
# 临床一路拿到的列：临床表去掉编号和诊断。
|
||||
CLINICAL_ROUTE_COLUMNS = tuple(c for c in CLINICAL.header if c not in ("Case_ID", "Diagnosis"))
|
||||
|
||||
# 用数字编码、需要映射表才知道含义的列。`Diagnosis` 同时管临床表的 `Diagnosis`
|
||||
# 和两张报告表的诊断列，三列编码相同。
|
||||
CODED_COLUMNS = (
|
||||
"Sex",
|
||||
"Diagnosis",
|
||||
*SYMPTOM_COLUMNS,
|
||||
*HISTORY_COLUMNS,
|
||||
*(level for _, level in LAB_PAIRS),
|
||||
)
|
||||
|
||||
|
||||
def case_dir_path(case_no: str) -> str:
|
||||
"""病例目录相对交付包根目录的路径。"""
|
||||
return f"CEUS_agent/{case_no}"
|
||||
|
||||
|
||||
def video_id(case_no: str) -> str:
|
||||
"""病例号对应的视频号（`U` 加三位病例号）。"""
|
||||
return f"U{case_no}"
|
||||
@@ -0,0 +1,84 @@
|
||||
"""读工作簿；整张表的检查 T1–T4，不过就整次构建停下。"""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import openpyxl
|
||||
|
||||
from cadx.delivery._shape import CLINICAL, MR, SUMMARY_TABLES, ULTRASOUND, TableShape
|
||||
|
||||
# 单元格按 openpyxl 读出的原样保留，不做类型转换；类型检查在各条 C 检查里做。
|
||||
type Cell = object
|
||||
|
||||
CASE_DIR_NAME = re.compile(r"[0-9]{3}")
|
||||
|
||||
|
||||
class DeliveryError(Exception):
|
||||
"""整张表读不了：交付方式变了，一例也读不出来。消息以检查编号开头。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Row:
|
||||
"""汇总表的一行：Excel 行号与「列名 → 单元格」。"""
|
||||
|
||||
number: int
|
||||
cells: dict[str, Cell]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Sheets:
|
||||
"""一个工作簿：全部工作表名，以及第一张表逐行的单元格（第 1 行是表头）。"""
|
||||
|
||||
names: tuple[str, ...]
|
||||
rows: tuple[tuple[Cell, ...], ...]
|
||||
|
||||
|
||||
def open_workbook(path: Path) -> Sheets:
|
||||
"""读出工作簿的工作表名和第一张工作表的全部行。"""
|
||||
workbook = openpyxl.load_workbook(path, read_only=True, data_only=True)
|
||||
names = tuple(workbook.sheetnames)
|
||||
rows = tuple(tuple(row) for row in workbook.worksheets[0].iter_rows(values_only=True))
|
||||
workbook.close()
|
||||
return Sheets(names=names, rows=rows)
|
||||
|
||||
|
||||
def _read_summary(delivery_dir: Path, table: TableShape) -> list[Row]:
|
||||
sheets = open_workbook(delivery_dir / table.path)
|
||||
if sheets.names != (table.sheet,):
|
||||
raise DeliveryError(
|
||||
f"T2 {table.path} 的工作表应恰好是 [{table.sheet}]，实际 {sheets.names}"
|
||||
)
|
||||
if not sheets.rows or sheets.rows[0] != table.header:
|
||||
raise DeliveryError(f"T3 {table.path} 第 1 行表头与约定不一致")
|
||||
return [
|
||||
Row(number=number, cells=dict(zip(table.header, values, strict=True)))
|
||||
for number, values in enumerate(sheets.rows[1:], start=2)
|
||||
]
|
||||
|
||||
|
||||
def _check_layout(delivery_dir: Path) -> list[str]:
|
||||
"""T4：三个子目录里没有多余的东西。返回病例目录名（按名排序）。"""
|
||||
for table in (CLINICAL, MR):
|
||||
folder = (delivery_dir / table.path).parent
|
||||
names = sorted(p.name for p in folder.iterdir())
|
||||
if names != [Path(table.path).name]:
|
||||
raise DeliveryError(f"T4 {folder.name}/ 下应只有 {Path(table.path).name}，实际 {names}")
|
||||
ceus = (delivery_dir / ULTRASOUND.path).parent
|
||||
case_dirs: list[str] = []
|
||||
for entry in sorted(ceus.iterdir()):
|
||||
if entry.name == Path(ULTRASOUND.path).name:
|
||||
continue
|
||||
if not (entry.is_dir() and CASE_DIR_NAME.fullmatch(entry.name)):
|
||||
raise DeliveryError(f"T4 {ceus.name}/ 下有不认识的条目 {entry.name}")
|
||||
case_dirs.append(entry.name)
|
||||
return case_dirs
|
||||
|
||||
|
||||
def read_tables(delivery_dir: Path) -> tuple[dict[TableShape, list[Row]], list[str]]:
|
||||
"""依次做 T1–T4，交出三张汇总表的数据行和病例目录名。"""
|
||||
for table in SUMMARY_TABLES:
|
||||
if not (delivery_dir / table.path).is_file():
|
||||
raise DeliveryError(f"T1 缺汇总表 {table.path}")
|
||||
rows = {table: _read_summary(delivery_dir, table) for table in SUMMARY_TABLES}
|
||||
return rows, _check_layout(delivery_dir)
|
||||
@@ -0,0 +1,102 @@
|
||||
"""报告文字的规则：结论段标题、亚型名、人口学信息、复核用的措辞。
|
||||
|
||||
都先做 NFKC 归一（全角字母、数字、标点转半角），这样全角写法绕不过去。
|
||||
"""
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
# 结论段标题的关键词。长的写在前面，正则按顺序尝试，命中「超声提示」时不会只报「提示」。
|
||||
CONCLUSION_KEYWORDS = (
|
||||
"超声提示",
|
||||
"超声诊断",
|
||||
"影像学诊断",
|
||||
"检查结论",
|
||||
"诊断意见",
|
||||
"印象",
|
||||
"结论",
|
||||
"诊断",
|
||||
"意见",
|
||||
"提示",
|
||||
)
|
||||
# 段首 = 全文开头或换行之后；沿用旧代码，允许标题前有空格或制表符缩进。
|
||||
# 换行认 str.splitlines 认的全部行边界（含 \x0b、\x0c、\x85、U+2028、U+2029），
|
||||
# 冒号除 NFKC 后的「:」外也认比例号「∶」（U+2236，NFKC 不把它转成冒号）。
|
||||
_LINE_BREAKS = "\r\n\x0b\x0c\x1c\x1d\x1e\x85\u2028\u2029"
|
||||
_CONCLUSION_HEADING = re.compile(
|
||||
rf"(?:^|(?<=[{_LINE_BREAKS}]))[ \t]*(" + "|".join(CONCLUSION_KEYWORDS) + r")\s*[:∶]"
|
||||
)
|
||||
|
||||
# 归一并转大写后按子串查。「浆液性」「黏液性」单独出现是描述囊液的正常用语，不查。
|
||||
# 词根覆盖了更长的写法（「SPT」覆盖「SPTP」，「导管内乳头」覆盖「导管内乳头状」），长的不再单列。
|
||||
SUBTYPE_TERMS = (
|
||||
"IPMN",
|
||||
"IPMT",
|
||||
"SPN",
|
||||
"SPT",
|
||||
"MCN",
|
||||
"SCN",
|
||||
"导管内乳头",
|
||||
"实性假乳头",
|
||||
"黏液性囊",
|
||||
"粘液性囊",
|
||||
"黏液囊性",
|
||||
"粘液囊性",
|
||||
"浆液性囊",
|
||||
"浆液性微囊",
|
||||
"囊腺瘤",
|
||||
)
|
||||
|
||||
# 阿拉伯数字或中文数字，可带「周」，简繁「岁」都认。
|
||||
_AGE = re.compile(r"[0-9一二三四五六七八九十百]+\s*周?[岁歲]")
|
||||
SEX_CHARACTERS = ("男", "女")
|
||||
|
||||
# 不带冒号出现时不算检查不过，只列进复核清单。
|
||||
REVIEW_WORDS = ("考虑", "可能", "倾向", "提示")
|
||||
|
||||
_SENTENCE = re.compile(r"[^。;]+")
|
||||
|
||||
|
||||
def normalize(text: str) -> str:
|
||||
"""NFKC 归一。"""
|
||||
return unicodedata.normalize("NFKC", text)
|
||||
|
||||
|
||||
def _sentence_at(normalized: str, position: int) -> str:
|
||||
for match in _SENTENCE.finditer(normalized):
|
||||
if match.start() <= position < match.end():
|
||||
return match.group()
|
||||
raise ValueError(f"位置 {position} 不在任何句子里（落在分隔符上）")
|
||||
|
||||
|
||||
def conclusion_headings(text: str) -> list[tuple[str, str]]:
|
||||
"""段首的结论段标题：每处命中给出（关键词，所在句子）。"""
|
||||
normalized = normalize(text)
|
||||
return [
|
||||
(m.group(1), _sentence_at(normalized, m.start(1)))
|
||||
for m in _CONCLUSION_HEADING.finditer(normalized)
|
||||
]
|
||||
|
||||
|
||||
def subtype_terms(text: str) -> list[str]:
|
||||
"""报告文字里出现的亚型名。"""
|
||||
upper = normalize(text).upper()
|
||||
return [term for term in SUBTYPE_TERMS if term in upper]
|
||||
|
||||
|
||||
def demographic_terms(text: str) -> list[str]:
|
||||
"""报告文字里出现的「数字岁」和性别字。"""
|
||||
normalized = normalize(text)
|
||||
ages = [m.group() for m in _AGE.finditer(normalized)]
|
||||
return ages + [ch for ch in SEX_CHARACTERS if ch in normalized]
|
||||
|
||||
|
||||
def review_sentences(text: str) -> list[tuple[str, str]]:
|
||||
"""含复核措辞的句子：每句一条（命中的词用「、」连接，句子）。按「。」「;」切句。"""
|
||||
found: list[tuple[str, str]] = []
|
||||
for match in _SENTENCE.finditer(normalize(text)):
|
||||
sentence = match.group()
|
||||
words = [w for w in REVIEW_WORDS if w in sentence]
|
||||
if words:
|
||||
found.append(("、".join(words), sentence))
|
||||
return found
|
||||
@@ -0,0 +1,30 @@
|
||||
"""library：给专家的资料库的目录布局、文件格式、写入和按路读取。
|
||||
|
||||
拒绝读取名字以 `.partial` 结尾的目录；读每个文件时核对清单里的 SHA-256。
|
||||
"""
|
||||
|
||||
from cadx.library._read import (
|
||||
ClinicalRoute,
|
||||
MrRoute,
|
||||
UltrasoundRoute,
|
||||
list_aliases,
|
||||
read_clinical,
|
||||
read_mr,
|
||||
read_ultrasound,
|
||||
)
|
||||
from cadx.library._scan import LeakHit, scan_for_leaks
|
||||
from cadx.library._write import LibraryCase, write_library
|
||||
|
||||
__all__ = [
|
||||
"ClinicalRoute",
|
||||
"LeakHit",
|
||||
"LibraryCase",
|
||||
"MrRoute",
|
||||
"UltrasoundRoute",
|
||||
"list_aliases",
|
||||
"read_clinical",
|
||||
"read_mr",
|
||||
"read_ultrasound",
|
||||
"scan_for_leaks",
|
||||
"write_library",
|
||||
]
|
||||
@@ -0,0 +1,83 @@
|
||||
"""按路读出资料库，读时核对清单里的 SHA-256。"""
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
from cadx.library._write import (
|
||||
CLINICAL_JSON,
|
||||
KEY_IMAGE,
|
||||
MANIFEST,
|
||||
MR_REPORT,
|
||||
ULTRASOUND_REPORT,
|
||||
case_file,
|
||||
sha256_file,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ClinicalRoute:
|
||||
"""临床一路：性别、年龄、症状、既往史、化验。"""
|
||||
|
||||
codes_decoded: bool
|
||||
fields: dict[str, str | int | float | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UltrasoundRoute:
|
||||
"""超声一路：报告「所见」原文和医生截的关键图（给路径）。"""
|
||||
|
||||
report: str
|
||||
key_image: Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MrRoute:
|
||||
"""MR 一路：报告「所见」原文。"""
|
||||
|
||||
report: str
|
||||
|
||||
|
||||
def _manifest(build_dir: Path) -> dict[str, Any]:
|
||||
if build_dir.name.endswith(".partial"):
|
||||
raise ValueError(f"{build_dir} 是没写完的构建（.partial），不能读")
|
||||
return cast(dict[str, Any], json.loads((build_dir / MANIFEST).read_text(encoding="utf-8")))
|
||||
|
||||
|
||||
def _verified(build_dir: Path, alias: str, relative: str) -> Path:
|
||||
manifest = _manifest(build_dir)
|
||||
if alias not in manifest["aliases"]:
|
||||
raise ValueError(f"构建 {build_dir.name} 里没有化名 {alias}")
|
||||
key = case_file(alias, relative)
|
||||
path = build_dir / key
|
||||
if sha256_file(path) != manifest["files"][key]:
|
||||
raise ValueError(f"{key} 的 SHA-256 与清单不一致，资料库被改动过")
|
||||
return path
|
||||
|
||||
|
||||
def _text(path: Path) -> str:
|
||||
# 按字节解码，不做换行转换，读出的就是写进去的原文。
|
||||
return path.read_bytes().decode("utf-8")
|
||||
|
||||
|
||||
def list_aliases(build_dir: Path) -> tuple[str, ...]:
|
||||
"""一次构建的全部化名（按化名排序）。"""
|
||||
return tuple(_manifest(build_dir)["aliases"])
|
||||
|
||||
|
||||
def read_clinical(build_dir: Path, alias: str) -> ClinicalRoute:
|
||||
"""读一例的临床一路。"""
|
||||
content = json.loads(_text(_verified(build_dir, alias, CLINICAL_JSON)))
|
||||
return ClinicalRoute(codes_decoded=content["codes_decoded"], fields=content["fields"])
|
||||
|
||||
|
||||
def read_ultrasound(build_dir: Path, alias: str) -> UltrasoundRoute:
|
||||
"""读一例的超声一路。"""
|
||||
report = _text(_verified(build_dir, alias, ULTRASOUND_REPORT))
|
||||
return UltrasoundRoute(report=report, key_image=_verified(build_dir, alias, KEY_IMAGE))
|
||||
|
||||
|
||||
def read_mr(build_dir: Path, alias: str) -> MrRoute:
|
||||
"""读一例的 MR 一路。"""
|
||||
return MrRoute(report=_text(_verified(build_dir, alias, MR_REPORT)))
|
||||
@@ -0,0 +1,68 @@
|
||||
"""泄露扫描：在一次构建的文字文件里找不该出现的字符串和 JSON 键名。"""
|
||||
|
||||
import json
|
||||
import unicodedata
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LeakHit:
|
||||
"""一处命中。`kind` 是「字符串」或「键名」。"""
|
||||
|
||||
file: str # 相对构建目录
|
||||
kind: str
|
||||
term: str
|
||||
|
||||
|
||||
def _fold(text: str) -> str:
|
||||
"""NFKC 再 casefold：全角、大小写写法都按同一个词比。"""
|
||||
return unicodedata.normalize("NFKC", text).casefold()
|
||||
|
||||
|
||||
def _json_keys_and_strings(node: object, keys: list[str], strings: list[str]) -> None:
|
||||
# json.loads 出来的对象：dict 的键都是字符串。
|
||||
if isinstance(node, dict):
|
||||
for key, value in cast(dict[str, object], node).items():
|
||||
keys.append(key)
|
||||
_json_keys_and_strings(value, keys, strings)
|
||||
elif isinstance(node, list):
|
||||
for item in cast(list[object], node):
|
||||
_json_keys_and_strings(item, keys, strings)
|
||||
elif isinstance(node, str):
|
||||
strings.append(node)
|
||||
|
||||
|
||||
def scan_for_leaks(
|
||||
build_dir: Path, forbidden_strings: Sequence[str], forbidden_keys: Sequence[str]
|
||||
) -> list[LeakHit]:
|
||||
"""扫构建目录下全部 `.txt`、`.json`（含清单），不扫 PNG。
|
||||
|
||||
字符串做 NFKC 并 casefold 后按子串比较；JSON 文件除原文外还扫解析后的键和字符串值，
|
||||
这样以 `\\uXXXX` 转义写出的中文也查得到。键名按整个键名、不分大小写比较。
|
||||
"""
|
||||
hits: list[LeakHit] = []
|
||||
banned_keys = {key.casefold() for key in forbidden_keys}
|
||||
paths = sorted(p for p in build_dir.rglob("*") if p.suffix in (".txt", ".json"))
|
||||
for path in paths:
|
||||
relative = path.relative_to(build_dir).as_posix()
|
||||
text = path.read_bytes().decode("utf-8")
|
||||
haystacks = [text]
|
||||
if path.suffix == ".json":
|
||||
keys: list[str] = []
|
||||
_json_keys_and_strings(json.loads(text), keys, haystacks)
|
||||
hits.extend(
|
||||
LeakHit(relative, "键名", key)
|
||||
for key in sorted(set(keys))
|
||||
if key.casefold() in banned_keys
|
||||
)
|
||||
haystacks.extend(keys)
|
||||
folded = [_fold(h) for h in haystacks]
|
||||
hits.extend(
|
||||
LeakHit(relative, "字符串", term)
|
||||
for term in forbidden_strings
|
||||
if any(_fold(term) in h for h in folded)
|
||||
)
|
||||
return hits
|
||||
@@ -0,0 +1,119 @@
|
||||
"""资料库的目录布局与写入。
|
||||
|
||||
```
|
||||
<构建目录>/
|
||||
manifest.json
|
||||
cases/<化名>/clinical/clinical.json
|
||||
cases/<化名>/ultrasound/report.txt
|
||||
cases/<化名>/ultrasound/key.png
|
||||
cases/<化名>/mr/report.txt
|
||||
```
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
MANIFEST = "manifest.json"
|
||||
CLINICAL_JSON = "clinical/clinical.json"
|
||||
ULTRASOUND_REPORT = "ultrasound/report.txt"
|
||||
KEY_IMAGE = "ultrasound/key.png"
|
||||
MR_REPORT = "mr/report.txt"
|
||||
|
||||
_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
# 能逐像素保真重编码的 (位深, PNG 颜色类型)：8 位的灰度、RGB、灰度加透明、RGBA，以及 1 位灰度。
|
||||
# 16 位图 Pillow 读 RGB 时会悄悄降成 8 位；调色板图按像素重编码会丢调色板。40 例都是 8 位 RGB。
|
||||
_FAITHFUL = frozenset({(8, 0), (8, 2), (8, 4), (8, 6), (1, 0)})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LibraryCase:
|
||||
"""要写进资料库的一例：三路各自的内容。"""
|
||||
|
||||
alias: str
|
||||
clinical_fields: Mapping[str, str | int | float | None]
|
||||
ultrasound_report: str
|
||||
key_image_source: Path
|
||||
mr_report: str
|
||||
|
||||
|
||||
def case_file(alias: str, relative: str) -> str:
|
||||
"""病例文件相对构建目录的路径。"""
|
||||
return f"cases/{alias}/{relative}"
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
"""文件内容的 SHA-256。"""
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _write_json(path: Path, content: object) -> None:
|
||||
# ensure_ascii=False：中文原样写出，泄露扫描按子串才查得到。
|
||||
path.write_text(json.dumps(content, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _write_clean_png(source: Path, target: Path) -> None:
|
||||
"""只保留像素和颜色模式重新编码：文本块、EXIF、ICC 等附加数据都不带过去。
|
||||
|
||||
不能逐像素保真的图（非 PNG、位深或颜色类型不在 `_FAITHFUL` 里、带 tRNS 透明色）报错停下，
|
||||
不悄悄降位或丢透明。
|
||||
"""
|
||||
header = source.read_bytes()[:26]
|
||||
if header[:8] != _PNG_SIGNATURE or header[12:16] != b"IHDR":
|
||||
raise ValueError(f"关键图不是 PNG：{source}")
|
||||
bit_depth, color_type = header[24], header[25]
|
||||
if (bit_depth, color_type) not in _FAITHFUL:
|
||||
raise ValueError(
|
||||
f"关键图位深 {bit_depth}、颜色类型 {color_type} 不能逐像素保真重编码：{source}"
|
||||
)
|
||||
with Image.open(source) as image:
|
||||
if "transparency" in image.info:
|
||||
raise ValueError(f"关键图带 tRNS 透明色，重编码会丢透明：{source}")
|
||||
clean = Image.frombytes(image.mode, image.size, image.tobytes())
|
||||
clean.save(target, format="PNG")
|
||||
|
||||
|
||||
def write_library(
|
||||
build_dir: Path,
|
||||
build_id: str,
|
||||
git_commit: str,
|
||||
git_clean: bool,
|
||||
config_sha256: Mapping[str, str],
|
||||
codes_decoded: bool,
|
||||
uncompared: Sequence[str],
|
||||
ceus_source: str | None,
|
||||
cases: Sequence[LibraryCase],
|
||||
) -> None:
|
||||
"""把一次构建写进已经存在的空目录 `build_dir`，最后写清单。"""
|
||||
files: dict[str, str] = {}
|
||||
for case in cases:
|
||||
case_dir = build_dir / "cases" / case.alias
|
||||
for relative in (CLINICAL_JSON, ULTRASOUND_REPORT, MR_REPORT):
|
||||
(case_dir / relative).parent.mkdir(parents=True, exist_ok=True)
|
||||
_write_json(
|
||||
case_dir / CLINICAL_JSON,
|
||||
{"codes_decoded": codes_decoded, "fields": dict(case.clinical_fields)},
|
||||
)
|
||||
(case_dir / ULTRASOUND_REPORT).write_text(case.ultrasound_report, encoding="utf-8")
|
||||
_write_clean_png(case.key_image_source, case_dir / KEY_IMAGE)
|
||||
(case_dir / MR_REPORT).write_text(case.mr_report, encoding="utf-8")
|
||||
for relative in (CLINICAL_JSON, ULTRASOUND_REPORT, KEY_IMAGE, MR_REPORT):
|
||||
files[case_file(case.alias, relative)] = sha256_file(case_dir / relative)
|
||||
_write_json(
|
||||
build_dir / MANIFEST,
|
||||
{
|
||||
"build_id": build_id,
|
||||
"git_commit": git_commit,
|
||||
"git_clean": git_clean,
|
||||
"config_sha256": dict(config_sha256),
|
||||
"codes_decoded": codes_decoded,
|
||||
"uncompared": list(uncompared),
|
||||
"ceus_source": ceus_source,
|
||||
"aliases": sorted(case.alias for case in cases),
|
||||
"files": dict(sorted(files.items())),
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,5 @@
|
||||
"""runs：构建号怎么取、运行快照记什么。会诊运行目录等会诊那一步再加。"""
|
||||
|
||||
from cadx.runs._snapshot import GitState, make_build_id, read_git_state, write_snapshot
|
||||
|
||||
__all__ = ["GitState", "make_build_id", "read_git_state", "write_snapshot"]
|
||||
@@ -0,0 +1,81 @@
|
||||
"""构建号与运行快照：一次构建用的是哪份代码、哪份配置、哪个环境。"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GitState:
|
||||
"""仓库当前的提交和工作区状态。"""
|
||||
|
||||
commit: str
|
||||
short_commit: str
|
||||
clean: bool # `git status --porcelain` 为空（含未跟踪文件）
|
||||
diff_sha256: str # `git diff HEAD` 输出的 SHA-256
|
||||
|
||||
|
||||
def _git(repo_root: Path, *args: str) -> bytes:
|
||||
git = shutil.which("git")
|
||||
if git is None:
|
||||
raise RuntimeError("找不到 git，无法记录构建用的代码版本")
|
||||
return subprocess.run( # noqa: S603 参数是写死的 git 子命令，不含外部输入
|
||||
[git, *args], cwd=repo_root, check=True, capture_output=True
|
||||
).stdout
|
||||
|
||||
|
||||
def read_git_state(repo_root: Path) -> GitState:
|
||||
"""读出 `repo_root` 所在仓库的提交号、是否干净、未提交改动的哈希。"""
|
||||
return GitState(
|
||||
commit=_git(repo_root, "rev-parse", "HEAD").decode().strip(),
|
||||
short_commit=_git(repo_root, "rev-parse", "--short", "HEAD").decode().strip(),
|
||||
clean=_git(repo_root, "status", "--porcelain").strip() == b"",
|
||||
diff_sha256=hashlib.sha256(_git(repo_root, "diff", "HEAD")).hexdigest(),
|
||||
)
|
||||
|
||||
|
||||
def make_build_id(now: datetime, git: GitState) -> str:
|
||||
"""`YYYYMMDD-HHMMSS-<git 短哈希>`，工作区不干净时末尾加 `-dirty`。`now` 用本地时间。"""
|
||||
build_id = f"{now:%Y%m%d-%H%M%S}-{git.short_commit}"
|
||||
return build_id if git.clean else f"{build_id}-dirty"
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def write_snapshot(
|
||||
snapshot_dir: Path,
|
||||
build_id: str,
|
||||
git: GitState,
|
||||
started_at: datetime,
|
||||
finished_at: datetime,
|
||||
uv_lock: Path,
|
||||
configs: Mapping[str, Path],
|
||||
) -> None:
|
||||
"""建 `snapshot_dir`，写 snapshot.json，并把 `configs`（快照里的文件名 → 原文件）原样复制进去。"""
|
||||
snapshot_dir.mkdir(exist_ok=False)
|
||||
copied: dict[str, str] = {}
|
||||
for name, source in configs.items():
|
||||
shutil.copyfile(source, snapshot_dir / name)
|
||||
copied[name] = _sha256(snapshot_dir / name)
|
||||
snapshot = {
|
||||
"build_id": build_id,
|
||||
"git_commit": git.commit,
|
||||
"git_clean": git.clean,
|
||||
"git_diff_head_sha256": git.diff_sha256,
|
||||
"python_version": sys.version,
|
||||
"uv_lock_sha256": _sha256(uv_lock),
|
||||
"started_at": started_at.isoformat(),
|
||||
"finished_at": finished_at.isoformat(),
|
||||
"config_sha256": copied,
|
||||
}
|
||||
(snapshot_dir / "snapshot.json").write_text(
|
||||
json.dumps(snapshot, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
Reference in New Issue
Block a user