feat: 任务书 05，runner 与 tasks 两个模块
runner 用 Docker 跑一条命令并交回结局和资源用量；tasks 读 task.json、准备运行目录、给出正式运行/校验器/轻量检查的运行定义，并从原始产物独立判分。任务目录从参考项目搬到 tasks/circle_packing/，新增 task.json；vocab 的 VERDICTS 删掉"判不了"，ProcessResult 的 cancelled 改为"容器从外面被杀了"。 全新上下文审阅后的两处修复：计量 shell 以容器内 root 运行，命令经 setpriv 降权，结果写在容器自己的文件层、结束后 docker cp 取出，并加 no-new-privileges，候选无法伪造计量结果把超时记成正常退出；harness 开跑前删掉旧的 solution.json、metrics.json，旧产物不会被当成本次的值。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
// runner：用 Docker 把一条命令跑完，交回它怎么结束的、用了多少资源（docs/结构.md 第二部分）。
|
||||
export { Runner, type ContainerStatus, type StartOutcome } from "./runner.ts";
|
||||
@@ -0,0 +1,269 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { closeSync, openSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { Measured, ProcessResult, RunDefinition } from "../vocab/index.ts";
|
||||
|
||||
// 用 Docker 把一条命令跑完：建镜像、起容器、等结束、查状态、删容器（docs/结构.md 第二部分）。
|
||||
// runner 不碰账本：登记要跑、确认启动、写结果都由 flow 做（方法.md 第九部分）。
|
||||
// 本身不存状态，全部依据都在 Docker 里，所以另一个进程里新建的 Runner 也能接着等一个早先启动的容器。
|
||||
|
||||
/** 查状态的五种回答。unreachable 是 Docker 不应答，flow 据此记"状态未知"并停下（方法.md 第七、第九部分）。 */
|
||||
export type ContainerStatus = "absent" | "created" | "running" | "exited" | "unreachable";
|
||||
|
||||
/** 启动的结果：启动了，或者没启动和 Docker 的报错原文。 */
|
||||
export type StartOutcome = { started: true } | { started: false; reason: string };
|
||||
|
||||
/** Docker 对容器名的要求（与 Docker 源码里的 RestrictedNamePattern 相同）。 */
|
||||
const CONTAINER_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]+$/;
|
||||
|
||||
/** 容器里的固定主机名：默认主机名是容器编号前 12 位，每次不同，镜像编号钉不住它。 */
|
||||
const HOSTNAME = "inforsi";
|
||||
|
||||
/**
|
||||
* 计量结果在容器自己文件层里的位置，不挂到宿主。目录由容器里的 root 建、只有 root 可写，
|
||||
* 降权后的命令既写不进去也改不了权限，伪造不了结果。
|
||||
*/
|
||||
const RECORD_DIR = "/inforsi-record";
|
||||
const USAGE_FILE = "usage";
|
||||
|
||||
/**
|
||||
* 容器里的第一个进程，以容器内 root 身份运行。参数依次是时限（纳秒）、时限（秒，交给 timeout）、
|
||||
* 降权到的用户编号和组编号、真正的命令。
|
||||
* - timeout 和这段 shell 是 root，被 timeout 包着的 setpriv 降到宿主当前用户再执行命令：
|
||||
* 命令写进运行目录的文件在宿主上归当前用户，宿主能读产物和清理；命令也没法向 root 进程发信号。
|
||||
* setpriv 不改环境变量，命令的环境仍只有镜像的和 docker run 给的。
|
||||
* - 用 KILL 结束超时的命令：它不能被捕获，候选不能拖延；timeout 此时的退出码是 128+9=137。
|
||||
* - 超时由这段 shell 自己确认：退出码 137 且用时不短于时限。命令自己以 124 或 137 结束时，
|
||||
* 用时不到时限，不会被当成超时。
|
||||
* - 资源用量读容器自己的 cgroup v2（cgroup 命名空间下 /sys/fs/cgroup 就是本容器），
|
||||
* 命令被 timeout 杀掉之后照样读得到；从外面杀掉整个容器时这段 shell 也一起没了，结果文件不会出现。
|
||||
* - 先写临时文件再改名：结果文件要么完整，要么不存在。
|
||||
*/
|
||||
const MEASURE_SCRIPT = `limit_ns=$1; limit=$2; uid=$3; gid=$4; shift 4
|
||||
mkdir -m 700 ${RECORD_DIR}
|
||||
start=$(date +%s%N)
|
||||
timeout -s KILL "$limit" setpriv --reuid="$uid" --regid="$gid" --clear-groups -- "$@"
|
||||
code=$?
|
||||
end=$(date +%s%N)
|
||||
wall=$((end - start))
|
||||
ending=exited
|
||||
if [ "$code" -eq 137 ] && [ "$wall" -ge "$limit_ns" ]; then ending=timeout; fi
|
||||
cpu=$(sed -n 's/^usage_usec //p' /sys/fs/cgroup/cpu.stat)
|
||||
mem=$(cat /sys/fs/cgroup/memory.peak)
|
||||
echo "$ending $code $wall $cpu $mem" > ${RECORD_DIR}/${USAGE_FILE}.tmp
|
||||
mv ${RECORD_DIR}/${USAGE_FILE}.tmp ${RECORD_DIR}/${USAGE_FILE}
|
||||
`;
|
||||
|
||||
/** 结果文件的格式：结局、退出码、墙钟纳秒、CPU 微秒、峰值内存字节，空格分隔，一行。 */
|
||||
const USAGE_LINE = /^(exited|timeout) (\d+) (\d+) (\d+) (\d+)\n$/;
|
||||
|
||||
/** docker cp 找不到容器里的文件时的报错。 */
|
||||
const NO_SUCH_FILE = "Could not find the file";
|
||||
|
||||
/** 认得出的"Docker 不应答"报错：本地套接字连不上、TCP 地址连不上两种写法。 */
|
||||
const UNREACHABLE = /failed to connect to the docker API|Cannot connect to the Docker daemon/;
|
||||
|
||||
interface Done {
|
||||
code: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
export class Runner {
|
||||
readonly #hostArgs: string[];
|
||||
|
||||
/**
|
||||
* dockerHost：Docker 守护进程的地址，例如 unix:///var/run/docker.sock；
|
||||
* null 表示不加 -H，由 docker 命令自己按 DOCKER_HOST 环境变量和当前 context 决定。
|
||||
*/
|
||||
constructor(dockerHost: string | null) {
|
||||
this.#hostArgs = dockerHost === null ? [] : ["-H", dockerHost];
|
||||
}
|
||||
|
||||
/** 用目录里的 Dockerfile 建镜像，打上 tag，交回镜像编号（`docker image inspect -f '{{.Id}}'` 的值）。 */
|
||||
async buildImage(contextDir: string, tag: string): Promise<string> {
|
||||
const built = await this.#docker(["build", "-t", tag, contextDir]);
|
||||
if (built.code !== 0) throw new Error(`docker build ${contextDir} 失败：\n${built.stderr}`);
|
||||
const inspected = await this.#docker(["image", "inspect", "--format", "{{.Id}}", tag]);
|
||||
if (inspected.code !== 0) throw new Error(`docker image inspect ${tag} 失败：${inspected.stderr}`);
|
||||
const digest = inspected.stdout.trim();
|
||||
if (!/^sha256:[0-9a-f]{64}$/.test(digest)) {
|
||||
throw new Error(`docker image inspect ${tag} 交回的不是镜像编号：${inspected.stdout}`);
|
||||
}
|
||||
return digest;
|
||||
}
|
||||
|
||||
/** 本机有没有这个镜像编号。Docker 不应答或报别的错，直接报错。 */
|
||||
async hasImage(digest: string): Promise<boolean> {
|
||||
const inspected = await this.#docker(["image", "inspect", "--format", "{{.Id}}", digest]);
|
||||
if (inspected.code === 0) return true;
|
||||
if (inspected.stderr.includes("No such image")) return false;
|
||||
throw new Error(`docker image inspect ${digest} 失败：${inspected.stderr}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* 起一个容器跑运行定义里的命令，runDir 挂到 definition.workdir。
|
||||
* docker run 失败时按名字再查一次（方法.md 第七部分"不确定有没有启动时，先核对"）：
|
||||
* 容器不存在或建了没启动，算没启动（建了的顺手删掉）；在跑或已结束，算启动了。
|
||||
*/
|
||||
async start(name: string, definition: RunDefinition, runDir: string): Promise<StartOutcome> {
|
||||
checkName(name);
|
||||
const getuid = process.getuid;
|
||||
const getgid = process.getgid;
|
||||
if (getuid === undefined || getgid === undefined) throw new Error("runner 只能在 POSIX 系统上用：读不到当前用户编号");
|
||||
const wallNs = Math.ceil(definition.wallSeconds * 1e9);
|
||||
const args = [
|
||||
"run",
|
||||
"--detach",
|
||||
"--name",
|
||||
name,
|
||||
// 本机 Docker 的默认运行时不一定是 runc（这台机器是 nvidia），跟着默认走，换台机器环境就悄悄变了。
|
||||
"--runtime",
|
||||
definition.image.runtime,
|
||||
// 镜像只按编号用本机已有的，不去仓库拉：拉到的可能是另一份内容。
|
||||
"--pull",
|
||||
"never",
|
||||
// 第一个进程是 root，好让计量结果不归命令管；命令本身由计量 shell 降到宿主当前用户再跑。
|
||||
"--user",
|
||||
"0:0",
|
||||
// 镜像里有 su、mount 这类 setuid 程序；禁止靠它们重新拿到 root，降权后的命令就碰不到计量结果。
|
||||
// setpriv 降权是 root 主动换身份，不受这条限制。
|
||||
"--security-opt",
|
||||
"no-new-privileges",
|
||||
"--hostname",
|
||||
HOSTNAME,
|
||||
// 方法要求运行时不联网。
|
||||
"--network",
|
||||
"none",
|
||||
// --cpus 限总量，不固定到某个核。PYTHON_CPU_COUNT 让 os.cpu_count() 报这个核数（Python 3.13 起），
|
||||
// 否则它报宿主核数，候选会开一堆进程抢一个核。写在 env 之后：同名时 docker 取后一个。
|
||||
`--cpus=${definition.cores}`,
|
||||
// 降权后的用户在镜像里没有家目录；HOME 不写明的话会是 root 的 /root。写在 env 之前，运行定义可以改。
|
||||
"--env",
|
||||
"HOME=/",
|
||||
...Object.entries(definition.env).flatMap(([key, value]) => ["--env", `${key}=${value}`]),
|
||||
"--env",
|
||||
`PYTHON_CPU_COUNT=${definition.cores}`,
|
||||
// 用 --mount 而不是 -v：-v 遇到相对路径会悄悄建一个同名数据卷，--mount 直接报错。
|
||||
"--mount",
|
||||
`type=bind,source=${runDir},target=${definition.workdir}`,
|
||||
"--workdir",
|
||||
definition.workdir,
|
||||
// 第一个进程固定是计量 shell，不受镜像 ENTRYPOINT 影响。
|
||||
"--entrypoint",
|
||||
"/bin/sh",
|
||||
// 不加 --rm：中断恢复时要按名字找到这个容器。
|
||||
definition.image.digest,
|
||||
"-c",
|
||||
MEASURE_SCRIPT,
|
||||
"inforsi-measure",
|
||||
String(wallNs),
|
||||
String(definition.wallSeconds),
|
||||
String(getuid.call(process)),
|
||||
String(getgid.call(process)),
|
||||
...definition.command,
|
||||
];
|
||||
const run = await this.#docker(args);
|
||||
if (run.code === 0) return { started: true };
|
||||
const status = await this.status(name);
|
||||
if (status === "running" || status === "exited") return { started: true };
|
||||
if (status === "unreachable") {
|
||||
throw new Error(`docker run ${name} 失败，之后查状态时 Docker 不应答，不知道有没有启动。docker run 的报错：${run.stderr}`);
|
||||
}
|
||||
if (status === "created") await this.remove(name);
|
||||
return { started: false, reason: run.stderr.trim() };
|
||||
}
|
||||
|
||||
/**
|
||||
* 等容器结束，交回进程层面的结果。标准输出、标准错误存成 logDir 下的 stdout.log、stderr.log，
|
||||
* 计量结果文件从容器里拷出来存成 logDir 下的 usage（容器结束后 docker cp 照样能拷）。
|
||||
* 只靠名字，能接上别的进程启动的容器；容器早已结束时 docker wait 立刻返回。
|
||||
*/
|
||||
async wait(name: string, logDir: string): Promise<ProcessResult> {
|
||||
checkName(name);
|
||||
const waited = await this.#docker(["wait", name]);
|
||||
if (waited.code !== 0) throw new Error(`docker wait ${name} 失败：${waited.stderr}`);
|
||||
await this.#saveLogs(name, logDir);
|
||||
const usagePath = join(logDir, USAGE_FILE);
|
||||
const copied = await this.#docker(["cp", `${name}:${RECORD_DIR}/${USAGE_FILE}`, usagePath]);
|
||||
if (copied.code !== 0) {
|
||||
if (!copied.stderr.includes(NO_SUCH_FILE)) throw new Error(`docker cp ${name} 的计量结果失败：${copied.stderr}`);
|
||||
const reason = `容器从外面被结束（容器退出码 ${waited.stdout.trim()}），计量 shell 没有写出结果文件`;
|
||||
const unmeasured: Measured = { status: "unmeasured", reason };
|
||||
return { ending: "cancelled", cpuSeconds: unmeasured, wallSeconds: unmeasured, peakMemoryBytes: unmeasured };
|
||||
}
|
||||
const text = readFileSync(usagePath, "utf8");
|
||||
const match = USAGE_LINE.exec(text);
|
||||
if (match === null) throw new Error(`${usagePath} 的格式认不出：${JSON.stringify(text)}`);
|
||||
const [, ending, code, wallNs, cpuUsec, memBytes] = match as unknown as [string, string, string, string, string, string];
|
||||
const usage = {
|
||||
cpuSeconds: measured(Number(cpuUsec) / 1e6),
|
||||
wallSeconds: measured(Number(wallNs) / 1e9),
|
||||
peakMemoryBytes: measured(Number(memBytes)),
|
||||
};
|
||||
if (ending === "timeout") return { ending: "timeout", ...usage };
|
||||
return { ending: "exited", exitCode: Number(code), ...usage };
|
||||
}
|
||||
|
||||
/** 按名字查容器的状态。Docker 报出这五种以外的状态（paused、restarting、dead 等）时报错。 */
|
||||
async status(name: string): Promise<ContainerStatus> {
|
||||
checkName(name);
|
||||
const inspected = await this.#docker(["container", "inspect", "--format", "{{.State.Status}}", name]);
|
||||
if (inspected.code !== 0) {
|
||||
if (inspected.stderr.includes("No such container")) return "absent";
|
||||
if (UNREACHABLE.test(inspected.stderr)) return "unreachable";
|
||||
throw new Error(`docker container inspect ${name} 失败：${inspected.stderr}`);
|
||||
}
|
||||
const state = inspected.stdout.trim();
|
||||
if (state === "created" || state === "running" || state === "exited") return state;
|
||||
throw new Error(`容器 ${name} 处于认不出的状态：${state}`);
|
||||
}
|
||||
|
||||
/** 删掉一个已结束或没启动的容器。还在跑的容器 docker 拒绝删，这里照样报错。 */
|
||||
async remove(name: string): Promise<void> {
|
||||
checkName(name);
|
||||
const removed = await this.#docker(["rm", name]);
|
||||
if (removed.code !== 0) throw new Error(`docker rm ${name} 失败：${removed.stderr}`);
|
||||
}
|
||||
|
||||
/** 两条流直接写进文件，不经过内存：候选可能打印很多内容。恢复时可能再等一次，所以覆盖已有的文件。 */
|
||||
async #saveLogs(name: string, logDir: string): Promise<void> {
|
||||
const out = openSync(join(logDir, "stdout.log"), "w");
|
||||
const err = openSync(join(logDir, "stderr.log"), "w");
|
||||
try {
|
||||
const code = await new Promise<number | null>((resolve, reject) => {
|
||||
const child = spawn("docker", [...this.#hostArgs, "logs", name], { stdio: ["ignore", out, err] });
|
||||
child.on("error", reject);
|
||||
child.on("close", resolve);
|
||||
});
|
||||
if (code !== 0) throw new Error(`docker logs ${name} 失败，退出码 ${code}，详情见 ${logDir}/stderr.log`);
|
||||
} finally {
|
||||
closeSync(out);
|
||||
closeSync(err);
|
||||
}
|
||||
}
|
||||
|
||||
/** 调一次 docker 命令，收齐输出。docker 命令本身起不来或被信号杀掉时报错。 */
|
||||
#docker(args: string[]): Promise<Done> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn("docker", [...this.#hostArgs, ...args], { stdio: ["ignore", "pipe", "pipe"] });
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.setEncoding("utf8").on("data", (chunk: string) => (stdout += chunk));
|
||||
child.stderr.setEncoding("utf8").on("data", (chunk: string) => (stderr += chunk));
|
||||
child.on("error", reject);
|
||||
child.on("close", (code, signal) => {
|
||||
if (code === null) reject(new Error(`docker ${args[0]} 被信号 ${signal} 结束`));
|
||||
else resolve({ code, stdout, stderr });
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function checkName(name: string): void {
|
||||
if (!CONTAINER_NAME.test(name)) throw new Error(`容器名 ${JSON.stringify(name)} 不符合 Docker 的要求 ${CONTAINER_NAME}`);
|
||||
}
|
||||
|
||||
function measured(value: number): Measured {
|
||||
return { status: "measured", value };
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
import { copyFileSync, cpSync, existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { Type, type Static } from "typebox";
|
||||
import { Value } from "typebox/value";
|
||||
import type {
|
||||
HypothesisMetric,
|
||||
Json,
|
||||
JudgedResult,
|
||||
Measured,
|
||||
MetricDefinition,
|
||||
ProcessResult,
|
||||
RunDefinition,
|
||||
} from "../vocab/index.ts";
|
||||
import type { TaskFacts } from "./facts.ts";
|
||||
import { measured, readMetricsFile, unmeasured } from "./metrics-file.ts";
|
||||
import type { LightCheckOutcome, Task, TaskImplementation } from "./types.ts";
|
||||
|
||||
// 填圆：在单位正方形里放 26 个圆，最大化半径之和。任务说明见 tasks/circle_packing/TASK.md。
|
||||
// 凡是要执行任务代码的事（正式运行、校验器、轻量检查）都只交出运行定义，由调用方交给 runner 在任务镜像里跑；
|
||||
// 这里只读写文件、解读产物（docs/结构.md 第二部分"runner 只管怎么跑，tasks 只管怎么判"）。
|
||||
|
||||
const CIRCLES = 26;
|
||||
const ENTRY = "initial.py";
|
||||
const HARNESS = "harness.py";
|
||||
const VALIDATOR = "validate.py";
|
||||
const CHECK_SCRIPT = "check-candidate.py";
|
||||
const METRICS_SPEC = "metrics-spec.json";
|
||||
const METRICS = "metrics.json";
|
||||
const SOLUTION = "solution.json";
|
||||
const VERDICT = "verdict.json";
|
||||
const CHECK_RESULT = "check.json";
|
||||
/** 轻量检查目录里放候选代码的子目录；检查脚本放在它的上一层，免得扫到自己。 */
|
||||
const CHECK_CANDIDATE = "candidate";
|
||||
/** 运行目录挂进容器的位置。 */
|
||||
const WORKDIR = "/work";
|
||||
/** 校验器的时限。26 个圆两两比较加合法化，实测一秒以内；它超时说明宿主的工具出了问题。 */
|
||||
const VALIDATOR_SECONDS = 60;
|
||||
/** 合法化前先按容差 0 判原产物（方法.md 第八部分"合法化"第 1 条）。 */
|
||||
const TOLERANCE = "0";
|
||||
|
||||
// 常规辅助指标的编号（方法.md 第五部分"指标"）。前三个来自 runner 的资源用量，后四个来自校验器。
|
||||
const WALL = "wall-seconds";
|
||||
const CPU = "cpu-seconds";
|
||||
const MEMORY = "peak-memory-bytes";
|
||||
const CLEARANCE = "min-pairwise-clearance";
|
||||
const RAW_SCORE = "raw-score";
|
||||
const RAW_VIOLATION = "raw-violation";
|
||||
const DEDUCTION = "legalization-deduction";
|
||||
const FROM_VALIDATOR = [CLEARANCE, RAW_SCORE, RAW_VIOLATION, DEDUCTION];
|
||||
|
||||
const SIDE = "单位正方形的边长";
|
||||
const PER_RUN = "每次运行结束后由宿主读一次，直接取值";
|
||||
|
||||
function auxiliary(
|
||||
id: string,
|
||||
meaning: string,
|
||||
unit: string,
|
||||
direction: "maximize" | "minimize",
|
||||
howMeasured: string,
|
||||
): MetricDefinition {
|
||||
return { id, layer: "auxiliary", meaning, unit, direction, howMeasured, missing: "recordMissing" };
|
||||
}
|
||||
|
||||
function metricDefinitions(facts: TaskFacts): MetricDefinition[] {
|
||||
const validator = `宿主的校验器从 ${SOLUTION} 重算，${PER_RUN}`;
|
||||
return [
|
||||
{
|
||||
id: facts.finalMetric.id,
|
||||
layer: "final",
|
||||
meaning:
|
||||
"合法摆法的半径之和，由宿主的独立校验器从实际产物重算；原产物差一点不合法时，先按 TASK.md 的只缩不移规则修，再按修后的半径算",
|
||||
unit: SIDE,
|
||||
direction: facts.finalMetric.direction,
|
||||
howMeasured: validator,
|
||||
missing: "invalidateRun",
|
||||
},
|
||||
auxiliary(WALL, "正式运行的墙钟时间", "秒", "minimize", `容器里的计量程序记录，${PER_RUN}`),
|
||||
auxiliary(CPU, "正式运行用掉的 CPU 秒（容器 cgroup 的 usage_usec）", "秒", "minimize", `容器里的计量程序记录，${PER_RUN}`),
|
||||
auxiliary(MEMORY, "正式运行的峰值内存（容器 cgroup 的 memory.peak）", "字节", "minimize", `容器里的计量程序记录，${PER_RUN}`),
|
||||
auxiliary(CLEARANCE, "所有圆对中「圆心距减两半径之和」的最小值，大于等于 0 表示不重叠；修过的取修后的摆法", SIDE, "maximize", validator),
|
||||
auxiliary(RAW_SCORE, "合法化之前原产物的半径之和", SIDE, "maximize", validator),
|
||||
auxiliary(RAW_VIOLATION, "原产物的最大违反量：两圆重叠的量和圆出边的量里最大的一个，0 表示没有违反", SIDE, "minimize", validator),
|
||||
auxiliary(DEDUCTION, "只缩不移合法化扣掉的分：原分数减最终成绩，原产物合法时是 0", SIDE, "minimize", validator),
|
||||
];
|
||||
}
|
||||
|
||||
/** 正式运行、校验器、轻量检查共用的运行定义。数学库的线程数和核数一致，否则会开多个线程抢核。 */
|
||||
function definition(task: Task, imageDigest: string, command: string[], wallSeconds: number, env: Record<string, string>): RunDefinition {
|
||||
const cores = String(task.facts.run.cores);
|
||||
return {
|
||||
command,
|
||||
workdir: WORKDIR,
|
||||
image: { digest: imageDigest, runtime: task.facts.run.runtime },
|
||||
wallSeconds,
|
||||
cores: task.facts.run.cores,
|
||||
inheritEnv: false,
|
||||
env: { OMP_NUM_THREADS: cores, OPENBLAS_NUM_THREADS: cores, MKL_NUM_THREADS: cores, ...env },
|
||||
};
|
||||
}
|
||||
|
||||
// 解释器不加 -I：-I 会忽略全部 PYTHON* 环境变量，连同 runner 设的 PYTHON_CPU_COUNT，
|
||||
// 于是 os.cpu_count() 报回宿主核数（本机实测 16）。容器里的环境变量全部由运行定义给出，用不着 -I。
|
||||
// -B：不写 __pycache__，不往运行目录里添文件。
|
||||
|
||||
function prepareRun(task: Task, candidateDir: string, runDir: string, hypothesisMetrics: HypothesisMetric[]): void {
|
||||
cpSync(candidateDir, runDir, { recursive: true });
|
||||
// harness 和指标说明以宿主的为准，候选目录里有同名文件也被盖掉（这两个名字在 task.json 的不许改名单里）。
|
||||
copyFileSync(join(task.dir, HARNESS), join(runDir, HARNESS));
|
||||
writeFileSync(join(runDir, METRICS_SPEC), `${JSON.stringify(hypothesisMetrics, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function formalRun(task: Task, imageDigest: string, seed: string): RunDefinition {
|
||||
const command = ["python", "-B", `${WORKDIR}/${HARNESS}`, "--program", ENTRY, "--output", SOLUTION];
|
||||
return definition(task, imageDigest, command, task.facts.run.wallSeconds, { INFORSI_SEED: seed });
|
||||
}
|
||||
|
||||
function lightCheck(task: Task, candidateDir: string, checkDir: string, imageDigest: string, seconds: number): RunDefinition {
|
||||
cpSync(candidateDir, join(checkDir, CHECK_CANDIDATE), { recursive: true });
|
||||
copyFileSync(join(task.dir, CHECK_SCRIPT), join(checkDir, CHECK_SCRIPT));
|
||||
const command = ["python", "-B", CHECK_SCRIPT, CHECK_CANDIDATE, ENTRY, CHECK_RESULT];
|
||||
return definition(task, imageDigest, command, seconds, {});
|
||||
}
|
||||
|
||||
/** 检查脚本写的结果文件。valid 与 reasons 是否为空必须一致。 */
|
||||
const CheckResult = Type.Object({ valid: Type.Boolean(), reasons: Type.Array(Type.String()) });
|
||||
|
||||
/** 宿主的工具（检查脚本、校验器）没正常跑完时报错停下：这不是候选的失败（任务书 05 第三部分）。 */
|
||||
function requireExit(what: string, result: ProcessResult, codes: number[]): number {
|
||||
if (result.ending === "exited" && codes.includes(result.exitCode)) return result.exitCode;
|
||||
const how = result.ending === "notStarted" ? `没启动：${result.reason}` : result.ending === "exited" ? `退出码 ${result.exitCode}` : result.ending;
|
||||
throw new Error(`${what}没有正常跑完（${how}），这是宿主一侧的工具出了问题，不能当成候选的失败`);
|
||||
}
|
||||
|
||||
function readJson(path: string): unknown {
|
||||
try {
|
||||
return JSON.parse(readFileSync(path, "utf8"));
|
||||
} catch (error) {
|
||||
throw new Error(`读不出 ${path}：${(error as Error).message}`, { cause: error });
|
||||
}
|
||||
}
|
||||
|
||||
function readLightCheck(checkDir: string, result: ProcessResult): LightCheckOutcome {
|
||||
requireExit("轻量检查脚本", result, [0]);
|
||||
const path = join(checkDir, CHECK_RESULT);
|
||||
const check = readJson(path);
|
||||
if (!Value.Check(CheckResult, check) || check.valid !== (check.reasons.length === 0)) {
|
||||
throw new Error(`${path} 的内容认不出：${JSON.stringify(check)}`);
|
||||
}
|
||||
return check.valid ? { passed: true } : { passed: false, reasons: check.reasons };
|
||||
}
|
||||
|
||||
/** 正式运行没正常结束的原因；正常退出（退出码 0）时为 null。 */
|
||||
function processFailure(task: Task, formal: Exclude<ProcessResult, { ending: "notStarted" }>): string | null {
|
||||
if (formal.ending === "timeout") return `超过单次时限 ${task.facts.run.wallSeconds} 秒，被结束`;
|
||||
if (formal.ending === "cancelled") return "容器从外面被结束（关机、手动 docker kill 等），没有留下计量结果";
|
||||
if (formal.exitCode !== 0) return `正式运行以退出码 ${formal.exitCode} 结束`;
|
||||
return null;
|
||||
}
|
||||
|
||||
function started(formal: ProcessResult): Exclude<ProcessResult, { ending: "notStarted" }> {
|
||||
if (formal.ending === "notStarted") throw new Error(`没启动的运行不判分：${formal.reason}`);
|
||||
return formal;
|
||||
}
|
||||
|
||||
/**
|
||||
* 拼出 JudgedResult：资源用量来自正式运行，假设专用指标和两个保留键来自候选写的 metrics.json。
|
||||
* 有"缺了整次作废"的假设专用指标测不到时，原本算数的也改判不算数（方法.md 第五部分"指标"）。
|
||||
*/
|
||||
function assemble(
|
||||
runDir: string,
|
||||
formal: Exclude<ProcessResult, { ending: "notStarted" }>,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
verdict: { valid: boolean; reason: string },
|
||||
score: Measured,
|
||||
fromValidator: Record<string, Measured>,
|
||||
extra: Json,
|
||||
): JudgedResult {
|
||||
const file = readMetricsFile(join(runDir, METRICS), METRICS, hypothesisMetrics);
|
||||
const voided = hypothesisMetrics.flatMap((m) => {
|
||||
const value = file.hypothesis[m.id]!;
|
||||
return m.missing === "invalidateRun" && value.status === "unmeasured" ? [`${m.id}（${value.reason}）`] : [];
|
||||
});
|
||||
const result = verdict.valid && voided.length > 0
|
||||
? { valid: false, reason: `登记为"缺了整次作废"的假设专用指标测不到：${voided.join("；")}` }
|
||||
: verdict;
|
||||
return {
|
||||
verdict: result.valid ? "valid" : "invalid",
|
||||
verdictReason: result.reason,
|
||||
score,
|
||||
metrics: {
|
||||
[WALL]: formal.wallSeconds,
|
||||
[CPU]: formal.cpuSeconds,
|
||||
[MEMORY]: formal.peakMemoryBytes,
|
||||
...fromValidator,
|
||||
...file.hypothesis,
|
||||
},
|
||||
answerPath: file.answerPath,
|
||||
safeguardTriggers: file.safeguardTriggers,
|
||||
extra,
|
||||
};
|
||||
}
|
||||
|
||||
function prepareJudging(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
imageDigest: string,
|
||||
formal: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): ReturnType<TaskImplementation["prepareJudging"]> {
|
||||
const run = started(formal);
|
||||
const solution = join(runDir, SOLUTION);
|
||||
const failure = processFailure(task, run) ?? (existsSync(solution) ? null : `没有产物：运行目录里没有 ${SOLUTION}`);
|
||||
if (failure !== null) {
|
||||
const notValidated = Object.fromEntries(FROM_VALIDATOR.map((id) => [id, unmeasured(`没有跑校验器：${failure}`)]));
|
||||
const result = assemble(runDir, run, hypothesisMetrics, { valid: false, reason: failure }, unmeasured(failure), notValidated, {
|
||||
legalization: null,
|
||||
});
|
||||
return { status: "judged", result };
|
||||
}
|
||||
copyFileSync(solution, join(judgeDir, SOLUTION));
|
||||
copyFileSync(join(task.dir, VALIDATOR), join(judgeDir, VALIDATOR));
|
||||
const command = [
|
||||
"python",
|
||||
"-B",
|
||||
VALIDATOR,
|
||||
"--circles",
|
||||
String(CIRCLES),
|
||||
"--solution",
|
||||
SOLUTION,
|
||||
"--atol",
|
||||
TOLERANCE,
|
||||
"--legalize",
|
||||
"--verdict",
|
||||
VERDICT,
|
||||
];
|
||||
return { status: "validate", definition: definition(task, imageDigest, command, VALIDATOR_SECONDS, {}) };
|
||||
}
|
||||
|
||||
const NumberOrNull = Type.Union([Type.Number(), Type.Null()]);
|
||||
|
||||
/** validate.py --legalize 写的判定里，这里要读的部分；其余键（evidence、legal_radii 等）不读。 */
|
||||
const ValidatorVerdict = Type.Object({
|
||||
valid: Type.Boolean(),
|
||||
computed_sum: NumberOrNull,
|
||||
reasons: Type.Array(Type.String()),
|
||||
diagnostics: Type.Object({ min_pairwise_clearance: NumberOrNull }),
|
||||
legalization: Type.Object({
|
||||
raw_valid: Type.Boolean(),
|
||||
raw_score: NumberOrNull,
|
||||
raw_violation: NumberOrNull,
|
||||
applied: Type.Boolean(),
|
||||
repaired: Type.Boolean(),
|
||||
shrunk_circles: Type.Array(Type.Integer()),
|
||||
reasons: Type.Array(Type.String()),
|
||||
}),
|
||||
});
|
||||
type ValidatorVerdict = Static<typeof ValidatorVerdict>;
|
||||
|
||||
/** 校验器在形状不对、含非有限数或负半径时提前返回，几何量没算。 */
|
||||
const NOT_COMPUTED = "原产物形状不对、含非有限数或负半径，校验器没算这个量";
|
||||
|
||||
function orUnmeasured(value: number | null, reason: string): Measured {
|
||||
return value === null ? unmeasured(reason) : measured(value);
|
||||
}
|
||||
|
||||
function readVerdict(judgeDir: string, exitCode: number): ValidatorVerdict {
|
||||
const path = join(judgeDir, VERDICT);
|
||||
const verdict = readJson(path);
|
||||
if (!Value.Check(ValidatorVerdict, verdict)) throw new Error(`${path} 的内容认不出：${JSON.stringify(verdict)}`);
|
||||
// 退出码 0 = 合法、1 = 不合法；合法化之后的结论就是最终判定。对不上说明跑的不是这份校验器。
|
||||
const consistent =
|
||||
verdict.valid === (exitCode === 0) &&
|
||||
verdict.legalization.repaired === verdict.valid &&
|
||||
(!verdict.valid || verdict.computed_sum !== null);
|
||||
if (!consistent) throw new Error(`${path} 与退出码 ${exitCode} 或自身前后对不上：${JSON.stringify(verdict)}`);
|
||||
return verdict;
|
||||
}
|
||||
|
||||
function judge(
|
||||
_task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
formal: ProcessResult,
|
||||
validator: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgedResult {
|
||||
const run = started(formal);
|
||||
const verdict = readVerdict(judgeDir, requireExit("校验器", validator, [0, 1]));
|
||||
const legal = verdict.legalization;
|
||||
let deduction: Measured;
|
||||
let score: Measured;
|
||||
if (verdict.valid) {
|
||||
const final = verdict.computed_sum!;
|
||||
score = measured(final);
|
||||
// legal.repaired 为真时 raw_score 一定有值：原产物能算出分数，才可能合法或被修好。
|
||||
const raw = legal.raw_score!;
|
||||
// 只缩半径只会降分；修后反而更高，说明修法写错了（方法.md 第八部分"合法化"第 3 条）。
|
||||
if (final > raw) throw new Error(`合法化后分数 ${final} 高于原分数 ${raw}：只缩半径只会降分，修法出错了`);
|
||||
deduction = measured(raw - final);
|
||||
} else {
|
||||
const reason = `不算数：${verdict.reasons.concat(legal.reasons).join("；")}`;
|
||||
score = unmeasured(reason);
|
||||
deduction = unmeasured(`修不了，没有修后分数：${legal.reasons.join("；")}`);
|
||||
}
|
||||
const rawScore = orUnmeasured(legal.raw_score, NOT_COMPUTED);
|
||||
const rawViolation = orUnmeasured(legal.raw_violation, NOT_COMPUTED);
|
||||
const fromValidator = {
|
||||
[CLEARANCE]: orUnmeasured(verdict.diagnostics.min_pairwise_clearance, NOT_COMPUTED),
|
||||
[RAW_SCORE]: rawScore,
|
||||
[RAW_VIOLATION]: rawViolation,
|
||||
[DEDUCTION]: deduction,
|
||||
};
|
||||
const reason = verdict.valid
|
||||
? legal.raw_valid
|
||||
? "校验器按容差 0 判原产物合法"
|
||||
: `原产物不合法，按只缩不移规则修好后合法（缩了 ${legal.shrunk_circles.length} 个圆）`
|
||||
: `校验器判不合法：${verdict.reasons.concat(legal.reasons).join("；")}`;
|
||||
const extra = {
|
||||
legalization: {
|
||||
rawValid: legal.raw_valid,
|
||||
applied: legal.applied,
|
||||
repaired: legal.repaired,
|
||||
shrunkCircles: legal.shrunk_circles,
|
||||
reasons: legal.reasons,
|
||||
rawScore,
|
||||
rawViolation,
|
||||
deduction,
|
||||
},
|
||||
};
|
||||
return assemble(runDir, run, hypothesisMetrics, { valid: verdict.valid, reason }, score, fromValidator, extra);
|
||||
}
|
||||
|
||||
export const circlePacking: TaskImplementation = {
|
||||
metricDefinitions,
|
||||
prepareRun,
|
||||
formalRun,
|
||||
lightCheck,
|
||||
readLightCheck,
|
||||
prepareJudging,
|
||||
judge,
|
||||
};
|
||||
@@ -0,0 +1,90 @@
|
||||
import { Type, type Static } from "typebox";
|
||||
import { Settings } from "typebox/system";
|
||||
import { Value } from "typebox/value";
|
||||
import { Direction, Id } from "../vocab/index.ts";
|
||||
|
||||
// task.json 的字段定义与检查。task.json 是用户提供的东西，属于系统边界（CLAUDE.md"罕见情况怎么办"），
|
||||
// 所以每个字段必填、不认识的字段报错，一次列出全部错误，每条写明字段路径。可以没有的值写 null。
|
||||
|
||||
/** 任务种类。每一种在 task.ts 的对照表里对应一份实现；nanochat、AI4AI 以后按同样的形状加。 */
|
||||
export const TASK_KINDS = ["circle_packing"] as const;
|
||||
export type TaskKind = (typeof TASK_KINDS)[number];
|
||||
|
||||
const CLOSED = { additionalProperties: false } as const;
|
||||
|
||||
export const TaskFacts = Type.Object(
|
||||
{
|
||||
kind: Type.Enum(TASK_KINDS, { type: "string", description: "任务种类，决定用哪一份实现" }),
|
||||
finalMetric: Type.Object(
|
||||
{
|
||||
id: Id,
|
||||
direction: Direction,
|
||||
},
|
||||
{ ...CLOSED, description: "最终指标的编号和方向（方法.md 第十部分第 1 条）" },
|
||||
),
|
||||
epsilon: Type.Number({ exclusiveMinimum: 0, description: "精度 ε（方法.md 6.11）" }),
|
||||
knownBest: Type.Union([Type.Number(), Type.Null()], {
|
||||
description: "已知最好成绩；没有就写 null（方法.md 6.11）",
|
||||
}),
|
||||
run: Type.Object(
|
||||
{
|
||||
wallSeconds: Type.Number({ exclusiveMinimum: 0, description: "单次正式运行的墙钟时限（秒）" }),
|
||||
cores: Type.Integer({ minimum: 1, description: "单次正式运行的核数" }),
|
||||
runtime: Type.String({ minLength: 1, description: "Docker 运行时，必须写明，例如 runc" }),
|
||||
},
|
||||
{ ...CLOSED, description: "同批候选共用的比较条件（方法.md 第八部分）" },
|
||||
),
|
||||
protectedFiles: Type.Array(Type.String({ minLength: 1 }), {
|
||||
description: "不许改的文件，路径相对候选代码目录（方法.md 第十部分第 2 条）",
|
||||
}),
|
||||
},
|
||||
CLOSED,
|
||||
);
|
||||
export type TaskFacts = Static<typeof TaskFacts>;
|
||||
|
||||
type SchemaError = ReturnType<typeof Value.Errors>[number];
|
||||
|
||||
const TYPES: Readonly<Record<string, string>> = { number: "数字", integer: "整数", string: "字符串", object: "对象", array: "数组" };
|
||||
|
||||
/** 本文件的字段用得到的几种错误，写成中文；其余照 typebox 的原文。 */
|
||||
const REASONS: Readonly<Record<string, (p: Record<string, unknown>) => string>> = {
|
||||
type: (p) => `应为${TYPES[String(p.type)] ?? String(p.type)}`,
|
||||
enum: (p) => `只能是 ${(p.allowedValues as unknown[]).map((v) => JSON.stringify(v)).join("、")} 之一`,
|
||||
exclusiveMinimum: (p) => `必须大于 ${String(p.limit)}`,
|
||||
minimum: (p) => `不能小于 ${String(p.limit)}`,
|
||||
minLength: () => "不能为空",
|
||||
pattern: (p) => `格式不对，应符合 ${String(p.pattern)}`,
|
||||
};
|
||||
|
||||
function errorLines(errors: SchemaError[]): string[] {
|
||||
const lines: string[] = [];
|
||||
for (const e of errors) {
|
||||
const p = e.params as Record<string, unknown>;
|
||||
const at = e.instancePath || "/";
|
||||
// 唯一的几选一是 knownBest（数字或 null）；各分支自己的错误不再逐条列。
|
||||
if (e.schemaPath.includes("/anyOf/")) continue;
|
||||
if (e.keyword === "anyOf") lines.push(`${at}：应为数字，没有就写 null`);
|
||||
else if (e.keyword === "required") {
|
||||
for (const name of p.requiredProperties as string[]) lines.push(`${e.instancePath}/${name}：缺少这一项（每个字段都必填）`);
|
||||
} else if (e.keyword === "additionalProperties") {
|
||||
for (const name of p.additionalProperties as string[]) lines.push(`${e.instancePath}/${name}：不认识的字段`);
|
||||
} else lines.push(`${at}：${REASONS[e.keyword]?.(p) ?? e.message}`);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
/** 检查一份已经解析好的 task.json；合格就原样返回，不合格就报错，列出全部错误。 */
|
||||
export function checkTaskFacts(value: unknown): TaskFacts {
|
||||
if (Value.Check(TaskFacts, value)) return value;
|
||||
// typebox 默认最多收 8 条错误，不放开就会漏报；放开只在这一次调用里。
|
||||
const cap = Settings.Get().maxErrors;
|
||||
Settings.Set({ maxErrors: Infinity });
|
||||
let errors: SchemaError[];
|
||||
try {
|
||||
errors = [...Value.Errors(TaskFacts, value)];
|
||||
} finally {
|
||||
Settings.Set({ maxErrors: cap });
|
||||
}
|
||||
const lines = errorLines(errors);
|
||||
throw new Error(`task.json 不合格，共 ${lines.length} 处：\n${lines.join("\n")}`);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// tasks：任务目录里的事实和文件、怎么准备一次运行、怎么从原始产物独立判分（docs/结构.md 第二部分）。
|
||||
// 不起子进程：要执行任务代码的事都只交出运行定义，由调用方交给 runner 在任务镜像里跑。
|
||||
export { checkTaskFacts, TASK_KINDS, TaskFacts, type TaskKind } from "./facts.ts";
|
||||
export {
|
||||
formalRun,
|
||||
judge,
|
||||
lightCheck,
|
||||
loadTask,
|
||||
metricDefinitions,
|
||||
prepareJudging,
|
||||
prepareRun,
|
||||
protectedChanges,
|
||||
readLightCheck,
|
||||
} from "./task.ts";
|
||||
export type { JudgingStep, LightCheckOutcome, Task } from "./types.ts";
|
||||
@@ -0,0 +1,88 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import {
|
||||
ANSWER_PATH_KEY,
|
||||
SAFEGUARD_TRIGGERS_KEY,
|
||||
type AnswerPath,
|
||||
type HypothesisMetric,
|
||||
type Measured,
|
||||
type SafeguardTriggers,
|
||||
} from "../vocab/index.ts";
|
||||
|
||||
// 读候选写的指标文件：假设专用指标的取值，加两个保留键（方法.md 第五部分"指标""答案路径"）。
|
||||
// 这个文件是候选的产物，读不到、写错了都记成明确的状态并附原因，不补 0、不补空对象，也不报错停下。
|
||||
|
||||
export interface MetricsFileReading {
|
||||
/** 每条登记的假设专用指标一项，键是指标编号。 */
|
||||
hypothesis: Record<string, Measured>;
|
||||
answerPath: AnswerPath;
|
||||
safeguardTriggers: SafeguardTriggers;
|
||||
}
|
||||
|
||||
export function unmeasured(reason: string): Measured {
|
||||
return { status: "unmeasured", reason };
|
||||
}
|
||||
|
||||
export function measured(value: number): Measured {
|
||||
return { status: "measured", value };
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
/** 整个文件都读不到时，每一项都记同一个原因。 */
|
||||
function allUnreadable(reason: string, metrics: HypothesisMetric[]): MetricsFileReading {
|
||||
return {
|
||||
hypothesis: Object.fromEntries(metrics.map((m) => [m.id, unmeasured(reason)])),
|
||||
answerPath: { status: "unreadable", reason },
|
||||
safeguardTriggers: { status: "unreadable", reason },
|
||||
};
|
||||
}
|
||||
|
||||
function readMetric(values: Record<string, unknown>, id: string, name: string): Measured {
|
||||
const value = values[id];
|
||||
if (value === undefined) return unmeasured(`${name} 里没有指标 ${id}`);
|
||||
if (value === null) return unmeasured(`候选写的是 null：这一次没测到 ${id}`);
|
||||
// JSON.parse 会把 1e999 读成 Infinity，所以要查有限。
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) {
|
||||
return unmeasured(`${name} 里指标 ${id} 的值不是有限数：${JSON.stringify(value)}`);
|
||||
}
|
||||
return measured(value);
|
||||
}
|
||||
|
||||
function readAnswerPath(values: Record<string, unknown>, name: string): AnswerPath {
|
||||
const value = values[ANSWER_PATH_KEY];
|
||||
if (value === undefined) return { status: "unreadable", reason: `${name} 里没写 ${ANSWER_PATH_KEY}` };
|
||||
if (typeof value !== "string") {
|
||||
return { status: "unreadable", reason: `${name} 里 ${ANSWER_PATH_KEY} 不是字符串：${JSON.stringify(value)}` };
|
||||
}
|
||||
return { status: "read", value };
|
||||
}
|
||||
|
||||
function readTriggers(values: Record<string, unknown>, name: string): SafeguardTriggers {
|
||||
const value = values[SAFEGUARD_TRIGGERS_KEY];
|
||||
if (value === undefined) return { status: "unreadable", reason: `${name} 里没写 ${SAFEGUARD_TRIGGERS_KEY}` };
|
||||
const counts = isRecord(value) ? Object.entries(value) : null;
|
||||
if (counts === null || counts.some(([, n]) => !Number.isInteger(n) || (n as number) < 0)) {
|
||||
const reason = `${name} 里 ${SAFEGUARD_TRIGGERS_KEY} 应为"保障短名 → 非负整数"的对象，实际是 ${JSON.stringify(value)}`;
|
||||
return { status: "unreadable", reason };
|
||||
}
|
||||
return { status: "read", counts: Object.fromEntries(counts) as Record<string, number> };
|
||||
}
|
||||
|
||||
/** 读指标文件。metrics 是这次登记的假设专用指标，只按它们的编号取值，多写的键不读。 */
|
||||
export function readMetricsFile(path: string, name: string, metrics: HypothesisMetric[]): MetricsFileReading {
|
||||
if (!existsSync(path)) return allUnreadable(`${name} 不存在`, metrics);
|
||||
let values: unknown;
|
||||
try {
|
||||
values = JSON.parse(readFileSync(path, "utf8"));
|
||||
} catch (error) {
|
||||
return allUnreadable(`${name} 不是合法的 JSON：${(error as Error).message}`, metrics);
|
||||
}
|
||||
if (!isRecord(values)) return allUnreadable(`${name} 应为 JSON 对象（指标编号 → 值）`, metrics);
|
||||
return {
|
||||
hypothesis: Object.fromEntries(metrics.map((m) => [m.id, readMetric(values, m.id, name)])),
|
||||
answerPath: readAnswerPath(values, name),
|
||||
safeguardTriggers: readTriggers(values, name),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import type { HypothesisMetric, JudgedResult, MetricDefinition, ProcessResult, RunDefinition } from "../vocab/index.ts";
|
||||
import { circlePacking } from "./circle-packing.ts";
|
||||
import { checkTaskFacts, type TaskKind } from "./facts.ts";
|
||||
import type { JudgingStep, LightCheckOutcome, Task, TaskImplementation } from "./types.ts";
|
||||
|
||||
// tasks 对外的函数。任务目录的约定：task.json（任务事实）、TASK.md（给实验员的说明）、
|
||||
// Dockerfile 与依赖清单（建镜像）、baseline/（基线代码），以及各任务自己的 Python 工具。
|
||||
// 每个函数按 task.facts.kind 从对照表里取出那种任务的实现来做。
|
||||
|
||||
/** 任务种类 → 实现。加一种任务就在这里加一行。 */
|
||||
const IMPLEMENTATIONS: Readonly<Record<TaskKind, TaskImplementation>> = {
|
||||
circle_packing: circlePacking,
|
||||
};
|
||||
|
||||
function implementation(task: Task): TaskImplementation {
|
||||
return IMPLEMENTATIONS[task.facts.kind];
|
||||
}
|
||||
|
||||
/** 读一个任务目录：严格检查 task.json，同时交回 Dockerfile 所在目录、基线目录和 TASK.md 全文。 */
|
||||
export function loadTask(taskDir: string): Task {
|
||||
const dir = resolve(taskDir);
|
||||
const path = join(dir, "task.json");
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(readFileSync(path, "utf8"));
|
||||
} catch (error) {
|
||||
throw new Error(`${path} 读不出来或不是合法的 JSON：${(error as Error).message}`, { cause: error });
|
||||
}
|
||||
return {
|
||||
facts: checkTaskFacts(value),
|
||||
dir,
|
||||
baselineDir: join(dir, "baseline"),
|
||||
guide: readFileSync(join(dir, "TASK.md"), "utf8"),
|
||||
};
|
||||
}
|
||||
|
||||
/** 最终指标与常规辅助指标的定义。最终指标的编号和方向取自 task.json。 */
|
||||
export function metricDefinitions(task: Task): MetricDefinition[] {
|
||||
return implementation(task).metricDefinitions(task.facts);
|
||||
}
|
||||
|
||||
/**
|
||||
* 准备一次正式运行的运行目录：把候选代码、任务的 harness 和这次要测的假设专用指标说明
|
||||
* （metrics-spec.json）放进去。runDir 由调用方建好，应为空目录。
|
||||
* 消融格里传进来的指标应当已经全部改成"缺了只记缺失"（vocab 指标说明），这里不区分。
|
||||
*/
|
||||
export function prepareRun(task: Task, candidateDir: string, runDir: string, hypothesisMetrics: HypothesisMetric[]): void {
|
||||
implementation(task).prepareRun(task, candidateDir, runDir, hypothesisMetrics);
|
||||
}
|
||||
|
||||
/**
|
||||
* 正式运行的运行定义。种子由调用方给，同一份研究的全部运行用同一个种子：
|
||||
* 评测要确定，消融不重跑基线也靠这一点（方法.md 第十部分第 9 条）。
|
||||
*/
|
||||
export function formalRun(task: Task, imageDigest: string, seed: string): RunDefinition {
|
||||
return implementation(task).formalRun(task, imageDigest, seed);
|
||||
}
|
||||
|
||||
/**
|
||||
* 轻量检查：把候选代码和检查脚本放进 checkDir（调用方建好的空目录），交回运行定义。
|
||||
* 只看语法和入口函数，不执行候选代码。seconds 来自配置；次数上限不归这里管。
|
||||
*/
|
||||
export function lightCheck(task: Task, candidateDir: string, checkDir: string, imageDigest: string, seconds: number): RunDefinition {
|
||||
return implementation(task).lightCheck(task, candidateDir, checkDir, imageDigest, seconds);
|
||||
}
|
||||
|
||||
/** 解读轻量检查：通过，或者不通过及原因。检查脚本自己没正常跑完时报错停下。 */
|
||||
export function readLightCheck(task: Task, checkDir: string, result: ProcessResult): LightCheckOutcome {
|
||||
return implementation(task).readLightCheck(checkDir, result);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判分第一步。正式运行没正常结束、或者没有产物时直接判不算数；否则把产物和宿主保管的校验器
|
||||
* 复制进 judgeDir（调用方建好的空目录），交回校验器的运行定义。
|
||||
*/
|
||||
export function prepareJudging(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
imageDigest: string,
|
||||
formal: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgingStep {
|
||||
return implementation(task).prepareJudging(task, runDir, judgeDir, imageDigest, formal, hypothesisMetrics);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判分第二步：读校验器写的判定、运行目录里候选写的指标文件，加上正式运行的结果，交回判分结果。
|
||||
* 校验器自己没正常跑完、判定文件认不出、修后分数高于原分数时报错停下：这些是宿主一侧的问题。
|
||||
*/
|
||||
export function judge(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
formal: ProcessResult,
|
||||
validator: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgedResult {
|
||||
return implementation(task).judge(task, runDir, judgeDir, formal, validator, hypothesisMetrics);
|
||||
}
|
||||
|
||||
/** 改动过的文件里，属于 task.json"不许改"名单的那些。路径都相对候选代码目录。 */
|
||||
export function protectedChanges(task: Task, changedFiles: string[]): string[] {
|
||||
return changedFiles.filter((file) => task.facts.protectedFiles.includes(file));
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { HypothesisMetric, JudgedResult, MetricDefinition, ProcessResult, RunDefinition } from "../vocab/index.ts";
|
||||
import type { TaskFacts } from "./facts.ts";
|
||||
|
||||
/** 读进来的一个任务。 */
|
||||
export interface Task {
|
||||
/** task.json 的内容，已严格检查。 */
|
||||
facts: TaskFacts;
|
||||
/** 任务目录的绝对路径，里面有 Dockerfile 和依赖清单，runner 拿它建镜像。 */
|
||||
dir: string;
|
||||
/** 基线代码目录。 */
|
||||
baselineDir: string;
|
||||
/** TASK.md 全文，给实验员看的任务说明。 */
|
||||
guide: string;
|
||||
}
|
||||
|
||||
/** 轻量检查的结论：通过，或者不通过及原因。 */
|
||||
export type LightCheckOutcome = { passed: true } | { passed: false; reasons: string[] };
|
||||
|
||||
/**
|
||||
* 判分第一步的结果：要么不用跑校验器就已判定（正式运行没正常结束、没有产物），
|
||||
* 要么交回校验器的运行定义，由调用方交给 runner 去跑。
|
||||
*/
|
||||
export type JudgingStep = { status: "judged"; result: JudgedResult } | { status: "validate"; definition: RunDefinition };
|
||||
|
||||
/** 一种任务的实现。对外函数按 task.facts.kind 从对照表里取出它来调。参数的含义见 task.ts 里同名的对外函数。 */
|
||||
export interface TaskImplementation {
|
||||
/** 最终指标（编号和方向取自 task.json）与常规辅助指标的定义。 */
|
||||
metricDefinitions(facts: TaskFacts): MetricDefinition[];
|
||||
prepareRun(task: Task, candidateDir: string, runDir: string, hypothesisMetrics: HypothesisMetric[]): void;
|
||||
formalRun(task: Task, imageDigest: string, seed: string): RunDefinition;
|
||||
lightCheck(task: Task, candidateDir: string, checkDir: string, imageDigest: string, seconds: number): RunDefinition;
|
||||
readLightCheck(checkDir: string, result: ProcessResult): LightCheckOutcome;
|
||||
prepareJudging(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
imageDigest: string,
|
||||
formal: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgingStep;
|
||||
judge(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
formal: ProcessResult,
|
||||
validator: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgedResult;
|
||||
}
|
||||
+4
-3
@@ -128,7 +128,8 @@ const RESOURCE_USAGE = {
|
||||
/**
|
||||
* 以什么方式结束。有没有启动由它决定：只有 notStarted 是没启动，
|
||||
* 其余三种都启动过、要扣次数（方法.md 第七部分）。
|
||||
* 启动前就取消的记 notStarted；启动后取消的记 cancelled。
|
||||
* 启动前就取消的记 notStarted；cancelled 是容器从外面被杀了（关机、手动 docker kill、
|
||||
* 计量的 shell 自己被杀），没有留下计量结果，三项资源都记测不到。
|
||||
* 退出码只有正常退出才有，资源用量只有启动过才有，所以分成四种形状。
|
||||
*/
|
||||
export const ProcessResult = Type.Union([
|
||||
@@ -168,8 +169,8 @@ export type ProcessResult = Static<typeof ProcessResult>;
|
||||
// 运行结果第二层：tasks 从原始产物独立判分
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 判定三选一：算数、不算数、判不了（判定所需的证据没拿到）。 */
|
||||
export const VERDICTS = ["valid", "invalid", "undecidable"] as const;
|
||||
/** 判定二选一：算数、不算数。 */
|
||||
export const VERDICTS = ["valid", "invalid"] as const;
|
||||
export type Verdict = (typeof VERDICTS)[number];
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user