feat: 任务书 05，runner 与 tasks 两个模块
runner 用 Docker 跑一条命令并交回结局和资源用量；tasks 读 task.json、准备运行目录、给出正式运行/校验器/轻量检查的运行定义，并从原始产物独立判分。任务目录从参考项目搬到 tasks/circle_packing/，新增 task.json；vocab 的 VERDICTS 删掉"判不了"，ProcessResult 的 cancelled 改为"容器从外面被杀了"。 全新上下文审阅后的两处修复：计量 shell 以容器内 root 运行，命令经 setpriv 降权，结果写在容器自己的文件层、结束后 docker cp 取出，并加 no-new-privileges，候选无法伪造计量结果把超时记成正常退出；harness 开跑前删掉旧的 solution.json、metrics.json，旧产物不会被当成本次的值。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1 @@
|
||||
{"centers": [[0.100484056446833, 0.07485776462601139], [0.10728235286470139, 0.2738532442085582], [0.086108914434533, 0.5223893812779171], [0.10580193867928778, 0.6806936405144427], [0.0856033942977713, 0.9222111281543698], [0.3120328263981882, 0.10480425665863033], [0.32716502959347243, 0.3067558057750085], [0.2897009711447359, 0.47507350932200926], [0.2935854248884542, 0.7104275384094612], [0.27237639359908744, 0.8951262487554578], [0.4914744630822173, 0.08989291879330835], [0.5100326585018874, 0.29884106286561407], [0.5213978072978597, 0.4946336955389669], [0.5151435556465788, 0.7026215638204316], [0.49451516606288104, 0.901030906906145], [0.721963590658637, 0.11913688062063861], [0.7296948042290226, 0.2744414856202059], [0.7002013752356513, 0.5129578551833195], [0.7069421474478524, 0.7067135961639011], [0.7178150742617755, 0.9143740466217667], [0.8917011179459332, 0.11647248318508979], [0.9275227999594169, 0.2848229492760654], [0.9028755468513529, 0.5143032303369243], [0.9178606846823245, 0.681413920131626], [0.9136434762753853, 0.8933655911924985], [0.18562920767437355, 0.7769807813776922]], "radii": [0.06361218681134884, 0.0441172439828679, 0.03913703570214593, 0.027074375269719862, 0.07778887184563021, 0.05730567971921061, 0.029286768433871176, 0.058739899233888004, 0.050016382123936116, 0.07485971141413426, 0.07105090920173604, 0.04742662315759923, 0.039360492244268, 0.05993669686613301, 0.0893218390023967, 0.06428677645874871, 0.08948690383353296, 0.09702962465194793, 0.09684333677627958, 0.08562595337823331, 0.1020164679124565, 0.07010288699954595, 0.09090324918001258, 0.07687796847999327, 0.08635652372461466, 0.07171254287625847], "reported_sum": 1.76027694928051}
|
||||
@@ -0,0 +1,224 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { after, test, type TestContext } from "node:test";
|
||||
import { Runner } from "../src/runner/index.ts";
|
||||
import type { ProcessResult, RunDefinition } from "../src/vocab/index.ts";
|
||||
|
||||
// 用真实 Docker 测（任务书 05 第五部分）：没有 Docker 时整个文件失败，不跳过。
|
||||
// 镜像、容器名一律 inforsi-test- 开头的固定名字；本机还有别的项目在建删容器，只动自己前缀的。
|
||||
|
||||
const TASK_DIR = join(import.meta.dirname, "..", "tasks", "circle_packing");
|
||||
const IMAGE_TAG = "inforsi-test-circle";
|
||||
const MISSING_DIGEST = `sha256:${"0".repeat(64)}`;
|
||||
|
||||
const runner = new Runner(null);
|
||||
const digest = await runner.buildImage(TASK_DIR, IMAGE_TAG);
|
||||
after(() => docker(["rmi", IMAGE_TAG]));
|
||||
|
||||
function docker(args: string[]): { status: number | null; stdout: string; stderr: string } {
|
||||
return spawnSync("docker", args, { encoding: "utf8" });
|
||||
}
|
||||
|
||||
/** 删掉测试自己的容器；不存在不算错，别的错误照样报。 */
|
||||
function forceRemove(name: string): void {
|
||||
const removed = docker(["rm", "-f", name]);
|
||||
if (removed.status !== 0 && !removed.stderr.includes("No such container")) throw new Error(removed.stderr);
|
||||
}
|
||||
|
||||
/** 一个测试用的容器名：先清掉上次没删干净的同名容器，测试结束后再删。 */
|
||||
function containerName(t: TestContext, suffix: string): string {
|
||||
const name = `inforsi-test-${suffix}`;
|
||||
forceRemove(name);
|
||||
t.after(() => forceRemove(name));
|
||||
return name;
|
||||
}
|
||||
|
||||
function dirs(t: TestContext): { runDir: string; logDir: string } {
|
||||
const root = mkdtempSync(join(tmpdir(), "inforsi-runner-"));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const runDir = join(root, "run");
|
||||
const logDir = join(root, "log");
|
||||
mkdirSync(runDir);
|
||||
mkdirSync(logDir);
|
||||
return { runDir, logDir };
|
||||
}
|
||||
|
||||
function definition(command: string[], wallSeconds: number, image: string): RunDefinition {
|
||||
return {
|
||||
command,
|
||||
workdir: "/work",
|
||||
image: { digest: image, runtime: "runc" },
|
||||
wallSeconds,
|
||||
cores: 1,
|
||||
inheritEnv: false,
|
||||
env: {},
|
||||
};
|
||||
}
|
||||
|
||||
const python = (code: string): string[] => ["python3", "-c", code];
|
||||
|
||||
/** 起一个容器并等它结束，交回结果和日志目录。 */
|
||||
async function runToEnd(t: TestContext, suffix: string, command: string[], wallSeconds: number) {
|
||||
const name = containerName(t, suffix);
|
||||
const { runDir, logDir } = dirs(t);
|
||||
const outcome = await runner.start(name, definition(command, wallSeconds, digest), runDir);
|
||||
assert.deepEqual(outcome, { started: true });
|
||||
const result = await runner.wait(name, logDir);
|
||||
return { name, result, logDir };
|
||||
}
|
||||
|
||||
function value(result: ProcessResult, key: "cpuSeconds" | "wallSeconds" | "peakMemoryBytes"): number {
|
||||
assert.notEqual(result.ending, "notStarted");
|
||||
if (result.ending === "notStarted") throw new Error("unreachable");
|
||||
const item = result[key];
|
||||
assert.equal(item.status, "measured", `${key} 应当测到`);
|
||||
if (item.status !== "measured") throw new Error("unreachable");
|
||||
return item.value;
|
||||
}
|
||||
|
||||
test("建镜像命中参考项目的缓存；查镜像：有的答有，没有的答没有", async () => {
|
||||
assert.match(digest, /^sha256:7ed52793365c/);
|
||||
assert.equal(await runner.hasImage(digest), true);
|
||||
assert.equal(await runner.hasImage(MISSING_DIGEST), false);
|
||||
});
|
||||
|
||||
test("正常退出：带退出码和三项资源用量，标准输出、标准错误存成两个文件", async (t) => {
|
||||
const code = "import sys; print('out'); print('err', file=sys.stderr); sys.exit(3)";
|
||||
const { result, logDir } = await runToEnd(t, "exit", python(code), 30);
|
||||
assert.equal(result.ending, "exited");
|
||||
assert.equal(result.ending === "exited" && result.exitCode, 3);
|
||||
assert.ok(value(result, "cpuSeconds") > 0);
|
||||
assert.ok(value(result, "wallSeconds") > 0 && value(result, "wallSeconds") < 30);
|
||||
assert.ok(value(result, "peakMemoryBytes") > 0);
|
||||
assert.equal(readFileSync(join(logDir, "stdout.log"), "utf8"), "out\n");
|
||||
assert.equal(readFileSync(join(logDir, "stderr.log"), "utf8"), "err\n");
|
||||
});
|
||||
|
||||
test("超时：结局 timeout，CPU 秒接近时限，墙钟不短于时限", async (t) => {
|
||||
const { result } = await runToEnd(t, "timeout", python("while True: pass"), 2);
|
||||
assert.equal(result.ending, "timeout");
|
||||
const cpu = value(result, "cpuSeconds");
|
||||
assert.ok(cpu > 1.7 && cpu < 2.3, `CPU 秒 ${cpu}`);
|
||||
assert.ok(value(result, "wallSeconds") >= 2);
|
||||
});
|
||||
|
||||
test("候选伪造计量结果再死循环：伪造不成，结局仍是 timeout，资源用量是真实测到的；写出的文件归当前用户", async (t) => {
|
||||
// 审阅时实测过的手法：先写一份假的 usage，再把目录改成只读，然后死循环到超时。
|
||||
const code = [
|
||||
"import os",
|
||||
"for attempt in (lambda: open('/inforsi-record/usage', 'w').write('exited 0 1 1 1\\n'), lambda: os.chmod('/inforsi-record', 0o555)):",
|
||||
" try:",
|
||||
" attempt()",
|
||||
" except OSError as error:",
|
||||
" print(type(error).__name__, flush=True)",
|
||||
"open('out.txt', 'w').write('x')",
|
||||
"while True: pass",
|
||||
].join("\n");
|
||||
const name = containerName(t, "forge");
|
||||
const { runDir, logDir } = dirs(t);
|
||||
assert.deepEqual(await runner.start(name, definition(python(code), 2, digest), runDir), { started: true });
|
||||
const result = await runner.wait(name, logDir);
|
||||
assert.equal(readFileSync(join(logDir, "stdout.log"), "utf8"), "PermissionError\nPermissionError\n");
|
||||
assert.equal(result.ending, "timeout");
|
||||
const cpu = value(result, "cpuSeconds");
|
||||
t.diagnostic(`结局 ${result.ending}，CPU 秒 ${cpu}，墙钟秒 ${value(result, "wallSeconds")}`);
|
||||
assert.ok(cpu > 1.7 && cpu < 2.3, `CPU 秒 ${cpu}`);
|
||||
assert.ok(value(result, "wallSeconds") >= 2);
|
||||
assert.equal(statSync(join(runDir, "out.txt")).uid, process.getuid?.());
|
||||
});
|
||||
|
||||
test("命令自己以 124 或 137 退出，不判成超时", async (t) => {
|
||||
const own124 = await runToEnd(t, "exit124", python("import sys; sys.exit(124)"), 30);
|
||||
assert.equal(own124.result.ending, "exited");
|
||||
assert.equal(own124.result.ending === "exited" && own124.result.exitCode, 124);
|
||||
const own137 = await runToEnd(t, "exit137", python("import os; os.kill(os.getpid(), 9)"), 30);
|
||||
assert.equal(own137.result.ending, "exited");
|
||||
assert.equal(own137.result.ending === "exited" && own137.result.exitCode, 137);
|
||||
});
|
||||
|
||||
test("从外面 docker kill：结局 cancelled，三项资源都测不到并写明原因", async (t) => {
|
||||
const name = containerName(t, "killed");
|
||||
const { runDir, logDir } = dirs(t);
|
||||
const outcome = await runner.start(name, definition(python("import time; time.sleep(60)"), 120, digest), runDir);
|
||||
assert.deepEqual(outcome, { started: true });
|
||||
assert.equal(await runner.status(name), "running");
|
||||
const killed = docker(["kill", name]);
|
||||
assert.equal(killed.status, 0, killed.stderr);
|
||||
const result = await runner.wait(name, logDir);
|
||||
assert.equal(result.ending, "cancelled");
|
||||
if (result.ending !== "cancelled") return;
|
||||
for (const item of [result.cpuSeconds, result.wallSeconds, result.peakMemoryBytes]) {
|
||||
assert.equal(item.status, "unmeasured");
|
||||
assert.match(item.status === "unmeasured" ? item.reason : "", /从外面被结束/);
|
||||
}
|
||||
});
|
||||
|
||||
test("镜像编号不存在：没启动，原因是 Docker 的报错原文，也没留下容器", async (t) => {
|
||||
const name = containerName(t, "noimage");
|
||||
const { runDir } = dirs(t);
|
||||
const outcome = await runner.start(name, definition(python("pass"), 30, MISSING_DIGEST), runDir);
|
||||
assert.equal(outcome.started, false);
|
||||
assert.match(outcome.started ? "" : outcome.reason, /No such image/);
|
||||
assert.equal(await runner.status(name), "absent");
|
||||
});
|
||||
|
||||
test("断网：容器里连不上外网", async (t) => {
|
||||
const code = [
|
||||
"import socket, sys",
|
||||
"try:",
|
||||
" socket.create_connection(('1.1.1.1', 53), timeout=3)",
|
||||
"except OSError as error:",
|
||||
" print(type(error).__name__); sys.exit(0)",
|
||||
"sys.exit(1)",
|
||||
].join("\n");
|
||||
const { result } = await runToEnd(t, "network", python(code), 30);
|
||||
assert.equal(result.ending === "exited" && result.exitCode, 0);
|
||||
});
|
||||
|
||||
test("核数：容器里 os.cpu_count() 报 1", async (t) => {
|
||||
const { logDir } = await runToEnd(t, "cores", python("import os; print(os.cpu_count())"), 30);
|
||||
assert.equal(readFileSync(join(logDir, "stdout.log"), "utf8"), "1\n");
|
||||
});
|
||||
|
||||
test("查状态：没有这个容器、建了没启动、在跑、已结束、问不到", async (t) => {
|
||||
assert.equal(await runner.status(containerName(t, "status-absent")), "absent");
|
||||
|
||||
const created = containerName(t, "status-created");
|
||||
const create = docker(["create", "--name", created, digest, "true"]);
|
||||
assert.equal(create.status, 0, create.stderr);
|
||||
assert.equal(await runner.status(created), "created");
|
||||
|
||||
const name = containerName(t, "status-running");
|
||||
const { runDir, logDir } = dirs(t);
|
||||
await runner.start(name, definition(python("import time; time.sleep(3)"), 30, digest), runDir);
|
||||
assert.equal(await runner.status(name), "running");
|
||||
await runner.wait(name, logDir);
|
||||
assert.equal(await runner.status(name), "exited");
|
||||
|
||||
const nowhere = new Runner("unix:///nonexistent/docker.sock");
|
||||
assert.equal(await nowhere.status(name), "unreachable");
|
||||
|
||||
await runner.remove(name);
|
||||
assert.equal(await runner.status(name), "absent");
|
||||
});
|
||||
|
||||
test("等结束能接上另一个 Runner 实例启动的容器；已结束的容器再等一次，结果相同", async (t) => {
|
||||
const name = containerName(t, "resume");
|
||||
const { runDir, logDir } = dirs(t);
|
||||
const starter = new Runner(null);
|
||||
await starter.start(name, definition(python("import time; time.sleep(2); print('done')"), 30, digest), runDir);
|
||||
const resumed = new Runner(null);
|
||||
const first = await resumed.wait(name, logDir);
|
||||
assert.equal(first.ending === "exited" && first.exitCode, 0);
|
||||
assert.equal(readFileSync(join(logDir, "stdout.log"), "utf8"), "done\n");
|
||||
const again = await new Runner(null).wait(name, logDir);
|
||||
assert.deepEqual(again, first);
|
||||
});
|
||||
|
||||
test("容器名不符合 Docker 的要求时报错", async () => {
|
||||
await assert.rejects(runner.status("inforsi test"), /不符合 Docker 的要求/);
|
||||
await assert.rejects(runner.status("-x"), /不符合 Docker 的要求/);
|
||||
});
|
||||
@@ -0,0 +1,460 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { after, test, type TestContext } from "node:test";
|
||||
import { Runner } from "../src/runner/index.ts";
|
||||
import {
|
||||
checkTaskFacts,
|
||||
formalRun,
|
||||
judge,
|
||||
lightCheck,
|
||||
loadTask,
|
||||
metricDefinitions,
|
||||
prepareJudging,
|
||||
prepareRun,
|
||||
protectedChanges,
|
||||
readLightCheck,
|
||||
} from "../src/tasks/index.ts";
|
||||
import type { HypothesisMetric, JudgedResult, Measured, ProcessResult, RunDefinition } from "../src/vocab/index.ts";
|
||||
|
||||
// tasks 的测试（任务书 05 第五部分）。凡是要执行任务代码的，都经真实 Docker 在任务镜像里跑：
|
||||
// 没有 Docker 时整个文件失败，不跳过。镜像和容器一律用 inforsi-test-tasks- 开头的固定名字，
|
||||
// 与 runner 的测试（inforsi-test-circle、inforsi-test-<用途>）错开，两个文件并行跑也不互相删。
|
||||
// 判分的预期值是手算的 2 的幂有理数布局（借自参考项目 tests/circle-validator.test.ts），
|
||||
// 加一份参考项目真实运行里差一点不合法的产物（tests/fixtures/legalization/）。
|
||||
|
||||
const TASK_DIR = join(import.meta.dirname, "..", "tasks", "circle_packing");
|
||||
const IMAGE_TAG = "inforsi-test-tasks";
|
||||
const FIXTURE = join(import.meta.dirname, "fixtures", "legalization", "14-real-1-keep-grid-start.json");
|
||||
|
||||
const task = loadTask(TASK_DIR);
|
||||
const runner = new Runner(null);
|
||||
const digest = await runner.buildImage(task.dir, IMAGE_TAG);
|
||||
after(() => docker(["rmi", IMAGE_TAG]));
|
||||
|
||||
function docker(args: string[]): { status: number | null; stderr: string } {
|
||||
return spawnSync("docker", args, { encoding: "utf8" });
|
||||
}
|
||||
|
||||
function forceRemove(name: string): void {
|
||||
const removed = docker(["rm", "-f", name]);
|
||||
if (removed.status !== 0 && !removed.stderr.includes("No such container")) throw new Error(removed.stderr);
|
||||
}
|
||||
|
||||
function tempDir(t: TestContext, label: string): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), `inforsi-tasks-${label}-`));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
|
||||
/** 让 runner 在任务镜像里跑一份运行定义，交回结果。 */
|
||||
async function run(t: TestContext, suffix: string, definition: RunDefinition, dir: string): Promise<ProcessResult> {
|
||||
const name = `inforsi-test-tasks-${suffix}`;
|
||||
forceRemove(name);
|
||||
t.after(() => forceRemove(name));
|
||||
const logDir = tempDir(t, "log");
|
||||
const outcome = await runner.start(name, definition, dir);
|
||||
assert.deepEqual(outcome, { started: true });
|
||||
return runner.wait(name, logDir);
|
||||
}
|
||||
|
||||
const m = (value: number): Measured => ({ status: "measured", value });
|
||||
|
||||
/** 一次正常结束的正式运行；资源用量是随手写的，判分只是原样转记。 */
|
||||
const EXITED_OK: ProcessResult = { ending: "exited", exitCode: 0, cpuSeconds: m(1.5), wallSeconds: m(2.5), peakMemoryBytes: m(1024) };
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 手算布局（2 的幂的有理数，双精度里精确）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface Solution {
|
||||
centers: number[][];
|
||||
radii: number[];
|
||||
reported_sum: number;
|
||||
}
|
||||
|
||||
/** 一行等间距圆：圆心 (firstX + i·spacing, 0.5)，半径统一。 */
|
||||
function rowLayout(radius: number, spacing: number, firstX: number): Solution {
|
||||
const centers: number[][] = [];
|
||||
const radii: number[] = [];
|
||||
for (let i = 0; i < 26; i += 1) {
|
||||
centers.push([firstX + i * spacing, 0.5]);
|
||||
radii.push(radius);
|
||||
}
|
||||
return { centers, radii, reported_sum: 99 };
|
||||
}
|
||||
|
||||
// 宽松布局：半径 1/128，间距 1/32，首个圆心 7/64。相邻间隙 1/32 − 1/64 = 1/64 > 0；
|
||||
// 最右 7/64 + 25/32 + 1/128 = 0.8984375 < 1。半径和 26/128 = 0.203125。
|
||||
const loose = (): Solution => rowLayout(1 / 128, 1 / 32, 7 / 64);
|
||||
// 相切布局：半径 1/64，间距 1/32 恰好等于两半径之和，间隙恰好 0。半径和 26/64 = 0.40625。
|
||||
const tangent = (): Solution => rowLayout(1 / 64, 1 / 32, 7 / 64);
|
||||
|
||||
/** 判分走完两步：有产物就在镜像里跑校验器。runDir 里放好 solution.json（或不放）。 */
|
||||
async function judgeRun(
|
||||
t: TestContext,
|
||||
suffix: string,
|
||||
runDir: string,
|
||||
formal: ProcessResult,
|
||||
metrics: HypothesisMetric[],
|
||||
): Promise<JudgedResult> {
|
||||
const judgeDir = tempDir(t, "judge");
|
||||
const step = prepareJudging(task, runDir, judgeDir, digest, formal, metrics);
|
||||
if (step.status === "judged") return step.result;
|
||||
const validator = await run(t, suffix, step.definition, judgeDir);
|
||||
return judge(task, runDir, judgeDir, formal, validator, metrics);
|
||||
}
|
||||
|
||||
function runDirWith(t: TestContext, solution: Solution | string | null): string {
|
||||
const dir = tempDir(t, "run");
|
||||
if (solution !== null) {
|
||||
writeFileSync(join(dir, "solution.json"), typeof solution === "string" ? solution : JSON.stringify(solution));
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
function value(item: Measured | undefined): number {
|
||||
assert.equal(item?.status, "measured", JSON.stringify(item));
|
||||
return (item as { value: number }).value;
|
||||
}
|
||||
|
||||
function reason(item: Measured | undefined): string {
|
||||
assert.equal(item?.status, "unmeasured", JSON.stringify(item));
|
||||
return (item as { reason: string }).reason;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 任务目录与 task.json
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("基线三个文件的 sha256 等于 SOURCES.md 表上的值", () => {
|
||||
// 期望值从 baseline/SOURCES.md 的表格手抄，不调项目里的哈希函数。
|
||||
const expected: Array<[string, string]> = [
|
||||
["initial.py", "09da8428a85792a94a9c6cc2193b6a0db1090ce598a17b75bc0db48d0953d098"],
|
||||
["evaluate.py", "eb8b9787dc25dee24a5f5305d1e3bd876e5c336cded1c9bab7fb204612928b4d"],
|
||||
["LICENSE", "c3e5b1c26607caf7208d5a105c113fe240630bb873fdb8ca5b4c2e32c721ddaf"],
|
||||
];
|
||||
for (const [name, sha] of expected) {
|
||||
assert.equal(createHash("sha256").update(readFileSync(join(task.baselineDir, name))).digest("hex"), sha, name);
|
||||
}
|
||||
});
|
||||
|
||||
test("读任务：task.json 的值、Dockerfile 所在目录、TASK.md 全文；最终指标和辅助指标的定义", () => {
|
||||
assert.deepEqual(task.facts, JSON.parse(readFileSync(join(TASK_DIR, "task.json"), "utf8")));
|
||||
assert.equal(task.facts.finalMetric.id, "sum-of-radii");
|
||||
assert.equal(task.facts.epsilon, 0.01);
|
||||
assert.equal(task.facts.knownBest, 2.635983);
|
||||
assert.deepEqual(task.facts.run, { wallSeconds: 120, cores: 1, runtime: "runc" });
|
||||
assert.ok(existsSync(join(task.dir, "Dockerfile")));
|
||||
assert.match(task.guide, /^# 填圆任务/);
|
||||
const ids = metricDefinitions(task).map((d) => `${d.layer}:${d.id}`);
|
||||
assert.deepEqual(ids, [
|
||||
"final:sum-of-radii",
|
||||
"auxiliary:wall-seconds",
|
||||
"auxiliary:cpu-seconds",
|
||||
"auxiliary:peak-memory-bytes",
|
||||
"auxiliary:min-pairwise-clearance",
|
||||
"auxiliary:raw-score",
|
||||
"auxiliary:raw-violation",
|
||||
"auxiliary:legalization-deduction",
|
||||
]);
|
||||
});
|
||||
|
||||
test("task.json：缺字段、多字段、类型不对，一次全部列出，带字段路径", () => {
|
||||
const good = JSON.parse(readFileSync(join(TASK_DIR, "task.json"), "utf8")) as Record<string, unknown>;
|
||||
const bad = {
|
||||
...good,
|
||||
epsilon: "0.01",
|
||||
knownBest: "unknown",
|
||||
run: { wallSeconds: 120, cores: 1.5 },
|
||||
extra: true,
|
||||
} as Record<string, unknown>;
|
||||
delete bad.protectedFiles;
|
||||
assert.throws(
|
||||
() => checkTaskFacts(bad),
|
||||
(error: Error) => {
|
||||
const lines = error.message.split("\n").slice(1);
|
||||
assert.match(error.message, /^task\.json 不合格，共 6 处/);
|
||||
assert.ok(lines.some((l) => l.startsWith("/protectedFiles：缺少")), error.message);
|
||||
assert.ok(lines.some((l) => l.startsWith("/extra：不认识的字段")), error.message);
|
||||
assert.ok(lines.some((l) => l.startsWith("/epsilon：")), error.message);
|
||||
assert.ok(lines.some((l) => l.startsWith("/knownBest：应为数字，没有就写 null")), error.message);
|
||||
assert.ok(lines.some((l) => l.startsWith("/run/cores：")), error.message);
|
||||
assert.ok(lines.some((l) => l.startsWith("/run/runtime：缺少")), error.message);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
assert.deepEqual(checkTaskFacts({ ...good, knownBest: null }).knownBest, null);
|
||||
});
|
||||
|
||||
test("读任务：task.json 不是合法 JSON 时报错带文件路径", (t) => {
|
||||
const dir = tempDir(t, "badjson");
|
||||
writeFileSync(join(dir, "task.json"), "{ not json");
|
||||
assert.throws(() => loadTask(dir), /task\.json 读不出来或不是合法的 JSON/);
|
||||
});
|
||||
|
||||
test("不许改的文件：只交回改动名单里属于 task.json 名单的那些", () => {
|
||||
assert.deepEqual(protectedChanges(task, ["initial.py", "harness.py", "lib/util.py", "validate.py"]), ["harness.py", "validate.py"]);
|
||||
assert.deepEqual(protectedChanges(task, ["initial.py"]), []);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 判定表（任务书 05 第三部分"判定怎么定"）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("判定表 1：正式运行超时、被从外面结束、退出码非 0，不算数，不跑校验器，资源用量照记", async (t) => {
|
||||
const un: Measured = { status: "unmeasured", reason: "容器从外面被结束" };
|
||||
const cases: Array<[ProcessResult, RegExp]> = [
|
||||
[{ ending: "timeout", cpuSeconds: m(119), wallSeconds: m(120), peakMemoryBytes: m(5) }, /超过单次时限 120 秒/],
|
||||
[{ ending: "cancelled", cpuSeconds: un, wallSeconds: un, peakMemoryBytes: un }, /容器从外面被结束/],
|
||||
[{ ...EXITED_OK, exitCode: 1 }, /退出码 1/],
|
||||
];
|
||||
for (const [formal, why] of cases) {
|
||||
// 有一份合法的产物也不跑校验器：运行没正常结束就不算数。
|
||||
const result = await judgeRun(t, "unused", runDirWith(t, loose()), formal, []);
|
||||
assert.equal(result.verdict, "invalid");
|
||||
assert.match(result.verdictReason, why);
|
||||
assert.match(reason(result.score), why);
|
||||
assert.match(reason(result.metrics["raw-score"]), /没有跑校验器/);
|
||||
assert.deepEqual(result.extra, { legalization: null });
|
||||
if (formal.ending !== "notStarted") assert.deepEqual(result.metrics["cpu-seconds"], formal.cpuSeconds);
|
||||
}
|
||||
});
|
||||
|
||||
test("判定表 2：没有 solution.json，不算数：没有产物", async (t) => {
|
||||
const result = await judgeRun(t, "unused", runDirWith(t, null), EXITED_OK, []);
|
||||
assert.equal(result.verdict, "invalid");
|
||||
assert.match(result.verdictReason, /^没有产物/);
|
||||
});
|
||||
|
||||
test("判定表 3：校验器判不合法（形状不对、非有限数、圆心出界修不了），不算数，带校验器的原因", async (t) => {
|
||||
const short = loose();
|
||||
short.centers.pop();
|
||||
short.radii.pop();
|
||||
const shape = await judgeRun(t, "shape", runDirWith(t, short), EXITED_OK, []);
|
||||
assert.equal(shape.verdict, "invalid");
|
||||
assert.match(shape.verdictReason, /shape incorrect/);
|
||||
assert.match(reason(shape.metrics["raw-score"]), /形状不对/);
|
||||
assert.match(reason(shape.metrics["min-pairwise-clearance"]), /形状不对/);
|
||||
|
||||
const nan = JSON.stringify(loose()).replace('"radii":[0.0078125', '"radii":[NaN');
|
||||
const nonFinite = await judgeRun(t, "nan", runDirWith(t, nan), EXITED_OK, []);
|
||||
assert.equal(nonFinite.verdict, "invalid");
|
||||
assert.match(nonFinite.verdictReason, /non-finite/);
|
||||
|
||||
// 0 号圆心挪到 x = −1/64：圆心在正方形外，只缩半径修不了。
|
||||
const outside = loose();
|
||||
outside.centers[0] = [-1 / 64, 0.5];
|
||||
const result = await judgeRun(t, "outside", runDirWith(t, outside), EXITED_OK, []);
|
||||
assert.equal(result.verdict, "invalid");
|
||||
assert.match(result.verdictReason, /圆心在正方形外面/);
|
||||
assert.match(reason(result.score), /圆心在正方形外面/);
|
||||
assert.match(reason(result.metrics["legalization-deduction"]), /修不了/);
|
||||
// 原分数照样算得出：26 个圆半径都没动，0.203125。
|
||||
assert.equal(value(result.metrics["raw-score"]), 0.203125);
|
||||
});
|
||||
|
||||
test("判定表 4：校验器判合法，算数，分数取校验器算的（宽松 0.203125、相切 0.40625，扣 0）", async (t) => {
|
||||
const result = await judgeRun(t, "loose", runDirWith(t, loose()), EXITED_OK, []);
|
||||
assert.equal(result.verdict, "valid");
|
||||
assert.equal(value(result.score), 0.203125);
|
||||
assert.equal(value(result.metrics["min-pairwise-clearance"]), 0.015625);
|
||||
assert.equal(value(result.metrics["raw-score"]), 0.203125);
|
||||
assert.equal(value(result.metrics["raw-violation"]), 0);
|
||||
assert.equal(value(result.metrics["legalization-deduction"]), 0);
|
||||
// 资源用量原样来自正式运行。
|
||||
assert.equal(value(result.metrics["wall-seconds"]), 2.5);
|
||||
assert.equal(value(result.metrics["peak-memory-bytes"]), 1024);
|
||||
|
||||
const touching = await judgeRun(t, "tangent", runDirWith(t, tangent()), EXITED_OK, []);
|
||||
assert.equal(touching.verdict, "valid");
|
||||
assert.equal(value(touching.score), 0.40625);
|
||||
assert.equal(value(touching.metrics["min-pairwise-clearance"]), 0);
|
||||
});
|
||||
|
||||
test("判定表 4（合法化）：差一点不合法的修好后算数，修后分数不高于原分数，扣分等于原分数减最终分数", async (t) => {
|
||||
// 0、1 号半径放大到 3/128：重叠 3/64 − 1/32 = 1/64；1、2 号间隙恰好 0。原分数 30/128 = 0.234375。
|
||||
// 只缩参与违反的圆：0、1 号各减 1/64 回到 1/128，修后 0.203125，扣 1/32 = 0.03125。
|
||||
const overlap = loose();
|
||||
overlap.radii[0] = 3 / 128;
|
||||
overlap.radii[1] = 3 / 128;
|
||||
const shrunk = await judgeRun(t, "overlap", runDirWith(t, overlap), EXITED_OK, []);
|
||||
assert.equal(shrunk.verdict, "valid");
|
||||
assert.equal(value(shrunk.score), 0.203125);
|
||||
assert.equal(value(shrunk.metrics["raw-score"]), 0.234375);
|
||||
assert.equal(value(shrunk.metrics["raw-violation"]), 0.015625);
|
||||
assert.equal(value(shrunk.metrics["legalization-deduction"]), 0.03125);
|
||||
const extra = shrunk.extra as { legalization: { shrunkCircles: number[]; rawValid: boolean; deduction: Measured } };
|
||||
assert.deepEqual(extra.legalization.shrunkCircles, [0, 1]);
|
||||
assert.equal(extra.legalization.rawValid, false);
|
||||
assert.equal(value(extra.legalization.deduction), 0.03125);
|
||||
|
||||
// 0 号圆心挪到 x = 1/256，出左边 1/128 − 1/256 = 1/256；修后半径 1/256，和 51/256 = 0.19921875。
|
||||
const edge = loose();
|
||||
edge.centers[0] = [1 / 256, 0.5];
|
||||
const edged = await judgeRun(t, "edge", runDirWith(t, edge), EXITED_OK, []);
|
||||
assert.equal(value(edged.score), 0.19921875);
|
||||
assert.equal(value(edged.metrics["raw-violation"]), 1 / 256);
|
||||
assert.equal(value(edged.metrics["legalization-deduction"]), 0.203125 - 0.19921875);
|
||||
|
||||
// 参考项目真实运行 14-real-1 的产物：重叠约 2.78e-17，修后 1.7602769492805097（参考项目设计-15 7.1 节记的数）。
|
||||
const real = await judgeRun(t, "real", runDirWith(t, readFileSync(FIXTURE, "utf8")), EXITED_OK, []);
|
||||
assert.equal(real.verdict, "valid");
|
||||
assert.equal(value(real.score), 1.7602769492805097);
|
||||
const raw = value(real.metrics["raw-score"]);
|
||||
assert.equal(raw, 1.76027694928051);
|
||||
assert.ok(value(real.score) <= raw);
|
||||
assert.equal(value(real.metrics["legalization-deduction"]), raw - value(real.score));
|
||||
assert.ok(Math.abs(value(real.metrics["raw-violation"]) - 2.78e-17) < 0.01e-17);
|
||||
});
|
||||
|
||||
test("判定表 5：缺了整次作废的假设专用指标测不到，不算数；缺了只记缺失的照常算数", async (t) => {
|
||||
const spec = (id: string, missing: HypothesisMetric["missing"]): HypothesisMetric => ({
|
||||
id,
|
||||
meaning: "测试用",
|
||||
unit: "无",
|
||||
direction: "maximize",
|
||||
howMeasured: "测试用",
|
||||
missing,
|
||||
});
|
||||
const runDir = runDirWith(t, loose());
|
||||
writeFileSync(
|
||||
join(runDir, "metrics.json"),
|
||||
JSON.stringify({ spread: 0.5, gap: null, "answer-path": "p1", "safeguard-triggers": { "shrink-repair": 2 } }),
|
||||
);
|
||||
const kept = await judgeRun(t, "keep", runDir, EXITED_OK, [spec("spread", "invalidateRun"), spec("gap", "recordMissing")]);
|
||||
assert.equal(kept.verdict, "valid");
|
||||
assert.equal(value(kept.metrics.spread), 0.5);
|
||||
assert.match(reason(kept.metrics.gap), /null/);
|
||||
assert.deepEqual(kept.answerPath, { status: "read", value: "p1" });
|
||||
assert.deepEqual(kept.safeguardTriggers, { status: "read", counts: { "shrink-repair": 2 } });
|
||||
|
||||
const voided = await judgeRun(t, "void", runDir, EXITED_OK, [spec("gap", "invalidateRun")]);
|
||||
assert.equal(voided.verdict, "invalid");
|
||||
assert.match(voided.verdictReason, /缺了整次作废.*gap/);
|
||||
// 成绩照样算出来记下，只是这次运行不算数。
|
||||
assert.equal(value(voided.score), 0.203125);
|
||||
});
|
||||
|
||||
test("答案路径、保障动手次数：读不到时记读不到及原因，不作废运行", async (t) => {
|
||||
const missing = await judgeRun(t, "nometrics", runDirWith(t, loose()), EXITED_OK, []);
|
||||
assert.equal(missing.verdict, "valid");
|
||||
assert.deepEqual(missing.answerPath, { status: "unreadable", reason: "metrics.json 不存在" });
|
||||
|
||||
const runDir = runDirWith(t, loose());
|
||||
writeFileSync(join(runDir, "metrics.json"), JSON.stringify({ "answer-path": 3, "safeguard-triggers": { a: -1 } }));
|
||||
const wrong = await judgeRun(t, "wrongkeys", runDir, EXITED_OK, []);
|
||||
assert.equal(wrong.verdict, "valid");
|
||||
assert.equal(wrong.answerPath.status, "unreadable");
|
||||
assert.equal(wrong.safeguardTriggers.status, "unreadable");
|
||||
});
|
||||
|
||||
test("校验器自己没正常跑完、判定认不出、修后分数高于原分数：报错停下，不当成候选的失败", (t) => {
|
||||
const runDir = runDirWith(t, loose());
|
||||
const judgeDir = tempDir(t, "judge");
|
||||
const un: Measured = { status: "unmeasured", reason: "x" };
|
||||
const timeout: ProcessResult = { ending: "timeout", cpuSeconds: un, wallSeconds: un, peakMemoryBytes: un };
|
||||
assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, timeout, []), /校验器没有正常跑完（timeout）/);
|
||||
// 退出码 2 是用法或读取出错，这时校验器不写判定文件。
|
||||
assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, { ...EXITED_OK, exitCode: 2 }, []), /校验器没有正常跑完（退出码 2）/);
|
||||
assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, EXITED_OK, []), /读不出 .*verdict\.json/);
|
||||
|
||||
const verdict = {
|
||||
valid: true,
|
||||
computed_sum: 0.3,
|
||||
reasons: [],
|
||||
diagnostics: { min_pairwise_clearance: 0 },
|
||||
legalization: { raw_valid: false, raw_score: 0.2, raw_violation: 0.1, applied: true, repaired: true, shrunk_circles: [0], reasons: [] },
|
||||
};
|
||||
writeFileSync(join(judgeDir, "verdict.json"), JSON.stringify(verdict));
|
||||
assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, EXITED_OK, []), /合法化后分数 0\.3 高于原分数 0\.2/);
|
||||
// 退出码说不合法、判定说合法：对不上。
|
||||
assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, { ...EXITED_OK, exitCode: 1 }, []), /对不上/);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 轻量检查
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function check(t: TestContext, suffix: string, files: Record<string, string>) {
|
||||
const candidate = tempDir(t, "candidate");
|
||||
for (const [name, text] of Object.entries(files)) writeFileSync(join(candidate, name), text);
|
||||
const checkDir = tempDir(t, "check");
|
||||
const definition = lightCheck(task, candidate, checkDir, digest, 5);
|
||||
return readLightCheck(task, checkDir, await run(t, suffix, definition, checkDir));
|
||||
}
|
||||
|
||||
test("轻量检查：语法错的、没有入口函数的不通过，正常的通过", async (t) => {
|
||||
const syntax = await check(t, "check-syntax", { "initial.py": "def run_packing(:\n pass\n" });
|
||||
assert.equal(syntax.passed, false);
|
||||
assert.match(syntax.passed ? "" : syntax.reasons.join(), /initial\.py 解析失败/);
|
||||
|
||||
const noEntry = await check(t, "check-noentry", { "initial.py": "def solve():\n return [], [], 0.0\n" });
|
||||
assert.equal(noEntry.passed, false);
|
||||
assert.match(noEntry.passed ? "" : noEntry.reasons.join(), /run_packing\(\) 应当恰好有一个定义，实际有 0 个/);
|
||||
|
||||
const baseline = tempDir(t, "baseline");
|
||||
cpSync(task.baselineDir, baseline, { recursive: true });
|
||||
const checkDir = tempDir(t, "check");
|
||||
const ok = readLightCheck(task, checkDir, await run(t, "check-ok", lightCheck(task, baseline, checkDir, digest, 5), checkDir));
|
||||
assert.deepEqual(ok, { passed: true });
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 从头到尾
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 基线 initial.py 准备运行目录 → 正式运行 → 跑校验器 → 判分。 */
|
||||
async function endToEnd(t: TestContext, suffix: string): Promise<JudgedResult> {
|
||||
const runDir = tempDir(t, "e2e");
|
||||
prepareRun(task, task.baselineDir, runDir, []);
|
||||
assert.equal(readFileSync(join(runDir, "metrics-spec.json"), "utf8"), "[]\n");
|
||||
assert.ok(existsSync(join(runDir, "harness.py")) && existsSync(join(runDir, "initial.py")));
|
||||
const formal = await run(t, `e2e-${suffix}`, formalRun(task, digest, "inforsi-test-seed"), runDir);
|
||||
assert.equal(formal.ending, "exited", JSON.stringify(formal));
|
||||
return judgeRun(t, `e2e-${suffix}-judge`, runDir, formal, []);
|
||||
}
|
||||
|
||||
test("从头到尾：基线跑两次都判为算数，分数逐位相同", async (t) => {
|
||||
const first = await endToEnd(t, "1");
|
||||
const second = await endToEnd(t, "2");
|
||||
for (const result of [first, second]) {
|
||||
assert.equal(result.verdict, "valid", result.verdictReason);
|
||||
assert.equal(result.score.status, "measured");
|
||||
}
|
||||
assert.ok(Object.is(value(first.score), value(second.score)), `${value(first.score)} 与 ${value(second.score)}`);
|
||||
for (const [label, result] of [["第一次", first], ["第二次", second]] as const) {
|
||||
t.diagnostic(
|
||||
`${label}：verdict=${result.verdict}（${result.verdictReason}） score=${value(result.score)} ` +
|
||||
`cpu=${JSON.stringify(result.metrics["cpu-seconds"])} wall=${JSON.stringify(result.metrics["wall-seconds"])} ` +
|
||||
`mem=${JSON.stringify(result.metrics["peak-memory-bytes"])} deduction=${JSON.stringify(result.metrics["legalization-deduction"])}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("候选目录里遗留的旧 metrics.json 不会被当成这次的产物：整次作废的指标这次没写，判为不算数", async (t) => {
|
||||
const candidate = tempDir(t, "candidate");
|
||||
cpSync(task.baselineDir, candidate, { recursive: true });
|
||||
// 基线代码不写 metrics.json；这份是以前某次运行留下的。
|
||||
writeFileSync(join(candidate, "metrics.json"), JSON.stringify({ spread: 0.5, "answer-path": "main", "safeguard-triggers": {} }));
|
||||
const spread: HypothesisMetric = {
|
||||
id: "spread",
|
||||
meaning: "测试用",
|
||||
unit: "无",
|
||||
direction: "maximize",
|
||||
howMeasured: "测试用",
|
||||
missing: "invalidateRun",
|
||||
};
|
||||
const runDir = tempDir(t, "stale");
|
||||
prepareRun(task, candidate, runDir, [spread]);
|
||||
const formal = await run(t, "stale", formalRun(task, digest, "inforsi-test-seed"), runDir);
|
||||
assert.equal(formal.ending, "exited", JSON.stringify(formal));
|
||||
const result = await judgeRun(t, "stale-judge", runDir, formal, [spread]);
|
||||
t.diagnostic(`verdict=${result.verdict}（${result.verdictReason}） spread=${JSON.stringify(result.metrics.spread)}`);
|
||||
assert.equal(result.metrics.spread?.status, "unmeasured");
|
||||
assert.equal(result.verdict, "invalid");
|
||||
assert.match(result.verdictReason, /缺了整次作废.*spread/);
|
||||
});
|
||||
Reference in New Issue
Block a user