import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { after, test, type TestContext } from "node:test"; import { Runner } from "../src/runner/index.ts"; import { checkTaskFacts, formalRun, judge, lightCheck, loadTask, metricDefinitions, prepareJudging, prepareRun, protectedChanges, readLightCheck, } from "../src/tasks/index.ts"; import type { HypothesisMetric, JudgedResult, Measured, ProcessResult, RunDefinition } from "../src/vocab/index.ts"; // tasks 的测试(任务书 05 第五部分)。凡是要执行任务代码的,都经真实 Docker 在任务镜像里跑: // 没有 Docker 时整个文件失败,不跳过。镜像和容器一律用 inforsi-test-tasks- 开头的固定名字, // 与 runner 的测试(inforsi-test-circle、inforsi-test-<用途>)错开,两个文件并行跑也不互相删。 // 判分的预期值是手算的 2 的幂有理数布局(借自参考项目 tests/circle-validator.test.ts), // 加一份参考项目真实运行里差一点不合法的产物(tests/fixtures/legalization/)。 const TASK_DIR = join(import.meta.dirname, "..", "tasks", "circle_packing"); const IMAGE_TAG = "inforsi-test-tasks"; const FIXTURE = join(import.meta.dirname, "fixtures", "legalization", "14-real-1-keep-grid-start.json"); const task = loadTask(TASK_DIR); const runner = new Runner(null); const digest = await runner.buildImage(task.dir, IMAGE_TAG); after(() => docker(["rmi", IMAGE_TAG])); function docker(args: string[]): { status: number | null; stderr: string } { return spawnSync("docker", args, { encoding: "utf8" }); } function forceRemove(name: string): void { const removed = docker(["rm", "-f", name]); if (removed.status !== 0 && !removed.stderr.includes("No such container")) throw new Error(removed.stderr); } function tempDir(t: TestContext, label: string): string { const dir = mkdtempSync(join(tmpdir(), `inforsi-tasks-${label}-`)); t.after(() => rmSync(dir, { recursive: true, force: true })); return dir; } /** 让 runner 在任务镜像里跑一份运行定义,交回结果。 */ async function run(t: TestContext, suffix: string, definition: RunDefinition, dir: string): Promise { const name = `inforsi-test-tasks-${suffix}`; forceRemove(name); t.after(() => forceRemove(name)); const logDir = tempDir(t, "log"); const outcome = await runner.start(name, definition, dir); assert.deepEqual(outcome, { started: true }); return runner.wait(name, logDir); } const m = (value: number): Measured => ({ status: "measured", value }); /** 一次正常结束的正式运行;资源用量是随手写的,判分只是原样转记。 */ const EXITED_OK: ProcessResult = { ending: "exited", exitCode: 0, cpuSeconds: m(1.5), wallSeconds: m(2.5), peakMemoryBytes: m(1024) }; // --------------------------------------------------------------------------- // 手算布局(2 的幂的有理数,双精度里精确) // --------------------------------------------------------------------------- interface Solution { centers: number[][]; radii: number[]; reported_sum: number; } /** 一行等间距圆:圆心 (firstX + i·spacing, 0.5),半径统一。 */ function rowLayout(radius: number, spacing: number, firstX: number): Solution { const centers: number[][] = []; const radii: number[] = []; for (let i = 0; i < 26; i += 1) { centers.push([firstX + i * spacing, 0.5]); radii.push(radius); } return { centers, radii, reported_sum: 99 }; } // 宽松布局:半径 1/128,间距 1/32,首个圆心 7/64。相邻间隙 1/32 − 1/64 = 1/64 > 0; // 最右 7/64 + 25/32 + 1/128 = 0.8984375 < 1。半径和 26/128 = 0.203125。 const loose = (): Solution => rowLayout(1 / 128, 1 / 32, 7 / 64); // 相切布局:半径 1/64,间距 1/32 恰好等于两半径之和,间隙恰好 0。半径和 26/64 = 0.40625。 const tangent = (): Solution => rowLayout(1 / 64, 1 / 32, 7 / 64); /** 判分走完两步:有产物就在镜像里跑校验器。runDir 里放好 solution.json(或不放)。 */ async function judgeRun( t: TestContext, suffix: string, runDir: string, formal: ProcessResult, metrics: HypothesisMetric[], ): Promise { const judgeDir = tempDir(t, "judge"); const step = prepareJudging(task, runDir, judgeDir, digest, formal, metrics); if (step.status === "judged") return step.result; const validator = await run(t, suffix, step.definition, judgeDir); return judge(task, runDir, judgeDir, formal, validator, metrics); } function runDirWith(t: TestContext, solution: Solution | string | null): string { const dir = tempDir(t, "run"); if (solution !== null) { writeFileSync(join(dir, "solution.json"), typeof solution === "string" ? solution : JSON.stringify(solution)); } return dir; } function value(item: Measured | undefined): number { assert.equal(item?.status, "measured", JSON.stringify(item)); return (item as { value: number }).value; } function reason(item: Measured | undefined): string { assert.equal(item?.status, "unmeasured", JSON.stringify(item)); return (item as { reason: string }).reason; } // --------------------------------------------------------------------------- // 任务目录与 task.json // --------------------------------------------------------------------------- test("基线三个文件的 sha256 等于 SOURCES.md 表上的值", () => { // 期望值从 baseline/SOURCES.md 的表格手抄,不调项目里的哈希函数。 const expected: Array<[string, string]> = [ ["initial.py", "09da8428a85792a94a9c6cc2193b6a0db1090ce598a17b75bc0db48d0953d098"], ["evaluate.py", "eb8b9787dc25dee24a5f5305d1e3bd876e5c336cded1c9bab7fb204612928b4d"], ["LICENSE", "c3e5b1c26607caf7208d5a105c113fe240630bb873fdb8ca5b4c2e32c721ddaf"], ]; for (const [name, sha] of expected) { assert.equal(createHash("sha256").update(readFileSync(join(task.baselineDir, name))).digest("hex"), sha, name); } }); test("读任务:task.json 的值、Dockerfile 所在目录、TASK.md 全文;最终指标和辅助指标的定义", () => { assert.deepEqual(task.facts, JSON.parse(readFileSync(join(TASK_DIR, "task.json"), "utf8"))); assert.equal(task.facts.finalMetric.id, "sum-of-radii"); assert.equal(task.facts.epsilon, 0.01); assert.equal(task.facts.knownBest, 2.635983); assert.deepEqual(task.facts.run, { wallSeconds: 120, cores: 1, runtime: "runc" }); assert.deepEqual(task.facts.unversionedNames, ["__pycache__", ".git", "solution.json", "metrics.json"]); assert.ok(existsSync(join(task.dir, "Dockerfile"))); assert.match(task.guide, /^# 填圆任务/); const ids = metricDefinitions(task).map((d) => `${d.layer}:${d.id}`); assert.deepEqual(ids, [ "final:sum-of-radii", "auxiliary:wall-seconds", "auxiliary:cpu-seconds", "auxiliary:peak-memory-bytes", "auxiliary:min-pairwise-clearance", "auxiliary:raw-score", "auxiliary:raw-violation", "auxiliary:legalization-deduction", ]); }); test("task.json:缺字段、多字段、类型不对,一次全部列出,带字段路径", () => { const good = JSON.parse(readFileSync(join(TASK_DIR, "task.json"), "utf8")) as Record; const bad = { ...good, epsilon: "0.01", knownBest: "unknown", run: { wallSeconds: 120, cores: 1.5 }, extra: true, } as Record; delete bad.protectedFiles; delete bad.unversionedNames; assert.throws( () => checkTaskFacts(bad), (error: Error) => { const lines = error.message.split("\n").slice(1); assert.match(error.message, /^task\.json 不合格,共 7 处/); assert.ok(lines.some((l) => l.startsWith("/protectedFiles:缺少")), error.message); assert.ok(lines.some((l) => l.startsWith("/unversionedNames:缺少")), error.message); assert.ok(lines.some((l) => l.startsWith("/extra:不认识的字段")), error.message); assert.ok(lines.some((l) => l.startsWith("/epsilon:")), error.message); assert.ok(lines.some((l) => l.startsWith("/knownBest:应为数字,没有就写 null")), error.message); assert.ok(lines.some((l) => l.startsWith("/run/cores:")), error.message); assert.ok(lines.some((l) => l.startsWith("/run/runtime:缺少")), error.message); return true; }, ); assert.deepEqual(checkTaskFacts({ ...good, knownBest: null }).knownBest, null); // "不存"清单只写名字:带 / 的写法在任何一层都对不上,报错。 assert.throws(() => checkTaskFacts({ ...good, unversionedNames: ["lib/__pycache__"] }), /\/unversionedNames\/0:格式不对/); }); test("读任务:task.json 不是合法 JSON 时报错带文件路径", (t) => { const dir = tempDir(t, "badjson"); writeFileSync(join(dir, "task.json"), "{ not json"); assert.throws(() => loadTask(dir), /task\.json 读不出来或不是合法的 JSON/); }); test("不许改的文件:只交回改动名单里属于 task.json 名单的那些", () => { assert.deepEqual(protectedChanges(task, ["initial.py", "harness.py", "lib/util.py", "validate.py"]), ["harness.py", "validate.py"]); assert.deepEqual(protectedChanges(task, ["initial.py"]), []); }); // --------------------------------------------------------------------------- // 判定表(任务书 05 第三部分"判定怎么定") // --------------------------------------------------------------------------- test("判定表 1:正式运行超时、被从外面结束、退出码非 0,不算数,不跑校验器,资源用量照记", async (t) => { const un: Measured = { status: "unmeasured", reason: "容器从外面被结束" }; const cases: Array<[ProcessResult, RegExp]> = [ [{ ending: "timeout", cpuSeconds: m(119), wallSeconds: m(120), peakMemoryBytes: m(5) }, /超过单次时限 120 秒/], [{ ending: "cancelled", cpuSeconds: un, wallSeconds: un, peakMemoryBytes: un }, /容器从外面被结束/], [{ ...EXITED_OK, exitCode: 1 }, /退出码 1/], ]; for (const [formal, why] of cases) { // 有一份合法的产物也不跑校验器:运行没正常结束就不算数。 const result = await judgeRun(t, "unused", runDirWith(t, loose()), formal, []); assert.equal(result.verdict, "invalid"); assert.match(result.verdictReason, why); assert.match(reason(result.score), why); assert.match(reason(result.metrics["raw-score"]), /没有跑校验器/); assert.deepEqual(result.extra, { legalization: null }); if (formal.ending !== "notStarted") assert.deepEqual(result.metrics["cpu-seconds"], formal.cpuSeconds); } }); test("判定表 2:没有 solution.json,不算数:没有产物", async (t) => { const result = await judgeRun(t, "unused", runDirWith(t, null), EXITED_OK, []); assert.equal(result.verdict, "invalid"); assert.match(result.verdictReason, /^没有产物/); }); test("判定表 3:校验器判不合法(形状不对、非有限数、圆心出界修不了),不算数,带校验器的原因", async (t) => { const short = loose(); short.centers.pop(); short.radii.pop(); const shape = await judgeRun(t, "shape", runDirWith(t, short), EXITED_OK, []); assert.equal(shape.verdict, "invalid"); assert.match(shape.verdictReason, /shape incorrect/); assert.match(reason(shape.metrics["raw-score"]), /形状不对/); assert.match(reason(shape.metrics["min-pairwise-clearance"]), /形状不对/); const nan = JSON.stringify(loose()).replace('"radii":[0.0078125', '"radii":[NaN'); const nonFinite = await judgeRun(t, "nan", runDirWith(t, nan), EXITED_OK, []); assert.equal(nonFinite.verdict, "invalid"); assert.match(nonFinite.verdictReason, /non-finite/); // 0 号圆心挪到 x = −1/64:圆心在正方形外,只缩半径修不了。 const outside = loose(); outside.centers[0] = [-1 / 64, 0.5]; const result = await judgeRun(t, "outside", runDirWith(t, outside), EXITED_OK, []); assert.equal(result.verdict, "invalid"); assert.match(result.verdictReason, /圆心在正方形外面/); assert.match(reason(result.score), /圆心在正方形外面/); assert.match(reason(result.metrics["legalization-deduction"]), /修不了/); // 原分数照样算得出:26 个圆半径都没动,0.203125。 assert.equal(value(result.metrics["raw-score"]), 0.203125); }); test("判定表 4:校验器判合法,算数,分数取校验器算的(宽松 0.203125、相切 0.40625,扣 0)", async (t) => { const result = await judgeRun(t, "loose", runDirWith(t, loose()), EXITED_OK, []); assert.equal(result.verdict, "valid"); assert.equal(value(result.score), 0.203125); assert.equal(value(result.metrics["min-pairwise-clearance"]), 0.015625); assert.equal(value(result.metrics["raw-score"]), 0.203125); assert.equal(value(result.metrics["raw-violation"]), 0); assert.equal(value(result.metrics["legalization-deduction"]), 0); // 资源用量原样来自正式运行。 assert.equal(value(result.metrics["wall-seconds"]), 2.5); assert.equal(value(result.metrics["peak-memory-bytes"]), 1024); const touching = await judgeRun(t, "tangent", runDirWith(t, tangent()), EXITED_OK, []); assert.equal(touching.verdict, "valid"); assert.equal(value(touching.score), 0.40625); assert.equal(value(touching.metrics["min-pairwise-clearance"]), 0); }); test("判定表 4(合法化):差一点不合法的修好后算数,修后分数不高于原分数,扣分等于原分数减最终分数", async (t) => { // 0、1 号半径放大到 3/128:重叠 3/64 − 1/32 = 1/64;1、2 号间隙恰好 0。原分数 30/128 = 0.234375。 // 只缩参与违反的圆:0、1 号各减 1/64 回到 1/128,修后 0.203125,扣 1/32 = 0.03125。 const overlap = loose(); overlap.radii[0] = 3 / 128; overlap.radii[1] = 3 / 128; const shrunk = await judgeRun(t, "overlap", runDirWith(t, overlap), EXITED_OK, []); assert.equal(shrunk.verdict, "valid"); assert.equal(value(shrunk.score), 0.203125); assert.equal(value(shrunk.metrics["raw-score"]), 0.234375); assert.equal(value(shrunk.metrics["raw-violation"]), 0.015625); assert.equal(value(shrunk.metrics["legalization-deduction"]), 0.03125); const extra = shrunk.extra as { legalization: { shrunkCircles: number[]; rawValid: boolean; deduction: Measured } }; assert.deepEqual(extra.legalization.shrunkCircles, [0, 1]); assert.equal(extra.legalization.rawValid, false); assert.equal(value(extra.legalization.deduction), 0.03125); // 0 号圆心挪到 x = 1/256,出左边 1/128 − 1/256 = 1/256;修后半径 1/256,和 51/256 = 0.19921875。 const edge = loose(); edge.centers[0] = [1 / 256, 0.5]; const edged = await judgeRun(t, "edge", runDirWith(t, edge), EXITED_OK, []); assert.equal(value(edged.score), 0.19921875); assert.equal(value(edged.metrics["raw-violation"]), 1 / 256); assert.equal(value(edged.metrics["legalization-deduction"]), 0.203125 - 0.19921875); // 参考项目真实运行 14-real-1 的产物:重叠约 2.78e-17,修后 1.7602769492805097(参考项目设计-15 7.1 节记的数)。 const real = await judgeRun(t, "real", runDirWith(t, readFileSync(FIXTURE, "utf8")), EXITED_OK, []); assert.equal(real.verdict, "valid"); assert.equal(value(real.score), 1.7602769492805097); const raw = value(real.metrics["raw-score"]); assert.equal(raw, 1.76027694928051); assert.ok(value(real.score) <= raw); assert.equal(value(real.metrics["legalization-deduction"]), raw - value(real.score)); assert.ok(Math.abs(value(real.metrics["raw-violation"]) - 2.78e-17) < 0.01e-17); }); test("判定表 5:缺了整次作废的假设专用指标测不到,不算数;缺了只记缺失的照常算数", async (t) => { const spec = (id: string, missing: HypothesisMetric["missing"]): HypothesisMetric => ({ id, meaning: "测试用", unit: "无", direction: "maximize", howMeasured: "测试用", missing, }); const runDir = runDirWith(t, loose()); writeFileSync( join(runDir, "metrics.json"), JSON.stringify({ spread: 0.5, gap: null, "answer-path": "p1", "safeguard-triggers": { "shrink-repair": 2 } }), ); const kept = await judgeRun(t, "keep", runDir, EXITED_OK, [spec("spread", "invalidateRun"), spec("gap", "recordMissing")]); assert.equal(kept.verdict, "valid"); assert.equal(value(kept.metrics.spread), 0.5); assert.match(reason(kept.metrics.gap), /null/); assert.deepEqual(kept.answerPath, { status: "read", value: "p1" }); assert.deepEqual(kept.safeguardTriggers, { status: "read", counts: { "shrink-repair": 2 } }); const voided = await judgeRun(t, "void", runDir, EXITED_OK, [spec("gap", "invalidateRun")]); assert.equal(voided.verdict, "invalid"); assert.match(voided.verdictReason, /缺了整次作废.*gap/); // 成绩照样算出来记下,只是这次运行不算数。 assert.equal(value(voided.score), 0.203125); }); test("答案路径、保障动手次数:读不到时记读不到及原因,不作废运行", async (t) => { const missing = await judgeRun(t, "nometrics", runDirWith(t, loose()), EXITED_OK, []); assert.equal(missing.verdict, "valid"); assert.deepEqual(missing.answerPath, { status: "unreadable", reason: "metrics.json 不存在" }); const runDir = runDirWith(t, loose()); writeFileSync(join(runDir, "metrics.json"), JSON.stringify({ "answer-path": 3, "safeguard-triggers": { a: -1 } })); const wrong = await judgeRun(t, "wrongkeys", runDir, EXITED_OK, []); assert.equal(wrong.verdict, "valid"); assert.equal(wrong.answerPath.status, "unreadable"); assert.equal(wrong.safeguardTriggers.status, "unreadable"); }); test("校验器自己没正常跑完、判定认不出、修后分数高于原分数:报错停下,不当成候选的失败", (t) => { const runDir = runDirWith(t, loose()); const judgeDir = tempDir(t, "judge"); const un: Measured = { status: "unmeasured", reason: "x" }; const timeout: ProcessResult = { ending: "timeout", cpuSeconds: un, wallSeconds: un, peakMemoryBytes: un }; assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, timeout, []), /校验器没有正常跑完(timeout)/); // 退出码 2 是用法或读取出错,这时校验器不写判定文件。 assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, { ...EXITED_OK, exitCode: 2 }, []), /校验器没有正常跑完(退出码 2)/); assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, EXITED_OK, []), /读不出 .*verdict\.json/); const verdict = { valid: true, computed_sum: 0.3, reasons: [], diagnostics: { min_pairwise_clearance: 0 }, legalization: { raw_valid: false, raw_score: 0.2, raw_violation: 0.1, applied: true, repaired: true, shrunk_circles: [0], reasons: [] }, }; writeFileSync(join(judgeDir, "verdict.json"), JSON.stringify(verdict)); assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, EXITED_OK, []), /合法化后分数 0\.3 高于原分数 0\.2/); // 退出码说不合法、判定说合法:对不上。 assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, { ...EXITED_OK, exitCode: 1 }, []), /对不上/); }); // --------------------------------------------------------------------------- // 轻量检查 // --------------------------------------------------------------------------- async function check(t: TestContext, suffix: string, files: Record) { const candidate = tempDir(t, "candidate"); for (const [name, text] of Object.entries(files)) writeFileSync(join(candidate, name), text); const checkDir = tempDir(t, "check"); const definition = lightCheck(task, candidate, checkDir, digest, 5); return readLightCheck(task, checkDir, await run(t, suffix, definition, checkDir)); } test("轻量检查:语法错的、没有入口函数的不通过,正常的通过", async (t) => { const syntax = await check(t, "check-syntax", { "initial.py": "def run_packing(:\n pass\n" }); assert.equal(syntax.passed, false); assert.match(syntax.passed ? "" : syntax.reasons.join(), /initial\.py 解析失败/); const noEntry = await check(t, "check-noentry", { "initial.py": "def solve():\n return [], [], 0.0\n" }); assert.equal(noEntry.passed, false); assert.match(noEntry.passed ? "" : noEntry.reasons.join(), /run_packing\(\) 应当恰好有一个定义,实际有 0 个/); const baseline = tempDir(t, "baseline"); cpSync(task.baselineDir, baseline, { recursive: true }); const checkDir = tempDir(t, "check"); const ok = readLightCheck(task, checkDir, await run(t, "check-ok", lightCheck(task, baseline, checkDir, digest, 5), checkDir)); assert.deepEqual(ok, { passed: true }); }); // --------------------------------------------------------------------------- // 从头到尾 // --------------------------------------------------------------------------- /** 基线 initial.py 准备运行目录 → 正式运行 → 跑校验器 → 判分。 */ async function endToEnd(t: TestContext, suffix: string): Promise { const runDir = tempDir(t, "e2e"); prepareRun(task, task.baselineDir, runDir, []); assert.equal(readFileSync(join(runDir, "metrics-spec.json"), "utf8"), "[]\n"); assert.ok(existsSync(join(runDir, "harness.py")) && existsSync(join(runDir, "initial.py"))); const formal = await run(t, `e2e-${suffix}`, formalRun(task, digest, "inforsi-test-seed"), runDir); assert.equal(formal.ending, "exited", JSON.stringify(formal)); return judgeRun(t, `e2e-${suffix}-judge`, runDir, formal, []); } test("从头到尾:基线跑两次都判为算数,分数逐位相同", async (t) => { const first = await endToEnd(t, "1"); const second = await endToEnd(t, "2"); for (const result of [first, second]) { assert.equal(result.verdict, "valid", result.verdictReason); assert.equal(result.score.status, "measured"); } assert.ok(Object.is(value(first.score), value(second.score)), `${value(first.score)} 与 ${value(second.score)}`); for (const [label, result] of [["第一次", first], ["第二次", second]] as const) { t.diagnostic( `${label}:verdict=${result.verdict}(${result.verdictReason}) score=${value(result.score)} ` + `cpu=${JSON.stringify(result.metrics["cpu-seconds"])} wall=${JSON.stringify(result.metrics["wall-seconds"])} ` + `mem=${JSON.stringify(result.metrics["peak-memory-bytes"])} deduction=${JSON.stringify(result.metrics["legalization-deduction"])}`, ); } }); test("候选目录里遗留的旧 metrics.json 不会被当成这次的产物:整次作废的指标这次没写,判为不算数", async (t) => { const candidate = tempDir(t, "candidate"); cpSync(task.baselineDir, candidate, { recursive: true }); // 基线代码不写 metrics.json;这份是以前某次运行留下的。 writeFileSync(join(candidate, "metrics.json"), JSON.stringify({ spread: 0.5, "answer-path": "main", "safeguard-triggers": {} })); const spread: HypothesisMetric = { id: "spread", meaning: "测试用", unit: "无", direction: "maximize", howMeasured: "测试用", missing: "invalidateRun", }; const runDir = tempDir(t, "stale"); prepareRun(task, candidate, runDir, [spread]); const formal = await run(t, "stale", formalRun(task, digest, "inforsi-test-seed"), runDir); assert.equal(formal.ending, "exited", JSON.stringify(formal)); const result = await judgeRun(t, "stale-judge", runDir, formal, [spread]); t.diagnostic(`verdict=${result.verdict}(${result.verdictReason}) spread=${JSON.stringify(result.metrics.spread)}`); assert.equal(result.metrics.spread?.status, "unmeasured"); assert.equal(result.verdict, "invalid"); assert.match(result.verdictReason, /缺了整次作废.*spread/); });