iomgaa
13a7e60355
- 假设专用指标短名不能含"."（最长 64），冻结时由 materials 的 freezeProtocol 改为全名"候选编号.短名"；可引用清单用全名，已占用清单只剩任务指标和两个保留键。 - 消融格核验不通过重做时，工作目录是上一次交的这一格代码的副本，能读的版本加上它，完整代码照旧列出；消融实现提示词同步改。 - jev 判重材料说明补 keySettings 一句。 - 结构.md 第八部分 vocab、materials、jev，WORK_LOG 同步更新。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
476 lines
24 KiB
TypeScript
476 lines
24 KiB
TypeScript
import assert from "node:assert/strict";
|
||
import { spawnSync } from "node:child_process";
|
||
import { createHash } from "node:crypto";
|
||
import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||
import { tmpdir } from "node:os";
|
||
import { join } from "node:path";
|
||
import { after, test, type TestContext } from "node:test";
|
||
import { Runner } from "../src/runner/index.ts";
|
||
import {
|
||
checkTaskFacts,
|
||
formalRun,
|
||
judge,
|
||
lightCheck,
|
||
loadTask,
|
||
metricDefinitions,
|
||
prepareJudging,
|
||
prepareRun,
|
||
protectedChanges,
|
||
readLightCheck,
|
||
} from "../src/tasks/index.ts";
|
||
import type { HypothesisMetric, JudgedResult, Measured, ProcessResult, RunDefinition } from "../src/vocab/index.ts";
|
||
|
||
// tasks 的测试（任务书 05 第五部分）。凡是要执行任务代码的，都经真实 Docker 在任务镜像里跑：
|
||
// 没有 Docker 时整个文件失败，不跳过。镜像和容器一律用 inforsi-test-tasks- 开头的固定名字，
|
||
// 与 runner 的测试（inforsi-test-circle、inforsi-test-<用途>）错开，两个文件并行跑也不互相删。
|
||
// 判分的预期值是手算的 2 的幂有理数布局（借自参考项目 tests/circle-validator.test.ts），
|
||
// 加一份参考项目真实运行里差一点不合法的产物（tests/fixtures/legalization/）。
|
||
|
||
const TASK_DIR = join(import.meta.dirname, "..", "tasks", "circle_packing");
|
||
const IMAGE_TAG = "inforsi-test-tasks";
|
||
const FIXTURE = join(import.meta.dirname, "fixtures", "legalization", "14-real-1-keep-grid-start.json");
|
||
|
||
const task = loadTask(TASK_DIR);
|
||
const runner = new Runner(null);
|
||
const digest = await runner.buildImage(task.dir, IMAGE_TAG);
|
||
after(() => docker(["rmi", IMAGE_TAG]));
|
||
|
||
function docker(args: string[]): { status: number | null; stderr: string } {
|
||
return spawnSync("docker", args, { encoding: "utf8" });
|
||
}
|
||
|
||
function forceRemove(name: string): void {
|
||
const removed = docker(["rm", "-f", name]);
|
||
if (removed.status !== 0 && !removed.stderr.includes("No such container")) throw new Error(removed.stderr);
|
||
}
|
||
|
||
function tempDir(t: TestContext, label: string): string {
|
||
const dir = mkdtempSync(join(tmpdir(), `inforsi-tasks-${label}-`));
|
||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||
return dir;
|
||
}
|
||
|
||
/** 让 runner 在任务镜像里跑一份运行定义，交回结果。 */
|
||
async function run(t: TestContext, suffix: string, definition: RunDefinition, dir: string): Promise<ProcessResult> {
|
||
const name = `inforsi-test-tasks-${suffix}`;
|
||
forceRemove(name);
|
||
t.after(() => forceRemove(name));
|
||
const logDir = tempDir(t, "log");
|
||
const outcome = await runner.start(name, definition, dir);
|
||
assert.deepEqual(outcome, { started: true });
|
||
return runner.wait(name, logDir);
|
||
}
|
||
|
||
const m = (value: number): Measured => ({ status: "measured", value });
|
||
|
||
/** 一次正常结束的正式运行；资源用量是随手写的，判分只是原样转记。 */
|
||
const EXITED_OK: ProcessResult = { ending: "exited", exitCode: 0, cpuSeconds: m(1.5), wallSeconds: m(2.5), peakMemoryBytes: m(1024) };
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 手算布局（2 的幂的有理数，双精度里精确）
|
||
// ---------------------------------------------------------------------------
|
||
|
||
interface Solution {
|
||
centers: number[][];
|
||
radii: number[];
|
||
reported_sum: number;
|
||
}
|
||
|
||
/** 一行等间距圆：圆心 (firstX + i·spacing, 0.5)，半径统一。 */
|
||
function rowLayout(radius: number, spacing: number, firstX: number): Solution {
|
||
const centers: number[][] = [];
|
||
const radii: number[] = [];
|
||
for (let i = 0; i < 26; i += 1) {
|
||
centers.push([firstX + i * spacing, 0.5]);
|
||
radii.push(radius);
|
||
}
|
||
return { centers, radii, reported_sum: 99 };
|
||
}
|
||
|
||
// 宽松布局：半径 1/128，间距 1/32，首个圆心 7/64。相邻间隙 1/32 − 1/64 = 1/64 > 0；
|
||
// 最右 7/64 + 25/32 + 1/128 = 0.8984375 < 1。半径和 26/128 = 0.203125。
|
||
const loose = (): Solution => rowLayout(1 / 128, 1 / 32, 7 / 64);
|
||
// 相切布局：半径 1/64，间距 1/32 恰好等于两半径之和，间隙恰好 0。半径和 26/64 = 0.40625。
|
||
const tangent = (): Solution => rowLayout(1 / 64, 1 / 32, 7 / 64);
|
||
|
||
/** 判分走完两步：有产物就在镜像里跑校验器。runDir 里放好 solution.json（或不放）。 */
|
||
async function judgeRun(
|
||
t: TestContext,
|
||
suffix: string,
|
||
runDir: string,
|
||
formal: ProcessResult,
|
||
metrics: HypothesisMetric[],
|
||
): Promise<JudgedResult> {
|
||
const judgeDir = tempDir(t, "judge");
|
||
const step = prepareJudging(task, runDir, judgeDir, digest, formal, metrics);
|
||
if (step.status === "judged") return step.result;
|
||
const validator = await run(t, suffix, step.definition, judgeDir);
|
||
return judge(task, runDir, judgeDir, formal, validator, metrics);
|
||
}
|
||
|
||
function runDirWith(t: TestContext, solution: Solution | string | null): string {
|
||
const dir = tempDir(t, "run");
|
||
if (solution !== null) {
|
||
writeFileSync(join(dir, "solution.json"), typeof solution === "string" ? solution : JSON.stringify(solution));
|
||
}
|
||
return dir;
|
||
}
|
||
|
||
function value(item: Measured | undefined): number {
|
||
assert.equal(item?.status, "measured", JSON.stringify(item));
|
||
return (item as { value: number }).value;
|
||
}
|
||
|
||
function reason(item: Measured | undefined): string {
|
||
assert.equal(item?.status, "unmeasured", JSON.stringify(item));
|
||
return (item as { reason: string }).reason;
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 任务目录与 task.json
|
||
// ---------------------------------------------------------------------------
|
||
|
||
test("基线三个文件的 sha256 等于 SOURCES.md 表上的值", () => {
|
||
// 期望值从 baseline/SOURCES.md 的表格手抄，不调项目里的哈希函数。
|
||
const expected: Array<[string, string]> = [
|
||
["initial.py", "09da8428a85792a94a9c6cc2193b6a0db1090ce598a17b75bc0db48d0953d098"],
|
||
["evaluate.py", "eb8b9787dc25dee24a5f5305d1e3bd876e5c336cded1c9bab7fb204612928b4d"],
|
||
["LICENSE", "c3e5b1c26607caf7208d5a105c113fe240630bb873fdb8ca5b4c2e32c721ddaf"],
|
||
];
|
||
for (const [name, sha] of expected) {
|
||
assert.equal(createHash("sha256").update(readFileSync(join(task.baselineDir, name))).digest("hex"), sha, name);
|
||
}
|
||
});
|
||
|
||
test("读任务：task.json 的值、Dockerfile 所在目录、TASK.md 全文；最终指标和辅助指标的定义", () => {
|
||
assert.deepEqual(task.facts, JSON.parse(readFileSync(join(TASK_DIR, "task.json"), "utf8")));
|
||
assert.equal(task.facts.finalMetric.id, "sum-of-radii");
|
||
assert.equal(task.facts.epsilon, 0.01);
|
||
assert.equal(task.facts.knownBest, 2.635983);
|
||
assert.deepEqual(task.facts.run, { wallSeconds: 120, cores: 1, runtime: "runc" });
|
||
assert.deepEqual(task.facts.unversionedNames, ["__pycache__", ".git", "solution.json", "metrics.json"]);
|
||
assert.ok(existsSync(join(task.dir, "Dockerfile")));
|
||
assert.match(task.guide, /^# 填圆任务/);
|
||
const ids = metricDefinitions(task).map((d) => `${d.layer}:${d.id}`);
|
||
assert.deepEqual(ids, [
|
||
"final:sum-of-radii",
|
||
"auxiliary:wall-seconds",
|
||
"auxiliary:cpu-seconds",
|
||
"auxiliary:peak-memory-bytes",
|
||
"auxiliary:min-pairwise-clearance",
|
||
"auxiliary:raw-score",
|
||
"auxiliary:raw-violation",
|
||
"auxiliary:legalization-deduction",
|
||
]);
|
||
});
|
||
|
||
test("task.json：缺字段、多字段、类型不对，一次全部列出，带字段路径", () => {
|
||
const good = JSON.parse(readFileSync(join(TASK_DIR, "task.json"), "utf8")) as Record<string, unknown>;
|
||
const bad = {
|
||
...good,
|
||
epsilon: "0.01",
|
||
knownBest: "unknown",
|
||
run: { wallSeconds: 120, cores: 1.5 },
|
||
extra: true,
|
||
} as Record<string, unknown>;
|
||
delete bad.protectedFiles;
|
||
delete bad.unversionedNames;
|
||
assert.throws(
|
||
() => checkTaskFacts(bad),
|
||
(error: Error) => {
|
||
const lines = error.message.split("\n").slice(1);
|
||
assert.match(error.message, /^task\.json 不合格，共 7 处/);
|
||
assert.ok(lines.some((l) => l.startsWith("/protectedFiles：缺少")), error.message);
|
||
assert.ok(lines.some((l) => l.startsWith("/unversionedNames：缺少")), error.message);
|
||
assert.ok(lines.some((l) => l.startsWith("/extra：不认识的字段")), error.message);
|
||
assert.ok(lines.some((l) => l.startsWith("/epsilon：")), error.message);
|
||
assert.ok(lines.some((l) => l.startsWith("/knownBest：应为数字，没有就写 null")), error.message);
|
||
assert.ok(lines.some((l) => l.startsWith("/run/cores：")), error.message);
|
||
assert.ok(lines.some((l) => l.startsWith("/run/runtime：缺少")), error.message);
|
||
return true;
|
||
},
|
||
);
|
||
assert.deepEqual(checkTaskFacts({ ...good, knownBest: null }).knownBest, null);
|
||
// "不存"清单只写名字：带 / 的写法在任何一层都对不上，报错。
|
||
assert.throws(() => checkTaskFacts({ ...good, unversionedNames: ["lib/__pycache__"] }), /\/unversionedNames\/0：格式不对/);
|
||
});
|
||
|
||
test("读任务：task.json 不是合法 JSON 时报错带文件路径", (t) => {
|
||
const dir = tempDir(t, "badjson");
|
||
writeFileSync(join(dir, "task.json"), "{ not json");
|
||
assert.throws(() => loadTask(dir), /task\.json 读不出来或不是合法的 JSON/);
|
||
});
|
||
|
||
test("不许改的文件：只交回改动名单里属于 task.json 名单的那些", () => {
|
||
assert.deepEqual(protectedChanges(task, ["initial.py", "harness.py", "lib/util.py", "validate.py"]), ["harness.py", "validate.py"]);
|
||
assert.deepEqual(protectedChanges(task, ["initial.py"]), []);
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 判定表（任务书 05 第三部分"判定怎么定"）
|
||
// ---------------------------------------------------------------------------
|
||
|
||
test("判定表 1：正式运行超时、被从外面结束、退出码非 0，不算数，不跑校验器，资源用量照记", async (t) => {
|
||
const un: Measured = { status: "unmeasured", reason: "容器从外面被结束" };
|
||
const cases: Array<[ProcessResult, RegExp]> = [
|
||
[{ ending: "timeout", cpuSeconds: m(119), wallSeconds: m(120), peakMemoryBytes: m(5) }, /超过单次时限 120 秒/],
|
||
[{ ending: "cancelled", cpuSeconds: un, wallSeconds: un, peakMemoryBytes: un }, /容器从外面被结束/],
|
||
[{ ...EXITED_OK, exitCode: 1 }, /退出码 1/],
|
||
];
|
||
for (const [formal, why] of cases) {
|
||
// 有一份合法的产物也不跑校验器：运行没正常结束就不算数。
|
||
const result = await judgeRun(t, "unused", runDirWith(t, loose()), formal, []);
|
||
assert.equal(result.verdict, "invalid");
|
||
assert.match(result.verdictReason, why);
|
||
assert.match(reason(result.score), why);
|
||
assert.match(reason(result.metrics["raw-score"]), /没有跑校验器/);
|
||
assert.deepEqual(result.extra, { legalization: null });
|
||
if (formal.ending !== "notStarted") assert.deepEqual(result.metrics["cpu-seconds"], formal.cpuSeconds);
|
||
}
|
||
});
|
||
|
||
test("判定表 2：没有 solution.json，不算数：没有产物", async (t) => {
|
||
const result = await judgeRun(t, "unused", runDirWith(t, null), EXITED_OK, []);
|
||
assert.equal(result.verdict, "invalid");
|
||
assert.match(result.verdictReason, /^没有产物/);
|
||
});
|
||
|
||
test("判定表 3：校验器判不合法（形状不对、非有限数、圆心出界修不了），不算数，带校验器的原因", async (t) => {
|
||
const short = loose();
|
||
short.centers.pop();
|
||
short.radii.pop();
|
||
const shape = await judgeRun(t, "shape", runDirWith(t, short), EXITED_OK, []);
|
||
assert.equal(shape.verdict, "invalid");
|
||
assert.match(shape.verdictReason, /shape incorrect/);
|
||
assert.match(reason(shape.metrics["raw-score"]), /形状不对/);
|
||
assert.match(reason(shape.metrics["min-pairwise-clearance"]), /形状不对/);
|
||
|
||
const nan = JSON.stringify(loose()).replace('"radii":[0.0078125', '"radii":[NaN');
|
||
const nonFinite = await judgeRun(t, "nan", runDirWith(t, nan), EXITED_OK, []);
|
||
assert.equal(nonFinite.verdict, "invalid");
|
||
assert.match(nonFinite.verdictReason, /non-finite/);
|
||
|
||
// 0 号圆心挪到 x = −1/64：圆心在正方形外，只缩半径修不了。
|
||
const outside = loose();
|
||
outside.centers[0] = [-1 / 64, 0.5];
|
||
const result = await judgeRun(t, "outside", runDirWith(t, outside), EXITED_OK, []);
|
||
assert.equal(result.verdict, "invalid");
|
||
assert.match(result.verdictReason, /圆心在正方形外面/);
|
||
assert.match(reason(result.score), /圆心在正方形外面/);
|
||
assert.match(reason(result.metrics["legalization-deduction"]), /修不了/);
|
||
// 原分数照样算得出：26 个圆半径都没动，0.203125。
|
||
assert.equal(value(result.metrics["raw-score"]), 0.203125);
|
||
});
|
||
|
||
test("判定表 4：校验器判合法，算数，分数取校验器算的（宽松 0.203125、相切 0.40625，扣 0）", async (t) => {
|
||
const result = await judgeRun(t, "loose", runDirWith(t, loose()), EXITED_OK, []);
|
||
assert.equal(result.verdict, "valid");
|
||
assert.equal(value(result.score), 0.203125);
|
||
assert.equal(value(result.metrics["min-pairwise-clearance"]), 0.015625);
|
||
assert.equal(value(result.metrics["raw-score"]), 0.203125);
|
||
assert.equal(value(result.metrics["raw-violation"]), 0);
|
||
assert.equal(value(result.metrics["legalization-deduction"]), 0);
|
||
// 资源用量原样来自正式运行。
|
||
assert.equal(value(result.metrics["wall-seconds"]), 2.5);
|
||
assert.equal(value(result.metrics["peak-memory-bytes"]), 1024);
|
||
|
||
const touching = await judgeRun(t, "tangent", runDirWith(t, tangent()), EXITED_OK, []);
|
||
assert.equal(touching.verdict, "valid");
|
||
assert.equal(value(touching.score), 0.40625);
|
||
assert.equal(value(touching.metrics["min-pairwise-clearance"]), 0);
|
||
});
|
||
|
||
test("判定表 4（合法化）：差一点不合法的修好后算数，修后分数不高于原分数，扣分等于原分数减最终分数", async (t) => {
|
||
// 0、1 号半径放大到 3/128：重叠 3/64 − 1/32 = 1/64；1、2 号间隙恰好 0。原分数 30/128 = 0.234375。
|
||
// 只缩参与违反的圆：0、1 号各减 1/64 回到 1/128，修后 0.203125，扣 1/32 = 0.03125。
|
||
const overlap = loose();
|
||
overlap.radii[0] = 3 / 128;
|
||
overlap.radii[1] = 3 / 128;
|
||
const shrunk = await judgeRun(t, "overlap", runDirWith(t, overlap), EXITED_OK, []);
|
||
assert.equal(shrunk.verdict, "valid");
|
||
assert.equal(value(shrunk.score), 0.203125);
|
||
assert.equal(value(shrunk.metrics["raw-score"]), 0.234375);
|
||
assert.equal(value(shrunk.metrics["raw-violation"]), 0.015625);
|
||
assert.equal(value(shrunk.metrics["legalization-deduction"]), 0.03125);
|
||
const extra = shrunk.extra as { legalization: { shrunkCircles: number[]; rawValid: boolean; deduction: Measured } };
|
||
assert.deepEqual(extra.legalization.shrunkCircles, [0, 1]);
|
||
assert.equal(extra.legalization.rawValid, false);
|
||
assert.equal(value(extra.legalization.deduction), 0.03125);
|
||
|
||
// 0 号圆心挪到 x = 1/256，出左边 1/128 − 1/256 = 1/256；修后半径 1/256，和 51/256 = 0.19921875。
|
||
const edge = loose();
|
||
edge.centers[0] = [1 / 256, 0.5];
|
||
const edged = await judgeRun(t, "edge", runDirWith(t, edge), EXITED_OK, []);
|
||
assert.equal(value(edged.score), 0.19921875);
|
||
assert.equal(value(edged.metrics["raw-violation"]), 1 / 256);
|
||
assert.equal(value(edged.metrics["legalization-deduction"]), 0.203125 - 0.19921875);
|
||
|
||
// 参考项目真实运行 14-real-1 的产物：重叠约 2.78e-17，修后 1.7602769492805097（参考项目设计-15 7.1 节记的数）。
|
||
const real = await judgeRun(t, "real", runDirWith(t, readFileSync(FIXTURE, "utf8")), EXITED_OK, []);
|
||
assert.equal(real.verdict, "valid");
|
||
assert.equal(value(real.score), 1.7602769492805097);
|
||
const raw = value(real.metrics["raw-score"]);
|
||
assert.equal(raw, 1.76027694928051);
|
||
assert.ok(value(real.score) <= raw);
|
||
assert.equal(value(real.metrics["legalization-deduction"]), raw - value(real.score));
|
||
assert.ok(Math.abs(value(real.metrics["raw-violation"]) - 2.78e-17) < 0.01e-17);
|
||
});
|
||
|
||
test("判定表 5：缺了整次作废的假设专用指标测不到，不算数；缺了只记缺失的照常算数", async (t) => {
|
||
const spec = (id: string, missing: HypothesisMetric["missing"]): HypothesisMetric => ({
|
||
id,
|
||
meaning: "测试用",
|
||
unit: "无",
|
||
direction: "maximize",
|
||
howMeasured: "测试用",
|
||
missing,
|
||
});
|
||
const runDir = runDirWith(t, loose());
|
||
writeFileSync(
|
||
join(runDir, "metrics.json"),
|
||
JSON.stringify({ spread: 0.5, gap: null, "answer-path": "p1", "safeguard-triggers": { "shrink-repair": 2 } }),
|
||
);
|
||
const kept = await judgeRun(t, "keep", runDir, EXITED_OK, [spec("spread", "invalidateRun"), spec("gap", "recordMissing")]);
|
||
assert.equal(kept.verdict, "valid");
|
||
assert.equal(value(kept.metrics.spread), 0.5);
|
||
assert.match(reason(kept.metrics.gap), /null/);
|
||
assert.deepEqual(kept.answerPath, { status: "read", value: "p1" });
|
||
assert.deepEqual(kept.safeguardTriggers, { status: "read", counts: { "shrink-repair": 2 } });
|
||
|
||
const voided = await judgeRun(t, "void", runDir, EXITED_OK, [spec("gap", "invalidateRun")]);
|
||
assert.equal(voided.verdict, "invalid");
|
||
assert.match(voided.verdictReason, /缺了整次作废.*gap/);
|
||
// 成绩照样算出来记下，只是这次运行不算数。
|
||
assert.equal(value(voided.score), 0.203125);
|
||
});
|
||
|
||
test("假设专用指标用冻结后的全名\"候选编号.短名\"当键，读得到值（任务书 11 第一部分）", async (t) => {
|
||
const full: HypothesisMetric = { id: "r2-c1.restart-gain", meaning: "测试用", unit: "无", direction: "maximize", howMeasured: "测试用", missing: "invalidateRun" };
|
||
const runDir = runDirWith(t, loose());
|
||
writeFileSync(join(runDir, "metrics.json"), JSON.stringify({ "r2-c1.restart-gain": 0.125, "restart-gain": 9, "answer-path": "main", "safeguard-triggers": {} }));
|
||
const result = await judgeRun(t, "fullname", runDir, EXITED_OK, [full]);
|
||
assert.equal(result.verdict, "valid");
|
||
assert.deepEqual(Object.keys(result.metrics).filter((k) => k.includes("restart")), ["r2-c1.restart-gain"]);
|
||
assert.equal(value(result.metrics["r2-c1.restart-gain"]), 0.125);
|
||
});
|
||
|
||
test("答案路径、保障动手次数：读不到时记读不到及原因，不作废运行", async (t) => {
|
||
const missing = await judgeRun(t, "nometrics", runDirWith(t, loose()), EXITED_OK, []);
|
||
assert.equal(missing.verdict, "valid");
|
||
assert.deepEqual(missing.answerPath, { status: "unreadable", reason: "metrics.json 不存在" });
|
||
|
||
const runDir = runDirWith(t, loose());
|
||
writeFileSync(join(runDir, "metrics.json"), JSON.stringify({ "answer-path": 3, "safeguard-triggers": { a: -1 } }));
|
||
const wrong = await judgeRun(t, "wrongkeys", runDir, EXITED_OK, []);
|
||
assert.equal(wrong.verdict, "valid");
|
||
assert.equal(wrong.answerPath.status, "unreadable");
|
||
assert.equal(wrong.safeguardTriggers.status, "unreadable");
|
||
});
|
||
|
||
test("校验器自己没正常跑完、判定认不出、修后分数高于原分数：报错停下，不当成候选的失败", (t) => {
|
||
const runDir = runDirWith(t, loose());
|
||
const judgeDir = tempDir(t, "judge");
|
||
const un: Measured = { status: "unmeasured", reason: "x" };
|
||
const timeout: ProcessResult = { ending: "timeout", cpuSeconds: un, wallSeconds: un, peakMemoryBytes: un };
|
||
assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, timeout, []), /校验器没有正常跑完（timeout）/);
|
||
// 退出码 2 是用法或读取出错，这时校验器不写判定文件。
|
||
assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, { ...EXITED_OK, exitCode: 2 }, []), /校验器没有正常跑完（退出码 2）/);
|
||
assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, EXITED_OK, []), /读不出 .*verdict\.json/);
|
||
|
||
const verdict = {
|
||
valid: true,
|
||
computed_sum: 0.3,
|
||
reasons: [],
|
||
diagnostics: { min_pairwise_clearance: 0 },
|
||
legalization: { raw_valid: false, raw_score: 0.2, raw_violation: 0.1, applied: true, repaired: true, shrunk_circles: [0], reasons: [] },
|
||
};
|
||
writeFileSync(join(judgeDir, "verdict.json"), JSON.stringify(verdict));
|
||
assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, EXITED_OK, []), /合法化后分数 0\.3 高于原分数 0\.2/);
|
||
// 退出码说不合法、判定说合法：对不上。
|
||
assert.throws(() => judge(task, runDir, judgeDir, EXITED_OK, { ...EXITED_OK, exitCode: 1 }, []), /对不上/);
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 轻量检查
|
||
// ---------------------------------------------------------------------------
|
||
|
||
async function check(t: TestContext, suffix: string, files: Record<string, string>) {
|
||
const candidate = tempDir(t, "candidate");
|
||
for (const [name, text] of Object.entries(files)) writeFileSync(join(candidate, name), text);
|
||
const checkDir = tempDir(t, "check");
|
||
const definition = lightCheck(task, candidate, checkDir, digest, 5);
|
||
return readLightCheck(task, checkDir, await run(t, suffix, definition, checkDir));
|
||
}
|
||
|
||
test("轻量检查：语法错的、没有入口函数的不通过，正常的通过", async (t) => {
|
||
const syntax = await check(t, "check-syntax", { "initial.py": "def run_packing(:\n pass\n" });
|
||
assert.equal(syntax.passed, false);
|
||
assert.match(syntax.passed ? "" : syntax.reasons.join(), /initial\.py 解析失败/);
|
||
|
||
const noEntry = await check(t, "check-noentry", { "initial.py": "def solve():\n return [], [], 0.0\n" });
|
||
assert.equal(noEntry.passed, false);
|
||
assert.match(noEntry.passed ? "" : noEntry.reasons.join(), /run_packing\(\) 应当恰好有一个定义，实际有 0 个/);
|
||
|
||
const baseline = tempDir(t, "baseline");
|
||
cpSync(task.baselineDir, baseline, { recursive: true });
|
||
const checkDir = tempDir(t, "check");
|
||
const ok = readLightCheck(task, checkDir, await run(t, "check-ok", lightCheck(task, baseline, checkDir, digest, 5), checkDir));
|
||
assert.deepEqual(ok, { passed: true });
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 从头到尾
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/** 基线 initial.py 准备运行目录 → 正式运行 → 跑校验器 → 判分。 */
|
||
async function endToEnd(t: TestContext, suffix: string): Promise<JudgedResult> {
|
||
const runDir = tempDir(t, "e2e");
|
||
prepareRun(task, task.baselineDir, runDir, []);
|
||
assert.equal(readFileSync(join(runDir, "metrics-spec.json"), "utf8"), "[]\n");
|
||
assert.ok(existsSync(join(runDir, "harness.py")) && existsSync(join(runDir, "initial.py")));
|
||
const formal = await run(t, `e2e-${suffix}`, formalRun(task, digest, "inforsi-test-seed"), runDir);
|
||
assert.equal(formal.ending, "exited", JSON.stringify(formal));
|
||
return judgeRun(t, `e2e-${suffix}-judge`, runDir, formal, []);
|
||
}
|
||
|
||
test("从头到尾：基线跑两次都判为算数，分数逐位相同", async (t) => {
|
||
const first = await endToEnd(t, "1");
|
||
const second = await endToEnd(t, "2");
|
||
for (const result of [first, second]) {
|
||
assert.equal(result.verdict, "valid", result.verdictReason);
|
||
assert.equal(result.score.status, "measured");
|
||
}
|
||
assert.ok(Object.is(value(first.score), value(second.score)), `${value(first.score)} 与 ${value(second.score)}`);
|
||
for (const [label, result] of [["第一次", first], ["第二次", second]] as const) {
|
||
t.diagnostic(
|
||
`${label}：verdict=${result.verdict}（${result.verdictReason}） score=${value(result.score)} ` +
|
||
`cpu=${JSON.stringify(result.metrics["cpu-seconds"])} wall=${JSON.stringify(result.metrics["wall-seconds"])} ` +
|
||
`mem=${JSON.stringify(result.metrics["peak-memory-bytes"])} deduction=${JSON.stringify(result.metrics["legalization-deduction"])}`,
|
||
);
|
||
}
|
||
});
|
||
|
||
test("候选目录里遗留的旧 metrics.json 不会被当成这次的产物：整次作废的指标这次没写，判为不算数", async (t) => {
|
||
const candidate = tempDir(t, "candidate");
|
||
cpSync(task.baselineDir, candidate, { recursive: true });
|
||
// 基线代码不写 metrics.json；这份是以前某次运行留下的。
|
||
writeFileSync(join(candidate, "metrics.json"), JSON.stringify({ spread: 0.5, "answer-path": "main", "safeguard-triggers": {} }));
|
||
const spread: HypothesisMetric = {
|
||
id: "spread",
|
||
meaning: "测试用",
|
||
unit: "无",
|
||
direction: "maximize",
|
||
howMeasured: "测试用",
|
||
missing: "invalidateRun",
|
||
};
|
||
const runDir = tempDir(t, "stale");
|
||
prepareRun(task, candidate, runDir, [spread]);
|
||
const formal = await run(t, "stale", formalRun(task, digest, "inforsi-test-seed"), runDir);
|
||
assert.equal(formal.ending, "exited", JSON.stringify(formal));
|
||
const result = await judgeRun(t, "stale-judge", runDir, formal, [spread]);
|
||
t.diagnostic(`verdict=${result.verdict}（${result.verdictReason}） spread=${JSON.stringify(result.metrics.spread)}`);
|
||
assert.equal(result.metrics.spread?.status, "unmeasured");
|
||
assert.equal(result.verdict, "invalid");
|
||
assert.match(result.verdictReason, /缺了整次作废.*spread/);
|
||
});
|