feat: 任务书 05,runner 与 tasks 两个模块

runner 用 Docker 跑一条命令并交回结局和资源用量;tasks 读 task.json、准备运行目录、给出正式运行/校验器/轻量检查的运行定义,并从原始产物独立判分。任务目录从参考项目搬到 tasks/circle_packing/,新增 task.json;vocab 的 VERDICTS 删掉"判不了",ProcessResult 的 cancelled 改为"容器从外面被杀了"。

全新上下文审阅后的两处修复:计量 shell 以容器内 root 运行,命令经 setpriv 降权,结果写在容器自己的文件层、结束后 docker cp 取出,并加 no-new-privileges,候选无法伪造计量结果把超时记成正常退出;harness 开跑前删掉旧的 solution.json、metrics.json,旧产物不会被当成本次的值。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-02 13:55:47 -04:00
parent 6005fb8ec2
commit cf6bc4c844
25 changed files with 3205 additions and 6 deletions
+342
View File
@@ -0,0 +1,342 @@
import { copyFileSync, cpSync, existsSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { Type, type Static } from "typebox";
import { Value } from "typebox/value";
import type {
HypothesisMetric,
Json,
JudgedResult,
Measured,
MetricDefinition,
ProcessResult,
RunDefinition,
} from "../vocab/index.ts";
import type { TaskFacts } from "./facts.ts";
import { measured, readMetricsFile, unmeasured } from "./metrics-file.ts";
import type { LightCheckOutcome, Task, TaskImplementation } from "./types.ts";
// 填圆:在单位正方形里放 26 个圆,最大化半径之和。任务说明见 tasks/circle_packing/TASK.md。
// 凡是要执行任务代码的事(正式运行、校验器、轻量检查)都只交出运行定义,由调用方交给 runner 在任务镜像里跑;
// 这里只读写文件、解读产物(docs/结构.md 第二部分"runner 只管怎么跑,tasks 只管怎么判")。
const CIRCLES = 26;
const ENTRY = "initial.py";
const HARNESS = "harness.py";
const VALIDATOR = "validate.py";
const CHECK_SCRIPT = "check-candidate.py";
const METRICS_SPEC = "metrics-spec.json";
const METRICS = "metrics.json";
const SOLUTION = "solution.json";
const VERDICT = "verdict.json";
const CHECK_RESULT = "check.json";
/** 轻量检查目录里放候选代码的子目录;检查脚本放在它的上一层,免得扫到自己。 */
const CHECK_CANDIDATE = "candidate";
/** 运行目录挂进容器的位置。 */
const WORKDIR = "/work";
/** 校验器的时限。26 个圆两两比较加合法化,实测一秒以内;它超时说明宿主的工具出了问题。 */
const VALIDATOR_SECONDS = 60;
/** 合法化前先按容差 0 判原产物(方法.md 第八部分"合法化"第 1 条)。 */
const TOLERANCE = "0";
// 常规辅助指标的编号(方法.md 第五部分"指标")。前三个来自 runner 的资源用量,后四个来自校验器。
const WALL = "wall-seconds";
const CPU = "cpu-seconds";
const MEMORY = "peak-memory-bytes";
const CLEARANCE = "min-pairwise-clearance";
const RAW_SCORE = "raw-score";
const RAW_VIOLATION = "raw-violation";
const DEDUCTION = "legalization-deduction";
const FROM_VALIDATOR = [CLEARANCE, RAW_SCORE, RAW_VIOLATION, DEDUCTION];
const SIDE = "单位正方形的边长";
const PER_RUN = "每次运行结束后由宿主读一次,直接取值";
function auxiliary(
id: string,
meaning: string,
unit: string,
direction: "maximize" | "minimize",
howMeasured: string,
): MetricDefinition {
return { id, layer: "auxiliary", meaning, unit, direction, howMeasured, missing: "recordMissing" };
}
function metricDefinitions(facts: TaskFacts): MetricDefinition[] {
const validator = `宿主的校验器从 ${SOLUTION} 重算,${PER_RUN}`;
return [
{
id: facts.finalMetric.id,
layer: "final",
meaning:
"合法摆法的半径之和,由宿主的独立校验器从实际产物重算;原产物差一点不合法时,先按 TASK.md 的只缩不移规则修,再按修后的半径算",
unit: SIDE,
direction: facts.finalMetric.direction,
howMeasured: validator,
missing: "invalidateRun",
},
auxiliary(WALL, "正式运行的墙钟时间", "秒", "minimize", `容器里的计量程序记录,${PER_RUN}`),
auxiliary(CPU, "正式运行用掉的 CPU 秒(容器 cgroup 的 usage_usec)", "秒", "minimize", `容器里的计量程序记录,${PER_RUN}`),
auxiliary(MEMORY, "正式运行的峰值内存(容器 cgroup 的 memory.peak)", "字节", "minimize", `容器里的计量程序记录,${PER_RUN}`),
auxiliary(CLEARANCE, "所有圆对中「圆心距减两半径之和」的最小值,大于等于 0 表示不重叠;修过的取修后的摆法", SIDE, "maximize", validator),
auxiliary(RAW_SCORE, "合法化之前原产物的半径之和", SIDE, "maximize", validator),
auxiliary(RAW_VIOLATION, "原产物的最大违反量:两圆重叠的量和圆出边的量里最大的一个,0 表示没有违反", SIDE, "minimize", validator),
auxiliary(DEDUCTION, "只缩不移合法化扣掉的分:原分数减最终成绩,原产物合法时是 0", SIDE, "minimize", validator),
];
}
/** 正式运行、校验器、轻量检查共用的运行定义。数学库的线程数和核数一致,否则会开多个线程抢核。 */
function definition(task: Task, imageDigest: string, command: string[], wallSeconds: number, env: Record<string, string>): RunDefinition {
const cores = String(task.facts.run.cores);
return {
command,
workdir: WORKDIR,
image: { digest: imageDigest, runtime: task.facts.run.runtime },
wallSeconds,
cores: task.facts.run.cores,
inheritEnv: false,
env: { OMP_NUM_THREADS: cores, OPENBLAS_NUM_THREADS: cores, MKL_NUM_THREADS: cores, ...env },
};
}
// 解释器不加 -I:-I 会忽略全部 PYTHON* 环境变量,连同 runner 设的 PYTHON_CPU_COUNT,
// 于是 os.cpu_count() 报回宿主核数(本机实测 16)。容器里的环境变量全部由运行定义给出,用不着 -I。
// -B:不写 __pycache__,不往运行目录里添文件。
function prepareRun(task: Task, candidateDir: string, runDir: string, hypothesisMetrics: HypothesisMetric[]): void {
cpSync(candidateDir, runDir, { recursive: true });
// harness 和指标说明以宿主的为准,候选目录里有同名文件也被盖掉(这两个名字在 task.json 的不许改名单里)。
copyFileSync(join(task.dir, HARNESS), join(runDir, HARNESS));
writeFileSync(join(runDir, METRICS_SPEC), `${JSON.stringify(hypothesisMetrics, null, 2)}\n`);
}
function formalRun(task: Task, imageDigest: string, seed: string): RunDefinition {
const command = ["python", "-B", `${WORKDIR}/${HARNESS}`, "--program", ENTRY, "--output", SOLUTION];
return definition(task, imageDigest, command, task.facts.run.wallSeconds, { INFORSI_SEED: seed });
}
function lightCheck(task: Task, candidateDir: string, checkDir: string, imageDigest: string, seconds: number): RunDefinition {
cpSync(candidateDir, join(checkDir, CHECK_CANDIDATE), { recursive: true });
copyFileSync(join(task.dir, CHECK_SCRIPT), join(checkDir, CHECK_SCRIPT));
const command = ["python", "-B", CHECK_SCRIPT, CHECK_CANDIDATE, ENTRY, CHECK_RESULT];
return definition(task, imageDigest, command, seconds, {});
}
/** 检查脚本写的结果文件。valid 与 reasons 是否为空必须一致。 */
const CheckResult = Type.Object({ valid: Type.Boolean(), reasons: Type.Array(Type.String()) });
/** 宿主的工具(检查脚本、校验器)没正常跑完时报错停下:这不是候选的失败(任务书 05 第三部分)。 */
function requireExit(what: string, result: ProcessResult, codes: number[]): number {
if (result.ending === "exited" && codes.includes(result.exitCode)) return result.exitCode;
const how = result.ending === "notStarted" ? `没启动:${result.reason}` : result.ending === "exited" ? `退出码 ${result.exitCode}` : result.ending;
throw new Error(`${what}没有正常跑完(${how}),这是宿主一侧的工具出了问题,不能当成候选的失败`);
}
function readJson(path: string): unknown {
try {
return JSON.parse(readFileSync(path, "utf8"));
} catch (error) {
throw new Error(`读不出 ${path}:${(error as Error).message}`, { cause: error });
}
}
function readLightCheck(checkDir: string, result: ProcessResult): LightCheckOutcome {
requireExit("轻量检查脚本", result, [0]);
const path = join(checkDir, CHECK_RESULT);
const check = readJson(path);
if (!Value.Check(CheckResult, check) || check.valid !== (check.reasons.length === 0)) {
throw new Error(`${path} 的内容认不出:${JSON.stringify(check)}`);
}
return check.valid ? { passed: true } : { passed: false, reasons: check.reasons };
}
/** 正式运行没正常结束的原因;正常退出(退出码 0)时为 null。 */
function processFailure(task: Task, formal: Exclude<ProcessResult, { ending: "notStarted" }>): string | null {
if (formal.ending === "timeout") return `超过单次时限 ${task.facts.run.wallSeconds} 秒,被结束`;
if (formal.ending === "cancelled") return "容器从外面被结束(关机、手动 docker kill 等),没有留下计量结果";
if (formal.exitCode !== 0) return `正式运行以退出码 ${formal.exitCode} 结束`;
return null;
}
function started(formal: ProcessResult): Exclude<ProcessResult, { ending: "notStarted" }> {
if (formal.ending === "notStarted") throw new Error(`没启动的运行不判分:${formal.reason}`);
return formal;
}
/**
* 拼出 JudgedResult:资源用量来自正式运行,假设专用指标和两个保留键来自候选写的 metrics.json。
* 有"缺了整次作废"的假设专用指标测不到时,原本算数的也改判不算数(方法.md 第五部分"指标")。
*/
function assemble(
runDir: string,
formal: Exclude<ProcessResult, { ending: "notStarted" }>,
hypothesisMetrics: HypothesisMetric[],
verdict: { valid: boolean; reason: string },
score: Measured,
fromValidator: Record<string, Measured>,
extra: Json,
): JudgedResult {
const file = readMetricsFile(join(runDir, METRICS), METRICS, hypothesisMetrics);
const voided = hypothesisMetrics.flatMap((m) => {
const value = file.hypothesis[m.id]!;
return m.missing === "invalidateRun" && value.status === "unmeasured" ? [`${m.id}(${value.reason})`] : [];
});
const result = verdict.valid && voided.length > 0
? { valid: false, reason: `登记为"缺了整次作废"的假设专用指标测不到:${voided.join(";")}` }
: verdict;
return {
verdict: result.valid ? "valid" : "invalid",
verdictReason: result.reason,
score,
metrics: {
[WALL]: formal.wallSeconds,
[CPU]: formal.cpuSeconds,
[MEMORY]: formal.peakMemoryBytes,
...fromValidator,
...file.hypothesis,
},
answerPath: file.answerPath,
safeguardTriggers: file.safeguardTriggers,
extra,
};
}
function prepareJudging(
task: Task,
runDir: string,
judgeDir: string,
imageDigest: string,
formal: ProcessResult,
hypothesisMetrics: HypothesisMetric[],
): ReturnType<TaskImplementation["prepareJudging"]> {
const run = started(formal);
const solution = join(runDir, SOLUTION);
const failure = processFailure(task, run) ?? (existsSync(solution) ? null : `没有产物:运行目录里没有 ${SOLUTION}`);
if (failure !== null) {
const notValidated = Object.fromEntries(FROM_VALIDATOR.map((id) => [id, unmeasured(`没有跑校验器:${failure}`)]));
const result = assemble(runDir, run, hypothesisMetrics, { valid: false, reason: failure }, unmeasured(failure), notValidated, {
legalization: null,
});
return { status: "judged", result };
}
copyFileSync(solution, join(judgeDir, SOLUTION));
copyFileSync(join(task.dir, VALIDATOR), join(judgeDir, VALIDATOR));
const command = [
"python",
"-B",
VALIDATOR,
"--circles",
String(CIRCLES),
"--solution",
SOLUTION,
"--atol",
TOLERANCE,
"--legalize",
"--verdict",
VERDICT,
];
return { status: "validate", definition: definition(task, imageDigest, command, VALIDATOR_SECONDS, {}) };
}
const NumberOrNull = Type.Union([Type.Number(), Type.Null()]);
/** validate.py --legalize 写的判定里,这里要读的部分;其余键(evidence、legal_radii 等)不读。 */
const ValidatorVerdict = Type.Object({
valid: Type.Boolean(),
computed_sum: NumberOrNull,
reasons: Type.Array(Type.String()),
diagnostics: Type.Object({ min_pairwise_clearance: NumberOrNull }),
legalization: Type.Object({
raw_valid: Type.Boolean(),
raw_score: NumberOrNull,
raw_violation: NumberOrNull,
applied: Type.Boolean(),
repaired: Type.Boolean(),
shrunk_circles: Type.Array(Type.Integer()),
reasons: Type.Array(Type.String()),
}),
});
type ValidatorVerdict = Static<typeof ValidatorVerdict>;
/** 校验器在形状不对、含非有限数或负半径时提前返回,几何量没算。 */
const NOT_COMPUTED = "原产物形状不对、含非有限数或负半径,校验器没算这个量";
function orUnmeasured(value: number | null, reason: string): Measured {
return value === null ? unmeasured(reason) : measured(value);
}
function readVerdict(judgeDir: string, exitCode: number): ValidatorVerdict {
const path = join(judgeDir, VERDICT);
const verdict = readJson(path);
if (!Value.Check(ValidatorVerdict, verdict)) throw new Error(`${path} 的内容认不出:${JSON.stringify(verdict)}`);
// 退出码 0 = 合法、1 = 不合法;合法化之后的结论就是最终判定。对不上说明跑的不是这份校验器。
const consistent =
verdict.valid === (exitCode === 0) &&
verdict.legalization.repaired === verdict.valid &&
(!verdict.valid || verdict.computed_sum !== null);
if (!consistent) throw new Error(`${path} 与退出码 ${exitCode} 或自身前后对不上:${JSON.stringify(verdict)}`);
return verdict;
}
function judge(
_task: Task,
runDir: string,
judgeDir: string,
formal: ProcessResult,
validator: ProcessResult,
hypothesisMetrics: HypothesisMetric[],
): JudgedResult {
const run = started(formal);
const verdict = readVerdict(judgeDir, requireExit("校验器", validator, [0, 1]));
const legal = verdict.legalization;
let deduction: Measured;
let score: Measured;
if (verdict.valid) {
const final = verdict.computed_sum!;
score = measured(final);
// legal.repaired 为真时 raw_score 一定有值:原产物能算出分数,才可能合法或被修好。
const raw = legal.raw_score!;
// 只缩半径只会降分;修后反而更高,说明修法写错了(方法.md 第八部分"合法化"第 3 条)。
if (final > raw) throw new Error(`合法化后分数 ${final} 高于原分数 ${raw}:只缩半径只会降分,修法出错了`);
deduction = measured(raw - final);
} else {
const reason = `不算数:${verdict.reasons.concat(legal.reasons).join(";")}`;
score = unmeasured(reason);
deduction = unmeasured(`修不了,没有修后分数:${legal.reasons.join(";")}`);
}
const rawScore = orUnmeasured(legal.raw_score, NOT_COMPUTED);
const rawViolation = orUnmeasured(legal.raw_violation, NOT_COMPUTED);
const fromValidator = {
[CLEARANCE]: orUnmeasured(verdict.diagnostics.min_pairwise_clearance, NOT_COMPUTED),
[RAW_SCORE]: rawScore,
[RAW_VIOLATION]: rawViolation,
[DEDUCTION]: deduction,
};
const reason = verdict.valid
? legal.raw_valid
? "校验器按容差 0 判原产物合法"
: `原产物不合法,按只缩不移规则修好后合法(缩了 ${legal.shrunk_circles.length} 个圆)`
: `校验器判不合法:${verdict.reasons.concat(legal.reasons).join(";")}`;
const extra = {
legalization: {
rawValid: legal.raw_valid,
applied: legal.applied,
repaired: legal.repaired,
shrunkCircles: legal.shrunk_circles,
reasons: legal.reasons,
rawScore,
rawViolation,
deduction,
},
};
return assemble(runDir, run, hypothesisMetrics, { valid: verdict.valid, reason }, score, fromValidator, extra);
}
export const circlePacking: TaskImplementation = {
metricDefinitions,
prepareRun,
formalRun,
lightCheck,
readLightCheck,
prepareJudging,
judge,
};
+90
View File
@@ -0,0 +1,90 @@
import { Type, type Static } from "typebox";
import { Settings } from "typebox/system";
import { Value } from "typebox/value";
import { Direction, Id } from "../vocab/index.ts";
// task.json 的字段定义与检查。task.json 是用户提供的东西,属于系统边界(CLAUDE.md"罕见情况怎么办"),
// 所以每个字段必填、不认识的字段报错,一次列出全部错误,每条写明字段路径。可以没有的值写 null。
/** 任务种类。每一种在 task.ts 的对照表里对应一份实现;nanochat、AI4AI 以后按同样的形状加。 */
export const TASK_KINDS = ["circle_packing"] as const;
export type TaskKind = (typeof TASK_KINDS)[number];
const CLOSED = { additionalProperties: false } as const;
export const TaskFacts = Type.Object(
{
kind: Type.Enum(TASK_KINDS, { type: "string", description: "任务种类,决定用哪一份实现" }),
finalMetric: Type.Object(
{
id: Id,
direction: Direction,
},
{ ...CLOSED, description: "最终指标的编号和方向(方法.md 第十部分第 1 条)" },
),
epsilon: Type.Number({ exclusiveMinimum: 0, description: "精度 ε(方法.md 6.11)" }),
knownBest: Type.Union([Type.Number(), Type.Null()], {
description: "已知最好成绩;没有就写 null(方法.md 6.11)",
}),
run: Type.Object(
{
wallSeconds: Type.Number({ exclusiveMinimum: 0, description: "单次正式运行的墙钟时限(秒)" }),
cores: Type.Integer({ minimum: 1, description: "单次正式运行的核数" }),
runtime: Type.String({ minLength: 1, description: "Docker 运行时,必须写明,例如 runc" }),
},
{ ...CLOSED, description: "同批候选共用的比较条件(方法.md 第八部分)" },
),
protectedFiles: Type.Array(Type.String({ minLength: 1 }), {
description: "不许改的文件,路径相对候选代码目录(方法.md 第十部分第 2 条)",
}),
},
CLOSED,
);
export type TaskFacts = Static<typeof TaskFacts>;
type SchemaError = ReturnType<typeof Value.Errors>[number];
const TYPES: Readonly<Record<string, string>> = { number: "数字", integer: "整数", string: "字符串", object: "对象", array: "数组" };
/** 本文件的字段用得到的几种错误,写成中文;其余照 typebox 的原文。 */
const REASONS: Readonly<Record<string, (p: Record<string, unknown>) => string>> = {
type: (p) => `应为${TYPES[String(p.type)] ?? String(p.type)}`,
enum: (p) => `只能是 ${(p.allowedValues as unknown[]).map((v) => JSON.stringify(v)).join("、")} 之一`,
exclusiveMinimum: (p) => `必须大于 ${String(p.limit)}`,
minimum: (p) => `不能小于 ${String(p.limit)}`,
minLength: () => "不能为空",
pattern: (p) => `格式不对,应符合 ${String(p.pattern)}`,
};
function errorLines(errors: SchemaError[]): string[] {
const lines: string[] = [];
for (const e of errors) {
const p = e.params as Record<string, unknown>;
const at = e.instancePath || "/";
// 唯一的几选一是 knownBest(数字或 null);各分支自己的错误不再逐条列。
if (e.schemaPath.includes("/anyOf/")) continue;
if (e.keyword === "anyOf") lines.push(`${at}:应为数字,没有就写 null`);
else if (e.keyword === "required") {
for (const name of p.requiredProperties as string[]) lines.push(`${e.instancePath}/${name}:缺少这一项(每个字段都必填)`);
} else if (e.keyword === "additionalProperties") {
for (const name of p.additionalProperties as string[]) lines.push(`${e.instancePath}/${name}:不认识的字段`);
} else lines.push(`${at}:${REASONS[e.keyword]?.(p) ?? e.message}`);
}
return lines;
}
/** 检查一份已经解析好的 task.json;合格就原样返回,不合格就报错,列出全部错误。 */
export function checkTaskFacts(value: unknown): TaskFacts {
if (Value.Check(TaskFacts, value)) return value;
// typebox 默认最多收 8 条错误,不放开就会漏报;放开只在这一次调用里。
const cap = Settings.Get().maxErrors;
Settings.Set({ maxErrors: Infinity });
let errors: SchemaError[];
try {
errors = [...Value.Errors(TaskFacts, value)];
} finally {
Settings.Set({ maxErrors: cap });
}
const lines = errorLines(errors);
throw new Error(`task.json 不合格,共 ${lines.length} 处:\n${lines.join("\n")}`);
}
+15
View File
@@ -0,0 +1,15 @@
// tasks:任务目录里的事实和文件、怎么准备一次运行、怎么从原始产物独立判分(docs/结构.md 第二部分)。
// 不起子进程:要执行任务代码的事都只交出运行定义,由调用方交给 runner 在任务镜像里跑。
export { checkTaskFacts, TASK_KINDS, TaskFacts, type TaskKind } from "./facts.ts";
export {
formalRun,
judge,
lightCheck,
loadTask,
metricDefinitions,
prepareJudging,
prepareRun,
protectedChanges,
readLightCheck,
} from "./task.ts";
export type { JudgingStep, LightCheckOutcome, Task } from "./types.ts";
+88
View File
@@ -0,0 +1,88 @@
import { existsSync, readFileSync } from "node:fs";
import {
ANSWER_PATH_KEY,
SAFEGUARD_TRIGGERS_KEY,
type AnswerPath,
type HypothesisMetric,
type Measured,
type SafeguardTriggers,
} from "../vocab/index.ts";
// 读候选写的指标文件:假设专用指标的取值,加两个保留键(方法.md 第五部分"指标""答案路径")。
// 这个文件是候选的产物,读不到、写错了都记成明确的状态并附原因,不补 0、不补空对象,也不报错停下。
export interface MetricsFileReading {
/** 每条登记的假设专用指标一项,键是指标编号。 */
hypothesis: Record<string, Measured>;
answerPath: AnswerPath;
safeguardTriggers: SafeguardTriggers;
}
export function unmeasured(reason: string): Measured {
return { status: "unmeasured", reason };
}
export function measured(value: number): Measured {
return { status: "measured", value };
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
/** 整个文件都读不到时,每一项都记同一个原因。 */
function allUnreadable(reason: string, metrics: HypothesisMetric[]): MetricsFileReading {
return {
hypothesis: Object.fromEntries(metrics.map((m) => [m.id, unmeasured(reason)])),
answerPath: { status: "unreadable", reason },
safeguardTriggers: { status: "unreadable", reason },
};
}
function readMetric(values: Record<string, unknown>, id: string, name: string): Measured {
const value = values[id];
if (value === undefined) return unmeasured(`${name} 里没有指标 ${id}`);
if (value === null) return unmeasured(`候选写的是 null:这一次没测到 ${id}`);
// JSON.parse 会把 1e999 读成 Infinity,所以要查有限。
if (typeof value !== "number" || !Number.isFinite(value)) {
return unmeasured(`${name} 里指标 ${id} 的值不是有限数:${JSON.stringify(value)}`);
}
return measured(value);
}
function readAnswerPath(values: Record<string, unknown>, name: string): AnswerPath {
const value = values[ANSWER_PATH_KEY];
if (value === undefined) return { status: "unreadable", reason: `${name} 里没写 ${ANSWER_PATH_KEY}` };
if (typeof value !== "string") {
return { status: "unreadable", reason: `${name} 里 ${ANSWER_PATH_KEY} 不是字符串:${JSON.stringify(value)}` };
}
return { status: "read", value };
}
function readTriggers(values: Record<string, unknown>, name: string): SafeguardTriggers {
const value = values[SAFEGUARD_TRIGGERS_KEY];
if (value === undefined) return { status: "unreadable", reason: `${name} 里没写 ${SAFEGUARD_TRIGGERS_KEY}` };
const counts = isRecord(value) ? Object.entries(value) : null;
if (counts === null || counts.some(([, n]) => !Number.isInteger(n) || (n as number) < 0)) {
const reason = `${name} 里 ${SAFEGUARD_TRIGGERS_KEY} 应为"保障短名 → 非负整数"的对象,实际是 ${JSON.stringify(value)}`;
return { status: "unreadable", reason };
}
return { status: "read", counts: Object.fromEntries(counts) as Record<string, number> };
}
/** 读指标文件。metrics 是这次登记的假设专用指标,只按它们的编号取值,多写的键不读。 */
export function readMetricsFile(path: string, name: string, metrics: HypothesisMetric[]): MetricsFileReading {
if (!existsSync(path)) return allUnreadable(`${name} 不存在`, metrics);
let values: unknown;
try {
values = JSON.parse(readFileSync(path, "utf8"));
} catch (error) {
return allUnreadable(`${name} 不是合法的 JSON:${(error as Error).message}`, metrics);
}
if (!isRecord(values)) return allUnreadable(`${name} 应为 JSON 对象(指标编号 → 值)`, metrics);
return {
hypothesis: Object.fromEntries(metrics.map((m) => [m.id, readMetric(values, m.id, name)])),
answerPath: readAnswerPath(values, name),
safeguardTriggers: readTriggers(values, name),
};
}
+107
View File
@@ -0,0 +1,107 @@
import { readFileSync } from "node:fs";
import { join, resolve } from "node:path";
import type { HypothesisMetric, JudgedResult, MetricDefinition, ProcessResult, RunDefinition } from "../vocab/index.ts";
import { circlePacking } from "./circle-packing.ts";
import { checkTaskFacts, type TaskKind } from "./facts.ts";
import type { JudgingStep, LightCheckOutcome, Task, TaskImplementation } from "./types.ts";
// tasks 对外的函数。任务目录的约定:task.json(任务事实)、TASK.md(给实验员的说明)、
// Dockerfile 与依赖清单(建镜像)、baseline/(基线代码),以及各任务自己的 Python 工具。
// 每个函数按 task.facts.kind 从对照表里取出那种任务的实现来做。
/** 任务种类 → 实现。加一种任务就在这里加一行。 */
const IMPLEMENTATIONS: Readonly<Record<TaskKind, TaskImplementation>> = {
circle_packing: circlePacking,
};
function implementation(task: Task): TaskImplementation {
return IMPLEMENTATIONS[task.facts.kind];
}
/** 读一个任务目录:严格检查 task.json,同时交回 Dockerfile 所在目录、基线目录和 TASK.md 全文。 */
export function loadTask(taskDir: string): Task {
const dir = resolve(taskDir);
const path = join(dir, "task.json");
let value: unknown;
try {
value = JSON.parse(readFileSync(path, "utf8"));
} catch (error) {
throw new Error(`${path} 读不出来或不是合法的 JSON:${(error as Error).message}`, { cause: error });
}
return {
facts: checkTaskFacts(value),
dir,
baselineDir: join(dir, "baseline"),
guide: readFileSync(join(dir, "TASK.md"), "utf8"),
};
}
/** 最终指标与常规辅助指标的定义。最终指标的编号和方向取自 task.json。 */
export function metricDefinitions(task: Task): MetricDefinition[] {
return implementation(task).metricDefinitions(task.facts);
}
/**
* 准备一次正式运行的运行目录:把候选代码、任务的 harness 和这次要测的假设专用指标说明
* (metrics-spec.json)放进去。runDir 由调用方建好,应为空目录。
* 消融格里传进来的指标应当已经全部改成"缺了只记缺失"(vocab 指标说明),这里不区分。
*/
export function prepareRun(task: Task, candidateDir: string, runDir: string, hypothesisMetrics: HypothesisMetric[]): void {
implementation(task).prepareRun(task, candidateDir, runDir, hypothesisMetrics);
}
/**
* 正式运行的运行定义。种子由调用方给,同一份研究的全部运行用同一个种子:
* 评测要确定,消融不重跑基线也靠这一点(方法.md 第十部分第 9 条)。
*/
export function formalRun(task: Task, imageDigest: string, seed: string): RunDefinition {
return implementation(task).formalRun(task, imageDigest, seed);
}
/**
* 轻量检查:把候选代码和检查脚本放进 checkDir(调用方建好的空目录),交回运行定义。
* 只看语法和入口函数,不执行候选代码。seconds 来自配置;次数上限不归这里管。
*/
export function lightCheck(task: Task, candidateDir: string, checkDir: string, imageDigest: string, seconds: number): RunDefinition {
return implementation(task).lightCheck(task, candidateDir, checkDir, imageDigest, seconds);
}
/** 解读轻量检查:通过,或者不通过及原因。检查脚本自己没正常跑完时报错停下。 */
export function readLightCheck(task: Task, checkDir: string, result: ProcessResult): LightCheckOutcome {
return implementation(task).readLightCheck(checkDir, result);
}
/**
* 判分第一步。正式运行没正常结束、或者没有产物时直接判不算数;否则把产物和宿主保管的校验器
* 复制进 judgeDir(调用方建好的空目录),交回校验器的运行定义。
*/
export function prepareJudging(
task: Task,
runDir: string,
judgeDir: string,
imageDigest: string,
formal: ProcessResult,
hypothesisMetrics: HypothesisMetric[],
): JudgingStep {
return implementation(task).prepareJudging(task, runDir, judgeDir, imageDigest, formal, hypothesisMetrics);
}
/**
* 判分第二步:读校验器写的判定、运行目录里候选写的指标文件,加上正式运行的结果,交回判分结果。
* 校验器自己没正常跑完、判定文件认不出、修后分数高于原分数时报错停下:这些是宿主一侧的问题。
*/
export function judge(
task: Task,
runDir: string,
judgeDir: string,
formal: ProcessResult,
validator: ProcessResult,
hypothesisMetrics: HypothesisMetric[],
): JudgedResult {
return implementation(task).judge(task, runDir, judgeDir, formal, validator, hypothesisMetrics);
}
/** 改动过的文件里,属于 task.json"不许改"名单的那些。路径都相对候选代码目录。 */
export function protectedChanges(task: Task, changedFiles: string[]): string[] {
return changedFiles.filter((file) => task.facts.protectedFiles.includes(file));
}
+49
View File
@@ -0,0 +1,49 @@
import type { HypothesisMetric, JudgedResult, MetricDefinition, ProcessResult, RunDefinition } from "../vocab/index.ts";
import type { TaskFacts } from "./facts.ts";
/** 读进来的一个任务。 */
export interface Task {
/** task.json 的内容,已严格检查。 */
facts: TaskFacts;
/** 任务目录的绝对路径,里面有 Dockerfile 和依赖清单,runner 拿它建镜像。 */
dir: string;
/** 基线代码目录。 */
baselineDir: string;
/** TASK.md 全文,给实验员看的任务说明。 */
guide: string;
}
/** 轻量检查的结论:通过,或者不通过及原因。 */
export type LightCheckOutcome = { passed: true } | { passed: false; reasons: string[] };
/**
* 判分第一步的结果:要么不用跑校验器就已判定(正式运行没正常结束、没有产物),
* 要么交回校验器的运行定义,由调用方交给 runner 去跑。
*/
export type JudgingStep = { status: "judged"; result: JudgedResult } | { status: "validate"; definition: RunDefinition };
/** 一种任务的实现。对外函数按 task.facts.kind 从对照表里取出它来调。参数的含义见 task.ts 里同名的对外函数。 */
export interface TaskImplementation {
/** 最终指标(编号和方向取自 task.json)与常规辅助指标的定义。 */
metricDefinitions(facts: TaskFacts): MetricDefinition[];
prepareRun(task: Task, candidateDir: string, runDir: string, hypothesisMetrics: HypothesisMetric[]): void;
formalRun(task: Task, imageDigest: string, seed: string): RunDefinition;
lightCheck(task: Task, candidateDir: string, checkDir: string, imageDigest: string, seconds: number): RunDefinition;
readLightCheck(checkDir: string, result: ProcessResult): LightCheckOutcome;
prepareJudging(
task: Task,
runDir: string,
judgeDir: string,
imageDigest: string,
formal: ProcessResult,
hypothesisMetrics: HypothesisMetric[],
): JudgingStep;
judge(
task: Task,
runDir: string,
judgeDir: string,
formal: ProcessResult,
validator: ProcessResult,
hypothesisMetrics: HypothesisMetric[],
): JudgedResult;
}