feat: 任务书 05,runner 与 tasks 两个模块
runner 用 Docker 跑一条命令并交回结局和资源用量;tasks 读 task.json、准备运行目录、给出正式运行/校验器/轻量检查的运行定义,并从原始产物独立判分。任务目录从参考项目搬到 tasks/circle_packing/,新增 task.json;vocab 的 VERDICTS 删掉"判不了",ProcessResult 的 cancelled 改为"容器从外面被杀了"。 全新上下文审阅后的两处修复:计量 shell 以容器内 root 运行,命令经 setpriv 降权,结果写在容器自己的文件层、结束后 docker cp 取出,并加 no-new-privileges,候选无法伪造计量结果把超时记成正常退出;harness 开跑前删掉旧的 solution.json、metrics.json,旧产物不会被当成本次的值。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,342 @@
|
||||
import { copyFileSync, cpSync, existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { Type, type Static } from "typebox";
|
||||
import { Value } from "typebox/value";
|
||||
import type {
|
||||
HypothesisMetric,
|
||||
Json,
|
||||
JudgedResult,
|
||||
Measured,
|
||||
MetricDefinition,
|
||||
ProcessResult,
|
||||
RunDefinition,
|
||||
} from "../vocab/index.ts";
|
||||
import type { TaskFacts } from "./facts.ts";
|
||||
import { measured, readMetricsFile, unmeasured } from "./metrics-file.ts";
|
||||
import type { LightCheckOutcome, Task, TaskImplementation } from "./types.ts";
|
||||
|
||||
// 填圆:在单位正方形里放 26 个圆,最大化半径之和。任务说明见 tasks/circle_packing/TASK.md。
|
||||
// 凡是要执行任务代码的事(正式运行、校验器、轻量检查)都只交出运行定义,由调用方交给 runner 在任务镜像里跑;
|
||||
// 这里只读写文件、解读产物(docs/结构.md 第二部分"runner 只管怎么跑,tasks 只管怎么判")。
|
||||
|
||||
const CIRCLES = 26;
|
||||
const ENTRY = "initial.py";
|
||||
const HARNESS = "harness.py";
|
||||
const VALIDATOR = "validate.py";
|
||||
const CHECK_SCRIPT = "check-candidate.py";
|
||||
const METRICS_SPEC = "metrics-spec.json";
|
||||
const METRICS = "metrics.json";
|
||||
const SOLUTION = "solution.json";
|
||||
const VERDICT = "verdict.json";
|
||||
const CHECK_RESULT = "check.json";
|
||||
/** 轻量检查目录里放候选代码的子目录;检查脚本放在它的上一层,免得扫到自己。 */
|
||||
const CHECK_CANDIDATE = "candidate";
|
||||
/** 运行目录挂进容器的位置。 */
|
||||
const WORKDIR = "/work";
|
||||
/** 校验器的时限。26 个圆两两比较加合法化,实测一秒以内;它超时说明宿主的工具出了问题。 */
|
||||
const VALIDATOR_SECONDS = 60;
|
||||
/** 合法化前先按容差 0 判原产物(方法.md 第八部分"合法化"第 1 条)。 */
|
||||
const TOLERANCE = "0";
|
||||
|
||||
// 常规辅助指标的编号(方法.md 第五部分"指标")。前三个来自 runner 的资源用量,后四个来自校验器。
|
||||
const WALL = "wall-seconds";
|
||||
const CPU = "cpu-seconds";
|
||||
const MEMORY = "peak-memory-bytes";
|
||||
const CLEARANCE = "min-pairwise-clearance";
|
||||
const RAW_SCORE = "raw-score";
|
||||
const RAW_VIOLATION = "raw-violation";
|
||||
const DEDUCTION = "legalization-deduction";
|
||||
const FROM_VALIDATOR = [CLEARANCE, RAW_SCORE, RAW_VIOLATION, DEDUCTION];
|
||||
|
||||
const SIDE = "单位正方形的边长";
|
||||
const PER_RUN = "每次运行结束后由宿主读一次,直接取值";
|
||||
|
||||
function auxiliary(
|
||||
id: string,
|
||||
meaning: string,
|
||||
unit: string,
|
||||
direction: "maximize" | "minimize",
|
||||
howMeasured: string,
|
||||
): MetricDefinition {
|
||||
return { id, layer: "auxiliary", meaning, unit, direction, howMeasured, missing: "recordMissing" };
|
||||
}
|
||||
|
||||
function metricDefinitions(facts: TaskFacts): MetricDefinition[] {
|
||||
const validator = `宿主的校验器从 ${SOLUTION} 重算,${PER_RUN}`;
|
||||
return [
|
||||
{
|
||||
id: facts.finalMetric.id,
|
||||
layer: "final",
|
||||
meaning:
|
||||
"合法摆法的半径之和,由宿主的独立校验器从实际产物重算;原产物差一点不合法时,先按 TASK.md 的只缩不移规则修,再按修后的半径算",
|
||||
unit: SIDE,
|
||||
direction: facts.finalMetric.direction,
|
||||
howMeasured: validator,
|
||||
missing: "invalidateRun",
|
||||
},
|
||||
auxiliary(WALL, "正式运行的墙钟时间", "秒", "minimize", `容器里的计量程序记录,${PER_RUN}`),
|
||||
auxiliary(CPU, "正式运行用掉的 CPU 秒(容器 cgroup 的 usage_usec)", "秒", "minimize", `容器里的计量程序记录,${PER_RUN}`),
|
||||
auxiliary(MEMORY, "正式运行的峰值内存(容器 cgroup 的 memory.peak)", "字节", "minimize", `容器里的计量程序记录,${PER_RUN}`),
|
||||
auxiliary(CLEARANCE, "所有圆对中「圆心距减两半径之和」的最小值,大于等于 0 表示不重叠;修过的取修后的摆法", SIDE, "maximize", validator),
|
||||
auxiliary(RAW_SCORE, "合法化之前原产物的半径之和", SIDE, "maximize", validator),
|
||||
auxiliary(RAW_VIOLATION, "原产物的最大违反量:两圆重叠的量和圆出边的量里最大的一个,0 表示没有违反", SIDE, "minimize", validator),
|
||||
auxiliary(DEDUCTION, "只缩不移合法化扣掉的分:原分数减最终成绩,原产物合法时是 0", SIDE, "minimize", validator),
|
||||
];
|
||||
}
|
||||
|
||||
/** 正式运行、校验器、轻量检查共用的运行定义。数学库的线程数和核数一致,否则会开多个线程抢核。 */
|
||||
function definition(task: Task, imageDigest: string, command: string[], wallSeconds: number, env: Record<string, string>): RunDefinition {
|
||||
const cores = String(task.facts.run.cores);
|
||||
return {
|
||||
command,
|
||||
workdir: WORKDIR,
|
||||
image: { digest: imageDigest, runtime: task.facts.run.runtime },
|
||||
wallSeconds,
|
||||
cores: task.facts.run.cores,
|
||||
inheritEnv: false,
|
||||
env: { OMP_NUM_THREADS: cores, OPENBLAS_NUM_THREADS: cores, MKL_NUM_THREADS: cores, ...env },
|
||||
};
|
||||
}
|
||||
|
||||
// 解释器不加 -I:-I 会忽略全部 PYTHON* 环境变量,连同 runner 设的 PYTHON_CPU_COUNT,
|
||||
// 于是 os.cpu_count() 报回宿主核数(本机实测 16)。容器里的环境变量全部由运行定义给出,用不着 -I。
|
||||
// -B:不写 __pycache__,不往运行目录里添文件。
|
||||
|
||||
function prepareRun(task: Task, candidateDir: string, runDir: string, hypothesisMetrics: HypothesisMetric[]): void {
|
||||
cpSync(candidateDir, runDir, { recursive: true });
|
||||
// harness 和指标说明以宿主的为准,候选目录里有同名文件也被盖掉(这两个名字在 task.json 的不许改名单里)。
|
||||
copyFileSync(join(task.dir, HARNESS), join(runDir, HARNESS));
|
||||
writeFileSync(join(runDir, METRICS_SPEC), `${JSON.stringify(hypothesisMetrics, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function formalRun(task: Task, imageDigest: string, seed: string): RunDefinition {
|
||||
const command = ["python", "-B", `${WORKDIR}/${HARNESS}`, "--program", ENTRY, "--output", SOLUTION];
|
||||
return definition(task, imageDigest, command, task.facts.run.wallSeconds, { INFORSI_SEED: seed });
|
||||
}
|
||||
|
||||
function lightCheck(task: Task, candidateDir: string, checkDir: string, imageDigest: string, seconds: number): RunDefinition {
|
||||
cpSync(candidateDir, join(checkDir, CHECK_CANDIDATE), { recursive: true });
|
||||
copyFileSync(join(task.dir, CHECK_SCRIPT), join(checkDir, CHECK_SCRIPT));
|
||||
const command = ["python", "-B", CHECK_SCRIPT, CHECK_CANDIDATE, ENTRY, CHECK_RESULT];
|
||||
return definition(task, imageDigest, command, seconds, {});
|
||||
}
|
||||
|
||||
/** 检查脚本写的结果文件。valid 与 reasons 是否为空必须一致。 */
|
||||
const CheckResult = Type.Object({ valid: Type.Boolean(), reasons: Type.Array(Type.String()) });
|
||||
|
||||
/** 宿主的工具(检查脚本、校验器)没正常跑完时报错停下:这不是候选的失败(任务书 05 第三部分)。 */
|
||||
function requireExit(what: string, result: ProcessResult, codes: number[]): number {
|
||||
if (result.ending === "exited" && codes.includes(result.exitCode)) return result.exitCode;
|
||||
const how = result.ending === "notStarted" ? `没启动:${result.reason}` : result.ending === "exited" ? `退出码 ${result.exitCode}` : result.ending;
|
||||
throw new Error(`${what}没有正常跑完(${how}),这是宿主一侧的工具出了问题,不能当成候选的失败`);
|
||||
}
|
||||
|
||||
function readJson(path: string): unknown {
|
||||
try {
|
||||
return JSON.parse(readFileSync(path, "utf8"));
|
||||
} catch (error) {
|
||||
throw new Error(`读不出 ${path}:${(error as Error).message}`, { cause: error });
|
||||
}
|
||||
}
|
||||
|
||||
function readLightCheck(checkDir: string, result: ProcessResult): LightCheckOutcome {
|
||||
requireExit("轻量检查脚本", result, [0]);
|
||||
const path = join(checkDir, CHECK_RESULT);
|
||||
const check = readJson(path);
|
||||
if (!Value.Check(CheckResult, check) || check.valid !== (check.reasons.length === 0)) {
|
||||
throw new Error(`${path} 的内容认不出:${JSON.stringify(check)}`);
|
||||
}
|
||||
return check.valid ? { passed: true } : { passed: false, reasons: check.reasons };
|
||||
}
|
||||
|
||||
/** 正式运行没正常结束的原因;正常退出(退出码 0)时为 null。 */
|
||||
function processFailure(task: Task, formal: Exclude<ProcessResult, { ending: "notStarted" }>): string | null {
|
||||
if (formal.ending === "timeout") return `超过单次时限 ${task.facts.run.wallSeconds} 秒,被结束`;
|
||||
if (formal.ending === "cancelled") return "容器从外面被结束(关机、手动 docker kill 等),没有留下计量结果";
|
||||
if (formal.exitCode !== 0) return `正式运行以退出码 ${formal.exitCode} 结束`;
|
||||
return null;
|
||||
}
|
||||
|
||||
function started(formal: ProcessResult): Exclude<ProcessResult, { ending: "notStarted" }> {
|
||||
if (formal.ending === "notStarted") throw new Error(`没启动的运行不判分:${formal.reason}`);
|
||||
return formal;
|
||||
}
|
||||
|
||||
/**
|
||||
* 拼出 JudgedResult:资源用量来自正式运行,假设专用指标和两个保留键来自候选写的 metrics.json。
|
||||
* 有"缺了整次作废"的假设专用指标测不到时,原本算数的也改判不算数(方法.md 第五部分"指标")。
|
||||
*/
|
||||
function assemble(
|
||||
runDir: string,
|
||||
formal: Exclude<ProcessResult, { ending: "notStarted" }>,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
verdict: { valid: boolean; reason: string },
|
||||
score: Measured,
|
||||
fromValidator: Record<string, Measured>,
|
||||
extra: Json,
|
||||
): JudgedResult {
|
||||
const file = readMetricsFile(join(runDir, METRICS), METRICS, hypothesisMetrics);
|
||||
const voided = hypothesisMetrics.flatMap((m) => {
|
||||
const value = file.hypothesis[m.id]!;
|
||||
return m.missing === "invalidateRun" && value.status === "unmeasured" ? [`${m.id}(${value.reason})`] : [];
|
||||
});
|
||||
const result = verdict.valid && voided.length > 0
|
||||
? { valid: false, reason: `登记为"缺了整次作废"的假设专用指标测不到:${voided.join(";")}` }
|
||||
: verdict;
|
||||
return {
|
||||
verdict: result.valid ? "valid" : "invalid",
|
||||
verdictReason: result.reason,
|
||||
score,
|
||||
metrics: {
|
||||
[WALL]: formal.wallSeconds,
|
||||
[CPU]: formal.cpuSeconds,
|
||||
[MEMORY]: formal.peakMemoryBytes,
|
||||
...fromValidator,
|
||||
...file.hypothesis,
|
||||
},
|
||||
answerPath: file.answerPath,
|
||||
safeguardTriggers: file.safeguardTriggers,
|
||||
extra,
|
||||
};
|
||||
}
|
||||
|
||||
function prepareJudging(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
imageDigest: string,
|
||||
formal: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): ReturnType<TaskImplementation["prepareJudging"]> {
|
||||
const run = started(formal);
|
||||
const solution = join(runDir, SOLUTION);
|
||||
const failure = processFailure(task, run) ?? (existsSync(solution) ? null : `没有产物:运行目录里没有 ${SOLUTION}`);
|
||||
if (failure !== null) {
|
||||
const notValidated = Object.fromEntries(FROM_VALIDATOR.map((id) => [id, unmeasured(`没有跑校验器:${failure}`)]));
|
||||
const result = assemble(runDir, run, hypothesisMetrics, { valid: false, reason: failure }, unmeasured(failure), notValidated, {
|
||||
legalization: null,
|
||||
});
|
||||
return { status: "judged", result };
|
||||
}
|
||||
copyFileSync(solution, join(judgeDir, SOLUTION));
|
||||
copyFileSync(join(task.dir, VALIDATOR), join(judgeDir, VALIDATOR));
|
||||
const command = [
|
||||
"python",
|
||||
"-B",
|
||||
VALIDATOR,
|
||||
"--circles",
|
||||
String(CIRCLES),
|
||||
"--solution",
|
||||
SOLUTION,
|
||||
"--atol",
|
||||
TOLERANCE,
|
||||
"--legalize",
|
||||
"--verdict",
|
||||
VERDICT,
|
||||
];
|
||||
return { status: "validate", definition: definition(task, imageDigest, command, VALIDATOR_SECONDS, {}) };
|
||||
}
|
||||
|
||||
const NumberOrNull = Type.Union([Type.Number(), Type.Null()]);
|
||||
|
||||
/** validate.py --legalize 写的判定里,这里要读的部分;其余键(evidence、legal_radii 等)不读。 */
|
||||
const ValidatorVerdict = Type.Object({
|
||||
valid: Type.Boolean(),
|
||||
computed_sum: NumberOrNull,
|
||||
reasons: Type.Array(Type.String()),
|
||||
diagnostics: Type.Object({ min_pairwise_clearance: NumberOrNull }),
|
||||
legalization: Type.Object({
|
||||
raw_valid: Type.Boolean(),
|
||||
raw_score: NumberOrNull,
|
||||
raw_violation: NumberOrNull,
|
||||
applied: Type.Boolean(),
|
||||
repaired: Type.Boolean(),
|
||||
shrunk_circles: Type.Array(Type.Integer()),
|
||||
reasons: Type.Array(Type.String()),
|
||||
}),
|
||||
});
|
||||
type ValidatorVerdict = Static<typeof ValidatorVerdict>;
|
||||
|
||||
/** 校验器在形状不对、含非有限数或负半径时提前返回,几何量没算。 */
|
||||
const NOT_COMPUTED = "原产物形状不对、含非有限数或负半径,校验器没算这个量";
|
||||
|
||||
function orUnmeasured(value: number | null, reason: string): Measured {
|
||||
return value === null ? unmeasured(reason) : measured(value);
|
||||
}
|
||||
|
||||
function readVerdict(judgeDir: string, exitCode: number): ValidatorVerdict {
|
||||
const path = join(judgeDir, VERDICT);
|
||||
const verdict = readJson(path);
|
||||
if (!Value.Check(ValidatorVerdict, verdict)) throw new Error(`${path} 的内容认不出:${JSON.stringify(verdict)}`);
|
||||
// 退出码 0 = 合法、1 = 不合法;合法化之后的结论就是最终判定。对不上说明跑的不是这份校验器。
|
||||
const consistent =
|
||||
verdict.valid === (exitCode === 0) &&
|
||||
verdict.legalization.repaired === verdict.valid &&
|
||||
(!verdict.valid || verdict.computed_sum !== null);
|
||||
if (!consistent) throw new Error(`${path} 与退出码 ${exitCode} 或自身前后对不上:${JSON.stringify(verdict)}`);
|
||||
return verdict;
|
||||
}
|
||||
|
||||
function judge(
|
||||
_task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
formal: ProcessResult,
|
||||
validator: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgedResult {
|
||||
const run = started(formal);
|
||||
const verdict = readVerdict(judgeDir, requireExit("校验器", validator, [0, 1]));
|
||||
const legal = verdict.legalization;
|
||||
let deduction: Measured;
|
||||
let score: Measured;
|
||||
if (verdict.valid) {
|
||||
const final = verdict.computed_sum!;
|
||||
score = measured(final);
|
||||
// legal.repaired 为真时 raw_score 一定有值:原产物能算出分数,才可能合法或被修好。
|
||||
const raw = legal.raw_score!;
|
||||
// 只缩半径只会降分;修后反而更高,说明修法写错了(方法.md 第八部分"合法化"第 3 条)。
|
||||
if (final > raw) throw new Error(`合法化后分数 ${final} 高于原分数 ${raw}:只缩半径只会降分,修法出错了`);
|
||||
deduction = measured(raw - final);
|
||||
} else {
|
||||
const reason = `不算数:${verdict.reasons.concat(legal.reasons).join(";")}`;
|
||||
score = unmeasured(reason);
|
||||
deduction = unmeasured(`修不了,没有修后分数:${legal.reasons.join(";")}`);
|
||||
}
|
||||
const rawScore = orUnmeasured(legal.raw_score, NOT_COMPUTED);
|
||||
const rawViolation = orUnmeasured(legal.raw_violation, NOT_COMPUTED);
|
||||
const fromValidator = {
|
||||
[CLEARANCE]: orUnmeasured(verdict.diagnostics.min_pairwise_clearance, NOT_COMPUTED),
|
||||
[RAW_SCORE]: rawScore,
|
||||
[RAW_VIOLATION]: rawViolation,
|
||||
[DEDUCTION]: deduction,
|
||||
};
|
||||
const reason = verdict.valid
|
||||
? legal.raw_valid
|
||||
? "校验器按容差 0 判原产物合法"
|
||||
: `原产物不合法,按只缩不移规则修好后合法(缩了 ${legal.shrunk_circles.length} 个圆)`
|
||||
: `校验器判不合法:${verdict.reasons.concat(legal.reasons).join(";")}`;
|
||||
const extra = {
|
||||
legalization: {
|
||||
rawValid: legal.raw_valid,
|
||||
applied: legal.applied,
|
||||
repaired: legal.repaired,
|
||||
shrunkCircles: legal.shrunk_circles,
|
||||
reasons: legal.reasons,
|
||||
rawScore,
|
||||
rawViolation,
|
||||
deduction,
|
||||
},
|
||||
};
|
||||
return assemble(runDir, run, hypothesisMetrics, { valid: verdict.valid, reason }, score, fromValidator, extra);
|
||||
}
|
||||
|
||||
export const circlePacking: TaskImplementation = {
|
||||
metricDefinitions,
|
||||
prepareRun,
|
||||
formalRun,
|
||||
lightCheck,
|
||||
readLightCheck,
|
||||
prepareJudging,
|
||||
judge,
|
||||
};
|
||||
@@ -0,0 +1,90 @@
|
||||
import { Type, type Static } from "typebox";
|
||||
import { Settings } from "typebox/system";
|
||||
import { Value } from "typebox/value";
|
||||
import { Direction, Id } from "../vocab/index.ts";
|
||||
|
||||
// task.json 的字段定义与检查。task.json 是用户提供的东西,属于系统边界(CLAUDE.md"罕见情况怎么办"),
|
||||
// 所以每个字段必填、不认识的字段报错,一次列出全部错误,每条写明字段路径。可以没有的值写 null。
|
||||
|
||||
/** 任务种类。每一种在 task.ts 的对照表里对应一份实现;nanochat、AI4AI 以后按同样的形状加。 */
|
||||
export const TASK_KINDS = ["circle_packing"] as const;
|
||||
export type TaskKind = (typeof TASK_KINDS)[number];
|
||||
|
||||
const CLOSED = { additionalProperties: false } as const;
|
||||
|
||||
export const TaskFacts = Type.Object(
|
||||
{
|
||||
kind: Type.Enum(TASK_KINDS, { type: "string", description: "任务种类,决定用哪一份实现" }),
|
||||
finalMetric: Type.Object(
|
||||
{
|
||||
id: Id,
|
||||
direction: Direction,
|
||||
},
|
||||
{ ...CLOSED, description: "最终指标的编号和方向(方法.md 第十部分第 1 条)" },
|
||||
),
|
||||
epsilon: Type.Number({ exclusiveMinimum: 0, description: "精度 ε(方法.md 6.11)" }),
|
||||
knownBest: Type.Union([Type.Number(), Type.Null()], {
|
||||
description: "已知最好成绩;没有就写 null(方法.md 6.11)",
|
||||
}),
|
||||
run: Type.Object(
|
||||
{
|
||||
wallSeconds: Type.Number({ exclusiveMinimum: 0, description: "单次正式运行的墙钟时限(秒)" }),
|
||||
cores: Type.Integer({ minimum: 1, description: "单次正式运行的核数" }),
|
||||
runtime: Type.String({ minLength: 1, description: "Docker 运行时,必须写明,例如 runc" }),
|
||||
},
|
||||
{ ...CLOSED, description: "同批候选共用的比较条件(方法.md 第八部分)" },
|
||||
),
|
||||
protectedFiles: Type.Array(Type.String({ minLength: 1 }), {
|
||||
description: "不许改的文件,路径相对候选代码目录(方法.md 第十部分第 2 条)",
|
||||
}),
|
||||
},
|
||||
CLOSED,
|
||||
);
|
||||
export type TaskFacts = Static<typeof TaskFacts>;
|
||||
|
||||
type SchemaError = ReturnType<typeof Value.Errors>[number];
|
||||
|
||||
const TYPES: Readonly<Record<string, string>> = { number: "数字", integer: "整数", string: "字符串", object: "对象", array: "数组" };
|
||||
|
||||
/** 本文件的字段用得到的几种错误,写成中文;其余照 typebox 的原文。 */
|
||||
const REASONS: Readonly<Record<string, (p: Record<string, unknown>) => string>> = {
|
||||
type: (p) => `应为${TYPES[String(p.type)] ?? String(p.type)}`,
|
||||
enum: (p) => `只能是 ${(p.allowedValues as unknown[]).map((v) => JSON.stringify(v)).join("、")} 之一`,
|
||||
exclusiveMinimum: (p) => `必须大于 ${String(p.limit)}`,
|
||||
minimum: (p) => `不能小于 ${String(p.limit)}`,
|
||||
minLength: () => "不能为空",
|
||||
pattern: (p) => `格式不对,应符合 ${String(p.pattern)}`,
|
||||
};
|
||||
|
||||
function errorLines(errors: SchemaError[]): string[] {
|
||||
const lines: string[] = [];
|
||||
for (const e of errors) {
|
||||
const p = e.params as Record<string, unknown>;
|
||||
const at = e.instancePath || "/";
|
||||
// 唯一的几选一是 knownBest(数字或 null);各分支自己的错误不再逐条列。
|
||||
if (e.schemaPath.includes("/anyOf/")) continue;
|
||||
if (e.keyword === "anyOf") lines.push(`${at}:应为数字,没有就写 null`);
|
||||
else if (e.keyword === "required") {
|
||||
for (const name of p.requiredProperties as string[]) lines.push(`${e.instancePath}/${name}:缺少这一项(每个字段都必填)`);
|
||||
} else if (e.keyword === "additionalProperties") {
|
||||
for (const name of p.additionalProperties as string[]) lines.push(`${e.instancePath}/${name}:不认识的字段`);
|
||||
} else lines.push(`${at}:${REASONS[e.keyword]?.(p) ?? e.message}`);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
/** 检查一份已经解析好的 task.json;合格就原样返回,不合格就报错,列出全部错误。 */
|
||||
export function checkTaskFacts(value: unknown): TaskFacts {
|
||||
if (Value.Check(TaskFacts, value)) return value;
|
||||
// typebox 默认最多收 8 条错误,不放开就会漏报;放开只在这一次调用里。
|
||||
const cap = Settings.Get().maxErrors;
|
||||
Settings.Set({ maxErrors: Infinity });
|
||||
let errors: SchemaError[];
|
||||
try {
|
||||
errors = [...Value.Errors(TaskFacts, value)];
|
||||
} finally {
|
||||
Settings.Set({ maxErrors: cap });
|
||||
}
|
||||
const lines = errorLines(errors);
|
||||
throw new Error(`task.json 不合格,共 ${lines.length} 处:\n${lines.join("\n")}`);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// tasks:任务目录里的事实和文件、怎么准备一次运行、怎么从原始产物独立判分(docs/结构.md 第二部分)。
|
||||
// 不起子进程:要执行任务代码的事都只交出运行定义,由调用方交给 runner 在任务镜像里跑。
|
||||
export { checkTaskFacts, TASK_KINDS, TaskFacts, type TaskKind } from "./facts.ts";
|
||||
export {
|
||||
formalRun,
|
||||
judge,
|
||||
lightCheck,
|
||||
loadTask,
|
||||
metricDefinitions,
|
||||
prepareJudging,
|
||||
prepareRun,
|
||||
protectedChanges,
|
||||
readLightCheck,
|
||||
} from "./task.ts";
|
||||
export type { JudgingStep, LightCheckOutcome, Task } from "./types.ts";
|
||||
@@ -0,0 +1,88 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import {
|
||||
ANSWER_PATH_KEY,
|
||||
SAFEGUARD_TRIGGERS_KEY,
|
||||
type AnswerPath,
|
||||
type HypothesisMetric,
|
||||
type Measured,
|
||||
type SafeguardTriggers,
|
||||
} from "../vocab/index.ts";
|
||||
|
||||
// 读候选写的指标文件:假设专用指标的取值,加两个保留键(方法.md 第五部分"指标""答案路径")。
|
||||
// 这个文件是候选的产物,读不到、写错了都记成明确的状态并附原因,不补 0、不补空对象,也不报错停下。
|
||||
|
||||
export interface MetricsFileReading {
|
||||
/** 每条登记的假设专用指标一项,键是指标编号。 */
|
||||
hypothesis: Record<string, Measured>;
|
||||
answerPath: AnswerPath;
|
||||
safeguardTriggers: SafeguardTriggers;
|
||||
}
|
||||
|
||||
export function unmeasured(reason: string): Measured {
|
||||
return { status: "unmeasured", reason };
|
||||
}
|
||||
|
||||
export function measured(value: number): Measured {
|
||||
return { status: "measured", value };
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
/** 整个文件都读不到时,每一项都记同一个原因。 */
|
||||
function allUnreadable(reason: string, metrics: HypothesisMetric[]): MetricsFileReading {
|
||||
return {
|
||||
hypothesis: Object.fromEntries(metrics.map((m) => [m.id, unmeasured(reason)])),
|
||||
answerPath: { status: "unreadable", reason },
|
||||
safeguardTriggers: { status: "unreadable", reason },
|
||||
};
|
||||
}
|
||||
|
||||
function readMetric(values: Record<string, unknown>, id: string, name: string): Measured {
|
||||
const value = values[id];
|
||||
if (value === undefined) return unmeasured(`${name} 里没有指标 ${id}`);
|
||||
if (value === null) return unmeasured(`候选写的是 null:这一次没测到 ${id}`);
|
||||
// JSON.parse 会把 1e999 读成 Infinity,所以要查有限。
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) {
|
||||
return unmeasured(`${name} 里指标 ${id} 的值不是有限数:${JSON.stringify(value)}`);
|
||||
}
|
||||
return measured(value);
|
||||
}
|
||||
|
||||
function readAnswerPath(values: Record<string, unknown>, name: string): AnswerPath {
|
||||
const value = values[ANSWER_PATH_KEY];
|
||||
if (value === undefined) return { status: "unreadable", reason: `${name} 里没写 ${ANSWER_PATH_KEY}` };
|
||||
if (typeof value !== "string") {
|
||||
return { status: "unreadable", reason: `${name} 里 ${ANSWER_PATH_KEY} 不是字符串:${JSON.stringify(value)}` };
|
||||
}
|
||||
return { status: "read", value };
|
||||
}
|
||||
|
||||
function readTriggers(values: Record<string, unknown>, name: string): SafeguardTriggers {
|
||||
const value = values[SAFEGUARD_TRIGGERS_KEY];
|
||||
if (value === undefined) return { status: "unreadable", reason: `${name} 里没写 ${SAFEGUARD_TRIGGERS_KEY}` };
|
||||
const counts = isRecord(value) ? Object.entries(value) : null;
|
||||
if (counts === null || counts.some(([, n]) => !Number.isInteger(n) || (n as number) < 0)) {
|
||||
const reason = `${name} 里 ${SAFEGUARD_TRIGGERS_KEY} 应为"保障短名 → 非负整数"的对象,实际是 ${JSON.stringify(value)}`;
|
||||
return { status: "unreadable", reason };
|
||||
}
|
||||
return { status: "read", counts: Object.fromEntries(counts) as Record<string, number> };
|
||||
}
|
||||
|
||||
/** 读指标文件。metrics 是这次登记的假设专用指标,只按它们的编号取值,多写的键不读。 */
|
||||
export function readMetricsFile(path: string, name: string, metrics: HypothesisMetric[]): MetricsFileReading {
|
||||
if (!existsSync(path)) return allUnreadable(`${name} 不存在`, metrics);
|
||||
let values: unknown;
|
||||
try {
|
||||
values = JSON.parse(readFileSync(path, "utf8"));
|
||||
} catch (error) {
|
||||
return allUnreadable(`${name} 不是合法的 JSON:${(error as Error).message}`, metrics);
|
||||
}
|
||||
if (!isRecord(values)) return allUnreadable(`${name} 应为 JSON 对象(指标编号 → 值)`, metrics);
|
||||
return {
|
||||
hypothesis: Object.fromEntries(metrics.map((m) => [m.id, readMetric(values, m.id, name)])),
|
||||
answerPath: readAnswerPath(values, name),
|
||||
safeguardTriggers: readTriggers(values, name),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import type { HypothesisMetric, JudgedResult, MetricDefinition, ProcessResult, RunDefinition } from "../vocab/index.ts";
|
||||
import { circlePacking } from "./circle-packing.ts";
|
||||
import { checkTaskFacts, type TaskKind } from "./facts.ts";
|
||||
import type { JudgingStep, LightCheckOutcome, Task, TaskImplementation } from "./types.ts";
|
||||
|
||||
// tasks 对外的函数。任务目录的约定:task.json(任务事实)、TASK.md(给实验员的说明)、
|
||||
// Dockerfile 与依赖清单(建镜像)、baseline/(基线代码),以及各任务自己的 Python 工具。
|
||||
// 每个函数按 task.facts.kind 从对照表里取出那种任务的实现来做。
|
||||
|
||||
/** 任务种类 → 实现。加一种任务就在这里加一行。 */
|
||||
const IMPLEMENTATIONS: Readonly<Record<TaskKind, TaskImplementation>> = {
|
||||
circle_packing: circlePacking,
|
||||
};
|
||||
|
||||
function implementation(task: Task): TaskImplementation {
|
||||
return IMPLEMENTATIONS[task.facts.kind];
|
||||
}
|
||||
|
||||
/** 读一个任务目录:严格检查 task.json,同时交回 Dockerfile 所在目录、基线目录和 TASK.md 全文。 */
|
||||
export function loadTask(taskDir: string): Task {
|
||||
const dir = resolve(taskDir);
|
||||
const path = join(dir, "task.json");
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(readFileSync(path, "utf8"));
|
||||
} catch (error) {
|
||||
throw new Error(`${path} 读不出来或不是合法的 JSON:${(error as Error).message}`, { cause: error });
|
||||
}
|
||||
return {
|
||||
facts: checkTaskFacts(value),
|
||||
dir,
|
||||
baselineDir: join(dir, "baseline"),
|
||||
guide: readFileSync(join(dir, "TASK.md"), "utf8"),
|
||||
};
|
||||
}
|
||||
|
||||
/** 最终指标与常规辅助指标的定义。最终指标的编号和方向取自 task.json。 */
|
||||
export function metricDefinitions(task: Task): MetricDefinition[] {
|
||||
return implementation(task).metricDefinitions(task.facts);
|
||||
}
|
||||
|
||||
/**
|
||||
* 准备一次正式运行的运行目录:把候选代码、任务的 harness 和这次要测的假设专用指标说明
|
||||
* (metrics-spec.json)放进去。runDir 由调用方建好,应为空目录。
|
||||
* 消融格里传进来的指标应当已经全部改成"缺了只记缺失"(vocab 指标说明),这里不区分。
|
||||
*/
|
||||
export function prepareRun(task: Task, candidateDir: string, runDir: string, hypothesisMetrics: HypothesisMetric[]): void {
|
||||
implementation(task).prepareRun(task, candidateDir, runDir, hypothesisMetrics);
|
||||
}
|
||||
|
||||
/**
|
||||
* 正式运行的运行定义。种子由调用方给,同一份研究的全部运行用同一个种子:
|
||||
* 评测要确定,消融不重跑基线也靠这一点(方法.md 第十部分第 9 条)。
|
||||
*/
|
||||
export function formalRun(task: Task, imageDigest: string, seed: string): RunDefinition {
|
||||
return implementation(task).formalRun(task, imageDigest, seed);
|
||||
}
|
||||
|
||||
/**
|
||||
* 轻量检查:把候选代码和检查脚本放进 checkDir(调用方建好的空目录),交回运行定义。
|
||||
* 只看语法和入口函数,不执行候选代码。seconds 来自配置;次数上限不归这里管。
|
||||
*/
|
||||
export function lightCheck(task: Task, candidateDir: string, checkDir: string, imageDigest: string, seconds: number): RunDefinition {
|
||||
return implementation(task).lightCheck(task, candidateDir, checkDir, imageDigest, seconds);
|
||||
}
|
||||
|
||||
/** 解读轻量检查:通过,或者不通过及原因。检查脚本自己没正常跑完时报错停下。 */
|
||||
export function readLightCheck(task: Task, checkDir: string, result: ProcessResult): LightCheckOutcome {
|
||||
return implementation(task).readLightCheck(checkDir, result);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判分第一步。正式运行没正常结束、或者没有产物时直接判不算数;否则把产物和宿主保管的校验器
|
||||
* 复制进 judgeDir(调用方建好的空目录),交回校验器的运行定义。
|
||||
*/
|
||||
export function prepareJudging(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
imageDigest: string,
|
||||
formal: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgingStep {
|
||||
return implementation(task).prepareJudging(task, runDir, judgeDir, imageDigest, formal, hypothesisMetrics);
|
||||
}
|
||||
|
||||
/**
|
||||
* 判分第二步:读校验器写的判定、运行目录里候选写的指标文件,加上正式运行的结果,交回判分结果。
|
||||
* 校验器自己没正常跑完、判定文件认不出、修后分数高于原分数时报错停下:这些是宿主一侧的问题。
|
||||
*/
|
||||
export function judge(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
formal: ProcessResult,
|
||||
validator: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgedResult {
|
||||
return implementation(task).judge(task, runDir, judgeDir, formal, validator, hypothesisMetrics);
|
||||
}
|
||||
|
||||
/** 改动过的文件里,属于 task.json"不许改"名单的那些。路径都相对候选代码目录。 */
|
||||
export function protectedChanges(task: Task, changedFiles: string[]): string[] {
|
||||
return changedFiles.filter((file) => task.facts.protectedFiles.includes(file));
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { HypothesisMetric, JudgedResult, MetricDefinition, ProcessResult, RunDefinition } from "../vocab/index.ts";
|
||||
import type { TaskFacts } from "./facts.ts";
|
||||
|
||||
/** 读进来的一个任务。 */
|
||||
export interface Task {
|
||||
/** task.json 的内容,已严格检查。 */
|
||||
facts: TaskFacts;
|
||||
/** 任务目录的绝对路径,里面有 Dockerfile 和依赖清单,runner 拿它建镜像。 */
|
||||
dir: string;
|
||||
/** 基线代码目录。 */
|
||||
baselineDir: string;
|
||||
/** TASK.md 全文,给实验员看的任务说明。 */
|
||||
guide: string;
|
||||
}
|
||||
|
||||
/** 轻量检查的结论:通过,或者不通过及原因。 */
|
||||
export type LightCheckOutcome = { passed: true } | { passed: false; reasons: string[] };
|
||||
|
||||
/**
|
||||
* 判分第一步的结果:要么不用跑校验器就已判定(正式运行没正常结束、没有产物),
|
||||
* 要么交回校验器的运行定义,由调用方交给 runner 去跑。
|
||||
*/
|
||||
export type JudgingStep = { status: "judged"; result: JudgedResult } | { status: "validate"; definition: RunDefinition };
|
||||
|
||||
/** 一种任务的实现。对外函数按 task.facts.kind 从对照表里取出它来调。参数的含义见 task.ts 里同名的对外函数。 */
|
||||
export interface TaskImplementation {
|
||||
/** 最终指标(编号和方向取自 task.json)与常规辅助指标的定义。 */
|
||||
metricDefinitions(facts: TaskFacts): MetricDefinition[];
|
||||
prepareRun(task: Task, candidateDir: string, runDir: string, hypothesisMetrics: HypothesisMetric[]): void;
|
||||
formalRun(task: Task, imageDigest: string, seed: string): RunDefinition;
|
||||
lightCheck(task: Task, candidateDir: string, checkDir: string, imageDigest: string, seconds: number): RunDefinition;
|
||||
readLightCheck(checkDir: string, result: ProcessResult): LightCheckOutcome;
|
||||
prepareJudging(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
imageDigest: string,
|
||||
formal: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgingStep;
|
||||
judge(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
formal: ProcessResult,
|
||||
validator: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgedResult;
|
||||
}
|
||||
Reference in New Issue
Block a user