feat: 诊断（任务书 18 第一块）：诊断数学、四种提交、一次运行跑多个实验、账本与材料、flow 诊断各步
消融默认关，关掉时解释阶段一并跳过；选诊断实验用信息价值减成本。 eval/diagnosis/research.json 为第三块配置。vocab 字段说明的出口文字随提示词一起改，见下一次提交的 prompts/ 与同文件改动（本次含在内）。 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
+227
-13
@@ -1,4 +1,4 @@
|
||||
import { copyFileSync, cpSync, existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { chmodSync, copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { Type, type Static } from "typebox";
|
||||
import { Value } from "typebox/value";
|
||||
@@ -13,7 +13,16 @@ import type {
|
||||
} from "../vocab/index.ts";
|
||||
import type { TaskFacts } from "./facts.ts";
|
||||
import { measured, readMetricsFile, unmeasured } from "./metrics-file.ts";
|
||||
import type { LightCheckOutcome, Task, TaskImplementation } from "./types.ts";
|
||||
import type {
|
||||
DiagnosisEnding,
|
||||
DiagnosisExperiment,
|
||||
DiagnosisExperimentJudged,
|
||||
DiagnosisJudged,
|
||||
DiagnosisJudgingStep,
|
||||
LightCheckOutcome,
|
||||
Task,
|
||||
TaskImplementation,
|
||||
} from "./types.ts";
|
||||
|
||||
// 填圆：在单位正方形里放 26 个圆，最大化半径之和。任务说明见 tasks/circle_packing/TASK.md。
|
||||
// 凡是要执行任务代码的事（正式运行、校验器、轻量检查）都只交出运行定义，由调用方交给 runner 在任务镜像里跑；
|
||||
@@ -22,6 +31,7 @@ import type { LightCheckOutcome, Task, TaskImplementation } from "./types.ts";
|
||||
const CIRCLES = 26;
|
||||
const ENTRY = "initial.py";
|
||||
const HARNESS = "harness.py";
|
||||
const DIAGNOSIS_HARNESS = "harness-diagnosis.py";
|
||||
const VALIDATOR = "validate.py";
|
||||
const CHECK_SCRIPT = "check-candidate.py";
|
||||
const METRICS_SPEC = "metrics-spec.json";
|
||||
@@ -29,6 +39,12 @@ const METRICS = "metrics.json";
|
||||
const SOLUTION = "solution.json";
|
||||
const VERDICT = "verdict.json";
|
||||
const CHECK_RESULT = "check.json";
|
||||
/** 诊断运行：原始产物目录、每个实验的答案目录、harness 写的每实验用时报告（路径都相对运行目录）。 */
|
||||
const BASELINE_ARTIFACT = "baseline-artifact";
|
||||
const ANSWERS = "answers";
|
||||
const DIAGNOSIS_REPORT = "diagnosis.json";
|
||||
/** 判分目录里每个实验写的校验器退出码文件（同一个目录下还有这个实验的 solution.json、verdict.json）。 */
|
||||
const EXIT_FILE = "exit";
|
||||
/** 轻量检查目录里放候选代码的子目录；检查脚本放在它的上一层，免得扫到自己。 */
|
||||
const CHECK_CANDIDATE = "candidate";
|
||||
/** 运行目录挂进容器的位置。 */
|
||||
@@ -277,16 +293,16 @@ function readVerdict(judgeDir: string, exitCode: number): ValidatorVerdict {
|
||||
return verdict;
|
||||
}
|
||||
|
||||
function judge(
|
||||
_task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
formal: ProcessResult,
|
||||
validator: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgedResult {
|
||||
const run = started(formal);
|
||||
const verdict = readVerdict(judgeDir, requireExit("校验器", validator, [0, 1]));
|
||||
/** 校验器判定读成的结论：算不算数、理由、最终分数、校验器给的辅助指标、合法化明细。 */
|
||||
interface Scored {
|
||||
valid: boolean;
|
||||
reason: string;
|
||||
score: Measured;
|
||||
fromValidator: Record<string, Measured>;
|
||||
extra: Json;
|
||||
}
|
||||
|
||||
function scoreVerdict(verdict: ValidatorVerdict): Scored {
|
||||
const legal = verdict.legalization;
|
||||
let deduction: Measured;
|
||||
let score: Measured;
|
||||
@@ -328,7 +344,199 @@ function judge(
|
||||
deduction,
|
||||
},
|
||||
};
|
||||
return assemble(runDir, run, hypothesisMetrics, { valid: verdict.valid, reason }, score, fromValidator, extra);
|
||||
return { valid: verdict.valid, reason, score, fromValidator, extra };
|
||||
}
|
||||
|
||||
function judge(
|
||||
_task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
formal: ProcessResult,
|
||||
validator: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgedResult {
|
||||
const run = started(formal);
|
||||
const scored = scoreVerdict(readVerdict(judgeDir, requireExit("校验器", validator, [0, 1])));
|
||||
return assemble(runDir, run, hypothesisMetrics, { valid: scored.valid, reason: scored.reason }, scored.score, scored.fromValidator, scored.extra);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 诊断运行（方法.md 4.3"诊断"一段）：一个容器、一次登记、一次扣次，里面按顺序跑几个实验。
|
||||
// 全部是新增的一套函数，用自己的 harness（harness-diagnosis.py），不碰上面正式运行的任何一步。
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 诊断程序存版本时跳过的名字：正式运行那份加上原始产物目录（它是宿主放进工作目录的，不是实验员的代码）。 */
|
||||
function diagnosisUnversionedNames(facts: TaskFacts): string[] {
|
||||
return [...facts.unversionedNames, BASELINE_ARTIFACT];
|
||||
}
|
||||
|
||||
/**
|
||||
* 把产出当前基线的那次正式运行的原始产物（solution.json）放进 dir 下的 baseline-artifact/，设成只读。
|
||||
* 实验员的工作目录和诊断运行的运行目录都用它：实验员能看到实验要接着算的那份答案，运行时实验读的是同一份。
|
||||
* 只读靠文件权限，防的是实验顺手改了它，不是安全边界。
|
||||
*/
|
||||
function placeBaselineArtifact(_task: Task, baselineRunDir: string, dir: string): void {
|
||||
const target = join(dir, BASELINE_ARTIFACT);
|
||||
mkdirSync(target);
|
||||
const file = join(target, SOLUTION);
|
||||
copyFileSync(join(baselineRunDir, SOLUTION), file);
|
||||
chmodSync(file, 0o444);
|
||||
}
|
||||
|
||||
function prepareDiagnosisRun(task: Task, programDir: string, baselineRunDir: string, runDir: string, experiments: DiagnosisExperiment[]): void {
|
||||
cpSync(programDir, runDir, { recursive: true });
|
||||
// 两份 harness 以宿主的为准；指标说明是全部实验登记的指标合在一起，每个实验只写自己的那几条。
|
||||
copyFileSync(join(task.dir, HARNESS), join(runDir, HARNESS));
|
||||
copyFileSync(join(task.dir, DIAGNOSIS_HARNESS), join(runDir, DIAGNOSIS_HARNESS));
|
||||
writeFileSync(join(runDir, METRICS_SPEC), `${JSON.stringify(experiments.flatMap((e) => e.metrics), null, 2)}\n`);
|
||||
placeBaselineArtifact(task, baselineRunDir, runDir);
|
||||
mkdirSync(join(runDir, ANSWERS));
|
||||
}
|
||||
|
||||
/** 总时限、核数照任务规定，实验按顺序跑，每个实验分到总时限 ÷ 实验数（由 harness 执行）。 */
|
||||
function diagnosisRun(task: Task, imageDigest: string, seed: string, experimentIds: string[]): RunDefinition {
|
||||
const eachSeconds = task.facts.run.wallSeconds / experimentIds.length;
|
||||
const command = [
|
||||
"python",
|
||||
"-B",
|
||||
`${WORKDIR}/${DIAGNOSIS_HARNESS}`,
|
||||
"--program",
|
||||
ENTRY,
|
||||
"--baseline",
|
||||
BASELINE_ARTIFACT,
|
||||
"--answers",
|
||||
ANSWERS,
|
||||
"--seconds-each",
|
||||
String(eachSeconds),
|
||||
"--report",
|
||||
DIAGNOSIS_REPORT,
|
||||
"--experiments",
|
||||
...experimentIds,
|
||||
];
|
||||
return definition(task, imageDigest, command, task.facts.run.wallSeconds, { INFORSI_SEED: seed });
|
||||
}
|
||||
|
||||
/** 一个实验在运行目录里的结局：harness 报告里的一项，或者整次运行先结束、没轮到它。 */
|
||||
type ExperimentOutcome = { ending: "finished" | "crashed" | "timeout"; wallSeconds: number } | { ending: "notRun"; reason: string };
|
||||
|
||||
/** harness 写的报告（我们自己的程序，按约定的格式直接读）。 */
|
||||
interface ReportEntry {
|
||||
id: string;
|
||||
status: "finished" | "crashed" | "timeout";
|
||||
wall_seconds: number;
|
||||
}
|
||||
|
||||
function experimentOutcomes(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
process: Exclude<ProcessResult, { ending: "notStarted" }>,
|
||||
experiments: DiagnosisExperiment[],
|
||||
): ExperimentOutcome[] {
|
||||
const path = join(runDir, DIAGNOSIS_REPORT);
|
||||
const report = existsSync(path) ? (readJson(path) as ReportEntry[]) : [];
|
||||
return experiments.map((e) => {
|
||||
const entry = report.find((r) => r.id === e.id);
|
||||
if (entry !== undefined) return { ending: entry.status, wallSeconds: entry.wall_seconds };
|
||||
const failure = processFailure(task, process);
|
||||
if (failure === null) throw new Error(`诊断运行正常退出，但 ${DIAGNOSIS_REPORT} 里没有实验 ${e.id}`);
|
||||
return { ending: "notRun", reason: `没轮到：整次诊断运行${failure}` };
|
||||
});
|
||||
}
|
||||
|
||||
/** 没有答案可判的实验：不算数，辅助指标和假设专用指标都记测不到并附原因。 */
|
||||
function unjudgedExperiment(
|
||||
id: string,
|
||||
ending: Exclude<DiagnosisEnding, "finished">,
|
||||
wall: Measured,
|
||||
reason: string,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): DiagnosisExperimentJudged {
|
||||
const metrics = {
|
||||
...Object.fromEntries(FROM_VALIDATOR.map((m) => [m, unmeasured(`没有跑校验器：${reason}`)])),
|
||||
...Object.fromEntries(hypothesisMetrics.map((m) => [m.id, unmeasured(`没有结果：${reason}`)])),
|
||||
};
|
||||
return { id, ending, wallSeconds: wall, verdict: "invalid", verdictReason: reason, score: unmeasured(reason), metrics, extra: { legalization: null } };
|
||||
}
|
||||
|
||||
function judgedExperiments(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
experiments: DiagnosisExperiment[],
|
||||
outcomes: ExperimentOutcome[],
|
||||
): DiagnosisJudged {
|
||||
const eachSeconds = task.facts.run.wallSeconds / experiments.length;
|
||||
return {
|
||||
experiments: experiments.map((e, i): DiagnosisExperimentJudged => {
|
||||
const outcome = outcomes[i]!;
|
||||
if (outcome.ending === "notRun") return unjudgedExperiment(e.id, "notRun", unmeasured(outcome.reason), outcome.reason, e.metrics);
|
||||
const wall = measured(outcome.wallSeconds);
|
||||
if (outcome.ending === "timeout") {
|
||||
return unjudgedExperiment(e.id, "timeout", wall, `超过分到的时间 ${eachSeconds} 秒，被结束`, e.metrics);
|
||||
}
|
||||
if (outcome.ending === "crashed") {
|
||||
return unjudgedExperiment(e.id, "crashed", wall, `实验的进程以非零退出码结束（报错见这次运行的 stderr.log）`, e.metrics);
|
||||
}
|
||||
const dir = join(judgeDir, e.id);
|
||||
const exitCode = Number(readFileSync(join(dir, EXIT_FILE), "utf8").trim());
|
||||
const scored = scoreVerdict(readVerdict(dir, exitCode));
|
||||
const hypothesis = readMetricsFile(join(runDir, ANSWERS, e.id, METRICS), `${ANSWERS}/${e.id}/${METRICS}`, e.metrics).hypothesis;
|
||||
return {
|
||||
id: e.id,
|
||||
ending: "finished",
|
||||
wallSeconds: wall,
|
||||
verdict: scored.valid ? "valid" : "invalid",
|
||||
verdictReason: scored.reason,
|
||||
score: scored.score,
|
||||
metrics: { ...scored.fromValidator, ...hypothesis },
|
||||
extra: scored.extra,
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/** 逐个实验判分。校验器一个容器、按顺序对每个有答案的实验各跑一次，退出码各存一个文件。 */
|
||||
const VALIDATE_EACH = `for id in "$@"; do
|
||||
python -B ${VALIDATOR} --circles ${CIRCLES} --solution "$id/${SOLUTION}" --atol ${TOLERANCE} --legalize --verdict "$id/${VERDICT}"
|
||||
echo $? > "$id/${EXIT_FILE}"
|
||||
done`;
|
||||
|
||||
function prepareDiagnosisJudging(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
imageDigest: string,
|
||||
process: ProcessResult,
|
||||
experiments: DiagnosisExperiment[],
|
||||
): DiagnosisJudgingStep {
|
||||
const run = started(process);
|
||||
const outcomes = experimentOutcomes(task, runDir, run, experiments);
|
||||
const finished = experiments.filter((_, i) => outcomes[i]!.ending === "finished");
|
||||
if (finished.length === 0) return { status: "judged", result: judgedExperiments(task, runDir, judgeDir, experiments, outcomes) };
|
||||
for (const e of finished) {
|
||||
mkdirSync(join(judgeDir, e.id));
|
||||
copyFileSync(join(runDir, ANSWERS, e.id, SOLUTION), join(judgeDir, e.id, SOLUTION));
|
||||
}
|
||||
copyFileSync(join(task.dir, VALIDATOR), join(judgeDir, VALIDATOR));
|
||||
const command = ["sh", "-c", VALIDATE_EACH, "inforsi-validate-each", ...finished.map((e) => e.id)];
|
||||
return { status: "validate", definition: definition(task, imageDigest, command, VALIDATOR_SECONDS * finished.length, {}) };
|
||||
}
|
||||
|
||||
function judgeDiagnosis(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
process: ProcessResult,
|
||||
validator: ProcessResult | null,
|
||||
experiments: DiagnosisExperiment[],
|
||||
): DiagnosisJudged {
|
||||
const run = started(process);
|
||||
const outcomes = experimentOutcomes(task, runDir, run, experiments);
|
||||
if (outcomes.some((o) => o.ending === "finished")) {
|
||||
if (validator === null) throw new Error("有实验交出了答案，却没有校验器的运行结果");
|
||||
requireExit("校验器", validator, [0]);
|
||||
}
|
||||
return judgedExperiments(task, runDir, judgeDir, experiments, outcomes);
|
||||
}
|
||||
|
||||
export const circlePacking: TaskImplementation = {
|
||||
@@ -339,4 +547,10 @@ export const circlePacking: TaskImplementation = {
|
||||
readLightCheck,
|
||||
prepareJudging,
|
||||
judge,
|
||||
diagnosisUnversionedNames,
|
||||
placeBaselineArtifact,
|
||||
prepareDiagnosisRun,
|
||||
diagnosisRun,
|
||||
prepareDiagnosisJudging,
|
||||
judgeDiagnosis,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user