eval: 完整研究的运行、统计与审阅导出（任务书 17 第三块的检验代码）
- eval/full/run.ts：配置照 eval/round-zero/research.json 只换编号，真实跑到停下； launches/machine/timeline/outcome 照任务书 16 惯例；--kill-after-first-ablation-start 在第一个消融运行确认启动后、结果落账前 SIGKILL 控制进程，恢复续跑到底， kill-report.json 写三条检查（同一容器不重跑、只扣一次、结果照常写入）。 - eval/lib/full.ts：账目核对 fullAccounts——逐轮走向（候选分数、采纳、选格、每格 核验重做次数与扣次、结算、押的因子）加 12 项程序账目检查（阶段开关成对、 扣次=确认启动数、余额为 0、解释冻结早于首格启动、每轮最多一条结算、解封在 收尾关闭之后且覆盖全部选中候选、跑不起来/不跑的格零扣次、第 2 轮起基线链 重算、停在"导出还没写"等）与 CPU/墙钟比（<0.9 列出并附同时段负载，只列事实）。 - eval/full/stats.ts、export.ts：统计输出与按轮按格的审阅材料导出。 - 不沿用 eval/forward/acceptance.ts 的"启动区间互不重叠"（这次 3 份同时跑）。 - tests/full-eval.test.ts 4 条：真实账本 + rules 真实计算走完一轮含消融的研究， 撑几成与配合关系标签按方法.md 6.6 手算对照。npm run check 通过，npm test 476 条通过。
This commit is contained in:
@@ -0,0 +1,448 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { test, type TestContext } from "node:test";
|
||||
import { cpuRatio, fullAccounts, FULL_STOP } from "../eval/lib/full.ts";
|
||||
import { Ledger } from "../src/ledger/index.ts";
|
||||
import {
|
||||
cellId,
|
||||
cellNotRunEvent,
|
||||
type Settlement,
|
||||
} from "../src/materials/index.ts";
|
||||
import {
|
||||
chooseBet,
|
||||
runCellRule,
|
||||
settleAblation,
|
||||
type AblationModelInput,
|
||||
type CellRuleInput,
|
||||
type SettleInput,
|
||||
} from "../src/rules/index.ts";
|
||||
import { checkTaskFacts } from "../src/tasks/index.ts";
|
||||
import { createRepository, saveVersion } from "../src/workspace/index.ts";
|
||||
import { fullConfig } from "./config-helpers.ts";
|
||||
import { definition, exited, tempDir } from "./ledger-helpers.ts";
|
||||
import { batch, judged, protocolOf, r, Writer } from "./materials-helpers.ts";
|
||||
import type { Submission } from "../src/vocab/index.ts";
|
||||
|
||||
// eval/full 的账目统计与审阅导出：在真实账本里写一份走完一轮（含消融）的研究，跑 stats 和 export，看产物。
|
||||
// 选格、结算、押的因子的记录用 rules 的真实函数现算（与 flow 同一个入口），预期值按方法.md 6.6 手算对照。
|
||||
|
||||
const facts = checkTaskFacts(
|
||||
JSON.parse(
|
||||
readFileSync(
|
||||
join(import.meta.dirname, "..", "tasks", "circle_packing", "task.json"),
|
||||
"utf8",
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const TWO_FACTORS: Submission<"proposal"> = {
|
||||
factors: [
|
||||
{
|
||||
name: "多起点重启",
|
||||
method: "做法一",
|
||||
keySettings: "设定一",
|
||||
continues: null,
|
||||
},
|
||||
{
|
||||
name: "停滞扰动",
|
||||
method: "做法二",
|
||||
keySettings: "设定二",
|
||||
continues: null,
|
||||
},
|
||||
],
|
||||
safeguards: [],
|
||||
metrics: { defined: [], reused: [] },
|
||||
authorEstimate: r(1.0, 1.1, 1.2),
|
||||
body: "提案正文",
|
||||
};
|
||||
|
||||
const EXPLANATIONS = [
|
||||
{
|
||||
author: "a1",
|
||||
cells: [
|
||||
{ factor: "p1", range: r(1.0, 1.1, 1.2) },
|
||||
{ factor: "p2", range: r(1.15, 1.2, 1.24) },
|
||||
],
|
||||
},
|
||||
{
|
||||
author: "a2",
|
||||
cells: [
|
||||
{ factor: "p1", range: r(0.9, 1.05, 1.2) },
|
||||
{ factor: "p2", range: r(1.1, 1.18, 1.25) },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const modelInput = (): AblationModelInput => ({
|
||||
baseline: 0.96,
|
||||
full: 1.25,
|
||||
direction: "maximize",
|
||||
knownBest: facts.knownBest,
|
||||
epsilon: facts.epsilon,
|
||||
lambda: 0.1,
|
||||
factors: ["p1", "p2"],
|
||||
explanations: EXPLANATIONS.map((e, i) => ({
|
||||
id: `r1-${e.author}-x${1}`,
|
||||
author: e.author,
|
||||
cells: e.cells,
|
||||
})),
|
||||
});
|
||||
|
||||
const verificationOf = (off: string): Submission<"ablationVerification"> => ({
|
||||
factors: [
|
||||
{ factor: "p1", off: off === "p1", where: "solve.py:10-20" },
|
||||
{ factor: "p2", off: off === "p2", where: "solve.py:30-40" },
|
||||
],
|
||||
});
|
||||
|
||||
/** 一份走完的研究：基线 0.96，第 1 轮采纳两因子的 r1-c1（1.25），消融两格测 1.10、1.20，收尾解封后停在导出。 */
|
||||
function studyDir(t: TestContext): {
|
||||
dir: string;
|
||||
ledger: Ledger;
|
||||
sealSeq: number;
|
||||
} {
|
||||
const root = tempDir(t);
|
||||
const dir = join(root, "run");
|
||||
const study = join(dir, "study");
|
||||
mkdirSync(study, { recursive: true });
|
||||
const ledger = Ledger.open(join(study, "ledger.sqlite"));
|
||||
t.after(() => ledger.close());
|
||||
const w = new Writer(ledger);
|
||||
// 预算 4 次：基线、正向、两格消融，走完余额正好是 0。
|
||||
ledger.append("config", {
|
||||
config: { ...fullConfig(), runs: 4 },
|
||||
task: facts as never,
|
||||
});
|
||||
|
||||
const repo = join(study, "repo.git");
|
||||
const work = join(root, "work");
|
||||
createRepository(repo);
|
||||
mkdirSync(work);
|
||||
const version = (text: string) => {
|
||||
writeFileSync(join(work, "solve.py"), text);
|
||||
const result = saveVersion(repo, work, []);
|
||||
assert.ok(result.saved);
|
||||
return result.version;
|
||||
};
|
||||
|
||||
ledger.append("code", {
|
||||
object: "base",
|
||||
version: version("value = 0\n"),
|
||||
locations: null,
|
||||
});
|
||||
w.run("base-1", "baseline", 0, null, judged(0.96, "main"));
|
||||
|
||||
w.open("r1-proposal", "proposal", 1, ["a1", "a2"]);
|
||||
const proposalSeq = w.sub(
|
||||
"r1-proposal",
|
||||
"a1",
|
||||
"proposal",
|
||||
null,
|
||||
"r1-a1-1",
|
||||
TWO_FACTORS,
|
||||
);
|
||||
w.close("r1-proposal");
|
||||
w.decision("dedup", 1, null, {
|
||||
fresh: [{ candidate: "r1-c1", proposals: ["r1-a1-1"] }],
|
||||
carried: [],
|
||||
});
|
||||
w.protocol(
|
||||
"r1-c1",
|
||||
protocolOf("r1-c1", TWO_FACTORS, 1, "base", ["r1-a1-1"], ["c1", "c2"]),
|
||||
);
|
||||
|
||||
w.open("r1-sp", "secondPrediction", 1, ["a1", "a2"]);
|
||||
w.sub("r1-sp", "a1", "secondPrediction", "r1-c1", null, {
|
||||
range: r(1.05, 1.15, 1.25),
|
||||
body: "理由",
|
||||
});
|
||||
w.sub("r1-sp", "a2", "secondPrediction", "r1-c1", null, {
|
||||
range: r(1.0, 1.1, 1.2),
|
||||
body: "理由",
|
||||
});
|
||||
w.close("r1-sp");
|
||||
const b = batch(1, 0.96, {
|
||||
"r1-c1": { a1: r(1.05, 1.15, 1.25), a2: r(1.0, 1.1, 1.2) },
|
||||
});
|
||||
w.decision("batch", 1, b.input, b.output);
|
||||
|
||||
w.open("r1-implementation", "implementation", 1, ["x1"]);
|
||||
w.sub("r1-implementation", "x1", "implementation", "r1-c1", null, {
|
||||
factorLocations: [
|
||||
{ factor: "p1", locations: ["solve.py:10-20"] },
|
||||
{ factor: "p2", locations: ["solve.py:30-40"] },
|
||||
],
|
||||
safeguardLocations: [],
|
||||
unassigned: [],
|
||||
concerns: "无",
|
||||
body: "实现说明",
|
||||
});
|
||||
ledger.append("code", {
|
||||
object: "r1-c1",
|
||||
version: version("value = 1\n"),
|
||||
locations: null,
|
||||
});
|
||||
w.close("r1-implementation");
|
||||
w.open("r1-verification-1", "verification", 1, ["v1"]);
|
||||
w.sub("r1-verification-1", "v1", "verification", "r1-c1", null, {
|
||||
clauses: [{ clause: "c1", found: true, where: "solve.py:10" }],
|
||||
factors: [
|
||||
{ factor: "p1", found: true, where: "solve.py:10-20" },
|
||||
{ factor: "p2", found: true, where: "solve.py:30-40" },
|
||||
],
|
||||
safeguards: [],
|
||||
unassigned: [],
|
||||
body: "逐项核对",
|
||||
});
|
||||
w.close("r1-verification-1");
|
||||
w.run("r1-forward-r1-c1-1", "forward", 1, "r1-c1", judged(1.25, "main"));
|
||||
w.decision("adopt", 1, null, { adopted: "r1-c1" });
|
||||
|
||||
w.open("r1-explanation", "explanation", 1, ["a1", "a2"]);
|
||||
for (const e of EXPLANATIONS) {
|
||||
w.sub("r1-explanation", e.author, "explanation", null, null, {
|
||||
explanations: [{ body: `机制（${e.author}）`, cells: e.cells }],
|
||||
});
|
||||
}
|
||||
w.close("r1-explanation");
|
||||
|
||||
const cellInput: CellRuleInput = {
|
||||
...modelInput(),
|
||||
balance: 8,
|
||||
batchCap: 2,
|
||||
u: 0.5,
|
||||
decay: 0.5,
|
||||
seed: "seed",
|
||||
};
|
||||
const { rule, ...cellOutput } = runCellRule("all", cellInput);
|
||||
ledger.append("decision", {
|
||||
decision: "cells",
|
||||
round: 1,
|
||||
rule: rule.name,
|
||||
ruleVersion: rule.version,
|
||||
params: cellOutput.params,
|
||||
seed: "seed",
|
||||
input: cellInput as never,
|
||||
output: cellOutput as never,
|
||||
});
|
||||
|
||||
w.open(`r1-ablationImplementation-1`, "ablationImplementation", 1, ["x1"]);
|
||||
for (const [key, text] of [
|
||||
["p1", "value = 2\n"],
|
||||
["p2", "value = 3\n"],
|
||||
] as const) {
|
||||
const cell = cellId("r1-c1", key);
|
||||
w.sub(
|
||||
`r1-ablationImplementation-1`,
|
||||
"x1",
|
||||
"ablationImplementation",
|
||||
cell,
|
||||
null,
|
||||
{ alsoOff: [], cannotRun: null, concerns: "无" },
|
||||
);
|
||||
ledger.append("code", {
|
||||
object: cell,
|
||||
version: version(text),
|
||||
locations: null,
|
||||
});
|
||||
}
|
||||
w.close(`r1-ablationImplementation-1`);
|
||||
w.open(`r1-ablationVerification-1`, "ablationVerification", 1, ["v1"]);
|
||||
for (const key of ["p1", "p2"] as const) {
|
||||
w.sub(
|
||||
`r1-ablationVerification-1`,
|
||||
"v1",
|
||||
"ablationVerification",
|
||||
cellId("r1-c1", key),
|
||||
null,
|
||||
verificationOf(key),
|
||||
);
|
||||
}
|
||||
w.close(`r1-ablationVerification-1`);
|
||||
for (const [key, score] of [
|
||||
["p1", 1.1],
|
||||
["p2", 1.2],
|
||||
] as const) {
|
||||
const cell = cellId("r1-c1", key);
|
||||
const addRun = (id: string) =>
|
||||
w.run(id, "ablation", 1, cell, judged(score, "main"));
|
||||
addRun(`r1-ablation-${cell}-1`);
|
||||
}
|
||||
|
||||
const settleInput: SettleInput = {
|
||||
...modelInput(),
|
||||
cells: [
|
||||
{ factor: "p1", outcome: { kind: "valid", score: 1.1 } },
|
||||
{ factor: "p2", outcome: { kind: "valid", score: 1.2 } },
|
||||
],
|
||||
};
|
||||
const settled: Settlement = {
|
||||
candidate: "r1-c1",
|
||||
input: settleInput,
|
||||
output: settleAblation(settleInput),
|
||||
};
|
||||
ledger.append("settlement", { round: 1, result: settled as never });
|
||||
const betInput = { ...modelInput(), cells: settleInput.cells };
|
||||
w.decision("bet", 1, betInput, chooseBet(betInput));
|
||||
|
||||
w.open("r1-wrapUp", "wrapUp", 1, ["a1", "a2"]);
|
||||
w.sub("r1-wrapUp", "a1", "wrapUp", null, null, {
|
||||
body: "甲的收尾：p1 撑了大半",
|
||||
});
|
||||
w.sub("r1-wrapUp", "a2", "wrapUp", null, null, { body: "乙的收尾" });
|
||||
w.close("r1-wrapUp");
|
||||
ledger.append("unseal", { candidate: "r1-c1", seals: [proposalSeq + 1] });
|
||||
ledger.append("event", { event: "halted", reason: FULL_STOP, objects: [] });
|
||||
return { dir, ledger, sealSeq: proposalSeq + 1 };
|
||||
}
|
||||
|
||||
test("full 账目：一轮含消融的研究全部核对通过；撑几成、押的因子按手算对上", (t) => {
|
||||
const { ledger } = studyDir(t);
|
||||
const accounts = fullAccounts(ledger.records());
|
||||
assert.deepEqual(accounts.confirmedStarts, {
|
||||
total: 4,
|
||||
baseline: 1,
|
||||
forward: 1,
|
||||
ablation: 2,
|
||||
});
|
||||
assert.deepEqual(accounts.balance, { charged: 4, reserved: 0, balance: 0 });
|
||||
assert.ok(
|
||||
Object.values(accounts.checks).every(Boolean),
|
||||
JSON.stringify(accounts.checks),
|
||||
);
|
||||
const round = accounts.rounds[0]!;
|
||||
assert.equal(round.adopted, "r1-c1");
|
||||
assert.equal(round.adoptedScore, 1.25);
|
||||
assert.ok(Math.abs(round.knownBestGap! - (2.635983 - 1.25)) < 1e-12);
|
||||
assert.equal(round.cells.length, 2);
|
||||
// 撑几成（方法.md 6.6）：(1.25 − 1.10) / 0.29 与 (1.25 − 1.20) / 0.29。
|
||||
assert.ok(Math.abs(round.cells[0]!.support! - 0.15 / 0.29) < 1e-12);
|
||||
assert.ok(Math.abs(round.cells[1]!.support! - 0.05 / 0.29) < 1e-12);
|
||||
assert.equal(round.label!.label, "substitute"); // i = 0.690 − 1 < −0.2
|
||||
assert.equal(round.bet!.decision, "bet");
|
||||
assert.equal(round.bet!.factor, "p1");
|
||||
assert.equal(round.unseal.pass, true);
|
||||
assert.equal(round.explanationBeforeCells, true);
|
||||
assert.deepEqual(
|
||||
round.candidates[0]!.authorEstimates.map((u) => u.range),
|
||||
[r(1.0, 1.1, 1.2)],
|
||||
);
|
||||
});
|
||||
|
||||
test("full 账目：缺解封、不跑的格被扣次都过不了核对", (t) => {
|
||||
const { ledger } = studyDir(t);
|
||||
const before = fullAccounts(ledger.records());
|
||||
assert.equal(before.checks.unsealAfterWrapUp, true);
|
||||
// 去掉解封记录重建一份账本太重，这里反过来验：另写一份没有解封的账。
|
||||
const plain = ledger
|
||||
.records()
|
||||
.filter((x) => x.kind !== "unseal" && x.kind !== "event");
|
||||
assert.equal(fullAccounts(plain).checks.unsealAfterWrapUp, false);
|
||||
assert.equal(fullAccounts(plain).checks.haltedAtExport, false);
|
||||
// "不跑"的格被扣次：给它补一条运行登记和启动，silentCellsNotCharged 应变 false。
|
||||
ledger.append(
|
||||
"event",
|
||||
cellNotRunEvent(cellId("r1-c1", "p2"), "测试：核验不通过"),
|
||||
);
|
||||
ledger.append("runPlanned", {
|
||||
run: "r1-ablation-r1-c1-p2-2",
|
||||
purpose: "ablation",
|
||||
round: 1,
|
||||
object: "r1-c1-p2",
|
||||
definition,
|
||||
});
|
||||
ledger.append("runStarted", { run: "r1-ablation-r1-c1-p2-2" });
|
||||
const after = fullAccounts(ledger.records());
|
||||
assert.equal(after.checks.silentCellsNotCharged, false);
|
||||
assert.equal(after.checks.chargedEqualsStarts, true); // 账目本身仍平：5 次启动 5 次扣
|
||||
});
|
||||
|
||||
test("full stats/export：从研究输出目录产出 stats.json/txt 和逐轮审阅材料", (t) => {
|
||||
const { dir } = studyDir(t);
|
||||
const statsOut = join(tempDir(t), "stats");
|
||||
const stats = spawnSync(
|
||||
process.execPath,
|
||||
["eval/full/stats.ts", statsOut, dir],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
assert.equal(stats.status, 0, stats.stderr);
|
||||
const text = readFileSync(join(statsOut, "stats.txt"), "utf8");
|
||||
assert.match(text, /确认启动：基线 1、正向 1、消融 2，共 4；扣次 4，余额 0/);
|
||||
assert.match(text, /采纳：r1-c1，分数 1\.250000，离已知最好 1\.385983/);
|
||||
assert.match(
|
||||
text,
|
||||
/r1-c1-p1：核验 1 次、重做 0 次，跑没跑 跑了（扣次 1），实测 1\.100000，撑几成 0\.517/,
|
||||
);
|
||||
assert.match(text, /押的因子 p1/);
|
||||
const summaries = JSON.parse(
|
||||
readFileSync(join(statsOut, "stats.json"), "utf8"),
|
||||
);
|
||||
assert.equal(summaries[0].pass, true);
|
||||
assert.equal(summaries[0].kill, null);
|
||||
assert.equal(summaries[0].ratios.length, 4);
|
||||
assert.ok(summaries[0].ratios.every((x: { flagged: boolean }) => !x.flagged));
|
||||
|
||||
const review = join(tempDir(t), "review");
|
||||
const exported = spawnSync(
|
||||
process.execPath,
|
||||
["eval/full/export.ts", dir, review],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
assert.equal(exported.status, 0, exported.stderr);
|
||||
const roundDir = join(review, "round-1");
|
||||
assert.match(
|
||||
readFileSync(join(roundDir, "explanations.txt"), "utf8"),
|
||||
/机制（a1）/,
|
||||
);
|
||||
assert.match(
|
||||
readFileSync(join(roundDir, "wrapups.txt"), "utf8"),
|
||||
/甲的收尾：p1 撑了大半/,
|
||||
);
|
||||
assert.match(
|
||||
readFileSync(join(roundDir, "proposals.txt"), "utf8"),
|
||||
/多起点重启/,
|
||||
);
|
||||
assert.equal(
|
||||
readFileSync(
|
||||
join(roundDir, "cells", "r1-c1-p1", "full", "solve.py"),
|
||||
"utf8",
|
||||
),
|
||||
"value = 1\n",
|
||||
);
|
||||
assert.equal(
|
||||
readFileSync(
|
||||
join(roundDir, "cells", "r1-c1-p1", "cell", "solve.py"),
|
||||
"utf8",
|
||||
),
|
||||
"value = 2\n",
|
||||
);
|
||||
const verifications = JSON.parse(
|
||||
readFileSync(
|
||||
join(roundDir, "cells", "r1-c1-p1", "verifications.json"),
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
assert.equal(verifications[0].outcome.passed, true);
|
||||
const settlement = JSON.parse(
|
||||
readFileSync(join(roundDir, "settlement.json"), "utf8"),
|
||||
);
|
||||
assert.equal(settlement.bet.content.output.factor, "p1");
|
||||
assert.match(readFileSync(join(roundDir, "review.txt"), "utf8"), /第3样/);
|
||||
});
|
||||
|
||||
test("full CPU/墙钟比：测不到或没启动记 null，正常退出算出比值", () => {
|
||||
assert.equal(cpuRatio({ process: { ending: "notStarted" } }), null);
|
||||
assert.ok(Math.abs(cpuRatio({ process: exited })! - 100 / 101) < 1e-12);
|
||||
assert.equal(
|
||||
cpuRatio({
|
||||
process: {
|
||||
ending: "exited",
|
||||
cpuSeconds: { status: "unmeasured" },
|
||||
wallSeconds: { status: "measured", value: 1 },
|
||||
},
|
||||
}),
|
||||
null,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user