import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { test, type TestContext } from "node:test"; import { cpuRatio, fullAccounts, FULL_STOP } from "../eval/lib/full.ts"; import { Ledger } from "../src/ledger/index.ts"; import { cellId, cellNotRunEvent, type Settlement, } from "../src/materials/index.ts"; import { chooseBet, runCellRule, settleAblation, type AblationModelInput, type CellRuleInput, type SettleInput, } from "../src/rules/index.ts"; import { checkTaskFacts } from "../src/tasks/index.ts"; import { createRepository, saveVersion } from "../src/workspace/index.ts"; import { fullConfig } from "./config-helpers.ts"; import { definition, exited, tempDir } from "./ledger-helpers.ts"; import { batch, judged, protocolOf, r, Writer } from "./materials-helpers.ts"; import type { Submission } from "../src/vocab/index.ts"; // eval/full 的账目统计与审阅导出:在真实账本里写一份走完一轮(含消融)的研究,跑 stats 和 export,看产物。 // 选格、结算、押的因子的记录用 rules 的真实函数现算(与 flow 同一个入口),预期值按方法.md 6.6 手算对照。 const facts = checkTaskFacts( JSON.parse( readFileSync( join(import.meta.dirname, "..", "tasks", "circle_packing", "task.json"), "utf8", ), ), ); const TWO_FACTORS: Submission<"proposal"> = { factors: [ { name: "多起点重启", method: "做法一", keySettings: "设定一", continues: null, }, { name: "停滞扰动", method: "做法二", keySettings: "设定二", continues: null, }, ], safeguards: [], metrics: { defined: [], reused: [] }, authorEstimate: r(1.0, 1.1, 1.2), body: "提案正文", }; const EXPLANATIONS = [ { author: "a1", cells: [ { factor: "p1", range: r(1.0, 1.1, 1.2) }, { factor: "p2", range: r(1.15, 1.2, 1.24) }, ], }, { author: "a2", cells: [ { factor: "p1", range: r(0.9, 1.05, 1.2) }, { factor: "p2", range: r(1.1, 1.18, 1.25) }, ], }, ]; const modelInput = (): AblationModelInput => ({ baseline: 0.96, full: 1.25, direction: "maximize", knownBest: facts.knownBest, epsilon: facts.epsilon, lambda: 0.1, factors: ["p1", "p2"], explanations: EXPLANATIONS.map((e, i) => ({ id: `r1-${e.author}-x${1}`, author: e.author, cells: e.cells, })), }); const verificationOf = (off: string): Submission<"ablationVerification"> => ({ factors: [ { factor: "p1", off: off === "p1", where: "solve.py:10-20" }, { factor: "p2", off: off === "p2", where: "solve.py:30-40" }, ], }); /** 一份走完的研究:基线 0.96,第 1 轮采纳两因子的 r1-c1(1.25),消融两格测 1.10、1.20,收尾解封后停在导出。 */ function studyDir(t: TestContext): { dir: string; ledger: Ledger; sealSeq: number; } { const root = tempDir(t); const dir = join(root, "run"); const study = join(dir, "study"); mkdirSync(study, { recursive: true }); const ledger = Ledger.open(join(study, "ledger.sqlite")); t.after(() => ledger.close()); const w = new Writer(ledger); // 预算 4 次:基线、正向、两格消融,走完余额正好是 0。 ledger.append("config", { config: { ...fullConfig(), runs: 4 }, task: facts as never, }); const repo = join(study, "repo.git"); const work = join(root, "work"); createRepository(repo); mkdirSync(work); const version = (text: string) => { writeFileSync(join(work, "solve.py"), text); const result = saveVersion(repo, work, []); assert.ok(result.saved); return result.version; }; ledger.append("code", { object: "base", version: version("value = 0\n"), locations: null, }); w.run("base-1", "baseline", 0, null, judged(0.96, "main")); w.open("r1-proposal", "proposal", 1, ["a1", "a2"]); const proposalSeq = w.sub( "r1-proposal", "a1", "proposal", null, "r1-a1-1", TWO_FACTORS, ); w.close("r1-proposal"); w.decision("dedup", 1, null, { fresh: [{ candidate: "r1-c1", proposals: ["r1-a1-1"] }], carried: [], }); w.protocol( "r1-c1", protocolOf("r1-c1", TWO_FACTORS, 1, "base", ["r1-a1-1"], ["c1", "c2"]), ); w.open("r1-sp", "secondPrediction", 1, ["a1", "a2"]); w.sub("r1-sp", "a1", "secondPrediction", "r1-c1", null, { range: r(1.05, 1.15, 1.25), body: "理由", }); w.sub("r1-sp", "a2", "secondPrediction", "r1-c1", null, { range: r(1.0, 1.1, 1.2), body: "理由", }); w.close("r1-sp"); const b = batch(1, 0.96, { "r1-c1": { a1: r(1.05, 1.15, 1.25), a2: r(1.0, 1.1, 1.2) }, }); w.decision("batch", 1, b.input, b.output); w.open("r1-implementation", "implementation", 1, ["x1"]); w.sub("r1-implementation", "x1", "implementation", "r1-c1", null, { factorLocations: [ { factor: "p1", locations: ["solve.py:10-20"] }, { factor: "p2", locations: ["solve.py:30-40"] }, ], safeguardLocations: [], unassigned: [], concerns: "无", body: "实现说明", }); ledger.append("code", { object: "r1-c1", version: version("value = 1\n"), locations: null, }); w.close("r1-implementation"); w.open("r1-verification-1", "verification", 1, ["v1"]); w.sub("r1-verification-1", "v1", "verification", "r1-c1", null, { clauses: [{ clause: "c1", found: true, where: "solve.py:10" }], factors: [ { factor: "p1", found: true, where: "solve.py:10-20" }, { factor: "p2", found: true, where: "solve.py:30-40" }, ], safeguards: [], unassigned: [], body: "逐项核对", }); w.close("r1-verification-1"); w.run("r1-forward-r1-c1-1", "forward", 1, "r1-c1", judged(1.25, "main")); w.decision("adopt", 1, null, { adopted: "r1-c1" }); w.open("r1-explanation", "explanation", 1, ["a1", "a2"]); for (const e of EXPLANATIONS) { w.sub("r1-explanation", e.author, "explanation", null, null, { explanations: [{ body: `机制(${e.author})`, cells: e.cells }], }); } w.close("r1-explanation"); const cellInput: CellRuleInput = { ...modelInput(), balance: 8, batchCap: 2, u: 0.5, decay: 0.5, seed: "seed", }; const { rule, ...cellOutput } = runCellRule("all", cellInput); ledger.append("decision", { decision: "cells", round: 1, rule: rule.name, ruleVersion: rule.version, params: cellOutput.params, seed: "seed", input: cellInput as never, output: cellOutput as never, }); w.open(`r1-ablationImplementation-1`, "ablationImplementation", 1, ["x1"]); for (const [key, text] of [ ["p1", "value = 2\n"], ["p2", "value = 3\n"], ] as const) { const cell = cellId("r1-c1", key); w.sub( `r1-ablationImplementation-1`, "x1", "ablationImplementation", cell, null, { alsoOff: [], cannotRun: null, concerns: "无" }, ); ledger.append("code", { object: cell, version: version(text), locations: null, }); } w.close(`r1-ablationImplementation-1`); w.open(`r1-ablationVerification-1`, "ablationVerification", 1, ["v1"]); for (const key of ["p1", "p2"] as const) { w.sub( `r1-ablationVerification-1`, "v1", "ablationVerification", cellId("r1-c1", key), null, verificationOf(key), ); } w.close(`r1-ablationVerification-1`); for (const [key, score] of [ ["p1", 1.1], ["p2", 1.2], ] as const) { const cell = cellId("r1-c1", key); const addRun = (id: string) => w.run(id, "ablation", 1, cell, judged(score, "main")); addRun(`r1-ablation-${cell}-1`); } const settleInput: SettleInput = { ...modelInput(), cells: [ { factor: "p1", outcome: { kind: "valid", score: 1.1 } }, { factor: "p2", outcome: { kind: "valid", score: 1.2 } }, ], }; const settled: Settlement = { candidate: "r1-c1", input: settleInput, output: settleAblation(settleInput), }; ledger.append("settlement", { round: 1, result: settled as never }); const betInput = { ...modelInput(), cells: settleInput.cells }; w.decision("bet", 1, betInput, chooseBet(betInput)); w.open("r1-wrapUp", "wrapUp", 1, ["a1", "a2"]); w.sub("r1-wrapUp", "a1", "wrapUp", null, null, { body: "甲的收尾:p1 撑了大半", }); w.sub("r1-wrapUp", "a2", "wrapUp", null, null, { body: "乙的收尾" }); w.close("r1-wrapUp"); ledger.append("unseal", { candidate: "r1-c1", seals: [proposalSeq + 1] }); ledger.append("event", { event: "halted", reason: FULL_STOP, objects: [] }); return { dir, ledger, sealSeq: proposalSeq + 1 }; } test("full 账目:一轮含消融的研究全部核对通过;撑几成、押的因子按手算对上", (t) => { const { ledger } = studyDir(t); const accounts = fullAccounts(ledger.records()); assert.deepEqual(accounts.confirmedStarts, { total: 4, baseline: 1, forward: 1, ablation: 2, }); assert.deepEqual(accounts.balance, { charged: 4, reserved: 0, balance: 0 }); assert.ok( Object.values(accounts.checks).every(Boolean), JSON.stringify(accounts.checks), ); const round = accounts.rounds[0]!; assert.equal(round.adopted, "r1-c1"); assert.equal(round.adoptedScore, 1.25); assert.ok(Math.abs(round.knownBestGap! - (2.635983 - 1.25)) < 1e-12); assert.equal(round.cells.length, 2); // 撑几成(方法.md 6.6):(1.25 − 1.10) / 0.29 与 (1.25 − 1.20) / 0.29。 assert.ok(Math.abs(round.cells[0]!.support! - 0.15 / 0.29) < 1e-12); assert.ok(Math.abs(round.cells[1]!.support! - 0.05 / 0.29) < 1e-12); assert.equal(round.label!.label, "substitute"); // i = 0.690 − 1 < −0.2 assert.equal(round.bet!.decision, "bet"); assert.equal(round.bet!.factor, "p1"); assert.equal(round.unseal.pass, true); assert.equal(round.explanationBeforeCells, true); assert.deepEqual( round.candidates[0]!.authorEstimates.map((u) => u.range), [r(1.0, 1.1, 1.2)], ); }); test("full 账目:缺解封、不跑的格被扣次都过不了核对", (t) => { const { ledger } = studyDir(t); const before = fullAccounts(ledger.records()); assert.equal(before.checks.unsealAfterWrapUp, true); // 去掉解封记录重建一份账本太重,这里反过来验:另写一份没有解封的账。 const plain = ledger .records() .filter((x) => x.kind !== "unseal" && x.kind !== "event"); assert.equal(fullAccounts(plain).checks.unsealAfterWrapUp, false); assert.equal(fullAccounts(plain).checks.haltedAtExport, false); // "不跑"的格被扣次:给它补一条运行登记和启动,silentCellsNotCharged 应变 false。 ledger.append( "event", cellNotRunEvent(cellId("r1-c1", "p2"), "测试:核验不通过"), ); ledger.append("runPlanned", { run: "r1-ablation-r1-c1-p2-2", purpose: "ablation", round: 1, object: "r1-c1-p2", definition, }); ledger.append("runStarted", { run: "r1-ablation-r1-c1-p2-2" }); const after = fullAccounts(ledger.records()); assert.equal(after.checks.silentCellsNotCharged, false); assert.equal(after.checks.chargedEqualsStarts, true); // 账目本身仍平:5 次启动 5 次扣 }); test("full stats/export:从研究输出目录产出 stats.json/txt 和逐轮审阅材料", (t) => { const { dir } = studyDir(t); const statsOut = join(tempDir(t), "stats"); const stats = spawnSync( process.execPath, ["eval/full/stats.ts", statsOut, dir], { encoding: "utf8" }, ); assert.equal(stats.status, 0, stats.stderr); const text = readFileSync(join(statsOut, "stats.txt"), "utf8"); assert.match(text, /确认启动:基线 1、正向 1、消融 2,共 4;扣次 4,余额 0/); assert.match(text, /采纳:r1-c1,分数 1\.250000,离已知最好 1\.385983/); assert.match( text, /r1-c1-p1:核验 1 次、重做 0 次,跑没跑 跑了(扣次 1),实测 1\.100000,撑几成 0\.517/, ); assert.match(text, /押的因子 p1/); const summaries = JSON.parse( readFileSync(join(statsOut, "stats.json"), "utf8"), ); assert.equal(summaries[0].pass, true); assert.equal(summaries[0].kill, null); assert.equal(summaries[0].ratios.length, 4); assert.ok(summaries[0].ratios.every((x: { flagged: boolean }) => !x.flagged)); const review = join(tempDir(t), "review"); const exported = spawnSync( process.execPath, ["eval/full/export.ts", dir, review], { encoding: "utf8" }, ); assert.equal(exported.status, 0, exported.stderr); const roundDir = join(review, "round-1"); assert.match( readFileSync(join(roundDir, "explanations.txt"), "utf8"), /机制(a1)/, ); assert.match( readFileSync(join(roundDir, "wrapups.txt"), "utf8"), /甲的收尾:p1 撑了大半/, ); assert.match( readFileSync(join(roundDir, "proposals.txt"), "utf8"), /多起点重启/, ); assert.equal( readFileSync( join(roundDir, "cells", "r1-c1-p1", "full", "solve.py"), "utf8", ), "value = 1\n", ); assert.equal( readFileSync( join(roundDir, "cells", "r1-c1-p1", "cell", "solve.py"), "utf8", ), "value = 2\n", ); const verifications = JSON.parse( readFileSync( join(roundDir, "cells", "r1-c1-p1", "verifications.json"), "utf8", ), ); assert.equal(verifications[0].outcome.passed, true); const settlement = JSON.parse( readFileSync(join(roundDir, "settlement.json"), "utf8"), ); assert.equal(settlement.bet.content.output.factor, "p1"); assert.match(readFileSync(join(roundDir, "review.txt"), "utf8"), /第3样/); }); test("full CPU/墙钟比:测不到或没启动记 null,正常退出算出比值", () => { assert.equal(cpuRatio({ process: { ending: "notStarted" } }), null); assert.ok(Math.abs(cpuRatio({ process: exited })! - 100 / 101) < 1e-12); assert.equal( cpuRatio({ process: { ending: "exited", cpuSeconds: { status: "unmeasured" }, wallSeconds: { status: "measured", value: 1 }, }, }), null, ); });