import assert from "node:assert/strict"; import { mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { test, type TestContext } from "node:test"; import { adoptBatch } from "../src/flow/steps/forward.ts"; import { prepareSubmission, taskCode } from "../src/flow/task-code.ts"; import type { Study } from "../src/flow/study.ts"; import { Board, buildMaterial, protocolMetrics, type FrozenProtocol } from "../src/materials/index.ts"; import { Ledger } from "../src/ledger/index.ts"; import { loadTask } from "../src/tasks/index.ts"; import { checkoutVersion, createRepository, saveVersion } from "../src/workspace/index.ts"; import type { Submission } from "../src/vocab/index.ts"; import { fullConfig } from "./config-helpers.ts"; import { implementation, tempDir } from "./ledger-helpers.ts"; import { judged, prompts, study, Writer } from "./materials-helpers.ts"; function setup(t: TestContext): { s: Study; w: Writer } { const dir = tempDir(t); const ledger = Ledger.open(join(dir, "ledger.sqlite")); t.after(() => ledger.close()); const task = loadTask(join(import.meta.dirname, "..", "tasks", "circle_packing")); const s = { dir, ledger, task, prompts, facts: study, frozen: { config: fullConfig(), facts: task.facts } } as Study; return { s, w: new Writer(ledger) }; } // 预期值手算:基线 1,ε=1/64,已知最好 2;线=1/64。1+1/64 正好在线上,不采纳。 for (const [label, scores, adopted, gain] of [ ["高于线且同分:取自然编号小的", [1.125, 1.125], "r1-c2", 0.125], ["正好等于线:也写 null 决定", [1.015625, 1], null, 0.015625], ["整批没有有效分数:也写 null 决定", [], null, null], ] as const) { test(`采纳写账:${label},写决定后整批公开`, (t) => { const { s, w } = setup(t); s.frozen.facts = { ...s.frozen.facts, epsilon: 0.015625, knownBest: 2 }; w.run("base-1", "baseline", 0, null, judged(1, "main")); w.decision("batch", 1, null, { chosen: ["r1-c10", "r1-c2", "r1-c3"] }); scores.forEach((score, i) => w.run(["run-first", "run-second"][i]!, "forward", 1, i === 0 ? "r1-c10" : "r1-c2", judged(score, "main"))); w.run("invalid", "forward", 1, "r1-c3", { ...judged(99, "main"), verdict: "invalid", verdictReason: "不合法" }); // 不在本批的运行不参与,即使它分数更高;上轮不能混进本轮。 w.run("other", "forward", 0, "old-candidate", judged(100, "main")); const before = new Board(s.ledger.records()); assert.ok(before.runs().filter((r) => r.plan.round === 1).every((r) => !before.runPublic(r))); adoptBatch(s, 1); const record = s.ledger.records().at(-1)!; assert.equal(record.kind, "decision"); if (record.kind !== "decision") throw new Error("应写采纳决定"); assert.deepEqual(record.content, { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: { direction: "maximize", epsilon: 0.015625, knownBest: 2 }, seed: null, input: { candidates: scores.map((score, i) => ({ id: i === 0 ? "r1-c10" : "r1-c2", score })), baseline: 1, direction: "maximize", epsilon: 0.015625, knownBest: 2 }, output: { line: 0.015625, quarter: 0.25, adopted, best: scores.length === 0 ? null : { id: adopted ?? "r1-c10", score: scores[0], improvement: gain } }, }); const after = new Board(s.ledger.records()); assert.ok(after.runs().filter((r) => r.plan.round === 1).every((r) => after.runPublic(r))); assert.equal(after.baselineOf(2), adopted ?? "base"); }); } const metric = { id: "old-c.metric", meaning: "试验值", unit: "次", direction: "maximize" as const, howMeasured: "按原登记计数", missing: "invalidateRun" as const }; const oldProtocol: FrozenProtocol = { round: 0, baseline: "base", proposals: [], factors: [], safeguards: [], metrics: { defined: [metric], reused: [] }, clauses: [], body: "原登记" }; const protocol: FrozenProtocol = { round: 1, baseline: "base", proposals: [], factors: [{ name: "改动", method: "照协议改动", keySettings: "无", continues: null }], safeguards: [], metrics: { defined: [], reused: [metric.id] }, clauses: ["照协议"], body: "冻结正文" }; function codeStudy(t: TestContext) { const { s, w } = setup(t); const repo = join(s.dir, "repo.git"); createRepository(repo); const workDir = join(s.dir, "work"); mkdirSync(workDir); writeFileSync(join(workDir, "initial.py"), "first = 1\nsecond = 2\n"); writeFileSync(join(workDir, "evaluate.py"), "protected = True\n"); const baseline = saveVersion(repo, workDir, []); if (!baseline.saved) throw new Error(baseline.reason); s.ledger.append("code", { object: "base", version: baseline.version, locations: null }); w.protocol("old-c", oldProtocol); w.protocol("r1-c1", protocol); w.open("r1-implementation", "implementation", 1, ["experimenter-r1-c1"]); return { s, w, repo, workDir, baseline: baseline.version }; } const impl: Submission<"implementation"> = { ...implementation("实现说明"), factorLocations: [{ factor: "p1", locations: ["initial.py:1-2"] }] }; const ref = { actor: "experimenter-r1-c1", object: "r1-c1", kind: "implementation" as const, stageId: "r1-implementation" }; test("提交准备:快照、保护文件、差异块和引用指标同源;修复取上版,但差异永远相对本批基线", (t) => { const { s, w, repo, workDir, baseline } = codeStudy(t); assert.deepEqual(taskCode(s, ref), { changes: null, startVersion: baseline }); writeFileSync(join(workDir, "initial.py"), "first = 10\nsecond = 2\n"); const prepared = prepareSubmission(s, ref, impl, 0, workDir); assert.ok(prepared.code); const known = prepared.known as { code: { saved: true; protectedFiles: string[]; hunks: { text: string }[]; protocol: unknown } }; assert.deepEqual(known.code.protectedFiles, []); assert.deepEqual(known.code.protocol, { body: "冻结正文", metrics: [metric] }); assert.match(known.code.hunks[0]!.text, /-first = 1\n\+first = 10/); s.ledger.transaction(() => { w.sub(ref.stageId, ref.actor, ref.kind, ref.object, null, impl); s.ledger.append("code", prepared.code!); }); w.close(ref.stageId); w.open("r1-verification-1", "verification", 1, ["verifier-r1-c1"]); w.sub("r1-verification-1", "verifier-r1-c1", "verification", "r1-c1", null, { clauses: [{ clause: "c1", found: false, where: "第二行未实现" }], factors: [{ factor: "p1", found: false, where: "第二行未实现" }], safeguards: [], unassigned: [], body: "只修第二行" }); w.close("r1-verification-1"); w.open("r1-repair-1", "implementation", 1, [ref.actor]); const repair = { ...ref, kind: "repair" as const, stageId: "r1-repair-1" }; const previous = taskCode(s, repair); assert.equal(previous.startVersion, prepared.code.version); assert.match(previous.changes!.diff, /-first = 1\n\+first = 10/); const repairMaterial = buildMaterial({ ...repair, ledger: s.ledger, study, prompts, codeRoot: join(s.dir, "readable"), changes: previous.changes, artifacts: null }); assert.match(repairMaterial.firstMessage, /工作目录是上一次提交的代码的副本/); assert.match(repairMaterial.firstMessage, /第二行未实现/); writeFileSync(join(workDir, "initial.py"), "first = 10\nsecond = 20\n"); const fixed = prepareSubmission(s, repair, impl, 0, workDir); const fixedKnown = fixed.known as typeof known; assert.match(fixedKnown.code.hunks[0]!.text, /-first = 1/); assert.match(fixedKnown.code.hunks[0]!.text, /\+second = 20/); s.ledger.transaction(() => { w.sub(repair.stageId, repair.actor, repair.kind, repair.object, null, impl); s.ledger.append("code", fixed.code!); }); w.close(repair.stageId); w.open("r1-verification-2", "verification", 1, ["verifier-r1-c1"]); const verification = { ...ref, kind: "verification" as const, stageId: "r1-verification-2" }; const compare = taskCode(s, verification); assert.equal(compare.startVersion, null); assert.match(compare.changes!.diff, /-first = 1/); assert.match(compare.changes!.diff, /\+second = 20/); assert.deepEqual(protocolMetrics(new Board(s.ledger.records()), protocol), [metric]); const snapshot = join(s.dir, "snapshot"); mkdirSync(snapshot); checkoutVersion(repo, prepared.code.version, snapshot); assert.equal(readFileSync(join(snapshot, "initial.py"), "utf8"), "first = 10\nsecond = 2\n"); }); test("存不成版本的原原因交 submissions;改了保护文件时不能被漏掉", (t) => { const { s, workDir } = codeStudy(t); writeFileSync(join(workDir, "evaluate.py"), "protected = False\n"); const prepared = prepareSubmission(s, ref, impl, 0, workDir); assert.deepEqual((prepared.known as { code: { protectedFiles: string[] } }).code.protectedFiles, ["evaluate.py"]); symlinkSync("initial.py", join(workDir, "link.py")); const refused = prepareSubmission(s, ref, impl, 0, workDir); assert.equal(refused.code, null); const code = (refused.known as { code: { saved: false; path: string; reason: string } }).code; assert.equal(code.saved, false); assert.equal(code.path, "link.py"); assert.match(code.reason, /符号链接/); });