InfoRSI/tests/flow-forward.test.ts

import assert from "node:assert/strict";
import { mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { test, type TestContext } from "node:test";
import { adoptBatch } from "../src/flow/steps/forward.ts";
import { prepareSubmission, taskCode } from "../src/flow/task-code.ts";
import type { Study } from "../src/flow/study.ts";
import { Board, buildMaterial, protocolMetrics, type FrozenProtocol } from "../src/materials/index.ts";
import { Ledger } from "../src/ledger/index.ts";
import { loadTask } from "../src/tasks/index.ts";
import { checkoutVersion, createRepository, saveVersion } from "../src/workspace/index.ts";
import type { Submission } from "../src/vocab/index.ts";
import { fullConfig } from "./config-helpers.ts";
import { implementation, tempDir } from "./ledger-helpers.ts";
import { judged, prompts, study, Writer } from "./materials-helpers.ts";

function setup(t: TestContext): { s: Study; w: Writer } {
  const dir = tempDir(t);
  const ledger = Ledger.open(join(dir, "ledger.sqlite"));
  t.after(() => ledger.close());
  const task = loadTask(join(import.meta.dirname, "..", "tasks", "circle_packing"));
  const s = { dir, ledger, task, prompts, facts: study, frozen: { config: fullConfig(), facts: task.facts } } as Study;
  return { s, w: new Writer(ledger) };
}

// 预期值手算：基线 1，ε=1/64，已知最好 2；线=1/64。1+1/64 正好在线上，不采纳。
for (const [label, scores, adopted, gain] of [
  ["高于线且同分：取自然编号小的", [1.125, 1.125], "r1-c2", 0.125],
  ["正好等于线：也写 null 决定", [1.015625, 1], null, 0.015625],
  ["整批没有有效分数：也写 null 决定", [], null, null],
] as const) {
  test(`采纳写账：${label}，写决定后整批公开`, (t) => {
    const { s, w } = setup(t);
    s.frozen.facts = { ...s.frozen.facts, epsilon: 0.015625, knownBest: 2 };
    w.run("base-1", "baseline", 0, null, judged(1, "main"));
    w.decision("batch", 1, null, { chosen: ["r1-c10", "r1-c2", "r1-c3"] });
    scores.forEach((score, i) => w.run(["run-first", "run-second"][i]!, "forward", 1, i === 0 ? "r1-c10" : "r1-c2", judged(score, "main")));
    w.run("invalid", "forward", 1, "r1-c3", { ...judged(99, "main"), verdict: "invalid", verdictReason: "不合法" });
    // 不在本批的运行不参与，即使它分数更高；上轮不能混进本轮。
    w.run("other", "forward", 0, "old-candidate", judged(100, "main"));
    const before = new Board(s.ledger.records());
    assert.ok(before.runs().filter((r) => r.plan.round === 1).every((r) => !before.runPublic(r)));
    adoptBatch(s, 1);
    const record = s.ledger.records().at(-1)!;
    assert.equal(record.kind, "decision");
    if (record.kind !== "decision") throw new Error("应写采纳决定");
    assert.deepEqual(record.content, {
      decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1,
      params: { direction: "maximize", epsilon: 0.015625, knownBest: 2 }, seed: null,
      input: { candidates: scores.map((score, i) => ({ id: i === 0 ? "r1-c10" : "r1-c2", score })), baseline: 1, direction: "maximize", epsilon: 0.015625, knownBest: 2 },
      output: { line: 0.015625, quarter: 0.25, adopted,
        best: scores.length === 0 ? null : { id: adopted ?? "r1-c10", score: scores[0], improvement: gain } },
    });
    const after = new Board(s.ledger.records());
    assert.ok(after.runs().filter((r) => r.plan.round === 1).every((r) => after.runPublic(r)));
    assert.equal(after.baselineOf(2), adopted ?? "base");
  });
}

const metric = { id: "old-c.metric", meaning: "试验值", unit: "次", direction: "maximize" as const, howMeasured: "按原登记计数", missing: "invalidateRun" as const };
const oldProtocol: FrozenProtocol = { round: 0, baseline: "base", proposals: [], factors: [], safeguards: [], metrics: { defined: [metric], reused: [] }, clauses: [], body: "原登记" };
const protocol: FrozenProtocol = { round: 1, baseline: "base", proposals: [], factors: [{ name: "改动", method: "照协议改动", keySettings: "无", continues: null }], safeguards: [], metrics: { defined: [], reused: [metric.id] }, clauses: ["照协议"], body: "冻结正文" };

function codeStudy(t: TestContext) {
  const { s, w } = setup(t);
  const repo = join(s.dir, "repo.git");
  createRepository(repo);
  const workDir = join(s.dir, "work");
  mkdirSync(workDir);
  writeFileSync(join(workDir, "initial.py"), "first = 1\nsecond = 2\n");
  writeFileSync(join(workDir, "evaluate.py"), "protected = True\n");
  const baseline = saveVersion(repo, workDir, []);
  if (!baseline.saved) throw new Error(baseline.reason);
  s.ledger.append("code", { object: "base", version: baseline.version, locations: null });
  w.protocol("old-c", oldProtocol);
  w.protocol("r1-c1", protocol);
  w.open("r1-implementation", "implementation", 1, ["experimenter-r1-c1"]);
  return { s, w, repo, workDir, baseline: baseline.version };
}

const impl: Submission<"implementation"> = { ...implementation("实现说明"), factorLocations: [{ factor: "p1", locations: ["initial.py:1-2"] }] };
const ref = { actor: "experimenter-r1-c1", object: "r1-c1", kind: "implementation" as const, stageId: "r1-implementation" };

test("提交准备：快照、保护文件、差异块和引用指标同源；修复取上版，但差异永远相对本批基线", (t) => {
  const { s, w, repo, workDir, baseline } = codeStudy(t);
  assert.deepEqual(taskCode(s, ref), { changes: null, startVersion: baseline });
  writeFileSync(join(workDir, "initial.py"), "first = 10\nsecond = 2\n");
  const prepared = prepareSubmission(s, ref, impl, 0, workDir);
  assert.ok(prepared.code);
  const known = prepared.known as { code: { saved: true; protectedFiles: string[]; hunks: { text: string }[]; protocol: unknown } };
  assert.deepEqual(known.code.protectedFiles, []);
  assert.deepEqual(known.code.protocol, { body: "冻结正文", metrics: [metric] });
  assert.match(known.code.hunks[0]!.text, /-first = 1\n\+first = 10/);
  s.ledger.transaction(() => {
    w.sub(ref.stageId, ref.actor, ref.kind, ref.object, null, impl);
    s.ledger.append("code", prepared.code!);
  });
  w.close(ref.stageId);
  w.open("r1-verification-1", "verification", 1, ["verifier-r1-c1"]);
  w.sub("r1-verification-1", "verifier-r1-c1", "verification", "r1-c1", null, { clauses: [{ clause: "c1", found: false, where: "第二行未实现" }], factors: [{ factor: "p1", found: false, where: "第二行未实现" }], safeguards: [], unassigned: [], body: "只修第二行" });
  w.close("r1-verification-1");
  w.open("r1-repair-1", "implementation", 1, [ref.actor]);
  const repair = { ...ref, kind: "repair" as const, stageId: "r1-repair-1" };
  const previous = taskCode(s, repair);
  assert.equal(previous.startVersion, prepared.code.version);
  assert.match(previous.changes!.diff, /-first = 1\n\+first = 10/);
  const repairMaterial = buildMaterial({ ...repair, ledger: s.ledger, study, prompts, codeRoot: join(s.dir, "readable"), changes: previous.changes, artifacts: null });
  assert.match(repairMaterial.firstMessage, /工作目录是上一次提交的代码的副本/);
  assert.match(repairMaterial.firstMessage, /第二行未实现/);
  writeFileSync(join(workDir, "initial.py"), "first = 10\nsecond = 20\n");
  const fixed = prepareSubmission(s, repair, impl, 0, workDir);
  const fixedKnown = fixed.known as typeof known;
  assert.match(fixedKnown.code.hunks[0]!.text, /-first = 1/);
  assert.match(fixedKnown.code.hunks[0]!.text, /\+second = 20/);
  s.ledger.transaction(() => {
    w.sub(repair.stageId, repair.actor, repair.kind, repair.object, null, impl);
    s.ledger.append("code", fixed.code!);
  });
  w.close(repair.stageId);
  w.open("r1-verification-2", "verification", 1, ["verifier-r1-c1"]);
  const verification = { ...ref, kind: "verification" as const, stageId: "r1-verification-2" };
  const compare = taskCode(s, verification);
  assert.equal(compare.startVersion, null);
  assert.match(compare.changes!.diff, /-first = 1/);
  assert.match(compare.changes!.diff, /\+second = 20/);
  assert.deepEqual(protocolMetrics(new Board(s.ledger.records()), protocol), [metric]);
  const snapshot = join(s.dir, "snapshot");
  mkdirSync(snapshot);
  checkoutVersion(repo, prepared.code.version, snapshot);
  assert.equal(readFileSync(join(snapshot, "initial.py"), "utf8"), "first = 10\nsecond = 2\n");
});

test("存不成版本的原原因交 submissions；改了保护文件时不能被漏掉", (t) => {
  const { s, workDir } = codeStudy(t);
  writeFileSync(join(workDir, "evaluate.py"), "protected = False\n");
  const prepared = prepareSubmission(s, ref, impl, 0, workDir);
  assert.deepEqual((prepared.known as { code: { protectedFiles: string[] } }).code.protectedFiles, ["evaluate.py"]);
  symlinkSync("initial.py", join(workDir, "link.py"));
  const refused = prepareSubmission(s, ref, impl, 0, workDir);
  assert.equal(refused.code, null);
  const code = (refused.known as { code: { saved: false; path: string; reason: string } }).code;
  assert.equal(code.saved, false);
  assert.equal(code.path, "link.py");
  assert.match(code.reason, /符号链接/);
});