InfoRSI/tests/forward-eval.test.ts

import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { test } from "node:test";
import {
  candidateHistory,
  forwardAccounts,
  FORWARD_STOP,
  scoreBand,
} from "../eval/lib/forward.ts";
import { lightCheckUsage } from "../eval/lib/light-checks.ts";
import { referenceScores } from "../eval/lib/reference-scores.ts";
import { Ledger } from "../src/ledger/index.ts";
import { forwardNotRunEvent } from "../src/materials/index.ts";
import { createRepository, saveVersion } from "../src/workspace/index.ts";
import { fullConfig } from "./config-helpers.ts";
import {
  definition,
  exited,
  implementation,
  notStarted,
  proposal,
  tempDir,
} from "./ledger-helpers.ts";
import { judged, protocolOf, Writer } from "./materials-helpers.ts";

const verification = (found: boolean) => ({
  clauses: [{ clause: "c1", found, where: "solve.py:1" }],
  factors: [{ factor: "p1", found, where: "solve.py:1" }],
  safeguards: [],
  unassigned: [],
  body: "逐项核对",
});

test("forward 导出/统计：历史代码配对、两次核验一次修复、封存自评与真实分数保持可追溯", (t) => {
  const root = tempDir(t),
    dir = join(root, "run"),
    study = join(dir, "study");
  mkdirSync(study, { recursive: true });
  const ledger = Ledger.open(join(study, "ledger.sqlite"));
  t.after(() => ledger.close());
  const w = new Writer(ledger),
    config = fullConfig();
  ledger.append("config", { config });
  const repo = join(study, "repo.git"),
    work = join(root, "work");
  createRepository(repo);
  mkdirSync(work);
  const version = (text: string) => {
    writeFileSync(join(work, "solve.py"), text);
    const result = saveVersion(repo, work, []);
    assert.ok(result.saved);
    return result.version;
  };
  ledger.append("code", {
    object: "base",
    version: version("value = 0\n"),
    locations: null,
  });
  w.run("base-1", "baseline", 0, null, judged(1, "main"));
  w.open("proposal", "proposal", 1, ["a"]);
  const p = proposal({ q1: 1, q5: 2, q9: 3 }, "协议正文");
  w.sub("proposal", "a", "proposal", null, "p", p);
  w.close("proposal");
  w.protocol("r1-c1", protocolOf("r1-c1", p, 1, "base", ["p"], ["改value"]));
  w.open("second", "secondPrediction", 1, ["a"]);
  w.sub("second", "a", "secondPrediction", "r1-c1", null, {
    range: { q1: 1, q5: 2, q9: 3 },
    body: "估计",
  });
  w.close("second");
  w.decision("batch", 1, null, {
    chosen: ["r1-c1"],
    candidates: [{ id: "r1-c1", merged: { q1: 1, q5: 2, q9: 3 } }],
  });
  const v1 = version("value = 1\n"),
    v2 = version("value = 2\n");
  w.open("impl", "implementation", 1, ["x"]);
  w.sub("impl", "x", "implementation", "r1-c1", null, implementation("首稿"));
  ledger.append("code", { object: "r1-c1", version: v1, locations: null });
  w.close("impl");
  w.open("verify1", "verification", 1, ["v"]);
  w.sub("verify1", "v", "verification", "r1-c1", null, verification(false));
  w.close("verify1");
  w.open("repair", "implementation", 1, ["x"]);
  w.sub("repair", "x", "repair", "r1-c1", null, implementation("只修value"));
  ledger.append("code", { object: "r1-c1", version: v2, locations: null });
  w.close("repair");
  w.open("verify2", "verification", 1, ["v"]);
  w.sub("verify2", "v", "verification", "r1-c1", null, verification(true));
  w.close("verify2");
  ledger.append("runPlanned", {
    run: "forward-1",
    purpose: "forward",
    round: 1,
    object: "r1-c1",
    definition,
  });
  ledger.append("runStarted", { run: "forward-1" });
  const atKill = ledger.records();
  writeFileSync(join(dir, "ledger-at-kill.json"), JSON.stringify(atKill));
  writeFileSync(
    join(dir, "kill.json"),
    JSON.stringify({
      run: "forward-1",
      candidate: "r1-c1",
      beforeKillSeq: atKill.at(-1)!.seq,
    }),
  );
  for (const part of ["before", "after-kill", "after-resume"])
    writeFileSync(
      join(dir, `container-${part}.json`),
      JSON.stringify({
        Id: "same-container",
        State: { Running: true, StartedAt: "2026-10-04T00:00:00Z" },
      }),
    );
  ledger.append("runResult", {
    run: "forward-1",
    process: exited,
    judged: judged(2.5, "main"),
  });
  w.decision("adopt", 1, null, { adopted: "r1-c1" });
  ledger.append("event", {
    event: "halted",
    objects: [],
    reason: FORWARD_STOP,
  });
  const records = ledger.records(),
    accounts = forwardAccounts(records);
  assert.deepEqual(accounts.balance, {
    charged: 2,
    reserved: 0,
    balance: config.runs - 2,
  });
  assert.ok(Object.values(accounts.checks).every(Boolean));
  assert.equal(accounts.candidates[0]!.verificationCount, 2);
  assert.equal(accounts.candidates[0]!.repairCount, 1);
  assert.deepEqual(
    candidateHistory(records, "r1-c1").verifications.map(
      (v) => v.code.content.version,
    ),
    [v1, v2],
  );
  const exported = join(root, "export");
  const exportResult = spawnSync(
    process.execPath,
    ["eval/forward/export.ts", study, exported],
    { encoding: "utf8" },
  );
  assert.equal(exportResult.status, 0, exportResult.stderr);
  assert.equal(
    readFileSync(join(exported, "r1-c1/final/solve.py"), "utf8"),
    "value = 2\n",
  );
  assert.match(
    readFileSync(join(exported, "r1-c1/attempts.json"), "utf8"),
    /只修value/,
  );
  assert.match(
    readFileSync(join(exported, "r1-c1/final-implementation.json"), "utf8"),
    /concerns/,
  );
  const refs = join(root, "refs");
  mkdirSync(refs);
  for (const name of [
    "16-real-1",
    "15-real-1",
    "15-tight-2",
    "14-real-1",
    "12-real-2",
  ])
    referenceFixture(join(refs, name));
  const statsOut = join(root, "stats");
  const statsResult = spawnSync(
    process.execPath,
    ["eval/forward/stats.ts", statsOut, refs, dir],
    { encoding: "utf8" },
  );
  assert.equal(statsResult.status, 0, statsResult.stderr);
  const scores = JSON.parse(
    readFileSync(join(statsOut, "predictions-scores.json"), "utf8"),
  );
  assert.deepEqual(scores[0].authorEstimates[0].range, { q1: 1, q5: 2, q9: 3 });
  assert.equal(scores[0].runs[0].measuredMinusQ5, 0.5);
  assert.equal(scores[0].runs[0].band, "[q5,q9]");
  const statistics = JSON.parse(
    readFileSync(join(statsOut, "stats.json"), "utf8"),
  );
  assert.deepEqual(statistics[0].kill.checks, {
    sameContainer: true,
    chargedOnceNoRerun: true,
    resultAndContinued: true,
  });
  writeFileSync(
    join(dir, "container-after-resume.json"),
    JSON.stringify({
      Id: "replacement",
      State: { Running: true, StartedAt: "2026-10-04T00:01:00Z" },
    }),
  );
  const badOut = join(root, "bad-stats");
  const bad = spawnSync(
    process.execPath,
    ["eval/forward/stats.ts", badOut, refs, dir],
    { encoding: "utf8" },
  );
  assert.equal(bad.status, 1, bad.stderr);
  assert.equal(
    JSON.parse(readFileSync(join(badOut, "stats.json"), "utf8"))[0].kill.checks
      .sameContainer,
    false,
  );
});

test("forward 账目：没启动退还、修复用尽不扣次；缺不跑事件或结果不能通过", (t) => {
  const ledger = Ledger.open(join(tempDir(t), "ledger.sqlite"));
  t.after(() => ledger.close());
  const w = new Writer(ledger),
    config = { ...fullConfig(), maxRepairs: 0 };
  ledger.append("config", { config });
  ledger.append("runPlanned", {
    run: "base-0",
    purpose: "baseline",
    round: 0,
    object: null,
    definition,
  });
  ledger.append("runResult", {
    run: "base-0",
    process: notStarted,
    judged: null,
  });
  w.run("base-1", "baseline", 0, null, judged(1, "main"));
  w.decision("batch", 1, null, { chosen: ["c"] });
  ledger.append("code", { object: "c", version: "v", locations: null });
  w.open("v", "verification", 1, ["v"]);
  w.sub("v", "v", "verification", "c", null, verification(false));
  w.close("v");
  assert.equal(forwardAccounts(ledger.records()).candidates[0]!.pass, false);
  ledger.append("event", forwardNotRunEvent("c", "修复用尽"));
  w.decision("adopt", 1, null, { adopted: null });
  ledger.append("event", {
    event: "halted",
    reason: FORWARD_STOP,
    objects: [],
  });
  const accounts = forwardAccounts(ledger.records());
  assert.deepEqual(accounts.balance, {
    charged: 1,
    reserved: 0,
    balance: config.runs - 1,
  });
  assert.ok(Object.values(accounts.checks).every(Boolean));
  ledger.append("runPlanned", {
    run: "unfinished",
    purpose: "forward",
    round: 1,
    object: "c",
    definition,
  });
  ledger.append("runStarted", { run: "unfinished" });
  assert.equal(forwardAccounts(ledger.records()).checks.runsFinished, false);
  assert.equal(forwardAccounts(ledger.records()).candidates[0]!.pass, false);
});

test("forward 分位区间不把无效分数当零，三条边界按既有placement", () => {
  assert.deepEqual(
    [null, 0, 1, 2, 3, 4].map((s) => scoreBand(s, { q1: 1, q5: 2, q9: 3 })),
    [null, "<q1", "[q1,q5)", "[q5,q9]", "[q5,q9]", ">q9"],
  );
});

test("forward 轻量检查：跨会话按工具编号配对，拒绝超限不算执行，缺结果不伪装零次", (t) => {
  const dir = tempDir(t);
  const request = (id: string) => ({
    type: "message",
    message: {
      role: "assistant",
      content: [{ type: "toolCall", name: "research_check", id }],
    },
  });
  const result = (id: string, text: string) => ({
    type: "message",
    message: {
      role: "toolResult",
      toolName: "research_check",
      toolCallId: id,
      content: [{ type: "text", text }],
    },
  });
  writeFileSync(join(dir, "1.jsonl"), JSON.stringify(request("a")) + "\n");
  writeFileSync(
    join(dir, "2.jsonl"),
    [
      result(
        "a",
        JSON.stringify({ used: 1, maxCount: 1, seconds: 5, result: {} }),
      ),
      request("b"),
      result("b", "本次调用的轻量检查已用完（上限 1 次），不能再查。"),
    ]
      .map((r) => JSON.stringify(r))
      .join("\n"),
  );
  const usage = lightCheckUsage(dir, 1);
  assert.equal(usage.requested, 2);
  assert.equal(usage.executed, 1);
  assert.equal(usage.deniedLimit, 1);
  assert.equal(usage.pass, true);
  writeFileSync(join(dir, "3.jsonl"), JSON.stringify(request("c")));
  assert.equal(lightCheckUsage(dir, 1).pass, false);
});

function referenceFixture(dir: string) {
  mkdirSync(dir);
  const db = new DatabaseSync(join(dir, "board.sqlite"));
  db.exec("CREATE TABLE facts (created_seq INTEGER, kind TEXT, payload TEXT)");
  const study = dir.split("/").at(-1)!;
  const insert = (seq: number, kind: string, payload: unknown) =>
    db
      .prepare("INSERT INTO facts VALUES (?,?,?)")
      .run(seq, kind, JSON.stringify(payload));
  insert(1, "run-link", {
    purpose: "forward",
    roundId: `${study}.r1`,
    runId: "r",
    candidateId: "c",
  });
  insert(2, "run-observation", {
    runId: "r",
    status: "incomplete",
    finalMetric: { value: 99 },
    reasons: ["无效"],
  });
  insert(3, "adoption", {
    roundId: `${study}.r1`,
    adoptedRunId: null,
    newBaselineId: "B0",
  });
  db.close();
}

test("forward 参考分数：incomplete 原值保留但有效分数为空", (t) => {
  const dir = join(tempDir(t), "reference");
  referenceFixture(dir);
  const rows = referenceScores(join(dir, "board.sqlite"));
  assert.equal(rows[0]!.rawScore, 99);
  assert.equal(rows[0]!.validScore, null);
  assert.equal(rows[0]!.adopted, false);
  assert.equal(rows[0]!.observationSeq, 2);
});