iomgaa
b7be524cdc
新增单份串行启动入口、首个正向运行杀进程恢复证据、逐候选审阅导出与账目统计；参考只读副本分数对照。456项免费测试通过，尚未启动三份真实验收。同步结构说明与当前工作状态。
355 lines
11 KiB
TypeScript
355 lines
11 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { spawnSync } from "node:child_process";
|
|
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { DatabaseSync } from "node:sqlite";
|
|
import { test } from "node:test";
|
|
import {
|
|
candidateHistory,
|
|
forwardAccounts,
|
|
FORWARD_STOP,
|
|
scoreBand,
|
|
} from "../eval/lib/forward.ts";
|
|
import { lightCheckUsage } from "../eval/lib/light-checks.ts";
|
|
import { referenceScores } from "../eval/lib/reference-scores.ts";
|
|
import { Ledger } from "../src/ledger/index.ts";
|
|
import { forwardNotRunEvent } from "../src/materials/index.ts";
|
|
import { createRepository, saveVersion } from "../src/workspace/index.ts";
|
|
import { fullConfig } from "./config-helpers.ts";
|
|
import {
|
|
definition,
|
|
exited,
|
|
implementation,
|
|
notStarted,
|
|
proposal,
|
|
tempDir,
|
|
} from "./ledger-helpers.ts";
|
|
import { judged, protocolOf, Writer } from "./materials-helpers.ts";
|
|
|
|
const verification = (found: boolean) => ({
|
|
clauses: [{ clause: "c1", found, where: "solve.py:1" }],
|
|
factors: [{ factor: "p1", found, where: "solve.py:1" }],
|
|
safeguards: [],
|
|
unassigned: [],
|
|
body: "逐项核对",
|
|
});
|
|
|
|
test("forward 导出/统计：历史代码配对、两次核验一次修复、封存自评与真实分数保持可追溯", (t) => {
|
|
const root = tempDir(t),
|
|
dir = join(root, "run"),
|
|
study = join(dir, "study");
|
|
mkdirSync(study, { recursive: true });
|
|
const ledger = Ledger.open(join(study, "ledger.sqlite"));
|
|
t.after(() => ledger.close());
|
|
const w = new Writer(ledger),
|
|
config = fullConfig();
|
|
ledger.append("config", { config });
|
|
const repo = join(study, "repo.git"),
|
|
work = join(root, "work");
|
|
createRepository(repo);
|
|
mkdirSync(work);
|
|
const version = (text: string) => {
|
|
writeFileSync(join(work, "solve.py"), text);
|
|
const result = saveVersion(repo, work, []);
|
|
assert.ok(result.saved);
|
|
return result.version;
|
|
};
|
|
ledger.append("code", {
|
|
object: "base",
|
|
version: version("value = 0\n"),
|
|
locations: null,
|
|
});
|
|
w.run("base-1", "baseline", 0, null, judged(1, "main"));
|
|
w.open("proposal", "proposal", 1, ["a"]);
|
|
const p = proposal({ q1: 1, q5: 2, q9: 3 }, "协议正文");
|
|
w.sub("proposal", "a", "proposal", null, "p", p);
|
|
w.close("proposal");
|
|
w.protocol("r1-c1", protocolOf("r1-c1", p, 1, "base", ["p"], ["改value"]));
|
|
w.open("second", "secondPrediction", 1, ["a"]);
|
|
w.sub("second", "a", "secondPrediction", "r1-c1", null, {
|
|
range: { q1: 1, q5: 2, q9: 3 },
|
|
body: "估计",
|
|
});
|
|
w.close("second");
|
|
w.decision("batch", 1, null, {
|
|
chosen: ["r1-c1"],
|
|
candidates: [{ id: "r1-c1", merged: { q1: 1, q5: 2, q9: 3 } }],
|
|
});
|
|
const v1 = version("value = 1\n"),
|
|
v2 = version("value = 2\n");
|
|
w.open("impl", "implementation", 1, ["x"]);
|
|
w.sub("impl", "x", "implementation", "r1-c1", null, implementation("首稿"));
|
|
ledger.append("code", { object: "r1-c1", version: v1, locations: null });
|
|
w.close("impl");
|
|
w.open("verify1", "verification", 1, ["v"]);
|
|
w.sub("verify1", "v", "verification", "r1-c1", null, verification(false));
|
|
w.close("verify1");
|
|
w.open("repair", "implementation", 1, ["x"]);
|
|
w.sub("repair", "x", "repair", "r1-c1", null, implementation("只修value"));
|
|
ledger.append("code", { object: "r1-c1", version: v2, locations: null });
|
|
w.close("repair");
|
|
w.open("verify2", "verification", 1, ["v"]);
|
|
w.sub("verify2", "v", "verification", "r1-c1", null, verification(true));
|
|
w.close("verify2");
|
|
ledger.append("runPlanned", {
|
|
run: "forward-1",
|
|
purpose: "forward",
|
|
round: 1,
|
|
object: "r1-c1",
|
|
definition,
|
|
});
|
|
ledger.append("runStarted", { run: "forward-1" });
|
|
const atKill = ledger.records();
|
|
writeFileSync(join(dir, "ledger-at-kill.json"), JSON.stringify(atKill));
|
|
writeFileSync(
|
|
join(dir, "kill.json"),
|
|
JSON.stringify({
|
|
run: "forward-1",
|
|
candidate: "r1-c1",
|
|
beforeKillSeq: atKill.at(-1)!.seq,
|
|
}),
|
|
);
|
|
for (const part of ["before", "after-kill", "after-resume"])
|
|
writeFileSync(
|
|
join(dir, `container-${part}.json`),
|
|
JSON.stringify({
|
|
Id: "same-container",
|
|
State: { Running: true, StartedAt: "2026-10-04T00:00:00Z" },
|
|
}),
|
|
);
|
|
ledger.append("runResult", {
|
|
run: "forward-1",
|
|
process: exited,
|
|
judged: judged(2.5, "main"),
|
|
});
|
|
w.decision("adopt", 1, null, { adopted: "r1-c1" });
|
|
ledger.append("event", {
|
|
event: "halted",
|
|
objects: [],
|
|
reason: FORWARD_STOP,
|
|
});
|
|
const records = ledger.records(),
|
|
accounts = forwardAccounts(records);
|
|
assert.deepEqual(accounts.balance, {
|
|
charged: 2,
|
|
reserved: 0,
|
|
balance: config.runs - 2,
|
|
});
|
|
assert.ok(Object.values(accounts.checks).every(Boolean));
|
|
assert.equal(accounts.candidates[0]!.verificationCount, 2);
|
|
assert.equal(accounts.candidates[0]!.repairCount, 1);
|
|
assert.deepEqual(
|
|
candidateHistory(records, "r1-c1").verifications.map(
|
|
(v) => v.code.content.version,
|
|
),
|
|
[v1, v2],
|
|
);
|
|
const exported = join(root, "export");
|
|
const exportResult = spawnSync(
|
|
process.execPath,
|
|
["eval/forward/export.ts", study, exported],
|
|
{ encoding: "utf8" },
|
|
);
|
|
assert.equal(exportResult.status, 0, exportResult.stderr);
|
|
assert.equal(
|
|
readFileSync(join(exported, "r1-c1/final/solve.py"), "utf8"),
|
|
"value = 2\n",
|
|
);
|
|
assert.match(
|
|
readFileSync(join(exported, "r1-c1/attempts.json"), "utf8"),
|
|
/只修value/,
|
|
);
|
|
assert.match(
|
|
readFileSync(join(exported, "r1-c1/final-implementation.json"), "utf8"),
|
|
/concerns/,
|
|
);
|
|
const refs = join(root, "refs");
|
|
mkdirSync(refs);
|
|
for (const name of [
|
|
"16-real-1",
|
|
"15-real-1",
|
|
"15-tight-2",
|
|
"14-real-1",
|
|
"12-real-2",
|
|
])
|
|
referenceFixture(join(refs, name));
|
|
const statsOut = join(root, "stats");
|
|
const statsResult = spawnSync(
|
|
process.execPath,
|
|
["eval/forward/stats.ts", statsOut, refs, dir],
|
|
{ encoding: "utf8" },
|
|
);
|
|
assert.equal(statsResult.status, 0, statsResult.stderr);
|
|
const scores = JSON.parse(
|
|
readFileSync(join(statsOut, "predictions-scores.json"), "utf8"),
|
|
);
|
|
assert.deepEqual(scores[0].authorEstimates[0].range, { q1: 1, q5: 2, q9: 3 });
|
|
assert.equal(scores[0].runs[0].measuredMinusQ5, 0.5);
|
|
assert.equal(scores[0].runs[0].band, "[q5,q9]");
|
|
const statistics = JSON.parse(
|
|
readFileSync(join(statsOut, "stats.json"), "utf8"),
|
|
);
|
|
assert.deepEqual(statistics[0].kill.checks, {
|
|
sameContainer: true,
|
|
chargedOnceNoRerun: true,
|
|
resultAndContinued: true,
|
|
});
|
|
writeFileSync(
|
|
join(dir, "container-after-resume.json"),
|
|
JSON.stringify({
|
|
Id: "replacement",
|
|
State: { Running: true, StartedAt: "2026-10-04T00:01:00Z" },
|
|
}),
|
|
);
|
|
const badOut = join(root, "bad-stats");
|
|
const bad = spawnSync(
|
|
process.execPath,
|
|
["eval/forward/stats.ts", badOut, refs, dir],
|
|
{ encoding: "utf8" },
|
|
);
|
|
assert.equal(bad.status, 1, bad.stderr);
|
|
assert.equal(
|
|
JSON.parse(readFileSync(join(badOut, "stats.json"), "utf8"))[0].kill.checks
|
|
.sameContainer,
|
|
false,
|
|
);
|
|
});
|
|
|
|
test("forward 账目：没启动退还、修复用尽不扣次；缺不跑事件或结果不能通过", (t) => {
|
|
const ledger = Ledger.open(join(tempDir(t), "ledger.sqlite"));
|
|
t.after(() => ledger.close());
|
|
const w = new Writer(ledger),
|
|
config = { ...fullConfig(), maxRepairs: 0 };
|
|
ledger.append("config", { config });
|
|
ledger.append("runPlanned", {
|
|
run: "base-0",
|
|
purpose: "baseline",
|
|
round: 0,
|
|
object: null,
|
|
definition,
|
|
});
|
|
ledger.append("runResult", {
|
|
run: "base-0",
|
|
process: notStarted,
|
|
judged: null,
|
|
});
|
|
w.run("base-1", "baseline", 0, null, judged(1, "main"));
|
|
w.decision("batch", 1, null, { chosen: ["c"] });
|
|
ledger.append("code", { object: "c", version: "v", locations: null });
|
|
w.open("v", "verification", 1, ["v"]);
|
|
w.sub("v", "v", "verification", "c", null, verification(false));
|
|
w.close("v");
|
|
assert.equal(forwardAccounts(ledger.records()).candidates[0]!.pass, false);
|
|
ledger.append("event", forwardNotRunEvent("c", "修复用尽"));
|
|
w.decision("adopt", 1, null, { adopted: null });
|
|
ledger.append("event", {
|
|
event: "halted",
|
|
reason: FORWARD_STOP,
|
|
objects: [],
|
|
});
|
|
const accounts = forwardAccounts(ledger.records());
|
|
assert.deepEqual(accounts.balance, {
|
|
charged: 1,
|
|
reserved: 0,
|
|
balance: config.runs - 1,
|
|
});
|
|
assert.ok(Object.values(accounts.checks).every(Boolean));
|
|
ledger.append("runPlanned", {
|
|
run: "unfinished",
|
|
purpose: "forward",
|
|
round: 1,
|
|
object: "c",
|
|
definition,
|
|
});
|
|
ledger.append("runStarted", { run: "unfinished" });
|
|
assert.equal(forwardAccounts(ledger.records()).checks.runsFinished, false);
|
|
assert.equal(forwardAccounts(ledger.records()).candidates[0]!.pass, false);
|
|
});
|
|
|
|
test("forward 分位区间不把无效分数当零，三条边界按既有placement", () => {
|
|
assert.deepEqual(
|
|
[null, 0, 1, 2, 3, 4].map((s) => scoreBand(s, { q1: 1, q5: 2, q9: 3 })),
|
|
[null, "<q1", "[q1,q5)", "[q5,q9]", "[q5,q9]", ">q9"],
|
|
);
|
|
});
|
|
|
|
test("forward 轻量检查：跨会话按工具编号配对，拒绝超限不算执行，缺结果不伪装零次", (t) => {
|
|
const dir = tempDir(t);
|
|
const request = (id: string) => ({
|
|
type: "message",
|
|
message: {
|
|
role: "assistant",
|
|
content: [{ type: "toolCall", name: "research_check", id }],
|
|
},
|
|
});
|
|
const result = (id: string, text: string) => ({
|
|
type: "message",
|
|
message: {
|
|
role: "toolResult",
|
|
toolName: "research_check",
|
|
toolCallId: id,
|
|
content: [{ type: "text", text }],
|
|
},
|
|
});
|
|
writeFileSync(join(dir, "1.jsonl"), JSON.stringify(request("a")) + "\n");
|
|
writeFileSync(
|
|
join(dir, "2.jsonl"),
|
|
[
|
|
result(
|
|
"a",
|
|
JSON.stringify({ used: 1, maxCount: 1, seconds: 5, result: {} }),
|
|
),
|
|
request("b"),
|
|
result("b", "本次调用的轻量检查已用完（上限 1 次），不能再查。"),
|
|
]
|
|
.map((r) => JSON.stringify(r))
|
|
.join("\n"),
|
|
);
|
|
const usage = lightCheckUsage(dir, 1);
|
|
assert.equal(usage.requested, 2);
|
|
assert.equal(usage.executed, 1);
|
|
assert.equal(usage.deniedLimit, 1);
|
|
assert.equal(usage.pass, true);
|
|
writeFileSync(join(dir, "3.jsonl"), JSON.stringify(request("c")));
|
|
assert.equal(lightCheckUsage(dir, 1).pass, false);
|
|
});
|
|
|
|
function referenceFixture(dir: string) {
|
|
mkdirSync(dir);
|
|
const db = new DatabaseSync(join(dir, "board.sqlite"));
|
|
db.exec("CREATE TABLE facts (created_seq INTEGER, kind TEXT, payload TEXT)");
|
|
const study = dir.split("/").at(-1)!;
|
|
const insert = (seq: number, kind: string, payload: unknown) =>
|
|
db
|
|
.prepare("INSERT INTO facts VALUES (?,?,?)")
|
|
.run(seq, kind, JSON.stringify(payload));
|
|
insert(1, "run-link", {
|
|
purpose: "forward",
|
|
roundId: `${study}.r1`,
|
|
runId: "r",
|
|
candidateId: "c",
|
|
});
|
|
insert(2, "run-observation", {
|
|
runId: "r",
|
|
status: "incomplete",
|
|
finalMetric: { value: 99 },
|
|
reasons: ["无效"],
|
|
});
|
|
insert(3, "adoption", {
|
|
roundId: `${study}.r1`,
|
|
adoptedRunId: null,
|
|
newBaselineId: "B0",
|
|
});
|
|
db.close();
|
|
}
|
|
|
|
test("forward 参考分数：incomplete 原值保留但有效分数为空", (t) => {
|
|
const dir = join(tempDir(t), "reference");
|
|
referenceFixture(dir);
|
|
const rows = referenceScores(join(dir, "board.sqlite"));
|
|
assert.equal(rows[0]!.rawScore, 99);
|
|
assert.equal(rows[0]!.validScore, null);
|
|
assert.equal(rows[0]!.adopted, false);
|
|
assert.equal(rows[0]!.observationSeq, 2);
|
|
});
|