eval: 准备正向采纳的真实运行恢复与质量评估
新增单份串行启动入口、首个正向运行杀进程恢复证据、逐候选审阅导出与账目统计；参考只读副本分数对照。456项免费测试通过，尚未启动三份真实验收。同步结构说明与当前工作状态。
This commit is contained in:
@@ -0,0 +1,354 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { test } from "node:test";
|
||||
import {
|
||||
candidateHistory,
|
||||
forwardAccounts,
|
||||
FORWARD_STOP,
|
||||
scoreBand,
|
||||
} from "../eval/lib/forward.ts";
|
||||
import { lightCheckUsage } from "../eval/lib/light-checks.ts";
|
||||
import { referenceScores } from "../eval/lib/reference-scores.ts";
|
||||
import { Ledger } from "../src/ledger/index.ts";
|
||||
import { forwardNotRunEvent } from "../src/materials/index.ts";
|
||||
import { createRepository, saveVersion } from "../src/workspace/index.ts";
|
||||
import { fullConfig } from "./config-helpers.ts";
|
||||
import {
|
||||
definition,
|
||||
exited,
|
||||
implementation,
|
||||
notStarted,
|
||||
proposal,
|
||||
tempDir,
|
||||
} from "./ledger-helpers.ts";
|
||||
import { judged, protocolOf, Writer } from "./materials-helpers.ts";
|
||||
|
||||
const verification = (found: boolean) => ({
|
||||
clauses: [{ clause: "c1", found, where: "solve.py:1" }],
|
||||
factors: [{ factor: "p1", found, where: "solve.py:1" }],
|
||||
safeguards: [],
|
||||
unassigned: [],
|
||||
body: "逐项核对",
|
||||
});
|
||||
|
||||
test("forward 导出/统计：历史代码配对、两次核验一次修复、封存自评与真实分数保持可追溯", (t) => {
|
||||
const root = tempDir(t),
|
||||
dir = join(root, "run"),
|
||||
study = join(dir, "study");
|
||||
mkdirSync(study, { recursive: true });
|
||||
const ledger = Ledger.open(join(study, "ledger.sqlite"));
|
||||
t.after(() => ledger.close());
|
||||
const w = new Writer(ledger),
|
||||
config = fullConfig();
|
||||
ledger.append("config", { config });
|
||||
const repo = join(study, "repo.git"),
|
||||
work = join(root, "work");
|
||||
createRepository(repo);
|
||||
mkdirSync(work);
|
||||
const version = (text: string) => {
|
||||
writeFileSync(join(work, "solve.py"), text);
|
||||
const result = saveVersion(repo, work, []);
|
||||
assert.ok(result.saved);
|
||||
return result.version;
|
||||
};
|
||||
ledger.append("code", {
|
||||
object: "base",
|
||||
version: version("value = 0\n"),
|
||||
locations: null,
|
||||
});
|
||||
w.run("base-1", "baseline", 0, null, judged(1, "main"));
|
||||
w.open("proposal", "proposal", 1, ["a"]);
|
||||
const p = proposal({ q1: 1, q5: 2, q9: 3 }, "协议正文");
|
||||
w.sub("proposal", "a", "proposal", null, "p", p);
|
||||
w.close("proposal");
|
||||
w.protocol("r1-c1", protocolOf("r1-c1", p, 1, "base", ["p"], ["改value"]));
|
||||
w.open("second", "secondPrediction", 1, ["a"]);
|
||||
w.sub("second", "a", "secondPrediction", "r1-c1", null, {
|
||||
range: { q1: 1, q5: 2, q9: 3 },
|
||||
body: "估计",
|
||||
});
|
||||
w.close("second");
|
||||
w.decision("batch", 1, null, {
|
||||
chosen: ["r1-c1"],
|
||||
candidates: [{ id: "r1-c1", merged: { q1: 1, q5: 2, q9: 3 } }],
|
||||
});
|
||||
const v1 = version("value = 1\n"),
|
||||
v2 = version("value = 2\n");
|
||||
w.open("impl", "implementation", 1, ["x"]);
|
||||
w.sub("impl", "x", "implementation", "r1-c1", null, implementation("首稿"));
|
||||
ledger.append("code", { object: "r1-c1", version: v1, locations: null });
|
||||
w.close("impl");
|
||||
w.open("verify1", "verification", 1, ["v"]);
|
||||
w.sub("verify1", "v", "verification", "r1-c1", null, verification(false));
|
||||
w.close("verify1");
|
||||
w.open("repair", "implementation", 1, ["x"]);
|
||||
w.sub("repair", "x", "repair", "r1-c1", null, implementation("只修value"));
|
||||
ledger.append("code", { object: "r1-c1", version: v2, locations: null });
|
||||
w.close("repair");
|
||||
w.open("verify2", "verification", 1, ["v"]);
|
||||
w.sub("verify2", "v", "verification", "r1-c1", null, verification(true));
|
||||
w.close("verify2");
|
||||
ledger.append("runPlanned", {
|
||||
run: "forward-1",
|
||||
purpose: "forward",
|
||||
round: 1,
|
||||
object: "r1-c1",
|
||||
definition,
|
||||
});
|
||||
ledger.append("runStarted", { run: "forward-1" });
|
||||
const atKill = ledger.records();
|
||||
writeFileSync(join(dir, "ledger-at-kill.json"), JSON.stringify(atKill));
|
||||
writeFileSync(
|
||||
join(dir, "kill.json"),
|
||||
JSON.stringify({
|
||||
run: "forward-1",
|
||||
candidate: "r1-c1",
|
||||
beforeKillSeq: atKill.at(-1)!.seq,
|
||||
}),
|
||||
);
|
||||
for (const part of ["before", "after-kill", "after-resume"])
|
||||
writeFileSync(
|
||||
join(dir, `container-${part}.json`),
|
||||
JSON.stringify({
|
||||
Id: "same-container",
|
||||
State: { Running: true, StartedAt: "2026-10-04T00:00:00Z" },
|
||||
}),
|
||||
);
|
||||
ledger.append("runResult", {
|
||||
run: "forward-1",
|
||||
process: exited,
|
||||
judged: judged(2.5, "main"),
|
||||
});
|
||||
w.decision("adopt", 1, null, { adopted: "r1-c1" });
|
||||
ledger.append("event", {
|
||||
event: "halted",
|
||||
objects: [],
|
||||
reason: FORWARD_STOP,
|
||||
});
|
||||
const records = ledger.records(),
|
||||
accounts = forwardAccounts(records);
|
||||
assert.deepEqual(accounts.balance, {
|
||||
charged: 2,
|
||||
reserved: 0,
|
||||
balance: config.runs - 2,
|
||||
});
|
||||
assert.ok(Object.values(accounts.checks).every(Boolean));
|
||||
assert.equal(accounts.candidates[0]!.verificationCount, 2);
|
||||
assert.equal(accounts.candidates[0]!.repairCount, 1);
|
||||
assert.deepEqual(
|
||||
candidateHistory(records, "r1-c1").verifications.map(
|
||||
(v) => v.code.content.version,
|
||||
),
|
||||
[v1, v2],
|
||||
);
|
||||
const exported = join(root, "export");
|
||||
const exportResult = spawnSync(
|
||||
process.execPath,
|
||||
["eval/forward/export.ts", study, exported],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
assert.equal(exportResult.status, 0, exportResult.stderr);
|
||||
assert.equal(
|
||||
readFileSync(join(exported, "r1-c1/final/solve.py"), "utf8"),
|
||||
"value = 2\n",
|
||||
);
|
||||
assert.match(
|
||||
readFileSync(join(exported, "r1-c1/attempts.json"), "utf8"),
|
||||
/只修value/,
|
||||
);
|
||||
assert.match(
|
||||
readFileSync(join(exported, "r1-c1/final-implementation.json"), "utf8"),
|
||||
/concerns/,
|
||||
);
|
||||
const refs = join(root, "refs");
|
||||
mkdirSync(refs);
|
||||
for (const name of [
|
||||
"16-real-1",
|
||||
"15-real-1",
|
||||
"15-tight-2",
|
||||
"14-real-1",
|
||||
"12-real-2",
|
||||
])
|
||||
referenceFixture(join(refs, name));
|
||||
const statsOut = join(root, "stats");
|
||||
const statsResult = spawnSync(
|
||||
process.execPath,
|
||||
["eval/forward/stats.ts", statsOut, refs, dir],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
assert.equal(statsResult.status, 0, statsResult.stderr);
|
||||
const scores = JSON.parse(
|
||||
readFileSync(join(statsOut, "predictions-scores.json"), "utf8"),
|
||||
);
|
||||
assert.deepEqual(scores[0].authorEstimates[0].range, { q1: 1, q5: 2, q9: 3 });
|
||||
assert.equal(scores[0].runs[0].measuredMinusQ5, 0.5);
|
||||
assert.equal(scores[0].runs[0].band, "[q5,q9]");
|
||||
const statistics = JSON.parse(
|
||||
readFileSync(join(statsOut, "stats.json"), "utf8"),
|
||||
);
|
||||
assert.deepEqual(statistics[0].kill.checks, {
|
||||
sameContainer: true,
|
||||
chargedOnceNoRerun: true,
|
||||
resultAndContinued: true,
|
||||
});
|
||||
writeFileSync(
|
||||
join(dir, "container-after-resume.json"),
|
||||
JSON.stringify({
|
||||
Id: "replacement",
|
||||
State: { Running: true, StartedAt: "2026-10-04T00:01:00Z" },
|
||||
}),
|
||||
);
|
||||
const badOut = join(root, "bad-stats");
|
||||
const bad = spawnSync(
|
||||
process.execPath,
|
||||
["eval/forward/stats.ts", badOut, refs, dir],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
assert.equal(bad.status, 1, bad.stderr);
|
||||
assert.equal(
|
||||
JSON.parse(readFileSync(join(badOut, "stats.json"), "utf8"))[0].kill.checks
|
||||
.sameContainer,
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("forward 账目：没启动退还、修复用尽不扣次；缺不跑事件或结果不能通过", (t) => {
|
||||
const ledger = Ledger.open(join(tempDir(t), "ledger.sqlite"));
|
||||
t.after(() => ledger.close());
|
||||
const w = new Writer(ledger),
|
||||
config = { ...fullConfig(), maxRepairs: 0 };
|
||||
ledger.append("config", { config });
|
||||
ledger.append("runPlanned", {
|
||||
run: "base-0",
|
||||
purpose: "baseline",
|
||||
round: 0,
|
||||
object: null,
|
||||
definition,
|
||||
});
|
||||
ledger.append("runResult", {
|
||||
run: "base-0",
|
||||
process: notStarted,
|
||||
judged: null,
|
||||
});
|
||||
w.run("base-1", "baseline", 0, null, judged(1, "main"));
|
||||
w.decision("batch", 1, null, { chosen: ["c"] });
|
||||
ledger.append("code", { object: "c", version: "v", locations: null });
|
||||
w.open("v", "verification", 1, ["v"]);
|
||||
w.sub("v", "v", "verification", "c", null, verification(false));
|
||||
w.close("v");
|
||||
assert.equal(forwardAccounts(ledger.records()).candidates[0]!.pass, false);
|
||||
ledger.append("event", forwardNotRunEvent("c", "修复用尽"));
|
||||
w.decision("adopt", 1, null, { adopted: null });
|
||||
ledger.append("event", {
|
||||
event: "halted",
|
||||
reason: FORWARD_STOP,
|
||||
objects: [],
|
||||
});
|
||||
const accounts = forwardAccounts(ledger.records());
|
||||
assert.deepEqual(accounts.balance, {
|
||||
charged: 1,
|
||||
reserved: 0,
|
||||
balance: config.runs - 1,
|
||||
});
|
||||
assert.ok(Object.values(accounts.checks).every(Boolean));
|
||||
ledger.append("runPlanned", {
|
||||
run: "unfinished",
|
||||
purpose: "forward",
|
||||
round: 1,
|
||||
object: "c",
|
||||
definition,
|
||||
});
|
||||
ledger.append("runStarted", { run: "unfinished" });
|
||||
assert.equal(forwardAccounts(ledger.records()).checks.runsFinished, false);
|
||||
assert.equal(forwardAccounts(ledger.records()).candidates[0]!.pass, false);
|
||||
});
|
||||
|
||||
test("forward 分位区间不把无效分数当零，三条边界按既有placement", () => {
|
||||
assert.deepEqual(
|
||||
[null, 0, 1, 2, 3, 4].map((s) => scoreBand(s, { q1: 1, q5: 2, q9: 3 })),
|
||||
[null, "<q1", "[q1,q5)", "[q5,q9]", "[q5,q9]", ">q9"],
|
||||
);
|
||||
});
|
||||
|
||||
test("forward 轻量检查：跨会话按工具编号配对，拒绝超限不算执行，缺结果不伪装零次", (t) => {
|
||||
const dir = tempDir(t);
|
||||
const request = (id: string) => ({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: [{ type: "toolCall", name: "research_check", id }],
|
||||
},
|
||||
});
|
||||
const result = (id: string, text: string) => ({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "toolResult",
|
||||
toolName: "research_check",
|
||||
toolCallId: id,
|
||||
content: [{ type: "text", text }],
|
||||
},
|
||||
});
|
||||
writeFileSync(join(dir, "1.jsonl"), JSON.stringify(request("a")) + "\n");
|
||||
writeFileSync(
|
||||
join(dir, "2.jsonl"),
|
||||
[
|
||||
result(
|
||||
"a",
|
||||
JSON.stringify({ used: 1, maxCount: 1, seconds: 5, result: {} }),
|
||||
),
|
||||
request("b"),
|
||||
result("b", "本次调用的轻量检查已用完（上限 1 次），不能再查。"),
|
||||
]
|
||||
.map((r) => JSON.stringify(r))
|
||||
.join("\n"),
|
||||
);
|
||||
const usage = lightCheckUsage(dir, 1);
|
||||
assert.equal(usage.requested, 2);
|
||||
assert.equal(usage.executed, 1);
|
||||
assert.equal(usage.deniedLimit, 1);
|
||||
assert.equal(usage.pass, true);
|
||||
writeFileSync(join(dir, "3.jsonl"), JSON.stringify(request("c")));
|
||||
assert.equal(lightCheckUsage(dir, 1).pass, false);
|
||||
});
|
||||
|
||||
function referenceFixture(dir: string) {
|
||||
mkdirSync(dir);
|
||||
const db = new DatabaseSync(join(dir, "board.sqlite"));
|
||||
db.exec("CREATE TABLE facts (created_seq INTEGER, kind TEXT, payload TEXT)");
|
||||
const study = dir.split("/").at(-1)!;
|
||||
const insert = (seq: number, kind: string, payload: unknown) =>
|
||||
db
|
||||
.prepare("INSERT INTO facts VALUES (?,?,?)")
|
||||
.run(seq, kind, JSON.stringify(payload));
|
||||
insert(1, "run-link", {
|
||||
purpose: "forward",
|
||||
roundId: `${study}.r1`,
|
||||
runId: "r",
|
||||
candidateId: "c",
|
||||
});
|
||||
insert(2, "run-observation", {
|
||||
runId: "r",
|
||||
status: "incomplete",
|
||||
finalMetric: { value: 99 },
|
||||
reasons: ["无效"],
|
||||
});
|
||||
insert(3, "adoption", {
|
||||
roundId: `${study}.r1`,
|
||||
adoptedRunId: null,
|
||||
newBaselineId: "B0",
|
||||
});
|
||||
db.close();
|
||||
}
|
||||
|
||||
test("forward 参考分数：incomplete 原值保留但有效分数为空", (t) => {
|
||||
const dir = join(tempDir(t), "reference");
|
||||
referenceFixture(dir);
|
||||
const rows = referenceScores(join(dir, "board.sqlite"));
|
||||
assert.equal(rows[0]!.rawScore, 99);
|
||||
assert.equal(rows[0]!.validScore, null);
|
||||
assert.equal(rows[0]!.adopted, false);
|
||||
assert.equal(rows[0]!.observationSeq, 2);
|
||||
});
|
||||
Reference in New Issue
Block a user