feat: 任务书 08,jev 模块与账本记 Jev 判断

src/jev 3 个文件 515 行:判重、查冻结条款、查代码改动三个函数,九问问法和阈值照参考项目,只改说法和字段名,阈值注明未在新项目材料上校准,等真实数据。
新增 npm run test:jev(真实 Jev 三处各问一次)。vocab 加 JevJudgment。
ledger 表结构版本 3:callEnded 四种结局对上 agents,墙钟时间用 Measured(恢复时补写出错、原因"进程中断"、测不到);退回、提交记录都带 jev。
config 加必填的 models.jev。

用户原话:"这次就给提交记录加一个和退回记录同样的 Jev 判断字段";"墙钟时间改用 Measured,恢复时写出错加测不到(原因'进程中断')"。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-03 02:10:55 -04:00
parent 68e49fd01a
commit 0ac1e01613
18 changed files with 1228 additions and 57 deletions
+16
View File
@@ -91,6 +91,21 @@ test("类型或取值不对报错,带字段路径", () => {
}
});
test("Jev 模型必填:少了、写 null、写空白都报错,不补 jev-latest", () => {
const missing = fullConfig() as unknown as Obj;
delete (missing.models as Obj).jev;
assert.match(errorOf(missing), /^\/models\/jev:缺少这一项/m);
const nulled = fullConfig() as unknown as Obj;
(nulled.models as Obj).jev = null;
assert.match(errorOf(nulled), /^\/models\/jev:应为字符串$/m);
const blank = fullConfig() as unknown as Obj;
(blank.models as Obj).jev = " ";
assert.match(errorOf(blank), /^\/models\/jev:不能为空,也不能只有空白$/m);
const naive = naiveConfig() as unknown as Obj;
delete (naive.models as Obj).jev;
assert.match(errorOf(naive), /^\/models\/jev:缺少这一项/m);
});
test("一次列出多处错误,超过 typebox 默认的 8 条、错在 Union 分支里的也都列出", () => {
const config = fullConfig() as unknown as Obj;
config.runs = -1;
@@ -196,6 +211,7 @@ test("字段定义里每个字段都有中文说明,并标明用户配置还
assert.equal(category(["runs"]), "用户配置");
assert.equal(category(["factorLimit"]), "方法参数");
assert.equal(category(["flow"]), "方法参数");
assert.equal(category(["models", "jev"]), "用户配置");
assert.ok(seen.includes("/models/researchers/0/thinking"));
assert.ok(seen.includes("/flow/ablation/cellRule"));
});
+1
View File
@@ -13,6 +13,7 @@ export function fullConfig(): ResearchConfig {
researchers: [{ ...glm }, { ...glm }, { ...glm }],
experimenter: { ...glm },
verifier: { ...glm },
jev: "jev-latest",
},
proposalLimit: 1,
batchLimit: 2,
+72
View File
@@ -0,0 +1,72 @@
import assert from "node:assert/strict";
import { checkHunk, checkProtocol, judgeSameChange } from "../src/jev/index.ts";
import type { JevJudgment } from "../src/vocab/index.ts";
// Jev 的真实接线测试(任务书 08 第七部分),npm run test:jev -- <模型名>:要联网、要密钥 TYPESAFE_API_KEY,不放进 npm test。
// 三处各问一次真实 Jev,材料是随手写的简单填圆材料。只确认请求发得出去、九问都拿回 0 到 1 之间的概率、
// 请求编号读得到;不判断概率对不对。
const model = process.argv[2];
if (model === undefined || process.argv.length !== 3) throw new Error("用法:npm run test:jev -- <Jev 模型名>,例如 jev-latest");
const body =
"### 不许改什么\n\nharness.py、validate.py 不改。\n\n### 随机性\n\n不用随机数。\n\n" +
"### 何时停\n\n程序跑完一次 construct_packing 就停。\n\n### 什么运行不算数\n\n校验不通过的运行不算数。";
const factor = {
name: "外圈环半径缩到 0.45",
method: "把 construct_packing 里外圈环半径常数 0.7 改为 0.45。",
keySettings: "环半径 0.45",
continues: null,
};
const all: Array<[string, JevJudgment]> = [];
const same = await judgeSameChange(model, [factor], [{ name: "外圈收紧", method: "construct_packing 中外圈 8 个圆所在环的半径从 0.7 改成 0.45。" }]);
all.push(["判重", same]);
const frozen = await checkProtocol(model, {
factors: [factor],
safeguards: [],
metrics: { defined: [], reused: [] },
body,
clauses: ["initial.py 的 construct_packing 里,外圈环半径常数 0.7 改为 0.45"],
});
for (const j of frozen.judgments) all.push(["冻结条款", j]);
const hunk = await checkHunk(
model,
{
path: "initial.py",
change: "modified",
text:
"@@ -33,7 +33,7 @@ def construct_packing():\n" +
" for i in range(8):\n" +
" angle = 2 * np.pi * i / 8\n" +
"- centers[i + 1] = [0.5 + 0.7 * np.cos(angle), 0.5 + 0.7 * np.sin(angle)]\n" +
"+ centers[i + 1] = [0.5 + 0.45 * np.cos(angle), 0.5 + 0.45 * np.sin(angle)]\n" +
" \n" +
" for i in range(16):\n" +
" angle = 2 * np.pi * i / 16",
},
{ body, metrics: [] },
);
for (const j of hunk.judgments) all.push(["代码改动", j]);
for (const [site, j] of all) {
console.log(`${site}\t${j.question}\t概率 ${j.probability}\t阈值 ${j.threshold}\t${j.yes ? "是" : "否"}\t请求编号 ${j.requestId}`);
}
console.log(`冻结条款有问题:${frozen.flagged};代码改动有问题:${hunk.flagged}`);
assert.deepEqual(
all.map(([, j]) => j.question),
["same_change", "self_contradictory", "changes_measurement", "too_vague", "unseeded_randomness", "reads_clock", "touches_measurement", "early_exit", "zero_for_unmeasured"],
);
for (const [, j] of all) {
assert.ok(j.probability >= 0 && j.probability <= 1, `${j.question} 的概率不在 0 到 1 之间`);
assert.ok(j.requestId.length > 0, `${j.question} 没有请求编号`);
}
// 冻结三问各发一次请求,请求编号各不相同;代码改动五问一次请求,编号相同。
assert.equal(new Set(frozen.judgments.map((j) => j.requestId)).size, 3);
assert.equal(new Set(hunk.judgments.map((j) => j.requestId)).size, 1);
console.log("九问都拿回了 0 到 1 之间的概率和请求编号");
+308
View File
@@ -0,0 +1,308 @@
import assert from "node:assert/strict";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { test, type TestContext } from "node:test";
import { checkHunk, checkProtocol, codeHunks, judgeSameChange, type FrozenProtocol, type ProtocolRules } from "../src/jev/index.ts";
import { hash, type Json } from "../src/vocab/index.ts";
import { compareVersions, createRepository, saveVersion } from "../src/workspace/index.ts";
// jev 的测试(任务书 08 第七部分),不花钱:假 Jev 在测试自己的环境里造——把全局 fetch 换成假的,
// 产品代码照常向真实地址发请求,拿到的是这里造的回答。重试的等待把全局 setTimeout 换成立刻回调,并记下要等多久。
type Reply = { status: number; body: string; requestId: string | null };
type Sent = { url: string; auth: string | null; body: { model: string; state: any; questions: Record<string, any> } };
/** 装一个假 Jev:每次请求交给 answer 决定回什么,并记下发出去的请求。 */
function fakeJev(t: TestContext, answer: (body: Sent["body"], n: number) => Reply | Error): Sent[] {
const sent: Sent[] = [];
const key = process.env["TYPESAFE_API_KEY"];
process.env["TYPESAFE_API_KEY"] = "test-key";
t.after(() => {
if (key === undefined) delete process.env["TYPESAFE_API_KEY"];
else process.env["TYPESAFE_API_KEY"] = key;
});
t.mock.method(globalThis, "fetch", async (url: string, init: RequestInit) => {
const body = JSON.parse(init.body as string);
sent.push({ url, auth: new Headers(init.headers).get("authorization"), body });
const reply = answer(body, sent.length);
if (reply instanceof Error) throw reply;
const headers = reply.requestId === null ? {} : { "x-typesafe-request-id": reply.requestId };
return new Response(reply.body, { status: reply.status, headers });
});
return sent;
}
/** 正常回答:每一问的概率由 p 给,请求编号是 req_<第几次请求>。 */
function answerAll(p: (name: string) => number): (body: Sent["body"], n: number) => Reply {
return (body, n) => ({
status: 200,
requestId: `req_${n}`,
body: JSON.stringify({
model: "jev-test",
answers: Object.fromEntries(Object.keys(body.questions).map((name) => [name, { type: "noul", noul: p(name) }])),
}),
});
}
/** 把重试前的等待换成立刻继续,记下每次要等的毫秒数。 */
function instantWaits(t: TestContext): number[] {
const waits: number[] = [];
t.mock.method(globalThis, "setTimeout", (fn: () => void, ms: number) => {
waits.push(ms);
setImmediate(fn);
});
return waits;
}
const PROTOCOL: FrozenProtocol = {
factors: [
{
name: "外圈环半径缩到 0.45",
method: "把 construct_packing 里外圈环半径常数 0.7 改为 0.45。",
keySettings: "环半径 0.45",
offBehavior: "回到 0.7",
continues: null,
},
],
safeguards: ["clip-radii"],
metrics: {
defined: [
{ id: "min-gap", meaning: "最小间隙", unit: "无", direction: "maximize", howMeasured: "最终解上所有圆对的最小间隙,测一次", missing: "recordMissing" },
],
reused: ["c1.overlap"],
},
body: "### 不许改什么\n\nharness.py 不改。",
clauses: ["外圈环半径常数改为 0.45(initial.py:36)"],
};
const RULES: ProtocolRules = {
body: PROTOCOL.body,
metrics: [{ id: "min-gap", howMeasured: "最终解上所有圆对的最小间隙,测一次", missing: "recordMissing" }],
};
const HUNK = { path: "initial.py", change: "modified", text: "@@ -1 +1 @@\n-a\n+b" } as const;
/** 参考项目 src/judge/questions.ts 的 JEV_THRESHOLDS 填圆那一套里这九问的值,抄自参考项目源码。 */
const REFERENCE_THRESHOLDS: Record<string, number> = {
same_change: 0.3,
self_contradictory: 0.5,
changes_measurement: 0.21,
too_vague: 0.78,
unseeded_randomness: 0.55,
reads_clock: 0.55,
touches_measurement: 0.65,
early_exit: 0.63,
zero_for_unmeasured: 0.62,
};
test("概率超过阈值判是,等于或低于阈值判否;阈值是参考项目的值", async (t) => {
// 冻结三问:第一问比阈值高 0.01,第二问正好等于阈值,第三问低 0.01。
const offsets: Record<string, number> = { self_contradictory: 0.01, changes_measurement: 0, too_vague: -0.01 };
const sent = fakeJev(t, answerAll((name) => Math.round(((REFERENCE_THRESHOLDS[name] as number) + (offsets[name] as number)) * 100) / 100));
const result = await checkProtocol("jev-x", PROTOCOL);
assert.deepEqual(
result.judgments.map((j) => [j.question, j.threshold, j.yes, j.requestId]),
[
["self_contradictory", 0.5, true, "req_1"],
["changes_measurement", 0.21, false, "req_2"],
["too_vague", 0.78, false, "req_3"],
],
);
assert.equal(result.flagged, true);
// 冻结三问每问单独一次请求,材料相同;模型名照传;密钥只在请求头里。
assert.deepEqual(
sent.map((s) => Object.keys(s.body.questions)),
[["self_contradictory"], ["changes_measurement"], ["too_vague"]],
);
for (const s of sent) {
assert.deepEqual(s.body.state, sent[0]?.body.state);
assert.equal(s.body.model, "jev-x");
assert.equal(s.auth, "Bearer test-key");
assert.equal(s.url, "https://api.typesafe.ai/v1/systemone");
assert.ok(!JSON.stringify(s.body).includes("test-key"));
}
});
test("一个差异块五问一次请求,判重一对一次请求;阈值都是参考项目的值", async (t) => {
const sent = fakeJev(t, answerAll((name) => (name === "early_exit" ? 0.64 : 0.2)));
const hunk = await checkHunk("jev-x", HUNK, RULES);
assert.equal(sent.length, 1);
assert.equal(hunk.flagged, true);
assert.deepEqual(
hunk.judgments.map((j) => [j.question, j.threshold, j.yes]),
[
["unseeded_randomness", 0.55, false],
["reads_clock", 0.55, false],
["touches_measurement", 0.65, false],
["early_exit", 0.63, true],
["zero_for_unmeasured", 0.62, false],
],
);
const same = await judgeSameChange("jev-x", PROTOCOL.factors, PROTOCOL.factors);
assert.equal(sent.length, 2);
assert.deepEqual(same, { question: "same_change", questionHash: same.questionHash, threshold: 0.3, probability: 0.2, yes: false, requestId: "req_2" });
});
test("回答读不懂就报错,带上请求编号", async (t) => {
const cases: Array<[string, Reply, RegExp]> = [
["没有概率", { status: 200, requestId: "req_a", body: JSON.stringify({ answers: {} }) }, /same_change 没有 0 到 1 之间的概率(request-id: req_a)/],
[
"概率大于 1",
{ status: 200, requestId: "req_b", body: JSON.stringify({ answers: { same_change: { noul: 1.2 } } }) },
/same_change 没有 0 到 1 之间的概率(request-id: req_b)/,
],
[
"概率不是数",
{ status: 200, requestId: "req_c", body: JSON.stringify({ answers: { same_change: { noul: "0.4" } } }) },
/same_change 没有 0 到 1 之间的概率(request-id: req_c)/,
],
["不是 JSON", { status: 200, requestId: "req_d", body: "<html>" }, /不是 JSON(request-id: req_d)/],
["没带请求编号", { status: 200, requestId: null, body: JSON.stringify({ answers: { same_change: { noul: 0.1 } } }) }, /没带请求编号/],
];
for (const [what, reply, message] of cases) {
await t.test(what, async (t) => {
fakeJev(t, () => reply);
await assert.rejects(judgeSameChange("jev-x", PROTOCOL.factors, PROTOCOL.factors), message);
});
}
});
test("调不通时按 2、10、30 秒重试,发满 4 次再报错", async (t) => {
const waits = instantWaits(t);
const sent = fakeJev(t, () => new TypeError("fetch failed"));
await assert.rejects(checkHunk("jev-x", HUNK, RULES), /发了 4 次都失败/);
assert.equal(sent.length, 4);
assert.deepEqual(waits, [2_000, 10_000, 30_000]);
});
test("服务暂时出错后恢复就照常判;请求本身被拒不重试;没有密钥直接报错", async (t) => {
await t.test("503 后恢复", async (t) => {
const waits = instantWaits(t);
const ok = answerAll(() => 0.9);
const sent = fakeJev(t, (body, n) => (n === 1 ? { status: 503, requestId: "req_x", body: "busy" } : ok(body, n)));
const j = await judgeSameChange("jev-x", PROTOCOL.factors, PROTOCOL.factors);
assert.deepEqual([j.yes, j.requestId], [true, "req_2"]);
assert.equal(sent.length, 2);
assert.deepEqual(waits, [2_000]);
});
await t.test("422 直接报错", async (t) => {
const sent = fakeJev(t, () => ({ status: 422, requestId: "req_y", body: "bad" }));
await assert.rejects(judgeSameChange("jev-x", PROTOCOL.factors, PROTOCOL.factors), /拒绝了请求,不重试:HTTP 422(request-id: req_y)/);
assert.equal(sent.length, 1);
});
await t.test("没有密钥", async (t) => {
const sent = fakeJev(t, answerAll(() => 0.1));
delete process.env["TYPESAFE_API_KEY"];
await assert.rejects(judgeSameChange("jev-x", PROTOCOL.factors, PROTOCOL.factors), /TYPESAFE_API_KEY/);
assert.equal(sent.length, 0);
});
});
test("问题哈希把材料说明、措辞、示例、阈值一起算,改一个字就变", async (t) => {
const sent = fakeJev(t, answerAll(() => 0.1));
const judgments = [
await judgeSameChange("jev-x", PROTOCOL.factors, PROTOCOL.factors),
...(await checkProtocol("jev-x", PROTOCOL)).judgments,
...(await checkHunk("jev-x", HUNK, RULES)).judgments,
];
assert.equal(judgments.length, 9);
assert.equal(new Set(judgments.map((j) => j.questionHash)).size, 9);
// 从实际发出去的请求里取回每一问的材料说明和问题原文。
const asked = new Map(sent.flatMap((s) => Object.entries(s.body.questions).map(([name, noul]) => [name, { task: s.body.state.task, noul }])));
for (const j of judgments) {
const { task, noul } = asked.get(j.question) as { task: string; noul: Json };
const content = { task, name: j.question, noul, threshold: j.threshold };
assert.equal(j.questionHash, hash(content));
const changed = JSON.parse(JSON.stringify(content));
changed.noul.criteria.true.examples[0] = `${changed.noul.criteria.true.examples[0]}x`;
assert.notEqual(hash(changed), j.questionHash);
assert.notEqual(hash({ ...content, threshold: j.threshold + 0.01 }), j.questionHash);
assert.notEqual(hash({ ...content, task: `${task} ` }), j.questionHash);
}
});
test("材料:给定带类型的输入,发出去的材料与手写预期一致", async (t) => {
const sent = fakeJev(t, answerAll(() => 0.1));
const other = [{ name: "六行六边形网格", method: "用 6 行六边形网格摆圆心。", keySettings: "6 行", continues: "f2" }];
await judgeSameChange("jev-x", PROTOCOL.factors, other);
await checkProtocol("jev-x", PROTOCOL);
await checkHunk("jev-x", { path: "a.py", change: "added", text: "@@ -0,0 +1 @@\n+x" }, RULES);
const states = sent.map((s) => s.body.state);
const [dedup, freeze, , , diff] = states;
for (const state of states) assert.equal(typeof state.task, "string");
// 判重只放名称和做法:关键设定、关掉时换成什么、延续哪个因子都不放。
assert.deepEqual(dedup, {
task: dedup.task,
proposal_a: { factors: [{ name: "外圈环半径缩到 0.45", method: "把 construct_packing 里外圈环半径常数 0.7 改为 0.45。" }] },
proposal_b: { factors: [{ name: "六行六边形网格", method: "用 6 行六边形网格摆圆心。" }] },
});
assert.deepEqual(freeze, {
task: freeze.task,
protocol: {
factors: [
{
name: "外圈环半径缩到 0.45",
method: "把 construct_packing 里外圈环半径常数 0.7 改为 0.45。",
keySettings: "环半径 0.45",
offBehavior: "回到 0.7",
continues: null,
},
],
safeguards: ["clip-radii"],
body: "### 不许改什么\n\nharness.py 不改。",
metrics: {
defined: [
{ id: "min-gap", meaning: "最小间隙", unit: "无", direction: "maximize", howMeasured: "最终解上所有圆对的最小间隙,测一次", missing: "recordMissing" },
],
reused: ["c1.overlap"],
},
clauses: ["外圈环半径常数改为 0.45(initial.py:36)"],
},
});
assert.deepEqual(diff, {
task: diff.task,
protocol: {
body: "### 不许改什么\n\nharness.py 不改。",
metrics: [{ id: "min-gap", howMeasured: "最终解上所有圆对的最小间隙,测一次", missing: "recordMissing" }],
},
change: { path: "a.py", change: "added", diff: "@@ -0,0 +1 @@\n+x" },
});
assert.equal(new Set(states.map((s) => s.task)).size, 3);
});
function tempDir(t: TestContext): string {
const dir = mkdtempSync(join(tmpdir(), "inforsi-jev-"));
t.after(() => rmSync(dir, { recursive: true, force: true }));
return dir;
}
test("把 workspace 的比较结果切成差异块,每块从 @@ 行起", (t) => {
const root = tempDir(t);
const repo = join(root, "v.git");
createRepository(repo);
const dir = join(root, "code");
mkdirSync(dir);
const lines = Array.from({ length: 30 }, (_, i) => `line ${i + 1}`);
writeFileSync(join(dir, "a.py"), `${lines.join("\n")}\n`);
writeFileSync(join(dir, "bin.dat"), Buffer.from([0, 1, 2]));
const v1 = saveVersion(repo, dir, []);
lines[1] = "changed 2";
lines[25] = "changed 26";
writeFileSync(join(dir, "a.py"), `${lines.join("\n")}\n`);
writeFileSync(join(dir, "bin.dat"), Buffer.from([0, 9, 2]));
writeFileSync(join(dir, "new.py"), "x = 1\n");
const v2 = saveVersion(repo, dir, []);
assert.ok(v1.saved && v2.saved);
const hunks = codeHunks(compareVersions(repo, v1.version, v2.version));
assert.deepEqual(
hunks.map((h) => [h.path, h.change, h.text.split("\n")[0]]),
[
["a.py", "modified", "@@ -1,5 +1,5 @@"],
["a.py", "modified", "@@ -23,7 +23,7 @@ line 22"],
["new.py", "added", "@@ -0,0 +1 @@"],
],
);
assert.equal(hunks[0]?.text, "@@ -1,5 +1,5 @@\n line 1\n-line 2\n+changed 2\n line 3\n line 4\n line 5");
assert.equal(hunks[2]?.text, "@@ -0,0 +1 @@\n+x = 1");
});
+26 -2
View File
@@ -33,14 +33,38 @@ test("模型调用:开始了没结束的被找出来,带着这次任务的
ledger.append("callStarted", callStart("k1", "r1", "s1"));
ledger.append("callStarted", callStart("k2", "e1"));
ledger.append("callUsage", { call: "k1", reply: 1, usage });
ledger.append("callEnded", { call: "k2", ending: "failed", reason: "接口超时" });
ledger.append("callEnded", { call: "k2", wallSeconds: { status: "measured", value: 30 }, status: "failed", reason: "接口超时" });
const calls = ledger.pending().calls;
assert.deepEqual(calls.map((c) => c.content.call), ["k1"]);
assert.deepEqual(calls[0]!.content.materials, callStart("k1").materials);
ledger.append("callEnded", { call: "k1", ending: "completed" });
ledger.append("callEnded", { call: "k1", wallSeconds: { status: "measured", value: 600 }, status: "notSubmitted", reason: "模型自己停下了" });
assert.deepEqual(ledger.pending(), { runs: [], calls: [] });
});
test("中断后恢复:给没结束的旧调用补写出错加测不到(进程中断),补写后不再算未完成", (t) => {
const path = join(tempDir(t), "ledger.db");
const ledger = Ledger.open(path);
ledger.append("callStarted", callStart("k1", "r1", "s1"));
ledger.append("callUsage", { call: "k1", reply: 1, usage });
ledger.close(); // 控制进程被系统杀掉
const reopened = Ledger.open(path);
t.after(() => reopened.close());
assert.deepEqual(reopened.pending().calls.map((c) => c.content.call), ["k1"]);
reopened.append("callEnded", {
call: "k1",
wallSeconds: { status: "unmeasured", reason: "进程中断" },
status: "failed",
reason: "进程中断",
});
assert.deepEqual(reopened.pending(), { runs: [], calls: [] });
assert.deepEqual(reopened.records().at(-1)!.content, {
call: "k1",
wallSeconds: { status: "unmeasured", reason: "进程中断" },
status: "failed",
reason: "进程中断",
});
});
test("关掉再打开:未完成的动作照样找得出来", (t) => {
const path = join(tempDir(t), "ledger.db");
const ledger = Ledger.open(path);
+19 -19
View File
@@ -24,7 +24,7 @@ import {
// 4. 这一轮消融已经有格确认启动了,才来关这一轮的解释阶段 → 拒收(方法.md 4.3 ⑭)。
function submitWrapUp(ledger: Ledger, stageId: string, author: string, version: number, body = "收尾") {
return ledger.append("submission", { stageId, author, target: null, item: null, version, kind: "wrapUp", content: wrapUp(body) });
return ledger.append("submission", { stageId, author, target: null, item: null, version, jev: null, kind: "wrapUp", content: wrapUp(body) });
}
test("拒收 1:只收开着的阶段的提交:没开过的不收,关了以后不收、替换也不收;开着时收", (t) => {
@@ -42,9 +42,9 @@ test("拒收 2 的正常情况:替换本人在这个阶段、对同一个对
const { ledger } = freshLedger(t);
openStage(ledger, "s1", "firstPrediction", 1);
const slot = { stageId: "s1", author: "r1", target: "c1", item: null, kind: "firstPrediction" as const };
ledger.append("submission", { ...slot, version: 1, content: prediction(range(1, 2, 3)) });
ledger.append("submission", { ...slot, version: 2, content: prediction(range(1, 2, 4)) });
ledger.append("submission", { ...slot, version: 3, content: prediction(range(1, 2, 5)) });
ledger.append("submission", { ...slot, version: 1, jev: null, content: prediction(range(1, 2, 3)) });
ledger.append("submission", { ...slot, version: 2, jev: null, content: prediction(range(1, 2, 4)) });
ledger.append("submission", { ...slot, version: 3, jev: null, content: prediction(range(1, 2, 5)) });
assert.equal(ledger.latestSubmission(slot)?.content.version, 3);
});
@@ -53,13 +53,13 @@ test("拒收 2:要替换的那一份不是最新一版、不是本人的、不
openStage(ledger, "s1", "firstPrediction", 1);
openStage(ledger, "s2", "firstPrediction", 1);
const slot = { stageId: "s1", author: "r1", target: "c1", item: null, kind: "firstPrediction" as const };
ledger.append("submission", { ...slot, version: 1, content: prediction(range(1, 2, 3)) });
ledger.append("submission", { ...slot, version: 2, content: prediction(range(1, 2, 4)) });
const next = { ...slot, version: 3, content: prediction(range(1, 2, 6)) };
ledger.append("submission", { ...slot, version: 1, jev: null, content: prediction(range(1, 2, 3)) });
ledger.append("submission", { ...slot, version: 2, jev: null, content: prediction(range(1, 2, 4)) });
const next = { ...slot, version: 3, jev: null, content: prediction(range(1, 2, 6)) };
// 想替换第 1 版,但最新是第 2 版。
assert.throws(() => ledger.append("submission", { ...slot, version: 2, content: prediction(range(0, 2, 4)) }), /最新是第 2 版/);
assert.throws(() => ledger.append("submission", { ...slot, version: 2, jev: null, content: prediction(range(0, 2, 4)) }), /最新是第 2 版/);
// 要替换的第 3 版不存在。
assert.throws(() => ledger.append("submission", { ...slot, version: 4, content: prediction(range(1, 2, 6)) }), /最新是第 2 版/);
assert.throws(() => ledger.append("submission", { ...slot, version: 4, jev: null, content: prediction(range(1, 2, 6)) }), /最新是第 2 版/);
// 不是本人的、不是同一个对象的、不在这个阶段的:本人在那里还没有交过。
assert.throws(() => ledger.append("submission", { ...next, author: "r2" }), /还没有交过/);
assert.throws(() => ledger.append("submission", { ...next, target: "c2" }), /还没有交过/);
@@ -71,10 +71,10 @@ test("拒收 2:同一个人对同一个候选的几份挑刺按各自的编号
const { ledger } = freshLedger(t);
openStage(ledger, "s1", "critique", 1);
const slot = { stageId: "s1", author: "r1", target: "c1", kind: "critique" as const };
ledger.append("submission", { ...slot, item: "k1", version: 1, content: critique("第一份") });
ledger.append("submission", { ...slot, item: "k2", version: 1, content: critique("第二份") });
ledger.append("submission", { ...slot, item: "k1", version: 2, content: critique("第一份改") });
assert.throws(() => ledger.append("submission", { ...slot, item: "k3", version: 2, content: critique("x") }), /还没有交过/);
ledger.append("submission", { ...slot, item: "k1", version: 1, jev: null, content: critique("第一份") });
ledger.append("submission", { ...slot, item: "k2", version: 1, jev: null, content: critique("第二份") });
ledger.append("submission", { ...slot, item: "k1", version: 2, jev: null, content: critique("第一份改") });
assert.throws(() => ledger.append("submission", { ...slot, item: "k3", version: 2, jev: null, content: critique("x") }), /还没有交过/);
});
test("拒收 2:实现、修复、核验、消融实现、消融核验只收第一版,不能替换", (t) => {
@@ -95,8 +95,8 @@ test("拒收 2:实现、修复、核验、消融实现、消融核验只收第
{ ...base, author: "v1", stageId: "aver", target: "cell1", kind: "ablationVerification" as const, content: { factors: [] } },
];
for (const c of cases) {
ledger.append("submission", { ...c, version: 1 });
assert.throws(() => ledger.append("submission", { ...c, version: 2 }), /不能替换/, c.kind);
ledger.append("submission", { ...c, version: 1, jev: null });
assert.throws(() => ledger.append("submission", { ...c, version: 2, jev: null }), /不能替换/, c.kind);
}
});
@@ -105,14 +105,14 @@ test("拒收 2:版本号只在同一种提交里接:不能替换的实现不
const { ledger } = freshLedger(t);
openStage(ledger, "impl", "implementation", 1, ["e1"]);
const implSlot = { stageId: "impl", author: "e1", target: "c1", item: null };
ledger.append("submission", { ...implSlot, version: 1, kind: "implementation", content: implementation() });
assert.throws(() => ledger.append("submission", { ...implSlot, version: 2, kind: "wrapUp", content: wrapUp() }), /还没有交过/);
ledger.append("submission", { ...implSlot, version: 1, jev: null, kind: "implementation", content: implementation() });
assert.throws(() => ledger.append("submission", { ...implSlot, version: 2, jev: null, kind: "wrapUp", content: wrapUp() }), /还没有交过/);
assert.equal(ledger.latestSubmission({ ...implSlot, kind: "implementation" })?.content.kind, "implementation");
openStage(ledger, "mix", "proposal", 1);
const mixSlot = { stageId: "mix", author: "r1", target: null, item: "x1" };
ledger.append("submission", { ...mixSlot, version: 1, kind: "proposal", content: proposal(range(1, 2, 3)) });
assert.throws(() => ledger.append("submission", { ...mixSlot, version: 2, kind: "critique", content: critique() }), /还没有交过/);
ledger.append("submission", { ...mixSlot, version: 1, jev: null, kind: "proposal", content: proposal(range(1, 2, 3)) });
assert.throws(() => ledger.append("submission", { ...mixSlot, version: 2, jev: null, kind: "critique", content: critique() }), /还没有交过/);
assert.equal(ledger.latestSubmission({ ...mixSlot, kind: "critique" }), undefined);
});
+184 -17
View File
@@ -3,7 +3,7 @@ import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { test } from "node:test";
import { Ledger, type LedgerRecord } from "../src/ledger/index.ts";
import { hash } from "../src/vocab/index.ts";
import { hash, type JevJudgment } from "../src/vocab/index.ts";
import {
callStart,
closeStage,
@@ -111,8 +111,11 @@ test("唯一约束:同一次模型调用开始两次报错", (t) => {
test("唯一约束:同一次模型调用结束两次报错", (t) => {
const { ledger, path } = freshLedger(t);
ledger.append("callStarted", callStart("k1"));
ledger.append("callEnded", { call: "k1", ending: "completed" });
assert.throws(() => ledger.append("callEnded", { call: "k1", ending: "failed", reason: "又报了一次" }), /call_ended_once/);
ledger.append("callEnded", { call: "k1", wallSeconds: { status: "measured", value: 12.5 }, status: "submitted" });
assert.throws(
() => ledger.append("callEnded", { call: "k1", wallSeconds: { status: "measured", value: 3 }, status: "failed", reason: "又报了一次" }),
/call_ended_once/,
);
assert.equal(countRows(path), 2);
});
@@ -162,19 +165,19 @@ test("内容不符合定义时不写入:多出的字段、提交内容不合
openStage(ledger, "s1", "wrapUp", 1);
assert.throws(() => ledger.append("runStarted", { run: "r1", extra: 1 } as never), /runStarted/);
assert.throws(
() => ledger.append("submission", { stageId: "s1", author: "r1", target: null, item: null, version: 1, kind: "wrapUp", content: { body: "" } }),
() => ledger.append("submission", { stageId: "s1", author: "r1", target: null, item: null, version: 1, jev: null, kind: "wrapUp", content: { body: "" } }),
/wrapUp/,
);
// 提案、挑刺一人能交好几份,必须带编号;实现一人对一个候选只交一份,编号必须为空,
// 否则另起一个编号就能绕开"实现不能替换"。
openStage(ledger, "prop", "proposal", 1);
assert.throws(
() => ledger.append("submission", { stageId: "prop", author: "r1", target: null, item: null as never, version: 1, kind: "proposal", content: proposal(range(1, 2, 3)) }),
() => ledger.append("submission", { stageId: "prop", author: "r1", target: null, item: null as never, version: 1, jev: null, kind: "proposal", content: proposal(range(1, 2, 3)) }),
/proposal/,
);
openStage(ledger, "impl", "implementation", 1, ["e1"]);
assert.throws(
() => ledger.append("submission", { stageId: "impl", author: "e1", target: "c1", item: "x" as never, version: 1, kind: "implementation", content: implementation() }),
() => ledger.append("submission", { stageId: "impl", author: "e1", target: "c1", item: "x" as never, version: 1, jev: null, kind: "implementation", content: implementation() }),
/implementation/,
);
assert.equal(countRows(path), 3); // 三条开阶段
@@ -184,7 +187,7 @@ test("写提案:作者自评拆成一条封存,提案记录里不带它", (t
const { ledger } = freshLedger(t);
openStage(ledger, "s1", "proposal", 1);
const record = ledger.append("submission", {
stageId: "s1", author: "r1", target: null, item: "r1-1", version: 1, kind: "proposal", content: proposal(range(1, 2, 3)),
stageId: "s1", author: "r1", target: null, item: "r1-1", version: 1, kind: "proposal", content: proposal(range(1, 2, 3)), jev: null,
});
const all = ledger.records();
const stored = all.find((r) => r.kind === "submission")!;
@@ -197,17 +200,174 @@ test("写提案:作者自评拆成一条封存,提案记录里不带它", (t
assert.equal(seal.seq, record.seq + 1);
});
test("退回:被退回的内容原样存,带退回理由;来自 Jev 的带原始概率和结论", (t) => {
const { ledger } = freshLedger(t);
// Jev 判断的样子照 src/vocab/jev.ts 的 JevJudgment 手写:问题名、问题全文哈希(64 位十六进制)、阈值、原始概率、结论、请求编号。
const judgment = (question: string, probability: number, threshold: number, requestId: string): JevJudgment => ({
question,
questionHash: "ab".repeat(32),
threshold,
probability,
yes: probability > threshold,
requestId,
});
test("退回:被退回的内容原样存;程序检查退回的 jev 为 null,Jev 退回的原样存它用到的几条判断", (t) => {
const path = join(tempDir(t), "ledger.db");
const ledger = Ledger.open(path);
const malformed = { factors: "不是数组" };
// 冻结条款一次问三问,退回时三条判断都存,包括判"否"的两条。
const clauses = [
judgment("changes_measurement", 0.97, 0.5, "req-1"),
judgment("self_contradictory", 0.02, 0.6, "req-2"),
judgment("too_vague", 0.1, 0.7, "req-3"),
];
ledger.append("rejection", { stageId: "s1", author: "r1", kind: "proposal", submitted: malformed, reason: "因子清单格式不对", jev: null });
ledger.append("rejection", {
stageId: "s6", author: "r2", kind: "response", submitted: { body: "改了评测" }, reason: "冻结条款改动了评测",
jev: { probabilities: { yes: 0.97, no: 0.03 }, conclusion: "changesEvaluation" },
stageId: "s6", author: "r2", kind: "response", submitted: { body: "改了评测" }, reason: "冻结条款改动了评测", jev: clauses,
});
const rejections = ledger.records().filter((r) => r.kind === "rejection");
ledger.close();
const reopened = Ledger.open(path);
t.after(() => reopened.close());
const rejections = reopened.records().filter((r) => r.kind === "rejection");
assert.deepEqual(rejections.map((r) => r.content.submitted), [malformed, { body: "改了评测" }]);
assert.deepEqual(rejections[1]!.content.jev, { probabilities: { yes: 0.97, no: 0.03 }, conclusion: "changesEvaluation" });
assert.equal(rejections[0]!.content.jev, null);
assert.deepEqual(rejections[1]!.content.jev, [
{ question: "changes_measurement", questionHash: "ab".repeat(32), threshold: 0.5, probability: 0.97, yes: true, requestId: "req-1" },
{ question: "self_contradictory", questionHash: "ab".repeat(32), threshold: 0.6, probability: 0.02, yes: false, requestId: "req-2" },
{ question: "too_vague", questionHash: "ab".repeat(32), threshold: 0.7, probability: 0.1, yes: false, requestId: "req-3" },
]);
});
test("退回:jev 不合 JevJudgment 的拒收,一条都不写", (t) => {
const { ledger, path } = freshLedger(t);
const good = judgment("same_change", 0.9, 0.5, "req-1");
const { requestId: _, ...withoutRequest } = good;
const bad: unknown[] = [
[], // 空数组:Jev 退回至少有一条判断,程序退回写 null
{ probabilities: { yes: 0.97, no: 0.03 }, conclusion: "changesEvaluation" }, // 旧写法:不是数组
good, // 一条判断也要放进数组
[{ ...good, probability: 1.2 }], // 概率超出 0 到 1
[{ ...good, threshold: -0.1 }], // 阈值超出 0 到 1
[withoutRequest], // 缺请求编号
[{ ...good, requestId: "" }], // 请求编号为空
[{ ...good, questionHash: "abc" }], // 哈希不是 64 位十六进制
[{ ...good, question: "Same Change" }], // 问题名格式不对
[{ ...good, yes: "true" }], // 结论不是布尔值
[{ ...good, note: "多写一项" }], // 不认识的字段
];
for (const jev of bad) {
assert.throws(
() => ledger.append("rejection", { stageId: "s1", author: "r1", kind: "proposal", submitted: {}, reason: "判为同一个改法", jev } as never),
/账本记录"rejection"的内容不符合定义/,
JSON.stringify(jev),
);
}
assert.equal(countRows(path), 0);
});
test("模型调用结束:四种结局各存一条,墙钟时间(秒)测到带值、测不到带原因,关掉再打开读出不变", (t) => {
const path = join(tempDir(t), "ledger.db");
const ledger = Ledger.open(path);
const endings = [
{ call: "k1", wallSeconds: { status: "measured", value: 812.25 }, status: "submitted" },
{ call: "k2", wallSeconds: { status: "measured", value: 95 }, status: "notSubmitted", reason: "模型自己停下了,一份都没交" },
{ call: "k3", wallSeconds: { status: "unmeasured", reason: "进程中断" }, status: "failed", reason: "进程中断,恢复时补写" },
{ call: "k4", wallSeconds: { status: "measured", value: 0 }, status: "stopped" },
] as const;
for (const e of endings) {
ledger.append("callStarted", callStart(e.call));
ledger.append("callEnded", e);
}
ledger.close();
const reopened = Ledger.open(path);
t.after(() => reopened.close());
const ended = reopened.records().filter((r) => r.kind === "callEnded").map((r) => r.content);
assert.deepEqual(ended, [
{ call: "k1", wallSeconds: { status: "measured", value: 812.25 }, status: "submitted" },
{ call: "k2", wallSeconds: { status: "measured", value: 95 }, status: "notSubmitted", reason: "模型自己停下了,一份都没交" },
{ call: "k3", wallSeconds: { status: "unmeasured", reason: "进程中断" }, status: "failed", reason: "进程中断,恢复时补写" },
{ call: "k4", wallSeconds: { status: "measured", value: 0 }, status: "stopped" },
]);
assert.deepEqual(reopened.pending().calls, []);
});
test("模型调用结束:结局和原因对不上、墙钟时间缺失或不合 Measured、测到的值为负,都拒收", (t) => {
const { ledger, path } = freshLedger(t);
const s3 = { status: "measured", value: 3 };
const bad: unknown[] = [
{ call: "k1", wallSeconds: s3, status: "notSubmitted" }, // 没交要带原因
{ call: "k1", wallSeconds: s3, status: "failed" }, // 出错要带原因
{ call: "k1", wallSeconds: s3, status: "failed", reason: "" }, // 原因不能为空
{ call: "k1", wallSeconds: s3, status: "submitted", reason: "交了" }, // 交了不带原因
{ call: "k1", wallSeconds: s3, status: "stopped", reason: "用户停下" }, // 被停下不带原因
{ call: "k1", wallSeconds: s3, status: "completed" }, // 旧写法
{ call: "k1", status: "submitted" }, // 缺墙钟时间
{ call: "k1", wallSeconds: 3, status: "submitted" }, // 裸数字,不是 Measured
{ call: "k1", wallSeconds: null, status: "submitted" }, // 测不到不能写 null
{ call: "k1", wallSeconds: { status: "measured", value: "3" }, status: "submitted" }, // 值不是数
{ call: "k1", wallSeconds: { status: "measured" }, status: "submitted" }, // 测到却没带值
{ call: "k1", wallSeconds: { status: "unmeasured" }, status: "failed", reason: "进程中断" }, // 测不到却没带原因
{ call: "k1", wallSeconds: { status: "unmeasured", reason: "" }, status: "failed", reason: "进程中断" }, // 原因为空
{ call: "k1", wallSeconds: { status: "unmeasured", reason: "进程中断", value: 0 }, status: "failed", reason: "进程中断" }, // 测不到又带值
];
for (const content of bad) {
assert.throws(() => ledger.append("callEnded", content as never), /账本记录"callEnded"的内容不符合定义/, JSON.stringify(content));
}
assert.equal(countRows(path), 0);
});
test("提交:jev 在外层与 content 并列,null 或 JevJudgment 数组都能存能读;提案拆封存不受影响", (t) => {
const path = join(tempDir(t), "ledger.db");
const ledger = Ledger.open(path);
openStage(ledger, "s1", "proposal", 1);
openStage(ledger, "s9", "implementation", 1);
// 代码改动五问都判"否"才收下,五条判断连同原始概率都存。
const checks = [
judgment("unseeded_randomness", 0.01, 0.5, "req-1"),
judgment("reads_clock", 0.02, 0.5, "req-2"),
judgment("touches_measurement", 0.2, 0.6, "req-3"),
judgment("early_exit", 0.05, 0.5, "req-4"),
judgment("zero_for_unmeasured", 0.3, 0.7, "req-5"),
];
const p = ledger.append("submission", { stageId: "s1", author: "r1", target: null, item: "r1-1", version: 1, kind: "proposal", content: proposal(range(1, 2, 3)), jev: null });
ledger.append("submission", { stageId: "s9", author: "e1", target: "c1", item: null, version: 1, kind: "implementation", content: implementation(), jev: checks });
ledger.close();
const reopened = Ledger.open(path);
t.after(() => reopened.close());
const all = reopened.records();
const subs = all.filter((r) => r.kind === "submission");
assert.deepEqual(subs.map((r) => r.content.jev), [
null,
[
{ question: "unseeded_randomness", questionHash: "ab".repeat(32), threshold: 0.5, probability: 0.01, yes: false, requestId: "req-1" },
{ question: "reads_clock", questionHash: "ab".repeat(32), threshold: 0.5, probability: 0.02, yes: false, requestId: "req-2" },
{ question: "touches_measurement", questionHash: "ab".repeat(32), threshold: 0.6, probability: 0.2, yes: false, requestId: "req-3" },
{ question: "early_exit", questionHash: "ab".repeat(32), threshold: 0.5, probability: 0.05, yes: false, requestId: "req-4" },
{ question: "zero_for_unmeasured", questionHash: "ab".repeat(32), threshold: 0.7, probability: 0.3, yes: false, requestId: "req-5" },
],
]);
assert.equal("jev" in (subs[1]!.content.content as object), false);
const seal = all.find((r) => r.kind === "seal")!;
assert.deepEqual(seal.content, { proposal: p.seq, range: range(1, 2, 3) });
assert.equal(seal.seq, p.seq + 1);
});
test("提交:缺 jev、jev 不合 JevJudgment 数组、jev 写进 content 里,都拒收", (t) => {
const { ledger, path } = freshLedger(t);
openStage(ledger, "s1", "wrapUp", 1);
const good = judgment("same_change", 0.1, 0.5, "req-1");
const base = { stageId: "s1", author: "r1", target: null, item: null, version: 1, kind: "wrapUp", content: { body: "收尾" } };
const bad: unknown[] = [
base, // 缺 jev:没用到 Jev 要写 null,不靠省略
{ ...base, jev: [] }, // 空数组
{ ...base, jev: good }, // 一条也要放进数组
{ ...base, jev: [{ ...good, probability: -0.1 }] }, // 概率超出 0 到 1
{ ...base, jev: [{ ...good, extra: 1 }] }, // 不认识的字段
{ ...base, jev: null, content: { body: "收尾", jev: null } }, // jev 不进 content
];
for (const content of bad) {
assert.throws(() => ledger.append("submission", content as never), /账本记录"submission(提交种类 wrapUp)"的内容不符合定义/, JSON.stringify(content));
}
assert.equal(countRows(path), 1);
});
test("事务:中途出错,一条都没写进去;中途被拒收,前面写的也撤销;正常结束则全部写进去", (t) => {
@@ -253,7 +413,7 @@ test("关掉再打开:读出的记录不变,序号接着往下排", (t) => {
const ledger = Ledger.open(path);
ledger.append("config", { researchers: ["r1", "r2"] });
openStage(ledger, "s1", "proposal", 1);
ledger.append("submission", { stageId: "s1", author: "r1", target: null, item: "r1-1", version: 1, kind: "proposal", content: proposal(range(1, 2, 3)) });
ledger.append("submission", { stageId: "s1", author: "r1", target: null, item: "r1-1", version: 1, jev: null, kind: "proposal", content: proposal(range(1, 2, 3)) });
ledger.append("note", { author: "r2", body: "笔记" });
closeStage(ledger, "s1");
planRun(ledger, "run1", "forward", 1);
@@ -276,6 +436,13 @@ test("打开:表结构版本不对报错;不是空文件也不是账本的
db.close();
assert.throws(() => Ledger.open(wrongVersion), /版本是 99/);
// 版本 2 是"模型调用结束"还没有墙钟时间、退回还没有 Jev 判断数组时的账本,不迁移,打开报错。
const previous = join(dir, "v2.db");
const v2 = new DatabaseSync(previous);
v2.exec("PRAGMA user_version = 2");
v2.close();
assert.throws(() => Ledger.open(previous), /版本是 2.*只认版本 3/s);
const foreign = join(dir, "foreign.db");
const other = new DatabaseSync(foreign);
other.exec("CREATE TABLE t (x INTEGER)");
@@ -311,9 +478,9 @@ test("查询:阶段关了没有、某份提交的最新一版(同一人对
assert.equal(ledger.stageClosed("s1"), false);
const slot = { stageId: "s1", author: "r1", kind: "critique" as const, target: "c1", item: "k1" };
assert.equal(ledger.latestSubmission(slot), undefined);
ledger.append("submission", { ...slot, version: 1, content: { target: "whole", body: "第一版" } });
const v2 = ledger.append("submission", { ...slot, version: 2, content: { target: "whole", body: "第二版" } });
ledger.append("submission", { ...slot, item: "k2", version: 1, content: { target: "p1", body: "另一份" } });
ledger.append("submission", { ...slot, version: 1, jev: null, content: { target: "whole", body: "第一版" } });
const v2 = ledger.append("submission", { ...slot, version: 2, jev: null, content: { target: "whole", body: "第二版" } });
ledger.append("submission", { ...slot, item: "k2", version: 1, jev: null, content: { target: "p1", body: "另一份" } });
assert.deepEqual(ledger.latestSubmission(slot), v2);
closeStage(ledger, "s1");
assert.equal(ledger.stageClosed("s1"), true);