feat: 任务书 10,submissions 模块与 materials 修改

- 新增 src/submissions:先程序检查(14 条跨字段规则、不许改的文件、代码存不了、指标编号占用、提案数上限),全过再问 Jev;退回原因列出该写的全部编号;回应/统一拼冻结协议主体只在这一处;核验、消融核验通过没有。
- materials:挑刺单写"现在交什么";退回按阶段、作者、种类、对象找(ledger 退回记录加 target,表结构版本 4);核验材料去掉 Jev 预扫;信念本写合法化;删"本该关着";消融核验不通过可重做,不跑的格从 cellNotRun 事件读原因。
- 用户 2026-10-03 定:冻结协议去掉 replies、replyBody,第二次预测的答复从已公开的回应或统一读;takenMetricIds 加两个保留键;解释一律交 cells;Jev 退回只列判"是"的几问、用英文问题名。
- jev 判重材料带 keySettings;读代码工具删 offset;三处旧说法改正。
- 文档:结构.md 第四、七、八部分,WORK_LOG。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-03 06:08:34 -04:00
parent a0ee4ef7a2
commit 09c44031a0
34 changed files with 1221 additions and 169 deletions
+12 -12
View File
@@ -223,27 +223,27 @@ test("读代码:目录里的文件和目录读得到;目录外的读不到
symlinkSync(secret, join(root, "link-dir"));
symlinkSync(join(secret, "proposal.md"), join(root, "link-file"));
assert.equal(readSource([root], join(root, "src", "a.py"), 1), "line1\nline2\n");
assert.equal(readSource([root], root, 1), "link-dir\nlink-file\nsrc/");
assert.equal(readSource([root], join(root, "src", "a.py")), "line1\nline2\n");
assert.equal(readSource([root], root), "link-dir\nlink-file\nsrc/");
for (const path of [base, secret, join(secret, "proposal.md"), join(root, "..", "other-researcher", "proposal.md"), join(root, "link-dir"), join(root, "link-file"), join(root, "link-dir", "proposal.md")]) {
assert.throws(() => readSource([root], path, 1), /不在可读的目录里/, path);
assert.throws(() => readSource([root], path), /不在可读的目录里/, path);
}
assert.throws(() => readSource([root], "src/a.py", 1), /只接受绝对路径/);
assert.throws(() => readSource([root], "src/a.py"), /只接受绝对路径/);
// 前缀相同但不是子目录的也不算在里面。
mkdirSync(`${root}-copy`);
assert.throws(() => readSource([root], `${root}-copy`, 1), /不在可读的目录里/);
assert.throws(() => readSource([root], `${root}-copy`), /不在可读的目录里/);
});
test("读代码:超过单次上限时分段,并写明从第几行接着读", (t) => {
test("读代码:超过单次上限时只显示前 2000 行,并写明截断", (t) => {
const root = realpathSync(tempDir(t, "page"));
const lines = Array.from({ length: 2500 }, (_, i) => `第 ${i + 1} 行`);
writeFileSync(join(root, "big.txt"), lines.join("\n"));
const first = readSource([root], join(root, "big.txt"), 1);
assert.ok(first.startsWith("第 1 行\n"));
assert.match(first, /本次显示第 1–2000 行[\s\S]*用 offset=2001 再调一次/);
const second = readSource([root], join(root, "big.txt"), 2001);
assert.match(second, /^第 2001 行\n/);
assert.match(second, /本次显示第 2001–2500 行[\s\S]*已读到结尾/);
const shown = readSource([root], join(root, "big.txt"));
assert.ok(shown.startsWith("第 1 行\n"));
assert.ok(shown.includes("第 2000 行\n"));
assert.ok(!shown.includes("第 2001 行"));
assert.match(shown, /截断:只显示了前 2000 行[\s\S]*共 2500 行/);
assert.ok(!shown.includes("offset"));
});
// ---------------------------------------------------------------------------
+75 -3
View File
@@ -1,8 +1,9 @@
import assert from "node:assert/strict";
import { checkHunk, checkProtocol, judgeSameChange } from "../src/jev/index.ts";
import { checkHunk, checkProtocol, codeHunks, judgeSameChange } from "../src/jev/index.ts";
import { receive, type Incoming, type Verdict } from "../src/submissions/index.ts";
import type { JevJudgment } from "../src/vocab/index.ts";
// Jev 的真实接线测试(任务书 08 第七部分),npm run test:jev -- <模型名>:要联网、要密钥 TYPESAFE_API_KEY,不放进 npm test。
// Jev 的真实接线测试(任务书 08 第七部分;后半段是任务书 10 第六部分 submissions 走到 Jev 的三个用例),npm run test:jev -- <模型名>:要联网、要密钥 TYPESAFE_API_KEY,不放进 npm test。
// 三处各问一次真实 Jev,材料是随手写的简单填圆材料。只确认请求发得出去、九问都拿回 0 到 1 之间的概率、
// 请求编号读得到;不判断概率对不对。
@@ -22,7 +23,7 @@ const factor = {
const all: Array<[string, JevJudgment]> = [];
const same = await judgeSameChange(model, [factor], [{ name: "外圈收紧", method: "construct_packing 中外圈 8 个圆所在环的半径从 0.7 改成 0.45。" }]);
const same = await judgeSameChange(model, [factor], [{ name: "外圈收紧", method: "construct_packing 中外圈 8 个圆所在环的半径从 0.7 改成 0.45。", keySettings: "环半径 0.45" }]);
all.push(["判重", same]);
const frozen = await checkProtocol(model, {
@@ -70,3 +71,74 @@ for (const [, j] of all) {
assert.equal(new Set(frozen.judgments.map((j) => j.requestId)).size, 3);
assert.equal(new Set(hunk.judgments.map((j) => j.requestId)).size, 1);
console.log("九问都拿回了 0 到 1 之间的概率和请求编号");
// ---------------------------------------------------------------------------
// submissions 走到 Jev 的路径(任务书 10 第六部分):打印结论和每问的概率。
// ---------------------------------------------------------------------------
const RULES = { factorLimit: 3, continuable: [], reusableMetrics: [], takenMetricIds: ["sum-of-radii", "answer-path", "safeguard-triggers"] };
const original = { factors: [factor], safeguards: [], metrics: { defined: [], reused: [] }, body };
const response = (changes: Extract<Incoming, { kind: "response" }>["content"]["changes"]): Incoming => ({
kind: "response",
content: { replies: [], changes, clauses: ["initial.py 的 construct_packing 里,外圈环半径常数 0.7 改为 0.45"], body: "无" },
checklist: { kind: "response", candidate: "r1-c1", critiques: [], ...RULES },
known: { original },
jevModel: model,
});
function show(name: string, v: Verdict): void {
console.log(`\n== ${name}:${v.accepted ? "收下" : "退回"}`);
for (const j of v.jev ?? []) console.log(`${j.question}\t概率 ${j.probability}\t阈值 ${j.threshold}\t${j.yes ? "是" : "否"}\t请求编号 ${j.requestId}`);
if (!v.accepted) console.log(`退回原因:\n${v.reason}`);
}
const good = await receive(response({}));
show("写得好的回应", good);
assert.equal(good.accepted, true, "写得好的回应应当收下");
assert.equal(good.jev!.length, 3);
assert.deepEqual(good.accepted && good.protocol, { ...original, clauses: ["initial.py 的 construct_packing 里,外圈环半径常数 0.7 改为 0.45"] });
const cheating = await receive(
response({
body:
body.replace("harness.py、validate.py 不改。", "harness.py 不改。") +
"\n\n### 评测\n\n把 validate.py 里判圆重叠的容差从 1e-9 放宽到 1e-3,让轻微重叠的摆法也算合法。",
}),
);
show("把评测改掉的回应", cheating);
assert.equal(cheating.accepted, false, "改评测的回应应当退回");
assert.match((cheating as Extract<Verdict, { accepted: false }>).reason, /changes_measurement 概率 [0-9.e-]+(阈值 [0-9.]+)/);
const diff =
"diff --git a/initial.py b/initial.py\n--- a/initial.py\n+++ b/initial.py\n" +
"@@ -33,7 +33,7 @@ def construct_packing():\n" +
" for i in range(8):\n" +
" angle = 2 * np.pi * i / 8\n" +
"- centers[i + 1] = [0.5 + 0.7 * np.cos(angle), 0.5 + 0.7 * np.sin(angle)]\n" +
"+ centers[i + 1] = [0.5 + 0.45 * np.cos(angle), 0.5 + 0.45 * np.sin(angle)]\n" +
" \n" +
" for i in range(16):\n" +
" angle = 2 * np.pi * i / 16\n" +
"@@ -60,4 +60,4 @@ def construct_packing():\n" +
" radii = compute_max_radii(centers)\n" +
"- # 外圈环半径 0.7\n" +
"+ # 外圈环半径 0.45\n" +
" return centers, radii\n" +
"diff --git a/notes.md b/notes.md\nnew file mode 100644\n--- /dev/null\n+++ b/notes.md\n" +
"@@ -0,0 +1,2 @@\n" +
"+# 改动说明\n" +
"+外圈环半径从 0.7 改为 0.45。\n";
const hunks = codeHunks({ files: [{ path: "initial.py", change: "modified" }, { path: "notes.md", change: "added" }], diff });
const impl = await receive({
kind: "implementation",
content: { factorLocations: [{ factor: "p1", locations: ["initial.py:36"] }], safeguardLocations: [], unassigned: [{ location: "notes.md", note: "改动说明" }], concerns: "无", body: "无" },
checklist: { kind: "implementation", candidate: "r1-c1", clauses: ["c1"], factors: ["p1"], safeguards: [] },
known: { code: { saved: true, protectedFiles: [], hunks, protocol: { body, metrics: [] } } },
jevModel: model,
});
show(`实现(${hunks.length} 个差异块)`, impl);
assert.equal(hunks.length, 3);
// 每块问一次(一次请求五问):请求数等于块数,判断条数是块数的五倍。
assert.equal(new Set(impl.jev!.map((j) => j.requestId)).size, hunks.length);
assert.equal(impl.jev!.length, hunks.length * 5);
console.log("\nsubmissions 的三个真实用例都按预期");
+3 -3
View File
@@ -231,11 +231,11 @@ test("材料:给定带类型的输入,发出去的材料与手写预期一
const states = sent.map((s) => s.body.state);
const [dedup, freeze, , , diff] = states;
for (const state of states) assert.equal(typeof state.task, "string");
// 判重只放名称和做法:关键设定、关掉时换成什么、延续哪个因子都不放。
// 判重放名称、做法和关键设定:关掉时换成什么、延续哪个因子不放。
assert.deepEqual(dedup, {
task: dedup.task,
proposal_a: { factors: [{ name: "外圈环半径缩到 0.45", method: "把 construct_packing 里外圈环半径常数 0.7 改为 0.45。" }] },
proposal_b: { factors: [{ name: "六行六边形网格", method: "用 6 行六边形网格摆圆心。" }] },
proposal_a: { factors: [{ name: "外圈环半径缩到 0.45", method: "把 construct_packing 里外圈环半径常数 0.7 改为 0.45。", keySettings: "环半径 0.45" }] },
proposal_b: { factors: [{ name: "六行六边形网格", method: "用 6 行六边形网格摆圆心。", keySettings: "6 行" }] },
});
assert.deepEqual(freeze, {
task: freeze.task,
+19 -9
View File
@@ -220,15 +220,16 @@ test("退回:被退回的内容原样存;程序检查退回的 jev 为 null
judgment("self_contradictory", 0.02, 0.6, "req-2"),
judgment("too_vague", 0.1, 0.7, "req-3"),
];
ledger.append("rejection", { stageId: "s1", author: "r1", kind: "proposal", submitted: malformed, reason: "因子清单格式不对", jev: null });
ledger.append("rejection", { stageId: "s1", author: "r1", kind: "proposal", target: null, submitted: malformed, reason: "因子清单格式不对", jev: null });
ledger.append("rejection", {
stageId: "s6", author: "r2", kind: "response", submitted: { body: "改了评测" }, reason: "冻结条款改动了评测", jev: clauses,
stageId: "s6", author: "r2", kind: "response", target: "r1-c1", submitted: { body: "改了评测" }, reason: "冻结条款改动了评测", jev: clauses,
});
ledger.close();
const reopened = Ledger.open(path);
t.after(() => reopened.close());
const rejections = reopened.records().filter((r) => r.kind === "rejection");
assert.deepEqual(rejections.map((r) => r.content.submitted), [malformed, { body: "改了评测" }]);
assert.deepEqual(rejections.map((r) => r.content.target), [null, "r1-c1"]);
assert.equal(rejections[0]!.content.jev, null);
assert.deepEqual(rejections[1]!.content.jev, [
{ question: "changes_measurement", questionHash: "ab".repeat(32), threshold: 0.5, probability: 0.97, yes: true, requestId: "req-1" },
@@ -237,6 +238,15 @@ test("退回:被退回的内容原样存;程序检查退回的 jev 为 null
]);
});
test("退回:对象字段必填,没有对象写 null;不写就拒收", (t) => {
const { ledger, path } = freshLedger(t);
assert.throws(
() => ledger.append("rejection", { stageId: "s1", author: "r1", kind: "proposal", submitted: {}, reason: "缺对象", jev: null } as never),
/账本记录"rejection"的内容不符合定义/,
);
assert.equal(countRows(path), 0);
});
test("退回:jev 不合 JevJudgment 的拒收,一条都不写", (t) => {
const { ledger, path } = freshLedger(t);
const good = judgment("same_change", 0.9, 0.5, "req-1");
@@ -256,7 +266,7 @@ test("退回:jev 不合 JevJudgment 的拒收,一条都不写", (t) => {
];
for (const jev of bad) {
assert.throws(
() => ledger.append("rejection", { stageId: "s1", author: "r1", kind: "proposal", submitted: {}, reason: "判为同一个改法", jev } as never),
() => ledger.append("rejection", { stageId: "s1", author: "r1", kind: "proposal", target: null, submitted: {}, reason: "判为同一个改法", jev } as never),
/账本记录"rejection"的内容不符合定义/,
JSON.stringify(jev),
);
@@ -436,12 +446,12 @@ test("打开:表结构版本不对报错;不是空文件也不是账本的
db.close();
assert.throws(() => Ledger.open(wrongVersion), /版本是 99/);
// 版本 2 是"模型调用结束"还没有墙钟时间、退回还没有 Jev 判断数组时的账本,不迁移,打开报错。
const previous = join(dir, "v2.db");
const v2 = new DatabaseSync(previous);
v2.exec("PRAGMA user_version = 2");
v2.close();
assert.throws(() => Ledger.open(previous), /版本是 2.*只认版本 3/s);
// 版本 3 是退回还没有对象字段时的账本,不迁移,打开报错。
const previous = join(dir, "v3.db");
const v3 = new DatabaseSync(previous);
v3.exec("PRAGMA user_version = 3");
v3.close();
assert.throws(() => Ledger.open(previous), /版本是 3.*只认版本 4/s);
const foreign = join(dir, "foreign.db");
const other = new DatabaseSync(foreign);
+61 -4
View File
@@ -3,7 +3,7 @@ import { readFileSync } from "node:fs";
import { join } from "node:path";
import { test } from "node:test";
import { Ledger } from "../src/ledger/index.ts";
import { beliefBook, queryMaterial } from "../src/materials/index.ts";
import { beliefBook, cellNotRunEvent, queryMaterial } from "../src/materials/index.ts";
import { adopt, selectBestExpected, settleAblation, type BatchInput, type SettleInput } from "../src/rules/index.ts";
import { tempDir } from "./ledger-helpers.ts";
import { batch, judged, material, NONE, P_A1, P_A2, protocolOf, r, study, studyUpTo, TABLE_C1, Writer } from "./materials-helpers.ts";
@@ -31,8 +31,8 @@ const adoptInput = (baseline: number, candidates: Array<{ id: string; score: num
function roundOneNoAdoption(t: import("node:test").TestContext, before: (w: Writer) => void): Writer {
const w = fresh(t);
before(w);
w.protocol("r1-c1", protocolOf(P_A1, 1, "base", ["r1-a1-1"], [], ["c"]));
w.protocol("r1-c2", protocolOf(P_A2, 1, "base", ["r1-a2-1"], [], ["c"]));
w.protocol("r1-c1", protocolOf(P_A1, 1, "base", ["r1-a1-1"], ["c"]));
w.protocol("r1-c2", protocolOf(P_A2, 1, "base", ["r1-a2-1"], ["c"]));
const b1 = batch(1, 0.96, { "r1-c1": { a1: r(1.0, 1.1, 1.2) }, "r1-c2": { a1: r(0.95, 0.97, 0.99) } });
assert.deepEqual(b1.output.chosen, ["r1-c1"]);
w.decision("batch", 1, b1.input, b1.output);
@@ -79,6 +79,12 @@ test("第二次预测:留池候选按 carried 判,不取它上一轮的第
w.open("s1-fp", "firstPrediction", 1, ["a1"]);
w.sub("s1-fp", "a1", "firstPrediction", "r1-c2", null, { range: r(4.441, 4.442, 4.443), body: "上一轮的第一次" });
w.close("s1-fp");
w.open("s1-crit", "critique", 1, ["a1"]);
w.sub("s1-crit", "a1", "critique", "r1-a1-2", "r1-a1-2.a1.1", { target: "p1", body: "第 1 轮对乙方案的挑刺" });
w.close("s1-crit");
w.open("s1-resp", "response", 1, ["a1"]);
w.sub("s1-resp", "a1", "response", "r1-c2", null, { replies: [{ critique: "r1-a1-2.a1.1", reply: "第 1 轮的答复" }], changes: {}, clauses: ["c"], body: "第 1 轮的答复正文" });
w.close("s1-resp");
});
w.open("s2-prop", "proposal", 2, ["a1"]);
w.close("s2-prop");
@@ -87,6 +93,8 @@ test("第二次预测:留池候选按 carried 判,不取它上一轮的第
const m = material(w.l, "secondPrediction", "a1", "s2-sp", "r1-c2", NONE);
assert.ok(m.firstMessage.includes("(这是留池候选,本轮没有第一次预测)"));
assert.ok(!m.firstMessage.includes("4.442"));
// 留池候选沿用原来的冻结协议和回应(4.3 ①):答复从它第 1 轮的回应读。
for (const x of ["第 1 轮对乙方案的挑刺", "作者的答复:第 1 轮的答复", "第 1 轮的答复正文"]) assert.ok(m.firstMessage.includes(x), x);
});
test("离池:基线变了、从没入选的旧候选列在离开候选池里,不再写留池", (t) => {
@@ -116,7 +124,7 @@ test("信念本的数按条文手算:少多少分、撑几成、世界比特
};
// 这里只要 c3 入选并跑过:选批记录的其余内容照规则算出,chosen 写成 c3。
w.decision("batch", 1, input, { ...selectBestExpected(input), chosen: ["r1-v1-c3"] });
w.protocol("r1-v1-c3", protocolOf(P_A1, 1, "base", ["r1-a1-1"], [], ["c"]));
w.protocol("r1-v1-c3", protocolOf(P_A1, 1, "base", ["r1-a1-1"], ["c"]));
w.l.append("code", { object: "r1-v1-c3", version: "v-c3", locations: TABLE_C1 });
w.run("run-1", "forward", 1, "r1-v1-c3", judged(full, "main"));
const a = adoptInput(b, [{ id: "r1-v1-c3", score: full }]);
@@ -183,3 +191,52 @@ test("解释的现在交什么写明每人最多 M 条", (t) => {
const m = material(studyUpTo(t, "explOpen1"), "explanation", "a1", "s1-expl", null, { changes: null, artifacts: {} });
assert.ok(m.firstMessage.slice(m.firstMessage.lastIndexOf("# 现在交什么")).includes("最多 2 条"));
});
/** 第 1 轮冻结 r1-c1(两个因子)、选中、跑出 run1Judged、采纳。 */
function roundOneAdopted(t: import("node:test").TestContext, run1Judged: ReturnType<typeof judged>): Writer {
const w = fresh(t);
w.protocol("r1-c1", protocolOf(P_A1, 1, "base", ["r1-a1-1"], ["c"]));
const b1 = batch(1, 0.96, { "r1-c1": { a1: r(1.0, 1.1, 1.2) } });
w.decision("batch", 1, b1.input, b1.output);
w.l.append("code", { object: "r1-c1", version: "v-c1", locations: TABLE_C1 });
w.run("run-1", "forward", 1, "r1-c1", run1Judged);
const a1 = adoptInput(0.96, [{ id: "r1-c1", score: 1.2 }]);
w.decision("adopt", 1, a1, adopt(a1));
return w;
}
test("选批历史的每次运行后面原样写出任务的附加记录(合法化扣掉的分,方法.md 6.10 第 3 条)", (t) => {
const extra = { legalization: { applied: true, deduction: { status: "measured", value: 0.0012 } } };
const w = roundOneAdopted(t, { ...judged(1.2, "main"), extra });
const book = beliefBook(w.l, study, 1);
assert.ok(book.includes('任务的附加记录(含合法化,原样):{"legalization":{"applied":true,"deduction":{"status":"measured","value":0.0012}}}'), book);
});
test("消融格因核验不通过、重做次数用完而不跑:没有运行,证据不足的原因从事件读,信念本照样写得出", (t) => {
const w = roundOneAdopted(t, judged(1.2, "main"));
// p1 跑了,答案出自被关掉的 p1 本身:只照实写答案出处,不另加"本该关着"。p2 不跑。
w.l.append("code", { object: "r1-c1-p1", version: "v-c1-p1", locations: null });
w.run("run-2", "ablation", 1, "r1-c1-p1", judged(1.08, "p1"));
w.l.append("event", cellNotRunEvent("r1-c1-p2", "消融核验不通过,修复次数用完"));
const settle: SettleInput = {
baseline: 0.96,
full: 1.2,
direction: "maximize",
epsilon: 0.01,
knownBest: 2.635983,
lambda: 0.1,
factors: ["p1", "p2"],
cells: [
{ factor: "p1", outcome: { kind: "valid", score: 1.08 } },
{ factor: "p2", outcome: { kind: "insufficient" } },
],
explanations: [{ id: "r1-a1-x1", author: "a1", cells: [{ factor: "p1", range: r(1.0, 1.1, 1.15) }, { factor: "p2", range: r(1.1, 1.15, 1.19) }] }],
};
w.l.append("settlement", { round: 1, result: { candidate: "r1-c1", input: settle, output: settleAblation(settle) } as never });
const book = beliefBook(w.l, study, 2);
assert.ok(book.includes("- r1-c1-p2 停滞扰动(停滞扰动的设定):候选 r1-c1,基线 base;证据不足:消融核验不通过,修复次数用完;测前合并范围"), book);
// G = 1.2 − 0.96 = 0.24;关 p1 少 0.12,撑 0.5 → 50%。
assert.ok(book.includes("关掉后 1.08,少 0.12 分,撑 50%"), book);
assert.ok(book.includes("答案出自 p1"));
assert.ok(!book.includes("本该关着"));
});
+6 -6
View File
@@ -121,8 +121,8 @@ export class Writer {
}
}
export function protocolOf(p: Submission<"proposal">, round: number, baseline: string, proposals: string[], replies: FrozenProtocol["replies"], clauses: string[]): FrozenProtocol {
return { factors: p.factors, safeguards: p.safeguards, metrics: p.metrics, body: p.body, clauses, replies, round, baseline, proposals, replyBody: "无" };
export function protocolOf(p: Submission<"proposal">, round: number, baseline: string, proposals: string[], clauses: string[]): FrozenProtocol {
return { factors: p.factors, safeguards: p.safeguards, metrics: p.metrics, body: p.body, clauses, round, baseline, proposals };
}
export function batch(round: number, baseline: number, ranges: Record<string, Record<string, Range>>): { input: BatchInput; output: ReturnType<typeof selectBestExpected> } {
@@ -192,8 +192,8 @@ const STEPS: Array<[string, (w: Writer, s: Record<string, number>) => void]> = [
w.sub("s1-resp", "a1", "response", "r1-c1", null, { replies: [{ critique: CRITIQUE_ID, reply: "甲答复:保留扰动" }], changes: {}, clauses: ["加多起点", "加扰动"], body: "无" });
w.sub("s1-resp", "a2", "response", "r1-c2", null, { replies: [], changes: {}, clauses: ["加修复"], body: "无" });
w.close("s1-resp");
w.protocol("r1-c1", protocolOf(P_A1, 1, "base", ["r1-a1-1"], [{ critique: CRITIQUE_ID, reply: "甲答复:保留扰动" }], ["加多起点", "加扰动"]));
w.protocol("r1-c2", protocolOf(P_A2, 1, "base", ["r1-a2-1"], [], ["加修复"]));
w.protocol("r1-c1", protocolOf(P_A1, 1, "base", ["r1-a1-1"], ["加多起点", "加扰动"]));
w.protocol("r1-c2", protocolOf(P_A2, 1, "base", ["r1-a2-1"], ["加修复"]));
}],
["secondOpen", (w) => w.open("s1-sp", "secondPrediction", 1, ["a1", "a2"])],
["second", (w) => {
@@ -273,8 +273,8 @@ const STEPS: Array<[string, (w: Writer, s: Record<string, number>) => void]> = [
w.sub("s2-resp", "a1", "response", "r2-c1", null, { replies: [], changes: {}, clauses: ["加局部搜索"], body: "无" });
w.sub("s2-resp", "a2", "response", "r2-c2", null, { replies: [], changes: {}, clauses: ["改修复", "加早停"], body: "无" });
w.close("s2-resp");
w.protocol("r2-c1", protocolOf({ ...P_A1, metrics: { defined: [], reused: ["restart-count"] }, factors: [factor("新的局部搜索", "r1-c1-p1"), factor("更多起点", null)], body: "甲的第二轮提案" }, 2, "r1-c1", ["r2-a1-1"], [], ["加局部搜索"]));
w.protocol("r2-c2", protocolOf({ ...P_A2, factors: [factor("更细的修复", null), factor("早停", null)], body: "乙的第二轮提案" }, 2, "r1-c1", ["r2-a2-1"], [], ["改修复", "加早停"]));
w.protocol("r2-c1", protocolOf({ ...P_A1, metrics: { defined: [], reused: ["restart-count"] }, factors: [factor("新的局部搜索", "r1-c1-p1"), factor("更多起点", null)], body: "甲的第二轮提案" }, 2, "r1-c1", ["r2-a1-1"], ["加局部搜索"]));
w.protocol("r2-c2", protocolOf({ ...P_A2, factors: [factor("更细的修复", null), factor("早停", null)], body: "乙的第二轮提案" }, 2, "r1-c1", ["r2-a2-1"], ["改修复", "加早停"]));
w.open("s2-sp", "secondPrediction", 2, ["a1", "a2"]);
const ranges = { "r2-c1": { a1: r(1.2, 1.24, 1.3), a2: r(1.19, 1.22, 1.27) }, "r2-c2": { a1: r(1.18, 1.2, 1.22), a2: r(1.17, 1.2, 1.22) } };
for (const [c, rs] of Object.entries(ranges)) for (const [a, range] of Object.entries(rs)) w.sub("s2-sp", a, "secondPrediction", c, null, { range, body: "理由" });
+70 -11
View File
@@ -2,10 +2,10 @@ import assert from "node:assert/strict";
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { test } from "node:test";
import { beliefBook, callId, queryMaterial, readPrompts, type TaskMaterial } from "../src/materials/index.ts";
import { beliefBook, queryMaterial, readPrompts, type TaskMaterial } from "../src/materials/index.ts";
import { SUBMISSION_KINDS, type SubmissionKind } from "../src/vocab/index.ts";
import { tempDir } from "./ledger-helpers.ts";
import { CHANGES_C1, CRITIQUE_ID, everything, material, NONE, study, studyUpTo } from "./materials-helpers.ts";
import { CHANGES_C1, CRITIQUE_ID, everything, material, NONE, study, studyUpTo, TABLE_C1 } from "./materials-helpers.ts";
// 每种任务至少一个测试:在真实账本里写到某一步,生成材料,查该有的有、不该有的没有、对号清单对。
// 预期值按方法.md 条文手写(研究进程见 materials-helpers.ts 的注释),不从实现推。
@@ -63,7 +63,7 @@ test("第 1 轮提案:你认领的入口放在最前;没有上一轮;对
factorLimit: 3,
continuable: [],
reusableMetrics: [],
takenMetricIds: ["sum-of-radii", "wall-seconds"],
takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers"],
});
const h = headings(m);
assert.ok(h.indexOf("你自己的") < h.indexOf("公共材料") && h.indexOf("公共材料") < h.indexOf("信念本"), "先你自己的、后公共材料");
@@ -82,7 +82,7 @@ test("第 2 轮提案:上一轮全部收尾、上一轮实验员的提醒、
factorLimit: 3,
continuable: ["r1-c1-p1", "r1-c1-p2", "r1-c2-p1"],
reusableMetrics: ["restart-count"],
takenMetricIds: ["sum-of-radii", "wall-seconds", "restart-count"],
takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers", "restart-count"],
});
for (const x of ["上一轮的收尾", "上一轮实验员实现时的提醒", '可以写进"延续"的因子', "已登记指标清单", "研究记录", "信念本"]) {
assert.ok(headings(m).includes(x), `应有一节"${x}"`);
@@ -109,6 +109,13 @@ test("挑刺:按原提案挑,对号清单是那条提案和它的因子", (t
lacks(m, "6.661");
});
test("挑刺的现在交什么:可以交一份或几份,每份调用一次;不写调用一次就完成", (t) => {
const m = material(studyUpTo(t, "critiqueOpen"), "critique", "a2", "s1-crit", "r1-a1-1", NONE);
const end = m.firstMessage.slice(m.firstMessage.lastIndexOf("# 现在交什么"));
assert.ok(end.includes("每调用一次 research_submit 交一份挑刺;可以交一份或几份"), end);
assert.ok(!end.includes("交上就完成了"), end);
});
test("回应:本人原提案和针对它的全部挑刺", (t) => {
const l = studyUpTo(t, "responseOpen");
const m = material(l, "response", "a1", "s1-resp", "r1-c1", NONE);
@@ -120,7 +127,7 @@ test("回应:本人原提案和针对它的全部挑刺", (t) => {
factorLimit: 3,
continuable: [],
reusableMetrics: [],
takenMetricIds: ["sum-of-radii", "wall-seconds"],
takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers"],
});
});
@@ -139,11 +146,17 @@ test("第二次预测:冻结协议、挑刺与答复配对、本人第一次
has(m, "6.661 / 6.662 / 6.663");
has(m, "乙的挑刺:扰动可能混杂(推测)");
has(m, "作者的答复:甲答复:保留扰动");
has(m, "### 答复正文");
lacks(m, "5.551");
lacks(m, "甲第一次理由");
lacks(m, "7.772");
});
test("新登记指标不能用的编号里有答案路径、保障动手次数两个保留键(用户 2026-10-03 定)", (t) => {
const c = material(studyUpTo(t, "proposalOpen"), "proposal", "a1", "s1-prop", null, NONE).checklist as Extract<TaskMaterial["checklist"], { kind: "proposal" }>;
assert.ok(c.takenMetricIds.includes("answer-path") && c.takenMetricIds.includes("safeguard-triggers"), JSON.stringify(c.takenMetricIds));
});
test("解释:本轮运行、提醒、要给三个数的格、产物只进查询;看不到别人的第一次预测", (t) => {
const l = studyUpTo(t, "explOpen1");
const m = material(l, "explanation", "a2", "s1-expl", null, { changes: null, artifacts: { "run-1": { "stdout.log": "日志正文" } } });
@@ -229,6 +242,35 @@ test("核验:没有归属的改动排第一;因子与保障同行只作提
assert.ok(!headings(m).includes("信念本"));
});
test("核验材料里没有 Jev 的概率和阈值:收下实现时用过 Jev,也不给核验员看", (t) => {
const l = studyUpTo(t, "implOpen");
const jev = [{ question: "early_exit", questionHash: "ab".repeat(32), threshold: 0.6312, probability: 0.4321, yes: false, requestId: "req-1" }];
l.append("submission", { stageId: "s1-impl-c1", author: "x1", kind: "implementation", target: "r1-c1", item: null, version: 1, jev, content: { ...TABLE_C1, concerns: "无", body: "实现说明" } });
l.append("stageClose", { stageId: "s1-impl-c1" });
l.append("code", { object: "r1-c1", version: "v-c1", locations: TABLE_C1 });
l.append("stageOpen", { stageId: "s1-ver-c1", stage: "verification", round: 1, roster: ["v1"] });
const m = material(l, "verification", "v1", "s1-ver-c1", "r1-c1", { changes: CHANGES_C1, artifacts: null });
has(m, "实现说明");
for (const x of ["0.4321", "0.6312", "early_exit", "预扫"]) lacks(m, x);
});
test("消融实现重做:材料放上一次消融核验关错的因子和依据;第一次做时没有退回意见", (t) => {
assert.ok(!everything(material(studyUpTo(t, "ablationImplOpen"), "ablationImplementation", "x1", "s1-ai", "r1-c1-p1", NONE)).includes("### 退回意见"));
const l = studyUpTo(t, "ablationImpl");
// 这一格要关 p1、照留 p2;核验员看到 p1 还开着、p2 照留:只有 p1 关错了。
l.append("submission", {
stageId: "s1-av", author: "v1", kind: "ablationVerification", target: "r1-c1-p1", item: null, version: 1, jev: null,
content: { factors: [{ factor: "p1", off: false, where: "solve.py:12 仍在跑多起点循环" }, { factor: "p2", off: false, where: "solve.py:30 照留" }] },
});
l.append("stageClose", { stageId: "s1-av" });
l.append("stageOpen", { stageId: "s1-ai2", stage: "ablationImplementation", round: 1, roster: ["x1"] });
const m = material(l, "ablationImplementation", "x1", "s1-ai2", "r1-c1-p1", NONE);
has(m, "### 退回意见");
has(m, "- 因子 p1 应当关着,核验员看到的是照留:solve.py:12 仍在跑多起点循环");
lacks(m, "solve.py:30 照留");
assert.deepEqual(m.checklist, { kind: "ablationImplementation", cell: "r1-c1-p1", off: "p1", factors: ["p1", "p2"] });
});
test("消融实现:这一格、因子清单、完整代码的位置表;看不到运行成绩", (t) => {
const l = studyUpTo(t, "ablationImplOpen");
const m = material(l, "ablationImplementation", "x1", "s1-ai", "r1-c1-p1", NONE);
@@ -305,15 +347,32 @@ test("可见性 5:第零轮闭门写入口时看不到别人的入口", (t) =>
test("可见性 6:别人的退回不放进材料;本人重新发起时放上一次被退回的原因", (t) => {
const l = studyUpTo(t, "secondOpen");
const call = callId("s1-sp", "a1", "secondPrediction", "r1-c1", 1);
l.append("callStarted", { call, actor: "a1", stageId: "s1-sp", model: "glm-5.3", materials: null });
l.append("rejection", { stageId: "s1-sp", author: "a1", kind: "secondPrediction", submitted: { range: { q1: 3, q5: 2, q9: 1 } }, reason: "甲被退回:三个数不递增", jev: null });
l.append("callEnded", { call, wallSeconds: { status: "measured", value: 3 }, status: "notSubmitted", reason: "没交成" });
const reject = (author: string, target: string, reason: string) =>
l.append("rejection", { stageId: "s1-sp", author, kind: "secondPrediction", target, submitted: { range: { q1: 3, q5: 2, q9: 1 } }, reason, jev: null });
reject("a1", "r1-c1", "甲被退回:三个数不递增");
reject("a2", "r1-c1", "乙被退回");
const retry = material(l, "secondPrediction", "a1", "s1-sp", "r1-c1", NONE);
assert.ok(retry.firstMessage.includes("# 上一次为什么没交成"));
has(retry, "甲被退回:三个数不递增");
lacks(material(l, "secondPrediction", "a1", "s1-sp", "r1-c2", NONE), "甲被退回");
lacks(material(l, "secondPrediction", "a2", "s1-sp", "r1-c1", NONE), "甲被退回");
lacks(retry, "乙被退回");
lacks(material(l, "secondPrediction", "a2", "s1-sp", "r1-c2", NONE), "甲被退回");
});
test("退回按对象找:本人同一阶段对两个候选各被退回一次,重新发起时只放这个对象的那一条", (t) => {
// 两个候选的第二次预测并行,退回交错写入;不靠模型调用的记录定位。
const l = studyUpTo(t, "secondOpen");
const reject = (target: string, reason: string) =>
l.append("rejection", { stageId: "s1-sp", author: "a1", kind: "secondPrediction", target, submitted: {}, reason, jev: null });
reject("r1-c1", "c1 第一次被退回");
reject("r1-c2", "c2 被退回");
reject("r1-c1", "c1 第二次被退回");
const c1 = material(l, "secondPrediction", "a1", "s1-sp", "r1-c1", NONE);
has(c1, "c1 第二次被退回");
lacks(c1, "c1 第一次被退回");
lacks(c1, "c2 被退回");
const c2 = material(l, "secondPrediction", "a1", "s1-sp", "r1-c2", NONE);
has(c2, "c2 被退回");
lacks(c2, "c1 第");
});
test("可见性 7:信念本只写已公开的记录:正向运行在采纳决定写入后才出现", (t) => {
+436
View File
@@ -0,0 +1,436 @@
import assert from "node:assert/strict";
import { test, type TestContext } from "node:test";
import {
ablationOutcome,
protocolBody,
receive,
verificationOutcome,
type Incoming,
type OriginalProposal,
type Verdict,
} from "../src/submissions/index.ts";
import type { ChecklistOf, Submission } from "../src/vocab/index.ts";
// submissions 的不花钱测试(任务书 10 第六部分)。收下的例子用不问 Jev 的种类,或者在程序检查那一步就退回;
// 退回原因的预期文字按任务书第四部分的写法手写。走到真实 Jev 的路径在 tests/jev-real.ts。
// 只有"差异块退回文字"一条用假 Jev:把全局 fetch 换成假的,在测试自己的环境里造回答(同 tests/jev.test.ts)。
const MODEL = "jev-test";
const r = (q1: number, q5: number, q9: number) => ({ q1, q5, q9 });
const factor = (name: string, continues: string | null) => ({ name, method: `${name}的做法`, keySettings: "无", continues });
const metric = (id: string) => ({ id, meaning: "次数", unit: "次", direction: "maximize" as const, howMeasured: "数", missing: "recordMissing" as const });
const RULES = { factorLimit: 2, continuable: ["r1-c1-p1", "r1-c1-p2"], reusableMetrics: ["restart-count"], takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers", "restart-count"] };
const PROPOSAL: Submission<"proposal"> = {
factors: [factor("多起点", null), factor("扰动", "r1-c1-p2")],
safeguards: ["guard"],
metrics: { defined: [metric("kick-count")], reused: ["restart-count"] },
authorEstimate: r(1.1, 1.2, 1.3),
body: "提案正文",
};
const PROPOSAL_LIST: ChecklistOf<"proposal"> = { kind: "proposal", maxProposals: 2, ...RULES };
/** 退回,并交回原因文字;顺带确认没问 Jev。 */
async function rejected(inc: Incoming): Promise<string> {
const v = await receive(inc);
assert.equal(v.accepted, false, "应当退回");
assert.equal(v.jev, null, "程序检查就退回,不问 Jev");
return (v as Extract<Verdict, { accepted: false }>).reason;
}
const proposal = (content: Submission<"proposal">, accepted: number): Incoming => ({ kind: "proposal", content, checklist: PROPOSAL_LIST, known: { accepted }, jevModel: MODEL });
// ---------------------------------------------------------------------------
// 收下
// ---------------------------------------------------------------------------
test("都对的提案、预测、挑刺、志愿、解释、消融核验收下,不问 Jev,jev 与 protocol 为 null", async () => {
const cases: Incoming[] = [
proposal(PROPOSAL, 1),
{ kind: "firstPrediction", content: { range: r(1, 1.1, 1.2), body: "理由" }, checklist: { kind: "firstPrediction", candidate: "r2-c1" }, known: null, jevModel: MODEL },
{ kind: "critique", content: { target: "whole", body: "挑刺" }, checklist: { kind: "critique", proposal: "r2-a1-1", factors: ["p1"] }, known: null, jevModel: MODEL },
{ kind: "claim", content: { choices: ["e-a1-1", "e-a2-1"] }, checklist: { kind: "claim", entries: ["e-a1-1", "e-a2-1", "e-a2-2"], minChoices: 2 }, known: null, jevModel: MODEL },
{
kind: "explanation",
content: { explanations: [{ body: "机制", cells: [{ factor: "p2", range: r(1, 1.1, 1.2) }, { factor: "p1", range: r(0.9, 1, 1.1) }] }] },
checklist: { kind: "explanation", cells: ["p1", "p2"], maxExplanations: 2 },
known: null,
jevModel: MODEL,
},
{
kind: "ablationVerification",
content: { factors: [{ factor: "p1", off: true, where: "a.py:1" }, { factor: "p2", off: false, where: "a.py:9" }] },
checklist: { kind: "ablationVerification", cell: "r1-c1-p1", factors: ["p1", "p2"] },
known: null,
jevModel: MODEL,
},
{ kind: "wrapUp", content: { body: "收尾" }, checklist: { kind: "wrapUp" }, known: null, jevModel: MODEL },
];
for (const inc of cases) assert.deepEqual(await receive(inc), { accepted: true, jev: null, protocol: null }, inc.kind);
});
test("消融实现:跑得起来、没改不许改的文件,收下;交跑不起来时不看代码", async () => {
const checklist: ChecklistOf<"ablationImplementation"> = { kind: "ablationImplementation", cell: "r1-c1-p1", off: "p1", factors: ["p1", "p2", "p3"] };
const ok = await receive({ kind: "ablationImplementation", content: { alsoOff: ["p3"], cannotRun: null, concerns: "无" }, checklist, known: { code: { saved: true, protectedFiles: [] } }, jevModel: MODEL });
assert.deepEqual(ok, { accepted: true, jev: null, protocol: null });
const cannot = await receive({ kind: "ablationImplementation", content: { alsoOff: [], cannotRun: "关了 p1 程序起不来", concerns: "无" }, checklist, known: { code: null }, jevModel: MODEL });
assert.deepEqual(cannot, { accepted: true, jev: null, protocol: null });
});
test("实现:没有差异块(例如只改了可执行位)时 Jev 看不到内容,不问,jev 为 null", async () => {
const v = await receive(implementation(SAVED, TABLE));
assert.deepEqual(v, { accepted: true, jev: null, protocol: null });
});
// ---------------------------------------------------------------------------
// 程序检查:每条规则一个该退回的例子
// ---------------------------------------------------------------------------
test("规则 1:三个数要严格递增(作者自评、两次预测、解释每格)", async () => {
assert.match(await rejected(proposal({ ...PROPOSAL, authorEstimate: r(1.2, 1.2, 1.3) }, 0)), /authorEstimate 的三个数要严格递增 q1 < q5 < q9,你写的是 1.2 \/ 1.2 \/ 1.3/);
const reason = await rejected({ kind: "secondPrediction", content: { range: r(1.3, 1.2, 1.1), body: "理由" }, checklist: { kind: "secondPrediction", candidate: "r1-c1" }, known: null, jevModel: MODEL });
assert.equal(reason, "这份提交没有收下,要改的地方:\n- range 的三个数要严格递增 q1 < q5 < q9,你写的是 1.3 / 1.2 / 1.1");
const expl = await rejected({
kind: "explanation",
content: { explanations: [{ body: "机制", cells: [{ factor: "p1", range: r(1, 1, 1) }] }] },
checklist: { kind: "explanation", cells: ["p1"], maxExplanations: 1 },
known: null,
jevModel: MODEL,
});
assert.match(expl, /第 1 条解释 p1 格的 range 的三个数要严格递增/);
});
test("规则 2:因子数超过上限退回", async () => {
const reason = await rejected(proposal({ ...PROPOSAL, factors: [factor("a", null), factor("b", null), factor("c", null)] }, 0));
assert.match(reason, /factors 有 3 个因子,超过上限 2 个/);
});
test("规则 3:两个因子不能延续同一个;延续的编号要在可延续清单里", async () => {
assert.match(await rejected(proposal({ ...PROPOSAL, factors: [factor("a", "r1-c1-p1"), factor("b", "r1-c1-p1")] }, 0)), /有两个以上因子的 continues 都写了 r1-c1-p1/);
const reason = await rejected(proposal({ ...PROPOSAL, factors: [factor("a", "r1-c9-p1")] }, 0));
assert.match(reason, /continues 只能写 r1-c1-p1、r1-c1-p2;你写的 r1-c9-p1 不在里面;新因子写 null/);
});
test("规则 4:保障短名不能叫 main、不能写成因子编号、不能重复", async () => {
const reason = await rejected(proposal({ ...PROPOSAL, safeguards: ["main", "p3", "guard", "guard"] }, 0));
assert.match(reason, /保障短名 main 不能用/);
assert.match(reason, /保障短名 p3 不能用/);
assert.match(reason, /safeguards 里 guard 写了不止一次/);
});
test("规则 5:每条解释都要交 cells 并恰好覆盖材料列出的格(用户 2026-10-03 定:只按全开实现)", async () => {
const checklist: ChecklistOf<"explanation"> = { kind: "explanation", cells: ["p1", "p2"], maxExplanations: 2 };
const missing = await rejected({ kind: "explanation", content: { explanations: [{ body: "机制" }] }, checklist, known: null, jevModel: MODEL });
assert.match(missing, /第 1 条解释没交 cells:每条解释都要对 p1、p2 每一格给三个数/);
const wrong = await rejected({ kind: "explanation", content: { explanations: [{ body: "机制", cells: [{ factor: "p1", range: r(1, 2, 3) }, { factor: "p3", range: r(1, 2, 3) }] }] }, checklist, known: null, jevModel: MODEL });
assert.match(wrong, /第 1 条解释 cells 的格应当是 p1、p2;你写的 p3 不在里面;还缺 p2/);
});
test("规则 6:解释最多 M 条", async () => {
const one = { body: "机制", cells: [{ factor: "p1", range: r(1, 2, 3) }] };
const reason = await rejected({ kind: "explanation", content: { explanations: [one, one] }, checklist: { kind: "explanation", cells: ["p1"], maxExplanations: 1 }, known: null, jevModel: MODEL });
assert.match(reason, /交了 2 条解释,最多 1 条/);
});
test("规则 7:志愿个数够、不重复、都在入口清单里", async () => {
const checklist: ChecklistOf<"claim"> = { kind: "claim", entries: ["e-a1-1", "e-a2-1"], minChoices: 2 };
assert.match(await rejected({ kind: "claim", content: { choices: ["e-a1-1"] }, checklist, known: null, jevModel: MODEL }), /choices 至少要排 2 个入口,你排了 1 个/);
const reason = await rejected({ kind: "claim", content: { choices: ["e-a1-1", "e-a1-1", "e-a3-1"] }, checklist, known: null, jevModel: MODEL });
assert.match(reason, /choices 只能写 e-a1-1、e-a2-1;你写的 e-a3-1 不在里面/);
assert.match(reason, /choices 里 e-a1-1 写了不止一次/);
});
test("规则 8:挑刺针对的因子编号在被挑提案的因子范围内", async () => {
const reason = await rejected({ kind: "critique", content: { target: "p3", body: "挑刺" }, checklist: { kind: "critique", proposal: "r1-a1-1", factors: ["p1", "p2"] }, known: null, jevModel: MODEL });
assert.match(reason, /target 只能写 whole、p1、p2;你写的 p3 不在里面/);
});
const ORIGINAL: OriginalProposal = { factors: PROPOSAL.factors, safeguards: PROPOSAL.safeguards, metrics: PROPOSAL.metrics, body: PROPOSAL.body };
const RESPONSE_LIST: ChecklistOf<"response"> = { kind: "response", candidate: "r1-c1", critiques: ["r1-a1-1.a2.1", "r1-a1-1.a3.1"], ...RULES };
const response = (content: Submission<"response">): Incoming => ({ kind: "response", content, checklist: RESPONSE_LIST, known: { original: ORIGINAL }, jevModel: MODEL });
test("规则 9a:回应的答复恰好覆盖全部挑刺;原因里列出该写的全部编号(参考项目提交 378fe68)", async () => {
const reason = await rejected(response({ replies: [{ critique: "r1-a1-1.a2.2", reply: "改了" }], changes: {}, clauses: ["c"], body: "无" }));
assert.equal(reason, "这份提交没有收下,要改的地方:\n- replies 的挑刺编号应当是 r1-a1-1.a2.1、r1-a1-1.a3.1;你写的 r1-a1-1.a2.2 不在里面;还缺 r1-a1-1.a2.1、r1-a1-1.a3.1");
});
const UNIFICATION: Submission<"unification"> = {
...ORIGINAL,
replies: [],
clauses: ["c1 条"],
sources: [{ proposal: "r1-a1-1", coverage: "全在" }],
};
const unification = (content: Submission<"unification">): Incoming => ({
kind: "unification",
content,
checklist: { kind: "unification", candidate: "r1-c1", critiques: [], sources: ["r1-a1-1", "r1-a2-1"], ...RULES },
known: null,
jevModel: MODEL,
});
test("规则 9b:统一的 sources 恰好覆盖被合并的原提案;没有挑刺时说交空数组", async () => {
const reason = await rejected(unification({ ...UNIFICATION, replies: [{ critique: "x.a1.1", reply: "答" }] }));
assert.match(reason, /replies 的挑刺编号应当是 没有(交空数组);你写的 x.a1.1 不在里面/);
assert.match(reason, /sources 的原提案编号应当是 r1-a1-1、r1-a2-1;还缺 r1-a2-1/);
});
test("规则 2、3、4、10 对拼好的协议查:回应改出来的因子超过上限也退回", async () => {
const reason = await rejected(response({ replies: [{ critique: "r1-a1-1.a2.1", reply: "答" }, { critique: "r1-a1-1.a3.1", reply: "答" }], changes: { factors: [factor("a", null), factor("b", null), factor("c", null)] }, clauses: ["c"], body: "无" }));
assert.match(reason, /factors 有 3 个因子,超过上限 2 个/);
});
test("规则 2、3、4、10 对统一自己的字段查:因子超上限、保障叫 main、登记已占用的编号都退回", async () => {
const reason = await rejected(unification({ ...UNIFICATION, factors: [factor("a", null), factor("b", null), factor("c", null)], safeguards: ["main"], metrics: { defined: [metric("wall-seconds")], reused: [] }, sources: [{ proposal: "r1-a1-1", coverage: "全在" }, { proposal: "r1-a2-1", coverage: "全在" }] }));
assert.match(reason, /factors 有 3 个因子,超过上限 2 个/);
assert.match(reason, /main/);
assert.match(reason, /metrics.defined 的编号 wall-seconds 已被占用/);
});
test("规则 10:reused 要在可引用清单里;新登记的编号不重复,也不能和已占用的重名(含答案路径两个保留键)", async () => {
assert.match(await rejected(proposal({ ...PROPOSAL, metrics: { defined: [], reused: ["kick-count"] } }, 0)), /metrics.reused 只能写 restart-count;你写的 kick-count 不在里面/);
const reason = await rejected(proposal({ ...PROPOSAL, metrics: { defined: [metric("a"), metric("a"), metric("wall-seconds"), metric("answer-path")], reused: [] } }, 0));
assert.match(reason, /metrics.defined 里编号 a 写了不止一次/);
assert.match(reason, /metrics.defined 的编号 wall-seconds 已被占用/);
assert.match(reason, /metrics.defined 的编号 answer-path 已被占用/);
});
test("提案条数:已收下 M 条再交就退回", async () => {
assert.match(await rejected(proposal(PROPOSAL, 2)), /本阶段你已经收下 2 条提案,每人最多 2 条/);
});
const TABLE = {
factorLocations: [{ factor: "p1", locations: ["solve.py:10-20"] }, { factor: "p2", locations: ["solve.py:30"] }],
safeguardLocations: [{ safeguard: "guard", locations: ["solve.py"] }],
unassigned: [],
concerns: "无",
body: "无",
};
const FROZEN = { candidate: "r1-c1", clauses: ["c1", "c2"], factors: ["p1", "p2"], safeguards: ["guard"] };
const implementation = (code: Extract<Incoming, { kind: "implementation" }>["known"]["code"], content: Submission<"implementation">): Incoming => ({
kind: "implementation",
content,
checklist: { kind: "implementation", ...FROZEN },
known: { code },
jevModel: MODEL,
});
const SAVED = { saved: true as const, protectedFiles: [], hunks: [], protocol: { body: "协议", metrics: [] } };
test("规则 11:位置表覆盖每个因子和每条保障;起始行不大于结束行", async () => {
const reason = await rejected(
implementation(SAVED, { ...TABLE, factorLocations: [{ factor: "p1", locations: ["solve.py:20-10"] }], safeguardLocations: [], unassigned: [{ location: "util.py:5-3", note: "日志" }] }),
);
assert.match(reason, /factorLocations 的因子应当是 p1、p2;还缺 p2/);
assert.match(reason, /safeguardLocations 的保障应当是 guard;还缺 guard/);
assert.match(reason, /代码位置 solve.py:20-10 的起始行大于结束行/);
assert.match(reason, /代码位置 util.py:5-3 的起始行大于结束行/);
});
test("改了不许改的文件:退回,原因里写出是哪几个文件(实现、修复、消融实现都查)", async () => {
const reason = await rejected(implementation({ ...SAVED, protectedFiles: ["validate.py", "harness.py"] }, TABLE));
assert.equal(reason, "这份提交没有收下,要改的地方:\n- 改了不许改的文件:validate.py、harness.py。把它们改回本批基线的样子再交");
const repair = await rejected({ kind: "repair", content: TABLE, checklist: { kind: "repair", ...FROZEN }, known: { code: { ...SAVED, protectedFiles: ["validate.py"] } }, jevModel: MODEL });
assert.match(repair, /改了不许改的文件:validate.py/);
const ablation = await rejected({
kind: "ablationImplementation",
content: { alsoOff: [], cannotRun: null, concerns: "无" },
checklist: { kind: "ablationImplementation", cell: "r1-c1-p1", off: "p1", factors: ["p1", "p2"] },
known: { code: { saved: true, protectedFiles: ["validate.py"] } },
jevModel: MODEL,
});
assert.match(ablation, /改了不许改的文件:validate.py/);
});
test("代码存不了:退回,原样写 workspace 给的路径和原因", async () => {
const reason = await rejected(implementation({ saved: false, path: "data/link", reason: "符号链接,代码版本只存普通文件和目录" }, TABLE));
assert.equal(reason, "这份提交没有收下,要改的地方:\n- 代码存不成版本:data/link(符号链接,代码版本只存普通文件和目录)。改掉它再交");
});
test("规则 12:核验的条款、因子、保障、没有归属的改动逐项全覆盖", async () => {
const reason = await rejected({
kind: "verification",
content: {
clauses: [{ clause: "c1", found: true, where: "a" }],
factors: [{ factor: "p1", found: true, where: "a" }, { factor: "p2", found: true, where: "a" }, { factor: "p2", found: true, where: "a" }],
safeguards: [],
unassigned: [{ location: "util.py:3", outOfScope: false, reason: "日志" }],
body: "无",
},
checklist: { kind: "verification", ...FROZEN, unassigned: ["util.py:1-4"] },
known: null,
jevModel: MODEL,
});
assert.match(reason, /clauses 的实现条款应当是 c1、c2;还缺 c2/);
assert.match(reason, /factors 的因子应当是 p1、p2;p2 写了不止一次/);
assert.match(reason, /safeguards 的保障应当是 guard;还缺 guard/);
assert.match(reason, /unassigned 的改动位置应当是 util.py:1-4;你写的 util.py:3 不在里面;还缺 util.py:1-4/);
});
test("规则 13:cannotRun 时 alsoOff 为空;alsoOff 不含本格的因子、都在因子清单里、不重复", async () => {
const checklist: ChecklistOf<"ablationImplementation"> = { kind: "ablationImplementation", cell: "r1-c1-p1", off: "p1", factors: ["p1", "p2"] };
const both = await rejected({ kind: "ablationImplementation", content: { alsoOff: ["p2"], cannotRun: "跑不起来", concerns: "无" }, checklist, known: { code: null }, jevModel: MODEL });
assert.match(both, /写了 cannotRun(这一格跑不起来)时 alsoOff 要交空数组/);
const reason = await rejected({ kind: "ablationImplementation", content: { alsoOff: ["p1", "p5", "p2", "p2"], cannotRun: null, concerns: "无" }, checklist, known: { code: { saved: true, protectedFiles: [] } }, jevModel: MODEL });
assert.match(reason, /alsoOff 不能写本格要关的因子 p1/);
assert.match(reason, /alsoOff 只能写 p2;你写的 p5 不在里面/);
assert.doesNotMatch(reason, /只能写 p1/);
assert.match(reason, /alsoOff 里 p2 写了不止一次/);
});
test("规则 14:消融核验覆盖每个因子", async () => {
const reason = await rejected({ kind: "ablationVerification", content: { factors: [{ factor: "p1", off: true, where: "a" }] }, checklist: { kind: "ablationVerification", cell: "r1-c1-p1", factors: ["p1", "p2"] }, known: null, jevModel: MODEL });
assert.match(reason, /factors 的因子应当是 p1、p2;还缺 p2/);
});
test("一份提交的几处问题一次列全,每条一行", async () => {
const reason = await rejected(proposal({ ...PROPOSAL, safeguards: ["main"], authorEstimate: r(3, 2, 1) }, 2));
const lines = reason.split("\n");
assert.equal(lines.length, 4, reason);
assert.equal(lines[0], "这份提交没有收下,要改的地方:");
assert.ok(lines.slice(1).every((l) => l.startsWith("- ")), reason);
assert.match(reason, /main/);
assert.match(reason, /严格递增/);
});
// ---------------------------------------------------------------------------
// 拼冻结协议
// ---------------------------------------------------------------------------
test("回应拼协议:changes 只改保障时,因子、指标、正文取原提案的,实现条款取回应自己的;答复不进协议", () => {
const p = protocolBody({ kind: "response", content: { replies: [{ critique: "x", reply: "答" }], changes: { safeguards: ["clip"] }, clauses: ["条款一"], body: "答复正文" }, original: ORIGINAL });
assert.deepEqual(p, {
factors: [factor("多起点", null), factor("扰动", "r1-c1-p2")],
safeguards: ["clip"],
metrics: { defined: [metric("kick-count")], reused: ["restart-count"] },
body: "提案正文",
clauses: ["条款一"],
});
});
test("回应拼协议:changes 改了正文时用新正文", () => {
const p = protocolBody({ kind: "response", content: { replies: [], changes: { body: "新正文" }, clauses: ["条款一"], body: "无" }, original: ORIGINAL });
assert.equal(p.body, "新正文");
assert.deepEqual(p.safeguards, ["guard"]);
});
test("统一拼协议:全用统一提交自己的字段,不叠原提案,不带 replies、sources", () => {
const content: Submission<"unification"> = { factors: [factor("合并后的因子", null)], safeguards: [], metrics: { defined: [], reused: [] }, body: "统一正文", replies: [{ critique: "x", reply: "答" }], clauses: ["统一条款"], sources: [{ proposal: "r1-a1-1", coverage: "全在" }] };
assert.deepEqual(protocolBody({ kind: "unification", content }), { factors: [factor("合并后的因子", null)], safeguards: [], metrics: { defined: [], reused: [] }, body: "统一正文", clauses: ["统一条款"] });
});
// ---------------------------------------------------------------------------
// 核验、消融核验通过没有
// ---------------------------------------------------------------------------
const VERIFICATION: Submission<"verification"> = {
clauses: [{ clause: "c1", found: true, where: "a" }],
factors: [{ factor: "p1", found: true, where: "a" }],
safeguards: [{ safeguard: "guard", found: true, where: "a" }],
unassigned: [{ location: "util.py:3", outOfScope: false, reason: "日志" }],
body: "无",
};
test("核验:全部找到、没有越界就通过", () => {
assert.deepEqual(verificationOutcome(VERIFICATION), { passed: true, clauses: [], factors: [], safeguards: [], outOfScope: [] });
});
test("核验:一项没落实或一条判为越界就不通过,分开列出", () => {
const v = { ...VERIFICATION, safeguards: [{ safeguard: "guard", found: false, where: "看了 solve.py" }], unassigned: [{ location: "util.py:3", outOfScope: true, reason: "改了计时" }] };
assert.deepEqual(verificationOutcome(v), { passed: false, clauses: [], factors: [], safeguards: ["guard"], outOfScope: ["util.py:3"] });
});
const ABLATION_V = (offs: Record<string, boolean>): Submission<"ablationVerification"> => ({
factors: Object.entries(offs).map(([factor, off]) => ({ factor, off, where: "a" })),
});
test("消融核验:要关的和一起关的都关着、其余照留,就通过", () => {
assert.deepEqual(ablationOutcome(ABLATION_V({ p1: true, p2: false, p3: true }), "p1", ["p3"]), { passed: true, leftOn: [], wronglyOff: [] });
});
test("消融核验:该关的照留、该留的关了,就不通过", () => {
assert.deepEqual(ablationOutcome(ABLATION_V({ p1: false, p2: true, p3: false }), "p1", []), { passed: false, leftOn: ["p1"], wronglyOff: ["p2"] });
});
// ---------------------------------------------------------------------------
// 差异块的退回文字(假 Jev)
// ---------------------------------------------------------------------------
function fakeJev(t: TestContext, yes: (question: string, diff: string) => boolean): void {
const key = process.env["TYPESAFE_API_KEY"];
process.env["TYPESAFE_API_KEY"] = "test-key";
t.after(() => {
if (key === undefined) delete process.env["TYPESAFE_API_KEY"];
else process.env["TYPESAFE_API_KEY"] = key;
});
let n = 0;
t.mock.method(globalThis, "fetch", async (_url: string, init: RequestInit) => {
const body = JSON.parse(init.body as string);
const answers = Object.fromEntries(Object.keys(body.questions).map((q) => [q, { noul: yes(q, body.state.change?.diff ?? "") ? 0.99 : 0.01 }]));
n += 1;
return new Response(JSON.stringify({ answers }), { status: 200, headers: { "x-typesafe-request-id": `req_${n}` } });
});
}
test("实现:每块问一次,判断全交回;退回只列判是的几问,写出文件、第几块、附差异原文,原文含 ``` 时围栏加长", async (t) => {
fakeJev(t, (q, diff) => q === "unseeded_randomness" && diff.includes("random"));
const hunks = [
{ path: "solve.py", change: "modified" as const, text: "@@ -1 +1 @@\n-a\n+b" },
{ path: "solve.py", change: "modified" as const, text: "@@ -9 +9 @@\n-x = 1\n+x = random() # ```" },
];
const v = await receive(implementation({ ...SAVED, hunks }, TABLE));
assert.equal(v.accepted, false);
assert.equal(v.jev!.length, 10, "两块各五问");
assert.equal(
(v as Extract<Verdict, { accepted: false }>).reason,
[
"这份提交没有收下,Jev 判出下面几处有问题:",
"",
"- solve.py(改动的文件)的第 2 块:unseeded_randomness 概率 0.99(阈值 " + v.jev!.find((j) => j.question === "unseeded_randomness")!.threshold + ")",
"",
"````diff",
"@@ -9 +9 @@\n-x = 1\n+x = random() # ```",
"````",
].join("\n"),
);
});
test("回应全判否:收下,交回拼好的协议和三问的判断", async (t) => {
fakeJev(t, () => false);
const v = await receive(response({ replies: [{ critique: "r1-a1-1.a2.1", reply: "答" }, { critique: "r1-a1-1.a3.1", reply: "答" }], changes: { body: "新正文" }, clauses: ["条款一"], body: "无" }));
assert.equal(v.accepted, true);
assert.equal(v.jev!.length, 3);
assert.deepEqual(v.accepted && v.protocol, {
factors: [factor("多起点", null), factor("扰动", "r1-c1-p2")],
safeguards: ["guard"],
metrics: { defined: [metric("kick-count")], reused: ["restart-count"] },
body: "新正文",
clauses: ["条款一"],
});
});
test("统一的冻结协议有一问判是:退回,原因只列这一问", async (t) => {
fakeJev(t, (q) => q === "changes_measurement");
const v = await receive(unification({ ...UNIFICATION, sources: [{ proposal: "r1-a1-1", coverage: "全在" }, { proposal: "r1-a2-1", coverage: "全在" }] }));
assert.equal(v.accepted, false);
assert.equal(v.jev!.length, 3);
const threshold = v.jev!.find((j) => j.question === "changes_measurement")!.threshold;
assert.equal((v as Extract<Verdict, { accepted: false }>).reason, `这份提交没有收下,Jev 判出下面几处有问题:\n\n- 冻结协议:changes_measurement 概率 0.99(阈值 ${threshold})`);
});
test("实现有差异块、全判否:收下,jev 是全部判断", async (t) => {
fakeJev(t, () => false);
const v = await receive(implementation({ ...SAVED, hunks: [{ path: "solve.py", change: "modified", text: "@@ -1 +1 @@\n-a\n+b" }] }, TABLE));
assert.equal(v.accepted, true);
assert.equal(v.jev!.length, 5);
});
test("消融实现的代码存不成版本:退回", async () => {
const reason = await rejected({
kind: "ablationImplementation",
content: { alsoOff: [], cannotRun: null, concerns: "无" },
checklist: { kind: "ablationImplementation", cell: "r1-c1-p1", off: "p1", factors: ["p1", "p2"] },
known: { code: { saved: false, path: "bin/x", reason: "符号链接,代码版本只存普通文件和目录" } },
jevModel: MODEL,
});
assert.match(reason, /代码存不成版本:bin\/x(符号链接/);
});