import assert from "node:assert/strict"; import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { test } from "node:test"; import { beliefBook, freezeProtocol, queryMaterial, readPrompts, type TaskMaterial } from "../src/materials/index.ts"; import { SUBMISSION_KINDS, type SubmissionKind } from "../src/vocab/index.ts"; import { tempDir } from "./ledger-helpers.ts"; import { CHANGES_C1, CRITIQUE_ID, everything, material, NONE, study, studyUpTo, TABLE_C1 } from "./materials-helpers.ts"; // 每种任务至少一个测试:在真实账本里写到某一步,生成材料,查该有的有、不该有的没有、对号清单对。 // 预期值按方法.md 条文手写(研究进程见 materials-helpers.ts 的注释),不从实现推。 const headings = (m: TaskMaterial) => m.sections.map((s) => s.heading); const has = (m: TaskMaterial, text: string) => assert.ok(everything(m).includes(text), `材料里应有"${text}"`); const lacks = (m: TaskMaterial, text: string) => assert.ok(!everything(m).includes(text), `材料里不该有"${text}"`); // --------------------------------------------------------------------------- // 第零轮(方法.md 4.2) // --------------------------------------------------------------------------- test("入口:只有任务与基线;看不到别人已交的入口(4.2 闭门写入口)", (t) => { const l = studyUpTo(t, "entryA1"); const m = material(l, "entry", "a2", "s0-entry", null, NONE); assert.deepEqual(m.checklist, { kind: "entry", entries: { min: 2, max: 3 } }); has(m, "本轮基线 base:最终指标 0.96"); assert.deepEqual(m.readable, [{ version: "v-base", label: "本批基线", dir: "/code/v-base" }]); lacks(m, "入口甲一"); assert.ok(!headings(m).includes("信念本"), "第零轮不放信念本(4.2:信念本不变)"); }); test("讨论:全部入口标明作者,编号按作者从 1 编", (t) => { const l = studyUpTo(t, "discussionOpen"); const m = material(l, "discussion", "a1", "s0-d1", null, NONE); assert.deepEqual(m.checklist, { kind: "discussion", entries: { min: 2, max: 3 } }); for (const x of ["e-a1-1:入口甲一", "e-a1-2:入口甲二", "e-a2-1:入口乙一", "e-a2-2:入口乙二"]) has(m, x); assert.ok(headings(m).includes("入口编号对照")); }); test("认领:整份替换后新清单接着编号,旧编号作废;至少排 min(N, 入口总数)", (t) => { const l = studyUpTo(t, "claimOpen"); const m = material(l, "claim", "a2", "s0-claim", null, NONE); // a1 讨论时整份换成两个新入口,接着 e-a1-2 往后编:e-a1-3、e-a1-4;a2 没换。N = 2,入口 4 个。 assert.deepEqual(m.checklist, { kind: "claim", entries: ["e-a1-3", "e-a1-4", "e-a2-1", "e-a2-2"], minChoices: 2 }); has(m, "e-a1-1(作废):入口甲一"); has(m, "甲的讨论"); has(m, "乙的讨论"); }); // --------------------------------------------------------------------------- // 第 r 轮(方法.md 4.3) // --------------------------------------------------------------------------- test("第 1 轮提案:你认领的入口放在最前;没有上一轮;对号清单", (t) => { const l = studyUpTo(t, "proposalOpen"); const m = material(l, "proposal", "a1", "s1-prop", null, NONE); assert.deepEqual(headings(m).slice(0, 2), ["你自己的", "你认领的入口"]); has(m, "e-a1-3:入口甲三"); has(m, "从这里开始看"); has(m, "这是第 1 轮,没有上一轮"); assert.deepEqual(m.checklist, { kind: "proposal", maxProposals: 2, factorLimit: 3, continuable: [], reusableMetrics: [], takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers"], }); const h = headings(m); assert.ok(h.indexOf("你自己的") < h.indexOf("公共材料") && h.indexOf("公共材料") < h.indexOf("信念本"), "先你自己的、后公共材料"); }); test("第 2 轮提案:上一轮全部收尾、上一轮实验员的提醒、可延续的因子、已登记指标;入口标为第 1 轮的起点", (t) => { const l = studyUpTo(t, "proposal2Open"); const m = material(l, "proposal", "a2", "s2-prop", null, NONE); has(m, "甲的收尾"); has(m, "乙的收尾"); has(m, "guard 可能挡掉一些摆法"); has(m, "第 1 轮的起点;之后是否沿用由你定"); assert.deepEqual(m.checklist, { kind: "proposal", maxProposals: 2, factorLimit: 3, continuable: ["r1-c1-p1", "r1-c1-p2", "r1-c2-p1"], reusableMetrics: ["r1-c1.restart-count", "r1-c2.restart-count"], takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers"], }); for (const x of ["上一轮的收尾", "上一轮实验员实现时的提醒", '可以写进"延续"的因子', "已登记指标清单", "研究记录", "信念本"]) { assert.ok(headings(m).includes(x), `应有一节"${x}"`); } }); test("第一次预测:给这个候选的原始提案;看不到别人刚交的第一次预测,也看不到作者自评", (t) => { const l = studyUpTo(t, "firstA1"); const m = material(l, "firstPrediction", "a2", "s1-fp", "r1-c1", NONE); assert.deepEqual(m.checklist, { kind: "firstPrediction", candidate: "r1-c1" }); has(m, "甲的提案正文"); lacks(m, "5.551"); lacks(m, "甲第一次理由"); lacks(m, "7.772"); for (const x of ["研究记录", "信念本"]) assert.ok(headings(m).includes(x)); }); test("挑刺:按原提案挑,对号清单是那条提案和它的因子", (t) => { const l = studyUpTo(t, "critiqueOpen"); const m = material(l, "critique", "a2", "s1-crit", "r1-a1-1", NONE); assert.deepEqual(m.checklist, { kind: "critique", proposal: "r1-a1-1", factors: ["p1", "p2"] }); has(m, "甲的提案正文"); lacks(m, "5.551"); lacks(m, "6.661"); }); test("挑刺的现在交什么:可以交一份或几份,每份调用一次;不写调用一次就完成", (t) => { const m = material(studyUpTo(t, "critiqueOpen"), "critique", "a2", "s1-crit", "r1-a1-1", NONE); const end = m.firstMessage.slice(m.firstMessage.lastIndexOf("# 现在交什么")); assert.ok(end.includes("每调用一次 research_submit 交一份挑刺;可以交一份或几份"), end); assert.ok(!end.includes("交上就完成了"), end); }); test("回应:本人原提案和针对它的全部挑刺", (t) => { const l = studyUpTo(t, "responseOpen"); const m = material(l, "response", "a1", "s1-resp", "r1-c1", NONE); has(m, "乙的挑刺:扰动可能混杂(推测)"); assert.deepEqual(m.checklist, { kind: "response", candidate: "r1-c1", critiques: [CRITIQUE_ID], factorLimit: 3, continuable: [], reusableMetrics: [], takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers"], }); }); test("统一:全部被合并的原提案、针对它们的挑刺、要逐条体现的原提案", (t) => { const l = studyUpTo(t, "responseOpen"); const m = material(l, "unification", "a1", "s1-resp", "r1-c1", NONE); const c = m.checklist as Extract; assert.deepEqual([c.sources, c.critiques], [["r1-a1-1"], [CRITIQUE_ID]]); has(m, "甲的提案正文"); }); test("第二次预测:冻结协议、挑刺与答复配对、本人第一次的三个数;看不到别人的第一次和作者自评(4.3 ⑦)", (t) => { const l = studyUpTo(t, "secondOpen"); const m = material(l, "secondPrediction", "a2", "s1-sp", "r1-c1", NONE); assert.deepEqual(m.checklist, { kind: "secondPrediction", candidate: "r1-c1" }); has(m, "6.661 / 6.662 / 6.663"); has(m, "乙的挑刺:扰动可能混杂(推测)"); has(m, "作者的答复:甲答复:保留扰动"); has(m, "### 答复正文"); lacks(m, "5.551"); lacks(m, "甲第一次理由"); lacks(m, "7.772"); }); test("新登记指标不能用的编号里有答案路径、保障动手次数两个保留键(用户 2026-10-03 定)", (t) => { const c = material(studyUpTo(t, "proposalOpen"), "proposal", "a1", "s1-prop", null, NONE).checklist as Extract; assert.ok(c.takenMetricIds.includes("answer-path") && c.takenMetricIds.includes("safeguard-triggers"), JSON.stringify(c.takenMetricIds)); }); test("冻结时新登记的指标改全名:同一轮两个候选登记同一个短名、定义不同,冻结后是两个不同的全名", () => { const body = (meaning: string) => ({ factors: [{ name: "因子", method: "做法", keySettings: "无", continues: null }], safeguards: [], metrics: { defined: [{ id: "restart-gain", meaning, unit: "无", direction: "maximize" as const, howMeasured: "测一次", missing: "recordMissing" as const }], reused: ["r1-c1.min-gap"] }, body: "正文", clauses: ["条款"], }); const a = freezeProtocol(body("重启段内最好 Σr 减进入重启段时的 Σr"), "r2-c1", 2, "r1-c1", ["r2-a1-1"]); const b = freezeProtocol(body("最终 Σr 减池阶段最好 Σr"), "r2-c2", 2, "r1-c1", ["r2-a2-1", "r2-a3-1"]); assert.deepEqual(a, { ...body("重启段内最好 Σr 减进入重启段时的 Σr"), metrics: { defined: [{ ...body("").metrics.defined[0]!, id: "r2-c1.restart-gain", meaning: "重启段内最好 Σr 减进入重启段时的 Σr" }], reused: ["r1-c1.min-gap"] }, round: 2, baseline: "r1-c1", proposals: ["r2-a1-1"] }); assert.deepEqual([b.metrics.defined[0]!.id, b.metrics.defined[0]!.meaning, b.proposals], ["r2-c2.restart-gain", "最终 Σr 减池阶段最好 Σr", ["r2-a2-1", "r2-a3-1"]]); }); test("下一轮:同短名的两个全名都能引用,各自对应自己的定义;引用的指标在实验员材料里是原登记的定义", (t) => { const m = material(studyUpTo(t, "proposal2Open"), "proposal", "a2", "s2-prop", null, NONE); const list = m.sections.find((s) => s.heading === "已登记指标清单")!.text; assert.match(list, /- r1-c1\.restart-count(候选 r1-c1 登记,越大越好,单位 次,测不到时只记缺失):重启次数;/); assert.match(list, /- r1-c2\.restart-count(候选 r1-c2 登记,越小越好,单位 次,测不到时只记缺失):修复阶段重启了几次;/); const impl = material(studyUpTo(t, "implOpen2"), "implementation", "x1", "s2-impl-c1", "r2-c1", NONE); const defs = impl.sections.find((s) => s.heading === "要实现的指标定义")!.text; assert.ok(defs.includes('"id": "r1-c1.restart-count"') && defs.includes("重启次数"), defs); assert.ok(!defs.includes("修复阶段重启了几次"), defs); }); test("解释:本轮运行、提醒、要给三个数的格、产物只进查询;看不到别人的第一次预测", (t) => { const l = studyUpTo(t, "explOpen1"); const m = material(l, "explanation", "a2", "s1-expl", null, { changes: null, artifacts: { "run-1": { "stdout.log": "日志正文" } } }); assert.deepEqual(m.checklist, { kind: "explanation", cells: ["p1", "p2"], maxExplanations: 2 }); has(m, "分数 1.2"); // 正向运行的答案出自保障:换一句;方法原句只用在消融格(见收尾的测试)。 has(m, "这次答案出自保障 guard,分数可能主要是这条保障的效果。"); lacks(m, "本格答案出自保障"); has(m, "guard 可能挡掉一些摆法"); assert.ok(!m.firstMessage.includes("日志正文"), "产物不进第一条消息"); assert.equal(queryMaterial(m, "run-1/stdout.log"), "运行 run-1 的产物 stdout.log\n\n日志正文"); lacks(m, "5.551"); lacks(m, "8.882"); }); test("收尾:本轮采纳、选格、押的因子", (t) => { const l = studyUpTo(t, "wrapOpen"); const m = material(l, "wrapUp", "a1", "s1-wrap", null, NONE); assert.deepEqual(m.checklist, { kind: "wrapUp" }); has(m, "本轮采纳候选 r1-c1"); has(m, "本轮选格:测 p1"); has(m, "程序按方向收益押的因子:r1-c1-p1"); has(m, "r1-c1-p2 停滞扰动(停滞扰动的设定):候选 r1-c1,基线 base;没测:剩余次数不够"); has(m, "本格答案出自保障 guard,量到的可能是这条保障的效果,而不是关掉这个因子的效果;保障动手次数"); lacks(m, "7.772"); }); test("实现:冻结协议、指标定义、要交的位置表、起点代码;没有信念本、成绩和别的候选", (t) => { const l = studyUpTo(t, "implOpen"); const m = material(l, "implementation", "x1", "s1-impl-c1", "r1-c1", NONE); assert.deepEqual(m.checklist, { kind: "implementation", candidate: "r1-c1", clauses: ["c1", "c2"], factors: ["p1", "p2"], safeguards: ["guard"] }); assert.deepEqual(m.readable.map((r) => r.version), ["v-base"]); for (const x of ["任务与本轮", "基线代码", "本任务要处理的东西", "要实现的指标定义", "要交的因子位置表"]) assert.ok(headings(m).includes(x)); assert.ok(!headings(m).includes("信念本")); has(m, '"id": "r1-c1.restart-count"'); has(m, "键照抄这里的 id(全名"); lacks(m, "乙的提案正文"); lacks(m, "收缩半径修复"); lacks(m, "6.661"); }); test("修复:退回意见、上一次的位置表、上一次的改动", (t) => { const l = studyUpTo(t, "verifyOpen"); l.append("submission", { stageId: "s1-ver-c1", author: "v1", kind: "verification", target: "r1-c1", item: null, version: 1, jev: null, content: { clauses: [{ clause: "c1", found: true, where: "solve.py:10" }, { clause: "c2", found: true, where: "solve.py:30" }], factors: [{ factor: "p1", found: true, where: "solve.py:10" }, { factor: "p2", found: false, where: "看了 solve.py 全文" }], safeguards: [{ safeguard: "guard", found: true, where: "solve.py:40" }], unassigned: [], body: "p2 没找到", }, }); l.append("stageClose", { stageId: "s1-ver-c1" }); l.append("stageOpen", { stageId: "s1-repair-c1", stage: "implementation", round: 1, roster: ["x1"] }); const m = material(l, "repair", "x1", "s1-repair-c1", "r1-c1", { changes: CHANGES_C1, artifacts: null }); assert.deepEqual(m.checklist, { kind: "repair", candidate: "r1-c1", clauses: ["c1", "c2"], factors: ["p1", "p2"], safeguards: ["guard"] }); has(m, "因子 p2 没落实:看了 solve.py 全文"); has(m, "+多起点循环"); assert.deepEqual(m.readable.map((r) => r.version), ["v-base", "v-c1"]); }); test("核验:没有归属的改动排第一;因子与保障同行只作提醒;对号清单带没有归属的改动", (t) => { const l = studyUpTo(t, "verifyOpen"); const m = material(l, "verification", "v1", "s1-ver-c1", "r1-c1", { changes: CHANGES_C1, artifacts: null }); assert.deepEqual(headings(m).slice(0, 2), ["分给你的材料", "没归到任何因子或保障的改动"]); // 改动是候选第 10–22 行,位置表 p1 盖住 10–20:21–22 没有归属。p2 的 30–40 与 guard 的 40–45 在第 40 行重叠。 assert.deepEqual(m.checklist, { kind: "verification", unassigned: ["solve.py:21-22"], candidate: "r1-c1", clauses: ["c1", "c2"], factors: ["p1", "p2"], safeguards: ["guard"], }); has(m, "solve.py:40:因子 p2、保障 guard"); assert.deepEqual(m.readable.map((r) => r.version), ["v-base", "v-c1"]); assert.ok(!headings(m).includes("信念本")); }); test("核验材料里没有 Jev 的概率和阈值:收下实现时用过 Jev,也不给核验员看", (t) => { const l = studyUpTo(t, "implOpen"); const jev = [{ question: "early_exit", questionHash: "ab".repeat(32), threshold: 0.6312, probability: 0.4321, yes: false, requestId: "req-1" }]; l.append("submission", { stageId: "s1-impl-c1", author: "x1", kind: "implementation", target: "r1-c1", item: null, version: 1, jev, content: { ...TABLE_C1, concerns: "无", body: "实现说明" } }); l.append("stageClose", { stageId: "s1-impl-c1" }); l.append("code", { object: "r1-c1", version: "v-c1", locations: TABLE_C1 }); l.append("stageOpen", { stageId: "s1-ver-c1", stage: "verification", round: 1, roster: ["v1"] }); const m = material(l, "verification", "v1", "s1-ver-c1", "r1-c1", { changes: CHANGES_C1, artifacts: null }); has(m, "实现说明"); for (const x of ["0.4321", "0.6312", "early_exit", "预扫"]) lacks(m, x); }); test("消融实现重做:材料放上一次消融核验关错的因子和依据;第一次做时没有退回意见", (t) => { assert.ok(!everything(material(studyUpTo(t, "ablationImplOpen"), "ablationImplementation", "x1", "s1-ai", "r1-c1-p1", NONE)).includes("### 退回意见")); const l = studyUpTo(t, "ablationImpl"); // 这一格要关 p1、照留 p2;核验员看到 p1 还开着、p2 照留:只有 p1 关错了。 l.append("submission", { stageId: "s1-av", author: "v1", kind: "ablationVerification", target: "r1-c1-p1", item: null, version: 1, jev: null, content: { factors: [{ factor: "p1", off: false, where: "solve.py:12 仍在跑多起点循环" }, { factor: "p2", off: false, where: "solve.py:30 照留" }] }, }); l.append("stageClose", { stageId: "s1-av" }); l.append("stageOpen", { stageId: "s1-ai2", stage: "ablationImplementation", round: 1, roster: ["x1"] }); const m = material(l, "ablationImplementation", "x1", "s1-ai2", "r1-c1-p1", NONE); has(m, "### 退回意见"); has(m, "- 因子 p1 应当关着,核验员看到的是照留:solve.py:12 仍在跑多起点循环"); lacks(m, "solve.py:30 照留"); assert.deepEqual(m.checklist, { kind: "ablationImplementation", cell: "r1-c1-p1", off: "p1", factors: ["p1", "p2"] }); // 重做从上一次交的这一格代码接着改(用户 2026-10-03 定):工作目录是它的副本,能读的版本里有它,也有完整代码作对照。 assert.deepEqual(m.readable.map((r) => [r.version, r.label]), [ ["v-c1", "完整代码"], ["v-base", "本批基线(因子没写关掉时换成什么,就回到它在这件事上的做法)"], ["v-c1-p1", "你上一次交的这一格代码"], ]); has(m, "你的工作目录是上一次交的这一格代码的副本,从它接着改"); lacks(m, "你的工作目录是完整代码的副本"); }); test("消融实现:这一格、因子清单、完整代码的位置表;看不到运行成绩", (t) => { const l = studyUpTo(t, "ablationImplOpen"); const m = material(l, "ablationImplementation", "x1", "s1-ai", "r1-c1-p1", NONE); assert.deepEqual(m.checklist, { kind: "ablationImplementation", cell: "r1-c1-p1", off: "p1", factors: ["p1", "p2"] }); assert.ok(headings(m).includes("完整代码(消融的起点)")); assert.deepEqual(m.readable.map((r) => r.version), ["v-c1", "v-base"]); has(m, "你的工作目录是完整代码的副本"); lacks(m, "1.2"); lacks(m, "甲的解释"); }); test("消融核验:实际要关的因子、改动全文", (t) => { const l = studyUpTo(t, "ablationImpl"); const m = material(l, "ablationVerification", "v1", "s1-av", "r1-c1-p1", { changes: CHANGES_C1, artifacts: null }); assert.deepEqual(m.checklist, { kind: "ablationVerification", cell: "r1-c1-p1", factors: ["p1", "p2"] }); has(m, "p1(只关指定的因子)"); assert.deepEqual(m.readable.map((r) => r.version), ["v-c1", "v-base", "v-c1-p1"]); }); // --------------------------------------------------------------------------- // 七条可见性规则(任务书 09 第二部分),每条一个"不该看到" // --------------------------------------------------------------------------- test("可见性 1:同一阶段没关之前,别人看不到已交的提交", (t) => { const l = studyUpTo(t, "proposalA1"); const m = material(l, "proposal", "a2", "s1-prop", null, NONE); lacks(m, "甲的提案正文"); lacks(m, "多起点重启"); // 本人自己当前阶段已交的看得到 has(material(l, "proposal", "a1", "s1-prop", null, NONE), "甲的提案正文"); }); test("可见性 2:作者自评到本轮结束解封前,任何任务的材料里都没有", (t) => { const cases: Array<[string, SubmissionKind, string, string, string | null]> = [ ["firstOpen", "firstPrediction", "a1", "s1-fp", "r1-c1"], ["critiqueOpen", "critique", "a2", "s1-crit", "r1-a1-1"], ["responseOpen", "response", "a1", "s1-resp", "r1-c1"], ["secondOpen", "secondPrediction", "a1", "s1-sp", "r1-c1"], ["implOpen", "implementation", "x1", "s1-impl-c1", "r1-c1"], ["wrapOpen", "wrapUp", "a1", "s1-wrap", null], ]; for (const [step, kind, actor, stage, object] of cases) { const l = studyUpTo(t, step); const m = material(l, kind, actor, stage, object, NONE); for (const sealed of ["7.771", "7.772", "8.881", "8.882"]) lacks(m, sealed); } // 解封以后(跑过的 r1-c1)进战绩;没跑的 r1-c2 继续封存。 const after = beliefBook(studyUpTo(t, "proposal2Open"), study, 2); assert.ok(after.includes("作者自评 vs 大家合并:r1-c1(作者 a1)")); assert.ok(!after.includes("8.882")); }); test("可见性 3:第一次预测只有本人看得到,之后各阶段也不给别人", (t) => { const l = studyUpTo(t, "wrapOpen"); for (const m of [material(l, "wrapUp", "a2", "s1-wrap", null, NONE), material(l, "wrapUp", "a1", "s1-wrap", null, NONE)]) { lacks(m, "5.551"); lacks(m, "6.661"); } }); test("可见性 4:第二次预测看不到别人第一次的数", (t) => { const l = studyUpTo(t, "secondOpen"); const m = material(l, "secondPrediction", "a1", "s1-sp", "r1-c1", NONE); has(m, "5.551 / 5.552 / 5.553"); lacks(m, "6.661"); lacks(m, "乙第一次理由"); }); test("可见性 5:第零轮闭门写入口时看不到别人的入口", (t) => { const l = studyUpTo(t, "entryA2"); const m = material(l, "entry", "a1", "s0-entry", null, NONE); lacks(m, "入口乙一"); has(m, "入口甲一"); }); test("可见性 6:别人的退回不放进材料;本人重新发起时放上一次被退回的原因", (t) => { const l = studyUpTo(t, "secondOpen"); const reject = (author: string, target: string, reason: string) => l.append("rejection", { stageId: "s1-sp", author, kind: "secondPrediction", target, submitted: { range: { q1: 3, q5: 2, q9: 1 } }, reason, jev: null }); reject("a1", "r1-c1", "甲被退回:三个数不递增"); reject("a2", "r1-c1", "乙被退回"); const retry = material(l, "secondPrediction", "a1", "s1-sp", "r1-c1", NONE); assert.ok(retry.firstMessage.includes("# 上一次为什么没交成")); has(retry, "甲被退回:三个数不递增"); lacks(retry, "乙被退回"); lacks(material(l, "secondPrediction", "a2", "s1-sp", "r1-c2", NONE), "甲被退回"); }); test("退回按对象找:本人同一阶段对两个候选各被退回一次,重新发起时只放这个对象的那一条", (t) => { // 两个候选的第二次预测并行,退回交错写入;不靠模型调用的记录定位。 const l = studyUpTo(t, "secondOpen"); const reject = (target: string, reason: string) => l.append("rejection", { stageId: "s1-sp", author: "a1", kind: "secondPrediction", target, submitted: {}, reason, jev: null }); reject("r1-c1", "c1 第一次被退回"); reject("r1-c2", "c2 被退回"); reject("r1-c1", "c1 第二次被退回"); const c1 = material(l, "secondPrediction", "a1", "s1-sp", "r1-c1", NONE); has(c1, "c1 第二次被退回"); lacks(c1, "c1 第一次被退回"); lacks(c1, "c2 被退回"); const c2 = material(l, "secondPrediction", "a1", "s1-sp", "r1-c2", NONE); has(c2, "c2 被退回"); lacks(c2, "c1 第"); }); test("可见性 7:信念本只写已公开的记录:正向运行在采纳决定写入后才出现", (t) => { const before = beliefBook(studyUpTo(t, "run1"), study, 1); assert.ok(!before.includes("run-1")); const after = beliefBook(studyUpTo(t, "adopt1"), study, 1); assert.ok(after.includes("运行 run-1:算数(校验器判合法),分数 1.2")); }); test("实验员、核验员看不到别的候选", (t) => { const l = studyUpTo(t, "verifyOpen"); for (const m of [ material(l, "verification", "v1", "s1-ver-c1", "r1-c1", { changes: CHANGES_C1, artifacts: null }), material(studyUpTo(t, "implOpen"), "implementation", "x1", "s1-impl-c1", "r1-c1", NONE), ]) { lacks(m, "r1-c2"); lacks(m, "乙的"); assert.equal(queryMaterial(m, "r1-c2"), null); } }); test("没有归属的改动:手写两个版本和位置表,算出的清单与手算的一致", (t) => { const l = studyUpTo(t, "verifyOpen"); // 位置表:p1 a.py:1-5;p2 a.py:20 和整个 b.py;保障 guard a.py:5-6。 l.append("code", { object: "r1-c1", version: "v-c1", locations: { factorLocations: [{ factor: "p1", locations: ["a.py:1-5"] }, { factor: "p2", locations: ["a.py:20", "b.py"] }], safeguardLocations: [{ safeguard: "guard", locations: ["a.py:5-6"] }], unassigned: [{ location: "a.py:7", note: "实验员说是 import" }], }, }); // 旧版本 → 新版本:a.py 新版本第 1–8 行是改过的;删了旧第 11–12 行(新版本第 10 行之后);删了旧第 21 行(新版本第 19 行之后)。 // b.py 改了第 3–4 行;c.bin 新增二进制;run.sh 只改可执行位;empty.txt 新增空文件。 const changes = { files: [ { path: "a.py", change: "modified" as const, modeOnly: false, binary: false as const, hunks: [ { from: { start: 1, end: 3 }, to: { start: 1, end: 8 } }, { from: { start: 11, end: 12 }, to: { after: 10 } }, { from: { start: 21, end: 21 }, to: { after: 19 } }, ] }, { path: "b.py", change: "modified" as const, modeOnly: false, binary: false as const, hunks: [{ from: { start: 3, end: 4 }, to: { start: 3, end: 4 } }] }, { path: "c.bin", change: "added" as const, modeOnly: false, binary: true as const }, { path: "empty.txt", change: "added" as const, modeOnly: false, binary: false as const, hunks: [] }, { path: "run.sh", change: "modified" as const, modeOnly: true, binary: false as const, hunks: [] }, ], diff: "", }; const m = material(l, "verification", "v1", "s1-ver-c1", "r1-c1", { changes, artifacts: null }); // 手算:a.py 第 6 行归 guard,7–8 行没有归属(实验员自己的说明不算归属);删除点 10 前后(10、11)都没被盖住; // 删除点 19 后一行 20 归 p2;b.py 整个文件归 p2;其余三个文件按整个文件算,位置表都没写到。 const c = m.checklist as Extract; assert.deepEqual(c.unassigned, ["a.py:7-8", "a.py:10", "c.bin", "empty.txt", "run.sh"]); has(m, "- a.py:10(删除 2 行(旧版本第 11–12 行))"); has(m, "- c.bin(新增(二进制文件,按整个文件算))"); has(m, "- run.sh(修改(只改了可执行位,按整个文件算))"); has(m, "- a.py:7:实验员说是 import"); // 因子与保障同一行:p1 的 1–5 与 guard 的 5–6 在第 5 行。 has(m, "- a.py:5:因子 p1、保障 guard"); lacks(m, "因子 p2、保障 guard"); }); test("答案路径不在位置表里、读不到,都写明状态", (t) => { const l = studyUpTo(t, "run1"); l.append("runPlanned", { run: "run-x", purpose: "forward", round: 1, object: "r1-c2", definition: { command: ["python"], workdir: "/w", image: { digest: `sha256:${"0".repeat(64)}`, runtime: "runc" }, wallSeconds: 120, cores: 1, inheritEnv: false, env: {} } }); l.append("runStarted", { run: "run-x" }); l.append("runResult", { run: "run-x", process: { ending: "exited", exitCode: 0, cpuSeconds: { status: "measured", value: 1 }, wallSeconds: { status: "measured", value: 1 }, peakMemoryBytes: { status: "measured", value: 1 } }, judged: { verdict: "valid", verdictReason: "合法", score: { status: "measured", value: 1.1 }, metrics: {}, extra: null, answerPath: { status: "read", value: "p9" }, safeguardTriggers: { status: "unreadable", reason: "没写 safeguard-triggers" }, }, }); l.append("code", { object: "r1-c2", version: "v-c2", locations: { factorLocations: [{ factor: "p1", locations: ["x.py"] }], safeguardLocations: [], unassigned: [] } }); const input = { baseline: 0.96, knownBest: 2.635983, epsilon: 0.01, direction: "maximize" as const, candidates: [{ id: "r1-c1", score: 1.2 }] }; l.append("decision", { decision: "adopt", round: 1, rule: "test", ruleVersion: 1, params: null, seed: null, input: input as never, output: { line: 0.01, quarter: 0.4, best: { id: "r1-c1", score: 1.2, improvement: 0.24 }, adopted: "r1-c1" } }); l.append("stageOpen", { stageId: "s1-expl", stage: "explanation", round: 1, roster: ["a1", "a2"] }); const m = material(l, "explanation", "a1", "s1-expl", null, { changes: null, artifacts: {} }); const run = queryMaterial(m, "run-x")!; assert.ok(run.includes("答案路径写的是 p9,不在位置表里")); assert.ok(run.includes("保障动手次数读不到:没写 safeguard-triggers")); }); // --------------------------------------------------------------------------- // 查询、提示词 // --------------------------------------------------------------------------- test("查询:材料里的编号查得到;同一账本里材料外的编号查不到", (t) => { const l = studyUpTo(t, "secondOpen"); const m = material(l, "secondPrediction", "a2", "s1-sp", "r1-c1", NONE); assert.ok(queryMaterial(m, "r1-a1-1")!.includes("甲的提案正文")); assert.ok(queryMaterial(m, "task")!.includes("填圆任务说明")); assert.equal(queryMaterial(m, "s1-sp"), null); assert.equal(queryMaterial(m, "run-0"), null); assert.equal(queryMaterial(m, "不存在"), null); const impl = material(studyUpTo(t, "implOpen"), "implementation", "x1", "s1-impl-c1", "r1-c1", NONE); assert.equal(queryMaterial(impl, "r1-a1-1"), null); }); test("提示词:每种提交种类都有任务说明文件,每个角色都有角色提示词", () => { const p = readPrompts(join(import.meta.dirname, "..", "prompts")); assert.deepEqual(Object.keys(p.tasks).sort(), Object.keys(SUBMISSION_KINDS).sort()); for (const f of [...Object.values(p.tasks), ...Object.values(p.roles)]) assert.match(f.text, /\S/); }); test("提示词哈希是文件原始字节的 sha256", (t) => { const dir = tempDir(t); mkdirSync(join(dir, "roles")); mkdirSync(join(dir, "tasks")); const bytes = Buffer.from("x\r\n中", "utf8"); for (const r of ["researcher", "experimenter", "verifier"]) writeFileSync(join(dir, "roles", `${r}.md`), bytes); for (const k of Object.keys(SUBMISSION_KINDS)) writeFileSync(join(dir, "tasks", `${k}.md`), bytes); const p = readPrompts(dir); // printf 'x\r\n中' | sha256sum assert.equal(p.roles.researcher.hash, "965da20bc965975b4eae297a49cabf4be539ef62bec455fe932093535de2c593"); assert.equal(p.tasks.wrapUp.hash, "965da20bc965975b4eae297a49cabf4be539ef62bec455fe932093535de2c593"); assert.equal(p.tasks.wrapUp.text, "x\r\n中"); }); test("第一条消息:任务说明文件在最前,现在交什么在最后并列出提交字段", (t) => { const l = studyUpTo(t, "proposalOpen"); const m = material(l, "proposal", "a1", "s1-prop", null, NONE); const p = readPrompts(join(import.meta.dirname, "..", "prompts")); assert.ok(m.firstMessage.startsWith(p.tasks.proposal.text.trimEnd())); const end = m.firstMessage.slice(m.firstMessage.lastIndexOf("# 现在交什么")); for (const f of ["`factors`", "`safeguards`", "`metrics`", "`authorEstimate`", "`body`", "最多交 2 条"]) assert.ok(end.includes(f), f); });