iomgaa
09c44031a0
- 新增 src/submissions：先程序检查（14 条跨字段规则、不许改的文件、代码存不了、指标编号占用、提案数上限），全过再问 Jev；退回原因列出该写的全部编号；回应/统一拼冻结协议主体只在这一处；核验、消融核验通过没有。 - materials：挑刺单写"现在交什么"；退回按阶段、作者、种类、对象找（ledger 退回记录加 target，表结构版本 4）；核验材料去掉 Jev 预扫；信念本写合法化；删"本该关着"；消融核验不通过可重做，不跑的格从 cellNotRun 事件读原因。 - 用户 2026-10-03 定：冻结协议去掉 replies、replyBody，第二次预测的答复从已公开的回应或统一读；takenMetricIds 加两个保留键；解释一律交 cells；Jev 退回只列判"是"的几问、用英文问题名。 - jev 判重材料带 keySettings；读代码工具删 offset；三处旧说法改正。 - 文档：结构.md 第四、七、八部分，WORK_LOG。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
506 lines
28 KiB
TypeScript
506 lines
28 KiB
TypeScript
import assert from "node:assert/strict";
|
||
import { mkdirSync, writeFileSync } from "node:fs";
|
||
import { join } from "node:path";
|
||
import { test } from "node:test";
|
||
import { beliefBook, queryMaterial, readPrompts, type TaskMaterial } from "../src/materials/index.ts";
|
||
import { SUBMISSION_KINDS, type SubmissionKind } from "../src/vocab/index.ts";
|
||
import { tempDir } from "./ledger-helpers.ts";
|
||
import { CHANGES_C1, CRITIQUE_ID, everything, material, NONE, study, studyUpTo, TABLE_C1 } from "./materials-helpers.ts";
|
||
|
||
// 每种任务至少一个测试：在真实账本里写到某一步，生成材料，查该有的有、不该有的没有、对号清单对。
|
||
// 预期值按方法.md 条文手写（研究进程见 materials-helpers.ts 的注释），不从实现推。
|
||
|
||
const headings = (m: TaskMaterial) => m.sections.map((s) => s.heading);
|
||
const has = (m: TaskMaterial, text: string) => assert.ok(everything(m).includes(text), `材料里应有"${text}"`);
|
||
const lacks = (m: TaskMaterial, text: string) => assert.ok(!everything(m).includes(text), `材料里不该有"${text}"`);
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 第零轮（方法.md 4.2）
|
||
// ---------------------------------------------------------------------------
|
||
|
||
test("入口：只有任务与基线；看不到别人已交的入口（4.2 闭门写入口）", (t) => {
|
||
const l = studyUpTo(t, "entryA1");
|
||
const m = material(l, "entry", "a2", "s0-entry", null, NONE);
|
||
assert.deepEqual(m.checklist, { kind: "entry", entries: { min: 2, max: 3 } });
|
||
has(m, "本轮基线 base：最终指标 0.96");
|
||
assert.deepEqual(m.readable, [{ version: "v-base", label: "本批基线", dir: "/code/v-base" }]);
|
||
lacks(m, "入口甲一");
|
||
assert.ok(!headings(m).includes("信念本"), "第零轮不放信念本（4.2：信念本不变）");
|
||
});
|
||
|
||
test("讨论：全部入口标明作者，编号按作者从 1 编", (t) => {
|
||
const l = studyUpTo(t, "discussionOpen");
|
||
const m = material(l, "discussion", "a1", "s0-d1", null, NONE);
|
||
assert.deepEqual(m.checklist, { kind: "discussion", entries: { min: 2, max: 3 } });
|
||
for (const x of ["e-a1-1：入口甲一", "e-a1-2：入口甲二", "e-a2-1：入口乙一", "e-a2-2：入口乙二"]) has(m, x);
|
||
assert.ok(headings(m).includes("入口编号对照"));
|
||
});
|
||
|
||
test("认领：整份替换后新清单接着编号，旧编号作废；至少排 min(N, 入口总数)", (t) => {
|
||
const l = studyUpTo(t, "claimOpen");
|
||
const m = material(l, "claim", "a2", "s0-claim", null, NONE);
|
||
// a1 讨论时整份换成两个新入口，接着 e-a1-2 往后编：e-a1-3、e-a1-4；a2 没换。N = 2，入口 4 个。
|
||
assert.deepEqual(m.checklist, { kind: "claim", entries: ["e-a1-3", "e-a1-4", "e-a2-1", "e-a2-2"], minChoices: 2 });
|
||
has(m, "e-a1-1（作废）：入口甲一");
|
||
has(m, "甲的讨论");
|
||
has(m, "乙的讨论");
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 第 r 轮（方法.md 4.3）
|
||
// ---------------------------------------------------------------------------
|
||
|
||
test("第 1 轮提案：你认领的入口放在最前；没有上一轮；对号清单", (t) => {
|
||
const l = studyUpTo(t, "proposalOpen");
|
||
const m = material(l, "proposal", "a1", "s1-prop", null, NONE);
|
||
assert.deepEqual(headings(m).slice(0, 2), ["你自己的", "你认领的入口"]);
|
||
has(m, "e-a1-3：入口甲三");
|
||
has(m, "从这里开始看");
|
||
has(m, "这是第 1 轮，没有上一轮");
|
||
assert.deepEqual(m.checklist, {
|
||
kind: "proposal",
|
||
maxProposals: 2,
|
||
factorLimit: 3,
|
||
continuable: [],
|
||
reusableMetrics: [],
|
||
takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers"],
|
||
});
|
||
const h = headings(m);
|
||
assert.ok(h.indexOf("你自己的") < h.indexOf("公共材料") && h.indexOf("公共材料") < h.indexOf("信念本"), "先你自己的、后公共材料");
|
||
});
|
||
|
||
test("第 2 轮提案：上一轮全部收尾、上一轮实验员的提醒、可延续的因子、已登记指标；入口标为第 1 轮的起点", (t) => {
|
||
const l = studyUpTo(t, "proposal2Open");
|
||
const m = material(l, "proposal", "a2", "s2-prop", null, NONE);
|
||
has(m, "甲的收尾");
|
||
has(m, "乙的收尾");
|
||
has(m, "guard 可能挡掉一些摆法");
|
||
has(m, "第 1 轮的起点；之后是否沿用由你定");
|
||
assert.deepEqual(m.checklist, {
|
||
kind: "proposal",
|
||
maxProposals: 2,
|
||
factorLimit: 3,
|
||
continuable: ["r1-c1-p1", "r1-c1-p2", "r1-c2-p1"],
|
||
reusableMetrics: ["restart-count"],
|
||
takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers", "restart-count"],
|
||
});
|
||
for (const x of ["上一轮的收尾", "上一轮实验员实现时的提醒", '可以写进"延续"的因子', "已登记指标清单", "研究记录", "信念本"]) {
|
||
assert.ok(headings(m).includes(x), `应有一节"${x}"`);
|
||
}
|
||
});
|
||
|
||
test("第一次预测：给这个候选的原始提案；看不到别人刚交的第一次预测，也看不到作者自评", (t) => {
|
||
const l = studyUpTo(t, "firstA1");
|
||
const m = material(l, "firstPrediction", "a2", "s1-fp", "r1-c1", NONE);
|
||
assert.deepEqual(m.checklist, { kind: "firstPrediction", candidate: "r1-c1" });
|
||
has(m, "甲的提案正文");
|
||
lacks(m, "5.551");
|
||
lacks(m, "甲第一次理由");
|
||
lacks(m, "7.772");
|
||
for (const x of ["研究记录", "信念本"]) assert.ok(headings(m).includes(x));
|
||
});
|
||
|
||
test("挑刺：按原提案挑，对号清单是那条提案和它的因子", (t) => {
|
||
const l = studyUpTo(t, "critiqueOpen");
|
||
const m = material(l, "critique", "a2", "s1-crit", "r1-a1-1", NONE);
|
||
assert.deepEqual(m.checklist, { kind: "critique", proposal: "r1-a1-1", factors: ["p1", "p2"] });
|
||
has(m, "甲的提案正文");
|
||
lacks(m, "5.551");
|
||
lacks(m, "6.661");
|
||
});
|
||
|
||
test("挑刺的现在交什么：可以交一份或几份，每份调用一次；不写调用一次就完成", (t) => {
|
||
const m = material(studyUpTo(t, "critiqueOpen"), "critique", "a2", "s1-crit", "r1-a1-1", NONE);
|
||
const end = m.firstMessage.slice(m.firstMessage.lastIndexOf("# 现在交什么"));
|
||
assert.ok(end.includes("每调用一次 research_submit 交一份挑刺；可以交一份或几份"), end);
|
||
assert.ok(!end.includes("交上就完成了"), end);
|
||
});
|
||
|
||
test("回应：本人原提案和针对它的全部挑刺", (t) => {
|
||
const l = studyUpTo(t, "responseOpen");
|
||
const m = material(l, "response", "a1", "s1-resp", "r1-c1", NONE);
|
||
has(m, "乙的挑刺：扰动可能混杂（推测）");
|
||
assert.deepEqual(m.checklist, {
|
||
kind: "response",
|
||
candidate: "r1-c1",
|
||
critiques: [CRITIQUE_ID],
|
||
factorLimit: 3,
|
||
continuable: [],
|
||
reusableMetrics: [],
|
||
takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers"],
|
||
});
|
||
});
|
||
|
||
test("统一：全部被合并的原提案、针对它们的挑刺、要逐条体现的原提案", (t) => {
|
||
const l = studyUpTo(t, "responseOpen");
|
||
const m = material(l, "unification", "a1", "s1-resp", "r1-c1", NONE);
|
||
const c = m.checklist as Extract<TaskMaterial["checklist"], { kind: "unification" }>;
|
||
assert.deepEqual([c.sources, c.critiques], [["r1-a1-1"], [CRITIQUE_ID]]);
|
||
has(m, "甲的提案正文");
|
||
});
|
||
|
||
test("第二次预测：冻结协议、挑刺与答复配对、本人第一次的三个数；看不到别人的第一次和作者自评（4.3 ⑦）", (t) => {
|
||
const l = studyUpTo(t, "secondOpen");
|
||
const m = material(l, "secondPrediction", "a2", "s1-sp", "r1-c1", NONE);
|
||
assert.deepEqual(m.checklist, { kind: "secondPrediction", candidate: "r1-c1" });
|
||
has(m, "6.661 / 6.662 / 6.663");
|
||
has(m, "乙的挑刺：扰动可能混杂（推测）");
|
||
has(m, "作者的答复：甲答复：保留扰动");
|
||
has(m, "### 答复正文");
|
||
lacks(m, "5.551");
|
||
lacks(m, "甲第一次理由");
|
||
lacks(m, "7.772");
|
||
});
|
||
|
||
test("新登记指标不能用的编号里有答案路径、保障动手次数两个保留键（用户 2026-10-03 定）", (t) => {
|
||
const c = material(studyUpTo(t, "proposalOpen"), "proposal", "a1", "s1-prop", null, NONE).checklist as Extract<TaskMaterial["checklist"], { kind: "proposal" }>;
|
||
assert.ok(c.takenMetricIds.includes("answer-path") && c.takenMetricIds.includes("safeguard-triggers"), JSON.stringify(c.takenMetricIds));
|
||
});
|
||
|
||
test("解释：本轮运行、提醒、要给三个数的格、产物只进查询；看不到别人的第一次预测", (t) => {
|
||
const l = studyUpTo(t, "explOpen1");
|
||
const m = material(l, "explanation", "a2", "s1-expl", null, { changes: null, artifacts: { "run-1": { "stdout.log": "日志正文" } } });
|
||
assert.deepEqual(m.checklist, { kind: "explanation", cells: ["p1", "p2"], maxExplanations: 2 });
|
||
has(m, "分数 1.2");
|
||
// 正向运行的答案出自保障：换一句；方法原句只用在消融格（见收尾的测试）。
|
||
has(m, "这次答案出自保障 guard，分数可能主要是这条保障的效果。");
|
||
lacks(m, "本格答案出自保障");
|
||
has(m, "guard 可能挡掉一些摆法");
|
||
assert.ok(!m.firstMessage.includes("日志正文"), "产物不进第一条消息");
|
||
assert.equal(queryMaterial(m, "run-1/stdout.log"), "运行 run-1 的产物 stdout.log\n\n日志正文");
|
||
lacks(m, "5.551");
|
||
lacks(m, "8.882");
|
||
});
|
||
|
||
test("收尾：本轮采纳、选格、押的因子", (t) => {
|
||
const l = studyUpTo(t, "wrapOpen");
|
||
const m = material(l, "wrapUp", "a1", "s1-wrap", null, NONE);
|
||
assert.deepEqual(m.checklist, { kind: "wrapUp" });
|
||
has(m, "本轮采纳候选 r1-c1");
|
||
has(m, "本轮选格：测 p1");
|
||
has(m, "程序按方向收益押的因子：r1-c1-p1");
|
||
has(m, "r1-c1-p2 停滞扰动（停滞扰动的设定）：候选 r1-c1，基线 base；没测：剩余次数不够");
|
||
has(m, "本格答案出自保障 guard，量到的可能是这条保障的效果，而不是关掉这个因子的效果；保障动手次数");
|
||
lacks(m, "7.772");
|
||
});
|
||
|
||
test("实现：冻结协议、指标定义、要交的位置表、起点代码；没有信念本、成绩和别的候选", (t) => {
|
||
const l = studyUpTo(t, "implOpen");
|
||
const m = material(l, "implementation", "x1", "s1-impl-c1", "r1-c1", NONE);
|
||
assert.deepEqual(m.checklist, { kind: "implementation", candidate: "r1-c1", clauses: ["c1", "c2"], factors: ["p1", "p2"], safeguards: ["guard"] });
|
||
assert.deepEqual(m.readable.map((r) => r.version), ["v-base"]);
|
||
for (const x of ["任务与本轮", "基线代码", "本任务要处理的东西", "要实现的指标定义", "要交的因子位置表"]) assert.ok(headings(m).includes(x));
|
||
assert.ok(!headings(m).includes("信念本"));
|
||
has(m, "restart-count");
|
||
lacks(m, "乙的提案正文");
|
||
lacks(m, "收缩半径修复");
|
||
lacks(m, "6.661");
|
||
});
|
||
|
||
test("修复：退回意见、上一次的位置表、上一次的改动", (t) => {
|
||
const l = studyUpTo(t, "verifyOpen");
|
||
l.append("submission", {
|
||
stageId: "s1-ver-c1",
|
||
author: "v1",
|
||
kind: "verification",
|
||
target: "r1-c1",
|
||
item: null,
|
||
version: 1,
|
||
jev: null,
|
||
content: {
|
||
clauses: [{ clause: "c1", found: true, where: "solve.py:10" }, { clause: "c2", found: true, where: "solve.py:30" }],
|
||
factors: [{ factor: "p1", found: true, where: "solve.py:10" }, { factor: "p2", found: false, where: "看了 solve.py 全文" }],
|
||
safeguards: [{ safeguard: "guard", found: true, where: "solve.py:40" }],
|
||
unassigned: [],
|
||
body: "p2 没找到",
|
||
},
|
||
});
|
||
l.append("stageClose", { stageId: "s1-ver-c1" });
|
||
l.append("stageOpen", { stageId: "s1-repair-c1", stage: "implementation", round: 1, roster: ["x1"] });
|
||
const m = material(l, "repair", "x1", "s1-repair-c1", "r1-c1", { changes: CHANGES_C1, artifacts: null });
|
||
assert.deepEqual(m.checklist, { kind: "repair", candidate: "r1-c1", clauses: ["c1", "c2"], factors: ["p1", "p2"], safeguards: ["guard"] });
|
||
has(m, "因子 p2 没落实：看了 solve.py 全文");
|
||
has(m, "+多起点循环");
|
||
assert.deepEqual(m.readable.map((r) => r.version), ["v-base", "v-c1"]);
|
||
});
|
||
|
||
test("核验：没有归属的改动排第一；因子与保障同行只作提醒；对号清单带没有归属的改动", (t) => {
|
||
const l = studyUpTo(t, "verifyOpen");
|
||
const m = material(l, "verification", "v1", "s1-ver-c1", "r1-c1", { changes: CHANGES_C1, artifacts: null });
|
||
assert.deepEqual(headings(m).slice(0, 2), ["分给你的材料", "没归到任何因子或保障的改动"]);
|
||
// 改动是候选第 10–22 行，位置表 p1 盖住 10–20：21–22 没有归属。p2 的 30–40 与 guard 的 40–45 在第 40 行重叠。
|
||
assert.deepEqual(m.checklist, {
|
||
kind: "verification",
|
||
unassigned: ["solve.py:21-22"],
|
||
candidate: "r1-c1",
|
||
clauses: ["c1", "c2"],
|
||
factors: ["p1", "p2"],
|
||
safeguards: ["guard"],
|
||
});
|
||
has(m, "solve.py:40：因子 p2、保障 guard");
|
||
assert.deepEqual(m.readable.map((r) => r.version), ["v-base", "v-c1"]);
|
||
assert.ok(!headings(m).includes("信念本"));
|
||
});
|
||
|
||
test("核验材料里没有 Jev 的概率和阈值：收下实现时用过 Jev，也不给核验员看", (t) => {
|
||
const l = studyUpTo(t, "implOpen");
|
||
const jev = [{ question: "early_exit", questionHash: "ab".repeat(32), threshold: 0.6312, probability: 0.4321, yes: false, requestId: "req-1" }];
|
||
l.append("submission", { stageId: "s1-impl-c1", author: "x1", kind: "implementation", target: "r1-c1", item: null, version: 1, jev, content: { ...TABLE_C1, concerns: "无", body: "实现说明" } });
|
||
l.append("stageClose", { stageId: "s1-impl-c1" });
|
||
l.append("code", { object: "r1-c1", version: "v-c1", locations: TABLE_C1 });
|
||
l.append("stageOpen", { stageId: "s1-ver-c1", stage: "verification", round: 1, roster: ["v1"] });
|
||
const m = material(l, "verification", "v1", "s1-ver-c1", "r1-c1", { changes: CHANGES_C1, artifacts: null });
|
||
has(m, "实现说明");
|
||
for (const x of ["0.4321", "0.6312", "early_exit", "预扫"]) lacks(m, x);
|
||
});
|
||
|
||
test("消融实现重做：材料放上一次消融核验关错的因子和依据；第一次做时没有退回意见", (t) => {
|
||
assert.ok(!everything(material(studyUpTo(t, "ablationImplOpen"), "ablationImplementation", "x1", "s1-ai", "r1-c1-p1", NONE)).includes("### 退回意见"));
|
||
const l = studyUpTo(t, "ablationImpl");
|
||
// 这一格要关 p1、照留 p2；核验员看到 p1 还开着、p2 照留：只有 p1 关错了。
|
||
l.append("submission", {
|
||
stageId: "s1-av", author: "v1", kind: "ablationVerification", target: "r1-c1-p1", item: null, version: 1, jev: null,
|
||
content: { factors: [{ factor: "p1", off: false, where: "solve.py:12 仍在跑多起点循环" }, { factor: "p2", off: false, where: "solve.py:30 照留" }] },
|
||
});
|
||
l.append("stageClose", { stageId: "s1-av" });
|
||
l.append("stageOpen", { stageId: "s1-ai2", stage: "ablationImplementation", round: 1, roster: ["x1"] });
|
||
const m = material(l, "ablationImplementation", "x1", "s1-ai2", "r1-c1-p1", NONE);
|
||
has(m, "### 退回意见");
|
||
has(m, "- 因子 p1 应当关着，核验员看到的是照留：solve.py:12 仍在跑多起点循环");
|
||
lacks(m, "solve.py:30 照留");
|
||
assert.deepEqual(m.checklist, { kind: "ablationImplementation", cell: "r1-c1-p1", off: "p1", factors: ["p1", "p2"] });
|
||
});
|
||
|
||
test("消融实现：这一格、因子清单、完整代码的位置表；看不到运行成绩", (t) => {
|
||
const l = studyUpTo(t, "ablationImplOpen");
|
||
const m = material(l, "ablationImplementation", "x1", "s1-ai", "r1-c1-p1", NONE);
|
||
assert.deepEqual(m.checklist, { kind: "ablationImplementation", cell: "r1-c1-p1", off: "p1", factors: ["p1", "p2"] });
|
||
assert.ok(headings(m).includes("完整代码（消融的起点）"));
|
||
assert.deepEqual(m.readable.map((r) => r.version), ["v-c1", "v-base"]);
|
||
lacks(m, "1.2");
|
||
lacks(m, "甲的解释");
|
||
});
|
||
|
||
test("消融核验：实际要关的因子、改动全文", (t) => {
|
||
const l = studyUpTo(t, "ablationImpl");
|
||
const m = material(l, "ablationVerification", "v1", "s1-av", "r1-c1-p1", { changes: CHANGES_C1, artifacts: null });
|
||
assert.deepEqual(m.checklist, { kind: "ablationVerification", cell: "r1-c1-p1", factors: ["p1", "p2"] });
|
||
has(m, "p1（只关指定的因子）");
|
||
assert.deepEqual(m.readable.map((r) => r.version), ["v-c1", "v-base", "v-c1-p1"]);
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 七条可见性规则（任务书 09 第二部分），每条一个"不该看到"
|
||
// ---------------------------------------------------------------------------
|
||
|
||
test("可见性 1：同一阶段没关之前，别人看不到已交的提交", (t) => {
|
||
const l = studyUpTo(t, "proposalA1");
|
||
const m = material(l, "proposal", "a2", "s1-prop", null, NONE);
|
||
lacks(m, "甲的提案正文");
|
||
lacks(m, "多起点重启");
|
||
// 本人自己当前阶段已交的看得到
|
||
has(material(l, "proposal", "a1", "s1-prop", null, NONE), "甲的提案正文");
|
||
});
|
||
|
||
test("可见性 2：作者自评到本轮结束解封前，任何任务的材料里都没有", (t) => {
|
||
const cases: Array<[string, SubmissionKind, string, string, string | null]> = [
|
||
["firstOpen", "firstPrediction", "a1", "s1-fp", "r1-c1"],
|
||
["critiqueOpen", "critique", "a2", "s1-crit", "r1-a1-1"],
|
||
["responseOpen", "response", "a1", "s1-resp", "r1-c1"],
|
||
["secondOpen", "secondPrediction", "a1", "s1-sp", "r1-c1"],
|
||
["implOpen", "implementation", "x1", "s1-impl-c1", "r1-c1"],
|
||
["wrapOpen", "wrapUp", "a1", "s1-wrap", null],
|
||
];
|
||
for (const [step, kind, actor, stage, object] of cases) {
|
||
const l = studyUpTo(t, step);
|
||
const m = material(l, kind, actor, stage, object, NONE);
|
||
for (const sealed of ["7.771", "7.772", "8.881", "8.882"]) lacks(m, sealed);
|
||
}
|
||
// 解封以后（跑过的 r1-c1）进战绩；没跑的 r1-c2 继续封存。
|
||
const after = beliefBook(studyUpTo(t, "proposal2Open"), study, 2);
|
||
assert.ok(after.includes("作者自评 vs 大家合并：r1-c1（作者 a1）"));
|
||
assert.ok(!after.includes("8.882"));
|
||
});
|
||
|
||
test("可见性 3：第一次预测只有本人看得到，之后各阶段也不给别人", (t) => {
|
||
const l = studyUpTo(t, "wrapOpen");
|
||
for (const m of [material(l, "wrapUp", "a2", "s1-wrap", null, NONE), material(l, "wrapUp", "a1", "s1-wrap", null, NONE)]) {
|
||
lacks(m, "5.551");
|
||
lacks(m, "6.661");
|
||
}
|
||
});
|
||
|
||
test("可见性 4：第二次预测看不到别人第一次的数", (t) => {
|
||
const l = studyUpTo(t, "secondOpen");
|
||
const m = material(l, "secondPrediction", "a1", "s1-sp", "r1-c1", NONE);
|
||
has(m, "5.551 / 5.552 / 5.553");
|
||
lacks(m, "6.661");
|
||
lacks(m, "乙第一次理由");
|
||
});
|
||
|
||
test("可见性 5：第零轮闭门写入口时看不到别人的入口", (t) => {
|
||
const l = studyUpTo(t, "entryA2");
|
||
const m = material(l, "entry", "a1", "s0-entry", null, NONE);
|
||
lacks(m, "入口乙一");
|
||
has(m, "入口甲一");
|
||
});
|
||
|
||
test("可见性 6：别人的退回不放进材料；本人重新发起时放上一次被退回的原因", (t) => {
|
||
const l = studyUpTo(t, "secondOpen");
|
||
const reject = (author: string, target: string, reason: string) =>
|
||
l.append("rejection", { stageId: "s1-sp", author, kind: "secondPrediction", target, submitted: { range: { q1: 3, q5: 2, q9: 1 } }, reason, jev: null });
|
||
reject("a1", "r1-c1", "甲被退回：三个数不递增");
|
||
reject("a2", "r1-c1", "乙被退回");
|
||
const retry = material(l, "secondPrediction", "a1", "s1-sp", "r1-c1", NONE);
|
||
assert.ok(retry.firstMessage.includes("# 上一次为什么没交成"));
|
||
has(retry, "甲被退回：三个数不递增");
|
||
lacks(retry, "乙被退回");
|
||
lacks(material(l, "secondPrediction", "a2", "s1-sp", "r1-c2", NONE), "甲被退回");
|
||
});
|
||
|
||
test("退回按对象找：本人同一阶段对两个候选各被退回一次，重新发起时只放这个对象的那一条", (t) => {
|
||
// 两个候选的第二次预测并行，退回交错写入；不靠模型调用的记录定位。
|
||
const l = studyUpTo(t, "secondOpen");
|
||
const reject = (target: string, reason: string) =>
|
||
l.append("rejection", { stageId: "s1-sp", author: "a1", kind: "secondPrediction", target, submitted: {}, reason, jev: null });
|
||
reject("r1-c1", "c1 第一次被退回");
|
||
reject("r1-c2", "c2 被退回");
|
||
reject("r1-c1", "c1 第二次被退回");
|
||
const c1 = material(l, "secondPrediction", "a1", "s1-sp", "r1-c1", NONE);
|
||
has(c1, "c1 第二次被退回");
|
||
lacks(c1, "c1 第一次被退回");
|
||
lacks(c1, "c2 被退回");
|
||
const c2 = material(l, "secondPrediction", "a1", "s1-sp", "r1-c2", NONE);
|
||
has(c2, "c2 被退回");
|
||
lacks(c2, "c1 第");
|
||
});
|
||
|
||
test("可见性 7：信念本只写已公开的记录：正向运行在采纳决定写入后才出现", (t) => {
|
||
const before = beliefBook(studyUpTo(t, "run1"), study, 1);
|
||
assert.ok(!before.includes("run-1"));
|
||
const after = beliefBook(studyUpTo(t, "adopt1"), study, 1);
|
||
assert.ok(after.includes("运行 run-1：算数（校验器判合法），分数 1.2"));
|
||
});
|
||
|
||
test("实验员、核验员看不到别的候选", (t) => {
|
||
const l = studyUpTo(t, "verifyOpen");
|
||
for (const m of [
|
||
material(l, "verification", "v1", "s1-ver-c1", "r1-c1", { changes: CHANGES_C1, artifacts: null }),
|
||
material(studyUpTo(t, "implOpen"), "implementation", "x1", "s1-impl-c1", "r1-c1", NONE),
|
||
]) {
|
||
lacks(m, "r1-c2");
|
||
lacks(m, "乙的");
|
||
assert.equal(queryMaterial(m, "r1-c2"), null);
|
||
}
|
||
});
|
||
|
||
test("没有归属的改动：手写两个版本和位置表，算出的清单与手算的一致", (t) => {
|
||
const l = studyUpTo(t, "verifyOpen");
|
||
// 位置表：p1 a.py:1-5；p2 a.py:20 和整个 b.py；保障 guard a.py:5-6。
|
||
l.append("code", {
|
||
object: "r1-c1",
|
||
version: "v-c1",
|
||
locations: {
|
||
factorLocations: [{ factor: "p1", locations: ["a.py:1-5"] }, { factor: "p2", locations: ["a.py:20", "b.py"] }],
|
||
safeguardLocations: [{ safeguard: "guard", locations: ["a.py:5-6"] }],
|
||
unassigned: [{ location: "a.py:7", note: "实验员说是 import" }],
|
||
},
|
||
});
|
||
// 旧版本 → 新版本：a.py 新版本第 1–8 行是改过的；删了旧第 11–12 行（新版本第 10 行之后）；删了旧第 21 行（新版本第 19 行之后）。
|
||
// b.py 改了第 3–4 行；c.bin 新增二进制；run.sh 只改可执行位；empty.txt 新增空文件。
|
||
const changes = {
|
||
files: [
|
||
{ path: "a.py", change: "modified" as const, modeOnly: false, binary: false as const, hunks: [
|
||
{ from: { start: 1, end: 3 }, to: { start: 1, end: 8 } },
|
||
{ from: { start: 11, end: 12 }, to: { after: 10 } },
|
||
{ from: { start: 21, end: 21 }, to: { after: 19 } },
|
||
] },
|
||
{ path: "b.py", change: "modified" as const, modeOnly: false, binary: false as const, hunks: [{ from: { start: 3, end: 4 }, to: { start: 3, end: 4 } }] },
|
||
{ path: "c.bin", change: "added" as const, modeOnly: false, binary: true as const },
|
||
{ path: "empty.txt", change: "added" as const, modeOnly: false, binary: false as const, hunks: [] },
|
||
{ path: "run.sh", change: "modified" as const, modeOnly: true, binary: false as const, hunks: [] },
|
||
],
|
||
diff: "",
|
||
};
|
||
const m = material(l, "verification", "v1", "s1-ver-c1", "r1-c1", { changes, artifacts: null });
|
||
// 手算：a.py 第 6 行归 guard，7–8 行没有归属（实验员自己的说明不算归属）；删除点 10 前后（10、11）都没被盖住；
|
||
// 删除点 19 后一行 20 归 p2；b.py 整个文件归 p2；其余三个文件按整个文件算，位置表都没写到。
|
||
const c = m.checklist as Extract<TaskMaterial["checklist"], { kind: "verification" }>;
|
||
assert.deepEqual(c.unassigned, ["a.py:7-8", "a.py:10", "c.bin", "empty.txt", "run.sh"]);
|
||
has(m, "- a.py:10（删除 2 行（旧版本第 11–12 行））");
|
||
has(m, "- c.bin（新增（二进制文件，按整个文件算））");
|
||
has(m, "- run.sh（修改（只改了可执行位，按整个文件算））");
|
||
has(m, "- a.py:7：实验员说是 import");
|
||
// 因子与保障同一行：p1 的 1–5 与 guard 的 5–6 在第 5 行。
|
||
has(m, "- a.py:5：因子 p1、保障 guard");
|
||
lacks(m, "因子 p2、保障 guard");
|
||
});
|
||
|
||
test("答案路径不在位置表里、读不到，都写明状态", (t) => {
|
||
const l = studyUpTo(t, "run1");
|
||
l.append("runPlanned", { run: "run-x", purpose: "forward", round: 1, object: "r1-c2", definition: { command: ["python"], workdir: "/w", image: { digest: `sha256:${"0".repeat(64)}`, runtime: "runc" }, wallSeconds: 120, cores: 1, inheritEnv: false, env: {} } });
|
||
l.append("runStarted", { run: "run-x" });
|
||
l.append("runResult", {
|
||
run: "run-x",
|
||
process: { ending: "exited", exitCode: 0, cpuSeconds: { status: "measured", value: 1 }, wallSeconds: { status: "measured", value: 1 }, peakMemoryBytes: { status: "measured", value: 1 } },
|
||
judged: {
|
||
verdict: "valid", verdictReason: "合法", score: { status: "measured", value: 1.1 }, metrics: {}, extra: null,
|
||
answerPath: { status: "read", value: "p9" },
|
||
safeguardTriggers: { status: "unreadable", reason: "没写 safeguard-triggers" },
|
||
},
|
||
});
|
||
l.append("code", { object: "r1-c2", version: "v-c2", locations: { factorLocations: [{ factor: "p1", locations: ["x.py"] }], safeguardLocations: [], unassigned: [] } });
|
||
const input = { baseline: 0.96, knownBest: 2.635983, epsilon: 0.01, direction: "maximize" as const, candidates: [{ id: "r1-c1", score: 1.2 }] };
|
||
l.append("decision", { decision: "adopt", round: 1, rule: "test", ruleVersion: 1, params: null, seed: null, input: input as never, output: { line: 0.01, quarter: 0.4, best: { id: "r1-c1", score: 1.2, improvement: 0.24 }, adopted: "r1-c1" } });
|
||
l.append("stageOpen", { stageId: "s1-expl", stage: "explanation", round: 1, roster: ["a1", "a2"] });
|
||
const m = material(l, "explanation", "a1", "s1-expl", null, { changes: null, artifacts: {} });
|
||
const run = queryMaterial(m, "run-x")!;
|
||
assert.ok(run.includes("答案路径写的是 p9，不在位置表里"));
|
||
assert.ok(run.includes("保障动手次数读不到：没写 safeguard-triggers"));
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 查询、提示词
|
||
// ---------------------------------------------------------------------------
|
||
|
||
test("查询：材料里的编号查得到；同一账本里材料外的编号查不到", (t) => {
|
||
const l = studyUpTo(t, "secondOpen");
|
||
const m = material(l, "secondPrediction", "a2", "s1-sp", "r1-c1", NONE);
|
||
assert.ok(queryMaterial(m, "r1-a1-1")!.includes("甲的提案正文"));
|
||
assert.ok(queryMaterial(m, "task")!.includes("填圆任务说明"));
|
||
assert.equal(queryMaterial(m, "s1-sp"), null);
|
||
assert.equal(queryMaterial(m, "run-0"), null);
|
||
assert.equal(queryMaterial(m, "不存在"), null);
|
||
const impl = material(studyUpTo(t, "implOpen"), "implementation", "x1", "s1-impl-c1", "r1-c1", NONE);
|
||
assert.equal(queryMaterial(impl, "r1-a1-1"), null);
|
||
});
|
||
|
||
test("提示词：每种提交种类都有任务说明文件，每个角色都有角色提示词", () => {
|
||
const p = readPrompts(join(import.meta.dirname, "..", "prompts"));
|
||
assert.deepEqual(Object.keys(p.tasks).sort(), Object.keys(SUBMISSION_KINDS).sort());
|
||
for (const f of [...Object.values(p.tasks), ...Object.values(p.roles)]) assert.match(f.text, /\S/);
|
||
});
|
||
|
||
test("提示词哈希是文件原始字节的 sha256", (t) => {
|
||
const dir = tempDir(t);
|
||
mkdirSync(join(dir, "roles"));
|
||
mkdirSync(join(dir, "tasks"));
|
||
const bytes = Buffer.from("x\r\n中", "utf8");
|
||
for (const r of ["researcher", "experimenter", "verifier"]) writeFileSync(join(dir, "roles", `${r}.md`), bytes);
|
||
for (const k of Object.keys(SUBMISSION_KINDS)) writeFileSync(join(dir, "tasks", `${k}.md`), bytes);
|
||
const p = readPrompts(dir);
|
||
// printf 'x\r\n中' | sha256sum
|
||
assert.equal(p.roles.researcher.hash, "965da20bc965975b4eae297a49cabf4be539ef62bec455fe932093535de2c593");
|
||
assert.equal(p.tasks.wrapUp.hash, "965da20bc965975b4eae297a49cabf4be539ef62bec455fe932093535de2c593");
|
||
assert.equal(p.tasks.wrapUp.text, "x\r\n中");
|
||
});
|
||
|
||
test("第一条消息：任务说明文件在最前，现在交什么在最后并列出提交字段", (t) => {
|
||
const l = studyUpTo(t, "proposalOpen");
|
||
const m = material(l, "proposal", "a1", "s1-prop", null, NONE);
|
||
const p = readPrompts(join(import.meta.dirname, "..", "prompts"));
|
||
assert.ok(m.firstMessage.startsWith(p.tasks.proposal.text.trimEnd()));
|
||
const end = m.firstMessage.slice(m.firstMessage.lastIndexOf("# 现在交什么"));
|
||
for (const f of ["`factors`", "`safeguards`", "`metrics`", "`authorEstimate`", "`body`", "最多交 2 条"]) assert.ok(end.includes(f), f);
|
||
});
|