feat: 任务书 09，materials 模块与提示词
- src/materials：每种提交一个材料函数，生成给模型看的材料、第一条消息、对号清单、能读的代码版本；查询只在材料内按编号找；读提示词并算原始字节哈希；信念本按方法.md 6.10 现算。 - 账本里只存成 Json 的冻结协议、位置表、决定、结算的形状和编号规则由 materials 定义并导出。 - vocab 加 Checklist（对号清单）。 - prompts/：参考项目 3 份角色提示词、16 份任务说明，只做必要改动。 - TASK.md 补一句 unversionedNames 不存进代码版本。 - 方法.md 4.3 ① 改为"研究者任务"放信念本；结构.md vocab 上限 1150、补 materials 对外内容。 用户 2026-10-03 开工前的四个决定（选的都是推荐项）："甲：上限改 1150，只放对号清单"；"甲：materials 定义并导出"；"不放，改方法.md ① 的措辞"；"按原提案挑"。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,446 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { beliefBook, callId, queryMaterial, readPrompts, type TaskMaterial } from "../src/materials/index.ts";
|
||||
import { SUBMISSION_KINDS, type SubmissionKind } from "../src/vocab/index.ts";
|
||||
import { tempDir } from "./ledger-helpers.ts";
|
||||
import { CHANGES_C1, CRITIQUE_ID, everything, material, NONE, study, studyUpTo } from "./materials-helpers.ts";
|
||||
|
||||
// 每种任务至少一个测试：在真实账本里写到某一步，生成材料，查该有的有、不该有的没有、对号清单对。
|
||||
// 预期值按方法.md 条文手写（研究进程见 materials-helpers.ts 的注释），不从实现推。
|
||||
|
||||
const headings = (m: TaskMaterial) => m.sections.map((s) => s.heading);
|
||||
const has = (m: TaskMaterial, text: string) => assert.ok(everything(m).includes(text), `材料里应有"${text}"`);
|
||||
const lacks = (m: TaskMaterial, text: string) => assert.ok(!everything(m).includes(text), `材料里不该有"${text}"`);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 第零轮（方法.md 4.2）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("入口：只有任务与基线；看不到别人已交的入口（4.2 闭门写入口）", (t) => {
|
||||
const l = studyUpTo(t, "entryA1");
|
||||
const m = material(l, "entry", "a2", "s0-entry", null, NONE);
|
||||
assert.deepEqual(m.checklist, { kind: "entry", entries: { min: 2, max: 3 } });
|
||||
has(m, "本轮基线 base：最终指标 0.96");
|
||||
assert.deepEqual(m.readable, [{ version: "v-base", label: "本批基线", dir: "/code/v-base" }]);
|
||||
lacks(m, "入口甲一");
|
||||
assert.ok(!headings(m).includes("信念本"), "第零轮不放信念本（4.2：信念本不变）");
|
||||
});
|
||||
|
||||
test("讨论：全部入口标明作者，编号按作者从 1 编", (t) => {
|
||||
const l = studyUpTo(t, "discussionOpen");
|
||||
const m = material(l, "discussion", "a1", "s0-d1", null, NONE);
|
||||
assert.deepEqual(m.checklist, { kind: "discussion", entries: { min: 2, max: 3 } });
|
||||
for (const x of ["e-a1-1：入口甲一", "e-a1-2：入口甲二", "e-a2-1：入口乙一", "e-a2-2：入口乙二"]) has(m, x);
|
||||
assert.ok(headings(m).includes("入口编号对照"));
|
||||
});
|
||||
|
||||
test("认领：整份替换后新清单接着编号，旧编号作废；至少排 min(N, 入口总数)", (t) => {
|
||||
const l = studyUpTo(t, "claimOpen");
|
||||
const m = material(l, "claim", "a2", "s0-claim", null, NONE);
|
||||
// a1 讨论时整份换成两个新入口，接着 e-a1-2 往后编：e-a1-3、e-a1-4；a2 没换。N = 2，入口 4 个。
|
||||
assert.deepEqual(m.checklist, { kind: "claim", entries: ["e-a1-3", "e-a1-4", "e-a2-1", "e-a2-2"], minChoices: 2 });
|
||||
has(m, "e-a1-1（作废）：入口甲一");
|
||||
has(m, "甲的讨论");
|
||||
has(m, "乙的讨论");
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 第 r 轮（方法.md 4.3）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("第 1 轮提案：你认领的入口放在最前；没有上一轮；对号清单", (t) => {
|
||||
const l = studyUpTo(t, "proposalOpen");
|
||||
const m = material(l, "proposal", "a1", "s1-prop", null, NONE);
|
||||
assert.deepEqual(headings(m).slice(0, 2), ["你自己的", "你认领的入口"]);
|
||||
has(m, "e-a1-3：入口甲三");
|
||||
has(m, "从这里开始看");
|
||||
has(m, "这是第 1 轮，没有上一轮");
|
||||
assert.deepEqual(m.checklist, {
|
||||
kind: "proposal",
|
||||
maxProposals: 2,
|
||||
factorLimit: 3,
|
||||
continuable: [],
|
||||
reusableMetrics: [],
|
||||
takenMetricIds: ["sum-of-radii", "wall-seconds"],
|
||||
});
|
||||
const h = headings(m);
|
||||
assert.ok(h.indexOf("你自己的") < h.indexOf("公共材料") && h.indexOf("公共材料") < h.indexOf("信念本"), "先你自己的、后公共材料");
|
||||
});
|
||||
|
||||
test("第 2 轮提案：上一轮全部收尾、上一轮实验员的提醒、可延续的因子、已登记指标；入口标为第 1 轮的起点", (t) => {
|
||||
const l = studyUpTo(t, "proposal2Open");
|
||||
const m = material(l, "proposal", "a2", "s2-prop", null, NONE);
|
||||
has(m, "甲的收尾");
|
||||
has(m, "乙的收尾");
|
||||
has(m, "guard 可能挡掉一些摆法");
|
||||
has(m, "第 1 轮的起点；之后是否沿用由你定");
|
||||
assert.deepEqual(m.checklist, {
|
||||
kind: "proposal",
|
||||
maxProposals: 2,
|
||||
factorLimit: 3,
|
||||
continuable: ["r1-c1-p1", "r1-c1-p2", "r1-c2-p1"],
|
||||
reusableMetrics: ["restart-count"],
|
||||
takenMetricIds: ["sum-of-radii", "wall-seconds", "restart-count"],
|
||||
});
|
||||
for (const x of ["上一轮的收尾", "上一轮实验员实现时的提醒", '可以写进"延续"的因子', "已登记指标清单", "研究记录", "信念本"]) {
|
||||
assert.ok(headings(m).includes(x), `应有一节"${x}"`);
|
||||
}
|
||||
});
|
||||
|
||||
test("第一次预测：给这个候选的原始提案；看不到别人刚交的第一次预测，也看不到作者自评", (t) => {
|
||||
const l = studyUpTo(t, "firstA1");
|
||||
const m = material(l, "firstPrediction", "a2", "s1-fp", "r1-c1", NONE);
|
||||
assert.deepEqual(m.checklist, { kind: "firstPrediction", candidate: "r1-c1" });
|
||||
has(m, "甲的提案正文");
|
||||
lacks(m, "5.551");
|
||||
lacks(m, "甲第一次理由");
|
||||
lacks(m, "7.772");
|
||||
for (const x of ["研究记录", "信念本"]) assert.ok(headings(m).includes(x));
|
||||
});
|
||||
|
||||
test("挑刺：按原提案挑，对号清单是那条提案和它的因子", (t) => {
|
||||
const l = studyUpTo(t, "critiqueOpen");
|
||||
const m = material(l, "critique", "a2", "s1-crit", "r1-a1-1", NONE);
|
||||
assert.deepEqual(m.checklist, { kind: "critique", proposal: "r1-a1-1", factors: ["p1", "p2"] });
|
||||
has(m, "甲的提案正文");
|
||||
lacks(m, "5.551");
|
||||
lacks(m, "6.661");
|
||||
});
|
||||
|
||||
test("回应：本人原提案和针对它的全部挑刺", (t) => {
|
||||
const l = studyUpTo(t, "responseOpen");
|
||||
const m = material(l, "response", "a1", "s1-resp", "r1-c1", NONE);
|
||||
has(m, "乙的挑刺：扰动可能混杂（推测）");
|
||||
assert.deepEqual(m.checklist, {
|
||||
kind: "response",
|
||||
candidate: "r1-c1",
|
||||
critiques: [CRITIQUE_ID],
|
||||
factorLimit: 3,
|
||||
continuable: [],
|
||||
reusableMetrics: [],
|
||||
takenMetricIds: ["sum-of-radii", "wall-seconds"],
|
||||
});
|
||||
});
|
||||
|
||||
test("统一：全部被合并的原提案、针对它们的挑刺、要逐条体现的原提案", (t) => {
|
||||
const l = studyUpTo(t, "responseOpen");
|
||||
const m = material(l, "unification", "a1", "s1-resp", "r1-c1", NONE);
|
||||
const c = m.checklist as Extract<TaskMaterial["checklist"], { kind: "unification" }>;
|
||||
assert.deepEqual([c.sources, c.critiques], [["r1-a1-1"], [CRITIQUE_ID]]);
|
||||
has(m, "甲的提案正文");
|
||||
});
|
||||
|
||||
test("第二次预测：冻结协议、挑刺与答复配对、本人第一次的三个数；看不到别人的第一次和作者自评（4.3 ⑦）", (t) => {
|
||||
const l = studyUpTo(t, "secondOpen");
|
||||
const m = material(l, "secondPrediction", "a2", "s1-sp", "r1-c1", NONE);
|
||||
assert.deepEqual(m.checklist, { kind: "secondPrediction", candidate: "r1-c1" });
|
||||
has(m, "6.661 / 6.662 / 6.663");
|
||||
has(m, "乙的挑刺：扰动可能混杂（推测）");
|
||||
has(m, "作者的答复：甲答复：保留扰动");
|
||||
lacks(m, "5.551");
|
||||
lacks(m, "甲第一次理由");
|
||||
lacks(m, "7.772");
|
||||
});
|
||||
|
||||
test("解释：本轮运行、提醒、要给三个数的格、产物只进查询；看不到别人的第一次预测", (t) => {
|
||||
const l = studyUpTo(t, "explOpen1");
|
||||
const m = material(l, "explanation", "a2", "s1-expl", null, { changes: null, artifacts: { "run-1": { "stdout.log": "日志正文" } } });
|
||||
assert.deepEqual(m.checklist, { kind: "explanation", cells: ["p1", "p2"], maxExplanations: 2 });
|
||||
has(m, "分数 1.2");
|
||||
// 正向运行的答案出自保障：换一句；方法原句只用在消融格（见收尾的测试）。
|
||||
has(m, "这次答案出自保障 guard，分数可能主要是这条保障的效果。");
|
||||
lacks(m, "本格答案出自保障");
|
||||
has(m, "guard 可能挡掉一些摆法");
|
||||
assert.ok(!m.firstMessage.includes("日志正文"), "产物不进第一条消息");
|
||||
assert.equal(queryMaterial(m, "run-1/stdout.log"), "运行 run-1 的产物 stdout.log\n\n日志正文");
|
||||
lacks(m, "5.551");
|
||||
lacks(m, "8.882");
|
||||
});
|
||||
|
||||
test("收尾：本轮采纳、选格、押的因子", (t) => {
|
||||
const l = studyUpTo(t, "wrapOpen");
|
||||
const m = material(l, "wrapUp", "a1", "s1-wrap", null, NONE);
|
||||
assert.deepEqual(m.checklist, { kind: "wrapUp" });
|
||||
has(m, "本轮采纳候选 r1-c1");
|
||||
has(m, "本轮选格：测 p1");
|
||||
has(m, "程序按方向收益押的因子：r1-c1-p1");
|
||||
has(m, "r1-c1-p2 停滞扰动（停滞扰动的设定）：候选 r1-c1，基线 base；没测：剩余次数不够");
|
||||
has(m, "本格答案出自保障 guard，量到的可能是这条保障的效果，而不是关掉这个因子的效果；保障动手次数");
|
||||
lacks(m, "7.772");
|
||||
});
|
||||
|
||||
test("实现：冻结协议、指标定义、要交的位置表、起点代码；没有信念本、成绩和别的候选", (t) => {
|
||||
const l = studyUpTo(t, "implOpen");
|
||||
const m = material(l, "implementation", "x1", "s1-impl-c1", "r1-c1", NONE);
|
||||
assert.deepEqual(m.checklist, { kind: "implementation", candidate: "r1-c1", clauses: ["c1", "c2"], factors: ["p1", "p2"], safeguards: ["guard"] });
|
||||
assert.deepEqual(m.readable.map((r) => r.version), ["v-base"]);
|
||||
for (const x of ["任务与本轮", "基线代码", "本任务要处理的东西", "要实现的指标定义", "要交的因子位置表"]) assert.ok(headings(m).includes(x));
|
||||
assert.ok(!headings(m).includes("信念本"));
|
||||
has(m, "restart-count");
|
||||
lacks(m, "乙的提案正文");
|
||||
lacks(m, "收缩半径修复");
|
||||
lacks(m, "6.661");
|
||||
});
|
||||
|
||||
test("修复：退回意见、上一次的位置表、上一次的改动", (t) => {
|
||||
const l = studyUpTo(t, "verifyOpen");
|
||||
l.append("submission", {
|
||||
stageId: "s1-ver-c1",
|
||||
author: "v1",
|
||||
kind: "verification",
|
||||
target: "r1-c1",
|
||||
item: null,
|
||||
version: 1,
|
||||
jev: null,
|
||||
content: {
|
||||
clauses: [{ clause: "c1", found: true, where: "solve.py:10" }, { clause: "c2", found: true, where: "solve.py:30" }],
|
||||
factors: [{ factor: "p1", found: true, where: "solve.py:10" }, { factor: "p2", found: false, where: "看了 solve.py 全文" }],
|
||||
safeguards: [{ safeguard: "guard", found: true, where: "solve.py:40" }],
|
||||
unassigned: [],
|
||||
body: "p2 没找到",
|
||||
},
|
||||
});
|
||||
l.append("stageClose", { stageId: "s1-ver-c1" });
|
||||
l.append("stageOpen", { stageId: "s1-repair-c1", stage: "implementation", round: 1, roster: ["x1"] });
|
||||
const m = material(l, "repair", "x1", "s1-repair-c1", "r1-c1", { changes: CHANGES_C1, artifacts: null });
|
||||
assert.deepEqual(m.checklist, { kind: "repair", candidate: "r1-c1", clauses: ["c1", "c2"], factors: ["p1", "p2"], safeguards: ["guard"] });
|
||||
has(m, "因子 p2 没落实：看了 solve.py 全文");
|
||||
has(m, "+多起点循环");
|
||||
assert.deepEqual(m.readable.map((r) => r.version), ["v-base", "v-c1"]);
|
||||
});
|
||||
|
||||
test("核验：没有归属的改动排第一；因子与保障同行只作提醒；对号清单带没有归属的改动", (t) => {
|
||||
const l = studyUpTo(t, "verifyOpen");
|
||||
const m = material(l, "verification", "v1", "s1-ver-c1", "r1-c1", { changes: CHANGES_C1, artifacts: null });
|
||||
assert.deepEqual(headings(m).slice(0, 2), ["分给你的材料", "没归到任何因子或保障的改动"]);
|
||||
// 改动是候选第 10–22 行，位置表 p1 盖住 10–20：21–22 没有归属。p2 的 30–40 与 guard 的 40–45 在第 40 行重叠。
|
||||
assert.deepEqual(m.checklist, {
|
||||
kind: "verification",
|
||||
unassigned: ["solve.py:21-22"],
|
||||
candidate: "r1-c1",
|
||||
clauses: ["c1", "c2"],
|
||||
factors: ["p1", "p2"],
|
||||
safeguards: ["guard"],
|
||||
});
|
||||
has(m, "solve.py:40：因子 p2、保障 guard");
|
||||
assert.deepEqual(m.readable.map((r) => r.version), ["v-base", "v-c1"]);
|
||||
assert.ok(!headings(m).includes("信念本"));
|
||||
});
|
||||
|
||||
test("消融实现：这一格、因子清单、完整代码的位置表；看不到运行成绩", (t) => {
|
||||
const l = studyUpTo(t, "ablationImplOpen");
|
||||
const m = material(l, "ablationImplementation", "x1", "s1-ai", "r1-c1-p1", NONE);
|
||||
assert.deepEqual(m.checklist, { kind: "ablationImplementation", cell: "r1-c1-p1", off: "p1", factors: ["p1", "p2"] });
|
||||
assert.ok(headings(m).includes("完整代码（消融的起点）"));
|
||||
assert.deepEqual(m.readable.map((r) => r.version), ["v-c1", "v-base"]);
|
||||
lacks(m, "1.2");
|
||||
lacks(m, "甲的解释");
|
||||
});
|
||||
|
||||
test("消融核验：实际要关的因子、改动全文", (t) => {
|
||||
const l = studyUpTo(t, "ablationImpl");
|
||||
const m = material(l, "ablationVerification", "v1", "s1-av", "r1-c1-p1", { changes: CHANGES_C1, artifacts: null });
|
||||
assert.deepEqual(m.checklist, { kind: "ablationVerification", cell: "r1-c1-p1", factors: ["p1", "p2"] });
|
||||
has(m, "p1（只关指定的因子）");
|
||||
assert.deepEqual(m.readable.map((r) => r.version), ["v-c1", "v-base", "v-c1-p1"]);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 七条可见性规则（任务书 09 第二部分），每条一个"不该看到"
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("可见性 1：同一阶段没关之前，别人看不到已交的提交", (t) => {
|
||||
const l = studyUpTo(t, "proposalA1");
|
||||
const m = material(l, "proposal", "a2", "s1-prop", null, NONE);
|
||||
lacks(m, "甲的提案正文");
|
||||
lacks(m, "多起点重启");
|
||||
// 本人自己当前阶段已交的看得到
|
||||
has(material(l, "proposal", "a1", "s1-prop", null, NONE), "甲的提案正文");
|
||||
});
|
||||
|
||||
test("可见性 2：作者自评到本轮结束解封前，任何任务的材料里都没有", (t) => {
|
||||
const cases: Array<[string, SubmissionKind, string, string, string | null]> = [
|
||||
["firstOpen", "firstPrediction", "a1", "s1-fp", "r1-c1"],
|
||||
["critiqueOpen", "critique", "a2", "s1-crit", "r1-a1-1"],
|
||||
["responseOpen", "response", "a1", "s1-resp", "r1-c1"],
|
||||
["secondOpen", "secondPrediction", "a1", "s1-sp", "r1-c1"],
|
||||
["implOpen", "implementation", "x1", "s1-impl-c1", "r1-c1"],
|
||||
["wrapOpen", "wrapUp", "a1", "s1-wrap", null],
|
||||
];
|
||||
for (const [step, kind, actor, stage, object] of cases) {
|
||||
const l = studyUpTo(t, step);
|
||||
const m = material(l, kind, actor, stage, object, NONE);
|
||||
for (const sealed of ["7.771", "7.772", "8.881", "8.882"]) lacks(m, sealed);
|
||||
}
|
||||
// 解封以后（跑过的 r1-c1）进战绩；没跑的 r1-c2 继续封存。
|
||||
const after = beliefBook(studyUpTo(t, "proposal2Open"), study, 2);
|
||||
assert.ok(after.includes("作者自评 vs 大家合并：r1-c1（作者 a1）"));
|
||||
assert.ok(!after.includes("8.882"));
|
||||
});
|
||||
|
||||
test("可见性 3：第一次预测只有本人看得到，之后各阶段也不给别人", (t) => {
|
||||
const l = studyUpTo(t, "wrapOpen");
|
||||
for (const m of [material(l, "wrapUp", "a2", "s1-wrap", null, NONE), material(l, "wrapUp", "a1", "s1-wrap", null, NONE)]) {
|
||||
lacks(m, "5.551");
|
||||
lacks(m, "6.661");
|
||||
}
|
||||
});
|
||||
|
||||
test("可见性 4：第二次预测看不到别人第一次的数", (t) => {
|
||||
const l = studyUpTo(t, "secondOpen");
|
||||
const m = material(l, "secondPrediction", "a1", "s1-sp", "r1-c1", NONE);
|
||||
has(m, "5.551 / 5.552 / 5.553");
|
||||
lacks(m, "6.661");
|
||||
lacks(m, "乙第一次理由");
|
||||
});
|
||||
|
||||
test("可见性 5：第零轮闭门写入口时看不到别人的入口", (t) => {
|
||||
const l = studyUpTo(t, "entryA2");
|
||||
const m = material(l, "entry", "a1", "s0-entry", null, NONE);
|
||||
lacks(m, "入口乙一");
|
||||
has(m, "入口甲一");
|
||||
});
|
||||
|
||||
test("可见性 6：别人的退回不放进材料；本人重新发起时放上一次被退回的原因", (t) => {
|
||||
const l = studyUpTo(t, "secondOpen");
|
||||
const call = callId("s1-sp", "a1", "secondPrediction", "r1-c1", 1);
|
||||
l.append("callStarted", { call, actor: "a1", stageId: "s1-sp", model: "glm-5.3", materials: null });
|
||||
l.append("rejection", { stageId: "s1-sp", author: "a1", kind: "secondPrediction", submitted: { range: { q1: 3, q5: 2, q9: 1 } }, reason: "甲被退回：三个数不递增", jev: null });
|
||||
l.append("callEnded", { call, wallSeconds: { status: "measured", value: 3 }, status: "notSubmitted", reason: "没交成" });
|
||||
const retry = material(l, "secondPrediction", "a1", "s1-sp", "r1-c1", NONE);
|
||||
assert.ok(retry.firstMessage.includes("# 上一次为什么没交成"));
|
||||
has(retry, "甲被退回：三个数不递增");
|
||||
lacks(material(l, "secondPrediction", "a1", "s1-sp", "r1-c2", NONE), "甲被退回");
|
||||
lacks(material(l, "secondPrediction", "a2", "s1-sp", "r1-c1", NONE), "甲被退回");
|
||||
});
|
||||
|
||||
test("可见性 7：信念本只写已公开的记录：正向运行在采纳决定写入后才出现", (t) => {
|
||||
const before = beliefBook(studyUpTo(t, "run1"), study, 1);
|
||||
assert.ok(!before.includes("run-1"));
|
||||
const after = beliefBook(studyUpTo(t, "adopt1"), study, 1);
|
||||
assert.ok(after.includes("运行 run-1：算数（校验器判合法），分数 1.2"));
|
||||
});
|
||||
|
||||
test("实验员、核验员看不到别的候选", (t) => {
|
||||
const l = studyUpTo(t, "verifyOpen");
|
||||
for (const m of [
|
||||
material(l, "verification", "v1", "s1-ver-c1", "r1-c1", { changes: CHANGES_C1, artifacts: null }),
|
||||
material(studyUpTo(t, "implOpen"), "implementation", "x1", "s1-impl-c1", "r1-c1", NONE),
|
||||
]) {
|
||||
lacks(m, "r1-c2");
|
||||
lacks(m, "乙的");
|
||||
assert.equal(queryMaterial(m, "r1-c2"), null);
|
||||
}
|
||||
});
|
||||
|
||||
test("没有归属的改动：手写两个版本和位置表，算出的清单与手算的一致", (t) => {
|
||||
const l = studyUpTo(t, "verifyOpen");
|
||||
// 位置表：p1 a.py:1-5；p2 a.py:20 和整个 b.py；保障 guard a.py:5-6。
|
||||
l.append("code", {
|
||||
object: "r1-c1",
|
||||
version: "v-c1",
|
||||
locations: {
|
||||
factorLocations: [{ factor: "p1", locations: ["a.py:1-5"] }, { factor: "p2", locations: ["a.py:20", "b.py"] }],
|
||||
safeguardLocations: [{ safeguard: "guard", locations: ["a.py:5-6"] }],
|
||||
unassigned: [{ location: "a.py:7", note: "实验员说是 import" }],
|
||||
},
|
||||
});
|
||||
// 旧版本 → 新版本：a.py 新版本第 1–8 行是改过的；删了旧第 11–12 行（新版本第 10 行之后）；删了旧第 21 行（新版本第 19 行之后）。
|
||||
// b.py 改了第 3–4 行；c.bin 新增二进制；run.sh 只改可执行位；empty.txt 新增空文件。
|
||||
const changes = {
|
||||
files: [
|
||||
{ path: "a.py", change: "modified" as const, modeOnly: false, binary: false as const, hunks: [
|
||||
{ from: { start: 1, end: 3 }, to: { start: 1, end: 8 } },
|
||||
{ from: { start: 11, end: 12 }, to: { after: 10 } },
|
||||
{ from: { start: 21, end: 21 }, to: { after: 19 } },
|
||||
] },
|
||||
{ path: "b.py", change: "modified" as const, modeOnly: false, binary: false as const, hunks: [{ from: { start: 3, end: 4 }, to: { start: 3, end: 4 } }] },
|
||||
{ path: "c.bin", change: "added" as const, modeOnly: false, binary: true as const },
|
||||
{ path: "empty.txt", change: "added" as const, modeOnly: false, binary: false as const, hunks: [] },
|
||||
{ path: "run.sh", change: "modified" as const, modeOnly: true, binary: false as const, hunks: [] },
|
||||
],
|
||||
diff: "",
|
||||
};
|
||||
const m = material(l, "verification", "v1", "s1-ver-c1", "r1-c1", { changes, artifacts: null });
|
||||
// 手算：a.py 第 6 行归 guard，7–8 行没有归属（实验员自己的说明不算归属）；删除点 10 前后（10、11）都没被盖住；
|
||||
// 删除点 19 后一行 20 归 p2；b.py 整个文件归 p2；其余三个文件按整个文件算，位置表都没写到。
|
||||
const c = m.checklist as Extract<TaskMaterial["checklist"], { kind: "verification" }>;
|
||||
assert.deepEqual(c.unassigned, ["a.py:7-8", "a.py:10", "c.bin", "empty.txt", "run.sh"]);
|
||||
has(m, "- a.py:10（删除 2 行（旧版本第 11–12 行））");
|
||||
has(m, "- c.bin（新增（二进制文件，按整个文件算））");
|
||||
has(m, "- run.sh（修改（只改了可执行位，按整个文件算））");
|
||||
has(m, "- a.py:7：实验员说是 import");
|
||||
// 因子与保障同一行：p1 的 1–5 与 guard 的 5–6 在第 5 行。
|
||||
has(m, "- a.py:5：因子 p1、保障 guard");
|
||||
lacks(m, "因子 p2、保障 guard");
|
||||
});
|
||||
|
||||
test("答案路径不在位置表里、读不到，都写明状态", (t) => {
|
||||
const l = studyUpTo(t, "run1");
|
||||
l.append("runPlanned", { run: "run-x", purpose: "forward", round: 1, object: "r1-c2", definition: { command: ["python"], workdir: "/w", image: { digest: `sha256:${"0".repeat(64)}`, runtime: "runc" }, wallSeconds: 120, cores: 1, inheritEnv: false, env: {} } });
|
||||
l.append("runStarted", { run: "run-x" });
|
||||
l.append("runResult", {
|
||||
run: "run-x",
|
||||
process: { ending: "exited", exitCode: 0, cpuSeconds: { status: "measured", value: 1 }, wallSeconds: { status: "measured", value: 1 }, peakMemoryBytes: { status: "measured", value: 1 } },
|
||||
judged: {
|
||||
verdict: "valid", verdictReason: "合法", score: { status: "measured", value: 1.1 }, metrics: {}, extra: null,
|
||||
answerPath: { status: "read", value: "p9" },
|
||||
safeguardTriggers: { status: "unreadable", reason: "没写 safeguard-triggers" },
|
||||
},
|
||||
});
|
||||
l.append("code", { object: "r1-c2", version: "v-c2", locations: { factorLocations: [{ factor: "p1", locations: ["x.py"] }], safeguardLocations: [], unassigned: [] } });
|
||||
const input = { baseline: 0.96, knownBest: 2.635983, epsilon: 0.01, direction: "maximize" as const, candidates: [{ id: "r1-c1", score: 1.2 }] };
|
||||
l.append("decision", { decision: "adopt", round: 1, rule: "test", ruleVersion: 1, params: null, seed: null, input: input as never, output: { line: 0.01, quarter: 0.4, best: { id: "r1-c1", score: 1.2, improvement: 0.24 }, adopted: "r1-c1" } });
|
||||
l.append("stageOpen", { stageId: "s1-expl", stage: "explanation", round: 1, roster: ["a1", "a2"] });
|
||||
const m = material(l, "explanation", "a1", "s1-expl", null, { changes: null, artifacts: {} });
|
||||
const run = queryMaterial(m, "run-x")!;
|
||||
assert.ok(run.includes("答案路径写的是 p9，不在位置表里"));
|
||||
assert.ok(run.includes("保障动手次数读不到：没写 safeguard-triggers"));
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 查询、提示词
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test("查询：材料里的编号查得到；同一账本里材料外的编号查不到", (t) => {
|
||||
const l = studyUpTo(t, "secondOpen");
|
||||
const m = material(l, "secondPrediction", "a2", "s1-sp", "r1-c1", NONE);
|
||||
assert.ok(queryMaterial(m, "r1-a1-1")!.includes("甲的提案正文"));
|
||||
assert.ok(queryMaterial(m, "task")!.includes("填圆任务说明"));
|
||||
assert.equal(queryMaterial(m, "s1-sp"), null);
|
||||
assert.equal(queryMaterial(m, "run-0"), null);
|
||||
assert.equal(queryMaterial(m, "不存在"), null);
|
||||
const impl = material(studyUpTo(t, "implOpen"), "implementation", "x1", "s1-impl-c1", "r1-c1", NONE);
|
||||
assert.equal(queryMaterial(impl, "r1-a1-1"), null);
|
||||
});
|
||||
|
||||
test("提示词：每种提交种类都有任务说明文件，每个角色都有角色提示词", () => {
|
||||
const p = readPrompts(join(import.meta.dirname, "..", "prompts"));
|
||||
assert.deepEqual(Object.keys(p.tasks).sort(), Object.keys(SUBMISSION_KINDS).sort());
|
||||
for (const f of [...Object.values(p.tasks), ...Object.values(p.roles)]) assert.match(f.text, /\S/);
|
||||
});
|
||||
|
||||
test("提示词哈希是文件原始字节的 sha256", (t) => {
|
||||
const dir = tempDir(t);
|
||||
mkdirSync(join(dir, "roles"));
|
||||
mkdirSync(join(dir, "tasks"));
|
||||
const bytes = Buffer.from("x\r\n中", "utf8");
|
||||
for (const r of ["researcher", "experimenter", "verifier"]) writeFileSync(join(dir, "roles", `${r}.md`), bytes);
|
||||
for (const k of Object.keys(SUBMISSION_KINDS)) writeFileSync(join(dir, "tasks", `${k}.md`), bytes);
|
||||
const p = readPrompts(dir);
|
||||
// printf 'x\r\n中' | sha256sum
|
||||
assert.equal(p.roles.researcher.hash, "965da20bc965975b4eae297a49cabf4be539ef62bec455fe932093535de2c593");
|
||||
assert.equal(p.tasks.wrapUp.hash, "965da20bc965975b4eae297a49cabf4be539ef62bec455fe932093535de2c593");
|
||||
assert.equal(p.tasks.wrapUp.text, "x\r\n中");
|
||||
});
|
||||
|
||||
test("第一条消息：任务说明文件在最前，现在交什么在最后并列出提交字段", (t) => {
|
||||
const l = studyUpTo(t, "proposalOpen");
|
||||
const m = material(l, "proposal", "a1", "s1-prop", null, NONE);
|
||||
const p = readPrompts(join(import.meta.dirname, "..", "prompts"));
|
||||
assert.ok(m.firstMessage.startsWith(p.tasks.proposal.text.trimEnd()));
|
||||
const end = m.firstMessage.slice(m.firstMessage.lastIndexOf("# 现在交什么"));
|
||||
for (const f of ["`factors`", "`safeguards`", "`metrics`", "`authorEstimate`", "`body`", "最多交 2 条"]) assert.ok(end.includes(f), f);
|
||||
});
|
||||
Reference in New Issue
Block a user