Files
InfoRSI/tests/materials-diagnosis.test.ts
T
iomgaa 852600df02 feat: 诊断(任务书 18 第一块):诊断数学、四种提交、一次运行跑多个实验、账本与材料、flow 诊断各步
消融默认关,关掉时解释阶段一并跳过;选诊断实验用信息价值减成本。
eval/diagnosis/research.json 为第三块配置。vocab 字段说明的出口文字随提示词一起改,见下一次提交的 prompts/ 与同文件改动(本次含在内)。

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-05 10:36:20 -04:00

338 lines
21 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import assert from "node:assert/strict";
import { test, type TestContext } from "node:test";
import { Ledger } from "../src/ledger/index.ts";
import {
Board,
buildMaterial,
type CodeChanges,
diagnosisExplanationId,
diagnosisId,
diagnosisNotRunEvent,
type DiagnosisSettlement,
type StudyFacts,
type TaskMaterial,
} from "../src/materials/index.ts";
import { selectDiagnosis, settleDiagnosis, type DiagnosisRuleInput, type SettleDiagnosisInput } from "../src/rules/index.ts";
import type { SubmissionKind } from "../src/vocab/index.ts";
import { freshLedger } from "./ledger-helpers.ts";
import { batch, CHANGES_C1, everything, judged, material, NONE, P_A1, prompts, protocolOf, r, study, TABLE_C1, Writer } from "./materials-helpers.ts";
// 诊断的材料(任务书 18 第 1.5 节、方法.md 第三部分、4.3 诊 1–诊 4):在真实账本里写一轮诊断,看每个任务该有的有、不该有的没有。
// 研究者 a1、a2,实验员 x1,核验员 v1。基线 0.96。三个合并后的诊断实验:d1(a1、a2 的"强起点"合并)、d2(a1 的)、d3(a2 的第二个)。
// a1 的诊断预测数 7.771…、a2 的 8.881…是好认的数:任何别人的材料里都不该出现。
// 选实验的决定、结算都用 rules 的函数算(不手写数);这里只查材料怎么读它们。
const SEALED = { a1: ["7.771", "7.772", "7.773"], a2: ["8.881", "8.882", "8.883"] };
const D = diagnosisId(1);
const headings = (m: TaskMaterial) => m.sections.map((s) => s.heading);
const has = (m: TaskMaterial, text: string) => assert.ok(everything(m).includes(text), `材料里应有"${text}"`);
const lacks = (m: TaskMaterial, text: string) => assert.ok(!everything(m).includes(text), `材料里不该有"${text}"`);
const experiment = (name: string, spot: string) => ({ name, spot, body: `${name}:${spot}上怎么动`, metrics: [] });
const gapExperiment = { ...experiment("拿掉收尾", "solve.py 收尾"), metrics: [{ id: "gap", meaning: "间隙", unit: "无", direction: "minimize" as const, howMeasured: "最后量一次", missing: "recordMissing" as const }] };
/** 两条解释对三个实验的三个数:a1 说 d1 能涨很多,a2 说 d1 涨不了;d2、d3 两人差不多。 */
const ranges = {
[diagnosisExplanationId(1, "a1", 1)]: { author: "a1", d1: r(1.1, 1.25, 1.3), d2: r(0.95, 1.0, 1.05), d3: r(0.95, 1.0, 1.05) },
[diagnosisExplanationId(1, "a2", 1)]: { author: "a2", d1: r(0.9, 0.95, 0.99), d2: r(0.95, 1.0, 1.05), d3: r(0.95, 1.0, 1.05) },
};
const ruleInput = (balance: number, same: boolean): DiagnosisRuleInput => ({
baseline: 0.96,
direction: "maximize",
lambda: 0.1,
experiments: ["d1", "d2", "d3"],
explanations: Object.entries(ranges).map(([id, x]) => ({
id,
author: x.author,
experiments: (["d1", "d2", "d3"] as const).map((e) => ({ experiment: e, range: same ? ranges[diagnosisExplanationId(1, "a1", 1)]![e] : x[e] })),
})),
balance,
batchCap: 1,
u: 0.5,
decay: 0.5,
maxExperiments: 2,
});
type Step = "diagProposalOpen" | "diagProposalClosed" | "diagPredictionOpen" | "diagPredictionClosed" | "selected" | "implemented" | "verifiedFailed" | "repairOpen" | "run" | "settled";
const ORDER: Step[] = ["diagProposalOpen", "diagProposalClosed", "diagPredictionOpen", "diagPredictionClosed", "selected", "implemented", "verifiedFailed", "repairOpen", "run", "settled"];
/** 第 1 轮诊断写到某一步(含)为止。选中的是净值最大的两个实验;d 里验证不通过的那个在结算里记证据不足。 */
function diagnosisStudy(t: TestContext, upTo: Step): { l: Ledger; w: Writer; chosen: string[] } {
const { ledger: l } = freshLedger(t);
const w = new Writer(l);
const reached = (s: Step) => ORDER.indexOf(upTo) >= ORDER.indexOf(s);
l.append("code", { object: "base", version: "v-base", locations: null });
w.run("run-0", "baseline", 0, null, judged(0.96, "main"));
w.open("s1-dp", "diagnosisProposal", 1, ["a1", "a2"]);
w.sub("s1-dp", "a1", "diagnosisProposal", null, null, {
explanations: [{ body: "甲的诊断解释:卡在起点 UNIQUE-A1" }],
experiments: [experiment("强起点", "solve.py 起点"), gapExperiment],
});
if (!reached("diagProposalClosed")) return { l, w, chosen: [] };
w.sub("s1-dp", "a2", "diagnosisProposal", null, null, {
explanations: [{ body: "乙的诊断解释:卡在收尾 UNIQUE-A2" }],
experiments: [experiment("强起点", "solve.py 起点(乙的写法)"), experiment("放宽间隙", "solve.py 间隙限制")],
});
w.close("s1-dp");
w.decision("diagnosisDedup", 1, null, {
experiments: [
{ experiment: "d1", sources: [{ author: "a1", index: 0 }, { author: "a2", index: 0 }] },
{ experiment: "d2", sources: [{ author: "a1", index: 1 }] },
{ experiment: "d3", sources: [{ author: "a2", index: 1 }] },
],
});
w.open("s1-dpred", "diagnosisPrediction", 1, ["a1", "a2"]);
const pred = (author: "a1" | "a2", numbers: string[]) =>
(["d1", "d2", "d3"] as const).map((e) => ({ explanation: diagnosisExplanationId(1, author, 1), experiment: e, range: e === "d1" ? r(...(numbers.map(Number) as [number, number, number])) : ranges[diagnosisExplanationId(1, author, 1)]![e] }));
w.sub("s1-dpred", "a1", "diagnosisPrediction", null, null, { predictions: pred("a1", SEALED.a1), body: "甲的预测理由" });
if (!reached("diagPredictionClosed")) return { l, w, chosen: [] };
w.sub("s1-dpred", "a2", "diagnosisPrediction", null, null, { predictions: pred("a2", SEALED.a2), body: "乙的预测理由" });
w.close("s1-dpred");
const input = ruleInput(9, false);
const decision = selectDiagnosis(input);
assert.equal(decision.diagnose, true);
const chosen = decision.chosen;
assert.equal(chosen.length, 2);
if (!reached("selected")) return { l, w, chosen };
w.decision("selectDiagnosis", 1, input, decision);
w.open("s1-dimpl", "diagnosisImplementation", 1, ["x1"]);
const table = chosen.map((experiment, i) => ({ experiment, locations: [`solve.py:${10 + 10 * i}-${19 + 10 * i}`] }));
w.sub("s1-dimpl", "x1", "diagnosisImplementation", D, null, { locations: table, concerns: "收尾里有个旧开关", body: "诊断实现说明 UNIQUE-X1" });
w.close("s1-dimpl");
l.append("code", { object: D, version: "v-diag", locations: table as never });
if (!reached("implemented")) return { l, w, chosen };
w.open("s1-dver", "diagnosisVerification", 1, ["v1"]);
w.sub("s1-dver", "v1", "diagnosisVerification", D, null, {
experiments: chosen.map((experiment, i) => ({ experiment, followed: i === 0, nothingElse: true, where: `solve.py:${10 + 10 * i}` })),
body: "核验员:第二个实验没照写法做",
});
w.close("s1-dver");
if (!reached("repairOpen")) return { l, w, chosen };
w.open("s1-dimpl2", "diagnosisImplementation", 1, ["x1"]);
if (!reached("run")) return { l, w, chosen };
w.run("run-1", "diagnosis", 1, D, null); // 诊断运行的记录里没有判分:逐个实验的结果在结算里
if (!reached("settled")) return { l, w, chosen };
const settleInput: SettleDiagnosisInput = {
baseline: 0.96,
direction: "maximize",
epsilon: 0.01,
lambda: 0.1,
experiments: ["d1", "d2", "d3"].map((experiment) => ({
experiment,
outcome: experiment === chosen[0] ? { kind: "valid", score: 1.1 } : experiment === chosen[1] ? { kind: "insufficient" } : { kind: "untested" },
})),
explanations: ruleInput(9, false).explanations,
};
const settlement: DiagnosisSettlement = {
run: "run-1",
judged: {
experiments: [
{
id: chosen[0]!, ending: "finished", wallSeconds: { status: "measured", value: 31.5 }, verdict: "valid", verdictReason: "校验器判合法",
score: { status: "measured", value: 1.1 }, metrics: { "sum-of-radii": { status: "measured", value: 1.1 }, [`${chosen[0]}.gap`]: { status: "measured", value: 0.02 } }, extra: null,
},
],
},
dropped: [{ experiment: chosen[1]!, reason: "第二个实验两次核验都没落实" }],
input: settleInput,
output: settleDiagnosis(settleInput),
};
l.append("diagnosisSettlement", { round: 1, result: settlement as never });
return { l, w, chosen };
}
const mat = (l: Ledger, kind: SubmissionKind, actor: string, stageId: string, object: string | null, extra: { changes: CodeChanges | null; artifacts: Record<string, Record<string, string>> | null } = NONE, facts: StudyFacts = study) =>
buildMaterial({ kind, actor, stageId, object, ledger: l, study: facts, prompts, codeRoot: "/code", ...extra });
test("诊 1:基线代码、基线那次运行的结果和答案、没有别人这一轮的诊断;上限取自配置(A、M)", (t) => {
const { l } = diagnosisStudy(t, "diagProposalOpen");
const m = mat(l, "diagnosisProposal", "a2", "s1-dp", null, { changes: null, artifacts: { "run-0": { "solution.json": "{\"centers\": []}" } } });
assert.deepEqual(m.checklist, {
kind: "diagnosisProposal",
maxExplanations: 2,
maxExperiments: 2,
takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers"],
});
assert.deepEqual(m.readable, [{ version: "v-base", label: "本批基线", dir: "/code/v-base" }]);
has(m, "运行 run-0:任务给的原始代码;算数(校验器判合法);分数 0.96");
has(m, "run-0/solution.json");
assert.equal(m.records.find((x) => x.id === "run-0/solution.json")!.text, "{\"centers\": []}");
assert.ok(headings(m).includes("上一轮的收尾"));
assert.ok(!headings(m).includes("本轮诊断结果"), "诊断这一段还没走完,不放本轮诊断结果");
lacks(m, "UNIQUE-A1"); // a1 在同一阶段已经交了,阶段没关,别人读不到
lacks(m, "强起点");
});
test("诊 2:合并后的实验标明作者、全部诊断解释、自己的解释另列并给对号清单;看不到别人在这一步交的数", (t) => {
const { l } = diagnosisStudy(t, "diagPredictionOpen");
const m = mat(l, "diagnosisPrediction", "a2", "s1-dpred", null);
assert.deepEqual(m.checklist, { kind: "diagnosisPrediction", explanations: ["r1-a2-dx1"], experiments: ["d1", "d2", "d3"] });
has(m, "#### d1 强起点(作者 a1、a2)");
has(m, "动哪一处:solve.py 起点\n"); // 合并后的正文取作者里编号最小的那一份(a1)
lacks(m, "solve.py 起点(乙的写法)");
has(m, "#### d3 放宽间隙(作者 a2)");
has(m, "UNIQUE-A1");
has(m, "UNIQUE-A2");
assert.ok(m.sections.some((s) => s.heading === "你的诊断解释" && s.text.includes("r1-a2-dx1")));
for (const x of SEALED.a1) lacks(m, x); // a1 已经交了诊断预测,阶段还开着
});
test("诊断预测的数:关闭之后,也不进任何别人的材料(诊 2 本人、实现、提案都没有)", (t) => {
const { l } = diagnosisStudy(t, "settled");
l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] });
const proposals = [mat(l, "proposal", "a1", "s1-prop", null), mat(l, "proposal", "a2", "s1-prop", null)];
for (const m of proposals) {
for (const x of [...SEALED.a1, ...SEALED.a2]) lacks(m, x);
}
const own = mat(l, "diagnosisPrediction", "a1", "s1-dpred", null);
for (const x of SEALED.a2) lacks(own, x);
});
test("诊断实现:选中的实验原文、基线代码、原始产物的位置;不给信念本和成绩", (t) => {
const { l, chosen } = diagnosisStudy(t, "selected");
const m = mat(l, "diagnosisImplementation", "x1", "s1-dimpl", D);
assert.deepEqual(m.checklist, { kind: "diagnosisImplementation", experiments: chosen });
assert.deepEqual(m.readable, [{ version: "v-base", label: "起点代码(当前基线)", dir: "/code/v-base" }]);
has(m, "产出当前基线的那次正式运行(run-0)的原始产物已经只读地放进你的工作目录");
for (const id of chosen) has(m, `### ${id} `);
assert.ok(!headings(m).includes("信念本") && !headings(m).includes("本批基线那次运行的结果"));
lacks(m, "UNIQUE-A2"); // 诊断解释不给实验员:只给选中的实验
});
test("诊断修复:实现的材料加退回意见(没通过的实验、依据、上一次位置表和改动)、能读上一次的诊断程序;通过的实验不要动", (t) => {
const { l, chosen } = diagnosisStudy(t, "repairOpen");
const m = mat(l, "diagnosisRepair", "x1", "s1-dimpl2", D, { changes: CHANGES_C1, artifacts: null });
assert.deepEqual(m.checklist, { kind: "diagnosisRepair", experiments: chosen });
assert.deepEqual(m.readable.map((x) => x.version), ["v-base", "v-diag"]);
const opinion = m.sections.find((s) => s.heading === "退回意见")!.text;
assert.ok(opinion.includes(`实验 ${chosen[1]}:没照写法做;solve.py:20`));
assert.ok(!opinion.includes(`实验 ${chosen[0]}:`), "通过的实验不列");
assert.ok(opinion.includes("核验员:第二个实验没照写法做"));
assert.ok(opinion.includes(`"experiment": "${chosen[0]}"`) && opinion.includes("多起点循环"));
});
test("诊断核验:选中的实验原文、基线到诊断程序的改动、位置表;不给信念本和实验员的说明", (t) => {
const { l, chosen } = diagnosisStudy(t, "implemented");
const m = mat(l, "diagnosisVerification", "v1", "s1-dver", D, { changes: CHANGES_C1, artifacts: null });
assert.deepEqual(m.checklist, { kind: "diagnosisVerification", experiments: chosen });
assert.deepEqual(m.readable.map((x) => x.version), ["v-base", "v-diag"]);
has(m, "多起点循环");
has(m, "solve.py:10-19");
for (const id of chosen) has(m, `### ${id} `);
assert.ok(!headings(m).includes("信念本"));
lacks(m, "UNIQUE-X1");
});
test("本轮诊断结果:提案材料里每个实验的做法、实测、用时、结局,每条解释确认或否定,推出的方向;排在基线之后、研究记录之前", (t) => {
const { l, chosen } = diagnosisStudy(t, "settled");
l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] });
const m = mat(l, "proposal", "a1", "s1-prop", null);
const h = headings(m);
assert.ok(h.indexOf("基线") < h.indexOf("本轮诊断结果") && h.indexOf("本轮诊断结果") < h.indexOf("研究记录"));
const section = m.sections.find((s) => s.heading === "本轮诊断结果")!.text;
const [first, second] = chosen as [string, string];
assert.ok(section.includes(`诊断运行 run-1:在基线 base(最终指标 0.96)上,一次运行做了 1 个实验,只扣 1 次`));
const line = (id: string) => section.split("\n").find((x) => x.startsWith(`- ${id} `))!;
for (const part of ["实测 1.1,改进量 +0.14", "用时 31.5 秒", "其余指标:d1.gap 0.02", "测前合并范围"]) assert.ok(line(first!).includes(part), part);
assert.ok(line(second!).includes("证据不足:核验不通过,第二个实验两次核验都没落实"));
assert.match(section, /没做:一次诊断最多做 D 个实验,放不下/);
assert.match(section, /- r1-a1-dx1(作者 a1):确认 .*;否定 /);
assert.ok(section.includes("甲的诊断解释:卡在起点 UNIQUE-A1") && section.includes("乙的诊断解释:卡在收尾 UNIQUE-A2"));
assert.match(section, /程序按诊断推出的方向:d\d(实测 \+0\.14)。这只是按公式推出来的参考,你可以不跟。/);
assert.ok(m.records.some((x) => x.id === "diagnosis-r1" && x.text.includes("run-1") && !x.text.includes("7.771")));
assert.ok(m.readable.some((x) => x.version === "v-diag" && x.label === "第 1 轮诊断程序的代码"));
});
/** 只有基线、第 1 轮提案阶段开着的账本:本轮没有诊断记录。 */
function bare(t: TestContext): Ledger {
const { ledger: l } = freshLedger(t);
const w = new Writer(l);
l.append("code", { object: "base", version: "v-base", locations: null });
w.run("run-0", "baseline", 0, null, judged(0.96, "main"));
w.open("s1-prop", "proposal", 1, ["a1", "a2"]);
return l;
}
const diagnosisText = (m: TaskMaterial) => m.sections.find((s) => s.heading === "本轮诊断结果")!.text;
test("本轮没有诊断:写原因。余额不到 2 次、配置关掉(余额够却没有决定)", (t) => {
const l = bare(t);
// 基线占 1 次:运行总数 2 时余额 1(不到 2),总数 10 时余额 9。
assert.equal(diagnosisText(mat(l, "proposal", "a1", "s1-prop", null, NONE, { ...study, totalRuns: 2 })), "本轮没有诊断:余额只剩 1 次,不到 2 次,要留 1 次给本轮正向");
assert.equal(diagnosisText(mat(l, "proposal", "a1", "s1-prop", null)), "本轮没有诊断:配置关掉了诊断");
});
test("本轮没有诊断:信息价值不抵成本时,原因带最好的组合净值;选中的实验一个都没剩下时,原因取事件", (t) => {
const l = bare(t);
const w = new Writer(l);
const input = ruleInput(9, true); // 两条解释对每个实验预测一样:信息价值只剩开放预留那一点,抵不过成本
const decision = selectDiagnosis(input);
assert.equal(decision.diagnose, false);
w.decision("selectDiagnosis", 1, input, decision);
const best = Math.max(...decision.combos.map((c) => c.net));
const text = diagnosisText(mat(l, "proposal", "a1", "s1-prop", null));
assert.equal(text, `本轮没有诊断:信息价值减成本都不大于 0,最好的组合净值 ${Number(best.toPrecision(4))}`);
// 同一轮另一份账本:决定是做,但核验都没通过、修复用完。
const l2 = bare(t);
const w2 = new Writer(l2);
const done = selectDiagnosis(ruleInput(9, false));
w2.decision("selectDiagnosis", 1, ruleInput(9, false), done);
l2.append("event", diagnosisNotRunEvent(1, "d1、d2 都没通过核验"));
assert.equal(diagnosisText(mat(l2, "proposal", "a1", "s1-prop", null)), "本轮没有诊断:选中的实验核验都没通过、修复次数用完,一个都没跑:d1、d2 都没通过核验");
});
test("诊断运行在诊断结算写入之前不公开,写入之后公开(和正向的采纳、消融的结算同一类规则)", (t) => {
const run = (l: Ledger) => new Board(l.records()).runs().find((x) => x.plan.run === "run-1")!;
const running = diagnosisStudy(t, "run").l;
assert.equal(new Board(running.records()).runPublic(run(running)), false);
const settled = diagnosisStudy(t, "settled").l;
assert.equal(new Board(settled.records()).runPublic(run(settled)), true);
});
test("信念本的诊断历史:各轮的实验、确认与否定、方向;战绩按人加一行诊断;诊断实验的实测不算手里最好", (t) => {
const { l, chosen } = diagnosisStudy(t, "settled");
l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] });
const book = mat(l, "proposal", "a1", "s1-prop", null).sections.find((s) => s.heading === "信念本")!.text;
assert.ok(book.includes("**诊断历史**"));
assert.ok(book.includes(`- 第 1 轮:\n 诊断运行 run-1:在基线 base`));
assert.ok(book.includes(` - ${chosen[0]} `) && book.includes("确认 d1;否定 无"));
assert.ok(book.includes("手里最好 0.96"), "诊断实验实测 1.1,但诊断的答案不采纳,手里最好仍是基线");
// 战绩:a1、a2 各一条诊断:只有 d1 一个有效实验,样本各 1 个。a1 的 q5 = 1.25 > 实测 1.1 偏高;a2 的 q5 = 0.95 < 1.1 偏低。
assert.ok(book.includes("- a1,诊断 1 次:中位数偏高 1、偏低 0"));
assert.ok(book.includes("- a2,诊断 1 次:中位数偏高 0、偏低 1"));
});
test("信念本的诊断历史:第 2 轮诊断关着时,第 1 轮的历史还在,第 2 轮没走完不列;第 2 轮诊断任务读不到第 2 轮的诊断结果", (t) => {
const { l } = diagnosisStudy(t, "settled");
l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] });
l.append("stageClose", { stageId: "s1-prop" });
// 第 1 轮采纳 r1-c1 作基线要写一大串记录,这里只看第 2 轮的诊断任务:基线仍是 base,所以直接开第 2 轮的诊断阶段。
l.append("stageOpen", { stageId: "s2-dp", stage: "diagnosisProposal", round: 2, roster: ["a1", "a2"] });
const m = mat(l, "diagnosisProposal", "a1", "s2-dp", null, { changes: null, artifacts: {} });
assert.ok(!headings(m).includes("本轮诊断结果"));
const book = m.sections.find((s) => s.heading === "信念本")!.text;
assert.ok(book.includes("- 第 1 轮:") && !book.includes("- 第 2 轮:"));
});
test("实现正向候选的实验员:本轮有诊断时能读诊断程序(只读,注明冻结协议引用诊断实验时作参考);没有诊断时没有", (t) => {
const { l } = diagnosisStudy(t, "settled");
const w = new Writer(l);
l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] });
w.protocol("r1-c1", protocolOf("r1-c1", P_A1, 1, "base", ["r1-a1-1"], ["加多起点"]));
w.open("s1-impl-c1", "implementation", 1, ["x1"]);
const m = mat(l, "implementation", "x1", "s1-impl-c1", "r1-c1");
assert.deepEqual(m.readable, [
{ version: "v-base", label: "起点代码(本批基线)", dir: "/code/v-base" },
{ version: "v-diag", label: "本轮诊断程序(只读,冻结协议引用诊断实验时作参考)", dir: "/code/v-diag" },
]);
has(m, "冻结协议引用诊断实验时作参考");
lacks(m, "UNIQUE-A1"); // 只给代码,不给诊断解释和成绩
lacks(m, "实测 1.1");
const l2 = bare(t);
const w2 = new Writer(l2);
w2.protocol("r1-c1", protocolOf("r1-c1", P_A1, 1, "base", ["r1-a1-1"], ["加多起点"]));
w2.open("s1-impl-c1", "implementation", 1, ["x1"]);
assert.deepEqual(mat(l2, "implementation", "x1", "s1-impl-c1", "r1-c1").readable.map((x) => x.version), ["v-base"]);
});