import assert from "node:assert/strict"; import { test, type TestContext } from "node:test"; import { Ledger } from "../src/ledger/index.ts"; import { Board, buildMaterial, type CodeChanges, diagnosisExplanationId, diagnosisId, diagnosisNotRunEvent, type DiagnosisSettlement, type StudyFacts, type TaskMaterial, } from "../src/materials/index.ts"; import { selectDiagnosis, settleDiagnosis, type DiagnosisRuleInput, type SettleDiagnosisInput } from "../src/rules/index.ts"; import type { SubmissionKind } from "../src/vocab/index.ts"; import { freshLedger } from "./ledger-helpers.ts"; import { batch, CHANGES_C1, everything, judged, material, NONE, P_A1, prompts, protocolOf, r, study, TABLE_C1, Writer } from "./materials-helpers.ts"; // 诊断的材料(任务书 18 第 1.5 节、方法.md 第三部分、4.3 诊 1–诊 4):在真实账本里写一轮诊断,看每个任务该有的有、不该有的没有。 // 研究者 a1、a2,实验员 x1,核验员 v1。基线 0.96。三个合并后的诊断实验:d1(a1、a2 的"强起点"合并)、d2(a1 的)、d3(a2 的第二个)。 // a1 的诊断预测数 7.771…、a2 的 8.881…是好认的数:任何别人的材料里都不该出现。 // 选实验的决定、结算都用 rules 的函数算(不手写数);这里只查材料怎么读它们。 const SEALED = { a1: ["7.771", "7.772", "7.773"], a2: ["8.881", "8.882", "8.883"] }; const D = diagnosisId(1); const headings = (m: TaskMaterial) => m.sections.map((s) => s.heading); const has = (m: TaskMaterial, text: string) => assert.ok(everything(m).includes(text), `材料里应有"${text}"`); const lacks = (m: TaskMaterial, text: string) => assert.ok(!everything(m).includes(text), `材料里不该有"${text}"`); const experiment = (name: string, spot: string) => ({ name, spot, body: `${name}:${spot}上怎么动`, metrics: [] }); const gapExperiment = { ...experiment("拿掉收尾", "solve.py 收尾"), metrics: [{ id: "gap", meaning: "间隙", unit: "无", direction: "minimize" as const, howMeasured: "最后量一次", missing: "recordMissing" as const }] }; /** 两条解释对三个实验的三个数:a1 说 d1 能涨很多,a2 说 d1 涨不了;d2、d3 两人差不多。 */ const ranges = { [diagnosisExplanationId(1, "a1", 1)]: { author: "a1", d1: r(1.1, 1.25, 1.3), d2: r(0.95, 1.0, 1.05), d3: r(0.95, 1.0, 1.05) }, [diagnosisExplanationId(1, "a2", 1)]: { author: "a2", d1: r(0.9, 0.95, 0.99), d2: r(0.95, 1.0, 1.05), d3: r(0.95, 1.0, 1.05) }, }; const ruleInput = (balance: number, same: boolean): DiagnosisRuleInput => ({ baseline: 0.96, direction: "maximize", lambda: 0.1, experiments: ["d1", "d2", "d3"], explanations: Object.entries(ranges).map(([id, x]) => ({ id, author: x.author, experiments: (["d1", "d2", "d3"] as const).map((e) => ({ experiment: e, range: same ? ranges[diagnosisExplanationId(1, "a1", 1)]![e] : x[e] })), })), balance, batchCap: 1, u: 0.5, decay: 0.5, maxExperiments: 2, }); type Step = "diagProposalOpen" | "diagProposalClosed" | "diagPredictionOpen" | "diagPredictionClosed" | "selected" | "implemented" | "verifiedFailed" | "repairOpen" | "run" | "settled"; const ORDER: Step[] = ["diagProposalOpen", "diagProposalClosed", "diagPredictionOpen", "diagPredictionClosed", "selected", "implemented", "verifiedFailed", "repairOpen", "run", "settled"]; /** 第 1 轮诊断写到某一步(含)为止。选中的是净值最大的两个实验;d 里验证不通过的那个在结算里记证据不足。 */ function diagnosisStudy(t: TestContext, upTo: Step): { l: Ledger; w: Writer; chosen: string[] } { const { ledger: l } = freshLedger(t); const w = new Writer(l); const reached = (s: Step) => ORDER.indexOf(upTo) >= ORDER.indexOf(s); l.append("code", { object: "base", version: "v-base", locations: null }); w.run("run-0", "baseline", 0, null, judged(0.96, "main")); w.open("s1-dp", "diagnosisProposal", 1, ["a1", "a2"]); w.sub("s1-dp", "a1", "diagnosisProposal", null, null, { explanations: [{ body: "甲的诊断解释:卡在起点 UNIQUE-A1" }], experiments: [experiment("强起点", "solve.py 起点"), gapExperiment], }); if (!reached("diagProposalClosed")) return { l, w, chosen: [] }; w.sub("s1-dp", "a2", "diagnosisProposal", null, null, { explanations: [{ body: "乙的诊断解释:卡在收尾 UNIQUE-A2" }], experiments: [experiment("强起点", "solve.py 起点(乙的写法)"), experiment("放宽间隙", "solve.py 间隙限制")], }); w.close("s1-dp"); w.decision("diagnosisDedup", 1, null, { experiments: [ { experiment: "d1", sources: [{ author: "a1", index: 0 }, { author: "a2", index: 0 }] }, { experiment: "d2", sources: [{ author: "a1", index: 1 }] }, { experiment: "d3", sources: [{ author: "a2", index: 1 }] }, ], }); w.open("s1-dpred", "diagnosisPrediction", 1, ["a1", "a2"]); const pred = (author: "a1" | "a2", numbers: string[]) => (["d1", "d2", "d3"] as const).map((e) => ({ explanation: diagnosisExplanationId(1, author, 1), experiment: e, range: e === "d1" ? r(...(numbers.map(Number) as [number, number, number])) : ranges[diagnosisExplanationId(1, author, 1)]![e] })); w.sub("s1-dpred", "a1", "diagnosisPrediction", null, null, { predictions: pred("a1", SEALED.a1), body: "甲的预测理由" }); if (!reached("diagPredictionClosed")) return { l, w, chosen: [] }; w.sub("s1-dpred", "a2", "diagnosisPrediction", null, null, { predictions: pred("a2", SEALED.a2), body: "乙的预测理由" }); w.close("s1-dpred"); const input = ruleInput(9, false); const decision = selectDiagnosis(input); assert.equal(decision.diagnose, true); const chosen = decision.chosen; assert.equal(chosen.length, 2); if (!reached("selected")) return { l, w, chosen }; w.decision("selectDiagnosis", 1, input, decision); w.open("s1-dimpl", "diagnosisImplementation", 1, ["x1"]); const table = chosen.map((experiment, i) => ({ experiment, locations: [`solve.py:${10 + 10 * i}-${19 + 10 * i}`] })); w.sub("s1-dimpl", "x1", "diagnosisImplementation", D, null, { locations: table, concerns: "收尾里有个旧开关", body: "诊断实现说明 UNIQUE-X1" }); w.close("s1-dimpl"); l.append("code", { object: D, version: "v-diag", locations: table as never }); if (!reached("implemented")) return { l, w, chosen }; w.open("s1-dver", "diagnosisVerification", 1, ["v1"]); w.sub("s1-dver", "v1", "diagnosisVerification", D, null, { experiments: chosen.map((experiment, i) => ({ experiment, followed: i === 0, nothingElse: true, where: `solve.py:${10 + 10 * i}` })), body: "核验员:第二个实验没照写法做", }); w.close("s1-dver"); if (!reached("repairOpen")) return { l, w, chosen }; w.open("s1-dimpl2", "diagnosisImplementation", 1, ["x1"]); if (!reached("run")) return { l, w, chosen }; w.run("run-1", "diagnosis", 1, D, null); // 诊断运行的记录里没有判分:逐个实验的结果在结算里 if (!reached("settled")) return { l, w, chosen }; const settleInput: SettleDiagnosisInput = { baseline: 0.96, direction: "maximize", epsilon: 0.01, lambda: 0.1, experiments: ["d1", "d2", "d3"].map((experiment) => ({ experiment, outcome: experiment === chosen[0] ? { kind: "valid", score: 1.1 } : experiment === chosen[1] ? { kind: "insufficient" } : { kind: "untested" }, })), explanations: ruleInput(9, false).explanations, }; const settlement: DiagnosisSettlement = { run: "run-1", judged: { experiments: [ { id: chosen[0]!, ending: "finished", wallSeconds: { status: "measured", value: 31.5 }, verdict: "valid", verdictReason: "校验器判合法", score: { status: "measured", value: 1.1 }, metrics: { "sum-of-radii": { status: "measured", value: 1.1 }, [`${chosen[0]}.gap`]: { status: "measured", value: 0.02 } }, extra: null, }, ], }, dropped: [{ experiment: chosen[1]!, reason: "第二个实验两次核验都没落实" }], input: settleInput, output: settleDiagnosis(settleInput), }; l.append("diagnosisSettlement", { round: 1, result: settlement as never }); return { l, w, chosen }; } const mat = (l: Ledger, kind: SubmissionKind, actor: string, stageId: string, object: string | null, extra: { changes: CodeChanges | null; artifacts: Record> | null } = NONE, facts: StudyFacts = study) => buildMaterial({ kind, actor, stageId, object, ledger: l, study: facts, prompts, codeRoot: "/code", ...extra }); test("诊 1:基线代码、基线那次运行的结果和答案、没有别人这一轮的诊断;上限取自配置(A、M)", (t) => { const { l } = diagnosisStudy(t, "diagProposalOpen"); const m = mat(l, "diagnosisProposal", "a2", "s1-dp", null, { changes: null, artifacts: { "run-0": { "solution.json": "{\"centers\": []}" } } }); assert.deepEqual(m.checklist, { kind: "diagnosisProposal", maxExplanations: 2, maxExperiments: 2, takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers"], }); assert.deepEqual(m.readable, [{ version: "v-base", label: "本批基线", dir: "/code/v-base" }]); has(m, "运行 run-0:任务给的原始代码;算数(校验器判合法);分数 0.96"); has(m, "run-0/solution.json"); assert.equal(m.records.find((x) => x.id === "run-0/solution.json")!.text, "{\"centers\": []}"); assert.ok(headings(m).includes("上一轮的收尾")); assert.ok(!headings(m).includes("本轮诊断结果"), "诊断这一段还没走完,不放本轮诊断结果"); lacks(m, "UNIQUE-A1"); // a1 在同一阶段已经交了,阶段没关,别人读不到 lacks(m, "强起点"); }); test("诊 2:合并后的实验标明作者、全部诊断解释、自己的解释另列并给对号清单;看不到别人在这一步交的数", (t) => { const { l } = diagnosisStudy(t, "diagPredictionOpen"); const m = mat(l, "diagnosisPrediction", "a2", "s1-dpred", null); assert.deepEqual(m.checklist, { kind: "diagnosisPrediction", explanations: ["r1-a2-dx1"], experiments: ["d1", "d2", "d3"] }); has(m, "#### d1 强起点(作者 a1、a2)"); has(m, "动哪一处:solve.py 起点\n"); // 合并后的正文取作者里编号最小的那一份(a1) lacks(m, "solve.py 起点(乙的写法)"); has(m, "#### d3 放宽间隙(作者 a2)"); has(m, "UNIQUE-A1"); has(m, "UNIQUE-A2"); assert.ok(m.sections.some((s) => s.heading === "你的诊断解释" && s.text.includes("r1-a2-dx1"))); for (const x of SEALED.a1) lacks(m, x); // a1 已经交了诊断预测,阶段还开着 }); test("诊断预测的数:关闭之后,也不进任何别人的材料(诊 2 本人、实现、提案都没有)", (t) => { const { l } = diagnosisStudy(t, "settled"); l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] }); const proposals = [mat(l, "proposal", "a1", "s1-prop", null), mat(l, "proposal", "a2", "s1-prop", null)]; for (const m of proposals) { for (const x of [...SEALED.a1, ...SEALED.a2]) lacks(m, x); } const own = mat(l, "diagnosisPrediction", "a1", "s1-dpred", null); for (const x of SEALED.a2) lacks(own, x); }); test("诊断实现:选中的实验原文、基线代码、原始产物的位置;不给信念本和成绩", (t) => { const { l, chosen } = diagnosisStudy(t, "selected"); const m = mat(l, "diagnosisImplementation", "x1", "s1-dimpl", D); assert.deepEqual(m.checklist, { kind: "diagnosisImplementation", experiments: chosen }); assert.deepEqual(m.readable, [{ version: "v-base", label: "起点代码(当前基线)", dir: "/code/v-base" }]); has(m, "产出当前基线的那次正式运行(run-0)的原始产物已经只读地放进你的工作目录"); for (const id of chosen) has(m, `### ${id} `); assert.ok(!headings(m).includes("信念本") && !headings(m).includes("本批基线那次运行的结果")); lacks(m, "UNIQUE-A2"); // 诊断解释不给实验员:只给选中的实验 }); test("诊断修复:实现的材料加退回意见(没通过的实验、依据、上一次位置表和改动)、能读上一次的诊断程序;通过的实验不要动", (t) => { const { l, chosen } = diagnosisStudy(t, "repairOpen"); const m = mat(l, "diagnosisRepair", "x1", "s1-dimpl2", D, { changes: CHANGES_C1, artifacts: null }); assert.deepEqual(m.checklist, { kind: "diagnosisRepair", experiments: chosen }); assert.deepEqual(m.readable.map((x) => x.version), ["v-base", "v-diag"]); const opinion = m.sections.find((s) => s.heading === "退回意见")!.text; assert.ok(opinion.includes(`实验 ${chosen[1]}:没照写法做;solve.py:20`)); assert.ok(!opinion.includes(`实验 ${chosen[0]}:`), "通过的实验不列"); assert.ok(opinion.includes("核验员:第二个实验没照写法做")); assert.ok(opinion.includes(`"experiment": "${chosen[0]}"`) && opinion.includes("多起点循环")); }); test("诊断核验:选中的实验原文、基线到诊断程序的改动、位置表;不给信念本和实验员的说明", (t) => { const { l, chosen } = diagnosisStudy(t, "implemented"); const m = mat(l, "diagnosisVerification", "v1", "s1-dver", D, { changes: CHANGES_C1, artifacts: null }); assert.deepEqual(m.checklist, { kind: "diagnosisVerification", experiments: chosen }); assert.deepEqual(m.readable.map((x) => x.version), ["v-base", "v-diag"]); has(m, "多起点循环"); has(m, "solve.py:10-19"); for (const id of chosen) has(m, `### ${id} `); assert.ok(!headings(m).includes("信念本")); lacks(m, "UNIQUE-X1"); }); test("本轮诊断结果:提案材料里每个实验的做法、实测、用时、结局,每条解释确认或否定,推出的方向;排在基线之后、研究记录之前", (t) => { const { l, chosen } = diagnosisStudy(t, "settled"); l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] }); const m = mat(l, "proposal", "a1", "s1-prop", null); const h = headings(m); assert.ok(h.indexOf("基线") < h.indexOf("本轮诊断结果") && h.indexOf("本轮诊断结果") < h.indexOf("研究记录")); const section = m.sections.find((s) => s.heading === "本轮诊断结果")!.text; const [first, second] = chosen as [string, string]; assert.ok(section.includes(`诊断运行 run-1:在基线 base(最终指标 0.96)上,一次运行做了 1 个实验,只扣 1 次`)); const line = (id: string) => section.split("\n").find((x) => x.startsWith(`- ${id} `))!; for (const part of ["实测 1.1,改进量 +0.14", "用时 31.5 秒", "其余指标:d1.gap 0.02", "测前合并范围"]) assert.ok(line(first!).includes(part), part); assert.ok(line(second!).includes("证据不足:核验不通过,第二个实验两次核验都没落实")); assert.match(section, /没做:一次诊断最多做 D 个实验,放不下/); assert.match(section, /- r1-a1-dx1(作者 a1):确认 .*;否定 /); assert.ok(section.includes("甲的诊断解释:卡在起点 UNIQUE-A1") && section.includes("乙的诊断解释:卡在收尾 UNIQUE-A2")); assert.match(section, /程序按诊断推出的方向:d\d(实测 \+0\.14)。这只是按公式推出来的参考,你可以不跟。/); assert.ok(m.records.some((x) => x.id === "diagnosis-r1" && x.text.includes("run-1") && !x.text.includes("7.771"))); assert.ok(m.readable.some((x) => x.version === "v-diag" && x.label === "第 1 轮诊断程序的代码")); }); /** 只有基线、第 1 轮提案阶段开着的账本:本轮没有诊断记录。 */ function bare(t: TestContext): Ledger { const { ledger: l } = freshLedger(t); const w = new Writer(l); l.append("code", { object: "base", version: "v-base", locations: null }); w.run("run-0", "baseline", 0, null, judged(0.96, "main")); w.open("s1-prop", "proposal", 1, ["a1", "a2"]); return l; } const diagnosisText = (m: TaskMaterial) => m.sections.find((s) => s.heading === "本轮诊断结果")!.text; test("本轮没有诊断:写原因。余额不到 2 次、配置关掉(余额够却没有决定)", (t) => { const l = bare(t); // 基线占 1 次:运行总数 2 时余额 1(不到 2),总数 10 时余额 9。 assert.equal(diagnosisText(mat(l, "proposal", "a1", "s1-prop", null, NONE, { ...study, totalRuns: 2 })), "本轮没有诊断:余额只剩 1 次,不到 2 次,要留 1 次给本轮正向"); assert.equal(diagnosisText(mat(l, "proposal", "a1", "s1-prop", null)), "本轮没有诊断:配置关掉了诊断"); }); test("本轮没有诊断:信息价值不抵成本时,原因带最好的组合净值;选中的实验一个都没剩下时,原因取事件", (t) => { const l = bare(t); const w = new Writer(l); const input = ruleInput(9, true); // 两条解释对每个实验预测一样:信息价值只剩开放预留那一点,抵不过成本 const decision = selectDiagnosis(input); assert.equal(decision.diagnose, false); w.decision("selectDiagnosis", 1, input, decision); const best = Math.max(...decision.combos.map((c) => c.net)); const text = diagnosisText(mat(l, "proposal", "a1", "s1-prop", null)); assert.equal(text, `本轮没有诊断:信息价值减成本都不大于 0,最好的组合净值 ${Number(best.toPrecision(4))}`); // 同一轮另一份账本:决定是做,但核验都没通过、修复用完。 const l2 = bare(t); const w2 = new Writer(l2); const done = selectDiagnosis(ruleInput(9, false)); w2.decision("selectDiagnosis", 1, ruleInput(9, false), done); l2.append("event", diagnosisNotRunEvent(1, "d1、d2 都没通过核验")); assert.equal(diagnosisText(mat(l2, "proposal", "a1", "s1-prop", null)), "本轮没有诊断:选中的实验核验都没通过、修复次数用完,一个都没跑:d1、d2 都没通过核验"); }); test("诊断运行在诊断结算写入之前不公开,写入之后公开(和正向的采纳、消融的结算同一类规则)", (t) => { const run = (l: Ledger) => new Board(l.records()).runs().find((x) => x.plan.run === "run-1")!; const running = diagnosisStudy(t, "run").l; assert.equal(new Board(running.records()).runPublic(run(running)), false); const settled = diagnosisStudy(t, "settled").l; assert.equal(new Board(settled.records()).runPublic(run(settled)), true); }); test("信念本的诊断历史:各轮的实验、确认与否定、方向;战绩按人加一行诊断;诊断实验的实测不算手里最好", (t) => { const { l, chosen } = diagnosisStudy(t, "settled"); l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] }); const book = mat(l, "proposal", "a1", "s1-prop", null).sections.find((s) => s.heading === "信念本")!.text; assert.ok(book.includes("**诊断历史**")); assert.ok(book.includes(`- 第 1 轮:\n 诊断运行 run-1:在基线 base`)); assert.ok(book.includes(` - ${chosen[0]} `) && book.includes("确认 d1;否定 无")); assert.ok(book.includes("手里最好 0.96"), "诊断实验实测 1.1,但诊断的答案不采纳,手里最好仍是基线"); // 战绩:a1、a2 各一条诊断:只有 d1 一个有效实验,样本各 1 个。a1 的 q5 = 1.25 > 实测 1.1 偏高;a2 的 q5 = 0.95 < 1.1 偏低。 assert.ok(book.includes("- a1,诊断 1 次:中位数偏高 1、偏低 0")); assert.ok(book.includes("- a2,诊断 1 次:中位数偏高 0、偏低 1")); }); test("信念本的诊断历史:第 2 轮诊断关着时,第 1 轮的历史还在,第 2 轮没走完不列;第 2 轮诊断任务读不到第 2 轮的诊断结果", (t) => { const { l } = diagnosisStudy(t, "settled"); l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] }); l.append("stageClose", { stageId: "s1-prop" }); // 第 1 轮采纳 r1-c1 作基线要写一大串记录,这里只看第 2 轮的诊断任务:基线仍是 base,所以直接开第 2 轮的诊断阶段。 l.append("stageOpen", { stageId: "s2-dp", stage: "diagnosisProposal", round: 2, roster: ["a1", "a2"] }); const m = mat(l, "diagnosisProposal", "a1", "s2-dp", null, { changes: null, artifacts: {} }); assert.ok(!headings(m).includes("本轮诊断结果")); const book = m.sections.find((s) => s.heading === "信念本")!.text; assert.ok(book.includes("- 第 1 轮:") && !book.includes("- 第 2 轮:")); }); test("实现正向候选的实验员:本轮有诊断时能读诊断程序(只读,注明冻结协议引用诊断实验时作参考);没有诊断时没有", (t) => { const { l } = diagnosisStudy(t, "settled"); const w = new Writer(l); l.append("stageOpen", { stageId: "s1-prop", stage: "proposal", round: 1, roster: ["a1", "a2"] }); w.protocol("r1-c1", protocolOf("r1-c1", P_A1, 1, "base", ["r1-a1-1"], ["加多起点"])); w.open("s1-impl-c1", "implementation", 1, ["x1"]); const m = mat(l, "implementation", "x1", "s1-impl-c1", "r1-c1"); assert.deepEqual(m.readable, [ { version: "v-base", label: "起点代码(本批基线)", dir: "/code/v-base" }, { version: "v-diag", label: "本轮诊断程序(只读,冻结协议引用诊断实验时作参考)", dir: "/code/v-diag" }, ]); has(m, "冻结协议引用诊断实验时作参考"); lacks(m, "UNIQUE-A1"); // 只给代码,不给诊断解释和成绩 lacks(m, "实测 1.1"); const l2 = bare(t); const w2 = new Writer(l2); w2.protocol("r1-c1", protocolOf("r1-c1", P_A1, 1, "base", ["r1-a1-1"], ["加多起点"])); w2.open("s1-impl-c1", "implementation", 1, ["x1"]); assert.deepEqual(mat(l2, "implementation", "x1", "s1-impl-c1", "r1-c1").readable.map((x) => x.version), ["v-base"]); });