feat: 诊断(任务书 18 第一块):诊断数学、四种提交、一次运行跑多个实验、账本与材料、flow 诊断各步

消融默认关,关掉时解释阶段一并跳过;选诊断实验用信息价值减成本。
eval/diagnosis/research.json 为第三块配置。vocab 字段说明的出口文字随提示词一起改,见下一次提交的 prompts/ 与同文件改动(本次含在内)。

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-05 10:36:20 -04:00
parent f39f8410e7
commit 852600df02
69 changed files with 6607 additions and 214 deletions
+217 -13
View File
@@ -4,11 +4,11 @@ import { join } from "node:path";
import { test } from "node:test";
import { stepList, type ResearchConfig } from "../src/config/index.ts";
import type { AppendKind, Content, LedgerRecord } from "../src/ledger/index.ts";
import { Board, forwardNotRunEvent, type StudyRecord } from "../src/materials/index.ts";
import { Board, diagnosisNotRunEvent, forwardNotRunEvent, type StudyRecord } from "../src/materials/index.ts";
import { runBalance } from "../src/rules/index.ts";
import { definition, exited } from "./ledger-helpers.ts";
import { judged } from "./materials-helpers.ts";
import { nextStep } from "../src/flow/next-step.ts";
import { diagnosisProgram, nextStep } from "../src/flow/next-step.ts";
import type { Json, Submission, SubmissionKind } from "../src/vocab/index.ts";
import { fullConfig } from "./config-helpers.ts";
@@ -217,10 +217,30 @@ function assigned(book: Book): Book {
return book.add("decision", { decision: "entries", round: 0, rule: "assignEntries", ruleVersion: 1, params: null, seed: null, input: null, output });
}
test("认领关了:分配;分配做完:开第 1 轮的提案阶段,名单是全部研究者", () => {
const DIAGNOSIS_STEPS = [
"diagnosisProposal", "diagnosisDedupe", "diagnosisPrediction", "selectDiagnosis",
"diagnosisImplementation", "diagnosisRun", "diagnosisSettle",
];
/** 诊断关着(fullConfig 的写法):第 round 轮开头的诊断七步各记一条按配置跳过(方法.md 第十一部分"步骤开关")。 */
function diagnosisSkipped(book: Book, round: number): Book {
for (const step of DIAGNOSIS_STEPS) {
assert.deepEqual(nextStep(book.records), { do: "skip", step: `r${round}-${step}`, reason: "配置关掉了诊断" });
book.add("event", { event: "skippedByConfig", reason: "配置关掉了诊断", objects: [`r${round}-${step}`] });
}
return book;
}
/** 第零轮做完、入口分配好,第 1 轮的诊断七步也记完跳过。 */
function inRoundOne(book: Book): Book {
return diagnosisSkipped(assigned(book), 1);
}
test("认领关了:分配;分配做完:诊断关着,七步各记按配置跳过,再开第 1 轮的提案阶段,名单是全部研究者", () => {
const book = afterRoundZero(fullConfig());
assert.deepEqual(nextStep(book.records), { do: "assignEntries" });
assigned(book);
diagnosisSkipped(book, 1);
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-proposal", stage: "proposal", round: 1, roster: R });
});
@@ -246,7 +266,7 @@ const each = (kind: SubmissionKind, objects: string[]) => R.flatMap((actor) => o
/** 第 1 轮写到 last 这一步做完为止。 */
function roundOne(last: "proposal" | "dedupe" | "firstPrediction" | "critique" | "response" | "freezeProtocol" | "secondPrediction" | "selectBatch"): Book {
const book = assigned(afterRoundZero(fullConfig()));
const book = inRoundOne(afterRoundZero(fullConfig()));
const order = ["proposal", "dedupe", "firstPrediction", "critique", "response", "freezeProtocol", "secondPrediction", "selectBatch"];
const steps: Record<string, () => void> = {
proposal: () => stage(book, "r1-proposal", "proposal", [["researcher-1", "proposal", null, P1], ["researcher-2", "proposal", null, P2], ["researcher-3", "proposal", null, P3]]),
@@ -278,7 +298,7 @@ function roundOne(last: "proposal" | "dedupe" | "firstPrediction" | "critique" |
}
test("提案阶段一人交了两条、两人没交:只派没交的两人,提案没有对象", () => {
const book = assigned(afterRoundZero(fullConfig())).open("r1-proposal", "proposal", 1, R).sub("r1-proposal", "researcher-1", "proposal", null, P1);
const book = inRoundOne(afterRoundZero(fullConfig())).open("r1-proposal", "proposal", 1, R).sub("r1-proposal", "researcher-1", "proposal", null, P1);
book.sub("r1-proposal", "researcher-1", "proposal", null, "r1-researcher-1-2");
const tasks = ["researcher-2", "researcher-3"].map((actor) => ({ actor, kind: "proposal", object: null }));
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-proposal", tasks });
@@ -350,7 +370,7 @@ test("关掉第一次预测:第 1 轮写一条按配置跳过(对象 r1-firs
const config = fullConfig();
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
config.flow.firstPrediction = false;
const book = assigned(afterRoundZero(config));
const book = inRoundOne(afterRoundZero(config));
stage(book, "r1-proposal", "proposal", [["researcher-1", "proposal", null, P1], ["researcher-2", "proposal", null, P2], ["researcher-3", "proposal", null, P3]]);
const output = { fresh: [{ candidate: "r1-c1", proposals: [P1] }, { candidate: "r1-c2", proposals: [P2, P3] }], carried: [] };
book.add("decision", { decision: "dedup", round: 1, rule: "jevSameChange", ruleVersion: 1, params: null, seed: null, input: null, output });
@@ -648,7 +668,8 @@ test("结算和押的因子写完:开收尾;收尾关闭后解封本轮选
book.add("unseal", { candidate: "r1-c1", seals: [1] });
assert.deepEqual(nextStep(book.records), { do: "unsealChosen", round: 1 });
book.add("unseal", { candidate: "r1-c2", seals: [2] });
// 余额还有(已扣 5 次):开第 2 轮。
// 余额还有(已扣 5 次):开第 2 轮,先是诊断七步的按配置跳过。
diagnosisSkipped(book, 2);
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r2-proposal", stage: "proposal", round: 2, roster: R });
});
@@ -679,7 +700,7 @@ test("余额为 0:不开下一轮,走到导出停下,原因写明", () =>
assert.deepEqual(nextStep(book.records), { do: "halt", reason: "导出还没写", objects: [] });
});
test("配置关掉消融、又采纳了多因子改法:先记按配置跳过,再在解释停下", () => {
test("配置关掉消融、又采纳了多因子改法:⑬–⑰ 整段(含解释)各记按配置跳过,再开收尾", () => {
const book = firstVerification(batchTwo(1), true, true);
for (const c of C) recordForward(book, c, 1, "valid");
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: ADOPTED } });
@@ -693,9 +714,192 @@ test("配置关掉消融、又采纳了多因子改法:先记按配置跳过
study.config = config as unknown as Json;
study.steps = stepList(config) as unknown as Json;
const reason = "配置关掉了消融";
assert.deepEqual(nextStep(book.records), { do: "skip", step: "r1-listCells", reason });
book.add("event", { event: "skippedByConfig", reason, objects: ["r1-listCells"] });
const next = nextStep(book.records);
assert.equal(next.do, "halt");
assert.match((next as { reason: string }).reason, /不交三个数的解释还没写/);
// 方法.md 4.3 ⑫:解释是 ⑭,和列格、选格、消融、结算一起不做;没有"解释停下"的情形。
for (const step of ["listCells", "explanation", "selectCells", "ablationImplementation", "ablationVerification", "ablationRun", "settle"]) {
assert.deepEqual(nextStep(book.records), { do: "skip", step: `r1-${step}`, reason });
book.add("event", { event: "skippedByConfig", reason, objects: [`r1-${step}`] });
}
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-wrapUp", stage: "wrapUp", round: 1, roster: R });
});
// ---------------------------------------------------------------------------
// 诊断(任务书 18 第 1.6 节,方法.md 4.3 诊 1–诊 4):预期动作手写,不调模型、Jev 或 Docker。
// ---------------------------------------------------------------------------
const DX = "r1-diagnosis";
const DX_ACTOR = "experimenter-r1-diagnosis";
const DX_VERIFIER = "verifier-r1-diagnosis";
/** 诊断开着的配置:A = 2、D = 4(方法.md 6.11 现行值),消融关;maxRepairs 和运行次数可调。 */
function diagnosisConfig(maxRepairs: number, runs: number): ResearchConfig {
const config = { ...fullConfig(), maxRepairs, runs };
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
config.flow = { ...config.flow, diagnosis: { enabled: true, perResearcher: 2, maxExperiments: 4 }, ablation: { enabled: false } };
return config;
}
/** 第零轮做完、入口分配好;诊断开着,第 1 轮的诊断从 r1-diagnosisProposal 开始。 */
function diagnosisRoundOne(maxRepairs: number, runs: number): Book {
return assigned(afterRoundZero(diagnosisConfig(maxRepairs, runs)));
}
const dxTasks = (kind: SubmissionKind) => R.map((actor) => ({ actor, kind, object: null }));
/** 诊 1、诊 2 阶段关闭,两个实验合并成 d1、d2,诊 3 的决定写下。 */
function selected(book: Book, diagnose: boolean, chosen: string[]): Book {
for (const [stageId, stage, kind] of [["r1-diagnosisProposal", "diagnosisProposal", "diagnosisProposal"], ["r1-diagnosisPrediction", "diagnosisPrediction", "diagnosisPrediction"]] as const) {
book.open(stageId, stage, 1, R);
for (const r of R) book.sub(stageId, r, kind, null, null);
book.add("stageClose", { stageId });
if (kind === "diagnosisProposal") {
book.add("decision", { decision: "diagnosisDedup", round: 1, rule: "jevSameExperiment", ruleVersion: 1, params: null, seed: null, input: null, output: { experiments: [] } });
}
}
return book.add("decision", { decision: "selectDiagnosis", round: 1, rule: "valueMinusCost", ruleVersion: 1, params: null, seed: null, input: null, output: { diagnose, chosen } });
}
/** 诊断实现(n = 1)或修复(n ≥ 2)阶段:实验员交一份,关闭。 */
function dxWork(book: Book, n: number): Book {
const stageId = n === 1 ? "r1-diagnosisImplementation-1" : `r1-diagnosisRepair-${n - 1}`;
book.open(stageId, "diagnosisImplementation", 1, [DX_ACTOR]);
book.sub(stageId, DX_ACTOR, n === 1 ? "diagnosisImplementation" : "diagnosisRepair", DX, null);
return book.add("stageClose", { stageId });
}
/** 第 n 次诊断核验:results 是每个选中实验"是否通过"。 */
function dxVerified(book: Book, n: number, results: Record<string, boolean>): Book {
const stageId = `r1-diagnosisVerification-${n}`;
book.open(stageId, "diagnosisVerification", 1, [DX_VERIFIER]);
const content: Submission<"diagnosisVerification"> = {
experiments: Object.entries(results).map(([experiment, ok]) => ({ experiment, followed: ok, nothingElse: true, where: ok ? "initial.py:10" : "没找到" })),
body: "无",
};
book.add("submission", { stageId, author: DX_VERIFIER, kind: "diagnosisVerification", target: DX, item: null, version: 1, content, jev: null });
return book.add("stageClose", { stageId });
}
/** 一次诊断运行:没启动,或启动了(运行记录不带判分)。 */
function dxRun(book: Book, n: number, ending: "valid" | "notStarted"): Book {
const run = `r1-diagnosis-${n}`;
book.add("runPlanned", { run, purpose: "diagnosis", round: 1, object: DX, definition });
if (ending === "notStarted") return book.add("runResult", { run, process: { ending, reason: "Docker 未启动" }, judged: null });
book.add("runStarted", { run });
return book.add("runResult", { run, process: exited, judged: null });
}
const openProposal = { do: "openStage", stageId: "r1-proposal", stage: "proposal", round: 1, roster: R };
test("诊断开着:诊 1 开阶段,每人一个任务、没有对象;交齐关闭后判重合并,再开诊断预测,同样每人一个任务", () => {
const book = diagnosisRoundOne(1, 13);
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisProposal", stage: "diagnosisProposal", round: 1, roster: R });
book.open("r1-diagnosisProposal", "diagnosisProposal", 1, R);
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisProposal", tasks: dxTasks("diagnosisProposal") });
for (const r of R) book.sub("r1-diagnosisProposal", r, "diagnosisProposal", null, null);
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-diagnosisProposal" });
book.add("stageClose", { stageId: "r1-diagnosisProposal" });
assert.deepEqual(nextStep(book.records), { do: "diagnosisDedupe", round: 1 });
book.add("decision", { decision: "diagnosisDedup", round: 1, rule: "jevSameExperiment", ruleVersion: 1, params: null, seed: null, input: null, output: { experiments: [] } });
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisPrediction", stage: "diagnosisPrediction", round: 1, roster: R });
book.open("r1-diagnosisPrediction", "diagnosisPrediction", 1, R);
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisPrediction", tasks: dxTasks("diagnosisPrediction") });
for (const r of R) book.sub("r1-diagnosisPrediction", r, "diagnosisPrediction", null, null);
book.add("stageClose", { stageId: "r1-diagnosisPrediction" });
assert.deepEqual(nextStep(book.records), { do: "selectDiagnosis", round: 1 });
});
test("余额不到 2 次:诊 1–诊 4 都不做,不开阶段、不写记录,直接开提案(运行 2 次,基线用掉 1 次,余额 1)", () => {
const book = diagnosisRoundOne(1, 2);
assert.deepEqual(nextStep(book.records), openProposal);
});
test("诊 3 定了不诊断:诊 4 的几步都不做,直接开提案;决定记录照写", () => {
const book = selected(diagnosisRoundOne(1, 13), false, []);
assert.deepEqual(nextStep(book.records), openProposal);
});
test("诊断实现与核验:一位实验员对本轮诊断,一位核验员逐个实验交结果;全通过就运行,没启动起新编号,已登记没结果要先恢复", () => {
const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]);
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisImplementation-1", stage: "diagnosisImplementation", round: 1, roster: [DX_ACTOR] });
book.open("r1-diagnosisImplementation-1", "diagnosisImplementation", 1, [DX_ACTOR]);
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisImplementation-1", tasks: [{ actor: DX_ACTOR, kind: "diagnosisImplementation", object: DX }] });
book.sub("r1-diagnosisImplementation-1", DX_ACTOR, "diagnosisImplementation", DX, null);
book.add("stageClose", { stageId: "r1-diagnosisImplementation-1" });
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisVerification-1", stage: "diagnosisVerification", round: 1, roster: [DX_VERIFIER] });
dxVerified(book, 1, { d1: true, d2: true });
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" });
dxRun(book, 1, "notStarted");
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-2" });
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 });
book.add("runPlanned", { run: "r1-diagnosis-2", purpose: "diagnosis", round: 1, object: DX, definition });
assert.throws(() => nextStep(book.records), /诊断运行 r1-diagnosis-2 登记了还没有结果:恢复入口应当已经核对过/);
book.add("runStarted", { run: "r1-diagnosis-2" });
book.add("runResult", { run: "r1-diagnosis-2", process: exited, judged: null });
assert.deepEqual(nextStep(book.records), { do: "diagnosisSettle", round: 1 });
// 没启动的那次退还名额:基线 1 次、诊断 1 次,诊断运行只扣一次。
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 2, reserved: 0, balance: 11 });
book.add("diagnosisSettlement", { round: 1, result: null });
assert.deepEqual(nextStep(book.records), openProposal);
});
test("核验有实验不通过:只退回修复,只重验一次;用完修复仍不通过的实验拿掉,跑的只是剩下的实验", () => {
const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]);
dxWork(book, 1);
dxVerified(book, 1, { d1: true, d2: false });
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisRepair-1", stage: "diagnosisImplementation", round: 1, roster: [DX_ACTOR] });
book.open("r1-diagnosisRepair-1", "diagnosisImplementation", 1, [DX_ACTOR]);
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisRepair-1", tasks: [{ actor: DX_ACTOR, kind: "diagnosisRepair", object: DX }] });
book.sub("r1-diagnosisRepair-1", DX_ACTOR, "diagnosisRepair", DX, null);
book.add("stageClose", { stageId: "r1-diagnosisRepair-1" });
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisVerification-2", stage: "diagnosisVerification", round: 1, roster: [DX_VERIFIER] });
dxVerified(book, 2, { d1: true, d2: false });
// 修复次数上限 1 已用完:d2 拿掉,d1 照跑;没有"不跑"事件。
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" });
const dropped = diagnosisProgram(new Board(book.records), 1, 1);
assert.deepEqual(dropped.experiments, ["d1"]);
assert.deepEqual(dropped.dropped.map((d) => d.experiment), ["d2"]);
assert.match(dropped.dropped[0]!.reason, /第 2 次核验仍不通过,修复次数上限 1 已用完:没照写法做;没找到/);
});
test("修复后上次通过的实验又不通过:以最近一次核验为准,修复前通过过也不能跑", () => {
const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]);
dxWork(book, 1);
dxVerified(book, 1, { d1: true, d2: false });
dxWork(book, 2);
dxVerified(book, 2, { d1: false, d2: true });
assert.deepEqual(diagnosisProgram(new Board(book.records), 1, 1).experiments, ["d2"]);
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" });
});
test("maxRepairs = 0:第一次核验不通过的实验就拿掉,不开修复阶段", () => {
const book = selected(diagnosisRoundOne(0, 13), true, ["d1", "d2"]);
dxWork(book, 1);
dxVerified(book, 1, { d1: false, d2: true });
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" });
assert.deepEqual(diagnosisProgram(new Board(book.records), 1, 0).experiments, ["d2"]);
});
test("选中的实验一个都不剩:写不跑事件(一次),不登记运行、不扣次,不结算,直接开提案", () => {
const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]);
dxWork(book, 1);
dxVerified(book, 1, { d1: false, d2: false });
dxWork(book, 2);
dxVerified(book, 2, { d1: false, d2: false });
const next = nextStep(book.records);
assert.equal(next.do, "diagnosisNotRun");
if (next.do !== "diagnosisNotRun") throw new Error("应写不跑事件");
assert.equal(next.round, 1);
assert.match(next.reason, /选中的实验第 2 次核验都不通过,修复次数上限 1 已用完:d1:.*;d2:/);
book.add("event", diagnosisNotRunEvent(1, next.reason));
assert.deepEqual(nextStep(book.records), openProposal);
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 });
});
test("诊断运行扣次以后:余额只算本轮开始之前的运行,诊断这一段的后几步不会算成不诊断", () => {
// 运行 3 次:本轮开始前余额 2,诊断运行扣 1 次后余额 1。诊断这一段的后几步照常走,不会因余额变小算成"不诊断"。
const book = selected(diagnosisRoundOne(1, 3), true, ["d1"]);
dxWork(book, 1);
dxVerified(book, 1, { d1: true });
dxRun(book, 1, "valid");
assert.equal(runBalance(3, new Board(book.records).runStates()).balance, 1);
assert.deepEqual(nextStep(book.records), { do: "diagnosisSettle", round: 1 });
});