iomgaa
852600df02
消融默认关，关掉时解释阶段一并跳过；选诊断实验用信息价值减成本。 eval/diagnosis/research.json 为第三块配置。vocab 字段说明的出口文字随提示词一起改，见下一次提交的 prompts/ 与同文件改动（本次含在内）。 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
906 lines
57 KiB
TypeScript
906 lines
57 KiB
TypeScript
import assert from "node:assert/strict";
|
||
import { readFileSync } from "node:fs";
|
||
import { join } from "node:path";
|
||
import { test } from "node:test";
|
||
import { stepList, type ResearchConfig } from "../src/config/index.ts";
|
||
import type { AppendKind, Content, LedgerRecord } from "../src/ledger/index.ts";
|
||
import { Board, diagnosisNotRunEvent, forwardNotRunEvent, type StudyRecord } from "../src/materials/index.ts";
|
||
import { runBalance } from "../src/rules/index.ts";
|
||
import { definition, exited } from "./ledger-helpers.ts";
|
||
import { judged } from "./materials-helpers.ts";
|
||
import { diagnosisProgram, nextStep } from "../src/flow/next-step.ts";
|
||
import type { Json, Submission, SubmissionKind } from "../src/vocab/index.ts";
|
||
import { fullConfig } from "./config-helpers.ts";
|
||
|
||
// flow 的"下一步做什么"（任务书 12 第六部分）：纯函数，手写账本记录，看它给出的下一步。
|
||
// 其余都由真实运行证明（任务书 12 第五部分），这里不写假模型、假 Docker。
|
||
|
||
const TASK_FACTS = JSON.parse(readFileSync(join(import.meta.dirname, "..", "tasks", "circle_packing", "task.json"), "utf8")) as Json;
|
||
const HASH = "0".repeat(64);
|
||
const measured = (value: number) => ({ status: "measured", value }) as const;
|
||
|
||
/** 调用结束记录去掉调用编号（四种结局各自去掉）。 */
|
||
type Ended = Content<"callEnded"> extends infer E ? (E extends unknown ? Omit<E, "call"> : never) : never;
|
||
|
||
/** 一本手写的账本：按写入先后编序号。内容照账本的形状写，nextStep 不再检查。 */
|
||
class Book {
|
||
readonly records: LedgerRecord[] = [];
|
||
|
||
add<K extends AppendKind>(kind: K, content: Content<K>): this {
|
||
this.records.push({ seq: this.records.length + 1, kind, hash: HASH, content } as LedgerRecord);
|
||
return this;
|
||
}
|
||
|
||
config(config: ResearchConfig): this {
|
||
const record: StudyRecord = {
|
||
config: config as unknown as Json,
|
||
steps: stepList(config) as unknown as Json,
|
||
prompts: { roles: {} as StudyRecord["prompts"]["roles"], tasks: {} as StudyRecord["prompts"]["tasks"] },
|
||
image: `sha256:${HASH}`,
|
||
task: TASK_FACTS,
|
||
seed: 1,
|
||
};
|
||
return this.add("config", record as unknown as Json);
|
||
}
|
||
|
||
baselineCode(): this {
|
||
return this.add("code", { object: "base", version: "v-base", locations: null });
|
||
}
|
||
|
||
/** 一次基线运行：没启动，或启动了、判分算数或不算数。 */
|
||
baselineRun(run: string, outcome: "notStarted" | "valid" | "invalid"): this {
|
||
const definition: Content<"runPlanned">["definition"] = {
|
||
command: ["python", "harness.py"],
|
||
workdir: "/work",
|
||
image: { digest: `sha256:${HASH}`, runtime: "runc" },
|
||
wallSeconds: 120,
|
||
cores: 1,
|
||
inheritEnv: false,
|
||
env: {},
|
||
};
|
||
this.add("runPlanned", { run, purpose: "baseline", round: 0, object: null, definition });
|
||
if (outcome === "notStarted") {
|
||
return this.add("runResult", { run, process: { ending: "notStarted", reason: "docker run 失败" }, judged: null });
|
||
}
|
||
this.add("runStarted", { run });
|
||
const process = { ending: "exited", exitCode: 0, cpuSeconds: measured(10), wallSeconds: measured(10), peakMemoryBytes: measured(1) } as const;
|
||
const judged = {
|
||
verdict: outcome,
|
||
verdictReason: outcome === "valid" ? "合法" : "不合法：圆重叠",
|
||
score: measured(0.9597642169962064),
|
||
metrics: {},
|
||
answerPath: { status: "unmeasured", reason: "基线不写" },
|
||
safeguardTriggers: { status: "unmeasured", reason: "基线不写" },
|
||
extra: null,
|
||
} as unknown as Content<"runResult">["judged"];
|
||
return this.add("runResult", { run, process, judged });
|
||
}
|
||
|
||
open(stageId: string, stage: Content<"stageOpen">["stage"], round: number, roster: string[]): this {
|
||
return this.add("stageOpen", { stageId, stage, round, roster });
|
||
}
|
||
|
||
/** 一份被收下的提交。nextStep 只看这个人在这个阶段有没有被收下的提交，种类一律写成入口。 */
|
||
entry(stageId: string, author: string): this {
|
||
const content = { entries: [{ name: "入口一", body: "从这里开始看" }, { name: "入口二", body: "从那里开始看" }] };
|
||
return this.add("submission", { stageId, author, target: null, item: null, version: 1, kind: "entry", content, jev: null });
|
||
}
|
||
|
||
/** 一份被收下的提交，内容 nextStep 不读。 */
|
||
sub(stageId: string, author: string, kind: SubmissionKind, target: string | null, item: string | null): this {
|
||
return this.add("submission", { stageId, author, target, item, version: 1, kind, content: {}, jev: null } as unknown as Content<"submission">);
|
||
}
|
||
|
||
/** 一次模型调用：开始和结束。"模型调用开始"里的材料只写 nextStep 认任务用的种类和对象。 */
|
||
call(stageId: string, actor: string, kind: SubmissionKind, object: string | null, n: number, ended: Ended): this {
|
||
const call = `${stageId}.${actor}.${kind}.${object ?? "none"}.${n}`;
|
||
this.add("callStarted", { call, actor, stageId, model: "iomgaa/glm-5.3", materials: { kind, object } });
|
||
return this.add("callEnded", { call, ...ended } as Content<"callEnded">);
|
||
}
|
||
}
|
||
|
||
const R = ["researcher-1", "researcher-2", "researcher-3"];
|
||
const entryTasks = (actors: string[]) => actors.map((actor) => ({ actor, kind: "entry", object: null }));
|
||
|
||
/** 冻结了配置、基线跑完算数的账本。 */
|
||
function afterBaseline(config: ResearchConfig = fullConfig()): Book {
|
||
return new Book().config(config).baselineCode().baselineRun("base-1", "valid");
|
||
}
|
||
|
||
test("空账本：冻结配置", () => {
|
||
assert.deepEqual(nextStep([]), { do: "freezeConfig" });
|
||
});
|
||
|
||
test("有配置、没有基线：先存基线代码，再用 base-1 跑基线", () => {
|
||
const book = new Book().config(fullConfig());
|
||
assert.deepEqual(nextStep(book.records), { do: "saveBaseline" });
|
||
book.baselineCode();
|
||
assert.deepEqual(nextStep(book.records), { do: "baselineRun", run: "base-1" });
|
||
});
|
||
|
||
test("基线最近一次是\"没启动\"：起新编号再跑", () => {
|
||
const book = new Book().config(fullConfig()).baselineCode().baselineRun("base-1", "notStarted");
|
||
assert.deepEqual(nextStep(book.records), { do: "baselineRun", run: "base-2" });
|
||
book.baselineRun("base-2", "notStarted");
|
||
assert.deepEqual(nextStep(book.records), { do: "baselineRun", run: "base-3" });
|
||
});
|
||
|
||
test("基线判为不算数：停下，原因写明", () => {
|
||
const book = new Book().config(fullConfig()).baselineCode().baselineRun("base-1", "invalid");
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "halt");
|
||
assert.match((next as { reason: string }).reason, /base-1 判为不算数（不合法：圆重叠）/);
|
||
});
|
||
|
||
test("基线算数之后：开写入口阶段，名单是全部研究者", () => {
|
||
assert.deepEqual(nextStep(afterBaseline().records), { do: "openStage", stageId: "r0-entryDraft", stage: "entryDraft", round: 0, roster: R });
|
||
});
|
||
|
||
test("阶段开着、一人已交、两人没交：派那两人", () => {
|
||
const book = afterBaseline().open("r0-entryDraft", "entryDraft", 0, R).entry("r0-entryDraft", "researcher-1");
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r0-entryDraft", tasks: entryTasks(["researcher-2", "researcher-3"]) });
|
||
});
|
||
|
||
test("都交了：关阶段；关了以后开第 1 次讨论", () => {
|
||
const book = afterBaseline().open("r0-entryDraft", "entryDraft", 0, R);
|
||
for (const r of R) book.entry("r0-entryDraft", r);
|
||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r0-entryDraft" });
|
||
book.add("stageClose", { stageId: "r0-entryDraft" });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r0-discussion1", stage: "discussion", round: 0, roster: R });
|
||
});
|
||
|
||
test("有人最近一次是\"没交\"：开发入口关着先派能派的、再停下；开着重新派", () => {
|
||
const stuck = (devEntry: boolean) =>
|
||
new Book()
|
||
.config({ ...fullConfig(), devEntry })
|
||
.baselineCode()
|
||
.baselineRun("base-1", "valid")
|
||
.open("r0-entryDraft", "entryDraft", 0, R)
|
||
.entry("r0-entryDraft", "researcher-1")
|
||
.call("r0-entryDraft", "researcher-2", "entry", null, 1, { wallSeconds: measured(30), status: "notSubmitted", reason: "模型结束了回复，没有一份提交被收下" });
|
||
// researcher-3 还没派过，先派他。
|
||
assert.deepEqual(nextStep(stuck(false).records), { do: "dispatch", stageId: "r0-entryDraft", tasks: entryTasks(["researcher-3"]) });
|
||
// researcher-3 交上以后只剩"没交"的 researcher-2：停下，附调用编号。
|
||
const closed = stuck(false).entry("r0-entryDraft", "researcher-3");
|
||
const next = nextStep(closed.records);
|
||
assert.equal(next.do, "halt");
|
||
assert.match((next as { reason: string }).reason, /^交不出合格提交/);
|
||
assert.deepEqual((next as { objects: string[] }).objects, ["r0-entryDraft.researcher-2.entry.none.1"]);
|
||
// 开发入口开着：重新派。
|
||
const dev = stuck(true).entry("r0-entryDraft", "researcher-3");
|
||
assert.deepEqual(nextStep(dev.records), { do: "dispatch", stageId: "r0-entryDraft", tasks: entryTasks(["researcher-2"]) });
|
||
});
|
||
|
||
test("有人最近一次是\"出错\"或\"进程中断\"：重新派；只看最近一次调用", () => {
|
||
const book = afterBaseline()
|
||
.open("r0-entryDraft", "entryDraft", 0, R)
|
||
.entry("r0-entryDraft", "researcher-1")
|
||
.call("r0-entryDraft", "researcher-2", "entry", null, 1, { wallSeconds: measured(5), status: "notSubmitted", reason: "没交" })
|
||
.call("r0-entryDraft", "researcher-2", "entry", null, 2, { wallSeconds: measured(5), status: "failed", reason: "模型服务报错" })
|
||
.call("r0-entryDraft", "researcher-3", "entry", null, 1, { wallSeconds: { status: "unmeasured", reason: "进程中断" }, status: "failed", reason: "进程中断" });
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r0-entryDraft", tasks: entryTasks(["researcher-2", "researcher-3"]) });
|
||
});
|
||
|
||
test("只有一位研究者：两次讨论各写一条按配置跳过，接着开认领", () => {
|
||
const one = fullConfig();
|
||
one.models.researchers = [one.models.researchers[0]!];
|
||
const book = afterBaseline(one).open("r0-entryDraft", "entryDraft", 0, ["researcher-1"]).entry("r0-entryDraft", "researcher-1");
|
||
book.add("stageClose", { stageId: "r0-entryDraft" });
|
||
const reason = "只有一位研究者，第零轮不讨论（方法.md 4.2）";
|
||
assert.deepEqual(nextStep(book.records), { do: "skip", step: "discussion1", reason });
|
||
book.add("event", { event: "skippedByConfig", reason, objects: ["discussion1"] });
|
||
assert.deepEqual(nextStep(book.records), { do: "skip", step: "discussion2", reason });
|
||
book.add("event", { event: "skippedByConfig", reason, objects: ["discussion2"] });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r0-claim", stage: "claim", round: 0, roster: ["researcher-1"] });
|
||
});
|
||
|
||
/** 第零轮做完：四个阶段都交齐关闭，分配决定已写。 */
|
||
function afterRoundZero(config: ResearchConfig): Book {
|
||
const book = afterBaseline(config);
|
||
const stages = [
|
||
["r0-entryDraft", "entryDraft"],
|
||
["r0-discussion1", "discussion"],
|
||
["r0-discussion2", "discussion"],
|
||
["r0-claim", "claim"],
|
||
] as const;
|
||
for (const [stageId, stage] of stages) {
|
||
book.open(stageId, stage, 0, R);
|
||
for (const r of R) book.entry(stageId, r);
|
||
book.add("stageClose", { stageId });
|
||
}
|
||
return book;
|
||
}
|
||
|
||
/** 分配决定。 */
|
||
function assigned(book: Book): Book {
|
||
const output = R.map((researcher, i) => ({ researcher, entry: `e-${researcher}-1`, rank: i + 1 }));
|
||
return book.add("decision", { decision: "entries", round: 0, rule: "assignEntries", ruleVersion: 1, params: null, seed: null, input: null, output });
|
||
}
|
||
|
||
const DIAGNOSIS_STEPS = [
|
||
"diagnosisProposal", "diagnosisDedupe", "diagnosisPrediction", "selectDiagnosis",
|
||
"diagnosisImplementation", "diagnosisRun", "diagnosisSettle",
|
||
];
|
||
|
||
/** 诊断关着（fullConfig 的写法）：第 round 轮开头的诊断七步各记一条按配置跳过（方法.md 第十一部分"步骤开关"）。 */
|
||
function diagnosisSkipped(book: Book, round: number): Book {
|
||
for (const step of DIAGNOSIS_STEPS) {
|
||
assert.deepEqual(nextStep(book.records), { do: "skip", step: `r${round}-${step}`, reason: "配置关掉了诊断" });
|
||
book.add("event", { event: "skippedByConfig", reason: "配置关掉了诊断", objects: [`r${round}-${step}`] });
|
||
}
|
||
return book;
|
||
}
|
||
|
||
/** 第零轮做完、入口分配好，第 1 轮的诊断七步也记完跳过。 */
|
||
function inRoundOne(book: Book): Book {
|
||
return diagnosisSkipped(assigned(book), 1);
|
||
}
|
||
|
||
test("认领关了：分配；分配做完：诊断关着，七步各记按配置跳过，再开第 1 轮的提案阶段，名单是全部研究者", () => {
|
||
const book = afterRoundZero(fullConfig());
|
||
assert.deepEqual(nextStep(book.records), { do: "assignEntries" });
|
||
assigned(book);
|
||
diagnosisSkipped(book, 1);
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-proposal", stage: "proposal", round: 1, roster: R });
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 第 1 轮：开轮到选批（任务书 14 第 2.2、2.3 节）
|
||
// 三条原提案：researcher-1 的单独成候选 r1-c1；researcher-2、researcher-3 的判为同一个改法，合成 r1-c2。
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const P1 = "r1-researcher-1-1";
|
||
const P2 = "r1-researcher-2-1";
|
||
const P3 = "r1-researcher-3-1";
|
||
const C = ["r1-c1", "r1-c2"];
|
||
|
||
/** 一个研究者阶段：开、每个任务交一份、关。tasks 是 [作者, 种类, 对象, 编号]。 */
|
||
function stage(book: Book, stageId: string, kind: Content<"stageOpen">["stage"], tasks: Array<[string, SubmissionKind, string | null, string | null]>): Book {
|
||
book.open(stageId, kind, 1, R);
|
||
for (const [author, k, target, item] of tasks) book.sub(stageId, author, k, target, item);
|
||
return book.add("stageClose", { stageId });
|
||
}
|
||
|
||
/** 每人 × 每个对象的任务。 */
|
||
const each = (kind: SubmissionKind, objects: string[]) => R.flatMap((actor) => objects.map((object) => ({ actor, kind, object })));
|
||
|
||
/** 第 1 轮写到 last 这一步做完为止。 */
|
||
function roundOne(last: "proposal" | "dedupe" | "firstPrediction" | "critique" | "response" | "freezeProtocol" | "secondPrediction" | "selectBatch"): Book {
|
||
const book = inRoundOne(afterRoundZero(fullConfig()));
|
||
const order = ["proposal", "dedupe", "firstPrediction", "critique", "response", "freezeProtocol", "secondPrediction", "selectBatch"];
|
||
const steps: Record<string, () => void> = {
|
||
proposal: () => stage(book, "r1-proposal", "proposal", [["researcher-1", "proposal", null, P1], ["researcher-2", "proposal", null, P2], ["researcher-3", "proposal", null, P3]]),
|
||
dedupe: () => {
|
||
const output = { fresh: [{ candidate: "r1-c1", proposals: [P1] }, { candidate: "r1-c2", proposals: [P2, P3] }], carried: [] };
|
||
book.add("decision", { decision: "dedup", round: 1, rule: "jevSameChange", ruleVersion: 1, params: null, seed: null, input: null, output });
|
||
},
|
||
firstPrediction: () => stage(book, "r1-firstPrediction", "firstPrediction", R.flatMap((a) => C.map((c) => [a, "firstPrediction", c, null] as [string, SubmissionKind, string, null]))),
|
||
critique: () =>
|
||
stage(book, "r1-critique", "critique", [
|
||
["researcher-1", "critique", P2, `${P2}.researcher-1.1`],
|
||
["researcher-1", "critique", P3, `${P3}.researcher-1.1`],
|
||
["researcher-2", "critique", P1, `${P1}.researcher-2.1`],
|
||
["researcher-2", "critique", P3, `${P3}.researcher-2.1`],
|
||
["researcher-3", "critique", P1, `${P1}.researcher-3.1`],
|
||
["researcher-3", "critique", P2, `${P2}.researcher-3.1`],
|
||
]),
|
||
response: () => stage(book, "r1-response", "response", [["researcher-1", "response", "r1-c1", null], ["researcher-2", "unification", "r1-c2", null]]),
|
||
freezeProtocol: () => {
|
||
// r1-c1 一个因子（采纳它不做消融），r1-c2 两个因子（采纳它走 ⑬–⑰）。
|
||
book.add("protocol", { candidate: "r1-c1", protocol: { round: 1, baseline: "base", proposals: [], factors: [{}] } });
|
||
book.add("protocol", { candidate: "r1-c2", protocol: { round: 1, baseline: "base", proposals: [], factors: [{}, {}] } });
|
||
},
|
||
secondPrediction: () => stage(book, "r1-secondPrediction", "secondPrediction", R.flatMap((a) => C.map((c) => [a, "secondPrediction", c, null] as [string, SubmissionKind, string, null]))),
|
||
selectBatch: () => book.add("decision", { decision: "batch", round: 1, rule: "bestExpected", ruleVersion: 1, params: null, seed: "s", input: null, output: { chosen: ["r1-c1"] } }),
|
||
};
|
||
for (const step of order.slice(0, order.indexOf(last) + 1)) steps[step]!();
|
||
return book;
|
||
}
|
||
|
||
test("提案阶段一人交了两条、两人没交：只派没交的两人，提案没有对象", () => {
|
||
const book = inRoundOne(afterRoundZero(fullConfig())).open("r1-proposal", "proposal", 1, R).sub("r1-proposal", "researcher-1", "proposal", null, P1);
|
||
book.sub("r1-proposal", "researcher-1", "proposal", null, "r1-researcher-1-2");
|
||
const tasks = ["researcher-2", "researcher-3"].map((actor) => ({ actor, kind: "proposal", object: null }));
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-proposal", tasks });
|
||
});
|
||
|
||
test("提案交齐关闭后：判重合并", () => {
|
||
assert.deepEqual(nextStep(roundOne("proposal").records), { do: "dedupe", round: 1 });
|
||
});
|
||
|
||
test("判重做完：开第一次预测；任务是每人 × 每个新候选（含自己的），同一个人的按候选编号排", () => {
|
||
const book = roundOne("dedupe");
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-firstPrediction", stage: "firstPrediction", round: 1, roster: R });
|
||
book.open("r1-firstPrediction", "firstPrediction", 1, R);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-firstPrediction", tasks: each("firstPrediction", C) });
|
||
});
|
||
|
||
test("同一个人的任务按对象分开算：对 r1-c1 没交的停住，r1-c2 照派；能派的派完才停下", () => {
|
||
const notSubmitted = { wallSeconds: measured(30), status: "notSubmitted", reason: "模型结束了回复，没有一份提交被收下" } as const;
|
||
const book = roundOne("dedupe").open("r1-firstPrediction", "firstPrediction", 1, R);
|
||
for (const a of ["researcher-1", "researcher-3"]) for (const c of C) book.sub("r1-firstPrediction", a, "firstPrediction", c, null);
|
||
book.call("r1-firstPrediction", "researcher-2", "firstPrediction", "r1-c1", 1, notSubmitted);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-firstPrediction", tasks: [{ actor: "researcher-2", kind: "firstPrediction", object: "r1-c2" }] });
|
||
book.sub("r1-firstPrediction", "researcher-2", "firstPrediction", "r1-c2", null);
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "halt");
|
||
assert.deepEqual((next as { objects: string[] }).objects, ["r1-firstPrediction.researcher-2.firstPrediction.r1-c1.1"]);
|
||
});
|
||
|
||
test("挑刺阶段：每人 × 每条不是自己写的原提案，没有挑自己的", () => {
|
||
const book = roundOne("firstPrediction").open("r1-critique", "critique", 1, R);
|
||
const expected = [
|
||
{ actor: "researcher-1", kind: "critique", object: P2 },
|
||
{ actor: "researcher-1", kind: "critique", object: P3 },
|
||
{ actor: "researcher-2", kind: "critique", object: P1 },
|
||
{ actor: "researcher-2", kind: "critique", object: P3 },
|
||
{ actor: "researcher-3", kind: "critique", object: P1 },
|
||
{ actor: "researcher-3", kind: "critique", object: P2 },
|
||
];
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-critique", tasks: expected });
|
||
});
|
||
|
||
test("回应阶段：单条原提案的候选派作者交回应，合并而成的派编号最小的作者交统一", () => {
|
||
const book = roundOne("critique").open("r1-response", "response", 1, R);
|
||
const expected = [
|
||
{ actor: "researcher-1", kind: "response", object: "r1-c1" },
|
||
{ actor: "researcher-2", kind: "unification", object: "r1-c2" },
|
||
];
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-response", tasks: expected });
|
||
});
|
||
|
||
test("回应阶段关闭、还没有冻结协议：写冻结协议；写完开第二次预测，任务是每人 × 池里每个候选", () => {
|
||
const book = roundOne("response");
|
||
assert.deepEqual(nextStep(book.records), { do: "freezeProtocols", round: 1 });
|
||
const frozen = roundOne("freezeProtocol");
|
||
assert.deepEqual(nextStep(frozen.records), { do: "openStage", stageId: "r1-secondPrediction", stage: "secondPrediction", round: 1, roster: R });
|
||
frozen.open("r1-secondPrediction", "secondPrediction", 1, R);
|
||
assert.deepEqual(nextStep(frozen.records), { do: "dispatch", stageId: "r1-secondPrediction", tasks: each("secondPrediction", C) });
|
||
});
|
||
|
||
test("第二次预测关闭：选批", () => {
|
||
assert.deepEqual(nextStep(roundOne("secondPrediction").records), { do: "selectBatch", round: 1 });
|
||
});
|
||
|
||
test("选批做完：开实现阶段，每个入选候选固定一位实验员", () => {
|
||
assert.deepEqual(nextStep(roundOne("selectBatch").records), { do: "openStage", stageId: "r1-implementation", stage: "implementation", round: 1, roster: ["experimenter-r1-c1"] });
|
||
});
|
||
|
||
test("关掉第一次预测：第 1 轮写一条按配置跳过（对象 r1-firstPrediction），接着开挑刺", () => {
|
||
const config = fullConfig();
|
||
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
|
||
config.flow.firstPrediction = false;
|
||
const book = inRoundOne(afterRoundZero(config));
|
||
stage(book, "r1-proposal", "proposal", [["researcher-1", "proposal", null, P1], ["researcher-2", "proposal", null, P2], ["researcher-3", "proposal", null, P3]]);
|
||
const output = { fresh: [{ candidate: "r1-c1", proposals: [P1] }, { candidate: "r1-c2", proposals: [P2, P3] }], carried: [] };
|
||
book.add("decision", { decision: "dedup", round: 1, rule: "jevSameChange", ruleVersion: 1, params: null, seed: null, input: null, output });
|
||
const reason = "配置关掉了第一次预测";
|
||
assert.deepEqual(nextStep(book.records), { do: "skip", step: "r1-firstPrediction", reason });
|
||
book.add("event", { event: "skippedByConfig", reason, objects: ["r1-firstPrediction"] });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-critique", stage: "critique", round: 1, roster: R });
|
||
});
|
||
|
||
|
||
// ⑨–⑫：预期动作手写，不调模型、Jev 或 Docker。
|
||
const X = ["experimenter-r1-c1", "experimenter-r1-c2"];
|
||
const V = ["verifier-r1-c1", "verifier-r1-c2"];
|
||
|
||
function batchTwo(maxRepairs: number): Book {
|
||
const book = roundOne("secondPrediction");
|
||
const record = book.records.find((r) => r.kind === "config")!;
|
||
(record.content as unknown as StudyRecord).config = { ...fullConfig(), maxRepairs } as unknown as Json;
|
||
return book.add("decision", { decision: "batch", round: 1, rule: "bestExpected", ruleVersion: 1, params: null, seed: "s", input: null, output: { chosen: C } });
|
||
}
|
||
|
||
function implemented(book: Book): Book {
|
||
book.open("r1-implementation", "implementation", 1, X);
|
||
for (const [i, candidate] of C.entries()) book.sub("r1-implementation", X[i]!, "implementation", candidate, null);
|
||
return book.add("stageClose", { stageId: "r1-implementation" });
|
||
}
|
||
|
||
function verified(book: Book, n: number, candidate: string, passed: boolean): Book {
|
||
// 对应真实 15-tight-2 的 submissions created_seq=173（墙约束行和右端错位）、194（修复后通过）。
|
||
// 只转录驱动程序所需的逐项 found，不伪造 Jev 的判断或模型运行。
|
||
const content: Submission<"verification"> = {
|
||
clauses: [{ clause: "c1", found: passed, where: passed ? "initial.py 墙约束行和右端已对齐" : "initial.py:225-228 与 243-246：墙约束行和右端错位" }],
|
||
factors: [{ factor: "p1", found: passed, where: "LP 墙约束" }],
|
||
safeguards: [], unassigned: [], body: passed ? "通过" : "解的不是协议写的线性规划",
|
||
};
|
||
return book.add("submission", { stageId: `r1-verification-${n}`, author: `verifier-${candidate}`, kind: "verification", target: candidate, item: null, version: 1, content, jev: null });
|
||
}
|
||
|
||
function firstVerification(book: Book, a: boolean, b: boolean): Book {
|
||
implemented(book).open("r1-verification-1", "verification", 1, V);
|
||
verified(book, 1, C[0]!, a);
|
||
verified(book, 1, C[1]!, b);
|
||
return book.add("stageClose", { stageId: "r1-verification-1" });
|
||
}
|
||
|
||
function recordForward(book: Book, candidate: string, n: number, ending: "valid" | "notStarted" | "timeout"): Book {
|
||
const run = `r1-forward-${candidate}-${n}`;
|
||
book.add("runPlanned", { run, purpose: "forward", round: 1, object: candidate, definition });
|
||
if (ending === "notStarted") return book.add("runResult", { run, process: { ending, reason: "Docker 未启动" }, judged: null });
|
||
book.add("runStarted", { run });
|
||
const process = ending === "timeout" ? { ending: "timeout" as const, wallSeconds: measured(120), cpuSeconds: measured(100), peakMemoryBytes: measured(1024) } : exited;
|
||
return book.add("runResult", { run, process, judged: ending === "valid" ? judged(1.25, "main") : null });
|
||
}
|
||
|
||
const forward = (candidate: string, n: number) => ({ do: "forwardRun", round: 1, candidate, run: `r1-forward-${candidate}-${n}` });
|
||
|
||
test("实现的任务同时可派；已提交后中断不重派，等全体交齐关阶段后才核验", () => {
|
||
const book = batchTwo(1).open("r1-implementation", "implementation", 1, X);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-implementation", tasks: C.map((object, i) => ({ actor: X[i], kind: "implementation", object })) });
|
||
book.sub("r1-implementation", X[0]!, "implementation", C[0]!, null);
|
||
book.call("r1-implementation", X[0]!, "implementation", C[0]!, 1, { wallSeconds: { status: "unmeasured", reason: "进程中断" }, status: "failed", reason: "进程中断" });
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-implementation", tasks: [{ actor: X[1], kind: "implementation", object: C[1] }] });
|
||
book.sub("r1-implementation", X[1]!, "implementation", C[1]!, null);
|
||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-implementation" });
|
||
book.add("stageClose", { stageId: "r1-implementation" });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-verification-1", stage: "verification", round: 1, roster: V });
|
||
});
|
||
|
||
test("15-tight-2 的一次修复路径：核验关后才派原实验员，修复关后只重验修过的候选", () => {
|
||
const book = implemented(batchTwo(1)).open("r1-verification-1", "verification", 1, V);
|
||
verified(book, 1, C[0]!, false);
|
||
verified(book, 1, C[1]!, true);
|
||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-verification-1" });
|
||
book.add("stageClose", { stageId: "r1-verification-1" });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-repair-1", stage: "implementation", round: 1, roster: [X[0]] });
|
||
book.open("r1-repair-1", "implementation", 1, [X[0]!]);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-repair-1", tasks: [{ actor: X[0], kind: "repair", object: C[0] }] });
|
||
book.sub("r1-repair-1", X[0]!, "repair", C[0]!, null);
|
||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-repair-1" });
|
||
book.add("stageClose", { stageId: "r1-repair-1" });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-verification-2", stage: "verification", round: 1, roster: [V[0]] });
|
||
book.open("r1-verification-2", "verification", 1, [V[0]!]);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-verification-2", tasks: [{ actor: V[0], kind: "verification", object: C[0] }] });
|
||
verified(book, 2, C[0]!, true);
|
||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-verification-2" });
|
||
book.add("stageClose", { stageId: "r1-verification-2" });
|
||
assert.deepEqual(nextStep(book.records), forward(C[0]!, 1));
|
||
});
|
||
|
||
test("修复次数从关闭的阶段数出：两次修复后仍失败，逐候选写不跑事件，不占预算", () => {
|
||
const book = firstVerification(batchTwo(2), false, false);
|
||
for (let n = 1; n <= 2; n++) {
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: `r1-repair-${n}`, stage: "implementation", round: 1, roster: X });
|
||
book.open(`r1-repair-${n}`, "implementation", 1, X);
|
||
for (const [i, candidate] of C.entries()) book.sub(`r1-repair-${n}`, X[i]!, "repair", candidate, null);
|
||
book.add("stageClose", { stageId: `r1-repair-${n}` });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: `r1-verification-${n + 1}`, stage: "verification", round: 1, roster: V });
|
||
book.open(`r1-verification-${n + 1}`, "verification", 1, V);
|
||
for (const candidate of C) verified(book, n + 1, candidate, false);
|
||
book.add("stageClose", { stageId: `r1-verification-${n + 1}` });
|
||
}
|
||
for (const candidate of C) {
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "forwardNotRun");
|
||
if (next.do !== "forwardNotRun") throw new Error("应写不跑事件");
|
||
assert.equal(next.candidate, candidate);
|
||
assert.match(next.reason, /第 3 次核验仍不通过，修复次数上限 2 已用完/);
|
||
assert.match(next.reason, /解的不是协议写的线性规划/);
|
||
book.add("event", forwardNotRunEvent(candidate, next.reason));
|
||
}
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 });
|
||
assert.deepEqual(nextStep(book.records), { do: "adopt", round: 1 });
|
||
});
|
||
|
||
test("maxRepairs=0：第一次核验失败就记不跑，另一候选仍可串行运行", () => {
|
||
const book = firstVerification(batchTwo(0), false, true);
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "forwardNotRun");
|
||
if (next.do !== "forwardNotRun") throw new Error("应写不跑事件");
|
||
book.add("event", forwardNotRunEvent(next.candidate, next.reason));
|
||
assert.deepEqual(nextStep(book.records), forward(C[1]!, 1));
|
||
});
|
||
|
||
test("正向串行：上次未启动起新编号；已登记未结束必须先恢复；超时也扣次且不重跑；采纳前不公开", () => {
|
||
const book = firstVerification(batchTwo(1), true, true);
|
||
assert.deepEqual(nextStep(book.records), forward(C[0]!, 1));
|
||
recordForward(book, C[0]!, 1, "notStarted");
|
||
assert.deepEqual(nextStep(book.records), forward(C[0]!, 2));
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 });
|
||
const run = `r1-forward-${C[0]}-2`;
|
||
book.add("runPlanned", { run, purpose: "forward", round: 1, object: C[0]!, definition });
|
||
assert.throws(() => nextStep(book.records), /登记了还没有结果：恢复入口应当已经核对过/);
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 1, balance: 11 });
|
||
book.add("runStarted", { run });
|
||
assert.throws(() => nextStep(book.records), /登记了还没有结果/);
|
||
book.add("runResult", { run, process: exited, judged: judged(1.25, "main") });
|
||
assert.deepEqual(nextStep(book.records), forward(C[1]!, 1));
|
||
recordForward(book, C[1]!, 1, "timeout");
|
||
assert.deepEqual(nextStep(book.records), { do: "adopt", round: 1 });
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 3, reserved: 0, balance: 10 });
|
||
const before = new Board(book.records);
|
||
assert.ok(before.runs().filter((r) => r.plan.purpose === "forward").every((r) => !before.runPublic(r)));
|
||
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: null } });
|
||
const after = new Board(book.records);
|
||
assert.ok(after.runs().filter((r) => r.plan.purpose === "forward").every((r) => after.runPublic(r)));
|
||
// 没有采纳：不开解释阶段，直接开收尾（方法.md 4.3 ⑫）。
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-wrapUp", stage: "wrapUp", round: 1, roster: R });
|
||
});
|
||
|
||
test("采纳只有一个因子的候选：不做消融，直接开收尾（方法.md 4.3 ⑫）", () => {
|
||
const book = firstVerification(batchTwo(1), true, true);
|
||
for (const c of C) recordForward(book, c, 1, "valid");
|
||
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: C[0]! } });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-wrapUp", stage: "wrapUp", round: 1, roster: R });
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// ⑬–⑱：消融到收尾、轮末解封、下一轮和研究结束（任务书 17 第 2.2、2.3 节）。预期动作手写，不调模型、Jev 或 Docker。
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const ADOPTED = "r1-c2"; // 两个因子：p1、p2
|
||
const CELL1 = "r1-c2-p1";
|
||
const CELL2 = "r1-c2-p2";
|
||
|
||
/** 采纳了两个因子的 r1-c2。 */
|
||
function adoptedBook(): Book {
|
||
const book = firstVerification(batchTwo(1), true, true);
|
||
for (const c of C) recordForward(book, c, 1, "valid");
|
||
return book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: ADOPTED } });
|
||
}
|
||
|
||
/** 解释阶段：开、三人各交一份、关。 */
|
||
function explained(book: Book): Book {
|
||
return stage(book, "r1-explanation", "explanation", R.map((a) => [a, "explanation", null, null]));
|
||
}
|
||
|
||
/** 选格决定。 */
|
||
function cellsChosen(book: Book, chosen: string[]): Book {
|
||
return book.add("decision", { decision: "cells", round: 1, rule: "valueMinusCost", ruleVersion: 1, params: null, seed: "s2", input: null, output: { chosen } });
|
||
}
|
||
|
||
/** 一次消融实现阶段：subs 是 [格, cannotRun, alsoOff]。 */
|
||
function implStage(book: Book, n: number, subs: Array<[string, string | null, string[]?]>): Book {
|
||
const stageId = `r1-ablationImplementation-${n}`;
|
||
book.open(stageId, "ablationImplementation", 1, subs.map(([cell]) => `experimenter-${cell}`));
|
||
for (const [cell, cannotRun, alsoOff] of subs) {
|
||
book.add("submission", { stageId, author: `experimenter-${cell}`, kind: "ablationImplementation", target: cell, item: null, version: 1, content: { alsoOff: alsoOff ?? [], cannotRun, concerns: "无" }, jev: null });
|
||
}
|
||
return book.add("stageClose", { stageId });
|
||
}
|
||
|
||
/** 一次消融核验阶段：subs 是 [格, [因子, 关没关]]。 */
|
||
function verStage(book: Book, n: number, subs: Array<[string, Array<[string, boolean]>]>): Book {
|
||
const stageId = `r1-ablationVerification-${n}`;
|
||
book.open(stageId, "ablationVerification", 1, subs.map(([cell]) => `verifier-${cell}`));
|
||
for (const [cell, offs] of subs) {
|
||
book.add("submission", { stageId, author: `verifier-${cell}`, kind: "ablationVerification", target: cell, item: null, version: 1, content: { factors: offs.map(([factor, off]) => ({ factor, off, where: "initial.py:1" })) }, jev: null });
|
||
}
|
||
return book.add("stageClose", { stageId });
|
||
}
|
||
|
||
/** 一次消融运行：没启动，或启动了、判分算数。 */
|
||
function recordAblation(book: Book, cell: string, n: number, ending: "valid" | "notStarted"): Book {
|
||
const run = `r1-ablation-${cell}-${n}`;
|
||
book.add("runPlanned", { run, purpose: "ablation", round: 1, object: cell, definition });
|
||
if (ending === "notStarted") return book.add("runResult", { run, process: { ending, reason: "Docker 未启动" }, judged: null });
|
||
book.add("runStarted", { run });
|
||
return book.add("runResult", { run, process: exited, judged: judged(1.2, "main") });
|
||
}
|
||
|
||
test("采纳多因子改法：先开解释阶段，每人一个任务、没有对象；关闭后选格", () => {
|
||
const book = adoptedBook();
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-explanation", stage: "explanation", round: 1, roster: R });
|
||
book.open("r1-explanation", "explanation", 1, R);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-explanation", tasks: R.map((actor) => ({ actor, kind: "explanation", object: null })) });
|
||
for (const a of R) book.sub("r1-explanation", a, "explanation", null, null);
|
||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-explanation" });
|
||
book.add("stageClose", { stageId: "r1-explanation" });
|
||
assert.deepEqual(nextStep(book.records), { do: "selectCells", round: 1 });
|
||
});
|
||
|
||
test("选格做完：每格固定一位实验员；交了跑不起来的格不核验、不运行，没启动的下次起新编号", () => {
|
||
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-ablationImplementation-1", stage: "ablationImplementation", round: 1, roster: [`experimenter-${CELL1}`, `experimenter-${CELL2}`] });
|
||
implStage(book, 1, [[CELL1, null], [CELL2, "离了 p2 跑不起来"]]);
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-ablationVerification-1", stage: "ablationVerification", round: 1, roster: [`verifier-${CELL1}`] });
|
||
verStage(book, 1, [[CELL1, [["p1", true], ["p2", false]]]]);
|
||
assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL1, run: `r1-ablation-${CELL1}-1` });
|
||
recordAblation(book, CELL1, 1, "notStarted");
|
||
assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL1, run: `r1-ablation-${CELL1}-2` });
|
||
recordAblation(book, CELL1, 2, "valid");
|
||
assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 });
|
||
});
|
||
|
||
test("核验不通过：退回原实验员重做，只重验重做的格；次数用完写不跑事件，不登记运行", () => {
|
||
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||
implStage(book, 1, [[CELL1, null], [CELL2, null]]);
|
||
verStage(book, 1, [[CELL1, [["p1", false], ["p2", false]]], [CELL2, [["p1", false], ["p2", true]]]]);
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-ablationImplementation-2", stage: "ablationImplementation", round: 1, roster: [`experimenter-${CELL1}`] });
|
||
implStage(book, 2, [[CELL1, null]]);
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-ablationVerification-2", stage: "ablationVerification", round: 1, roster: [`verifier-${CELL1}`] });
|
||
verStage(book, 2, [[CELL1, [["p1", false], ["p2", false]]]]);
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "cellNotRun");
|
||
if (next.do !== "cellNotRun") throw new Error("应写不跑事件");
|
||
assert.equal(next.cell, CELL1);
|
||
assert.match(next.reason, /第 2 次核验仍不通过，修复次数上限 1 已用完/);
|
||
book.add("event", { event: "cellNotRun", reason: next.reason, objects: [next.cell] });
|
||
// p1 不跑不扣次；只跑 p2。
|
||
assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL2, run: `r1-ablation-${CELL2}-1` });
|
||
});
|
||
|
||
test("几格实际关掉的因子完全相同：沿用第一格，不重复跑", () => {
|
||
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||
implStage(book, 1, [[CELL1, null, ["p2"]], [CELL2, null, ["p1"]]]);
|
||
verStage(book, 1, [[CELL1, [["p1", true], ["p2", true]]], [CELL2, [["p1", true], ["p2", true]]]]);
|
||
// 两格实际关的都是 {p1, p2}：只跑第一格。
|
||
assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL1, run: `r1-ablation-${CELL1}-1` });
|
||
recordAblation(book, CELL1, 1, "valid");
|
||
assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 });
|
||
});
|
||
|
||
test("全部交跑不起来：不核验、不运行，直接结算", () => {
|
||
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||
implStage(book, 1, [[CELL1, "跑不起来一"], [CELL2, "跑不起来二"]]);
|
||
assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 });
|
||
});
|
||
|
||
/** 结算和押的因子写完、收尾关闭、解封也写完的账本（余额还有 8 次）。 */
|
||
function afterWrapUp(): Book {
|
||
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||
implStage(book, 1, [[CELL1, null], [CELL2, null]]);
|
||
verStage(book, 1, [[CELL1, [["p1", true], ["p2", false]]], [CELL2, [["p1", false], ["p2", true]]]]);
|
||
recordAblation(book, CELL1, 1, "valid");
|
||
recordAblation(book, CELL2, 1, "valid");
|
||
book.add("settlement", { round: 1, result: { candidate: ADOPTED, input: null, output: null } });
|
||
book.add("decision", { decision: "bet", round: 1, rule: "chooseBet", ruleVersion: 1, params: null, seed: null, input: null, output: null });
|
||
return stage(book, "r1-wrapUp", "wrapUp", R.map((a) => [a, "wrapUp", null, null]));
|
||
}
|
||
|
||
test("结算和押的因子写完：开收尾；收尾关闭后解封本轮选中的全部候选，然后开下一轮", () => {
|
||
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||
implStage(book, 1, [[CELL1, null], [CELL2, null]]);
|
||
verStage(book, 1, [[CELL1, [["p1", true], ["p2", false]]], [CELL2, [["p1", false], ["p2", true]]]]);
|
||
recordAblation(book, CELL1, 1, "valid");
|
||
assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL2, run: `r1-ablation-${CELL2}-1` });
|
||
recordAblation(book, CELL2, 1, "valid");
|
||
assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 });
|
||
book.add("settlement", { round: 1, result: { candidate: ADOPTED, input: null, output: null } });
|
||
book.add("decision", { decision: "bet", round: 1, rule: "chooseBet", ruleVersion: 1, params: null, seed: null, input: null, output: null });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-wrapUp", stage: "wrapUp", round: 1, roster: R });
|
||
stage(book, "r1-wrapUp", "wrapUp", R.map((a) => [a, "wrapUp", null, null]));
|
||
// 收尾关闭后：解封本轮选批选中的 r1-c1、r1-c2；还有一个没解封就继续。
|
||
assert.deepEqual(nextStep(book.records), { do: "unsealChosen", round: 1 });
|
||
book.add("unseal", { candidate: "r1-c1", seals: [1] });
|
||
assert.deepEqual(nextStep(book.records), { do: "unsealChosen", round: 1 });
|
||
book.add("unseal", { candidate: "r1-c2", seals: [2] });
|
||
// 余额还有（已扣 5 次）：开第 2 轮，先是诊断七步的按配置跳过。
|
||
diagnosisSkipped(book, 2);
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r2-proposal", stage: "proposal", round: 2, roster: R });
|
||
});
|
||
|
||
test("最后一次正向用掉全部余额：进行中的这一轮照常走完采纳与收尾，不开下一轮", () => {
|
||
// 复现 full17-b 第 5 轮的缺陷：两个正向运行把余额用到 0 后，采纳决定、收尾和解封被整轮跳过，
|
||
// 直接走到导出停下。余额为 0 只是不开【下一轮】（方法.md 第七部分“什么时候开下一轮”）。
|
||
const book = firstVerification(batchTwo(1), true, true);
|
||
for (const c of C) recordForward(book, c, 1, "valid");
|
||
// 已扣 3 次（基线 1、正向 2）：把冻结配置的运行次数改成 3，余额为 0。
|
||
const record = book.records.find((r) => r.kind === "config")!;
|
||
const study = record.content as unknown as StudyRecord;
|
||
study.config = { ...fullConfig(), runs: 3 } as unknown as Json;
|
||
// 这一轮已经开始（r1- 的阶段都在账上）：下一步是写采纳决定，不是停下。
|
||
assert.deepEqual(nextStep(book.records), { do: "adopt", round: 1 });
|
||
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: null } });
|
||
// 没有采纳：不做消融，照样开收尾阶段。
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-wrapUp", stage: "wrapUp", round: 1, roster: R });
|
||
});
|
||
|
||
test("余额为 0：不开下一轮，走到导出停下，原因写明", () => {
|
||
const book = afterWrapUp();
|
||
book.add("unseal", { candidate: "r1-c1", seals: [1] });
|
||
book.add("unseal", { candidate: "r1-c2", seals: [2] });
|
||
// 已扣 5 次（基线 1、正向 2、消融 2）：把冻结配置的运行次数改成 5，余额为 0。
|
||
const record = book.records.find((r) => r.kind === "config")!;
|
||
const study = record.content as unknown as StudyRecord;
|
||
study.config = { ...fullConfig(), runs: 5 } as unknown as Json;
|
||
assert.deepEqual(nextStep(book.records), { do: "halt", reason: "导出还没写", objects: [] });
|
||
});
|
||
|
||
test("配置关掉消融、又采纳了多因子改法：⑬–⑰ 整段（含解释）各记按配置跳过，再开收尾", () => {
|
||
const book = firstVerification(batchTwo(1), true, true);
|
||
for (const c of C) recordForward(book, c, 1, "valid");
|
||
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: ADOPTED } });
|
||
// 冻结的配置改为关掉消融：配置和步骤清单一起换（步骤清单在研究开始时冻结）。
|
||
const config = fullConfig();
|
||
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
|
||
// 配置关掉消融时不写 cellRule（它只有开着消融才用得到）。
|
||
config.flow.ablation = { enabled: false };
|
||
const record = book.records.find((r) => r.kind === "config")!;
|
||
const study = record.content as unknown as StudyRecord;
|
||
study.config = config as unknown as Json;
|
||
study.steps = stepList(config) as unknown as Json;
|
||
const reason = "配置关掉了消融";
|
||
// 方法.md 4.3 ⑫：解释是 ⑭，和列格、选格、消融、结算一起不做；没有"解释停下"的情形。
|
||
for (const step of ["listCells", "explanation", "selectCells", "ablationImplementation", "ablationVerification", "ablationRun", "settle"]) {
|
||
assert.deepEqual(nextStep(book.records), { do: "skip", step: `r1-${step}`, reason });
|
||
book.add("event", { event: "skippedByConfig", reason, objects: [`r1-${step}`] });
|
||
}
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-wrapUp", stage: "wrapUp", round: 1, roster: R });
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 诊断（任务书 18 第 1.6 节，方法.md 4.3 诊 1–诊 4）：预期动作手写，不调模型、Jev 或 Docker。
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const DX = "r1-diagnosis";
|
||
const DX_ACTOR = "experimenter-r1-diagnosis";
|
||
const DX_VERIFIER = "verifier-r1-diagnosis";
|
||
|
||
/** 诊断开着的配置：A = 2、D = 4（方法.md 6.11 现行值），消融关；maxRepairs 和运行次数可调。 */
|
||
function diagnosisConfig(maxRepairs: number, runs: number): ResearchConfig {
|
||
const config = { ...fullConfig(), maxRepairs, runs };
|
||
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
|
||
config.flow = { ...config.flow, diagnosis: { enabled: true, perResearcher: 2, maxExperiments: 4 }, ablation: { enabled: false } };
|
||
return config;
|
||
}
|
||
|
||
/** 第零轮做完、入口分配好；诊断开着，第 1 轮的诊断从 r1-diagnosisProposal 开始。 */
|
||
function diagnosisRoundOne(maxRepairs: number, runs: number): Book {
|
||
return assigned(afterRoundZero(diagnosisConfig(maxRepairs, runs)));
|
||
}
|
||
|
||
const dxTasks = (kind: SubmissionKind) => R.map((actor) => ({ actor, kind, object: null }));
|
||
|
||
/** 诊 1、诊 2 阶段关闭，两个实验合并成 d1、d2，诊 3 的决定写下。 */
|
||
function selected(book: Book, diagnose: boolean, chosen: string[]): Book {
|
||
for (const [stageId, stage, kind] of [["r1-diagnosisProposal", "diagnosisProposal", "diagnosisProposal"], ["r1-diagnosisPrediction", "diagnosisPrediction", "diagnosisPrediction"]] as const) {
|
||
book.open(stageId, stage, 1, R);
|
||
for (const r of R) book.sub(stageId, r, kind, null, null);
|
||
book.add("stageClose", { stageId });
|
||
if (kind === "diagnosisProposal") {
|
||
book.add("decision", { decision: "diagnosisDedup", round: 1, rule: "jevSameExperiment", ruleVersion: 1, params: null, seed: null, input: null, output: { experiments: [] } });
|
||
}
|
||
}
|
||
return book.add("decision", { decision: "selectDiagnosis", round: 1, rule: "valueMinusCost", ruleVersion: 1, params: null, seed: null, input: null, output: { diagnose, chosen } });
|
||
}
|
||
|
||
/** 诊断实现（n = 1）或修复（n ≥ 2）阶段：实验员交一份，关闭。 */
|
||
function dxWork(book: Book, n: number): Book {
|
||
const stageId = n === 1 ? "r1-diagnosisImplementation-1" : `r1-diagnosisRepair-${n - 1}`;
|
||
book.open(stageId, "diagnosisImplementation", 1, [DX_ACTOR]);
|
||
book.sub(stageId, DX_ACTOR, n === 1 ? "diagnosisImplementation" : "diagnosisRepair", DX, null);
|
||
return book.add("stageClose", { stageId });
|
||
}
|
||
|
||
/** 第 n 次诊断核验：results 是每个选中实验"是否通过"。 */
|
||
function dxVerified(book: Book, n: number, results: Record<string, boolean>): Book {
|
||
const stageId = `r1-diagnosisVerification-${n}`;
|
||
book.open(stageId, "diagnosisVerification", 1, [DX_VERIFIER]);
|
||
const content: Submission<"diagnosisVerification"> = {
|
||
experiments: Object.entries(results).map(([experiment, ok]) => ({ experiment, followed: ok, nothingElse: true, where: ok ? "initial.py:10" : "没找到" })),
|
||
body: "无",
|
||
};
|
||
book.add("submission", { stageId, author: DX_VERIFIER, kind: "diagnosisVerification", target: DX, item: null, version: 1, content, jev: null });
|
||
return book.add("stageClose", { stageId });
|
||
}
|
||
|
||
/** 一次诊断运行：没启动，或启动了（运行记录不带判分）。 */
|
||
function dxRun(book: Book, n: number, ending: "valid" | "notStarted"): Book {
|
||
const run = `r1-diagnosis-${n}`;
|
||
book.add("runPlanned", { run, purpose: "diagnosis", round: 1, object: DX, definition });
|
||
if (ending === "notStarted") return book.add("runResult", { run, process: { ending, reason: "Docker 未启动" }, judged: null });
|
||
book.add("runStarted", { run });
|
||
return book.add("runResult", { run, process: exited, judged: null });
|
||
}
|
||
|
||
const openProposal = { do: "openStage", stageId: "r1-proposal", stage: "proposal", round: 1, roster: R };
|
||
|
||
test("诊断开着：诊 1 开阶段，每人一个任务、没有对象；交齐关闭后判重合并，再开诊断预测，同样每人一个任务", () => {
|
||
const book = diagnosisRoundOne(1, 13);
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisProposal", stage: "diagnosisProposal", round: 1, roster: R });
|
||
book.open("r1-diagnosisProposal", "diagnosisProposal", 1, R);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisProposal", tasks: dxTasks("diagnosisProposal") });
|
||
for (const r of R) book.sub("r1-diagnosisProposal", r, "diagnosisProposal", null, null);
|
||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-diagnosisProposal" });
|
||
book.add("stageClose", { stageId: "r1-diagnosisProposal" });
|
||
assert.deepEqual(nextStep(book.records), { do: "diagnosisDedupe", round: 1 });
|
||
book.add("decision", { decision: "diagnosisDedup", round: 1, rule: "jevSameExperiment", ruleVersion: 1, params: null, seed: null, input: null, output: { experiments: [] } });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisPrediction", stage: "diagnosisPrediction", round: 1, roster: R });
|
||
book.open("r1-diagnosisPrediction", "diagnosisPrediction", 1, R);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisPrediction", tasks: dxTasks("diagnosisPrediction") });
|
||
for (const r of R) book.sub("r1-diagnosisPrediction", r, "diagnosisPrediction", null, null);
|
||
book.add("stageClose", { stageId: "r1-diagnosisPrediction" });
|
||
assert.deepEqual(nextStep(book.records), { do: "selectDiagnosis", round: 1 });
|
||
});
|
||
|
||
test("余额不到 2 次：诊 1–诊 4 都不做，不开阶段、不写记录，直接开提案（运行 2 次，基线用掉 1 次，余额 1）", () => {
|
||
const book = diagnosisRoundOne(1, 2);
|
||
assert.deepEqual(nextStep(book.records), openProposal);
|
||
});
|
||
|
||
test("诊 3 定了不诊断：诊 4 的几步都不做，直接开提案；决定记录照写", () => {
|
||
const book = selected(diagnosisRoundOne(1, 13), false, []);
|
||
assert.deepEqual(nextStep(book.records), openProposal);
|
||
});
|
||
|
||
test("诊断实现与核验：一位实验员对本轮诊断，一位核验员逐个实验交结果；全通过就运行，没启动起新编号，已登记没结果要先恢复", () => {
|
||
const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]);
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisImplementation-1", stage: "diagnosisImplementation", round: 1, roster: [DX_ACTOR] });
|
||
book.open("r1-diagnosisImplementation-1", "diagnosisImplementation", 1, [DX_ACTOR]);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisImplementation-1", tasks: [{ actor: DX_ACTOR, kind: "diagnosisImplementation", object: DX }] });
|
||
book.sub("r1-diagnosisImplementation-1", DX_ACTOR, "diagnosisImplementation", DX, null);
|
||
book.add("stageClose", { stageId: "r1-diagnosisImplementation-1" });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisVerification-1", stage: "diagnosisVerification", round: 1, roster: [DX_VERIFIER] });
|
||
dxVerified(book, 1, { d1: true, d2: true });
|
||
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" });
|
||
dxRun(book, 1, "notStarted");
|
||
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-2" });
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 });
|
||
book.add("runPlanned", { run: "r1-diagnosis-2", purpose: "diagnosis", round: 1, object: DX, definition });
|
||
assert.throws(() => nextStep(book.records), /诊断运行 r1-diagnosis-2 登记了还没有结果：恢复入口应当已经核对过/);
|
||
book.add("runStarted", { run: "r1-diagnosis-2" });
|
||
book.add("runResult", { run: "r1-diagnosis-2", process: exited, judged: null });
|
||
assert.deepEqual(nextStep(book.records), { do: "diagnosisSettle", round: 1 });
|
||
// 没启动的那次退还名额：基线 1 次、诊断 1 次，诊断运行只扣一次。
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 2, reserved: 0, balance: 11 });
|
||
book.add("diagnosisSettlement", { round: 1, result: null });
|
||
assert.deepEqual(nextStep(book.records), openProposal);
|
||
});
|
||
|
||
test("核验有实验不通过：只退回修复，只重验一次；用完修复仍不通过的实验拿掉，跑的只是剩下的实验", () => {
|
||
const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]);
|
||
dxWork(book, 1);
|
||
dxVerified(book, 1, { d1: true, d2: false });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisRepair-1", stage: "diagnosisImplementation", round: 1, roster: [DX_ACTOR] });
|
||
book.open("r1-diagnosisRepair-1", "diagnosisImplementation", 1, [DX_ACTOR]);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-diagnosisRepair-1", tasks: [{ actor: DX_ACTOR, kind: "diagnosisRepair", object: DX }] });
|
||
book.sub("r1-diagnosisRepair-1", DX_ACTOR, "diagnosisRepair", DX, null);
|
||
book.add("stageClose", { stageId: "r1-diagnosisRepair-1" });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-diagnosisVerification-2", stage: "diagnosisVerification", round: 1, roster: [DX_VERIFIER] });
|
||
dxVerified(book, 2, { d1: true, d2: false });
|
||
// 修复次数上限 1 已用完：d2 拿掉，d1 照跑；没有"不跑"事件。
|
||
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" });
|
||
const dropped = diagnosisProgram(new Board(book.records), 1, 1);
|
||
assert.deepEqual(dropped.experiments, ["d1"]);
|
||
assert.deepEqual(dropped.dropped.map((d) => d.experiment), ["d2"]);
|
||
assert.match(dropped.dropped[0]!.reason, /第 2 次核验仍不通过，修复次数上限 1 已用完：没照写法做；没找到/);
|
||
});
|
||
|
||
test("修复后上次通过的实验又不通过：以最近一次核验为准，修复前通过过也不能跑", () => {
|
||
const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]);
|
||
dxWork(book, 1);
|
||
dxVerified(book, 1, { d1: true, d2: false });
|
||
dxWork(book, 2);
|
||
dxVerified(book, 2, { d1: false, d2: true });
|
||
assert.deepEqual(diagnosisProgram(new Board(book.records), 1, 1).experiments, ["d2"]);
|
||
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" });
|
||
});
|
||
|
||
test("maxRepairs = 0：第一次核验不通过的实验就拿掉，不开修复阶段", () => {
|
||
const book = selected(diagnosisRoundOne(0, 13), true, ["d1", "d2"]);
|
||
dxWork(book, 1);
|
||
dxVerified(book, 1, { d1: false, d2: true });
|
||
assert.deepEqual(nextStep(book.records), { do: "diagnosisRun", round: 1, run: "r1-diagnosis-1" });
|
||
assert.deepEqual(diagnosisProgram(new Board(book.records), 1, 0).experiments, ["d2"]);
|
||
});
|
||
|
||
test("选中的实验一个都不剩：写不跑事件（一次），不登记运行、不扣次，不结算，直接开提案", () => {
|
||
const book = selected(diagnosisRoundOne(1, 13), true, ["d1", "d2"]);
|
||
dxWork(book, 1);
|
||
dxVerified(book, 1, { d1: false, d2: false });
|
||
dxWork(book, 2);
|
||
dxVerified(book, 2, { d1: false, d2: false });
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "diagnosisNotRun");
|
||
if (next.do !== "diagnosisNotRun") throw new Error("应写不跑事件");
|
||
assert.equal(next.round, 1);
|
||
assert.match(next.reason, /选中的实验第 2 次核验都不通过，修复次数上限 1 已用完：d1：.*；d2：/);
|
||
book.add("event", diagnosisNotRunEvent(1, next.reason));
|
||
assert.deepEqual(nextStep(book.records), openProposal);
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 });
|
||
});
|
||
|
||
test("诊断运行扣次以后：余额只算本轮开始之前的运行，诊断这一段的后几步不会算成不诊断", () => {
|
||
// 运行 3 次：本轮开始前余额 2，诊断运行扣 1 次后余额 1。诊断这一段的后几步照常走，不会因余额变小算成"不诊断"。
|
||
const book = selected(diagnosisRoundOne(1, 3), true, ["d1"]);
|
||
dxWork(book, 1);
|
||
dxVerified(book, 1, { d1: true });
|
||
dxRun(book, 1, "valid");
|
||
assert.equal(runBalance(3, new Board(book.records).runStates()).balance, 1);
|
||
assert.deepEqual(nextStep(book.records), { do: "diagnosisSettle", round: 1 });
|
||
});
|