1669 lines
48 KiB
TypeScript
1669 lines
48 KiB
TypeScript
import assert from "node:assert/strict";
|
||
import { readFileSync } from "node:fs";
|
||
import { join } from "node:path";
|
||
import { test } from "node:test";
|
||
import { stepList, type ResearchConfig } from "../src/config/index.ts";
|
||
import type { AppendKind, Content, LedgerRecord } from "../src/ledger/index.ts";
|
||
import {
|
||
Board,
|
||
forwardNotRunEvent,
|
||
type StudyRecord,
|
||
} from "../src/materials/index.ts";
|
||
import { runBalance } from "../src/rules/index.ts";
|
||
import { definition, exited } from "./ledger-helpers.ts";
|
||
import { judged } from "./materials-helpers.ts";
|
||
import { nextStep } from "../src/flow/next-step.ts";
|
||
import type { Json, Submission, SubmissionKind } from "../src/vocab/index.ts";
|
||
import { fullConfig } from "./config-helpers.ts";
|
||
|
||
// flow 的"下一步做什么"（任务书 12 第六部分）：纯函数，手写账本记录，看它给出的下一步。
|
||
// 其余都由真实运行证明（任务书 12 第五部分），这里不写假模型、假 Docker。
|
||
|
||
const TASK_FACTS = JSON.parse(
|
||
readFileSync(
|
||
join(import.meta.dirname, "..", "tasks", "circle_packing", "task.json"),
|
||
"utf8",
|
||
),
|
||
) as Json;
|
||
const HASH = "0".repeat(64);
|
||
const measured = (value: number) => ({ status: "measured", value }) as const;
|
||
|
||
/** 调用结束记录去掉调用编号（四种结局各自去掉）。 */
|
||
type Ended =
|
||
Content<"callEnded"> extends infer E
|
||
? E extends unknown
|
||
? Omit<E, "call">
|
||
: never
|
||
: never;
|
||
|
||
/** 一本手写的账本：按写入先后编序号。内容照账本的形状写，nextStep 不再检查。 */
|
||
class Book {
|
||
readonly records: LedgerRecord[] = [];
|
||
|
||
add<K extends AppendKind>(kind: K, content: Content<K>): this {
|
||
this.records.push({
|
||
seq: this.records.length + 1,
|
||
kind,
|
||
hash: HASH,
|
||
content,
|
||
} as LedgerRecord);
|
||
return this;
|
||
}
|
||
|
||
config(config: ResearchConfig): this {
|
||
const record: StudyRecord = {
|
||
config: config as unknown as Json,
|
||
steps: stepList(config) as unknown as Json,
|
||
prompts: {
|
||
roles: {} as StudyRecord["prompts"]["roles"],
|
||
tasks: {} as StudyRecord["prompts"]["tasks"],
|
||
},
|
||
image: `sha256:${HASH}`,
|
||
task: TASK_FACTS,
|
||
seed: 1,
|
||
};
|
||
return this.add("config", record as unknown as Json);
|
||
}
|
||
|
||
baselineCode(): this {
|
||
return this.add("code", {
|
||
object: "base",
|
||
version: "v-base",
|
||
locations: null,
|
||
});
|
||
}
|
||
|
||
/** 一次基线运行：没启动，或启动了、判分算数或不算数。 */
|
||
baselineRun(run: string, outcome: "notStarted" | "valid" | "invalid"): this {
|
||
const definition: Content<"runPlanned">["definition"] = {
|
||
command: ["python", "harness.py"],
|
||
workdir: "/work",
|
||
image: { digest: `sha256:${HASH}`, runtime: "runc" },
|
||
wallSeconds: 120,
|
||
cores: 1,
|
||
inheritEnv: false,
|
||
env: {},
|
||
};
|
||
this.add("runPlanned", {
|
||
run,
|
||
purpose: "baseline",
|
||
round: 0,
|
||
object: null,
|
||
definition,
|
||
});
|
||
if (outcome === "notStarted") {
|
||
return this.add("runResult", {
|
||
run,
|
||
process: { ending: "notStarted", reason: "docker run 失败" },
|
||
judged: null,
|
||
});
|
||
}
|
||
this.add("runStarted", { run });
|
||
const process = {
|
||
ending: "exited",
|
||
exitCode: 0,
|
||
cpuSeconds: measured(10),
|
||
wallSeconds: measured(10),
|
||
peakMemoryBytes: measured(1),
|
||
} as const;
|
||
const judged = {
|
||
verdict: outcome,
|
||
verdictReason: outcome === "valid" ? "合法" : "不合法：圆重叠",
|
||
score: measured(0.9597642169962064),
|
||
metrics: {},
|
||
answerPath: { status: "unmeasured", reason: "基线不写" },
|
||
safeguardTriggers: { status: "unmeasured", reason: "基线不写" },
|
||
extra: null,
|
||
} as unknown as Content<"runResult">["judged"];
|
||
return this.add("runResult", { run, process, judged });
|
||
}
|
||
|
||
open(
|
||
stageId: string,
|
||
stage: Content<"stageOpen">["stage"],
|
||
round: number,
|
||
roster: string[],
|
||
): this {
|
||
return this.add("stageOpen", { stageId, stage, round, roster });
|
||
}
|
||
|
||
/** 一份被收下的提交。nextStep 只看这个人在这个阶段有没有被收下的提交，种类一律写成入口。 */
|
||
entry(stageId: string, author: string): this {
|
||
const content = {
|
||
entries: [
|
||
{ name: "入口一", body: "从这里开始看" },
|
||
{ name: "入口二", body: "从那里开始看" },
|
||
],
|
||
};
|
||
return this.add("submission", {
|
||
stageId,
|
||
author,
|
||
target: null,
|
||
item: null,
|
||
version: 1,
|
||
kind: "entry",
|
||
content,
|
||
jev: null,
|
||
});
|
||
}
|
||
|
||
/** 一份被收下的提交，内容 nextStep 不读。 */
|
||
sub(
|
||
stageId: string,
|
||
author: string,
|
||
kind: SubmissionKind,
|
||
target: string | null,
|
||
item: string | null,
|
||
): this {
|
||
return this.add("submission", {
|
||
stageId,
|
||
author,
|
||
target,
|
||
item,
|
||
version: 1,
|
||
kind,
|
||
content: {},
|
||
jev: null,
|
||
} as unknown as Content<"submission">);
|
||
}
|
||
|
||
/** 一次模型调用：开始和结束。"模型调用开始"里的材料只写 nextStep 认任务用的种类和对象。 */
|
||
call(
|
||
stageId: string,
|
||
actor: string,
|
||
kind: SubmissionKind,
|
||
object: string | null,
|
||
n: number,
|
||
ended: Ended,
|
||
): this {
|
||
const call = `${stageId}.${actor}.${kind}.${object ?? "none"}.${n}`;
|
||
this.add("callStarted", {
|
||
call,
|
||
actor,
|
||
stageId,
|
||
model: "iomgaa/glm-5.3",
|
||
materials: { kind, object },
|
||
});
|
||
return this.add("callEnded", { call, ...ended } as Content<"callEnded">);
|
||
}
|
||
}
|
||
|
||
const R = ["researcher-1", "researcher-2", "researcher-3"];
|
||
const entryTasks = (actors: string[]) =>
|
||
actors.map((actor) => ({ actor, kind: "entry", object: null }));
|
||
|
||
/** 冻结了配置、基线跑完算数的账本。 */
|
||
function afterBaseline(config: ResearchConfig = fullConfig()): Book {
|
||
return new Book()
|
||
.config(config)
|
||
.baselineCode()
|
||
.baselineRun("base-1", "valid");
|
||
}
|
||
|
||
test("空账本：冻结配置", () => {
|
||
assert.deepEqual(nextStep([]), { do: "freezeConfig" });
|
||
});
|
||
|
||
test("有配置、没有基线：先存基线代码，再用 base-1 跑基线", () => {
|
||
const book = new Book().config(fullConfig());
|
||
assert.deepEqual(nextStep(book.records), { do: "saveBaseline" });
|
||
book.baselineCode();
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "baselineRun",
|
||
run: "base-1",
|
||
});
|
||
});
|
||
|
||
test('基线最近一次是"没启动"：起新编号再跑', () => {
|
||
const book = new Book()
|
||
.config(fullConfig())
|
||
.baselineCode()
|
||
.baselineRun("base-1", "notStarted");
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "baselineRun",
|
||
run: "base-2",
|
||
});
|
||
book.baselineRun("base-2", "notStarted");
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "baselineRun",
|
||
run: "base-3",
|
||
});
|
||
});
|
||
|
||
test("基线判为不算数：停下，原因写明", () => {
|
||
const book = new Book()
|
||
.config(fullConfig())
|
||
.baselineCode()
|
||
.baselineRun("base-1", "invalid");
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "halt");
|
||
assert.match(
|
||
(next as { reason: string }).reason,
|
||
/base-1 判为不算数（不合法：圆重叠）/,
|
||
);
|
||
});
|
||
|
||
test("基线算数之后：开写入口阶段，名单是全部研究者", () => {
|
||
assert.deepEqual(nextStep(afterBaseline().records), {
|
||
do: "openStage",
|
||
stageId: "r0-entryDraft",
|
||
stage: "entryDraft",
|
||
round: 0,
|
||
roster: R,
|
||
});
|
||
});
|
||
|
||
test("阶段开着、一人已交、两人没交：派那两人", () => {
|
||
const book = afterBaseline()
|
||
.open("r0-entryDraft", "entryDraft", 0, R)
|
||
.entry("r0-entryDraft", "researcher-1");
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "dispatch",
|
||
stageId: "r0-entryDraft",
|
||
tasks: entryTasks(["researcher-2", "researcher-3"]),
|
||
});
|
||
});
|
||
|
||
test("都交了：关阶段；关了以后开第 1 次讨论", () => {
|
||
const book = afterBaseline().open("r0-entryDraft", "entryDraft", 0, R);
|
||
for (const r of R) book.entry("r0-entryDraft", r);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "closeStage",
|
||
stageId: "r0-entryDraft",
|
||
});
|
||
book.add("stageClose", { stageId: "r0-entryDraft" });
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r0-discussion1",
|
||
stage: "discussion",
|
||
round: 0,
|
||
roster: R,
|
||
});
|
||
});
|
||
|
||
test('有人最近一次是"没交"：开发入口关着先派能派的、再停下；开着重新派', () => {
|
||
const stuck = (devEntry: boolean) =>
|
||
new Book()
|
||
.config({ ...fullConfig(), devEntry })
|
||
.baselineCode()
|
||
.baselineRun("base-1", "valid")
|
||
.open("r0-entryDraft", "entryDraft", 0, R)
|
||
.entry("r0-entryDraft", "researcher-1")
|
||
.call("r0-entryDraft", "researcher-2", "entry", null, 1, {
|
||
wallSeconds: measured(30),
|
||
status: "notSubmitted",
|
||
reason: "模型结束了回复，没有一份提交被收下",
|
||
});
|
||
// researcher-3 还没派过，先派他。
|
||
assert.deepEqual(nextStep(stuck(false).records), {
|
||
do: "dispatch",
|
||
stageId: "r0-entryDraft",
|
||
tasks: entryTasks(["researcher-3"]),
|
||
});
|
||
// researcher-3 交上以后只剩"没交"的 researcher-2：停下，附调用编号。
|
||
const closed = stuck(false).entry("r0-entryDraft", "researcher-3");
|
||
const next = nextStep(closed.records);
|
||
assert.equal(next.do, "halt");
|
||
assert.match((next as { reason: string }).reason, /^交不出合格提交/);
|
||
assert.deepEqual((next as { objects: string[] }).objects, [
|
||
"r0-entryDraft.researcher-2.entry.none.1",
|
||
]);
|
||
// 开发入口开着：重新派。
|
||
const dev = stuck(true).entry("r0-entryDraft", "researcher-3");
|
||
assert.deepEqual(nextStep(dev.records), {
|
||
do: "dispatch",
|
||
stageId: "r0-entryDraft",
|
||
tasks: entryTasks(["researcher-2"]),
|
||
});
|
||
});
|
||
|
||
test('有人最近一次是"出错"或"进程中断"：重新派；只看最近一次调用', () => {
|
||
const book = afterBaseline()
|
||
.open("r0-entryDraft", "entryDraft", 0, R)
|
||
.entry("r0-entryDraft", "researcher-1")
|
||
.call("r0-entryDraft", "researcher-2", "entry", null, 1, {
|
||
wallSeconds: measured(5),
|
||
status: "notSubmitted",
|
||
reason: "没交",
|
||
})
|
||
.call("r0-entryDraft", "researcher-2", "entry", null, 2, {
|
||
wallSeconds: measured(5),
|
||
status: "failed",
|
||
reason: "模型服务报错",
|
||
})
|
||
.call("r0-entryDraft", "researcher-3", "entry", null, 1, {
|
||
wallSeconds: { status: "unmeasured", reason: "进程中断" },
|
||
status: "failed",
|
||
reason: "进程中断",
|
||
});
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "dispatch",
|
||
stageId: "r0-entryDraft",
|
||
tasks: entryTasks(["researcher-2", "researcher-3"]),
|
||
});
|
||
});
|
||
|
||
test("只有一位研究者：两次讨论各写一条按配置跳过，接着开认领", () => {
|
||
const one = fullConfig();
|
||
one.models.researchers = [one.models.researchers[0]!];
|
||
const book = afterBaseline(one)
|
||
.open("r0-entryDraft", "entryDraft", 0, ["researcher-1"])
|
||
.entry("r0-entryDraft", "researcher-1");
|
||
book.add("stageClose", { stageId: "r0-entryDraft" });
|
||
const reason = "只有一位研究者，第零轮不讨论（方法.md 4.2）";
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "skip",
|
||
step: "discussion1",
|
||
reason,
|
||
});
|
||
book.add("event", {
|
||
event: "skippedByConfig",
|
||
reason,
|
||
objects: ["discussion1"],
|
||
});
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "skip",
|
||
step: "discussion2",
|
||
reason,
|
||
});
|
||
book.add("event", {
|
||
event: "skippedByConfig",
|
||
reason,
|
||
objects: ["discussion2"],
|
||
});
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r0-claim",
|
||
stage: "claim",
|
||
round: 0,
|
||
roster: ["researcher-1"],
|
||
});
|
||
});
|
||
|
||
/** 第零轮做完：四个阶段都交齐关闭，分配决定已写。 */
|
||
function afterRoundZero(config: ResearchConfig): Book {
|
||
const book = afterBaseline(config);
|
||
const stages = [
|
||
["r0-entryDraft", "entryDraft"],
|
||
["r0-discussion1", "discussion"],
|
||
["r0-discussion2", "discussion"],
|
||
["r0-claim", "claim"],
|
||
] as const;
|
||
for (const [stageId, stage] of stages) {
|
||
book.open(stageId, stage, 0, R);
|
||
for (const r of R) book.entry(stageId, r);
|
||
book.add("stageClose", { stageId });
|
||
}
|
||
return book;
|
||
}
|
||
|
||
/** 分配决定。 */
|
||
function assigned(book: Book): Book {
|
||
const output = R.map((researcher, i) => ({
|
||
researcher,
|
||
entry: `e-${researcher}-1`,
|
||
rank: i + 1,
|
||
}));
|
||
return book.add("decision", {
|
||
decision: "entries",
|
||
round: 0,
|
||
rule: "assignEntries",
|
||
ruleVersion: 1,
|
||
params: null,
|
||
seed: null,
|
||
input: null,
|
||
output,
|
||
});
|
||
}
|
||
|
||
test("认领关了：分配；分配做完：开第 1 轮的提案阶段，名单是全部研究者", () => {
|
||
const book = afterRoundZero(fullConfig());
|
||
assert.deepEqual(nextStep(book.records), { do: "assignEntries" });
|
||
assigned(book);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-proposal",
|
||
stage: "proposal",
|
||
round: 1,
|
||
roster: R,
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 第 1 轮：开轮到选批（任务书 14 第 2.2、2.3 节）
|
||
// 三条原提案：researcher-1 的单独成候选 r1-c1；researcher-2、researcher-3 的判为同一个改法，合成 r1-c2。
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const P1 = "r1-researcher-1-1";
|
||
const P2 = "r1-researcher-2-1";
|
||
const P3 = "r1-researcher-3-1";
|
||
const C = ["r1-c1", "r1-c2"];
|
||
|
||
/** 一个研究者阶段：开、每个任务交一份、关。tasks 是 [作者, 种类, 对象, 编号]。 */
|
||
function stage(
|
||
book: Book,
|
||
stageId: string,
|
||
kind: Content<"stageOpen">["stage"],
|
||
tasks: Array<[string, SubmissionKind, string | null, string | null]>,
|
||
): Book {
|
||
book.open(stageId, kind, 1, R);
|
||
for (const [author, k, target, item] of tasks)
|
||
book.sub(stageId, author, k, target, item);
|
||
return book.add("stageClose", { stageId });
|
||
}
|
||
|
||
/** 每人 × 每个对象的任务。 */
|
||
const each = (kind: SubmissionKind, objects: string[]) =>
|
||
R.flatMap((actor) => objects.map((object) => ({ actor, kind, object })));
|
||
|
||
/** 第 1 轮写到 last 这一步做完为止。 */
|
||
function roundOne(
|
||
last:
|
||
| "proposal"
|
||
| "dedupe"
|
||
| "firstPrediction"
|
||
| "critique"
|
||
| "response"
|
||
| "freezeProtocol"
|
||
| "secondPrediction"
|
||
| "selectBatch",
|
||
): Book {
|
||
const book = assigned(afterRoundZero(fullConfig()));
|
||
const order = [
|
||
"proposal",
|
||
"dedupe",
|
||
"firstPrediction",
|
||
"critique",
|
||
"response",
|
||
"freezeProtocol",
|
||
"secondPrediction",
|
||
"selectBatch",
|
||
];
|
||
const steps: Record<string, () => void> = {
|
||
proposal: () =>
|
||
stage(book, "r1-proposal", "proposal", [
|
||
["researcher-1", "proposal", null, P1],
|
||
["researcher-2", "proposal", null, P2],
|
||
["researcher-3", "proposal", null, P3],
|
||
]),
|
||
dedupe: () => {
|
||
const output = {
|
||
fresh: [
|
||
{ candidate: "r1-c1", proposals: [P1] },
|
||
{ candidate: "r1-c2", proposals: [P2, P3] },
|
||
],
|
||
carried: [],
|
||
};
|
||
book.add("decision", {
|
||
decision: "dedup",
|
||
round: 1,
|
||
rule: "jevSameChange",
|
||
ruleVersion: 1,
|
||
params: null,
|
||
seed: null,
|
||
input: null,
|
||
output,
|
||
});
|
||
},
|
||
firstPrediction: () =>
|
||
stage(
|
||
book,
|
||
"r1-firstPrediction",
|
||
"firstPrediction",
|
||
R.flatMap((a) =>
|
||
C.map(
|
||
(c) =>
|
||
[a, "firstPrediction", c, null] as [
|
||
string,
|
||
SubmissionKind,
|
||
string,
|
||
null,
|
||
],
|
||
),
|
||
),
|
||
),
|
||
critique: () =>
|
||
stage(book, "r1-critique", "critique", [
|
||
["researcher-1", "critique", P2, `${P2}.researcher-1.1`],
|
||
["researcher-1", "critique", P3, `${P3}.researcher-1.1`],
|
||
["researcher-2", "critique", P1, `${P1}.researcher-2.1`],
|
||
["researcher-2", "critique", P3, `${P3}.researcher-2.1`],
|
||
["researcher-3", "critique", P1, `${P1}.researcher-3.1`],
|
||
["researcher-3", "critique", P2, `${P2}.researcher-3.1`],
|
||
]),
|
||
response: () =>
|
||
stage(book, "r1-response", "response", [
|
||
["researcher-1", "response", "r1-c1", null],
|
||
["researcher-2", "unification", "r1-c2", null],
|
||
]),
|
||
freezeProtocol: () => {
|
||
// r1-c1 一个因子（采纳它不做消融），r1-c2 两个因子（采纳它走 ⑬–⑰）。
|
||
book.add("protocol", {
|
||
candidate: "r1-c1",
|
||
protocol: { round: 1, baseline: "base", proposals: [], factors: [{}] },
|
||
});
|
||
book.add("protocol", {
|
||
candidate: "r1-c2",
|
||
protocol: {
|
||
round: 1,
|
||
baseline: "base",
|
||
proposals: [],
|
||
factors: [{}, {}],
|
||
},
|
||
});
|
||
},
|
||
secondPrediction: () =>
|
||
stage(
|
||
book,
|
||
"r1-secondPrediction",
|
||
"secondPrediction",
|
||
R.flatMap((a) =>
|
||
C.map(
|
||
(c) =>
|
||
[a, "secondPrediction", c, null] as [
|
||
string,
|
||
SubmissionKind,
|
||
string,
|
||
null,
|
||
],
|
||
),
|
||
),
|
||
),
|
||
selectBatch: () =>
|
||
book.add("decision", {
|
||
decision: "batch",
|
||
round: 1,
|
||
rule: "bestExpected",
|
||
ruleVersion: 1,
|
||
params: null,
|
||
seed: "s",
|
||
input: null,
|
||
output: { chosen: ["r1-c1"] },
|
||
}),
|
||
};
|
||
for (const step of order.slice(0, order.indexOf(last) + 1)) steps[step]!();
|
||
return book;
|
||
}
|
||
|
||
test("提案阶段一人交了两条、两人没交：只派没交的两人，提案没有对象", () => {
|
||
const book = assigned(afterRoundZero(fullConfig()))
|
||
.open("r1-proposal", "proposal", 1, R)
|
||
.sub("r1-proposal", "researcher-1", "proposal", null, P1);
|
||
book.sub(
|
||
"r1-proposal",
|
||
"researcher-1",
|
||
"proposal",
|
||
null,
|
||
"r1-researcher-1-2",
|
||
);
|
||
const tasks = ["researcher-2", "researcher-3"].map((actor) => ({
|
||
actor,
|
||
kind: "proposal",
|
||
object: null,
|
||
}));
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "dispatch",
|
||
stageId: "r1-proposal",
|
||
tasks,
|
||
});
|
||
});
|
||
|
||
test("提案交齐关闭后：判重合并", () => {
|
||
assert.deepEqual(nextStep(roundOne("proposal").records), {
|
||
do: "dedupe",
|
||
round: 1,
|
||
});
|
||
});
|
||
|
||
test("判重做完：开第一次预测；任务是每人 × 每个新候选（含自己的），同一个人的按候选编号排", () => {
|
||
const book = roundOne("dedupe");
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-firstPrediction",
|
||
stage: "firstPrediction",
|
||
round: 1,
|
||
roster: R,
|
||
});
|
||
book.open("r1-firstPrediction", "firstPrediction", 1, R);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "dispatch",
|
||
stageId: "r1-firstPrediction",
|
||
tasks: each("firstPrediction", C),
|
||
});
|
||
});
|
||
|
||
test("同一个人的任务按对象分开算：对 r1-c1 没交的停住，r1-c2 照派；能派的派完才停下", () => {
|
||
const notSubmitted = {
|
||
wallSeconds: measured(30),
|
||
status: "notSubmitted",
|
||
reason: "模型结束了回复，没有一份提交被收下",
|
||
} as const;
|
||
const book = roundOne("dedupe").open(
|
||
"r1-firstPrediction",
|
||
"firstPrediction",
|
||
1,
|
||
R,
|
||
);
|
||
for (const a of ["researcher-1", "researcher-3"])
|
||
for (const c of C)
|
||
book.sub("r1-firstPrediction", a, "firstPrediction", c, null);
|
||
book.call(
|
||
"r1-firstPrediction",
|
||
"researcher-2",
|
||
"firstPrediction",
|
||
"r1-c1",
|
||
1,
|
||
notSubmitted,
|
||
);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "dispatch",
|
||
stageId: "r1-firstPrediction",
|
||
tasks: [
|
||
{ actor: "researcher-2", kind: "firstPrediction", object: "r1-c2" },
|
||
],
|
||
});
|
||
book.sub(
|
||
"r1-firstPrediction",
|
||
"researcher-2",
|
||
"firstPrediction",
|
||
"r1-c2",
|
||
null,
|
||
);
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "halt");
|
||
assert.deepEqual((next as { objects: string[] }).objects, [
|
||
"r1-firstPrediction.researcher-2.firstPrediction.r1-c1.1",
|
||
]);
|
||
});
|
||
|
||
test("挑刺阶段：每人 × 每条不是自己写的原提案，没有挑自己的", () => {
|
||
const book = roundOne("firstPrediction").open(
|
||
"r1-critique",
|
||
"critique",
|
||
1,
|
||
R,
|
||
);
|
||
const expected = [
|
||
{ actor: "researcher-1", kind: "critique", object: P2 },
|
||
{ actor: "researcher-1", kind: "critique", object: P3 },
|
||
{ actor: "researcher-2", kind: "critique", object: P1 },
|
||
{ actor: "researcher-2", kind: "critique", object: P3 },
|
||
{ actor: "researcher-3", kind: "critique", object: P1 },
|
||
{ actor: "researcher-3", kind: "critique", object: P2 },
|
||
];
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "dispatch",
|
||
stageId: "r1-critique",
|
||
tasks: expected,
|
||
});
|
||
});
|
||
|
||
test("回应阶段：单条原提案的候选派作者交回应，合并而成的派编号最小的作者交统一", () => {
|
||
const book = roundOne("critique").open("r1-response", "response", 1, R);
|
||
const expected = [
|
||
{ actor: "researcher-1", kind: "response", object: "r1-c1" },
|
||
{ actor: "researcher-2", kind: "unification", object: "r1-c2" },
|
||
];
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "dispatch",
|
||
stageId: "r1-response",
|
||
tasks: expected,
|
||
});
|
||
});
|
||
|
||
test("回应阶段关闭、还没有冻结协议：写冻结协议；写完开第二次预测，任务是每人 × 池里每个候选", () => {
|
||
const book = roundOne("response");
|
||
assert.deepEqual(nextStep(book.records), { do: "freezeProtocols", round: 1 });
|
||
const frozen = roundOne("freezeProtocol");
|
||
assert.deepEqual(nextStep(frozen.records), {
|
||
do: "openStage",
|
||
stageId: "r1-secondPrediction",
|
||
stage: "secondPrediction",
|
||
round: 1,
|
||
roster: R,
|
||
});
|
||
frozen.open("r1-secondPrediction", "secondPrediction", 1, R);
|
||
assert.deepEqual(nextStep(frozen.records), {
|
||
do: "dispatch",
|
||
stageId: "r1-secondPrediction",
|
||
tasks: each("secondPrediction", C),
|
||
});
|
||
});
|
||
|
||
test("第二次预测关闭：选批", () => {
|
||
assert.deepEqual(nextStep(roundOne("secondPrediction").records), {
|
||
do: "selectBatch",
|
||
round: 1,
|
||
});
|
||
});
|
||
|
||
test("选批做完：开实现阶段，每个入选候选固定一位实验员", () => {
|
||
assert.deepEqual(nextStep(roundOne("selectBatch").records), {
|
||
do: "openStage",
|
||
stageId: "r1-implementation",
|
||
stage: "implementation",
|
||
round: 1,
|
||
roster: ["experimenter-r1-c1"],
|
||
});
|
||
});
|
||
|
||
test("关掉第一次预测：第 1 轮写一条按配置跳过（对象 r1-firstPrediction），接着开挑刺", () => {
|
||
const config = fullConfig();
|
||
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
|
||
config.flow.firstPrediction = false;
|
||
const book = assigned(afterRoundZero(config));
|
||
stage(book, "r1-proposal", "proposal", [
|
||
["researcher-1", "proposal", null, P1],
|
||
["researcher-2", "proposal", null, P2],
|
||
["researcher-3", "proposal", null, P3],
|
||
]);
|
||
const output = {
|
||
fresh: [
|
||
{ candidate: "r1-c1", proposals: [P1] },
|
||
{ candidate: "r1-c2", proposals: [P2, P3] },
|
||
],
|
||
carried: [],
|
||
};
|
||
book.add("decision", {
|
||
decision: "dedup",
|
||
round: 1,
|
||
rule: "jevSameChange",
|
||
ruleVersion: 1,
|
||
params: null,
|
||
seed: null,
|
||
input: null,
|
||
output,
|
||
});
|
||
const reason = "配置关掉了第一次预测";
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "skip",
|
||
step: "r1-firstPrediction",
|
||
reason,
|
||
});
|
||
book.add("event", {
|
||
event: "skippedByConfig",
|
||
reason,
|
||
objects: ["r1-firstPrediction"],
|
||
});
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-critique",
|
||
stage: "critique",
|
||
round: 1,
|
||
roster: R,
|
||
});
|
||
});
|
||
|
||
// ⑨–⑫：预期动作手写，不调模型、Jev 或 Docker。
|
||
const X = ["experimenter-r1-c1", "experimenter-r1-c2"];
|
||
const V = ["verifier-r1-c1", "verifier-r1-c2"];
|
||
|
||
function batchTwo(maxRepairs: number): Book {
|
||
const book = roundOne("secondPrediction");
|
||
const record = book.records.find((r) => r.kind === "config")!;
|
||
(record.content as unknown as StudyRecord).config = {
|
||
...fullConfig(),
|
||
maxRepairs,
|
||
} as unknown as Json;
|
||
return book.add("decision", {
|
||
decision: "batch",
|
||
round: 1,
|
||
rule: "bestExpected",
|
||
ruleVersion: 1,
|
||
params: null,
|
||
seed: "s",
|
||
input: null,
|
||
output: { chosen: C },
|
||
});
|
||
}
|
||
|
||
function implemented(book: Book): Book {
|
||
book.open("r1-implementation", "implementation", 1, X);
|
||
for (const [i, candidate] of C.entries())
|
||
book.sub("r1-implementation", X[i]!, "implementation", candidate, null);
|
||
return book.add("stageClose", { stageId: "r1-implementation" });
|
||
}
|
||
|
||
function verified(
|
||
book: Book,
|
||
n: number,
|
||
candidate: string,
|
||
passed: boolean,
|
||
): Book {
|
||
// 对应真实 15-tight-2 的 submissions created_seq=173（墙约束行和右端错位）、194（修复后通过）。
|
||
// 只转录驱动程序所需的逐项 found，不伪造 Jev 的判断或模型运行。
|
||
const content: Submission<"verification"> = {
|
||
clauses: [
|
||
{
|
||
clause: "c1",
|
||
found: passed,
|
||
where: passed
|
||
? "initial.py 墙约束行和右端已对齐"
|
||
: "initial.py:225-228 与 243-246：墙约束行和右端错位",
|
||
},
|
||
],
|
||
factors: [{ factor: "p1", found: passed, where: "LP 墙约束" }],
|
||
safeguards: [],
|
||
unassigned: [],
|
||
body: passed ? "通过" : "解的不是协议写的线性规划",
|
||
};
|
||
return book.add("submission", {
|
||
stageId: `r1-verification-${n}`,
|
||
author: `verifier-${candidate}`,
|
||
kind: "verification",
|
||
target: candidate,
|
||
item: null,
|
||
version: 1,
|
||
content,
|
||
jev: null,
|
||
});
|
||
}
|
||
|
||
function firstVerification(book: Book, a: boolean, b: boolean): Book {
|
||
implemented(book).open("r1-verification-1", "verification", 1, V);
|
||
verified(book, 1, C[0]!, a);
|
||
verified(book, 1, C[1]!, b);
|
||
return book.add("stageClose", { stageId: "r1-verification-1" });
|
||
}
|
||
|
||
function recordForward(
|
||
book: Book,
|
||
candidate: string,
|
||
n: number,
|
||
ending: "valid" | "notStarted" | "timeout",
|
||
): Book {
|
||
const run = `r1-forward-${candidate}-${n}`;
|
||
book.add("runPlanned", {
|
||
run,
|
||
purpose: "forward",
|
||
round: 1,
|
||
object: candidate,
|
||
definition,
|
||
});
|
||
if (ending === "notStarted")
|
||
return book.add("runResult", {
|
||
run,
|
||
process: { ending, reason: "Docker 未启动" },
|
||
judged: null,
|
||
});
|
||
book.add("runStarted", { run });
|
||
const process =
|
||
ending === "timeout"
|
||
? {
|
||
ending: "timeout" as const,
|
||
wallSeconds: measured(120),
|
||
cpuSeconds: measured(100),
|
||
peakMemoryBytes: measured(1024),
|
||
}
|
||
: exited;
|
||
return book.add("runResult", {
|
||
run,
|
||
process,
|
||
judged: ending === "valid" ? judged(1.25, "main") : null,
|
||
});
|
||
}
|
||
|
||
const forward = (candidate: string, n: number) => ({
|
||
do: "forwardRun",
|
||
round: 1,
|
||
candidate,
|
||
run: `r1-forward-${candidate}-${n}`,
|
||
});
|
||
|
||
test("实现的任务同时可派；已提交后中断不重派，等全体交齐关阶段后才核验", () => {
|
||
const book = batchTwo(1).open("r1-implementation", "implementation", 1, X);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "dispatch",
|
||
stageId: "r1-implementation",
|
||
tasks: C.map((object, i) => ({
|
||
actor: X[i],
|
||
kind: "implementation",
|
||
object,
|
||
})),
|
||
});
|
||
book.sub("r1-implementation", X[0]!, "implementation", C[0]!, null);
|
||
book.call("r1-implementation", X[0]!, "implementation", C[0]!, 1, {
|
||
wallSeconds: { status: "unmeasured", reason: "进程中断" },
|
||
status: "failed",
|
||
reason: "进程中断",
|
||
});
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "dispatch",
|
||
stageId: "r1-implementation",
|
||
tasks: [{ actor: X[1], kind: "implementation", object: C[1] }],
|
||
});
|
||
book.sub("r1-implementation", X[1]!, "implementation", C[1]!, null);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "closeStage",
|
||
stageId: "r1-implementation",
|
||
});
|
||
book.add("stageClose", { stageId: "r1-implementation" });
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-verification-1",
|
||
stage: "verification",
|
||
round: 1,
|
||
roster: V,
|
||
});
|
||
});
|
||
|
||
test("15-tight-2 的一次修复路径：核验关后才派原实验员，修复关后只重验修过的候选", () => {
|
||
const book = implemented(batchTwo(1)).open(
|
||
"r1-verification-1",
|
||
"verification",
|
||
1,
|
||
V,
|
||
);
|
||
verified(book, 1, C[0]!, false);
|
||
verified(book, 1, C[1]!, true);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "closeStage",
|
||
stageId: "r1-verification-1",
|
||
});
|
||
book.add("stageClose", { stageId: "r1-verification-1" });
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-repair-1",
|
||
stage: "implementation",
|
||
round: 1,
|
||
roster: [X[0]],
|
||
});
|
||
book.open("r1-repair-1", "implementation", 1, [X[0]!]);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "dispatch",
|
||
stageId: "r1-repair-1",
|
||
tasks: [{ actor: X[0], kind: "repair", object: C[0] }],
|
||
});
|
||
book.sub("r1-repair-1", X[0]!, "repair", C[0]!, null);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "closeStage",
|
||
stageId: "r1-repair-1",
|
||
});
|
||
book.add("stageClose", { stageId: "r1-repair-1" });
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-verification-2",
|
||
stage: "verification",
|
||
round: 1,
|
||
roster: [V[0]],
|
||
});
|
||
book.open("r1-verification-2", "verification", 1, [V[0]!]);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "dispatch",
|
||
stageId: "r1-verification-2",
|
||
tasks: [{ actor: V[0], kind: "verification", object: C[0] }],
|
||
});
|
||
verified(book, 2, C[0]!, true);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "closeStage",
|
||
stageId: "r1-verification-2",
|
||
});
|
||
book.add("stageClose", { stageId: "r1-verification-2" });
|
||
assert.deepEqual(nextStep(book.records), forward(C[0]!, 1));
|
||
});
|
||
|
||
test("修复次数从关闭的阶段数出：两次修复后仍失败，逐候选写不跑事件，不占预算", () => {
|
||
const book = firstVerification(batchTwo(2), false, false);
|
||
for (let n = 1; n <= 2; n++) {
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: `r1-repair-${n}`,
|
||
stage: "implementation",
|
||
round: 1,
|
||
roster: X,
|
||
});
|
||
book.open(`r1-repair-${n}`, "implementation", 1, X);
|
||
for (const [i, candidate] of C.entries())
|
||
book.sub(`r1-repair-${n}`, X[i]!, "repair", candidate, null);
|
||
book.add("stageClose", { stageId: `r1-repair-${n}` });
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: `r1-verification-${n + 1}`,
|
||
stage: "verification",
|
||
round: 1,
|
||
roster: V,
|
||
});
|
||
book.open(`r1-verification-${n + 1}`, "verification", 1, V);
|
||
for (const candidate of C) verified(book, n + 1, candidate, false);
|
||
book.add("stageClose", { stageId: `r1-verification-${n + 1}` });
|
||
}
|
||
for (const candidate of C) {
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "forwardNotRun");
|
||
if (next.do !== "forwardNotRun") throw new Error("应写不跑事件");
|
||
assert.equal(next.candidate, candidate);
|
||
assert.match(next.reason, /第 3 次核验仍不通过，修复次数上限 2 已用完/);
|
||
assert.match(next.reason, /解的不是协议写的线性规划/);
|
||
book.add("event", forwardNotRunEvent(candidate, next.reason));
|
||
}
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), {
|
||
charged: 1,
|
||
reserved: 0,
|
||
balance: 12,
|
||
});
|
||
assert.deepEqual(nextStep(book.records), { do: "adopt", round: 1 });
|
||
});
|
||
|
||
test("maxRepairs=0：第一次核验失败就记不跑，另一候选仍可串行运行", () => {
|
||
const book = firstVerification(batchTwo(0), false, true);
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "forwardNotRun");
|
||
if (next.do !== "forwardNotRun") throw new Error("应写不跑事件");
|
||
book.add("event", forwardNotRunEvent(next.candidate, next.reason));
|
||
assert.deepEqual(nextStep(book.records), forward(C[1]!, 1));
|
||
});
|
||
|
||
test("正向串行：上次未启动起新编号；已登记未结束必须先恢复；超时也扣次且不重跑；采纳前不公开", () => {
|
||
const book = firstVerification(batchTwo(1), true, true);
|
||
assert.deepEqual(nextStep(book.records), forward(C[0]!, 1));
|
||
recordForward(book, C[0]!, 1, "notStarted");
|
||
assert.deepEqual(nextStep(book.records), forward(C[0]!, 2));
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), {
|
||
charged: 1,
|
||
reserved: 0,
|
||
balance: 12,
|
||
});
|
||
const run = `r1-forward-${C[0]}-2`;
|
||
book.add("runPlanned", {
|
||
run,
|
||
purpose: "forward",
|
||
round: 1,
|
||
object: C[0]!,
|
||
definition,
|
||
});
|
||
assert.throws(
|
||
() => nextStep(book.records),
|
||
/登记了还没有结果：恢复入口应当已经核对过/,
|
||
);
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), {
|
||
charged: 1,
|
||
reserved: 1,
|
||
balance: 11,
|
||
});
|
||
book.add("runStarted", { run });
|
||
assert.throws(() => nextStep(book.records), /登记了还没有结果/);
|
||
book.add("runResult", { run, process: exited, judged: judged(1.25, "main") });
|
||
assert.deepEqual(nextStep(book.records), forward(C[1]!, 1));
|
||
recordForward(book, C[1]!, 1, "timeout");
|
||
assert.deepEqual(nextStep(book.records), { do: "adopt", round: 1 });
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), {
|
||
charged: 3,
|
||
reserved: 0,
|
||
balance: 10,
|
||
});
|
||
const before = new Board(book.records);
|
||
assert.ok(
|
||
before
|
||
.runs()
|
||
.filter((r) => r.plan.purpose === "forward")
|
||
.every((r) => !before.runPublic(r)),
|
||
);
|
||
book.add("decision", {
|
||
decision: "adopt",
|
||
round: 1,
|
||
rule: "adopt",
|
||
ruleVersion: 1,
|
||
params: null,
|
||
seed: null,
|
||
input: null,
|
||
output: { adopted: null },
|
||
});
|
||
const after = new Board(book.records);
|
||
assert.ok(
|
||
after
|
||
.runs()
|
||
.filter((r) => r.plan.purpose === "forward")
|
||
.every((r) => after.runPublic(r)),
|
||
);
|
||
// 没有采纳：不开解释阶段，直接开收尾（方法.md 4.3 ⑫）。
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-wrapUp",
|
||
stage: "wrapUp",
|
||
round: 1,
|
||
roster: R,
|
||
});
|
||
});
|
||
|
||
test("采纳只有一个因子的候选：不做消融，直接开收尾（方法.md 4.3 ⑫）", () => {
|
||
const book = firstVerification(batchTwo(1), true, true);
|
||
for (const c of C) recordForward(book, c, 1, "valid");
|
||
book.add("decision", {
|
||
decision: "adopt",
|
||
round: 1,
|
||
rule: "adopt",
|
||
ruleVersion: 1,
|
||
params: null,
|
||
seed: null,
|
||
input: null,
|
||
output: { adopted: C[0]! },
|
||
});
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-wrapUp",
|
||
stage: "wrapUp",
|
||
round: 1,
|
||
roster: R,
|
||
});
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// ⑬–⑱：消融到收尾、轮末解封、下一轮和研究结束（任务书 17 第 2.2、2.3 节）。预期动作手写，不调模型、Jev 或 Docker。
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const ADOPTED = "r1-c2"; // 两个因子：p1、p2
|
||
const CELL1 = "r1-c2-p1";
|
||
const CELL2 = "r1-c2-p2";
|
||
|
||
/** 采纳了两个因子的 r1-c2。 */
|
||
function adoptedBook(): Book {
|
||
const book = firstVerification(batchTwo(1), true, true);
|
||
for (const c of C) recordForward(book, c, 1, "valid");
|
||
return book.add("decision", {
|
||
decision: "adopt",
|
||
round: 1,
|
||
rule: "adopt",
|
||
ruleVersion: 1,
|
||
params: null,
|
||
seed: null,
|
||
input: null,
|
||
output: { adopted: ADOPTED },
|
||
});
|
||
}
|
||
|
||
/** 解释阶段：开、三人各交一份、关。 */
|
||
function explained(book: Book): Book {
|
||
return stage(
|
||
book,
|
||
"r1-explanation",
|
||
"explanation",
|
||
R.map((a) => [a, "explanation", null, null]),
|
||
);
|
||
}
|
||
|
||
/** 选格决定。 */
|
||
function cellsChosen(book: Book, chosen: string[]): Book {
|
||
return book.add("decision", {
|
||
decision: "cells",
|
||
round: 1,
|
||
rule: "valueMinusCost",
|
||
ruleVersion: 1,
|
||
params: null,
|
||
seed: "s2",
|
||
input: null,
|
||
output: { chosen },
|
||
});
|
||
}
|
||
|
||
/** 一次消融实现阶段：subs 是 [格, cannotRun, alsoOff]。 */
|
||
function implStage(
|
||
book: Book,
|
||
n: number,
|
||
subs: Array<[string, string | null, string[]?]>,
|
||
): Book {
|
||
const stageId = `r1-ablationImplementation-${n}`;
|
||
book.open(
|
||
stageId,
|
||
"ablationImplementation",
|
||
1,
|
||
subs.map(([cell]) => `experimenter-${cell}`),
|
||
);
|
||
for (const [cell, cannotRun, alsoOff] of subs) {
|
||
book.add("submission", {
|
||
stageId,
|
||
author: `experimenter-${cell}`,
|
||
kind: "ablationImplementation",
|
||
target: cell,
|
||
item: null,
|
||
version: 1,
|
||
content: { alsoOff: alsoOff ?? [], cannotRun, concerns: "无" },
|
||
jev: null,
|
||
});
|
||
}
|
||
return book.add("stageClose", { stageId });
|
||
}
|
||
|
||
/** 一次消融核验阶段：subs 是 [格, [因子, 关没关]]。 */
|
||
function verStage(
|
||
book: Book,
|
||
n: number,
|
||
subs: Array<[string, Array<[string, boolean]>]>,
|
||
): Book {
|
||
const stageId = `r1-ablationVerification-${n}`;
|
||
book.open(
|
||
stageId,
|
||
"ablationVerification",
|
||
1,
|
||
subs.map(([cell]) => `verifier-${cell}`),
|
||
);
|
||
for (const [cell, offs] of subs) {
|
||
book.add("submission", {
|
||
stageId,
|
||
author: `verifier-${cell}`,
|
||
kind: "ablationVerification",
|
||
target: cell,
|
||
item: null,
|
||
version: 1,
|
||
content: {
|
||
factors: offs.map(([factor, off]) => ({
|
||
factor,
|
||
off,
|
||
where: "initial.py:1",
|
||
})),
|
||
},
|
||
jev: null,
|
||
});
|
||
}
|
||
return book.add("stageClose", { stageId });
|
||
}
|
||
|
||
/** 一次消融运行：没启动，或启动了、判分算数。 */
|
||
function recordAblation(
|
||
book: Book,
|
||
cell: string,
|
||
n: number,
|
||
ending: "valid" | "notStarted",
|
||
): Book {
|
||
const run = `r1-ablation-${cell}-${n}`;
|
||
book.add("runPlanned", {
|
||
run,
|
||
purpose: "ablation",
|
||
round: 1,
|
||
object: cell,
|
||
definition,
|
||
});
|
||
if (ending === "notStarted")
|
||
return book.add("runResult", {
|
||
run,
|
||
process: { ending, reason: "Docker 未启动" },
|
||
judged: null,
|
||
});
|
||
book.add("runStarted", { run });
|
||
return book.add("runResult", {
|
||
run,
|
||
process: exited,
|
||
judged: judged(1.2, "main"),
|
||
});
|
||
}
|
||
|
||
test("采纳多因子改法：先开解释阶段，每人一个任务、没有对象；关闭后选格", () => {
|
||
const book = adoptedBook();
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-explanation",
|
||
stage: "explanation",
|
||
round: 1,
|
||
roster: R,
|
||
});
|
||
book.open("r1-explanation", "explanation", 1, R);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "dispatch",
|
||
stageId: "r1-explanation",
|
||
tasks: R.map((actor) => ({ actor, kind: "explanation", object: null })),
|
||
});
|
||
for (const a of R) book.sub("r1-explanation", a, "explanation", null, null);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "closeStage",
|
||
stageId: "r1-explanation",
|
||
});
|
||
book.add("stageClose", { stageId: "r1-explanation" });
|
||
assert.deepEqual(nextStep(book.records), { do: "selectCells", round: 1 });
|
||
});
|
||
|
||
test("选格做完：每格固定一位实验员；交了跑不起来的格不核验、不运行，没启动的下次起新编号", () => {
|
||
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-ablationImplementation-1",
|
||
stage: "ablationImplementation",
|
||
round: 1,
|
||
roster: [`experimenter-${CELL1}`, `experimenter-${CELL2}`],
|
||
});
|
||
implStage(book, 1, [
|
||
[CELL1, null],
|
||
[CELL2, "离了 p2 跑不起来"],
|
||
]);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-ablationVerification-1",
|
||
stage: "ablationVerification",
|
||
round: 1,
|
||
roster: [`verifier-${CELL1}`],
|
||
});
|
||
verStage(book, 1, [
|
||
[
|
||
CELL1,
|
||
[
|
||
["p1", true],
|
||
["p2", false],
|
||
],
|
||
],
|
||
]);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "ablationRun",
|
||
round: 1,
|
||
cell: CELL1,
|
||
run: `r1-ablation-${CELL1}-1`,
|
||
});
|
||
recordAblation(book, CELL1, 1, "notStarted");
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "ablationRun",
|
||
round: 1,
|
||
cell: CELL1,
|
||
run: `r1-ablation-${CELL1}-2`,
|
||
});
|
||
recordAblation(book, CELL1, 2, "valid");
|
||
assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 });
|
||
});
|
||
|
||
test("核验不通过：退回原实验员重做，只重验重做的格；次数用完写不跑事件，不登记运行", () => {
|
||
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||
implStage(book, 1, [
|
||
[CELL1, null],
|
||
[CELL2, null],
|
||
]);
|
||
verStage(book, 1, [
|
||
[
|
||
CELL1,
|
||
[
|
||
["p1", false],
|
||
["p2", false],
|
||
],
|
||
],
|
||
[
|
||
CELL2,
|
||
[
|
||
["p1", false],
|
||
["p2", true],
|
||
],
|
||
],
|
||
]);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-ablationImplementation-2",
|
||
stage: "ablationImplementation",
|
||
round: 1,
|
||
roster: [`experimenter-${CELL1}`],
|
||
});
|
||
implStage(book, 2, [[CELL1, null]]);
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-ablationVerification-2",
|
||
stage: "ablationVerification",
|
||
round: 1,
|
||
roster: [`verifier-${CELL1}`],
|
||
});
|
||
verStage(book, 2, [
|
||
[
|
||
CELL1,
|
||
[
|
||
["p1", false],
|
||
["p2", false],
|
||
],
|
||
],
|
||
]);
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "cellNotRun");
|
||
if (next.do !== "cellNotRun") throw new Error("应写不跑事件");
|
||
assert.equal(next.cell, CELL1);
|
||
assert.match(next.reason, /第 2 次核验仍不通过，修复次数上限 1 已用完/);
|
||
book.add("event", {
|
||
event: "cellNotRun",
|
||
reason: next.reason,
|
||
objects: [next.cell],
|
||
});
|
||
// p1 不跑不扣次；只跑 p2。
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "ablationRun",
|
||
round: 1,
|
||
cell: CELL2,
|
||
run: `r1-ablation-${CELL2}-1`,
|
||
});
|
||
});
|
||
|
||
test("几格实际关掉的因子完全相同：沿用第一格，不重复跑", () => {
|
||
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||
implStage(book, 1, [
|
||
[CELL1, null, ["p2"]],
|
||
[CELL2, null, ["p1"]],
|
||
]);
|
||
verStage(book, 1, [
|
||
[
|
||
CELL1,
|
||
[
|
||
["p1", true],
|
||
["p2", true],
|
||
],
|
||
],
|
||
[
|
||
CELL2,
|
||
[
|
||
["p1", true],
|
||
["p2", true],
|
||
],
|
||
],
|
||
]);
|
||
// 两格实际关的都是 {p1, p2}：只跑第一格。
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "ablationRun",
|
||
round: 1,
|
||
cell: CELL1,
|
||
run: `r1-ablation-${CELL1}-1`,
|
||
});
|
||
recordAblation(book, CELL1, 1, "valid");
|
||
assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 });
|
||
});
|
||
|
||
test("全部交跑不起来：不核验、不运行，直接结算", () => {
|
||
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||
implStage(book, 1, [
|
||
[CELL1, "跑不起来一"],
|
||
[CELL2, "跑不起来二"],
|
||
]);
|
||
assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 });
|
||
});
|
||
|
||
/** 结算和押的因子写完、收尾关闭、解封也写完的账本（余额还有 8 次）。 */
|
||
function afterWrapUp(): Book {
|
||
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||
implStage(book, 1, [
|
||
[CELL1, null],
|
||
[CELL2, null],
|
||
]);
|
||
verStage(book, 1, [
|
||
[
|
||
CELL1,
|
||
[
|
||
["p1", true],
|
||
["p2", false],
|
||
],
|
||
],
|
||
[
|
||
CELL2,
|
||
[
|
||
["p1", false],
|
||
["p2", true],
|
||
],
|
||
],
|
||
]);
|
||
recordAblation(book, CELL1, 1, "valid");
|
||
recordAblation(book, CELL2, 1, "valid");
|
||
book.add("settlement", {
|
||
round: 1,
|
||
result: { candidate: ADOPTED, input: null, output: null },
|
||
});
|
||
book.add("decision", {
|
||
decision: "bet",
|
||
round: 1,
|
||
rule: "chooseBet",
|
||
ruleVersion: 1,
|
||
params: null,
|
||
seed: null,
|
||
input: null,
|
||
output: null,
|
||
});
|
||
return stage(
|
||
book,
|
||
"r1-wrapUp",
|
||
"wrapUp",
|
||
R.map((a) => [a, "wrapUp", null, null]),
|
||
);
|
||
}
|
||
|
||
test("结算和押的因子写完：开收尾；收尾关闭后解封本轮选中的全部候选，然后开下一轮", () => {
|
||
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||
implStage(book, 1, [
|
||
[CELL1, null],
|
||
[CELL2, null],
|
||
]);
|
||
verStage(book, 1, [
|
||
[
|
||
CELL1,
|
||
[
|
||
["p1", true],
|
||
["p2", false],
|
||
],
|
||
],
|
||
[
|
||
CELL2,
|
||
[
|
||
["p1", false],
|
||
["p2", true],
|
||
],
|
||
],
|
||
]);
|
||
recordAblation(book, CELL1, 1, "valid");
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "ablationRun",
|
||
round: 1,
|
||
cell: CELL2,
|
||
run: `r1-ablation-${CELL2}-1`,
|
||
});
|
||
recordAblation(book, CELL2, 1, "valid");
|
||
assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 });
|
||
book.add("settlement", {
|
||
round: 1,
|
||
result: { candidate: ADOPTED, input: null, output: null },
|
||
});
|
||
book.add("decision", {
|
||
decision: "bet",
|
||
round: 1,
|
||
rule: "chooseBet",
|
||
ruleVersion: 1,
|
||
params: null,
|
||
seed: null,
|
||
input: null,
|
||
output: null,
|
||
});
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-wrapUp",
|
||
stage: "wrapUp",
|
||
round: 1,
|
||
roster: R,
|
||
});
|
||
stage(
|
||
book,
|
||
"r1-wrapUp",
|
||
"wrapUp",
|
||
R.map((a) => [a, "wrapUp", null, null]),
|
||
);
|
||
// 收尾关闭后：解封本轮选批选中的 r1-c1、r1-c2；还有一个没解封就继续。
|
||
assert.deepEqual(nextStep(book.records), { do: "unsealChosen", round: 1 });
|
||
book.add("unseal", { candidate: "r1-c1", seals: [1] });
|
||
assert.deepEqual(nextStep(book.records), { do: "unsealChosen", round: 1 });
|
||
book.add("unseal", { candidate: "r1-c2", seals: [2] });
|
||
// 余额还有（已扣 5 次）：开第 2 轮。
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r2-proposal",
|
||
stage: "proposal",
|
||
round: 2,
|
||
roster: R,
|
||
});
|
||
});
|
||
|
||
test("最后一次正向用掉全部余额：进行中的这一轮照常走完采纳与收尾，不开下一轮", () => {
|
||
// 复现 full17-b 第 5 轮的缺陷：两个正向运行把余额用到 0 后，采纳决定、收尾和解封被整轮跳过，
|
||
// 直接走到导出停下。余额为 0 只是不开【下一轮】（方法.md 第七部分“什么时候开下一轮”）。
|
||
const book = firstVerification(batchTwo(1), true, true);
|
||
for (const c of C) recordForward(book, c, 1, "valid");
|
||
// 已扣 3 次（基线 1、正向 2）：把冻结配置的运行次数改成 3，余额为 0。
|
||
const record = book.records.find((r) => r.kind === "config")!;
|
||
const study = record.content as unknown as StudyRecord;
|
||
study.config = { ...fullConfig(), runs: 3 } as unknown as Json;
|
||
// 这一轮已经开始（r1- 的阶段都在账上）：下一步是写采纳决定，不是停下。
|
||
assert.deepEqual(nextStep(book.records), { do: "adopt", round: 1 });
|
||
book.add("decision", {
|
||
decision: "adopt",
|
||
round: 1,
|
||
rule: "adopt",
|
||
ruleVersion: 1,
|
||
params: null,
|
||
seed: null,
|
||
input: null,
|
||
output: { adopted: null },
|
||
});
|
||
// 没有采纳：不做消融，照样开收尾阶段。
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "openStage",
|
||
stageId: "r1-wrapUp",
|
||
stage: "wrapUp",
|
||
round: 1,
|
||
roster: R,
|
||
});
|
||
});
|
||
|
||
test("余额为 0：不开下一轮，走到导出停下，原因写明", () => {
|
||
const book = afterWrapUp();
|
||
book.add("unseal", { candidate: "r1-c1", seals: [1] });
|
||
book.add("unseal", { candidate: "r1-c2", seals: [2] });
|
||
// 已扣 5 次（基线 1、正向 2、消融 2）：把冻结配置的运行次数改成 5，余额为 0。
|
||
const record = book.records.find((r) => r.kind === "config")!;
|
||
const study = record.content as unknown as StudyRecord;
|
||
study.config = { ...fullConfig(), runs: 5 } as unknown as Json;
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "halt",
|
||
reason: "导出还没写",
|
||
objects: [],
|
||
});
|
||
});
|
||
|
||
test("配置关掉消融、又采纳了多因子改法：先记按配置跳过，再在解释停下", () => {
|
||
const book = firstVerification(batchTwo(1), true, true);
|
||
for (const c of C) recordForward(book, c, 1, "valid");
|
||
book.add("decision", {
|
||
decision: "adopt",
|
||
round: 1,
|
||
rule: "adopt",
|
||
ruleVersion: 1,
|
||
params: null,
|
||
seed: null,
|
||
input: null,
|
||
output: { adopted: ADOPTED },
|
||
});
|
||
// 冻结的配置改为关掉消融：配置和步骤清单一起换（步骤清单在研究开始时冻结）。
|
||
const config = fullConfig();
|
||
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
|
||
// 配置关掉消融时不写 cellRule（它只有开着消融才用得到）。
|
||
config.flow.ablation = { enabled: false };
|
||
const record = book.records.find((r) => r.kind === "config")!;
|
||
const study = record.content as unknown as StudyRecord;
|
||
study.config = config as unknown as Json;
|
||
study.steps = stepList(config) as unknown as Json;
|
||
const reason = "配置关掉了消融";
|
||
assert.deepEqual(nextStep(book.records), {
|
||
do: "skip",
|
||
step: "r1-listCells",
|
||
reason,
|
||
});
|
||
book.add("event", {
|
||
event: "skippedByConfig",
|
||
reason,
|
||
objects: ["r1-listCells"],
|
||
});
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "halt");
|
||
assert.match((next as { reason: string }).reason, /不交三个数的解释还没写/);
|
||
});
|