iomgaa
bf5b3f2358
任务书16第二块：角色分派与每调用轻量检查，提交及代码原子记账，按次数核验修复且耗尽不跑，串行正式运行及恢复引用指标，采纳含null统一公开。451项免费测试通过，全新上下文独立审阅未发现问题；真实研究验收仍待进行。
506 lines
30 KiB
TypeScript
506 lines
30 KiB
TypeScript
import assert from "node:assert/strict";
|
||
import { readFileSync } from "node:fs";
|
||
import { join } from "node:path";
|
||
import { test } from "node:test";
|
||
import { stepList, type ResearchConfig } from "../src/config/index.ts";
|
||
import type { AppendKind, Content, LedgerRecord } from "../src/ledger/index.ts";
|
||
import { Board, forwardNotRunEvent, type StudyRecord } from "../src/materials/index.ts";
|
||
import { runBalance } from "../src/rules/index.ts";
|
||
import { definition, exited } from "./ledger-helpers.ts";
|
||
import { judged } from "./materials-helpers.ts";
|
||
import { nextStep } from "../src/flow/next-step.ts";
|
||
import type { Json, Submission, SubmissionKind } from "../src/vocab/index.ts";
|
||
import { fullConfig } from "./config-helpers.ts";
|
||
|
||
// flow 的"下一步做什么"（任务书 12 第六部分）：纯函数，手写账本记录，看它给出的下一步。
|
||
// 其余都由真实运行证明（任务书 12 第五部分），这里不写假模型、假 Docker。
|
||
|
||
const TASK_FACTS = JSON.parse(readFileSync(join(import.meta.dirname, "..", "tasks", "circle_packing", "task.json"), "utf8")) as Json;
|
||
const HASH = "0".repeat(64);
|
||
const measured = (value: number) => ({ status: "measured", value }) as const;
|
||
|
||
/** 调用结束记录去掉调用编号（四种结局各自去掉）。 */
|
||
type Ended = Content<"callEnded"> extends infer E ? (E extends unknown ? Omit<E, "call"> : never) : never;
|
||
|
||
/** 一本手写的账本：按写入先后编序号。内容照账本的形状写，nextStep 不再检查。 */
|
||
class Book {
|
||
readonly records: LedgerRecord[] = [];
|
||
|
||
add<K extends AppendKind>(kind: K, content: Content<K>): this {
|
||
this.records.push({ seq: this.records.length + 1, kind, hash: HASH, content } as LedgerRecord);
|
||
return this;
|
||
}
|
||
|
||
config(config: ResearchConfig): this {
|
||
const record: StudyRecord = {
|
||
config: config as unknown as Json,
|
||
steps: stepList(config) as unknown as Json,
|
||
prompts: { roles: {} as StudyRecord["prompts"]["roles"], tasks: {} as StudyRecord["prompts"]["tasks"] },
|
||
image: `sha256:${HASH}`,
|
||
task: TASK_FACTS,
|
||
seed: 1,
|
||
};
|
||
return this.add("config", record as unknown as Json);
|
||
}
|
||
|
||
baselineCode(): this {
|
||
return this.add("code", { object: "base", version: "v-base", locations: null });
|
||
}
|
||
|
||
/** 一次基线运行：没启动，或启动了、判分算数或不算数。 */
|
||
baselineRun(run: string, outcome: "notStarted" | "valid" | "invalid"): this {
|
||
const definition: Content<"runPlanned">["definition"] = {
|
||
command: ["python", "harness.py"],
|
||
workdir: "/work",
|
||
image: { digest: `sha256:${HASH}`, runtime: "runc" },
|
||
wallSeconds: 120,
|
||
cores: 1,
|
||
inheritEnv: false,
|
||
env: {},
|
||
};
|
||
this.add("runPlanned", { run, purpose: "baseline", round: 0, object: null, definition });
|
||
if (outcome === "notStarted") {
|
||
return this.add("runResult", { run, process: { ending: "notStarted", reason: "docker run 失败" }, judged: null });
|
||
}
|
||
this.add("runStarted", { run });
|
||
const process = { ending: "exited", exitCode: 0, cpuSeconds: measured(10), wallSeconds: measured(10), peakMemoryBytes: measured(1) } as const;
|
||
const judged = {
|
||
verdict: outcome,
|
||
verdictReason: outcome === "valid" ? "合法" : "不合法：圆重叠",
|
||
score: measured(0.9597642169962064),
|
||
metrics: {},
|
||
answerPath: { status: "unmeasured", reason: "基线不写" },
|
||
safeguardTriggers: { status: "unmeasured", reason: "基线不写" },
|
||
extra: null,
|
||
} as unknown as Content<"runResult">["judged"];
|
||
return this.add("runResult", { run, process, judged });
|
||
}
|
||
|
||
open(stageId: string, stage: Content<"stageOpen">["stage"], round: number, roster: string[]): this {
|
||
return this.add("stageOpen", { stageId, stage, round, roster });
|
||
}
|
||
|
||
/** 一份被收下的提交。nextStep 只看这个人在这个阶段有没有被收下的提交，种类一律写成入口。 */
|
||
entry(stageId: string, author: string): this {
|
||
const content = { entries: [{ name: "入口一", body: "从这里开始看" }, { name: "入口二", body: "从那里开始看" }] };
|
||
return this.add("submission", { stageId, author, target: null, item: null, version: 1, kind: "entry", content, jev: null });
|
||
}
|
||
|
||
/** 一份被收下的提交，内容 nextStep 不读。 */
|
||
sub(stageId: string, author: string, kind: SubmissionKind, target: string | null, item: string | null): this {
|
||
return this.add("submission", { stageId, author, target, item, version: 1, kind, content: {}, jev: null } as unknown as Content<"submission">);
|
||
}
|
||
|
||
/** 一次模型调用：开始和结束。"模型调用开始"里的材料只写 nextStep 认任务用的种类和对象。 */
|
||
call(stageId: string, actor: string, kind: SubmissionKind, object: string | null, n: number, ended: Ended): this {
|
||
const call = `${stageId}.${actor}.${kind}.${object ?? "none"}.${n}`;
|
||
this.add("callStarted", { call, actor, stageId, model: "iomgaa/glm-5.3", materials: { kind, object } });
|
||
return this.add("callEnded", { call, ...ended } as Content<"callEnded">);
|
||
}
|
||
}
|
||
|
||
const R = ["researcher-1", "researcher-2", "researcher-3"];
|
||
const entryTasks = (actors: string[]) => actors.map((actor) => ({ actor, kind: "entry", object: null }));
|
||
|
||
/** 冻结了配置、基线跑完算数的账本。 */
|
||
function afterBaseline(config: ResearchConfig = fullConfig()): Book {
|
||
return new Book().config(config).baselineCode().baselineRun("base-1", "valid");
|
||
}
|
||
|
||
test("空账本：冻结配置", () => {
|
||
assert.deepEqual(nextStep([]), { do: "freezeConfig" });
|
||
});
|
||
|
||
test("有配置、没有基线：先存基线代码，再用 base-1 跑基线", () => {
|
||
const book = new Book().config(fullConfig());
|
||
assert.deepEqual(nextStep(book.records), { do: "saveBaseline" });
|
||
book.baselineCode();
|
||
assert.deepEqual(nextStep(book.records), { do: "baselineRun", run: "base-1" });
|
||
});
|
||
|
||
test("基线最近一次是\"没启动\"：起新编号再跑", () => {
|
||
const book = new Book().config(fullConfig()).baselineCode().baselineRun("base-1", "notStarted");
|
||
assert.deepEqual(nextStep(book.records), { do: "baselineRun", run: "base-2" });
|
||
book.baselineRun("base-2", "notStarted");
|
||
assert.deepEqual(nextStep(book.records), { do: "baselineRun", run: "base-3" });
|
||
});
|
||
|
||
test("基线判为不算数：停下，原因写明", () => {
|
||
const book = new Book().config(fullConfig()).baselineCode().baselineRun("base-1", "invalid");
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "halt");
|
||
assert.match((next as { reason: string }).reason, /base-1 判为不算数（不合法：圆重叠）/);
|
||
});
|
||
|
||
test("基线算数之后：开写入口阶段，名单是全部研究者", () => {
|
||
assert.deepEqual(nextStep(afterBaseline().records), { do: "openStage", stageId: "r0-entryDraft", stage: "entryDraft", round: 0, roster: R });
|
||
});
|
||
|
||
test("阶段开着、一人已交、两人没交：派那两人", () => {
|
||
const book = afterBaseline().open("r0-entryDraft", "entryDraft", 0, R).entry("r0-entryDraft", "researcher-1");
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r0-entryDraft", tasks: entryTasks(["researcher-2", "researcher-3"]) });
|
||
});
|
||
|
||
test("都交了：关阶段；关了以后开第 1 次讨论", () => {
|
||
const book = afterBaseline().open("r0-entryDraft", "entryDraft", 0, R);
|
||
for (const r of R) book.entry("r0-entryDraft", r);
|
||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r0-entryDraft" });
|
||
book.add("stageClose", { stageId: "r0-entryDraft" });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r0-discussion1", stage: "discussion", round: 0, roster: R });
|
||
});
|
||
|
||
test("有人最近一次是\"没交\"：开发入口关着先派能派的、再停下；开着重新派", () => {
|
||
const stuck = (devEntry: boolean) =>
|
||
new Book()
|
||
.config({ ...fullConfig(), devEntry })
|
||
.baselineCode()
|
||
.baselineRun("base-1", "valid")
|
||
.open("r0-entryDraft", "entryDraft", 0, R)
|
||
.entry("r0-entryDraft", "researcher-1")
|
||
.call("r0-entryDraft", "researcher-2", "entry", null, 1, { wallSeconds: measured(30), status: "notSubmitted", reason: "模型结束了回复，没有一份提交被收下" });
|
||
// researcher-3 还没派过，先派他。
|
||
assert.deepEqual(nextStep(stuck(false).records), { do: "dispatch", stageId: "r0-entryDraft", tasks: entryTasks(["researcher-3"]) });
|
||
// researcher-3 交上以后只剩"没交"的 researcher-2：停下，附调用编号。
|
||
const closed = stuck(false).entry("r0-entryDraft", "researcher-3");
|
||
const next = nextStep(closed.records);
|
||
assert.equal(next.do, "halt");
|
||
assert.match((next as { reason: string }).reason, /^交不出合格提交/);
|
||
assert.deepEqual((next as { objects: string[] }).objects, ["r0-entryDraft.researcher-2.entry.none.1"]);
|
||
// 开发入口开着：重新派。
|
||
const dev = stuck(true).entry("r0-entryDraft", "researcher-3");
|
||
assert.deepEqual(nextStep(dev.records), { do: "dispatch", stageId: "r0-entryDraft", tasks: entryTasks(["researcher-2"]) });
|
||
});
|
||
|
||
test("有人最近一次是\"出错\"或\"进程中断\"：重新派；只看最近一次调用", () => {
|
||
const book = afterBaseline()
|
||
.open("r0-entryDraft", "entryDraft", 0, R)
|
||
.entry("r0-entryDraft", "researcher-1")
|
||
.call("r0-entryDraft", "researcher-2", "entry", null, 1, { wallSeconds: measured(5), status: "notSubmitted", reason: "没交" })
|
||
.call("r0-entryDraft", "researcher-2", "entry", null, 2, { wallSeconds: measured(5), status: "failed", reason: "模型服务报错" })
|
||
.call("r0-entryDraft", "researcher-3", "entry", null, 1, { wallSeconds: { status: "unmeasured", reason: "进程中断" }, status: "failed", reason: "进程中断" });
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r0-entryDraft", tasks: entryTasks(["researcher-2", "researcher-3"]) });
|
||
});
|
||
|
||
test("只有一位研究者：两次讨论各写一条按配置跳过，接着开认领", () => {
|
||
const one = fullConfig();
|
||
one.models.researchers = [one.models.researchers[0]!];
|
||
const book = afterBaseline(one).open("r0-entryDraft", "entryDraft", 0, ["researcher-1"]).entry("r0-entryDraft", "researcher-1");
|
||
book.add("stageClose", { stageId: "r0-entryDraft" });
|
||
const reason = "只有一位研究者，第零轮不讨论（方法.md 4.2）";
|
||
assert.deepEqual(nextStep(book.records), { do: "skip", step: "discussion1", reason });
|
||
book.add("event", { event: "skippedByConfig", reason, objects: ["discussion1"] });
|
||
assert.deepEqual(nextStep(book.records), { do: "skip", step: "discussion2", reason });
|
||
book.add("event", { event: "skippedByConfig", reason, objects: ["discussion2"] });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r0-claim", stage: "claim", round: 0, roster: ["researcher-1"] });
|
||
});
|
||
|
||
/** 第零轮做完：四个阶段都交齐关闭，分配决定已写。 */
|
||
function afterRoundZero(config: ResearchConfig): Book {
|
||
const book = afterBaseline(config);
|
||
const stages = [
|
||
["r0-entryDraft", "entryDraft"],
|
||
["r0-discussion1", "discussion"],
|
||
["r0-discussion2", "discussion"],
|
||
["r0-claim", "claim"],
|
||
] as const;
|
||
for (const [stageId, stage] of stages) {
|
||
book.open(stageId, stage, 0, R);
|
||
for (const r of R) book.entry(stageId, r);
|
||
book.add("stageClose", { stageId });
|
||
}
|
||
return book;
|
||
}
|
||
|
||
/** 分配决定。 */
|
||
function assigned(book: Book): Book {
|
||
const output = R.map((researcher, i) => ({ researcher, entry: `e-${researcher}-1`, rank: i + 1 }));
|
||
return book.add("decision", { decision: "entries", round: 0, rule: "assignEntries", ruleVersion: 1, params: null, seed: null, input: null, output });
|
||
}
|
||
|
||
test("认领关了：分配；分配做完：开第 1 轮的提案阶段，名单是全部研究者", () => {
|
||
const book = afterRoundZero(fullConfig());
|
||
assert.deepEqual(nextStep(book.records), { do: "assignEntries" });
|
||
assigned(book);
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-proposal", stage: "proposal", round: 1, roster: R });
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 第 1 轮：开轮到选批（任务书 14 第 2.2、2.3 节）
|
||
// 三条原提案：researcher-1 的单独成候选 r1-c1；researcher-2、researcher-3 的判为同一个改法，合成 r1-c2。
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const P1 = "r1-researcher-1-1";
|
||
const P2 = "r1-researcher-2-1";
|
||
const P3 = "r1-researcher-3-1";
|
||
const C = ["r1-c1", "r1-c2"];
|
||
|
||
/** 一个研究者阶段：开、每个任务交一份、关。tasks 是 [作者, 种类, 对象, 编号]。 */
|
||
function stage(book: Book, stageId: string, kind: Content<"stageOpen">["stage"], tasks: Array<[string, SubmissionKind, string | null, string | null]>): Book {
|
||
book.open(stageId, kind, 1, R);
|
||
for (const [author, k, target, item] of tasks) book.sub(stageId, author, k, target, item);
|
||
return book.add("stageClose", { stageId });
|
||
}
|
||
|
||
/** 每人 × 每个对象的任务。 */
|
||
const each = (kind: SubmissionKind, objects: string[]) => R.flatMap((actor) => objects.map((object) => ({ actor, kind, object })));
|
||
|
||
/** 第 1 轮写到 last 这一步做完为止。 */
|
||
function roundOne(last: "proposal" | "dedupe" | "firstPrediction" | "critique" | "response" | "freezeProtocol" | "secondPrediction" | "selectBatch"): Book {
|
||
const book = assigned(afterRoundZero(fullConfig()));
|
||
const order = ["proposal", "dedupe", "firstPrediction", "critique", "response", "freezeProtocol", "secondPrediction", "selectBatch"];
|
||
const steps: Record<string, () => void> = {
|
||
proposal: () => stage(book, "r1-proposal", "proposal", [["researcher-1", "proposal", null, P1], ["researcher-2", "proposal", null, P2], ["researcher-3", "proposal", null, P3]]),
|
||
dedupe: () => {
|
||
const output = { fresh: [{ candidate: "r1-c1", proposals: [P1] }, { candidate: "r1-c2", proposals: [P2, P3] }], carried: [] };
|
||
book.add("decision", { decision: "dedup", round: 1, rule: "jevSameChange", ruleVersion: 1, params: null, seed: null, input: null, output });
|
||
},
|
||
firstPrediction: () => stage(book, "r1-firstPrediction", "firstPrediction", R.flatMap((a) => C.map((c) => [a, "firstPrediction", c, null] as [string, SubmissionKind, string, null]))),
|
||
critique: () =>
|
||
stage(book, "r1-critique", "critique", [
|
||
["researcher-1", "critique", P2, `${P2}.researcher-1.1`],
|
||
["researcher-1", "critique", P3, `${P3}.researcher-1.1`],
|
||
["researcher-2", "critique", P1, `${P1}.researcher-2.1`],
|
||
["researcher-2", "critique", P3, `${P3}.researcher-2.1`],
|
||
["researcher-3", "critique", P1, `${P1}.researcher-3.1`],
|
||
["researcher-3", "critique", P2, `${P2}.researcher-3.1`],
|
||
]),
|
||
response: () => stage(book, "r1-response", "response", [["researcher-1", "response", "r1-c1", null], ["researcher-2", "unification", "r1-c2", null]]),
|
||
freezeProtocol: () => {
|
||
for (const candidate of C) book.add("protocol", { candidate, protocol: { round: 1, baseline: "base", proposals: [] } });
|
||
},
|
||
secondPrediction: () => stage(book, "r1-secondPrediction", "secondPrediction", R.flatMap((a) => C.map((c) => [a, "secondPrediction", c, null] as [string, SubmissionKind, string, null]))),
|
||
selectBatch: () => book.add("decision", { decision: "batch", round: 1, rule: "bestExpected", ruleVersion: 1, params: null, seed: "s", input: null, output: { chosen: ["r1-c1"] } }),
|
||
};
|
||
for (const step of order.slice(0, order.indexOf(last) + 1)) steps[step]!();
|
||
return book;
|
||
}
|
||
|
||
test("提案阶段一人交了两条、两人没交：只派没交的两人，提案没有对象", () => {
|
||
const book = assigned(afterRoundZero(fullConfig())).open("r1-proposal", "proposal", 1, R).sub("r1-proposal", "researcher-1", "proposal", null, P1);
|
||
book.sub("r1-proposal", "researcher-1", "proposal", null, "r1-researcher-1-2");
|
||
const tasks = ["researcher-2", "researcher-3"].map((actor) => ({ actor, kind: "proposal", object: null }));
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-proposal", tasks });
|
||
});
|
||
|
||
test("提案交齐关闭后：判重合并", () => {
|
||
assert.deepEqual(nextStep(roundOne("proposal").records), { do: "dedupe", round: 1 });
|
||
});
|
||
|
||
test("判重做完：开第一次预测；任务是每人 × 每个新候选（含自己的），同一个人的按候选编号排", () => {
|
||
const book = roundOne("dedupe");
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-firstPrediction", stage: "firstPrediction", round: 1, roster: R });
|
||
book.open("r1-firstPrediction", "firstPrediction", 1, R);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-firstPrediction", tasks: each("firstPrediction", C) });
|
||
});
|
||
|
||
test("同一个人的任务按对象分开算：对 r1-c1 没交的停住，r1-c2 照派；能派的派完才停下", () => {
|
||
const notSubmitted = { wallSeconds: measured(30), status: "notSubmitted", reason: "模型结束了回复，没有一份提交被收下" } as const;
|
||
const book = roundOne("dedupe").open("r1-firstPrediction", "firstPrediction", 1, R);
|
||
for (const a of ["researcher-1", "researcher-3"]) for (const c of C) book.sub("r1-firstPrediction", a, "firstPrediction", c, null);
|
||
book.call("r1-firstPrediction", "researcher-2", "firstPrediction", "r1-c1", 1, notSubmitted);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-firstPrediction", tasks: [{ actor: "researcher-2", kind: "firstPrediction", object: "r1-c2" }] });
|
||
book.sub("r1-firstPrediction", "researcher-2", "firstPrediction", "r1-c2", null);
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "halt");
|
||
assert.deepEqual((next as { objects: string[] }).objects, ["r1-firstPrediction.researcher-2.firstPrediction.r1-c1.1"]);
|
||
});
|
||
|
||
test("挑刺阶段：每人 × 每条不是自己写的原提案，没有挑自己的", () => {
|
||
const book = roundOne("firstPrediction").open("r1-critique", "critique", 1, R);
|
||
const expected = [
|
||
{ actor: "researcher-1", kind: "critique", object: P2 },
|
||
{ actor: "researcher-1", kind: "critique", object: P3 },
|
||
{ actor: "researcher-2", kind: "critique", object: P1 },
|
||
{ actor: "researcher-2", kind: "critique", object: P3 },
|
||
{ actor: "researcher-3", kind: "critique", object: P1 },
|
||
{ actor: "researcher-3", kind: "critique", object: P2 },
|
||
];
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-critique", tasks: expected });
|
||
});
|
||
|
||
test("回应阶段：单条原提案的候选派作者交回应，合并而成的派编号最小的作者交统一", () => {
|
||
const book = roundOne("critique").open("r1-response", "response", 1, R);
|
||
const expected = [
|
||
{ actor: "researcher-1", kind: "response", object: "r1-c1" },
|
||
{ actor: "researcher-2", kind: "unification", object: "r1-c2" },
|
||
];
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-response", tasks: expected });
|
||
});
|
||
|
||
test("回应阶段关闭、还没有冻结协议：写冻结协议；写完开第二次预测，任务是每人 × 池里每个候选", () => {
|
||
const book = roundOne("response");
|
||
assert.deepEqual(nextStep(book.records), { do: "freezeProtocols", round: 1 });
|
||
const frozen = roundOne("freezeProtocol");
|
||
assert.deepEqual(nextStep(frozen.records), { do: "openStage", stageId: "r1-secondPrediction", stage: "secondPrediction", round: 1, roster: R });
|
||
frozen.open("r1-secondPrediction", "secondPrediction", 1, R);
|
||
assert.deepEqual(nextStep(frozen.records), { do: "dispatch", stageId: "r1-secondPrediction", tasks: each("secondPrediction", C) });
|
||
});
|
||
|
||
test("第二次预测关闭：选批", () => {
|
||
assert.deepEqual(nextStep(roundOne("secondPrediction").records), { do: "selectBatch", round: 1 });
|
||
});
|
||
|
||
test("选批做完：开实现阶段，每个入选候选固定一位实验员", () => {
|
||
assert.deepEqual(nextStep(roundOne("selectBatch").records), { do: "openStage", stageId: "r1-implementation", stage: "implementation", round: 1, roster: ["experimenter-r1-c1"] });
|
||
});
|
||
|
||
test("关掉第一次预测：第 1 轮写一条按配置跳过（对象 r1-firstPrediction），接着开挑刺", () => {
|
||
const config = fullConfig();
|
||
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
|
||
config.flow.firstPrediction = false;
|
||
const book = assigned(afterRoundZero(config));
|
||
stage(book, "r1-proposal", "proposal", [["researcher-1", "proposal", null, P1], ["researcher-2", "proposal", null, P2], ["researcher-3", "proposal", null, P3]]);
|
||
const output = { fresh: [{ candidate: "r1-c1", proposals: [P1] }, { candidate: "r1-c2", proposals: [P2, P3] }], carried: [] };
|
||
book.add("decision", { decision: "dedup", round: 1, rule: "jevSameChange", ruleVersion: 1, params: null, seed: null, input: null, output });
|
||
const reason = "配置关掉了第一次预测";
|
||
assert.deepEqual(nextStep(book.records), { do: "skip", step: "r1-firstPrediction", reason });
|
||
book.add("event", { event: "skippedByConfig", reason, objects: ["r1-firstPrediction"] });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-critique", stage: "critique", round: 1, roster: R });
|
||
});
|
||
|
||
|
||
// ⑨–⑫：预期动作手写，不调模型、Jev 或 Docker。
|
||
const X = ["experimenter-r1-c1", "experimenter-r1-c2"];
|
||
const V = ["verifier-r1-c1", "verifier-r1-c2"];
|
||
|
||
function batchTwo(maxRepairs: number): Book {
|
||
const book = roundOne("secondPrediction");
|
||
const record = book.records.find((r) => r.kind === "config")!;
|
||
(record.content as unknown as StudyRecord).config = { ...fullConfig(), maxRepairs } as unknown as Json;
|
||
return book.add("decision", { decision: "batch", round: 1, rule: "bestExpected", ruleVersion: 1, params: null, seed: "s", input: null, output: { chosen: C } });
|
||
}
|
||
|
||
function implemented(book: Book): Book {
|
||
book.open("r1-implementation", "implementation", 1, X);
|
||
for (const [i, candidate] of C.entries()) book.sub("r1-implementation", X[i]!, "implementation", candidate, null);
|
||
return book.add("stageClose", { stageId: "r1-implementation" });
|
||
}
|
||
|
||
function verified(book: Book, n: number, candidate: string, passed: boolean): Book {
|
||
// 对应真实 15-tight-2 的 submissions created_seq=173（墙约束行和右端错位）、194（修复后通过）。
|
||
// 只转录驱动程序所需的逐项 found，不伪造 Jev 的判断或模型运行。
|
||
const content: Submission<"verification"> = {
|
||
clauses: [{ clause: "c1", found: passed, where: passed ? "initial.py 墙约束行和右端已对齐" : "initial.py:225-228 与 243-246：墙约束行和右端错位" }],
|
||
factors: [{ factor: "p1", found: passed, where: "LP 墙约束" }],
|
||
safeguards: [], unassigned: [], body: passed ? "通过" : "解的不是协议写的线性规划",
|
||
};
|
||
return book.add("submission", { stageId: `r1-verification-${n}`, author: `verifier-${candidate}`, kind: "verification", target: candidate, item: null, version: 1, content, jev: null });
|
||
}
|
||
|
||
function firstVerification(book: Book, a: boolean, b: boolean): Book {
|
||
implemented(book).open("r1-verification-1", "verification", 1, V);
|
||
verified(book, 1, C[0]!, a);
|
||
verified(book, 1, C[1]!, b);
|
||
return book.add("stageClose", { stageId: "r1-verification-1" });
|
||
}
|
||
|
||
function recordForward(book: Book, candidate: string, n: number, ending: "valid" | "notStarted" | "timeout"): Book {
|
||
const run = `r1-forward-${candidate}-${n}`;
|
||
book.add("runPlanned", { run, purpose: "forward", round: 1, object: candidate, definition });
|
||
if (ending === "notStarted") return book.add("runResult", { run, process: { ending, reason: "Docker 未启动" }, judged: null });
|
||
book.add("runStarted", { run });
|
||
const process = ending === "timeout" ? { ending: "timeout" as const, wallSeconds: measured(120), cpuSeconds: measured(100), peakMemoryBytes: measured(1024) } : exited;
|
||
return book.add("runResult", { run, process, judged: ending === "valid" ? judged(1.25, "main") : null });
|
||
}
|
||
|
||
const forward = (candidate: string, n: number) => ({ do: "forwardRun", round: 1, candidate, run: `r1-forward-${candidate}-${n}` });
|
||
|
||
test("实现的任务同时可派；已提交后中断不重派，等全体交齐关阶段后才核验", () => {
|
||
const book = batchTwo(1).open("r1-implementation", "implementation", 1, X);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-implementation", tasks: C.map((object, i) => ({ actor: X[i], kind: "implementation", object })) });
|
||
book.sub("r1-implementation", X[0]!, "implementation", C[0]!, null);
|
||
book.call("r1-implementation", X[0]!, "implementation", C[0]!, 1, { wallSeconds: { status: "unmeasured", reason: "进程中断" }, status: "failed", reason: "进程中断" });
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-implementation", tasks: [{ actor: X[1], kind: "implementation", object: C[1] }] });
|
||
book.sub("r1-implementation", X[1]!, "implementation", C[1]!, null);
|
||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-implementation" });
|
||
book.add("stageClose", { stageId: "r1-implementation" });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-verification-1", stage: "verification", round: 1, roster: V });
|
||
});
|
||
|
||
test("15-tight-2 的一次修复路径：核验关后才派原实验员，修复关后只重验修过的候选", () => {
|
||
const book = implemented(batchTwo(1)).open("r1-verification-1", "verification", 1, V);
|
||
verified(book, 1, C[0]!, false);
|
||
verified(book, 1, C[1]!, true);
|
||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-verification-1" });
|
||
book.add("stageClose", { stageId: "r1-verification-1" });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-repair-1", stage: "implementation", round: 1, roster: [X[0]] });
|
||
book.open("r1-repair-1", "implementation", 1, [X[0]!]);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-repair-1", tasks: [{ actor: X[0], kind: "repair", object: C[0] }] });
|
||
book.sub("r1-repair-1", X[0]!, "repair", C[0]!, null);
|
||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-repair-1" });
|
||
book.add("stageClose", { stageId: "r1-repair-1" });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-verification-2", stage: "verification", round: 1, roster: [V[0]] });
|
||
book.open("r1-verification-2", "verification", 1, [V[0]!]);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-verification-2", tasks: [{ actor: V[0], kind: "verification", object: C[0] }] });
|
||
verified(book, 2, C[0]!, true);
|
||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-verification-2" });
|
||
book.add("stageClose", { stageId: "r1-verification-2" });
|
||
assert.deepEqual(nextStep(book.records), forward(C[0]!, 1));
|
||
});
|
||
|
||
test("修复次数从关闭的阶段数出：两次修复后仍失败，逐候选写不跑事件，不占预算", () => {
|
||
const book = firstVerification(batchTwo(2), false, false);
|
||
for (let n = 1; n <= 2; n++) {
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: `r1-repair-${n}`, stage: "implementation", round: 1, roster: X });
|
||
book.open(`r1-repair-${n}`, "implementation", 1, X);
|
||
for (const [i, candidate] of C.entries()) book.sub(`r1-repair-${n}`, X[i]!, "repair", candidate, null);
|
||
book.add("stageClose", { stageId: `r1-repair-${n}` });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: `r1-verification-${n + 1}`, stage: "verification", round: 1, roster: V });
|
||
book.open(`r1-verification-${n + 1}`, "verification", 1, V);
|
||
for (const candidate of C) verified(book, n + 1, candidate, false);
|
||
book.add("stageClose", { stageId: `r1-verification-${n + 1}` });
|
||
}
|
||
for (const candidate of C) {
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "forwardNotRun");
|
||
if (next.do !== "forwardNotRun") throw new Error("应写不跑事件");
|
||
assert.equal(next.candidate, candidate);
|
||
assert.match(next.reason, /第 3 次核验仍不通过，修复次数上限 2 已用完/);
|
||
assert.match(next.reason, /解的不是协议写的线性规划/);
|
||
book.add("event", forwardNotRunEvent(candidate, next.reason));
|
||
}
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 });
|
||
assert.deepEqual(nextStep(book.records), { do: "adopt", round: 1 });
|
||
});
|
||
|
||
test("maxRepairs=0：第一次核验失败就记不跑，另一候选仍可串行运行", () => {
|
||
const book = firstVerification(batchTwo(0), false, true);
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "forwardNotRun");
|
||
if (next.do !== "forwardNotRun") throw new Error("应写不跑事件");
|
||
book.add("event", forwardNotRunEvent(next.candidate, next.reason));
|
||
assert.deepEqual(nextStep(book.records), forward(C[1]!, 1));
|
||
});
|
||
|
||
test("正向串行：上次未启动起新编号；已登记未结束必须先恢复；超时也扣次且不重跑；采纳前不公开", () => {
|
||
const book = firstVerification(batchTwo(1), true, true);
|
||
assert.deepEqual(nextStep(book.records), forward(C[0]!, 1));
|
||
recordForward(book, C[0]!, 1, "notStarted");
|
||
assert.deepEqual(nextStep(book.records), forward(C[0]!, 2));
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 });
|
||
const run = `r1-forward-${C[0]}-2`;
|
||
book.add("runPlanned", { run, purpose: "forward", round: 1, object: C[0]!, definition });
|
||
assert.throws(() => nextStep(book.records), /登记了还没有结果：恢复入口应当已经核对过/);
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 1, balance: 11 });
|
||
book.add("runStarted", { run });
|
||
assert.throws(() => nextStep(book.records), /登记了还没有结果/);
|
||
book.add("runResult", { run, process: exited, judged: judged(1.25, "main") });
|
||
assert.deepEqual(nextStep(book.records), forward(C[1]!, 1));
|
||
recordForward(book, C[1]!, 1, "timeout");
|
||
assert.deepEqual(nextStep(book.records), { do: "adopt", round: 1 });
|
||
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 3, reserved: 0, balance: 10 });
|
||
const before = new Board(book.records);
|
||
assert.ok(before.runs().filter((r) => r.plan.purpose === "forward").every((r) => !before.runPublic(r)));
|
||
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: null } });
|
||
const after = new Board(book.records);
|
||
assert.ok(after.runs().filter((r) => r.plan.purpose === "forward").every((r) => after.runPublic(r)));
|
||
assert.deepEqual(nextStep(book.records), { do: "halt", reason: "消融还没写（任务书 17）", objects: [] });
|
||
});
|
||
|
||
test("采纳只有一个因子的候选也统一在 listCells 停下，不擅自开收尾", () => {
|
||
const book = firstVerification(batchTwo(1), true, true);
|
||
for (const c of C) recordForward(book, c, 1, "valid");
|
||
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: C[0]! } });
|
||
assert.deepEqual(nextStep(book.records), { do: "halt", reason: "消融还没写（任务书 17）", objects: [] });
|
||
});
|