357 lines
20 KiB
TypeScript
357 lines
20 KiB
TypeScript
import assert from "node:assert/strict";
|
||
import { readFileSync } from "node:fs";
|
||
import { join } from "node:path";
|
||
import { test } from "node:test";
|
||
import { stepList, type ResearchConfig } from "../src/config/index.ts";
|
||
import type { AppendKind, Content, LedgerRecord } from "../src/ledger/index.ts";
|
||
import type { StudyRecord } from "../src/materials/index.ts";
|
||
import { nextStep } from "../src/flow/next-step.ts";
|
||
import type { Json, SubmissionKind } from "../src/vocab/index.ts";
|
||
import { fullConfig } from "./config-helpers.ts";
|
||
|
||
// flow 的"下一步做什么"（任务书 12 第六部分）：纯函数，手写账本记录，看它给出的下一步。
|
||
// 其余都由真实运行证明（任务书 12 第五部分），这里不写假模型、假 Docker。
|
||
|
||
const TASK_FACTS = JSON.parse(readFileSync(join(import.meta.dirname, "..", "tasks", "circle_packing", "task.json"), "utf8")) as Json;
|
||
const HASH = "0".repeat(64);
|
||
const measured = (value: number) => ({ status: "measured", value }) as const;
|
||
|
||
/** 调用结束记录去掉调用编号（四种结局各自去掉）。 */
|
||
type Ended = Content<"callEnded"> extends infer E ? (E extends unknown ? Omit<E, "call"> : never) : never;
|
||
|
||
/** 一本手写的账本：按写入先后编序号。内容照账本的形状写，nextStep 不再检查。 */
|
||
class Book {
|
||
readonly records: LedgerRecord[] = [];
|
||
|
||
add<K extends AppendKind>(kind: K, content: Content<K>): this {
|
||
this.records.push({ seq: this.records.length + 1, kind, hash: HASH, content } as LedgerRecord);
|
||
return this;
|
||
}
|
||
|
||
config(config: ResearchConfig): this {
|
||
const record: StudyRecord = {
|
||
config: config as unknown as Json,
|
||
steps: stepList(config) as unknown as Json,
|
||
prompts: { roles: {} as StudyRecord["prompts"]["roles"], tasks: {} as StudyRecord["prompts"]["tasks"] },
|
||
image: `sha256:${HASH}`,
|
||
task: TASK_FACTS,
|
||
seed: 1,
|
||
};
|
||
return this.add("config", record as unknown as Json);
|
||
}
|
||
|
||
baselineCode(): this {
|
||
return this.add("code", { object: "base", version: "v-base", locations: null });
|
||
}
|
||
|
||
/** 一次基线运行：没启动，或启动了、判分算数或不算数。 */
|
||
baselineRun(run: string, outcome: "notStarted" | "valid" | "invalid"): this {
|
||
const definition: Content<"runPlanned">["definition"] = {
|
||
command: ["python", "harness.py"],
|
||
workdir: "/work",
|
||
image: { digest: `sha256:${HASH}`, runtime: "runc" },
|
||
wallSeconds: 120,
|
||
cores: 1,
|
||
inheritEnv: false,
|
||
env: {},
|
||
};
|
||
this.add("runPlanned", { run, purpose: "baseline", round: 0, object: null, definition });
|
||
if (outcome === "notStarted") {
|
||
return this.add("runResult", { run, process: { ending: "notStarted", reason: "docker run 失败" }, judged: null });
|
||
}
|
||
this.add("runStarted", { run });
|
||
const process = { ending: "exited", exitCode: 0, cpuSeconds: measured(10), wallSeconds: measured(10), peakMemoryBytes: measured(1) } as const;
|
||
const judged = {
|
||
verdict: outcome,
|
||
verdictReason: outcome === "valid" ? "合法" : "不合法：圆重叠",
|
||
score: measured(0.9597642169962064),
|
||
metrics: {},
|
||
answerPath: { status: "unmeasured", reason: "基线不写" },
|
||
safeguardTriggers: { status: "unmeasured", reason: "基线不写" },
|
||
extra: null,
|
||
} as unknown as Content<"runResult">["judged"];
|
||
return this.add("runResult", { run, process, judged });
|
||
}
|
||
|
||
open(stageId: string, stage: Content<"stageOpen">["stage"], round: number, roster: string[]): this {
|
||
return this.add("stageOpen", { stageId, stage, round, roster });
|
||
}
|
||
|
||
/** 一份被收下的提交。nextStep 只看这个人在这个阶段有没有被收下的提交，种类一律写成入口。 */
|
||
entry(stageId: string, author: string): this {
|
||
const content = { entries: [{ name: "入口一", body: "从这里开始看" }, { name: "入口二", body: "从那里开始看" }] };
|
||
return this.add("submission", { stageId, author, target: null, item: null, version: 1, kind: "entry", content, jev: null });
|
||
}
|
||
|
||
/** 一份被收下的提交，内容 nextStep 不读。 */
|
||
sub(stageId: string, author: string, kind: SubmissionKind, target: string | null, item: string | null): this {
|
||
return this.add("submission", { stageId, author, target, item, version: 1, kind, content: {}, jev: null } as unknown as Content<"submission">);
|
||
}
|
||
|
||
/** 一次模型调用：开始和结束。"模型调用开始"里的材料只写 nextStep 认任务用的种类和对象。 */
|
||
call(stageId: string, actor: string, kind: SubmissionKind, object: string | null, n: number, ended: Ended): this {
|
||
const call = `${stageId}.${actor}.${kind}.${object ?? "none"}.${n}`;
|
||
this.add("callStarted", { call, actor, stageId, model: "iomgaa/glm-5.3", materials: { kind, object } });
|
||
return this.add("callEnded", { call, ...ended } as Content<"callEnded">);
|
||
}
|
||
}
|
||
|
||
const R = ["researcher-1", "researcher-2", "researcher-3"];
|
||
const entryTasks = (actors: string[]) => actors.map((actor) => ({ actor, kind: "entry", object: null }));
|
||
|
||
/** 冻结了配置、基线跑完算数的账本。 */
|
||
function afterBaseline(config: ResearchConfig = fullConfig()): Book {
|
||
return new Book().config(config).baselineCode().baselineRun("base-1", "valid");
|
||
}
|
||
|
||
test("空账本：冻结配置", () => {
|
||
assert.deepEqual(nextStep([]), { do: "freezeConfig" });
|
||
});
|
||
|
||
test("有配置、没有基线：先存基线代码，再用 base-1 跑基线", () => {
|
||
const book = new Book().config(fullConfig());
|
||
assert.deepEqual(nextStep(book.records), { do: "saveBaseline" });
|
||
book.baselineCode();
|
||
assert.deepEqual(nextStep(book.records), { do: "baselineRun", run: "base-1" });
|
||
});
|
||
|
||
test("基线最近一次是\"没启动\"：起新编号再跑", () => {
|
||
const book = new Book().config(fullConfig()).baselineCode().baselineRun("base-1", "notStarted");
|
||
assert.deepEqual(nextStep(book.records), { do: "baselineRun", run: "base-2" });
|
||
book.baselineRun("base-2", "notStarted");
|
||
assert.deepEqual(nextStep(book.records), { do: "baselineRun", run: "base-3" });
|
||
});
|
||
|
||
test("基线判为不算数：停下，原因写明", () => {
|
||
const book = new Book().config(fullConfig()).baselineCode().baselineRun("base-1", "invalid");
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "halt");
|
||
assert.match((next as { reason: string }).reason, /base-1 判为不算数（不合法：圆重叠）/);
|
||
});
|
||
|
||
test("基线算数之后：开写入口阶段，名单是全部研究者", () => {
|
||
assert.deepEqual(nextStep(afterBaseline().records), { do: "openStage", stageId: "r0-entryDraft", stage: "entryDraft", round: 0, roster: R });
|
||
});
|
||
|
||
test("阶段开着、一人已交、两人没交：派那两人", () => {
|
||
const book = afterBaseline().open("r0-entryDraft", "entryDraft", 0, R).entry("r0-entryDraft", "researcher-1");
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r0-entryDraft", tasks: entryTasks(["researcher-2", "researcher-3"]) });
|
||
});
|
||
|
||
test("都交了：关阶段；关了以后开第 1 次讨论", () => {
|
||
const book = afterBaseline().open("r0-entryDraft", "entryDraft", 0, R);
|
||
for (const r of R) book.entry("r0-entryDraft", r);
|
||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r0-entryDraft" });
|
||
book.add("stageClose", { stageId: "r0-entryDraft" });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r0-discussion1", stage: "discussion", round: 0, roster: R });
|
||
});
|
||
|
||
test("有人最近一次是\"没交\"：开发入口关着先派能派的、再停下；开着重新派", () => {
|
||
const stuck = (devEntry: boolean) =>
|
||
new Book()
|
||
.config({ ...fullConfig(), devEntry })
|
||
.baselineCode()
|
||
.baselineRun("base-1", "valid")
|
||
.open("r0-entryDraft", "entryDraft", 0, R)
|
||
.entry("r0-entryDraft", "researcher-1")
|
||
.call("r0-entryDraft", "researcher-2", "entry", null, 1, { wallSeconds: measured(30), status: "notSubmitted", reason: "模型结束了回复，没有一份提交被收下" });
|
||
// researcher-3 还没派过，先派他。
|
||
assert.deepEqual(nextStep(stuck(false).records), { do: "dispatch", stageId: "r0-entryDraft", tasks: entryTasks(["researcher-3"]) });
|
||
// researcher-3 交上以后只剩"没交"的 researcher-2：停下，附调用编号。
|
||
const closed = stuck(false).entry("r0-entryDraft", "researcher-3");
|
||
const next = nextStep(closed.records);
|
||
assert.equal(next.do, "halt");
|
||
assert.match((next as { reason: string }).reason, /^交不出合格提交/);
|
||
assert.deepEqual((next as { objects: string[] }).objects, ["r0-entryDraft.researcher-2.entry.none.1"]);
|
||
// 开发入口开着：重新派。
|
||
const dev = stuck(true).entry("r0-entryDraft", "researcher-3");
|
||
assert.deepEqual(nextStep(dev.records), { do: "dispatch", stageId: "r0-entryDraft", tasks: entryTasks(["researcher-2"]) });
|
||
});
|
||
|
||
test("有人最近一次是\"出错\"或\"进程中断\"：重新派；只看最近一次调用", () => {
|
||
const book = afterBaseline()
|
||
.open("r0-entryDraft", "entryDraft", 0, R)
|
||
.entry("r0-entryDraft", "researcher-1")
|
||
.call("r0-entryDraft", "researcher-2", "entry", null, 1, { wallSeconds: measured(5), status: "notSubmitted", reason: "没交" })
|
||
.call("r0-entryDraft", "researcher-2", "entry", null, 2, { wallSeconds: measured(5), status: "failed", reason: "模型服务报错" })
|
||
.call("r0-entryDraft", "researcher-3", "entry", null, 1, { wallSeconds: { status: "unmeasured", reason: "进程中断" }, status: "failed", reason: "进程中断" });
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r0-entryDraft", tasks: entryTasks(["researcher-2", "researcher-3"]) });
|
||
});
|
||
|
||
test("只有一位研究者：两次讨论各写一条按配置跳过，接着开认领", () => {
|
||
const one = fullConfig();
|
||
one.models.researchers = [one.models.researchers[0]!];
|
||
const book = afterBaseline(one).open("r0-entryDraft", "entryDraft", 0, ["researcher-1"]).entry("r0-entryDraft", "researcher-1");
|
||
book.add("stageClose", { stageId: "r0-entryDraft" });
|
||
const reason = "只有一位研究者，第零轮不讨论（方法.md 4.2）";
|
||
assert.deepEqual(nextStep(book.records), { do: "skip", step: "discussion1", reason });
|
||
book.add("event", { event: "skippedByConfig", reason, objects: ["discussion1"] });
|
||
assert.deepEqual(nextStep(book.records), { do: "skip", step: "discussion2", reason });
|
||
book.add("event", { event: "skippedByConfig", reason, objects: ["discussion2"] });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r0-claim", stage: "claim", round: 0, roster: ["researcher-1"] });
|
||
});
|
||
|
||
/** 第零轮做完：四个阶段都交齐关闭，分配决定已写。 */
|
||
function afterRoundZero(config: ResearchConfig): Book {
|
||
const book = afterBaseline(config);
|
||
const stages = [
|
||
["r0-entryDraft", "entryDraft"],
|
||
["r0-discussion1", "discussion"],
|
||
["r0-discussion2", "discussion"],
|
||
["r0-claim", "claim"],
|
||
] as const;
|
||
for (const [stageId, stage] of stages) {
|
||
book.open(stageId, stage, 0, R);
|
||
for (const r of R) book.entry(stageId, r);
|
||
book.add("stageClose", { stageId });
|
||
}
|
||
return book;
|
||
}
|
||
|
||
/** 分配决定。 */
|
||
function assigned(book: Book): Book {
|
||
const output = R.map((researcher, i) => ({ researcher, entry: `e-${researcher}-1`, rank: i + 1 }));
|
||
return book.add("decision", { decision: "entries", round: 0, rule: "assignEntries", ruleVersion: 1, params: null, seed: null, input: null, output });
|
||
}
|
||
|
||
test("认领关了：分配；分配做完：开第 1 轮的提案阶段，名单是全部研究者", () => {
|
||
const book = afterRoundZero(fullConfig());
|
||
assert.deepEqual(nextStep(book.records), { do: "assignEntries" });
|
||
assigned(book);
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-proposal", stage: "proposal", round: 1, roster: R });
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 第 1 轮：开轮到选批（任务书 14 第 2.2、2.3 节）
|
||
// 三条原提案：researcher-1 的单独成候选 r1-c1；researcher-2、researcher-3 的判为同一个改法，合成 r1-c2。
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const P1 = "r1-researcher-1-1";
|
||
const P2 = "r1-researcher-2-1";
|
||
const P3 = "r1-researcher-3-1";
|
||
const C = ["r1-c1", "r1-c2"];
|
||
|
||
/** 一个研究者阶段：开、每个任务交一份、关。tasks 是 [作者, 种类, 对象, 编号]。 */
|
||
function stage(book: Book, stageId: string, kind: Content<"stageOpen">["stage"], tasks: Array<[string, SubmissionKind, string | null, string | null]>): Book {
|
||
book.open(stageId, kind, 1, R);
|
||
for (const [author, k, target, item] of tasks) book.sub(stageId, author, k, target, item);
|
||
return book.add("stageClose", { stageId });
|
||
}
|
||
|
||
/** 每人 × 每个对象的任务。 */
|
||
const each = (kind: SubmissionKind, objects: string[]) => R.flatMap((actor) => objects.map((object) => ({ actor, kind, object })));
|
||
|
||
/** 第 1 轮写到 last 这一步做完为止。 */
|
||
function roundOne(last: "proposal" | "dedupe" | "firstPrediction" | "critique" | "response" | "freezeProtocol" | "secondPrediction" | "selectBatch"): Book {
|
||
const book = assigned(afterRoundZero(fullConfig()));
|
||
const order = ["proposal", "dedupe", "firstPrediction", "critique", "response", "freezeProtocol", "secondPrediction", "selectBatch"];
|
||
const steps: Record<string, () => void> = {
|
||
proposal: () => stage(book, "r1-proposal", "proposal", [["researcher-1", "proposal", null, P1], ["researcher-2", "proposal", null, P2], ["researcher-3", "proposal", null, P3]]),
|
||
dedupe: () => {
|
||
const output = { fresh: [{ candidate: "r1-c1", proposals: [P1] }, { candidate: "r1-c2", proposals: [P2, P3] }], carried: [] };
|
||
book.add("decision", { decision: "dedup", round: 1, rule: "jevSameChange", ruleVersion: 1, params: null, seed: null, input: null, output });
|
||
},
|
||
firstPrediction: () => stage(book, "r1-firstPrediction", "firstPrediction", R.flatMap((a) => C.map((c) => [a, "firstPrediction", c, null] as [string, SubmissionKind, string, null]))),
|
||
critique: () =>
|
||
stage(book, "r1-critique", "critique", [
|
||
["researcher-1", "critique", P2, `${P2}.researcher-1.1`],
|
||
["researcher-1", "critique", P3, `${P3}.researcher-1.1`],
|
||
["researcher-2", "critique", P1, `${P1}.researcher-2.1`],
|
||
["researcher-2", "critique", P3, `${P3}.researcher-2.1`],
|
||
["researcher-3", "critique", P1, `${P1}.researcher-3.1`],
|
||
["researcher-3", "critique", P2, `${P2}.researcher-3.1`],
|
||
]),
|
||
response: () => stage(book, "r1-response", "response", [["researcher-1", "response", "r1-c1", null], ["researcher-2", "unification", "r1-c2", null]]),
|
||
freezeProtocol: () => {
|
||
for (const candidate of C) book.add("protocol", { candidate, protocol: { round: 1, baseline: "base", proposals: [] } });
|
||
},
|
||
secondPrediction: () => stage(book, "r1-secondPrediction", "secondPrediction", R.flatMap((a) => C.map((c) => [a, "secondPrediction", c, null] as [string, SubmissionKind, string, null]))),
|
||
selectBatch: () => book.add("decision", { decision: "batch", round: 1, rule: "bestExpected", ruleVersion: 1, params: null, seed: "s", input: null, output: { chosen: ["r1-c1"] } }),
|
||
};
|
||
for (const step of order.slice(0, order.indexOf(last) + 1)) steps[step]!();
|
||
return book;
|
||
}
|
||
|
||
test("提案阶段一人交了两条、两人没交：只派没交的两人，提案没有对象", () => {
|
||
const book = assigned(afterRoundZero(fullConfig())).open("r1-proposal", "proposal", 1, R).sub("r1-proposal", "researcher-1", "proposal", null, P1);
|
||
book.sub("r1-proposal", "researcher-1", "proposal", null, "r1-researcher-1-2");
|
||
const tasks = ["researcher-2", "researcher-3"].map((actor) => ({ actor, kind: "proposal", object: null }));
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-proposal", tasks });
|
||
});
|
||
|
||
test("提案交齐关闭后：判重合并", () => {
|
||
assert.deepEqual(nextStep(roundOne("proposal").records), { do: "dedupe", round: 1 });
|
||
});
|
||
|
||
test("判重做完：开第一次预测；任务是每人 × 每个新候选（含自己的），同一个人的按候选编号排", () => {
|
||
const book = roundOne("dedupe");
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-firstPrediction", stage: "firstPrediction", round: 1, roster: R });
|
||
book.open("r1-firstPrediction", "firstPrediction", 1, R);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-firstPrediction", tasks: each("firstPrediction", C) });
|
||
});
|
||
|
||
test("同一个人的任务按对象分开算：对 r1-c1 没交的停住，r1-c2 照派；能派的派完才停下", () => {
|
||
const notSubmitted = { wallSeconds: measured(30), status: "notSubmitted", reason: "模型结束了回复，没有一份提交被收下" } as const;
|
||
const book = roundOne("dedupe").open("r1-firstPrediction", "firstPrediction", 1, R);
|
||
for (const a of ["researcher-1", "researcher-3"]) for (const c of C) book.sub("r1-firstPrediction", a, "firstPrediction", c, null);
|
||
book.call("r1-firstPrediction", "researcher-2", "firstPrediction", "r1-c1", 1, notSubmitted);
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-firstPrediction", tasks: [{ actor: "researcher-2", kind: "firstPrediction", object: "r1-c2" }] });
|
||
book.sub("r1-firstPrediction", "researcher-2", "firstPrediction", "r1-c2", null);
|
||
const next = nextStep(book.records);
|
||
assert.equal(next.do, "halt");
|
||
assert.deepEqual((next as { objects: string[] }).objects, ["r1-firstPrediction.researcher-2.firstPrediction.r1-c1.1"]);
|
||
});
|
||
|
||
test("挑刺阶段：每人 × 每条不是自己写的原提案，没有挑自己的", () => {
|
||
const book = roundOne("firstPrediction").open("r1-critique", "critique", 1, R);
|
||
const expected = [
|
||
{ actor: "researcher-1", kind: "critique", object: P2 },
|
||
{ actor: "researcher-1", kind: "critique", object: P3 },
|
||
{ actor: "researcher-2", kind: "critique", object: P1 },
|
||
{ actor: "researcher-2", kind: "critique", object: P3 },
|
||
{ actor: "researcher-3", kind: "critique", object: P1 },
|
||
{ actor: "researcher-3", kind: "critique", object: P2 },
|
||
];
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-critique", tasks: expected });
|
||
});
|
||
|
||
test("回应阶段：单条原提案的候选派作者交回应，合并而成的派编号最小的作者交统一", () => {
|
||
const book = roundOne("critique").open("r1-response", "response", 1, R);
|
||
const expected = [
|
||
{ actor: "researcher-1", kind: "response", object: "r1-c1" },
|
||
{ actor: "researcher-2", kind: "unification", object: "r1-c2" },
|
||
];
|
||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-response", tasks: expected });
|
||
});
|
||
|
||
test("回应阶段关闭、还没有冻结协议：写冻结协议；写完开第二次预测，任务是每人 × 池里每个候选", () => {
|
||
const book = roundOne("response");
|
||
assert.deepEqual(nextStep(book.records), { do: "freezeProtocols", round: 1 });
|
||
const frozen = roundOne("freezeProtocol");
|
||
assert.deepEqual(nextStep(frozen.records), { do: "openStage", stageId: "r1-secondPrediction", stage: "secondPrediction", round: 1, roster: R });
|
||
frozen.open("r1-secondPrediction", "secondPrediction", 1, R);
|
||
assert.deepEqual(nextStep(frozen.records), { do: "dispatch", stageId: "r1-secondPrediction", tasks: each("secondPrediction", C) });
|
||
});
|
||
|
||
test("第二次预测关闭：选批", () => {
|
||
assert.deepEqual(nextStep(roundOne("secondPrediction").records), { do: "selectBatch", round: 1 });
|
||
});
|
||
|
||
test("选批做完：停下，原因\"实现还没写\"", () => {
|
||
assert.deepEqual(nextStep(roundOne("selectBatch").records), { do: "halt", reason: "实现还没写（任务书 16）", objects: [] });
|
||
});
|
||
|
||
test("关掉第一次预测：第 1 轮写一条按配置跳过（对象 r1-firstPrediction），接着开挑刺", () => {
|
||
const config = fullConfig();
|
||
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
|
||
config.flow.firstPrediction = false;
|
||
const book = assigned(afterRoundZero(config));
|
||
stage(book, "r1-proposal", "proposal", [["researcher-1", "proposal", null, P1], ["researcher-2", "proposal", null, P2], ["researcher-3", "proposal", null, P3]]);
|
||
const output = { fresh: [{ candidate: "r1-c1", proposals: [P1] }, { candidate: "r1-c2", proposals: [P2, P3] }], carried: [] };
|
||
book.add("decision", { decision: "dedup", round: 1, rule: "jevSameChange", ruleVersion: 1, params: null, seed: null, input: null, output });
|
||
const reason = "配置关掉了第一次预测";
|
||
assert.deepEqual(nextStep(book.records), { do: "skip", step: "r1-firstPrediction", reason });
|
||
book.add("event", { event: "skippedByConfig", reason, objects: ["r1-firstPrediction"] });
|
||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-critique", stage: "critique", round: 1, roster: R });
|
||
});
|