feat: flow 第 1 部分（骨架、研究开始、第零轮）与第零轮检验
按任务书 12 实现 src/flow：从账本现算下一步的纯函数、主循环、恢复入口、派模型任务和跑正式运行两个共用动作、冻结配置、基线、第零轮三个研究者阶段和入口分配，走到第 1 轮时停下。materials 导出 finalEntries、加 researcherId 和 StudyRecord。 审阅采纳两条：研究编号本机已用过时冻结前报错；docker run 没启动时写完结果就停下，不在本进程里无限起新编号。 eval/ 新增第零轮检验：起研究命令行、回放假模型、检验用 Jev 客户端、六问质量评估、16-real-1 回放、杀进程与两份同时跑。tsc 与 depcruise 也检查 eval/，新增 eval 只经 index.ts 用正式代码的规则。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,78 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test, type TestContext } from "node:test";
|
||||
import { runStudy } from "../src/flow/index.ts";
|
||||
import { Ledger } from "../src/ledger/index.ts";
|
||||
import { Runner } from "../src/runner/index.ts";
|
||||
import { fullConfig } from "./config-helpers.ts";
|
||||
|
||||
// flow 碰到 Docker 的两处边界，用真实 Docker 测，不花模型的钱：走不到派模型任务就停下。
|
||||
// 镜像一律 inforsi-test- 开头的固定名字；本机还有别的项目在建删容器和镜像，只动自己前缀的。
|
||||
|
||||
const TASK_DIR = join(import.meta.dirname, "..", "tasks", "circle_packing");
|
||||
|
||||
function docker(args: string[]): { status: number | null; stdout: string; stderr: string } {
|
||||
return spawnSync("docker", args, { encoding: "utf8" });
|
||||
}
|
||||
|
||||
/** 删掉测试自己的镜像标签；不存在不算错，别的错误照样报。 */
|
||||
function removeImage(tag: string): void {
|
||||
const removed = docker(["rmi", tag]);
|
||||
if (removed.status !== 0 && !removed.stderr.includes("No such image")) throw new Error(removed.stderr);
|
||||
}
|
||||
|
||||
/** 一个测试用的研究编号：先清掉上次没删干净的同名镜像，测试结束后再删。 */
|
||||
function studyId(t: TestContext, suffix: string): string {
|
||||
const id = `test-flow-${suffix}`;
|
||||
removeImage(`inforsi-${id}`);
|
||||
t.after(() => removeImage(`inforsi-${id}`));
|
||||
return id;
|
||||
}
|
||||
|
||||
function tempDir(t: TestContext): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "inforsi-flow-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
|
||||
function ledgerKinds(studyDir: string): string[] {
|
||||
const ledger = Ledger.open(join(studyDir, "ledger.sqlite"));
|
||||
try {
|
||||
return ledger.records().map((r) => r.kind);
|
||||
} finally {
|
||||
ledger.close();
|
||||
}
|
||||
}
|
||||
|
||||
test("研究编号本机用过（已有 inforsi-<研究编号> 镜像）：冻结配置前报错停下，不建镜像、不写研究配置记录", async (t) => {
|
||||
const id = studyId(t, "dup");
|
||||
// 一份旧研究冻结配置时建下的镜像。
|
||||
await new Runner(null).buildImage(TASK_DIR, `inforsi-${id}`);
|
||||
const dir = tempDir(t);
|
||||
await assert.rejects(
|
||||
runStudy({ studyDir: dir, config: { ...fullConfig(), studyId: id }, stop: new AbortController().signal }),
|
||||
new RegExp(`研究编号 ${id} 已被本机用过（已有镜像 inforsi-${id}）`),
|
||||
);
|
||||
assert.deepEqual(ledgerKinds(dir), []);
|
||||
});
|
||||
|
||||
test("docker run 没启动：写结果\"没启动\"后停下，原因带 Docker 报错原文，不在本进程里起新编号重跑", { timeout: 120_000 }, async (t) => {
|
||||
// 任务事实里写一个本机没有的 Docker 运行时，docker run 每次都失败，也不建容器。
|
||||
const task = join(tempDir(t), "task");
|
||||
cpSync(TASK_DIR, task, { recursive: true });
|
||||
const facts = JSON.parse(readFileSync(join(task, "task.json"), "utf8")) as { run: { runtime: string } };
|
||||
facts.run.runtime = "inforsi-test-no-such-runtime";
|
||||
writeFileSync(join(task, "task.json"), JSON.stringify(facts));
|
||||
const dir = tempDir(t);
|
||||
const ending = await runStudy({
|
||||
studyDir: dir,
|
||||
config: { ...fullConfig(), studyId: studyId(t, "not-started"), task },
|
||||
stop: new AbortController().signal,
|
||||
});
|
||||
assert.equal(ending.status, "halted");
|
||||
assert.match((ending as { reason: string }).reason, /运行 base-1 没有启动.*unknown or invalid runtime name: inforsi-test-no-such-runtime/);
|
||||
assert.deepEqual(ledgerKinds(dir), ["config", "code", "runPlanned", "runResult", "event"]);
|
||||
});
|
||||
@@ -0,0 +1,205 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { stepList, type ResearchConfig } from "../src/config/index.ts";
|
||||
import type { AppendKind, Content, LedgerRecord } from "../src/ledger/index.ts";
|
||||
import type { StudyRecord } from "../src/materials/index.ts";
|
||||
import { nextStep } from "../src/flow/next-step.ts";
|
||||
import type { Json } from "../src/vocab/index.ts";
|
||||
import { fullConfig } from "./config-helpers.ts";
|
||||
|
||||
// flow 的"下一步做什么"（任务书 12 第六部分）：纯函数，手写账本记录，看它给出的下一步。
|
||||
// 其余都由真实运行证明（任务书 12 第五部分），这里不写假模型、假 Docker。
|
||||
|
||||
const TASK_FACTS = JSON.parse(readFileSync(join(import.meta.dirname, "..", "tasks", "circle_packing", "task.json"), "utf8")) as Json;
|
||||
const HASH = "0".repeat(64);
|
||||
const measured = (value: number) => ({ status: "measured", value }) as const;
|
||||
|
||||
/** 调用结束记录去掉调用编号（四种结局各自去掉）。 */
|
||||
type Ended = Content<"callEnded"> extends infer E ? (E extends unknown ? Omit<E, "call"> : never) : never;
|
||||
|
||||
/** 一本手写的账本：按写入先后编序号。内容照账本的形状写，nextStep 不再检查。 */
|
||||
class Book {
|
||||
readonly records: LedgerRecord[] = [];
|
||||
|
||||
add<K extends AppendKind>(kind: K, content: Content<K>): this {
|
||||
this.records.push({ seq: this.records.length + 1, kind, hash: HASH, content } as LedgerRecord);
|
||||
return this;
|
||||
}
|
||||
|
||||
config(config: ResearchConfig): this {
|
||||
const record: StudyRecord = {
|
||||
config: config as unknown as Json,
|
||||
steps: stepList(config) as unknown as Json,
|
||||
prompts: { roles: {} as StudyRecord["prompts"]["roles"], tasks: {} as StudyRecord["prompts"]["tasks"] },
|
||||
image: `sha256:${HASH}`,
|
||||
task: TASK_FACTS,
|
||||
seed: 1,
|
||||
};
|
||||
return this.add("config", record as unknown as Json);
|
||||
}
|
||||
|
||||
baselineCode(): this {
|
||||
return this.add("code", { object: "base", version: "v-base", locations: null });
|
||||
}
|
||||
|
||||
/** 一次基线运行：没启动，或启动了、判分算数或不算数。 */
|
||||
baselineRun(run: string, outcome: "notStarted" | "valid" | "invalid"): this {
|
||||
const definition: Content<"runPlanned">["definition"] = {
|
||||
command: ["python", "harness.py"],
|
||||
workdir: "/work",
|
||||
image: { digest: `sha256:${HASH}`, runtime: "runc" },
|
||||
wallSeconds: 120,
|
||||
cores: 1,
|
||||
inheritEnv: false,
|
||||
env: {},
|
||||
};
|
||||
this.add("runPlanned", { run, purpose: "baseline", round: 0, object: null, definition });
|
||||
if (outcome === "notStarted") {
|
||||
return this.add("runResult", { run, process: { ending: "notStarted", reason: "docker run 失败" }, judged: null });
|
||||
}
|
||||
this.add("runStarted", { run });
|
||||
const process = { ending: "exited", exitCode: 0, cpuSeconds: measured(10), wallSeconds: measured(10), peakMemoryBytes: measured(1) } as const;
|
||||
const judged = {
|
||||
verdict: outcome,
|
||||
verdictReason: outcome === "valid" ? "合法" : "不合法：圆重叠",
|
||||
score: measured(0.9597642169962064),
|
||||
metrics: {},
|
||||
answerPath: { status: "unmeasured", reason: "基线不写" },
|
||||
safeguardTriggers: { status: "unmeasured", reason: "基线不写" },
|
||||
extra: null,
|
||||
} as unknown as Content<"runResult">["judged"];
|
||||
return this.add("runResult", { run, process, judged });
|
||||
}
|
||||
|
||||
open(stageId: string, stage: Content<"stageOpen">["stage"], roster: string[]): this {
|
||||
return this.add("stageOpen", { stageId, stage, round: 0, roster });
|
||||
}
|
||||
|
||||
/** 一份被收下的提交。nextStep 只看这个人在这个阶段有没有被收下的提交，种类一律写成入口。 */
|
||||
entry(stageId: string, author: string): this {
|
||||
const content = { entries: [{ name: "入口一", body: "从这里开始看" }, { name: "入口二", body: "从那里开始看" }] };
|
||||
return this.add("submission", { stageId, author, target: null, item: null, version: 1, kind: "entry", content, jev: null });
|
||||
}
|
||||
|
||||
/** 一次模型调用：开始和结束。 */
|
||||
call(stageId: string, actor: string, n: number, ended: Ended): this {
|
||||
const call = `${stageId}.${actor}.entry.none.${n}`;
|
||||
this.add("callStarted", { call, actor, stageId, model: "iomgaa/glm-5.3", materials: null });
|
||||
return this.add("callEnded", { call, ...ended } as Content<"callEnded">);
|
||||
}
|
||||
}
|
||||
|
||||
const R = ["researcher-1", "researcher-2", "researcher-3"];
|
||||
|
||||
/** 冻结了配置、基线跑完算数的账本。 */
|
||||
function afterBaseline(config: ResearchConfig = fullConfig()): Book {
|
||||
return new Book().config(config).baselineCode().baselineRun("base-1", "valid");
|
||||
}
|
||||
|
||||
test("空账本：冻结配置", () => {
|
||||
assert.deepEqual(nextStep([]), { do: "freezeConfig" });
|
||||
});
|
||||
|
||||
test("有配置、没有基线：先存基线代码，再用 base-1 跑基线", () => {
|
||||
const book = new Book().config(fullConfig());
|
||||
assert.deepEqual(nextStep(book.records), { do: "saveBaseline" });
|
||||
book.baselineCode();
|
||||
assert.deepEqual(nextStep(book.records), { do: "baselineRun", run: "base-1" });
|
||||
});
|
||||
|
||||
test("基线最近一次是\"没启动\"：起新编号再跑", () => {
|
||||
const book = new Book().config(fullConfig()).baselineCode().baselineRun("base-1", "notStarted");
|
||||
assert.deepEqual(nextStep(book.records), { do: "baselineRun", run: "base-2" });
|
||||
book.baselineRun("base-2", "notStarted");
|
||||
assert.deepEqual(nextStep(book.records), { do: "baselineRun", run: "base-3" });
|
||||
});
|
||||
|
||||
test("基线判为不算数：停下，原因写明", () => {
|
||||
const book = new Book().config(fullConfig()).baselineCode().baselineRun("base-1", "invalid");
|
||||
const next = nextStep(book.records);
|
||||
assert.equal(next.do, "halt");
|
||||
assert.match((next as { reason: string }).reason, /base-1 判为不算数（不合法：圆重叠）/);
|
||||
});
|
||||
|
||||
test("基线算数之后：开写入口阶段，名单是全部研究者", () => {
|
||||
assert.deepEqual(nextStep(afterBaseline().records), { do: "openStage", stageId: "r0-entryDraft", stage: "entryDraft", roster: R });
|
||||
});
|
||||
|
||||
test("阶段开着、一人已交、两人没交：派那两人", () => {
|
||||
const book = afterBaseline().open("r0-entryDraft", "entryDraft", R).entry("r0-entryDraft", "researcher-1");
|
||||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r0-entryDraft", kind: "entry", actors: ["researcher-2", "researcher-3"] });
|
||||
});
|
||||
|
||||
test("都交了：关阶段；关了以后开第 1 次讨论", () => {
|
||||
const book = afterBaseline().open("r0-entryDraft", "entryDraft", R);
|
||||
for (const r of R) book.entry("r0-entryDraft", r);
|
||||
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r0-entryDraft" });
|
||||
book.add("stageClose", { stageId: "r0-entryDraft" });
|
||||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r0-discussion1", stage: "discussion", roster: R });
|
||||
});
|
||||
|
||||
test("有人最近一次是\"没交\"：开发入口关着先派能派的、再停下；开着重新派", () => {
|
||||
const stuck = (devEntry: boolean) =>
|
||||
new Book()
|
||||
.config({ ...fullConfig(), devEntry })
|
||||
.baselineCode()
|
||||
.baselineRun("base-1", "valid")
|
||||
.open("r0-entryDraft", "entryDraft", R)
|
||||
.entry("r0-entryDraft", "researcher-1")
|
||||
.call("r0-entryDraft", "researcher-2", 1, { wallSeconds: measured(30), status: "notSubmitted", reason: "模型结束了回复，没有一份提交被收下" });
|
||||
// researcher-3 还没派过，先派他。
|
||||
assert.deepEqual(nextStep(stuck(false).records), { do: "dispatch", stageId: "r0-entryDraft", kind: "entry", actors: ["researcher-3"] });
|
||||
// researcher-3 交上以后只剩"没交"的 researcher-2：停下，附调用编号。
|
||||
const closed = stuck(false).entry("r0-entryDraft", "researcher-3");
|
||||
const next = nextStep(closed.records);
|
||||
assert.equal(next.do, "halt");
|
||||
assert.match((next as { reason: string }).reason, /^交不出合格提交/);
|
||||
assert.deepEqual((next as { objects: string[] }).objects, ["r0-entryDraft.researcher-2.entry.none.1"]);
|
||||
// 开发入口开着：重新派。
|
||||
const dev = stuck(true).entry("r0-entryDraft", "researcher-3");
|
||||
assert.deepEqual(nextStep(dev.records), { do: "dispatch", stageId: "r0-entryDraft", kind: "entry", actors: ["researcher-2"] });
|
||||
});
|
||||
|
||||
test("有人最近一次是\"出错\"或\"进程中断\"：重新派；只看最近一次调用", () => {
|
||||
const book = afterBaseline()
|
||||
.open("r0-entryDraft", "entryDraft", R)
|
||||
.entry("r0-entryDraft", "researcher-1")
|
||||
.call("r0-entryDraft", "researcher-2", 1, { wallSeconds: measured(5), status: "notSubmitted", reason: "没交" })
|
||||
.call("r0-entryDraft", "researcher-2", 2, { wallSeconds: measured(5), status: "failed", reason: "模型服务报错" })
|
||||
.call("r0-entryDraft", "researcher-3", 1, { wallSeconds: { status: "unmeasured", reason: "进程中断" }, status: "failed", reason: "进程中断" });
|
||||
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r0-entryDraft", kind: "entry", actors: ["researcher-2", "researcher-3"] });
|
||||
});
|
||||
|
||||
test("只有一位研究者：两次讨论各写一条按配置跳过，接着开认领", () => {
|
||||
const one = fullConfig();
|
||||
one.models.researchers = [one.models.researchers[0]!];
|
||||
const book = afterBaseline(one).open("r0-entryDraft", "entryDraft", ["researcher-1"]).entry("r0-entryDraft", "researcher-1");
|
||||
book.add("stageClose", { stageId: "r0-entryDraft" });
|
||||
const reason = "只有一位研究者，第零轮不讨论（方法.md 4.2）";
|
||||
assert.deepEqual(nextStep(book.records), { do: "skip", step: "discussion1", reason });
|
||||
book.add("event", { event: "skippedByConfig", reason, objects: ["discussion1"] });
|
||||
assert.deepEqual(nextStep(book.records), { do: "skip", step: "discussion2", reason });
|
||||
book.add("event", { event: "skippedByConfig", reason, objects: ["discussion2"] });
|
||||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r0-claim", stage: "claim", roster: ["researcher-1"] });
|
||||
});
|
||||
|
||||
test("认领关了：分配；分配做完：停下，原因\"第 1 轮还没写\"", () => {
|
||||
const book = afterBaseline();
|
||||
const stages = [
|
||||
["r0-entryDraft", "entryDraft"],
|
||||
["r0-discussion1", "discussion"],
|
||||
["r0-discussion2", "discussion"],
|
||||
["r0-claim", "claim"],
|
||||
] as const;
|
||||
for (const [stageId, stage] of stages) {
|
||||
book.open(stageId, stage, R);
|
||||
for (const r of R) book.entry(stageId, r);
|
||||
book.add("stageClose", { stageId });
|
||||
}
|
||||
assert.deepEqual(nextStep(book.records), { do: "assignEntries" });
|
||||
const output = R.map((researcher, i) => ({ researcher, entry: `e-${researcher}-1`, rank: i + 1 }));
|
||||
book.add("decision", { decision: "entries", round: 0, rule: "assignEntries", ruleVersion: 1, params: null, seed: null, input: null, output });
|
||||
assert.deepEqual(nextStep(book.records), { do: "halt", reason: "第 1 轮还没写（任务书 13）", objects: [] });
|
||||
});
|
||||
Reference in New Issue
Block a user