feat: flow 第 1 部分(骨架、研究开始、第零轮)与第零轮检验

按任务书 12 实现 src/flow:从账本现算下一步的纯函数、主循环、恢复入口、派模型任务和跑正式运行两个共用动作、冻结配置、基线、第零轮三个研究者阶段和入口分配,走到第 1 轮时停下。materials 导出 finalEntries、加 researcherId 和 StudyRecord。

审阅采纳两条:研究编号本机已用过时冻结前报错;docker run 没启动时写完结果就停下,不在本进程里无限起新编号。

eval/ 新增第零轮检验:起研究命令行、回放假模型、检验用 Jev 客户端、六问质量评估、16-real-1 回放、杀进程与两份同时跑。tsc 与 depcruise 也检查 eval/,新增 eval 只经 index.ts 用正式代码的规则。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-03 10:58:52 -04:00
parent 8491191a2b
commit 75d7cffd27
36 changed files with 2456 additions and 14 deletions
+90
View File
@@ -0,0 +1,90 @@
import { rmSync } from "node:fs";
import type { Content } from "../ledger/index.ts";
import { formalRun as runDefinition, judge, prepareJudging, prepareRun } from "../tasks/index.ts";
import type { HypothesisMetric, JudgedResult, ProcessResult } from "../vocab/index.ts";
import { checkoutVersion } from "../workspace/index.ts";
import { containerName, freshDir, judgeContainerName, runPaths, workspaceRepo } from "./paths.ts";
import type { Study } from "./study.ts";
// 共用动作二:跑一次正式运行(任务书 12 第 2.7 节)。先登记、再启动、再写结果,前后顺序由这个入口保证
// (方法.md 第七、九部分):登记占名额,确认启动才扣次,确认没启动的写结果"没启动"、退还名额。
// 判分不扣次数,不登记。恢复时接着等一次已经启动的运行,走同一个 finishRun。
export type RunPlan = Omit<Content<"runPlanned">, "definition">;
/**
* 一次正式运行怎么结束:写完结果;被用户停下;或 docker run 没启动,附 Docker 的报错原文。
* 没启动是外部错误,本进程不重试(CLAUDE.md"错误怎么处理",重试 0 次):调用方写停下事件,
* 下次启动时"下一步做什么"照旧起新编号重跑。
*/
export type RunOutcome = "done" | "stopped" | { notStarted: string };
/** 跑一次正式运行:取出 version 的代码,在任务镜像里跑,判分,写结果。用户停下时不等容器,交回 "stopped"。 */
export async function formalRun(s: Study, plan: RunPlan, version: string, metrics: HypothesisMetric[]): Promise<RunOutcome> {
const definition = runDefinition(s.task, s.frozen.image, String(s.frozen.seed));
s.ledger.append("runPlanned", { ...plan, definition });
const paths = runPaths(s.dir, plan.run);
freshDir(paths.base);
checkoutVersion(workspaceRepo(s.dir), version, freshDir(paths.code));
prepareRun(s.task, paths.code, freshDir(paths.run), metrics);
const started = await s.runner.start(containerName(s.frozen.config.studyId, plan.run), definition, paths.run);
if (!started.started) {
s.ledger.append("runResult", { run: plan.run, process: { ending: "notStarted", reason: started.reason }, judged: null });
return { notStarted: started.reason };
}
s.ledger.append("runStarted", { run: plan.run });
return finishRun(s, plan.run, metrics);
}
/**
* 等一次已经确认启动的运行结束,判分,写结果,删掉容器。日志写进这次运行自己的日志目录。
* 收到"停下"信号就不等了,直接交回 "stopped";容器不管它,下次启动时恢复入口再来接着等。
*/
export async function finishRun(s: Study, run: string, metrics: HypothesisMetric[]): Promise<"done" | "stopped"> {
const name = containerName(s.frozen.config.studyId, run);
const paths = runPaths(s.dir, run);
const waited = await untilStopped(s.stop, s.runner.wait(name, emptyDir(paths.logs)));
if (waited === "stopped") return "stopped";
const judged = await judgeRun(s, run, waited, metrics);
s.ledger.append("runResult", { run, process: waited, judged });
await s.runner.remove(name);
return "done";
}
/**
* 判分:tasks 的 prepareJudging,要跑校验器就让 runner 在镜像里跑,再交给 judge。
* 判分目录每次重新建空的(恢复时会再判一次);判分容器还在就先删掉再用。
*/
async function judgeRun(s: Study, run: string, formal: ProcessResult, metrics: HypothesisMetric[]): Promise<JudgedResult> {
const paths = runPaths(s.dir, run);
const step = prepareJudging(s.task, paths.run, emptyDir(paths.judge), s.frozen.image, formal, metrics);
if (step.status === "judged") return step.result;
const name = judgeContainerName(s.frozen.config.studyId, run);
if ((await s.runner.status(name)) !== "absent") await s.runner.remove(name);
const started = await s.runner.start(name, step.definition, paths.judge);
if (!started.started) throw new Error(`运行 ${run} 的校验器容器没有启动:${started.reason}`);
const validator = await s.runner.wait(name, emptyDir(paths.judgeLogs));
await s.runner.remove(name);
return judge(s.task, paths.run, paths.judge, formal, validator, metrics);
}
/** 清空重建一个目录:恢复时会再等一次、再判一次,上一次没写完的日志和判分目录不留。 */
function emptyDir(dir: string): string {
rmSync(dir, { recursive: true, force: true });
return freshDir(dir);
}
/** 等一件事做完,或者收到"停下"信号先交回 "stopped"。被放下的那个 Promise 之后的结果没人用。 */
async function untilStopped<T>(stop: AbortSignal, work: Promise<T>): Promise<T | "stopped"> {
let onStop = (): void => undefined;
const stopped = new Promise<"stopped">((resolve) => {
onStop = () => resolve("stopped");
if (stop.aborted) onStop();
else stop.addEventListener("abort", onStop, { once: true });
});
try {
return await Promise.race([work, stopped]);
} finally {
stop.removeEventListener("abort", onStop);
}
}
+41
View File
@@ -0,0 +1,41 @@
import { checkConfig, type ResearchConfig, type StepList } from "../config/index.ts";
import type { LedgerRecord } from "../ledger/index.ts";
import type { StudyRecord } from "../materials/index.ts";
import { checkTaskFacts, type TaskFacts } from "../tasks/index.ts";
// 从账本读回冻结的研究配置(方法.md 4.1:之后的流程只读这份冻结的配置)。纯函数,"下一步做什么"也用它。
/** 完整流程的配置。朴素循环和步骤开关以后再做(docs/结构.md 第五部分),flow 现在只走完整流程。 */
export type FullConfig = ResearchConfig & { flow: Extract<ResearchConfig["flow"], { preset: "full" }> };
export function fullConfig(config: ResearchConfig): FullConfig {
if (config.flow.preset !== "full") {
throw new Error("flow 现在只实现完整流程,朴素循环还没写(docs/结构.md 第五部分\"步骤开关和对照组\")");
}
return config as FullConfig;
}
/** 冻结的研究配置:配置、任务事实再检查过一遍;步骤清单是 config 的 stepList 写下的,照原样用。 */
export type Frozen = {
config: FullConfig;
steps: StepList;
facts: TaskFacts;
prompts: StudyRecord["prompts"];
image: string;
seed: number;
};
/** 账本里还没有研究配置记录就是新研究,交回 null。 */
export function readFrozen(records: readonly LedgerRecord[]): Frozen | null {
const record = records.find((r): r is LedgerRecord<"config"> => r.kind === "config");
if (record === undefined) return null;
const c = record.content as unknown as StudyRecord;
return {
config: fullConfig(checkConfig(c.config)),
steps: c.steps as unknown as StepList,
facts: checkTaskFacts(c.task),
prompts: c.prompts,
image: c.image,
seed: c.seed,
};
}
+4
View File
@@ -0,0 +1,4 @@
// flow:按冻结的步骤清单推进一份研究;每次启动走同一条恢复路径(docs/结构.md 第二部分)。
// 对外只有 runStudy,以后 plugin 也用它。
export { runStudy, type StudyInput } from "./loop.ts";
export type { StudyEnding } from "./study.ts";
+99
View File
@@ -0,0 +1,99 @@
import { mkdirSync } from "node:fs";
import { resolve } from "node:path";
import type { ResearchConfig } from "../config/index.ts";
import { Ledger } from "../ledger/index.ts";
import { Runner } from "../runner/index.ts";
import { fullConfig, readFrozen } from "./frozen.ts";
import { EVENTS, nextStep, type Next } from "./next-step.ts";
import { ledgerFile } from "./paths.ts";
import { recover } from "./recover.ts";
import { baselineRun, saveBaseline } from "./steps/baseline.ts";
import { assignEntries, dispatchStage } from "./steps/round-zero.ts";
import { freezeConfig, halt, openStudy, USER_STOPPED, type Study, type StudyEnding } from "./study.ts";
// 主循环:读账本 → 问下一步 → 做这一步 → 回到开头(任务书 12 第 2.4 节)。每做完一步都重新读账本。
// 每次启动走同一条路:新研究先冻结配置;之后把研究接上(核对镜像和提示词)、核对未完成的动作,再进入主循环。
export type StudyInput = {
/** 研究目录:账本、代码仓库、每次运行和每次模型调用的目录都在这里(任务书 12 第 2.1 节)。 */
studyDir: string;
/** 新研究交 checkConfig 检查过的配置;接着跑一份已有的研究写 null,以账本里冻结的那份为准。 */
config: ResearchConfig | null;
/** 用户停下研究:正在跑的模型会话立即停,正在跑的容器不管它,下次启动时由恢复入口接着等。 */
stop: AbortSignal;
};
export async function runStudy(input: StudyInput): Promise<StudyEnding> {
const dir = resolve(input.studyDir);
const config = input.config === null ? null : fullConfig(input.config);
mkdirSync(dir, { recursive: true });
const ledger = Ledger.open(ledgerFile(dir));
try {
const frozen = readFrozen(ledger.records()) !== null;
if (!frozen && config === null) throw new Error(`研究目录 ${dir} 里还没有研究,配置不能写 null`);
if (frozen && config !== null) throw new Error(`研究目录 ${dir} 里已经有冻结的配置:接着跑时配置写 null,以账本里冻结的那份为准`);
// Docker 守护进程的地址交给 docker 命令按自己的环境变量和 context 决定。
const runner = new Runner(null);
// 恢复入口:把研究接上,核对未完成的动作,之后才问下一步(任务书 12 第 2.5 节)。新研究冻结配置之后走同一条路。
const begin = async (): Promise<Study | StudyEnding> => {
const opened = await openStudy(dir, ledger, runner, input.stop);
if ("halt" in opened) return halt(ledger, opened.halt, []);
const recovered = await recover(opened);
if (recovered === "stopped") return USER_STOPPED;
if (recovered !== null) return { status: "halted", reason: recovered.halt };
return opened;
};
let study: Study | null = null;
if (frozen) {
const begun = await begin();
if ("status" in begun) return begun;
study = begun;
}
for (;;) {
if (input.stop.aborted) return USER_STOPPED;
const next = nextStep(ledger.records());
if (next.do === "freezeConfig") {
await freezeConfig(dir, ledger, runner, config!);
const begun = await begin();
if ("status" in begun) return begun;
study = begun;
continue;
}
const ended = await doStep(study!, next);
if (ended !== null) return ended;
}
} finally {
ledger.close();
}
}
/** 做一步。交回 null 表示接着走下一步。 */
async function doStep(s: Study, next: Exclude<Next, { do: "freezeConfig" }>): Promise<StudyEnding | null> {
switch (next.do) {
case "saveBaseline":
saveBaseline(s);
return null;
case "baselineRun": {
const ran = await baselineRun(s, next.run);
if (ran === "stopped") return USER_STOPPED;
if (ran === "done") return null;
return halt(s.ledger, `运行 ${next.run} 没有启动,名额已退还;本进程不重跑,下次启动时起新编号重跑。Docker 的报错:${ran.notStarted}`, [next.run]);
}
case "skip":
s.ledger.append("event", { event: EVENTS.skipped, reason: next.reason, objects: [next.step] });
return null;
case "openStage":
s.ledger.append("stageOpen", { stageId: next.stageId, stage: next.stage, round: 0, roster: next.roster });
return null;
case "closeStage":
s.ledger.append("stageClose", { stageId: next.stageId });
return null;
case "dispatch":
return dispatchStage(s, next);
case "assignEntries":
assignEntries(s);
return null;
case "halt":
return halt(s.ledger, next.reason, next.objects);
}
}
+121
View File
@@ -0,0 +1,121 @@
import { runSession, type KindOf, type SessionEnding, type SubmitAttempt, type SubmitReceipt } from "../agents/index.ts";
import { buildMaterial, callId, queryMaterial, researcherId, type TaskMaterial } from "../materials/index.ts";
import { receive, type Incoming, type KnownOf } from "../submissions/index.ts";
import type { JevJudgment, Json, Submission } from "../vocab/index.ts";
import { checkoutVersion } from "../workspace/index.ts";
import { callPaths, freshDir, workspaceRepo } from "./paths.ts";
import type { Study } from "./study.ts";
// 共用动作一:派一个模型任务(任务书 12 第 2.6 节)。先写"模型调用开始"(材料整份存进去),再开会话,
// 最后写"模型调用结束",前后顺序由这个入口保证(方法.md 第九部分"先登记、再执行、再登记结果")。
// 提交收不收只由 submissions 判;形状不合格的由 agents 报上来,这里直接记退回("给 flow")。
/** 一个研究者任务:哪种提交、谁、哪个阶段、对象;known 是交上来之后才知道的东西(submissions 的 KnownOf)。 */
export type ModelTask<K extends KindOf<"researcher">> = {
kind: K;
actor: string;
stageId: string;
object: string | null;
known(content: Submission<K>): KnownOf<K>;
};
/** 派一次任务,等会话结束,交回调用编号和结局。 */
export async function runModelTask<K extends KindOf<"researcher">>(s: Study, t: ModelTask<K>): Promise<{ call: string; ending: SessionEnding }> {
const { ledger } = s;
const started = new Set(ledger.records().flatMap((r) => (r.kind === "callStarted" ? [r.content.call] : [])));
let attempt = 1;
while (started.has(callId(t.stageId, t.actor, t.kind, t.object, attempt))) attempt++;
const call = callId(t.stageId, t.actor, t.kind, t.object, attempt);
const paths = callPaths(s.dir, call);
const material = buildMaterial({
kind: t.kind,
actor: t.actor,
stageId: t.stageId,
object: t.object,
ledger,
study: s.facts,
prompts: s.prompts,
codeRoot: paths.code,
changes: null,
artifacts: null,
});
const model = researcherModel(s, t.actor);
// 先登记、后建目录:调用编号只在登记之后才算用掉,被杀在两者之间时下次的编号不会撞上旧目录。
ledger.append("callStarted", {
call,
actor: t.actor,
stageId: t.stageId,
model: `${model.provider}/${model.model}(思考档位 ${model.thinking})`,
materials: material as unknown as Json,
});
freshDir(paths.base);
freshDir(paths.code);
for (const r of material.readable) checkoutVersion(workspaceRepo(s.dir), r.version, freshDir(r.dir));
let reply = 0;
const outcome = await runSession({
role: "researcher",
model,
systemPrompt: s.prompts.roles.researcher.text,
firstMessage: material.firstMessage,
workDir: freshDir(paths.work),
sessionDir: freshDir(paths.sessions),
kind: t.kind,
readableDirs: material.readable.map((r) => r.dir),
query: (ref) => queryMaterial(material, ref),
submit: (attempt) => submit(s, t, material, attempt as SubmitAttempt<K>),
usage: (usage) => void ledger.append("callUsage", { call, reply: ++reply, usage }),
writeNote: (body) => void ledger.append("note", { author: t.actor, body }),
stop: s.stop,
});
// 会话文件就在这次调用的 sessions 目录里,不另记。
const { wallSeconds, sessionFile: _sessionFile, ...ending } = outcome;
ledger.append("callEnded", { call, wallSeconds: { status: "measured", value: wallSeconds }, ...ending });
return { call, ending };
}
/** 研究者用的模型:第 i 位研究者(researcherId(i))用配置里第 i 个。 */
function researcherModel(s: Study, actor: string) {
const models = s.frozen.config.models.researchers;
const i = models.findIndex((_, k) => researcherId(k) === actor);
if (i < 0) throw new Error(`${actor} 不是这份研究的研究者`);
return models[i]!;
}
/**
* 一份提交收不收。这个任务只有一份提交(第零轮每人每阶段一个任务、一份提交):
* 回给模型的提交编号就是这一份的位置,不分版本;替换时写它,接在最新一版后面。
* 第一次交以后不写 replaces 又交,或 replaces 写的不是这一份,都退回并说明该怎么写。
*/
async function submit<K extends KindOf<"researcher">>(s: Study, t: ModelTask<K>, material: TaskMaterial, attempt: SubmitAttempt<K>): Promise<SubmitReceipt> {
const { ledger } = s;
const slot = { stageId: t.stageId, author: t.actor, kind: t.kind, target: t.object, item: null };
const reject = (submitted: Json, reason: string, jev: JevJudgment[] | null): SubmitReceipt => {
ledger.append("rejection", { stageId: t.stageId, author: t.actor, kind: t.kind, target: t.object, submitted, reason, jev });
return { accepted: false, reason };
};
if (attempt.shape === "invalid") return reject(attempt.raw, attempt.reason, null);
const id = receiptId(t);
const latest = ledger.latestSubmission(slot)?.content.version ?? 0;
const content = attempt.content as unknown as Json;
if (attempt.replaces === null && latest > 0) {
return reject(content, `本阶段你已经交过这一份(提交编号 ${id})。要改就在 replaces 里写 ${id} 整份替换,不要另交`, null);
}
if (attempt.replaces !== null && (attempt.replaces !== id || latest === 0)) {
const which = latest === 0 ? "你本阶段还没有收下的提交,第一次交时不要带 replaces" : `要替换就写 ${id}`;
return reject(content, `replaces 写的 ${attempt.replaces} 不是你本阶段交过的提交:${which}`, null);
}
const verdict = await receive({
kind: t.kind,
content: attempt.content,
checklist: material.checklist,
known: t.known(attempt.content),
jevModel: s.frozen.config.models.jev,
} as Incoming);
if (!verdict.accepted) return reject(content, verdict.reason, verdict.jev);
ledger.append("submission", { ...slot, version: latest + 1, content: attempt.content, jev: verdict.jev } as never);
return { accepted: true, id };
}
/** 回给模型的提交编号:阶段、本人、提交种类、对象。 */
const receiptId = (t: { stageId: string; actor: string; kind: string; object: string | null }): string =>
`${t.stageId}.${t.actor}.${t.kind}.${t.object ?? "none"}`;
+160
View File
@@ -0,0 +1,160 @@
import type { StudyStepId } from "../config/index.ts";
import type { Content, LedgerRecord } from "../ledger/index.ts";
import { ORIGINAL_BASELINE, researcherId } from "../materials/index.ts";
import type { Stage } from "../vocab/index.ts";
import { readFrozen, type Frozen } from "./frozen.ts";
// "下一步做什么"(docs/结构.md 第二部分"几条边界"):步骤的先后只写在这里。
// 纯函数:输入账本记录(冻结的步骤清单就在其中的研究配置记录里),输出下一步要做的一件事;
// 不读文件、不调模型,同样的记录永远得到同样的下一步。账本不存"进行到第几步",每次从记录现算。
//
// 每一步做完以哪条记录为准,见 docs/结构.md 第八部分 flow 一节的表;下面每个步骤的分支各注一行。
// 没写成那条记录的步骤,下次会被重做或报错,不靠"这条记录公开了没有"这类隐含约定去猜。
/** 第零轮的研究者阶段:步骤 → 阶段名(vocab STAGES)和这个阶段交的提交种类。 */
const ROUND_ZERO_STAGES = {
entryDraft: { stage: "entryDraft", kind: "entry" },
discussion1: { stage: "discussion", kind: "discussion" },
discussion2: { stage: "discussion", kind: "discussion" },
claim: { stage: "claim", kind: "claim" },
} as const satisfies Partial<Record<StudyStepId, { stage: Stage; kind: string }>>;
type RoundZeroStep = keyof typeof ROUND_ZERO_STAGES;
export type RoundZeroKind = (typeof ROUND_ZERO_STAGES)[RoundZeroStep]["kind"];
/** 阶段编号由 flow 起:第零轮是 r0-<步骤>,一个步骤一个阶段。 */
export const stageIdOf = (step: RoundZeroStep): string => `r0-${step}`;
/** 基线运行编号:没启动就起下一个编号重跑(docs/结构.md 第四部分)。 */
export const baselineRunId = (n: number): string => `base-${n}`;
/** flow 写的事件种类(账本记录 event)。 */
export const EVENTS = {
/** 按配置跳过的步骤,objects 是步骤编号,reason 是 config 给的原因。 */
skipped: "skippedByConfig",
/** 研究停下,reason 是停下的原因,objects 是相关的调用或运行。 */
halted: "halted",
/** 恢复时问不到 Docker,不知道一次运行有没有启动(方法.md 第七部分"状态未知"),objects 是运行编号。 */
runStatusUnknown: "runStatusUnknown",
} as const;
export type Next =
| { do: "freezeConfig" }
| { do: "saveBaseline" }
| { do: "baselineRun"; run: string }
| { do: "skip"; step: StudyStepId; reason: string }
| { do: "openStage"; stageId: string; stage: Stage; roster: string[] }
| { do: "closeStage"; stageId: string }
| { do: "dispatch"; stageId: string; kind: RoundZeroKind; actors: string[] }
| { do: "assignEntries" }
| { do: "halt"; reason: string; objects: string[] };
export function nextStep(records: readonly LedgerRecord[]): Next {
// 冻结配置 → 研究配置记录。
const frozen = readFrozen(records);
if (frozen === null) return { do: "freezeConfig" };
const view = new View(records);
for (const step of frozen.steps.study) {
if (step.status === "skip") {
// 按配置跳过的步骤 → 一条"按配置跳过"事件,写明步骤和原因。
if (!view.events(EVENTS.skipped).some((e) => e.objects[0] === step.id)) return { do: "skip", step: step.id, reason: step.reason };
continue;
}
const next = stepNext(step.id, view, frozen);
if (next !== null) return next;
}
throw new Error("步骤清单走完了却没有停下:rounds 一步应当停下");
}
/** 一个要执行的步骤还要做什么;做完了交回 null。 */
function stepNext(step: StudyStepId, view: View, frozen: Frozen): Next | null {
switch (step) {
case "freezeConfig":
return null;
case "baselineRun":
return baselineNext(view);
case "entryDraft":
case "discussion1":
case "discussion2":
case "claim":
return stageNext(step, view, frozen);
case "assignEntries":
// 分配 → 第 0 轮的入口分配决定(decision,种类 entries)。
return view.of("decision").some((d) => d.decision === "entries" && d.round === 0) ? null : { do: "assignEntries" };
case "rounds":
return { do: "halt", reason: "第 1 轮还没写(任务书 13)", objects: [] };
case "export":
throw new Error("走到了导出:rounds 一步应当先停下");
}
}
/**
* 基线 → 基线代码记录(对象 ORIGINAL_BASELINE),而且最近一次基线运行的结果不是"没启动"。
* 没启动的起新编号重跑;判为不算数的停下(方法.md 4.1)。
*/
function baselineNext(view: View): Next | null {
if (!view.of("code").some((c) => c.object === ORIGINAL_BASELINE)) return { do: "saveBaseline" };
const plans = view.of("runPlanned").filter((p) => p.purpose === "baseline");
const last = plans.at(-1);
if (last === undefined) return { do: "baselineRun", run: baselineRunId(1) };
const result = view.of("runResult").find((r) => r.run === last.run);
if (result === undefined) throw new Error(`基线运行 ${last.run} 登记了还没有结果:恢复入口应当已经核对过`);
if (result.process.ending === "notStarted") return { do: "baselineRun", run: baselineRunId(plans.length + 1) };
if (result.judged?.verdict === "valid") return null;
const why = result.judged === null ? result.process.ending : result.judged.verdictReason;
return { do: "halt", reason: `基线运行 ${last.run} 判为不算数(${why}),不能当基线`, objects: [last.run] };
}
/**
* 研究者阶段 → 这个阶段的关闭记录(stageClose)。没开就开;名单上的人都有被收下的提交才关;
* 没交上的人按最近一次调用怎么结束的来定派不派(任务书 12 第 2.3 节的表)。能派的先全派,派完还有人停在"没交"才停下。
*/
function stageNext(step: RoundZeroStep, view: View, frozen: Frozen): Next | null {
const stageId = stageIdOf(step);
const { stage, kind } = ROUND_ZERO_STAGES[step];
const open = view.of("stageOpen").find((s) => s.stageId === stageId);
if (open === undefined) {
return { do: "openStage", stageId, stage, roster: frozen.config.models.researchers.map((_, i) => researcherId(i)) };
}
if (view.of("stageClose").some((s) => s.stageId === stageId)) return null;
const missing = open.roster.filter((a) => !view.of("submission").some((s) => s.stageId === stageId && s.author === a));
if (missing.length === 0) return { do: "closeStage", stageId };
const actors: string[] = [];
const stuck: Array<{ call: string; reason: string }> = [];
for (const actor of missing) {
const last = view.lastCall(stageId, actor);
// 没有调用、被用户停下、出错(含恢复时补写的"进程中断")都派;没交的只在开发入口开着时派(方法.md 第九部分"开发入口")。
if (last?.status === "notSubmitted" && !frozen.config.devEntry) stuck.push({ call: last.call, reason: last.reason });
else actors.push(actor);
}
if (actors.length > 0) return { do: "dispatch", stageId, kind, actors };
return {
do: "halt",
reason: `交不出合格提交(方法.md 4.3 ②:按内部错误停下):${stuck.map((s) => `调用 ${s.call},${s.reason}`).join(";")}`,
objects: stuck.map((s) => s.call),
};
}
/** 按种类取记录内容的小工具。 */
class View {
readonly #records: readonly LedgerRecord[];
constructor(records: readonly LedgerRecord[]) {
this.#records = records;
}
of<K extends LedgerRecord["kind"]>(kind: K): Array<Content<K>> {
return (this.#records.filter((r) => r.kind === kind) as Array<LedgerRecord<K>>).map((r) => r.content);
}
events(event: string): Array<Content<"event">> {
return this.of("event").filter((e) => e.event === event);
}
/** 某人在某阶段最近一次模型调用的结束记录,没有调用为 null。开始了没结束的,恢复入口应当已经补写。 */
lastCall(stageId: string, actor: string): Content<"callEnded"> | null {
const started = this.of("callStarted").filter((c) => c.stageId === stageId && c.actor === actor).at(-1);
if (started === undefined) return null;
const ended = this.of("callEnded").find((e) => e.call === started.call);
if (ended === undefined) throw new Error(`模型调用 ${started.call} 开始了还没有结束记录:恢复入口应当已经补写`);
return ended;
}
}
+35
View File
@@ -0,0 +1,35 @@
import { mkdirSync } from "node:fs";
import { dirname, join } from "node:path";
// 研究目录里放什么(任务书 12 第 2.1 节),全由 flow 管:
// ledger.sqlite 账本
// repo.git workspace 的裸仓库
// runs/<运行编号>/ 一次正式运行:code 取出的代码、run 运行目录、logs 日志、judge 判分目录、judge-logs 校验器日志
// calls/<调用编号>/ 一次模型调用:work 工作目录、sessions 会话文件、code 能读的代码版本(materials 的 codeRoot)
export const ledgerFile = (dir: string): string => join(dir, "ledger.sqlite");
export const workspaceRepo = (dir: string): string => join(dir, "repo.git");
export function runPaths(dir: string, run: string) {
const base = join(dir, "runs", run);
return { base, code: join(base, "code"), run: join(base, "run"), logs: join(base, "logs"), judge: join(base, "judge"), judgeLogs: join(base, "judge-logs") };
}
export function callPaths(dir: string, call: string) {
const base = join(dir, "calls", call);
return { base, work: join(base, "work"), sessions: join(base, "sessions"), code: join(base, "code") };
}
/** 容器名带研究编号,在本机所有研究之间唯一(参考项目 9b 两份研究撞过名);判分容器另起名字。 */
export const containerName = (studyId: string, run: string): string => `inforsi-${studyId}-${run}`;
export const judgeContainerName = (studyId: string, run: string): string => `inforsi-${studyId}-${run}-judge`;
/**
* 新建一个空目录:上一层没有就建,目录本身已经存在就报错。取出代码、建运行目录、判分目录时目标都必须是空的
* (workspace、tasks 不查);每次运行、每次调用都有新编号,所以不会撞上旧目录。
*/
export function freshDir(path: string): string {
mkdirSync(dirname(path), { recursive: true });
mkdirSync(path);
return path;
}
+48
View File
@@ -0,0 +1,48 @@
import type { Content } from "../ledger/index.ts";
import type { HypothesisMetric } from "../vocab/index.ts";
import { finishRun } from "./formal-run.ts";
import { EVENTS } from "./next-step.ts";
import { containerName } from "./paths.ts";
import type { Study } from "./study.ts";
// 恢复入口第 3、4 步(任务书 12 第 2.5 节):每次启动都走这里,不分第一次启动、正常重启还是被杀后重启。
// 从账本找出已登记要做、还没有结果的动作,逐个核对,只核对不重做(CLAUDE.md"中断后怎么恢复")。
/** 核对完交回 null;问不到 Docker 时写"状态未知"事件,交回停下的原因;等运行时被用户停下交回 "stopped"。 */
export async function recover(s: Study): Promise<{ halt: string } | "stopped" | null> {
const pending = s.ledger.pending();
// 开始了还没结束的模型调用:控制进程中断了,补写结束记录,墙钟时间测不到(docs/结构.md 第四部分第 10 种)。
for (const c of pending.calls) {
const unmeasured = { status: "unmeasured", reason: "进程中断" } as const;
s.ledger.append("callEnded", { call: c.content.call, wallSeconds: unmeasured, status: "failed", reason: "进程中断" });
}
// 登记了还没结果的正式运行:按容器名问 Docker(方法.md 第七部分"不确定有没有启动时,先核对")。
for (const { plan, started } of pending.runs) {
const { run } = plan.content;
const name = containerName(s.frozen.config.studyId, run);
const status = await s.runner.status(name);
if (status === "unreachable") {
const reason = `恢复时问不到 Docker,不知道运行 ${run}(容器 ${name})有没有启动:状态未知,等用户核对(方法.md 第七部分)`;
s.ledger.append("event", { event: EVENTS.runStatusUnknown, reason, objects: [run] });
return { halt: reason };
}
if (!started && (status === "absent" || status === "created")) {
// 确认没启动:写结果"没启动",退还名额;建了没启动的删掉。
if (status === "created") await s.runner.remove(name);
const reason = `恢复时核对:容器 ${name} ${status === "absent" ? "不存在" : "建了没有启动,已删掉"},这次运行没有启动`;
s.ledger.append("runResult", { run, process: { ending: "notStarted", reason }, judged: null });
continue;
}
// 在跑或已结束:没写"确认启动"的补写(扣次),然后等它、判分、写结果。
// 已经确认启动、容器却不见了,真实运行里没出现过,不专门写:runner.wait 会报错,按内部错误停下。
if (!started) s.ledger.append("runStarted", { run });
if ((await finishRun(s, run, metricsOf(plan.content))) === "stopped") return "stopped";
}
return null;
}
/** 一次运行要测的假设专用指标。第 1 部分只有基线运行,它不测假设专用指标;正向、消融运行由第 3、4 部分补。 */
function metricsOf(plan: Content<"runPlanned">): HypothesisMetric[] {
if (plan.purpose !== "baseline") throw new Error(`运行 ${plan.run} 是 ${plan.purpose} 运行,flow 第 1 部分还不会恢复它`);
return [];
}
+21
View File
@@ -0,0 +1,21 @@
import { ORIGINAL_BASELINE } from "../../materials/index.ts";
import { saveVersion } from "../../workspace/index.ts";
import { formalRun, type RunOutcome } from "../formal-run.ts";
import { workspaceRepo } from "../paths.ts";
import type { Study } from "../study.ts";
// 基线(方法.md 4.1):存任务的基线代码,写代码记录,再用共用动作二跑一次;每份研究都跑,计入运行次数。
/** 存任务的基线目录("不存"清单取冻结的任务事实),写代码记录:对象 ORIGINAL_BASELINE,位置表 null。 */
export function saveBaseline(s: Study): void {
const saved = saveVersion(workspaceRepo(s.dir), s.task.baselineDir, s.frozen.facts.unversionedNames);
if (!saved.saved) throw new Error(`任务的基线目录 ${s.task.baselineDir} 存不成版本:${saved.path},${saved.reason}`);
s.ledger.append("code", { object: ORIGINAL_BASELINE, version: saved.version, locations: null });
}
/** 跑一次基线运行。基线不测假设专用指标。 */
export function baselineRun(s: Study, run: string): Promise<RunOutcome> {
const code = s.ledger.records().findLast((r) => r.kind === "code" && r.content.object === ORIGINAL_BASELINE);
const version = (code!.content as { version: string }).version;
return formalRun(s, { run, purpose: "baseline", round: 0, object: null }, version, []);
}
+44
View File
@@ -0,0 +1,44 @@
import { finalEntries } from "../../materials/index.ts";
import { assignEntries as assign } from "../../rules/index.ts";
import type { Json } from "../../vocab/index.ts";
import { runModelTask } from "../model-task.ts";
import type { Next } from "../next-step.ts";
import { halt, USER_STOPPED, type Study, type StudyEnding } from "../study.ts";
// 第零轮(方法.md 4.2):三个研究者阶段派任务,认领之后由程序分配入口。
/**
* 派一个研究者阶段里要派的人,同时跑,各自一个会话(任务书 12 第 2.6 节)。第零轮的提交种类交上来之后没有要另算的东西。
* 有人没交上(没交或出错),等这次派发的会话全部结束后停下,不打断别人(第 2.4 节)。
* 某个会话抛出内部错误时,也等别人结束再把它抛出去。
*/
export async function dispatchStage(s: Study, next: Extract<Next, { do: "dispatch" }>): Promise<StudyEnding | null> {
const settled = await Promise.allSettled(
next.actors.map((actor) => runModelTask(s, { kind: next.kind, actor, stageId: next.stageId, object: null, known: () => null })),
);
const failure = settled.find((r) => r.status === "rejected");
if (failure !== undefined) throw failure.reason;
if (s.stop.aborted) return USER_STOPPED;
const results = settled.map((r) => (r as PromiseFulfilledResult<Awaited<ReturnType<typeof runModelTask>>>).value);
const missed = results.flatMap(({ call, ending }) =>
ending.status === "notSubmitted" || ending.status === "failed" ? [{ call, text: `${ending.status === "failed" ? "出错" : "没交"}:${ending.reason}` }] : [],
);
if (missed.length === 0) return null;
return halt(s.ledger, `阶段 ${next.stageId} 有人没交上,等这次派发的会话全部结束后停下:${missed.map((m) => `调用 ${m.call},${m.text}`).join(";")}`, missed.map((m) => m.call));
}
/**
* 分配(方法.md 4.2"分配规则"):每人最后一份认领和最终有效的入口交给 rules 的 assignEntries,写一条入口分配决定。
* assignEntries 不在规则登记表里,规则名就写它,版本 1;没有方法参数,也不用随机。
*/
export function assignEntries(s: Study): void {
const roster = s.ledger.records().find((r) => r.kind === "stageOpen" && r.content.stage === "claim" && r.content.round === 0);
const claim = roster!.content as { stageId: string; roster: string[] };
const researchers = claim.roster.map((id) => {
const last = s.ledger.latestSubmission({ stageId: claim.stageId, author: id, kind: "claim", target: null, item: null });
return { id, choices: (last!.content.content as { choices: string[] }).choices };
});
const entries = finalEntries(s.ledger).map((e) => ({ id: e.id, author: e.author }));
const input = { researchers, entries };
s.ledger.append("decision", { decision: "entries", round: 0, rule: "assignEntries", ruleVersion: 1, params: null, seed: null, input, output: assign(input) as unknown as Json });
}
+112
View File
@@ -0,0 +1,112 @@
import { stepList } from "../config/index.ts";
import type { Ledger } from "../ledger/index.ts";
import { readPrompts, type Prompts, type StudyFacts, type StudyRecord } from "../materials/index.ts";
import type { Runner } from "../runner/index.ts";
import { loadTask, metricDefinitions, type Task } from "../tasks/index.ts";
import { hash, ROLES, SUBMISSION_KINDS, type Json, type SubmissionKind } from "../vocab/index.ts";
import { createRepository } from "../workspace/index.ts";
import { readFrozen, type FullConfig, type Frozen } from "./frozen.ts";
import { EVENTS } from "./next-step.ts";
import { workspaceRepo } from "./paths.ts";
// 一份研究跑起来要的东西:研究目录、账本、Docker、冻结的配置,以及按冻结配置读进来的任务和提示词。
// 研究开始时冻结配置(方法.md 4.1);之后每次启动都从账本里那份冻结的配置重建,不再读配置文件和 task.json。
export type Study = {
dir: string;
ledger: Ledger;
runner: Runner;
stop: AbortSignal;
frozen: Frozen;
task: Task;
prompts: Prompts;
facts: StudyFacts;
};
/**
* 研究开始:核对研究编号没被用过,读任务和提示词、建镜像、建裸仓库,一次写进研究配置记录(任务书 12 第 2.8 节)。
* 研究编号来自用户配置,是边界:容器名是 inforsi-<研究编号>-<运行编号>,编号重复的新研究会把旧研究遗留的容器
* 当成自己的运行(docker run 重名失败后按名字查到它,算作启动了),扣次并记下别人的结果。
* 每份研究冻结配置时都建 inforsi-<研究编号> 的镜像,本机已有这个标签就说明编号用过了,报错停下。
*/
export async function freezeConfig(dir: string, ledger: Ledger, runner: Runner, config: FullConfig): Promise<void> {
const tag = `inforsi-${config.studyId}`;
if (await runner.hasImage(tag)) {
throw new Error(`研究编号 ${config.studyId} 已被本机用过(已有镜像 ${tag}),换一个研究编号再开新研究`);
}
const task = loadTask(config.task);
const prompts = readPrompts(config.promptDir);
// 冻结的是镜像编号,标签以后被别的构建挪走也不影响(docs/结构.md"依赖跟着任务走")。
const image = await runner.buildImage(task.dir, tag);
createRepository(workspaceRepo(dir));
const record: StudyRecord = {
config: config as unknown as Json,
steps: stepList(config) as unknown as Json,
prompts: promptHashes(prompts),
image,
task: task.facts as unknown as Json,
// 同一份研究的全部运行用同一个种子:研究编号的哈希取前 8 位十六进制(任务书 12 第 2.7 节)。
seed: Number.parseInt(hash(config.studyId).slice(0, 8), 16),
};
ledger.append("config", record as unknown as Json);
}
/**
* 按账本里冻结的配置把研究接上(恢复入口第 2 步):镜像还在、提示词没变,才往下走。
* 镜像不在就停下,不重建(docs/结构.md"依赖跟着任务走");提示词变了而开发入口关着也停下(方法.md 第九部分)。
* 交回研究,或停下的原因。
*/
export async function openStudy(dir: string, ledger: Ledger, runner: Runner, stop: AbortSignal): Promise<Study | { halt: string }> {
const frozen = readFrozen(ledger.records())!;
if (!(await runner.hasImage(frozen.image))) {
return { halt: `本机没有冻结的任务镜像 ${frozen.image}:一份研究不能自己改依赖,不重建(docs/结构.md 第二部分"依赖跟着任务走")` };
}
const prompts = readPrompts(frozen.config.promptDir);
const now = promptHashes(prompts);
const changed = [
...ROLES.filter((r) => now.roles[r] !== frozen.prompts.roles[r]).map((r) => `roles/${r}.md`),
...KINDS.filter((k) => now.tasks[k] !== frozen.prompts.tasks[k]).map((k) => `tasks/${k}.md`),
];
if (changed.length > 0 && !frozen.config.devEntry) {
return { halt: `提示词和冻结时不一样:${changed.join("、")};开发入口关着,不能换提示词(方法.md 第九部分)` };
}
// 任务事实只认冻结的那份;任务目录里的基线、harness、校验器和 TASK.md 照目录读。
const task: Task = { ...loadTask(frozen.config.task), facts: frozen.facts };
return { dir, ledger, runner, stop, frozen, task, prompts, facts: studyFacts(frozen, task) };
}
const KINDS = Object.keys(SUBMISSION_KINDS) as SubmissionKind[];
function promptHashes(prompts: Prompts): StudyRecord["prompts"] {
const pick = <K extends string>(files: Record<K, { hash: string }>) =>
Object.fromEntries(Object.entries<{ hash: string }>(files).map(([k, f]) => [k, f.hash])) as Record<K, string>;
return { roles: pick(prompts.roles), tasks: pick(prompts.tasks) };
}
/** materials 要的研究事实,取自冻结的配置和任务。 */
function studyFacts(frozen: Frozen, task: Task): StudyFacts {
const { config, facts } = frozen;
return {
finalMetric: facts.finalMetric.id,
direction: facts.finalMetric.direction,
epsilon: facts.epsilon,
knownBest: facts.knownBest,
lambda: config.flow.openReserve,
totalRuns: config.runs,
proposalLimit: config.proposalLimit,
factorLimit: config.factorLimit,
metrics: metricDefinitions(task),
taskText: task.guide,
wallSeconds: facts.run.wallSeconds,
};
}
/** runStudy 的两种结局:停下(附原因),或被用户停下。"研究结束"第 4 部分才走得到。 */
export type StudyEnding = { status: "halted"; reason: string } | { status: "userStopped" };
export const USER_STOPPED: StudyEnding = { status: "userStopped" };
/** 停下:写一条停下事件,交回停下的结局。 */
export function halt(ledger: Ledger, reason: string, objects: string[]): StudyEnding {
ledger.append("event", { event: EVENTS.halted, reason, objects });
return { status: "halted", reason };
}