diff --git a/src/flow/loop.ts b/src/flow/loop.ts index eb88661..39fe3f1 100644 --- a/src/flow/loop.ts +++ b/src/flow/loop.ts @@ -1,6 +1,7 @@ import { mkdirSync } from "node:fs"; import { resolve } from "node:path"; import type { ResearchConfig } from "../config/index.ts"; +import { forwardNotRunEvent } from "../materials/index.ts"; import { Ledger } from "../ledger/index.ts"; import { Runner } from "../runner/index.ts"; import { fullConfig, readFrozen } from "./frozen.ts"; @@ -8,6 +9,7 @@ import { EVENTS, nextStep, type Next } from "./next-step.ts"; import { ledgerFile } from "./paths.ts"; import { recover } from "./recover.ts"; import { baselineRun, saveBaseline } from "./steps/baseline.ts"; +import { forwardRun, adoptBatch } from "./steps/forward.ts"; import { dispatchStage } from "./steps/dispatch.ts"; import { dedupe, freezeProtocols, selectBatch } from "./steps/round.ts"; import { assignEntries } from "./steps/round-zero.ts"; @@ -75,8 +77,9 @@ async function doStep(s: Study, next: Exclude): Pr case "saveBaseline": saveBaseline(s); return null; - case "baselineRun": { - const ran = await baselineRun(s, next.run); + case "baselineRun": + case "forwardRun": { + const ran = next.do === "baselineRun" ? await baselineRun(s, next.run) : await forwardRun(s, next.round, next.candidate, next.run); if (ran === "stopped") return USER_STOPPED; if (ran === "done") return null; return halt(s.ledger, `运行 ${next.run} 没有启动，名额已退还；本进程不重跑，下次启动时起新编号重跑。Docker 的报错：${ran.notStarted}`, [next.run]); @@ -104,6 +107,12 @@ async function doStep(s: Study, next: Exclude): Pr case "selectBatch": selectBatch(s, next.round); return null; + case "forwardNotRun": + s.ledger.append("event", forwardNotRunEvent(next.candidate, next.reason)); + return null; + case "adopt": + adoptBatch(s, next.round); + return null; case "halt": return halt(s.ledger, next.reason, next.objects); } diff --git a/src/flow/model-task.ts b/src/flow/model-task.ts index d2c66d2..14f98e3 100644 --- a/src/flow/model-task.ts +++ b/src/flow/model-task.ts @@ -1,8 +1,9 @@ -import { runSession, type KindOf, type SessionEnding, type SubmitAttempt, type SubmitReceipt } from "../agents/index.ts"; +import { runSession, type SessionTask, type SessionEnding, type SubmitAttempt, type SubmitReceipt } from "../agents/index.ts"; import { buildMaterial, callId, queryMaterial, researcherId, type TaskMaterial } from "../materials/index.ts"; -import { receive, type Incoming, type KnownOf } from "../submissions/index.ts"; -import type { JevJudgment, Json, Submission } from "../vocab/index.ts"; +import { receive, type Incoming } from "../submissions/index.ts"; +import { SUBMISSION_KINDS, type JevJudgment, type Json, type SubmissionKind } from "../vocab/index.ts"; import { checkoutVersion } from "../workspace/index.ts"; +import { prepareSubmission, taskCode, lightChecker } from "./task-code.ts"; import { callPaths, freshDir, workspaceRepo } from "./paths.ts"; import type { Study } from "./study.ts"; @@ -10,26 +11,26 @@ import type { Study } from "./study.ts"; // 最后写"模型调用结束"，前后顺序由这个入口保证（方法.md 第九部分"先登记、再执行、再登记结果"）。 // 提交收不收只由 submissions 判；形状不合格的由 agents 报上来，这里直接记退回（"给 flow"）。 -/** 一个研究者任务：哪种提交、谁、哪个阶段、对象。 */ -export type ModelTask> = { +/** 一个模型任务：哪种提交、谁、哪个阶段、对象。 */ +export type ModelTask = { kind: K; actor: string; stageId: string; object: string | null; /** 一个任务里能交几份的种类（提案、挑刺）：第 n 份的编号（materials 的 proposalId、critiqueId），收下时分配；只交一份的为 null。 */ item: ((n: number) => string) | null; - /** 交上来之后才知道的东西（submissions 的 KnownOf）。others 是本任务已收下、而且不是这次要替换的那几份的份数。 */ - known(content: Submission, others: number): KnownOf; }; /** 派一次任务，等会话结束，交回调用编号和结局。 */ -export async function runModelTask>(s: Study, t: ModelTask): Promise<{ call: string; ending: SessionEnding }> { +export async function runModelTask(s: Study, t: ModelTask): Promise<{ call: string; ending: SessionEnding }> { const { ledger } = s; const started = new Set(ledger.records().flatMap((r) => (r.kind === "callStarted" ? [r.content.call] : []))); let attempt = 1; while (started.has(callId(t.stageId, t.actor, t.kind, t.object, attempt))) attempt++; const call = callId(t.stageId, t.actor, t.kind, t.object, attempt); const paths = callPaths(s.dir, call); + const { changes, startVersion } = taskCode(s, t); + const role = SUBMISSION_KINDS[t.kind].role; const material = buildMaterial({ kind: t.kind, actor: t.actor, @@ -39,37 +40,44 @@ export async function runModelTask>(s: Study, t: study: s.facts, prompts: s.prompts, codeRoot: paths.code, - changes: null, + changes, artifacts: null, }); - const model = researcherModel(s, t.actor); + const model = role === "researcher" ? researcherModel(s, t.actor) : s.frozen.config.models[role]; // 先登记、后建目录：调用编号只在登记之后才算用掉，被杀在两者之间时下次的编号不会撞上旧目录。 ledger.append("callStarted", { call, actor: t.actor, stageId: t.stageId, model: `${model.provider}/${model.model}（思考档位 ${model.thinking}）`, + // SAFETY: TaskMaterial 的全部字段都是可序列化数据，账本在写入时校验 Json。 materials: material as unknown as Json, }); freshDir(paths.base); freshDir(paths.code); for (const r of material.readable) checkoutVersion(workspaceRepo(s.dir), r.version, freshDir(r.dir)); + const workDir = freshDir(paths.work); + if (startVersion !== null) checkoutVersion(workspaceRepo(s.dir), startVersion, workDir); let reply = 0; - const outcome = await runSession({ - role: "researcher", + const common = { model, - systemPrompt: s.prompts.roles.researcher.text, + systemPrompt: s.prompts.roles[role].text, firstMessage: material.firstMessage, - workDir: freshDir(paths.work), + workDir, sessionDir: freshDir(paths.sessions), kind: t.kind, - readableDirs: material.readable.map((r) => r.dir), - query: (ref) => queryMaterial(material, ref), - submit: (attempt) => submit(s, t, material, attempt as SubmitAttempt), - usage: (usage) => void ledger.append("callUsage", { call, reply: ++reply, usage }), - writeNote: (body) => void ledger.append("note", { author: t.actor, body }), + query: (ref: string) => queryMaterial(material, ref), + submit: (attempt: SubmitAttempt) => submit(s, t, material, attempt, workDir), + usage: (usage: Parameters[0]) => void ledger.append("callUsage", { call, reply: ++reply, usage }), stop: s.stop, - }); + }; + const readableDirs = material.readable.map((r) => r.dir); + const task = role === "experimenter" + ? { ...common, role, lightCheck: lightChecker(s, call, workDir) } + : role === "verifier" + ? { ...common, role, readableDirs } + : { ...common, role, readableDirs, writeNote: (body: string) => void ledger.append("note", { author: t.actor, body }) }; + const outcome = await runSession(task as SessionTask); // 会话文件就在这次调用的 sessions 目录里，不另记。 const { wallSeconds, sessionFile: _sessionFile, ...ending } = outcome; ledger.append("callEnded", { call, wallSeconds: { status: "measured", value: wallSeconds }, ...ending }); @@ -85,26 +93,32 @@ function researcherModel(s: Study, actor: string) { } /** 一份提交收不收：先定写到哪一份，形状合格的再交给 submissions 判。 */ -async function submit>(s: Study, t: ModelTask, material: TaskMaterial, attempt: SubmitAttempt): Promise { +async function submit(s: Study, t: ModelTask, material: TaskMaterial, attempt: SubmitAttempt, workDir: string): Promise { const { ledger } = s; const reject = (submitted: Json, reason: string, jev: JevJudgment[] | null): SubmitReceipt => { ledger.append("rejection", { stageId: t.stageId, author: t.actor, kind: t.kind, target: t.object, submitted, reason, jev }); return { accepted: false, reason }; }; if (attempt.shape === "invalid") return reject(attempt.raw, attempt.reason, null); + // SAFETY: agents 已按 vocab 的提交字段校验，提交内容只含 Json 值。 const content = attempt.content as unknown as Json; const slot = slotOf(s, t, attempt.replaces); if ("refuse" in slot) return reject(content, slot.refuse, null); + const prepared = prepareSubmission(s, t, attempt.content, slot.others, workDir); const verdict = await receive({ kind: t.kind, content: attempt.content, checklist: material.checklist, - known: t.known(attempt.content, slot.others), + known: prepared.known, jevModel: s.frozen.config.models.jev, } as Incoming); if (!verdict.accepted) return reject(content, verdict.reason, verdict.jev); const { stageId, actor: author, kind, object: target } = t; - ledger.append("submission", { stageId, author, kind, target, item: slot.item, version: slot.version, content: attempt.content, jev: verdict.jev } as never); + // 代码在提交时冻结；提交和代码记录同进同退，恢复不能看到“交了却没快照”的半截状态。 + ledger.transaction(() => { + ledger.append("submission", { stageId, author, kind, target, item: slot.item, version: slot.version, content: attempt.content, jev: verdict.jev } as never); + if (prepared.code !== null) ledger.append("code", prepared.code); + }); return { accepted: true, id: slot.id }; } @@ -113,7 +127,7 @@ async function submit>(s: Study, t: ModelTask, * 提交编号：能交几份的种类就是这一份的编号，只交一份的是"阶段.本人.种类.对象"。 * 不写 replaces 是交新的一份，只交一份的种类已经交过就退回并说明该怎么写；写了 replaces 就必须是本任务已收下的某一份。 */ -function slotOf>( +function slotOf( s: Study, t: ModelTask, replaces: string | null, diff --git a/src/flow/next-step.ts b/src/flow/next-step.ts index a925a2a..2e8bfd2 100644 --- a/src/flow/next-step.ts +++ b/src/flow/next-step.ts @@ -1,9 +1,9 @@ -import type { KindOf } from "../agents/index.ts"; import type { RoundStepId, StudyStepId } from "../config/index.ts"; import type { Content, LedgerRecord } from "../ledger/index.ts"; -import { Board, ORIGINAL_BASELINE, researcherId } from "../materials/index.ts"; +import { Board, experimenterId, verifierId, FORWARD_NOT_RUN, ORIGINAL_BASELINE, researcherId } from "../materials/index.ts"; import { compareIds } from "../rules/index.ts"; -import type { Stage } from "../vocab/index.ts"; +import { verificationOutcome } from "../submissions/index.ts"; +import { SUBMISSION_KINDS, type Stage, type SubmissionKind } from "../vocab/index.ts"; import { readFrozen, type Frozen } from "./frozen.ts"; // "下一步做什么"（docs/结构.md 第二部分"几条边界"）：步骤的先后只写在这里。 @@ -40,8 +40,8 @@ export const EVENTS = { runStatusUnknown: "runStatusUnknown", } as const; -/** 一个研究者任务：一个人对一个对象（候选、原提案；提案和第零轮的任务没有对象）交一种提交。 */ -export type TaskRef = { actor: string; kind: KindOf<"researcher">; object: string | null }; +/** 一个任务：一个人对一个对象交一种提交；提案和第零轮任务没有对象。 */ +export type TaskRef = { actor: string; kind: SubmissionKind; object: string | null }; export type Next = | { do: "freezeConfig" } @@ -55,6 +55,9 @@ export type Next = | { do: "dedupe"; round: number } | { do: "freezeProtocols"; round: number } | { do: "selectBatch"; round: number } + | { do: "forwardNotRun"; candidate: string; reason: string } + | { do: "forwardRun"; round: number; candidate: string; run: string } + | { do: "adopt"; round: number } | { do: "halt"; reason: string; objects: string[] }; export function nextStep(records: readonly LedgerRecord[]): Next { @@ -87,7 +90,7 @@ function studyStepNext(step: Exclude, view: View, frozen: case "discussion2": case "claim": { const { stage, kind } = ROUND_ZERO_STAGES[step]; - return stageNext(stageIdOf(0, step), stage, 0, (roster) => roster.map((actor) => ({ actor, kind, object: null })), view, frozen); + return stageNext(stageIdOf(0, step), stage, 0, (roster) => roster.map((actor) => ({ actor, kind, object: null })), view, frozen, researchers(frozen)); } case "assignEntries": // 分配 → 第 0 轮的入口分配决定（decision，种类 entries）。 @@ -99,12 +102,12 @@ function studyStepNext(step: Exclude, view: View, frozen: /** * 一轮接一轮（结构.md 第三部分"每一轮"）：第一个没做完的轮次里第一个没做完的步骤。 - * 每一步做完的记录都带轮次，做完的轮次整轮跳过；现在走到实现一步就停下，所以循环一定会返回。 + * 每一步做完的记录都带轮次，做完的轮次整轮跳过；现在走到列消融格就停下，所以循环一定会返回。 */ function roundsNext(view: View, frozen: Frozen): Next { for (let round = 1; ; round++) { for (const step of frozen.steps.round) { - const next = step.status === "skip" ? skipNext(stageIdOf(round, step.id), step.reason, view) : roundStepNext(step.id, round, view, frozen); + const next = step.status === "skip" && step.id !== "listCells" ? skipNext(stageIdOf(round, step.id), step.reason, view) : roundStepNext(step.id, round, view, frozen); if (next !== null) return next; } } @@ -122,7 +125,7 @@ function roundStepNext(step: RoundStepId, round: number, view: View, frozen: Fro case "critique": case "response": case "secondPrediction": - return stageNext(stageIdOf(round, step), step, round, (roster) => roundTasks(step, round, roster, view.board), view, frozen); + return stageNext(stageIdOf(round, step), step, round, (roster) => roundTasks(step, round, roster, view.board), view, frozen, researchers(frozen)); case "dedupe": // 判重合并 → 本轮的判重合并决定（decision，种类 dedup）。 return view.board.decision("dedup", round) === undefined ? { do: "dedupe", round } : null; @@ -138,10 +141,25 @@ function roundStepNext(step: RoundStepId, round: number, view: View, frozen: Fro case "selectBatch": // 选批 → 本轮的选批决定（decision，种类 batch）。 return view.board.decision("batch", round) === undefined ? { do: "selectBatch", round } : null; - case "implementation": - return { do: "halt", reason: "实现还没写（任务书 16）", objects: [] }; + case "implementation": { + const candidates = view.board.decision("batch", round)!.output.chosen; + return assignedStage("implementation", round, "implementation", candidates, view, frozen); + } + case "verification": + case "repair": + return verificationNext(round, view, frozen); + case "forwardRun": + return forwardNext(round, view); + case "publish": + case "informationAccount": + // 不另写记录：整批运行已结束。采纳决定使结果整批公开，信息账从预测和结果现算。 + return null; + case "adopt": + return view.board.decision("adopt", round) === undefined ? { do: "adopt", round } : null; + case "listCells": + return { do: "halt", reason: "消融还没写（任务书 17）", objects: [] }; default: - throw new Error(`走到了第 ${round} 轮的 ${step}：implementation 一步应当先停下`); + throw new Error(`走到了第 ${round} 轮的 ${step}：listCells 一步应当先停下`); } } @@ -215,10 +233,11 @@ function stageNext( tasksOf: (roster: readonly string[]) => TaskRef[], view: View, frozen: Frozen, + roster: string[], ): Next | null { const open = view.board.stages.get(stageId); if (open === undefined) { - return { do: "openStage", stageId, stage, round, roster: frozen.config.models.researchers.map((_, i) => researcherId(i)) }; + return { do: "openStage", stageId, stage, round, roster }; } if (open.closed) return null; const todo = tasksOf(open.roster).filter((t) => !view.done(stageId, t)); @@ -239,6 +258,63 @@ function stageNext( }; } +/** 研究者阶段的冻结名单。实验员、核验员阶段则按本阶段的任务列名单。 */ +const researchers = (frozen: Frozen): string[] => frozen.config.models.researchers.map((_, i) => researcherId(i)); + +/** 正向的实现、修复、核验共用阶段开关和派发规则，每个候选固定一个人。 */ +function assignedStage( + step: string, + round: number, + kind: "implementation" | "repair" | "verification", + candidates: readonly string[], + view: View, + frozen: Frozen, +): Next | null { + const actorOf = kind === "verification" ? verifierId : experimenterId; + const tasks = candidates.map((object) => ({ actor: actorOf(object), kind, object })); + return stageNext(stageIdOf(round, step), SUBMISSION_KINDS[kind].stage, round, () => tasks, view, frozen, tasks.map((t) => t.actor)); +} + +/** + * ⑨：实现全部关闭后开核验；核验全部关闭后，失败的才修复。每次只重验上次修复的候选。 + * 第几次由已开关的阶段推出来；次数用完先写不跑事件，不登记运行（方法.md 4.3 ⑨）。 + */ +function verificationNext(round: number, view: View, frozen: Frozen): Next | null { + let candidates = view.board.decision("batch", round)!.output.chosen; + for (let n = 1; candidates.length > 0; n++) { + const verificationId = stageIdOf(round, `verification-${n}`); + const verify = assignedStage(`verification-${n}`, round, "verification", candidates, view, frozen); + if (verify !== null) return verify; + const failed = view.board.published("verification") + .filter((s) => s.stageId === verificationId && !verificationOutcome(s.content).passed); + if (n > frozen.config.maxRepairs) { + for (const s of failed) { + if (view.events(FORWARD_NOT_RUN).some((e) => e.objects.includes(s.target!))) continue; + return { do: "forwardNotRun", candidate: s.target!, reason: `第 ${n} 次核验仍不通过，修复次数上限 ${frozen.config.maxRepairs} 已用完：${JSON.stringify(verificationOutcome(s.content))}；${s.content.body}` }; + } + return null; + } + candidates = failed.map((s) => s.target!); + if (candidates.length === 0) return null; + const repair = assignedStage(`repair-${n}`, round, "repair", candidates, view, frozen); + if (repair !== null) return repair; + } + return null; +} + +/** ⑩：按选批顺序一次只交回一个运行；没启动的下次启动起新编号，不会把半批提前公开。 */ +function forwardNext(round: number, view: View): Next | null { + for (const candidate of view.board.decision("batch", round)!.output.chosen) { + if (view.events(FORWARD_NOT_RUN).some((e) => e.objects.includes(candidate))) continue; + const runs = view.board.runs().filter((r) => r.plan.purpose === "forward" && r.plan.round === round && r.plan.object === candidate); + const last = runs.at(-1); + if (last !== undefined && last.result === null) throw new Error(`正向运行 ${last.plan.run} 登记了还没有结果：恢复入口应当已经核对过`); + if (last !== undefined && last.result!.process.ending !== "notStarted") continue; + return { do: "forwardRun", round, candidate, run: `r${round}-forward-${candidate}-${runs.length + 1}` }; + } + return null; +} + /** 按种类取记录内容的小工具，加上 materials 读账本的那一份（Board）。 */ class View { readonly #records: readonly LedgerRecord[]; diff --git a/src/flow/recover.ts b/src/flow/recover.ts index 1017894..0831e36 100644 --- a/src/flow/recover.ts +++ b/src/flow/recover.ts @@ -1,4 +1,5 @@ import type { Content } from "../ledger/index.ts"; +import { Board, protocolMetrics } from "../materials/index.ts"; import type { HypothesisMetric } from "../vocab/index.ts"; import { finishRun } from "./formal-run.ts"; import { EVENTS } from "./next-step.ts"; @@ -36,13 +37,17 @@ export async function recover(s: Study): Promise<{ halt: string } | "stopped" | // 在跑或已结束：没写"确认启动"的补写（扣次），然后等它、判分、写结果。 // 已经确认启动、容器却不见了，真实运行里没出现过，不专门写：runner.wait 会报错，按内部错误停下。 if (!started) s.ledger.append("runStarted", { run }); - if ((await finishRun(s, run, metricsOf(plan.content))) === "stopped") return "stopped"; + if ((await finishRun(s, run, metricsOf(s, plan.content))) === "stopped") return "stopped"; } return null; } -/** 一次运行要测的假设专用指标。第 1 部分只有基线运行，它不测假设专用指标；正向、消融运行由第 3、4 部分补。 */ -function metricsOf(plan: Content<"runPlanned">): HypothesisMetric[] { - if (plan.purpose !== "baseline") throw new Error(`运行 ${plan.run} 是 ${plan.purpose} 运行，flow 第 1 部分还不会恢复它`); - return []; +/** 恢复和第一次运行共用原登记指标定义（引用指标也保留原来的缺失规则）。 */ +function metricsOf(s: Study, plan: Content<"runPlanned">): HypothesisMetric[] { + if (plan.purpose === "baseline") return []; + if (plan.purpose === "forward") { + const board = new Board(s.ledger.records()); + return protocolMetrics(board, board.protocol(plan.object!)); + } + throw new Error(`运行 ${plan.run} 是消融运行，消融还没写（任务书 17）`); } diff --git a/src/flow/steps/dispatch.ts b/src/flow/steps/dispatch.ts index 09763ae..4fd5038 100644 --- a/src/flow/steps/dispatch.ts +++ b/src/flow/steps/dispatch.ts @@ -3,7 +3,7 @@ import { runModelTask, type ModelTask } from "../model-task.ts"; import type { Next, TaskRef } from "../next-step.ts"; import { halt, USER_STOPPED, type Study, type StudyEnding } from "../study.ts"; -// 派一个研究者阶段里要派的任务（任务书 12 第 2.6 节、任务书 14 第 2.3 节）： +// 派一个阶段里要派的任务（任务书 12 第 2.6 节、任务书 14 第 2.3 节）： // 同一个人的几个任务按先后跑，不同的人同时跑，各自一个会话。 /** @@ -40,26 +40,10 @@ export async function dispatchStage(s: Study, next: Extract `调用 ${m.call}，${m.text}`).join("；")}`, missed.map((m) => m.call)); } -/** - * 一个任务交上来之后才知道的东西和分份编号（submissions 的 KnownOf、任务书 14 第 2.3 节）： - * 提案是本人本阶段已收下几条，按 proposalId 分份；挑刺按 critiqueId 分份；回应是作者那条原提案；其余没有。 - */ +/** 一个任务的分份编号；实现、核验、修复都只交一份。 */ function modelTask(s: Study, stageId: string, t: TaskRef): ModelTask { - const base = { kind: t.kind, actor: t.actor, stageId, object: t.object }; - const board = new Board(s.ledger.records()); - switch (t.kind) { - case "proposal": { - const round = board.stage(stageId).round; - return { ...base, item: (n) => proposalId(round, t.actor, n), known: (_, others) => ({ accepted: others }) } as ModelTask<"proposal">; - } - case "critique": - return { ...base, item: (n) => critiqueId(t.object!, t.actor, n), known: () => null }; - case "response": { - // 回应的候选只由一条原提案而来（几条合成的交统一）。 - const original = board.proposal(board.proposalsOf(t.object!)[0]!).content; - return { ...base, item: null, known: () => ({ original }) } as ModelTask<"response">; - } - default: - return { ...base, item: null, known: () => null }; - } + const round = new Board(s.ledger.records()).stage(stageId).round; + const item = t.kind === "proposal" ? (n: number) => proposalId(round, t.actor, n) + : t.kind === "critique" ? (n: number) => critiqueId(t.object!, t.actor, n) : null; + return { ...t, stageId, item }; } diff --git a/src/flow/steps/forward.ts b/src/flow/steps/forward.ts new file mode 100644 index 0000000..a50dc0c --- /dev/null +++ b/src/flow/steps/forward.ts @@ -0,0 +1,34 @@ +import { Board, protocolMetrics, validScore } from "../../materials/index.ts"; +import { adopt } from "../../rules/index.ts"; +import type { Json } from "../../vocab/index.ts"; +import { formalRun, type RunOutcome } from "../formal-run.ts"; +import type { Study } from "../study.ts"; + +/** ⑩ 正向运行：用核验过的快照和原登记指标定义，复用正式运行的登记、扣次、判分入口。 */ +export function forwardRun(s: Study, round: number, candidate: string, run: string): Promise { + const board = new Board(s.ledger.records()); + return formalRun(s, { run, purpose: "forward", round, object: candidate }, board.code(candidate).version, protocolMetrics(board, board.protocol(candidate))); +} + +/** ⑫ 采纳：只把本批有效实测分数交给 rules（adopt 内部用 adoptionLine）；null 也写，整批运行据此公开。 */ +export function adoptBatch(s: Study, round: number): void { + const board = new Board(s.ledger.records()); + const candidates = board.decision("batch", round)!.output.chosen.flatMap((id) => { + const run = board.runs().filter((r) => r.plan.purpose === "forward" && r.plan.round === round && r.plan.object === id).at(-1); + const score = run === undefined ? null : validScore(run.result!.judged); + return score === null ? [] : [{ id, score }]; + }); + const input = { + candidates, + baseline: board.baselineScore(board.baselineOf(round)), + direction: s.frozen.facts.finalMetric.direction, + epsilon: s.frozen.facts.epsilon, + knownBest: s.frozen.facts.knownBest, + }; + const output = adopt(input); + s.ledger.append("decision", { + decision: "adopt", round, rule: "adopt", ruleVersion: 1, + params: { direction: input.direction, epsilon: input.epsilon, knownBest: input.knownBest }, seed: null, + input: input as Json, output: output as Json, + }); +} diff --git a/src/flow/task-code.ts b/src/flow/task-code.ts new file mode 100644 index 0000000..200e353 --- /dev/null +++ b/src/flow/task-code.ts @@ -0,0 +1,75 @@ +import { join } from "node:path"; +import type { Content } from "../ledger/index.ts"; +import { codeHunks } from "../jev/index.ts"; +import { Board, protocolMetrics, type CodeChanges } from "../materials/index.ts"; +import type { KnownOf } from "../submissions/index.ts"; +import { lightCheck, protectedChanges, readLightCheck } from "../tasks/index.ts"; +import type { Json, Submission, SubmissionKind } from "../vocab/index.ts"; +import { compareVersions, saveVersion } from "../workspace/index.ts"; +import type { TaskRef } from "./next-step.ts"; +import { callPaths, containerName, freshDir, workspaceRepo } from "./paths.ts"; +import type { Study } from "./study.ts"; + +/** 实现取本批基线；修复取上次提交。核验、修复的材料差异始终相对本批基线。 */ +export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes: CodeChanges | null; startVersion: string | null } { + if (t.kind !== "implementation" && t.kind !== "repair" && t.kind !== "verification") return { changes: null, startVersion: null }; + const board = new Board(s.ledger.records()); + const baseline = board.code(board.baselineOf(board.stage(t.stageId).round)).version; + if (t.kind === "implementation") return { changes: null, startVersion: baseline }; + const version = board.code(t.object!).version; + return { changes: compareVersions(workspaceRepo(s.dir), baseline, version), startVersion: t.kind === "repair" ? version : null }; +} + +/** + * 提交时才知道的东西：提案份数、回应原提案，或实验员当前工作目录的快照和相对本批基线的改动。 + * 快照写入 workspace 后才问 Jev；收下才把 code 和 submission 在同一个事务写进账本。 + */ +export function prepareSubmission( + s: Study, + t: TaskRef & { stageId: string }, + content: Submission, + others: number, + workDir: string, +): { known: KnownOf; code: Content<"code"> | null } { + const board = new Board(s.ledger.records()); + if (t.kind === "proposal") return { known: { accepted: others }, code: null }; + if (t.kind === "response") return { known: { original: board.proposal(board.proposalsOf(t.object!)[0]!).content }, code: null }; + if (t.kind !== "implementation" && t.kind !== "repair") return { known: null, code: null }; + const saved = saveVersion(workspaceRepo(s.dir), workDir, s.frozen.facts.unversionedNames); + if (!saved.saved) return { known: { code: saved }, code: null }; + const baseline = board.code(board.baselineOf(board.stage(t.stageId).round)).version; + const changes = compareVersions(workspaceRepo(s.dir), baseline, saved.version); + const protocol = board.protocol(t.object!); + const impl = content as Submission<"implementation">; + return { + known: { code: { + saved: true, + protectedFiles: protectedChanges(s.task, changes.files.map((f) => f.path)), + hunks: codeHunks(changes), + protocol: { body: protocol.body, metrics: protocolMetrics(board, protocol) }, + } }, + code: { + object: t.object!, version: saved.version, + locations: { factorLocations: impl.factorLocations, safeguardLocations: impl.safeguardLocations, unassigned: impl.unassigned } as Json, + }, + }; +} + +/** 每次模型调用自己的计次器；会话里记录检查结果，不记正式运行、不扣次数（任务书 16 第 2.2 节）。 */ +export function lightChecker(s: Study, call: string, workDir: string): () => Promise { + let used = 0; + const { maxCount, seconds } = s.frozen.config.lightChecks; + return async () => { + if (used >= maxCount) return `本次调用的轻量检查已用完（上限 ${maxCount} 次），不能再查。`; + const n = ++used; + const paths = callPaths(s.dir, call); + const checkDir = freshDir(join(paths.base, `check-${n}`)); + const definition = lightCheck(s.task, workDir, checkDir, s.frozen.image, seconds); + const name = containerName(s.frozen.config.studyId, `${call}-check-${n}`); + const started = await s.runner.start(name, definition, checkDir); + if (!started.started) throw new Error(`调用 ${call} 第 ${n} 次轻量检查没有启动：${started.reason}`); + const result = await s.runner.wait(name, freshDir(join(paths.base, `check-${n}-logs`))); + await s.runner.remove(name); + return JSON.stringify({ used, maxCount, seconds, result: readLightCheck(s.task, checkDir, result) }); + }; +} diff --git a/src/materials/assigned.ts b/src/materials/assigned.ts index 613a81d..0502432 100644 --- a/src/materials/assigned.ts +++ b/src/materials/assigned.ts @@ -1,5 +1,6 @@ -import type { ChecklistOf } from "../vocab/index.ts"; -import { Draft, factorLines, KIND_TEXT, offText, registeredMetrics, taskPoints } from "./blocks.ts"; +import type { ChecklistOf, HypothesisMetric } from "../vocab/index.ts"; +import { type Draft, factorLines, KIND_TEXT, offText, registeredMetrics, taskPoints } from "./blocks.ts"; +import type { Board } from "./read.ts"; import { fenced, json } from "./format.ts"; import { unassignedChanges } from "./locations.ts"; import { clauseKey, factorKey, splitCell, type FrozenProtocol, type LocationTable } from "./shapes.ts"; @@ -27,9 +28,16 @@ function taskSection(d: Draft, role: string, what: string): void { } /** 协议要采集的指标：新登记的全文，引用的取原登记（方法.md 第五部分"引用别人登记过的指标时，沿用原登记的选法"）。 */ +export function protocolMetrics(board: Board, p: FrozenProtocol): HypothesisMetric[] { + const registered = registeredMetrics(board); + return [...p.metrics.defined, ...p.metrics.reused.map((id) => { + const { candidate: _candidate, ...metric } = registered.find((x) => x.id === id)!; + return metric; + })]; +} + function metricSection(d: Draft, p: FrozenProtocol): void { - const registered = registeredMetrics(d.board); - const all = [...p.metrics.defined, ...p.metrics.reused.map((id) => registered.find((x) => x.id === id)!)]; + const all = protocolMetrics(d.board, p); d.section("要实现的指标定义", [ all.length === 0 ? "（这份协议没有要实现的假设专用指标）" : json(all), "", diff --git a/src/materials/index.ts b/src/materials/index.ts index 7fdeec1..2764725 100644 --- a/src/materials/index.ts +++ b/src/materials/index.ts @@ -1,12 +1,13 @@ // materials：每个任务能看到哪些材料、第一条消息怎么写、查询在材料范围内回答什么（docs/结构.md 第二部分）， // 以及账本里只存成 Json 的几样内容的形状和编号规则（用户 2026-10-03 决定 2）。 +export { protocolMetrics } from "./assigned.ts"; export { beliefBook } from "./belief-book.ts"; export { finalEntries, type MaterialRecord, type MaterialSection, type ReadableVersion, type TaskMaterial } from "./blocks.ts"; export { buildMaterial } from "./build.ts"; -export { type CodeChanges, type PromptFile, type Prompts, type StudyFacts, type TaskRequest } from "./inputs.ts"; +export type { CodeChanges, PromptFile, Prompts, StudyFacts, TaskRequest } from "./inputs.ts"; export { queryMaterial, readPrompts } from "./prompts.ts"; // flow 推进时读账本也用这一份（本批基线、基线分数、留池候选、各候选合并了哪些原提案），不另写一份。 -export { Board } from "./read.ts"; +export { Board, validScore } from "./read.ts"; export { callId, candidateId, @@ -17,6 +18,10 @@ export { critiqueId, DECISION_KINDS, entryId, + experimenterId, + verifierId, + FORWARD_NOT_RUN, + forwardNotRunEvent, explanationId, factorId, factorKey, diff --git a/src/materials/shapes.ts b/src/materials/shapes.ts index ec5f205..373be7e 100644 --- a/src/materials/shapes.ts +++ b/src/materials/shapes.ts @@ -25,6 +25,9 @@ export const ORIGINAL_BASELINE = "base"; /** 研究者：第 i 位（从 0 数）是 researcher-(i+1)，与参考项目的写法相同，回放时不用改编号。 */ export const researcherId = (i: number): string => `researcher-${i + 1}`; +/** 每个候选固定一位实验员和核验员，修复、再次核验沿用编号。 */ +export const experimenterId = (candidate: string): string => `experimenter-${candidate}`; +export const verifierId = (candidate: string): string => `verifier-${candidate}`; /** 入口：作者在第零轮交的第 n 个入口，讨论时整份替换的新清单接着往后编（方法.md 4.2）。 */ export const entryId = (author: string, n: number): string => `e-${author}-${n}`; /** 提案：第 round 轮作者交的第 n 条，也是提交记录的 item。 */ @@ -145,3 +148,7 @@ export type Settlement = { */ export const CELL_NOT_RUN = "cellNotRun"; export const cellNotRunEvent = (cell: string, reason: string): Content<"event"> => ({ event: CELL_NOT_RUN, reason, objects: [cell] }); + +/** 正向候选核验不通过、修复次数用完而不跑，不登记正式运行、不扣次（方法.md 4.3 ⑨）。 */ +export const FORWARD_NOT_RUN = "forwardNotRun"; +export const forwardNotRunEvent = (candidate: string, reason: string): Content<"event"> => ({ event: FORWARD_NOT_RUN, reason, objects: [candidate] }); diff --git a/tests/flow-docker.test.ts b/tests/flow-docker.test.ts index bf8767c..e2910ca 100644 --- a/tests/flow-docker.test.ts +++ b/tests/flow-docker.test.ts @@ -1,9 +1,16 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test, type TestContext } from "node:test"; +import { lightChecker } from "../src/flow/task-code.ts"; +import { recover } from "../src/flow/recover.ts"; +import { containerName, freshDir, runPaths } from "../src/flow/paths.ts"; +import type { Study } from "../src/flow/study.ts"; +import { Board, protocolMetrics, type FrozenProtocol } from "../src/materials/index.ts"; +import { loadTask, formalRun as runDefinition, prepareRun } from "../src/tasks/index.ts"; +import { runBalance } from "../src/rules/index.ts"; import { runStudy } from "../src/flow/index.ts"; import { Ledger } from "../src/ledger/index.ts"; import { Runner } from "../src/runner/index.ts"; @@ -76,3 +83,67 @@ test("docker run 没启动：写结果\"没启动\"后停下，原因带 Docker assert.match((ending as { reason: string }).reason, /运行 base-1 没有启动.*unknown or invalid runtime name: inforsi-test-no-such-runtime/); assert.deepEqual(ledgerKinds(dir), ["config", "code", "runPlanned", "runResult", "event"]); }); + + +/** 不派模型，只接真实任务镜像、账本和运行器，检查 flow 的两处 Docker 接线。 */ +async function runtimeStudy(t: TestContext, suffix: string): Promise { + const dir = tempDir(t); + const id = studyId(t, suffix); + const task = loadTask(TASK_DIR); + const runner = new Runner(null); + const image = await runner.buildImage(task.dir, `inforsi-${id}`); + const ledger = Ledger.open(join(dir, "ledger.sqlite")); + t.after(() => ledger.close()); + return { dir, task, runner, ledger, stop: new AbortController().signal, + frozen: { config: { ...fullConfig(), studyId: id, lightChecks: { maxCount: 1, seconds: 7 } }, facts: task.facts, image, seed: 1 } } as Study; +} + +test("flow 轻量检查：每次调用各自限次和限时，真实 Docker 只检查当前目录，不登记正式运行", { timeout: 120_000 }, async (t) => { + const s = await runtimeStudy(t, "light"); + const work = join(s.dir, "work"); + mkdirSync(work); + writeFileSync(join(work, "initial.py"), "raise RuntimeError('不应执行候选代码')\ndef run_packing():\n pass\n"); + for (const call of ["first-call", "second-call"]) { + freshDir(join(s.dir, "calls", call)); + const check = lightChecker(s, call, work); + const result = JSON.parse(await check()); + assert.deepEqual(result, { used: 1, maxCount: 1, seconds: 7, result: { passed: true } }); + assert.match(await check(), /已用完（上限 1 次）/); + assert.equal(await s.runner.status(containerName(s.frozen.config.studyId, `${call}-check-1`)), "absent"); + } + assert.deepEqual(s.ledger.records(), []); +}); + +test("正向恢复：接上原容器不重跑，补启动只扣一次，引用指标沿用原登记缺失作废规则", { timeout: 120_000 }, async (t) => { + const s = await runtimeStudy(t, "forward-recover"); + const metric = { id: "old-c.required", meaning: "必测量", unit: "次", direction: "maximize" as const, howMeasured: "原登记定义", missing: "invalidateRun" as const }; + const original: FrozenProtocol = { round: 0, baseline: "base", proposals: [], factors: [], safeguards: [], metrics: { defined: [metric], reused: [] }, clauses: [], body: "登记" }; + const current: FrozenProtocol = { ...original, round: 1, metrics: { defined: [], reused: [metric.id] } }; + s.ledger.append("protocol", { candidate: "old-c", protocol: original }); + s.ledger.append("protocol", { candidate: "r1-c1", protocol: current }); + const board = new Board(s.ledger.records()); + const run = "forward-1"; + const paths = runPaths(s.dir, run); + const definition = runDefinition(s.task, s.frozen.image, "1"); + s.ledger.append("runPlanned", { run, purpose: "forward", round: 1, object: "r1-c1", definition }); + freshDir(paths.base); + prepareRun(s.task, s.task.baselineDir, freshDir(paths.run), protocolMetrics(board, current)); + const name = containerName(s.frozen.config.studyId, run); + t.after(() => { const result = docker(["rm", "-f", name]); if (result.status !== 0 && !result.stderr.includes("No such container")) throw new Error(result.stderr); }); + assert.deepEqual(await s.runner.start(name, definition, paths.run), { started: true }); + // 中断点：容器已启动，账本还没写 runStarted。恢复不调用 formalRun，而是找到原容器等它。 + assert.equal(await recover(s), null); + const records = s.ledger.records(); + assert.equal(records.filter((r) => r.kind === "runPlanned").length, 1); + assert.equal(records.filter((r) => r.kind === "runStarted").length, 1); + assert.equal(records.filter((r) => r.kind === "runResult").length, 1); + const result = records.find((r) => r.kind === "runResult")!; + if (result.kind !== "runResult") throw new Error("缺运行结果"); + assert.equal(result.content.judged!.verdict, "invalid"); + assert.match(result.content.judged!.verdictReason, /old-c.required/); + assert.equal(result.content.judged!.metrics[metric.id]!.status, "unmeasured"); + assert.deepEqual(runBalance(13, new Board(records).runStates()), { charged: 1, reserved: 0, balance: 12 }); + assert.equal(await s.runner.status(name), "absent"); + assert.equal(await recover(s), null); + assert.equal(s.ledger.records().length, records.length); +}); diff --git a/tests/flow-forward.test.ts b/tests/flow-forward.test.ts new file mode 100644 index 0000000..28d204b --- /dev/null +++ b/tests/flow-forward.test.ts @@ -0,0 +1,145 @@ +import assert from "node:assert/strict"; +import { mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { test, type TestContext } from "node:test"; +import { adoptBatch } from "../src/flow/steps/forward.ts"; +import { prepareSubmission, taskCode } from "../src/flow/task-code.ts"; +import type { Study } from "../src/flow/study.ts"; +import { Board, buildMaterial, protocolMetrics, type FrozenProtocol } from "../src/materials/index.ts"; +import { Ledger } from "../src/ledger/index.ts"; +import { loadTask } from "../src/tasks/index.ts"; +import { checkoutVersion, createRepository, saveVersion } from "../src/workspace/index.ts"; +import type { Submission } from "../src/vocab/index.ts"; +import { fullConfig } from "./config-helpers.ts"; +import { implementation, tempDir } from "./ledger-helpers.ts"; +import { judged, prompts, study, Writer } from "./materials-helpers.ts"; + +function setup(t: TestContext): { s: Study; w: Writer } { + const dir = tempDir(t); + const ledger = Ledger.open(join(dir, "ledger.sqlite")); + t.after(() => ledger.close()); + const task = loadTask(join(import.meta.dirname, "..", "tasks", "circle_packing")); + const s = { dir, ledger, task, prompts, facts: study, frozen: { config: fullConfig(), facts: task.facts } } as Study; + return { s, w: new Writer(ledger) }; +} + +// 预期值手算：基线 1，ε=1/64，已知最好 2；线=1/64。1+1/64 正好在线上，不采纳。 +for (const [label, scores, adopted, gain] of [ + ["高于线且同分：取自然编号小的", [1.125, 1.125], "r1-c2", 0.125], + ["正好等于线：也写 null 决定", [1.015625, 1], null, 0.015625], + ["整批没有有效分数：也写 null 决定", [], null, null], +] as const) { + test(`采纳写账：${label}，写决定后整批公开`, (t) => { + const { s, w } = setup(t); + s.frozen.facts = { ...s.frozen.facts, epsilon: 0.015625, knownBest: 2 }; + w.run("base-1", "baseline", 0, null, judged(1, "main")); + w.decision("batch", 1, null, { chosen: ["r1-c10", "r1-c2", "r1-c3"] }); + scores.forEach((score, i) => w.run(["run-first", "run-second"][i]!, "forward", 1, i === 0 ? "r1-c10" : "r1-c2", judged(score, "main"))); + w.run("invalid", "forward", 1, "r1-c3", { ...judged(99, "main"), verdict: "invalid", verdictReason: "不合法" }); + // 不在本批的运行不参与，即使它分数更高；上轮不能混进本轮。 + w.run("other", "forward", 0, "old-candidate", judged(100, "main")); + const before = new Board(s.ledger.records()); + assert.ok(before.runs().filter((r) => r.plan.round === 1).every((r) => !before.runPublic(r))); + adoptBatch(s, 1); + const record = s.ledger.records().at(-1)!; + assert.equal(record.kind, "decision"); + if (record.kind !== "decision") throw new Error("应写采纳决定"); + assert.deepEqual(record.content, { + decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, + params: { direction: "maximize", epsilon: 0.015625, knownBest: 2 }, seed: null, + input: { candidates: scores.map((score, i) => ({ id: i === 0 ? "r1-c10" : "r1-c2", score })), baseline: 1, direction: "maximize", epsilon: 0.015625, knownBest: 2 }, + output: { line: 0.015625, quarter: 0.25, adopted, + best: scores.length === 0 ? null : { id: adopted ?? "r1-c10", score: scores[0], improvement: gain } }, + }); + const after = new Board(s.ledger.records()); + assert.ok(after.runs().filter((r) => r.plan.round === 1).every((r) => after.runPublic(r))); + assert.equal(after.baselineOf(2), adopted ?? "base"); + }); +} + +const metric = { id: "old-c.metric", meaning: "试验值", unit: "次", direction: "maximize" as const, howMeasured: "按原登记计数", missing: "invalidateRun" as const }; +const oldProtocol: FrozenProtocol = { round: 0, baseline: "base", proposals: [], factors: [], safeguards: [], metrics: { defined: [metric], reused: [] }, clauses: [], body: "原登记" }; +const protocol: FrozenProtocol = { round: 1, baseline: "base", proposals: [], factors: [{ name: "改动", method: "照协议改动", keySettings: "无", continues: null }], safeguards: [], metrics: { defined: [], reused: [metric.id] }, clauses: ["照协议"], body: "冻结正文" }; + +function codeStudy(t: TestContext) { + const { s, w } = setup(t); + const repo = join(s.dir, "repo.git"); + createRepository(repo); + const workDir = join(s.dir, "work"); + mkdirSync(workDir); + writeFileSync(join(workDir, "initial.py"), "first = 1\nsecond = 2\n"); + writeFileSync(join(workDir, "evaluate.py"), "protected = True\n"); + const baseline = saveVersion(repo, workDir, []); + if (!baseline.saved) throw new Error(baseline.reason); + s.ledger.append("code", { object: "base", version: baseline.version, locations: null }); + w.protocol("old-c", oldProtocol); + w.protocol("r1-c1", protocol); + w.open("r1-implementation", "implementation", 1, ["experimenter-r1-c1"]); + return { s, w, repo, workDir, baseline: baseline.version }; +} + +const impl: Submission<"implementation"> = { ...implementation("实现说明"), factorLocations: [{ factor: "p1", locations: ["initial.py:1-2"] }] }; +const ref = { actor: "experimenter-r1-c1", object: "r1-c1", kind: "implementation" as const, stageId: "r1-implementation" }; + +test("提交准备：快照、保护文件、差异块和引用指标同源；修复取上版，但差异永远相对本批基线", (t) => { + const { s, w, repo, workDir, baseline } = codeStudy(t); + assert.deepEqual(taskCode(s, ref), { changes: null, startVersion: baseline }); + writeFileSync(join(workDir, "initial.py"), "first = 10\nsecond = 2\n"); + const prepared = prepareSubmission(s, ref, impl, 0, workDir); + assert.ok(prepared.code); + const known = prepared.known as { code: { saved: true; protectedFiles: string[]; hunks: { text: string }[]; protocol: unknown } }; + assert.deepEqual(known.code.protectedFiles, []); + assert.deepEqual(known.code.protocol, { body: "冻结正文", metrics: [metric] }); + assert.match(known.code.hunks[0]!.text, /-first = 1\n\+first = 10/); + s.ledger.transaction(() => { + w.sub(ref.stageId, ref.actor, ref.kind, ref.object, null, impl); + s.ledger.append("code", prepared.code!); + }); + w.close(ref.stageId); + w.open("r1-verification-1", "verification", 1, ["verifier-r1-c1"]); + w.sub("r1-verification-1", "verifier-r1-c1", "verification", "r1-c1", null, { clauses: [{ clause: "c1", found: false, where: "第二行未实现" }], factors: [{ factor: "p1", found: false, where: "第二行未实现" }], safeguards: [], unassigned: [], body: "只修第二行" }); + w.close("r1-verification-1"); + w.open("r1-repair-1", "implementation", 1, [ref.actor]); + const repair = { ...ref, kind: "repair" as const, stageId: "r1-repair-1" }; + const previous = taskCode(s, repair); + assert.equal(previous.startVersion, prepared.code.version); + assert.match(previous.changes!.diff, /-first = 1\n\+first = 10/); + const repairMaterial = buildMaterial({ ...repair, ledger: s.ledger, study, prompts, codeRoot: join(s.dir, "readable"), changes: previous.changes, artifacts: null }); + assert.match(repairMaterial.firstMessage, /工作目录是上一次提交的代码的副本/); + assert.match(repairMaterial.firstMessage, /第二行未实现/); + writeFileSync(join(workDir, "initial.py"), "first = 10\nsecond = 20\n"); + const fixed = prepareSubmission(s, repair, impl, 0, workDir); + const fixedKnown = fixed.known as typeof known; + assert.match(fixedKnown.code.hunks[0]!.text, /-first = 1/); + assert.match(fixedKnown.code.hunks[0]!.text, /\+second = 20/); + s.ledger.transaction(() => { + w.sub(repair.stageId, repair.actor, repair.kind, repair.object, null, impl); + s.ledger.append("code", fixed.code!); + }); + w.close(repair.stageId); + w.open("r1-verification-2", "verification", 1, ["verifier-r1-c1"]); + const verification = { ...ref, kind: "verification" as const, stageId: "r1-verification-2" }; + const compare = taskCode(s, verification); + assert.equal(compare.startVersion, null); + assert.match(compare.changes!.diff, /-first = 1/); + assert.match(compare.changes!.diff, /\+second = 20/); + assert.deepEqual(protocolMetrics(new Board(s.ledger.records()), protocol), [metric]); + const snapshot = join(s.dir, "snapshot"); + mkdirSync(snapshot); + checkoutVersion(repo, prepared.code.version, snapshot); + assert.equal(readFileSync(join(snapshot, "initial.py"), "utf8"), "first = 10\nsecond = 2\n"); +}); + +test("存不成版本的原原因交 submissions；改了保护文件时不能被漏掉", (t) => { + const { s, workDir } = codeStudy(t); + writeFileSync(join(workDir, "evaluate.py"), "protected = False\n"); + const prepared = prepareSubmission(s, ref, impl, 0, workDir); + assert.deepEqual((prepared.known as { code: { protectedFiles: string[] } }).code.protectedFiles, ["evaluate.py"]); + symlinkSync("initial.py", join(workDir, "link.py")); + const refused = prepareSubmission(s, ref, impl, 0, workDir); + assert.equal(refused.code, null); + const code = (refused.known as { code: { saved: false; path: string; reason: string } }).code; + assert.equal(code.saved, false); + assert.equal(code.path, "link.py"); + assert.match(code.reason, /符号链接/); +}); diff --git a/tests/flow-next-step.test.ts b/tests/flow-next-step.test.ts index a7c8f91..8723baa 100644 --- a/tests/flow-next-step.test.ts +++ b/tests/flow-next-step.test.ts @@ -4,9 +4,12 @@ import { join } from "node:path"; import { test } from "node:test"; import { stepList, type ResearchConfig } from "../src/config/index.ts"; import type { AppendKind, Content, LedgerRecord } from "../src/ledger/index.ts"; -import type { StudyRecord } from "../src/materials/index.ts"; +import { Board, forwardNotRunEvent, type StudyRecord } from "../src/materials/index.ts"; +import { runBalance } from "../src/rules/index.ts"; +import { definition, exited } from "./ledger-helpers.ts"; +import { judged } from "./materials-helpers.ts"; import { nextStep } from "../src/flow/next-step.ts"; -import type { Json, SubmissionKind } from "../src/vocab/index.ts"; +import type { Json, Submission, SubmissionKind } from "../src/vocab/index.ts"; import { fullConfig } from "./config-helpers.ts"; // flow 的"下一步做什么"（任务书 12 第六部分）：纯函数，手写账本记录，看它给出的下一步。 @@ -337,8 +340,8 @@ test("第二次预测关闭：选批", () => { assert.deepEqual(nextStep(roundOne("secondPrediction").records), { do: "selectBatch", round: 1 }); }); -test("选批做完：停下，原因\"实现还没写\"", () => { - assert.deepEqual(nextStep(roundOne("selectBatch").records), { do: "halt", reason: "实现还没写（任务书 16）", objects: [] }); +test("选批做完：开实现阶段，每个入选候选固定一位实验员", () => { + assert.deepEqual(nextStep(roundOne("selectBatch").records), { do: "openStage", stageId: "r1-implementation", stage: "implementation", round: 1, roster: ["experimenter-r1-c1"] }); }); test("关掉第一次预测：第 1 轮写一条按配置跳过（对象 r1-firstPrediction），接着开挑刺", () => { @@ -354,3 +357,149 @@ test("关掉第一次预测：第 1 轮写一条按配置跳过（对象 r1-firs book.add("event", { event: "skippedByConfig", reason, objects: ["r1-firstPrediction"] }); assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-critique", stage: "critique", round: 1, roster: R }); }); + + +// ⑨–⑫：预期动作手写，不调模型、Jev 或 Docker。 +const X = ["experimenter-r1-c1", "experimenter-r1-c2"]; +const V = ["verifier-r1-c1", "verifier-r1-c2"]; + +function batchTwo(maxRepairs: number): Book { + const book = roundOne("secondPrediction"); + const record = book.records.find((r) => r.kind === "config")!; + (record.content as unknown as StudyRecord).config = { ...fullConfig(), maxRepairs } as unknown as Json; + return book.add("decision", { decision: "batch", round: 1, rule: "bestExpected", ruleVersion: 1, params: null, seed: "s", input: null, output: { chosen: C } }); +} + +function implemented(book: Book): Book { + book.open("r1-implementation", "implementation", 1, X); + for (const [i, candidate] of C.entries()) book.sub("r1-implementation", X[i]!, "implementation", candidate, null); + return book.add("stageClose", { stageId: "r1-implementation" }); +} + +function verified(book: Book, n: number, candidate: string, passed: boolean): Book { + // 对应真实 15-tight-2 的 submissions created_seq=173（墙约束行和右端错位）、194（修复后通过）。 + // 只转录驱动程序所需的逐项 found，不伪造 Jev 的判断或模型运行。 + const content: Submission<"verification"> = { + clauses: [{ clause: "c1", found: passed, where: passed ? "initial.py 墙约束行和右端已对齐" : "initial.py:225-228 与 243-246：墙约束行和右端错位" }], + factors: [{ factor: "p1", found: passed, where: "LP 墙约束" }], + safeguards: [], unassigned: [], body: passed ? "通过" : "解的不是协议写的线性规划", + }; + return book.add("submission", { stageId: `r1-verification-${n}`, author: `verifier-${candidate}`, kind: "verification", target: candidate, item: null, version: 1, content, jev: null }); +} + +function firstVerification(book: Book, a: boolean, b: boolean): Book { + implemented(book).open("r1-verification-1", "verification", 1, V); + verified(book, 1, C[0]!, a); + verified(book, 1, C[1]!, b); + return book.add("stageClose", { stageId: "r1-verification-1" }); +} + +function recordForward(book: Book, candidate: string, n: number, ending: "valid" | "notStarted" | "timeout"): Book { + const run = `r1-forward-${candidate}-${n}`; + book.add("runPlanned", { run, purpose: "forward", round: 1, object: candidate, definition }); + if (ending === "notStarted") return book.add("runResult", { run, process: { ending, reason: "Docker 未启动" }, judged: null }); + book.add("runStarted", { run }); + const process = ending === "timeout" ? { ending: "timeout" as const, wallSeconds: measured(120), cpuSeconds: measured(100), peakMemoryBytes: measured(1024) } : exited; + return book.add("runResult", { run, process, judged: ending === "valid" ? judged(1.25, "main") : null }); +} + +const forward = (candidate: string, n: number) => ({ do: "forwardRun", round: 1, candidate, run: `r1-forward-${candidate}-${n}` }); + +test("实现的任务同时可派；已提交后中断不重派，等全体交齐关阶段后才核验", () => { + const book = batchTwo(1).open("r1-implementation", "implementation", 1, X); + assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-implementation", tasks: C.map((object, i) => ({ actor: X[i], kind: "implementation", object })) }); + book.sub("r1-implementation", X[0]!, "implementation", C[0]!, null); + book.call("r1-implementation", X[0]!, "implementation", C[0]!, 1, { wallSeconds: { status: "unmeasured", reason: "进程中断" }, status: "failed", reason: "进程中断" }); + assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-implementation", tasks: [{ actor: X[1], kind: "implementation", object: C[1] }] }); + book.sub("r1-implementation", X[1]!, "implementation", C[1]!, null); + assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-implementation" }); + book.add("stageClose", { stageId: "r1-implementation" }); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-verification-1", stage: "verification", round: 1, roster: V }); +}); + +test("15-tight-2 的一次修复路径：核验关后才派原实验员，修复关后只重验修过的候选", () => { + const book = implemented(batchTwo(1)).open("r1-verification-1", "verification", 1, V); + verified(book, 1, C[0]!, false); + verified(book, 1, C[1]!, true); + assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-verification-1" }); + book.add("stageClose", { stageId: "r1-verification-1" }); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-repair-1", stage: "implementation", round: 1, roster: [X[0]] }); + book.open("r1-repair-1", "implementation", 1, [X[0]!]); + assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-repair-1", tasks: [{ actor: X[0], kind: "repair", object: C[0] }] }); + book.sub("r1-repair-1", X[0]!, "repair", C[0]!, null); + assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-repair-1" }); + book.add("stageClose", { stageId: "r1-repair-1" }); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-verification-2", stage: "verification", round: 1, roster: [V[0]] }); + book.open("r1-verification-2", "verification", 1, [V[0]!]); + assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-verification-2", tasks: [{ actor: V[0], kind: "verification", object: C[0] }] }); + verified(book, 2, C[0]!, true); + assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-verification-2" }); + book.add("stageClose", { stageId: "r1-verification-2" }); + assert.deepEqual(nextStep(book.records), forward(C[0]!, 1)); +}); + +test("修复次数从关闭的阶段数出：两次修复后仍失败，逐候选写不跑事件，不占预算", () => { + const book = firstVerification(batchTwo(2), false, false); + for (let n = 1; n <= 2; n++) { + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: `r1-repair-${n}`, stage: "implementation", round: 1, roster: X }); + book.open(`r1-repair-${n}`, "implementation", 1, X); + for (const [i, candidate] of C.entries()) book.sub(`r1-repair-${n}`, X[i]!, "repair", candidate, null); + book.add("stageClose", { stageId: `r1-repair-${n}` }); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: `r1-verification-${n + 1}`, stage: "verification", round: 1, roster: V }); + book.open(`r1-verification-${n + 1}`, "verification", 1, V); + for (const candidate of C) verified(book, n + 1, candidate, false); + book.add("stageClose", { stageId: `r1-verification-${n + 1}` }); + } + for (const candidate of C) { + const next = nextStep(book.records); + assert.equal(next.do, "forwardNotRun"); + if (next.do !== "forwardNotRun") throw new Error("应写不跑事件"); + assert.equal(next.candidate, candidate); + assert.match(next.reason, /第 3 次核验仍不通过，修复次数上限 2 已用完/); + assert.match(next.reason, /解的不是协议写的线性规划/); + book.add("event", forwardNotRunEvent(candidate, next.reason)); + } + assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 }); + assert.deepEqual(nextStep(book.records), { do: "adopt", round: 1 }); +}); + +test("maxRepairs=0：第一次核验失败就记不跑，另一候选仍可串行运行", () => { + const book = firstVerification(batchTwo(0), false, true); + const next = nextStep(book.records); + assert.equal(next.do, "forwardNotRun"); + if (next.do !== "forwardNotRun") throw new Error("应写不跑事件"); + book.add("event", forwardNotRunEvent(next.candidate, next.reason)); + assert.deepEqual(nextStep(book.records), forward(C[1]!, 1)); +}); + +test("正向串行：上次未启动起新编号；已登记未结束必须先恢复；超时也扣次且不重跑；采纳前不公开", () => { + const book = firstVerification(batchTwo(1), true, true); + assert.deepEqual(nextStep(book.records), forward(C[0]!, 1)); + recordForward(book, C[0]!, 1, "notStarted"); + assert.deepEqual(nextStep(book.records), forward(C[0]!, 2)); + assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 }); + const run = `r1-forward-${C[0]}-2`; + book.add("runPlanned", { run, purpose: "forward", round: 1, object: C[0]!, definition }); + assert.throws(() => nextStep(book.records), /登记了还没有结果：恢复入口应当已经核对过/); + assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 1, balance: 11 }); + book.add("runStarted", { run }); + assert.throws(() => nextStep(book.records), /登记了还没有结果/); + book.add("runResult", { run, process: exited, judged: judged(1.25, "main") }); + assert.deepEqual(nextStep(book.records), forward(C[1]!, 1)); + recordForward(book, C[1]!, 1, "timeout"); + assert.deepEqual(nextStep(book.records), { do: "adopt", round: 1 }); + assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 3, reserved: 0, balance: 10 }); + const before = new Board(book.records); + assert.ok(before.runs().filter((r) => r.plan.purpose === "forward").every((r) => !before.runPublic(r))); + book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: null } }); + const after = new Board(book.records); + assert.ok(after.runs().filter((r) => r.plan.purpose === "forward").every((r) => after.runPublic(r))); + assert.deepEqual(nextStep(book.records), { do: "halt", reason: "消融还没写（任务书 17）", objects: [] }); +}); + +test("采纳只有一个因子的候选也统一在 listCells 停下，不擅自开收尾", () => { + const book = firstVerification(batchTwo(1), true, true); + for (const c of C) recordForward(book, c, 1, "valid"); + book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: C[0]! } }); + assert.deepEqual(nextStep(book.records), { do: "halt", reason: "消融还没写（任务书 17）", objects: [] }); +});