From 144b88480b8ecec49bbb510379f5d907292e80ab Mon Sep 17 00:00:00 2001 From: iomgaa Date: Sun, 4 Oct 2026 14:04:55 -0400 Subject: [PATCH] =?UTF-8?q?flow:=20=E6=B6=88=E8=9E=8D=E5=88=B0=E6=94=B6?= =?UTF-8?q?=E5=B0=BE=E3=80=81=E8=BD=AE=E6=9C=AB=E8=A7=A3=E5=B0=81=E3=80=81?= =?UTF-8?q?=E4=B8=80=E8=BD=AE=E6=8E=A5=E4=B8=80=E8=BD=AE=E4=B8=8E=E7=A0=94?= =?UTF-8?q?=E7=A9=B6=E7=BB=93=E6=9D=9F=EF=BC=88=E4=BB=BB=E5=8A=A1=E4=B9=A6?= =?UTF-8?q?=2017=20=E7=AC=AC=E4=BA=8C=E5=9D=97=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ⑬–⑱：listCells 从采纳决定和冻结协议现算（不写记录）；explanation 有消融才开 阶段；selectCells 照 CellRuleInput 拼输入（余额当时现算、decisionSeed 版本 2）， 写种类 cells 的决定，没选中的格不另写事件；消融实现/核验按第几次编号 （r<轮>-ablationImplementation-1、verification-1、-2……从账本数），重做从上一次 这一格代码接着改，次数用完写 cellNotRunEvent 不跑不扣次；cannotRun 传 null 不核验 不运行不扣次；alsoOff 用 sharedSettings 沿用同设置的第一格；消融运行一个接一个跑， 用途 ablation，登记/扣次/恢复与正向同一入口，恢复时指标缺失策略覆盖为只记缺失 （方法.md 第五部分）；settle 拼 SettleInput 调 settleAblation，结算与 chooseBet 的 bet 决定写在同一事务；wrapUp 阶段关闭后，本轮选中的每个候选在同一个事务里解封 作者自评（取最后一版提案的封存）。 - 一轮接一轮：每轮开头用 runBalance 现算余额，余额为 0 不开新轮；走到 export 停下， 原因"导出还没写"。去掉 roundsNext 里 step.id !== "listCells" 的临时写法。 - 消融中发现由 materials 的 Board.baselineOf 从结算输出读出，flow 不另写采纳记录 （已核实 materials 已实现）。 - 测试 +14：下一步分支 8 条（含收尾关闭后才解封、余额 0 走到导出停、关掉消融时 解释照方法停下）、消融步骤 6 条（选格输入逐字段手拼对账、结算四种结局、同设置 沿用、解封取最后一版封存）；npm run check 通过，npm test 472 条通过。 - 全新上下文审阅一轮（只看消融重做循环、登记扣次恢复、选格/结算/押的因子输入、 解封时机、下一轮与结束条件）：没有会污染账本的问题；采纳一条小修（删掉未用的 导入），其余笔记照列不改。 flow 1302 → 1662 行（上限 3500），materials 未动（1930 行）。 --- src/flow/loop.ts | 23 +++- src/flow/model-task.ts | 28 +++- src/flow/next-step.ts | 153 ++++++++++++++++++++-- src/flow/recover.ts | 13 +- src/flow/steps/ablation.ts | 162 +++++++++++++++++++++++ src/flow/steps/round.ts | 4 + src/flow/task-code.ts | 33 ++++- tests/flow-ablation-steps.test.ts | 207 ++++++++++++++++++++++++++++++ tests/flow-next-step.test.ts | 188 ++++++++++++++++++++++++++- 9 files changed, 779 insertions(+), 32 deletions(-) create mode 100644 src/flow/steps/ablation.ts create mode 100644 tests/flow-ablation-steps.test.ts diff --git a/src/flow/loop.ts b/src/flow/loop.ts index 39fe3f1..52a0aef 100644 --- a/src/flow/loop.ts +++ b/src/flow/loop.ts @@ -1,7 +1,7 @@ import { mkdirSync } from "node:fs"; import { resolve } from "node:path"; import type { ResearchConfig } from "../config/index.ts"; -import { forwardNotRunEvent } from "../materials/index.ts"; +import { forwardNotRunEvent, cellNotRunEvent } from "../materials/index.ts"; import { Ledger } from "../ledger/index.ts"; import { Runner } from "../runner/index.ts"; import { fullConfig, readFrozen } from "./frozen.ts"; @@ -9,6 +9,7 @@ import { EVENTS, nextStep, type Next } from "./next-step.ts"; import { ledgerFile } from "./paths.ts"; import { recover } from "./recover.ts"; import { baselineRun, saveBaseline } from "./steps/baseline.ts"; +import { ablationRun, selectCells, settle, unsealChosen } from "./steps/ablation.ts"; import { forwardRun, adoptBatch } from "./steps/forward.ts"; import { dispatchStage } from "./steps/dispatch.ts"; import { dedupe, freezeProtocols, selectBatch } from "./steps/round.ts"; @@ -78,8 +79,12 @@ async function doStep(s: Study, next: Exclude): Pr saveBaseline(s); return null; case "baselineRun": - case "forwardRun": { - const ran = next.do === "baselineRun" ? await baselineRun(s, next.run) : await forwardRun(s, next.round, next.candidate, next.run); + case "forwardRun": + case "ablationRun": { + const ran = + next.do === "baselineRun" ? await baselineRun(s, next.run) + : next.do === "forwardRun" ? await forwardRun(s, next.round, next.candidate, next.run) + : await ablationRun(s, next.round, next.cell, next.run); if (ran === "stopped") return USER_STOPPED; if (ran === "done") return null; return halt(s.ledger, `运行 ${next.run} 没有启动，名额已退还；本进程不重跑，下次启动时起新编号重跑。Docker 的报错：${ran.notStarted}`, [next.run]); @@ -113,6 +118,18 @@ async function doStep(s: Study, next: Exclude): Pr case "adopt": adoptBatch(s, next.round); return null; + case "selectCells": + selectCells(s, next.round); + return null; + case "cellNotRun": + s.ledger.append("event", cellNotRunEvent(next.cell, next.reason)); + return null; + case "settle": + settle(s, next.round); + return null; + case "unsealChosen": + unsealChosen(s, next.round); + return null; case "halt": return halt(s.ledger, next.reason, next.objects); } diff --git a/src/flow/model-task.ts b/src/flow/model-task.ts index 14f98e3..9eda06f 100644 --- a/src/flow/model-task.ts +++ b/src/flow/model-task.ts @@ -1,10 +1,12 @@ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; import { runSession, type SessionTask, type SessionEnding, type SubmitAttempt, type SubmitReceipt } from "../agents/index.ts"; -import { buildMaterial, callId, queryMaterial, researcherId, type TaskMaterial } from "../materials/index.ts"; +import { Board, buildMaterial, callId, queryMaterial, researcherId, type TaskMaterial } from "../materials/index.ts"; import { receive, type Incoming } from "../submissions/index.ts"; import { SUBMISSION_KINDS, type JevJudgment, type Json, type SubmissionKind } from "../vocab/index.ts"; import { checkoutVersion } from "../workspace/index.ts"; import { prepareSubmission, taskCode, lightChecker } from "./task-code.ts"; -import { callPaths, freshDir, workspaceRepo } from "./paths.ts"; +import { callPaths, freshDir, runPaths, workspaceRepo } from "./paths.ts"; import type { Study } from "./study.ts"; // 共用动作一：派一个模型任务（任务书 12 第 2.6 节）。先写"模型调用开始"（材料整份存进去），再开会话， @@ -41,7 +43,7 @@ export async function runModelTask(s: Study, t: ModelT prompts: s.prompts, codeRoot: paths.code, changes, - artifacts: null, + artifacts: t.kind === "explanation" ? runArtifacts(s, t.stageId) : null, }); const model = role === "researcher" ? researcherModel(s, t.actor) : s.frozen.config.models[role]; // 先登记、后建目录：调用编号只在登记之后才算用掉，被杀在两者之间时下次的编号不会撞上旧目录。 @@ -84,6 +86,26 @@ export async function runModelTask(s: Study, t: ModelT return { call, ending }; } +/** + * 解释任务要的产物（materials 的 TaskRequest.artifacts）：本轮每次已公开正向运行的 metrics.json、solution.json + * 和标准输出、标准错误日志（运行编号 → 文件名 → 文字）。完整代码那次的答案出自哪里，就靠这些产物判断。 + */ +function runArtifacts(s: Study, stageId: string): Record> { + const board = new Board(s.ledger.records()); + const round = board.stage(stageId).round; + const artifacts: Record> = {}; + for (const r of board.runs()) { + if (r.plan.purpose !== "forward" || r.plan.round !== round || !board.runPublic(r)) continue; + const paths = runPaths(s.dir, r.plan.run); + const files: Record = {}; + for (const [dir, name] of [[paths.run, "metrics.json"], [paths.run, "solution.json"], [paths.logs, "stdout.log"], [paths.logs, "stderr.log"]] as const) { + if (existsSync(join(dir, name))) files[name] = readFileSync(join(dir, name), "utf8"); + } + artifacts[r.plan.run] = files; + } + return artifacts; +} + /** 研究者用的模型：第 i 位研究者（researcherId(i)）用配置里第 i 个。 */ function researcherModel(s: Study, actor: string) { const models = s.frozen.config.models.researchers; diff --git a/src/flow/next-step.ts b/src/flow/next-step.ts index 2e8bfd2..602eadf 100644 --- a/src/flow/next-step.ts +++ b/src/flow/next-step.ts @@ -1,8 +1,8 @@ import type { RoundStepId, StudyStepId } from "../config/index.ts"; import type { Content, LedgerRecord } from "../ledger/index.ts"; -import { Board, experimenterId, verifierId, FORWARD_NOT_RUN, ORIGINAL_BASELINE, researcherId } from "../materials/index.ts"; -import { compareIds } from "../rules/index.ts"; -import { verificationOutcome } from "../submissions/index.ts"; +import { Board, CELL_NOT_RUN, cellId, experimenterId, factorKey, FORWARD_NOT_RUN, ORIGINAL_BASELINE, researcherId, splitCell, verifierId } from "../materials/index.ts"; +import { compareIds, listCells, runBalance, sharedSettings } from "../rules/index.ts"; +import { ablationOutcome, verificationOutcome } from "../submissions/index.ts"; import { SUBMISSION_KINDS, type Stage, type SubmissionKind } from "../vocab/index.ts"; import { readFrozen, type Frozen } from "./frozen.ts"; @@ -58,6 +58,11 @@ export type Next = | { do: "forwardNotRun"; candidate: string; reason: string } | { do: "forwardRun"; round: number; candidate: string; run: string } | { do: "adopt"; round: number } + | { do: "selectCells"; round: number } + | { do: "cellNotRun"; cell: string; reason: string } + | { do: "ablationRun"; round: number; cell: string; run: string } + | { do: "settle"; round: number } + | { do: "unsealChosen"; round: number } | { do: "halt"; reason: string; objects: string[] }; export function nextStep(records: readonly LedgerRecord[]): Next { @@ -70,7 +75,7 @@ export function nextStep(records: readonly LedgerRecord[]): Next { step.status === "skip" ? skipNext(step.id, step.reason, view) : step.id === "rounds" ? roundsNext(view, frozen) : studyStepNext(step.id, view, frozen); if (next !== null) return next; } - throw new Error("步骤清单走完了却没有停下：rounds 一步应当停下"); + throw new Error("步骤清单走完了却没有停下：export 一步应当停下"); } /** 按配置跳过的步骤 → 一条"按配置跳过"事件，写明步骤和原因。 */ @@ -96,18 +101,21 @@ function studyStepNext(step: Exclude, view: View, frozen: // 分配 → 第 0 轮的入口分配决定（decision，种类 entries）。 return view.board.decision("entries", 0) === undefined ? { do: "assignEntries" } : null; case "export": - throw new Error("走到了导出：rounds 一步应当先停下"); + // 余额用完、一轮接一轮走完后走到这里：导出还没写，停下（任务书 17 第 2.1 节）。 + return { do: "halt", reason: "导出还没写", objects: [] }; } } /** * 一轮接一轮（结构.md 第三部分"每一轮"）：第一个没做完的轮次里第一个没做完的步骤。 - * 每一步做完的记录都带轮次，做完的轮次整轮跳过；现在走到列消融格就停下，所以循环一定会返回。 + * 每一步做完的记录都带轮次，做完的轮次整轮跳过。余额为 0 不开下一轮（方法.md 第七部分"什么时候开下一轮"）： + * 交回 null，整份研究的下一步就是 export。 */ -function roundsNext(view: View, frozen: Frozen): Next { +function roundsNext(view: View, frozen: Frozen): Next | null { for (let round = 1; ; round++) { + if (runBalance(frozen.config.runs, view.board.runStates()).balance === 0) return null; for (const step of frozen.steps.round) { - const next = step.status === "skip" && step.id !== "listCells" ? skipNext(stageIdOf(round, step.id), step.reason, view) : roundStepNext(step.id, round, view, frozen); + const next = step.status === "skip" ? skipNext(stageIdOf(round, step.id), step.reason, view) : roundStepNext(step.id, round, view, frozen); if (next !== null) return next; } } @@ -157,9 +165,43 @@ function roundStepNext(step: RoundStepId, round: number, view: View, frozen: Fro case "adopt": return view.board.decision("adopt", round) === undefined ? { do: "adopt", round } : null; case "listCells": - return { do: "halt", reason: "消融还没写（任务书 17）", objects: [] }; + // 不写记录：格从采纳决定和冻结协议现算（方法.md 4.3 ⑫），格编号用 materials 的 cellId。 + return null; + case "explanation": { + // ⑭：这一轮有消融（采纳了、且被采纳改法有两个以上因子）才开解释阶段；否则直接到收尾（方法.md 4.3 ⑫）。 + // 解释 → 这一轮解释阶段的关闭记录。 + const adopted = view.board.decision("adopt", round)?.output.adopted ?? null; + if (adopted === null || view.board.protocol(adopted).factors.length < 2) return null; + if (frozen.steps.round.some((s) => s.id === "listCells" && s.status === "skip")) { + return { do: "halt", reason: "配置关掉了消融、又采纳了多因子改法：不交三个数的解释还没写（cells 只按完整流程全开实现，做步骤开关时再补）", objects: [] }; + } + return stageNext(stageIdOf(round, "explanation"), "explanation", round, (roster) => roster.map((actor) => ({ actor, kind: "explanation" as const, object: null })), view, frozen, researchers(frozen)); + } + case "selectCells": + // ⑮ → 本轮的选格决定（decision，种类 cells）。没选中的格不另写事件：原因都在这条决定里。 + if (ablationOf(view.board, round) === null) return null; + return view.board.decision("cells", round) === undefined ? { do: "selectCells", round } : null; + case "ablationImplementation": + case "ablationVerification": + // ⑯ → 每个选中的格：交了"跑不起来"，或最近一次核验通过，或重做次数用完、有"不跑"事件。 + return ablationOf(view.board, round) === null ? null : ablationCellsNext(round, view, frozen); + case "ablationRun": + // ⑯ → 每个通过核验的格，最近一次消融运行有结果，而且不是"没启动"。 + return ablationOf(view.board, round) === null ? null : ablationRunNext(round, view); + case "settle": + // ⑰ → 本轮的结算记录（settlement），以及同一事务里的押的因子决定（decision，种类 bet）。 + if (ablationOf(view.board, round) === null) return null; + return view.board.settlement(round) === undefined || view.board.decision("bet", round) === undefined ? { do: "settle", round } : null; + case "wrapUp": { + // ⑱ → 收尾阶段的关闭记录，以及本轮选中的每个候选都有解封记录。 + const stage = stageNext(stageIdOf(round, "wrapUp"), "wrapUp", round, (roster) => roster.map((actor) => ({ actor, kind: "wrapUp" as const, object: null })), view, frozen, researchers(frozen)); + if (stage !== null) return stage; + // 轮末解封（方法.md 4.3 ⑱）：只在收尾阶段关闭之后写，提前写会泄露。本轮全部解封写在同一个事务里。 + const chosen = view.board.decision("batch", round)!.output.chosen; + return chosen.every((c) => view.of("unseal").some((u) => u.candidate === c)) ? null : { do: "unsealChosen", round }; + } default: - throw new Error(`走到了第 ${round} 轮的 ${step}：listCells 一步应当先停下`); + throw new Error(`第 ${round} 轮的 ${step} 没有对应的下一步写法`); } } @@ -265,12 +307,12 @@ const researchers = (frozen: Frozen): string[] => frozen.config.models.researche function assignedStage( step: string, round: number, - kind: "implementation" | "repair" | "verification", + kind: "implementation" | "repair" | "verification" | "ablationImplementation" | "ablationVerification", candidates: readonly string[], view: View, frozen: Frozen, ): Next | null { - const actorOf = kind === "verification" ? verifierId : experimenterId; + const actorOf = SUBMISSION_KINDS[kind].role === "verifier" ? verifierId : experimenterId; const tasks = candidates.map((object) => ({ actor: actorOf(object), kind, object })); return stageNext(stageIdOf(round, step), SUBMISSION_KINDS[kind].stage, round, () => tasks, view, frozen, tasks.map((t) => t.actor)); } @@ -315,6 +357,93 @@ function forwardNext(round: number, view: View): Next | null { return null; } +/** + * 这一轮做不做消融、消融哪个候选的哪几个因子（方法.md 4.3 ⑫）：采纳了、且被采纳改法有两个以上因子。 + * 没采纳或只采纳了单因子改法时 ⑬–⑰ 都不做：这几步不写记录，从采纳决定和冻结协议就算得出。 + */ +export function ablationOf(board: Board, round: number): { candidate: string; keys: string[] } | null { + const adopted = board.decision("adopt", round)?.output.adopted ?? null; + if (adopted === null) return null; + const keys = listCells(board.protocol(adopted).factors.map((_, i) => factorKey(i))); + return keys.length < 2 ? null : { candidate: adopted, keys }; +} + +/** 选格决定选中要测的格（对象编号用 materials 的 cellId），按因子编号排。 */ +export function chosenCells(board: Board, round: number): string[] { + const { candidate } = ablationOf(board, round)!; + return board.decision("cells", round)!.output.chosen.map((key) => cellId(candidate, key)); +} + +/** 一格最近一次已公开的消融实现提交（阶段关闭即公开）；没有为 undefined。 */ +export function lastAblationImpl(board: Board, cell: string) { + return board.published("ablationImplementation").filter((s) => s.target === cell).at(-1); +} + +/** 选中的格里"活着"的（没交"跑不起来"、没有"不跑"事件），以及每格实际关掉的全部因子（指定的加实验员报的一起关的）。 */ +export function aliveCells(board: Board, events: readonly Content<"event">[], round: number): Array<{ cell: string; off: string[] }> { + return chosenCells(board, round) + .map((cell) => ({ cell, impl: lastAblationImpl(board, cell) })) + .filter((x) => x.impl !== undefined && x.impl.content.cannotRun === null && !events.some((e) => e.event === CELL_NOT_RUN && e.objects.includes(x.cell))) + .map((x) => ({ cell: x.cell, off: [splitCell(x.cell).key, ...x.impl!.content.alsoOff] })); +} + +/** + * 要跑消融运行的格：活着的格里同设置的第一格（方法.md 6.6"一起关"：几格实际关掉的因子完全相同就是同一种设置， + * 沿用第一格的结果，不重跑、不重复扣次、不重复记账）。 + */ +export function cellsToRun(board: Board, events: readonly Content<"event">[], round: number): string[] { + const alive = aliveCells(board, events, round); + const sameAs = new Map(sharedSettings(alive.map((x) => ({ factor: splitCell(x.cell).key, off: x.off }))).map((x) => [x.factor, x.sameAs])); + return alive.filter((x) => sameAs.get(splitCell(x.cell).key) === null).map((x) => x.cell); +} + +/** + * ⑯ 消融实现与核验（任务书 17 第 2.2 节）：实现全部关闭后开核验；核验全部关闭后，没通过的退回原实验员， + * 从它上一次交的这一格代码接着改，每次只重验刚重做的格。阶段编号按"第几次"排，第几次从已开关的阶段数出来，不另存。 + * 交了"跑不起来"的格不核验、不运行、不扣次；核验不通过、次数用完写"不跑"事件，不登记运行（方法.md 4.3 ⑯、6.6）。 + */ +function ablationCellsNext(round: number, view: View, frozen: Frozen): Next | null { + let cells = chosenCells(view.board, round).filter((cell) => !view.events(CELL_NOT_RUN).some((e) => e.objects.includes(cell))); + for (let n = 1; cells.length > 0; n++) { + const impl = assignedStage(`ablationImplementation-${n}`, round, "ablationImplementation", cells, view, frozen); + if (impl !== null) return impl; + const runnable = cells.filter((cell) => lastAblationImpl(view.board, cell)!.content.cannotRun === null); + const failed: Array<{ cell: string; reason: string }> = []; + if (runnable.length > 0) { + const ver = assignedStage(`ablationVerification-${n}`, round, "ablationVerification", runnable, view, frozen); + if (ver !== null) return ver; + for (const s of view.board.published("ablationVerification").filter((x) => x.stageId === stageIdOf(round, `ablationVerification-${n}`))) { + const outcome = ablationOutcome(s.content, splitCell(s.target!).key, lastAblationImpl(view.board, s.target!)!.content.alsoOff); + if (!outcome.passed) { + failed.push({ cell: s.target!, reason: `第 ${n} 次核验仍不通过，修复次数上限 ${frozen.config.maxRepairs} 已用完：${JSON.stringify({ leftOn: outcome.leftOn, wronglyOff: outcome.wronglyOff })}` }); + } + } + } + if (n > frozen.config.maxRepairs) { + for (const f of failed) { + if (view.events(CELL_NOT_RUN).some((e) => e.objects.includes(f.cell))) continue; + return { do: "cellNotRun", cell: f.cell, reason: f.reason }; + } + return null; + } + cells = failed.map((f) => f.cell); + if (cells.length === 0) return null; + } + return null; +} + +/** ⑯ 消融运行：按格编号顺序一次只交回一个运行（一份研究里的消融运行一个接一个跑，同正向）；没启动的下次启动起新编号。 */ +function ablationRunNext(round: number, view: View): Next | null { + for (const cell of cellsToRun(view.board, view.of("event"), round)) { + const runs = view.board.runs().filter((r) => r.plan.purpose === "ablation" && r.plan.round === round && r.plan.object === cell); + const last = runs.at(-1); + if (last !== undefined && last.result === null) throw new Error(`消融运行 ${last.plan.run} 登记了还没有结果：恢复入口应当已经核对过`); + if (last !== undefined && last.result!.process.ending !== "notStarted") continue; + return { do: "ablationRun", round, cell, run: `r${round}-ablation-${cell}-${runs.length + 1}` }; + } + return null; +} + /** 按种类取记录内容的小工具，加上 materials 读账本的那一份（Board）。 */ class View { readonly #records: readonly LedgerRecord[]; diff --git a/src/flow/recover.ts b/src/flow/recover.ts index 0831e36..9936b8e 100644 --- a/src/flow/recover.ts +++ b/src/flow/recover.ts @@ -1,9 +1,10 @@ import type { Content } from "../ledger/index.ts"; -import { Board, protocolMetrics } from "../materials/index.ts"; +import { Board, protocolMetrics, splitCell } from "../materials/index.ts"; import type { HypothesisMetric } from "../vocab/index.ts"; import { finishRun } from "./formal-run.ts"; import { EVENTS } from "./next-step.ts"; import { containerName } from "./paths.ts"; +import { ablationMetrics } from "./steps/ablation.ts"; import type { Study } from "./study.ts"; // 恢复入口第 3、4 步（任务书 12 第 2.5 节）：每次启动都走这里，不分第一次启动、正常重启还是被杀后重启。 @@ -42,12 +43,10 @@ export async function recover(s: Study): Promise<{ halt: string } | "stopped" | return null; } -/** 恢复和第一次运行共用原登记指标定义（引用指标也保留原来的缺失规则）。 */ +/** 恢复和第一次运行共用原登记指标定义（引用指标也保留原来的缺失规则）；消融格缺失一律只记缺失（方法.md 第五部分"指标"）。 */ function metricsOf(s: Study, plan: Content<"runPlanned">): HypothesisMetric[] { if (plan.purpose === "baseline") return []; - if (plan.purpose === "forward") { - const board = new Board(s.ledger.records()); - return protocolMetrics(board, board.protocol(plan.object!)); - } - throw new Error(`运行 ${plan.run} 是消融运行，消融还没写（任务书 17）`); + const board = new Board(s.ledger.records()); + if (plan.purpose === "forward") return protocolMetrics(board, board.protocol(plan.object!)); + return ablationMetrics(board, splitCell(plan.object!).candidate); } diff --git a/src/flow/steps/ablation.ts b/src/flow/steps/ablation.ts new file mode 100644 index 0000000..4514bea --- /dev/null +++ b/src/flow/steps/ablation.ts @@ -0,0 +1,162 @@ +import type { Content } from "../../ledger/index.ts"; +import { Board, cellId, CELL_NOT_RUN, explanationId, protocolMetrics, splitCell, validScore } from "../../materials/index.ts"; +import { + chooseBet, + decisionSeed, + runBalance, + runCellRule, + settleAblation, + sharedSettings, + type AblationExplanation, + type AblationModelInput, + type CellOutcome, + type CellRuleInput, + type CellRuleName, + type SettleInput, +} from "../../rules/index.ts"; +import type { HypothesisMetric, Json } from "../../vocab/index.ts"; +import { formalRun, type RunOutcome } from "../formal-run.ts"; +import { ablationOf, aliveCells, lastAblationImpl } from "../next-step.ts"; +import { SEED_VERSIONS } from "./round.ts"; +import type { Study } from "../study.ts"; + +// ⑮ 选格、⑯ 消融运行、⑰ 结算与押的因子、⑱ 轮末解封（任务书 17 第 2.2 节）。选格、结算和押的因子各写一条记录 +// （结算和押的因子同一个事务），写下之前被打断就从账本重新算一遍；写下之后恢复时照原样用，不重算（方法.md 第九部分）。 + +/** + * ⑮ 选格（方法.md 6.7）：基线、完整分数、方向、已知最好、ε、λ、因子、各条解释，加余额、每批上限、u、衰减和种子， + * 交给步骤清单里 selectCells 那一步的规则，写一条选格决定。种子用决定的版本 2（选批已用版本 1，方法.md 第九部分）。 + */ +export function selectCells(s: Study, round: number): void { + const board = new Board(s.ledger.records()); + const { config, steps } = s.frozen; + const seed = decisionSeed({ studyId: config.studyId, round, version: SEED_VERSIONS.cells }); + const input: CellRuleInput = { + ...modelInput(board, s, round), + balance: runBalance(config.runs, board.runStates()).balance, + batchCap: config.batchLimit, + u: config.flow.knowledgeUse, + decay: config.flow.decay, + seed, + }; + const rule = steps.round.find((x) => x.id === "selectCells")!.rule!.name as CellRuleName; + const { rule: used, ...output } = runCellRule(rule, input); + // SAFETY: rules 是纯计算，输入输出都是纯 Json 值；账本在写入时校验 Json。 + s.ledger.append("decision", { + decision: "cells", + round, + rule: used.name, + ruleVersion: used.version, + params: output.params, + seed, + input: input as unknown as Json, + output: output as unknown as Json, + }); +} + +/** ⑯ 消融运行：用核验过的这一格快照，复用正式运行的登记、扣次、判分入口。完整代码和基线的分数取本批已有的运行结果，不重跑。 */ +export function ablationRun(s: Study, round: number, cell: string, run: string): Promise { + const board = new Board(s.ledger.records()); + return formalRun(s, { run, purpose: "ablation", round, object: cell }, board.code(cell).version, ablationMetrics(board, splitCell(cell).candidate)); +} + +/** 消融格正式运行登记的指标：照被采纳候选的冻结协议取；消融格里假设专用指标缺失一律只记缺失、不作废这一格（方法.md 第五部分"指标"）。 */ +export function ablationMetrics(board: Board, candidate: string): HypothesisMetric[] { + return protocolMetrics(board, board.protocol(candidate)).map((m) => ({ ...m, missing: "recordMissing" })); +} + +/** + * ⑰ 结算（方法.md 4.3 ⑰、6.6）：选中的格全部有结果、"跑不起来"或"不跑"事件后，照 SettleInput 拼输入， + * 写一条结算记录；押的因子（6.8）在同一个事务里写成种类 bet 的决定。 + * "消融中发现"在 settleAblation 的输出里：materials 的 Board.baselineOf 据此把那一格的代码当成下一轮的基线，不另写采纳记录。 + */ +export function settle(s: Study, round: number): void { + const board = new Board(s.ledger.records()); + const { candidate, keys } = ablationOf(board, round)!; + const events = s.ledger.records().flatMap((r) => (r.kind === "event" ? [r.content] : [])); + const model = modelInput(board, s, round); + const input: SettleInput = { ...model, cells: keys.map((key) => ({ factor: key, outcome: outcomeOf(board, events, round, candidate, key) })) }; + const output = settleAblation(input); + const betInput = { ...model, cells: input.cells }; + const bet = chooseBet(betInput); + // SAFETY: rules 是纯计算，结算和决定的输入输出都是纯 Json 值；账本在写入时校验 Json。 + s.ledger.transaction(() => { + s.ledger.append("settlement", { round, result: { candidate, input, output } as unknown as Json }); + s.ledger.append("decision", { + decision: "bet", + round, + rule: "chooseBet", + ruleVersion: 1, + params: { direction: model.direction, epsilon: model.epsilon, knownBest: model.knownBest, lambda: model.lambda }, + seed: null, + input: betInput as unknown as Json, + output: bet as unknown as Json, + }); + }); +} + +/** 一格的结局（方法.md 6.6"一格的几种结局"）：没测、跑不起来、证据不足、一起关、有效分数。 */ +function outcomeOf(board: Board, events: readonly Content<"event">[], round: number, candidate: string, key: string): CellOutcome { + if (!board.decision("cells", round)!.output.chosen.includes(key)) return { kind: "untested" }; + const cell = cellId(candidate, key); + const impl = lastAblationImpl(board, cell)!; + if (impl.content.cannotRun !== null) return { kind: "cannotRun" }; + if (events.some((e) => e.event === CELL_NOT_RUN && e.objects.includes(cell))) return { kind: "insufficient" }; + const off = [key, ...impl.content.alsoOff]; + // 几格实际关掉的因子完全相同时，沿用同设置第一格的运行结果（不重跑，方法.md 6.6"一起关"）。 + const alive = aliveCells(board, events, round); + const sameAs = new Map(sharedSettings(alive.map((x) => ({ factor: splitCell(x.cell).key, off: x.off }))).map((x) => [x.factor, x.sameAs])); + const first = sameAs.get(key); + const ranCell = first === null || first === undefined ? cell : cellId(candidate, first); + const run = board.runs().filter((r) => r.plan.purpose === "ablation" && r.plan.round === round && r.plan.object === ranCell).at(-1)!; + const score = validScore(run.result!.judged); + if (off.length > 1) return { kind: "joint", off, score }; + return score === null ? { kind: "insufficient" } : { kind: "valid", score }; +} + +/** + * ⑱ 轮末解封（方法.md 4.3 ⑱）：本轮选批选中的每个候选（跑过的、入选了却没跑成的）解封作者自评， + * 全部写在同一个事务里。只在收尾阶段关闭之后走到这里（"下一步做什么"保证），提前写会泄露。 + * "入选了却没跑成的候选离开候选池"不写记录：Board.carried 只留从没进过任何一批的候选。 + */ +export function unsealChosen(s: Study, round: number): void { + const board = new Board(s.ledger.records()); + const records = s.ledger.records(); + const chosen = board.decision("batch", round)!.output.chosen; + s.ledger.transaction(() => { + for (const candidate of chosen) { + if (records.some((r) => r.kind === "unseal" && r.content.candidate === candidate)) continue; + const seals = board.proposalsOf(candidate).map((id) => { + const seq = board.proposal(id).seq; + const seal = records.find((r) => r.kind === "seal" && r.content.proposal === seq); + if (seal === undefined) throw new Error(`提案 ${id}（记录 ${seq}）没有封存记录：提案的作者自评在写入提案时一并封存`); + return seal.seq; + }); + s.ledger.append("unseal", { candidate, seals }); + } + }); +} + +/** 成分模型的输入（方法.md 6.7）：选格、结算和押的因子共用。完整代码和本批基线的分数取本批已有的运行结果。 */ +function modelInput(board: Board, s: Study, round: number): AblationModelInput { + const { candidate, keys } = ablationOf(board, round)!; + const { config, facts } = s.frozen; + return { + baseline: board.baselineScore(board.baselineOf(round)), + full: board.baselineScore(candidate), + direction: facts.finalMetric.direction, + knownBest: facts.knownBest, + epsilon: facts.epsilon, + lambda: config.flow.openReserve, + factors: keys, + explanations: explanationsOf(board, round), + }; +} + +/** 这一轮已冻结的各条解释：作者的第 i 条（materials 的 explanationId），每条对每一格给的三个数。 */ +function explanationsOf(board: Board, round: number): AblationExplanation[] { + return board + .published("explanation") + .filter((x) => x.round === round) + .flatMap((x) => x.content.explanations.map((e, i) => ({ id: explanationId(round, x.author, i + 1), author: x.author, cells: e.cells! }))); +} diff --git a/src/flow/steps/round.ts b/src/flow/steps/round.ts index 9117273..0678008 100644 --- a/src/flow/steps/round.ts +++ b/src/flow/steps/round.ts @@ -14,6 +14,7 @@ import type { Study } from "../study.ts"; * 两次抽取不会用同一串随机数。选格由 flow 第 4 部分接上。 */ const SEED_VERSIONS = { batch: 1, cells: 2 } as const; +export { SEED_VERSIONS }; /** 一对提案（或提案与留池候选）的 Jev 判断，原样存进判重合并决定的输入。 */ export type SamePair = { a: string; b: string; judgment: JevJudgment }; @@ -38,6 +39,7 @@ export async function dedupe(s: Study, round: number): Promise { const pairs: SamePair[] = []; for (const [a, b] of asks) pairs.push({ a, b, judgment: await judgeSameChange(model, factorsOf.get(a)!, factorsOf.get(b)!) }); const output = groupSameChange(round, proposals.map((p) => p.item!), carried, pairs); + // SAFETY: JevJudgment 和 DedupOutput 都是纯 Json 值；账本在写入时校验 Json。 s.ledger.append("decision", { decision: "dedup", round, @@ -97,6 +99,7 @@ export function freezeProtocols(s: Study, round: number): void { answer.kind === "response" ? protocolBody({ kind: "response", content: answer.content, original: board.proposal(proposals[0]!).content }) : protocolBody({ kind: "unification", content: answer.content }); + // SAFETY: FrozenProtocol 是纯 Json 值；账本在写入时校验 Json。 s.ledger.append("protocol", { candidate, protocol: freezeProtocol(body, candidate, round, baseline, proposals) as unknown as Json }); } }); @@ -125,6 +128,7 @@ export function selectBatch(s: Study, round: number): void { }; const rule = steps.round.find((x) => x.id === "selectBatch")!.rule!.name as BatchRuleName; const { rule: used, ...output } = runBatchRule(rule, input); + // SAFETY: rules 是纯计算，输入输出都是纯 Json 值；账本在写入时校验 Json。 s.ledger.append("decision", { decision: "batch", round, diff --git a/src/flow/task-code.ts b/src/flow/task-code.ts index 200e353..6e6385d 100644 --- a/src/flow/task-code.ts +++ b/src/flow/task-code.ts @@ -1,7 +1,7 @@ import { join } from "node:path"; import type { Content } from "../ledger/index.ts"; import { codeHunks } from "../jev/index.ts"; -import { Board, protocolMetrics, type CodeChanges } from "../materials/index.ts"; +import { Board, protocolMetrics, splitCell, type CodeChanges } from "../materials/index.ts"; import type { KnownOf } from "../submissions/index.ts"; import { lightCheck, protectedChanges, readLightCheck } from "../tasks/index.ts"; import type { Json, Submission, SubmissionKind } from "../vocab/index.ts"; @@ -10,10 +10,23 @@ import type { TaskRef } from "./next-step.ts"; import { callPaths, containerName, freshDir, workspaceRepo } from "./paths.ts"; import type { Study } from "./study.ts"; -/** 实现取本批基线；修复取上次提交。核验、修复的材料差异始终相对本批基线。 */ +/** + * 实现取本批基线；修复取上次提交。核验、修复的材料差异始终相对本批基线。 + * 消融实现第一次取被采纳的完整代码，重做取上一次交的这一格代码（materials 的消融实现材料就是这样写的）； + * 消融核验的材料差异是完整代码到这一格代码（方法.md 4.3 ⑯）。 + */ export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes: CodeChanges | null; startVersion: string | null } { - if (t.kind !== "implementation" && t.kind !== "repair" && t.kind !== "verification") return { changes: null, startVersion: null }; const board = new Board(s.ledger.records()); + if (t.kind === "ablationImplementation") { + const { candidate } = splitCell(t.object!); + const hasCellCode = s.ledger.records().some((r) => r.kind === "code" && r.content.object === t.object); + return { changes: null, startVersion: hasCellCode ? board.code(t.object!).version : board.code(candidate).version }; + } + if (t.kind === "ablationVerification") { + const { candidate } = splitCell(t.object!); + return { changes: compareVersions(workspaceRepo(s.dir), board.code(candidate).version, board.code(t.object!).version), startVersion: null }; + } + if (t.kind !== "implementation" && t.kind !== "repair" && t.kind !== "verification") return { changes: null, startVersion: null }; const baseline = board.code(board.baselineOf(board.stage(t.stageId).round)).version; if (t.kind === "implementation") return { changes: null, startVersion: baseline }; const version = board.code(t.object!).version; @@ -34,6 +47,20 @@ export function prepareSubmission( const board = new Board(s.ledger.records()); if (t.kind === "proposal") return { known: { accepted: others }, code: null }; if (t.kind === "response") return { known: { original: board.proposal(board.proposalsOf(t.object!)[0]!).content }, code: null }; + if (t.kind === "ablationImplementation") { + const impl = content as Submission<"ablationImplementation">; + // 交了"这一格跑不起来"：known.code 传 null，不存版本（任务书 17 第六部分），这一格不核验、不运行、不扣次。 + if (impl.cannotRun !== null) return { known: { code: null }, code: null }; + const saved = saveVersion(workspaceRepo(s.dir), workDir, s.frozen.facts.unversionedNames); + if (!saved.saved) return { known: { code: saved }, code: null }; + const { candidate } = splitCell(t.object!); + const changes = compareVersions(workspaceRepo(s.dir), board.code(candidate).version, saved.version); + return { + known: { code: { saved: true, protectedFiles: protectedChanges(s.task, changes.files.map((f) => f.path)) } }, + // 消融格的代码不记位置表：用完整代码那次实现交的位置表（materials 的 LocationTable 注释）。 + code: { object: t.object!, version: saved.version, locations: null }, + }; + } if (t.kind !== "implementation" && t.kind !== "repair") return { known: null, code: null }; const saved = saveVersion(workspaceRepo(s.dir), workDir, s.frozen.facts.unversionedNames); if (!saved.saved) return { known: { code: saved }, code: null }; diff --git a/tests/flow-ablation-steps.test.ts b/tests/flow-ablation-steps.test.ts new file mode 100644 index 0000000..447ff87 --- /dev/null +++ b/tests/flow-ablation-steps.test.ts @@ -0,0 +1,207 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { test, type TestContext } from "node:test"; +import { stepList } from "../src/config/index.ts"; +import { fullConfig as checkedFull } from "../src/flow/frozen.ts"; +import { selectCells, settle, unsealChosen } from "../src/flow/steps/ablation.ts"; +import type { Study } from "../src/flow/study.ts"; +import { Ledger } from "../src/ledger/index.ts"; +import { Board } from "../src/materials/index.ts"; +import { chooseBet, decisionSeed, runCellRule, settleAblation, type AblationModelInput, type CellRuleInput, type SettleInput } from "../src/rules/index.ts"; +import { checkTaskFacts } from "../src/tasks/index.ts"; +import { fullConfig } from "./config-helpers.ts"; +import { definition, exited, proposal, tempDir } from "./ledger-helpers.ts"; +import { judged, r, Writer } from "./materials-helpers.ts"; + +// 选格、结算与押的因子、轮末解封（任务书 17 第 2.2 节）：在真实账本里写到上一步做完，跑这一步，看写下的记录。 +// 预期值按条文手拼：选格输入照 rules 的 CellRuleInput 逐项拼，结算的各格结局照方法.md 6.6 的情形拼； +// 输出用 rules 的函数当对照（同选批测试的写法，rules 自己有独立预期值的测试）。 + +const config = fullConfig(); +const facts = checkTaskFacts(JSON.parse(readFileSync(join(import.meta.dirname, "..", "tasks", "circle_packing", "task.json"), "utf8"))); + +function study(t: TestContext): { s: Study; w: Writer } { + const ledger = Ledger.open(join(tempDir(t), "ledger.db")); + t.after(() => ledger.close()); + const frozen = { config: checkedFull(config), steps: stepList(config), facts, prompts: null, image: "", seed: 1 }; + return { s: { ledger, frozen } as unknown as Study, w: new Writer(ledger) }; +} + +/** 基线 0.96、被采纳候选 r1-c1（两个因子）完整代码 1.25。 */ +function adoptedBase(w: Writer): void { + w.l.append("code", { object: "base", version: "v-base", locations: null }); + w.run("base-1", "baseline", 0, null, judged(0.96, "main")); + w.l.append("code", { object: "r1-c1", version: "v-full", locations: null }); + w.run("r1-forward-r1-c1-1", "forward", 1, "r1-c1", judged(1.25, "main")); + w.protocol("r1-c1", { factors: [{ name: "因子一" }, { name: "因子二" }], safeguards: [], metrics: { defined: [], reused: [] }, body: "正文", clauses: [], round: 1, baseline: "base", proposals: ["r1-a1-1"] } as never); + w.decision("adopt", 1, null, { adopted: "r1-c1" }); +} + +const EXPLANATIONS = [ + { id: "r1-a1-x1", author: "a1", cells: [{ factor: "p1", range: r(1.0, 1.1, 1.2) }, { factor: "p2", range: r(1.15, 1.2, 1.24) }] }, + { id: "r1-a2-x1", author: "a2", cells: [{ factor: "p1", range: r(0.9, 1.05, 1.2) }, { factor: "p2", range: r(1.1, 1.18, 1.25) }] }, +] as const; + +/** 解释阶段关闭：a1、a2 各交一条，对两格都给了三个数。 */ +function explain(w: Writer): void { + w.open("r1-explanation", "explanation", 1, ["a1", "a2"]); + for (const e of EXPLANATIONS) { + w.sub("r1-explanation", e.author, "explanation", null, null, { explanations: [{ body: `机制（${e.author}）`, cells: e.cells.map((c) => ({ factor: c.factor, range: c.range })) }] }); + } + w.close("r1-explanation"); +} + +/** 成分模型的输入：选格、结算和押的因子共用的那一份。 */ +function modelInput(): AblationModelInput { + return { baseline: 0.96, full: 1.25, direction: "maximize", knownBest: facts.knownBest, epsilon: facts.epsilon, lambda: 0.1, factors: ["p1", "p2"], explanations: EXPLANATIONS.map((e) => ({ id: e.id, author: e.author, cells: e.cells.map((c) => ({ factor: c.factor, range: c.range })) })) }; +} + +test("选格：输入照 CellRuleInput 拼（基线、完整分数、各条解释、余额从运行记录算、种子用选格的版本 2），规则取步骤清单", (t) => { + const { s, w } = study(t); + adoptedBase(w); + explain(w); + selectCells(s, 1); + const record = s.ledger.records().find((x) => x.kind === "decision" && x.content.decision === "cells")!; + const seed = decisionSeed({ studyId: "circle-full-1", round: 1, version: 2 }); + const input: CellRuleInput = { + ...modelInput(), + // 13 次里基线和正向各确认启动 1 次，余额 11。 + balance: 11, + batchCap: 2, + u: 0.5, + decay: 0.5, + seed, + }; + const { rule: used, ...output } = runCellRule("valueMinusCost", input); + assert.deepEqual(record.content, { decision: "cells", round: 1, rule: used.name, ruleVersion: used.version, params: output.params, seed, input, output }); +}); + +/** 一次消融实现阶段：subs 是 [格, cannotRun, alsoOff]；cannotRun 为 null 表示交了代码。 */ +function impl(w: Writer, n: number, subs: Array<[string, string | null, string[]?]>): void { + w.open(`r1-ablationImplementation-${n}`, "ablationImplementation", 1, subs.map(([cell]) => `experimenter-${cell}`)); + for (const [cell, cannotRun, alsoOff] of subs) { + w.sub(`r1-ablationImplementation-${n}`, `experimenter-${cell}`, "ablationImplementation", cell, null, { alsoOff: alsoOff ?? [], cannotRun, concerns: "无" }); + } + w.close(`r1-ablationImplementation-${n}`); +} + +/** 结算一次：拼出 SettleInput，跑 settle，核对 settlement 和 bet 两条记录。 */ +function settled(t: TestContext, prepare: (w: Writer) => void, cells: SettleInput["cells"]): void { + const { s, w } = study(t); + adoptedBase(w); + explain(w); + w.decision("cells", 1, null, { chosen: ["p1", "p2"] }); + prepare(w); + settle(s, 1); + const input: SettleInput = { ...modelInput(), cells }; + const betInput = { ...modelInput(), cells }; + const settlement = s.ledger.records().find((x) => x.kind === "settlement")!; + assert.deepEqual(settlement.content, { round: 1, result: { candidate: "r1-c1", input, output: settleAblation(input) } }); + const bet = s.ledger.records().find((x) => x.kind === "decision" && x.content.decision === "bet")!; + assert.deepEqual(bet.content, { + decision: "bet", + round: 1, + rule: "chooseBet", + ruleVersion: 1, + params: { direction: "maximize", epsilon: facts.epsilon, knownBest: facts.knownBest, lambda: 0.1 }, + seed: null, + input: betInput, + output: chooseBet(betInput), + }); +} + +test("结算：有效分数、没测、跑不起来、超时记证据不足，各按 6.6 的结局拼", (t) => { + settled( + t, + (w) => { + impl(w, 1, [["r1-c1-p1", null], ["r1-c1-p2", null]]); + w.run("r1-ablation-r1-c1-p1-1", "ablation", 1, "r1-c1-p1", judged(1.1, "main")); + // p2 启动了但跑崩： judged 为 null，记证据不足（扣了次数）。 + w.l.append("runPlanned", { run: "r1-ablation-r1-c1-p2-1", purpose: "ablation", round: 1, object: "r1-c1-p2", definition }); + w.l.append("runStarted", { run: "r1-ablation-r1-c1-p2-1" }); + w.l.append("runResult", { run: "r1-ablation-r1-c1-p2-1", process: exited, judged: null }); + }, + [ + { factor: "p1", outcome: { kind: "valid", score: 1.1 } }, + { factor: "p2", outcome: { kind: "insufficient" } }, + ], + ); +}); + +test("结算：没选中的格记没测；交了跑不起来的记跑不起来", (t) => { + const { s, w } = study(t); + adoptedBase(w); + explain(w); + w.decision("cells", 1, null, { chosen: ["p1"] }); + impl(w, 1, [["r1-c1-p1", "离了 p1 跑不起来"]]); + settle(s, 1); + const input: SettleInput = { + ...modelInput(), + cells: [ + { factor: "p1", outcome: { kind: "cannotRun" } }, + { factor: "p2", outcome: { kind: "untested" } }, + ], + }; + const settlement = s.ledger.records().find((x) => x.kind === "settlement")!; + assert.deepEqual(settlement.content, { round: 1, result: { candidate: "r1-c1", input, output: settleAblation(input) } }); +}); + +test("结算：核验不通过、次数用完有事件的记证据不足；设置相同的两格都记一起关，沿用第一格的分数", (t) => { + settled( + t, + (w) => { + impl(w, 1, [["r1-c1-p1", null, ["p2"]]]); + w.run("r1-ablation-r1-c1-p1-1", "ablation", 1, "r1-c1-p1", judged(1.05, "main")); + impl(w, 2, [["r1-c1-p2", null]]); + w.l.append("event", { event: "cellNotRun", reason: "第 2 次核验仍不通过，修复次数上限 1 已用完", objects: ["r1-c1-p2"] }); + }, + [ + { factor: "p1", outcome: { kind: "joint", off: ["p1", "p2"], score: 1.05 } }, + { factor: "p2", outcome: { kind: "insufficient" } }, + ], + ); +}); + +test("结算：设置相同的两格都记一起关，沿用第一格的分数", (t) => { + settled( + t, + (w) => { + impl(w, 1, [["r1-c1-p1", null, ["p2"]]]); + w.run("r1-ablation-r1-c1-p1-1", "ablation", 1, "r1-c1-p1", judged(1.05, "main")); + impl(w, 2, [["r1-c1-p2", null, ["p1"]]]); + }, + [ + { factor: "p1", outcome: { kind: "joint", off: ["p1", "p2"], score: 1.05 } }, + { factor: "p2", outcome: { kind: "joint", off: ["p2", "p1"], score: 1.05 } }, + ], + ); +}); + +test("轮末解封：解封本轮选中的候选的全部作者自评（取最后一版的封存），没选中的继续封存", (t) => { + const { s, w } = study(t); + w.open("r1-proposal", "proposal", 1, ["a1", "a2"]); + w.sub("r1-proposal", "a1", "proposal", null, "r1-a1-1", proposal(r(1.0, 1.1, 1.2), "第一版")); + const v2 = s.ledger.append("submission", { + stageId: "r1-proposal", + author: "a1", + kind: "proposal", + target: null, + item: "r1-a1-1", + version: 2, + content: proposal(r(1.05, 1.15, 1.25), "第二版"), + jev: null, + } as never); + w.sub("r1-proposal", "a2", "proposal", null, "r1-a2-1", proposal(r(0.9, 1.0, 1.1))); + w.close("r1-proposal"); + w.decision("dedup", 1, null, { fresh: [{ candidate: "r1-c1", proposals: ["r1-a1-1"] }, { candidate: "r1-c2", proposals: ["r1-a2-1"] }], carried: [] }); + w.decision("batch", 1, null, { chosen: ["r1-c1"] }); + unsealChosen(s, 1); + const unseals = s.ledger.records().filter((x) => x.kind === "unseal"); + assert.equal(unseals.length, 1); + assert.equal(unseals[0]!.content.candidate, "r1-c1"); + // 解封的是最后一版（v2）的封存。 + const seal = s.ledger.records().find((x) => x.kind === "seal" && x.content.proposal === v2.seq)!; + assert.deepEqual(unseals[0]!.content.seals, [seal.seq]); + assert.deepEqual(new Board(s.ledger.records()).unsealed(), [{ candidate: "r1-c1", author: "a1", range: r(1.05, 1.15, 1.25) }]); +}); diff --git a/tests/flow-next-step.test.ts b/tests/flow-next-step.test.ts index 8723baa..4eebe8b 100644 --- a/tests/flow-next-step.test.ts +++ b/tests/flow-next-step.test.ts @@ -266,7 +266,9 @@ function roundOne(last: "proposal" | "dedupe" | "firstPrediction" | "critique" | ]), response: () => stage(book, "r1-response", "response", [["researcher-1", "response", "r1-c1", null], ["researcher-2", "unification", "r1-c2", null]]), freezeProtocol: () => { - for (const candidate of C) book.add("protocol", { candidate, protocol: { round: 1, baseline: "base", proposals: [] } }); + // r1-c1 一个因子（采纳它不做消融），r1-c2 两个因子（采纳它走 ⑬–⑰）。 + book.add("protocol", { candidate: "r1-c1", protocol: { round: 1, baseline: "base", proposals: [], factors: [{}] } }); + book.add("protocol", { candidate: "r1-c2", protocol: { round: 1, baseline: "base", proposals: [], factors: [{}, {}] } }); }, secondPrediction: () => stage(book, "r1-secondPrediction", "secondPrediction", R.flatMap((a) => C.map((c) => [a, "secondPrediction", c, null] as [string, SubmissionKind, string, null]))), selectBatch: () => book.add("decision", { decision: "batch", round: 1, rule: "bestExpected", ruleVersion: 1, params: null, seed: "s", input: null, output: { chosen: ["r1-c1"] } }), @@ -494,12 +496,190 @@ test("正向串行：上次未启动起新编号；已登记未结束必须先 book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: null } }); const after = new Board(book.records); assert.ok(after.runs().filter((r) => r.plan.purpose === "forward").every((r) => after.runPublic(r))); - assert.deepEqual(nextStep(book.records), { do: "halt", reason: "消融还没写（任务书 17）", objects: [] }); + // 没有采纳：不开解释阶段，直接开收尾（方法.md 4.3 ⑫）。 + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-wrapUp", stage: "wrapUp", round: 1, roster: R }); }); -test("采纳只有一个因子的候选也统一在 listCells 停下，不擅自开收尾", () => { +test("采纳只有一个因子的候选：不做消融，直接开收尾（方法.md 4.3 ⑫）", () => { const book = firstVerification(batchTwo(1), true, true); for (const c of C) recordForward(book, c, 1, "valid"); book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: C[0]! } }); - assert.deepEqual(nextStep(book.records), { do: "halt", reason: "消融还没写（任务书 17）", objects: [] }); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-wrapUp", stage: "wrapUp", round: 1, roster: R }); +}); + +// --------------------------------------------------------------------------- +// ⑬–⑱：消融到收尾、轮末解封、下一轮和研究结束（任务书 17 第 2.2、2.3 节）。预期动作手写，不调模型、Jev 或 Docker。 +// --------------------------------------------------------------------------- + +const ADOPTED = "r1-c2"; // 两个因子：p1、p2 +const CELL1 = "r1-c2-p1"; +const CELL2 = "r1-c2-p2"; + +/** 采纳了两个因子的 r1-c2。 */ +function adoptedBook(): Book { + const book = firstVerification(batchTwo(1), true, true); + for (const c of C) recordForward(book, c, 1, "valid"); + return book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: ADOPTED } }); +} + +/** 解释阶段：开、三人各交一份、关。 */ +function explained(book: Book): Book { + return stage(book, "r1-explanation", "explanation", R.map((a) => [a, "explanation", null, null])); +} + +/** 选格决定。 */ +function cellsChosen(book: Book, chosen: string[]): Book { + return book.add("decision", { decision: "cells", round: 1, rule: "valueMinusCost", ruleVersion: 1, params: null, seed: "s2", input: null, output: { chosen } }); +} + +/** 一次消融实现阶段：subs 是 [格, cannotRun, alsoOff]。 */ +function implStage(book: Book, n: number, subs: Array<[string, string | null, string[]?]>): Book { + const stageId = `r1-ablationImplementation-${n}`; + book.open(stageId, "ablationImplementation", 1, subs.map(([cell]) => `experimenter-${cell}`)); + for (const [cell, cannotRun, alsoOff] of subs) { + book.add("submission", { stageId, author: `experimenter-${cell}`, kind: "ablationImplementation", target: cell, item: null, version: 1, content: { alsoOff: alsoOff ?? [], cannotRun, concerns: "无" }, jev: null }); + } + return book.add("stageClose", { stageId }); +} + +/** 一次消融核验阶段：subs 是 [格, [因子, 关没关]]。 */ +function verStage(book: Book, n: number, subs: Array<[string, Array<[string, boolean]>]>): Book { + const stageId = `r1-ablationVerification-${n}`; + book.open(stageId, "ablationVerification", 1, subs.map(([cell]) => `verifier-${cell}`)); + for (const [cell, offs] of subs) { + book.add("submission", { stageId, author: `verifier-${cell}`, kind: "ablationVerification", target: cell, item: null, version: 1, content: { factors: offs.map(([factor, off]) => ({ factor, off, where: "initial.py:1" })) }, jev: null }); + } + return book.add("stageClose", { stageId }); +} + +/** 一次消融运行：没启动，或启动了、判分算数。 */ +function recordAblation(book: Book, cell: string, n: number, ending: "valid" | "notStarted"): Book { + const run = `r1-ablation-${cell}-${n}`; + book.add("runPlanned", { run, purpose: "ablation", round: 1, object: cell, definition }); + if (ending === "notStarted") return book.add("runResult", { run, process: { ending, reason: "Docker 未启动" }, judged: null }); + book.add("runStarted", { run }); + return book.add("runResult", { run, process: exited, judged: judged(1.2, "main") }); +} + +test("采纳多因子改法：先开解释阶段，每人一个任务、没有对象；关闭后选格", () => { + const book = adoptedBook(); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-explanation", stage: "explanation", round: 1, roster: R }); + book.open("r1-explanation", "explanation", 1, R); + assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-explanation", tasks: R.map((actor) => ({ actor, kind: "explanation", object: null })) }); + for (const a of R) book.sub("r1-explanation", a, "explanation", null, null); + assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-explanation" }); + book.add("stageClose", { stageId: "r1-explanation" }); + assert.deepEqual(nextStep(book.records), { do: "selectCells", round: 1 }); +}); + +test("选格做完：每格固定一位实验员；交了跑不起来的格不核验、不运行，没启动的下次起新编号", () => { + const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-ablationImplementation-1", stage: "ablationImplementation", round: 1, roster: [`experimenter-${CELL1}`, `experimenter-${CELL2}`] }); + implStage(book, 1, [[CELL1, null], [CELL2, "离了 p2 跑不起来"]]); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-ablationVerification-1", stage: "ablationVerification", round: 1, roster: [`verifier-${CELL1}`] }); + verStage(book, 1, [[CELL1, [["p1", true], ["p2", false]]]]); + assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL1, run: `r1-ablation-${CELL1}-1` }); + recordAblation(book, CELL1, 1, "notStarted"); + assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL1, run: `r1-ablation-${CELL1}-2` }); + recordAblation(book, CELL1, 2, "valid"); + assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 }); +}); + +test("核验不通过：退回原实验员重做，只重验重做的格；次数用完写不跑事件，不登记运行", () => { + const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]); + implStage(book, 1, [[CELL1, null], [CELL2, null]]); + verStage(book, 1, [[CELL1, [["p1", false], ["p2", false]]], [CELL2, [["p1", false], ["p2", true]]]]); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-ablationImplementation-2", stage: "ablationImplementation", round: 1, roster: [`experimenter-${CELL1}`] }); + implStage(book, 2, [[CELL1, null]]); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-ablationVerification-2", stage: "ablationVerification", round: 1, roster: [`verifier-${CELL1}`] }); + verStage(book, 2, [[CELL1, [["p1", false], ["p2", false]]]]); + const next = nextStep(book.records); + assert.equal(next.do, "cellNotRun"); + if (next.do !== "cellNotRun") throw new Error("应写不跑事件"); + assert.equal(next.cell, CELL1); + assert.match(next.reason, /第 2 次核验仍不通过，修复次数上限 1 已用完/); + book.add("event", { event: "cellNotRun", reason: next.reason, objects: [next.cell] }); + // p1 不跑不扣次；只跑 p2。 + assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL2, run: `r1-ablation-${CELL2}-1` }); +}); + +test("几格实际关掉的因子完全相同：沿用第一格，不重复跑", () => { + const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]); + implStage(book, 1, [[CELL1, null, ["p2"]], [CELL2, null, ["p1"]]]); + verStage(book, 1, [[CELL1, [["p1", true], ["p2", true]]], [CELL2, [["p1", true], ["p2", true]]]]); + // 两格实际关的都是 {p1, p2}：只跑第一格。 + assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL1, run: `r1-ablation-${CELL1}-1` }); + recordAblation(book, CELL1, 1, "valid"); + assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 }); +}); + +test("全部交跑不起来：不核验、不运行，直接结算", () => { + const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]); + implStage(book, 1, [[CELL1, "跑不起来一"], [CELL2, "跑不起来二"]]); + assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 }); +}); + +/** 结算和押的因子写完、收尾关闭、解封也写完的账本（余额还有 8 次）。 */ +function afterWrapUp(): Book { + const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]); + implStage(book, 1, [[CELL1, null], [CELL2, null]]); + verStage(book, 1, [[CELL1, [["p1", true], ["p2", false]]], [CELL2, [["p1", false], ["p2", true]]]]); + recordAblation(book, CELL1, 1, "valid"); + recordAblation(book, CELL2, 1, "valid"); + book.add("settlement", { round: 1, result: { candidate: ADOPTED, input: null, output: null } }); + book.add("decision", { decision: "bet", round: 1, rule: "chooseBet", ruleVersion: 1, params: null, seed: null, input: null, output: null }); + return stage(book, "r1-wrapUp", "wrapUp", R.map((a) => [a, "wrapUp", null, null])); +} + +test("结算和押的因子写完：开收尾；收尾关闭后解封本轮选中的全部候选，然后开下一轮", () => { + const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]); + implStage(book, 1, [[CELL1, null], [CELL2, null]]); + verStage(book, 1, [[CELL1, [["p1", true], ["p2", false]]], [CELL2, [["p1", false], ["p2", true]]]]); + recordAblation(book, CELL1, 1, "valid"); + assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL2, run: `r1-ablation-${CELL2}-1` }); + recordAblation(book, CELL2, 1, "valid"); + assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 }); + book.add("settlement", { round: 1, result: { candidate: ADOPTED, input: null, output: null } }); + book.add("decision", { decision: "bet", round: 1, rule: "chooseBet", ruleVersion: 1, params: null, seed: null, input: null, output: null }); + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-wrapUp", stage: "wrapUp", round: 1, roster: R }); + stage(book, "r1-wrapUp", "wrapUp", R.map((a) => [a, "wrapUp", null, null])); + // 收尾关闭后：解封本轮选批选中的 r1-c1、r1-c2；还有一个没解封就继续。 + assert.deepEqual(nextStep(book.records), { do: "unsealChosen", round: 1 }); + book.add("unseal", { candidate: "r1-c1", seals: [1] }); + assert.deepEqual(nextStep(book.records), { do: "unsealChosen", round: 1 }); + book.add("unseal", { candidate: "r1-c2", seals: [2] }); + // 余额还有（已扣 5 次）：开第 2 轮。 + assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r2-proposal", stage: "proposal", round: 2, roster: R }); +}); + +test("余额为 0：不开下一轮，走到导出停下，原因写明", () => { + const book = afterWrapUp(); + book.add("unseal", { candidate: "r1-c1", seals: [1] }); + book.add("unseal", { candidate: "r1-c2", seals: [2] }); + // 已扣 5 次（基线 1、正向 2、消融 2）：把冻结配置的运行次数改成 5，余额为 0。 + const record = book.records.find((r) => r.kind === "config")!; + const study = record.content as unknown as StudyRecord; + study.config = { ...fullConfig(), runs: 5 } as unknown as Json; + assert.deepEqual(nextStep(book.records), { do: "halt", reason: "导出还没写", objects: [] }); +}); + +test("配置关掉消融、又采纳了多因子改法：先记按配置跳过，再在解释停下", () => { + const book = firstVerification(batchTwo(1), true, true); + for (const c of C) recordForward(book, c, 1, "valid"); + book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: ADOPTED } }); + // 冻结的配置改为关掉消融：配置和步骤清单一起换（步骤清单在研究开始时冻结）。 + const config = fullConfig(); + if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程"); + // 配置关掉消融时不写 cellRule（它只有开着消融才用得到）。 + config.flow.ablation = { enabled: false }; + const record = book.records.find((r) => r.kind === "config")!; + const study = record.content as unknown as StudyRecord; + study.config = config as unknown as Json; + study.steps = stepList(config) as unknown as Json; + const reason = "配置关掉了消融"; + assert.deepEqual(nextStep(book.records), { do: "skip", step: "r1-listCells", reason }); + book.add("event", { event: "skippedByConfig", reason, objects: ["r1-listCells"] }); + const next = nextStep(book.records); + assert.equal(next.do, "halt"); + assert.match((next as { reason: string }).reason, /不交三个数的解释还没写/); });