flow: 消融到收尾、轮末解封、一轮接一轮与研究结束(任务书 17 第二块)
- ⑬–⑱:listCells 从采纳决定和冻结协议现算(不写记录);explanation 有消融才开 阶段;selectCells 照 CellRuleInput 拼输入(余额当时现算、decisionSeed 版本 2), 写种类 cells 的决定,没选中的格不另写事件;消融实现/核验按第几次编号 (r<轮>-ablationImplementation-1、verification-1、-2……从账本数),重做从上一次 这一格代码接着改,次数用完写 cellNotRunEvent 不跑不扣次;cannotRun 传 null 不核验 不运行不扣次;alsoOff 用 sharedSettings 沿用同设置的第一格;消融运行一个接一个跑, 用途 ablation,登记/扣次/恢复与正向同一入口,恢复时指标缺失策略覆盖为只记缺失 (方法.md 第五部分);settle 拼 SettleInput 调 settleAblation,结算与 chooseBet 的 bet 决定写在同一事务;wrapUp 阶段关闭后,本轮选中的每个候选在同一个事务里解封 作者自评(取最后一版提案的封存)。 - 一轮接一轮:每轮开头用 runBalance 现算余额,余额为 0 不开新轮;走到 export 停下, 原因"导出还没写"。去掉 roundsNext 里 step.id !== "listCells" 的临时写法。 - 消融中发现由 materials 的 Board.baselineOf 从结算输出读出,flow 不另写采纳记录 (已核实 materials 已实现)。 - 测试 +14:下一步分支 8 条(含收尾关闭后才解封、余额 0 走到导出停、关掉消融时 解释照方法停下)、消融步骤 6 条(选格输入逐字段手拼对账、结算四种结局、同设置 沿用、解封取最后一版封存);npm run check 通过,npm test 472 条通过。 - 全新上下文审阅一轮(只看消融重做循环、登记扣次恢复、选格/结算/押的因子输入、 解封时机、下一轮与结束条件):没有会污染账本的问题;采纳一条小修(删掉未用的 导入),其余笔记照列不改。 flow 1302 → 1662 行(上限 3500),materials 未动(1930 行)。
This commit is contained in:
+20
-3
@@ -1,7 +1,7 @@
|
|||||||
import { mkdirSync } from "node:fs";
|
import { mkdirSync } from "node:fs";
|
||||||
import { resolve } from "node:path";
|
import { resolve } from "node:path";
|
||||||
import type { ResearchConfig } from "../config/index.ts";
|
import type { ResearchConfig } from "../config/index.ts";
|
||||||
import { forwardNotRunEvent } from "../materials/index.ts";
|
import { forwardNotRunEvent, cellNotRunEvent } from "../materials/index.ts";
|
||||||
import { Ledger } from "../ledger/index.ts";
|
import { Ledger } from "../ledger/index.ts";
|
||||||
import { Runner } from "../runner/index.ts";
|
import { Runner } from "../runner/index.ts";
|
||||||
import { fullConfig, readFrozen } from "./frozen.ts";
|
import { fullConfig, readFrozen } from "./frozen.ts";
|
||||||
@@ -9,6 +9,7 @@ import { EVENTS, nextStep, type Next } from "./next-step.ts";
|
|||||||
import { ledgerFile } from "./paths.ts";
|
import { ledgerFile } from "./paths.ts";
|
||||||
import { recover } from "./recover.ts";
|
import { recover } from "./recover.ts";
|
||||||
import { baselineRun, saveBaseline } from "./steps/baseline.ts";
|
import { baselineRun, saveBaseline } from "./steps/baseline.ts";
|
||||||
|
import { ablationRun, selectCells, settle, unsealChosen } from "./steps/ablation.ts";
|
||||||
import { forwardRun, adoptBatch } from "./steps/forward.ts";
|
import { forwardRun, adoptBatch } from "./steps/forward.ts";
|
||||||
import { dispatchStage } from "./steps/dispatch.ts";
|
import { dispatchStage } from "./steps/dispatch.ts";
|
||||||
import { dedupe, freezeProtocols, selectBatch } from "./steps/round.ts";
|
import { dedupe, freezeProtocols, selectBatch } from "./steps/round.ts";
|
||||||
@@ -78,8 +79,12 @@ async function doStep(s: Study, next: Exclude<Next, { do: "freezeConfig" }>): Pr
|
|||||||
saveBaseline(s);
|
saveBaseline(s);
|
||||||
return null;
|
return null;
|
||||||
case "baselineRun":
|
case "baselineRun":
|
||||||
case "forwardRun": {
|
case "forwardRun":
|
||||||
const ran = next.do === "baselineRun" ? await baselineRun(s, next.run) : await forwardRun(s, next.round, next.candidate, next.run);
|
case "ablationRun": {
|
||||||
|
const ran =
|
||||||
|
next.do === "baselineRun" ? await baselineRun(s, next.run)
|
||||||
|
: next.do === "forwardRun" ? await forwardRun(s, next.round, next.candidate, next.run)
|
||||||
|
: await ablationRun(s, next.round, next.cell, next.run);
|
||||||
if (ran === "stopped") return USER_STOPPED;
|
if (ran === "stopped") return USER_STOPPED;
|
||||||
if (ran === "done") return null;
|
if (ran === "done") return null;
|
||||||
return halt(s.ledger, `运行 ${next.run} 没有启动,名额已退还;本进程不重跑,下次启动时起新编号重跑。Docker 的报错:${ran.notStarted}`, [next.run]);
|
return halt(s.ledger, `运行 ${next.run} 没有启动,名额已退还;本进程不重跑,下次启动时起新编号重跑。Docker 的报错:${ran.notStarted}`, [next.run]);
|
||||||
@@ -113,6 +118,18 @@ async function doStep(s: Study, next: Exclude<Next, { do: "freezeConfig" }>): Pr
|
|||||||
case "adopt":
|
case "adopt":
|
||||||
adoptBatch(s, next.round);
|
adoptBatch(s, next.round);
|
||||||
return null;
|
return null;
|
||||||
|
case "selectCells":
|
||||||
|
selectCells(s, next.round);
|
||||||
|
return null;
|
||||||
|
case "cellNotRun":
|
||||||
|
s.ledger.append("event", cellNotRunEvent(next.cell, next.reason));
|
||||||
|
return null;
|
||||||
|
case "settle":
|
||||||
|
settle(s, next.round);
|
||||||
|
return null;
|
||||||
|
case "unsealChosen":
|
||||||
|
unsealChosen(s, next.round);
|
||||||
|
return null;
|
||||||
case "halt":
|
case "halt":
|
||||||
return halt(s.ledger, next.reason, next.objects);
|
return halt(s.ledger, next.reason, next.objects);
|
||||||
}
|
}
|
||||||
|
|||||||
+25
-3
@@ -1,10 +1,12 @@
|
|||||||
|
import { existsSync, readFileSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
import { runSession, type SessionTask, type SessionEnding, type SubmitAttempt, type SubmitReceipt } from "../agents/index.ts";
|
import { runSession, type SessionTask, type SessionEnding, type SubmitAttempt, type SubmitReceipt } from "../agents/index.ts";
|
||||||
import { buildMaterial, callId, queryMaterial, researcherId, type TaskMaterial } from "../materials/index.ts";
|
import { Board, buildMaterial, callId, queryMaterial, researcherId, type TaskMaterial } from "../materials/index.ts";
|
||||||
import { receive, type Incoming } from "../submissions/index.ts";
|
import { receive, type Incoming } from "../submissions/index.ts";
|
||||||
import { SUBMISSION_KINDS, type JevJudgment, type Json, type SubmissionKind } from "../vocab/index.ts";
|
import { SUBMISSION_KINDS, type JevJudgment, type Json, type SubmissionKind } from "../vocab/index.ts";
|
||||||
import { checkoutVersion } from "../workspace/index.ts";
|
import { checkoutVersion } from "../workspace/index.ts";
|
||||||
import { prepareSubmission, taskCode, lightChecker } from "./task-code.ts";
|
import { prepareSubmission, taskCode, lightChecker } from "./task-code.ts";
|
||||||
import { callPaths, freshDir, workspaceRepo } from "./paths.ts";
|
import { callPaths, freshDir, runPaths, workspaceRepo } from "./paths.ts";
|
||||||
import type { Study } from "./study.ts";
|
import type { Study } from "./study.ts";
|
||||||
|
|
||||||
// 共用动作一:派一个模型任务(任务书 12 第 2.6 节)。先写"模型调用开始"(材料整份存进去),再开会话,
|
// 共用动作一:派一个模型任务(任务书 12 第 2.6 节)。先写"模型调用开始"(材料整份存进去),再开会话,
|
||||||
@@ -41,7 +43,7 @@ export async function runModelTask<K extends SubmissionKind>(s: Study, t: ModelT
|
|||||||
prompts: s.prompts,
|
prompts: s.prompts,
|
||||||
codeRoot: paths.code,
|
codeRoot: paths.code,
|
||||||
changes,
|
changes,
|
||||||
artifacts: null,
|
artifacts: t.kind === "explanation" ? runArtifacts(s, t.stageId) : null,
|
||||||
});
|
});
|
||||||
const model = role === "researcher" ? researcherModel(s, t.actor) : s.frozen.config.models[role];
|
const model = role === "researcher" ? researcherModel(s, t.actor) : s.frozen.config.models[role];
|
||||||
// 先登记、后建目录:调用编号只在登记之后才算用掉,被杀在两者之间时下次的编号不会撞上旧目录。
|
// 先登记、后建目录:调用编号只在登记之后才算用掉,被杀在两者之间时下次的编号不会撞上旧目录。
|
||||||
@@ -84,6 +86,26 @@ export async function runModelTask<K extends SubmissionKind>(s: Study, t: ModelT
|
|||||||
return { call, ending };
|
return { call, ending };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 解释任务要的产物(materials 的 TaskRequest.artifacts):本轮每次已公开正向运行的 metrics.json、solution.json
|
||||||
|
* 和标准输出、标准错误日志(运行编号 → 文件名 → 文字)。完整代码那次的答案出自哪里,就靠这些产物判断。
|
||||||
|
*/
|
||||||
|
function runArtifacts(s: Study, stageId: string): Record<string, Record<string, string>> {
|
||||||
|
const board = new Board(s.ledger.records());
|
||||||
|
const round = board.stage(stageId).round;
|
||||||
|
const artifacts: Record<string, Record<string, string>> = {};
|
||||||
|
for (const r of board.runs()) {
|
||||||
|
if (r.plan.purpose !== "forward" || r.plan.round !== round || !board.runPublic(r)) continue;
|
||||||
|
const paths = runPaths(s.dir, r.plan.run);
|
||||||
|
const files: Record<string, string> = {};
|
||||||
|
for (const [dir, name] of [[paths.run, "metrics.json"], [paths.run, "solution.json"], [paths.logs, "stdout.log"], [paths.logs, "stderr.log"]] as const) {
|
||||||
|
if (existsSync(join(dir, name))) files[name] = readFileSync(join(dir, name), "utf8");
|
||||||
|
}
|
||||||
|
artifacts[r.plan.run] = files;
|
||||||
|
}
|
||||||
|
return artifacts;
|
||||||
|
}
|
||||||
|
|
||||||
/** 研究者用的模型:第 i 位研究者(researcherId(i))用配置里第 i 个。 */
|
/** 研究者用的模型:第 i 位研究者(researcherId(i))用配置里第 i 个。 */
|
||||||
function researcherModel(s: Study, actor: string) {
|
function researcherModel(s: Study, actor: string) {
|
||||||
const models = s.frozen.config.models.researchers;
|
const models = s.frozen.config.models.researchers;
|
||||||
|
|||||||
+141
-12
@@ -1,8 +1,8 @@
|
|||||||
import type { RoundStepId, StudyStepId } from "../config/index.ts";
|
import type { RoundStepId, StudyStepId } from "../config/index.ts";
|
||||||
import type { Content, LedgerRecord } from "../ledger/index.ts";
|
import type { Content, LedgerRecord } from "../ledger/index.ts";
|
||||||
import { Board, experimenterId, verifierId, FORWARD_NOT_RUN, ORIGINAL_BASELINE, researcherId } from "../materials/index.ts";
|
import { Board, CELL_NOT_RUN, cellId, experimenterId, factorKey, FORWARD_NOT_RUN, ORIGINAL_BASELINE, researcherId, splitCell, verifierId } from "../materials/index.ts";
|
||||||
import { compareIds } from "../rules/index.ts";
|
import { compareIds, listCells, runBalance, sharedSettings } from "../rules/index.ts";
|
||||||
import { verificationOutcome } from "../submissions/index.ts";
|
import { ablationOutcome, verificationOutcome } from "../submissions/index.ts";
|
||||||
import { SUBMISSION_KINDS, type Stage, type SubmissionKind } from "../vocab/index.ts";
|
import { SUBMISSION_KINDS, type Stage, type SubmissionKind } from "../vocab/index.ts";
|
||||||
import { readFrozen, type Frozen } from "./frozen.ts";
|
import { readFrozen, type Frozen } from "./frozen.ts";
|
||||||
|
|
||||||
@@ -58,6 +58,11 @@ export type Next =
|
|||||||
| { do: "forwardNotRun"; candidate: string; reason: string }
|
| { do: "forwardNotRun"; candidate: string; reason: string }
|
||||||
| { do: "forwardRun"; round: number; candidate: string; run: string }
|
| { do: "forwardRun"; round: number; candidate: string; run: string }
|
||||||
| { do: "adopt"; round: number }
|
| { do: "adopt"; round: number }
|
||||||
|
| { do: "selectCells"; round: number }
|
||||||
|
| { do: "cellNotRun"; cell: string; reason: string }
|
||||||
|
| { do: "ablationRun"; round: number; cell: string; run: string }
|
||||||
|
| { do: "settle"; round: number }
|
||||||
|
| { do: "unsealChosen"; round: number }
|
||||||
| { do: "halt"; reason: string; objects: string[] };
|
| { do: "halt"; reason: string; objects: string[] };
|
||||||
|
|
||||||
export function nextStep(records: readonly LedgerRecord[]): Next {
|
export function nextStep(records: readonly LedgerRecord[]): Next {
|
||||||
@@ -70,7 +75,7 @@ export function nextStep(records: readonly LedgerRecord[]): Next {
|
|||||||
step.status === "skip" ? skipNext(step.id, step.reason, view) : step.id === "rounds" ? roundsNext(view, frozen) : studyStepNext(step.id, view, frozen);
|
step.status === "skip" ? skipNext(step.id, step.reason, view) : step.id === "rounds" ? roundsNext(view, frozen) : studyStepNext(step.id, view, frozen);
|
||||||
if (next !== null) return next;
|
if (next !== null) return next;
|
||||||
}
|
}
|
||||||
throw new Error("步骤清单走完了却没有停下:rounds 一步应当停下");
|
throw new Error("步骤清单走完了却没有停下:export 一步应当停下");
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 按配置跳过的步骤 → 一条"按配置跳过"事件,写明步骤和原因。 */
|
/** 按配置跳过的步骤 → 一条"按配置跳过"事件,写明步骤和原因。 */
|
||||||
@@ -96,18 +101,21 @@ function studyStepNext(step: Exclude<StudyStepId, "rounds">, view: View, frozen:
|
|||||||
// 分配 → 第 0 轮的入口分配决定(decision,种类 entries)。
|
// 分配 → 第 0 轮的入口分配决定(decision,种类 entries)。
|
||||||
return view.board.decision("entries", 0) === undefined ? { do: "assignEntries" } : null;
|
return view.board.decision("entries", 0) === undefined ? { do: "assignEntries" } : null;
|
||||||
case "export":
|
case "export":
|
||||||
throw new Error("走到了导出:rounds 一步应当先停下");
|
// 余额用完、一轮接一轮走完后走到这里:导出还没写,停下(任务书 17 第 2.1 节)。
|
||||||
|
return { do: "halt", reason: "导出还没写", objects: [] };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 一轮接一轮(结构.md 第三部分"每一轮"):第一个没做完的轮次里第一个没做完的步骤。
|
* 一轮接一轮(结构.md 第三部分"每一轮"):第一个没做完的轮次里第一个没做完的步骤。
|
||||||
* 每一步做完的记录都带轮次,做完的轮次整轮跳过;现在走到列消融格就停下,所以循环一定会返回。
|
* 每一步做完的记录都带轮次,做完的轮次整轮跳过。余额为 0 不开下一轮(方法.md 第七部分"什么时候开下一轮"):
|
||||||
|
* 交回 null,整份研究的下一步就是 export。
|
||||||
*/
|
*/
|
||||||
function roundsNext(view: View, frozen: Frozen): Next {
|
function roundsNext(view: View, frozen: Frozen): Next | null {
|
||||||
for (let round = 1; ; round++) {
|
for (let round = 1; ; round++) {
|
||||||
|
if (runBalance(frozen.config.runs, view.board.runStates()).balance === 0) return null;
|
||||||
for (const step of frozen.steps.round) {
|
for (const step of frozen.steps.round) {
|
||||||
const next = step.status === "skip" && step.id !== "listCells" ? skipNext(stageIdOf(round, step.id), step.reason, view) : roundStepNext(step.id, round, view, frozen);
|
const next = step.status === "skip" ? skipNext(stageIdOf(round, step.id), step.reason, view) : roundStepNext(step.id, round, view, frozen);
|
||||||
if (next !== null) return next;
|
if (next !== null) return next;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -157,9 +165,43 @@ function roundStepNext(step: RoundStepId, round: number, view: View, frozen: Fro
|
|||||||
case "adopt":
|
case "adopt":
|
||||||
return view.board.decision("adopt", round) === undefined ? { do: "adopt", round } : null;
|
return view.board.decision("adopt", round) === undefined ? { do: "adopt", round } : null;
|
||||||
case "listCells":
|
case "listCells":
|
||||||
return { do: "halt", reason: "消融还没写(任务书 17)", objects: [] };
|
// 不写记录:格从采纳决定和冻结协议现算(方法.md 4.3 ⑫),格编号用 materials 的 cellId。
|
||||||
|
return null;
|
||||||
|
case "explanation": {
|
||||||
|
// ⑭:这一轮有消融(采纳了、且被采纳改法有两个以上因子)才开解释阶段;否则直接到收尾(方法.md 4.3 ⑫)。
|
||||||
|
// 解释 → 这一轮解释阶段的关闭记录。
|
||||||
|
const adopted = view.board.decision("adopt", round)?.output.adopted ?? null;
|
||||||
|
if (adopted === null || view.board.protocol(adopted).factors.length < 2) return null;
|
||||||
|
if (frozen.steps.round.some((s) => s.id === "listCells" && s.status === "skip")) {
|
||||||
|
return { do: "halt", reason: "配置关掉了消融、又采纳了多因子改法:不交三个数的解释还没写(cells 只按完整流程全开实现,做步骤开关时再补)", objects: [] };
|
||||||
|
}
|
||||||
|
return stageNext(stageIdOf(round, "explanation"), "explanation", round, (roster) => roster.map((actor) => ({ actor, kind: "explanation" as const, object: null })), view, frozen, researchers(frozen));
|
||||||
|
}
|
||||||
|
case "selectCells":
|
||||||
|
// ⑮ → 本轮的选格决定(decision,种类 cells)。没选中的格不另写事件:原因都在这条决定里。
|
||||||
|
if (ablationOf(view.board, round) === null) return null;
|
||||||
|
return view.board.decision("cells", round) === undefined ? { do: "selectCells", round } : null;
|
||||||
|
case "ablationImplementation":
|
||||||
|
case "ablationVerification":
|
||||||
|
// ⑯ → 每个选中的格:交了"跑不起来",或最近一次核验通过,或重做次数用完、有"不跑"事件。
|
||||||
|
return ablationOf(view.board, round) === null ? null : ablationCellsNext(round, view, frozen);
|
||||||
|
case "ablationRun":
|
||||||
|
// ⑯ → 每个通过核验的格,最近一次消融运行有结果,而且不是"没启动"。
|
||||||
|
return ablationOf(view.board, round) === null ? null : ablationRunNext(round, view);
|
||||||
|
case "settle":
|
||||||
|
// ⑰ → 本轮的结算记录(settlement),以及同一事务里的押的因子决定(decision,种类 bet)。
|
||||||
|
if (ablationOf(view.board, round) === null) return null;
|
||||||
|
return view.board.settlement(round) === undefined || view.board.decision("bet", round) === undefined ? { do: "settle", round } : null;
|
||||||
|
case "wrapUp": {
|
||||||
|
// ⑱ → 收尾阶段的关闭记录,以及本轮选中的每个候选都有解封记录。
|
||||||
|
const stage = stageNext(stageIdOf(round, "wrapUp"), "wrapUp", round, (roster) => roster.map((actor) => ({ actor, kind: "wrapUp" as const, object: null })), view, frozen, researchers(frozen));
|
||||||
|
if (stage !== null) return stage;
|
||||||
|
// 轮末解封(方法.md 4.3 ⑱):只在收尾阶段关闭之后写,提前写会泄露。本轮全部解封写在同一个事务里。
|
||||||
|
const chosen = view.board.decision("batch", round)!.output.chosen;
|
||||||
|
return chosen.every((c) => view.of("unseal").some((u) => u.candidate === c)) ? null : { do: "unsealChosen", round };
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
throw new Error(`走到了第 ${round} 轮的 ${step}:listCells 一步应当先停下`);
|
throw new Error(`第 ${round} 轮的 ${step} 没有对应的下一步写法`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -265,12 +307,12 @@ const researchers = (frozen: Frozen): string[] => frozen.config.models.researche
|
|||||||
function assignedStage(
|
function assignedStage(
|
||||||
step: string,
|
step: string,
|
||||||
round: number,
|
round: number,
|
||||||
kind: "implementation" | "repair" | "verification",
|
kind: "implementation" | "repair" | "verification" | "ablationImplementation" | "ablationVerification",
|
||||||
candidates: readonly string[],
|
candidates: readonly string[],
|
||||||
view: View,
|
view: View,
|
||||||
frozen: Frozen,
|
frozen: Frozen,
|
||||||
): Next | null {
|
): Next | null {
|
||||||
const actorOf = kind === "verification" ? verifierId : experimenterId;
|
const actorOf = SUBMISSION_KINDS[kind].role === "verifier" ? verifierId : experimenterId;
|
||||||
const tasks = candidates.map((object) => ({ actor: actorOf(object), kind, object }));
|
const tasks = candidates.map((object) => ({ actor: actorOf(object), kind, object }));
|
||||||
return stageNext(stageIdOf(round, step), SUBMISSION_KINDS[kind].stage, round, () => tasks, view, frozen, tasks.map((t) => t.actor));
|
return stageNext(stageIdOf(round, step), SUBMISSION_KINDS[kind].stage, round, () => tasks, view, frozen, tasks.map((t) => t.actor));
|
||||||
}
|
}
|
||||||
@@ -315,6 +357,93 @@ function forwardNext(round: number, view: View): Next | null {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 这一轮做不做消融、消融哪个候选的哪几个因子(方法.md 4.3 ⑫):采纳了、且被采纳改法有两个以上因子。
|
||||||
|
* 没采纳或只采纳了单因子改法时 ⑬–⑰ 都不做:这几步不写记录,从采纳决定和冻结协议就算得出。
|
||||||
|
*/
|
||||||
|
export function ablationOf(board: Board, round: number): { candidate: string; keys: string[] } | null {
|
||||||
|
const adopted = board.decision("adopt", round)?.output.adopted ?? null;
|
||||||
|
if (adopted === null) return null;
|
||||||
|
const keys = listCells(board.protocol(adopted).factors.map((_, i) => factorKey(i)));
|
||||||
|
return keys.length < 2 ? null : { candidate: adopted, keys };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 选格决定选中要测的格(对象编号用 materials 的 cellId),按因子编号排。 */
|
||||||
|
export function chosenCells(board: Board, round: number): string[] {
|
||||||
|
const { candidate } = ablationOf(board, round)!;
|
||||||
|
return board.decision("cells", round)!.output.chosen.map((key) => cellId(candidate, key));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 一格最近一次已公开的消融实现提交(阶段关闭即公开);没有为 undefined。 */
|
||||||
|
export function lastAblationImpl(board: Board, cell: string) {
|
||||||
|
return board.published("ablationImplementation").filter((s) => s.target === cell).at(-1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 选中的格里"活着"的(没交"跑不起来"、没有"不跑"事件),以及每格实际关掉的全部因子(指定的加实验员报的一起关的)。 */
|
||||||
|
export function aliveCells(board: Board, events: readonly Content<"event">[], round: number): Array<{ cell: string; off: string[] }> {
|
||||||
|
return chosenCells(board, round)
|
||||||
|
.map((cell) => ({ cell, impl: lastAblationImpl(board, cell) }))
|
||||||
|
.filter((x) => x.impl !== undefined && x.impl.content.cannotRun === null && !events.some((e) => e.event === CELL_NOT_RUN && e.objects.includes(x.cell)))
|
||||||
|
.map((x) => ({ cell: x.cell, off: [splitCell(x.cell).key, ...x.impl!.content.alsoOff] }));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 要跑消融运行的格:活着的格里同设置的第一格(方法.md 6.6"一起关":几格实际关掉的因子完全相同就是同一种设置,
|
||||||
|
* 沿用第一格的结果,不重跑、不重复扣次、不重复记账)。
|
||||||
|
*/
|
||||||
|
export function cellsToRun(board: Board, events: readonly Content<"event">[], round: number): string[] {
|
||||||
|
const alive = aliveCells(board, events, round);
|
||||||
|
const sameAs = new Map(sharedSettings(alive.map((x) => ({ factor: splitCell(x.cell).key, off: x.off }))).map((x) => [x.factor, x.sameAs]));
|
||||||
|
return alive.filter((x) => sameAs.get(splitCell(x.cell).key) === null).map((x) => x.cell);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ⑯ 消融实现与核验(任务书 17 第 2.2 节):实现全部关闭后开核验;核验全部关闭后,没通过的退回原实验员,
|
||||||
|
* 从它上一次交的这一格代码接着改,每次只重验刚重做的格。阶段编号按"第几次"排,第几次从已开关的阶段数出来,不另存。
|
||||||
|
* 交了"跑不起来"的格不核验、不运行、不扣次;核验不通过、次数用完写"不跑"事件,不登记运行(方法.md 4.3 ⑯、6.6)。
|
||||||
|
*/
|
||||||
|
function ablationCellsNext(round: number, view: View, frozen: Frozen): Next | null {
|
||||||
|
let cells = chosenCells(view.board, round).filter((cell) => !view.events(CELL_NOT_RUN).some((e) => e.objects.includes(cell)));
|
||||||
|
for (let n = 1; cells.length > 0; n++) {
|
||||||
|
const impl = assignedStage(`ablationImplementation-${n}`, round, "ablationImplementation", cells, view, frozen);
|
||||||
|
if (impl !== null) return impl;
|
||||||
|
const runnable = cells.filter((cell) => lastAblationImpl(view.board, cell)!.content.cannotRun === null);
|
||||||
|
const failed: Array<{ cell: string; reason: string }> = [];
|
||||||
|
if (runnable.length > 0) {
|
||||||
|
const ver = assignedStage(`ablationVerification-${n}`, round, "ablationVerification", runnable, view, frozen);
|
||||||
|
if (ver !== null) return ver;
|
||||||
|
for (const s of view.board.published("ablationVerification").filter((x) => x.stageId === stageIdOf(round, `ablationVerification-${n}`))) {
|
||||||
|
const outcome = ablationOutcome(s.content, splitCell(s.target!).key, lastAblationImpl(view.board, s.target!)!.content.alsoOff);
|
||||||
|
if (!outcome.passed) {
|
||||||
|
failed.push({ cell: s.target!, reason: `第 ${n} 次核验仍不通过,修复次数上限 ${frozen.config.maxRepairs} 已用完:${JSON.stringify({ leftOn: outcome.leftOn, wronglyOff: outcome.wronglyOff })}` });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (n > frozen.config.maxRepairs) {
|
||||||
|
for (const f of failed) {
|
||||||
|
if (view.events(CELL_NOT_RUN).some((e) => e.objects.includes(f.cell))) continue;
|
||||||
|
return { do: "cellNotRun", cell: f.cell, reason: f.reason };
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
cells = failed.map((f) => f.cell);
|
||||||
|
if (cells.length === 0) return null;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** ⑯ 消融运行:按格编号顺序一次只交回一个运行(一份研究里的消融运行一个接一个跑,同正向);没启动的下次启动起新编号。 */
|
||||||
|
function ablationRunNext(round: number, view: View): Next | null {
|
||||||
|
for (const cell of cellsToRun(view.board, view.of("event"), round)) {
|
||||||
|
const runs = view.board.runs().filter((r) => r.plan.purpose === "ablation" && r.plan.round === round && r.plan.object === cell);
|
||||||
|
const last = runs.at(-1);
|
||||||
|
if (last !== undefined && last.result === null) throw new Error(`消融运行 ${last.plan.run} 登记了还没有结果:恢复入口应当已经核对过`);
|
||||||
|
if (last !== undefined && last.result!.process.ending !== "notStarted") continue;
|
||||||
|
return { do: "ablationRun", round, cell, run: `r${round}-ablation-${cell}-${runs.length + 1}` };
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/** 按种类取记录内容的小工具,加上 materials 读账本的那一份(Board)。 */
|
/** 按种类取记录内容的小工具,加上 materials 读账本的那一份(Board)。 */
|
||||||
class View {
|
class View {
|
||||||
readonly #records: readonly LedgerRecord[];
|
readonly #records: readonly LedgerRecord[];
|
||||||
|
|||||||
+6
-7
@@ -1,9 +1,10 @@
|
|||||||
import type { Content } from "../ledger/index.ts";
|
import type { Content } from "../ledger/index.ts";
|
||||||
import { Board, protocolMetrics } from "../materials/index.ts";
|
import { Board, protocolMetrics, splitCell } from "../materials/index.ts";
|
||||||
import type { HypothesisMetric } from "../vocab/index.ts";
|
import type { HypothesisMetric } from "../vocab/index.ts";
|
||||||
import { finishRun } from "./formal-run.ts";
|
import { finishRun } from "./formal-run.ts";
|
||||||
import { EVENTS } from "./next-step.ts";
|
import { EVENTS } from "./next-step.ts";
|
||||||
import { containerName } from "./paths.ts";
|
import { containerName } from "./paths.ts";
|
||||||
|
import { ablationMetrics } from "./steps/ablation.ts";
|
||||||
import type { Study } from "./study.ts";
|
import type { Study } from "./study.ts";
|
||||||
|
|
||||||
// 恢复入口第 3、4 步(任务书 12 第 2.5 节):每次启动都走这里,不分第一次启动、正常重启还是被杀后重启。
|
// 恢复入口第 3、4 步(任务书 12 第 2.5 节):每次启动都走这里,不分第一次启动、正常重启还是被杀后重启。
|
||||||
@@ -42,12 +43,10 @@ export async function recover(s: Study): Promise<{ halt: string } | "stopped" |
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 恢复和第一次运行共用原登记指标定义(引用指标也保留原来的缺失规则)。 */
|
/** 恢复和第一次运行共用原登记指标定义(引用指标也保留原来的缺失规则);消融格缺失一律只记缺失(方法.md 第五部分"指标")。 */
|
||||||
function metricsOf(s: Study, plan: Content<"runPlanned">): HypothesisMetric[] {
|
function metricsOf(s: Study, plan: Content<"runPlanned">): HypothesisMetric[] {
|
||||||
if (plan.purpose === "baseline") return [];
|
if (plan.purpose === "baseline") return [];
|
||||||
if (plan.purpose === "forward") {
|
const board = new Board(s.ledger.records());
|
||||||
const board = new Board(s.ledger.records());
|
if (plan.purpose === "forward") return protocolMetrics(board, board.protocol(plan.object!));
|
||||||
return protocolMetrics(board, board.protocol(plan.object!));
|
return ablationMetrics(board, splitCell(plan.object!).candidate);
|
||||||
}
|
|
||||||
throw new Error(`运行 ${plan.run} 是消融运行,消融还没写(任务书 17)`);
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,162 @@
|
|||||||
|
import type { Content } from "../../ledger/index.ts";
|
||||||
|
import { Board, cellId, CELL_NOT_RUN, explanationId, protocolMetrics, splitCell, validScore } from "../../materials/index.ts";
|
||||||
|
import {
|
||||||
|
chooseBet,
|
||||||
|
decisionSeed,
|
||||||
|
runBalance,
|
||||||
|
runCellRule,
|
||||||
|
settleAblation,
|
||||||
|
sharedSettings,
|
||||||
|
type AblationExplanation,
|
||||||
|
type AblationModelInput,
|
||||||
|
type CellOutcome,
|
||||||
|
type CellRuleInput,
|
||||||
|
type CellRuleName,
|
||||||
|
type SettleInput,
|
||||||
|
} from "../../rules/index.ts";
|
||||||
|
import type { HypothesisMetric, Json } from "../../vocab/index.ts";
|
||||||
|
import { formalRun, type RunOutcome } from "../formal-run.ts";
|
||||||
|
import { ablationOf, aliveCells, lastAblationImpl } from "../next-step.ts";
|
||||||
|
import { SEED_VERSIONS } from "./round.ts";
|
||||||
|
import type { Study } from "../study.ts";
|
||||||
|
|
||||||
|
// ⑮ 选格、⑯ 消融运行、⑰ 结算与押的因子、⑱ 轮末解封(任务书 17 第 2.2 节)。选格、结算和押的因子各写一条记录
|
||||||
|
// (结算和押的因子同一个事务),写下之前被打断就从账本重新算一遍;写下之后恢复时照原样用,不重算(方法.md 第九部分)。
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ⑮ 选格(方法.md 6.7):基线、完整分数、方向、已知最好、ε、λ、因子、各条解释,加余额、每批上限、u、衰减和种子,
|
||||||
|
* 交给步骤清单里 selectCells 那一步的规则,写一条选格决定。种子用决定的版本 2(选批已用版本 1,方法.md 第九部分)。
|
||||||
|
*/
|
||||||
|
export function selectCells(s: Study, round: number): void {
|
||||||
|
const board = new Board(s.ledger.records());
|
||||||
|
const { config, steps } = s.frozen;
|
||||||
|
const seed = decisionSeed({ studyId: config.studyId, round, version: SEED_VERSIONS.cells });
|
||||||
|
const input: CellRuleInput = {
|
||||||
|
...modelInput(board, s, round),
|
||||||
|
balance: runBalance(config.runs, board.runStates()).balance,
|
||||||
|
batchCap: config.batchLimit,
|
||||||
|
u: config.flow.knowledgeUse,
|
||||||
|
decay: config.flow.decay,
|
||||||
|
seed,
|
||||||
|
};
|
||||||
|
const rule = steps.round.find((x) => x.id === "selectCells")!.rule!.name as CellRuleName;
|
||||||
|
const { rule: used, ...output } = runCellRule(rule, input);
|
||||||
|
// SAFETY: rules 是纯计算,输入输出都是纯 Json 值;账本在写入时校验 Json。
|
||||||
|
s.ledger.append("decision", {
|
||||||
|
decision: "cells",
|
||||||
|
round,
|
||||||
|
rule: used.name,
|
||||||
|
ruleVersion: used.version,
|
||||||
|
params: output.params,
|
||||||
|
seed,
|
||||||
|
input: input as unknown as Json,
|
||||||
|
output: output as unknown as Json,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** ⑯ 消融运行:用核验过的这一格快照,复用正式运行的登记、扣次、判分入口。完整代码和基线的分数取本批已有的运行结果,不重跑。 */
|
||||||
|
export function ablationRun(s: Study, round: number, cell: string, run: string): Promise<RunOutcome> {
|
||||||
|
const board = new Board(s.ledger.records());
|
||||||
|
return formalRun(s, { run, purpose: "ablation", round, object: cell }, board.code(cell).version, ablationMetrics(board, splitCell(cell).candidate));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 消融格正式运行登记的指标:照被采纳候选的冻结协议取;消融格里假设专用指标缺失一律只记缺失、不作废这一格(方法.md 第五部分"指标")。 */
|
||||||
|
export function ablationMetrics(board: Board, candidate: string): HypothesisMetric[] {
|
||||||
|
return protocolMetrics(board, board.protocol(candidate)).map((m) => ({ ...m, missing: "recordMissing" }));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ⑰ 结算(方法.md 4.3 ⑰、6.6):选中的格全部有结果、"跑不起来"或"不跑"事件后,照 SettleInput 拼输入,
|
||||||
|
* 写一条结算记录;押的因子(6.8)在同一个事务里写成种类 bet 的决定。
|
||||||
|
* "消融中发现"在 settleAblation 的输出里:materials 的 Board.baselineOf 据此把那一格的代码当成下一轮的基线,不另写采纳记录。
|
||||||
|
*/
|
||||||
|
export function settle(s: Study, round: number): void {
|
||||||
|
const board = new Board(s.ledger.records());
|
||||||
|
const { candidate, keys } = ablationOf(board, round)!;
|
||||||
|
const events = s.ledger.records().flatMap((r) => (r.kind === "event" ? [r.content] : []));
|
||||||
|
const model = modelInput(board, s, round);
|
||||||
|
const input: SettleInput = { ...model, cells: keys.map((key) => ({ factor: key, outcome: outcomeOf(board, events, round, candidate, key) })) };
|
||||||
|
const output = settleAblation(input);
|
||||||
|
const betInput = { ...model, cells: input.cells };
|
||||||
|
const bet = chooseBet(betInput);
|
||||||
|
// SAFETY: rules 是纯计算,结算和决定的输入输出都是纯 Json 值;账本在写入时校验 Json。
|
||||||
|
s.ledger.transaction(() => {
|
||||||
|
s.ledger.append("settlement", { round, result: { candidate, input, output } as unknown as Json });
|
||||||
|
s.ledger.append("decision", {
|
||||||
|
decision: "bet",
|
||||||
|
round,
|
||||||
|
rule: "chooseBet",
|
||||||
|
ruleVersion: 1,
|
||||||
|
params: { direction: model.direction, epsilon: model.epsilon, knownBest: model.knownBest, lambda: model.lambda },
|
||||||
|
seed: null,
|
||||||
|
input: betInput as unknown as Json,
|
||||||
|
output: bet as unknown as Json,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 一格的结局(方法.md 6.6"一格的几种结局"):没测、跑不起来、证据不足、一起关、有效分数。 */
|
||||||
|
function outcomeOf(board: Board, events: readonly Content<"event">[], round: number, candidate: string, key: string): CellOutcome {
|
||||||
|
if (!board.decision("cells", round)!.output.chosen.includes(key)) return { kind: "untested" };
|
||||||
|
const cell = cellId(candidate, key);
|
||||||
|
const impl = lastAblationImpl(board, cell)!;
|
||||||
|
if (impl.content.cannotRun !== null) return { kind: "cannotRun" };
|
||||||
|
if (events.some((e) => e.event === CELL_NOT_RUN && e.objects.includes(cell))) return { kind: "insufficient" };
|
||||||
|
const off = [key, ...impl.content.alsoOff];
|
||||||
|
// 几格实际关掉的因子完全相同时,沿用同设置第一格的运行结果(不重跑,方法.md 6.6"一起关")。
|
||||||
|
const alive = aliveCells(board, events, round);
|
||||||
|
const sameAs = new Map(sharedSettings(alive.map((x) => ({ factor: splitCell(x.cell).key, off: x.off }))).map((x) => [x.factor, x.sameAs]));
|
||||||
|
const first = sameAs.get(key);
|
||||||
|
const ranCell = first === null || first === undefined ? cell : cellId(candidate, first);
|
||||||
|
const run = board.runs().filter((r) => r.plan.purpose === "ablation" && r.plan.round === round && r.plan.object === ranCell).at(-1)!;
|
||||||
|
const score = validScore(run.result!.judged);
|
||||||
|
if (off.length > 1) return { kind: "joint", off, score };
|
||||||
|
return score === null ? { kind: "insufficient" } : { kind: "valid", score };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ⑱ 轮末解封(方法.md 4.3 ⑱):本轮选批选中的每个候选(跑过的、入选了却没跑成的)解封作者自评,
|
||||||
|
* 全部写在同一个事务里。只在收尾阶段关闭之后走到这里("下一步做什么"保证),提前写会泄露。
|
||||||
|
* "入选了却没跑成的候选离开候选池"不写记录:Board.carried 只留从没进过任何一批的候选。
|
||||||
|
*/
|
||||||
|
export function unsealChosen(s: Study, round: number): void {
|
||||||
|
const board = new Board(s.ledger.records());
|
||||||
|
const records = s.ledger.records();
|
||||||
|
const chosen = board.decision("batch", round)!.output.chosen;
|
||||||
|
s.ledger.transaction(() => {
|
||||||
|
for (const candidate of chosen) {
|
||||||
|
if (records.some((r) => r.kind === "unseal" && r.content.candidate === candidate)) continue;
|
||||||
|
const seals = board.proposalsOf(candidate).map((id) => {
|
||||||
|
const seq = board.proposal(id).seq;
|
||||||
|
const seal = records.find((r) => r.kind === "seal" && r.content.proposal === seq);
|
||||||
|
if (seal === undefined) throw new Error(`提案 ${id}(记录 ${seq})没有封存记录:提案的作者自评在写入提案时一并封存`);
|
||||||
|
return seal.seq;
|
||||||
|
});
|
||||||
|
s.ledger.append("unseal", { candidate, seals });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 成分模型的输入(方法.md 6.7):选格、结算和押的因子共用。完整代码和本批基线的分数取本批已有的运行结果。 */
|
||||||
|
function modelInput(board: Board, s: Study, round: number): AblationModelInput {
|
||||||
|
const { candidate, keys } = ablationOf(board, round)!;
|
||||||
|
const { config, facts } = s.frozen;
|
||||||
|
return {
|
||||||
|
baseline: board.baselineScore(board.baselineOf(round)),
|
||||||
|
full: board.baselineScore(candidate),
|
||||||
|
direction: facts.finalMetric.direction,
|
||||||
|
knownBest: facts.knownBest,
|
||||||
|
epsilon: facts.epsilon,
|
||||||
|
lambda: config.flow.openReserve,
|
||||||
|
factors: keys,
|
||||||
|
explanations: explanationsOf(board, round),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 这一轮已冻结的各条解释:作者的第 i 条(materials 的 explanationId),每条对每一格给的三个数。 */
|
||||||
|
function explanationsOf(board: Board, round: number): AblationExplanation[] {
|
||||||
|
return board
|
||||||
|
.published("explanation")
|
||||||
|
.filter((x) => x.round === round)
|
||||||
|
.flatMap((x) => x.content.explanations.map((e, i) => ({ id: explanationId(round, x.author, i + 1), author: x.author, cells: e.cells! })));
|
||||||
|
}
|
||||||
@@ -14,6 +14,7 @@ import type { Study } from "../study.ts";
|
|||||||
* 两次抽取不会用同一串随机数。选格由 flow 第 4 部分接上。
|
* 两次抽取不会用同一串随机数。选格由 flow 第 4 部分接上。
|
||||||
*/
|
*/
|
||||||
const SEED_VERSIONS = { batch: 1, cells: 2 } as const;
|
const SEED_VERSIONS = { batch: 1, cells: 2 } as const;
|
||||||
|
export { SEED_VERSIONS };
|
||||||
|
|
||||||
/** 一对提案(或提案与留池候选)的 Jev 判断,原样存进判重合并决定的输入。 */
|
/** 一对提案(或提案与留池候选)的 Jev 判断,原样存进判重合并决定的输入。 */
|
||||||
export type SamePair = { a: string; b: string; judgment: JevJudgment };
|
export type SamePair = { a: string; b: string; judgment: JevJudgment };
|
||||||
@@ -38,6 +39,7 @@ export async function dedupe(s: Study, round: number): Promise<void> {
|
|||||||
const pairs: SamePair[] = [];
|
const pairs: SamePair[] = [];
|
||||||
for (const [a, b] of asks) pairs.push({ a, b, judgment: await judgeSameChange(model, factorsOf.get(a)!, factorsOf.get(b)!) });
|
for (const [a, b] of asks) pairs.push({ a, b, judgment: await judgeSameChange(model, factorsOf.get(a)!, factorsOf.get(b)!) });
|
||||||
const output = groupSameChange(round, proposals.map((p) => p.item!), carried, pairs);
|
const output = groupSameChange(round, proposals.map((p) => p.item!), carried, pairs);
|
||||||
|
// SAFETY: JevJudgment 和 DedupOutput 都是纯 Json 值;账本在写入时校验 Json。
|
||||||
s.ledger.append("decision", {
|
s.ledger.append("decision", {
|
||||||
decision: "dedup",
|
decision: "dedup",
|
||||||
round,
|
round,
|
||||||
@@ -97,6 +99,7 @@ export function freezeProtocols(s: Study, round: number): void {
|
|||||||
answer.kind === "response"
|
answer.kind === "response"
|
||||||
? protocolBody({ kind: "response", content: answer.content, original: board.proposal(proposals[0]!).content })
|
? protocolBody({ kind: "response", content: answer.content, original: board.proposal(proposals[0]!).content })
|
||||||
: protocolBody({ kind: "unification", content: answer.content });
|
: protocolBody({ kind: "unification", content: answer.content });
|
||||||
|
// SAFETY: FrozenProtocol 是纯 Json 值;账本在写入时校验 Json。
|
||||||
s.ledger.append("protocol", { candidate, protocol: freezeProtocol(body, candidate, round, baseline, proposals) as unknown as Json });
|
s.ledger.append("protocol", { candidate, protocol: freezeProtocol(body, candidate, round, baseline, proposals) as unknown as Json });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -125,6 +128,7 @@ export function selectBatch(s: Study, round: number): void {
|
|||||||
};
|
};
|
||||||
const rule = steps.round.find((x) => x.id === "selectBatch")!.rule!.name as BatchRuleName;
|
const rule = steps.round.find((x) => x.id === "selectBatch")!.rule!.name as BatchRuleName;
|
||||||
const { rule: used, ...output } = runBatchRule(rule, input);
|
const { rule: used, ...output } = runBatchRule(rule, input);
|
||||||
|
// SAFETY: rules 是纯计算,输入输出都是纯 Json 值;账本在写入时校验 Json。
|
||||||
s.ledger.append("decision", {
|
s.ledger.append("decision", {
|
||||||
decision: "batch",
|
decision: "batch",
|
||||||
round,
|
round,
|
||||||
|
|||||||
+30
-3
@@ -1,7 +1,7 @@
|
|||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import type { Content } from "../ledger/index.ts";
|
import type { Content } from "../ledger/index.ts";
|
||||||
import { codeHunks } from "../jev/index.ts";
|
import { codeHunks } from "../jev/index.ts";
|
||||||
import { Board, protocolMetrics, type CodeChanges } from "../materials/index.ts";
|
import { Board, protocolMetrics, splitCell, type CodeChanges } from "../materials/index.ts";
|
||||||
import type { KnownOf } from "../submissions/index.ts";
|
import type { KnownOf } from "../submissions/index.ts";
|
||||||
import { lightCheck, protectedChanges, readLightCheck } from "../tasks/index.ts";
|
import { lightCheck, protectedChanges, readLightCheck } from "../tasks/index.ts";
|
||||||
import type { Json, Submission, SubmissionKind } from "../vocab/index.ts";
|
import type { Json, Submission, SubmissionKind } from "../vocab/index.ts";
|
||||||
@@ -10,10 +10,23 @@ import type { TaskRef } from "./next-step.ts";
|
|||||||
import { callPaths, containerName, freshDir, workspaceRepo } from "./paths.ts";
|
import { callPaths, containerName, freshDir, workspaceRepo } from "./paths.ts";
|
||||||
import type { Study } from "./study.ts";
|
import type { Study } from "./study.ts";
|
||||||
|
|
||||||
/** 实现取本批基线;修复取上次提交。核验、修复的材料差异始终相对本批基线。 */
|
/**
|
||||||
|
* 实现取本批基线;修复取上次提交。核验、修复的材料差异始终相对本批基线。
|
||||||
|
* 消融实现第一次取被采纳的完整代码,重做取上一次交的这一格代码(materials 的消融实现材料就是这样写的);
|
||||||
|
* 消融核验的材料差异是完整代码到这一格代码(方法.md 4.3 ⑯)。
|
||||||
|
*/
|
||||||
export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes: CodeChanges | null; startVersion: string | null } {
|
export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes: CodeChanges | null; startVersion: string | null } {
|
||||||
if (t.kind !== "implementation" && t.kind !== "repair" && t.kind !== "verification") return { changes: null, startVersion: null };
|
|
||||||
const board = new Board(s.ledger.records());
|
const board = new Board(s.ledger.records());
|
||||||
|
if (t.kind === "ablationImplementation") {
|
||||||
|
const { candidate } = splitCell(t.object!);
|
||||||
|
const hasCellCode = s.ledger.records().some((r) => r.kind === "code" && r.content.object === t.object);
|
||||||
|
return { changes: null, startVersion: hasCellCode ? board.code(t.object!).version : board.code(candidate).version };
|
||||||
|
}
|
||||||
|
if (t.kind === "ablationVerification") {
|
||||||
|
const { candidate } = splitCell(t.object!);
|
||||||
|
return { changes: compareVersions(workspaceRepo(s.dir), board.code(candidate).version, board.code(t.object!).version), startVersion: null };
|
||||||
|
}
|
||||||
|
if (t.kind !== "implementation" && t.kind !== "repair" && t.kind !== "verification") return { changes: null, startVersion: null };
|
||||||
const baseline = board.code(board.baselineOf(board.stage(t.stageId).round)).version;
|
const baseline = board.code(board.baselineOf(board.stage(t.stageId).round)).version;
|
||||||
if (t.kind === "implementation") return { changes: null, startVersion: baseline };
|
if (t.kind === "implementation") return { changes: null, startVersion: baseline };
|
||||||
const version = board.code(t.object!).version;
|
const version = board.code(t.object!).version;
|
||||||
@@ -34,6 +47,20 @@ export function prepareSubmission(
|
|||||||
const board = new Board(s.ledger.records());
|
const board = new Board(s.ledger.records());
|
||||||
if (t.kind === "proposal") return { known: { accepted: others }, code: null };
|
if (t.kind === "proposal") return { known: { accepted: others }, code: null };
|
||||||
if (t.kind === "response") return { known: { original: board.proposal(board.proposalsOf(t.object!)[0]!).content }, code: null };
|
if (t.kind === "response") return { known: { original: board.proposal(board.proposalsOf(t.object!)[0]!).content }, code: null };
|
||||||
|
if (t.kind === "ablationImplementation") {
|
||||||
|
const impl = content as Submission<"ablationImplementation">;
|
||||||
|
// 交了"这一格跑不起来":known.code 传 null,不存版本(任务书 17 第六部分),这一格不核验、不运行、不扣次。
|
||||||
|
if (impl.cannotRun !== null) return { known: { code: null }, code: null };
|
||||||
|
const saved = saveVersion(workspaceRepo(s.dir), workDir, s.frozen.facts.unversionedNames);
|
||||||
|
if (!saved.saved) return { known: { code: saved }, code: null };
|
||||||
|
const { candidate } = splitCell(t.object!);
|
||||||
|
const changes = compareVersions(workspaceRepo(s.dir), board.code(candidate).version, saved.version);
|
||||||
|
return {
|
||||||
|
known: { code: { saved: true, protectedFiles: protectedChanges(s.task, changes.files.map((f) => f.path)) } },
|
||||||
|
// 消融格的代码不记位置表:用完整代码那次实现交的位置表(materials 的 LocationTable 注释)。
|
||||||
|
code: { object: t.object!, version: saved.version, locations: null },
|
||||||
|
};
|
||||||
|
}
|
||||||
if (t.kind !== "implementation" && t.kind !== "repair") return { known: null, code: null };
|
if (t.kind !== "implementation" && t.kind !== "repair") return { known: null, code: null };
|
||||||
const saved = saveVersion(workspaceRepo(s.dir), workDir, s.frozen.facts.unversionedNames);
|
const saved = saveVersion(workspaceRepo(s.dir), workDir, s.frozen.facts.unversionedNames);
|
||||||
if (!saved.saved) return { known: { code: saved }, code: null };
|
if (!saved.saved) return { known: { code: saved }, code: null };
|
||||||
|
|||||||
@@ -0,0 +1,207 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { test, type TestContext } from "node:test";
|
||||||
|
import { stepList } from "../src/config/index.ts";
|
||||||
|
import { fullConfig as checkedFull } from "../src/flow/frozen.ts";
|
||||||
|
import { selectCells, settle, unsealChosen } from "../src/flow/steps/ablation.ts";
|
||||||
|
import type { Study } from "../src/flow/study.ts";
|
||||||
|
import { Ledger } from "../src/ledger/index.ts";
|
||||||
|
import { Board } from "../src/materials/index.ts";
|
||||||
|
import { chooseBet, decisionSeed, runCellRule, settleAblation, type AblationModelInput, type CellRuleInput, type SettleInput } from "../src/rules/index.ts";
|
||||||
|
import { checkTaskFacts } from "../src/tasks/index.ts";
|
||||||
|
import { fullConfig } from "./config-helpers.ts";
|
||||||
|
import { definition, exited, proposal, tempDir } from "./ledger-helpers.ts";
|
||||||
|
import { judged, r, Writer } from "./materials-helpers.ts";
|
||||||
|
|
||||||
|
// 选格、结算与押的因子、轮末解封(任务书 17 第 2.2 节):在真实账本里写到上一步做完,跑这一步,看写下的记录。
|
||||||
|
// 预期值按条文手拼:选格输入照 rules 的 CellRuleInput 逐项拼,结算的各格结局照方法.md 6.6 的情形拼;
|
||||||
|
// 输出用 rules 的函数当对照(同选批测试的写法,rules 自己有独立预期值的测试)。
|
||||||
|
|
||||||
|
const config = fullConfig();
|
||||||
|
const facts = checkTaskFacts(JSON.parse(readFileSync(join(import.meta.dirname, "..", "tasks", "circle_packing", "task.json"), "utf8")));
|
||||||
|
|
||||||
|
function study(t: TestContext): { s: Study; w: Writer } {
|
||||||
|
const ledger = Ledger.open(join(tempDir(t), "ledger.db"));
|
||||||
|
t.after(() => ledger.close());
|
||||||
|
const frozen = { config: checkedFull(config), steps: stepList(config), facts, prompts: null, image: "", seed: 1 };
|
||||||
|
return { s: { ledger, frozen } as unknown as Study, w: new Writer(ledger) };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 基线 0.96、被采纳候选 r1-c1(两个因子)完整代码 1.25。 */
|
||||||
|
function adoptedBase(w: Writer): void {
|
||||||
|
w.l.append("code", { object: "base", version: "v-base", locations: null });
|
||||||
|
w.run("base-1", "baseline", 0, null, judged(0.96, "main"));
|
||||||
|
w.l.append("code", { object: "r1-c1", version: "v-full", locations: null });
|
||||||
|
w.run("r1-forward-r1-c1-1", "forward", 1, "r1-c1", judged(1.25, "main"));
|
||||||
|
w.protocol("r1-c1", { factors: [{ name: "因子一" }, { name: "因子二" }], safeguards: [], metrics: { defined: [], reused: [] }, body: "正文", clauses: [], round: 1, baseline: "base", proposals: ["r1-a1-1"] } as never);
|
||||||
|
w.decision("adopt", 1, null, { adopted: "r1-c1" });
|
||||||
|
}
|
||||||
|
|
||||||
|
const EXPLANATIONS = [
|
||||||
|
{ id: "r1-a1-x1", author: "a1", cells: [{ factor: "p1", range: r(1.0, 1.1, 1.2) }, { factor: "p2", range: r(1.15, 1.2, 1.24) }] },
|
||||||
|
{ id: "r1-a2-x1", author: "a2", cells: [{ factor: "p1", range: r(0.9, 1.05, 1.2) }, { factor: "p2", range: r(1.1, 1.18, 1.25) }] },
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
/** 解释阶段关闭:a1、a2 各交一条,对两格都给了三个数。 */
|
||||||
|
function explain(w: Writer): void {
|
||||||
|
w.open("r1-explanation", "explanation", 1, ["a1", "a2"]);
|
||||||
|
for (const e of EXPLANATIONS) {
|
||||||
|
w.sub("r1-explanation", e.author, "explanation", null, null, { explanations: [{ body: `机制(${e.author})`, cells: e.cells.map((c) => ({ factor: c.factor, range: c.range })) }] });
|
||||||
|
}
|
||||||
|
w.close("r1-explanation");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 成分模型的输入:选格、结算和押的因子共用的那一份。 */
|
||||||
|
function modelInput(): AblationModelInput {
|
||||||
|
return { baseline: 0.96, full: 1.25, direction: "maximize", knownBest: facts.knownBest, epsilon: facts.epsilon, lambda: 0.1, factors: ["p1", "p2"], explanations: EXPLANATIONS.map((e) => ({ id: e.id, author: e.author, cells: e.cells.map((c) => ({ factor: c.factor, range: c.range })) })) };
|
||||||
|
}
|
||||||
|
|
||||||
|
test("选格:输入照 CellRuleInput 拼(基线、完整分数、各条解释、余额从运行记录算、种子用选格的版本 2),规则取步骤清单", (t) => {
|
||||||
|
const { s, w } = study(t);
|
||||||
|
adoptedBase(w);
|
||||||
|
explain(w);
|
||||||
|
selectCells(s, 1);
|
||||||
|
const record = s.ledger.records().find((x) => x.kind === "decision" && x.content.decision === "cells")!;
|
||||||
|
const seed = decisionSeed({ studyId: "circle-full-1", round: 1, version: 2 });
|
||||||
|
const input: CellRuleInput = {
|
||||||
|
...modelInput(),
|
||||||
|
// 13 次里基线和正向各确认启动 1 次,余额 11。
|
||||||
|
balance: 11,
|
||||||
|
batchCap: 2,
|
||||||
|
u: 0.5,
|
||||||
|
decay: 0.5,
|
||||||
|
seed,
|
||||||
|
};
|
||||||
|
const { rule: used, ...output } = runCellRule("valueMinusCost", input);
|
||||||
|
assert.deepEqual(record.content, { decision: "cells", round: 1, rule: used.name, ruleVersion: used.version, params: output.params, seed, input, output });
|
||||||
|
});
|
||||||
|
|
||||||
|
/** 一次消融实现阶段:subs 是 [格, cannotRun, alsoOff];cannotRun 为 null 表示交了代码。 */
|
||||||
|
function impl(w: Writer, n: number, subs: Array<[string, string | null, string[]?]>): void {
|
||||||
|
w.open(`r1-ablationImplementation-${n}`, "ablationImplementation", 1, subs.map(([cell]) => `experimenter-${cell}`));
|
||||||
|
for (const [cell, cannotRun, alsoOff] of subs) {
|
||||||
|
w.sub(`r1-ablationImplementation-${n}`, `experimenter-${cell}`, "ablationImplementation", cell, null, { alsoOff: alsoOff ?? [], cannotRun, concerns: "无" });
|
||||||
|
}
|
||||||
|
w.close(`r1-ablationImplementation-${n}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 结算一次:拼出 SettleInput,跑 settle,核对 settlement 和 bet 两条记录。 */
|
||||||
|
function settled(t: TestContext, prepare: (w: Writer) => void, cells: SettleInput["cells"]): void {
|
||||||
|
const { s, w } = study(t);
|
||||||
|
adoptedBase(w);
|
||||||
|
explain(w);
|
||||||
|
w.decision("cells", 1, null, { chosen: ["p1", "p2"] });
|
||||||
|
prepare(w);
|
||||||
|
settle(s, 1);
|
||||||
|
const input: SettleInput = { ...modelInput(), cells };
|
||||||
|
const betInput = { ...modelInput(), cells };
|
||||||
|
const settlement = s.ledger.records().find((x) => x.kind === "settlement")!;
|
||||||
|
assert.deepEqual(settlement.content, { round: 1, result: { candidate: "r1-c1", input, output: settleAblation(input) } });
|
||||||
|
const bet = s.ledger.records().find((x) => x.kind === "decision" && x.content.decision === "bet")!;
|
||||||
|
assert.deepEqual(bet.content, {
|
||||||
|
decision: "bet",
|
||||||
|
round: 1,
|
||||||
|
rule: "chooseBet",
|
||||||
|
ruleVersion: 1,
|
||||||
|
params: { direction: "maximize", epsilon: facts.epsilon, knownBest: facts.knownBest, lambda: 0.1 },
|
||||||
|
seed: null,
|
||||||
|
input: betInput,
|
||||||
|
output: chooseBet(betInput),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
test("结算:有效分数、没测、跑不起来、超时记证据不足,各按 6.6 的结局拼", (t) => {
|
||||||
|
settled(
|
||||||
|
t,
|
||||||
|
(w) => {
|
||||||
|
impl(w, 1, [["r1-c1-p1", null], ["r1-c1-p2", null]]);
|
||||||
|
w.run("r1-ablation-r1-c1-p1-1", "ablation", 1, "r1-c1-p1", judged(1.1, "main"));
|
||||||
|
// p2 启动了但跑崩: judged 为 null,记证据不足(扣了次数)。
|
||||||
|
w.l.append("runPlanned", { run: "r1-ablation-r1-c1-p2-1", purpose: "ablation", round: 1, object: "r1-c1-p2", definition });
|
||||||
|
w.l.append("runStarted", { run: "r1-ablation-r1-c1-p2-1" });
|
||||||
|
w.l.append("runResult", { run: "r1-ablation-r1-c1-p2-1", process: exited, judged: null });
|
||||||
|
},
|
||||||
|
[
|
||||||
|
{ factor: "p1", outcome: { kind: "valid", score: 1.1 } },
|
||||||
|
{ factor: "p2", outcome: { kind: "insufficient" } },
|
||||||
|
],
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("结算:没选中的格记没测;交了跑不起来的记跑不起来", (t) => {
|
||||||
|
const { s, w } = study(t);
|
||||||
|
adoptedBase(w);
|
||||||
|
explain(w);
|
||||||
|
w.decision("cells", 1, null, { chosen: ["p1"] });
|
||||||
|
impl(w, 1, [["r1-c1-p1", "离了 p1 跑不起来"]]);
|
||||||
|
settle(s, 1);
|
||||||
|
const input: SettleInput = {
|
||||||
|
...modelInput(),
|
||||||
|
cells: [
|
||||||
|
{ factor: "p1", outcome: { kind: "cannotRun" } },
|
||||||
|
{ factor: "p2", outcome: { kind: "untested" } },
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const settlement = s.ledger.records().find((x) => x.kind === "settlement")!;
|
||||||
|
assert.deepEqual(settlement.content, { round: 1, result: { candidate: "r1-c1", input, output: settleAblation(input) } });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("结算:核验不通过、次数用完有事件的记证据不足;设置相同的两格都记一起关,沿用第一格的分数", (t) => {
|
||||||
|
settled(
|
||||||
|
t,
|
||||||
|
(w) => {
|
||||||
|
impl(w, 1, [["r1-c1-p1", null, ["p2"]]]);
|
||||||
|
w.run("r1-ablation-r1-c1-p1-1", "ablation", 1, "r1-c1-p1", judged(1.05, "main"));
|
||||||
|
impl(w, 2, [["r1-c1-p2", null]]);
|
||||||
|
w.l.append("event", { event: "cellNotRun", reason: "第 2 次核验仍不通过,修复次数上限 1 已用完", objects: ["r1-c1-p2"] });
|
||||||
|
},
|
||||||
|
[
|
||||||
|
{ factor: "p1", outcome: { kind: "joint", off: ["p1", "p2"], score: 1.05 } },
|
||||||
|
{ factor: "p2", outcome: { kind: "insufficient" } },
|
||||||
|
],
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("结算:设置相同的两格都记一起关,沿用第一格的分数", (t) => {
|
||||||
|
settled(
|
||||||
|
t,
|
||||||
|
(w) => {
|
||||||
|
impl(w, 1, [["r1-c1-p1", null, ["p2"]]]);
|
||||||
|
w.run("r1-ablation-r1-c1-p1-1", "ablation", 1, "r1-c1-p1", judged(1.05, "main"));
|
||||||
|
impl(w, 2, [["r1-c1-p2", null, ["p1"]]]);
|
||||||
|
},
|
||||||
|
[
|
||||||
|
{ factor: "p1", outcome: { kind: "joint", off: ["p1", "p2"], score: 1.05 } },
|
||||||
|
{ factor: "p2", outcome: { kind: "joint", off: ["p2", "p1"], score: 1.05 } },
|
||||||
|
],
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("轮末解封:解封本轮选中的候选的全部作者自评(取最后一版的封存),没选中的继续封存", (t) => {
|
||||||
|
const { s, w } = study(t);
|
||||||
|
w.open("r1-proposal", "proposal", 1, ["a1", "a2"]);
|
||||||
|
w.sub("r1-proposal", "a1", "proposal", null, "r1-a1-1", proposal(r(1.0, 1.1, 1.2), "第一版"));
|
||||||
|
const v2 = s.ledger.append("submission", {
|
||||||
|
stageId: "r1-proposal",
|
||||||
|
author: "a1",
|
||||||
|
kind: "proposal",
|
||||||
|
target: null,
|
||||||
|
item: "r1-a1-1",
|
||||||
|
version: 2,
|
||||||
|
content: proposal(r(1.05, 1.15, 1.25), "第二版"),
|
||||||
|
jev: null,
|
||||||
|
} as never);
|
||||||
|
w.sub("r1-proposal", "a2", "proposal", null, "r1-a2-1", proposal(r(0.9, 1.0, 1.1)));
|
||||||
|
w.close("r1-proposal");
|
||||||
|
w.decision("dedup", 1, null, { fresh: [{ candidate: "r1-c1", proposals: ["r1-a1-1"] }, { candidate: "r1-c2", proposals: ["r1-a2-1"] }], carried: [] });
|
||||||
|
w.decision("batch", 1, null, { chosen: ["r1-c1"] });
|
||||||
|
unsealChosen(s, 1);
|
||||||
|
const unseals = s.ledger.records().filter((x) => x.kind === "unseal");
|
||||||
|
assert.equal(unseals.length, 1);
|
||||||
|
assert.equal(unseals[0]!.content.candidate, "r1-c1");
|
||||||
|
// 解封的是最后一版(v2)的封存。
|
||||||
|
const seal = s.ledger.records().find((x) => x.kind === "seal" && x.content.proposal === v2.seq)!;
|
||||||
|
assert.deepEqual(unseals[0]!.content.seals, [seal.seq]);
|
||||||
|
assert.deepEqual(new Board(s.ledger.records()).unsealed(), [{ candidate: "r1-c1", author: "a1", range: r(1.05, 1.15, 1.25) }]);
|
||||||
|
});
|
||||||
@@ -266,7 +266,9 @@ function roundOne(last: "proposal" | "dedupe" | "firstPrediction" | "critique" |
|
|||||||
]),
|
]),
|
||||||
response: () => stage(book, "r1-response", "response", [["researcher-1", "response", "r1-c1", null], ["researcher-2", "unification", "r1-c2", null]]),
|
response: () => stage(book, "r1-response", "response", [["researcher-1", "response", "r1-c1", null], ["researcher-2", "unification", "r1-c2", null]]),
|
||||||
freezeProtocol: () => {
|
freezeProtocol: () => {
|
||||||
for (const candidate of C) book.add("protocol", { candidate, protocol: { round: 1, baseline: "base", proposals: [] } });
|
// r1-c1 一个因子(采纳它不做消融),r1-c2 两个因子(采纳它走 ⑬–⑰)。
|
||||||
|
book.add("protocol", { candidate: "r1-c1", protocol: { round: 1, baseline: "base", proposals: [], factors: [{}] } });
|
||||||
|
book.add("protocol", { candidate: "r1-c2", protocol: { round: 1, baseline: "base", proposals: [], factors: [{}, {}] } });
|
||||||
},
|
},
|
||||||
secondPrediction: () => stage(book, "r1-secondPrediction", "secondPrediction", R.flatMap((a) => C.map((c) => [a, "secondPrediction", c, null] as [string, SubmissionKind, string, null]))),
|
secondPrediction: () => stage(book, "r1-secondPrediction", "secondPrediction", R.flatMap((a) => C.map((c) => [a, "secondPrediction", c, null] as [string, SubmissionKind, string, null]))),
|
||||||
selectBatch: () => book.add("decision", { decision: "batch", round: 1, rule: "bestExpected", ruleVersion: 1, params: null, seed: "s", input: null, output: { chosen: ["r1-c1"] } }),
|
selectBatch: () => book.add("decision", { decision: "batch", round: 1, rule: "bestExpected", ruleVersion: 1, params: null, seed: "s", input: null, output: { chosen: ["r1-c1"] } }),
|
||||||
@@ -494,12 +496,190 @@ test("正向串行:上次未启动起新编号;已登记未结束必须先
|
|||||||
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: null } });
|
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: null } });
|
||||||
const after = new Board(book.records);
|
const after = new Board(book.records);
|
||||||
assert.ok(after.runs().filter((r) => r.plan.purpose === "forward").every((r) => after.runPublic(r)));
|
assert.ok(after.runs().filter((r) => r.plan.purpose === "forward").every((r) => after.runPublic(r)));
|
||||||
assert.deepEqual(nextStep(book.records), { do: "halt", reason: "消融还没写(任务书 17)", objects: [] });
|
// 没有采纳:不开解释阶段,直接开收尾(方法.md 4.3 ⑫)。
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-wrapUp", stage: "wrapUp", round: 1, roster: R });
|
||||||
});
|
});
|
||||||
|
|
||||||
test("采纳只有一个因子的候选也统一在 listCells 停下,不擅自开收尾", () => {
|
test("采纳只有一个因子的候选:不做消融,直接开收尾(方法.md 4.3 ⑫)", () => {
|
||||||
const book = firstVerification(batchTwo(1), true, true);
|
const book = firstVerification(batchTwo(1), true, true);
|
||||||
for (const c of C) recordForward(book, c, 1, "valid");
|
for (const c of C) recordForward(book, c, 1, "valid");
|
||||||
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: C[0]! } });
|
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: C[0]! } });
|
||||||
assert.deepEqual(nextStep(book.records), { do: "halt", reason: "消融还没写(任务书 17)", objects: [] });
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-wrapUp", stage: "wrapUp", round: 1, roster: R });
|
||||||
|
});
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// ⑬–⑱:消融到收尾、轮末解封、下一轮和研究结束(任务书 17 第 2.2、2.3 节)。预期动作手写,不调模型、Jev 或 Docker。
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const ADOPTED = "r1-c2"; // 两个因子:p1、p2
|
||||||
|
const CELL1 = "r1-c2-p1";
|
||||||
|
const CELL2 = "r1-c2-p2";
|
||||||
|
|
||||||
|
/** 采纳了两个因子的 r1-c2。 */
|
||||||
|
function adoptedBook(): Book {
|
||||||
|
const book = firstVerification(batchTwo(1), true, true);
|
||||||
|
for (const c of C) recordForward(book, c, 1, "valid");
|
||||||
|
return book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: ADOPTED } });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 解释阶段:开、三人各交一份、关。 */
|
||||||
|
function explained(book: Book): Book {
|
||||||
|
return stage(book, "r1-explanation", "explanation", R.map((a) => [a, "explanation", null, null]));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 选格决定。 */
|
||||||
|
function cellsChosen(book: Book, chosen: string[]): Book {
|
||||||
|
return book.add("decision", { decision: "cells", round: 1, rule: "valueMinusCost", ruleVersion: 1, params: null, seed: "s2", input: null, output: { chosen } });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 一次消融实现阶段:subs 是 [格, cannotRun, alsoOff]。 */
|
||||||
|
function implStage(book: Book, n: number, subs: Array<[string, string | null, string[]?]>): Book {
|
||||||
|
const stageId = `r1-ablationImplementation-${n}`;
|
||||||
|
book.open(stageId, "ablationImplementation", 1, subs.map(([cell]) => `experimenter-${cell}`));
|
||||||
|
for (const [cell, cannotRun, alsoOff] of subs) {
|
||||||
|
book.add("submission", { stageId, author: `experimenter-${cell}`, kind: "ablationImplementation", target: cell, item: null, version: 1, content: { alsoOff: alsoOff ?? [], cannotRun, concerns: "无" }, jev: null });
|
||||||
|
}
|
||||||
|
return book.add("stageClose", { stageId });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 一次消融核验阶段:subs 是 [格, [因子, 关没关]]。 */
|
||||||
|
function verStage(book: Book, n: number, subs: Array<[string, Array<[string, boolean]>]>): Book {
|
||||||
|
const stageId = `r1-ablationVerification-${n}`;
|
||||||
|
book.open(stageId, "ablationVerification", 1, subs.map(([cell]) => `verifier-${cell}`));
|
||||||
|
for (const [cell, offs] of subs) {
|
||||||
|
book.add("submission", { stageId, author: `verifier-${cell}`, kind: "ablationVerification", target: cell, item: null, version: 1, content: { factors: offs.map(([factor, off]) => ({ factor, off, where: "initial.py:1" })) }, jev: null });
|
||||||
|
}
|
||||||
|
return book.add("stageClose", { stageId });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 一次消融运行:没启动,或启动了、判分算数。 */
|
||||||
|
function recordAblation(book: Book, cell: string, n: number, ending: "valid" | "notStarted"): Book {
|
||||||
|
const run = `r1-ablation-${cell}-${n}`;
|
||||||
|
book.add("runPlanned", { run, purpose: "ablation", round: 1, object: cell, definition });
|
||||||
|
if (ending === "notStarted") return book.add("runResult", { run, process: { ending, reason: "Docker 未启动" }, judged: null });
|
||||||
|
book.add("runStarted", { run });
|
||||||
|
return book.add("runResult", { run, process: exited, judged: judged(1.2, "main") });
|
||||||
|
}
|
||||||
|
|
||||||
|
test("采纳多因子改法:先开解释阶段,每人一个任务、没有对象;关闭后选格", () => {
|
||||||
|
const book = adoptedBook();
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-explanation", stage: "explanation", round: 1, roster: R });
|
||||||
|
book.open("r1-explanation", "explanation", 1, R);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-explanation", tasks: R.map((actor) => ({ actor, kind: "explanation", object: null })) });
|
||||||
|
for (const a of R) book.sub("r1-explanation", a, "explanation", null, null);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-explanation" });
|
||||||
|
book.add("stageClose", { stageId: "r1-explanation" });
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "selectCells", round: 1 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("选格做完:每格固定一位实验员;交了跑不起来的格不核验、不运行,没启动的下次起新编号", () => {
|
||||||
|
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-ablationImplementation-1", stage: "ablationImplementation", round: 1, roster: [`experimenter-${CELL1}`, `experimenter-${CELL2}`] });
|
||||||
|
implStage(book, 1, [[CELL1, null], [CELL2, "离了 p2 跑不起来"]]);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-ablationVerification-1", stage: "ablationVerification", round: 1, roster: [`verifier-${CELL1}`] });
|
||||||
|
verStage(book, 1, [[CELL1, [["p1", true], ["p2", false]]]]);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL1, run: `r1-ablation-${CELL1}-1` });
|
||||||
|
recordAblation(book, CELL1, 1, "notStarted");
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL1, run: `r1-ablation-${CELL1}-2` });
|
||||||
|
recordAblation(book, CELL1, 2, "valid");
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("核验不通过:退回原实验员重做,只重验重做的格;次数用完写不跑事件,不登记运行", () => {
|
||||||
|
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||||||
|
implStage(book, 1, [[CELL1, null], [CELL2, null]]);
|
||||||
|
verStage(book, 1, [[CELL1, [["p1", false], ["p2", false]]], [CELL2, [["p1", false], ["p2", true]]]]);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-ablationImplementation-2", stage: "ablationImplementation", round: 1, roster: [`experimenter-${CELL1}`] });
|
||||||
|
implStage(book, 2, [[CELL1, null]]);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-ablationVerification-2", stage: "ablationVerification", round: 1, roster: [`verifier-${CELL1}`] });
|
||||||
|
verStage(book, 2, [[CELL1, [["p1", false], ["p2", false]]]]);
|
||||||
|
const next = nextStep(book.records);
|
||||||
|
assert.equal(next.do, "cellNotRun");
|
||||||
|
if (next.do !== "cellNotRun") throw new Error("应写不跑事件");
|
||||||
|
assert.equal(next.cell, CELL1);
|
||||||
|
assert.match(next.reason, /第 2 次核验仍不通过,修复次数上限 1 已用完/);
|
||||||
|
book.add("event", { event: "cellNotRun", reason: next.reason, objects: [next.cell] });
|
||||||
|
// p1 不跑不扣次;只跑 p2。
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL2, run: `r1-ablation-${CELL2}-1` });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("几格实际关掉的因子完全相同:沿用第一格,不重复跑", () => {
|
||||||
|
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||||||
|
implStage(book, 1, [[CELL1, null, ["p2"]], [CELL2, null, ["p1"]]]);
|
||||||
|
verStage(book, 1, [[CELL1, [["p1", true], ["p2", true]]], [CELL2, [["p1", true], ["p2", true]]]]);
|
||||||
|
// 两格实际关的都是 {p1, p2}:只跑第一格。
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL1, run: `r1-ablation-${CELL1}-1` });
|
||||||
|
recordAblation(book, CELL1, 1, "valid");
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("全部交跑不起来:不核验、不运行,直接结算", () => {
|
||||||
|
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||||||
|
implStage(book, 1, [[CELL1, "跑不起来一"], [CELL2, "跑不起来二"]]);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 });
|
||||||
|
});
|
||||||
|
|
||||||
|
/** 结算和押的因子写完、收尾关闭、解封也写完的账本(余额还有 8 次)。 */
|
||||||
|
function afterWrapUp(): Book {
|
||||||
|
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||||||
|
implStage(book, 1, [[CELL1, null], [CELL2, null]]);
|
||||||
|
verStage(book, 1, [[CELL1, [["p1", true], ["p2", false]]], [CELL2, [["p1", false], ["p2", true]]]]);
|
||||||
|
recordAblation(book, CELL1, 1, "valid");
|
||||||
|
recordAblation(book, CELL2, 1, "valid");
|
||||||
|
book.add("settlement", { round: 1, result: { candidate: ADOPTED, input: null, output: null } });
|
||||||
|
book.add("decision", { decision: "bet", round: 1, rule: "chooseBet", ruleVersion: 1, params: null, seed: null, input: null, output: null });
|
||||||
|
return stage(book, "r1-wrapUp", "wrapUp", R.map((a) => [a, "wrapUp", null, null]));
|
||||||
|
}
|
||||||
|
|
||||||
|
test("结算和押的因子写完:开收尾;收尾关闭后解封本轮选中的全部候选,然后开下一轮", () => {
|
||||||
|
const book = cellsChosen(explained(adoptedBook()), ["p1", "p2"]);
|
||||||
|
implStage(book, 1, [[CELL1, null], [CELL2, null]]);
|
||||||
|
verStage(book, 1, [[CELL1, [["p1", true], ["p2", false]]], [CELL2, [["p1", false], ["p2", true]]]]);
|
||||||
|
recordAblation(book, CELL1, 1, "valid");
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "ablationRun", round: 1, cell: CELL2, run: `r1-ablation-${CELL2}-1` });
|
||||||
|
recordAblation(book, CELL2, 1, "valid");
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "settle", round: 1 });
|
||||||
|
book.add("settlement", { round: 1, result: { candidate: ADOPTED, input: null, output: null } });
|
||||||
|
book.add("decision", { decision: "bet", round: 1, rule: "chooseBet", ruleVersion: 1, params: null, seed: null, input: null, output: null });
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-wrapUp", stage: "wrapUp", round: 1, roster: R });
|
||||||
|
stage(book, "r1-wrapUp", "wrapUp", R.map((a) => [a, "wrapUp", null, null]));
|
||||||
|
// 收尾关闭后:解封本轮选批选中的 r1-c1、r1-c2;还有一个没解封就继续。
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "unsealChosen", round: 1 });
|
||||||
|
book.add("unseal", { candidate: "r1-c1", seals: [1] });
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "unsealChosen", round: 1 });
|
||||||
|
book.add("unseal", { candidate: "r1-c2", seals: [2] });
|
||||||
|
// 余额还有(已扣 5 次):开第 2 轮。
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r2-proposal", stage: "proposal", round: 2, roster: R });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("余额为 0:不开下一轮,走到导出停下,原因写明", () => {
|
||||||
|
const book = afterWrapUp();
|
||||||
|
book.add("unseal", { candidate: "r1-c1", seals: [1] });
|
||||||
|
book.add("unseal", { candidate: "r1-c2", seals: [2] });
|
||||||
|
// 已扣 5 次(基线 1、正向 2、消融 2):把冻结配置的运行次数改成 5,余额为 0。
|
||||||
|
const record = book.records.find((r) => r.kind === "config")!;
|
||||||
|
const study = record.content as unknown as StudyRecord;
|
||||||
|
study.config = { ...fullConfig(), runs: 5 } as unknown as Json;
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "halt", reason: "导出还没写", objects: [] });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("配置关掉消融、又采纳了多因子改法:先记按配置跳过,再在解释停下", () => {
|
||||||
|
const book = firstVerification(batchTwo(1), true, true);
|
||||||
|
for (const c of C) recordForward(book, c, 1, "valid");
|
||||||
|
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: ADOPTED } });
|
||||||
|
// 冻结的配置改为关掉消融:配置和步骤清单一起换(步骤清单在研究开始时冻结)。
|
||||||
|
const config = fullConfig();
|
||||||
|
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
|
||||||
|
// 配置关掉消融时不写 cellRule(它只有开着消融才用得到)。
|
||||||
|
config.flow.ablation = { enabled: false };
|
||||||
|
const record = book.records.find((r) => r.kind === "config")!;
|
||||||
|
const study = record.content as unknown as StudyRecord;
|
||||||
|
study.config = config as unknown as Json;
|
||||||
|
study.steps = stepList(config) as unknown as Json;
|
||||||
|
const reason = "配置关掉了消融";
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "skip", step: "r1-listCells", reason });
|
||||||
|
book.add("event", { event: "skippedByConfig", reason, objects: ["r1-listCells"] });
|
||||||
|
const next = nextStep(book.records);
|
||||||
|
assert.equal(next.do, "halt");
|
||||||
|
assert.match((next as { reason: string }).reason, /不交三个数的解释还没写/);
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user