feat: 接通实现核验修复到正向运行采纳

任务书16第二块:角色分派与每调用轻量检查,提交及代码原子记账,按次数核验修复且耗尽不跑,串行正式运行及恢复引用指标,采纳含null统一公开。451项免费测试通过,全新上下文独立审阅未发现问题;真实研究验收仍待进行。
This commit is contained in:
2026-10-04 04:25:00 -04:00
parent 039b7bb3bc
commit bf5b3f2358
13 changed files with 659 additions and 77 deletions
+11 -2
View File
@@ -1,6 +1,7 @@
import { mkdirSync } from "node:fs"; import { mkdirSync } from "node:fs";
import { resolve } from "node:path"; import { resolve } from "node:path";
import type { ResearchConfig } from "../config/index.ts"; import type { ResearchConfig } from "../config/index.ts";
import { forwardNotRunEvent } from "../materials/index.ts";
import { Ledger } from "../ledger/index.ts"; import { Ledger } from "../ledger/index.ts";
import { Runner } from "../runner/index.ts"; import { Runner } from "../runner/index.ts";
import { fullConfig, readFrozen } from "./frozen.ts"; import { fullConfig, readFrozen } from "./frozen.ts";
@@ -8,6 +9,7 @@ import { EVENTS, nextStep, type Next } from "./next-step.ts";
import { ledgerFile } from "./paths.ts"; import { ledgerFile } from "./paths.ts";
import { recover } from "./recover.ts"; import { recover } from "./recover.ts";
import { baselineRun, saveBaseline } from "./steps/baseline.ts"; import { baselineRun, saveBaseline } from "./steps/baseline.ts";
import { forwardRun, adoptBatch } from "./steps/forward.ts";
import { dispatchStage } from "./steps/dispatch.ts"; import { dispatchStage } from "./steps/dispatch.ts";
import { dedupe, freezeProtocols, selectBatch } from "./steps/round.ts"; import { dedupe, freezeProtocols, selectBatch } from "./steps/round.ts";
import { assignEntries } from "./steps/round-zero.ts"; import { assignEntries } from "./steps/round-zero.ts";
@@ -75,8 +77,9 @@ async function doStep(s: Study, next: Exclude<Next, { do: "freezeConfig" }>): Pr
case "saveBaseline": case "saveBaseline":
saveBaseline(s); saveBaseline(s);
return null; return null;
case "baselineRun": { case "baselineRun":
const ran = await baselineRun(s, next.run); case "forwardRun": {
const ran = next.do === "baselineRun" ? await baselineRun(s, next.run) : await forwardRun(s, next.round, next.candidate, next.run);
if (ran === "stopped") return USER_STOPPED; if (ran === "stopped") return USER_STOPPED;
if (ran === "done") return null; if (ran === "done") return null;
return halt(s.ledger, `运行 ${next.run} 没有启动,名额已退还;本进程不重跑,下次启动时起新编号重跑。Docker 的报错:${ran.notStarted}`, [next.run]); return halt(s.ledger, `运行 ${next.run} 没有启动,名额已退还;本进程不重跑,下次启动时起新编号重跑。Docker 的报错:${ran.notStarted}`, [next.run]);
@@ -104,6 +107,12 @@ async function doStep(s: Study, next: Exclude<Next, { do: "freezeConfig" }>): Pr
case "selectBatch": case "selectBatch":
selectBatch(s, next.round); selectBatch(s, next.round);
return null; return null;
case "forwardNotRun":
s.ledger.append("event", forwardNotRunEvent(next.candidate, next.reason));
return null;
case "adopt":
adoptBatch(s, next.round);
return null;
case "halt": case "halt":
return halt(s.ledger, next.reason, next.objects); return halt(s.ledger, next.reason, next.objects);
} }
+38 -24
View File
@@ -1,8 +1,9 @@
import { runSession, type KindOf, type SessionEnding, type SubmitAttempt, type SubmitReceipt } from "../agents/index.ts"; import { runSession, type SessionTask, type SessionEnding, type SubmitAttempt, type SubmitReceipt } from "../agents/index.ts";
import { buildMaterial, callId, queryMaterial, researcherId, type TaskMaterial } from "../materials/index.ts"; import { buildMaterial, callId, queryMaterial, researcherId, type TaskMaterial } from "../materials/index.ts";
import { receive, type Incoming, type KnownOf } from "../submissions/index.ts"; import { receive, type Incoming } from "../submissions/index.ts";
import type { JevJudgment, Json, Submission } from "../vocab/index.ts"; import { SUBMISSION_KINDS, type JevJudgment, type Json, type SubmissionKind } from "../vocab/index.ts";
import { checkoutVersion } from "../workspace/index.ts"; import { checkoutVersion } from "../workspace/index.ts";
import { prepareSubmission, taskCode, lightChecker } from "./task-code.ts";
import { callPaths, freshDir, workspaceRepo } from "./paths.ts"; import { callPaths, freshDir, workspaceRepo } from "./paths.ts";
import type { Study } from "./study.ts"; import type { Study } from "./study.ts";
@@ -10,26 +11,26 @@ import type { Study } from "./study.ts";
// 最后写"模型调用结束",前后顺序由这个入口保证(方法.md 第九部分"先登记、再执行、再登记结果")。 // 最后写"模型调用结束",前后顺序由这个入口保证(方法.md 第九部分"先登记、再执行、再登记结果")。
// 提交收不收只由 submissions 判;形状不合格的由 agents 报上来,这里直接记退回("给 flow")。 // 提交收不收只由 submissions 判;形状不合格的由 agents 报上来,这里直接记退回("给 flow")。
/** 一个研究者任务:哪种提交、谁、哪个阶段、对象。 */ /** 一个模型任务:哪种提交、谁、哪个阶段、对象。 */
export type ModelTask<K extends KindOf<"researcher">> = { export type ModelTask<K extends SubmissionKind> = {
kind: K; kind: K;
actor: string; actor: string;
stageId: string; stageId: string;
object: string | null; object: string | null;
/** 一个任务里能交几份的种类(提案、挑刺):第 n 份的编号(materials 的 proposalId、critiqueId),收下时分配;只交一份的为 null。 */ /** 一个任务里能交几份的种类(提案、挑刺):第 n 份的编号(materials 的 proposalId、critiqueId),收下时分配;只交一份的为 null。 */
item: ((n: number) => string) | null; item: ((n: number) => string) | null;
/** 交上来之后才知道的东西(submissions 的 KnownOf)。others 是本任务已收下、而且不是这次要替换的那几份的份数。 */
known(content: Submission<K>, others: number): KnownOf<K>;
}; };
/** 派一次任务,等会话结束,交回调用编号和结局。 */ /** 派一次任务,等会话结束,交回调用编号和结局。 */
export async function runModelTask<K extends KindOf<"researcher">>(s: Study, t: ModelTask<K>): Promise<{ call: string; ending: SessionEnding }> { export async function runModelTask<K extends SubmissionKind>(s: Study, t: ModelTask<K>): Promise<{ call: string; ending: SessionEnding }> {
const { ledger } = s; const { ledger } = s;
const started = new Set(ledger.records().flatMap((r) => (r.kind === "callStarted" ? [r.content.call] : []))); const started = new Set(ledger.records().flatMap((r) => (r.kind === "callStarted" ? [r.content.call] : [])));
let attempt = 1; let attempt = 1;
while (started.has(callId(t.stageId, t.actor, t.kind, t.object, attempt))) attempt++; while (started.has(callId(t.stageId, t.actor, t.kind, t.object, attempt))) attempt++;
const call = callId(t.stageId, t.actor, t.kind, t.object, attempt); const call = callId(t.stageId, t.actor, t.kind, t.object, attempt);
const paths = callPaths(s.dir, call); const paths = callPaths(s.dir, call);
const { changes, startVersion } = taskCode(s, t);
const role = SUBMISSION_KINDS[t.kind].role;
const material = buildMaterial({ const material = buildMaterial({
kind: t.kind, kind: t.kind,
actor: t.actor, actor: t.actor,
@@ -39,37 +40,44 @@ export async function runModelTask<K extends KindOf<"researcher">>(s: Study, t:
study: s.facts, study: s.facts,
prompts: s.prompts, prompts: s.prompts,
codeRoot: paths.code, codeRoot: paths.code,
changes: null, changes,
artifacts: null, artifacts: null,
}); });
const model = researcherModel(s, t.actor); const model = role === "researcher" ? researcherModel(s, t.actor) : s.frozen.config.models[role];
// 先登记、后建目录:调用编号只在登记之后才算用掉,被杀在两者之间时下次的编号不会撞上旧目录。 // 先登记、后建目录:调用编号只在登记之后才算用掉,被杀在两者之间时下次的编号不会撞上旧目录。
ledger.append("callStarted", { ledger.append("callStarted", {
call, call,
actor: t.actor, actor: t.actor,
stageId: t.stageId, stageId: t.stageId,
model: `${model.provider}/${model.model}(思考档位 ${model.thinking})`, model: `${model.provider}/${model.model}(思考档位 ${model.thinking})`,
// SAFETY: TaskMaterial 的全部字段都是可序列化数据,账本在写入时校验 Json。
materials: material as unknown as Json, materials: material as unknown as Json,
}); });
freshDir(paths.base); freshDir(paths.base);
freshDir(paths.code); freshDir(paths.code);
for (const r of material.readable) checkoutVersion(workspaceRepo(s.dir), r.version, freshDir(r.dir)); for (const r of material.readable) checkoutVersion(workspaceRepo(s.dir), r.version, freshDir(r.dir));
const workDir = freshDir(paths.work);
if (startVersion !== null) checkoutVersion(workspaceRepo(s.dir), startVersion, workDir);
let reply = 0; let reply = 0;
const outcome = await runSession({ const common = {
role: "researcher",
model, model,
systemPrompt: s.prompts.roles.researcher.text, systemPrompt: s.prompts.roles[role].text,
firstMessage: material.firstMessage, firstMessage: material.firstMessage,
workDir: freshDir(paths.work), workDir,
sessionDir: freshDir(paths.sessions), sessionDir: freshDir(paths.sessions),
kind: t.kind, kind: t.kind,
readableDirs: material.readable.map((r) => r.dir), query: (ref: string) => queryMaterial(material, ref),
query: (ref) => queryMaterial(material, ref), submit: (attempt: SubmitAttempt<K>) => submit(s, t, material, attempt, workDir),
submit: (attempt) => submit(s, t, material, attempt as SubmitAttempt<K>), usage: (usage: Parameters<SessionTask["usage"]>[0]) => void ledger.append("callUsage", { call, reply: ++reply, usage }),
usage: (usage) => void ledger.append("callUsage", { call, reply: ++reply, usage }),
writeNote: (body) => void ledger.append("note", { author: t.actor, body }),
stop: s.stop, stop: s.stop,
}); };
const readableDirs = material.readable.map((r) => r.dir);
const task = role === "experimenter"
? { ...common, role, lightCheck: lightChecker(s, call, workDir) }
: role === "verifier"
? { ...common, role, readableDirs }
: { ...common, role, readableDirs, writeNote: (body: string) => void ledger.append("note", { author: t.actor, body }) };
const outcome = await runSession(task as SessionTask);
// 会话文件就在这次调用的 sessions 目录里,不另记。 // 会话文件就在这次调用的 sessions 目录里,不另记。
const { wallSeconds, sessionFile: _sessionFile, ...ending } = outcome; const { wallSeconds, sessionFile: _sessionFile, ...ending } = outcome;
ledger.append("callEnded", { call, wallSeconds: { status: "measured", value: wallSeconds }, ...ending }); ledger.append("callEnded", { call, wallSeconds: { status: "measured", value: wallSeconds }, ...ending });
@@ -85,26 +93,32 @@ function researcherModel(s: Study, actor: string) {
} }
/** 一份提交收不收:先定写到哪一份,形状合格的再交给 submissions 判。 */ /** 一份提交收不收:先定写到哪一份,形状合格的再交给 submissions 判。 */
async function submit<K extends KindOf<"researcher">>(s: Study, t: ModelTask<K>, material: TaskMaterial, attempt: SubmitAttempt<K>): Promise<SubmitReceipt> { async function submit<K extends SubmissionKind>(s: Study, t: ModelTask<K>, material: TaskMaterial, attempt: SubmitAttempt<K>, workDir: string): Promise<SubmitReceipt> {
const { ledger } = s; const { ledger } = s;
const reject = (submitted: Json, reason: string, jev: JevJudgment[] | null): SubmitReceipt => { const reject = (submitted: Json, reason: string, jev: JevJudgment[] | null): SubmitReceipt => {
ledger.append("rejection", { stageId: t.stageId, author: t.actor, kind: t.kind, target: t.object, submitted, reason, jev }); ledger.append("rejection", { stageId: t.stageId, author: t.actor, kind: t.kind, target: t.object, submitted, reason, jev });
return { accepted: false, reason }; return { accepted: false, reason };
}; };
if (attempt.shape === "invalid") return reject(attempt.raw, attempt.reason, null); if (attempt.shape === "invalid") return reject(attempt.raw, attempt.reason, null);
// SAFETY: agents 已按 vocab 的提交字段校验,提交内容只含 Json 值。
const content = attempt.content as unknown as Json; const content = attempt.content as unknown as Json;
const slot = slotOf(s, t, attempt.replaces); const slot = slotOf(s, t, attempt.replaces);
if ("refuse" in slot) return reject(content, slot.refuse, null); if ("refuse" in slot) return reject(content, slot.refuse, null);
const prepared = prepareSubmission(s, t, attempt.content, slot.others, workDir);
const verdict = await receive({ const verdict = await receive({
kind: t.kind, kind: t.kind,
content: attempt.content, content: attempt.content,
checklist: material.checklist, checklist: material.checklist,
known: t.known(attempt.content, slot.others), known: prepared.known,
jevModel: s.frozen.config.models.jev, jevModel: s.frozen.config.models.jev,
} as Incoming); } as Incoming);
if (!verdict.accepted) return reject(content, verdict.reason, verdict.jev); if (!verdict.accepted) return reject(content, verdict.reason, verdict.jev);
const { stageId, actor: author, kind, object: target } = t; const { stageId, actor: author, kind, object: target } = t;
ledger.append("submission", { stageId, author, kind, target, item: slot.item, version: slot.version, content: attempt.content, jev: verdict.jev } as never); // 代码在提交时冻结;提交和代码记录同进同退,恢复不能看到“交了却没快照”的半截状态。
ledger.transaction(() => {
ledger.append("submission", { stageId, author, kind, target, item: slot.item, version: slot.version, content: attempt.content, jev: verdict.jev } as never);
if (prepared.code !== null) ledger.append("code", prepared.code);
});
return { accepted: true, id: slot.id }; return { accepted: true, id: slot.id };
} }
@@ -113,7 +127,7 @@ async function submit<K extends KindOf<"researcher">>(s: Study, t: ModelTask<K>,
* 提交编号:能交几份的种类就是这一份的编号,只交一份的是"阶段.本人.种类.对象"。 * 提交编号:能交几份的种类就是这一份的编号,只交一份的是"阶段.本人.种类.对象"。
* 不写 replaces 是交新的一份,只交一份的种类已经交过就退回并说明该怎么写;写了 replaces 就必须是本任务已收下的某一份。 * 不写 replaces 是交新的一份,只交一份的种类已经交过就退回并说明该怎么写;写了 replaces 就必须是本任务已收下的某一份。
*/ */
function slotOf<K extends KindOf<"researcher">>( function slotOf<K extends SubmissionKind>(
s: Study, s: Study,
t: ModelTask<K>, t: ModelTask<K>,
replaces: string | null, replaces: string | null,
+89 -13
View File
@@ -1,9 +1,9 @@
import type { KindOf } from "../agents/index.ts";
import type { RoundStepId, StudyStepId } from "../config/index.ts"; import type { RoundStepId, StudyStepId } from "../config/index.ts";
import type { Content, LedgerRecord } from "../ledger/index.ts"; import type { Content, LedgerRecord } from "../ledger/index.ts";
import { Board, ORIGINAL_BASELINE, researcherId } from "../materials/index.ts"; import { Board, experimenterId, verifierId, FORWARD_NOT_RUN, ORIGINAL_BASELINE, researcherId } from "../materials/index.ts";
import { compareIds } from "../rules/index.ts"; import { compareIds } from "../rules/index.ts";
import type { Stage } from "../vocab/index.ts"; import { verificationOutcome } from "../submissions/index.ts";
import { SUBMISSION_KINDS, type Stage, type SubmissionKind } from "../vocab/index.ts";
import { readFrozen, type Frozen } from "./frozen.ts"; import { readFrozen, type Frozen } from "./frozen.ts";
// "下一步做什么"(docs/结构.md 第二部分"几条边界"):步骤的先后只写在这里。 // "下一步做什么"(docs/结构.md 第二部分"几条边界"):步骤的先后只写在这里。
@@ -40,8 +40,8 @@ export const EVENTS = {
runStatusUnknown: "runStatusUnknown", runStatusUnknown: "runStatusUnknown",
} as const; } as const;
/** 一个研究者任务:一个人对一个对象(候选、原提案;提案和第零轮的任务没有对象)交一种提交。 */ /** 一个任务:一个人对一个对象交一种提交;提案和第零轮任务没有对象。 */
export type TaskRef = { actor: string; kind: KindOf<"researcher">; object: string | null }; export type TaskRef = { actor: string; kind: SubmissionKind; object: string | null };
export type Next = export type Next =
| { do: "freezeConfig" } | { do: "freezeConfig" }
@@ -55,6 +55,9 @@ export type Next =
| { do: "dedupe"; round: number } | { do: "dedupe"; round: number }
| { do: "freezeProtocols"; round: number } | { do: "freezeProtocols"; round: number }
| { do: "selectBatch"; round: number } | { do: "selectBatch"; round: number }
| { do: "forwardNotRun"; candidate: string; reason: string }
| { do: "forwardRun"; round: number; candidate: string; run: string }
| { do: "adopt"; round: number }
| { do: "halt"; reason: string; objects: string[] }; | { do: "halt"; reason: string; objects: string[] };
export function nextStep(records: readonly LedgerRecord[]): Next { export function nextStep(records: readonly LedgerRecord[]): Next {
@@ -87,7 +90,7 @@ function studyStepNext(step: Exclude<StudyStepId, "rounds">, view: View, frozen:
case "discussion2": case "discussion2":
case "claim": { case "claim": {
const { stage, kind } = ROUND_ZERO_STAGES[step]; const { stage, kind } = ROUND_ZERO_STAGES[step];
return stageNext(stageIdOf(0, step), stage, 0, (roster) => roster.map((actor) => ({ actor, kind, object: null })), view, frozen); return stageNext(stageIdOf(0, step), stage, 0, (roster) => roster.map((actor) => ({ actor, kind, object: null })), view, frozen, researchers(frozen));
} }
case "assignEntries": case "assignEntries":
// 分配 → 第 0 轮的入口分配决定(decision,种类 entries)。 // 分配 → 第 0 轮的入口分配决定(decision,种类 entries)。
@@ -99,12 +102,12 @@ function studyStepNext(step: Exclude<StudyStepId, "rounds">, view: View, frozen:
/** /**
* 一轮接一轮(结构.md 第三部分"每一轮"):第一个没做完的轮次里第一个没做完的步骤。 * 一轮接一轮(结构.md 第三部分"每一轮"):第一个没做完的轮次里第一个没做完的步骤。
* 每一步做完的记录都带轮次,做完的轮次整轮跳过;现在走到实现一步就停下,所以循环一定会返回。 * 每一步做完的记录都带轮次,做完的轮次整轮跳过;现在走到列消融格就停下,所以循环一定会返回。
*/ */
function roundsNext(view: View, frozen: Frozen): Next { function roundsNext(view: View, frozen: Frozen): Next {
for (let round = 1; ; round++) { for (let round = 1; ; round++) {
for (const step of frozen.steps.round) { for (const step of frozen.steps.round) {
const next = step.status === "skip" ? skipNext(stageIdOf(round, step.id), step.reason, view) : roundStepNext(step.id, round, view, frozen); const next = step.status === "skip" && step.id !== "listCells" ? skipNext(stageIdOf(round, step.id), step.reason, view) : roundStepNext(step.id, round, view, frozen);
if (next !== null) return next; if (next !== null) return next;
} }
} }
@@ -122,7 +125,7 @@ function roundStepNext(step: RoundStepId, round: number, view: View, frozen: Fro
case "critique": case "critique":
case "response": case "response":
case "secondPrediction": case "secondPrediction":
return stageNext(stageIdOf(round, step), step, round, (roster) => roundTasks(step, round, roster, view.board), view, frozen); return stageNext(stageIdOf(round, step), step, round, (roster) => roundTasks(step, round, roster, view.board), view, frozen, researchers(frozen));
case "dedupe": case "dedupe":
// 判重合并 → 本轮的判重合并决定(decision,种类 dedup)。 // 判重合并 → 本轮的判重合并决定(decision,种类 dedup)。
return view.board.decision("dedup", round) === undefined ? { do: "dedupe", round } : null; return view.board.decision("dedup", round) === undefined ? { do: "dedupe", round } : null;
@@ -138,10 +141,25 @@ function roundStepNext(step: RoundStepId, round: number, view: View, frozen: Fro
case "selectBatch": case "selectBatch":
// 选批 → 本轮的选批决定(decision,种类 batch)。 // 选批 → 本轮的选批决定(decision,种类 batch)。
return view.board.decision("batch", round) === undefined ? { do: "selectBatch", round } : null; return view.board.decision("batch", round) === undefined ? { do: "selectBatch", round } : null;
case "implementation": case "implementation": {
return { do: "halt", reason: "实现还没写(任务书 16)", objects: [] }; const candidates = view.board.decision("batch", round)!.output.chosen;
return assignedStage("implementation", round, "implementation", candidates, view, frozen);
}
case "verification":
case "repair":
return verificationNext(round, view, frozen);
case "forwardRun":
return forwardNext(round, view);
case "publish":
case "informationAccount":
// 不另写记录:整批运行已结束。采纳决定使结果整批公开,信息账从预测和结果现算。
return null;
case "adopt":
return view.board.decision("adopt", round) === undefined ? { do: "adopt", round } : null;
case "listCells":
return { do: "halt", reason: "消融还没写(任务书 17)", objects: [] };
default: default:
throw new Error(`走到了第 ${round} 轮的 ${step}:implementation 一步应当先停下`); throw new Error(`走到了第 ${round} 轮的 ${step}:listCells 一步应当先停下`);
} }
} }
@@ -215,10 +233,11 @@ function stageNext(
tasksOf: (roster: readonly string[]) => TaskRef[], tasksOf: (roster: readonly string[]) => TaskRef[],
view: View, view: View,
frozen: Frozen, frozen: Frozen,
roster: string[],
): Next | null { ): Next | null {
const open = view.board.stages.get(stageId); const open = view.board.stages.get(stageId);
if (open === undefined) { if (open === undefined) {
return { do: "openStage", stageId, stage, round, roster: frozen.config.models.researchers.map((_, i) => researcherId(i)) }; return { do: "openStage", stageId, stage, round, roster };
} }
if (open.closed) return null; if (open.closed) return null;
const todo = tasksOf(open.roster).filter((t) => !view.done(stageId, t)); const todo = tasksOf(open.roster).filter((t) => !view.done(stageId, t));
@@ -239,6 +258,63 @@ function stageNext(
}; };
} }
/** 研究者阶段的冻结名单。实验员、核验员阶段则按本阶段的任务列名单。 */
const researchers = (frozen: Frozen): string[] => frozen.config.models.researchers.map((_, i) => researcherId(i));
/** 正向的实现、修复、核验共用阶段开关和派发规则,每个候选固定一个人。 */
function assignedStage(
step: string,
round: number,
kind: "implementation" | "repair" | "verification",
candidates: readonly string[],
view: View,
frozen: Frozen,
): Next | null {
const actorOf = kind === "verification" ? verifierId : experimenterId;
const tasks = candidates.map((object) => ({ actor: actorOf(object), kind, object }));
return stageNext(stageIdOf(round, step), SUBMISSION_KINDS[kind].stage, round, () => tasks, view, frozen, tasks.map((t) => t.actor));
}
/**
* ⑨:实现全部关闭后开核验;核验全部关闭后,失败的才修复。每次只重验上次修复的候选。
* 第几次由已开关的阶段推出来;次数用完先写不跑事件,不登记运行(方法.md 4.3 ⑨)。
*/
function verificationNext(round: number, view: View, frozen: Frozen): Next | null {
let candidates = view.board.decision("batch", round)!.output.chosen;
for (let n = 1; candidates.length > 0; n++) {
const verificationId = stageIdOf(round, `verification-${n}`);
const verify = assignedStage(`verification-${n}`, round, "verification", candidates, view, frozen);
if (verify !== null) return verify;
const failed = view.board.published("verification")
.filter((s) => s.stageId === verificationId && !verificationOutcome(s.content).passed);
if (n > frozen.config.maxRepairs) {
for (const s of failed) {
if (view.events(FORWARD_NOT_RUN).some((e) => e.objects.includes(s.target!))) continue;
return { do: "forwardNotRun", candidate: s.target!, reason: `第 ${n} 次核验仍不通过,修复次数上限 ${frozen.config.maxRepairs} 已用完:${JSON.stringify(verificationOutcome(s.content))};${s.content.body}` };
}
return null;
}
candidates = failed.map((s) => s.target!);
if (candidates.length === 0) return null;
const repair = assignedStage(`repair-${n}`, round, "repair", candidates, view, frozen);
if (repair !== null) return repair;
}
return null;
}
/** ⑩:按选批顺序一次只交回一个运行;没启动的下次启动起新编号,不会把半批提前公开。 */
function forwardNext(round: number, view: View): Next | null {
for (const candidate of view.board.decision("batch", round)!.output.chosen) {
if (view.events(FORWARD_NOT_RUN).some((e) => e.objects.includes(candidate))) continue;
const runs = view.board.runs().filter((r) => r.plan.purpose === "forward" && r.plan.round === round && r.plan.object === candidate);
const last = runs.at(-1);
if (last !== undefined && last.result === null) throw new Error(`正向运行 ${last.plan.run} 登记了还没有结果:恢复入口应当已经核对过`);
if (last !== undefined && last.result!.process.ending !== "notStarted") continue;
return { do: "forwardRun", round, candidate, run: `r${round}-forward-${candidate}-${runs.length + 1}` };
}
return null;
}
/** 按种类取记录内容的小工具,加上 materials 读账本的那一份(Board)。 */ /** 按种类取记录内容的小工具,加上 materials 读账本的那一份(Board)。 */
class View { class View {
readonly #records: readonly LedgerRecord[]; readonly #records: readonly LedgerRecord[];
+10 -5
View File
@@ -1,4 +1,5 @@
import type { Content } from "../ledger/index.ts"; import type { Content } from "../ledger/index.ts";
import { Board, protocolMetrics } from "../materials/index.ts";
import type { HypothesisMetric } from "../vocab/index.ts"; import type { HypothesisMetric } from "../vocab/index.ts";
import { finishRun } from "./formal-run.ts"; import { finishRun } from "./formal-run.ts";
import { EVENTS } from "./next-step.ts"; import { EVENTS } from "./next-step.ts";
@@ -36,13 +37,17 @@ export async function recover(s: Study): Promise<{ halt: string } | "stopped" |
// 在跑或已结束:没写"确认启动"的补写(扣次),然后等它、判分、写结果。 // 在跑或已结束:没写"确认启动"的补写(扣次),然后等它、判分、写结果。
// 已经确认启动、容器却不见了,真实运行里没出现过,不专门写:runner.wait 会报错,按内部错误停下。 // 已经确认启动、容器却不见了,真实运行里没出现过,不专门写:runner.wait 会报错,按内部错误停下。
if (!started) s.ledger.append("runStarted", { run }); if (!started) s.ledger.append("runStarted", { run });
if ((await finishRun(s, run, metricsOf(plan.content))) === "stopped") return "stopped"; if ((await finishRun(s, run, metricsOf(s, plan.content))) === "stopped") return "stopped";
} }
return null; return null;
} }
/** 一次运行要测的假设专用指标。第 1 部分只有基线运行,它不测假设专用指标;正向、消融运行由第 3、4 部分补。 */ /** 恢复和第一次运行共用原登记指标定义(引用指标也保留原来的缺失规则)。 */
function metricsOf(plan: Content<"runPlanned">): HypothesisMetric[] { function metricsOf(s: Study, plan: Content<"runPlanned">): HypothesisMetric[] {
if (plan.purpose !== "baseline") throw new Error(`运行 ${plan.run} 是 ${plan.purpose} 运行,flow 第 1 部分还不会恢复它`); if (plan.purpose === "baseline") return [];
return []; if (plan.purpose === "forward") {
const board = new Board(s.ledger.records());
return protocolMetrics(board, board.protocol(plan.object!));
}
throw new Error(`运行 ${plan.run} 是消融运行,消融还没写(任务书 17)`);
} }
+6 -22
View File
@@ -3,7 +3,7 @@ import { runModelTask, type ModelTask } from "../model-task.ts";
import type { Next, TaskRef } from "../next-step.ts"; import type { Next, TaskRef } from "../next-step.ts";
import { halt, USER_STOPPED, type Study, type StudyEnding } from "../study.ts"; import { halt, USER_STOPPED, type Study, type StudyEnding } from "../study.ts";
// 派一个研究者阶段里要派的任务(任务书 12 第 2.6 节、任务书 14 第 2.3 节): // 派一个阶段里要派的任务(任务书 12 第 2.6 节、任务书 14 第 2.3 节):
// 同一个人的几个任务按先后跑,不同的人同时跑,各自一个会话。 // 同一个人的几个任务按先后跑,不同的人同时跑,各自一个会话。
/** /**
@@ -40,26 +40,10 @@ export async function dispatchStage(s: Study, next: Extract<Next, { do: "dispatc
return halt(s.ledger, `阶段 ${next.stageId} 有任务没交上,等这次派发的会话全部结束后停下:${missed.map((m) => `调用 ${m.call},${m.text}`).join(";")}`, missed.map((m) => m.call)); return halt(s.ledger, `阶段 ${next.stageId} 有任务没交上,等这次派发的会话全部结束后停下:${missed.map((m) => `调用 ${m.call},${m.text}`).join(";")}`, missed.map((m) => m.call));
} }
/** /** 一个任务的分份编号;实现、核验、修复都只交一份。 */
* 一个任务交上来之后才知道的东西和分份编号(submissions 的 KnownOf、任务书 14 第 2.3 节):
* 提案是本人本阶段已收下几条,按 proposalId 分份;挑刺按 critiqueId 分份;回应是作者那条原提案;其余没有。
*/
function modelTask(s: Study, stageId: string, t: TaskRef): ModelTask<TaskRef["kind"]> { function modelTask(s: Study, stageId: string, t: TaskRef): ModelTask<TaskRef["kind"]> {
const base = { kind: t.kind, actor: t.actor, stageId, object: t.object }; const round = new Board(s.ledger.records()).stage(stageId).round;
const board = new Board(s.ledger.records()); const item = t.kind === "proposal" ? (n: number) => proposalId(round, t.actor, n)
switch (t.kind) { : t.kind === "critique" ? (n: number) => critiqueId(t.object!, t.actor, n) : null;
case "proposal": { return { ...t, stageId, item };
const round = board.stage(stageId).round;
return { ...base, item: (n) => proposalId(round, t.actor, n), known: (_, others) => ({ accepted: others }) } as ModelTask<"proposal">;
}
case "critique":
return { ...base, item: (n) => critiqueId(t.object!, t.actor, n), known: () => null };
case "response": {
// 回应的候选只由一条原提案而来(几条合成的交统一)。
const original = board.proposal(board.proposalsOf(t.object!)[0]!).content;
return { ...base, item: null, known: () => ({ original }) } as ModelTask<"response">;
}
default:
return { ...base, item: null, known: () => null };
}
} }
+34
View File
@@ -0,0 +1,34 @@
import { Board, protocolMetrics, validScore } from "../../materials/index.ts";
import { adopt } from "../../rules/index.ts";
import type { Json } from "../../vocab/index.ts";
import { formalRun, type RunOutcome } from "../formal-run.ts";
import type { Study } from "../study.ts";
/** ⑩ 正向运行:用核验过的快照和原登记指标定义,复用正式运行的登记、扣次、判分入口。 */
export function forwardRun(s: Study, round: number, candidate: string, run: string): Promise<RunOutcome> {
const board = new Board(s.ledger.records());
return formalRun(s, { run, purpose: "forward", round, object: candidate }, board.code(candidate).version, protocolMetrics(board, board.protocol(candidate)));
}
/** ⑫ 采纳:只把本批有效实测分数交给 rules(adopt 内部用 adoptionLine);null 也写,整批运行据此公开。 */
export function adoptBatch(s: Study, round: number): void {
const board = new Board(s.ledger.records());
const candidates = board.decision("batch", round)!.output.chosen.flatMap((id) => {
const run = board.runs().filter((r) => r.plan.purpose === "forward" && r.plan.round === round && r.plan.object === id).at(-1);
const score = run === undefined ? null : validScore(run.result!.judged);
return score === null ? [] : [{ id, score }];
});
const input = {
candidates,
baseline: board.baselineScore(board.baselineOf(round)),
direction: s.frozen.facts.finalMetric.direction,
epsilon: s.frozen.facts.epsilon,
knownBest: s.frozen.facts.knownBest,
};
const output = adopt(input);
s.ledger.append("decision", {
decision: "adopt", round, rule: "adopt", ruleVersion: 1,
params: { direction: input.direction, epsilon: input.epsilon, knownBest: input.knownBest }, seed: null,
input: input as Json, output: output as Json,
});
}
+75
View File
@@ -0,0 +1,75 @@
import { join } from "node:path";
import type { Content } from "../ledger/index.ts";
import { codeHunks } from "../jev/index.ts";
import { Board, protocolMetrics, type CodeChanges } from "../materials/index.ts";
import type { KnownOf } from "../submissions/index.ts";
import { lightCheck, protectedChanges, readLightCheck } from "../tasks/index.ts";
import type { Json, Submission, SubmissionKind } from "../vocab/index.ts";
import { compareVersions, saveVersion } from "../workspace/index.ts";
import type { TaskRef } from "./next-step.ts";
import { callPaths, containerName, freshDir, workspaceRepo } from "./paths.ts";
import type { Study } from "./study.ts";
/** 实现取本批基线;修复取上次提交。核验、修复的材料差异始终相对本批基线。 */
export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes: CodeChanges | null; startVersion: string | null } {
if (t.kind !== "implementation" && t.kind !== "repair" && t.kind !== "verification") return { changes: null, startVersion: null };
const board = new Board(s.ledger.records());
const baseline = board.code(board.baselineOf(board.stage(t.stageId).round)).version;
if (t.kind === "implementation") return { changes: null, startVersion: baseline };
const version = board.code(t.object!).version;
return { changes: compareVersions(workspaceRepo(s.dir), baseline, version), startVersion: t.kind === "repair" ? version : null };
}
/**
* 提交时才知道的东西:提案份数、回应原提案,或实验员当前工作目录的快照和相对本批基线的改动。
* 快照写入 workspace 后才问 Jev;收下才把 code 和 submission 在同一个事务写进账本。
*/
export function prepareSubmission(
s: Study,
t: TaskRef & { stageId: string },
content: Submission<SubmissionKind>,
others: number,
workDir: string,
): { known: KnownOf<SubmissionKind>; code: Content<"code"> | null } {
const board = new Board(s.ledger.records());
if (t.kind === "proposal") return { known: { accepted: others }, code: null };
if (t.kind === "response") return { known: { original: board.proposal(board.proposalsOf(t.object!)[0]!).content }, code: null };
if (t.kind !== "implementation" && t.kind !== "repair") return { known: null, code: null };
const saved = saveVersion(workspaceRepo(s.dir), workDir, s.frozen.facts.unversionedNames);
if (!saved.saved) return { known: { code: saved }, code: null };
const baseline = board.code(board.baselineOf(board.stage(t.stageId).round)).version;
const changes = compareVersions(workspaceRepo(s.dir), baseline, saved.version);
const protocol = board.protocol(t.object!);
const impl = content as Submission<"implementation">;
return {
known: { code: {
saved: true,
protectedFiles: protectedChanges(s.task, changes.files.map((f) => f.path)),
hunks: codeHunks(changes),
protocol: { body: protocol.body, metrics: protocolMetrics(board, protocol) },
} },
code: {
object: t.object!, version: saved.version,
locations: { factorLocations: impl.factorLocations, safeguardLocations: impl.safeguardLocations, unassigned: impl.unassigned } as Json,
},
};
}
/** 每次模型调用自己的计次器;会话里记录检查结果,不记正式运行、不扣次数(任务书 16 第 2.2 节)。 */
export function lightChecker(s: Study, call: string, workDir: string): () => Promise<string> {
let used = 0;
const { maxCount, seconds } = s.frozen.config.lightChecks;
return async () => {
if (used >= maxCount) return `本次调用的轻量检查已用完(上限 ${maxCount} 次),不能再查。`;
const n = ++used;
const paths = callPaths(s.dir, call);
const checkDir = freshDir(join(paths.base, `check-${n}`));
const definition = lightCheck(s.task, workDir, checkDir, s.frozen.image, seconds);
const name = containerName(s.frozen.config.studyId, `${call}-check-${n}`);
const started = await s.runner.start(name, definition, checkDir);
if (!started.started) throw new Error(`调用 ${call} 第 ${n} 次轻量检查没有启动:${started.reason}`);
const result = await s.runner.wait(name, freshDir(join(paths.base, `check-${n}-logs`)));
await s.runner.remove(name);
return JSON.stringify({ used, maxCount, seconds, result: readLightCheck(s.task, checkDir, result) });
};
}
+12 -4
View File
@@ -1,5 +1,6 @@
import type { ChecklistOf } from "../vocab/index.ts"; import type { ChecklistOf, HypothesisMetric } from "../vocab/index.ts";
import { Draft, factorLines, KIND_TEXT, offText, registeredMetrics, taskPoints } from "./blocks.ts"; import { type Draft, factorLines, KIND_TEXT, offText, registeredMetrics, taskPoints } from "./blocks.ts";
import type { Board } from "./read.ts";
import { fenced, json } from "./format.ts"; import { fenced, json } from "./format.ts";
import { unassignedChanges } from "./locations.ts"; import { unassignedChanges } from "./locations.ts";
import { clauseKey, factorKey, splitCell, type FrozenProtocol, type LocationTable } from "./shapes.ts"; import { clauseKey, factorKey, splitCell, type FrozenProtocol, type LocationTable } from "./shapes.ts";
@@ -27,9 +28,16 @@ function taskSection(d: Draft, role: string, what: string): void {
} }
/** 协议要采集的指标:新登记的全文,引用的取原登记(方法.md 第五部分"引用别人登记过的指标时,沿用原登记的选法")。 */ /** 协议要采集的指标:新登记的全文,引用的取原登记(方法.md 第五部分"引用别人登记过的指标时,沿用原登记的选法")。 */
export function protocolMetrics(board: Board, p: FrozenProtocol): HypothesisMetric[] {
const registered = registeredMetrics(board);
return [...p.metrics.defined, ...p.metrics.reused.map((id) => {
const { candidate: _candidate, ...metric } = registered.find((x) => x.id === id)!;
return metric;
})];
}
function metricSection(d: Draft, p: FrozenProtocol): void { function metricSection(d: Draft, p: FrozenProtocol): void {
const registered = registeredMetrics(d.board); const all = protocolMetrics(d.board, p);
const all = [...p.metrics.defined, ...p.metrics.reused.map((id) => registered.find((x) => x.id === id)!)];
d.section("要实现的指标定义", [ d.section("要实现的指标定义", [
all.length === 0 ? "(这份协议没有要实现的假设专用指标)" : json(all), all.length === 0 ? "(这份协议没有要实现的假设专用指标)" : json(all),
"", "",
+7 -2
View File
@@ -1,12 +1,13 @@
// materials:每个任务能看到哪些材料、第一条消息怎么写、查询在材料范围内回答什么(docs/结构.md 第二部分), // materials:每个任务能看到哪些材料、第一条消息怎么写、查询在材料范围内回答什么(docs/结构.md 第二部分),
// 以及账本里只存成 Json 的几样内容的形状和编号规则(用户 2026-10-03 决定 2)。 // 以及账本里只存成 Json 的几样内容的形状和编号规则(用户 2026-10-03 决定 2)。
export { protocolMetrics } from "./assigned.ts";
export { beliefBook } from "./belief-book.ts"; export { beliefBook } from "./belief-book.ts";
export { finalEntries, type MaterialRecord, type MaterialSection, type ReadableVersion, type TaskMaterial } from "./blocks.ts"; export { finalEntries, type MaterialRecord, type MaterialSection, type ReadableVersion, type TaskMaterial } from "./blocks.ts";
export { buildMaterial } from "./build.ts"; export { buildMaterial } from "./build.ts";
export { type CodeChanges, type PromptFile, type Prompts, type StudyFacts, type TaskRequest } from "./inputs.ts"; export type { CodeChanges, PromptFile, Prompts, StudyFacts, TaskRequest } from "./inputs.ts";
export { queryMaterial, readPrompts } from "./prompts.ts"; export { queryMaterial, readPrompts } from "./prompts.ts";
// flow 推进时读账本也用这一份(本批基线、基线分数、留池候选、各候选合并了哪些原提案),不另写一份。 // flow 推进时读账本也用这一份(本批基线、基线分数、留池候选、各候选合并了哪些原提案),不另写一份。
export { Board } from "./read.ts"; export { Board, validScore } from "./read.ts";
export { export {
callId, callId,
candidateId, candidateId,
@@ -17,6 +18,10 @@ export {
critiqueId, critiqueId,
DECISION_KINDS, DECISION_KINDS,
entryId, entryId,
experimenterId,
verifierId,
FORWARD_NOT_RUN,
forwardNotRunEvent,
explanationId, explanationId,
factorId, factorId,
factorKey, factorKey,
+7
View File
@@ -25,6 +25,9 @@ export const ORIGINAL_BASELINE = "base";
/** 研究者:第 i 位(从 0 数)是 researcher-(i+1),与参考项目的写法相同,回放时不用改编号。 */ /** 研究者:第 i 位(从 0 数)是 researcher-(i+1),与参考项目的写法相同,回放时不用改编号。 */
export const researcherId = (i: number): string => `researcher-${i + 1}`; export const researcherId = (i: number): string => `researcher-${i + 1}`;
/** 每个候选固定一位实验员和核验员,修复、再次核验沿用编号。 */
export const experimenterId = (candidate: string): string => `experimenter-${candidate}`;
export const verifierId = (candidate: string): string => `verifier-${candidate}`;
/** 入口:作者在第零轮交的第 n 个入口,讨论时整份替换的新清单接着往后编(方法.md 4.2)。 */ /** 入口:作者在第零轮交的第 n 个入口,讨论时整份替换的新清单接着往后编(方法.md 4.2)。 */
export const entryId = (author: string, n: number): string => `e-${author}-${n}`; export const entryId = (author: string, n: number): string => `e-${author}-${n}`;
/** 提案:第 round 轮作者交的第 n 条,也是提交记录的 item。 */ /** 提案:第 round 轮作者交的第 n 条,也是提交记录的 item。 */
@@ -145,3 +148,7 @@ export type Settlement = {
*/ */
export const CELL_NOT_RUN = "cellNotRun"; export const CELL_NOT_RUN = "cellNotRun";
export const cellNotRunEvent = (cell: string, reason: string): Content<"event"> => ({ event: CELL_NOT_RUN, reason, objects: [cell] }); export const cellNotRunEvent = (cell: string, reason: string): Content<"event"> => ({ event: CELL_NOT_RUN, reason, objects: [cell] });
/** 正向候选核验不通过、修复次数用完而不跑,不登记正式运行、不扣次(方法.md 4.3 ⑨)。 */
export const FORWARD_NOT_RUN = "forwardNotRun";
export const forwardNotRunEvent = (candidate: string, reason: string): Content<"event"> => ({ event: FORWARD_NOT_RUN, reason, objects: [candidate] });
+72 -1
View File
@@ -1,9 +1,16 @@
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { spawnSync } from "node:child_process"; import { spawnSync } from "node:child_process";
import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { join } from "node:path"; import { join } from "node:path";
import { test, type TestContext } from "node:test"; import { test, type TestContext } from "node:test";
import { lightChecker } from "../src/flow/task-code.ts";
import { recover } from "../src/flow/recover.ts";
import { containerName, freshDir, runPaths } from "../src/flow/paths.ts";
import type { Study } from "../src/flow/study.ts";
import { Board, protocolMetrics, type FrozenProtocol } from "../src/materials/index.ts";
import { loadTask, formalRun as runDefinition, prepareRun } from "../src/tasks/index.ts";
import { runBalance } from "../src/rules/index.ts";
import { runStudy } from "../src/flow/index.ts"; import { runStudy } from "../src/flow/index.ts";
import { Ledger } from "../src/ledger/index.ts"; import { Ledger } from "../src/ledger/index.ts";
import { Runner } from "../src/runner/index.ts"; import { Runner } from "../src/runner/index.ts";
@@ -76,3 +83,67 @@ test("docker run 没启动:写结果\"没启动\"后停下,原因带 Docker
assert.match((ending as { reason: string }).reason, /运行 base-1 没有启动.*unknown or invalid runtime name: inforsi-test-no-such-runtime/); assert.match((ending as { reason: string }).reason, /运行 base-1 没有启动.*unknown or invalid runtime name: inforsi-test-no-such-runtime/);
assert.deepEqual(ledgerKinds(dir), ["config", "code", "runPlanned", "runResult", "event"]); assert.deepEqual(ledgerKinds(dir), ["config", "code", "runPlanned", "runResult", "event"]);
}); });
/** 不派模型,只接真实任务镜像、账本和运行器,检查 flow 的两处 Docker 接线。 */
async function runtimeStudy(t: TestContext, suffix: string): Promise<Study> {
const dir = tempDir(t);
const id = studyId(t, suffix);
const task = loadTask(TASK_DIR);
const runner = new Runner(null);
const image = await runner.buildImage(task.dir, `inforsi-${id}`);
const ledger = Ledger.open(join(dir, "ledger.sqlite"));
t.after(() => ledger.close());
return { dir, task, runner, ledger, stop: new AbortController().signal,
frozen: { config: { ...fullConfig(), studyId: id, lightChecks: { maxCount: 1, seconds: 7 } }, facts: task.facts, image, seed: 1 } } as Study;
}
test("flow 轻量检查:每次调用各自限次和限时,真实 Docker 只检查当前目录,不登记正式运行", { timeout: 120_000 }, async (t) => {
const s = await runtimeStudy(t, "light");
const work = join(s.dir, "work");
mkdirSync(work);
writeFileSync(join(work, "initial.py"), "raise RuntimeError('不应执行候选代码')\ndef run_packing():\n pass\n");
for (const call of ["first-call", "second-call"]) {
freshDir(join(s.dir, "calls", call));
const check = lightChecker(s, call, work);
const result = JSON.parse(await check());
assert.deepEqual(result, { used: 1, maxCount: 1, seconds: 7, result: { passed: true } });
assert.match(await check(), /已用完(上限 1 次)/);
assert.equal(await s.runner.status(containerName(s.frozen.config.studyId, `${call}-check-1`)), "absent");
}
assert.deepEqual(s.ledger.records(), []);
});
test("正向恢复:接上原容器不重跑,补启动只扣一次,引用指标沿用原登记缺失作废规则", { timeout: 120_000 }, async (t) => {
const s = await runtimeStudy(t, "forward-recover");
const metric = { id: "old-c.required", meaning: "必测量", unit: "次", direction: "maximize" as const, howMeasured: "原登记定义", missing: "invalidateRun" as const };
const original: FrozenProtocol = { round: 0, baseline: "base", proposals: [], factors: [], safeguards: [], metrics: { defined: [metric], reused: [] }, clauses: [], body: "登记" };
const current: FrozenProtocol = { ...original, round: 1, metrics: { defined: [], reused: [metric.id] } };
s.ledger.append("protocol", { candidate: "old-c", protocol: original });
s.ledger.append("protocol", { candidate: "r1-c1", protocol: current });
const board = new Board(s.ledger.records());
const run = "forward-1";
const paths = runPaths(s.dir, run);
const definition = runDefinition(s.task, s.frozen.image, "1");
s.ledger.append("runPlanned", { run, purpose: "forward", round: 1, object: "r1-c1", definition });
freshDir(paths.base);
prepareRun(s.task, s.task.baselineDir, freshDir(paths.run), protocolMetrics(board, current));
const name = containerName(s.frozen.config.studyId, run);
t.after(() => { const result = docker(["rm", "-f", name]); if (result.status !== 0 && !result.stderr.includes("No such container")) throw new Error(result.stderr); });
assert.deepEqual(await s.runner.start(name, definition, paths.run), { started: true });
// 中断点:容器已启动,账本还没写 runStarted。恢复不调用 formalRun,而是找到原容器等它。
assert.equal(await recover(s), null);
const records = s.ledger.records();
assert.equal(records.filter((r) => r.kind === "runPlanned").length, 1);
assert.equal(records.filter((r) => r.kind === "runStarted").length, 1);
assert.equal(records.filter((r) => r.kind === "runResult").length, 1);
const result = records.find((r) => r.kind === "runResult")!;
if (result.kind !== "runResult") throw new Error("缺运行结果");
assert.equal(result.content.judged!.verdict, "invalid");
assert.match(result.content.judged!.verdictReason, /old-c.required/);
assert.equal(result.content.judged!.metrics[metric.id]!.status, "unmeasured");
assert.deepEqual(runBalance(13, new Board(records).runStates()), { charged: 1, reserved: 0, balance: 12 });
assert.equal(await s.runner.status(name), "absent");
assert.equal(await recover(s), null);
assert.equal(s.ledger.records().length, records.length);
});
+145
View File
@@ -0,0 +1,145 @@
import assert from "node:assert/strict";
import { mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { test, type TestContext } from "node:test";
import { adoptBatch } from "../src/flow/steps/forward.ts";
import { prepareSubmission, taskCode } from "../src/flow/task-code.ts";
import type { Study } from "../src/flow/study.ts";
import { Board, buildMaterial, protocolMetrics, type FrozenProtocol } from "../src/materials/index.ts";
import { Ledger } from "../src/ledger/index.ts";
import { loadTask } from "../src/tasks/index.ts";
import { checkoutVersion, createRepository, saveVersion } from "../src/workspace/index.ts";
import type { Submission } from "../src/vocab/index.ts";
import { fullConfig } from "./config-helpers.ts";
import { implementation, tempDir } from "./ledger-helpers.ts";
import { judged, prompts, study, Writer } from "./materials-helpers.ts";
function setup(t: TestContext): { s: Study; w: Writer } {
const dir = tempDir(t);
const ledger = Ledger.open(join(dir, "ledger.sqlite"));
t.after(() => ledger.close());
const task = loadTask(join(import.meta.dirname, "..", "tasks", "circle_packing"));
const s = { dir, ledger, task, prompts, facts: study, frozen: { config: fullConfig(), facts: task.facts } } as Study;
return { s, w: new Writer(ledger) };
}
// 预期值手算:基线 1,ε=1/64,已知最好 2;线=1/64。1+1/64 正好在线上,不采纳。
for (const [label, scores, adopted, gain] of [
["高于线且同分:取自然编号小的", [1.125, 1.125], "r1-c2", 0.125],
["正好等于线:也写 null 决定", [1.015625, 1], null, 0.015625],
["整批没有有效分数:也写 null 决定", [], null, null],
] as const) {
test(`采纳写账:${label},写决定后整批公开`, (t) => {
const { s, w } = setup(t);
s.frozen.facts = { ...s.frozen.facts, epsilon: 0.015625, knownBest: 2 };
w.run("base-1", "baseline", 0, null, judged(1, "main"));
w.decision("batch", 1, null, { chosen: ["r1-c10", "r1-c2", "r1-c3"] });
scores.forEach((score, i) => w.run(["run-first", "run-second"][i]!, "forward", 1, i === 0 ? "r1-c10" : "r1-c2", judged(score, "main")));
w.run("invalid", "forward", 1, "r1-c3", { ...judged(99, "main"), verdict: "invalid", verdictReason: "不合法" });
// 不在本批的运行不参与,即使它分数更高;上轮不能混进本轮。
w.run("other", "forward", 0, "old-candidate", judged(100, "main"));
const before = new Board(s.ledger.records());
assert.ok(before.runs().filter((r) => r.plan.round === 1).every((r) => !before.runPublic(r)));
adoptBatch(s, 1);
const record = s.ledger.records().at(-1)!;
assert.equal(record.kind, "decision");
if (record.kind !== "decision") throw new Error("应写采纳决定");
assert.deepEqual(record.content, {
decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1,
params: { direction: "maximize", epsilon: 0.015625, knownBest: 2 }, seed: null,
input: { candidates: scores.map((score, i) => ({ id: i === 0 ? "r1-c10" : "r1-c2", score })), baseline: 1, direction: "maximize", epsilon: 0.015625, knownBest: 2 },
output: { line: 0.015625, quarter: 0.25, adopted,
best: scores.length === 0 ? null : { id: adopted ?? "r1-c10", score: scores[0], improvement: gain } },
});
const after = new Board(s.ledger.records());
assert.ok(after.runs().filter((r) => r.plan.round === 1).every((r) => after.runPublic(r)));
assert.equal(after.baselineOf(2), adopted ?? "base");
});
}
const metric = { id: "old-c.metric", meaning: "试验值", unit: "次", direction: "maximize" as const, howMeasured: "按原登记计数", missing: "invalidateRun" as const };
const oldProtocol: FrozenProtocol = { round: 0, baseline: "base", proposals: [], factors: [], safeguards: [], metrics: { defined: [metric], reused: [] }, clauses: [], body: "原登记" };
const protocol: FrozenProtocol = { round: 1, baseline: "base", proposals: [], factors: [{ name: "改动", method: "照协议改动", keySettings: "无", continues: null }], safeguards: [], metrics: { defined: [], reused: [metric.id] }, clauses: ["照协议"], body: "冻结正文" };
function codeStudy(t: TestContext) {
const { s, w } = setup(t);
const repo = join(s.dir, "repo.git");
createRepository(repo);
const workDir = join(s.dir, "work");
mkdirSync(workDir);
writeFileSync(join(workDir, "initial.py"), "first = 1\nsecond = 2\n");
writeFileSync(join(workDir, "evaluate.py"), "protected = True\n");
const baseline = saveVersion(repo, workDir, []);
if (!baseline.saved) throw new Error(baseline.reason);
s.ledger.append("code", { object: "base", version: baseline.version, locations: null });
w.protocol("old-c", oldProtocol);
w.protocol("r1-c1", protocol);
w.open("r1-implementation", "implementation", 1, ["experimenter-r1-c1"]);
return { s, w, repo, workDir, baseline: baseline.version };
}
const impl: Submission<"implementation"> = { ...implementation("实现说明"), factorLocations: [{ factor: "p1", locations: ["initial.py:1-2"] }] };
const ref = { actor: "experimenter-r1-c1", object: "r1-c1", kind: "implementation" as const, stageId: "r1-implementation" };
test("提交准备:快照、保护文件、差异块和引用指标同源;修复取上版,但差异永远相对本批基线", (t) => {
const { s, w, repo, workDir, baseline } = codeStudy(t);
assert.deepEqual(taskCode(s, ref), { changes: null, startVersion: baseline });
writeFileSync(join(workDir, "initial.py"), "first = 10\nsecond = 2\n");
const prepared = prepareSubmission(s, ref, impl, 0, workDir);
assert.ok(prepared.code);
const known = prepared.known as { code: { saved: true; protectedFiles: string[]; hunks: { text: string }[]; protocol: unknown } };
assert.deepEqual(known.code.protectedFiles, []);
assert.deepEqual(known.code.protocol, { body: "冻结正文", metrics: [metric] });
assert.match(known.code.hunks[0]!.text, /-first = 1\n\+first = 10/);
s.ledger.transaction(() => {
w.sub(ref.stageId, ref.actor, ref.kind, ref.object, null, impl);
s.ledger.append("code", prepared.code!);
});
w.close(ref.stageId);
w.open("r1-verification-1", "verification", 1, ["verifier-r1-c1"]);
w.sub("r1-verification-1", "verifier-r1-c1", "verification", "r1-c1", null, { clauses: [{ clause: "c1", found: false, where: "第二行未实现" }], factors: [{ factor: "p1", found: false, where: "第二行未实现" }], safeguards: [], unassigned: [], body: "只修第二行" });
w.close("r1-verification-1");
w.open("r1-repair-1", "implementation", 1, [ref.actor]);
const repair = { ...ref, kind: "repair" as const, stageId: "r1-repair-1" };
const previous = taskCode(s, repair);
assert.equal(previous.startVersion, prepared.code.version);
assert.match(previous.changes!.diff, /-first = 1\n\+first = 10/);
const repairMaterial = buildMaterial({ ...repair, ledger: s.ledger, study, prompts, codeRoot: join(s.dir, "readable"), changes: previous.changes, artifacts: null });
assert.match(repairMaterial.firstMessage, /工作目录是上一次提交的代码的副本/);
assert.match(repairMaterial.firstMessage, /第二行未实现/);
writeFileSync(join(workDir, "initial.py"), "first = 10\nsecond = 20\n");
const fixed = prepareSubmission(s, repair, impl, 0, workDir);
const fixedKnown = fixed.known as typeof known;
assert.match(fixedKnown.code.hunks[0]!.text, /-first = 1/);
assert.match(fixedKnown.code.hunks[0]!.text, /\+second = 20/);
s.ledger.transaction(() => {
w.sub(repair.stageId, repair.actor, repair.kind, repair.object, null, impl);
s.ledger.append("code", fixed.code!);
});
w.close(repair.stageId);
w.open("r1-verification-2", "verification", 1, ["verifier-r1-c1"]);
const verification = { ...ref, kind: "verification" as const, stageId: "r1-verification-2" };
const compare = taskCode(s, verification);
assert.equal(compare.startVersion, null);
assert.match(compare.changes!.diff, /-first = 1/);
assert.match(compare.changes!.diff, /\+second = 20/);
assert.deepEqual(protocolMetrics(new Board(s.ledger.records()), protocol), [metric]);
const snapshot = join(s.dir, "snapshot");
mkdirSync(snapshot);
checkoutVersion(repo, prepared.code.version, snapshot);
assert.equal(readFileSync(join(snapshot, "initial.py"), "utf8"), "first = 10\nsecond = 2\n");
});
test("存不成版本的原原因交 submissions;改了保护文件时不能被漏掉", (t) => {
const { s, workDir } = codeStudy(t);
writeFileSync(join(workDir, "evaluate.py"), "protected = False\n");
const prepared = prepareSubmission(s, ref, impl, 0, workDir);
assert.deepEqual((prepared.known as { code: { protectedFiles: string[] } }).code.protectedFiles, ["evaluate.py"]);
symlinkSync("initial.py", join(workDir, "link.py"));
const refused = prepareSubmission(s, ref, impl, 0, workDir);
assert.equal(refused.code, null);
const code = (refused.known as { code: { saved: false; path: string; reason: string } }).code;
assert.equal(code.saved, false);
assert.equal(code.path, "link.py");
assert.match(code.reason, /符号链接/);
});
+153 -4
View File
@@ -4,9 +4,12 @@ import { join } from "node:path";
import { test } from "node:test"; import { test } from "node:test";
import { stepList, type ResearchConfig } from "../src/config/index.ts"; import { stepList, type ResearchConfig } from "../src/config/index.ts";
import type { AppendKind, Content, LedgerRecord } from "../src/ledger/index.ts"; import type { AppendKind, Content, LedgerRecord } from "../src/ledger/index.ts";
import type { StudyRecord } from "../src/materials/index.ts"; import { Board, forwardNotRunEvent, type StudyRecord } from "../src/materials/index.ts";
import { runBalance } from "../src/rules/index.ts";
import { definition, exited } from "./ledger-helpers.ts";
import { judged } from "./materials-helpers.ts";
import { nextStep } from "../src/flow/next-step.ts"; import { nextStep } from "../src/flow/next-step.ts";
import type { Json, SubmissionKind } from "../src/vocab/index.ts"; import type { Json, Submission, SubmissionKind } from "../src/vocab/index.ts";
import { fullConfig } from "./config-helpers.ts"; import { fullConfig } from "./config-helpers.ts";
// flow 的"下一步做什么"(任务书 12 第六部分):纯函数,手写账本记录,看它给出的下一步。 // flow 的"下一步做什么"(任务书 12 第六部分):纯函数,手写账本记录,看它给出的下一步。
@@ -337,8 +340,8 @@ test("第二次预测关闭:选批", () => {
assert.deepEqual(nextStep(roundOne("secondPrediction").records), { do: "selectBatch", round: 1 }); assert.deepEqual(nextStep(roundOne("secondPrediction").records), { do: "selectBatch", round: 1 });
}); });
test("选批做完:停下,原因\"实现还没写\"", () => { test("选批做完:开实现阶段,每个入选候选固定一位实验员", () => {
assert.deepEqual(nextStep(roundOne("selectBatch").records), { do: "halt", reason: "实现还没写(任务书 16)", objects: [] }); assert.deepEqual(nextStep(roundOne("selectBatch").records), { do: "openStage", stageId: "r1-implementation", stage: "implementation", round: 1, roster: ["experimenter-r1-c1"] });
}); });
test("关掉第一次预测:第 1 轮写一条按配置跳过(对象 r1-firstPrediction),接着开挑刺", () => { test("关掉第一次预测:第 1 轮写一条按配置跳过(对象 r1-firstPrediction),接着开挑刺", () => {
@@ -354,3 +357,149 @@ test("关掉第一次预测:第 1 轮写一条按配置跳过(对象 r1-firs
book.add("event", { event: "skippedByConfig", reason, objects: ["r1-firstPrediction"] }); book.add("event", { event: "skippedByConfig", reason, objects: ["r1-firstPrediction"] });
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-critique", stage: "critique", round: 1, roster: R }); assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-critique", stage: "critique", round: 1, roster: R });
}); });
// ⑨–⑫:预期动作手写,不调模型、Jev 或 Docker。
const X = ["experimenter-r1-c1", "experimenter-r1-c2"];
const V = ["verifier-r1-c1", "verifier-r1-c2"];
function batchTwo(maxRepairs: number): Book {
const book = roundOne("secondPrediction");
const record = book.records.find((r) => r.kind === "config")!;
(record.content as unknown as StudyRecord).config = { ...fullConfig(), maxRepairs } as unknown as Json;
return book.add("decision", { decision: "batch", round: 1, rule: "bestExpected", ruleVersion: 1, params: null, seed: "s", input: null, output: { chosen: C } });
}
function implemented(book: Book): Book {
book.open("r1-implementation", "implementation", 1, X);
for (const [i, candidate] of C.entries()) book.sub("r1-implementation", X[i]!, "implementation", candidate, null);
return book.add("stageClose", { stageId: "r1-implementation" });
}
function verified(book: Book, n: number, candidate: string, passed: boolean): Book {
// 对应真实 15-tight-2 的 submissions created_seq=173(墙约束行和右端错位)、194(修复后通过)。
// 只转录驱动程序所需的逐项 found,不伪造 Jev 的判断或模型运行。
const content: Submission<"verification"> = {
clauses: [{ clause: "c1", found: passed, where: passed ? "initial.py 墙约束行和右端已对齐" : "initial.py:225-228 与 243-246:墙约束行和右端错位" }],
factors: [{ factor: "p1", found: passed, where: "LP 墙约束" }],
safeguards: [], unassigned: [], body: passed ? "通过" : "解的不是协议写的线性规划",
};
return book.add("submission", { stageId: `r1-verification-${n}`, author: `verifier-${candidate}`, kind: "verification", target: candidate, item: null, version: 1, content, jev: null });
}
function firstVerification(book: Book, a: boolean, b: boolean): Book {
implemented(book).open("r1-verification-1", "verification", 1, V);
verified(book, 1, C[0]!, a);
verified(book, 1, C[1]!, b);
return book.add("stageClose", { stageId: "r1-verification-1" });
}
function recordForward(book: Book, candidate: string, n: number, ending: "valid" | "notStarted" | "timeout"): Book {
const run = `r1-forward-${candidate}-${n}`;
book.add("runPlanned", { run, purpose: "forward", round: 1, object: candidate, definition });
if (ending === "notStarted") return book.add("runResult", { run, process: { ending, reason: "Docker 未启动" }, judged: null });
book.add("runStarted", { run });
const process = ending === "timeout" ? { ending: "timeout" as const, wallSeconds: measured(120), cpuSeconds: measured(100), peakMemoryBytes: measured(1024) } : exited;
return book.add("runResult", { run, process, judged: ending === "valid" ? judged(1.25, "main") : null });
}
const forward = (candidate: string, n: number) => ({ do: "forwardRun", round: 1, candidate, run: `r1-forward-${candidate}-${n}` });
test("实现的任务同时可派;已提交后中断不重派,等全体交齐关阶段后才核验", () => {
const book = batchTwo(1).open("r1-implementation", "implementation", 1, X);
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-implementation", tasks: C.map((object, i) => ({ actor: X[i], kind: "implementation", object })) });
book.sub("r1-implementation", X[0]!, "implementation", C[0]!, null);
book.call("r1-implementation", X[0]!, "implementation", C[0]!, 1, { wallSeconds: { status: "unmeasured", reason: "进程中断" }, status: "failed", reason: "进程中断" });
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-implementation", tasks: [{ actor: X[1], kind: "implementation", object: C[1] }] });
book.sub("r1-implementation", X[1]!, "implementation", C[1]!, null);
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-implementation" });
book.add("stageClose", { stageId: "r1-implementation" });
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-verification-1", stage: "verification", round: 1, roster: V });
});
test("15-tight-2 的一次修复路径:核验关后才派原实验员,修复关后只重验修过的候选", () => {
const book = implemented(batchTwo(1)).open("r1-verification-1", "verification", 1, V);
verified(book, 1, C[0]!, false);
verified(book, 1, C[1]!, true);
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-verification-1" });
book.add("stageClose", { stageId: "r1-verification-1" });
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-repair-1", stage: "implementation", round: 1, roster: [X[0]] });
book.open("r1-repair-1", "implementation", 1, [X[0]!]);
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-repair-1", tasks: [{ actor: X[0], kind: "repair", object: C[0] }] });
book.sub("r1-repair-1", X[0]!, "repair", C[0]!, null);
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-repair-1" });
book.add("stageClose", { stageId: "r1-repair-1" });
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-verification-2", stage: "verification", round: 1, roster: [V[0]] });
book.open("r1-verification-2", "verification", 1, [V[0]!]);
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-verification-2", tasks: [{ actor: V[0], kind: "verification", object: C[0] }] });
verified(book, 2, C[0]!, true);
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-verification-2" });
book.add("stageClose", { stageId: "r1-verification-2" });
assert.deepEqual(nextStep(book.records), forward(C[0]!, 1));
});
test("修复次数从关闭的阶段数出:两次修复后仍失败,逐候选写不跑事件,不占预算", () => {
const book = firstVerification(batchTwo(2), false, false);
for (let n = 1; n <= 2; n++) {
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: `r1-repair-${n}`, stage: "implementation", round: 1, roster: X });
book.open(`r1-repair-${n}`, "implementation", 1, X);
for (const [i, candidate] of C.entries()) book.sub(`r1-repair-${n}`, X[i]!, "repair", candidate, null);
book.add("stageClose", { stageId: `r1-repair-${n}` });
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: `r1-verification-${n + 1}`, stage: "verification", round: 1, roster: V });
book.open(`r1-verification-${n + 1}`, "verification", 1, V);
for (const candidate of C) verified(book, n + 1, candidate, false);
book.add("stageClose", { stageId: `r1-verification-${n + 1}` });
}
for (const candidate of C) {
const next = nextStep(book.records);
assert.equal(next.do, "forwardNotRun");
if (next.do !== "forwardNotRun") throw new Error("应写不跑事件");
assert.equal(next.candidate, candidate);
assert.match(next.reason, /第 3 次核验仍不通过,修复次数上限 2 已用完/);
assert.match(next.reason, /解的不是协议写的线性规划/);
book.add("event", forwardNotRunEvent(candidate, next.reason));
}
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 });
assert.deepEqual(nextStep(book.records), { do: "adopt", round: 1 });
});
test("maxRepairs=0:第一次核验失败就记不跑,另一候选仍可串行运行", () => {
const book = firstVerification(batchTwo(0), false, true);
const next = nextStep(book.records);
assert.equal(next.do, "forwardNotRun");
if (next.do !== "forwardNotRun") throw new Error("应写不跑事件");
book.add("event", forwardNotRunEvent(next.candidate, next.reason));
assert.deepEqual(nextStep(book.records), forward(C[1]!, 1));
});
test("正向串行:上次未启动起新编号;已登记未结束必须先恢复;超时也扣次且不重跑;采纳前不公开", () => {
const book = firstVerification(batchTwo(1), true, true);
assert.deepEqual(nextStep(book.records), forward(C[0]!, 1));
recordForward(book, C[0]!, 1, "notStarted");
assert.deepEqual(nextStep(book.records), forward(C[0]!, 2));
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 });
const run = `r1-forward-${C[0]}-2`;
book.add("runPlanned", { run, purpose: "forward", round: 1, object: C[0]!, definition });
assert.throws(() => nextStep(book.records), /登记了还没有结果:恢复入口应当已经核对过/);
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 1, balance: 11 });
book.add("runStarted", { run });
assert.throws(() => nextStep(book.records), /登记了还没有结果/);
book.add("runResult", { run, process: exited, judged: judged(1.25, "main") });
assert.deepEqual(nextStep(book.records), forward(C[1]!, 1));
recordForward(book, C[1]!, 1, "timeout");
assert.deepEqual(nextStep(book.records), { do: "adopt", round: 1 });
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 3, reserved: 0, balance: 10 });
const before = new Board(book.records);
assert.ok(before.runs().filter((r) => r.plan.purpose === "forward").every((r) => !before.runPublic(r)));
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: null } });
const after = new Board(book.records);
assert.ok(after.runs().filter((r) => r.plan.purpose === "forward").every((r) => after.runPublic(r)));
assert.deepEqual(nextStep(book.records), { do: "halt", reason: "消融还没写(任务书 17)", objects: [] });
});
test("采纳只有一个因子的候选也统一在 listCells 停下,不擅自开收尾", () => {
const book = firstVerification(batchTwo(1), true, true);
for (const c of C) recordForward(book, c, 1, "valid");
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: C[0]! } });
assert.deepEqual(nextStep(book.records), { do: "halt", reason: "消融还没写(任务书 17)", objects: [] });
});