feat: 接通实现核验修复到正向运行采纳
任务书16第二块:角色分派与每调用轻量检查,提交及代码原子记账,按次数核验修复且耗尽不跑,串行正式运行及恢复引用指标,采纳含null统一公开。451项免费测试通过,全新上下文独立审阅未发现问题;真实研究验收仍待进行。
This commit is contained in:
+11
-2
@@ -1,6 +1,7 @@
|
|||||||
import { mkdirSync } from "node:fs";
|
import { mkdirSync } from "node:fs";
|
||||||
import { resolve } from "node:path";
|
import { resolve } from "node:path";
|
||||||
import type { ResearchConfig } from "../config/index.ts";
|
import type { ResearchConfig } from "../config/index.ts";
|
||||||
|
import { forwardNotRunEvent } from "../materials/index.ts";
|
||||||
import { Ledger } from "../ledger/index.ts";
|
import { Ledger } from "../ledger/index.ts";
|
||||||
import { Runner } from "../runner/index.ts";
|
import { Runner } from "../runner/index.ts";
|
||||||
import { fullConfig, readFrozen } from "./frozen.ts";
|
import { fullConfig, readFrozen } from "./frozen.ts";
|
||||||
@@ -8,6 +9,7 @@ import { EVENTS, nextStep, type Next } from "./next-step.ts";
|
|||||||
import { ledgerFile } from "./paths.ts";
|
import { ledgerFile } from "./paths.ts";
|
||||||
import { recover } from "./recover.ts";
|
import { recover } from "./recover.ts";
|
||||||
import { baselineRun, saveBaseline } from "./steps/baseline.ts";
|
import { baselineRun, saveBaseline } from "./steps/baseline.ts";
|
||||||
|
import { forwardRun, adoptBatch } from "./steps/forward.ts";
|
||||||
import { dispatchStage } from "./steps/dispatch.ts";
|
import { dispatchStage } from "./steps/dispatch.ts";
|
||||||
import { dedupe, freezeProtocols, selectBatch } from "./steps/round.ts";
|
import { dedupe, freezeProtocols, selectBatch } from "./steps/round.ts";
|
||||||
import { assignEntries } from "./steps/round-zero.ts";
|
import { assignEntries } from "./steps/round-zero.ts";
|
||||||
@@ -75,8 +77,9 @@ async function doStep(s: Study, next: Exclude<Next, { do: "freezeConfig" }>): Pr
|
|||||||
case "saveBaseline":
|
case "saveBaseline":
|
||||||
saveBaseline(s);
|
saveBaseline(s);
|
||||||
return null;
|
return null;
|
||||||
case "baselineRun": {
|
case "baselineRun":
|
||||||
const ran = await baselineRun(s, next.run);
|
case "forwardRun": {
|
||||||
|
const ran = next.do === "baselineRun" ? await baselineRun(s, next.run) : await forwardRun(s, next.round, next.candidate, next.run);
|
||||||
if (ran === "stopped") return USER_STOPPED;
|
if (ran === "stopped") return USER_STOPPED;
|
||||||
if (ran === "done") return null;
|
if (ran === "done") return null;
|
||||||
return halt(s.ledger, `运行 ${next.run} 没有启动,名额已退还;本进程不重跑,下次启动时起新编号重跑。Docker 的报错:${ran.notStarted}`, [next.run]);
|
return halt(s.ledger, `运行 ${next.run} 没有启动,名额已退还;本进程不重跑,下次启动时起新编号重跑。Docker 的报错:${ran.notStarted}`, [next.run]);
|
||||||
@@ -104,6 +107,12 @@ async function doStep(s: Study, next: Exclude<Next, { do: "freezeConfig" }>): Pr
|
|||||||
case "selectBatch":
|
case "selectBatch":
|
||||||
selectBatch(s, next.round);
|
selectBatch(s, next.round);
|
||||||
return null;
|
return null;
|
||||||
|
case "forwardNotRun":
|
||||||
|
s.ledger.append("event", forwardNotRunEvent(next.candidate, next.reason));
|
||||||
|
return null;
|
||||||
|
case "adopt":
|
||||||
|
adoptBatch(s, next.round);
|
||||||
|
return null;
|
||||||
case "halt":
|
case "halt":
|
||||||
return halt(s.ledger, next.reason, next.objects);
|
return halt(s.ledger, next.reason, next.objects);
|
||||||
}
|
}
|
||||||
|
|||||||
+38
-24
@@ -1,8 +1,9 @@
|
|||||||
import { runSession, type KindOf, type SessionEnding, type SubmitAttempt, type SubmitReceipt } from "../agents/index.ts";
|
import { runSession, type SessionTask, type SessionEnding, type SubmitAttempt, type SubmitReceipt } from "../agents/index.ts";
|
||||||
import { buildMaterial, callId, queryMaterial, researcherId, type TaskMaterial } from "../materials/index.ts";
|
import { buildMaterial, callId, queryMaterial, researcherId, type TaskMaterial } from "../materials/index.ts";
|
||||||
import { receive, type Incoming, type KnownOf } from "../submissions/index.ts";
|
import { receive, type Incoming } from "../submissions/index.ts";
|
||||||
import type { JevJudgment, Json, Submission } from "../vocab/index.ts";
|
import { SUBMISSION_KINDS, type JevJudgment, type Json, type SubmissionKind } from "../vocab/index.ts";
|
||||||
import { checkoutVersion } from "../workspace/index.ts";
|
import { checkoutVersion } from "../workspace/index.ts";
|
||||||
|
import { prepareSubmission, taskCode, lightChecker } from "./task-code.ts";
|
||||||
import { callPaths, freshDir, workspaceRepo } from "./paths.ts";
|
import { callPaths, freshDir, workspaceRepo } from "./paths.ts";
|
||||||
import type { Study } from "./study.ts";
|
import type { Study } from "./study.ts";
|
||||||
|
|
||||||
@@ -10,26 +11,26 @@ import type { Study } from "./study.ts";
|
|||||||
// 最后写"模型调用结束",前后顺序由这个入口保证(方法.md 第九部分"先登记、再执行、再登记结果")。
|
// 最后写"模型调用结束",前后顺序由这个入口保证(方法.md 第九部分"先登记、再执行、再登记结果")。
|
||||||
// 提交收不收只由 submissions 判;形状不合格的由 agents 报上来,这里直接记退回("给 flow")。
|
// 提交收不收只由 submissions 判;形状不合格的由 agents 报上来,这里直接记退回("给 flow")。
|
||||||
|
|
||||||
/** 一个研究者任务:哪种提交、谁、哪个阶段、对象。 */
|
/** 一个模型任务:哪种提交、谁、哪个阶段、对象。 */
|
||||||
export type ModelTask<K extends KindOf<"researcher">> = {
|
export type ModelTask<K extends SubmissionKind> = {
|
||||||
kind: K;
|
kind: K;
|
||||||
actor: string;
|
actor: string;
|
||||||
stageId: string;
|
stageId: string;
|
||||||
object: string | null;
|
object: string | null;
|
||||||
/** 一个任务里能交几份的种类(提案、挑刺):第 n 份的编号(materials 的 proposalId、critiqueId),收下时分配;只交一份的为 null。 */
|
/** 一个任务里能交几份的种类(提案、挑刺):第 n 份的编号(materials 的 proposalId、critiqueId),收下时分配;只交一份的为 null。 */
|
||||||
item: ((n: number) => string) | null;
|
item: ((n: number) => string) | null;
|
||||||
/** 交上来之后才知道的东西(submissions 的 KnownOf)。others 是本任务已收下、而且不是这次要替换的那几份的份数。 */
|
|
||||||
known(content: Submission<K>, others: number): KnownOf<K>;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/** 派一次任务,等会话结束,交回调用编号和结局。 */
|
/** 派一次任务,等会话结束,交回调用编号和结局。 */
|
||||||
export async function runModelTask<K extends KindOf<"researcher">>(s: Study, t: ModelTask<K>): Promise<{ call: string; ending: SessionEnding }> {
|
export async function runModelTask<K extends SubmissionKind>(s: Study, t: ModelTask<K>): Promise<{ call: string; ending: SessionEnding }> {
|
||||||
const { ledger } = s;
|
const { ledger } = s;
|
||||||
const started = new Set(ledger.records().flatMap((r) => (r.kind === "callStarted" ? [r.content.call] : [])));
|
const started = new Set(ledger.records().flatMap((r) => (r.kind === "callStarted" ? [r.content.call] : [])));
|
||||||
let attempt = 1;
|
let attempt = 1;
|
||||||
while (started.has(callId(t.stageId, t.actor, t.kind, t.object, attempt))) attempt++;
|
while (started.has(callId(t.stageId, t.actor, t.kind, t.object, attempt))) attempt++;
|
||||||
const call = callId(t.stageId, t.actor, t.kind, t.object, attempt);
|
const call = callId(t.stageId, t.actor, t.kind, t.object, attempt);
|
||||||
const paths = callPaths(s.dir, call);
|
const paths = callPaths(s.dir, call);
|
||||||
|
const { changes, startVersion } = taskCode(s, t);
|
||||||
|
const role = SUBMISSION_KINDS[t.kind].role;
|
||||||
const material = buildMaterial({
|
const material = buildMaterial({
|
||||||
kind: t.kind,
|
kind: t.kind,
|
||||||
actor: t.actor,
|
actor: t.actor,
|
||||||
@@ -39,37 +40,44 @@ export async function runModelTask<K extends KindOf<"researcher">>(s: Study, t:
|
|||||||
study: s.facts,
|
study: s.facts,
|
||||||
prompts: s.prompts,
|
prompts: s.prompts,
|
||||||
codeRoot: paths.code,
|
codeRoot: paths.code,
|
||||||
changes: null,
|
changes,
|
||||||
artifacts: null,
|
artifacts: null,
|
||||||
});
|
});
|
||||||
const model = researcherModel(s, t.actor);
|
const model = role === "researcher" ? researcherModel(s, t.actor) : s.frozen.config.models[role];
|
||||||
// 先登记、后建目录:调用编号只在登记之后才算用掉,被杀在两者之间时下次的编号不会撞上旧目录。
|
// 先登记、后建目录:调用编号只在登记之后才算用掉,被杀在两者之间时下次的编号不会撞上旧目录。
|
||||||
ledger.append("callStarted", {
|
ledger.append("callStarted", {
|
||||||
call,
|
call,
|
||||||
actor: t.actor,
|
actor: t.actor,
|
||||||
stageId: t.stageId,
|
stageId: t.stageId,
|
||||||
model: `${model.provider}/${model.model}(思考档位 ${model.thinking})`,
|
model: `${model.provider}/${model.model}(思考档位 ${model.thinking})`,
|
||||||
|
// SAFETY: TaskMaterial 的全部字段都是可序列化数据,账本在写入时校验 Json。
|
||||||
materials: material as unknown as Json,
|
materials: material as unknown as Json,
|
||||||
});
|
});
|
||||||
freshDir(paths.base);
|
freshDir(paths.base);
|
||||||
freshDir(paths.code);
|
freshDir(paths.code);
|
||||||
for (const r of material.readable) checkoutVersion(workspaceRepo(s.dir), r.version, freshDir(r.dir));
|
for (const r of material.readable) checkoutVersion(workspaceRepo(s.dir), r.version, freshDir(r.dir));
|
||||||
|
const workDir = freshDir(paths.work);
|
||||||
|
if (startVersion !== null) checkoutVersion(workspaceRepo(s.dir), startVersion, workDir);
|
||||||
let reply = 0;
|
let reply = 0;
|
||||||
const outcome = await runSession({
|
const common = {
|
||||||
role: "researcher",
|
|
||||||
model,
|
model,
|
||||||
systemPrompt: s.prompts.roles.researcher.text,
|
systemPrompt: s.prompts.roles[role].text,
|
||||||
firstMessage: material.firstMessage,
|
firstMessage: material.firstMessage,
|
||||||
workDir: freshDir(paths.work),
|
workDir,
|
||||||
sessionDir: freshDir(paths.sessions),
|
sessionDir: freshDir(paths.sessions),
|
||||||
kind: t.kind,
|
kind: t.kind,
|
||||||
readableDirs: material.readable.map((r) => r.dir),
|
query: (ref: string) => queryMaterial(material, ref),
|
||||||
query: (ref) => queryMaterial(material, ref),
|
submit: (attempt: SubmitAttempt<K>) => submit(s, t, material, attempt, workDir),
|
||||||
submit: (attempt) => submit(s, t, material, attempt as SubmitAttempt<K>),
|
usage: (usage: Parameters<SessionTask["usage"]>[0]) => void ledger.append("callUsage", { call, reply: ++reply, usage }),
|
||||||
usage: (usage) => void ledger.append("callUsage", { call, reply: ++reply, usage }),
|
|
||||||
writeNote: (body) => void ledger.append("note", { author: t.actor, body }),
|
|
||||||
stop: s.stop,
|
stop: s.stop,
|
||||||
});
|
};
|
||||||
|
const readableDirs = material.readable.map((r) => r.dir);
|
||||||
|
const task = role === "experimenter"
|
||||||
|
? { ...common, role, lightCheck: lightChecker(s, call, workDir) }
|
||||||
|
: role === "verifier"
|
||||||
|
? { ...common, role, readableDirs }
|
||||||
|
: { ...common, role, readableDirs, writeNote: (body: string) => void ledger.append("note", { author: t.actor, body }) };
|
||||||
|
const outcome = await runSession(task as SessionTask);
|
||||||
// 会话文件就在这次调用的 sessions 目录里,不另记。
|
// 会话文件就在这次调用的 sessions 目录里,不另记。
|
||||||
const { wallSeconds, sessionFile: _sessionFile, ...ending } = outcome;
|
const { wallSeconds, sessionFile: _sessionFile, ...ending } = outcome;
|
||||||
ledger.append("callEnded", { call, wallSeconds: { status: "measured", value: wallSeconds }, ...ending });
|
ledger.append("callEnded", { call, wallSeconds: { status: "measured", value: wallSeconds }, ...ending });
|
||||||
@@ -85,26 +93,32 @@ function researcherModel(s: Study, actor: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** 一份提交收不收:先定写到哪一份,形状合格的再交给 submissions 判。 */
|
/** 一份提交收不收:先定写到哪一份,形状合格的再交给 submissions 判。 */
|
||||||
async function submit<K extends KindOf<"researcher">>(s: Study, t: ModelTask<K>, material: TaskMaterial, attempt: SubmitAttempt<K>): Promise<SubmitReceipt> {
|
async function submit<K extends SubmissionKind>(s: Study, t: ModelTask<K>, material: TaskMaterial, attempt: SubmitAttempt<K>, workDir: string): Promise<SubmitReceipt> {
|
||||||
const { ledger } = s;
|
const { ledger } = s;
|
||||||
const reject = (submitted: Json, reason: string, jev: JevJudgment[] | null): SubmitReceipt => {
|
const reject = (submitted: Json, reason: string, jev: JevJudgment[] | null): SubmitReceipt => {
|
||||||
ledger.append("rejection", { stageId: t.stageId, author: t.actor, kind: t.kind, target: t.object, submitted, reason, jev });
|
ledger.append("rejection", { stageId: t.stageId, author: t.actor, kind: t.kind, target: t.object, submitted, reason, jev });
|
||||||
return { accepted: false, reason };
|
return { accepted: false, reason };
|
||||||
};
|
};
|
||||||
if (attempt.shape === "invalid") return reject(attempt.raw, attempt.reason, null);
|
if (attempt.shape === "invalid") return reject(attempt.raw, attempt.reason, null);
|
||||||
|
// SAFETY: agents 已按 vocab 的提交字段校验,提交内容只含 Json 值。
|
||||||
const content = attempt.content as unknown as Json;
|
const content = attempt.content as unknown as Json;
|
||||||
const slot = slotOf(s, t, attempt.replaces);
|
const slot = slotOf(s, t, attempt.replaces);
|
||||||
if ("refuse" in slot) return reject(content, slot.refuse, null);
|
if ("refuse" in slot) return reject(content, slot.refuse, null);
|
||||||
|
const prepared = prepareSubmission(s, t, attempt.content, slot.others, workDir);
|
||||||
const verdict = await receive({
|
const verdict = await receive({
|
||||||
kind: t.kind,
|
kind: t.kind,
|
||||||
content: attempt.content,
|
content: attempt.content,
|
||||||
checklist: material.checklist,
|
checklist: material.checklist,
|
||||||
known: t.known(attempt.content, slot.others),
|
known: prepared.known,
|
||||||
jevModel: s.frozen.config.models.jev,
|
jevModel: s.frozen.config.models.jev,
|
||||||
} as Incoming);
|
} as Incoming);
|
||||||
if (!verdict.accepted) return reject(content, verdict.reason, verdict.jev);
|
if (!verdict.accepted) return reject(content, verdict.reason, verdict.jev);
|
||||||
const { stageId, actor: author, kind, object: target } = t;
|
const { stageId, actor: author, kind, object: target } = t;
|
||||||
ledger.append("submission", { stageId, author, kind, target, item: slot.item, version: slot.version, content: attempt.content, jev: verdict.jev } as never);
|
// 代码在提交时冻结;提交和代码记录同进同退,恢复不能看到“交了却没快照”的半截状态。
|
||||||
|
ledger.transaction(() => {
|
||||||
|
ledger.append("submission", { stageId, author, kind, target, item: slot.item, version: slot.version, content: attempt.content, jev: verdict.jev } as never);
|
||||||
|
if (prepared.code !== null) ledger.append("code", prepared.code);
|
||||||
|
});
|
||||||
return { accepted: true, id: slot.id };
|
return { accepted: true, id: slot.id };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -113,7 +127,7 @@ async function submit<K extends KindOf<"researcher">>(s: Study, t: ModelTask<K>,
|
|||||||
* 提交编号:能交几份的种类就是这一份的编号,只交一份的是"阶段.本人.种类.对象"。
|
* 提交编号:能交几份的种类就是这一份的编号,只交一份的是"阶段.本人.种类.对象"。
|
||||||
* 不写 replaces 是交新的一份,只交一份的种类已经交过就退回并说明该怎么写;写了 replaces 就必须是本任务已收下的某一份。
|
* 不写 replaces 是交新的一份,只交一份的种类已经交过就退回并说明该怎么写;写了 replaces 就必须是本任务已收下的某一份。
|
||||||
*/
|
*/
|
||||||
function slotOf<K extends KindOf<"researcher">>(
|
function slotOf<K extends SubmissionKind>(
|
||||||
s: Study,
|
s: Study,
|
||||||
t: ModelTask<K>,
|
t: ModelTask<K>,
|
||||||
replaces: string | null,
|
replaces: string | null,
|
||||||
|
|||||||
+89
-13
@@ -1,9 +1,9 @@
|
|||||||
import type { KindOf } from "../agents/index.ts";
|
|
||||||
import type { RoundStepId, StudyStepId } from "../config/index.ts";
|
import type { RoundStepId, StudyStepId } from "../config/index.ts";
|
||||||
import type { Content, LedgerRecord } from "../ledger/index.ts";
|
import type { Content, LedgerRecord } from "../ledger/index.ts";
|
||||||
import { Board, ORIGINAL_BASELINE, researcherId } from "../materials/index.ts";
|
import { Board, experimenterId, verifierId, FORWARD_NOT_RUN, ORIGINAL_BASELINE, researcherId } from "../materials/index.ts";
|
||||||
import { compareIds } from "../rules/index.ts";
|
import { compareIds } from "../rules/index.ts";
|
||||||
import type { Stage } from "../vocab/index.ts";
|
import { verificationOutcome } from "../submissions/index.ts";
|
||||||
|
import { SUBMISSION_KINDS, type Stage, type SubmissionKind } from "../vocab/index.ts";
|
||||||
import { readFrozen, type Frozen } from "./frozen.ts";
|
import { readFrozen, type Frozen } from "./frozen.ts";
|
||||||
|
|
||||||
// "下一步做什么"(docs/结构.md 第二部分"几条边界"):步骤的先后只写在这里。
|
// "下一步做什么"(docs/结构.md 第二部分"几条边界"):步骤的先后只写在这里。
|
||||||
@@ -40,8 +40,8 @@ export const EVENTS = {
|
|||||||
runStatusUnknown: "runStatusUnknown",
|
runStatusUnknown: "runStatusUnknown",
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
/** 一个研究者任务:一个人对一个对象(候选、原提案;提案和第零轮的任务没有对象)交一种提交。 */
|
/** 一个任务:一个人对一个对象交一种提交;提案和第零轮任务没有对象。 */
|
||||||
export type TaskRef = { actor: string; kind: KindOf<"researcher">; object: string | null };
|
export type TaskRef = { actor: string; kind: SubmissionKind; object: string | null };
|
||||||
|
|
||||||
export type Next =
|
export type Next =
|
||||||
| { do: "freezeConfig" }
|
| { do: "freezeConfig" }
|
||||||
@@ -55,6 +55,9 @@ export type Next =
|
|||||||
| { do: "dedupe"; round: number }
|
| { do: "dedupe"; round: number }
|
||||||
| { do: "freezeProtocols"; round: number }
|
| { do: "freezeProtocols"; round: number }
|
||||||
| { do: "selectBatch"; round: number }
|
| { do: "selectBatch"; round: number }
|
||||||
|
| { do: "forwardNotRun"; candidate: string; reason: string }
|
||||||
|
| { do: "forwardRun"; round: number; candidate: string; run: string }
|
||||||
|
| { do: "adopt"; round: number }
|
||||||
| { do: "halt"; reason: string; objects: string[] };
|
| { do: "halt"; reason: string; objects: string[] };
|
||||||
|
|
||||||
export function nextStep(records: readonly LedgerRecord[]): Next {
|
export function nextStep(records: readonly LedgerRecord[]): Next {
|
||||||
@@ -87,7 +90,7 @@ function studyStepNext(step: Exclude<StudyStepId, "rounds">, view: View, frozen:
|
|||||||
case "discussion2":
|
case "discussion2":
|
||||||
case "claim": {
|
case "claim": {
|
||||||
const { stage, kind } = ROUND_ZERO_STAGES[step];
|
const { stage, kind } = ROUND_ZERO_STAGES[step];
|
||||||
return stageNext(stageIdOf(0, step), stage, 0, (roster) => roster.map((actor) => ({ actor, kind, object: null })), view, frozen);
|
return stageNext(stageIdOf(0, step), stage, 0, (roster) => roster.map((actor) => ({ actor, kind, object: null })), view, frozen, researchers(frozen));
|
||||||
}
|
}
|
||||||
case "assignEntries":
|
case "assignEntries":
|
||||||
// 分配 → 第 0 轮的入口分配决定(decision,种类 entries)。
|
// 分配 → 第 0 轮的入口分配决定(decision,种类 entries)。
|
||||||
@@ -99,12 +102,12 @@ function studyStepNext(step: Exclude<StudyStepId, "rounds">, view: View, frozen:
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 一轮接一轮(结构.md 第三部分"每一轮"):第一个没做完的轮次里第一个没做完的步骤。
|
* 一轮接一轮(结构.md 第三部分"每一轮"):第一个没做完的轮次里第一个没做完的步骤。
|
||||||
* 每一步做完的记录都带轮次,做完的轮次整轮跳过;现在走到实现一步就停下,所以循环一定会返回。
|
* 每一步做完的记录都带轮次,做完的轮次整轮跳过;现在走到列消融格就停下,所以循环一定会返回。
|
||||||
*/
|
*/
|
||||||
function roundsNext(view: View, frozen: Frozen): Next {
|
function roundsNext(view: View, frozen: Frozen): Next {
|
||||||
for (let round = 1; ; round++) {
|
for (let round = 1; ; round++) {
|
||||||
for (const step of frozen.steps.round) {
|
for (const step of frozen.steps.round) {
|
||||||
const next = step.status === "skip" ? skipNext(stageIdOf(round, step.id), step.reason, view) : roundStepNext(step.id, round, view, frozen);
|
const next = step.status === "skip" && step.id !== "listCells" ? skipNext(stageIdOf(round, step.id), step.reason, view) : roundStepNext(step.id, round, view, frozen);
|
||||||
if (next !== null) return next;
|
if (next !== null) return next;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -122,7 +125,7 @@ function roundStepNext(step: RoundStepId, round: number, view: View, frozen: Fro
|
|||||||
case "critique":
|
case "critique":
|
||||||
case "response":
|
case "response":
|
||||||
case "secondPrediction":
|
case "secondPrediction":
|
||||||
return stageNext(stageIdOf(round, step), step, round, (roster) => roundTasks(step, round, roster, view.board), view, frozen);
|
return stageNext(stageIdOf(round, step), step, round, (roster) => roundTasks(step, round, roster, view.board), view, frozen, researchers(frozen));
|
||||||
case "dedupe":
|
case "dedupe":
|
||||||
// 判重合并 → 本轮的判重合并决定(decision,种类 dedup)。
|
// 判重合并 → 本轮的判重合并决定(decision,种类 dedup)。
|
||||||
return view.board.decision("dedup", round) === undefined ? { do: "dedupe", round } : null;
|
return view.board.decision("dedup", round) === undefined ? { do: "dedupe", round } : null;
|
||||||
@@ -138,10 +141,25 @@ function roundStepNext(step: RoundStepId, round: number, view: View, frozen: Fro
|
|||||||
case "selectBatch":
|
case "selectBatch":
|
||||||
// 选批 → 本轮的选批决定(decision,种类 batch)。
|
// 选批 → 本轮的选批决定(decision,种类 batch)。
|
||||||
return view.board.decision("batch", round) === undefined ? { do: "selectBatch", round } : null;
|
return view.board.decision("batch", round) === undefined ? { do: "selectBatch", round } : null;
|
||||||
case "implementation":
|
case "implementation": {
|
||||||
return { do: "halt", reason: "实现还没写(任务书 16)", objects: [] };
|
const candidates = view.board.decision("batch", round)!.output.chosen;
|
||||||
|
return assignedStage("implementation", round, "implementation", candidates, view, frozen);
|
||||||
|
}
|
||||||
|
case "verification":
|
||||||
|
case "repair":
|
||||||
|
return verificationNext(round, view, frozen);
|
||||||
|
case "forwardRun":
|
||||||
|
return forwardNext(round, view);
|
||||||
|
case "publish":
|
||||||
|
case "informationAccount":
|
||||||
|
// 不另写记录:整批运行已结束。采纳决定使结果整批公开,信息账从预测和结果现算。
|
||||||
|
return null;
|
||||||
|
case "adopt":
|
||||||
|
return view.board.decision("adopt", round) === undefined ? { do: "adopt", round } : null;
|
||||||
|
case "listCells":
|
||||||
|
return { do: "halt", reason: "消融还没写(任务书 17)", objects: [] };
|
||||||
default:
|
default:
|
||||||
throw new Error(`走到了第 ${round} 轮的 ${step}:implementation 一步应当先停下`);
|
throw new Error(`走到了第 ${round} 轮的 ${step}:listCells 一步应当先停下`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -215,10 +233,11 @@ function stageNext(
|
|||||||
tasksOf: (roster: readonly string[]) => TaskRef[],
|
tasksOf: (roster: readonly string[]) => TaskRef[],
|
||||||
view: View,
|
view: View,
|
||||||
frozen: Frozen,
|
frozen: Frozen,
|
||||||
|
roster: string[],
|
||||||
): Next | null {
|
): Next | null {
|
||||||
const open = view.board.stages.get(stageId);
|
const open = view.board.stages.get(stageId);
|
||||||
if (open === undefined) {
|
if (open === undefined) {
|
||||||
return { do: "openStage", stageId, stage, round, roster: frozen.config.models.researchers.map((_, i) => researcherId(i)) };
|
return { do: "openStage", stageId, stage, round, roster };
|
||||||
}
|
}
|
||||||
if (open.closed) return null;
|
if (open.closed) return null;
|
||||||
const todo = tasksOf(open.roster).filter((t) => !view.done(stageId, t));
|
const todo = tasksOf(open.roster).filter((t) => !view.done(stageId, t));
|
||||||
@@ -239,6 +258,63 @@ function stageNext(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 研究者阶段的冻结名单。实验员、核验员阶段则按本阶段的任务列名单。 */
|
||||||
|
const researchers = (frozen: Frozen): string[] => frozen.config.models.researchers.map((_, i) => researcherId(i));
|
||||||
|
|
||||||
|
/** 正向的实现、修复、核验共用阶段开关和派发规则,每个候选固定一个人。 */
|
||||||
|
function assignedStage(
|
||||||
|
step: string,
|
||||||
|
round: number,
|
||||||
|
kind: "implementation" | "repair" | "verification",
|
||||||
|
candidates: readonly string[],
|
||||||
|
view: View,
|
||||||
|
frozen: Frozen,
|
||||||
|
): Next | null {
|
||||||
|
const actorOf = kind === "verification" ? verifierId : experimenterId;
|
||||||
|
const tasks = candidates.map((object) => ({ actor: actorOf(object), kind, object }));
|
||||||
|
return stageNext(stageIdOf(round, step), SUBMISSION_KINDS[kind].stage, round, () => tasks, view, frozen, tasks.map((t) => t.actor));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* ⑨:实现全部关闭后开核验;核验全部关闭后,失败的才修复。每次只重验上次修复的候选。
|
||||||
|
* 第几次由已开关的阶段推出来;次数用完先写不跑事件,不登记运行(方法.md 4.3 ⑨)。
|
||||||
|
*/
|
||||||
|
function verificationNext(round: number, view: View, frozen: Frozen): Next | null {
|
||||||
|
let candidates = view.board.decision("batch", round)!.output.chosen;
|
||||||
|
for (let n = 1; candidates.length > 0; n++) {
|
||||||
|
const verificationId = stageIdOf(round, `verification-${n}`);
|
||||||
|
const verify = assignedStage(`verification-${n}`, round, "verification", candidates, view, frozen);
|
||||||
|
if (verify !== null) return verify;
|
||||||
|
const failed = view.board.published("verification")
|
||||||
|
.filter((s) => s.stageId === verificationId && !verificationOutcome(s.content).passed);
|
||||||
|
if (n > frozen.config.maxRepairs) {
|
||||||
|
for (const s of failed) {
|
||||||
|
if (view.events(FORWARD_NOT_RUN).some((e) => e.objects.includes(s.target!))) continue;
|
||||||
|
return { do: "forwardNotRun", candidate: s.target!, reason: `第 ${n} 次核验仍不通过,修复次数上限 ${frozen.config.maxRepairs} 已用完:${JSON.stringify(verificationOutcome(s.content))};${s.content.body}` };
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
candidates = failed.map((s) => s.target!);
|
||||||
|
if (candidates.length === 0) return null;
|
||||||
|
const repair = assignedStage(`repair-${n}`, round, "repair", candidates, view, frozen);
|
||||||
|
if (repair !== null) return repair;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** ⑩:按选批顺序一次只交回一个运行;没启动的下次启动起新编号,不会把半批提前公开。 */
|
||||||
|
function forwardNext(round: number, view: View): Next | null {
|
||||||
|
for (const candidate of view.board.decision("batch", round)!.output.chosen) {
|
||||||
|
if (view.events(FORWARD_NOT_RUN).some((e) => e.objects.includes(candidate))) continue;
|
||||||
|
const runs = view.board.runs().filter((r) => r.plan.purpose === "forward" && r.plan.round === round && r.plan.object === candidate);
|
||||||
|
const last = runs.at(-1);
|
||||||
|
if (last !== undefined && last.result === null) throw new Error(`正向运行 ${last.plan.run} 登记了还没有结果:恢复入口应当已经核对过`);
|
||||||
|
if (last !== undefined && last.result!.process.ending !== "notStarted") continue;
|
||||||
|
return { do: "forwardRun", round, candidate, run: `r${round}-forward-${candidate}-${runs.length + 1}` };
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/** 按种类取记录内容的小工具,加上 materials 读账本的那一份(Board)。 */
|
/** 按种类取记录内容的小工具,加上 materials 读账本的那一份(Board)。 */
|
||||||
class View {
|
class View {
|
||||||
readonly #records: readonly LedgerRecord[];
|
readonly #records: readonly LedgerRecord[];
|
||||||
|
|||||||
+10
-5
@@ -1,4 +1,5 @@
|
|||||||
import type { Content } from "../ledger/index.ts";
|
import type { Content } from "../ledger/index.ts";
|
||||||
|
import { Board, protocolMetrics } from "../materials/index.ts";
|
||||||
import type { HypothesisMetric } from "../vocab/index.ts";
|
import type { HypothesisMetric } from "../vocab/index.ts";
|
||||||
import { finishRun } from "./formal-run.ts";
|
import { finishRun } from "./formal-run.ts";
|
||||||
import { EVENTS } from "./next-step.ts";
|
import { EVENTS } from "./next-step.ts";
|
||||||
@@ -36,13 +37,17 @@ export async function recover(s: Study): Promise<{ halt: string } | "stopped" |
|
|||||||
// 在跑或已结束:没写"确认启动"的补写(扣次),然后等它、判分、写结果。
|
// 在跑或已结束:没写"确认启动"的补写(扣次),然后等它、判分、写结果。
|
||||||
// 已经确认启动、容器却不见了,真实运行里没出现过,不专门写:runner.wait 会报错,按内部错误停下。
|
// 已经确认启动、容器却不见了,真实运行里没出现过,不专门写:runner.wait 会报错,按内部错误停下。
|
||||||
if (!started) s.ledger.append("runStarted", { run });
|
if (!started) s.ledger.append("runStarted", { run });
|
||||||
if ((await finishRun(s, run, metricsOf(plan.content))) === "stopped") return "stopped";
|
if ((await finishRun(s, run, metricsOf(s, plan.content))) === "stopped") return "stopped";
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 一次运行要测的假设专用指标。第 1 部分只有基线运行,它不测假设专用指标;正向、消融运行由第 3、4 部分补。 */
|
/** 恢复和第一次运行共用原登记指标定义(引用指标也保留原来的缺失规则)。 */
|
||||||
function metricsOf(plan: Content<"runPlanned">): HypothesisMetric[] {
|
function metricsOf(s: Study, plan: Content<"runPlanned">): HypothesisMetric[] {
|
||||||
if (plan.purpose !== "baseline") throw new Error(`运行 ${plan.run} 是 ${plan.purpose} 运行,flow 第 1 部分还不会恢复它`);
|
if (plan.purpose === "baseline") return [];
|
||||||
return [];
|
if (plan.purpose === "forward") {
|
||||||
|
const board = new Board(s.ledger.records());
|
||||||
|
return protocolMetrics(board, board.protocol(plan.object!));
|
||||||
|
}
|
||||||
|
throw new Error(`运行 ${plan.run} 是消融运行,消融还没写(任务书 17)`);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { runModelTask, type ModelTask } from "../model-task.ts";
|
|||||||
import type { Next, TaskRef } from "../next-step.ts";
|
import type { Next, TaskRef } from "../next-step.ts";
|
||||||
import { halt, USER_STOPPED, type Study, type StudyEnding } from "../study.ts";
|
import { halt, USER_STOPPED, type Study, type StudyEnding } from "../study.ts";
|
||||||
|
|
||||||
// 派一个研究者阶段里要派的任务(任务书 12 第 2.6 节、任务书 14 第 2.3 节):
|
// 派一个阶段里要派的任务(任务书 12 第 2.6 节、任务书 14 第 2.3 节):
|
||||||
// 同一个人的几个任务按先后跑,不同的人同时跑,各自一个会话。
|
// 同一个人的几个任务按先后跑,不同的人同时跑,各自一个会话。
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -40,26 +40,10 @@ export async function dispatchStage(s: Study, next: Extract<Next, { do: "dispatc
|
|||||||
return halt(s.ledger, `阶段 ${next.stageId} 有任务没交上,等这次派发的会话全部结束后停下:${missed.map((m) => `调用 ${m.call},${m.text}`).join(";")}`, missed.map((m) => m.call));
|
return halt(s.ledger, `阶段 ${next.stageId} 有任务没交上,等这次派发的会话全部结束后停下:${missed.map((m) => `调用 ${m.call},${m.text}`).join(";")}`, missed.map((m) => m.call));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** 一个任务的分份编号;实现、核验、修复都只交一份。 */
|
||||||
* 一个任务交上来之后才知道的东西和分份编号(submissions 的 KnownOf、任务书 14 第 2.3 节):
|
|
||||||
* 提案是本人本阶段已收下几条,按 proposalId 分份;挑刺按 critiqueId 分份;回应是作者那条原提案;其余没有。
|
|
||||||
*/
|
|
||||||
function modelTask(s: Study, stageId: string, t: TaskRef): ModelTask<TaskRef["kind"]> {
|
function modelTask(s: Study, stageId: string, t: TaskRef): ModelTask<TaskRef["kind"]> {
|
||||||
const base = { kind: t.kind, actor: t.actor, stageId, object: t.object };
|
const round = new Board(s.ledger.records()).stage(stageId).round;
|
||||||
const board = new Board(s.ledger.records());
|
const item = t.kind === "proposal" ? (n: number) => proposalId(round, t.actor, n)
|
||||||
switch (t.kind) {
|
: t.kind === "critique" ? (n: number) => critiqueId(t.object!, t.actor, n) : null;
|
||||||
case "proposal": {
|
return { ...t, stageId, item };
|
||||||
const round = board.stage(stageId).round;
|
|
||||||
return { ...base, item: (n) => proposalId(round, t.actor, n), known: (_, others) => ({ accepted: others }) } as ModelTask<"proposal">;
|
|
||||||
}
|
|
||||||
case "critique":
|
|
||||||
return { ...base, item: (n) => critiqueId(t.object!, t.actor, n), known: () => null };
|
|
||||||
case "response": {
|
|
||||||
// 回应的候选只由一条原提案而来(几条合成的交统一)。
|
|
||||||
const original = board.proposal(board.proposalsOf(t.object!)[0]!).content;
|
|
||||||
return { ...base, item: null, known: () => ({ original }) } as ModelTask<"response">;
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return { ...base, item: null, known: () => null };
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
import { Board, protocolMetrics, validScore } from "../../materials/index.ts";
|
||||||
|
import { adopt } from "../../rules/index.ts";
|
||||||
|
import type { Json } from "../../vocab/index.ts";
|
||||||
|
import { formalRun, type RunOutcome } from "../formal-run.ts";
|
||||||
|
import type { Study } from "../study.ts";
|
||||||
|
|
||||||
|
/** ⑩ 正向运行:用核验过的快照和原登记指标定义,复用正式运行的登记、扣次、判分入口。 */
|
||||||
|
export function forwardRun(s: Study, round: number, candidate: string, run: string): Promise<RunOutcome> {
|
||||||
|
const board = new Board(s.ledger.records());
|
||||||
|
return formalRun(s, { run, purpose: "forward", round, object: candidate }, board.code(candidate).version, protocolMetrics(board, board.protocol(candidate)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** ⑫ 采纳:只把本批有效实测分数交给 rules(adopt 内部用 adoptionLine);null 也写,整批运行据此公开。 */
|
||||||
|
export function adoptBatch(s: Study, round: number): void {
|
||||||
|
const board = new Board(s.ledger.records());
|
||||||
|
const candidates = board.decision("batch", round)!.output.chosen.flatMap((id) => {
|
||||||
|
const run = board.runs().filter((r) => r.plan.purpose === "forward" && r.plan.round === round && r.plan.object === id).at(-1);
|
||||||
|
const score = run === undefined ? null : validScore(run.result!.judged);
|
||||||
|
return score === null ? [] : [{ id, score }];
|
||||||
|
});
|
||||||
|
const input = {
|
||||||
|
candidates,
|
||||||
|
baseline: board.baselineScore(board.baselineOf(round)),
|
||||||
|
direction: s.frozen.facts.finalMetric.direction,
|
||||||
|
epsilon: s.frozen.facts.epsilon,
|
||||||
|
knownBest: s.frozen.facts.knownBest,
|
||||||
|
};
|
||||||
|
const output = adopt(input);
|
||||||
|
s.ledger.append("decision", {
|
||||||
|
decision: "adopt", round, rule: "adopt", ruleVersion: 1,
|
||||||
|
params: { direction: input.direction, epsilon: input.epsilon, knownBest: input.knownBest }, seed: null,
|
||||||
|
input: input as Json, output: output as Json,
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import { join } from "node:path";
|
||||||
|
import type { Content } from "../ledger/index.ts";
|
||||||
|
import { codeHunks } from "../jev/index.ts";
|
||||||
|
import { Board, protocolMetrics, type CodeChanges } from "../materials/index.ts";
|
||||||
|
import type { KnownOf } from "../submissions/index.ts";
|
||||||
|
import { lightCheck, protectedChanges, readLightCheck } from "../tasks/index.ts";
|
||||||
|
import type { Json, Submission, SubmissionKind } from "../vocab/index.ts";
|
||||||
|
import { compareVersions, saveVersion } from "../workspace/index.ts";
|
||||||
|
import type { TaskRef } from "./next-step.ts";
|
||||||
|
import { callPaths, containerName, freshDir, workspaceRepo } from "./paths.ts";
|
||||||
|
import type { Study } from "./study.ts";
|
||||||
|
|
||||||
|
/** 实现取本批基线;修复取上次提交。核验、修复的材料差异始终相对本批基线。 */
|
||||||
|
export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes: CodeChanges | null; startVersion: string | null } {
|
||||||
|
if (t.kind !== "implementation" && t.kind !== "repair" && t.kind !== "verification") return { changes: null, startVersion: null };
|
||||||
|
const board = new Board(s.ledger.records());
|
||||||
|
const baseline = board.code(board.baselineOf(board.stage(t.stageId).round)).version;
|
||||||
|
if (t.kind === "implementation") return { changes: null, startVersion: baseline };
|
||||||
|
const version = board.code(t.object!).version;
|
||||||
|
return { changes: compareVersions(workspaceRepo(s.dir), baseline, version), startVersion: t.kind === "repair" ? version : null };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 提交时才知道的东西:提案份数、回应原提案,或实验员当前工作目录的快照和相对本批基线的改动。
|
||||||
|
* 快照写入 workspace 后才问 Jev;收下才把 code 和 submission 在同一个事务写进账本。
|
||||||
|
*/
|
||||||
|
export function prepareSubmission(
|
||||||
|
s: Study,
|
||||||
|
t: TaskRef & { stageId: string },
|
||||||
|
content: Submission<SubmissionKind>,
|
||||||
|
others: number,
|
||||||
|
workDir: string,
|
||||||
|
): { known: KnownOf<SubmissionKind>; code: Content<"code"> | null } {
|
||||||
|
const board = new Board(s.ledger.records());
|
||||||
|
if (t.kind === "proposal") return { known: { accepted: others }, code: null };
|
||||||
|
if (t.kind === "response") return { known: { original: board.proposal(board.proposalsOf(t.object!)[0]!).content }, code: null };
|
||||||
|
if (t.kind !== "implementation" && t.kind !== "repair") return { known: null, code: null };
|
||||||
|
const saved = saveVersion(workspaceRepo(s.dir), workDir, s.frozen.facts.unversionedNames);
|
||||||
|
if (!saved.saved) return { known: { code: saved }, code: null };
|
||||||
|
const baseline = board.code(board.baselineOf(board.stage(t.stageId).round)).version;
|
||||||
|
const changes = compareVersions(workspaceRepo(s.dir), baseline, saved.version);
|
||||||
|
const protocol = board.protocol(t.object!);
|
||||||
|
const impl = content as Submission<"implementation">;
|
||||||
|
return {
|
||||||
|
known: { code: {
|
||||||
|
saved: true,
|
||||||
|
protectedFiles: protectedChanges(s.task, changes.files.map((f) => f.path)),
|
||||||
|
hunks: codeHunks(changes),
|
||||||
|
protocol: { body: protocol.body, metrics: protocolMetrics(board, protocol) },
|
||||||
|
} },
|
||||||
|
code: {
|
||||||
|
object: t.object!, version: saved.version,
|
||||||
|
locations: { factorLocations: impl.factorLocations, safeguardLocations: impl.safeguardLocations, unassigned: impl.unassigned } as Json,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 每次模型调用自己的计次器;会话里记录检查结果,不记正式运行、不扣次数(任务书 16 第 2.2 节)。 */
|
||||||
|
export function lightChecker(s: Study, call: string, workDir: string): () => Promise<string> {
|
||||||
|
let used = 0;
|
||||||
|
const { maxCount, seconds } = s.frozen.config.lightChecks;
|
||||||
|
return async () => {
|
||||||
|
if (used >= maxCount) return `本次调用的轻量检查已用完(上限 ${maxCount} 次),不能再查。`;
|
||||||
|
const n = ++used;
|
||||||
|
const paths = callPaths(s.dir, call);
|
||||||
|
const checkDir = freshDir(join(paths.base, `check-${n}`));
|
||||||
|
const definition = lightCheck(s.task, workDir, checkDir, s.frozen.image, seconds);
|
||||||
|
const name = containerName(s.frozen.config.studyId, `${call}-check-${n}`);
|
||||||
|
const started = await s.runner.start(name, definition, checkDir);
|
||||||
|
if (!started.started) throw new Error(`调用 ${call} 第 ${n} 次轻量检查没有启动:${started.reason}`);
|
||||||
|
const result = await s.runner.wait(name, freshDir(join(paths.base, `check-${n}-logs`)));
|
||||||
|
await s.runner.remove(name);
|
||||||
|
return JSON.stringify({ used, maxCount, seconds, result: readLightCheck(s.task, checkDir, result) });
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import type { ChecklistOf } from "../vocab/index.ts";
|
import type { ChecklistOf, HypothesisMetric } from "../vocab/index.ts";
|
||||||
import { Draft, factorLines, KIND_TEXT, offText, registeredMetrics, taskPoints } from "./blocks.ts";
|
import { type Draft, factorLines, KIND_TEXT, offText, registeredMetrics, taskPoints } from "./blocks.ts";
|
||||||
|
import type { Board } from "./read.ts";
|
||||||
import { fenced, json } from "./format.ts";
|
import { fenced, json } from "./format.ts";
|
||||||
import { unassignedChanges } from "./locations.ts";
|
import { unassignedChanges } from "./locations.ts";
|
||||||
import { clauseKey, factorKey, splitCell, type FrozenProtocol, type LocationTable } from "./shapes.ts";
|
import { clauseKey, factorKey, splitCell, type FrozenProtocol, type LocationTable } from "./shapes.ts";
|
||||||
@@ -27,9 +28,16 @@ function taskSection(d: Draft, role: string, what: string): void {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** 协议要采集的指标:新登记的全文,引用的取原登记(方法.md 第五部分"引用别人登记过的指标时,沿用原登记的选法")。 */
|
/** 协议要采集的指标:新登记的全文,引用的取原登记(方法.md 第五部分"引用别人登记过的指标时,沿用原登记的选法")。 */
|
||||||
|
export function protocolMetrics(board: Board, p: FrozenProtocol): HypothesisMetric[] {
|
||||||
|
const registered = registeredMetrics(board);
|
||||||
|
return [...p.metrics.defined, ...p.metrics.reused.map((id) => {
|
||||||
|
const { candidate: _candidate, ...metric } = registered.find((x) => x.id === id)!;
|
||||||
|
return metric;
|
||||||
|
})];
|
||||||
|
}
|
||||||
|
|
||||||
function metricSection(d: Draft, p: FrozenProtocol): void {
|
function metricSection(d: Draft, p: FrozenProtocol): void {
|
||||||
const registered = registeredMetrics(d.board);
|
const all = protocolMetrics(d.board, p);
|
||||||
const all = [...p.metrics.defined, ...p.metrics.reused.map((id) => registered.find((x) => x.id === id)!)];
|
|
||||||
d.section("要实现的指标定义", [
|
d.section("要实现的指标定义", [
|
||||||
all.length === 0 ? "(这份协议没有要实现的假设专用指标)" : json(all),
|
all.length === 0 ? "(这份协议没有要实现的假设专用指标)" : json(all),
|
||||||
"",
|
"",
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
// materials:每个任务能看到哪些材料、第一条消息怎么写、查询在材料范围内回答什么(docs/结构.md 第二部分),
|
// materials:每个任务能看到哪些材料、第一条消息怎么写、查询在材料范围内回答什么(docs/结构.md 第二部分),
|
||||||
// 以及账本里只存成 Json 的几样内容的形状和编号规则(用户 2026-10-03 决定 2)。
|
// 以及账本里只存成 Json 的几样内容的形状和编号规则(用户 2026-10-03 决定 2)。
|
||||||
|
export { protocolMetrics } from "./assigned.ts";
|
||||||
export { beliefBook } from "./belief-book.ts";
|
export { beliefBook } from "./belief-book.ts";
|
||||||
export { finalEntries, type MaterialRecord, type MaterialSection, type ReadableVersion, type TaskMaterial } from "./blocks.ts";
|
export { finalEntries, type MaterialRecord, type MaterialSection, type ReadableVersion, type TaskMaterial } from "./blocks.ts";
|
||||||
export { buildMaterial } from "./build.ts";
|
export { buildMaterial } from "./build.ts";
|
||||||
export { type CodeChanges, type PromptFile, type Prompts, type StudyFacts, type TaskRequest } from "./inputs.ts";
|
export type { CodeChanges, PromptFile, Prompts, StudyFacts, TaskRequest } from "./inputs.ts";
|
||||||
export { queryMaterial, readPrompts } from "./prompts.ts";
|
export { queryMaterial, readPrompts } from "./prompts.ts";
|
||||||
// flow 推进时读账本也用这一份(本批基线、基线分数、留池候选、各候选合并了哪些原提案),不另写一份。
|
// flow 推进时读账本也用这一份(本批基线、基线分数、留池候选、各候选合并了哪些原提案),不另写一份。
|
||||||
export { Board } from "./read.ts";
|
export { Board, validScore } from "./read.ts";
|
||||||
export {
|
export {
|
||||||
callId,
|
callId,
|
||||||
candidateId,
|
candidateId,
|
||||||
@@ -17,6 +18,10 @@ export {
|
|||||||
critiqueId,
|
critiqueId,
|
||||||
DECISION_KINDS,
|
DECISION_KINDS,
|
||||||
entryId,
|
entryId,
|
||||||
|
experimenterId,
|
||||||
|
verifierId,
|
||||||
|
FORWARD_NOT_RUN,
|
||||||
|
forwardNotRunEvent,
|
||||||
explanationId,
|
explanationId,
|
||||||
factorId,
|
factorId,
|
||||||
factorKey,
|
factorKey,
|
||||||
|
|||||||
@@ -25,6 +25,9 @@ export const ORIGINAL_BASELINE = "base";
|
|||||||
|
|
||||||
/** 研究者:第 i 位(从 0 数)是 researcher-(i+1),与参考项目的写法相同,回放时不用改编号。 */
|
/** 研究者:第 i 位(从 0 数)是 researcher-(i+1),与参考项目的写法相同,回放时不用改编号。 */
|
||||||
export const researcherId = (i: number): string => `researcher-${i + 1}`;
|
export const researcherId = (i: number): string => `researcher-${i + 1}`;
|
||||||
|
/** 每个候选固定一位实验员和核验员,修复、再次核验沿用编号。 */
|
||||||
|
export const experimenterId = (candidate: string): string => `experimenter-${candidate}`;
|
||||||
|
export const verifierId = (candidate: string): string => `verifier-${candidate}`;
|
||||||
/** 入口:作者在第零轮交的第 n 个入口,讨论时整份替换的新清单接着往后编(方法.md 4.2)。 */
|
/** 入口:作者在第零轮交的第 n 个入口,讨论时整份替换的新清单接着往后编(方法.md 4.2)。 */
|
||||||
export const entryId = (author: string, n: number): string => `e-${author}-${n}`;
|
export const entryId = (author: string, n: number): string => `e-${author}-${n}`;
|
||||||
/** 提案:第 round 轮作者交的第 n 条,也是提交记录的 item。 */
|
/** 提案:第 round 轮作者交的第 n 条,也是提交记录的 item。 */
|
||||||
@@ -145,3 +148,7 @@ export type Settlement = {
|
|||||||
*/
|
*/
|
||||||
export const CELL_NOT_RUN = "cellNotRun";
|
export const CELL_NOT_RUN = "cellNotRun";
|
||||||
export const cellNotRunEvent = (cell: string, reason: string): Content<"event"> => ({ event: CELL_NOT_RUN, reason, objects: [cell] });
|
export const cellNotRunEvent = (cell: string, reason: string): Content<"event"> => ({ event: CELL_NOT_RUN, reason, objects: [cell] });
|
||||||
|
|
||||||
|
/** 正向候选核验不通过、修复次数用完而不跑,不登记正式运行、不扣次(方法.md 4.3 ⑨)。 */
|
||||||
|
export const FORWARD_NOT_RUN = "forwardNotRun";
|
||||||
|
export const forwardNotRunEvent = (candidate: string, reason: string): Content<"event"> => ({ event: FORWARD_NOT_RUN, reason, objects: [candidate] });
|
||||||
|
|||||||
@@ -1,9 +1,16 @@
|
|||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { spawnSync } from "node:child_process";
|
import { spawnSync } from "node:child_process";
|
||||||
import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { test, type TestContext } from "node:test";
|
import { test, type TestContext } from "node:test";
|
||||||
|
import { lightChecker } from "../src/flow/task-code.ts";
|
||||||
|
import { recover } from "../src/flow/recover.ts";
|
||||||
|
import { containerName, freshDir, runPaths } from "../src/flow/paths.ts";
|
||||||
|
import type { Study } from "../src/flow/study.ts";
|
||||||
|
import { Board, protocolMetrics, type FrozenProtocol } from "../src/materials/index.ts";
|
||||||
|
import { loadTask, formalRun as runDefinition, prepareRun } from "../src/tasks/index.ts";
|
||||||
|
import { runBalance } from "../src/rules/index.ts";
|
||||||
import { runStudy } from "../src/flow/index.ts";
|
import { runStudy } from "../src/flow/index.ts";
|
||||||
import { Ledger } from "../src/ledger/index.ts";
|
import { Ledger } from "../src/ledger/index.ts";
|
||||||
import { Runner } from "../src/runner/index.ts";
|
import { Runner } from "../src/runner/index.ts";
|
||||||
@@ -76,3 +83,67 @@ test("docker run 没启动:写结果\"没启动\"后停下,原因带 Docker
|
|||||||
assert.match((ending as { reason: string }).reason, /运行 base-1 没有启动.*unknown or invalid runtime name: inforsi-test-no-such-runtime/);
|
assert.match((ending as { reason: string }).reason, /运行 base-1 没有启动.*unknown or invalid runtime name: inforsi-test-no-such-runtime/);
|
||||||
assert.deepEqual(ledgerKinds(dir), ["config", "code", "runPlanned", "runResult", "event"]);
|
assert.deepEqual(ledgerKinds(dir), ["config", "code", "runPlanned", "runResult", "event"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
/** 不派模型,只接真实任务镜像、账本和运行器,检查 flow 的两处 Docker 接线。 */
|
||||||
|
async function runtimeStudy(t: TestContext, suffix: string): Promise<Study> {
|
||||||
|
const dir = tempDir(t);
|
||||||
|
const id = studyId(t, suffix);
|
||||||
|
const task = loadTask(TASK_DIR);
|
||||||
|
const runner = new Runner(null);
|
||||||
|
const image = await runner.buildImage(task.dir, `inforsi-${id}`);
|
||||||
|
const ledger = Ledger.open(join(dir, "ledger.sqlite"));
|
||||||
|
t.after(() => ledger.close());
|
||||||
|
return { dir, task, runner, ledger, stop: new AbortController().signal,
|
||||||
|
frozen: { config: { ...fullConfig(), studyId: id, lightChecks: { maxCount: 1, seconds: 7 } }, facts: task.facts, image, seed: 1 } } as Study;
|
||||||
|
}
|
||||||
|
|
||||||
|
test("flow 轻量检查:每次调用各自限次和限时,真实 Docker 只检查当前目录,不登记正式运行", { timeout: 120_000 }, async (t) => {
|
||||||
|
const s = await runtimeStudy(t, "light");
|
||||||
|
const work = join(s.dir, "work");
|
||||||
|
mkdirSync(work);
|
||||||
|
writeFileSync(join(work, "initial.py"), "raise RuntimeError('不应执行候选代码')\ndef run_packing():\n pass\n");
|
||||||
|
for (const call of ["first-call", "second-call"]) {
|
||||||
|
freshDir(join(s.dir, "calls", call));
|
||||||
|
const check = lightChecker(s, call, work);
|
||||||
|
const result = JSON.parse(await check());
|
||||||
|
assert.deepEqual(result, { used: 1, maxCount: 1, seconds: 7, result: { passed: true } });
|
||||||
|
assert.match(await check(), /已用完(上限 1 次)/);
|
||||||
|
assert.equal(await s.runner.status(containerName(s.frozen.config.studyId, `${call}-check-1`)), "absent");
|
||||||
|
}
|
||||||
|
assert.deepEqual(s.ledger.records(), []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("正向恢复:接上原容器不重跑,补启动只扣一次,引用指标沿用原登记缺失作废规则", { timeout: 120_000 }, async (t) => {
|
||||||
|
const s = await runtimeStudy(t, "forward-recover");
|
||||||
|
const metric = { id: "old-c.required", meaning: "必测量", unit: "次", direction: "maximize" as const, howMeasured: "原登记定义", missing: "invalidateRun" as const };
|
||||||
|
const original: FrozenProtocol = { round: 0, baseline: "base", proposals: [], factors: [], safeguards: [], metrics: { defined: [metric], reused: [] }, clauses: [], body: "登记" };
|
||||||
|
const current: FrozenProtocol = { ...original, round: 1, metrics: { defined: [], reused: [metric.id] } };
|
||||||
|
s.ledger.append("protocol", { candidate: "old-c", protocol: original });
|
||||||
|
s.ledger.append("protocol", { candidate: "r1-c1", protocol: current });
|
||||||
|
const board = new Board(s.ledger.records());
|
||||||
|
const run = "forward-1";
|
||||||
|
const paths = runPaths(s.dir, run);
|
||||||
|
const definition = runDefinition(s.task, s.frozen.image, "1");
|
||||||
|
s.ledger.append("runPlanned", { run, purpose: "forward", round: 1, object: "r1-c1", definition });
|
||||||
|
freshDir(paths.base);
|
||||||
|
prepareRun(s.task, s.task.baselineDir, freshDir(paths.run), protocolMetrics(board, current));
|
||||||
|
const name = containerName(s.frozen.config.studyId, run);
|
||||||
|
t.after(() => { const result = docker(["rm", "-f", name]); if (result.status !== 0 && !result.stderr.includes("No such container")) throw new Error(result.stderr); });
|
||||||
|
assert.deepEqual(await s.runner.start(name, definition, paths.run), { started: true });
|
||||||
|
// 中断点:容器已启动,账本还没写 runStarted。恢复不调用 formalRun,而是找到原容器等它。
|
||||||
|
assert.equal(await recover(s), null);
|
||||||
|
const records = s.ledger.records();
|
||||||
|
assert.equal(records.filter((r) => r.kind === "runPlanned").length, 1);
|
||||||
|
assert.equal(records.filter((r) => r.kind === "runStarted").length, 1);
|
||||||
|
assert.equal(records.filter((r) => r.kind === "runResult").length, 1);
|
||||||
|
const result = records.find((r) => r.kind === "runResult")!;
|
||||||
|
if (result.kind !== "runResult") throw new Error("缺运行结果");
|
||||||
|
assert.equal(result.content.judged!.verdict, "invalid");
|
||||||
|
assert.match(result.content.judged!.verdictReason, /old-c.required/);
|
||||||
|
assert.equal(result.content.judged!.metrics[metric.id]!.status, "unmeasured");
|
||||||
|
assert.deepEqual(runBalance(13, new Board(records).runStates()), { charged: 1, reserved: 0, balance: 12 });
|
||||||
|
assert.equal(await s.runner.status(name), "absent");
|
||||||
|
assert.equal(await recover(s), null);
|
||||||
|
assert.equal(s.ledger.records().length, records.length);
|
||||||
|
});
|
||||||
|
|||||||
@@ -0,0 +1,145 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { test, type TestContext } from "node:test";
|
||||||
|
import { adoptBatch } from "../src/flow/steps/forward.ts";
|
||||||
|
import { prepareSubmission, taskCode } from "../src/flow/task-code.ts";
|
||||||
|
import type { Study } from "../src/flow/study.ts";
|
||||||
|
import { Board, buildMaterial, protocolMetrics, type FrozenProtocol } from "../src/materials/index.ts";
|
||||||
|
import { Ledger } from "../src/ledger/index.ts";
|
||||||
|
import { loadTask } from "../src/tasks/index.ts";
|
||||||
|
import { checkoutVersion, createRepository, saveVersion } from "../src/workspace/index.ts";
|
||||||
|
import type { Submission } from "../src/vocab/index.ts";
|
||||||
|
import { fullConfig } from "./config-helpers.ts";
|
||||||
|
import { implementation, tempDir } from "./ledger-helpers.ts";
|
||||||
|
import { judged, prompts, study, Writer } from "./materials-helpers.ts";
|
||||||
|
|
||||||
|
function setup(t: TestContext): { s: Study; w: Writer } {
|
||||||
|
const dir = tempDir(t);
|
||||||
|
const ledger = Ledger.open(join(dir, "ledger.sqlite"));
|
||||||
|
t.after(() => ledger.close());
|
||||||
|
const task = loadTask(join(import.meta.dirname, "..", "tasks", "circle_packing"));
|
||||||
|
const s = { dir, ledger, task, prompts, facts: study, frozen: { config: fullConfig(), facts: task.facts } } as Study;
|
||||||
|
return { s, w: new Writer(ledger) };
|
||||||
|
}
|
||||||
|
|
||||||
|
// 预期值手算:基线 1,ε=1/64,已知最好 2;线=1/64。1+1/64 正好在线上,不采纳。
|
||||||
|
for (const [label, scores, adopted, gain] of [
|
||||||
|
["高于线且同分:取自然编号小的", [1.125, 1.125], "r1-c2", 0.125],
|
||||||
|
["正好等于线:也写 null 决定", [1.015625, 1], null, 0.015625],
|
||||||
|
["整批没有有效分数:也写 null 决定", [], null, null],
|
||||||
|
] as const) {
|
||||||
|
test(`采纳写账:${label},写决定后整批公开`, (t) => {
|
||||||
|
const { s, w } = setup(t);
|
||||||
|
s.frozen.facts = { ...s.frozen.facts, epsilon: 0.015625, knownBest: 2 };
|
||||||
|
w.run("base-1", "baseline", 0, null, judged(1, "main"));
|
||||||
|
w.decision("batch", 1, null, { chosen: ["r1-c10", "r1-c2", "r1-c3"] });
|
||||||
|
scores.forEach((score, i) => w.run(["run-first", "run-second"][i]!, "forward", 1, i === 0 ? "r1-c10" : "r1-c2", judged(score, "main")));
|
||||||
|
w.run("invalid", "forward", 1, "r1-c3", { ...judged(99, "main"), verdict: "invalid", verdictReason: "不合法" });
|
||||||
|
// 不在本批的运行不参与,即使它分数更高;上轮不能混进本轮。
|
||||||
|
w.run("other", "forward", 0, "old-candidate", judged(100, "main"));
|
||||||
|
const before = new Board(s.ledger.records());
|
||||||
|
assert.ok(before.runs().filter((r) => r.plan.round === 1).every((r) => !before.runPublic(r)));
|
||||||
|
adoptBatch(s, 1);
|
||||||
|
const record = s.ledger.records().at(-1)!;
|
||||||
|
assert.equal(record.kind, "decision");
|
||||||
|
if (record.kind !== "decision") throw new Error("应写采纳决定");
|
||||||
|
assert.deepEqual(record.content, {
|
||||||
|
decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1,
|
||||||
|
params: { direction: "maximize", epsilon: 0.015625, knownBest: 2 }, seed: null,
|
||||||
|
input: { candidates: scores.map((score, i) => ({ id: i === 0 ? "r1-c10" : "r1-c2", score })), baseline: 1, direction: "maximize", epsilon: 0.015625, knownBest: 2 },
|
||||||
|
output: { line: 0.015625, quarter: 0.25, adopted,
|
||||||
|
best: scores.length === 0 ? null : { id: adopted ?? "r1-c10", score: scores[0], improvement: gain } },
|
||||||
|
});
|
||||||
|
const after = new Board(s.ledger.records());
|
||||||
|
assert.ok(after.runs().filter((r) => r.plan.round === 1).every((r) => after.runPublic(r)));
|
||||||
|
assert.equal(after.baselineOf(2), adopted ?? "base");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const metric = { id: "old-c.metric", meaning: "试验值", unit: "次", direction: "maximize" as const, howMeasured: "按原登记计数", missing: "invalidateRun" as const };
|
||||||
|
const oldProtocol: FrozenProtocol = { round: 0, baseline: "base", proposals: [], factors: [], safeguards: [], metrics: { defined: [metric], reused: [] }, clauses: [], body: "原登记" };
|
||||||
|
const protocol: FrozenProtocol = { round: 1, baseline: "base", proposals: [], factors: [{ name: "改动", method: "照协议改动", keySettings: "无", continues: null }], safeguards: [], metrics: { defined: [], reused: [metric.id] }, clauses: ["照协议"], body: "冻结正文" };
|
||||||
|
|
||||||
|
function codeStudy(t: TestContext) {
|
||||||
|
const { s, w } = setup(t);
|
||||||
|
const repo = join(s.dir, "repo.git");
|
||||||
|
createRepository(repo);
|
||||||
|
const workDir = join(s.dir, "work");
|
||||||
|
mkdirSync(workDir);
|
||||||
|
writeFileSync(join(workDir, "initial.py"), "first = 1\nsecond = 2\n");
|
||||||
|
writeFileSync(join(workDir, "evaluate.py"), "protected = True\n");
|
||||||
|
const baseline = saveVersion(repo, workDir, []);
|
||||||
|
if (!baseline.saved) throw new Error(baseline.reason);
|
||||||
|
s.ledger.append("code", { object: "base", version: baseline.version, locations: null });
|
||||||
|
w.protocol("old-c", oldProtocol);
|
||||||
|
w.protocol("r1-c1", protocol);
|
||||||
|
w.open("r1-implementation", "implementation", 1, ["experimenter-r1-c1"]);
|
||||||
|
return { s, w, repo, workDir, baseline: baseline.version };
|
||||||
|
}
|
||||||
|
|
||||||
|
const impl: Submission<"implementation"> = { ...implementation("实现说明"), factorLocations: [{ factor: "p1", locations: ["initial.py:1-2"] }] };
|
||||||
|
const ref = { actor: "experimenter-r1-c1", object: "r1-c1", kind: "implementation" as const, stageId: "r1-implementation" };
|
||||||
|
|
||||||
|
test("提交准备:快照、保护文件、差异块和引用指标同源;修复取上版,但差异永远相对本批基线", (t) => {
|
||||||
|
const { s, w, repo, workDir, baseline } = codeStudy(t);
|
||||||
|
assert.deepEqual(taskCode(s, ref), { changes: null, startVersion: baseline });
|
||||||
|
writeFileSync(join(workDir, "initial.py"), "first = 10\nsecond = 2\n");
|
||||||
|
const prepared = prepareSubmission(s, ref, impl, 0, workDir);
|
||||||
|
assert.ok(prepared.code);
|
||||||
|
const known = prepared.known as { code: { saved: true; protectedFiles: string[]; hunks: { text: string }[]; protocol: unknown } };
|
||||||
|
assert.deepEqual(known.code.protectedFiles, []);
|
||||||
|
assert.deepEqual(known.code.protocol, { body: "冻结正文", metrics: [metric] });
|
||||||
|
assert.match(known.code.hunks[0]!.text, /-first = 1\n\+first = 10/);
|
||||||
|
s.ledger.transaction(() => {
|
||||||
|
w.sub(ref.stageId, ref.actor, ref.kind, ref.object, null, impl);
|
||||||
|
s.ledger.append("code", prepared.code!);
|
||||||
|
});
|
||||||
|
w.close(ref.stageId);
|
||||||
|
w.open("r1-verification-1", "verification", 1, ["verifier-r1-c1"]);
|
||||||
|
w.sub("r1-verification-1", "verifier-r1-c1", "verification", "r1-c1", null, { clauses: [{ clause: "c1", found: false, where: "第二行未实现" }], factors: [{ factor: "p1", found: false, where: "第二行未实现" }], safeguards: [], unassigned: [], body: "只修第二行" });
|
||||||
|
w.close("r1-verification-1");
|
||||||
|
w.open("r1-repair-1", "implementation", 1, [ref.actor]);
|
||||||
|
const repair = { ...ref, kind: "repair" as const, stageId: "r1-repair-1" };
|
||||||
|
const previous = taskCode(s, repair);
|
||||||
|
assert.equal(previous.startVersion, prepared.code.version);
|
||||||
|
assert.match(previous.changes!.diff, /-first = 1\n\+first = 10/);
|
||||||
|
const repairMaterial = buildMaterial({ ...repair, ledger: s.ledger, study, prompts, codeRoot: join(s.dir, "readable"), changes: previous.changes, artifacts: null });
|
||||||
|
assert.match(repairMaterial.firstMessage, /工作目录是上一次提交的代码的副本/);
|
||||||
|
assert.match(repairMaterial.firstMessage, /第二行未实现/);
|
||||||
|
writeFileSync(join(workDir, "initial.py"), "first = 10\nsecond = 20\n");
|
||||||
|
const fixed = prepareSubmission(s, repair, impl, 0, workDir);
|
||||||
|
const fixedKnown = fixed.known as typeof known;
|
||||||
|
assert.match(fixedKnown.code.hunks[0]!.text, /-first = 1/);
|
||||||
|
assert.match(fixedKnown.code.hunks[0]!.text, /\+second = 20/);
|
||||||
|
s.ledger.transaction(() => {
|
||||||
|
w.sub(repair.stageId, repair.actor, repair.kind, repair.object, null, impl);
|
||||||
|
s.ledger.append("code", fixed.code!);
|
||||||
|
});
|
||||||
|
w.close(repair.stageId);
|
||||||
|
w.open("r1-verification-2", "verification", 1, ["verifier-r1-c1"]);
|
||||||
|
const verification = { ...ref, kind: "verification" as const, stageId: "r1-verification-2" };
|
||||||
|
const compare = taskCode(s, verification);
|
||||||
|
assert.equal(compare.startVersion, null);
|
||||||
|
assert.match(compare.changes!.diff, /-first = 1/);
|
||||||
|
assert.match(compare.changes!.diff, /\+second = 20/);
|
||||||
|
assert.deepEqual(protocolMetrics(new Board(s.ledger.records()), protocol), [metric]);
|
||||||
|
const snapshot = join(s.dir, "snapshot");
|
||||||
|
mkdirSync(snapshot);
|
||||||
|
checkoutVersion(repo, prepared.code.version, snapshot);
|
||||||
|
assert.equal(readFileSync(join(snapshot, "initial.py"), "utf8"), "first = 10\nsecond = 2\n");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("存不成版本的原原因交 submissions;改了保护文件时不能被漏掉", (t) => {
|
||||||
|
const { s, workDir } = codeStudy(t);
|
||||||
|
writeFileSync(join(workDir, "evaluate.py"), "protected = False\n");
|
||||||
|
const prepared = prepareSubmission(s, ref, impl, 0, workDir);
|
||||||
|
assert.deepEqual((prepared.known as { code: { protectedFiles: string[] } }).code.protectedFiles, ["evaluate.py"]);
|
||||||
|
symlinkSync("initial.py", join(workDir, "link.py"));
|
||||||
|
const refused = prepareSubmission(s, ref, impl, 0, workDir);
|
||||||
|
assert.equal(refused.code, null);
|
||||||
|
const code = (refused.known as { code: { saved: false; path: string; reason: string } }).code;
|
||||||
|
assert.equal(code.saved, false);
|
||||||
|
assert.equal(code.path, "link.py");
|
||||||
|
assert.match(code.reason, /符号链接/);
|
||||||
|
});
|
||||||
@@ -4,9 +4,12 @@ import { join } from "node:path";
|
|||||||
import { test } from "node:test";
|
import { test } from "node:test";
|
||||||
import { stepList, type ResearchConfig } from "../src/config/index.ts";
|
import { stepList, type ResearchConfig } from "../src/config/index.ts";
|
||||||
import type { AppendKind, Content, LedgerRecord } from "../src/ledger/index.ts";
|
import type { AppendKind, Content, LedgerRecord } from "../src/ledger/index.ts";
|
||||||
import type { StudyRecord } from "../src/materials/index.ts";
|
import { Board, forwardNotRunEvent, type StudyRecord } from "../src/materials/index.ts";
|
||||||
|
import { runBalance } from "../src/rules/index.ts";
|
||||||
|
import { definition, exited } from "./ledger-helpers.ts";
|
||||||
|
import { judged } from "./materials-helpers.ts";
|
||||||
import { nextStep } from "../src/flow/next-step.ts";
|
import { nextStep } from "../src/flow/next-step.ts";
|
||||||
import type { Json, SubmissionKind } from "../src/vocab/index.ts";
|
import type { Json, Submission, SubmissionKind } from "../src/vocab/index.ts";
|
||||||
import { fullConfig } from "./config-helpers.ts";
|
import { fullConfig } from "./config-helpers.ts";
|
||||||
|
|
||||||
// flow 的"下一步做什么"(任务书 12 第六部分):纯函数,手写账本记录,看它给出的下一步。
|
// flow 的"下一步做什么"(任务书 12 第六部分):纯函数,手写账本记录,看它给出的下一步。
|
||||||
@@ -337,8 +340,8 @@ test("第二次预测关闭:选批", () => {
|
|||||||
assert.deepEqual(nextStep(roundOne("secondPrediction").records), { do: "selectBatch", round: 1 });
|
assert.deepEqual(nextStep(roundOne("secondPrediction").records), { do: "selectBatch", round: 1 });
|
||||||
});
|
});
|
||||||
|
|
||||||
test("选批做完:停下,原因\"实现还没写\"", () => {
|
test("选批做完:开实现阶段,每个入选候选固定一位实验员", () => {
|
||||||
assert.deepEqual(nextStep(roundOne("selectBatch").records), { do: "halt", reason: "实现还没写(任务书 16)", objects: [] });
|
assert.deepEqual(nextStep(roundOne("selectBatch").records), { do: "openStage", stageId: "r1-implementation", stage: "implementation", round: 1, roster: ["experimenter-r1-c1"] });
|
||||||
});
|
});
|
||||||
|
|
||||||
test("关掉第一次预测:第 1 轮写一条按配置跳过(对象 r1-firstPrediction),接着开挑刺", () => {
|
test("关掉第一次预测:第 1 轮写一条按配置跳过(对象 r1-firstPrediction),接着开挑刺", () => {
|
||||||
@@ -354,3 +357,149 @@ test("关掉第一次预测:第 1 轮写一条按配置跳过(对象 r1-firs
|
|||||||
book.add("event", { event: "skippedByConfig", reason, objects: ["r1-firstPrediction"] });
|
book.add("event", { event: "skippedByConfig", reason, objects: ["r1-firstPrediction"] });
|
||||||
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-critique", stage: "critique", round: 1, roster: R });
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-critique", stage: "critique", round: 1, roster: R });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
// ⑨–⑫:预期动作手写,不调模型、Jev 或 Docker。
|
||||||
|
const X = ["experimenter-r1-c1", "experimenter-r1-c2"];
|
||||||
|
const V = ["verifier-r1-c1", "verifier-r1-c2"];
|
||||||
|
|
||||||
|
function batchTwo(maxRepairs: number): Book {
|
||||||
|
const book = roundOne("secondPrediction");
|
||||||
|
const record = book.records.find((r) => r.kind === "config")!;
|
||||||
|
(record.content as unknown as StudyRecord).config = { ...fullConfig(), maxRepairs } as unknown as Json;
|
||||||
|
return book.add("decision", { decision: "batch", round: 1, rule: "bestExpected", ruleVersion: 1, params: null, seed: "s", input: null, output: { chosen: C } });
|
||||||
|
}
|
||||||
|
|
||||||
|
function implemented(book: Book): Book {
|
||||||
|
book.open("r1-implementation", "implementation", 1, X);
|
||||||
|
for (const [i, candidate] of C.entries()) book.sub("r1-implementation", X[i]!, "implementation", candidate, null);
|
||||||
|
return book.add("stageClose", { stageId: "r1-implementation" });
|
||||||
|
}
|
||||||
|
|
||||||
|
function verified(book: Book, n: number, candidate: string, passed: boolean): Book {
|
||||||
|
// 对应真实 15-tight-2 的 submissions created_seq=173(墙约束行和右端错位)、194(修复后通过)。
|
||||||
|
// 只转录驱动程序所需的逐项 found,不伪造 Jev 的判断或模型运行。
|
||||||
|
const content: Submission<"verification"> = {
|
||||||
|
clauses: [{ clause: "c1", found: passed, where: passed ? "initial.py 墙约束行和右端已对齐" : "initial.py:225-228 与 243-246:墙约束行和右端错位" }],
|
||||||
|
factors: [{ factor: "p1", found: passed, where: "LP 墙约束" }],
|
||||||
|
safeguards: [], unassigned: [], body: passed ? "通过" : "解的不是协议写的线性规划",
|
||||||
|
};
|
||||||
|
return book.add("submission", { stageId: `r1-verification-${n}`, author: `verifier-${candidate}`, kind: "verification", target: candidate, item: null, version: 1, content, jev: null });
|
||||||
|
}
|
||||||
|
|
||||||
|
function firstVerification(book: Book, a: boolean, b: boolean): Book {
|
||||||
|
implemented(book).open("r1-verification-1", "verification", 1, V);
|
||||||
|
verified(book, 1, C[0]!, a);
|
||||||
|
verified(book, 1, C[1]!, b);
|
||||||
|
return book.add("stageClose", { stageId: "r1-verification-1" });
|
||||||
|
}
|
||||||
|
|
||||||
|
function recordForward(book: Book, candidate: string, n: number, ending: "valid" | "notStarted" | "timeout"): Book {
|
||||||
|
const run = `r1-forward-${candidate}-${n}`;
|
||||||
|
book.add("runPlanned", { run, purpose: "forward", round: 1, object: candidate, definition });
|
||||||
|
if (ending === "notStarted") return book.add("runResult", { run, process: { ending, reason: "Docker 未启动" }, judged: null });
|
||||||
|
book.add("runStarted", { run });
|
||||||
|
const process = ending === "timeout" ? { ending: "timeout" as const, wallSeconds: measured(120), cpuSeconds: measured(100), peakMemoryBytes: measured(1024) } : exited;
|
||||||
|
return book.add("runResult", { run, process, judged: ending === "valid" ? judged(1.25, "main") : null });
|
||||||
|
}
|
||||||
|
|
||||||
|
const forward = (candidate: string, n: number) => ({ do: "forwardRun", round: 1, candidate, run: `r1-forward-${candidate}-${n}` });
|
||||||
|
|
||||||
|
test("实现的任务同时可派;已提交后中断不重派,等全体交齐关阶段后才核验", () => {
|
||||||
|
const book = batchTwo(1).open("r1-implementation", "implementation", 1, X);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-implementation", tasks: C.map((object, i) => ({ actor: X[i], kind: "implementation", object })) });
|
||||||
|
book.sub("r1-implementation", X[0]!, "implementation", C[0]!, null);
|
||||||
|
book.call("r1-implementation", X[0]!, "implementation", C[0]!, 1, { wallSeconds: { status: "unmeasured", reason: "进程中断" }, status: "failed", reason: "进程中断" });
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-implementation", tasks: [{ actor: X[1], kind: "implementation", object: C[1] }] });
|
||||||
|
book.sub("r1-implementation", X[1]!, "implementation", C[1]!, null);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-implementation" });
|
||||||
|
book.add("stageClose", { stageId: "r1-implementation" });
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-verification-1", stage: "verification", round: 1, roster: V });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("15-tight-2 的一次修复路径:核验关后才派原实验员,修复关后只重验修过的候选", () => {
|
||||||
|
const book = implemented(batchTwo(1)).open("r1-verification-1", "verification", 1, V);
|
||||||
|
verified(book, 1, C[0]!, false);
|
||||||
|
verified(book, 1, C[1]!, true);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-verification-1" });
|
||||||
|
book.add("stageClose", { stageId: "r1-verification-1" });
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-repair-1", stage: "implementation", round: 1, roster: [X[0]] });
|
||||||
|
book.open("r1-repair-1", "implementation", 1, [X[0]!]);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-repair-1", tasks: [{ actor: X[0], kind: "repair", object: C[0] }] });
|
||||||
|
book.sub("r1-repair-1", X[0]!, "repair", C[0]!, null);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-repair-1" });
|
||||||
|
book.add("stageClose", { stageId: "r1-repair-1" });
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: "r1-verification-2", stage: "verification", round: 1, roster: [V[0]] });
|
||||||
|
book.open("r1-verification-2", "verification", 1, [V[0]!]);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "dispatch", stageId: "r1-verification-2", tasks: [{ actor: V[0], kind: "verification", object: C[0] }] });
|
||||||
|
verified(book, 2, C[0]!, true);
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "closeStage", stageId: "r1-verification-2" });
|
||||||
|
book.add("stageClose", { stageId: "r1-verification-2" });
|
||||||
|
assert.deepEqual(nextStep(book.records), forward(C[0]!, 1));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("修复次数从关闭的阶段数出:两次修复后仍失败,逐候选写不跑事件,不占预算", () => {
|
||||||
|
const book = firstVerification(batchTwo(2), false, false);
|
||||||
|
for (let n = 1; n <= 2; n++) {
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: `r1-repair-${n}`, stage: "implementation", round: 1, roster: X });
|
||||||
|
book.open(`r1-repair-${n}`, "implementation", 1, X);
|
||||||
|
for (const [i, candidate] of C.entries()) book.sub(`r1-repair-${n}`, X[i]!, "repair", candidate, null);
|
||||||
|
book.add("stageClose", { stageId: `r1-repair-${n}` });
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "openStage", stageId: `r1-verification-${n + 1}`, stage: "verification", round: 1, roster: V });
|
||||||
|
book.open(`r1-verification-${n + 1}`, "verification", 1, V);
|
||||||
|
for (const candidate of C) verified(book, n + 1, candidate, false);
|
||||||
|
book.add("stageClose", { stageId: `r1-verification-${n + 1}` });
|
||||||
|
}
|
||||||
|
for (const candidate of C) {
|
||||||
|
const next = nextStep(book.records);
|
||||||
|
assert.equal(next.do, "forwardNotRun");
|
||||||
|
if (next.do !== "forwardNotRun") throw new Error("应写不跑事件");
|
||||||
|
assert.equal(next.candidate, candidate);
|
||||||
|
assert.match(next.reason, /第 3 次核验仍不通过,修复次数上限 2 已用完/);
|
||||||
|
assert.match(next.reason, /解的不是协议写的线性规划/);
|
||||||
|
book.add("event", forwardNotRunEvent(candidate, next.reason));
|
||||||
|
}
|
||||||
|
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 });
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "adopt", round: 1 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("maxRepairs=0:第一次核验失败就记不跑,另一候选仍可串行运行", () => {
|
||||||
|
const book = firstVerification(batchTwo(0), false, true);
|
||||||
|
const next = nextStep(book.records);
|
||||||
|
assert.equal(next.do, "forwardNotRun");
|
||||||
|
if (next.do !== "forwardNotRun") throw new Error("应写不跑事件");
|
||||||
|
book.add("event", forwardNotRunEvent(next.candidate, next.reason));
|
||||||
|
assert.deepEqual(nextStep(book.records), forward(C[1]!, 1));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("正向串行:上次未启动起新编号;已登记未结束必须先恢复;超时也扣次且不重跑;采纳前不公开", () => {
|
||||||
|
const book = firstVerification(batchTwo(1), true, true);
|
||||||
|
assert.deepEqual(nextStep(book.records), forward(C[0]!, 1));
|
||||||
|
recordForward(book, C[0]!, 1, "notStarted");
|
||||||
|
assert.deepEqual(nextStep(book.records), forward(C[0]!, 2));
|
||||||
|
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 0, balance: 12 });
|
||||||
|
const run = `r1-forward-${C[0]}-2`;
|
||||||
|
book.add("runPlanned", { run, purpose: "forward", round: 1, object: C[0]!, definition });
|
||||||
|
assert.throws(() => nextStep(book.records), /登记了还没有结果:恢复入口应当已经核对过/);
|
||||||
|
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 1, reserved: 1, balance: 11 });
|
||||||
|
book.add("runStarted", { run });
|
||||||
|
assert.throws(() => nextStep(book.records), /登记了还没有结果/);
|
||||||
|
book.add("runResult", { run, process: exited, judged: judged(1.25, "main") });
|
||||||
|
assert.deepEqual(nextStep(book.records), forward(C[1]!, 1));
|
||||||
|
recordForward(book, C[1]!, 1, "timeout");
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "adopt", round: 1 });
|
||||||
|
assert.deepEqual(runBalance(13, new Board(book.records).runStates()), { charged: 3, reserved: 0, balance: 10 });
|
||||||
|
const before = new Board(book.records);
|
||||||
|
assert.ok(before.runs().filter((r) => r.plan.purpose === "forward").every((r) => !before.runPublic(r)));
|
||||||
|
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: null } });
|
||||||
|
const after = new Board(book.records);
|
||||||
|
assert.ok(after.runs().filter((r) => r.plan.purpose === "forward").every((r) => after.runPublic(r)));
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "halt", reason: "消融还没写(任务书 17)", objects: [] });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("采纳只有一个因子的候选也统一在 listCells 停下,不擅自开收尾", () => {
|
||||||
|
const book = firstVerification(batchTwo(1), true, true);
|
||||||
|
for (const c of C) recordForward(book, c, 1, "valid");
|
||||||
|
book.add("decision", { decision: "adopt", round: 1, rule: "adopt", ruleVersion: 1, params: null, seed: null, input: null, output: { adopted: C[0]! } });
|
||||||
|
assert.deepEqual(nextStep(book.records), { do: "halt", reason: "消融还没写(任务书 17)", objects: [] });
|
||||||
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user