feat: flow 第 1 部分(骨架、研究开始、第零轮)与第零轮检验
按任务书 12 实现 src/flow:从账本现算下一步的纯函数、主循环、恢复入口、派模型任务和跑正式运行两个共用动作、冻结配置、基线、第零轮三个研究者阶段和入口分配,走到第 1 轮时停下。materials 导出 finalEntries、加 researcherId 和 StudyRecord。 审阅采纳两条:研究编号本机已用过时冻结前报错;docker run 没启动时写完结果就停下,不在本进程里无限起新编号。 eval/ 新增第零轮检验:起研究命令行、回放假模型、检验用 Jev 客户端、六问质量评估、16-real-1 回放、杀进程与两份同时跑。tsc 与 depcruise 也检查 eval/,新增 eval 只经 index.ts 用正式代码的规则。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,121 @@
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { LedgerRecord } from "../../src/ledger/index.ts";
|
||||
import { entryDecision, finalSubmissions, ofKind, readLedger, sameAssignment, stages, type Assignment } from "../lib/ledger.ts";
|
||||
import { replayModels, startReplay, type ReplayScript } from "../lib/replay.ts";
|
||||
import { containers, launch, newEvalDir, sleep, waitForLedger, writeConfig } from "../lib/study.ts";
|
||||
|
||||
// 杀进程检验(任务书 12 第 5.4 节):在一处 SIGKILL 控制进程,再启动让它跑完第零轮,逐条查五件事。
|
||||
// node eval/round-zero/kill.ts baseline|drafting|stageFull
|
||||
// 模型一侧用 16-real-1 的回放(不花钱、结果确定,分配应和当年相同);基线照样用真 Docker。
|
||||
// 回放每次回复前等 DELAY_MS,留出"研究者正在写""最后一份刚收下、阶段还没关"的窗口。
|
||||
|
||||
const DELAY_MS = 3000;
|
||||
type Point = "baseline" | "drafting" | "stageFull";
|
||||
const point = process.argv[2] as Point;
|
||||
|
||||
/** 三处杀点:看到账本里出现这种状态就杀。 */
|
||||
const HITS: Record<Point, (r: LedgerRecord[]) => boolean> = {
|
||||
// 基线容器正在跑:基线运行确认启动了、还没有结果。
|
||||
baseline: (r) => {
|
||||
const base = new Set(ofKind(r, "runPlanned").filter((p) => p.content.purpose === "baseline").map((p) => p.content.run));
|
||||
const done = new Set(ofKind(r, "runResult").map((x) => x.content.run));
|
||||
return ofKind(r, "runStarted").some((s) => base.has(s.content.run) && !done.has(s.content.run));
|
||||
},
|
||||
// 研究者 2 正在写入口:他在写入口阶段的调用开始了、还没结束。
|
||||
drafting: (r) => {
|
||||
const draft = new Set(stages(r).filter((s) => s.stage === "entryDraft").map((s) => s.stageId));
|
||||
return openCalls(r).some((c) => c.content.actor === "researcher-2" && draft.has(c.content.stageId ?? ""));
|
||||
},
|
||||
// 某阶段最后一份提交刚收下、阶段还没关:名单上的人都交了,还没有关闭记录。
|
||||
stageFull: (r) => stages(r).some((s) => !s.closed && s.roster.every((a) => submitted(r, s.stageId, a))),
|
||||
};
|
||||
if (!(point in HITS)) throw new Error(`杀点只能是 ${Object.keys(HITS).join("、")} 之一`);
|
||||
|
||||
function openCalls(r: LedgerRecord[]): Array<LedgerRecord<"callStarted">> {
|
||||
const ended = new Set(ofKind(r, "callEnded").map((e) => e.content.call));
|
||||
return ofKind(r, "callStarted").filter((c) => !ended.has(c.content.call));
|
||||
}
|
||||
const submitted = (r: LedgerRecord[], stageId: string, author: string) =>
|
||||
ofKind(r, "submission").some((s) => s.content.stageId === stageId && s.content.author === author);
|
||||
|
||||
const script = JSON.parse(readFileSync("eval/round-zero/replay-16-real-1.json", "utf8")) as ReplayScript & { expected: Assignment[] };
|
||||
// 研究编号也是镜像标签的一部分,Docker 镜像名只许小写,所以杀点名转成小写。
|
||||
const name = `r0-kill-${point.toLowerCase()}`;
|
||||
const out = newEvalDir(name);
|
||||
const replay = await startReplay(script, join(out, "pi-agent"), DELAY_MS);
|
||||
const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, replayModels(Object.keys(script.models).sort(), "jev-latest"));
|
||||
const studyDir = join(out, "study");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 第一次启动,打中就杀
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const first = launch(config, studyDir, replay.env);
|
||||
await waitForLedger(studyDir, first.child);
|
||||
let triggered: LedgerRecord | undefined;
|
||||
while (first.child.exitCode === null) {
|
||||
const r = readLedger(studyDir);
|
||||
if (HITS[point](r)) {
|
||||
first.child.kill("SIGKILL");
|
||||
triggered = r.at(-1);
|
||||
break;
|
||||
}
|
||||
await sleep(20);
|
||||
}
|
||||
await first.done;
|
||||
if (triggered === undefined) throw new Error(`没打中 ${point}:研究进程自己结束了,账本最后一条是 ${JSON.stringify(readLedger(studyDir).at(-1))}`);
|
||||
// 进程已死,此刻的账本就是被杀时的样子。
|
||||
const atKill = readLedger(studyDir);
|
||||
const killSeq = atKill.at(-1)!.seq;
|
||||
const killedCalls = openCalls(atKill);
|
||||
const openStages = stages(atKill).filter((s) => !s.closed);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 再启动,跑完第零轮
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const second = await launch(null, studyDir, replay.env).done;
|
||||
await replay.close();
|
||||
const r = readLedger(studyDir);
|
||||
const studyId = name;
|
||||
|
||||
const baseRuns = new Set(ofKind(r, "runPlanned").filter((p) => p.content.purpose === "baseline").map((p) => p.content.run));
|
||||
const baseStarted = ofKind(r, "runStarted").filter((s) => baseRuns.has(s.content.run));
|
||||
const leftover = containers(`inforsi-${studyId}-`);
|
||||
const callEnds = killedCalls.map((c) => ({ call: c.content.call, end: ofKind(r, "callEnded").find((e) => e.content.call === c.content.call)?.content ?? null }));
|
||||
const redispatch = openStages.flatMap((s) =>
|
||||
s.roster.map((actor) => {
|
||||
const before = submitted(atKill, s.stageId, actor);
|
||||
const again = ofKind(r, "callStarted").some((c) => c.seq > killSeq && c.content.stageId === s.stageId && c.content.actor === actor);
|
||||
return { stageId: s.stageId, actor, submittedBeforeKill: before, dispatchedAfterRestart: again, ok: before !== again };
|
||||
}),
|
||||
);
|
||||
const finals = finalSubmissions(r);
|
||||
const perStage = stages(r).map((s) => ({
|
||||
stageId: s.stageId,
|
||||
closed: s.closed,
|
||||
finalPerPerson: Object.fromEntries(s.roster.map((a) => [a, finals.filter((f) => f.content.stageId === s.stageId && f.content.author === a).length])),
|
||||
}));
|
||||
const decision = entryDecision(r);
|
||||
const assignment = decision === undefined ? null : decision.output;
|
||||
|
||||
const checks = {
|
||||
"1 基线只扣了 1 次,Docker 里没有留下基线容器": { pass: baseStarted.length === 1 && leftover.length === 0, baseRunsPlanned: [...baseRuns], baseStarted: baseStarted.length, leftoverContainers: leftover },
|
||||
"2 被杀那次调用有结束记录,结局出错、原因进程中断": {
|
||||
pass: callEnds.every((c) => c.end !== null && c.end.status === "failed" && c.end.reason.includes("进程中断")),
|
||||
calls: callEnds,
|
||||
},
|
||||
"3 没交上的人被重新派了,交过的没被重派": { pass: redispatch.every((x) => x.ok), detail: redispatch },
|
||||
"4 每个阶段都关上了,每人每阶段只有一份最终收下的提交": {
|
||||
pass: perStage.every((s) => s.closed && Object.values(s.finalPerPerson).every((n) => n === 1)),
|
||||
detail: perStage,
|
||||
},
|
||||
"5 分配照常写出": { pass: assignment !== null, assignment, sameAsReference: assignment !== null && sameAssignment(assignment, script.expected) },
|
||||
};
|
||||
const report = { point, triggeredAt: triggered, lastRecordAtKill: atKill.at(-1), outcomeAfterRestart: second.outcome, checks };
|
||||
writeFileSync(join(out, "kill-report.json"), `${JSON.stringify(report, null, 2)}\n`);
|
||||
console.log(`\n杀点 ${point}:触发时账本最后一条 #${triggered.seq} ${triggered.kind};被杀时最后一条 #${killSeq} ${atKill.at(-1)!.kind} ${JSON.stringify(atKill.at(-1)!.content).slice(0, 200)}`);
|
||||
console.log(`再启动后的结局:${JSON.stringify(second.outcome)}`);
|
||||
for (const [k, v] of Object.entries(checks)) console.log(`${v.pass ? "通过" : "不通过"} ${k}`);
|
||||
console.log(`报告:${join(out, "kill-report.json")}`);
|
||||
Reference in New Issue
Block a user