import { readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import type { LedgerRecord } from "../../src/ledger/index.ts"; import { entryDecision, finalSubmissions, ofKind, readLedger, sameAssignment, stages, type Assignment } from "../lib/ledger.ts"; import { replayModels, startReplay, type ReplayScript } from "../lib/replay.ts"; import { containers, launch, newEvalDir, sleep, waitForLedger, writeConfig } from "../lib/study.ts"; // 杀进程检验(任务书 12 第 5.4 节):在一处 SIGKILL 控制进程,再启动让它跑完第零轮,逐条查五件事。 // node eval/round-zero/kill.ts baseline|drafting|stageFull // 模型一侧用 16-real-1 的回放(不花钱、结果确定,分配应和当年相同);基线照样用真 Docker。 // 回放每次回复前等 DELAY_MS,留出"研究者正在写""最后一份刚收下、阶段还没关"的窗口。 const DELAY_MS = 3000; type Point = "baseline" | "drafting" | "stageFull"; const point = process.argv[2] as Point; /** 三处杀点:看到账本里出现这种状态就杀。 */ const HITS: Record boolean> = { // 基线容器正在跑:基线运行确认启动了、还没有结果。 baseline: (r) => { const base = new Set(ofKind(r, "runPlanned").filter((p) => p.content.purpose === "baseline").map((p) => p.content.run)); const done = new Set(ofKind(r, "runResult").map((x) => x.content.run)); return ofKind(r, "runStarted").some((s) => base.has(s.content.run) && !done.has(s.content.run)); }, // 研究者 2 正在写入口:他在写入口阶段的调用开始了、还没结束。 drafting: (r) => { const draft = new Set(stages(r).filter((s) => s.stage === "entryDraft").map((s) => s.stageId)); return openCalls(r).some((c) => c.content.actor === "researcher-2" && draft.has(c.content.stageId ?? "")); }, // 某阶段最后一份提交刚收下、阶段还没关:名单上的人都交了,还没有关闭记录。 stageFull: (r) => stages(r).some((s) => !s.closed && s.roster.every((a) => submitted(r, s.stageId, a))), }; if (!(point in HITS)) throw new Error(`杀点只能是 ${Object.keys(HITS).join("、")} 之一`); function openCalls(r: LedgerRecord[]): Array> { const ended = new Set(ofKind(r, "callEnded").map((e) => e.content.call)); return ofKind(r, "callStarted").filter((c) => !ended.has(c.content.call)); } const submitted = (r: LedgerRecord[], stageId: string, author: string) => ofKind(r, "submission").some((s) => s.content.stageId === stageId && s.content.author === author); const script = JSON.parse(readFileSync("eval/round-zero/replay-16-real-1.json", "utf8")) as ReplayScript & { expected: Assignment[] }; // 研究编号也是镜像标签的一部分,Docker 镜像名只许小写,所以杀点名转成小写。 const name = `r0-kill-${point.toLowerCase()}`; const out = newEvalDir(name); const replay = await startReplay(script, join(out, "pi-agent"), DELAY_MS); const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, replayModels(Object.keys(script.models).sort(), "jev-latest")); const studyDir = join(out, "study"); // --------------------------------------------------------------------------- // 第一次启动,打中就杀 // --------------------------------------------------------------------------- const first = launch(config, studyDir, replay.env); await waitForLedger(studyDir, first.child); let triggered: LedgerRecord | undefined; while (first.child.exitCode === null) { const r = readLedger(studyDir); if (HITS[point](r)) { first.child.kill("SIGKILL"); triggered = r.at(-1); break; } await sleep(20); } await first.done; if (triggered === undefined) throw new Error(`没打中 ${point}:研究进程自己结束了,账本最后一条是 ${JSON.stringify(readLedger(studyDir).at(-1))}`); // 进程已死,此刻的账本就是被杀时的样子。 const atKill = readLedger(studyDir); const killSeq = atKill.at(-1)!.seq; const killedCalls = openCalls(atKill); const openStages = stages(atKill).filter((s) => !s.closed); // --------------------------------------------------------------------------- // 再启动,跑完第零轮 // --------------------------------------------------------------------------- const second = await launch(null, studyDir, replay.env).done; await replay.close(); const r = readLedger(studyDir); const studyId = name; const baseRuns = new Set(ofKind(r, "runPlanned").filter((p) => p.content.purpose === "baseline").map((p) => p.content.run)); const baseStarted = ofKind(r, "runStarted").filter((s) => baseRuns.has(s.content.run)); const leftover = containers(`inforsi-${studyId}-`); const callEnds = killedCalls.map((c) => ({ call: c.content.call, end: ofKind(r, "callEnded").find((e) => e.content.call === c.content.call)?.content ?? null })); const redispatch = openStages.flatMap((s) => s.roster.map((actor) => { const before = submitted(atKill, s.stageId, actor); const again = ofKind(r, "callStarted").some((c) => c.seq > killSeq && c.content.stageId === s.stageId && c.content.actor === actor); return { stageId: s.stageId, actor, submittedBeforeKill: before, dispatchedAfterRestart: again, ok: before !== again }; }), ); const finals = finalSubmissions(r); const perStage = stages(r).map((s) => ({ stageId: s.stageId, closed: s.closed, finalPerPerson: Object.fromEntries(s.roster.map((a) => [a, finals.filter((f) => f.content.stageId === s.stageId && f.content.author === a).length])), })); const decision = entryDecision(r); const assignment = decision === undefined ? null : decision.output; const checks = { "1 基线只扣了 1 次,Docker 里没有留下基线容器": { pass: baseStarted.length === 1 && leftover.length === 0, baseRunsPlanned: [...baseRuns], baseStarted: baseStarted.length, leftoverContainers: leftover }, "2 被杀那次调用有结束记录,结局出错、原因进程中断": { pass: callEnds.every((c) => c.end !== null && c.end.status === "failed" && c.end.reason.includes("进程中断")), calls: callEnds, }, "3 没交上的人被重新派了,交过的没被重派": { pass: redispatch.every((x) => x.ok), detail: redispatch }, "4 每个阶段都关上了,每人每阶段只有一份最终收下的提交": { pass: perStage.every((s) => s.closed && Object.values(s.finalPerPerson).every((n) => n === 1)), detail: perStage, }, "5 分配照常写出": { pass: assignment !== null, assignment, sameAsReference: assignment !== null && sameAssignment(assignment, script.expected) }, }; const report = { point, triggeredAt: triggered, lastRecordAtKill: atKill.at(-1), outcomeAfterRestart: second.outcome, checks }; writeFileSync(join(out, "kill-report.json"), `${JSON.stringify(report, null, 2)}\n`); console.log(`\n杀点 ${point}:触发时账本最后一条 #${triggered.seq} ${triggered.kind};被杀时最后一条 #${killSeq} ${atKill.at(-1)!.kind} ${JSON.stringify(atKill.at(-1)!.content).slice(0, 200)}`); console.log(`再启动后的结局:${JSON.stringify(second.outcome)}`); for (const [k, v] of Object.entries(checks)) console.log(`${v.pass ? "通过" : "不通过"} ${k}`); console.log(`报告:${join(out, "kill-report.json")}`);