InfoRSI/eval/round-zero/kill.ts

import { readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import type { LedgerRecord } from "../../src/ledger/index.ts";
import { entryDecision, finalSubmissions, ofKind, readLedger, sameAssignment, stages, type Assignment } from "../lib/ledger.ts";
import { replayModels, startReplay, type ReplayScript } from "../lib/replay.ts";
import { containers, launch, newEvalDir, sleep, waitForLedger, writeConfig } from "../lib/study.ts";

// 杀进程检验（任务书 12 第 5.4 节）：在一处 SIGKILL 控制进程，再启动让它跑完第零轮，逐条查五件事。
//   node eval/round-zero/kill.ts baseline|drafting|stageFull
// 模型一侧用 16-real-1 的回放（不花钱、结果确定，分配应和当年相同）；基线照样用真 Docker。
// 回放每次回复前等 DELAY_MS，留出"研究者正在写""最后一份刚收下、阶段还没关"的窗口。

const DELAY_MS = 3000;
type Point = "baseline" | "drafting" | "stageFull";
const point = process.argv[2] as Point;

/** 三处杀点：看到账本里出现这种状态就杀。 */
const HITS: Record<Point, (r: LedgerRecord[]) => boolean> = {
  // 基线容器正在跑：基线运行确认启动了、还没有结果。
  baseline: (r) => {
    const base = new Set(ofKind(r, "runPlanned").filter((p) => p.content.purpose === "baseline").map((p) => p.content.run));
    const done = new Set(ofKind(r, "runResult").map((x) => x.content.run));
    return ofKind(r, "runStarted").some((s) => base.has(s.content.run) && !done.has(s.content.run));
  },
  // 研究者 2 正在写入口：他在写入口阶段的调用开始了、还没结束。
  drafting: (r) => {
    const draft = new Set(stages(r).filter((s) => s.stage === "entryDraft").map((s) => s.stageId));
    return openCalls(r).some((c) => c.content.actor === "researcher-2" && draft.has(c.content.stageId ?? ""));
  },
  // 某阶段最后一份提交刚收下、阶段还没关：名单上的人都交了，还没有关闭记录。
  stageFull: (r) => stages(r).some((s) => !s.closed && s.roster.every((a) => submitted(r, s.stageId, a))),
};
if (!(point in HITS)) throw new Error(`杀点只能是 ${Object.keys(HITS).join("、")} 之一`);

function openCalls(r: LedgerRecord[]): Array<LedgerRecord<"callStarted">> {
  const ended = new Set(ofKind(r, "callEnded").map((e) => e.content.call));
  return ofKind(r, "callStarted").filter((c) => !ended.has(c.content.call));
}
const submitted = (r: LedgerRecord[], stageId: string, author: string) =>
  ofKind(r, "submission").some((s) => s.content.stageId === stageId && s.content.author === author);

const script = JSON.parse(readFileSync("eval/round-zero/replay-16-real-1.json", "utf8")) as ReplayScript & { expected: Assignment[] };
// 研究编号也是镜像标签的一部分，Docker 镜像名只许小写，所以杀点名转成小写。
const name = `r0-kill-${point.toLowerCase()}`;
const out = newEvalDir(name);
const replay = await startReplay(script, join(out, "pi-agent"), DELAY_MS);
const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, replayModels(Object.keys(script.models).sort(), "jev-latest"));
const studyDir = join(out, "study");

// ---------------------------------------------------------------------------
// 第一次启动，打中就杀
// ---------------------------------------------------------------------------

const first = launch(config, studyDir, replay.env);
await waitForLedger(studyDir, first.child);
let triggered: LedgerRecord | undefined;
while (first.child.exitCode === null) {
  const r = readLedger(studyDir);
  if (HITS[point](r)) {
    first.child.kill("SIGKILL");
    triggered = r.at(-1);
    break;
  }
  await sleep(20);
}
await first.done;
if (triggered === undefined) throw new Error(`没打中 ${point}：研究进程自己结束了，账本最后一条是 ${JSON.stringify(readLedger(studyDir).at(-1))}`);
// 进程已死，此刻的账本就是被杀时的样子。
const atKill = readLedger(studyDir);
const killSeq = atKill.at(-1)!.seq;
const killedCalls = openCalls(atKill);
const openStages = stages(atKill).filter((s) => !s.closed);

// ---------------------------------------------------------------------------
// 再启动，跑完第零轮
// ---------------------------------------------------------------------------

const second = await launch(null, studyDir, replay.env).done;
await replay.close();
const r = readLedger(studyDir);
const studyId = name;

const baseRuns = new Set(ofKind(r, "runPlanned").filter((p) => p.content.purpose === "baseline").map((p) => p.content.run));
const baseStarted = ofKind(r, "runStarted").filter((s) => baseRuns.has(s.content.run));
const leftover = containers(`inforsi-${studyId}-`);
const callEnds = killedCalls.map((c) => ({ call: c.content.call, end: ofKind(r, "callEnded").find((e) => e.content.call === c.content.call)?.content ?? null }));
const redispatch = openStages.flatMap((s) =>
  s.roster.map((actor) => {
    const before = submitted(atKill, s.stageId, actor);
    const again = ofKind(r, "callStarted").some((c) => c.seq > killSeq && c.content.stageId === s.stageId && c.content.actor === actor);
    return { stageId: s.stageId, actor, submittedBeforeKill: before, dispatchedAfterRestart: again, ok: before !== again };
  }),
);
const finals = finalSubmissions(r);
const perStage = stages(r).map((s) => ({
  stageId: s.stageId,
  closed: s.closed,
  finalPerPerson: Object.fromEntries(s.roster.map((a) => [a, finals.filter((f) => f.content.stageId === s.stageId && f.content.author === a).length])),
}));
const decision = entryDecision(r);
const assignment = decision === undefined ? null : decision.output;

const checks = {
  "1 基线只扣了 1 次，Docker 里没有留下基线容器": { pass: baseStarted.length === 1 && leftover.length === 0, baseRunsPlanned: [...baseRuns], baseStarted: baseStarted.length, leftoverContainers: leftover },
  "2 被杀那次调用有结束记录，结局出错、原因进程中断": {
    pass: callEnds.every((c) => c.end !== null && c.end.status === "failed" && c.end.reason.includes("进程中断")),
    calls: callEnds,
  },
  "3 没交上的人被重新派了，交过的没被重派": { pass: redispatch.every((x) => x.ok), detail: redispatch },
  "4 每个阶段都关上了，每人每阶段只有一份最终收下的提交": {
    pass: perStage.every((s) => s.closed && Object.values(s.finalPerPerson).every((n) => n === 1)),
    detail: perStage,
  },
  "5 分配照常写出": { pass: assignment !== null, assignment, sameAsReference: assignment !== null && sameAssignment(assignment, script.expected) },
};
const report = { point, triggeredAt: triggered, lastRecordAtKill: atKill.at(-1), outcomeAfterRestart: second.outcome, checks };
writeFileSync(join(out, "kill-report.json"), `${JSON.stringify(report, null, 2)}\n`);
console.log(`\n杀点 ${point}：触发时账本最后一条 #${triggered.seq} ${triggered.kind}；被杀时最后一条 #${killSeq} ${atKill.at(-1)!.kind} ${JSON.stringify(atKill.at(-1)!.content).slice(0, 200)}`);
console.log(`再启动后的结局：${JSON.stringify(second.outcome)}`);
for (const [k, v] of Object.entries(checks)) console.log(`${v.pass ? "通过" : "不通过"}  ${k}`);
console.log(`报告：${join(out, "kill-report.json")}`);