import { execFileSync } from "node:child_process"; import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync, } from "node:fs"; import { loadavg, freemem, totalmem } from "node:os"; import { join, resolve } from "node:path"; import { fullAccounts, FULL_STOP } from "../lib/full.ts"; import { jsonFile } from "../lib/forward.ts"; import { sameContainerThroughResult, type ContainerSnapshot, } from "../lib/forward-evidence.ts"; import { ofKind, readLedger } from "../lib/ledger.ts"; import { launch, sleep, waitForLedger, writeConfig } from "../lib/study.ts"; import { watchTimeline } from "../lib/timeline.ts"; // 任务书 17 第 3.1 节:从头真实跑一份完整研究,到余额用完停在"导出还没写"。三份同时跑,由父会话各起一个进程; // 容器名带研究编号,这里只认本研究的,不和别的研究互相核对。 // node eval/full/run.ts <研究编号> <全新绝对输出目录> [--kill-after-first-ablation-start] // 服务一时调不通等中断后的续跑:npm run eval:study -- --resume --dir <输出目录>/study(照任务书 16 的做法)。 const [name, output, ...extra] = process.argv.slice(2); const kill = extra.length === 1 && extra[0] === "--kill-after-first-ablation-start"; if ( !name || !output || extra.length > 1 || (extra.length === 1 && !kill) || output !== resolve(output) ) { throw new Error( "用法:node eval/full/run.ts <研究编号> <全新绝对输出目录> [--kill-after-first-ablation-start]", ); } const out = output; if (existsSync(out)) throw new Error(`输出目录已存在,不覆盖现场:${out}`); mkdirSync(out, { recursive: true }); const studyDir = join(out, "study"); const config = writeConfig( "eval/round-zero/research.json", join(out, "research.json"), name, null, ); const daemonTime = () => execFileSync("docker", ["info", "--format", "{{.SystemTime}}"], { encoding: "utf8", }).trim(); const inspect = ( container: string, ): ContainerSnapshot & { Name: string; Created: string; State: { Running: boolean; StartedAt: string; FinishedAt: string; Status: string; }; } => JSON.parse( execFileSync("docker", ["inspect", container], { encoding: "utf8" }), )[0]; /** 事件点的重快照(进程列表、容器列表),照任务书 16 的 machine.jsonl 做法。 */ function observe(label: string) { appendFileSync( join(out, "machine.jsonl"), `${JSON.stringify({ at: new Date().toISOString(), label, processes: execFileSync("ps", ["-eo", "pid,ppid,lstart,args"], { encoding: "utf8", }), containers: execFileSync("docker", ["ps", "--no-trunc"], { encoding: "utf8", }), })}\n`, ); } /** 运行期间的轻量负载采样,几秒一条(任务书 17 第 3.1 节:和每次运行的 CPU/墙钟比对照,只列事实)。 */ function sample() { appendFileSync( join(out, "machine.jsonl"), `${JSON.stringify({ at: new Date().toISOString(), label: "sample", load: loadavg(), memory: { total: totalmem(), free: freemem() } })}\n`, ); } function start(path: string | null) { observe(path === null ? "resume" : "start"); const study = launch(path, studyDir, {}); appendFileSync( join(out, "launches.jsonl"), `${JSON.stringify({ event: "start", pid: study.child.pid, at: new Date().toISOString(), config: path })}\n`, ); const interrupt = () => study.child.kill("SIGINT"); process.on("SIGINT", interrupt); study.done.then((end) => { process.off("SIGINT", interrupt); appendFileSync( join(out, "launches.jsonl"), `${JSON.stringify({ event: "end", pid: study.child.pid, at: new Date().toISOString(), ...end })}\n`, ); }); return study; } const sampler = setInterval(sample, 5000); sampler.unref(); let active = start(config); let killEvidence: { run: string; cell: string; round: number; killSeq: number; } | null = null; try { await waitForLedger(studyDir, active.child); let watching = watchTimeline( studyDir, active.child, join(out, "timeline.jsonl"), ); if (kill) { // 盯住账本:第一个用途为 ablation 的运行确认启动之后、结果落账之前杀控制进程(照 eval/forward/run.ts 的 kill 做法)。 let hit = false; while (active.child.exitCode === null && active.child.signalCode === null) { const records = readLedger(studyDir); const plan = ofKind(records, "runPlanned").find( (r) => r.content.purpose === "ablation", ); if (plan !== undefined) { const run = plan.content.run; if (ofKind(records, "runResult").some((r) => r.content.run === run)) { throw new Error( "第一个消融运行已经有结果了,未打中;保留现场,不改杀别的格", ); } const started = ofKind(records, "runStarted").find( (r) => r.content.run === run, ); if (started !== undefined) { // 容器名是正式入口写在结构文档里的约定(inforsi-<研究编号>-<运行编号>),不 import flow 内部文件。 const container = `inforsi-${name}-${run}`; // 从杀前划定历史事件区间,覆盖恢复启动到目标结果落账;不把 spawn 当成恢复已执行。 const lifecycleSince = daemonTime(); const before = inspect(container); jsonFile(join(out, "container-before.json"), before); if (!before.State.Running) throw new Error("第一个消融容器已不在运行,未打中"); jsonFile(join(out, "kill.json"), { run, cell: plan.content.object, round: plan.content.round, planSeq: plan.seq, startSeq: started.seq, beforeKillSeq: records.at(-1)!.seq, pid: active.child.pid, at: new Date().toISOString(), container, }); if (!active.child.kill("SIGKILL")) throw new Error("未发出 SIGKILL"); const killed = await active.done; await watching; if (killed.signal !== "SIGKILL") throw new Error("控制进程不是以 SIGKILL 结束"); const atKill = readLedger(studyDir); jsonFile(join(out, "ledger-at-kill.json"), atKill); if (ofKind(atKill, "runResult").some((r) => r.content.run === run)) { throw new Error("杀后核对发现目标运行已经写结果,未打中;保留现场"); } jsonFile(join(out, "container-after-kill.json"), inspect(container)); observe("after-kill"); jsonFile( join(out, "container-before-resume.json"), inspect(container), ); killEvidence = { run, cell: plan.content.object!, round: plan.content.round, killSeq: atKill.at(-1)!.seq, }; active = start(null); watching = watchTimeline( studyDir, active.child, join(out, "timeline.jsonl"), ); let targetResult; while (true) { targetResult = ofKind(readLedger(studyDir), "runResult").find( (r) => r.content.run === run, ); if ( targetResult !== undefined || active.child.exitCode !== null || active.child.signalCode !== null ) break; await sleep(100); } const lifecycleUntil = daemonTime(); // Docker CLI 按时间回读,不依赖事件订阅的启动时序;原文全存,stats 核对缓存上限和目标 die。 writeFileSync( join(out, "container-events.jsonl"), execFileSync( "docker", [ "events", "--since", lifecycleSince, "--until", lifecycleUntil, "--format", "{{json .}}", ], { encoding: "utf8" }, ), ); jsonFile(join(out, "container-events-window.json"), { since: lifecycleSince, until: lifecycleUntil, run, resultSeq: targetResult?.seq ?? null, }); if (targetResult === undefined) throw new Error( "恢复进程没有交出目标结果,生命周期证据不完整;保留现场", ); hit = true; break; } } await sleep(100); } if (!hit) throw new Error( `没有打中第一个消融运行:${JSON.stringify(await active.done)}`, ); } await watching; const end = await active.done; jsonFile(join(out, "outcome.json"), end); observe("end"); const records = readLedger(studyDir); const accounts = fullAccounts(records); jsonFile(join(out, "accounts.json"), accounts); // 杀进程的三条检查(任务书 17 第 3.1 节):①恢复按容器名接着等同一个容器、没有重跑;②只扣一次;③结果照常写入、结算和后续步骤走完。 if (killEvidence !== null) { const evidence = JSON.parse( readFileSync(join(out, "kill.json"), "utf8"), ) as { run: string; cell: string; round: number; container: string }; const snapshots = (["before", "after-kill", "before-resume"] as const).map( (part) => JSON.parse( readFileSync(join(out, `container-${part}.json`), "utf8"), ) as ContainerSnapshot, ); const lifecycle = readFileSync(join(out, "container-events.jsonl"), "utf8") .split("\n") .filter(Boolean) .map((line) => JSON.parse(line)); const lifecycleChecks = sameContainerThroughResult( snapshots, lifecycle, evidence.container, ); const target = accounts.runs.find( (r) => r.plan.content.run === evidence.run, )!; const cellPlans = accounts.runs.filter( (r) => r.plan.content.purpose === "ablation" && r.plan.content.object === evidence.cell, ); const report = { evidence, killSeq: killEvidence.killSeq, snapshotMeaning: "杀前、杀后、恢复启动前;没有把快照当成恢复完成的证据", lifecycleChecks, checks: { sameContainerNoRerun: Object.values(lifecycleChecks).every(Boolean) && cellPlans.length === 1 && target.starts.length === 1, chargedOnce: target.starts.length === 1, resultSettledAndFinished: target.results.length === 1 && target.results[0]!.seq > killEvidence.killSeq && ofKind(records, "settlement").some( (s) => s.content.round === evidence.round, ) && accounts.halt !== null && accounts.halt.reason === FULL_STOP, }, }; jsonFile(join(out, "kill-report.json"), report); } if ( end.code !== 0 || !accounts.checks.haltedAtExport || !accounts.checks.chargedEqualsStarts ) { throw new Error( `没有走到"导出还没写"或账目不平,保留现场;查明原因后用 npm run eval:study -- --resume --dir ${studyDir} 续跑:${JSON.stringify(end)}`, ); } console.log(`研究走完:${studyDir}。仍需 export、stats 和独立质量审阅。`); } finally { clearInterval(sampler); if (active.child.exitCode === null && active.child.signalCode === null) { active.child.kill("SIGINT"); await active.done; } }