diff --git a/eval/diagnosis/export.ts b/eval/diagnosis/export.ts new file mode 100644 index 0000000..4c485f4 --- /dev/null +++ b/eval/diagnosis/export.ts @@ -0,0 +1,223 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { + Board, + diagnosisId, + type TaskMaterial, +} from "../../src/materials/index.ts"; +import { diagnosisOutcome } from "../../src/submissions/index.ts"; +import { checkoutVersion, compareVersions } from "../../src/workspace/index.ts"; +import { jsonFile } from "../lib/forward.ts"; +import { finalSubmissions, ofKind, readLedger } from "../lib/ledger.ts"; + +// 任务书 18 第 3.2 节第 1、3、4 样的审阅材料。第 1 样的目录（blind/）只放诊断结算之前就存在的东西， +// 不放诊断结果；第 3 样（impl/）放诊断实现与核验的往来；第 4 样（direction/）放本轮结果和本轮提案、候选， +// 供子 Agent 标注"沿不沿诊断指出的方向"。只导出事实和原文，不替审阅者下结论。 +// node eval/diagnosis/export.ts <研究输出目录> <全新绝对审阅目录> +const [dir, output, ...extra] = process.argv.slice(2); +if (!dir || !output || extra.length || output !== resolve(output)) { + throw new Error( + "用法：node eval/diagnosis/export.ts <研究输出目录> <全新绝对审阅目录>", + ); +} +mkdirSync(output); +const study = join(dir, "study"); +const records = readLedger(study); +const board = new Board(records); +const repo = join(study, "repo.git"); +const finals = finalSubmissions(records); +const roundOf = (stageId: string) => board.stage(stageId).round; +const json = (v: unknown) => `${JSON.stringify(v, null, 2)}\n`; +const checkout = (version: string, path: string) => { + mkdirSync(path, { recursive: true }); + checkoutVersion(repo, version, path); +}; +const text = (path: string, body: string) => writeFileSync(path, body); + +const rounds = [ + ...new Set( + ofKind(records, "decision") + .filter((d) => d.content.decision === "selectDiagnosis") + .map((d) => d.content.round), + ), +].sort((a, b) => a - b); + +const index = []; +for (const round of rounds) { + const roundDir = join(output, `round-${round}`); + const id = diagnosisId(round); + const baseline = board.code(board.baselineOf(round)); + const baselineScore = board.baselineScore(board.baselineOf(round)); + const mine = (kind: string) => + finals.filter( + (s) => s.content.kind === kind && roundOf(s.content.stageId) === round, + ); + + // ---- 第 1 样（看不到诊断结果）---- + const blind = join(roundDir, "blind"); + mkdirSync(blind, { recursive: true }); + checkout(baseline.version, join(blind, "baseline-code")); + text( + join(blind, "baseline-score.txt"), + `本轮当前基线 ${board.baselineOf(round)} 的最终指标：${baselineScore}\n`, + ); + text( + join(blind, "diagnosis-proposals.txt"), + mine("diagnosisProposal") + .map( + (s) => + `${"=".repeat(20)} ${s.content.author}（记录 ${s.seq}）\n${json(s.content.content)}`, + ) + .join("\n"), + ); + text( + join(blind, "diagnosis-predictions.txt"), + mine("diagnosisPrediction") + .map( + (s) => + `${"=".repeat(20)} ${s.content.author}（记录 ${s.seq}）\n${json(s.content.content)}`, + ) + .join("\n"), + ); + jsonFile( + join(blind, "merged-experiments.json"), + board.diagnosisExperiments(round), + ); + // 研究者写诊断解释与实验时实际读到的材料（含信念本和上一轮收尾，不含本轮诊断结果）。 + for (const c of ofKind(records, "callStarted").filter((x) => { + const m = x.content.materials as TaskMaterial; + return m.kind === "diagnosisProposal" && m.round === round; + })) { + text( + join(blind, `researcher-material-${c.content.actor}.txt`), + (c.content.materials as TaskMaterial).firstMessage, + ); + } + text( + join(blind, "review.txt"), + `任务书18第3.2节第1样，只读本目录，中文直接回复，不写文件，看不到诊断结果。\n` + + `读 diagnosis-proposals.txt（每位研究者的诊断解释与实验）、diagnosis-predictions.txt（三个数）、baseline-code/ 与 baseline-score.txt、researcher-material-*.txt（研究者当时读到的材料，含以前轮次的结论）。回答：\n` + + `- 每条解释说出了具体的瓶颈吗？写了什么实测会推翻它吗？\n- 同一轮几位研究者的解释有没有真正不同？\n- 每个实验是不是只动一处？是不是真能把几种解释分开？\n- 有没有实验去测以前否掉过的路线的理由？\n`, + ); + + // ---- 选实验与结算（第 2、5 样的原始数据，供脚本和人读）---- + jsonFile( + join(roundDir, "select-decision.json"), + ofKind(records, "decision").find( + (d) => d.content.decision === "selectDiagnosis" && d.content.round === round, + ) ?? null, + ); + jsonFile( + join(roundDir, "settlement.json"), + ofKind(records, "diagnosisSettlement").find((s) => s.content.round === round) ?? null, + ); + + // ---- 第 3 样（诊断实现与核验）---- + const impl = join(roundDir, "impl"); + mkdirSync(impl, { recursive: true }); + const chosen = board.decision("selectDiagnosis", round)!.output; + if (chosen.diagnose) { + checkout(baseline.version, join(impl, "baseline")); + const attempts = finals + .filter( + (s) => + ["diagnosisImplementation", "diagnosisRepair"].includes(s.content.kind) && + s.content.target === id, + ) + .map((s) => ({ + submission: s, + code: ofKind(records, "code").find((c) => c.seq === s.seq + 1) ?? null, + })); + jsonFile(join(impl, "implementations.json"), attempts); + for (const a of attempts) { + if (a.code === null) continue; + checkout(a.code.content.version, join(impl, `program-${a.code.seq}`)); + jsonFile( + join(impl, `diff-${a.code.seq}.json`), + compareVersions(repo, baseline.version, a.code.content.version), + ); + } + const verifications = finals + .filter( + (s) => s.content.kind === "diagnosisVerification" && s.content.target === id, + ) + .map((s) => ({ + submission: s, + outcome: diagnosisOutcome( + s.content.content as Parameters[0], + ), + })); + jsonFile(join(impl, "verifications.json"), verifications); + jsonFile( + join(impl, "rejections.json"), + ofKind(records, "rejection").filter((r) => r.content.target === id), + ); + jsonFile( + join(impl, "selected-experiments.json"), + board.diagnosisExperiments(round).filter((e) => chosen.chosen.includes(e.id)), + ); + jsonFile( + join(impl, "calls.json"), + ofKind(records, "callStarted") + .filter((c) => { + const m = c.content.materials as TaskMaterial; + return ( + m.object === id && + ["diagnosisImplementation", "diagnosisRepair", "diagnosisVerification"].includes(m.kind) + ); + }) + .map((c) => ({ + record: c, + sessions: join(study, "calls", c.content.call, "sessions"), + })), + ); + text( + join(impl, "review.txt"), + `任务书18第3.2节第3样，只读本目录，中文直接回复，不写文件。\n` + + `读 selected-experiments.json（实验原文）、baseline/（当前基线代码）、program-<记录号>/ 与 diff-<记录号>.json（每次交上来的诊断程序与基线的差）、implementations.json（位置表、concerns、说明）、verifications.json（核验员逐实验结论）、rejections.json。回答：\n` + + `- 每个实验是不是只动了写的那一处，有没有偷偷改进别的地方；\n- 核验员该判不通过却通过的、不该判却判了的；有修复的，修复是否只改被指出的地方。\n`, + ); + } + + // ---- 第 4 样（沿不沿诊断指出的方向）---- + const direction = join(roundDir, "direction"); + mkdirSync(direction, { recursive: true }); + const settlement = board.diagnosisSettlement(round); + jsonFile(join(direction, "diagnosis-direction.json"), { + experiments: board.diagnosisExperiments(round), + direction: settlement?.output.direction ?? null, + outcomes: settlement?.output.experiments ?? null, + judged: settlement?.judged.experiments ?? null, + }); + const proposals = mine("proposal"); + text( + join(direction, "proposals.txt"), + proposals + .map( + (s) => + `${"=".repeat(20)} ${s.content.item}（${s.content.author}）\n${json(s.content.content)}`, + ) + .join("\n"), + ); + const candidates = ofKind(records, "protocol").filter( + (p) => (p.content.protocol as { round: number }).round === round, + ); + jsonFile( + join(direction, "candidates.json"), + candidates.map((p) => ({ + candidate: p.content.candidate, + proposals: board.proposalsOf(p.content.candidate), + protocol: p.content.protocol, + chosen: board.decision("batch", round)?.output.chosen.includes(p.content.candidate) ?? null, + })), + ); + text( + join(direction, "review.txt"), + `任务书18第3.2节第4样的标注，只读本目录，中文直接回复，最后给一段 JSON：{"proposals":{"<提案编号>":"along"|"not"},"candidates":{"<候选编号>":"along"|"not"}}。\n` + + `读 diagnosis-direction.json（本轮诊断推出的方向：direction.experiment 及它的实验原文、实测收益；found 为 false 时没有方向，全部标 not）、proposals.txt（本轮每位研究者交的原提案）、candidates.json（合并后的候选和冻结协议）。\n` + + `判断每条提案、每个候选是否"沿着诊断指出的方向"：它动的地方或要换的做法，和诊断里收益最大的那个实验所动的地方属于同一处或明确把它当起点；只是在别处小步调参或走别的路线，标 not。同时逐条写一句理由，并写明提案正文有没有引用诊断结果。\n`, + ); + index.push({ round, dir: roundDir, diagnosed: chosen.diagnose }); +} +jsonFile(join(output, "index.json"), { study: dir, rounds: index }); +console.log(`审阅材料：${output}`); diff --git a/eval/diagnosis/run.ts b/eval/diagnosis/run.ts new file mode 100644 index 0000000..0a98a37 --- /dev/null +++ b/eval/diagnosis/run.ts @@ -0,0 +1,352 @@ +import { execFileSync } from "node:child_process"; +import { + appendFileSync, + existsSync, + mkdirSync, + readFileSync, + writeFileSync, +} from "node:fs"; +import { loadavg, freemem, totalmem } from "node:os"; +import { join, resolve } from "node:path"; +import { fullAccounts, FULL_STOP } from "../lib/full.ts"; +import { jsonFile } from "../lib/forward.ts"; +import { + sameContainerThroughResult, + type ContainerSnapshot, +} from "../lib/forward-evidence.ts"; +import { Board } from "../../src/materials/index.ts"; +import { ofKind, readLedger } from "../lib/ledger.ts"; +import { launch, sleep, waitForLedger, writeConfig } from "../lib/study.ts"; +import { watchTimeline } from "../lib/timeline.ts"; + +// 任务书 18 第 3.1 节：从头真实跑一份诊断开、消融关的完整研究，到余额用完停在"导出还没写"。三份同时跑，由父会话各起一个进程； +// 容器名带研究编号，这里只认本研究的，不和别的研究互相核对。 +// node eval/diagnosis/run.ts <研究编号> <绝对输出目录> [--kill-after-first-diagnosis-start] [--resume] + +const [name, output, ...extra] = process.argv.slice(2); +// 选项：--kill-after-first-diagnosis-start（杀一次）、--resume（模型服务一时调不通而停下后接着跑，同一个研究目录）。 +const OPTIONS = ["--kill-after-first-diagnosis-start", "--resume"]; +const kill = extra.includes("--kill-after-first-diagnosis-start"); +const resume = extra.includes("--resume"); +if ( + !name || + !output || + extra.some((x) => !OPTIONS.includes(x)) || + new Set(extra).size !== extra.length || + output !== resolve(output) +) { + throw new Error( + "用法：node eval/diagnosis/run.ts <研究编号> <绝对输出目录> [--kill-after-first-diagnosis-start] [--resume]", + ); +} +const out = output; +const studyDir = join(out, "study"); +let config: string | null = null; +if (resume) { + if (!existsSync(studyDir)) throw new Error(`没有可续跑的研究：${studyDir}`); +} else { + if (existsSync(out)) throw new Error(`输出目录已存在，不覆盖现场：${out}`); + mkdirSync(out, { recursive: true }); + config = writeConfig( + "eval/diagnosis/research.json", + join(out, "research.json"), + name, + null, + ); +} + +const daemonTime = () => + execFileSync("docker", ["info", "--format", "{{.SystemTime}}"], { + encoding: "utf8", + }).trim(); +const inspect = ( + container: string, +): ContainerSnapshot & { + Name: string; + Created: string; + State: { + Running: boolean; + StartedAt: string; + FinishedAt: string; + Status: string; + }; +} => + JSON.parse( + execFileSync("docker", ["inspect", container], { encoding: "utf8" }), + )[0]; + +/** 事件点的重快照（进程列表、容器列表），照任务书 16 的 machine.jsonl 做法。 */ +function observe(label: string) { + appendFileSync( + join(out, "machine.jsonl"), + `${JSON.stringify({ + at: new Date().toISOString(), + label, + processes: execFileSync("ps", ["-eo", "pid,ppid,lstart,args"], { + encoding: "utf8", + }), + containers: execFileSync("docker", ["ps", "--no-trunc"], { + encoding: "utf8", + }), + })}\n`, + ); +} + +/** 运行期间的轻量负载采样，几秒一条（任务书 17 第 3.1 节：和每次运行的 CPU/墙钟比对照，只列事实）。 */ +function sample() { + appendFileSync( + join(out, "machine.jsonl"), + `${JSON.stringify({ at: new Date().toISOString(), label: "sample", load: loadavg(), memory: { total: totalmem(), free: freemem() } })}\n`, + ); +} + +function start(path: string | null) { + observe(path === null ? "resume" : "start"); + const study = launch(path, studyDir, {}); + appendFileSync( + join(out, "launches.jsonl"), + `${JSON.stringify({ event: "start", pid: study.child.pid, at: new Date().toISOString(), config: path })}\n`, + ); + const interrupt = () => study.child.kill("SIGINT"); + process.on("SIGINT", interrupt); + study.done.then((end) => { + process.off("SIGINT", interrupt); + appendFileSync( + join(out, "launches.jsonl"), + `${JSON.stringify({ event: "end", pid: study.child.pid, at: new Date().toISOString(), ...end })}\n`, + ); + }); + return study; +} + +const sampler = setInterval(sample, 5000); +sampler.unref(); +let active = start(config); +let killEvidence: { + run: string; + cell: string; + round: number; + killSeq: number; +} | null = null; +try { + await waitForLedger(studyDir, active.child); + let watching = watchTimeline( + studyDir, + active.child, + join(out, "timeline.jsonl"), + ); + if (kill) { + // 盯住账本：第一个用途为 diagnosis 的运行确认启动之后、结果落账之前杀控制进程（照 eval/forward/run.ts 的 kill 做法）。 + let hit = false; + while (active.child.exitCode === null && active.child.signalCode === null) { + const records = readLedger(studyDir); + const plan = ofKind(records, "runPlanned").find( + (r) => r.content.purpose === "diagnosis", + ); + if (plan !== undefined) { + const run = plan.content.run; + if (ofKind(records, "runResult").some((r) => r.content.run === run)) { + throw new Error( + "第一个诊断运行已经有结果了，未打中；保留现场，不改杀别的轮", + ); + } + const started = ofKind(records, "runStarted").find( + (r) => r.content.run === run, + ); + if (started !== undefined) { + // 容器名是正式入口写在结构文档里的约定（inforsi-<研究编号>-<运行编号>），不 import flow 内部文件。 + const container = `inforsi-${name}-${run}`; + // 从杀前划定历史事件区间，覆盖恢复启动到目标结果落账；不把 spawn 当成恢复已执行。 + const lifecycleSince = daemonTime(); + const before = inspect(container); + jsonFile(join(out, "container-before.json"), before); + if (!before.State.Running) + throw new Error("第一个诊断容器已不在运行，未打中"); + jsonFile(join(out, "kill.json"), { + run, + cell: plan.content.object, + round: plan.content.round, + planSeq: plan.seq, + startSeq: started.seq, + beforeKillSeq: records.at(-1)!.seq, + pid: active.child.pid, + at: new Date().toISOString(), + container, + }); + if (!active.child.kill("SIGKILL")) throw new Error("未发出 SIGKILL"); + const killed = await active.done; + await watching; + if (killed.signal !== "SIGKILL") + throw new Error("控制进程不是以 SIGKILL 结束"); + const atKill = readLedger(studyDir); + jsonFile(join(out, "ledger-at-kill.json"), atKill); + if (ofKind(atKill, "runResult").some((r) => r.content.run === run)) { + throw new Error("杀后核对发现目标运行已经写结果，未打中；保留现场"); + } + jsonFile(join(out, "container-after-kill.json"), inspect(container)); + observe("after-kill"); + jsonFile( + join(out, "container-before-resume.json"), + inspect(container), + ); + killEvidence = { + run, + cell: plan.content.object!, + round: plan.content.round, + killSeq: atKill.at(-1)!.seq, + }; + active = start(null); + watching = watchTimeline( + studyDir, + active.child, + join(out, "timeline.jsonl"), + ); + let targetResult; + while (true) { + targetResult = ofKind(readLedger(studyDir), "runResult").find( + (r) => r.content.run === run, + ); + if ( + targetResult !== undefined || + active.child.exitCode !== null || + active.child.signalCode !== null + ) + break; + await sleep(100); + } + const lifecycleUntil = daemonTime(); + // Docker CLI 按时间回读，不依赖事件订阅的启动时序；原文全存，stats 核对缓存上限和目标 die。 + writeFileSync( + join(out, "container-events.jsonl"), + execFileSync( + "docker", + [ + "events", + "--since", + lifecycleSince, + "--until", + lifecycleUntil, + "--format", + "{{json .}}", + ], + { encoding: "utf8" }, + ), + ); + jsonFile(join(out, "container-events-window.json"), { + since: lifecycleSince, + until: lifecycleUntil, + run, + resultSeq: targetResult?.seq ?? null, + }); + if (targetResult === undefined) + throw new Error( + "恢复进程没有交出目标结果，生命周期证据不完整；保留现场", + ); + hit = true; + break; + } + } + await sleep(100); + } + if (!hit) + throw new Error( + `没有打中第一个诊断运行：${JSON.stringify(await active.done)}`, + ); + } + await watching; + const end = await active.done; + jsonFile(join(out, "outcome.json"), end); + observe("end"); + const records = readLedger(studyDir); + const accounts = fullAccounts(records); + jsonFile(join(out, "accounts.json"), accounts); + + // 杀进程的三条检查（任务书 17 第 3.1 节）：①恢复按容器名接着等同一个容器、没有重跑；②只扣一次；③结果照常写入、结算和后续步骤走完。 + if (killEvidence !== null) { + const evidence = JSON.parse( + readFileSync(join(out, "kill.json"), "utf8"), + ) as { run: string; cell: string; round: number; container: string }; + const snapshots = (["before", "after-kill", "before-resume"] as const).map( + (part) => + JSON.parse( + readFileSync(join(out, `container-${part}.json`), "utf8"), + ) as ContainerSnapshot, + ); + const lifecycle = readFileSync(join(out, "container-events.jsonl"), "utf8") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); + const lifecycleChecks = sameContainerThroughResult( + snapshots, + lifecycle, + evidence.container, + ); + const target = accounts.runs.find( + (r) => r.plan.content.run === evidence.run, + )!; + const cellPlans = accounts.runs.filter( + (r) => + r.plan.content.purpose === "diagnosis" && + r.plan.content.object === evidence.cell, + ); + const board = new Board(records); + const settlement = board.diagnosisSettlement(evidence.round); + const settlementSeq = ofKind(records, "diagnosisSettlement").find( + (r) => r.content.round === evidence.round, + )!.seq; + // 选中的每个实验：核验不通过拿掉的记在 dropped，其余都在逐实验判分里有一行。 + const chosen = board.decision("selectDiagnosis", evidence.round)!.output.chosen; + const perExperimentRecorded = + settlement !== undefined && + chosen.every( + (e) => + settlement.dropped.some((d) => d.experiment === e) || + settlement.judged.experiments.some((x) => x.id === e), + ); + const report = { + evidence, + killSeq: killEvidence.killSeq, + snapshotMeaning: "杀前、杀后、恢复启动前；没有把快照当成恢复完成的证据", + lifecycleChecks, + checks: { + sameContainerNoRerun: + Object.values(lifecycleChecks).every(Boolean) && + cellPlans.length === 1 && + target.starts.length === 1, + chargedOnce: target.starts.length === 1, + resultSettledAndFinished: + target.results.length === 1 && + target.results[0]!.seq > killEvidence.killSeq && + settlement !== undefined && + settlement.run === evidence.run && + perExperimentRecorded && + ofKind(records, "stageOpen").some( + (o) => + o.content.stage === "proposal" && + o.content.round === evidence.round && + o.seq > settlementSeq, + ) && + accounts.halt !== null && + accounts.halt.reason === FULL_STOP, + }, + }; + jsonFile(join(out, "kill-report.json"), report); + } + + if ( + end.code !== 0 || + !accounts.checks.haltedAtExport || + !accounts.checks.chargedEqualsStarts + ) { + throw new Error( + `没有走到"导出还没写"或账目不平，保留现场；查明原因后用 npm run eval:study -- --resume --dir ${studyDir} 续跑：${JSON.stringify(end)}`, + ); + } + console.log(`研究走完：${studyDir}。仍需 export、stats 和独立质量审阅。`); +} finally { + clearInterval(sampler); + if (active.child.exitCode === null && active.child.signalCode === null) { + active.child.kill("SIGINT"); + await active.done; + } +} diff --git a/eval/diagnosis/stats.ts b/eval/diagnosis/stats.ts new file mode 100644 index 0000000..06d977a --- /dev/null +++ b/eval/diagnosis/stats.ts @@ -0,0 +1,400 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import type { TaskMaterial } from "../../src/materials/index.ts"; +import { diagnosisAccounts } from "../lib/diagnosis.ts"; +import { cpuRatio, CPU_RATIO_FLAG } from "../lib/full.ts"; +import { frozenConfig, jsonFile } from "../lib/forward.ts"; +import { ofKind, readLedger } from "../lib/ledger.ts"; +import { lightCheckUsage } from "../lib/light-checks.ts"; +import { readTimeline } from "../lib/timeline.ts"; + +// 任务书 18 第 3.2 节第 2、4、5、6、7 样里能由账本算出的数字。只算事实、不作质量判断 +// （第 1、3 样和"提案是否沿诊断方向"由全新上下文的子 Agent 读 eval/diagnosis/export.ts 的材料）。 +// 第 4 样"沿这个方向"的列来自子 Agent 的标注文件 labels.json（可选）： +// { "<研究编号>": { "<轮次>": { "proposals": { "<提案编号>": "along"|"not" }, "candidates": { "<候选编号>": "along"|"not" } } } } +// node eval/diagnosis/stats.ts <全新绝对统计输出目录> [--labels ] <研究输出目录>... +const args = process.argv.slice(2); +const output = args.shift(); +let labelsPath: string | null = null; +if (args[0] === "--labels") { + args.shift(); + labelsPath = args.shift() ?? null; +} +const dirs = args; +if (!output || dirs.length === 0 || output !== resolve(output)) { + throw new Error( + "用法：node eval/diagnosis/stats.ts <全新绝对统计输出目录> [--labels ] <研究输出目录>...", + ); +} +mkdirSync(output); + +type Label = "along" | "not"; +type Labels = Record< + string, + Record< + string, + { proposals: Record; candidates: Record } + > +>; +const labels: Labels = + labelsPath === null ? {} : JSON.parse(readFileSync(labelsPath, "utf8")); + +const f = (v: number | null | undefined, d = 6) => + v === null || v === undefined ? "—" : v.toFixed(d); +const e3 = (v: number | null | undefined) => + v === null || v === undefined ? "—" : v.toExponential(3); +const range = (r: { q1: number; q5: number; q9: number } | null) => + r === null ? "—" : `${f(r.q1)}/${f(r.q5)}/${f(r.q9)}`; + +// 旧研究（任务书 17 的 B、D）和参考项目逐轮采纳分数。 +const OLD = "/home/iomgaa/inforsi-eval/2026-10-04-task17"; +const oldTrajectory = (x: "b" | "d") => { + const stats = JSON.parse( + readFileSync(join(OLD, `stats-${x}`, "stats.json"), "utf8"), + ) as Array<{ + study: string; + accounts: { rounds: Array<{ round: number; adoptedScore: number | null }> }; + }>; + return { + name: stats[0]!.study, + scores: stats[0]!.accounts.rounds.map((r) => r.adoptedScore), + }; +}; +function referenceTrajectory(study: string) { + const db = new DatabaseSync( + `/home/iomgaa/rb-acceptance/${study}/board.sqlite`, + { readOnly: true }, + ); + try { + const rows = (kind: string) => + ( + db + .prepare("SELECT payload FROM facts WHERE kind = ? ORDER BY created_seq") + .all(kind) as Array<{ payload: string }> + ).map((r) => JSON.parse(r.payload)); + const obs = rows("run-observation") as Array<{ + runId: string; + finalMetric: { value: number | null }; + }>; + return ( + rows("adoption") as Array<{ adoptedRunId: string | null }> + ).map((a) => + a.adoptedRunId === null + ? null + : obs.find((o) => o.runId === a.adoptedRunId)!.finalMetric.value, + ); + } finally { + db.close(); + } +} + +const summaries = dirs.map((dir) => { + const studyDir = join(dir, "study"); + const records = readLedger(studyDir); + const config = frozenConfig(records); + const accounts = diagnosisAccounts(records); + const study = config.studyId; + const studyLabels = labels[study] ?? {}; + + // 轻量检查：实现/修复/诊断实现调用用了几次，有没有超过配置上限。 + const lightChecks = ofKind(records, "callStarted") + .filter((c) => + [ + "implementation", + "repair", + "diagnosisImplementation", + "diagnosisRepair", + ].includes((c.content.materials as TaskMaterial).kind), + ) + .map((c) => { + const sessions = join(studyDir, "calls", c.content.call, "sessions"); + return { + call: c.content.call, + seq: c.seq, + ...(existsSync(sessions) + ? lightCheckUsage(sessions, config.lightChecks.maxCount) + : { + maxCount: config.lightChecks.maxCount, + pass: false, + note: "会话目录不存在", + }), + }; + }); + + // CPU 时间 / 墙钟时间之比，并附同时段机器负载。 + const timeline = existsSync(join(dir, "timeline.jsonl")) + ? readTimeline(join(dir, "timeline.jsonl")) + : []; + const atOf = (seq: number) => timeline.find((t) => t.seq === seq)?.at ?? null; + const machine = existsSync(join(dir, "machine.jsonl")) + ? readFileSync(join(dir, "machine.jsonl"), "utf8") + .split("\n") + .filter(Boolean) + .map( + (line) => + JSON.parse(line) as { at: string; label: string; load?: number[] }, + ) + : []; + const ratios = accounts.runs.map((r) => { + const result = r.results[0] ?? null; + const ratio = result === null ? null : cpuRatio(result.content); + const start = r.starts[0] === undefined ? null : atOf(r.starts[0].seq); + const end = result === null ? null : atOf(result.seq); + const samples = + start === null || end === null + ? [] + : machine + .filter( + (m) => + m.label === "sample" && + Date.parse(m.at) >= start && + Date.parse(m.at) <= end, + ) + .map((m) => ({ at: m.at, load: m.load })); + return { + run: r.plan.content.run, + purpose: r.plan.content.purpose, + round: r.plan.content.round, + ratio, + flagged: ratio !== null && ratio < CPU_RATIO_FLAG, + start: start === null ? null : new Date(start).toISOString(), + end: end === null ? null : new Date(end).toISOString(), + meanLoad1: + samples.length === 0 + ? null + : samples.reduce((s, x) => s + x.load![0]!, 0) / samples.length, + }; + }); + const measuredRatios = ratios.filter((r) => r.ratio !== null); + + const killPath = join(dir, "kill-report.json"); + const kill = existsSync(killPath) + ? (JSON.parse(readFileSync(killPath, "utf8")) as { + evidence: unknown; + checks: Record; + }) + : null; + + // 第 4 样：每轮一行。 + const dr = accounts.diagnosisRounds; + const table4 = dr.map((r) => { + const d = r.direction; + const label = studyLabels[String(r.round)]; + const proposals = board4(records, r.round); + const proposalLabels = proposals.map((p) => label?.proposals[p.id] ?? null); + const candidateRows = r.forward.map((c) => ({ + ...c, + chosen: true, + label: label?.candidates[c.candidate] ?? null, + })); + const along = candidateRows.filter((c) => c.label === "along"); + const diagGain = d === null ? null : d.gain; + return { + round: r.round, + direction: + d === null + ? r.diagnosed + ? "诊断未结算" + : "本轮没有诊断" + : d.found + ? `${d.experiment}（${d.gains.find((g) => g.experiment === d.experiment)!.measured ? "实测" : "估计"}收益 ${f(d.gain)}）` + : `没有指出能涨分的方向（最大收益 ${f(d.gain)}，${d.experiment}）`, + found: d?.found ?? null, + diagnosisGain: diagGain, + proposalCount: proposals.length, + proposalsAlong: + label === undefined ? null : proposalLabels.filter((x) => x === "along").length, + proposalsCiteDiagnosis: proposals.filter((p) => p.cites).length, + candidatesAlong: label === undefined ? null : along.length, + alongChosen: label === undefined ? null : along.length, + candidates: candidateRows.map((c) => ({ + candidate: c.candidate, + label: c.label, + gain: c.gain, + vsDiagnosis: + c.gain === null || diagGain === null + ? null + : c.gain > diagGain + ? "高于诊断实测" + : "不高于诊断实测", + })), + adopted: accounts.rounds.find((x) => x.round === r.round)?.adopted ?? null, + // 诊断指出了方向却没有人沿它提（有标注才判）。 + directionNotFollowed: + label === undefined || d === null || !d.found + ? null + : proposalLabels.every((x) => x === "not"), + }; + }); + + const pass = + Object.values(accounts.diagnosisChecks).every(Boolean) && + lightChecks.every((c) => c.pass) && + (kill === null || Object.values(kill.checks).every(Boolean)); + return { + dir, + study, + accounts, + lightChecks, + ratios, + cpuRatioMin: + measuredRatios.length === 0 + ? null + : Math.min(...measuredRatios.map((r) => r.ratio!)), + kill, + table4, + pass, + }; +}); + +/** 本轮的原提案：编号和是否出现"诊断"字样（机械统计，不是"沿方向"的判断）。 */ +function board4( + records: ReturnType, + round: number, +): Array<{ id: string; cites: boolean }> { + const stage = `r${round}-proposal`; + const latest = new Map(); + for (const r of records) { + if (r.kind === "submission" && r.content.stageId === stage) { + latest.set(`${r.content.author}/${r.content.item}`, r); + } + } + return [...latest.values()].map((r) => { + if (r.kind !== "submission") throw new Error("不可能"); + return { + id: r.content.item!, + cites: /诊断|diagnos/i.test(JSON.stringify(r.content.content)), + }; + }); +} + +jsonFile(join(output, "stats.json"), summaries); + +const oldB = oldTrajectory("b"); +const oldD = oldTrajectory("d"); +const refs = ["15-real-1", "15-tight-2", "16-real-1"].map((s) => ({ + name: s, + scores: referenceTrajectory(s), +})); + +const text: string[] = []; +for (const s of summaries) { + const a = s.accounts; + text.push( + `${"=".repeat(70)}`, + `研究 ${s.study}（${s.dir}）`, + `第 7 样 程序账目`, + `确认启动：基线 ${a.confirmedStarts.baseline}、诊断 ${a.runs.filter((r) => r.plan.content.purpose === "diagnosis" && r.starts.length > 0).length}、正向 ${a.confirmedStarts.forward}、消融 ${a.confirmedStarts.ablation}，共 ${a.confirmedStarts.total}；扣次 ${a.balance.charged}，余额 ${a.balance.balance}，占用 ${a.balance.reserved}；已知最好 ${f(a.knownBest)}`, + `停下：${a.halt === null ? "（没有停下事件）" : a.halt.reason}`, + `账目核对：${Object.entries(a.diagnosisChecks) + .map(([k, v]) => `${k}=${v ? "过" : "不过"}`) + .join("、")}`, + `轻量检查：${s.lightChecks.every((c) => c.pass) ? "全部在上限内" : "有超限或会话缺失，见 stats.json"}`, + `CPU/墙钟比：${s.ratios.filter((r) => r.ratio !== null).length} 次运行有比值，最小 ${f(s.cpuRatioMin, 3)}；低于 ${CPU_RATIO_FLAG} 的：${s.ratios.filter((r) => r.flagged).map((r) => `${r.run}=${r.ratio!.toFixed(3)}（同时段 1 分钟负载均值 ${f(r.meanLoad1, 1)}）`).join("、") || "没有"}`, + s.kill === null + ? "杀进程：这份没做" + : `杀进程：${Object.entries(s.kill.checks).map(([k, v]) => `${k}=${v ? "过" : "不过"}`).join("、")}`, + "", + "每次诊断：轮 | 余额(开轮前) | 诊断 | 选中实验 | 核验次 | 修复次 | 拿掉 | 跑了 | 扣次 | 预测冻结早于启动 | 结算数", + ); + for (const r of a.diagnosisRounds) { + text.push( + `${r.round} | ${r.balanceBefore} | ${r.expectedDiagnosis ? (r.diagnosed ? "做" : "不做") : "无（余额<2）"} | ${r.chosen.join(",") || "—"} | ${r.verificationCount} | ${r.repairCount} | ${r.droppedCount} | ${r.ranCount} | ${r.charged} | ${r.predictionBeforeStart === null ? "—" : r.predictionBeforeStart ? "是" : "否"} | ${r.settlementCount}`, + ); + } + text.push( + "", + "每个实验：轮 | 实验 | 作者 | 动哪处 | 选中 | 结局 | 实测 | 改进量 | 用时秒 | ending | 世界比特 | 惊讶比特 | 没做原因/拿掉原因", + ); + for (const r of a.diagnosisRounds) { + for (const x of r.experiments) { + text.push( + `${r.round} | ${x.id} ${x.name} | ${x.authors.join("+")} | ${x.spot} | ${x.chosen ? "是" : "否"} | ${x.outcome ?? "—"} | ${f(x.score)} | ${f(x.gain)} | ${f(x.wallSeconds, 2)} | ${x.ending ?? "—"} | ${f(x.worldBits, 4)} | ${f(x.surprise, 4)} | ${x.dropped ?? (x.skipReason === null ? "—" : `${x.skipReason}（含它最好组合净值 ${e3(x.skipBestNet)}）`)}`, + ); + } + } + + text.push("", "第 2 样 选实验：每轮合并实验、事前期望收益、各组合价值/成本/净值、选了谁"); + for (const r of a.diagnosisRounds) { + if (!r.expectedDiagnosis) { + text.push(`第 ${r.round} 轮：余额 ${r.balanceBefore} < 2，无诊断记录`); + continue; + } + text.push( + `第 ${r.round} 轮（基线 ${f(r.baselineScore)}，参数 ${JSON.stringify(r.selectParams)}，ĝ₀ ${f(r.g0)}）：${r.diagnosed ? `做，选 ${r.chosen.join("、")}，净值 ${e3(r.chosenNet)}` : "不做（所有组合净值都不大于 1e-12）"}`, + ...r.experiments.map( + (x) => + ` ${x.id} ${x.name}：事前期望收益 ${f(x.priorGain)}；事前合并范围 ${range(x.merged)}`, + ), + ...r.combos.map( + (c) => + ` 组合 {${c.experiments.join(",")}}：VOI ${e3(c.voi)}，价值 ${e3(c.value)}，成本 ${e3(c.cost)}，净值 ${e3(c.net)}（R̂ ${c.rHat}，m′ ${f(c.mPrime, 4)}）`, + ), + ); + } + + text.push("", "第 4 样 诊断有没有改变走向（'沿方向'列需子 Agent 标注 labels.json，没有就写 —）"); + text.push("轮 | 诊断推出的方向 | 本轮提案数 | 提案引用'诊断'字样 | 沿方向的提案 | 沿方向的候选(选批进了几个) | 采纳 | 各入选候选：标注/正向改进量/对比诊断实测"); + for (const t of s.table4) { + text.push( + `${t.round} | ${t.direction} | ${t.proposalCount} | ${t.proposalsCiteDiagnosis} | ${t.proposalsAlong ?? "—"} | ${t.candidatesAlong ?? "—"} | ${t.adopted ?? "没有"} | ${t.candidates.map((c) => `${c.candidate}:${c.label ?? "—"}/${f(c.gain)}/${c.vsDiagnosis ?? "—"}`).join("；")}`, + ); + } + text.push( + `诊断指出方向却无人沿它提的轮次：${ + s.table4.some((t) => t.directionNotFollowed === null && t.found === true) + ? "待标注" + : s.table4.filter((t) => t.directionNotFollowed).map((t) => t.round).join("、") || "没有" + }`, + ); + + text.push("", "第 5 样 预测准不准、分不分得开"); + for (const r of a.diagnosisRounds) { + if (r.predictionRows.length === 0 && r.mergedRows.length === 0) { + text.push(`第 ${r.round} 轮：没有有效实验可对`); + continue; + } + const byAuthor = new Map(); + for (const p of r.predictionRows) { + byAuthor.set(p.author, [...(byAuthor.get(p.author) ?? []), p]); + } + text.push(`第 ${r.round} 轮：`); + for (const [author, rows] of byAuthor) { + const bands = { "q9": 0 } as Record; + for (const p of rows) bands[p.band] = bands[p.band]! + 1; + text.push( + ` ${author}：${rows.length} 对，落点 ${JSON.stringify(bands)}，覆盖率 ${rows.filter((p) => p.covered).length}/${rows.length}`, + ); + } + const mb = { "q9": 0 } as Record; + for (const p of r.mergedRows) mb[p.band] = mb[p.band]! + 1; + text.push( + ` 合并：${r.mergedRows.length} 个实验，落点 ${JSON.stringify(mb)}，覆盖率 ${r.mergedRows.filter((p) => p.covered).length}/${r.mergedRows.length}`, + ` 同轮各实验合并中位数差距：${r.experimentGap === null ? "—" : `最大减最小 ${f(r.experimentGap.maxMinusMin)}，相邻最小间隔 ${f(r.experimentGap.minPairwise)}`}；同轮正向候选合并中位数差距：${r.candidateGap === null ? "—" : `最大减最小 ${f(r.candidateGap.maxMinusMin)}，相邻最小间隔 ${f(r.candidateGap.minPairwise)}`}`, + ); + } + + text.push("", "第 6 样 逐轮走向与候选"); + for (const r of a.rounds) { + const dRound = a.diagnosisRounds.find((x) => x.round === r.round)!; + text.push( + `第 ${r.round} 轮（基线 ${r.baseline}）：诊断${dRound.diagnosed ? `做了，${dRound.ranCount} 个实验` : dRound.expectedDiagnosis ? "没做（净值不大于 0）" : "没有（余额<2）"}；采纳 ${r.adopted ?? "没有"}${r.adoptedScore === null ? "" : `，分数 ${f(r.adoptedScore)}，离已知最好 ${f(r.knownBestGap)}`}`, + ...r.candidates.map( + (c) => + ` 候选 ${c.candidate}：实测 ${f(c.score)}，合并 ${range(c.merged)}，落点 ${c.band ?? "—"}，作者自评 ${c.authorEstimates.map((u) => `${u.author} ${range(u.range)}`).join("、") || "—"}${c.notRunEvents.length > 0 ? "，不跑：" + c.notRunEvents[0]!.reason : ""}`, + ), + ); + } + const curve = a.rounds.map((r) => r.adoptedScore); + text.push( + `采纳分数曲线 ${s.study}：${curve.map((x) => f(x)).join(" → ")}`, + `并排：${oldB.name}（无诊断、消融开）${oldB.scores.map((x) => f(x)).join(" → ")}；${oldD.name} ${oldD.scores.map((x) => f(x)).join(" → ")}；${refs.map((x) => `${x.name} ${x.scores.map((y) => f(y)).join(" → ")}`).join("；")}`, + "", + ); +} +writeFileSync(join(output, "stats.txt"), `${text.join("\n")}\n`); +console.log( + `统计：${output}；${summaries.length} 份研究的逐研究账目检查${summaries.every((s) => s.pass) ? "通过" : "未通过"}。研究质量判断仍待独立审阅。`, +); +if (!summaries.every((s) => s.pass)) process.exitCode = 1; diff --git a/eval/lib/diagnosis.ts b/eval/lib/diagnosis.ts new file mode 100644 index 0000000..0872ab5 --- /dev/null +++ b/eval/lib/diagnosis.ts @@ -0,0 +1,310 @@ +import type { LedgerRecord } from "../../src/ledger/index.ts"; +import { + Board, + diagnosisId, + type StudyRecord, +} from "../../src/materials/index.ts"; +import { + covers, + improvement, + placement, + runBalance, +} from "../../src/rules/index.ts"; +import { checkTaskFacts } from "../../src/tasks/index.ts"; +import { fullAccounts } from "./full.ts"; +import { frozenConfig } from "./forward.ts"; +import { ofKind } from "./ledger.ts"; + +// 诊断开、消融关的完整研究（任务书 18 第三块）的账目与逐轮事实：只从账本记录重算，不作质量判断。 +// 第 4 样"沿这个方向"这类判断由子 Agent 读材料标注，这里只给能由账本算出的列。 + +/** 消融关着时，每轮都应当有"按配置跳过"的七个步骤（src/config/steps.ts 的 off 清单）。 */ +export const ABLATION_STEPS = [ + "listCells", + "explanation", + "selectCells", + "ablationImplementation", + "ablationVerification", + "ablationRun", + "settle", +] as const; + +const BAND = { + belowQ1: "q9", +} as const; + +export function diagnosisAccounts(records: LedgerRecord[]) { + const base = fullAccounts(records); + const board = new Board(records); + const config = frozenConfig(records); + const facts = checkTaskFacts( + (ofKind(records, "config")[0]!.content as unknown as StudyRecord).task, + ); + const events = ofKind(records, "event"); + const opens = ofKind(records, "stageOpen"); + const closes = ofKind(records, "stageClose"); + const closeSeq = (stageId: string) => + closes.find((c) => c.content.stageId === stageId)?.seq ?? null; + const planned = ofKind(records, "runPlanned"); + const started = ofKind(records, "runStarted"); + const results = ofKind(records, "runResult"); + + // 开过的轮次：有阶段、决定或按配置跳过事件的都算。 + const roundIds = base.rounds.map((r) => r.round); + const rounds = roundIds.map((round) => { + const roundOf = base.rounds.find((r) => r.round === round)!; + const balanceBefore = runBalance( + config.runs, + board.runStatesBefore(round), + ).balance; + const prefix = `r${round}-`; + const diagStages = opens.filter( + (o) => + o.content.round === round && + o.content.stageId.startsWith(`${prefix}diagnosis`), + ); + const select = board.decision("selectDiagnosis", round) ?? null; + const dedup = board.decision("diagnosisDedup", round) ?? null; + const settlement = board.diagnosisSettlement(round) ?? null; + const settlementRecords = ofKind(records, "diagnosisSettlement").filter( + (r) => r.content.round === round, + ); + const diagRuns = planned + .filter( + (p) => p.content.purpose === "diagnosis" && p.content.round === round, + ) + .map((p) => ({ + run: p.content.run, + planSeq: p.seq, + startSeqs: started + .filter((s) => s.content.run === p.content.run) + .map((s) => s.seq), + result: results.find((r) => r.content.run === p.content.run) ?? null, + })); + const anyDiagnosisRecord = + diagStages.length > 0 || + select !== null || + dedup !== null || + diagRuns.length > 0 || + settlementRecords.length > 0; + const merged = dedup === null ? [] : board.diagnosisExperiments(round); + const dropped = settlement?.dropped ?? []; + const verifications = board + .published("diagnosisVerification") + .filter((s) => s.round === round); + const repairs = board + .published("diagnosisRepair") + .filter((s) => s.round === round); + const predictionClose = closeSeq(`${prefix}diagnosisPrediction`); + const firstStart = diagRuns.flatMap((r) => r.startSeqs).sort((a, b) => a - b)[0]; + const experiments = merged.map((e) => { + const row = settlement?.output.experiments.find( + (x) => x.experiment === e.id, + ); + const judged = settlement?.judged.experiments.find((x) => x.id === e.id); + const skip = select?.output.experiments.find( + (x) => x.experiment === e.id, + ); + return { + id: e.id, + name: e.name, + spot: e.spot, + authors: e.authors, + chosen: select?.output.chosen.includes(e.id) ?? false, + skipReason: skip?.reason ?? null, + skipBestNet: skip?.bestCombo?.net ?? null, + priorGain: select?.output.priorGains.find((g) => g.experiment === e.id) + ?.gain ?? null, + outcome: row?.outcome ?? null, + score: + judged !== undefined && + judged.verdict === "valid" && + judged.score.status === "measured" + ? judged.score.value + : null, + scoreStatus: judged?.score.status ?? null, + gain: row?.gain ?? null, + merged: row?.merged ?? null, + ending: judged?.ending ?? null, + wallSeconds: + judged?.wallSeconds.status === "measured" + ? judged.wallSeconds.value + : null, + worldBits: row?.worldBits ?? null, + surprise: + row?.surprise?.bits.status === "measured" + ? row.surprise.bits.value + : null, + dropped: dropped.find((d) => d.experiment === e.id)?.reason ?? null, + }; + }); + + // 第 5 样：每位作者每条解释对每个有效实验的三个数，实测落点；合并范围的落点。 + const predictionRows = + settlement === null + ? [] + : settlement.input.explanations.flatMap((x) => + settlement.input.experiments.flatMap((e) => { + if (e.outcome.kind !== "valid") return []; + const range = x.experiments.find( + (y) => y.experiment === e.experiment, + )!.range; + return [ + { + explanation: x.id, + author: x.author, + experiment: e.experiment, + range, + actual: e.outcome.score, + band: BAND[placement(range, e.outcome.score)], + covered: covers(range, e.outcome.score), + }, + ]; + }), + ); + const mergedRows = + settlement === null + ? [] + : settlement.input.experiments.flatMap((e) => { + if (e.outcome.kind !== "valid") return []; + const range = settlement.output.experiments.find( + (x) => x.experiment === e.experiment, + )!.merged; + return [ + { + experiment: e.experiment, + range, + actual: e.outcome.score, + band: BAND[placement(range, e.outcome.score)], + covered: covers(range, e.outcome.score), + }, + ]; + }); + const medians = ( + settlement?.output.experiments.map((x) => x.merged.q5) ?? [] + ).sort((a, b) => a - b); + const experimentGap = + medians.length < 2 + ? null + : { + maxMinusMin: medians.at(-1)! - medians[0]!, + minPairwise: Math.min( + ...medians.slice(1).map((m, i) => m - medians[i]!), + ), + }; + const batch = board.decision("batch", round) ?? null; + const candMedians = (batch?.output.candidates ?? []) + .flatMap((c) => (c.merged === null ? [] : [c.merged.q5])) + .sort((a, b) => a - b); + const candidateGap = + candMedians.length < 2 + ? null + : { + maxMinusMin: candMedians.at(-1)! - candMedians[0]!, + minPairwise: Math.min( + ...candMedians.slice(1).map((m, i) => m - candMedians[i]!), + ), + }; + + // 本轮用于第 4 样的正向侧：每个候选相对本轮基线的改进量。 + const baseline = board.baselineOf(round); + const baselineScore = board.baselineScore(baseline); + const forward = roundOf.candidates.map((c) => ({ + candidate: c.candidate, + score: c.score, + gain: + c.score === null + ? null + : improvement(baselineScore, c.score, facts.finalMetric.direction), + })); + + // 本轮诊断结算按轮次唯一；运行扣次：确认启动的诊断运行数。 + const ablationAbsent = + opens.every( + (o) => + o.content.round !== round || + !["explanation", "ablationImplementation", "ablationVerification"].includes( + o.content.stage, + ), + ) && + planned.every( + (p) => p.content.purpose !== "ablation" || p.content.round !== round, + ) && + board.decision("cells", round) === undefined && + board.decision("bet", round) === undefined && + board.settlement(round) === undefined; + const skippedSteps = ABLATION_STEPS.map((step) => + events.some( + (e) => + e.content.event === "skippedByConfig" && + e.content.objects[0] === `${prefix}${step}`, + ), + ); + return { + round, + baseline, + baselineScore, + balanceBefore, + expectedDiagnosis: balanceBefore >= 2, + anyDiagnosisRecord, + diagnosed: select?.output.diagnose ?? false, + chosen: select?.output.chosen ?? [], + chosenNet: select?.output.chosenNet ?? null, + combos: select?.output.combos ?? [], + g0: select?.output.g0 ?? null, + selectParams: select?.output.params ?? null, + experiments, + verificationCount: verifications.length, + repairCount: repairs.length, + droppedCount: dropped.length, + ranCount: settlement?.judged.experiments.length ?? 0, + runs: diagRuns, + charged: diagRuns.filter((r) => r.startSeqs.length > 0).length, + settlementCount: settlementRecords.length, + direction: settlement?.output.direction ?? null, + judgments: settlement?.output.judgments ?? null, + predictionRows, + mergedRows, + experimentGap, + candidateGap, + forward, + predictionCloseSeq: predictionClose, + firstDiagnosisStartSeq: firstStart ?? null, + predictionBeforeStart: + firstStart === undefined + ? null + : predictionClose !== null && predictionClose < firstStart, + ablationAbsent, + ablationSkipEventsAll: skippedSteps.every(Boolean), + diagnosisNotRunEvents: events.filter( + (e) => + e.content.event === "diagnosisNotRun" && + e.content.objects.includes(diagnosisId(round)), + ).length, + }; + }); + + const checks = { + ...base.checks, + diagnosisChargedEqualsStarts: rounds.every( + (r) => r.runs.length === 0 || r.charged === r.runs.filter((x) => x.startSeqs.length > 0).length, + ), + oneDiagnosisSettlementPerRound: rounds.every((r) => r.settlementCount <= 1), + noDiagnosisBelowTwo: rounds.every( + (r) => r.expectedDiagnosis || !r.anyDiagnosisRecord, + ), + diagnosisWhenBalanceAtLeastTwo: rounds.every( + (r) => !r.expectedDiagnosis || r.anyDiagnosisRecord, + ), + predictionBeforeStart: rounds.every((r) => r.predictionBeforeStart !== false), + ablationOffClean: rounds.every((r) => r.ablationAbsent), + ablationSkipEvents: rounds.every((r) => r.ablationSkipEventsAll), + diagnosisRunPurposeCount: + planned.filter((p) => p.content.purpose === "diagnosis").length === + rounds.reduce((n, r) => n + r.runs.length, 0), + }; + return { ...base, diagnosisRounds: rounds, diagnosisChecks: checks }; +} diff --git a/package.json b/package.json index a11e661..30123e8 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,10 @@ "eval:forward:stats": "node eval/forward/stats.ts", "eval:full:run": "set -a && . /home/iomgaa/Projects/RivalBits/.env && set +a && node eval/full/run.ts", "eval:full:stats": "node eval/full/stats.ts", - "eval:full:export": "node eval/full/export.ts" + "eval:full:export": "node eval/full/export.ts", + "eval:diagnosis:run": "set -a && . /home/iomgaa/Projects/RivalBits/.env && set +a && node eval/diagnosis/run.ts", + "eval:diagnosis:stats": "node eval/diagnosis/stats.ts", + "eval:diagnosis:export": "node eval/diagnosis/export.ts" }, "dependencies": { "@earendil-works/pi-ai": "0.85.1",