diff --git a/eval/full/export.ts b/eval/full/export.ts new file mode 100644 index 0000000..f700b97 --- /dev/null +++ b/eval/full/export.ts @@ -0,0 +1,273 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { + Board, + cellId, + explanationId, + protocolMetrics, + splitCell, + type TaskMaterial, +} from "../../src/materials/index.ts"; +import { ablationOutcome } from "../../src/submissions/index.ts"; +import { checkoutVersion, compareVersions } from "../../src/workspace/index.ts"; +import { jsonFile } from "../lib/forward.ts"; +import { finalSubmissions, ofKind, readLedger } from "../lib/ledger.ts"; + +// 任务书 17 第 3.2 节研究质量评估的审阅材料:解释、选格决定、消融格(协议、位置表、代码与 diff、实现与核验往来)、 +// 结算、收尾、下一轮提案。只导出事实和原文,不替审阅者下结论。 +// node eval/full/export.ts <研究输出目录> <全新绝对审阅目录> +const [dir, output, ...extra] = process.argv.slice(2); +if (!dir || !output || extra.length || output !== resolve(output)) { + throw new Error( + "用法:node eval/full/export.ts <研究输出目录> <全新绝对审阅目录>", + ); +} +mkdirSync(output); // 不覆盖以前的审阅材料。 +const study = join(dir, "study"); +const records = readLedger(study); +const board = new Board(records); +const repo = join(study, "repo.git"); +const finals = finalSubmissions(records); +const roundOf = (stageId: string) => board.stage(stageId).round; +const roundIds = [ + ...new Set(ofKind(records, "decision").map((d) => d.content.round)), +] + .filter((r) => r >= 1) + .sort((a, b) => a - b); + +const checkout = (version: string, path: string) => { + mkdirSync(path); + checkoutVersion(repo, version, path); +}; +const json = (v: unknown) => `${JSON.stringify(v, null, 2)}\n`; + +const index = []; +for (const round of roundIds) { + const roundDir = join(output, `round-${round}`); + mkdirSync(roundDir); + + // 本轮提案(下一轮提案在 round- 里,审阅"收尾→接力"时把两份对着读)。 + const proposals = finals.filter( + (s) => + s.content.kind === "proposal" && + roundOf(s.content.stageId) === round, + ); + jsonFile(join(roundDir, "proposals.json"), proposals); + writeFileSync( + join(roundDir, "proposals.txt"), + proposals + .map( + (s) => + `${"=".repeat(20)} ${s.content.item}(${s.content.author},记录 ${s.seq})\n${json(s.content.content)}`, + ) + .join("\n"), + ); + + // 解释(任务书 17 第 3.2 节第 1 样):作者、正文、每格三个数。 + const explanations = finals.filter( + (s) => + s.content.kind === "explanation" && + roundOf(s.content.stageId) === round, + ); + jsonFile(join(roundDir, "explanations.json"), explanations); + writeFileSync( + join(roundDir, "explanations.txt"), + explanations + .flatMap((s) => + ( + s.content.content as { + explanations: Array<{ + body: string; + cells?: Array<{ factor: string; range: unknown }>; + }>; + } + ).explanations.map( + (e, i) => + `${"=".repeat(20)} ${explanationId(round, s.content.author, i + 1)}(${s.content.author} 的第 ${i + 1} 条,记录 ${s.seq})\n${e.body}\n三个数:${(e.cells ?? []).map((c) => `${c.factor} ${JSON.stringify(c.range)}`).join(";")}`, + ), + ) + .join("\n\n"), + ); + + // 选格决定全文(第 2 样):因子、合并范围、各组合的信息价值/成本/净值、选中与没选的原因都在这条决定里。 + const cellsDecision = + ofKind(records, "decision").find( + (d) => d.content.decision === "cells" && d.content.round === round, + ) ?? null; + const chosenCells = board.decision("cells", round)?.output.chosen ?? null; + jsonFile(join(roundDir, "cells-decision.json"), cellsDecision); + + // 结算全文(第 4 样):结算记录和同一事务里押的因子决定。 + const settlement = + ofKind(records, "settlement").find((s) => s.content.round === round) ?? + null; + const bet = + ofKind(records, "decision").find( + (d) => d.content.decision === "bet" && d.content.round === round, + ) ?? null; + jsonFile(join(roundDir, "settlement.json"), { settlement, bet }); + + // 收尾全文(第 5 样)。 + const wrapUps = finals.filter( + (s) => + s.content.kind === "wrapUp" && roundOf(s.content.stageId) === round, + ); + jsonFile(join(roundDir, "wrapups.json"), wrapUps); + writeFileSync( + join(roundDir, "wrapups.txt"), + wrapUps + .map( + (s) => + `${"=".repeat(20)} ${s.content.author}(记录 ${s.seq})\n${(s.content.content as { body: string }).body}`, + ) + .join("\n\n"), + ); + + // 消融格(第 3 样):冻结协议、位置表、完整代码与格代码及 diff、alsoOff、concerns、每次核验逐项结论与退回、重做记录。 + const cells = []; + if (chosenCells !== null) { + const adopt = board.decision("adopt", round)!; + const candidate = adopt.output.adopted!; + const full = board.code(candidate); + const baseline = board.code(board.baselineOf(round)); + for (const key of chosenCells) { + const cell = cellId(candidate, key); + const cellDir = join(roundDir, "cells", cell); + mkdirSync(cellDir, { recursive: true }); + jsonFile(join(cellDir, "protocol.json"), { + record: ofKind(records, "protocol") + .filter((p) => p.content.candidate === candidate) + .at(-1), + resolvedMetrics: protocolMetrics( + board, + board.protocol(candidate), + ), + factorLocations: full.locations, + baselineVersion: baseline.version, + fullVersion: full.version, + }); + checkout(full.version, join(cellDir, "full")); + checkout(baseline.version, join(cellDir, "baseline")); + // 每次实现和代码版本对号(同一事务里先写提交、紧接着写代码;交"跑不起来"时没有代码记录)。 + const impls = finals.filter( + (s) => + s.content.kind === "ablationImplementation" && + s.content.target === cell, + ); + const codes = ofKind(records, "code").filter( + (c) => c.content.object === cell, + ); + const attempts = impls.map((s) => ({ + submission: s, + code: codes.find((c) => c.seq === s.seq + 1) ?? null, + })); + jsonFile(join(cellDir, "implementations.json"), attempts); + const verifications = finals + .filter( + (s) => + s.content.kind === "ablationVerification" && + s.content.target === cell, + ) + .map((s) => { + const impl = + impls.filter((x) => x.seq < s.seq).at(-1) ?? null; + return { + submission: s, + judgedImplSeq: impl?.seq ?? null, + outcome: + impl === null + ? null + : ablationOutcome( + s.content.content as Parameters< + typeof ablationOutcome + >[0], + key, + ( + impl.content.content as { + alsoOff: string[]; + } + ).alsoOff, + ), + }; + }); + jsonFile(join(cellDir, "verifications.json"), verifications); + jsonFile( + join(cellDir, "rejections.json"), + ofKind(records, "rejection").filter( + (r) => r.content.target === cell, + ), + ); + const lastCode = codes.at(-1) ?? null; + if (lastCode !== null) { + checkout(lastCode.content.version, join(cellDir, "cell")); + jsonFile( + join(cellDir, "diff.json"), + compareVersions( + repo, + full.version, + lastCode.content.version, + ), + ); + } + for (const attempt of attempts) { + if (attempt.code === null) continue; + const path = join(cellDir, `code-${attempt.code.seq}`); + checkout(attempt.code.content.version, path); + jsonFile( + join(cellDir, `diff-${attempt.code.seq}.json`), + compareVersions( + repo, + full.version, + attempt.code.content.version, + ), + ); + } + jsonFile( + join(cellDir, "calls.json"), + ofKind(records, "callStarted") + .filter((c) => { + const material = c.content.materials as TaskMaterial; + return ( + material.object === cell && + [ + "ablationImplementation", + "ablationVerification", + ].includes(material.kind) + ); + }) + .map((c) => ({ + record: c, + sessions: join( + study, + "calls", + c.content.call, + "sessions", + ), + })), + ); + cells.push({ + cell, + factor: splitCell(cell).key, + dir: cellDir, + ran: lastCode !== null, + }); + } + } + + writeFileSync( + join(roundDir, "review.txt"), + `任务书17第3.2节,只读本目录,中文直接回复,不写文件。\n` + + `第1样(解释):读 explanations.txt、cells-decision.json、settlement.json。看不到消融结果地回答:机制是否具体、写了什么情况下不成立没有、三个数和正文是否一致、范围宽窄有没有依据、同轮几人的解释有没有真正不同、"没有具体机制判断"用得诚不诚实;再对 settlement.json 看每条解释在每格被确认还是否定。\n` + + `第2样(选格):读 cells-decision.json,列出各格合并范围、各组合信息价值/成本/净值、选中和没选的原因。\n` + + `第3样(消融实现与核验,每格一个全新上下文):读 cells/<格>/ 下的 protocol.json、full/、cell/、diff.json、implementations.json、verifications.json、rejections.json。判断:是不是只关了指定的因子;保障原样保留没有、关掉的因子的指标写 null 没有;答案路径标签和关掉以后的分支对不对得上;核验员每次逐项结论判得对不对;重做的只改了被指出的地方没有。\n` + + `第5样(收尾与接力):读 wrapups.txt、settlement.json,再对着 round-${round + 1}/proposals.txt(没有就是最后一轮)看下一轮提案用上了哪些发现。\n`, + ); + index.push({ + round, + dir: roundDir, + hasAblation: chosenCells !== null, + cells, + }); +} +jsonFile(join(output, "index.json"), { study: dir, rounds: index }); +console.log(`审阅材料:${output}`); diff --git a/eval/full/run.ts b/eval/full/run.ts new file mode 100644 index 0000000..5e4b167 --- /dev/null +++ b/eval/full/run.ts @@ -0,0 +1,325 @@ +import { execFileSync } from "node:child_process"; +import { + appendFileSync, + existsSync, + mkdirSync, + readFileSync, + writeFileSync, +} from "node:fs"; +import { loadavg, freemem, totalmem } from "node:os"; +import { join, resolve } from "node:path"; +import { fullAccounts, FULL_STOP } from "../lib/full.ts"; +import { jsonFile } from "../lib/forward.ts"; +import { + sameContainerThroughResult, + type ContainerSnapshot, +} from "../lib/forward-evidence.ts"; +import { ofKind, readLedger } from "../lib/ledger.ts"; +import { launch, sleep, waitForLedger, writeConfig } from "../lib/study.ts"; +import { watchTimeline } from "../lib/timeline.ts"; + +// 任务书 17 第 3.1 节:从头真实跑一份完整研究,到余额用完停在"导出还没写"。三份同时跑,由父会话各起一个进程; +// 容器名带研究编号,这里只认本研究的,不和别的研究互相核对。 +// node eval/full/run.ts <研究编号> <全新绝对输出目录> [--kill-after-first-ablation-start] +// 服务一时调不通等中断后的续跑:npm run eval:study -- --resume --dir <输出目录>/study(照任务书 16 的做法)。 + +const [name, output, ...extra] = process.argv.slice(2); +const kill = + extra.length === 1 && extra[0] === "--kill-after-first-ablation-start"; +if ( + !name || + !output || + extra.length > 1 || + (extra.length === 1 && !kill) || + output !== resolve(output) +) { + throw new Error( + "用法:node eval/full/run.ts <研究编号> <全新绝对输出目录> [--kill-after-first-ablation-start]", + ); +} +const out = output; +if (existsSync(out)) throw new Error(`输出目录已存在,不覆盖现场:${out}`); +mkdirSync(out, { recursive: true }); +const studyDir = join(out, "study"); +const config = writeConfig( + "eval/round-zero/research.json", + join(out, "research.json"), + name, + null, +); + +const daemonTime = () => + execFileSync("docker", ["info", "--format", "{{.SystemTime}}"], { + encoding: "utf8", + }).trim(); +const inspect = ( + container: string, +): ContainerSnapshot & { + Name: string; + Created: string; + State: { + Running: boolean; + StartedAt: string; + FinishedAt: string; + Status: string; + }; +} => + JSON.parse( + execFileSync("docker", ["inspect", container], { encoding: "utf8" }), + )[0]; + +/** 事件点的重快照(进程列表、容器列表),照任务书 16 的 machine.jsonl 做法。 */ +function observe(label: string) { + appendFileSync( + join(out, "machine.jsonl"), + `${JSON.stringify({ + at: new Date().toISOString(), + label, + processes: execFileSync("ps", ["-eo", "pid,ppid,lstart,args"], { + encoding: "utf8", + }), + containers: execFileSync("docker", ["ps", "--no-trunc"], { + encoding: "utf8", + }), + })}\n`, + ); +} + +/** 运行期间的轻量负载采样,几秒一条(任务书 17 第 3.1 节:和每次运行的 CPU/墙钟比对照,只列事实)。 */ +function sample() { + appendFileSync( + join(out, "machine.jsonl"), + `${JSON.stringify({ at: new Date().toISOString(), label: "sample", load: loadavg(), memory: { total: totalmem(), free: freemem() } })}\n`, + ); +} + +function start(path: string | null) { + observe(path === null ? "resume" : "start"); + const study = launch(path, studyDir, {}); + appendFileSync( + join(out, "launches.jsonl"), + `${JSON.stringify({ event: "start", pid: study.child.pid, at: new Date().toISOString(), config: path })}\n`, + ); + const interrupt = () => study.child.kill("SIGINT"); + process.on("SIGINT", interrupt); + study.done.then((end) => { + process.off("SIGINT", interrupt); + appendFileSync( + join(out, "launches.jsonl"), + `${JSON.stringify({ event: "end", pid: study.child.pid, at: new Date().toISOString(), ...end })}\n`, + ); + }); + return study; +} + +const sampler = setInterval(sample, 5000); +sampler.unref(); +let active = start(config); +let killEvidence: { + run: string; + cell: string; + round: number; + killSeq: number; +} | null = null; +try { + await waitForLedger(studyDir, active.child); + let watching = watchTimeline( + studyDir, + active.child, + join(out, "timeline.jsonl"), + ); + if (kill) { + // 盯住账本:第一个用途为 ablation 的运行确认启动之后、结果落账之前杀控制进程(照 eval/forward/run.ts 的 kill 做法)。 + let hit = false; + while (active.child.exitCode === null && active.child.signalCode === null) { + const records = readLedger(studyDir); + const plan = ofKind(records, "runPlanned").find( + (r) => r.content.purpose === "ablation", + ); + if (plan !== undefined) { + const run = plan.content.run; + if (ofKind(records, "runResult").some((r) => r.content.run === run)) { + throw new Error( + "第一个消融运行已经有结果了,未打中;保留现场,不改杀别的格", + ); + } + const started = ofKind(records, "runStarted").find( + (r) => r.content.run === run, + ); + if (started !== undefined) { + // 容器名是正式入口写在结构文档里的约定(inforsi-<研究编号>-<运行编号>),不 import flow 内部文件。 + const container = `inforsi-${name}-${run}`; + // 从杀前划定历史事件区间,覆盖恢复启动到目标结果落账;不把 spawn 当成恢复已执行。 + const lifecycleSince = daemonTime(); + const before = inspect(container); + jsonFile(join(out, "container-before.json"), before); + if (!before.State.Running) + throw new Error("第一个消融容器已不在运行,未打中"); + jsonFile(join(out, "kill.json"), { + run, + cell: plan.content.object, + round: plan.content.round, + planSeq: plan.seq, + startSeq: started.seq, + beforeKillSeq: records.at(-1)!.seq, + pid: active.child.pid, + at: new Date().toISOString(), + container, + }); + if (!active.child.kill("SIGKILL")) throw new Error("未发出 SIGKILL"); + const killed = await active.done; + await watching; + if (killed.signal !== "SIGKILL") + throw new Error("控制进程不是以 SIGKILL 结束"); + const atKill = readLedger(studyDir); + jsonFile(join(out, "ledger-at-kill.json"), atKill); + if (ofKind(atKill, "runResult").some((r) => r.content.run === run)) { + throw new Error("杀后核对发现目标运行已经写结果,未打中;保留现场"); + } + jsonFile(join(out, "container-after-kill.json"), inspect(container)); + observe("after-kill"); + jsonFile( + join(out, "container-before-resume.json"), + inspect(container), + ); + killEvidence = { + run, + cell: plan.content.object!, + round: plan.content.round, + killSeq: atKill.at(-1)!.seq, + }; + active = start(null); + watching = watchTimeline( + studyDir, + active.child, + join(out, "timeline.jsonl"), + ); + let targetResult; + while (true) { + targetResult = ofKind(readLedger(studyDir), "runResult").find( + (r) => r.content.run === run, + ); + if ( + targetResult !== undefined || + active.child.exitCode !== null || + active.child.signalCode !== null + ) + break; + await sleep(100); + } + const lifecycleUntil = daemonTime(); + // Docker CLI 按时间回读,不依赖事件订阅的启动时序;原文全存,stats 核对缓存上限和目标 die。 + writeFileSync( + join(out, "container-events.jsonl"), + execFileSync( + "docker", + [ + "events", + "--since", + lifecycleSince, + "--until", + lifecycleUntil, + "--format", + "{{json .}}", + ], + { encoding: "utf8" }, + ), + ); + jsonFile(join(out, "container-events-window.json"), { + since: lifecycleSince, + until: lifecycleUntil, + run, + resultSeq: targetResult?.seq ?? null, + }); + if (targetResult === undefined) + throw new Error( + "恢复进程没有交出目标结果,生命周期证据不完整;保留现场", + ); + hit = true; + break; + } + } + await sleep(100); + } + if (!hit) + throw new Error( + `没有打中第一个消融运行:${JSON.stringify(await active.done)}`, + ); + } + await watching; + const end = await active.done; + jsonFile(join(out, "outcome.json"), end); + observe("end"); + const records = readLedger(studyDir); + const accounts = fullAccounts(records); + jsonFile(join(out, "accounts.json"), accounts); + + // 杀进程的三条检查(任务书 17 第 3.1 节):①恢复按容器名接着等同一个容器、没有重跑;②只扣一次;③结果照常写入、结算和后续步骤走完。 + if (killEvidence !== null) { + const evidence = JSON.parse( + readFileSync(join(out, "kill.json"), "utf8"), + ) as { run: string; cell: string; round: number; container: string }; + const snapshots = (["before", "after-kill", "before-resume"] as const).map( + (part) => + JSON.parse( + readFileSync(join(out, `container-${part}.json`), "utf8"), + ) as ContainerSnapshot, + ); + const lifecycle = readFileSync(join(out, "container-events.jsonl"), "utf8") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); + const lifecycleChecks = sameContainerThroughResult( + snapshots, + lifecycle, + evidence.container, + ); + const target = accounts.runs.find( + (r) => r.plan.content.run === evidence.run, + )!; + const cellPlans = accounts.runs.filter( + (r) => + r.plan.content.purpose === "ablation" && + r.plan.content.object === evidence.cell, + ); + const report = { + evidence, + killSeq: killEvidence.killSeq, + snapshotMeaning: "杀前、杀后、恢复启动前;没有把快照当成恢复完成的证据", + lifecycleChecks, + checks: { + sameContainerNoRerun: + Object.values(lifecycleChecks).every(Boolean) && + cellPlans.length === 1 && + target.starts.length === 1, + chargedOnce: target.starts.length === 1, + resultSettledAndFinished: + target.results.length === 1 && + target.results[0]!.seq > killEvidence.killSeq && + ofKind(records, "settlement").some( + (s) => s.content.round === evidence.round, + ) && + accounts.halt !== null && + accounts.halt.reason === FULL_STOP, + }, + }; + jsonFile(join(out, "kill-report.json"), report); + } + + if ( + end.code !== 0 || + !accounts.checks.haltedAtExport || + !accounts.checks.chargedEqualsStarts + ) { + throw new Error( + `没有走到"导出还没写"或账目不平,保留现场;查明原因后用 npm run eval:study -- --resume --dir ${studyDir} 续跑:${JSON.stringify(end)}`, + ); + } + console.log(`研究走完:${studyDir}。仍需 export、stats 和独立质量审阅。`); +} finally { + clearInterval(sampler); + if (active.child.exitCode === null && active.child.signalCode === null) { + active.child.kill("SIGINT"); + await active.done; + } +} diff --git a/eval/full/stats.ts b/eval/full/stats.ts new file mode 100644 index 0000000..1c79f4c --- /dev/null +++ b/eval/full/stats.ts @@ -0,0 +1,180 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import type { TaskMaterial } from "../../src/materials/index.ts"; +import { cpuRatio, CPU_RATIO_FLAG, fullAccounts } from "../lib/full.ts"; +import { frozenConfig, jsonFile } from "../lib/forward.ts"; +import { ofKind, readLedger } from "../lib/ledger.ts"; +import { lightCheckUsage } from "../lib/light-checks.ts"; +import { readTimeline } from "../lib/timeline.ts"; + +// 任务书 17 第 3.2 节第 6、7 样的可计算部分:逐轮走向 + 程序账目 + CPU/墙钟比。只算事实、不作质量判断 +// (解释质量、核验判得对不对、收尾与接力由父会话派全新上下文的子 Agent 读 eval/full/export.ts 的产物)。 +// 这次三份研究同时跑,没有"启动区间互不重叠"这条(任务书 17 第 3.1 节);也不设总体 acceptance。 +// node eval/full/stats.ts <全新绝对统计输出目录> <研究输出目录>... +const [output, ...dirs] = process.argv.slice(2); +if (!output || dirs.length === 0 || output !== resolve(output)) { + throw new Error( + "用法:node eval/full/stats.ts <全新绝对统计输出目录> <研究输出目录>...", + ); +} +mkdirSync(output); + +const fmt = (v: number | null) => (v === null ? "—" : v.toFixed(6)); + +const summaries = dirs.map((dir) => { + const studyDir = join(dir, "study"); + const records = readLedger(studyDir); + const config = frozenConfig(records); + const accounts = fullAccounts(records); + + // 轻量检查:每次实现/修复/消融实现调用用了几次、有没有超过配置上限(任务书 17 第 3.2 节第 7 样)。 + const lightChecks = ofKind(records, "callStarted") + .filter((c) => + ["implementation", "repair", "ablationImplementation"].includes( + (c.content.materials as TaskMaterial).kind, + ), + ) + .map((c) => { + const sessions = join(studyDir, "calls", c.content.call, "sessions"); + return { + call: c.content.call, + seq: c.seq, + ...(existsSync(sessions) + ? lightCheckUsage(sessions, config.lightChecks.maxCount) + : { + maxCount: config.lightChecks.maxCount, + pass: false, + note: "会话目录不存在", + }), + }; + }); + + // 每次正式运行的 CPU 时间/墙钟时间之比;比值明显低于 1 的列出,并和 machine.jsonl 同时段负载放在一起(只列事实)。 + const timeline = existsSync(join(dir, "timeline.jsonl")) + ? readTimeline(join(dir, "timeline.jsonl")) + : []; + const atOf = (seq: number) => timeline.find((t) => t.seq === seq)?.at ?? null; + const machine = existsSync(join(dir, "machine.jsonl")) + ? readFileSync(join(dir, "machine.jsonl"), "utf8") + .split("\n") + .filter(Boolean) + .map( + (line) => + JSON.parse(line) as { at: string; label: string; load?: number[] }, + ) + : []; + const ratios = accounts.runs.map((r) => { + const result = r.results[0] ?? null; + const ratio = result === null ? null : cpuRatio(result.content); + const start = r.starts[0] === undefined ? null : atOf(r.starts[0].seq); + const end = result === null ? null : atOf(result.seq); + const samples = + start === null || end === null + ? [] + : machine + .filter( + (m) => + m.label === "sample" && + Date.parse(m.at) >= start && + Date.parse(m.at) <= end, + ) + .map((m) => ({ at: m.at, load: m.load })); + return { + run: r.plan.content.run, + purpose: r.plan.content.purpose, + round: r.plan.content.round, + object: r.plan.content.object, + ratio, + flagged: ratio !== null && ratio < CPU_RATIO_FLAG, + start: start === null ? null : new Date(start).toISOString(), + end: end === null ? null : new Date(end).toISOString(), + loadSamples: samples, + note: `比值低于 ${CPU_RATIO_FLAG} 列出并附同时段负载;只列事实,不作判断`, + }; + }); + + // 有 kill-report.json 的研究,把三条检查的结果抄进来(任务书 17 第 3.1 节)。 + const kill = existsSync(join(dir, "kill-report.json")) + ? (JSON.parse(readFileSync(join(dir, "kill-report.json"), "utf8")) as { + evidence: unknown; + checks: Record; + }) + : null; + + const pass = + Object.values(accounts.checks).every(Boolean) && + lightChecks.every((c) => c.pass) && + (kill === null || Object.values(kill.checks).every(Boolean)); + return { + dir, + study: config.studyId, + accounts, + lightChecks, + ratios, + kill, + pass, + }; +}); + +jsonFile(join(output, "stats.json"), summaries); + +const text = summaries.map((s) => { + const a = s.accounts; + const lines = [ + `研究 ${s.study}(${s.dir})`, + `确认启动:基线 ${a.confirmedStarts.baseline}、正向 ${a.confirmedStarts.forward}、消融 ${a.confirmedStarts.ablation},共 ${a.confirmedStarts.total};扣次 ${a.balance.charged},余额 ${a.balance.balance},占用 ${a.balance.reserved};已知最好 ${fmt(a.knownBest)}`, + `停下:${a.halt === null ? "(没有停下事件)" : a.halt.reason}`, + `账目核对:${Object.entries(a.checks) + .map(([k, v]) => `${k}=${v ? "过" : "不过"}`) + .join("、")}`, + `轻量检查:${s.lightChecks.every((c) => c.pass) ? "全部在上限内" : "有超限或会话缺失,见 stats.json"}`, + `CPU/墙钟比低于 ${CPU_RATIO_FLAG} 的运行:${ + s.ratios + .filter((r) => r.flagged) + .map((r) => `${r.run}=${r.ratio!.toFixed(3)}`) + .join("、") || "没有" + }`, + s.kill === null + ? "杀进程:这份没做" + : `杀进程:${Object.entries(s.kill.checks) + .map(([k, v]) => `${k}=${v ? "过" : "不过"}`) + .join("、")}`, + "", + ]; + for (const r of a.rounds) { + lines.push( + `第 ${r.round} 轮(基线 ${r.baseline})`, + ...r.candidates.map( + (c) => + ` 候选 ${c.candidate}:实测 ${fmt(c.score)},合并 ${c.merged === null ? "—" : `${fmt(c.merged.q1)}/${fmt(c.merged.q5)}/${fmt(c.merged.q9)}`},落点 ${c.band ?? "—"},作者自评 ${c.authorEstimates.map((u) => `${u.author} ${fmt(u.range.q1)}/${fmt(u.range.q5)}/${fmt(u.range.q9)}`).join("、") || "—"}${c.notRunEvents.length > 0 ? ",不跑:" + c.notRunEvents[0]!.reason : ""}`, + ), + ` 采纳:${r.adopted ?? "没有"}${r.adoptedScore === null ? "" : `,分数 ${fmt(r.adoptedScore)},离已知最好 ${fmt(r.knownBestGap)}`}`, + ); + if (r.hasAblation) { + lines.push( + ` 消融:选格 ${r.cellsDecision!.output.chosen.join("、") || "(一格都没选)"};配合关系 ${r.label === null ? "—" : r.label.label};押的因子 ${r.bet === null ? "—" : `${r.bet.decision === "bet" ? r.bet.factor : `不押(${r.bet.decision})`},预测收益 ${fmt(r.bet.predictedGain)}`}`, + ...r.cells.map( + (c) => + ` ${c.cell}:核验 ${c.verificationCount} 次、重做 ${c.redoCount} 次${c.cannotRun === null ? "" : ",跑不起来:" + c.cannotRun}${c.notRunEvents.length > 0 ? ",不跑:" + c.notRunEvents[0]!.reason : ""},跑没跑 ${c.ran ? "跑了" : "没跑"}(扣次 ${c.charged}),实测 ${fmt(c.measured)},撑几成 ${c.support === null ? "—" : c.support.toFixed(3)},合并 ${c.merged === null ? "—" : `${fmt(c.merged.q1)}/${fmt(c.merged.q5)}/${fmt(c.merged.q9)}`},结局 ${c.outcome ?? "—"}`, + ), + ...r + .cellsDecision!.output.cells.filter((c) => !c.tested) + .map( + (c) => + ` ${c.factor} 没测:${c.reason}(含它的最好组合净值 ${c.bestCombo === null ? "没有可选组合" : c.bestCombo.net.toExponential(3)})`, + ), + ); + } else lines.push(" 消融:这一轮没有"); + lines.push( + ` 解释关闭早于首格启动:${r.explanationBeforeCells === null ? "无消融" : r.explanationBeforeCells ? "是" : "否"};解封在收尾关闭之后且覆盖选中候选:${r.unseal.pass ? "是" : "否"}`, + ); + lines.push(""); + } + return lines.join("\n"); +}); +writeFileSync(join(output, "stats.txt"), `${text.join("\n")}\n`); + +console.log( + `统计:${output};所给 ${summaries.length} 份研究的逐研究账目检查${summaries.every((s) => s.pass) ? "通过" : "未通过"}。不设总体 acceptance(三份同时跑,无互不重叠条款);研究质量判断仍待独立审阅。`, +); +if (!summaries.every((s) => s.pass)) process.exitCode = 1; diff --git a/eval/lib/full.ts b/eval/lib/full.ts new file mode 100644 index 0000000..8bcdeb5 --- /dev/null +++ b/eval/lib/full.ts @@ -0,0 +1,376 @@ +import type { LedgerRecord } from "../../src/ledger/index.ts"; +import { + Board, + CELL_NOT_RUN, + cellId, + FORWARD_NOT_RUN, + ORIGINAL_BASELINE, + validScore, + type StudyRecord, +} from "../../src/materials/index.ts"; +import { improvement, runBalance } from "../../src/rules/index.ts"; +import { ablationOutcome } from "../../src/submissions/index.ts"; +import { checkTaskFacts } from "../../src/tasks/index.ts"; +import { frozenConfig, scoreBand } from "./forward.ts"; +import { ofKind } from "./ledger.ts"; + +// 完整研究(任务书 17 第三块)的账目核对:只从账本记录重算事实,不作质量判断。 +// 质量判断(解释好不好、核验判得对不对、收尾写得怎么样)由父会话派全新上下文的子 Agent 做。 + +/** 研究走完一轮接一轮、余额用完后的停点(任务书 17 第 2.1 节)。 */ +export const FULL_STOP = "导出还没写"; + +/** CPU 时间 / 墙钟时间低于这个比值就列出来和机器负载放在一起(任务书 17 第 3.1 节"明显低于 1";只列事实)。 */ +export const CPU_RATIO_FLAG = 0.9; + +/** 一份完整研究的账目:每轮的走向 + 全研究的程序账目核对(任务书 17 第 3.2 节第 6、7 样的可计算部分)。 */ +export function fullAccounts(records: LedgerRecord[]) { + const board = new Board(records); + const config = frozenConfig(records); + // SAFETY: 账本 config 记录的内容形状由 materials 的 StudyRecord 定义(docs/结构.md 第八部分),ledger 里只存 Json。 + const studyRecord = ofKind(records, "config")[0]! + .content as unknown as StudyRecord; + const facts = checkTaskFacts(studyRecord.task); + const events = ofKind(records, "event").map((r) => ({ + ...r.content, + seq: r.seq, + })); + const started = ofKind(records, "runStarted"); + const runs = ofKind(records, "runPlanned").map((p) => ({ + plan: p, + starts: started.filter((s) => s.content.run === p.content.run), + results: ofKind(records, "runResult").filter( + (r) => r.content.run === p.content.run, + ), + })); + const balance = runBalance(config.runs, board.runStates()); + + // 阶段开关成对:每个开恰有一个比它晚的关;没有没开的关。 + const opens = ofKind(records, "stageOpen"); + const closes = ofKind(records, "stageClose"); + const stages = opens.map((r) => { + const closing = closes.filter( + (c) => c.content.stageId === r.content.stageId, + ); + return { + ...r.content, + openSeq: r.seq, + closeSeq: closing[0]?.seq ?? null, + pass: closing.length === 1 && closing[0]!.seq > r.seq, + }; + }); + const orphanCloses = closes.filter( + (c) => !opens.some((o) => o.content.stageId === c.content.stageId), + ); + const closeSeqOf = (stageId: string) => + stages.find((s) => s.stageId === stageId)?.closeSeq ?? null; + + const roundIds = [ + ...new Set([ + ...opens.map((o) => o.content.round), + ...ofKind(records, "decision").map((d) => d.content.round), + ...ofKind(records, "settlement").map((s) => s.content.round), + ]), + ] + .filter((r) => r >= 1) + .sort((a, b) => a - b); + + const rounds = roundIds.map((round) => { + const batch = board.decision("batch", round); + const chosen = batch?.output.chosen ?? []; + // 每个入选候选一行(任务书 17 第 3.2 节第 6 样):路线代号不自动猜,留给人工填。 + const candidates = chosen.map((candidate) => { + const run = board.runOf(candidate); + const score = validScore(run?.result?.judged ?? null); + const merged = + batch!.output.candidates.find((c) => c.id === candidate)?.merged ?? + null; + return { + candidate, + score, + merged, + authorEstimates: board + .unsealed() + .filter((u) => u.candidate === candidate), + band: merged === null ? null : scoreBand(score, merged), + started: run !== undefined, + notRunEvents: events.filter( + (e) => e.event === FORWARD_NOT_RUN && e.objects.includes(candidate), + ), + route: "路线代号由父会话审阅时人工填,不自动按关键词贴", + }; + }); + const adopt = board.decision("adopt", round) ?? null; + const adopted = adopt?.output.adopted ?? null; + const adoptedScore = + adopted === null + ? null + : validScore(board.runOf(adopted)?.result?.judged ?? null); + const bestSoFar = + board.decision("batch", round) === undefined ? null : null; + void bestSoFar; + const knownBestGap = + adoptedScore === null || facts.knownBest === null + ? null + : improvement( + adoptedScore, + facts.knownBest, + facts.finalMetric.direction, + ); + + // 消融(任务书 17 第 3.2 节第 4 样的可计算部分和第 7 样的逐格账目)。 + const cellsDecision = board.decision("cells", round) ?? null; + const settlement = board.settlement(round) ?? null; + const bet = board.decision("bet", round) ?? null; + const cells = (cellsDecision?.output.chosen ?? []).map((key) => { + const cell = cellId(adopted!, key); + // published 只收已关阶段的最后一版;研究走完了,这些阶段都已关闭。 + const impls = board + .published("ablationImplementation") + .filter((s) => s.target === cell); + const lastImpl = impls.at(-1) ?? null; + const verifications = board + .published("ablationVerification") + .filter((s) => s.target === cell); + const lastVerification = verifications.at(-1) ?? null; + const lastOutcome = + lastVerification === null || lastImpl === null + ? null + : ablationOutcome( + lastVerification.content, + key, + lastImpl.content.alsoOff, + ); + const cellRuns = runs.filter( + (r) => + r.plan.content.purpose === "ablation" && + r.plan.content.round === round && + r.plan.content.object === cell, + ); + const lastRun = cellRuns.at(-1) ?? null; + const settled = + settlement?.output.cells.find((c) => c.factor === key) ?? null; + const settleInput = + settlement?.input.cells.find((c) => c.factor === key)?.outcome ?? null; + const notRun = events.filter( + (e) => e.event === CELL_NOT_RUN && e.objects.includes(cell), + ); + const cannotRun = lastImpl === null ? null : lastImpl.content.cannotRun; + return { + cell, + factor: key, + implementationCount: impls.length, + redoCount: Math.max(0, impls.length - 1), + verificationCount: verifications.length, + cannotRun, + notRunEvents: notRun, + lastVerificationPassed: + lastOutcome === null ? null : lastOutcome.passed, + charged: cellRuns.filter((r) => r.starts.length > 0).length, + ran: + lastRun !== null && + lastRun.results.length === 1 && + lastRun.results[0]!.content.process.ending !== "notStarted", + measured: + settleInput !== null && + (settleInput.kind === "valid" || settleInput.kind === "joint") + ? settleInput.score + : null, + alsoOff: + settleInput !== null && settleInput.kind === "joint" + ? settleInput.off + : [], + outcome: settled?.outcome ?? null, + support: settled?.support ?? null, + merged: settled?.merged ?? null, + worldBits: settled?.worldBits ?? null, + surprise: settled?.surprise ?? null, + // 每个选中的格都已结束这一段:交了"跑不起来",或核验不通过有"不跑"事件,或最近一次核验通过且跑出了结果。 + accounted: + lastImpl !== null && + (cannotRun !== null || + notRun.length > 0 || + (lastOutcome?.passed === true && + lastRun !== null && + lastRun.results.length === 1 && + lastRun.results[0]!.content.process.ending !== "notStarted")), + }; + }); + + // 解释阶段关闭早于本轮任何一格的确认启动(任务书 17 第 3.2 节第 7 样)。 + const explanationClose = closeSeqOf(`r${round}-explanation`); + const cellStartSeqs = runs + .filter( + (r) => + r.plan.content.purpose === "ablation" && + r.plan.content.round === round, + ) + .flatMap((r) => r.starts.map((s) => s.seq)); + const firstCellStart = + cellStartSeqs.length === 0 ? null : Math.min(...cellStartSeqs); + const explanationBeforeCells = + cellsDecision === null + ? null + : explanationClose !== null && + (firstCellStart === null || explanationClose < firstCellStart); + + // 轮末解封:本轮选中的每个候选都有解封记录,且写在收尾阶段关闭之后(任务书 17 第 3.2 节第 7 样)。 + const wrapUpClose = closeSeqOf(`r${round}-wrapUp`); + const unseals = ofKind(records, "unseal").filter((u) => + chosen.includes(u.content.candidate), + ); + const unseal = { + wrapUpClose, + perCandidate: chosen.map((candidate) => { + const u = unseals.find((x) => x.content.candidate === candidate); + return { + candidate, + unsealSeq: u?.seq ?? null, + afterWrapUpClose: + u !== undefined && wrapUpClose !== null && u.seq > wrapUpClose, + }; + }), + pass: + wrapUpClose !== null && + chosen.length > 0 && + chosen.every((candidate) => { + const u = unseals.find((x) => x.content.candidate === candidate); + return u !== undefined && u.seq > wrapUpClose; + }), + }; + + return { + round, + baseline: board.baselineOf(round), + candidates, + adopted, + adoptedScore, + knownBestGap, + hasAblation: cellsDecision !== null, + cellsDecision, + cells, + label: settlement?.output.label ?? null, + judgments: settlement?.output.judgments ?? null, + inseparable: settlement?.output.inseparable ?? null, + finding: settlement?.output.finding ?? null, + bet: + bet === null + ? null + : { + decision: bet.output.decision, + factor: bet.output.factor, + predictedGain: bet.output.predictedGain, + offBetter: bet.output.offBetter, + }, + settlementCount: board.settlements().filter((s) => s.round === round) + .length, + explanationBeforeCells, + unseal, + }; + }); + + // 第 2 轮起的基线:从采纳决定和结算的"消融中发现"逐轮重算,和 Board 现算的对照; + // 再核对第 r 轮冻结的协议都写在当时的基线上(方法.md 4.3 ①)。 + const chain: Array<{ round: number; baseline: string }> = [ + { round: 1, baseline: ORIGINAL_BASELINE }, + ]; + for (const round of roundIds) { + const adopted = board.decision("adopt", round)?.output.adopted ?? null; + const found = board.settlement(round)?.output.finding.chosen ?? null; + const previous = chain.at(-1)!.baseline; + chain.push({ + round: round + 1, + baseline: + found === null + ? adopted === null + ? previous + : adopted + : cellId(adopted!, found), + }); + } + const baselineChain = { + chain, + boardAgrees: chain.every((c) => board.baselineOf(c.round) === c.baseline), + protocolsOnBaseline: board + .protocols() + .filter((p) => p.protocol.round >= 2) + .every( + (p) => + p.protocol.baseline === + chain.find((c) => c.round === p.protocol.round)?.baseline, + ), + }; + + // "跑不起来"和"不跑"的格不扣次(任务书 17 第 3.2 节第 7 样):这些格没有任何运行登记。 + const silentCells = rounds.flatMap((r) => + r.cells.filter((c) => c.cannotRun !== null || c.notRunEvents.length > 0), + ); + const silentNotCharged = silentCells.every((c) => c.charged === 0); + + const halt = events.filter((e) => e.event === "halted").at(-1) ?? null; + const checks = { + stagesPaired: stages.every((s) => s.pass) && orphanCloses.length === 0, + chargedEqualsStarts: balance.charged === started.length, + finalBalanceZero: balance.balance === 0, + reservedZero: balance.reserved === 0, + runsFinished: runs.length > 0 && runs.every((r) => r.results.length === 1), + explanationBeforeCells: rounds.every( + (r) => r.explanationBeforeCells !== false, + ), + oneSettlementPerRound: rounds.every((r) => r.settlementCount <= 1), + unsealAfterWrapUp: rounds.every((r) => r.unseal.pass), + cellsAccounted: rounds.every((r) => r.cells.every((c) => c.accounted)), + silentCellsNotCharged: silentNotCharged, + baselineChain: + baselineChain.boardAgrees && baselineChain.protocolsOnBaseline, + haltedAtExport: + halt !== null && + halt.reason === FULL_STOP && + halt.seq === records.at(-1)!.seq, + }; + return { + study: config.studyId, + initial: config.runs, + knownBest: facts.knownBest, + confirmedStarts: { + total: started.length, + baseline: runs.filter( + (r) => r.plan.content.purpose === "baseline" && r.starts.length > 0, + ).length, + forward: runs.filter( + (r) => r.plan.content.purpose === "forward" && r.starts.length > 0, + ).length, + ablation: runs.filter( + (r) => r.plan.content.purpose === "ablation" && r.starts.length > 0, + ).length, + }, + balance, + runs, + rounds, + baselineChain, + stages, + orphanCloses, + halt, + checks, + }; +} + +/** 一次正式运行的 CPU 时间 / 墙钟时间(任务书 17 第 3.1 节);测不到或没启动为 null。 */ +export function cpuRatio(result: { + process: { + ending: string; + cpuSeconds?: { status: string; value?: number }; + wallSeconds?: { status: string; value?: number }; + }; +}): number | null { + const p = result.process; + if (p.ending === "notStarted") return null; + if ( + p.cpuSeconds?.status !== "measured" || + p.wallSeconds?.status !== "measured" + ) + return null; + return p.wallSeconds.value === 0 + ? null + : p.cpuSeconds.value! / p.wallSeconds.value!; +} diff --git a/package.json b/package.json index 2538b31..a11e661 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,10 @@ "eval:route:thinking": "node eval/route/thinking.ts", "eval:forward:run": "set -a && . /home/iomgaa/Projects/RivalBits/.env && set +a && node eval/forward/run.ts", "eval:forward:export": "node eval/forward/export.ts", - "eval:forward:stats": "node eval/forward/stats.ts" + "eval:forward:stats": "node eval/forward/stats.ts", + "eval:full:run": "set -a && . /home/iomgaa/Projects/RivalBits/.env && set +a && node eval/full/run.ts", + "eval:full:stats": "node eval/full/stats.ts", + "eval:full:export": "node eval/full/export.ts" }, "dependencies": { "@earendil-works/pi-ai": "0.85.1", diff --git a/tests/full-eval.test.ts b/tests/full-eval.test.ts new file mode 100644 index 0000000..f868560 --- /dev/null +++ b/tests/full-eval.test.ts @@ -0,0 +1,448 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { test, type TestContext } from "node:test"; +import { cpuRatio, fullAccounts, FULL_STOP } from "../eval/lib/full.ts"; +import { Ledger } from "../src/ledger/index.ts"; +import { + cellId, + cellNotRunEvent, + type Settlement, +} from "../src/materials/index.ts"; +import { + chooseBet, + runCellRule, + settleAblation, + type AblationModelInput, + type CellRuleInput, + type SettleInput, +} from "../src/rules/index.ts"; +import { checkTaskFacts } from "../src/tasks/index.ts"; +import { createRepository, saveVersion } from "../src/workspace/index.ts"; +import { fullConfig } from "./config-helpers.ts"; +import { definition, exited, tempDir } from "./ledger-helpers.ts"; +import { batch, judged, protocolOf, r, Writer } from "./materials-helpers.ts"; +import type { Submission } from "../src/vocab/index.ts"; + +// eval/full 的账目统计与审阅导出:在真实账本里写一份走完一轮(含消融)的研究,跑 stats 和 export,看产物。 +// 选格、结算、押的因子的记录用 rules 的真实函数现算(与 flow 同一个入口),预期值按方法.md 6.6 手算对照。 + +const facts = checkTaskFacts( + JSON.parse( + readFileSync( + join(import.meta.dirname, "..", "tasks", "circle_packing", "task.json"), + "utf8", + ), + ), +); + +const TWO_FACTORS: Submission<"proposal"> = { + factors: [ + { + name: "多起点重启", + method: "做法一", + keySettings: "设定一", + continues: null, + }, + { + name: "停滞扰动", + method: "做法二", + keySettings: "设定二", + continues: null, + }, + ], + safeguards: [], + metrics: { defined: [], reused: [] }, + authorEstimate: r(1.0, 1.1, 1.2), + body: "提案正文", +}; + +const EXPLANATIONS = [ + { + author: "a1", + cells: [ + { factor: "p1", range: r(1.0, 1.1, 1.2) }, + { factor: "p2", range: r(1.15, 1.2, 1.24) }, + ], + }, + { + author: "a2", + cells: [ + { factor: "p1", range: r(0.9, 1.05, 1.2) }, + { factor: "p2", range: r(1.1, 1.18, 1.25) }, + ], + }, +]; + +const modelInput = (): AblationModelInput => ({ + baseline: 0.96, + full: 1.25, + direction: "maximize", + knownBest: facts.knownBest, + epsilon: facts.epsilon, + lambda: 0.1, + factors: ["p1", "p2"], + explanations: EXPLANATIONS.map((e, i) => ({ + id: `r1-${e.author}-x${1}`, + author: e.author, + cells: e.cells, + })), +}); + +const verificationOf = (off: string): Submission<"ablationVerification"> => ({ + factors: [ + { factor: "p1", off: off === "p1", where: "solve.py:10-20" }, + { factor: "p2", off: off === "p2", where: "solve.py:30-40" }, + ], +}); + +/** 一份走完的研究:基线 0.96,第 1 轮采纳两因子的 r1-c1(1.25),消融两格测 1.10、1.20,收尾解封后停在导出。 */ +function studyDir(t: TestContext): { + dir: string; + ledger: Ledger; + sealSeq: number; +} { + const root = tempDir(t); + const dir = join(root, "run"); + const study = join(dir, "study"); + mkdirSync(study, { recursive: true }); + const ledger = Ledger.open(join(study, "ledger.sqlite")); + t.after(() => ledger.close()); + const w = new Writer(ledger); + // 预算 4 次:基线、正向、两格消融,走完余额正好是 0。 + ledger.append("config", { + config: { ...fullConfig(), runs: 4 }, + task: facts as never, + }); + + const repo = join(study, "repo.git"); + const work = join(root, "work"); + createRepository(repo); + mkdirSync(work); + const version = (text: string) => { + writeFileSync(join(work, "solve.py"), text); + const result = saveVersion(repo, work, []); + assert.ok(result.saved); + return result.version; + }; + + ledger.append("code", { + object: "base", + version: version("value = 0\n"), + locations: null, + }); + w.run("base-1", "baseline", 0, null, judged(0.96, "main")); + + w.open("r1-proposal", "proposal", 1, ["a1", "a2"]); + const proposalSeq = w.sub( + "r1-proposal", + "a1", + "proposal", + null, + "r1-a1-1", + TWO_FACTORS, + ); + w.close("r1-proposal"); + w.decision("dedup", 1, null, { + fresh: [{ candidate: "r1-c1", proposals: ["r1-a1-1"] }], + carried: [], + }); + w.protocol( + "r1-c1", + protocolOf("r1-c1", TWO_FACTORS, 1, "base", ["r1-a1-1"], ["c1", "c2"]), + ); + + w.open("r1-sp", "secondPrediction", 1, ["a1", "a2"]); + w.sub("r1-sp", "a1", "secondPrediction", "r1-c1", null, { + range: r(1.05, 1.15, 1.25), + body: "理由", + }); + w.sub("r1-sp", "a2", "secondPrediction", "r1-c1", null, { + range: r(1.0, 1.1, 1.2), + body: "理由", + }); + w.close("r1-sp"); + const b = batch(1, 0.96, { + "r1-c1": { a1: r(1.05, 1.15, 1.25), a2: r(1.0, 1.1, 1.2) }, + }); + w.decision("batch", 1, b.input, b.output); + + w.open("r1-implementation", "implementation", 1, ["x1"]); + w.sub("r1-implementation", "x1", "implementation", "r1-c1", null, { + factorLocations: [ + { factor: "p1", locations: ["solve.py:10-20"] }, + { factor: "p2", locations: ["solve.py:30-40"] }, + ], + safeguardLocations: [], + unassigned: [], + concerns: "无", + body: "实现说明", + }); + ledger.append("code", { + object: "r1-c1", + version: version("value = 1\n"), + locations: null, + }); + w.close("r1-implementation"); + w.open("r1-verification-1", "verification", 1, ["v1"]); + w.sub("r1-verification-1", "v1", "verification", "r1-c1", null, { + clauses: [{ clause: "c1", found: true, where: "solve.py:10" }], + factors: [ + { factor: "p1", found: true, where: "solve.py:10-20" }, + { factor: "p2", found: true, where: "solve.py:30-40" }, + ], + safeguards: [], + unassigned: [], + body: "逐项核对", + }); + w.close("r1-verification-1"); + w.run("r1-forward-r1-c1-1", "forward", 1, "r1-c1", judged(1.25, "main")); + w.decision("adopt", 1, null, { adopted: "r1-c1" }); + + w.open("r1-explanation", "explanation", 1, ["a1", "a2"]); + for (const e of EXPLANATIONS) { + w.sub("r1-explanation", e.author, "explanation", null, null, { + explanations: [{ body: `机制(${e.author})`, cells: e.cells }], + }); + } + w.close("r1-explanation"); + + const cellInput: CellRuleInput = { + ...modelInput(), + balance: 8, + batchCap: 2, + u: 0.5, + decay: 0.5, + seed: "seed", + }; + const { rule, ...cellOutput } = runCellRule("all", cellInput); + ledger.append("decision", { + decision: "cells", + round: 1, + rule: rule.name, + ruleVersion: rule.version, + params: cellOutput.params, + seed: "seed", + input: cellInput as never, + output: cellOutput as never, + }); + + w.open(`r1-ablationImplementation-1`, "ablationImplementation", 1, ["x1"]); + for (const [key, text] of [ + ["p1", "value = 2\n"], + ["p2", "value = 3\n"], + ] as const) { + const cell = cellId("r1-c1", key); + w.sub( + `r1-ablationImplementation-1`, + "x1", + "ablationImplementation", + cell, + null, + { alsoOff: [], cannotRun: null, concerns: "无" }, + ); + ledger.append("code", { + object: cell, + version: version(text), + locations: null, + }); + } + w.close(`r1-ablationImplementation-1`); + w.open(`r1-ablationVerification-1`, "ablationVerification", 1, ["v1"]); + for (const key of ["p1", "p2"] as const) { + w.sub( + `r1-ablationVerification-1`, + "v1", + "ablationVerification", + cellId("r1-c1", key), + null, + verificationOf(key), + ); + } + w.close(`r1-ablationVerification-1`); + for (const [key, score] of [ + ["p1", 1.1], + ["p2", 1.2], + ] as const) { + const cell = cellId("r1-c1", key); + const addRun = (id: string) => + w.run(id, "ablation", 1, cell, judged(score, "main")); + addRun(`r1-ablation-${cell}-1`); + } + + const settleInput: SettleInput = { + ...modelInput(), + cells: [ + { factor: "p1", outcome: { kind: "valid", score: 1.1 } }, + { factor: "p2", outcome: { kind: "valid", score: 1.2 } }, + ], + }; + const settled: Settlement = { + candidate: "r1-c1", + input: settleInput, + output: settleAblation(settleInput), + }; + ledger.append("settlement", { round: 1, result: settled as never }); + const betInput = { ...modelInput(), cells: settleInput.cells }; + w.decision("bet", 1, betInput, chooseBet(betInput)); + + w.open("r1-wrapUp", "wrapUp", 1, ["a1", "a2"]); + w.sub("r1-wrapUp", "a1", "wrapUp", null, null, { + body: "甲的收尾:p1 撑了大半", + }); + w.sub("r1-wrapUp", "a2", "wrapUp", null, null, { body: "乙的收尾" }); + w.close("r1-wrapUp"); + ledger.append("unseal", { candidate: "r1-c1", seals: [proposalSeq + 1] }); + ledger.append("event", { event: "halted", reason: FULL_STOP, objects: [] }); + return { dir, ledger, sealSeq: proposalSeq + 1 }; +} + +test("full 账目:一轮含消融的研究全部核对通过;撑几成、押的因子按手算对上", (t) => { + const { ledger } = studyDir(t); + const accounts = fullAccounts(ledger.records()); + assert.deepEqual(accounts.confirmedStarts, { + total: 4, + baseline: 1, + forward: 1, + ablation: 2, + }); + assert.deepEqual(accounts.balance, { charged: 4, reserved: 0, balance: 0 }); + assert.ok( + Object.values(accounts.checks).every(Boolean), + JSON.stringify(accounts.checks), + ); + const round = accounts.rounds[0]!; + assert.equal(round.adopted, "r1-c1"); + assert.equal(round.adoptedScore, 1.25); + assert.ok(Math.abs(round.knownBestGap! - (2.635983 - 1.25)) < 1e-12); + assert.equal(round.cells.length, 2); + // 撑几成(方法.md 6.6):(1.25 − 1.10) / 0.29 与 (1.25 − 1.20) / 0.29。 + assert.ok(Math.abs(round.cells[0]!.support! - 0.15 / 0.29) < 1e-12); + assert.ok(Math.abs(round.cells[1]!.support! - 0.05 / 0.29) < 1e-12); + assert.equal(round.label!.label, "substitute"); // i = 0.690 − 1 < −0.2 + assert.equal(round.bet!.decision, "bet"); + assert.equal(round.bet!.factor, "p1"); + assert.equal(round.unseal.pass, true); + assert.equal(round.explanationBeforeCells, true); + assert.deepEqual( + round.candidates[0]!.authorEstimates.map((u) => u.range), + [r(1.0, 1.1, 1.2)], + ); +}); + +test("full 账目:缺解封、不跑的格被扣次都过不了核对", (t) => { + const { ledger } = studyDir(t); + const before = fullAccounts(ledger.records()); + assert.equal(before.checks.unsealAfterWrapUp, true); + // 去掉解封记录重建一份账本太重,这里反过来验:另写一份没有解封的账。 + const plain = ledger + .records() + .filter((x) => x.kind !== "unseal" && x.kind !== "event"); + assert.equal(fullAccounts(plain).checks.unsealAfterWrapUp, false); + assert.equal(fullAccounts(plain).checks.haltedAtExport, false); + // "不跑"的格被扣次:给它补一条运行登记和启动,silentCellsNotCharged 应变 false。 + ledger.append( + "event", + cellNotRunEvent(cellId("r1-c1", "p2"), "测试:核验不通过"), + ); + ledger.append("runPlanned", { + run: "r1-ablation-r1-c1-p2-2", + purpose: "ablation", + round: 1, + object: "r1-c1-p2", + definition, + }); + ledger.append("runStarted", { run: "r1-ablation-r1-c1-p2-2" }); + const after = fullAccounts(ledger.records()); + assert.equal(after.checks.silentCellsNotCharged, false); + assert.equal(after.checks.chargedEqualsStarts, true); // 账目本身仍平:5 次启动 5 次扣 +}); + +test("full stats/export:从研究输出目录产出 stats.json/txt 和逐轮审阅材料", (t) => { + const { dir } = studyDir(t); + const statsOut = join(tempDir(t), "stats"); + const stats = spawnSync( + process.execPath, + ["eval/full/stats.ts", statsOut, dir], + { encoding: "utf8" }, + ); + assert.equal(stats.status, 0, stats.stderr); + const text = readFileSync(join(statsOut, "stats.txt"), "utf8"); + assert.match(text, /确认启动:基线 1、正向 1、消融 2,共 4;扣次 4,余额 0/); + assert.match(text, /采纳:r1-c1,分数 1\.250000,离已知最好 1\.385983/); + assert.match( + text, + /r1-c1-p1:核验 1 次、重做 0 次,跑没跑 跑了(扣次 1),实测 1\.100000,撑几成 0\.517/, + ); + assert.match(text, /押的因子 p1/); + const summaries = JSON.parse( + readFileSync(join(statsOut, "stats.json"), "utf8"), + ); + assert.equal(summaries[0].pass, true); + assert.equal(summaries[0].kill, null); + assert.equal(summaries[0].ratios.length, 4); + assert.ok(summaries[0].ratios.every((x: { flagged: boolean }) => !x.flagged)); + + const review = join(tempDir(t), "review"); + const exported = spawnSync( + process.execPath, + ["eval/full/export.ts", dir, review], + { encoding: "utf8" }, + ); + assert.equal(exported.status, 0, exported.stderr); + const roundDir = join(review, "round-1"); + assert.match( + readFileSync(join(roundDir, "explanations.txt"), "utf8"), + /机制(a1)/, + ); + assert.match( + readFileSync(join(roundDir, "wrapups.txt"), "utf8"), + /甲的收尾:p1 撑了大半/, + ); + assert.match( + readFileSync(join(roundDir, "proposals.txt"), "utf8"), + /多起点重启/, + ); + assert.equal( + readFileSync( + join(roundDir, "cells", "r1-c1-p1", "full", "solve.py"), + "utf8", + ), + "value = 1\n", + ); + assert.equal( + readFileSync( + join(roundDir, "cells", "r1-c1-p1", "cell", "solve.py"), + "utf8", + ), + "value = 2\n", + ); + const verifications = JSON.parse( + readFileSync( + join(roundDir, "cells", "r1-c1-p1", "verifications.json"), + "utf8", + ), + ); + assert.equal(verifications[0].outcome.passed, true); + const settlement = JSON.parse( + readFileSync(join(roundDir, "settlement.json"), "utf8"), + ); + assert.equal(settlement.bet.content.output.factor, "p1"); + assert.match(readFileSync(join(roundDir, "review.txt"), "utf8"), /第3样/); +}); + +test("full CPU/墙钟比:测不到或没启动记 null,正常退出算出比值", () => { + assert.equal(cpuRatio({ process: { ending: "notStarted" } }), null); + assert.ok(Math.abs(cpuRatio({ process: exited })! - 100 / 101) < 1e-12); + assert.equal( + cpuRatio({ + process: { + ending: "exited", + cpuSeconds: { status: "unmeasured" }, + wallSeconds: { status: "measured", value: 1 }, + }, + }), + null, + ); +});