diff --git a/eval/lib/timeline.ts b/eval/lib/timeline.ts new file mode 100644 index 0000000..6139a14 --- /dev/null +++ b/eval/lib/timeline.ts @@ -0,0 +1,52 @@ +import { appendFileSync, existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import type { ChildProcess } from "node:child_process"; +import { sleep } from "./study.ts"; + +// 账本里没有时间戳,墙钟时间由检验自己记:研究进程跑着的时候每隔一会儿看一眼账本, +// 新出现的每条记录记下"第一次看到它的时刻",写成 timeline.jsonl(每行 {seq, kind, at},at 是毫秒)。 +// 只读打开账本、只查序号和种类,不经 Ledger.records()(那会每次重读、重查全部内容)。 +// 时刻比真实写入晚最多一个间隔,用来算阶段级的墙钟足够。 + +const POLL_MS = 1000; + +export interface TimelineRow { + seq: number; + kind: string; + at: number; +} + +/** 已经记下的最大序号;文件还没有就是 0。接着跑时往同一个文件后面追加。 */ +function lastSeq(file: string): number { + if (!existsSync(file)) return 0; + return readTimeline(file).reduce((m, r) => Math.max(m, r.seq), 0); +} + +/** 研究进程退出前一直记;进程退出后再看最后一眼。调用前要先等 waitForLedger(表建好了才能读)。 */ +export async function watchTimeline(studyDir: string, child: ChildProcess, file: string): Promise { + let seen = lastSeq(file); + const look = () => { + const db = new DatabaseSync(join(studyDir, "ledger.sqlite"), { readOnly: true }); + try { + const rows = db.prepare("SELECT seq, kind FROM records WHERE seq > ? ORDER BY seq").all(seen) as Array<{ seq: number; kind: string }>; + const at = Date.now(); + for (const r of rows) appendFileSync(file, `${JSON.stringify({ seq: r.seq, kind: r.kind, at })}\n`); + if (rows.length > 0) seen = rows.at(-1)!.seq; + } finally { + db.close(); + } + }; + while (child.exitCode === null && child.signalCode === null) { + look(); + await sleep(POLL_MS); + } + look(); +} + +export function readTimeline(file: string): TimelineRow[] { + return readFileSync(file, "utf8") + .split("\n") + .filter((l) => l !== "") + .map((l) => JSON.parse(l) as TimelineRow); +} diff --git a/eval/round-one/blind.ts b/eval/round-one/blind.ts new file mode 100644 index 0000000..0a64e50 --- /dev/null +++ b/eval/round-one/blind.ts @@ -0,0 +1,30 @@ +import { copyFileSync, existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { randomInt } from "node:crypto"; + +// 盲评用的汇总(任务书 14 第 3.2 节第 1 步最后一条):把导出目录里每次运行的 text.txt 随机编成"运行 A、B……", +// 只把这些文本放进 <检验目录>/blind/;对照表(字母 → 运行名)另存到 blind 外面的 blind-key.json,审阅者看不到。 +// node eval/round-one/blind.ts + +const out = process.argv[2]; +if (out === undefined) throw new Error("要写 export.ts 写出的检验目录"); +const runs = readdirSync(join(out, "runs")).sort(); +if (runs.length > 26) throw new Error(`有 ${runs.length} 次运行,字母不够用`); +const blind = join(out, "blind"); +if (existsSync(blind)) throw new Error(`${blind} 已经有了:重新编号会让旧的对照表作废,先挪走`); +mkdirSync(blind); + +// Fisher–Yates 洗牌,用 crypto 的随机数:编号不能从运行名或目录顺序推出来。 +const order = [...runs]; +for (let i = order.length - 1; i > 0; i--) { + const j = randomInt(i + 1); + [order[i], order[j]] = [order[j]!, order[i]!]; +} +const key: Record = {}; +order.forEach((run, i) => { + const letter = String.fromCharCode(65 + i); + key[`运行 ${letter}`] = run; + copyFileSync(join(out, "runs", run, "text.txt"), join(blind, `运行-${letter}.txt`)); +}); +writeFileSync(join(out, "blind-key.json"), `${JSON.stringify(key, null, 2)}\n`); +console.log(`${order.length} 份文本 → ${blind}\n对照表 → ${join(out, "blind-key.json")}`); diff --git a/eval/round-one/export.ts b/eval/round-one/export.ts new file mode 100644 index 0000000..3cf76ed --- /dev/null +++ b/eval/round-one/export.ts @@ -0,0 +1,126 @@ +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; +import type { StudyRecord } from "../../src/materials/index.ts"; +import { ofKind, readLedger } from "../lib/ledger.ts"; +import { newEvalDir } from "../lib/study.ts"; +import { Neutral, render } from "./neutral.ts"; +import { readOurs, readReference, type Json, type Prediction, type RoundData } from "./round-data.ts"; + +// 把几次运行的第 1 轮导出成同一种纯文本(任务书 14 第 3.2 节第 1 步),按固定小标题排,去掉来历。 +// node eval/round-one/export.ts <检验名字> <来源>... +// 来源是我们的研究目录(里面有 ledger.sqlite),或参考项目的 board.sqlite(只读,先复制到输出目录)。 +// 每次运行写进 <检验目录>/runs/<名字>/:data.json(检验用的中间数据,带原编号)、text.txt(给审阅者的文本)、 +// names.json(原编号 → 中性名的映射,审阅者看不到)。之后用 blind.ts 随机编成"运行 A、B……"。 + +const [name, ...sources] = process.argv.slice(2); +if (name === undefined || sources.length === 0) throw new Error("用法:node eval/round-one/export.ts <检验名字> <研究目录或 board.sqlite>..."); +const out = newEvalDir(name); + +for (const source of sources) { + const ours = existsSync(join(source, "ledger.sqlite")); + // 名字取上一层目录名:我们的研究目录是 <检验目录>/study,取检验目录名;参考项目取运行名(board.sqlite 所在目录)。 + const label = basename(dirname(source)); + const dir = join(out, "runs", label); + if (existsSync(dir)) throw new Error(`两个来源的名字都是 ${label},分不开`); + mkdirSync(dir, { recursive: true }); + let data: RoundData; + if (ours) { + const records = readLedger(source); + const study = ofKind(records, "config")[0]?.content as StudyRecord | undefined; + if (study === undefined) throw new Error(`${source} 的账本里没有研究配置记录`); + data = readOurs(records, 1, source, (study.config as { studyId: string }).studyId); + } else { + data = readReference(source, join(dir, "board-copy")); + } + const nt = new Neutral(data.hiddenNames); + writeFileSync(join(dir, "data.json"), `${JSON.stringify(data, null, 2)}\n`); + writeFileSync(join(dir, "text.txt"), renderRound(data, nt)); + writeFileSync(join(dir, "names.json"), `${JSON.stringify(nt.table(), null, 2)}\n`); + console.log(`${label}:提案 ${data.proposals.length}、候选 ${data.dedup.groups.length}、挑刺 ${data.critiques.length}、退回 ${data.rejections.length} → ${dir}`); +} +console.log(`检验目录:${out}`); + +// --------------------------------------------------------------------------- +// 文本 +// --------------------------------------------------------------------------- + +function renderRound(d: RoundData, nt: Neutral): string { + const out: string[] = ["# 第 1 轮记录(开轮到选批)", ""]; + const range = (r: { q1: number; q5: number; q9: number }) => `q1 = ${r.q1},q5 = ${r.q5},q9 = ${r.q9}`; + const candidates = d.dedup.groups.map((g) => g.candidate); + const candidateOf = new Map(d.dedup.groups.flatMap((g) => g.proposals.map((p) => [p, g.candidate] as const))); + + // 先按"提案 → 候选 → 挑刺"的顺序给编号起中性名,正文里引用到的编号就和小标题一致。 + for (const p of d.proposals) { + nt.name("研究者", p.author); + nt.name("提案", p.id); + } + for (const g of d.dedup.groups) nt.name("候选", g.candidate); + for (const p of d.proposals) for (const c of d.critiques.filter((x) => x.proposal === p.id)) nt.name("挑刺", c.id); + + out.push("## 一、提案全文", ""); + for (const p of d.proposals) { + out.push(`### ${nt.text(p.id)}(${nt.text(p.author)})`, ""); + out.push(`作者自评:${p.authorEstimate === null ? "无" : range(p.authorEstimate)}`, ""); + out.push(render(p.content, nt), ""); + } + + out.push("## 二、判重结果", ""); + for (const g of d.dedup.groups) out.push(`- ${nt.text(g.candidate)}:${g.proposals.map((p) => nt.text(p)).join("、")}`); + out.push("", "并进留池候选的:", render(d.dedup.carried, nt), "", "判重时每一对的判断(原样):", render(d.dedup.judgments, nt), ""); + + out.push("## 三、挑刺和回应", ""); + for (const p of d.proposals) { + const these = d.critiques.filter((c) => c.proposal === p.id); + const candidate = candidateOf.get(p.id); + out.push(`### 对 ${nt.text(p.id)} 的挑刺${candidate === undefined ? "" : `(${nt.text(candidate)})`}`, ""); + if (these.length === 0) out.push("(没有挑刺)", ""); + for (const c of these) { + out.push(`#### ${nt.text(c.id)}(${nt.text(c.author)})`, "", render(c.content, nt), ""); + const replies = d.responses.flatMap((r) => r.replies.filter((x) => x.critique === c.id).map((x) => ({ ...x, author: r.author }))); + if (replies.length === 0) out.push("作者回应:(没有找到对这条挑刺的回应)", ""); + for (const x of replies) out.push(`作者回应(${nt.text(x.author)}):`, render(x.text, nt, " "), ""); + } + } + for (const r of d.responses) { + out.push(`### ${nt.text(r.candidate)} 回应里逐条答复以外的部分(${nt.text(r.author)},${r.kind === "unification" ? "统一" : "回应"})`, ""); + out.push(render(r.rest, nt), ""); + } + + out.push("## 四、冻结协议", ""); + for (const p of d.protocols) out.push(`### ${nt.text(p.candidate)}`, "", render(p.content, nt), ""); + + out.push("## 五、两次预测的三个数", ""); + const predictions = (title: string, list: Prediction[] | null, missing: string) => { + out.push(`### ${title}`, ""); + if (list === null) return out.push(missing, ""); + const order = (p: Prediction) => [candidates.indexOf(p.candidate), nt.text(p.researcher)] as const; + const sorted = [...list].sort((a, b) => order(a)[0] - order(b)[0] || order(a)[1].localeCompare(order(b)[1])); + for (const p of sorted) { + out.push(`- ${nt.text(p.candidate)},${nt.text(p.researcher)}:${range(p.range)}`); + if (p.note !== null) out.push(render(p.note as Json, nt, " ")); + } + out.push(""); + }; + predictions("第一次预测", d.firstPrediction, "无此阶段"); + predictions("第二次预测", d.secondPrediction, "不适用:这次运行用投票选批,不是范围预测。投票原样如下:"); + if (d.votes !== null) out.push(render(d.votes, nt), ""); + + out.push("## 六、选批结果", ""); + out.push(`选中:${d.batch.chosen.length === 0 ? "无" : d.batch.chosen.map((c) => nt.text(c)).join("、")}`, "", "选批记录(原样):", render(d.batch.detail, nt), ""); + + out.push("## 七、退回的次数和原因", ""); + const byStage = new Map(); + for (const r of d.rejections) byStage.set(r.stage, (byStage.get(r.stage) ?? 0) + 1); + out.push(`共 ${d.rejections.length} 次${byStage.size === 0 ? "" : `:${[...byStage].map(([s, n]) => `${s} ${n} 次`).join(",")}`}`, ""); + d.rejections.forEach((r, i) => { + out.push(`${i + 1}. ${r.stage},${nt.text(r.author)}${r.target === null ? "" : `,对象 ${nt.text(r.target)}`}:`, render(clip(r.reason), nt, " "), ""); + }); + return `${out.join("\n")}\n`; +} + +/** 退回原因只留开头 800 字:形状不合格的退回会把整份交上来的内容附在后面,全文在 data.json 里。 */ +function clip(s: string): string { + const keep = 800; + return s.length <= keep ? s : `${s.slice(0, keep)}\n(以下省略 ${s.length - keep} 字)`; +} diff --git a/eval/round-one/jevcheck.ts b/eval/round-one/jevcheck.ts new file mode 100644 index 0000000..9b9f63d --- /dev/null +++ b/eval/round-one/jevcheck.ts @@ -0,0 +1,97 @@ +import { readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { ask, THRESHOLD, THRESHOLD_NOTE, type Answer, type Noul } from "../lib/jev.ts"; +import { ADDRESSED, CARRIES_IDEA, SAME_CHANGE, STAYS_IN_DEAD_END, TASKS } from "./questions.ts"; +import type { RoundData } from "./round-data.ts"; + +// 第 1 轮的 Jev 封闭问题(任务书 14 第 3.2 节第 3 步),对 export.ts 导出的每次运行都做(我们的和参考项目的),读 data.json: +// a. 判重对不对:本轮提案两两、两个顺序各问一次"是不是同一个改法",取平均,和程序的判重决定对照; +// b. 挑刺有没有被回应:每条挑刺问一次(找不到回应的不问,直接记"没有回应"); +// c. 提案对应研究路径的哪一步:每条提案对 K1–K5(做没做这个想法)和 D1、D2(是不是整个停在这条死路)各问一次,只描述。 +// 阈值未校准,概率大于 0.5 判"是";原始概率全部存进 <检验目录>/jevcheck.json。要花钱(Jev)。 +// node eval/round-one/jevcheck.ts + +type Idea = { id: string; name: string; description: string }; +const ideas = JSON.parse(readFileSync("eval/round-zero/ideas.json", "utf8")) as { key: Idea[]; deadEnds: Idea[] }; + +const out = process.argv[2]; +if (out === undefined) throw new Error("要写 export.ts 写出的检验目录"); + +/** 同时最多几个请求在路上:一下发上百个容易被限流。 */ +const PARALLEL = 6; +let running = 0; +const waiting: Array<() => void> = []; +async function limited(fn: () => Promise): Promise { + if (running >= PARALLEL) await new Promise((ok) => waiting.push(ok)); + running++; + try { + return await fn(); + } finally { + running--; + waiting.shift()?.(); + } +} +async function one(state: Record, questions: Record): Promise { + const [answer] = await limited(() => ask(state, questions)); + return answer!; +} + +async function check(d: RoundData) { + const change = new Map(d.proposals.map((p) => [p.id, p.change])); + const groupOf = new Map(d.dedup.groups.flatMap((g) => g.proposals.map((p) => [p, g.candidate] as const))); + + // a. 判重 + const pairQuestion = (a: string, b: string) => one({ task: TASKS.pair, proposal_a: change.get(a), proposal_b: change.get(b) }, SAME_CHANGE); + const pairs = await Promise.all( + d.proposals.flatMap((a, i) => + d.proposals.slice(i + 1).map(async (b) => { + const [ab, ba] = await Promise.all([pairQuestion(a.id, b.id), pairQuestion(b.id, a.id)]); + const mean = (ab.probability + ba.probability) / 2; + return { pair: [a.id, b.id], ab, ba, mean, jevSame: mean > THRESHOLD, programMerged: groupOf.get(a.id) === groupOf.get(b.id) }; + }), + ), + ); + const dedup = { + pairs, + jevYesProgramNotMerged: pairs.filter((p) => p.jevSame && !p.programMerged).map((p) => p.pair), + jevNoProgramMerged: pairs.filter((p) => !p.jevSame && p.programMerged).map((p) => p.pair), + }; + + // b. 挑刺有没有被回应 + const critiques = await Promise.all( + d.critiques.map(async (c) => { + const response = d.responses.find((r) => r.replies.some((x) => x.critique === c.id)); + if (response === undefined) return { critique: c.id, author: c.author, proposal: c.proposal, replied: false as const, answer: null }; + const reply = response.replies.find((x) => x.critique === c.id)!.text; + const answer = await one({ task: TASKS.critique, critique: c.content, reply, revision: response.rest }, ADDRESSED); + return { critique: c.id, author: c.author, proposal: c.proposal, replied: true as const, answer }; + }), + ); + + // c. 研究路径 + const path = await Promise.all( + d.proposals.map(async (p) => { + const ask = (idea: Idea, questions: Record) => + one({ task: TASKS.idea, proposal: p.change, idea: { description: idea.description } }, questions).then((answer) => ({ idea: idea.id, answer })); + const answers = await Promise.all([...ideas.key.map((k) => ask(k, CARRIES_IDEA)), ...ideas.deadEnds.map((k) => ask(k, STAYS_IN_DEAD_END))]); + return { proposal: p.id, author: p.author, answers }; + }), + ); + return { dedup, critiques, path }; +} + +const report: Record> & { source: string }> = {}; +for (const run of readdirSync(join(out, "runs")).sort()) { + const d = JSON.parse(readFileSync(join(out, "runs", run, "data.json"), "utf8")) as RoundData; + report[run] = { source: d.source, ...(await check(d)) }; + const r = report[run]; + const p = (x: number) => x.toFixed(2); + console.log(`\n== ${run}`); + console.log(`a 判重:${r.dedup.pairs.map((x) => `${x.pair.join("~")} ${p(x.ab.probability)}/${p(x.ba.probability)} 平均 ${p(x.mean)}${x.programMerged ? "(程序合了)" : ""}`).join(";")}`); + console.log(` 检验判是而程序没合:${JSON.stringify(r.dedup.jevYesProgramNotMerged)};检验判否而程序合了:${JSON.stringify(r.dedup.jevNoProgramMerged)}`); + const answered = r.critiques.filter((c) => c.answer?.yes === true).length; + console.log(`b 挑刺被回应:${answered}/${r.critiques.length};${r.critiques.map((c) => `${c.critique.slice(0, 40)} ${c.replied ? p(c.answer!.probability) : "没有回应"}`).join(";")}`); + for (const x of r.path) console.log(`c ${x.proposal}:${x.answers.map((a) => `${a.idea} ${p(a.answer.probability)}${a.answer.yes ? "是" : ""}`).join(" ")}`); +} +writeFileSync(join(out, "jevcheck.json"), `${JSON.stringify({ thresholdNote: THRESHOLD_NOTE, runs: report }, null, 2)}\n`); +console.log(`\n(${THRESHOLD_NOTE})原始概率:${join(out, "jevcheck.json")}`); diff --git a/eval/round-one/kill.ts b/eval/round-one/kill.ts new file mode 100644 index 0000000..6e9f5ea --- /dev/null +++ b/eval/round-one/kill.ts @@ -0,0 +1,154 @@ +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { LedgerRecord } from "../../src/ledger/index.ts"; +import { callId, type DedupOutput } from "../../src/materials/index.ts"; +import { ofKind, readLedger, stages, type Of } from "../lib/ledger.ts"; +import { launch, newEvalDir, sleep, waitForLedger, writeConfig } from "../lib/study.ts"; +import { watchTimeline } from "../lib/timeline.ts"; + +// 真实运行里的杀进程检验(任务书 14 第 3.1 节):和 run.ts 一样从头跑一份研究(真模型、真 Docker、真 Jev,要花钱), +// 一边跑一边看账本;第 1 轮第一次预测阶段开着、已有一部分第一次预测被收下、还有人在写时,SIGKILL 控制进程, +// 再用恢复入口(--resume)启动让它跑完,查四条: +// 1 被杀那次调用补了结束记录(出错、原因"进程中断"); +// 2 交过的任务(人×候选)没被重派,没交的被重派; +// 3 没有重复收下的提交(同人同对象同阶段只有一份收下,除非是替换); +// 4 第一次预测阶段照常关闭,往下走到选批后停下(原因"实现还没写")。 +// node eval/round-one/kill.ts <名字> +// 产物和 run.ts 一样(study、timeline.jsonl、outcome.json),另有 kill-report.json;这份研究也算 3 次真实运行之一。 + +const POLL_MS = 3000; +const KIND = "firstPrediction"; +/** flow 写停下事件用的事件名(src/flow/next-step.ts 的 EVENTS.halted;flow 只导出 runStudy,这里照抄)。 */ +const HALTED = "halted"; + +const name = process.argv[2]; +if (name === undefined) throw new Error("要写检验名字,例如 r1-kill-1(只许小写字母、数字和 ._-,它也是研究编号)"); +const out = newEvalDir(name); +const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, null); +const studyDir = join(out, "study"); +const timeline = join(out, "timeline.jsonl"); + +function openCalls(r: LedgerRecord[]): Array> { + const ended = new Set(ofKind(r, "callEnded").map((e) => e.content.call)); + return ofKind(r, "callStarted").filter((c) => !ended.has(c.content.call)); +} +/** 第 1 轮第一次预测阶段(开着的或已关的);还没开是 undefined。 */ +const predictionStage = (r: LedgerRecord[]) => stages(r).find((s) => s.round === 1 && s.stage === KIND); +/** 第一次预测的任务:名单上每个人 × 本轮每个新候选(任务书 14 第 2.3 节)。新候选取第 1 轮的判重决定。 */ +function tasks(r: LedgerRecord[], roster: string[]): Array<{ actor: string; candidate: string }> { + const dedup = ofKind(r, "decision").find((d) => d.content.decision === "dedup" && d.content.round === 1); + if (dedup === undefined) throw new Error("第一次预测阶段开了,账本里却没有第 1 轮的判重决定"); + const fresh = (dedup.content.output as unknown as DedupOutput).fresh.map((g) => g.candidate); + return roster.flatMap((actor) => fresh.map((candidate) => ({ actor, candidate }))); +} +const accepted = (r: LedgerRecord[], stageId: string, actor: string, candidate: string) => + ofKind(r, "submission").some((s) => s.content.stageId === stageId && s.content.kind === KIND && s.content.author === actor && s.content.target === candidate); + +/** 打中的条件:阶段开着没关,已有至少一份收下,还有任务没交,而且有人正在写(这样第 1、2 条都查得到东西)。 */ +function hit(r: LedgerRecord[]): boolean { + const s = predictionStage(r); + if (s === undefined || s.closed) return false; + const all = tasks(r, s.roster); + const done = all.filter((t) => accepted(r, s.stageId, t.actor, t.candidate)).length; + return done > 0 && done < all.length && openCalls(r).some((c) => c.content.stageId === s.stageId); +} + +// --------------------------------------------------------------------------- +// 第一次启动,打中就杀 +// --------------------------------------------------------------------------- + +const first = launch(config, studyDir, {}); +await waitForLedger(studyDir, first.child); +const watching = watchTimeline(studyDir, first.child, timeline); +let triggered: LedgerRecord | undefined; +while (first.child.exitCode === null && first.child.signalCode === null) { + const r = readLedger(studyDir); + if (hit(r)) { + first.child.kill("SIGKILL"); + triggered = r.at(-1); + break; + } + await sleep(POLL_MS); +} +const firstEnd = await first.done; +await watching; +if (triggered === undefined) { + writeFileSync(join(out, "outcome.json"), `${JSON.stringify({ launches: [firstEnd] }, null, 2)}\n`); + throw new Error(`没打中:研究进程自己结束了(${JSON.stringify(firstEnd.outcome)}),账本最后一条是 ${JSON.stringify(readLedger(studyDir).at(-1))}`); +} +// 进程已死,此刻的账本就是被杀时的样子。 +const atKill = readLedger(studyDir); +const killSeq = atKill.at(-1)!.seq; +const killedCalls = openCalls(atKill).map((c) => c.content.call); +const stage = predictionStage(atKill)!; +const taskList = tasks(atKill, stage.roster).map((t) => ({ ...t, submittedBeforeKill: accepted(atKill, stage.stageId, t.actor, t.candidate) })); + +// --------------------------------------------------------------------------- +// 再启动,跑完 +// --------------------------------------------------------------------------- + +const second = launch(null, studyDir, {}); +process.on("SIGINT", () => second.child.kill("SIGINT")); +await watchTimeline(studyDir, second.child, timeline); +const secondEnd = await second.done; +writeFileSync(join(out, "outcome.json"), `${JSON.stringify({ launches: [firstEnd, secondEnd] }, null, 2)}\n`); +const r = readLedger(studyDir); + +// 1 被杀那次调用补了结束记录 +const ends = ofKind(r, "callEnded"); +const check1 = killedCalls.map((call) => { + const end = ends.find((e) => e.content.call === call); + return { call, end: end === undefined ? null : { seq: end.seq, ...end.content } }; +}); + +// 2 交过的没重派、没交的重派了。一次调用属于哪个任务,用 materials 的调用编号规则对:同一任务第 n 次发起是 callId(…, n)。 +const startedAfter = ofKind(r, "callStarted").filter((c) => c.seq > killSeq); +const attempts = ofKind(r, "callStarted").length; +const check2 = taskList.map((t) => { + const ids = new Set(Array.from({ length: attempts }, (_, i) => callId(stage.stageId, t.actor, KIND, t.candidate, i + 1))); + const again = startedAfter.filter((c) => ids.has(c.content.call)).map((c) => c.content.call); + return { ...t, dispatchedAfterRestart: again, ok: t.submittedBeforeKill === (again.length === 0) }; +}); + +// 3 没有重复收下:第 1 轮每份提交(阶段、作者、种类、对象、编号)的各版版本号是 1..n;第一次预测里被杀前已收下的,被杀后不能再多出一版 +const round1 = new Set(stages(r).filter((s) => s.round === 1).map((s) => s.stageId)); +const slots = new Map>(); +for (const s of ofKind(r, "submission")) { + const c = s.content; + if (!round1.has(c.stageId)) continue; + const key = JSON.stringify([c.stageId, c.author, c.kind, c.target, c.item]); + slots.set(key, [...(slots.get(key) ?? []), { seq: s.seq, version: c.version }]); +} +const check3 = [...slots].map(([slot, versions]) => { + const numbered = versions.map((v) => v.version).join(",") === versions.map((_, i) => i + 1).join(","); + const [stageId, , kind] = JSON.parse(slot) as [string, string, string]; + const acrossKill = stageId === stage.stageId && kind === KIND && versions.some((v) => v.seq <= killSeq) && versions.some((v) => v.seq > killSeq); + return { slot, versions, replaced: versions.length > 1, ok: numbered && !acrossKill }; +}); + +// 4 阶段照常关闭、往下走到选批后停下 +const closed = ofKind(r, "stageClose").some((c) => c.content.stageId === stage.stageId); +const batch = ofKind(r, "decision").some((d) => d.content.decision === "batch" && d.content.round === 1); +const lastHalt = ofKind(r, "event").filter((e) => e.content.event === HALTED).at(-1); + +const checks = { + "1 被杀那次调用补了结束记录(出错、进程中断)": { + pass: check1.length > 0 && check1.every((c) => c.end !== null && c.end.status === "failed" && c.end.reason.includes("进程中断")), + calls: check1, + }, + "2 交过的任务没被重派,没交的被重派": { pass: check2.every((t) => t.ok), tasks: check2 }, + "3 没有重复收下的提交(替换除外)": { pass: check3.every((s) => s.ok), problems: check3.filter((s) => !s.ok), replaced: check3.filter((s) => s.replaced) }, + "4 阶段照常关闭,往下走到选批后停下": { + pass: closed && batch && lastHalt !== undefined && lastHalt.content.reason.includes("实现还没写"), + firstPredictionClosed: closed, + batchDecision: batch, + lastHalt: lastHalt?.content ?? null, + }, +}; +const report = { triggeredAt: { seq: triggered.seq, kind: triggered.kind }, killSeq, lastRecordAtKill: atKill.at(-1), tasksAtKill: taskList, outcomeAfterRestart: secondEnd.outcome, checks }; +writeFileSync(join(out, "kill-report.json"), `${JSON.stringify(report, null, 2)}\n`); +console.log(`\n被杀时账本最后一条 #${killSeq} ${atKill.at(-1)!.kind};被杀时正在进行的调用:${killedCalls.join("、") || "无"}`); +console.log(`被杀时第一次预测:${taskList.filter((t) => t.submittedBeforeKill).length}/${taskList.length} 个任务已交`); +console.log(`再启动后的结局:${JSON.stringify(secondEnd.outcome)}`); +for (const [k, v] of Object.entries(checks)) console.log(`${v.pass ? "通过" : "不通过"} ${k}`); +console.log(`报告:${join(out, "kill-report.json")}`); diff --git a/eval/round-one/neutral.ts b/eval/round-one/neutral.ts new file mode 100644 index 0000000..6cd89a5 --- /dev/null +++ b/eval/round-one/neutral.ts @@ -0,0 +1,97 @@ +import type { Json } from "./round-data.ts"; + +// 给审阅者的文本去掉来历(任务书 14 第 3.2 节第 1 步):运行名、模型名、研究编号去掉; +// 研究者、提案、候选、挑刺、入口的编号改写成"研究者 1""提案 1"这类中性写法,按出现顺序编,一份文本一张映射表。 +// 两个项目的编号写法都认:我们的 r1-researcher-2-1、r1-c1、e-researcher-1-4,参考项目的 r1-v1-p-researcher-2-1、r1-v1-c1、 +// researcher-1-e4、挑刺的事实编号(uuid)。字段名(points 对 factors)不改。 + +type Family = "研究者" | "提案" | "候选" | "挑刺" | "入口"; + +/** 认编号的写法,按先后试:长的、带别的编号在里面的写法放前面。 */ +const PATTERNS: Array<[Family | "实验员" | "核验员", string]> = [ + ["挑刺", String.raw`r\d+-researcher-\d+-\d+\.researcher-\d+\.\d+`], + ["提案", String.raw`r\d+-v\d+-p-researcher-\d+-\d+`], + ["提案", String.raw`r\d+-researcher-\d+-\d+`], + ["实验员", String.raw`implementer-[A-Za-z0-9-]+`], + ["核验员", String.raw`verifier-[A-Za-z0-9-]+`], + ["入口", String.raw`e-researcher-\d+-\d+`], + ["入口", String.raw`researcher-\d+-e\d+`], + ["候选", String.raw`r\d+-v\d+-c\d+`], + ["候选", String.raw`r\d+-c\d+`], + ["研究者", String.raw`researcher-\d+`], + ["挑刺", String.raw`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`], +]; +const ID_RE = new RegExp(PATTERNS.map(([, p]) => `(? = [ + [/(?(); + private readonly counts = new Map(); + private readonly hidden: RegExp | null; + + constructor(hiddenNames: string[]) { + const esc = hiddenNames.map((n) => n.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")); + this.hidden = esc.length === 0 ? null : new RegExp(esc.sort((a, b) => b.length - a.length).join("|"), "g"); + } + + /** 一个编号的中性名:第一次见到时按这一类的出现顺序编号。 */ + name(family: Family | "实验员" | "核验员", raw: string): string { + const seen = this.names.get(raw); + if (seen !== undefined) return seen; + const n = (this.counts.get(family) ?? 0) + 1; + this.counts.set(family, n); + const named = `${family} ${n}`; + this.names.set(raw, named); + return named; + } + + /** 一段文字里的编号和来历都换掉。研究编号、运行名先换:我们的研究编号里可能带着像编号的片段。 */ + text(s: string): string { + let t = this.hidden === null ? s : s.replace(this.hidden, "某次运行"); + t = t.replace(ID_RE, (match, ...groups: unknown[]) => { + const i = groups.findIndex((g, k) => k < PATTERNS.length && g !== undefined); + return this.name(PATTERNS[i]![0], match); + }); + for (const [re, to] of SCRUB) t = t.replace(re, to); + return t; + } + + /** 映射表(原编号 → 中性名),存进 data 旁边,审阅者看不到。 */ + table(): Record { + return Object.fromEntries(this.names); + } +} + +/** 渲染时去掉的键:只说明来历的哈希、请求编号、轮次编号(编号类的值照常经 text 改写成中性名)。 */ +const DROP = new Set(["baselineCodeHash", "materialHash", "questionHash", "requestId", "roundId", "protocolId"]); + +/** 把 Json 写成缩进的纯文本:对象一键一行,多行文字缩进成块,数组逐项编号。 */ +export function render(value: Json, nt: Neutral, indent = ""): string { + if (value === null) return `${indent}(空)`; + if (typeof value !== "object") return block(nt.text(String(value)), indent); + if (Array.isArray(value)) { + if (value.length === 0) return `${indent}(无)`; + return value.map((v, i) => (isScalar(v) ? `${indent}${i + 1}. ${inline(v, nt)}` : `${indent}${i + 1}.\n${render(v, nt, `${indent} `)}`)).join("\n"); + } + const entries = Object.entries(value).filter(([k]) => !DROP.has(k)); + if (entries.length === 0) return `${indent}(无)`; + return entries + .map(([k, v]) => (isScalar(v) && !String(v).includes("\n") ? `${indent}${k}: ${inline(v, nt)}` : `${indent}${k}:\n${render(v, nt, `${indent} `)}`)) + .join("\n"); +} + +const isScalar = (v: Json) => v === null || typeof v !== "object"; +const inline = (v: Json, nt: Neutral) => (v === null ? "(空)" : nt.text(String(v))); +const block = (s: string, indent: string) => + s + .split("\n") + .map((l) => `${indent}${l}`) + .join("\n"); diff --git a/eval/round-one/path.md b/eval/round-one/path.md new file mode 100644 index 0000000..7b76553 --- /dev/null +++ b/eval/round-one/path.md @@ -0,0 +1,3 @@ +# 研究路径说明 + +26 圆填圆,基线 0.9598(圆心按同心环写死、半径逐对缩放),已知最好 2.635983。同类系统和参考项目的高分运行,大多沿着这样一条路往上走:结构化的起点或多起点 → 把圆心和半径一起交给连续优化器 → 在当前最好解附近加扰动再优化,或退火式接受 → 最后修到严格合法。辅助手段有:固定圆心用线性规划求最优半径、只动贴紧的那几个圆。死路有:只做几何构造(圆心按手排的规则摆、不移动),各系统都卡在 2.2–2.54;只改半径的分法、不动圆心,上限受布局限制。这是常见的一条路,不是唯一正确的路:一条清单上没有、但讲得通的路,同样算往前走了。第 1 轮能走多远取决于起点,参考项目第 1 轮采纳的分数在 1.55 到 2.632 之间。 diff --git a/eval/round-one/questions.ts b/eval/round-one/questions.ts new file mode 100644 index 0000000..b0818b1 --- /dev/null +++ b/eval/round-one/questions.ts @@ -0,0 +1,146 @@ +import type { Noul } from "../lib/jev.ts"; + +// 第 1 轮检验问 Jev 的问题(任务书 14 第 3.2 节第 3 步):英文、一问只问一件事、带材料说明、判"是"判"否"的说明和反例, +// 写法照 src/jev/questions.ts。这些是检验自己的问题,不进 src/jev;阈值没有校准,一律以概率大于 0.5 判"是"。 +// 提案、挑刺、回应原文是中文,原样放进材料;两个项目的字段名不同(factors 对 points、modification),材料说明里写明。 + +const STUDY = + "One round of a study that improves a program for the circle packing task: place 26 non-overlapping circles inside the unit square " + + "so that the sum of their radii is as large as possible (no tolerance: any overlap or circle outside the square is invalid). " + + "The baseline program puts the circles on fixed rings around the center and then shrinks radii pair by pair to remove overlaps; " + + "it scores about 0.96 and the best known score is about 2.636. " + + "In this round each researcher proposed one code change. A proposal's change is given as a list of separate edits " + + "(each with a `name`, a `method` saying what to change and how, and sometimes `keySettings`), or as one `modification` text. " + + "The texts are mostly in Chinese."; + +export const TASKS = { + pair: `${STUDY} \`proposal_a\` and \`proposal_b\` are the changes of two proposals from the same round.`, + idea: `${STUDY} \`proposal\` is the change of one proposal. \`idea\` is a technique from the literature on this task, described in English.`, + critique: + `${STUDY} After the proposals were written, other researchers wrote critiques of them, and each author replied to every critique ` + + "and could revise the proposal before it was frozen as the protocol an implementer must follow. " + + "`critique` is one critique of a proposal; `reply` is the author's reply to that critique; " + + "`revision` is the rest of the author's response: the changes made to the proposal and the final text, in whatever fields that system used.", +}; + +/** a. 两条提案是不是同一个改法(照方法.md 4.3 ③ 判重的意思:只差参数的不算同一个改法)。 */ +export const SAME_CHANGE: Record = { + same_change: { + type: "noul", + instructions: { + question: "Would carrying out `proposal_a` and carrying out `proposal_b` produce the same code change, so that a single implementation would serve as both?", + focus: + "Compare what each change does to the program: which parts it changes and by what method, reading every edit. " + + "Ignore wording, language, level of detail, names, and how the change is split into edits. " + + "The same method with only different settings (step sizes, counts, time budgets, layouts) is not the same change, because each setting needs its own run.", + }, + criteria: { + true: { + what: "Both describe the same algorithmic change to the same part of the program with the same settings; they differ only in wording, detail or how the change is split", + examples: [ + "a: 把圆心和半径一起交给约束优化器,从当前布局出发; b: optimize centers and radii jointly with SLSQP starting from the current layout", + "a: 半径改用线性规划求解; b: 固定圆心,用 linprog 求半径和最大", + ], + }, + false: { + what: "They change different parts of the program, change the same part by a different method, or use the same method with different settings", + not_for: "Two changes that only share the goal of a higher score, or only edit the same function in different ways", + examples: [ + "a: 半径改用线性规划求解; b: 把圆心和半径一起交给约束优化器", + "a: 6 行六角网格; b: 5 行六角网格", + "a: 多起点重启 20 次; b: 多起点重启 200 次", + ], + }, + }, + }, +}; + +/** b. 作者的回应有没有处理这条挑刺:改了协议,或者给了不改的具体理由。 */ +export const ADDRESSED: Record = { + critique_addressed: { + type: "noul", + instructions: { + question: + "Does `reply` deal with the concern raised in `critique`, either by a concrete change to the proposal (visible in `reply` or `revision`) or by a specific reason why no change is needed?", + focus: + "First find the concern in `critique`. Then check whether `reply` engages with that concern: says what was changed and how, or argues specifically why the concern does not apply. " + + "Agreeing without saying what changed, promising to consider it, or answering a different point does not deal with the concern. " + + "Whether the reply is right is not the question; only whether it deals with the concern.", + }, + criteria: { + true: { + what: "The reply states a concrete change that answers the concern, or gives a specific reason tied to the concern for not changing", + examples: [ + "critique: 停止条件没写锚点; reply: 采纳,t0 改为入口处取 perf_counter,并写进协议的停止条件一节", + "critique: 这个因子会改变评分; reply: 不改:评分只读 solution.json,这个因子只改内部的起点,具体见协议第 2 节", + ], + }, + false: { + what: "The reply ignores the concern, only agrees or thanks, promises to look later, or answers something else", + not_for: "Replies that disagree with the critique but give a specific reason tied to its concern", + examples: ["critique: 停止条件没写锚点; reply: 感谢意见,同意", "critique: 两个因子分不开; reply: 我们会在实现时注意"], + }, + }, + }, +}; + +/** c. 提案对研究路径上的关键想法(K1–K5):这条提案的改动是否做了这个想法。 */ +export const CARRIES_IDEA: Record = { + carries_idea: { + type: "noul", + instructions: { + question: "Does the code change in `proposal` carry out the technique described in `idea.description`, as the whole change or as one of its parts?", + focus: + "Read every edit of the proposal. Answer yes if one of the edits, or the change as a whole, does what the technique describes, possibly in other words or in Chinese. " + + "Mentioning the technique as a later plan, or sharing only the goal, is not enough.", + }, + criteria: { + true: { + what: "The proposal's change includes this technique", + examples: [ + "idea: optimize centers and radii together with a constrained optimizer; proposal: 把 78 个变量交给 SLSQP,约束是在正方形内、两两不重叠", + "idea: solve a linear program for the radii with fixed centers; proposal: 半径分配改成 linprog 求半径和最大", + ], + }, + false: { + what: "The proposal's change does not include this technique", + not_for: "Proposals that only mention the technique as future work, or only share the goal", + examples: [ + "idea: optimize centers and radii together with a constrained optimizer; proposal: 圆心按六角网格摆,半径用贪心缩放", + "idea: solve a linear program for the radii with fixed centers; proposal: 从当前最好解加扰动再跑 SLSQP", + ], + }, + }, + }, +}; + +/** c. 提案对死路(D1、D2):这条提案的改动是不是整个都停在这条死路里。 */ +export const STAYS_IN_DEAD_END: Record = { + stays_within_idea: { + type: "noul", + instructions: { + question: "Does the whole code change in `proposal` stay within the approach described in `idea.description`, with no part that goes beyond it?", + focus: + "Answer yes only if everything the proposal changes falls inside this approach. " + + "If any edit uses a continuous optimization that moves the circle centers, or uses the approach only as a helper step " + + "(a starting layout, or a way to score layouts inside a larger search), answer no.", + }, + criteria: { + true: { + what: "Everything the proposal changes is inside this approach", + examples: [ + "idea: hand-designed placement of centers only; proposal: 换成 6 行六角网格,枚举行数取最好,半径用贪心", + "idea: change only the radii with fixed centers; proposal: 圆心不动,半径改用线性规划", + ], + }, + false: { + what: "Some part of the proposal goes beyond this approach, or the approach is only one step of a larger change", + not_for: "Proposals that use this approach as a starting layout or a scoring helper inside a search that moves the centers", + examples: [ + "idea: hand-designed placement of centers only; proposal: 从六角起点出发,再用 SLSQP 同时移动圆心和半径", + "idea: change only the radii with fixed centers; proposal: 半径用线性规划打分,外层搜索圆心位置", + ], + }, + }, + }, +}; diff --git a/eval/round-one/round-data.ts b/eval/round-one/round-data.ts new file mode 100644 index 0000000..b7bc32e --- /dev/null +++ b/eval/round-one/round-data.ts @@ -0,0 +1,352 @@ +import { copyFileSync, mkdirSync, statSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import type { LedgerRecord } from "../../src/ledger/index.ts"; +import type { DedupOutput, FrozenProtocol } from "../../src/materials/index.ts"; +import { finalSubmissions, ofKind, stages } from "../lib/ledger.ts"; + +// 一次运行第 1 轮(开轮到选批)的记录,读成同一种形状,导出文本和 Jev 检验都用它(任务书 14 第 3.2 节)。 +// 两种来源:我们的账本(经 src/ledger 读,像用户一样)和参考项目的账本(只读,先复制到输出目录再打开)。 +// 内容按原样存 Json,字段名不统一(参考项目叫 points,我们叫 factors),只把检验要用的几样取到固定的键上。 +// 按名字读,读不到就报错停下:flow 第 2 部分还在写,读错了要立刻看出来,不猜。 + +export type Json = null | boolean | number | string | Json[] | { [key: string]: Json }; +export type JsonObject = { [key: string]: Json }; +export interface Range3 { + q1: number; + q5: number; + q9: number; +} +export interface Prediction { + researcher: string; + candidate: string; + range: Range3; + /** 预测附的理由(我们的 body、参考项目 14 的 rationale);没有就是 null。 */ + note: Json; +} + +export interface RoundData { + /** 来历(运行名、库路径),只进 data.json,不进给审阅者的文本。 */ + source: string; + /** 要从文本里去掉的名字:运行名、研究编号。 */ + hiddenNames: string[]; + proposals: Array<{ + id: string; + author: string; + content: JsonObject; + /** 判重看的"改了什么":我们是因子的名称、做法、关键设定;参考项目是 points 或 modification。 */ + change: Json; + authorEstimate: Range3 | null; + }>; + dedup: { + /** 本轮新候选各合并了哪几条原提案。 */ + groups: Array<{ candidate: string; proposals: string[] }>; + /** 并进留池候选的新提案。 */ + carried: Json; + /** 程序判重时 Jev 每一对的判断,原样。 */ + judgments: Json; + }; + critiques: Array<{ id: string; author: string; proposal: string; content: JsonObject }>; + responses: Array<{ + candidate: string; + author: string; + kind: string; + replies: Array<{ critique: string; text: string }>; + /** 回应里逐条答复以外的部分(改动、实现条款、正文)。 */ + rest: JsonObject; + }>; + protocols: Array<{ candidate: string; content: JsonObject }>; + /** null:这次运行没有第一次预测这个阶段。 */ + firstPrediction: Prediction[] | null; + /** null:这次运行用投票,不是范围预测,投票原样在 votes 里。 */ + secondPrediction: Prediction[] | null; + votes: Json; + batch: { chosen: string[]; detail: Json }; + rejections: Array<{ stage: string; author: string; target: string | null; reason: string }>; +} + +/** 中性的阶段名,两种来源共用。 */ +export const STAGE_NAMES = { + proposal: "提案", + firstPrediction: "第一次预测", + critique: "挑刺", + response: "回应与统一", + secondPrediction: "第二次预测", + vote: "投票", +} as const; +type StageName = keyof typeof STAGE_NAMES; + +function need(value: T | undefined, what: string): T { + if (value === undefined) throw new Error(`读不到:${what}`); + return value; +} +function field(obj: JsonObject, key: string, where: string): Json { + return need(obj[key], `${where} 的字段 ${key}`); +} +const omit = (obj: JsonObject, keys: string[]): JsonObject => Object.fromEntries(Object.entries(obj).filter(([k]) => !keys.includes(k))); +const asObject = (v: unknown) => v as JsonObject; + +// --------------------------------------------------------------------------- +// 我们的账本 +// --------------------------------------------------------------------------- + +/** 我们的研究第 round 轮。records 是 readLedger 读出的全部记录,studyId 是研究编号(要从文本里去掉)。 */ +export function readOurs(records: LedgerRecord[], round: number, source: string, studyId: string): RoundData { + const inRound = stages(records).filter((s) => s.round === round); + const stageOf = (name: string) => need(inRound.find((s) => s.stage === name), `第 ${round} 轮 ${name} 阶段的开记录`).stageId; + const stageName = new Map(inRound.map((s) => [s.stageId, s.stage])); + const finals = finalSubmissions(records); + const subs = (stage: string, kinds: string[]) => { + const id = stageOf(stage); + return finals.filter((s) => s.content.stageId === id && kinds.includes(s.content.kind)); + }; + const seals = ofKind(records, "seal"); + + const proposals = subs("proposal", ["proposal"]).map((s) => { + const content = asObject(s.content.content); + const factors = field(content, "factors", `提案 ${s.content.item}`) as JsonObject[]; + const seal = need(seals.find((x) => x.content.proposal === s.seq), `提案 ${s.content.item}(序号 ${s.seq})的作者自评封存`); + return { + id: need(s.content.item ?? undefined, `提案序号 ${s.seq} 的编号 item`), + author: s.content.author, + content, + change: factors.map((f) => ({ name: f["name"]!, method: f["method"]!, keySettings: f["keySettings"]! })), + authorEstimate: seal.content.range, + }; + }); + + const decision = (kind: string) => + need(ofKind(records, "decision").find((d) => d.content.decision === kind && d.content.round === round), `第 ${round} 轮的 ${kind} 决定`).content; + const dedup = decision("dedup"); + const dedupOut = dedup.output as unknown as DedupOutput; + need(dedupOut.fresh, "判重决定输出的 fresh"); + need(dedupOut.carried, "判重决定输出的 carried"); + + const critiques = subs("critique", ["critique"]).map((s) => ({ + id: need(s.content.item ?? undefined, `挑刺序号 ${s.seq} 的编号 item`), + author: s.content.author, + proposal: need(s.content.target ?? undefined, `挑刺序号 ${s.seq} 的对象 target`), + content: asObject(s.content.content), + })); + + const responses = subs("response", ["response", "unification"]).map((s) => { + const content = asObject(s.content.content); + const replies = field(content, "replies", `回应序号 ${s.seq}`) as JsonObject[]; + return { + candidate: need(s.content.target ?? undefined, `回应序号 ${s.seq} 的对象 target`), + author: s.content.author, + kind: s.content.kind, + replies: replies.map((r) => ({ critique: r["critique"] as string, text: r["reply"] as string })), + rest: omit(content, ["replies"]), + }; + }); + + const fresh = new Set(dedupOut.fresh.map((g) => g.candidate)); + const protocols = ofKind(records, "protocol") + .filter((p) => fresh.has(p.content.candidate) && (p.content.protocol as unknown as FrozenProtocol).round === round) + .map((p) => ({ candidate: p.content.candidate, content: asObject(p.content.protocol) })); + + const predictions = (stage: string): Prediction[] => + subs(stage, [stage]).map((s) => { + const content = asObject(s.content.content); + return { + researcher: s.content.author, + candidate: need(s.content.target ?? undefined, `${stage} 序号 ${s.seq} 的对象 target`), + range: field(content, "range", `${stage} 序号 ${s.seq}`) as unknown as Range3, + note: content["body"] ?? null, + }; + }); + + const batch = decision("batch"); + const chosen = need(asObject(batch.output)["chosen"], "选批决定输出的 chosen") as string[]; + + const rejections = ofKind(records, "rejection") + .filter((r) => stageName.has(r.content.stageId)) + .map((r) => ({ + stage: STAGE_NAMES[need(ourStage(stageName.get(r.content.stageId)!), `阶段 ${r.content.stageId} 的中性名`)], + author: r.content.author, + target: r.content.target, + reason: r.content.reason, + })); + + return { + source, + hiddenNames: [studyId], + proposals, + dedup: { groups: dedupOut.fresh, carried: dedupOut.carried as unknown as Json, judgments: dedup.input as Json }, + critiques, + responses, + protocols, + firstPrediction: predictions("firstPrediction"), + secondPrediction: predictions("secondPrediction"), + votes: null, + batch: { chosen, detail: batch.output as Json }, + rejections, + }; +} + +function ourStage(stage: string): StageName | undefined { + return stage in STAGE_NAMES ? (stage as StageName) : undefined; +} + +// --------------------------------------------------------------------------- +// 参考项目的账本(/home/iomgaa/rb-acceptance/<运行名>/board.sqlite) +// --------------------------------------------------------------------------- + +/** 参考项目的阶段名(stage 编号的最后一段)→ 中性名。挑刺和回应都在 phase review 里,按编号区分。 */ +const REF_STAGES: Record = { + proposal: "proposal", + "predict-blind": "firstPrediction", + critique: "critique", + response: "response", + predict: "secondPrediction", + vote: "vote", +}; + +/** 原库只读:库文件(和非空的 -wal)先复制到 copyDir,再以只读方式打开复制件,原目录里不写任何东西。 */ +function openCopy(board: string, copyDir: string): DatabaseSync { + mkdirSync(copyDir, { recursive: true }); + const copy = join(copyDir, "board.sqlite"); + copyFileSync(board, copy); + if (statSync(`${board}-wal`, { throwIfNoEntry: false })?.size) copyFileSync(`${board}-wal`, `${copy}-wal`); + return new DatabaseSync(copy, { readOnly: true }); +} + +const refRange = (r: JsonObject): Range3 => ({ q1: r["q10"] as number, q5: r["q50"] as number, q9: r["q90"] as number }); + +export function readReference(board: string, copyDir: string): RoundData { + const run = basename(dirname(board)); + const roundId = `${run}.r1`; + const db = openCopy(board, copyDir); + try { + const all = (sql: string, ...args: Array) => db.prepare(sql).all(...args) as T[]; + const stageRows = all<{ id: string }>("SELECT id FROM stages WHERE round_id = ? ORDER BY created_seq", roundId); + const stageOf = new Map(); + for (const s of stageRows) { + const name = REF_STAGES[s.id.split(".").at(-1)!]; + if (name !== undefined) { + if (stageOf.has(name)) throw new Error(`${board}:第 1 轮 ${name} 阶段不止一个(${stageOf.get(name)}、${s.id})`); + stageOf.set(name, s.id); + } + } + // 14、12 那两版的 submissions 表还没有 version 列(一人一个 slot 只有一份),按第 1 版读。 + const versioned = all<{ name: string }>("PRAGMA table_info(submissions)").some((c) => c.name === "version"); + /** 每份提交(阶段、人、slot)的最后一版,按第一次出现的先后排。 */ + const subs = (name: StageName) => { + const stage = need(stageOf.get(name), `${board} 第 1 轮 ${name} 阶段`); + const rows = all<{ actor: string; slot: string; version: number; payload: string }>( + `SELECT actor, slot, ${versioned ? "version" : "1 AS version"}, payload FROM submissions WHERE stage_id = ? ORDER BY created_seq`, + stage, + ); + const latest = new Map(); + for (const r of rows) { + const key = `${r.actor}\n${r.slot}`; + const seen = latest.get(key); + if (seen === undefined || seen.version < r.version) latest.set(key, { stage, actor: r.actor, slot: r.slot, version: r.version, payload: JSON.parse(r.payload) }); + } + return [...latest.values()]; + }; + const fact = (kind: string) => { + const rows = all<{ payload: string }>( + "SELECT payload FROM facts WHERE kind = ? AND id LIKE ? ORDER BY created_seq", + kind, + `${run}.${kind}.${roundId}.%`, + ); + return JSON.parse(need(rows.at(-1), `${board} 第 1 轮的 ${kind}`).payload) as JsonObject; + }; + + const proposals = subs("proposal").map((s) => { + const p = s.payload; + const points = p["points"] as JsonObject[] | undefined; + const change: Json = + points !== undefined + ? points.map((x) => ({ name: x["name"]!, method: x["method"]!, ...(x["keySettings"] === undefined ? {} : { keySettings: x["keySettings"] }) })) + : { modification: field(p, "modification", `${board} 提案 ${s.slot}`) }; + const estimate = (p["authorRange"] ?? p["range"]) as JsonObject | undefined; + return { + id: s.slot, + author: s.actor, + content: omit(p, ["authorId", "id", "roundId", "authorRange", "range"]), + change, + authorEstimate: estimate === undefined ? null : refRange(estimate), + }; + }); + + const dedup = fact("dedup-record"); + const groups = (field(dedup, "candidates", `${board} dedup-record`) as JsonObject[]).map((c) => ({ + candidate: c["candidateId"] as string, + proposals: c["proposalIds"] as string[], + })); + + // 一份挑刺提交(一人对一条原提案)是一条挑刺,编号是它的 review-submission 事实编号:回应里的 critiqueId 指的就是它。 + const critiqueStage = need(stageOf.get("critique"), `${board} 第 1 轮挑刺阶段`); + const critiqueIds = new Map( + all<{ id: string; slot: string }>("SELECT id, slot FROM facts WHERE kind = 'review-submission' AND slot LIKE ?", `${critiqueStage}/%`).map((f) => [f.slot, f.id]), + ); + const critiques = subs("critique").map((s) => ({ + id: need(critiqueIds.get(`${s.stage}/${s.actor}/${s.slot}`), `${board} 挑刺 ${s.actor}/${s.slot} 的事实编号`), + author: s.actor, + proposal: s.slot, + content: omit(s.payload, ["authorId", "proposalId", "roundId"]), + })); + + const responses = subs("response").map((s) => ({ + candidate: s.slot, + author: s.actor, + kind: "response", + replies: (field(s.payload, "critiqueResponses", `${board} 回应 ${s.slot}`) as JsonObject[]).map((r) => ({ + critique: r["critiqueId"] as string, + text: r["response"] as string, + })), + rest: omit(s.payload, ["authorId", "candidateId", "roundId", "critiqueResponses"]), + })); + + const protocols = all<{ payload: string }>( + "SELECT payload FROM facts WHERE kind = 'candidate-protocol' AND id LIKE ? ORDER BY created_seq", + `${run}.candidate-protocol.${roundId}.%`, + ).map((f) => { + const p = JSON.parse(f.payload) as JsonObject; + return { candidate: field(p, "candidateId", `${board} candidate-protocol`) as string, content: omit(p, ["author", "candidateId", "baselineCodeHash"]) }; + }); + + const predictions = (name: StageName): Prediction[] => + subs(name).map((s) => ({ + researcher: s.actor, + candidate: s.slot, + range: refRange(field(s.payload, "range", `${board} ${name} ${s.actor}/${s.slot}`) as JsonObject), + note: s.payload["rationale"] ?? null, + })); + const voting = stageOf.has("vote"); + + const batch = voting ? fact("vote-tally") : fact("batch-selection"); + const chosen = (field(batch, "candidates", `${board} 选批`) as JsonObject[]).filter((c) => c["selected"] === true).map((c) => c["candidateId"] as string); + + // 退回:audit 里的 submission-rejected,taskId 是"<轮编号>.v1.<阶段>.<人>.<对象>",只取开轮到选批这几个阶段的。 + const rejections = all<{ payload: string }>("SELECT payload FROM audit WHERE event = 'submission-rejected' ORDER BY seq").flatMap((a) => { + const p = JSON.parse(a.payload) as JsonObject; + const task = p["taskId"] as string; + const m = new RegExp(`^${roundId.replace(/\./g, "\\.")}\\.v\\d+\\.([a-z-]+)\\.([^.]+)\\.(.+)$`).exec(task); + const stage = m === null ? undefined : REF_STAGES[m[1]!]; + if (stage === undefined) return []; + const target = m![3]!; + return [{ stage: STAGE_NAMES[stage], author: m![2]!, target: target === m![2] ? null : target, reason: p["reason"] as string }]; + }); + + return { + source: board, + hiddenNames: [run], + proposals, + dedup: { groups, carried: dedup["pooled"] ?? null, judgments: dedup["pairs"] ?? null }, + critiques, + responses, + protocols, + firstPrediction: stageOf.has("firstPrediction") ? predictions("firstPrediction") : null, + secondPrediction: voting ? null : predictions("secondPrediction"), + votes: voting ? subs("vote").map((s) => ({ researcher: s.actor, candidate: s.slot, ...omit(s.payload, ["voterId", "candidateId", "protocolId"]) })) : null, + batch: { chosen, detail: batch }, + rejections, + }; + } finally { + db.close(); + } +} diff --git a/eval/round-one/run.ts b/eval/round-one/run.ts new file mode 100644 index 0000000..3b56ec7 --- /dev/null +++ b/eval/round-one/run.ts @@ -0,0 +1,23 @@ +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { launch, newEvalDir, waitForLedger, writeConfig } from "../lib/study.ts"; +import { watchTimeline } from "../lib/timeline.ts"; + +// 用真模型、真 Docker、真 Jev 从头跑一份研究,直到它自己停下(任务书 14 第 3.1 节;预期停在第 1 轮选批之后, +// 原因"实现还没写")。配置照 eval/round-zero/research.json,研究编号换成检验名字。要花钱。 +// node eval/round-one/run.ts <名字> +// 产物:<检验目录>/study(研究目录)、timeline.jsonl(每条记录第一次被看到的时刻,stats 用来算墙钟)、outcome.json(结局)。 +// 中途按 Ctrl-C 会把停下信号传给研究进程;接着跑用 node eval/lib/cli.ts --resume --dir <检验目录>/study。 + +const name = process.argv[2]; +if (name === undefined) throw new Error("要写检验名字,例如 r1-real-1(只许小写字母、数字和 ._-,它也是研究编号)"); +const out = newEvalDir(name); +const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, null); +const studyDir = join(out, "study"); +const study = launch(config, studyDir, {}); +process.on("SIGINT", () => study.child.kill("SIGINT")); +await waitForLedger(studyDir, study.child); +await watchTimeline(studyDir, study.child, join(out, "timeline.jsonl")); +const { code, signal, outcome } = await study.done; +writeFileSync(join(out, "outcome.json"), `${JSON.stringify({ launches: [{ code, signal, outcome }] }, null, 2)}\n`); +console.log(`\n结局:${JSON.stringify(outcome)}\n研究目录:${studyDir}`); diff --git a/eval/round-one/stats.ts b/eval/round-one/stats.ts new file mode 100644 index 0000000..4d1e2d5 --- /dev/null +++ b/eval/round-one/stats.ts @@ -0,0 +1,172 @@ +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { FrozenProtocol } from "../../src/materials/index.ts"; +import { finalSubmissions, ofKind, readLedger, stages } from "../lib/ledger.ts"; +import { readTimeline } from "../lib/timeline.ts"; + +// 第 1 轮的数字和过程(任务书 14 第 3.2 节第 4 步),只看我们的运行;另查程序部分(第 3.4 节): +// 阶段开关成对、冻结协议写在回应阶段关闭之后、最后停下的原因是"实现还没写"。不花钱,只读账本和 timeline.jsonl。 +// node eval/round-one/stats.ts <检验目录>...(run.ts、kill.ts 写出的目录:study/、timeline.jsonl) +// 每个目录写一份 stats.json,并打印摘要。 + +/** 已知最好和基线分数(任务书 14 第 3.2 节第 4 步给的数),预测超出这两头的逐条列出。 */ +const BEST_KNOWN = 2.635983; +const BASELINE = 0.9598; +/** flow 写停下事件用的事件名(src/flow/next-step.ts 的 EVENTS.halted;flow 只导出 runStudy,这里照抄)。 */ +const HALTED = "halted"; +const EXPECTED_STOP = "实现还没写"; + +const dirs = process.argv.slice(2); +if (dirs.length === 0) throw new Error("要写一个或几个检验目录(run.ts、kill.ts 写出的)"); + +type Range3 = { q1: number; q5: number; q9: number }; +const measured = (m: { status: string; value?: number }) => (m.status === "measured" ? m.value! : null); +const sum = (xs: Array) => ({ total: xs.reduce((a, x) => a + (x ?? 0), 0), unmeasured: xs.filter((x) => x === null).length }); + +function statsOf(dir: string) { + const records = readLedger(join(dir, "study")); + const timelineFile = join(dir, "timeline.jsonl"); + if (!existsSync(timelineFile)) throw new Error(`${timelineFile} 不存在:墙钟时间要靠 run.ts、kill.ts 记下的时刻`); + const seen = new Map(readTimeline(timelineFile).map((r) => [r.seq, r.at])); + const at = (seq: number) => { + const t = seen.get(seq); + if (t === undefined) throw new Error(`timeline.jsonl 里没有序号 ${seq} 的时刻`); + return t; + }; + const allStages = stages(records); + const opens = new Map(ofKind(records, "stageOpen").map((r) => [r.content.stageId, r.seq])); + const closes = new Map(ofKind(records, "stageClose").map((r) => [r.content.stageId, r.seq])); + const calls = ofKind(records, "callStarted"); + const ends = new Map(ofKind(records, "callEnded").map((r) => [r.content.call, r.content])); + const usage = ofKind(records, "callUsage"); + const rejections = ofKind(records, "rejection"); + + // ---- 每个阶段:调用、结局、墙钟、用量、退回 ---- + const perStage = allStages.map((s) => { + const these = calls.filter((c) => c.content.stageId === s.stageId); + const ids = new Set(these.map((c) => c.content.call)); + const endings: Record = {}; + for (const c of these) { + const e = ends.get(c.content.call); + const k = e === undefined ? "没有结束记录" : e.status === "failed" ? `failed:${e.reason}` : e.status; + endings[k] = (endings[k] ?? 0) + 1; + } + const u = usage.filter((x) => ids.has(x.content.call)).map((x) => x.content.usage); + const rej = rejections.filter((r) => r.content.stageId === s.stageId); + const byKind: Record = {}; + for (const r of rej) (byKind[r.content.kind] ??= []).push(`${r.content.author}${r.content.target === null ? "" : ` → ${r.content.target}`}:${r.content.reason}`); + const close = closes.get(s.stageId); + return { + stageId: s.stageId, + round: s.round, + stage: s.stage, + calls: these.length, + endings, + wallClockSeconds: close === undefined ? null : (at(close) - at(opens.get(s.stageId)!)) / 1000, + callWallSeconds: sum(these.map((c) => { + const e = ends.get(c.content.call); + return e === undefined ? null : measured(e.wallSeconds); + })), + tokens: { + input: sum(u.map((x) => measured(x.inputTokens))), + cacheRead: sum(u.map((x) => measured(x.cacheReadTokens))), + cacheWrite: sum(u.map((x) => measured(x.cacheWriteTokens))), + output: sum(u.map((x) => measured(x.outputTokens))), + cost: sum(u.map((x) => measured(x.cost))), + }, + rejections: { count: rej.length, byKind }, + }; + }); + + // ---- 停下、交不出、卡住 ---- + const events = ofKind(records, "event").map((e) => ({ seq: e.seq, ...e.content })); + const notSubmitted = calls.filter((c) => ends.get(c.content.call)?.status === "notSubmitted").map((c) => ({ call: c.content.call, end: ends.get(c.content.call) })); + const openCalls = calls.filter((c) => !ends.has(c.content.call)).map((c) => c.content.call); + const openStages = allStages.filter((s) => !s.closed).map((s) => s.stageId); + const outcome = existsSync(join(dir, "outcome.json")) ? JSON.parse(readFileSync(join(dir, "outcome.json"), "utf8")) : null; + + // ---- 预测 ---- + const finals = finalSubmissions(records); + const round1 = allStages.filter((s) => s.round === 1); + const stageId = (name: string) => { + const s = round1.find((x) => x.stage === name); + if (s === undefined) throw new Error(`第 1 轮没有 ${name} 阶段`); + return s.stageId; + }; + const predictions = (name: string) => + finals + .filter((f) => f.content.stageId === stageId(name) && f.content.kind === name) + .map((f) => ({ researcher: f.content.author, candidate: f.content.target!, range: (f.content.content as { range: Range3 }).range })); + const first = predictions("firstPrediction"); + const second = predictions("secondPrediction"); + const outOfRange = [...first.map((p) => ({ ...p, which: "第一次" })), ...second.map((p) => ({ ...p, which: "第二次" }))].flatMap((p) => + (["q1", "q5", "q9"] as const) + .filter((q) => p.range[q] > BEST_KNOWN || p.range[q] < BASELINE) + .map((q) => ({ which: p.which, researcher: p.researcher, candidate: p.candidate, q, value: p.range[q], side: p.range[q] > BEST_KNOWN ? "超过已知最好" : "低于基线" })), + ); + const changes = first.map((f) => { + const s = second.find((x) => x.researcher === f.researcher && x.candidate === f.candidate); + return { + researcher: f.researcher, + candidate: f.candidate, + first: f.range, + second: s?.range ?? null, + identical: s !== undefined && f.range.q1 === s.range.q1 && f.range.q5 === s.range.q5 && f.range.q9 === s.range.q9, + delta: s === undefined ? null : { q1: s.range.q1 - f.range.q1, q5: s.range.q5 - f.range.q5, q9: s.range.q9 - f.range.q9 }, + }; + }); + + // ---- 程序部分 ---- + const unpaired = [...opens.keys()].filter((id) => !closes.has(id)).concat([...closes.keys()].filter((id) => !opens.has(id))); + const protocols = ofKind(records, "protocol").map((p) => { + const round = (p.content.protocol as unknown as FrozenProtocol).round; + const response = allStages.find((s) => s.round === round && s.stage === "response"); + const close = response === undefined ? undefined : closes.get(response.stageId); + return { candidate: p.content.candidate, round, seq: p.seq, responseCloseSeq: close ?? null, after: close !== undefined && p.seq > close }; + }); + const lastHalt = events.filter((e) => e.event === HALTED).at(-1) ?? null; + const checks = { + "每个阶段开关成对": { pass: unpaired.length === 0, unpaired }, + "冻结协议都写在回应阶段关闭之后": { pass: protocols.length > 0 && protocols.every((p) => p.after), protocols }, + "最后停下原因是实现还没写": { + pass: lastHalt !== null && lastHalt.reason.includes(EXPECTED_STOP) && lastHalt.seq === records.at(-1)!.seq, + lastHalt, + lastRecord: { seq: records.at(-1)!.seq, kind: records.at(-1)!.kind }, + }, + }; + + const times = [...seen.values()]; + return { + study: join(dir, "study"), + totalWallClockSeconds: (Math.max(...times) - Math.min(...times)) / 1000, + perStage, + events, + notSubmitted, + stuck: { openCalls, openStages }, + outcome, + predictions: { bestKnown: BEST_KNOWN, baseline: BASELINE, outOfRange, changes }, + checks, + }; +} + +for (const dir of dirs) { + const s = statsOf(dir); + writeFileSync(join(dir, "stats.json"), `${JSON.stringify(s, null, 2)}\n`); + const f = (x: number) => x.toFixed(0); + console.log(`\n== ${dir}(总墙钟 ${f(s.totalWallClockSeconds)} 秒)`); + for (const p of s.perStage) { + const rej = Object.entries(p.rejections.byKind).map(([k, v]) => `${k} ${v.length}`).join("、") || "无"; + console.log( + `${p.stageId}:调用 ${p.calls}(${Object.entries(p.endings).map(([k, v]) => `${k} ${v}`).join(",")}),墙钟 ${p.wallClockSeconds === null ? "未关闭" : `${f(p.wallClockSeconds)} 秒`},` + + `输入 ${p.tokens.input.total}/输出 ${p.tokens.output.total} token,费用 ${p.tokens.cost.total.toFixed(2)}${p.tokens.cost.unmeasured > 0 ? `(${p.tokens.cost.unmeasured} 条测不到)` : ""},退回 ${rej}`, + ); + } + console.log(`事件:${s.events.map((e) => `#${e.seq} ${e.event}:${e.reason}`).join(";")}`); + console.log(`交不出:${s.notSubmitted.length};卡住:未结束的调用 ${JSON.stringify(s.stuck.openCalls)},未关闭的阶段 ${JSON.stringify(s.stuck.openStages)}`); + console.log(`超出范围的预测:${s.predictions.outOfRange.map((o) => `${o.which} ${o.researcher}→${o.candidate} ${o.q}=${o.value}(${o.side})`).join(";") || "无"}`); + for (const c of s.predictions.changes) { + const r = (x: Range3 | null) => (x === null ? "没有" : `${x.q1}/${x.q5}/${x.q9}`); + console.log(` ${c.researcher}→${c.candidate}:第一次 ${r(c.first)},第二次 ${r(c.second)}${c.identical ? "(完全相同)" : ""}`); + } + for (const [k, v] of Object.entries(s.checks)) console.log(`${v.pass ? "通过" : "不通过"} ${k}`); +} diff --git a/package.json b/package.json index ff9d883..82bb29b 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,13 @@ "eval:r0:extract": "node eval/round-zero/extract-16.ts", "eval:r0:replay": "node eval/round-zero/replay.ts", "eval:r0:kill": "node eval/round-zero/kill.ts", - "eval:r0:concurrent": "node eval/round-zero/concurrent.ts" + "eval:r0:concurrent": "node eval/round-zero/concurrent.ts", + "eval:r1:run": "set -a && . /home/iomgaa/Projects/RivalBits/.env && set +a && node eval/round-one/run.ts", + "eval:r1:kill": "set -a && . /home/iomgaa/Projects/RivalBits/.env && set +a && node eval/round-one/kill.ts", + "eval:r1:export": "node eval/round-one/export.ts", + "eval:r1:blind": "node eval/round-one/blind.ts", + "eval:r1:jevcheck": "set -a && . /home/iomgaa/Projects/RivalBits/.env && set +a && node eval/round-one/jevcheck.ts", + "eval:r1:stats": "node eval/round-one/stats.ts" }, "dependencies": { "@earendil-works/pi-ai": "0.85.1",