eval: 第 1 轮检验脚本(跑、杀进程、导出去来历、盲评编号、Jev 三问、数字与过程)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-03 13:59:54 -04:00
parent f98d664a16
commit a2a1392f01
12 changed files with 1259 additions and 1 deletions
+52
View File
@@ -0,0 +1,52 @@
import { appendFileSync, existsSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import type { ChildProcess } from "node:child_process";
import { sleep } from "./study.ts";
// 账本里没有时间戳,墙钟时间由检验自己记:研究进程跑着的时候每隔一会儿看一眼账本,
// 新出现的每条记录记下"第一次看到它的时刻",写成 timeline.jsonl(每行 {seq, kind, at},at 是毫秒)。
// 只读打开账本、只查序号和种类,不经 Ledger.records()(那会每次重读、重查全部内容)。
// 时刻比真实写入晚最多一个间隔,用来算阶段级的墙钟足够。
const POLL_MS = 1000;
export interface TimelineRow {
seq: number;
kind: string;
at: number;
}
/** 已经记下的最大序号;文件还没有就是 0。接着跑时往同一个文件后面追加。 */
function lastSeq(file: string): number {
if (!existsSync(file)) return 0;
return readTimeline(file).reduce((m, r) => Math.max(m, r.seq), 0);
}
/** 研究进程退出前一直记;进程退出后再看最后一眼。调用前要先等 waitForLedger(表建好了才能读)。 */
export async function watchTimeline(studyDir: string, child: ChildProcess, file: string): Promise<void> {
let seen = lastSeq(file);
const look = () => {
const db = new DatabaseSync(join(studyDir, "ledger.sqlite"), { readOnly: true });
try {
const rows = db.prepare("SELECT seq, kind FROM records WHERE seq > ? ORDER BY seq").all(seen) as Array<{ seq: number; kind: string }>;
const at = Date.now();
for (const r of rows) appendFileSync(file, `${JSON.stringify({ seq: r.seq, kind: r.kind, at })}\n`);
if (rows.length > 0) seen = rows.at(-1)!.seq;
} finally {
db.close();
}
};
while (child.exitCode === null && child.signalCode === null) {
look();
await sleep(POLL_MS);
}
look();
}
export function readTimeline(file: string): TimelineRow[] {
return readFileSync(file, "utf8")
.split("\n")
.filter((l) => l !== "")
.map((l) => JSON.parse(l) as TimelineRow);
}
+30
View File
@@ -0,0 +1,30 @@
import { copyFileSync, existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { randomInt } from "node:crypto";
// 盲评用的汇总(任务书 14 第 3.2 节第 1 步最后一条):把导出目录里每次运行的 text.txt 随机编成"运行 A、B……",
// 只把这些文本放进 <检验目录>/blind/;对照表(字母 → 运行名)另存到 blind 外面的 blind-key.json,审阅者看不到。
// node eval/round-one/blind.ts <export.ts 写出的检验目录>
const out = process.argv[2];
if (out === undefined) throw new Error("要写 export.ts 写出的检验目录");
const runs = readdirSync(join(out, "runs")).sort();
if (runs.length > 26) throw new Error(`有 ${runs.length} 次运行,字母不够用`);
const blind = join(out, "blind");
if (existsSync(blind)) throw new Error(`${blind} 已经有了:重新编号会让旧的对照表作废,先挪走`);
mkdirSync(blind);
// Fisher–Yates 洗牌,用 crypto 的随机数:编号不能从运行名或目录顺序推出来。
const order = [...runs];
for (let i = order.length - 1; i > 0; i--) {
const j = randomInt(i + 1);
[order[i], order[j]] = [order[j]!, order[i]!];
}
const key: Record<string, string> = {};
order.forEach((run, i) => {
const letter = String.fromCharCode(65 + i);
key[`运行 ${letter}`] = run;
copyFileSync(join(out, "runs", run, "text.txt"), join(blind, `运行-${letter}.txt`));
});
writeFileSync(join(out, "blind-key.json"), `${JSON.stringify(key, null, 2)}\n`);
console.log(`${order.length} 份文本 → ${blind}\n对照表 → ${join(out, "blind-key.json")}`);
+126
View File
@@ -0,0 +1,126 @@
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
import { basename, dirname, join } from "node:path";
import type { StudyRecord } from "../../src/materials/index.ts";
import { ofKind, readLedger } from "../lib/ledger.ts";
import { newEvalDir } from "../lib/study.ts";
import { Neutral, render } from "./neutral.ts";
import { readOurs, readReference, type Json, type Prediction, type RoundData } from "./round-data.ts";
// 把几次运行的第 1 轮导出成同一种纯文本(任务书 14 第 3.2 节第 1 步),按固定小标题排,去掉来历。
// node eval/round-one/export.ts <检验名字> <来源>...
// 来源是我们的研究目录(里面有 ledger.sqlite),或参考项目的 board.sqlite(只读,先复制到输出目录)。
// 每次运行写进 <检验目录>/runs/<名字>/:data.json(检验用的中间数据,带原编号)、text.txt(给审阅者的文本)、
// names.json(原编号 → 中性名的映射,审阅者看不到)。之后用 blind.ts 随机编成"运行 A、B……"。
const [name, ...sources] = process.argv.slice(2);
if (name === undefined || sources.length === 0) throw new Error("用法:node eval/round-one/export.ts <检验名字> <研究目录或 board.sqlite>...");
const out = newEvalDir(name);
for (const source of sources) {
const ours = existsSync(join(source, "ledger.sqlite"));
// 名字取上一层目录名:我们的研究目录是 <检验目录>/study,取检验目录名;参考项目取运行名(board.sqlite 所在目录)。
const label = basename(dirname(source));
const dir = join(out, "runs", label);
if (existsSync(dir)) throw new Error(`两个来源的名字都是 ${label},分不开`);
mkdirSync(dir, { recursive: true });
let data: RoundData;
if (ours) {
const records = readLedger(source);
const study = ofKind(records, "config")[0]?.content as StudyRecord | undefined;
if (study === undefined) throw new Error(`${source} 的账本里没有研究配置记录`);
data = readOurs(records, 1, source, (study.config as { studyId: string }).studyId);
} else {
data = readReference(source, join(dir, "board-copy"));
}
const nt = new Neutral(data.hiddenNames);
writeFileSync(join(dir, "data.json"), `${JSON.stringify(data, null, 2)}\n`);
writeFileSync(join(dir, "text.txt"), renderRound(data, nt));
writeFileSync(join(dir, "names.json"), `${JSON.stringify(nt.table(), null, 2)}\n`);
console.log(`${label}:提案 ${data.proposals.length}、候选 ${data.dedup.groups.length}、挑刺 ${data.critiques.length}、退回 ${data.rejections.length} → ${dir}`);
}
console.log(`检验目录:${out}`);
// ---------------------------------------------------------------------------
// 文本
// ---------------------------------------------------------------------------
function renderRound(d: RoundData, nt: Neutral): string {
const out: string[] = ["# 第 1 轮记录(开轮到选批)", ""];
const range = (r: { q1: number; q5: number; q9: number }) => `q1 = ${r.q1},q5 = ${r.q5},q9 = ${r.q9}`;
const candidates = d.dedup.groups.map((g) => g.candidate);
const candidateOf = new Map(d.dedup.groups.flatMap((g) => g.proposals.map((p) => [p, g.candidate] as const)));
// 先按"提案 → 候选 → 挑刺"的顺序给编号起中性名,正文里引用到的编号就和小标题一致。
for (const p of d.proposals) {
nt.name("研究者", p.author);
nt.name("提案", p.id);
}
for (const g of d.dedup.groups) nt.name("候选", g.candidate);
for (const p of d.proposals) for (const c of d.critiques.filter((x) => x.proposal === p.id)) nt.name("挑刺", c.id);
out.push("## 一、提案全文", "");
for (const p of d.proposals) {
out.push(`### ${nt.text(p.id)}(${nt.text(p.author)})`, "");
out.push(`作者自评:${p.authorEstimate === null ? "无" : range(p.authorEstimate)}`, "");
out.push(render(p.content, nt), "");
}
out.push("## 二、判重结果", "");
for (const g of d.dedup.groups) out.push(`- ${nt.text(g.candidate)}:${g.proposals.map((p) => nt.text(p)).join("、")}`);
out.push("", "并进留池候选的:", render(d.dedup.carried, nt), "", "判重时每一对的判断(原样):", render(d.dedup.judgments, nt), "");
out.push("## 三、挑刺和回应", "");
for (const p of d.proposals) {
const these = d.critiques.filter((c) => c.proposal === p.id);
const candidate = candidateOf.get(p.id);
out.push(`### 对 ${nt.text(p.id)} 的挑刺${candidate === undefined ? "" : `(${nt.text(candidate)})`}`, "");
if (these.length === 0) out.push("(没有挑刺)", "");
for (const c of these) {
out.push(`#### ${nt.text(c.id)}(${nt.text(c.author)})`, "", render(c.content, nt), "");
const replies = d.responses.flatMap((r) => r.replies.filter((x) => x.critique === c.id).map((x) => ({ ...x, author: r.author })));
if (replies.length === 0) out.push("作者回应:(没有找到对这条挑刺的回应)", "");
for (const x of replies) out.push(`作者回应(${nt.text(x.author)}):`, render(x.text, nt, " "), "");
}
}
for (const r of d.responses) {
out.push(`### ${nt.text(r.candidate)} 回应里逐条答复以外的部分(${nt.text(r.author)},${r.kind === "unification" ? "统一" : "回应"})`, "");
out.push(render(r.rest, nt), "");
}
out.push("## 四、冻结协议", "");
for (const p of d.protocols) out.push(`### ${nt.text(p.candidate)}`, "", render(p.content, nt), "");
out.push("## 五、两次预测的三个数", "");
const predictions = (title: string, list: Prediction[] | null, missing: string) => {
out.push(`### ${title}`, "");
if (list === null) return out.push(missing, "");
const order = (p: Prediction) => [candidates.indexOf(p.candidate), nt.text(p.researcher)] as const;
const sorted = [...list].sort((a, b) => order(a)[0] - order(b)[0] || order(a)[1].localeCompare(order(b)[1]));
for (const p of sorted) {
out.push(`- ${nt.text(p.candidate)},${nt.text(p.researcher)}:${range(p.range)}`);
if (p.note !== null) out.push(render(p.note as Json, nt, " "));
}
out.push("");
};
predictions("第一次预测", d.firstPrediction, "无此阶段");
predictions("第二次预测", d.secondPrediction, "不适用:这次运行用投票选批,不是范围预测。投票原样如下:");
if (d.votes !== null) out.push(render(d.votes, nt), "");
out.push("## 六、选批结果", "");
out.push(`选中:${d.batch.chosen.length === 0 ? "无" : d.batch.chosen.map((c) => nt.text(c)).join("、")}`, "", "选批记录(原样):", render(d.batch.detail, nt), "");
out.push("## 七、退回的次数和原因", "");
const byStage = new Map<string, number>();
for (const r of d.rejections) byStage.set(r.stage, (byStage.get(r.stage) ?? 0) + 1);
out.push(`共 ${d.rejections.length} 次${byStage.size === 0 ? "" : `:${[...byStage].map(([s, n]) => `${s} ${n} 次`).join(",")}`}`, "");
d.rejections.forEach((r, i) => {
out.push(`${i + 1}. ${r.stage},${nt.text(r.author)}${r.target === null ? "" : `,对象 ${nt.text(r.target)}`}:`, render(clip(r.reason), nt, " "), "");
});
return `${out.join("\n")}\n`;
}
/** 退回原因只留开头 800 字:形状不合格的退回会把整份交上来的内容附在后面,全文在 data.json 里。 */
function clip(s: string): string {
const keep = 800;
return s.length <= keep ? s : `${s.slice(0, keep)}\n(以下省略 ${s.length - keep} 字)`;
}
+97
View File
@@ -0,0 +1,97 @@
import { readdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { ask, THRESHOLD, THRESHOLD_NOTE, type Answer, type Noul } from "../lib/jev.ts";
import { ADDRESSED, CARRIES_IDEA, SAME_CHANGE, STAYS_IN_DEAD_END, TASKS } from "./questions.ts";
import type { RoundData } from "./round-data.ts";
// 第 1 轮的 Jev 封闭问题(任务书 14 第 3.2 节第 3 步),对 export.ts 导出的每次运行都做(我们的和参考项目的),读 data.json:
// a. 判重对不对:本轮提案两两、两个顺序各问一次"是不是同一个改法",取平均,和程序的判重决定对照;
// b. 挑刺有没有被回应:每条挑刺问一次(找不到回应的不问,直接记"没有回应");
// c. 提案对应研究路径的哪一步:每条提案对 K1–K5(做没做这个想法)和 D1、D2(是不是整个停在这条死路)各问一次,只描述。
// 阈值未校准,概率大于 0.5 判"是";原始概率全部存进 <检验目录>/jevcheck.json。要花钱(Jev)。
// node eval/round-one/jevcheck.ts <export.ts 写出的检验目录>
type Idea = { id: string; name: string; description: string };
const ideas = JSON.parse(readFileSync("eval/round-zero/ideas.json", "utf8")) as { key: Idea[]; deadEnds: Idea[] };
const out = process.argv[2];
if (out === undefined) throw new Error("要写 export.ts 写出的检验目录");
/** 同时最多几个请求在路上:一下发上百个容易被限流。 */
const PARALLEL = 6;
let running = 0;
const waiting: Array<() => void> = [];
async function limited<T>(fn: () => Promise<T>): Promise<T> {
if (running >= PARALLEL) await new Promise<void>((ok) => waiting.push(ok));
running++;
try {
return await fn();
} finally {
running--;
waiting.shift()?.();
}
}
async function one(state: Record<string, unknown>, questions: Record<string, Noul>): Promise<Answer> {
const [answer] = await limited(() => ask(state, questions));
return answer!;
}
async function check(d: RoundData) {
const change = new Map(d.proposals.map((p) => [p.id, p.change]));
const groupOf = new Map(d.dedup.groups.flatMap((g) => g.proposals.map((p) => [p, g.candidate] as const)));
// a. 判重
const pairQuestion = (a: string, b: string) => one({ task: TASKS.pair, proposal_a: change.get(a), proposal_b: change.get(b) }, SAME_CHANGE);
const pairs = await Promise.all(
d.proposals.flatMap((a, i) =>
d.proposals.slice(i + 1).map(async (b) => {
const [ab, ba] = await Promise.all([pairQuestion(a.id, b.id), pairQuestion(b.id, a.id)]);
const mean = (ab.probability + ba.probability) / 2;
return { pair: [a.id, b.id], ab, ba, mean, jevSame: mean > THRESHOLD, programMerged: groupOf.get(a.id) === groupOf.get(b.id) };
}),
),
);
const dedup = {
pairs,
jevYesProgramNotMerged: pairs.filter((p) => p.jevSame && !p.programMerged).map((p) => p.pair),
jevNoProgramMerged: pairs.filter((p) => !p.jevSame && p.programMerged).map((p) => p.pair),
};
// b. 挑刺有没有被回应
const critiques = await Promise.all(
d.critiques.map(async (c) => {
const response = d.responses.find((r) => r.replies.some((x) => x.critique === c.id));
if (response === undefined) return { critique: c.id, author: c.author, proposal: c.proposal, replied: false as const, answer: null };
const reply = response.replies.find((x) => x.critique === c.id)!.text;
const answer = await one({ task: TASKS.critique, critique: c.content, reply, revision: response.rest }, ADDRESSED);
return { critique: c.id, author: c.author, proposal: c.proposal, replied: true as const, answer };
}),
);
// c. 研究路径
const path = await Promise.all(
d.proposals.map(async (p) => {
const ask = (idea: Idea, questions: Record<string, Noul>) =>
one({ task: TASKS.idea, proposal: p.change, idea: { description: idea.description } }, questions).then((answer) => ({ idea: idea.id, answer }));
const answers = await Promise.all([...ideas.key.map((k) => ask(k, CARRIES_IDEA)), ...ideas.deadEnds.map((k) => ask(k, STAYS_IN_DEAD_END))]);
return { proposal: p.id, author: p.author, answers };
}),
);
return { dedup, critiques, path };
}
const report: Record<string, Awaited<ReturnType<typeof check>> & { source: string }> = {};
for (const run of readdirSync(join(out, "runs")).sort()) {
const d = JSON.parse(readFileSync(join(out, "runs", run, "data.json"), "utf8")) as RoundData;
report[run] = { source: d.source, ...(await check(d)) };
const r = report[run];
const p = (x: number) => x.toFixed(2);
console.log(`\n== ${run}`);
console.log(`a 判重:${r.dedup.pairs.map((x) => `${x.pair.join("~")} ${p(x.ab.probability)}/${p(x.ba.probability)} 平均 ${p(x.mean)}${x.programMerged ? "(程序合了)" : ""}`).join(";")}`);
console.log(` 检验判是而程序没合:${JSON.stringify(r.dedup.jevYesProgramNotMerged)};检验判否而程序合了:${JSON.stringify(r.dedup.jevNoProgramMerged)}`);
const answered = r.critiques.filter((c) => c.answer?.yes === true).length;
console.log(`b 挑刺被回应:${answered}/${r.critiques.length};${r.critiques.map((c) => `${c.critique.slice(0, 40)} ${c.replied ? p(c.answer!.probability) : "没有回应"}`).join(";")}`);
for (const x of r.path) console.log(`c ${x.proposal}:${x.answers.map((a) => `${a.idea} ${p(a.answer.probability)}${a.answer.yes ? "是" : ""}`).join(" ")}`);
}
writeFileSync(join(out, "jevcheck.json"), `${JSON.stringify({ thresholdNote: THRESHOLD_NOTE, runs: report }, null, 2)}\n`);
console.log(`\n(${THRESHOLD_NOTE})原始概率:${join(out, "jevcheck.json")}`);
+154
View File
@@ -0,0 +1,154 @@
import { writeFileSync } from "node:fs";
import { join } from "node:path";
import type { LedgerRecord } from "../../src/ledger/index.ts";
import { callId, type DedupOutput } from "../../src/materials/index.ts";
import { ofKind, readLedger, stages, type Of } from "../lib/ledger.ts";
import { launch, newEvalDir, sleep, waitForLedger, writeConfig } from "../lib/study.ts";
import { watchTimeline } from "../lib/timeline.ts";
// 真实运行里的杀进程检验(任务书 14 第 3.1 节):和 run.ts 一样从头跑一份研究(真模型、真 Docker、真 Jev,要花钱),
// 一边跑一边看账本;第 1 轮第一次预测阶段开着、已有一部分第一次预测被收下、还有人在写时,SIGKILL 控制进程,
// 再用恢复入口(--resume)启动让它跑完,查四条:
// 1 被杀那次调用补了结束记录(出错、原因"进程中断");
// 2 交过的任务(人×候选)没被重派,没交的被重派;
// 3 没有重复收下的提交(同人同对象同阶段只有一份收下,除非是替换);
// 4 第一次预测阶段照常关闭,往下走到选批后停下(原因"实现还没写")。
// node eval/round-one/kill.ts <名字>
// 产物和 run.ts 一样(study、timeline.jsonl、outcome.json),另有 kill-report.json;这份研究也算 3 次真实运行之一。
const POLL_MS = 3000;
const KIND = "firstPrediction";
/** flow 写停下事件用的事件名(src/flow/next-step.ts 的 EVENTS.halted;flow 只导出 runStudy,这里照抄)。 */
const HALTED = "halted";
const name = process.argv[2];
if (name === undefined) throw new Error("要写检验名字,例如 r1-kill-1(只许小写字母、数字和 ._-,它也是研究编号)");
const out = newEvalDir(name);
const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, null);
const studyDir = join(out, "study");
const timeline = join(out, "timeline.jsonl");
function openCalls(r: LedgerRecord[]): Array<Of<"callStarted">> {
const ended = new Set(ofKind(r, "callEnded").map((e) => e.content.call));
return ofKind(r, "callStarted").filter((c) => !ended.has(c.content.call));
}
/** 第 1 轮第一次预测阶段(开着的或已关的);还没开是 undefined。 */
const predictionStage = (r: LedgerRecord[]) => stages(r).find((s) => s.round === 1 && s.stage === KIND);
/** 第一次预测的任务:名单上每个人 × 本轮每个新候选(任务书 14 第 2.3 节)。新候选取第 1 轮的判重决定。 */
function tasks(r: LedgerRecord[], roster: string[]): Array<{ actor: string; candidate: string }> {
const dedup = ofKind(r, "decision").find((d) => d.content.decision === "dedup" && d.content.round === 1);
if (dedup === undefined) throw new Error("第一次预测阶段开了,账本里却没有第 1 轮的判重决定");
const fresh = (dedup.content.output as unknown as DedupOutput).fresh.map((g) => g.candidate);
return roster.flatMap((actor) => fresh.map((candidate) => ({ actor, candidate })));
}
const accepted = (r: LedgerRecord[], stageId: string, actor: string, candidate: string) =>
ofKind(r, "submission").some((s) => s.content.stageId === stageId && s.content.kind === KIND && s.content.author === actor && s.content.target === candidate);
/** 打中的条件:阶段开着没关,已有至少一份收下,还有任务没交,而且有人正在写(这样第 1、2 条都查得到东西)。 */
function hit(r: LedgerRecord[]): boolean {
const s = predictionStage(r);
if (s === undefined || s.closed) return false;
const all = tasks(r, s.roster);
const done = all.filter((t) => accepted(r, s.stageId, t.actor, t.candidate)).length;
return done > 0 && done < all.length && openCalls(r).some((c) => c.content.stageId === s.stageId);
}
// ---------------------------------------------------------------------------
// 第一次启动,打中就杀
// ---------------------------------------------------------------------------
const first = launch(config, studyDir, {});
await waitForLedger(studyDir, first.child);
const watching = watchTimeline(studyDir, first.child, timeline);
let triggered: LedgerRecord | undefined;
while (first.child.exitCode === null && first.child.signalCode === null) {
const r = readLedger(studyDir);
if (hit(r)) {
first.child.kill("SIGKILL");
triggered = r.at(-1);
break;
}
await sleep(POLL_MS);
}
const firstEnd = await first.done;
await watching;
if (triggered === undefined) {
writeFileSync(join(out, "outcome.json"), `${JSON.stringify({ launches: [firstEnd] }, null, 2)}\n`);
throw new Error(`没打中:研究进程自己结束了(${JSON.stringify(firstEnd.outcome)}),账本最后一条是 ${JSON.stringify(readLedger(studyDir).at(-1))}`);
}
// 进程已死,此刻的账本就是被杀时的样子。
const atKill = readLedger(studyDir);
const killSeq = atKill.at(-1)!.seq;
const killedCalls = openCalls(atKill).map((c) => c.content.call);
const stage = predictionStage(atKill)!;
const taskList = tasks(atKill, stage.roster).map((t) => ({ ...t, submittedBeforeKill: accepted(atKill, stage.stageId, t.actor, t.candidate) }));
// ---------------------------------------------------------------------------
// 再启动,跑完
// ---------------------------------------------------------------------------
const second = launch(null, studyDir, {});
process.on("SIGINT", () => second.child.kill("SIGINT"));
await watchTimeline(studyDir, second.child, timeline);
const secondEnd = await second.done;
writeFileSync(join(out, "outcome.json"), `${JSON.stringify({ launches: [firstEnd, secondEnd] }, null, 2)}\n`);
const r = readLedger(studyDir);
// 1 被杀那次调用补了结束记录
const ends = ofKind(r, "callEnded");
const check1 = killedCalls.map((call) => {
const end = ends.find((e) => e.content.call === call);
return { call, end: end === undefined ? null : { seq: end.seq, ...end.content } };
});
// 2 交过的没重派、没交的重派了。一次调用属于哪个任务,用 materials 的调用编号规则对:同一任务第 n 次发起是 callId(…, n)。
const startedAfter = ofKind(r, "callStarted").filter((c) => c.seq > killSeq);
const attempts = ofKind(r, "callStarted").length;
const check2 = taskList.map((t) => {
const ids = new Set(Array.from({ length: attempts }, (_, i) => callId(stage.stageId, t.actor, KIND, t.candidate, i + 1)));
const again = startedAfter.filter((c) => ids.has(c.content.call)).map((c) => c.content.call);
return { ...t, dispatchedAfterRestart: again, ok: t.submittedBeforeKill === (again.length === 0) };
});
// 3 没有重复收下:第 1 轮每份提交(阶段、作者、种类、对象、编号)的各版版本号是 1..n;第一次预测里被杀前已收下的,被杀后不能再多出一版
const round1 = new Set(stages(r).filter((s) => s.round === 1).map((s) => s.stageId));
const slots = new Map<string, Array<{ seq: number; version: number }>>();
for (const s of ofKind(r, "submission")) {
const c = s.content;
if (!round1.has(c.stageId)) continue;
const key = JSON.stringify([c.stageId, c.author, c.kind, c.target, c.item]);
slots.set(key, [...(slots.get(key) ?? []), { seq: s.seq, version: c.version }]);
}
const check3 = [...slots].map(([slot, versions]) => {
const numbered = versions.map((v) => v.version).join(",") === versions.map((_, i) => i + 1).join(",");
const [stageId, , kind] = JSON.parse(slot) as [string, string, string];
const acrossKill = stageId === stage.stageId && kind === KIND && versions.some((v) => v.seq <= killSeq) && versions.some((v) => v.seq > killSeq);
return { slot, versions, replaced: versions.length > 1, ok: numbered && !acrossKill };
});
// 4 阶段照常关闭、往下走到选批后停下
const closed = ofKind(r, "stageClose").some((c) => c.content.stageId === stage.stageId);
const batch = ofKind(r, "decision").some((d) => d.content.decision === "batch" && d.content.round === 1);
const lastHalt = ofKind(r, "event").filter((e) => e.content.event === HALTED).at(-1);
const checks = {
"1 被杀那次调用补了结束记录(出错、进程中断)": {
pass: check1.length > 0 && check1.every((c) => c.end !== null && c.end.status === "failed" && c.end.reason.includes("进程中断")),
calls: check1,
},
"2 交过的任务没被重派,没交的被重派": { pass: check2.every((t) => t.ok), tasks: check2 },
"3 没有重复收下的提交(替换除外)": { pass: check3.every((s) => s.ok), problems: check3.filter((s) => !s.ok), replaced: check3.filter((s) => s.replaced) },
"4 阶段照常关闭,往下走到选批后停下": {
pass: closed && batch && lastHalt !== undefined && lastHalt.content.reason.includes("实现还没写"),
firstPredictionClosed: closed,
batchDecision: batch,
lastHalt: lastHalt?.content ?? null,
},
};
const report = { triggeredAt: { seq: triggered.seq, kind: triggered.kind }, killSeq, lastRecordAtKill: atKill.at(-1), tasksAtKill: taskList, outcomeAfterRestart: secondEnd.outcome, checks };
writeFileSync(join(out, "kill-report.json"), `${JSON.stringify(report, null, 2)}\n`);
console.log(`\n被杀时账本最后一条 #${killSeq} ${atKill.at(-1)!.kind};被杀时正在进行的调用:${killedCalls.join("、") || "无"}`);
console.log(`被杀时第一次预测:${taskList.filter((t) => t.submittedBeforeKill).length}/${taskList.length} 个任务已交`);
console.log(`再启动后的结局:${JSON.stringify(secondEnd.outcome)}`);
for (const [k, v] of Object.entries(checks)) console.log(`${v.pass ? "通过" : "不通过"} ${k}`);
console.log(`报告:${join(out, "kill-report.json")}`);
+97
View File
@@ -0,0 +1,97 @@
import type { Json } from "./round-data.ts";
// 给审阅者的文本去掉来历(任务书 14 第 3.2 节第 1 步):运行名、模型名、研究编号去掉;
// 研究者、提案、候选、挑刺、入口的编号改写成"研究者 1""提案 1"这类中性写法,按出现顺序编,一份文本一张映射表。
// 两个项目的编号写法都认:我们的 r1-researcher-2-1、r1-c1、e-researcher-1-4,参考项目的 r1-v1-p-researcher-2-1、r1-v1-c1、
// researcher-1-e4、挑刺的事实编号(uuid)。字段名(points 对 factors)不改。
type Family = "研究者" | "提案" | "候选" | "挑刺" | "入口";
/** 认编号的写法,按先后试:长的、带别的编号在里面的写法放前面。 */
const PATTERNS: Array<[Family | "实验员" | "核验员", string]> = [
["挑刺", String.raw`r\d+-researcher-\d+-\d+\.researcher-\d+\.\d+`],
["提案", String.raw`r\d+-v\d+-p-researcher-\d+-\d+`],
["提案", String.raw`r\d+-researcher-\d+-\d+`],
["实验员", String.raw`implementer-[A-Za-z0-9-]+`],
["核验员", String.raw`verifier-[A-Za-z0-9-]+`],
["入口", String.raw`e-researcher-\d+-\d+`],
["入口", String.raw`researcher-\d+-e\d+`],
["候选", String.raw`r\d+-v\d+-c\d+`],
["候选", String.raw`r\d+-c\d+`],
["研究者", String.raw`researcher-\d+`],
["挑刺", String.raw`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`],
];
const ID_RE = new RegExp(PATTERNS.map(([, p]) => `(?<![A-Za-z0-9])(${p})(?![0-9])`).join("|"), "g");
/** 认不成编号、但能看出来历的东西。 */
const SCRUB: Array<[RegExp, string]> = [
[/(?<![A-Za-z0-9])\d+[a-z]?-(?:real|tight|evidence|headless|probe|replay)-\d+(?![0-9])/g, "某次运行"],
[/\b(?:glm|claude|gpt|kimi|deepseek|qwen|gemini|minimax)[-\w.]*/gi, "某模型"],
[/iomgaa|rivalbits|inforsi/gi, "某项目"],
[/req_[0-9a-f]{16,}/g, "某请求"],
[/\b[0-9a-f]{32,}\b/g, "某哈希"],
];
export class Neutral {
private readonly names = new Map<string, string>();
private readonly counts = new Map<string, number>();
private readonly hidden: RegExp | null;
constructor(hiddenNames: string[]) {
const esc = hiddenNames.map((n) => n.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
this.hidden = esc.length === 0 ? null : new RegExp(esc.sort((a, b) => b.length - a.length).join("|"), "g");
}
/** 一个编号的中性名:第一次见到时按这一类的出现顺序编号。 */
name(family: Family | "实验员" | "核验员", raw: string): string {
const seen = this.names.get(raw);
if (seen !== undefined) return seen;
const n = (this.counts.get(family) ?? 0) + 1;
this.counts.set(family, n);
const named = `${family} ${n}`;
this.names.set(raw, named);
return named;
}
/** 一段文字里的编号和来历都换掉。研究编号、运行名先换:我们的研究编号里可能带着像编号的片段。 */
text(s: string): string {
let t = this.hidden === null ? s : s.replace(this.hidden, "某次运行");
t = t.replace(ID_RE, (match, ...groups: unknown[]) => {
const i = groups.findIndex((g, k) => k < PATTERNS.length && g !== undefined);
return this.name(PATTERNS[i]![0], match);
});
for (const [re, to] of SCRUB) t = t.replace(re, to);
return t;
}
/** 映射表(原编号 → 中性名),存进 data 旁边,审阅者看不到。 */
table(): Record<string, string> {
return Object.fromEntries(this.names);
}
}
/** 渲染时去掉的键:只说明来历的哈希、请求编号、轮次编号(编号类的值照常经 text 改写成中性名)。 */
const DROP = new Set(["baselineCodeHash", "materialHash", "questionHash", "requestId", "roundId", "protocolId"]);
/** 把 Json 写成缩进的纯文本:对象一键一行,多行文字缩进成块,数组逐项编号。 */
export function render(value: Json, nt: Neutral, indent = ""): string {
if (value === null) return `${indent}(空)`;
if (typeof value !== "object") return block(nt.text(String(value)), indent);
if (Array.isArray(value)) {
if (value.length === 0) return `${indent}(无)`;
return value.map((v, i) => (isScalar(v) ? `${indent}${i + 1}. ${inline(v, nt)}` : `${indent}${i + 1}.\n${render(v, nt, `${indent} `)}`)).join("\n");
}
const entries = Object.entries(value).filter(([k]) => !DROP.has(k));
if (entries.length === 0) return `${indent}(无)`;
return entries
.map(([k, v]) => (isScalar(v) && !String(v).includes("\n") ? `${indent}${k}: ${inline(v, nt)}` : `${indent}${k}:\n${render(v, nt, `${indent} `)}`))
.join("\n");
}
const isScalar = (v: Json) => v === null || typeof v !== "object";
const inline = (v: Json, nt: Neutral) => (v === null ? "(空)" : nt.text(String(v)));
const block = (s: string, indent: string) =>
s
.split("\n")
.map((l) => `${indent}${l}`)
.join("\n");
+3
View File
@@ -0,0 +1,3 @@
# 研究路径说明
26 圆填圆,基线 0.9598(圆心按同心环写死、半径逐对缩放),已知最好 2.635983。同类系统和参考项目的高分运行,大多沿着这样一条路往上走:结构化的起点或多起点 → 把圆心和半径一起交给连续优化器 → 在当前最好解附近加扰动再优化,或退火式接受 → 最后修到严格合法。辅助手段有:固定圆心用线性规划求最优半径、只动贴紧的那几个圆。死路有:只做几何构造(圆心按手排的规则摆、不移动),各系统都卡在 2.2–2.54;只改半径的分法、不动圆心,上限受布局限制。这是常见的一条路,不是唯一正确的路:一条清单上没有、但讲得通的路,同样算往前走了。第 1 轮能走多远取决于起点,参考项目第 1 轮采纳的分数在 1.55 到 2.632 之间。
+146
View File
@@ -0,0 +1,146 @@
import type { Noul } from "../lib/jev.ts";
// 第 1 轮检验问 Jev 的问题(任务书 14 第 3.2 节第 3 步):英文、一问只问一件事、带材料说明、判"是"判"否"的说明和反例,
// 写法照 src/jev/questions.ts。这些是检验自己的问题,不进 src/jev;阈值没有校准,一律以概率大于 0.5 判"是"。
// 提案、挑刺、回应原文是中文,原样放进材料;两个项目的字段名不同(factors 对 points、modification),材料说明里写明。
const STUDY =
"One round of a study that improves a program for the circle packing task: place 26 non-overlapping circles inside the unit square " +
"so that the sum of their radii is as large as possible (no tolerance: any overlap or circle outside the square is invalid). " +
"The baseline program puts the circles on fixed rings around the center and then shrinks radii pair by pair to remove overlaps; " +
"it scores about 0.96 and the best known score is about 2.636. " +
"In this round each researcher proposed one code change. A proposal's change is given as a list of separate edits " +
"(each with a `name`, a `method` saying what to change and how, and sometimes `keySettings`), or as one `modification` text. " +
"The texts are mostly in Chinese.";
export const TASKS = {
pair: `${STUDY} \`proposal_a\` and \`proposal_b\` are the changes of two proposals from the same round.`,
idea: `${STUDY} \`proposal\` is the change of one proposal. \`idea\` is a technique from the literature on this task, described in English.`,
critique:
`${STUDY} After the proposals were written, other researchers wrote critiques of them, and each author replied to every critique ` +
"and could revise the proposal before it was frozen as the protocol an implementer must follow. " +
"`critique` is one critique of a proposal; `reply` is the author's reply to that critique; " +
"`revision` is the rest of the author's response: the changes made to the proposal and the final text, in whatever fields that system used.",
};
/** a. 两条提案是不是同一个改法(照方法.md 4.3 ③ 判重的意思:只差参数的不算同一个改法)。 */
export const SAME_CHANGE: Record<string, Noul> = {
same_change: {
type: "noul",
instructions: {
question: "Would carrying out `proposal_a` and carrying out `proposal_b` produce the same code change, so that a single implementation would serve as both?",
focus:
"Compare what each change does to the program: which parts it changes and by what method, reading every edit. " +
"Ignore wording, language, level of detail, names, and how the change is split into edits. " +
"The same method with only different settings (step sizes, counts, time budgets, layouts) is not the same change, because each setting needs its own run.",
},
criteria: {
true: {
what: "Both describe the same algorithmic change to the same part of the program with the same settings; they differ only in wording, detail or how the change is split",
examples: [
"a: 把圆心和半径一起交给约束优化器,从当前布局出发; b: optimize centers and radii jointly with SLSQP starting from the current layout",
"a: 半径改用线性规划求解; b: 固定圆心,用 linprog 求半径和最大",
],
},
false: {
what: "They change different parts of the program, change the same part by a different method, or use the same method with different settings",
not_for: "Two changes that only share the goal of a higher score, or only edit the same function in different ways",
examples: [
"a: 半径改用线性规划求解; b: 把圆心和半径一起交给约束优化器",
"a: 6 行六角网格; b: 5 行六角网格",
"a: 多起点重启 20 次; b: 多起点重启 200 次",
],
},
},
},
};
/** b. 作者的回应有没有处理这条挑刺:改了协议,或者给了不改的具体理由。 */
export const ADDRESSED: Record<string, Noul> = {
critique_addressed: {
type: "noul",
instructions: {
question:
"Does `reply` deal with the concern raised in `critique`, either by a concrete change to the proposal (visible in `reply` or `revision`) or by a specific reason why no change is needed?",
focus:
"First find the concern in `critique`. Then check whether `reply` engages with that concern: says what was changed and how, or argues specifically why the concern does not apply. " +
"Agreeing without saying what changed, promising to consider it, or answering a different point does not deal with the concern. " +
"Whether the reply is right is not the question; only whether it deals with the concern.",
},
criteria: {
true: {
what: "The reply states a concrete change that answers the concern, or gives a specific reason tied to the concern for not changing",
examples: [
"critique: 停止条件没写锚点; reply: 采纳,t0 改为入口处取 perf_counter,并写进协议的停止条件一节",
"critique: 这个因子会改变评分; reply: 不改:评分只读 solution.json,这个因子只改内部的起点,具体见协议第 2 节",
],
},
false: {
what: "The reply ignores the concern, only agrees or thanks, promises to look later, or answers something else",
not_for: "Replies that disagree with the critique but give a specific reason tied to its concern",
examples: ["critique: 停止条件没写锚点; reply: 感谢意见,同意", "critique: 两个因子分不开; reply: 我们会在实现时注意"],
},
},
},
};
/** c. 提案对研究路径上的关键想法(K1–K5):这条提案的改动是否做了这个想法。 */
export const CARRIES_IDEA: Record<string, Noul> = {
carries_idea: {
type: "noul",
instructions: {
question: "Does the code change in `proposal` carry out the technique described in `idea.description`, as the whole change or as one of its parts?",
focus:
"Read every edit of the proposal. Answer yes if one of the edits, or the change as a whole, does what the technique describes, possibly in other words or in Chinese. " +
"Mentioning the technique as a later plan, or sharing only the goal, is not enough.",
},
criteria: {
true: {
what: "The proposal's change includes this technique",
examples: [
"idea: optimize centers and radii together with a constrained optimizer; proposal: 把 78 个变量交给 SLSQP,约束是在正方形内、两两不重叠",
"idea: solve a linear program for the radii with fixed centers; proposal: 半径分配改成 linprog 求半径和最大",
],
},
false: {
what: "The proposal's change does not include this technique",
not_for: "Proposals that only mention the technique as future work, or only share the goal",
examples: [
"idea: optimize centers and radii together with a constrained optimizer; proposal: 圆心按六角网格摆,半径用贪心缩放",
"idea: solve a linear program for the radii with fixed centers; proposal: 从当前最好解加扰动再跑 SLSQP",
],
},
},
},
};
/** c. 提案对死路(D1、D2):这条提案的改动是不是整个都停在这条死路里。 */
export const STAYS_IN_DEAD_END: Record<string, Noul> = {
stays_within_idea: {
type: "noul",
instructions: {
question: "Does the whole code change in `proposal` stay within the approach described in `idea.description`, with no part that goes beyond it?",
focus:
"Answer yes only if everything the proposal changes falls inside this approach. " +
"If any edit uses a continuous optimization that moves the circle centers, or uses the approach only as a helper step " +
"(a starting layout, or a way to score layouts inside a larger search), answer no.",
},
criteria: {
true: {
what: "Everything the proposal changes is inside this approach",
examples: [
"idea: hand-designed placement of centers only; proposal: 换成 6 行六角网格,枚举行数取最好,半径用贪心",
"idea: change only the radii with fixed centers; proposal: 圆心不动,半径改用线性规划",
],
},
false: {
what: "Some part of the proposal goes beyond this approach, or the approach is only one step of a larger change",
not_for: "Proposals that use this approach as a starting layout or a scoring helper inside a search that moves the centers",
examples: [
"idea: hand-designed placement of centers only; proposal: 从六角起点出发,再用 SLSQP 同时移动圆心和半径",
"idea: change only the radii with fixed centers; proposal: 半径用线性规划打分,外层搜索圆心位置",
],
},
},
},
};
+352
View File
@@ -0,0 +1,352 @@
import { copyFileSync, mkdirSync, statSync } from "node:fs";
import { basename, dirname, join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import type { LedgerRecord } from "../../src/ledger/index.ts";
import type { DedupOutput, FrozenProtocol } from "../../src/materials/index.ts";
import { finalSubmissions, ofKind, stages } from "../lib/ledger.ts";
// 一次运行第 1 轮(开轮到选批)的记录,读成同一种形状,导出文本和 Jev 检验都用它(任务书 14 第 3.2 节)。
// 两种来源:我们的账本(经 src/ledger 读,像用户一样)和参考项目的账本(只读,先复制到输出目录再打开)。
// 内容按原样存 Json,字段名不统一(参考项目叫 points,我们叫 factors),只把检验要用的几样取到固定的键上。
// 按名字读,读不到就报错停下:flow 第 2 部分还在写,读错了要立刻看出来,不猜。
export type Json = null | boolean | number | string | Json[] | { [key: string]: Json };
export type JsonObject = { [key: string]: Json };
export interface Range3 {
q1: number;
q5: number;
q9: number;
}
export interface Prediction {
researcher: string;
candidate: string;
range: Range3;
/** 预测附的理由(我们的 body、参考项目 14 的 rationale);没有就是 null。 */
note: Json;
}
export interface RoundData {
/** 来历(运行名、库路径),只进 data.json,不进给审阅者的文本。 */
source: string;
/** 要从文本里去掉的名字:运行名、研究编号。 */
hiddenNames: string[];
proposals: Array<{
id: string;
author: string;
content: JsonObject;
/** 判重看的"改了什么":我们是因子的名称、做法、关键设定;参考项目是 points 或 modification。 */
change: Json;
authorEstimate: Range3 | null;
}>;
dedup: {
/** 本轮新候选各合并了哪几条原提案。 */
groups: Array<{ candidate: string; proposals: string[] }>;
/** 并进留池候选的新提案。 */
carried: Json;
/** 程序判重时 Jev 每一对的判断,原样。 */
judgments: Json;
};
critiques: Array<{ id: string; author: string; proposal: string; content: JsonObject }>;
responses: Array<{
candidate: string;
author: string;
kind: string;
replies: Array<{ critique: string; text: string }>;
/** 回应里逐条答复以外的部分(改动、实现条款、正文)。 */
rest: JsonObject;
}>;
protocols: Array<{ candidate: string; content: JsonObject }>;
/** null:这次运行没有第一次预测这个阶段。 */
firstPrediction: Prediction[] | null;
/** null:这次运行用投票,不是范围预测,投票原样在 votes 里。 */
secondPrediction: Prediction[] | null;
votes: Json;
batch: { chosen: string[]; detail: Json };
rejections: Array<{ stage: string; author: string; target: string | null; reason: string }>;
}
/** 中性的阶段名,两种来源共用。 */
export const STAGE_NAMES = {
proposal: "提案",
firstPrediction: "第一次预测",
critique: "挑刺",
response: "回应与统一",
secondPrediction: "第二次预测",
vote: "投票",
} as const;
type StageName = keyof typeof STAGE_NAMES;
function need<T>(value: T | undefined, what: string): T {
if (value === undefined) throw new Error(`读不到:${what}`);
return value;
}
function field(obj: JsonObject, key: string, where: string): Json {
return need(obj[key], `${where} 的字段 ${key}`);
}
const omit = (obj: JsonObject, keys: string[]): JsonObject => Object.fromEntries(Object.entries(obj).filter(([k]) => !keys.includes(k)));
const asObject = (v: unknown) => v as JsonObject;
// ---------------------------------------------------------------------------
// 我们的账本
// ---------------------------------------------------------------------------
/** 我们的研究第 round 轮。records 是 readLedger 读出的全部记录,studyId 是研究编号(要从文本里去掉)。 */
export function readOurs(records: LedgerRecord[], round: number, source: string, studyId: string): RoundData {
const inRound = stages(records).filter((s) => s.round === round);
const stageOf = (name: string) => need(inRound.find((s) => s.stage === name), `第 ${round} 轮 ${name} 阶段的开记录`).stageId;
const stageName = new Map(inRound.map((s) => [s.stageId, s.stage]));
const finals = finalSubmissions(records);
const subs = (stage: string, kinds: string[]) => {
const id = stageOf(stage);
return finals.filter((s) => s.content.stageId === id && kinds.includes(s.content.kind));
};
const seals = ofKind(records, "seal");
const proposals = subs("proposal", ["proposal"]).map((s) => {
const content = asObject(s.content.content);
const factors = field(content, "factors", `提案 ${s.content.item}`) as JsonObject[];
const seal = need(seals.find((x) => x.content.proposal === s.seq), `提案 ${s.content.item}(序号 ${s.seq})的作者自评封存`);
return {
id: need(s.content.item ?? undefined, `提案序号 ${s.seq} 的编号 item`),
author: s.content.author,
content,
change: factors.map((f) => ({ name: f["name"]!, method: f["method"]!, keySettings: f["keySettings"]! })),
authorEstimate: seal.content.range,
};
});
const decision = (kind: string) =>
need(ofKind(records, "decision").find((d) => d.content.decision === kind && d.content.round === round), `第 ${round} 轮的 ${kind} 决定`).content;
const dedup = decision("dedup");
const dedupOut = dedup.output as unknown as DedupOutput;
need(dedupOut.fresh, "判重决定输出的 fresh");
need(dedupOut.carried, "判重决定输出的 carried");
const critiques = subs("critique", ["critique"]).map((s) => ({
id: need(s.content.item ?? undefined, `挑刺序号 ${s.seq} 的编号 item`),
author: s.content.author,
proposal: need(s.content.target ?? undefined, `挑刺序号 ${s.seq} 的对象 target`),
content: asObject(s.content.content),
}));
const responses = subs("response", ["response", "unification"]).map((s) => {
const content = asObject(s.content.content);
const replies = field(content, "replies", `回应序号 ${s.seq}`) as JsonObject[];
return {
candidate: need(s.content.target ?? undefined, `回应序号 ${s.seq} 的对象 target`),
author: s.content.author,
kind: s.content.kind,
replies: replies.map((r) => ({ critique: r["critique"] as string, text: r["reply"] as string })),
rest: omit(content, ["replies"]),
};
});
const fresh = new Set(dedupOut.fresh.map((g) => g.candidate));
const protocols = ofKind(records, "protocol")
.filter((p) => fresh.has(p.content.candidate) && (p.content.protocol as unknown as FrozenProtocol).round === round)
.map((p) => ({ candidate: p.content.candidate, content: asObject(p.content.protocol) }));
const predictions = (stage: string): Prediction[] =>
subs(stage, [stage]).map((s) => {
const content = asObject(s.content.content);
return {
researcher: s.content.author,
candidate: need(s.content.target ?? undefined, `${stage} 序号 ${s.seq} 的对象 target`),
range: field(content, "range", `${stage} 序号 ${s.seq}`) as unknown as Range3,
note: content["body"] ?? null,
};
});
const batch = decision("batch");
const chosen = need(asObject(batch.output)["chosen"], "选批决定输出的 chosen") as string[];
const rejections = ofKind(records, "rejection")
.filter((r) => stageName.has(r.content.stageId))
.map((r) => ({
stage: STAGE_NAMES[need(ourStage(stageName.get(r.content.stageId)!), `阶段 ${r.content.stageId} 的中性名`)],
author: r.content.author,
target: r.content.target,
reason: r.content.reason,
}));
return {
source,
hiddenNames: [studyId],
proposals,
dedup: { groups: dedupOut.fresh, carried: dedupOut.carried as unknown as Json, judgments: dedup.input as Json },
critiques,
responses,
protocols,
firstPrediction: predictions("firstPrediction"),
secondPrediction: predictions("secondPrediction"),
votes: null,
batch: { chosen, detail: batch.output as Json },
rejections,
};
}
function ourStage(stage: string): StageName | undefined {
return stage in STAGE_NAMES ? (stage as StageName) : undefined;
}
// ---------------------------------------------------------------------------
// 参考项目的账本(/home/iomgaa/rb-acceptance/<运行名>/board.sqlite)
// ---------------------------------------------------------------------------
/** 参考项目的阶段名(stage 编号的最后一段)→ 中性名。挑刺和回应都在 phase review 里,按编号区分。 */
const REF_STAGES: Record<string, StageName> = {
proposal: "proposal",
"predict-blind": "firstPrediction",
critique: "critique",
response: "response",
predict: "secondPrediction",
vote: "vote",
};
/** 原库只读:库文件(和非空的 -wal)先复制到 copyDir,再以只读方式打开复制件,原目录里不写任何东西。 */
function openCopy(board: string, copyDir: string): DatabaseSync {
mkdirSync(copyDir, { recursive: true });
const copy = join(copyDir, "board.sqlite");
copyFileSync(board, copy);
if (statSync(`${board}-wal`, { throwIfNoEntry: false })?.size) copyFileSync(`${board}-wal`, `${copy}-wal`);
return new DatabaseSync(copy, { readOnly: true });
}
const refRange = (r: JsonObject): Range3 => ({ q1: r["q10"] as number, q5: r["q50"] as number, q9: r["q90"] as number });
export function readReference(board: string, copyDir: string): RoundData {
const run = basename(dirname(board));
const roundId = `${run}.r1`;
const db = openCopy(board, copyDir);
try {
const all = <T>(sql: string, ...args: Array<string | number>) => db.prepare(sql).all(...args) as T[];
const stageRows = all<{ id: string }>("SELECT id FROM stages WHERE round_id = ? ORDER BY created_seq", roundId);
const stageOf = new Map<StageName, string>();
for (const s of stageRows) {
const name = REF_STAGES[s.id.split(".").at(-1)!];
if (name !== undefined) {
if (stageOf.has(name)) throw new Error(`${board}:第 1 轮 ${name} 阶段不止一个(${stageOf.get(name)}、${s.id})`);
stageOf.set(name, s.id);
}
}
// 14、12 那两版的 submissions 表还没有 version 列(一人一个 slot 只有一份),按第 1 版读。
const versioned = all<{ name: string }>("PRAGMA table_info(submissions)").some((c) => c.name === "version");
/** 每份提交(阶段、人、slot)的最后一版,按第一次出现的先后排。 */
const subs = (name: StageName) => {
const stage = need(stageOf.get(name), `${board} 第 1 轮 ${name} 阶段`);
const rows = all<{ actor: string; slot: string; version: number; payload: string }>(
`SELECT actor, slot, ${versioned ? "version" : "1 AS version"}, payload FROM submissions WHERE stage_id = ? ORDER BY created_seq`,
stage,
);
const latest = new Map<string, { stage: string; actor: string; slot: string; version: number; payload: JsonObject }>();
for (const r of rows) {
const key = `${r.actor}\n${r.slot}`;
const seen = latest.get(key);
if (seen === undefined || seen.version < r.version) latest.set(key, { stage, actor: r.actor, slot: r.slot, version: r.version, payload: JSON.parse(r.payload) });
}
return [...latest.values()];
};
const fact = (kind: string) => {
const rows = all<{ payload: string }>(
"SELECT payload FROM facts WHERE kind = ? AND id LIKE ? ORDER BY created_seq",
kind,
`${run}.${kind}.${roundId}.%`,
);
return JSON.parse(need(rows.at(-1), `${board} 第 1 轮的 ${kind}`).payload) as JsonObject;
};
const proposals = subs("proposal").map((s) => {
const p = s.payload;
const points = p["points"] as JsonObject[] | undefined;
const change: Json =
points !== undefined
? points.map((x) => ({ name: x["name"]!, method: x["method"]!, ...(x["keySettings"] === undefined ? {} : { keySettings: x["keySettings"] }) }))
: { modification: field(p, "modification", `${board} 提案 ${s.slot}`) };
const estimate = (p["authorRange"] ?? p["range"]) as JsonObject | undefined;
return {
id: s.slot,
author: s.actor,
content: omit(p, ["authorId", "id", "roundId", "authorRange", "range"]),
change,
authorEstimate: estimate === undefined ? null : refRange(estimate),
};
});
const dedup = fact("dedup-record");
const groups = (field(dedup, "candidates", `${board} dedup-record`) as JsonObject[]).map((c) => ({
candidate: c["candidateId"] as string,
proposals: c["proposalIds"] as string[],
}));
// 一份挑刺提交(一人对一条原提案)是一条挑刺,编号是它的 review-submission 事实编号:回应里的 critiqueId 指的就是它。
const critiqueStage = need(stageOf.get("critique"), `${board} 第 1 轮挑刺阶段`);
const critiqueIds = new Map(
all<{ id: string; slot: string }>("SELECT id, slot FROM facts WHERE kind = 'review-submission' AND slot LIKE ?", `${critiqueStage}/%`).map((f) => [f.slot, f.id]),
);
const critiques = subs("critique").map((s) => ({
id: need(critiqueIds.get(`${s.stage}/${s.actor}/${s.slot}`), `${board} 挑刺 ${s.actor}/${s.slot} 的事实编号`),
author: s.actor,
proposal: s.slot,
content: omit(s.payload, ["authorId", "proposalId", "roundId"]),
}));
const responses = subs("response").map((s) => ({
candidate: s.slot,
author: s.actor,
kind: "response",
replies: (field(s.payload, "critiqueResponses", `${board} 回应 ${s.slot}`) as JsonObject[]).map((r) => ({
critique: r["critiqueId"] as string,
text: r["response"] as string,
})),
rest: omit(s.payload, ["authorId", "candidateId", "roundId", "critiqueResponses"]),
}));
const protocols = all<{ payload: string }>(
"SELECT payload FROM facts WHERE kind = 'candidate-protocol' AND id LIKE ? ORDER BY created_seq",
`${run}.candidate-protocol.${roundId}.%`,
).map((f) => {
const p = JSON.parse(f.payload) as JsonObject;
return { candidate: field(p, "candidateId", `${board} candidate-protocol`) as string, content: omit(p, ["author", "candidateId", "baselineCodeHash"]) };
});
const predictions = (name: StageName): Prediction[] =>
subs(name).map((s) => ({
researcher: s.actor,
candidate: s.slot,
range: refRange(field(s.payload, "range", `${board} ${name} ${s.actor}/${s.slot}`) as JsonObject),
note: s.payload["rationale"] ?? null,
}));
const voting = stageOf.has("vote");
const batch = voting ? fact("vote-tally") : fact("batch-selection");
const chosen = (field(batch, "candidates", `${board} 选批`) as JsonObject[]).filter((c) => c["selected"] === true).map((c) => c["candidateId"] as string);
// 退回:audit 里的 submission-rejected,taskId 是"<轮编号>.v1.<阶段>.<人>.<对象>",只取开轮到选批这几个阶段的。
const rejections = all<{ payload: string }>("SELECT payload FROM audit WHERE event = 'submission-rejected' ORDER BY seq").flatMap((a) => {
const p = JSON.parse(a.payload) as JsonObject;
const task = p["taskId"] as string;
const m = new RegExp(`^${roundId.replace(/\./g, "\\.")}\\.v\\d+\\.([a-z-]+)\\.([^.]+)\\.(.+)$`).exec(task);
const stage = m === null ? undefined : REF_STAGES[m[1]!];
if (stage === undefined) return [];
const target = m![3]!;
return [{ stage: STAGE_NAMES[stage], author: m![2]!, target: target === m![2] ? null : target, reason: p["reason"] as string }];
});
return {
source: board,
hiddenNames: [run],
proposals,
dedup: { groups, carried: dedup["pooled"] ?? null, judgments: dedup["pairs"] ?? null },
critiques,
responses,
protocols,
firstPrediction: stageOf.has("firstPrediction") ? predictions("firstPrediction") : null,
secondPrediction: voting ? null : predictions("secondPrediction"),
votes: voting ? subs("vote").map((s) => ({ researcher: s.actor, candidate: s.slot, ...omit(s.payload, ["voterId", "candidateId", "protocolId"]) })) : null,
batch: { chosen, detail: batch },
rejections,
};
} finally {
db.close();
}
}
+23
View File
@@ -0,0 +1,23 @@
import { writeFileSync } from "node:fs";
import { join } from "node:path";
import { launch, newEvalDir, waitForLedger, writeConfig } from "../lib/study.ts";
import { watchTimeline } from "../lib/timeline.ts";
// 用真模型、真 Docker、真 Jev 从头跑一份研究,直到它自己停下(任务书 14 第 3.1 节;预期停在第 1 轮选批之后,
// 原因"实现还没写")。配置照 eval/round-zero/research.json,研究编号换成检验名字。要花钱。
// node eval/round-one/run.ts <名字>
// 产物:<检验目录>/study(研究目录)、timeline.jsonl(每条记录第一次被看到的时刻,stats 用来算墙钟)、outcome.json(结局)。
// 中途按 Ctrl-C 会把停下信号传给研究进程;接着跑用 node eval/lib/cli.ts --resume --dir <检验目录>/study。
const name = process.argv[2];
if (name === undefined) throw new Error("要写检验名字,例如 r1-real-1(只许小写字母、数字和 ._-,它也是研究编号)");
const out = newEvalDir(name);
const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, null);
const studyDir = join(out, "study");
const study = launch(config, studyDir, {});
process.on("SIGINT", () => study.child.kill("SIGINT"));
await waitForLedger(studyDir, study.child);
await watchTimeline(studyDir, study.child, join(out, "timeline.jsonl"));
const { code, signal, outcome } = await study.done;
writeFileSync(join(out, "outcome.json"), `${JSON.stringify({ launches: [{ code, signal, outcome }] }, null, 2)}\n`);
console.log(`\n结局:${JSON.stringify(outcome)}\n研究目录:${studyDir}`);
+172
View File
@@ -0,0 +1,172 @@
import { existsSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import type { FrozenProtocol } from "../../src/materials/index.ts";
import { finalSubmissions, ofKind, readLedger, stages } from "../lib/ledger.ts";
import { readTimeline } from "../lib/timeline.ts";
// 第 1 轮的数字和过程(任务书 14 第 3.2 节第 4 步),只看我们的运行;另查程序部分(第 3.4 节):
// 阶段开关成对、冻结协议写在回应阶段关闭之后、最后停下的原因是"实现还没写"。不花钱,只读账本和 timeline.jsonl。
// node eval/round-one/stats.ts <检验目录>...(run.ts、kill.ts 写出的目录:study/、timeline.jsonl)
// 每个目录写一份 stats.json,并打印摘要。
/** 已知最好和基线分数(任务书 14 第 3.2 节第 4 步给的数),预测超出这两头的逐条列出。 */
const BEST_KNOWN = 2.635983;
const BASELINE = 0.9598;
/** flow 写停下事件用的事件名(src/flow/next-step.ts 的 EVENTS.halted;flow 只导出 runStudy,这里照抄)。 */
const HALTED = "halted";
const EXPECTED_STOP = "实现还没写";
const dirs = process.argv.slice(2);
if (dirs.length === 0) throw new Error("要写一个或几个检验目录(run.ts、kill.ts 写出的)");
type Range3 = { q1: number; q5: number; q9: number };
const measured = (m: { status: string; value?: number }) => (m.status === "measured" ? m.value! : null);
const sum = (xs: Array<number | null>) => ({ total: xs.reduce<number>((a, x) => a + (x ?? 0), 0), unmeasured: xs.filter((x) => x === null).length });
function statsOf(dir: string) {
const records = readLedger(join(dir, "study"));
const timelineFile = join(dir, "timeline.jsonl");
if (!existsSync(timelineFile)) throw new Error(`${timelineFile} 不存在:墙钟时间要靠 run.ts、kill.ts 记下的时刻`);
const seen = new Map(readTimeline(timelineFile).map((r) => [r.seq, r.at]));
const at = (seq: number) => {
const t = seen.get(seq);
if (t === undefined) throw new Error(`timeline.jsonl 里没有序号 ${seq} 的时刻`);
return t;
};
const allStages = stages(records);
const opens = new Map(ofKind(records, "stageOpen").map((r) => [r.content.stageId, r.seq]));
const closes = new Map(ofKind(records, "stageClose").map((r) => [r.content.stageId, r.seq]));
const calls = ofKind(records, "callStarted");
const ends = new Map(ofKind(records, "callEnded").map((r) => [r.content.call, r.content]));
const usage = ofKind(records, "callUsage");
const rejections = ofKind(records, "rejection");
// ---- 每个阶段:调用、结局、墙钟、用量、退回 ----
const perStage = allStages.map((s) => {
const these = calls.filter((c) => c.content.stageId === s.stageId);
const ids = new Set(these.map((c) => c.content.call));
const endings: Record<string, number> = {};
for (const c of these) {
const e = ends.get(c.content.call);
const k = e === undefined ? "没有结束记录" : e.status === "failed" ? `failed:${e.reason}` : e.status;
endings[k] = (endings[k] ?? 0) + 1;
}
const u = usage.filter((x) => ids.has(x.content.call)).map((x) => x.content.usage);
const rej = rejections.filter((r) => r.content.stageId === s.stageId);
const byKind: Record<string, string[]> = {};
for (const r of rej) (byKind[r.content.kind] ??= []).push(`${r.content.author}${r.content.target === null ? "" : ` → ${r.content.target}`}:${r.content.reason}`);
const close = closes.get(s.stageId);
return {
stageId: s.stageId,
round: s.round,
stage: s.stage,
calls: these.length,
endings,
wallClockSeconds: close === undefined ? null : (at(close) - at(opens.get(s.stageId)!)) / 1000,
callWallSeconds: sum(these.map((c) => {
const e = ends.get(c.content.call);
return e === undefined ? null : measured(e.wallSeconds);
})),
tokens: {
input: sum(u.map((x) => measured(x.inputTokens))),
cacheRead: sum(u.map((x) => measured(x.cacheReadTokens))),
cacheWrite: sum(u.map((x) => measured(x.cacheWriteTokens))),
output: sum(u.map((x) => measured(x.outputTokens))),
cost: sum(u.map((x) => measured(x.cost))),
},
rejections: { count: rej.length, byKind },
};
});
// ---- 停下、交不出、卡住 ----
const events = ofKind(records, "event").map((e) => ({ seq: e.seq, ...e.content }));
const notSubmitted = calls.filter((c) => ends.get(c.content.call)?.status === "notSubmitted").map((c) => ({ call: c.content.call, end: ends.get(c.content.call) }));
const openCalls = calls.filter((c) => !ends.has(c.content.call)).map((c) => c.content.call);
const openStages = allStages.filter((s) => !s.closed).map((s) => s.stageId);
const outcome = existsSync(join(dir, "outcome.json")) ? JSON.parse(readFileSync(join(dir, "outcome.json"), "utf8")) : null;
// ---- 预测 ----
const finals = finalSubmissions(records);
const round1 = allStages.filter((s) => s.round === 1);
const stageId = (name: string) => {
const s = round1.find((x) => x.stage === name);
if (s === undefined) throw new Error(`第 1 轮没有 ${name} 阶段`);
return s.stageId;
};
const predictions = (name: string) =>
finals
.filter((f) => f.content.stageId === stageId(name) && f.content.kind === name)
.map((f) => ({ researcher: f.content.author, candidate: f.content.target!, range: (f.content.content as { range: Range3 }).range }));
const first = predictions("firstPrediction");
const second = predictions("secondPrediction");
const outOfRange = [...first.map((p) => ({ ...p, which: "第一次" })), ...second.map((p) => ({ ...p, which: "第二次" }))].flatMap((p) =>
(["q1", "q5", "q9"] as const)
.filter((q) => p.range[q] > BEST_KNOWN || p.range[q] < BASELINE)
.map((q) => ({ which: p.which, researcher: p.researcher, candidate: p.candidate, q, value: p.range[q], side: p.range[q] > BEST_KNOWN ? "超过已知最好" : "低于基线" })),
);
const changes = first.map((f) => {
const s = second.find((x) => x.researcher === f.researcher && x.candidate === f.candidate);
return {
researcher: f.researcher,
candidate: f.candidate,
first: f.range,
second: s?.range ?? null,
identical: s !== undefined && f.range.q1 === s.range.q1 && f.range.q5 === s.range.q5 && f.range.q9 === s.range.q9,
delta: s === undefined ? null : { q1: s.range.q1 - f.range.q1, q5: s.range.q5 - f.range.q5, q9: s.range.q9 - f.range.q9 },
};
});
// ---- 程序部分 ----
const unpaired = [...opens.keys()].filter((id) => !closes.has(id)).concat([...closes.keys()].filter((id) => !opens.has(id)));
const protocols = ofKind(records, "protocol").map((p) => {
const round = (p.content.protocol as unknown as FrozenProtocol).round;
const response = allStages.find((s) => s.round === round && s.stage === "response");
const close = response === undefined ? undefined : closes.get(response.stageId);
return { candidate: p.content.candidate, round, seq: p.seq, responseCloseSeq: close ?? null, after: close !== undefined && p.seq > close };
});
const lastHalt = events.filter((e) => e.event === HALTED).at(-1) ?? null;
const checks = {
"每个阶段开关成对": { pass: unpaired.length === 0, unpaired },
"冻结协议都写在回应阶段关闭之后": { pass: protocols.length > 0 && protocols.every((p) => p.after), protocols },
"最后停下原因是实现还没写": {
pass: lastHalt !== null && lastHalt.reason.includes(EXPECTED_STOP) && lastHalt.seq === records.at(-1)!.seq,
lastHalt,
lastRecord: { seq: records.at(-1)!.seq, kind: records.at(-1)!.kind },
},
};
const times = [...seen.values()];
return {
study: join(dir, "study"),
totalWallClockSeconds: (Math.max(...times) - Math.min(...times)) / 1000,
perStage,
events,
notSubmitted,
stuck: { openCalls, openStages },
outcome,
predictions: { bestKnown: BEST_KNOWN, baseline: BASELINE, outOfRange, changes },
checks,
};
}
for (const dir of dirs) {
const s = statsOf(dir);
writeFileSync(join(dir, "stats.json"), `${JSON.stringify(s, null, 2)}\n`);
const f = (x: number) => x.toFixed(0);
console.log(`\n== ${dir}(总墙钟 ${f(s.totalWallClockSeconds)} 秒)`);
for (const p of s.perStage) {
const rej = Object.entries(p.rejections.byKind).map(([k, v]) => `${k} ${v.length}`).join("、") || "无";
console.log(
`${p.stageId}:调用 ${p.calls}(${Object.entries(p.endings).map(([k, v]) => `${k} ${v}`).join(",")}),墙钟 ${p.wallClockSeconds === null ? "未关闭" : `${f(p.wallClockSeconds)} 秒`},` +
`输入 ${p.tokens.input.total}/输出 ${p.tokens.output.total} token,费用 ${p.tokens.cost.total.toFixed(2)}${p.tokens.cost.unmeasured > 0 ? `(${p.tokens.cost.unmeasured} 条测不到)` : ""},退回 ${rej}`,
);
}
console.log(`事件:${s.events.map((e) => `#${e.seq} ${e.event}:${e.reason}`).join(";")}`);
console.log(`交不出:${s.notSubmitted.length};卡住:未结束的调用 ${JSON.stringify(s.stuck.openCalls)},未关闭的阶段 ${JSON.stringify(s.stuck.openStages)}`);
console.log(`超出范围的预测:${s.predictions.outOfRange.map((o) => `${o.which} ${o.researcher}→${o.candidate} ${o.q}=${o.value}(${o.side})`).join(";") || "无"}`);
for (const c of s.predictions.changes) {
const r = (x: Range3 | null) => (x === null ? "没有" : `${x.q1}/${x.q5}/${x.q9}`);
console.log(` ${c.researcher}→${c.candidate}:第一次 ${r(c.first)},第二次 ${r(c.second)}${c.identical ? "(完全相同)" : ""}`);
}
for (const [k, v] of Object.entries(s.checks)) console.log(`${v.pass ? "通过" : "不通过"} ${k}`);
}