eval: 第 1 轮检验脚本(跑、杀进程、导出去来历、盲评编号、Jev 三问、数字与过程)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,52 @@
|
||||
import { appendFileSync, existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import type { ChildProcess } from "node:child_process";
|
||||
import { sleep } from "./study.ts";
|
||||
|
||||
// 账本里没有时间戳,墙钟时间由检验自己记:研究进程跑着的时候每隔一会儿看一眼账本,
|
||||
// 新出现的每条记录记下"第一次看到它的时刻",写成 timeline.jsonl(每行 {seq, kind, at},at 是毫秒)。
|
||||
// 只读打开账本、只查序号和种类,不经 Ledger.records()(那会每次重读、重查全部内容)。
|
||||
// 时刻比真实写入晚最多一个间隔,用来算阶段级的墙钟足够。
|
||||
|
||||
const POLL_MS = 1000;
|
||||
|
||||
export interface TimelineRow {
|
||||
seq: number;
|
||||
kind: string;
|
||||
at: number;
|
||||
}
|
||||
|
||||
/** 已经记下的最大序号;文件还没有就是 0。接着跑时往同一个文件后面追加。 */
|
||||
function lastSeq(file: string): number {
|
||||
if (!existsSync(file)) return 0;
|
||||
return readTimeline(file).reduce((m, r) => Math.max(m, r.seq), 0);
|
||||
}
|
||||
|
||||
/** 研究进程退出前一直记;进程退出后再看最后一眼。调用前要先等 waitForLedger(表建好了才能读)。 */
|
||||
export async function watchTimeline(studyDir: string, child: ChildProcess, file: string): Promise<void> {
|
||||
let seen = lastSeq(file);
|
||||
const look = () => {
|
||||
const db = new DatabaseSync(join(studyDir, "ledger.sqlite"), { readOnly: true });
|
||||
try {
|
||||
const rows = db.prepare("SELECT seq, kind FROM records WHERE seq > ? ORDER BY seq").all(seen) as Array<{ seq: number; kind: string }>;
|
||||
const at = Date.now();
|
||||
for (const r of rows) appendFileSync(file, `${JSON.stringify({ seq: r.seq, kind: r.kind, at })}\n`);
|
||||
if (rows.length > 0) seen = rows.at(-1)!.seq;
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
};
|
||||
while (child.exitCode === null && child.signalCode === null) {
|
||||
look();
|
||||
await sleep(POLL_MS);
|
||||
}
|
||||
look();
|
||||
}
|
||||
|
||||
export function readTimeline(file: string): TimelineRow[] {
|
||||
return readFileSync(file, "utf8")
|
||||
.split("\n")
|
||||
.filter((l) => l !== "")
|
||||
.map((l) => JSON.parse(l) as TimelineRow);
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { copyFileSync, existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { randomInt } from "node:crypto";
|
||||
|
||||
// 盲评用的汇总(任务书 14 第 3.2 节第 1 步最后一条):把导出目录里每次运行的 text.txt 随机编成"运行 A、B……",
|
||||
// 只把这些文本放进 <检验目录>/blind/;对照表(字母 → 运行名)另存到 blind 外面的 blind-key.json,审阅者看不到。
|
||||
// node eval/round-one/blind.ts <export.ts 写出的检验目录>
|
||||
|
||||
const out = process.argv[2];
|
||||
if (out === undefined) throw new Error("要写 export.ts 写出的检验目录");
|
||||
const runs = readdirSync(join(out, "runs")).sort();
|
||||
if (runs.length > 26) throw new Error(`有 ${runs.length} 次运行,字母不够用`);
|
||||
const blind = join(out, "blind");
|
||||
if (existsSync(blind)) throw new Error(`${blind} 已经有了:重新编号会让旧的对照表作废,先挪走`);
|
||||
mkdirSync(blind);
|
||||
|
||||
// Fisher–Yates 洗牌,用 crypto 的随机数:编号不能从运行名或目录顺序推出来。
|
||||
const order = [...runs];
|
||||
for (let i = order.length - 1; i > 0; i--) {
|
||||
const j = randomInt(i + 1);
|
||||
[order[i], order[j]] = [order[j]!, order[i]!];
|
||||
}
|
||||
const key: Record<string, string> = {};
|
||||
order.forEach((run, i) => {
|
||||
const letter = String.fromCharCode(65 + i);
|
||||
key[`运行 ${letter}`] = run;
|
||||
copyFileSync(join(out, "runs", run, "text.txt"), join(blind, `运行-${letter}.txt`));
|
||||
});
|
||||
writeFileSync(join(out, "blind-key.json"), `${JSON.stringify(key, null, 2)}\n`);
|
||||
console.log(`${order.length} 份文本 → ${blind}\n对照表 → ${join(out, "blind-key.json")}`);
|
||||
@@ -0,0 +1,126 @@
|
||||
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import type { StudyRecord } from "../../src/materials/index.ts";
|
||||
import { ofKind, readLedger } from "../lib/ledger.ts";
|
||||
import { newEvalDir } from "../lib/study.ts";
|
||||
import { Neutral, render } from "./neutral.ts";
|
||||
import { readOurs, readReference, type Json, type Prediction, type RoundData } from "./round-data.ts";
|
||||
|
||||
// 把几次运行的第 1 轮导出成同一种纯文本(任务书 14 第 3.2 节第 1 步),按固定小标题排,去掉来历。
|
||||
// node eval/round-one/export.ts <检验名字> <来源>...
|
||||
// 来源是我们的研究目录(里面有 ledger.sqlite),或参考项目的 board.sqlite(只读,先复制到输出目录)。
|
||||
// 每次运行写进 <检验目录>/runs/<名字>/:data.json(检验用的中间数据,带原编号)、text.txt(给审阅者的文本)、
|
||||
// names.json(原编号 → 中性名的映射,审阅者看不到)。之后用 blind.ts 随机编成"运行 A、B……"。
|
||||
|
||||
const [name, ...sources] = process.argv.slice(2);
|
||||
if (name === undefined || sources.length === 0) throw new Error("用法:node eval/round-one/export.ts <检验名字> <研究目录或 board.sqlite>...");
|
||||
const out = newEvalDir(name);
|
||||
|
||||
for (const source of sources) {
|
||||
const ours = existsSync(join(source, "ledger.sqlite"));
|
||||
// 名字取上一层目录名:我们的研究目录是 <检验目录>/study,取检验目录名;参考项目取运行名(board.sqlite 所在目录)。
|
||||
const label = basename(dirname(source));
|
||||
const dir = join(out, "runs", label);
|
||||
if (existsSync(dir)) throw new Error(`两个来源的名字都是 ${label},分不开`);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
let data: RoundData;
|
||||
if (ours) {
|
||||
const records = readLedger(source);
|
||||
const study = ofKind(records, "config")[0]?.content as StudyRecord | undefined;
|
||||
if (study === undefined) throw new Error(`${source} 的账本里没有研究配置记录`);
|
||||
data = readOurs(records, 1, source, (study.config as { studyId: string }).studyId);
|
||||
} else {
|
||||
data = readReference(source, join(dir, "board-copy"));
|
||||
}
|
||||
const nt = new Neutral(data.hiddenNames);
|
||||
writeFileSync(join(dir, "data.json"), `${JSON.stringify(data, null, 2)}\n`);
|
||||
writeFileSync(join(dir, "text.txt"), renderRound(data, nt));
|
||||
writeFileSync(join(dir, "names.json"), `${JSON.stringify(nt.table(), null, 2)}\n`);
|
||||
console.log(`${label}:提案 ${data.proposals.length}、候选 ${data.dedup.groups.length}、挑刺 ${data.critiques.length}、退回 ${data.rejections.length} → ${dir}`);
|
||||
}
|
||||
console.log(`检验目录:${out}`);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 文本
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function renderRound(d: RoundData, nt: Neutral): string {
|
||||
const out: string[] = ["# 第 1 轮记录(开轮到选批)", ""];
|
||||
const range = (r: { q1: number; q5: number; q9: number }) => `q1 = ${r.q1},q5 = ${r.q5},q9 = ${r.q9}`;
|
||||
const candidates = d.dedup.groups.map((g) => g.candidate);
|
||||
const candidateOf = new Map(d.dedup.groups.flatMap((g) => g.proposals.map((p) => [p, g.candidate] as const)));
|
||||
|
||||
// 先按"提案 → 候选 → 挑刺"的顺序给编号起中性名,正文里引用到的编号就和小标题一致。
|
||||
for (const p of d.proposals) {
|
||||
nt.name("研究者", p.author);
|
||||
nt.name("提案", p.id);
|
||||
}
|
||||
for (const g of d.dedup.groups) nt.name("候选", g.candidate);
|
||||
for (const p of d.proposals) for (const c of d.critiques.filter((x) => x.proposal === p.id)) nt.name("挑刺", c.id);
|
||||
|
||||
out.push("## 一、提案全文", "");
|
||||
for (const p of d.proposals) {
|
||||
out.push(`### ${nt.text(p.id)}(${nt.text(p.author)})`, "");
|
||||
out.push(`作者自评:${p.authorEstimate === null ? "无" : range(p.authorEstimate)}`, "");
|
||||
out.push(render(p.content, nt), "");
|
||||
}
|
||||
|
||||
out.push("## 二、判重结果", "");
|
||||
for (const g of d.dedup.groups) out.push(`- ${nt.text(g.candidate)}:${g.proposals.map((p) => nt.text(p)).join("、")}`);
|
||||
out.push("", "并进留池候选的:", render(d.dedup.carried, nt), "", "判重时每一对的判断(原样):", render(d.dedup.judgments, nt), "");
|
||||
|
||||
out.push("## 三、挑刺和回应", "");
|
||||
for (const p of d.proposals) {
|
||||
const these = d.critiques.filter((c) => c.proposal === p.id);
|
||||
const candidate = candidateOf.get(p.id);
|
||||
out.push(`### 对 ${nt.text(p.id)} 的挑刺${candidate === undefined ? "" : `(${nt.text(candidate)})`}`, "");
|
||||
if (these.length === 0) out.push("(没有挑刺)", "");
|
||||
for (const c of these) {
|
||||
out.push(`#### ${nt.text(c.id)}(${nt.text(c.author)})`, "", render(c.content, nt), "");
|
||||
const replies = d.responses.flatMap((r) => r.replies.filter((x) => x.critique === c.id).map((x) => ({ ...x, author: r.author })));
|
||||
if (replies.length === 0) out.push("作者回应:(没有找到对这条挑刺的回应)", "");
|
||||
for (const x of replies) out.push(`作者回应(${nt.text(x.author)}):`, render(x.text, nt, " "), "");
|
||||
}
|
||||
}
|
||||
for (const r of d.responses) {
|
||||
out.push(`### ${nt.text(r.candidate)} 回应里逐条答复以外的部分(${nt.text(r.author)},${r.kind === "unification" ? "统一" : "回应"})`, "");
|
||||
out.push(render(r.rest, nt), "");
|
||||
}
|
||||
|
||||
out.push("## 四、冻结协议", "");
|
||||
for (const p of d.protocols) out.push(`### ${nt.text(p.candidate)}`, "", render(p.content, nt), "");
|
||||
|
||||
out.push("## 五、两次预测的三个数", "");
|
||||
const predictions = (title: string, list: Prediction[] | null, missing: string) => {
|
||||
out.push(`### ${title}`, "");
|
||||
if (list === null) return out.push(missing, "");
|
||||
const order = (p: Prediction) => [candidates.indexOf(p.candidate), nt.text(p.researcher)] as const;
|
||||
const sorted = [...list].sort((a, b) => order(a)[0] - order(b)[0] || order(a)[1].localeCompare(order(b)[1]));
|
||||
for (const p of sorted) {
|
||||
out.push(`- ${nt.text(p.candidate)},${nt.text(p.researcher)}:${range(p.range)}`);
|
||||
if (p.note !== null) out.push(render(p.note as Json, nt, " "));
|
||||
}
|
||||
out.push("");
|
||||
};
|
||||
predictions("第一次预测", d.firstPrediction, "无此阶段");
|
||||
predictions("第二次预测", d.secondPrediction, "不适用:这次运行用投票选批,不是范围预测。投票原样如下:");
|
||||
if (d.votes !== null) out.push(render(d.votes, nt), "");
|
||||
|
||||
out.push("## 六、选批结果", "");
|
||||
out.push(`选中:${d.batch.chosen.length === 0 ? "无" : d.batch.chosen.map((c) => nt.text(c)).join("、")}`, "", "选批记录(原样):", render(d.batch.detail, nt), "");
|
||||
|
||||
out.push("## 七、退回的次数和原因", "");
|
||||
const byStage = new Map<string, number>();
|
||||
for (const r of d.rejections) byStage.set(r.stage, (byStage.get(r.stage) ?? 0) + 1);
|
||||
out.push(`共 ${d.rejections.length} 次${byStage.size === 0 ? "" : `:${[...byStage].map(([s, n]) => `${s} ${n} 次`).join(",")}`}`, "");
|
||||
d.rejections.forEach((r, i) => {
|
||||
out.push(`${i + 1}. ${r.stage},${nt.text(r.author)}${r.target === null ? "" : `,对象 ${nt.text(r.target)}`}:`, render(clip(r.reason), nt, " "), "");
|
||||
});
|
||||
return `${out.join("\n")}\n`;
|
||||
}
|
||||
|
||||
/** 退回原因只留开头 800 字:形状不合格的退回会把整份交上来的内容附在后面,全文在 data.json 里。 */
|
||||
function clip(s: string): string {
|
||||
const keep = 800;
|
||||
return s.length <= keep ? s : `${s.slice(0, keep)}\n(以下省略 ${s.length - keep} 字)`;
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import { readdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { ask, THRESHOLD, THRESHOLD_NOTE, type Answer, type Noul } from "../lib/jev.ts";
|
||||
import { ADDRESSED, CARRIES_IDEA, SAME_CHANGE, STAYS_IN_DEAD_END, TASKS } from "./questions.ts";
|
||||
import type { RoundData } from "./round-data.ts";
|
||||
|
||||
// 第 1 轮的 Jev 封闭问题(任务书 14 第 3.2 节第 3 步),对 export.ts 导出的每次运行都做(我们的和参考项目的),读 data.json:
|
||||
// a. 判重对不对:本轮提案两两、两个顺序各问一次"是不是同一个改法",取平均,和程序的判重决定对照;
|
||||
// b. 挑刺有没有被回应:每条挑刺问一次(找不到回应的不问,直接记"没有回应");
|
||||
// c. 提案对应研究路径的哪一步:每条提案对 K1–K5(做没做这个想法)和 D1、D2(是不是整个停在这条死路)各问一次,只描述。
|
||||
// 阈值未校准,概率大于 0.5 判"是";原始概率全部存进 <检验目录>/jevcheck.json。要花钱(Jev)。
|
||||
// node eval/round-one/jevcheck.ts <export.ts 写出的检验目录>
|
||||
|
||||
type Idea = { id: string; name: string; description: string };
|
||||
const ideas = JSON.parse(readFileSync("eval/round-zero/ideas.json", "utf8")) as { key: Idea[]; deadEnds: Idea[] };
|
||||
|
||||
const out = process.argv[2];
|
||||
if (out === undefined) throw new Error("要写 export.ts 写出的检验目录");
|
||||
|
||||
/** 同时最多几个请求在路上:一下发上百个容易被限流。 */
|
||||
const PARALLEL = 6;
|
||||
let running = 0;
|
||||
const waiting: Array<() => void> = [];
|
||||
async function limited<T>(fn: () => Promise<T>): Promise<T> {
|
||||
if (running >= PARALLEL) await new Promise<void>((ok) => waiting.push(ok));
|
||||
running++;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
running--;
|
||||
waiting.shift()?.();
|
||||
}
|
||||
}
|
||||
async function one(state: Record<string, unknown>, questions: Record<string, Noul>): Promise<Answer> {
|
||||
const [answer] = await limited(() => ask(state, questions));
|
||||
return answer!;
|
||||
}
|
||||
|
||||
async function check(d: RoundData) {
|
||||
const change = new Map(d.proposals.map((p) => [p.id, p.change]));
|
||||
const groupOf = new Map(d.dedup.groups.flatMap((g) => g.proposals.map((p) => [p, g.candidate] as const)));
|
||||
|
||||
// a. 判重
|
||||
const pairQuestion = (a: string, b: string) => one({ task: TASKS.pair, proposal_a: change.get(a), proposal_b: change.get(b) }, SAME_CHANGE);
|
||||
const pairs = await Promise.all(
|
||||
d.proposals.flatMap((a, i) =>
|
||||
d.proposals.slice(i + 1).map(async (b) => {
|
||||
const [ab, ba] = await Promise.all([pairQuestion(a.id, b.id), pairQuestion(b.id, a.id)]);
|
||||
const mean = (ab.probability + ba.probability) / 2;
|
||||
return { pair: [a.id, b.id], ab, ba, mean, jevSame: mean > THRESHOLD, programMerged: groupOf.get(a.id) === groupOf.get(b.id) };
|
||||
}),
|
||||
),
|
||||
);
|
||||
const dedup = {
|
||||
pairs,
|
||||
jevYesProgramNotMerged: pairs.filter((p) => p.jevSame && !p.programMerged).map((p) => p.pair),
|
||||
jevNoProgramMerged: pairs.filter((p) => !p.jevSame && p.programMerged).map((p) => p.pair),
|
||||
};
|
||||
|
||||
// b. 挑刺有没有被回应
|
||||
const critiques = await Promise.all(
|
||||
d.critiques.map(async (c) => {
|
||||
const response = d.responses.find((r) => r.replies.some((x) => x.critique === c.id));
|
||||
if (response === undefined) return { critique: c.id, author: c.author, proposal: c.proposal, replied: false as const, answer: null };
|
||||
const reply = response.replies.find((x) => x.critique === c.id)!.text;
|
||||
const answer = await one({ task: TASKS.critique, critique: c.content, reply, revision: response.rest }, ADDRESSED);
|
||||
return { critique: c.id, author: c.author, proposal: c.proposal, replied: true as const, answer };
|
||||
}),
|
||||
);
|
||||
|
||||
// c. 研究路径
|
||||
const path = await Promise.all(
|
||||
d.proposals.map(async (p) => {
|
||||
const ask = (idea: Idea, questions: Record<string, Noul>) =>
|
||||
one({ task: TASKS.idea, proposal: p.change, idea: { description: idea.description } }, questions).then((answer) => ({ idea: idea.id, answer }));
|
||||
const answers = await Promise.all([...ideas.key.map((k) => ask(k, CARRIES_IDEA)), ...ideas.deadEnds.map((k) => ask(k, STAYS_IN_DEAD_END))]);
|
||||
return { proposal: p.id, author: p.author, answers };
|
||||
}),
|
||||
);
|
||||
return { dedup, critiques, path };
|
||||
}
|
||||
|
||||
const report: Record<string, Awaited<ReturnType<typeof check>> & { source: string }> = {};
|
||||
for (const run of readdirSync(join(out, "runs")).sort()) {
|
||||
const d = JSON.parse(readFileSync(join(out, "runs", run, "data.json"), "utf8")) as RoundData;
|
||||
report[run] = { source: d.source, ...(await check(d)) };
|
||||
const r = report[run];
|
||||
const p = (x: number) => x.toFixed(2);
|
||||
console.log(`\n== ${run}`);
|
||||
console.log(`a 判重:${r.dedup.pairs.map((x) => `${x.pair.join("~")} ${p(x.ab.probability)}/${p(x.ba.probability)} 平均 ${p(x.mean)}${x.programMerged ? "(程序合了)" : ""}`).join(";")}`);
|
||||
console.log(` 检验判是而程序没合:${JSON.stringify(r.dedup.jevYesProgramNotMerged)};检验判否而程序合了:${JSON.stringify(r.dedup.jevNoProgramMerged)}`);
|
||||
const answered = r.critiques.filter((c) => c.answer?.yes === true).length;
|
||||
console.log(`b 挑刺被回应:${answered}/${r.critiques.length};${r.critiques.map((c) => `${c.critique.slice(0, 40)} ${c.replied ? p(c.answer!.probability) : "没有回应"}`).join(";")}`);
|
||||
for (const x of r.path) console.log(`c ${x.proposal}:${x.answers.map((a) => `${a.idea} ${p(a.answer.probability)}${a.answer.yes ? "是" : ""}`).join(" ")}`);
|
||||
}
|
||||
writeFileSync(join(out, "jevcheck.json"), `${JSON.stringify({ thresholdNote: THRESHOLD_NOTE, runs: report }, null, 2)}\n`);
|
||||
console.log(`\n(${THRESHOLD_NOTE})原始概率:${join(out, "jevcheck.json")}`);
|
||||
@@ -0,0 +1,154 @@
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { LedgerRecord } from "../../src/ledger/index.ts";
|
||||
import { callId, type DedupOutput } from "../../src/materials/index.ts";
|
||||
import { ofKind, readLedger, stages, type Of } from "../lib/ledger.ts";
|
||||
import { launch, newEvalDir, sleep, waitForLedger, writeConfig } from "../lib/study.ts";
|
||||
import { watchTimeline } from "../lib/timeline.ts";
|
||||
|
||||
// 真实运行里的杀进程检验(任务书 14 第 3.1 节):和 run.ts 一样从头跑一份研究(真模型、真 Docker、真 Jev,要花钱),
|
||||
// 一边跑一边看账本;第 1 轮第一次预测阶段开着、已有一部分第一次预测被收下、还有人在写时,SIGKILL 控制进程,
|
||||
// 再用恢复入口(--resume)启动让它跑完,查四条:
|
||||
// 1 被杀那次调用补了结束记录(出错、原因"进程中断");
|
||||
// 2 交过的任务(人×候选)没被重派,没交的被重派;
|
||||
// 3 没有重复收下的提交(同人同对象同阶段只有一份收下,除非是替换);
|
||||
// 4 第一次预测阶段照常关闭,往下走到选批后停下(原因"实现还没写")。
|
||||
// node eval/round-one/kill.ts <名字>
|
||||
// 产物和 run.ts 一样(study、timeline.jsonl、outcome.json),另有 kill-report.json;这份研究也算 3 次真实运行之一。
|
||||
|
||||
const POLL_MS = 3000;
|
||||
const KIND = "firstPrediction";
|
||||
/** flow 写停下事件用的事件名(src/flow/next-step.ts 的 EVENTS.halted;flow 只导出 runStudy,这里照抄)。 */
|
||||
const HALTED = "halted";
|
||||
|
||||
const name = process.argv[2];
|
||||
if (name === undefined) throw new Error("要写检验名字,例如 r1-kill-1(只许小写字母、数字和 ._-,它也是研究编号)");
|
||||
const out = newEvalDir(name);
|
||||
const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, null);
|
||||
const studyDir = join(out, "study");
|
||||
const timeline = join(out, "timeline.jsonl");
|
||||
|
||||
function openCalls(r: LedgerRecord[]): Array<Of<"callStarted">> {
|
||||
const ended = new Set(ofKind(r, "callEnded").map((e) => e.content.call));
|
||||
return ofKind(r, "callStarted").filter((c) => !ended.has(c.content.call));
|
||||
}
|
||||
/** 第 1 轮第一次预测阶段(开着的或已关的);还没开是 undefined。 */
|
||||
const predictionStage = (r: LedgerRecord[]) => stages(r).find((s) => s.round === 1 && s.stage === KIND);
|
||||
/** 第一次预测的任务:名单上每个人 × 本轮每个新候选(任务书 14 第 2.3 节)。新候选取第 1 轮的判重决定。 */
|
||||
function tasks(r: LedgerRecord[], roster: string[]): Array<{ actor: string; candidate: string }> {
|
||||
const dedup = ofKind(r, "decision").find((d) => d.content.decision === "dedup" && d.content.round === 1);
|
||||
if (dedup === undefined) throw new Error("第一次预测阶段开了,账本里却没有第 1 轮的判重决定");
|
||||
const fresh = (dedup.content.output as unknown as DedupOutput).fresh.map((g) => g.candidate);
|
||||
return roster.flatMap((actor) => fresh.map((candidate) => ({ actor, candidate })));
|
||||
}
|
||||
const accepted = (r: LedgerRecord[], stageId: string, actor: string, candidate: string) =>
|
||||
ofKind(r, "submission").some((s) => s.content.stageId === stageId && s.content.kind === KIND && s.content.author === actor && s.content.target === candidate);
|
||||
|
||||
/** 打中的条件:阶段开着没关,已有至少一份收下,还有任务没交,而且有人正在写(这样第 1、2 条都查得到东西)。 */
|
||||
function hit(r: LedgerRecord[]): boolean {
|
||||
const s = predictionStage(r);
|
||||
if (s === undefined || s.closed) return false;
|
||||
const all = tasks(r, s.roster);
|
||||
const done = all.filter((t) => accepted(r, s.stageId, t.actor, t.candidate)).length;
|
||||
return done > 0 && done < all.length && openCalls(r).some((c) => c.content.stageId === s.stageId);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 第一次启动,打中就杀
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const first = launch(config, studyDir, {});
|
||||
await waitForLedger(studyDir, first.child);
|
||||
const watching = watchTimeline(studyDir, first.child, timeline);
|
||||
let triggered: LedgerRecord | undefined;
|
||||
while (first.child.exitCode === null && first.child.signalCode === null) {
|
||||
const r = readLedger(studyDir);
|
||||
if (hit(r)) {
|
||||
first.child.kill("SIGKILL");
|
||||
triggered = r.at(-1);
|
||||
break;
|
||||
}
|
||||
await sleep(POLL_MS);
|
||||
}
|
||||
const firstEnd = await first.done;
|
||||
await watching;
|
||||
if (triggered === undefined) {
|
||||
writeFileSync(join(out, "outcome.json"), `${JSON.stringify({ launches: [firstEnd] }, null, 2)}\n`);
|
||||
throw new Error(`没打中:研究进程自己结束了(${JSON.stringify(firstEnd.outcome)}),账本最后一条是 ${JSON.stringify(readLedger(studyDir).at(-1))}`);
|
||||
}
|
||||
// 进程已死,此刻的账本就是被杀时的样子。
|
||||
const atKill = readLedger(studyDir);
|
||||
const killSeq = atKill.at(-1)!.seq;
|
||||
const killedCalls = openCalls(atKill).map((c) => c.content.call);
|
||||
const stage = predictionStage(atKill)!;
|
||||
const taskList = tasks(atKill, stage.roster).map((t) => ({ ...t, submittedBeforeKill: accepted(atKill, stage.stageId, t.actor, t.candidate) }));
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 再启动,跑完
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const second = launch(null, studyDir, {});
|
||||
process.on("SIGINT", () => second.child.kill("SIGINT"));
|
||||
await watchTimeline(studyDir, second.child, timeline);
|
||||
const secondEnd = await second.done;
|
||||
writeFileSync(join(out, "outcome.json"), `${JSON.stringify({ launches: [firstEnd, secondEnd] }, null, 2)}\n`);
|
||||
const r = readLedger(studyDir);
|
||||
|
||||
// 1 被杀那次调用补了结束记录
|
||||
const ends = ofKind(r, "callEnded");
|
||||
const check1 = killedCalls.map((call) => {
|
||||
const end = ends.find((e) => e.content.call === call);
|
||||
return { call, end: end === undefined ? null : { seq: end.seq, ...end.content } };
|
||||
});
|
||||
|
||||
// 2 交过的没重派、没交的重派了。一次调用属于哪个任务,用 materials 的调用编号规则对:同一任务第 n 次发起是 callId(…, n)。
|
||||
const startedAfter = ofKind(r, "callStarted").filter((c) => c.seq > killSeq);
|
||||
const attempts = ofKind(r, "callStarted").length;
|
||||
const check2 = taskList.map((t) => {
|
||||
const ids = new Set(Array.from({ length: attempts }, (_, i) => callId(stage.stageId, t.actor, KIND, t.candidate, i + 1)));
|
||||
const again = startedAfter.filter((c) => ids.has(c.content.call)).map((c) => c.content.call);
|
||||
return { ...t, dispatchedAfterRestart: again, ok: t.submittedBeforeKill === (again.length === 0) };
|
||||
});
|
||||
|
||||
// 3 没有重复收下:第 1 轮每份提交(阶段、作者、种类、对象、编号)的各版版本号是 1..n;第一次预测里被杀前已收下的,被杀后不能再多出一版
|
||||
const round1 = new Set(stages(r).filter((s) => s.round === 1).map((s) => s.stageId));
|
||||
const slots = new Map<string, Array<{ seq: number; version: number }>>();
|
||||
for (const s of ofKind(r, "submission")) {
|
||||
const c = s.content;
|
||||
if (!round1.has(c.stageId)) continue;
|
||||
const key = JSON.stringify([c.stageId, c.author, c.kind, c.target, c.item]);
|
||||
slots.set(key, [...(slots.get(key) ?? []), { seq: s.seq, version: c.version }]);
|
||||
}
|
||||
const check3 = [...slots].map(([slot, versions]) => {
|
||||
const numbered = versions.map((v) => v.version).join(",") === versions.map((_, i) => i + 1).join(",");
|
||||
const [stageId, , kind] = JSON.parse(slot) as [string, string, string];
|
||||
const acrossKill = stageId === stage.stageId && kind === KIND && versions.some((v) => v.seq <= killSeq) && versions.some((v) => v.seq > killSeq);
|
||||
return { slot, versions, replaced: versions.length > 1, ok: numbered && !acrossKill };
|
||||
});
|
||||
|
||||
// 4 阶段照常关闭、往下走到选批后停下
|
||||
const closed = ofKind(r, "stageClose").some((c) => c.content.stageId === stage.stageId);
|
||||
const batch = ofKind(r, "decision").some((d) => d.content.decision === "batch" && d.content.round === 1);
|
||||
const lastHalt = ofKind(r, "event").filter((e) => e.content.event === HALTED).at(-1);
|
||||
|
||||
const checks = {
|
||||
"1 被杀那次调用补了结束记录(出错、进程中断)": {
|
||||
pass: check1.length > 0 && check1.every((c) => c.end !== null && c.end.status === "failed" && c.end.reason.includes("进程中断")),
|
||||
calls: check1,
|
||||
},
|
||||
"2 交过的任务没被重派,没交的被重派": { pass: check2.every((t) => t.ok), tasks: check2 },
|
||||
"3 没有重复收下的提交(替换除外)": { pass: check3.every((s) => s.ok), problems: check3.filter((s) => !s.ok), replaced: check3.filter((s) => s.replaced) },
|
||||
"4 阶段照常关闭,往下走到选批后停下": {
|
||||
pass: closed && batch && lastHalt !== undefined && lastHalt.content.reason.includes("实现还没写"),
|
||||
firstPredictionClosed: closed,
|
||||
batchDecision: batch,
|
||||
lastHalt: lastHalt?.content ?? null,
|
||||
},
|
||||
};
|
||||
const report = { triggeredAt: { seq: triggered.seq, kind: triggered.kind }, killSeq, lastRecordAtKill: atKill.at(-1), tasksAtKill: taskList, outcomeAfterRestart: secondEnd.outcome, checks };
|
||||
writeFileSync(join(out, "kill-report.json"), `${JSON.stringify(report, null, 2)}\n`);
|
||||
console.log(`\n被杀时账本最后一条 #${killSeq} ${atKill.at(-1)!.kind};被杀时正在进行的调用:${killedCalls.join("、") || "无"}`);
|
||||
console.log(`被杀时第一次预测:${taskList.filter((t) => t.submittedBeforeKill).length}/${taskList.length} 个任务已交`);
|
||||
console.log(`再启动后的结局:${JSON.stringify(secondEnd.outcome)}`);
|
||||
for (const [k, v] of Object.entries(checks)) console.log(`${v.pass ? "通过" : "不通过"} ${k}`);
|
||||
console.log(`报告:${join(out, "kill-report.json")}`);
|
||||
@@ -0,0 +1,97 @@
|
||||
import type { Json } from "./round-data.ts";
|
||||
|
||||
// 给审阅者的文本去掉来历(任务书 14 第 3.2 节第 1 步):运行名、模型名、研究编号去掉;
|
||||
// 研究者、提案、候选、挑刺、入口的编号改写成"研究者 1""提案 1"这类中性写法,按出现顺序编,一份文本一张映射表。
|
||||
// 两个项目的编号写法都认:我们的 r1-researcher-2-1、r1-c1、e-researcher-1-4,参考项目的 r1-v1-p-researcher-2-1、r1-v1-c1、
|
||||
// researcher-1-e4、挑刺的事实编号(uuid)。字段名(points 对 factors)不改。
|
||||
|
||||
type Family = "研究者" | "提案" | "候选" | "挑刺" | "入口";
|
||||
|
||||
/** 认编号的写法,按先后试:长的、带别的编号在里面的写法放前面。 */
|
||||
const PATTERNS: Array<[Family | "实验员" | "核验员", string]> = [
|
||||
["挑刺", String.raw`r\d+-researcher-\d+-\d+\.researcher-\d+\.\d+`],
|
||||
["提案", String.raw`r\d+-v\d+-p-researcher-\d+-\d+`],
|
||||
["提案", String.raw`r\d+-researcher-\d+-\d+`],
|
||||
["实验员", String.raw`implementer-[A-Za-z0-9-]+`],
|
||||
["核验员", String.raw`verifier-[A-Za-z0-9-]+`],
|
||||
["入口", String.raw`e-researcher-\d+-\d+`],
|
||||
["入口", String.raw`researcher-\d+-e\d+`],
|
||||
["候选", String.raw`r\d+-v\d+-c\d+`],
|
||||
["候选", String.raw`r\d+-c\d+`],
|
||||
["研究者", String.raw`researcher-\d+`],
|
||||
["挑刺", String.raw`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`],
|
||||
];
|
||||
const ID_RE = new RegExp(PATTERNS.map(([, p]) => `(?<![A-Za-z0-9])(${p})(?![0-9])`).join("|"), "g");
|
||||
|
||||
/** 认不成编号、但能看出来历的东西。 */
|
||||
const SCRUB: Array<[RegExp, string]> = [
|
||||
[/(?<![A-Za-z0-9])\d+[a-z]?-(?:real|tight|evidence|headless|probe|replay)-\d+(?![0-9])/g, "某次运行"],
|
||||
[/\b(?:glm|claude|gpt|kimi|deepseek|qwen|gemini|minimax)[-\w.]*/gi, "某模型"],
|
||||
[/iomgaa|rivalbits|inforsi/gi, "某项目"],
|
||||
[/req_[0-9a-f]{16,}/g, "某请求"],
|
||||
[/\b[0-9a-f]{32,}\b/g, "某哈希"],
|
||||
];
|
||||
|
||||
export class Neutral {
|
||||
private readonly names = new Map<string, string>();
|
||||
private readonly counts = new Map<string, number>();
|
||||
private readonly hidden: RegExp | null;
|
||||
|
||||
constructor(hiddenNames: string[]) {
|
||||
const esc = hiddenNames.map((n) => n.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
||||
this.hidden = esc.length === 0 ? null : new RegExp(esc.sort((a, b) => b.length - a.length).join("|"), "g");
|
||||
}
|
||||
|
||||
/** 一个编号的中性名:第一次见到时按这一类的出现顺序编号。 */
|
||||
name(family: Family | "实验员" | "核验员", raw: string): string {
|
||||
const seen = this.names.get(raw);
|
||||
if (seen !== undefined) return seen;
|
||||
const n = (this.counts.get(family) ?? 0) + 1;
|
||||
this.counts.set(family, n);
|
||||
const named = `${family} ${n}`;
|
||||
this.names.set(raw, named);
|
||||
return named;
|
||||
}
|
||||
|
||||
/** 一段文字里的编号和来历都换掉。研究编号、运行名先换:我们的研究编号里可能带着像编号的片段。 */
|
||||
text(s: string): string {
|
||||
let t = this.hidden === null ? s : s.replace(this.hidden, "某次运行");
|
||||
t = t.replace(ID_RE, (match, ...groups: unknown[]) => {
|
||||
const i = groups.findIndex((g, k) => k < PATTERNS.length && g !== undefined);
|
||||
return this.name(PATTERNS[i]![0], match);
|
||||
});
|
||||
for (const [re, to] of SCRUB) t = t.replace(re, to);
|
||||
return t;
|
||||
}
|
||||
|
||||
/** 映射表(原编号 → 中性名),存进 data 旁边,审阅者看不到。 */
|
||||
table(): Record<string, string> {
|
||||
return Object.fromEntries(this.names);
|
||||
}
|
||||
}
|
||||
|
||||
/** 渲染时去掉的键:只说明来历的哈希、请求编号、轮次编号(编号类的值照常经 text 改写成中性名)。 */
|
||||
const DROP = new Set(["baselineCodeHash", "materialHash", "questionHash", "requestId", "roundId", "protocolId"]);
|
||||
|
||||
/** 把 Json 写成缩进的纯文本:对象一键一行,多行文字缩进成块,数组逐项编号。 */
|
||||
export function render(value: Json, nt: Neutral, indent = ""): string {
|
||||
if (value === null) return `${indent}(空)`;
|
||||
if (typeof value !== "object") return block(nt.text(String(value)), indent);
|
||||
if (Array.isArray(value)) {
|
||||
if (value.length === 0) return `${indent}(无)`;
|
||||
return value.map((v, i) => (isScalar(v) ? `${indent}${i + 1}. ${inline(v, nt)}` : `${indent}${i + 1}.\n${render(v, nt, `${indent} `)}`)).join("\n");
|
||||
}
|
||||
const entries = Object.entries(value).filter(([k]) => !DROP.has(k));
|
||||
if (entries.length === 0) return `${indent}(无)`;
|
||||
return entries
|
||||
.map(([k, v]) => (isScalar(v) && !String(v).includes("\n") ? `${indent}${k}: ${inline(v, nt)}` : `${indent}${k}:\n${render(v, nt, `${indent} `)}`))
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
const isScalar = (v: Json) => v === null || typeof v !== "object";
|
||||
const inline = (v: Json, nt: Neutral) => (v === null ? "(空)" : nt.text(String(v)));
|
||||
const block = (s: string, indent: string) =>
|
||||
s
|
||||
.split("\n")
|
||||
.map((l) => `${indent}${l}`)
|
||||
.join("\n");
|
||||
@@ -0,0 +1,3 @@
|
||||
# 研究路径说明
|
||||
|
||||
26 圆填圆,基线 0.9598(圆心按同心环写死、半径逐对缩放),已知最好 2.635983。同类系统和参考项目的高分运行,大多沿着这样一条路往上走:结构化的起点或多起点 → 把圆心和半径一起交给连续优化器 → 在当前最好解附近加扰动再优化,或退火式接受 → 最后修到严格合法。辅助手段有:固定圆心用线性规划求最优半径、只动贴紧的那几个圆。死路有:只做几何构造(圆心按手排的规则摆、不移动),各系统都卡在 2.2–2.54;只改半径的分法、不动圆心,上限受布局限制。这是常见的一条路,不是唯一正确的路:一条清单上没有、但讲得通的路,同样算往前走了。第 1 轮能走多远取决于起点,参考项目第 1 轮采纳的分数在 1.55 到 2.632 之间。
|
||||
@@ -0,0 +1,146 @@
|
||||
import type { Noul } from "../lib/jev.ts";
|
||||
|
||||
// 第 1 轮检验问 Jev 的问题(任务书 14 第 3.2 节第 3 步):英文、一问只问一件事、带材料说明、判"是"判"否"的说明和反例,
|
||||
// 写法照 src/jev/questions.ts。这些是检验自己的问题,不进 src/jev;阈值没有校准,一律以概率大于 0.5 判"是"。
|
||||
// 提案、挑刺、回应原文是中文,原样放进材料;两个项目的字段名不同(factors 对 points、modification),材料说明里写明。
|
||||
|
||||
const STUDY =
|
||||
"One round of a study that improves a program for the circle packing task: place 26 non-overlapping circles inside the unit square " +
|
||||
"so that the sum of their radii is as large as possible (no tolerance: any overlap or circle outside the square is invalid). " +
|
||||
"The baseline program puts the circles on fixed rings around the center and then shrinks radii pair by pair to remove overlaps; " +
|
||||
"it scores about 0.96 and the best known score is about 2.636. " +
|
||||
"In this round each researcher proposed one code change. A proposal's change is given as a list of separate edits " +
|
||||
"(each with a `name`, a `method` saying what to change and how, and sometimes `keySettings`), or as one `modification` text. " +
|
||||
"The texts are mostly in Chinese.";
|
||||
|
||||
export const TASKS = {
|
||||
pair: `${STUDY} \`proposal_a\` and \`proposal_b\` are the changes of two proposals from the same round.`,
|
||||
idea: `${STUDY} \`proposal\` is the change of one proposal. \`idea\` is a technique from the literature on this task, described in English.`,
|
||||
critique:
|
||||
`${STUDY} After the proposals were written, other researchers wrote critiques of them, and each author replied to every critique ` +
|
||||
"and could revise the proposal before it was frozen as the protocol an implementer must follow. " +
|
||||
"`critique` is one critique of a proposal; `reply` is the author's reply to that critique; " +
|
||||
"`revision` is the rest of the author's response: the changes made to the proposal and the final text, in whatever fields that system used.",
|
||||
};
|
||||
|
||||
/** a. 两条提案是不是同一个改法(照方法.md 4.3 ③ 判重的意思:只差参数的不算同一个改法)。 */
|
||||
export const SAME_CHANGE: Record<string, Noul> = {
|
||||
same_change: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Would carrying out `proposal_a` and carrying out `proposal_b` produce the same code change, so that a single implementation would serve as both?",
|
||||
focus:
|
||||
"Compare what each change does to the program: which parts it changes and by what method, reading every edit. " +
|
||||
"Ignore wording, language, level of detail, names, and how the change is split into edits. " +
|
||||
"The same method with only different settings (step sizes, counts, time budgets, layouts) is not the same change, because each setting needs its own run.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "Both describe the same algorithmic change to the same part of the program with the same settings; they differ only in wording, detail or how the change is split",
|
||||
examples: [
|
||||
"a: 把圆心和半径一起交给约束优化器,从当前布局出发; b: optimize centers and radii jointly with SLSQP starting from the current layout",
|
||||
"a: 半径改用线性规划求解; b: 固定圆心,用 linprog 求半径和最大",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "They change different parts of the program, change the same part by a different method, or use the same method with different settings",
|
||||
not_for: "Two changes that only share the goal of a higher score, or only edit the same function in different ways",
|
||||
examples: [
|
||||
"a: 半径改用线性规划求解; b: 把圆心和半径一起交给约束优化器",
|
||||
"a: 6 行六角网格; b: 5 行六角网格",
|
||||
"a: 多起点重启 20 次; b: 多起点重启 200 次",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
/** b. 作者的回应有没有处理这条挑刺:改了协议,或者给了不改的具体理由。 */
|
||||
export const ADDRESSED: Record<string, Noul> = {
|
||||
critique_addressed: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question:
|
||||
"Does `reply` deal with the concern raised in `critique`, either by a concrete change to the proposal (visible in `reply` or `revision`) or by a specific reason why no change is needed?",
|
||||
focus:
|
||||
"First find the concern in `critique`. Then check whether `reply` engages with that concern: says what was changed and how, or argues specifically why the concern does not apply. " +
|
||||
"Agreeing without saying what changed, promising to consider it, or answering a different point does not deal with the concern. " +
|
||||
"Whether the reply is right is not the question; only whether it deals with the concern.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "The reply states a concrete change that answers the concern, or gives a specific reason tied to the concern for not changing",
|
||||
examples: [
|
||||
"critique: 停止条件没写锚点; reply: 采纳,t0 改为入口处取 perf_counter,并写进协议的停止条件一节",
|
||||
"critique: 这个因子会改变评分; reply: 不改:评分只读 solution.json,这个因子只改内部的起点,具体见协议第 2 节",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "The reply ignores the concern, only agrees or thanks, promises to look later, or answers something else",
|
||||
not_for: "Replies that disagree with the critique but give a specific reason tied to its concern",
|
||||
examples: ["critique: 停止条件没写锚点; reply: 感谢意见,同意", "critique: 两个因子分不开; reply: 我们会在实现时注意"],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
/** c. 提案对研究路径上的关键想法(K1–K5):这条提案的改动是否做了这个想法。 */
|
||||
export const CARRIES_IDEA: Record<string, Noul> = {
|
||||
carries_idea: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Does the code change in `proposal` carry out the technique described in `idea.description`, as the whole change or as one of its parts?",
|
||||
focus:
|
||||
"Read every edit of the proposal. Answer yes if one of the edits, or the change as a whole, does what the technique describes, possibly in other words or in Chinese. " +
|
||||
"Mentioning the technique as a later plan, or sharing only the goal, is not enough.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "The proposal's change includes this technique",
|
||||
examples: [
|
||||
"idea: optimize centers and radii together with a constrained optimizer; proposal: 把 78 个变量交给 SLSQP,约束是在正方形内、两两不重叠",
|
||||
"idea: solve a linear program for the radii with fixed centers; proposal: 半径分配改成 linprog 求半径和最大",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "The proposal's change does not include this technique",
|
||||
not_for: "Proposals that only mention the technique as future work, or only share the goal",
|
||||
examples: [
|
||||
"idea: optimize centers and radii together with a constrained optimizer; proposal: 圆心按六角网格摆,半径用贪心缩放",
|
||||
"idea: solve a linear program for the radii with fixed centers; proposal: 从当前最好解加扰动再跑 SLSQP",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
/** c. 提案对死路(D1、D2):这条提案的改动是不是整个都停在这条死路里。 */
|
||||
export const STAYS_IN_DEAD_END: Record<string, Noul> = {
|
||||
stays_within_idea: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Does the whole code change in `proposal` stay within the approach described in `idea.description`, with no part that goes beyond it?",
|
||||
focus:
|
||||
"Answer yes only if everything the proposal changes falls inside this approach. " +
|
||||
"If any edit uses a continuous optimization that moves the circle centers, or uses the approach only as a helper step " +
|
||||
"(a starting layout, or a way to score layouts inside a larger search), answer no.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "Everything the proposal changes is inside this approach",
|
||||
examples: [
|
||||
"idea: hand-designed placement of centers only; proposal: 换成 6 行六角网格,枚举行数取最好,半径用贪心",
|
||||
"idea: change only the radii with fixed centers; proposal: 圆心不动,半径改用线性规划",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "Some part of the proposal goes beyond this approach, or the approach is only one step of a larger change",
|
||||
not_for: "Proposals that use this approach as a starting layout or a scoring helper inside a search that moves the centers",
|
||||
examples: [
|
||||
"idea: hand-designed placement of centers only; proposal: 从六角起点出发,再用 SLSQP 同时移动圆心和半径",
|
||||
"idea: change only the radii with fixed centers; proposal: 半径用线性规划打分,外层搜索圆心位置",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,352 @@
|
||||
import { copyFileSync, mkdirSync, statSync } from "node:fs";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import type { LedgerRecord } from "../../src/ledger/index.ts";
|
||||
import type { DedupOutput, FrozenProtocol } from "../../src/materials/index.ts";
|
||||
import { finalSubmissions, ofKind, stages } from "../lib/ledger.ts";
|
||||
|
||||
// 一次运行第 1 轮(开轮到选批)的记录,读成同一种形状,导出文本和 Jev 检验都用它(任务书 14 第 3.2 节)。
|
||||
// 两种来源:我们的账本(经 src/ledger 读,像用户一样)和参考项目的账本(只读,先复制到输出目录再打开)。
|
||||
// 内容按原样存 Json,字段名不统一(参考项目叫 points,我们叫 factors),只把检验要用的几样取到固定的键上。
|
||||
// 按名字读,读不到就报错停下:flow 第 2 部分还在写,读错了要立刻看出来,不猜。
|
||||
|
||||
export type Json = null | boolean | number | string | Json[] | { [key: string]: Json };
|
||||
export type JsonObject = { [key: string]: Json };
|
||||
export interface Range3 {
|
||||
q1: number;
|
||||
q5: number;
|
||||
q9: number;
|
||||
}
|
||||
export interface Prediction {
|
||||
researcher: string;
|
||||
candidate: string;
|
||||
range: Range3;
|
||||
/** 预测附的理由(我们的 body、参考项目 14 的 rationale);没有就是 null。 */
|
||||
note: Json;
|
||||
}
|
||||
|
||||
export interface RoundData {
|
||||
/** 来历(运行名、库路径),只进 data.json,不进给审阅者的文本。 */
|
||||
source: string;
|
||||
/** 要从文本里去掉的名字:运行名、研究编号。 */
|
||||
hiddenNames: string[];
|
||||
proposals: Array<{
|
||||
id: string;
|
||||
author: string;
|
||||
content: JsonObject;
|
||||
/** 判重看的"改了什么":我们是因子的名称、做法、关键设定;参考项目是 points 或 modification。 */
|
||||
change: Json;
|
||||
authorEstimate: Range3 | null;
|
||||
}>;
|
||||
dedup: {
|
||||
/** 本轮新候选各合并了哪几条原提案。 */
|
||||
groups: Array<{ candidate: string; proposals: string[] }>;
|
||||
/** 并进留池候选的新提案。 */
|
||||
carried: Json;
|
||||
/** 程序判重时 Jev 每一对的判断,原样。 */
|
||||
judgments: Json;
|
||||
};
|
||||
critiques: Array<{ id: string; author: string; proposal: string; content: JsonObject }>;
|
||||
responses: Array<{
|
||||
candidate: string;
|
||||
author: string;
|
||||
kind: string;
|
||||
replies: Array<{ critique: string; text: string }>;
|
||||
/** 回应里逐条答复以外的部分(改动、实现条款、正文)。 */
|
||||
rest: JsonObject;
|
||||
}>;
|
||||
protocols: Array<{ candidate: string; content: JsonObject }>;
|
||||
/** null:这次运行没有第一次预测这个阶段。 */
|
||||
firstPrediction: Prediction[] | null;
|
||||
/** null:这次运行用投票,不是范围预测,投票原样在 votes 里。 */
|
||||
secondPrediction: Prediction[] | null;
|
||||
votes: Json;
|
||||
batch: { chosen: string[]; detail: Json };
|
||||
rejections: Array<{ stage: string; author: string; target: string | null; reason: string }>;
|
||||
}
|
||||
|
||||
/** 中性的阶段名,两种来源共用。 */
|
||||
export const STAGE_NAMES = {
|
||||
proposal: "提案",
|
||||
firstPrediction: "第一次预测",
|
||||
critique: "挑刺",
|
||||
response: "回应与统一",
|
||||
secondPrediction: "第二次预测",
|
||||
vote: "投票",
|
||||
} as const;
|
||||
type StageName = keyof typeof STAGE_NAMES;
|
||||
|
||||
function need<T>(value: T | undefined, what: string): T {
|
||||
if (value === undefined) throw new Error(`读不到:${what}`);
|
||||
return value;
|
||||
}
|
||||
function field(obj: JsonObject, key: string, where: string): Json {
|
||||
return need(obj[key], `${where} 的字段 ${key}`);
|
||||
}
|
||||
const omit = (obj: JsonObject, keys: string[]): JsonObject => Object.fromEntries(Object.entries(obj).filter(([k]) => !keys.includes(k)));
|
||||
const asObject = (v: unknown) => v as JsonObject;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 我们的账本
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 我们的研究第 round 轮。records 是 readLedger 读出的全部记录,studyId 是研究编号(要从文本里去掉)。 */
|
||||
export function readOurs(records: LedgerRecord[], round: number, source: string, studyId: string): RoundData {
|
||||
const inRound = stages(records).filter((s) => s.round === round);
|
||||
const stageOf = (name: string) => need(inRound.find((s) => s.stage === name), `第 ${round} 轮 ${name} 阶段的开记录`).stageId;
|
||||
const stageName = new Map(inRound.map((s) => [s.stageId, s.stage]));
|
||||
const finals = finalSubmissions(records);
|
||||
const subs = (stage: string, kinds: string[]) => {
|
||||
const id = stageOf(stage);
|
||||
return finals.filter((s) => s.content.stageId === id && kinds.includes(s.content.kind));
|
||||
};
|
||||
const seals = ofKind(records, "seal");
|
||||
|
||||
const proposals = subs("proposal", ["proposal"]).map((s) => {
|
||||
const content = asObject(s.content.content);
|
||||
const factors = field(content, "factors", `提案 ${s.content.item}`) as JsonObject[];
|
||||
const seal = need(seals.find((x) => x.content.proposal === s.seq), `提案 ${s.content.item}(序号 ${s.seq})的作者自评封存`);
|
||||
return {
|
||||
id: need(s.content.item ?? undefined, `提案序号 ${s.seq} 的编号 item`),
|
||||
author: s.content.author,
|
||||
content,
|
||||
change: factors.map((f) => ({ name: f["name"]!, method: f["method"]!, keySettings: f["keySettings"]! })),
|
||||
authorEstimate: seal.content.range,
|
||||
};
|
||||
});
|
||||
|
||||
const decision = (kind: string) =>
|
||||
need(ofKind(records, "decision").find((d) => d.content.decision === kind && d.content.round === round), `第 ${round} 轮的 ${kind} 决定`).content;
|
||||
const dedup = decision("dedup");
|
||||
const dedupOut = dedup.output as unknown as DedupOutput;
|
||||
need(dedupOut.fresh, "判重决定输出的 fresh");
|
||||
need(dedupOut.carried, "判重决定输出的 carried");
|
||||
|
||||
const critiques = subs("critique", ["critique"]).map((s) => ({
|
||||
id: need(s.content.item ?? undefined, `挑刺序号 ${s.seq} 的编号 item`),
|
||||
author: s.content.author,
|
||||
proposal: need(s.content.target ?? undefined, `挑刺序号 ${s.seq} 的对象 target`),
|
||||
content: asObject(s.content.content),
|
||||
}));
|
||||
|
||||
const responses = subs("response", ["response", "unification"]).map((s) => {
|
||||
const content = asObject(s.content.content);
|
||||
const replies = field(content, "replies", `回应序号 ${s.seq}`) as JsonObject[];
|
||||
return {
|
||||
candidate: need(s.content.target ?? undefined, `回应序号 ${s.seq} 的对象 target`),
|
||||
author: s.content.author,
|
||||
kind: s.content.kind,
|
||||
replies: replies.map((r) => ({ critique: r["critique"] as string, text: r["reply"] as string })),
|
||||
rest: omit(content, ["replies"]),
|
||||
};
|
||||
});
|
||||
|
||||
const fresh = new Set(dedupOut.fresh.map((g) => g.candidate));
|
||||
const protocols = ofKind(records, "protocol")
|
||||
.filter((p) => fresh.has(p.content.candidate) && (p.content.protocol as unknown as FrozenProtocol).round === round)
|
||||
.map((p) => ({ candidate: p.content.candidate, content: asObject(p.content.protocol) }));
|
||||
|
||||
const predictions = (stage: string): Prediction[] =>
|
||||
subs(stage, [stage]).map((s) => {
|
||||
const content = asObject(s.content.content);
|
||||
return {
|
||||
researcher: s.content.author,
|
||||
candidate: need(s.content.target ?? undefined, `${stage} 序号 ${s.seq} 的对象 target`),
|
||||
range: field(content, "range", `${stage} 序号 ${s.seq}`) as unknown as Range3,
|
||||
note: content["body"] ?? null,
|
||||
};
|
||||
});
|
||||
|
||||
const batch = decision("batch");
|
||||
const chosen = need(asObject(batch.output)["chosen"], "选批决定输出的 chosen") as string[];
|
||||
|
||||
const rejections = ofKind(records, "rejection")
|
||||
.filter((r) => stageName.has(r.content.stageId))
|
||||
.map((r) => ({
|
||||
stage: STAGE_NAMES[need(ourStage(stageName.get(r.content.stageId)!), `阶段 ${r.content.stageId} 的中性名`)],
|
||||
author: r.content.author,
|
||||
target: r.content.target,
|
||||
reason: r.content.reason,
|
||||
}));
|
||||
|
||||
return {
|
||||
source,
|
||||
hiddenNames: [studyId],
|
||||
proposals,
|
||||
dedup: { groups: dedupOut.fresh, carried: dedupOut.carried as unknown as Json, judgments: dedup.input as Json },
|
||||
critiques,
|
||||
responses,
|
||||
protocols,
|
||||
firstPrediction: predictions("firstPrediction"),
|
||||
secondPrediction: predictions("secondPrediction"),
|
||||
votes: null,
|
||||
batch: { chosen, detail: batch.output as Json },
|
||||
rejections,
|
||||
};
|
||||
}
|
||||
|
||||
function ourStage(stage: string): StageName | undefined {
|
||||
return stage in STAGE_NAMES ? (stage as StageName) : undefined;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 参考项目的账本(/home/iomgaa/rb-acceptance/<运行名>/board.sqlite)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 参考项目的阶段名(stage 编号的最后一段)→ 中性名。挑刺和回应都在 phase review 里,按编号区分。 */
|
||||
const REF_STAGES: Record<string, StageName> = {
|
||||
proposal: "proposal",
|
||||
"predict-blind": "firstPrediction",
|
||||
critique: "critique",
|
||||
response: "response",
|
||||
predict: "secondPrediction",
|
||||
vote: "vote",
|
||||
};
|
||||
|
||||
/** 原库只读:库文件(和非空的 -wal)先复制到 copyDir,再以只读方式打开复制件,原目录里不写任何东西。 */
|
||||
function openCopy(board: string, copyDir: string): DatabaseSync {
|
||||
mkdirSync(copyDir, { recursive: true });
|
||||
const copy = join(copyDir, "board.sqlite");
|
||||
copyFileSync(board, copy);
|
||||
if (statSync(`${board}-wal`, { throwIfNoEntry: false })?.size) copyFileSync(`${board}-wal`, `${copy}-wal`);
|
||||
return new DatabaseSync(copy, { readOnly: true });
|
||||
}
|
||||
|
||||
const refRange = (r: JsonObject): Range3 => ({ q1: r["q10"] as number, q5: r["q50"] as number, q9: r["q90"] as number });
|
||||
|
||||
export function readReference(board: string, copyDir: string): RoundData {
|
||||
const run = basename(dirname(board));
|
||||
const roundId = `${run}.r1`;
|
||||
const db = openCopy(board, copyDir);
|
||||
try {
|
||||
const all = <T>(sql: string, ...args: Array<string | number>) => db.prepare(sql).all(...args) as T[];
|
||||
const stageRows = all<{ id: string }>("SELECT id FROM stages WHERE round_id = ? ORDER BY created_seq", roundId);
|
||||
const stageOf = new Map<StageName, string>();
|
||||
for (const s of stageRows) {
|
||||
const name = REF_STAGES[s.id.split(".").at(-1)!];
|
||||
if (name !== undefined) {
|
||||
if (stageOf.has(name)) throw new Error(`${board}:第 1 轮 ${name} 阶段不止一个(${stageOf.get(name)}、${s.id})`);
|
||||
stageOf.set(name, s.id);
|
||||
}
|
||||
}
|
||||
// 14、12 那两版的 submissions 表还没有 version 列(一人一个 slot 只有一份),按第 1 版读。
|
||||
const versioned = all<{ name: string }>("PRAGMA table_info(submissions)").some((c) => c.name === "version");
|
||||
/** 每份提交(阶段、人、slot)的最后一版,按第一次出现的先后排。 */
|
||||
const subs = (name: StageName) => {
|
||||
const stage = need(stageOf.get(name), `${board} 第 1 轮 ${name} 阶段`);
|
||||
const rows = all<{ actor: string; slot: string; version: number; payload: string }>(
|
||||
`SELECT actor, slot, ${versioned ? "version" : "1 AS version"}, payload FROM submissions WHERE stage_id = ? ORDER BY created_seq`,
|
||||
stage,
|
||||
);
|
||||
const latest = new Map<string, { stage: string; actor: string; slot: string; version: number; payload: JsonObject }>();
|
||||
for (const r of rows) {
|
||||
const key = `${r.actor}\n${r.slot}`;
|
||||
const seen = latest.get(key);
|
||||
if (seen === undefined || seen.version < r.version) latest.set(key, { stage, actor: r.actor, slot: r.slot, version: r.version, payload: JSON.parse(r.payload) });
|
||||
}
|
||||
return [...latest.values()];
|
||||
};
|
||||
const fact = (kind: string) => {
|
||||
const rows = all<{ payload: string }>(
|
||||
"SELECT payload FROM facts WHERE kind = ? AND id LIKE ? ORDER BY created_seq",
|
||||
kind,
|
||||
`${run}.${kind}.${roundId}.%`,
|
||||
);
|
||||
return JSON.parse(need(rows.at(-1), `${board} 第 1 轮的 ${kind}`).payload) as JsonObject;
|
||||
};
|
||||
|
||||
const proposals = subs("proposal").map((s) => {
|
||||
const p = s.payload;
|
||||
const points = p["points"] as JsonObject[] | undefined;
|
||||
const change: Json =
|
||||
points !== undefined
|
||||
? points.map((x) => ({ name: x["name"]!, method: x["method"]!, ...(x["keySettings"] === undefined ? {} : { keySettings: x["keySettings"] }) }))
|
||||
: { modification: field(p, "modification", `${board} 提案 ${s.slot}`) };
|
||||
const estimate = (p["authorRange"] ?? p["range"]) as JsonObject | undefined;
|
||||
return {
|
||||
id: s.slot,
|
||||
author: s.actor,
|
||||
content: omit(p, ["authorId", "id", "roundId", "authorRange", "range"]),
|
||||
change,
|
||||
authorEstimate: estimate === undefined ? null : refRange(estimate),
|
||||
};
|
||||
});
|
||||
|
||||
const dedup = fact("dedup-record");
|
||||
const groups = (field(dedup, "candidates", `${board} dedup-record`) as JsonObject[]).map((c) => ({
|
||||
candidate: c["candidateId"] as string,
|
||||
proposals: c["proposalIds"] as string[],
|
||||
}));
|
||||
|
||||
// 一份挑刺提交(一人对一条原提案)是一条挑刺,编号是它的 review-submission 事实编号:回应里的 critiqueId 指的就是它。
|
||||
const critiqueStage = need(stageOf.get("critique"), `${board} 第 1 轮挑刺阶段`);
|
||||
const critiqueIds = new Map(
|
||||
all<{ id: string; slot: string }>("SELECT id, slot FROM facts WHERE kind = 'review-submission' AND slot LIKE ?", `${critiqueStage}/%`).map((f) => [f.slot, f.id]),
|
||||
);
|
||||
const critiques = subs("critique").map((s) => ({
|
||||
id: need(critiqueIds.get(`${s.stage}/${s.actor}/${s.slot}`), `${board} 挑刺 ${s.actor}/${s.slot} 的事实编号`),
|
||||
author: s.actor,
|
||||
proposal: s.slot,
|
||||
content: omit(s.payload, ["authorId", "proposalId", "roundId"]),
|
||||
}));
|
||||
|
||||
const responses = subs("response").map((s) => ({
|
||||
candidate: s.slot,
|
||||
author: s.actor,
|
||||
kind: "response",
|
||||
replies: (field(s.payload, "critiqueResponses", `${board} 回应 ${s.slot}`) as JsonObject[]).map((r) => ({
|
||||
critique: r["critiqueId"] as string,
|
||||
text: r["response"] as string,
|
||||
})),
|
||||
rest: omit(s.payload, ["authorId", "candidateId", "roundId", "critiqueResponses"]),
|
||||
}));
|
||||
|
||||
const protocols = all<{ payload: string }>(
|
||||
"SELECT payload FROM facts WHERE kind = 'candidate-protocol' AND id LIKE ? ORDER BY created_seq",
|
||||
`${run}.candidate-protocol.${roundId}.%`,
|
||||
).map((f) => {
|
||||
const p = JSON.parse(f.payload) as JsonObject;
|
||||
return { candidate: field(p, "candidateId", `${board} candidate-protocol`) as string, content: omit(p, ["author", "candidateId", "baselineCodeHash"]) };
|
||||
});
|
||||
|
||||
const predictions = (name: StageName): Prediction[] =>
|
||||
subs(name).map((s) => ({
|
||||
researcher: s.actor,
|
||||
candidate: s.slot,
|
||||
range: refRange(field(s.payload, "range", `${board} ${name} ${s.actor}/${s.slot}`) as JsonObject),
|
||||
note: s.payload["rationale"] ?? null,
|
||||
}));
|
||||
const voting = stageOf.has("vote");
|
||||
|
||||
const batch = voting ? fact("vote-tally") : fact("batch-selection");
|
||||
const chosen = (field(batch, "candidates", `${board} 选批`) as JsonObject[]).filter((c) => c["selected"] === true).map((c) => c["candidateId"] as string);
|
||||
|
||||
// 退回:audit 里的 submission-rejected,taskId 是"<轮编号>.v1.<阶段>.<人>.<对象>",只取开轮到选批这几个阶段的。
|
||||
const rejections = all<{ payload: string }>("SELECT payload FROM audit WHERE event = 'submission-rejected' ORDER BY seq").flatMap((a) => {
|
||||
const p = JSON.parse(a.payload) as JsonObject;
|
||||
const task = p["taskId"] as string;
|
||||
const m = new RegExp(`^${roundId.replace(/\./g, "\\.")}\\.v\\d+\\.([a-z-]+)\\.([^.]+)\\.(.+)$`).exec(task);
|
||||
const stage = m === null ? undefined : REF_STAGES[m[1]!];
|
||||
if (stage === undefined) return [];
|
||||
const target = m![3]!;
|
||||
return [{ stage: STAGE_NAMES[stage], author: m![2]!, target: target === m![2] ? null : target, reason: p["reason"] as string }];
|
||||
});
|
||||
|
||||
return {
|
||||
source: board,
|
||||
hiddenNames: [run],
|
||||
proposals,
|
||||
dedup: { groups, carried: dedup["pooled"] ?? null, judgments: dedup["pairs"] ?? null },
|
||||
critiques,
|
||||
responses,
|
||||
protocols,
|
||||
firstPrediction: stageOf.has("firstPrediction") ? predictions("firstPrediction") : null,
|
||||
secondPrediction: voting ? null : predictions("secondPrediction"),
|
||||
votes: voting ? subs("vote").map((s) => ({ researcher: s.actor, candidate: s.slot, ...omit(s.payload, ["voterId", "candidateId", "protocolId"]) })) : null,
|
||||
batch: { chosen, detail: batch },
|
||||
rejections,
|
||||
};
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { launch, newEvalDir, waitForLedger, writeConfig } from "../lib/study.ts";
|
||||
import { watchTimeline } from "../lib/timeline.ts";
|
||||
|
||||
// 用真模型、真 Docker、真 Jev 从头跑一份研究,直到它自己停下(任务书 14 第 3.1 节;预期停在第 1 轮选批之后,
|
||||
// 原因"实现还没写")。配置照 eval/round-zero/research.json,研究编号换成检验名字。要花钱。
|
||||
// node eval/round-one/run.ts <名字>
|
||||
// 产物:<检验目录>/study(研究目录)、timeline.jsonl(每条记录第一次被看到的时刻,stats 用来算墙钟)、outcome.json(结局)。
|
||||
// 中途按 Ctrl-C 会把停下信号传给研究进程;接着跑用 node eval/lib/cli.ts --resume --dir <检验目录>/study。
|
||||
|
||||
const name = process.argv[2];
|
||||
if (name === undefined) throw new Error("要写检验名字,例如 r1-real-1(只许小写字母、数字和 ._-,它也是研究编号)");
|
||||
const out = newEvalDir(name);
|
||||
const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, null);
|
||||
const studyDir = join(out, "study");
|
||||
const study = launch(config, studyDir, {});
|
||||
process.on("SIGINT", () => study.child.kill("SIGINT"));
|
||||
await waitForLedger(studyDir, study.child);
|
||||
await watchTimeline(studyDir, study.child, join(out, "timeline.jsonl"));
|
||||
const { code, signal, outcome } = await study.done;
|
||||
writeFileSync(join(out, "outcome.json"), `${JSON.stringify({ launches: [{ code, signal, outcome }] }, null, 2)}\n`);
|
||||
console.log(`\n结局:${JSON.stringify(outcome)}\n研究目录:${studyDir}`);
|
||||
@@ -0,0 +1,172 @@
|
||||
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { FrozenProtocol } from "../../src/materials/index.ts";
|
||||
import { finalSubmissions, ofKind, readLedger, stages } from "../lib/ledger.ts";
|
||||
import { readTimeline } from "../lib/timeline.ts";
|
||||
|
||||
// 第 1 轮的数字和过程(任务书 14 第 3.2 节第 4 步),只看我们的运行;另查程序部分(第 3.4 节):
|
||||
// 阶段开关成对、冻结协议写在回应阶段关闭之后、最后停下的原因是"实现还没写"。不花钱,只读账本和 timeline.jsonl。
|
||||
// node eval/round-one/stats.ts <检验目录>...(run.ts、kill.ts 写出的目录:study/、timeline.jsonl)
|
||||
// 每个目录写一份 stats.json,并打印摘要。
|
||||
|
||||
/** 已知最好和基线分数(任务书 14 第 3.2 节第 4 步给的数),预测超出这两头的逐条列出。 */
|
||||
const BEST_KNOWN = 2.635983;
|
||||
const BASELINE = 0.9598;
|
||||
/** flow 写停下事件用的事件名(src/flow/next-step.ts 的 EVENTS.halted;flow 只导出 runStudy,这里照抄)。 */
|
||||
const HALTED = "halted";
|
||||
const EXPECTED_STOP = "实现还没写";
|
||||
|
||||
const dirs = process.argv.slice(2);
|
||||
if (dirs.length === 0) throw new Error("要写一个或几个检验目录(run.ts、kill.ts 写出的)");
|
||||
|
||||
type Range3 = { q1: number; q5: number; q9: number };
|
||||
const measured = (m: { status: string; value?: number }) => (m.status === "measured" ? m.value! : null);
|
||||
const sum = (xs: Array<number | null>) => ({ total: xs.reduce<number>((a, x) => a + (x ?? 0), 0), unmeasured: xs.filter((x) => x === null).length });
|
||||
|
||||
function statsOf(dir: string) {
|
||||
const records = readLedger(join(dir, "study"));
|
||||
const timelineFile = join(dir, "timeline.jsonl");
|
||||
if (!existsSync(timelineFile)) throw new Error(`${timelineFile} 不存在:墙钟时间要靠 run.ts、kill.ts 记下的时刻`);
|
||||
const seen = new Map(readTimeline(timelineFile).map((r) => [r.seq, r.at]));
|
||||
const at = (seq: number) => {
|
||||
const t = seen.get(seq);
|
||||
if (t === undefined) throw new Error(`timeline.jsonl 里没有序号 ${seq} 的时刻`);
|
||||
return t;
|
||||
};
|
||||
const allStages = stages(records);
|
||||
const opens = new Map(ofKind(records, "stageOpen").map((r) => [r.content.stageId, r.seq]));
|
||||
const closes = new Map(ofKind(records, "stageClose").map((r) => [r.content.stageId, r.seq]));
|
||||
const calls = ofKind(records, "callStarted");
|
||||
const ends = new Map(ofKind(records, "callEnded").map((r) => [r.content.call, r.content]));
|
||||
const usage = ofKind(records, "callUsage");
|
||||
const rejections = ofKind(records, "rejection");
|
||||
|
||||
// ---- 每个阶段:调用、结局、墙钟、用量、退回 ----
|
||||
const perStage = allStages.map((s) => {
|
||||
const these = calls.filter((c) => c.content.stageId === s.stageId);
|
||||
const ids = new Set(these.map((c) => c.content.call));
|
||||
const endings: Record<string, number> = {};
|
||||
for (const c of these) {
|
||||
const e = ends.get(c.content.call);
|
||||
const k = e === undefined ? "没有结束记录" : e.status === "failed" ? `failed:${e.reason}` : e.status;
|
||||
endings[k] = (endings[k] ?? 0) + 1;
|
||||
}
|
||||
const u = usage.filter((x) => ids.has(x.content.call)).map((x) => x.content.usage);
|
||||
const rej = rejections.filter((r) => r.content.stageId === s.stageId);
|
||||
const byKind: Record<string, string[]> = {};
|
||||
for (const r of rej) (byKind[r.content.kind] ??= []).push(`${r.content.author}${r.content.target === null ? "" : ` → ${r.content.target}`}:${r.content.reason}`);
|
||||
const close = closes.get(s.stageId);
|
||||
return {
|
||||
stageId: s.stageId,
|
||||
round: s.round,
|
||||
stage: s.stage,
|
||||
calls: these.length,
|
||||
endings,
|
||||
wallClockSeconds: close === undefined ? null : (at(close) - at(opens.get(s.stageId)!)) / 1000,
|
||||
callWallSeconds: sum(these.map((c) => {
|
||||
const e = ends.get(c.content.call);
|
||||
return e === undefined ? null : measured(e.wallSeconds);
|
||||
})),
|
||||
tokens: {
|
||||
input: sum(u.map((x) => measured(x.inputTokens))),
|
||||
cacheRead: sum(u.map((x) => measured(x.cacheReadTokens))),
|
||||
cacheWrite: sum(u.map((x) => measured(x.cacheWriteTokens))),
|
||||
output: sum(u.map((x) => measured(x.outputTokens))),
|
||||
cost: sum(u.map((x) => measured(x.cost))),
|
||||
},
|
||||
rejections: { count: rej.length, byKind },
|
||||
};
|
||||
});
|
||||
|
||||
// ---- 停下、交不出、卡住 ----
|
||||
const events = ofKind(records, "event").map((e) => ({ seq: e.seq, ...e.content }));
|
||||
const notSubmitted = calls.filter((c) => ends.get(c.content.call)?.status === "notSubmitted").map((c) => ({ call: c.content.call, end: ends.get(c.content.call) }));
|
||||
const openCalls = calls.filter((c) => !ends.has(c.content.call)).map((c) => c.content.call);
|
||||
const openStages = allStages.filter((s) => !s.closed).map((s) => s.stageId);
|
||||
const outcome = existsSync(join(dir, "outcome.json")) ? JSON.parse(readFileSync(join(dir, "outcome.json"), "utf8")) : null;
|
||||
|
||||
// ---- 预测 ----
|
||||
const finals = finalSubmissions(records);
|
||||
const round1 = allStages.filter((s) => s.round === 1);
|
||||
const stageId = (name: string) => {
|
||||
const s = round1.find((x) => x.stage === name);
|
||||
if (s === undefined) throw new Error(`第 1 轮没有 ${name} 阶段`);
|
||||
return s.stageId;
|
||||
};
|
||||
const predictions = (name: string) =>
|
||||
finals
|
||||
.filter((f) => f.content.stageId === stageId(name) && f.content.kind === name)
|
||||
.map((f) => ({ researcher: f.content.author, candidate: f.content.target!, range: (f.content.content as { range: Range3 }).range }));
|
||||
const first = predictions("firstPrediction");
|
||||
const second = predictions("secondPrediction");
|
||||
const outOfRange = [...first.map((p) => ({ ...p, which: "第一次" })), ...second.map((p) => ({ ...p, which: "第二次" }))].flatMap((p) =>
|
||||
(["q1", "q5", "q9"] as const)
|
||||
.filter((q) => p.range[q] > BEST_KNOWN || p.range[q] < BASELINE)
|
||||
.map((q) => ({ which: p.which, researcher: p.researcher, candidate: p.candidate, q, value: p.range[q], side: p.range[q] > BEST_KNOWN ? "超过已知最好" : "低于基线" })),
|
||||
);
|
||||
const changes = first.map((f) => {
|
||||
const s = second.find((x) => x.researcher === f.researcher && x.candidate === f.candidate);
|
||||
return {
|
||||
researcher: f.researcher,
|
||||
candidate: f.candidate,
|
||||
first: f.range,
|
||||
second: s?.range ?? null,
|
||||
identical: s !== undefined && f.range.q1 === s.range.q1 && f.range.q5 === s.range.q5 && f.range.q9 === s.range.q9,
|
||||
delta: s === undefined ? null : { q1: s.range.q1 - f.range.q1, q5: s.range.q5 - f.range.q5, q9: s.range.q9 - f.range.q9 },
|
||||
};
|
||||
});
|
||||
|
||||
// ---- 程序部分 ----
|
||||
const unpaired = [...opens.keys()].filter((id) => !closes.has(id)).concat([...closes.keys()].filter((id) => !opens.has(id)));
|
||||
const protocols = ofKind(records, "protocol").map((p) => {
|
||||
const round = (p.content.protocol as unknown as FrozenProtocol).round;
|
||||
const response = allStages.find((s) => s.round === round && s.stage === "response");
|
||||
const close = response === undefined ? undefined : closes.get(response.stageId);
|
||||
return { candidate: p.content.candidate, round, seq: p.seq, responseCloseSeq: close ?? null, after: close !== undefined && p.seq > close };
|
||||
});
|
||||
const lastHalt = events.filter((e) => e.event === HALTED).at(-1) ?? null;
|
||||
const checks = {
|
||||
"每个阶段开关成对": { pass: unpaired.length === 0, unpaired },
|
||||
"冻结协议都写在回应阶段关闭之后": { pass: protocols.length > 0 && protocols.every((p) => p.after), protocols },
|
||||
"最后停下原因是实现还没写": {
|
||||
pass: lastHalt !== null && lastHalt.reason.includes(EXPECTED_STOP) && lastHalt.seq === records.at(-1)!.seq,
|
||||
lastHalt,
|
||||
lastRecord: { seq: records.at(-1)!.seq, kind: records.at(-1)!.kind },
|
||||
},
|
||||
};
|
||||
|
||||
const times = [...seen.values()];
|
||||
return {
|
||||
study: join(dir, "study"),
|
||||
totalWallClockSeconds: (Math.max(...times) - Math.min(...times)) / 1000,
|
||||
perStage,
|
||||
events,
|
||||
notSubmitted,
|
||||
stuck: { openCalls, openStages },
|
||||
outcome,
|
||||
predictions: { bestKnown: BEST_KNOWN, baseline: BASELINE, outOfRange, changes },
|
||||
checks,
|
||||
};
|
||||
}
|
||||
|
||||
for (const dir of dirs) {
|
||||
const s = statsOf(dir);
|
||||
writeFileSync(join(dir, "stats.json"), `${JSON.stringify(s, null, 2)}\n`);
|
||||
const f = (x: number) => x.toFixed(0);
|
||||
console.log(`\n== ${dir}(总墙钟 ${f(s.totalWallClockSeconds)} 秒)`);
|
||||
for (const p of s.perStage) {
|
||||
const rej = Object.entries(p.rejections.byKind).map(([k, v]) => `${k} ${v.length}`).join("、") || "无";
|
||||
console.log(
|
||||
`${p.stageId}:调用 ${p.calls}(${Object.entries(p.endings).map(([k, v]) => `${k} ${v}`).join(",")}),墙钟 ${p.wallClockSeconds === null ? "未关闭" : `${f(p.wallClockSeconds)} 秒`},` +
|
||||
`输入 ${p.tokens.input.total}/输出 ${p.tokens.output.total} token,费用 ${p.tokens.cost.total.toFixed(2)}${p.tokens.cost.unmeasured > 0 ? `(${p.tokens.cost.unmeasured} 条测不到)` : ""},退回 ${rej}`,
|
||||
);
|
||||
}
|
||||
console.log(`事件:${s.events.map((e) => `#${e.seq} ${e.event}:${e.reason}`).join(";")}`);
|
||||
console.log(`交不出:${s.notSubmitted.length};卡住:未结束的调用 ${JSON.stringify(s.stuck.openCalls)},未关闭的阶段 ${JSON.stringify(s.stuck.openStages)}`);
|
||||
console.log(`超出范围的预测:${s.predictions.outOfRange.map((o) => `${o.which} ${o.researcher}→${o.candidate} ${o.q}=${o.value}(${o.side})`).join(";") || "无"}`);
|
||||
for (const c of s.predictions.changes) {
|
||||
const r = (x: Range3 | null) => (x === null ? "没有" : `${x.q1}/${x.q5}/${x.q9}`);
|
||||
console.log(` ${c.researcher}→${c.candidate}:第一次 ${r(c.first)},第二次 ${r(c.second)}${c.identical ? "(完全相同)" : ""}`);
|
||||
}
|
||||
for (const [k, v] of Object.entries(s.checks)) console.log(`${v.pass ? "通过" : "不通过"} ${k}`);
|
||||
}
|
||||
+7
-1
@@ -17,7 +17,13 @@
|
||||
"eval:r0:extract": "node eval/round-zero/extract-16.ts",
|
||||
"eval:r0:replay": "node eval/round-zero/replay.ts",
|
||||
"eval:r0:kill": "node eval/round-zero/kill.ts",
|
||||
"eval:r0:concurrent": "node eval/round-zero/concurrent.ts"
|
||||
"eval:r0:concurrent": "node eval/round-zero/concurrent.ts",
|
||||
"eval:r1:run": "set -a && . /home/iomgaa/Projects/RivalBits/.env && set +a && node eval/round-one/run.ts",
|
||||
"eval:r1:kill": "set -a && . /home/iomgaa/Projects/RivalBits/.env && set +a && node eval/round-one/kill.ts",
|
||||
"eval:r1:export": "node eval/round-one/export.ts",
|
||||
"eval:r1:blind": "node eval/round-one/blind.ts",
|
||||
"eval:r1:jevcheck": "set -a && . /home/iomgaa/Projects/RivalBits/.env && set +a && node eval/round-one/jevcheck.ts",
|
||||
"eval:r1:stats": "node eval/round-one/stats.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-ai": "0.85.1",
|
||||
|
||||
Reference in New Issue
Block a user