eval: 完成任务书15路线对照与先想后读核查

This commit is contained in:
2026-10-04 01:31:50 -04:00
parent 18235cc9b1
commit 856845cd71
11 changed files with 448 additions and 4 deletions
+69
View File
@@ -0,0 +1,69 @@
import { writeFileSync } from "node:fs";
import { pyDumps, pyRepr, readRun, type J } from "./py.ts";
// 任务书 15 第 4.1 节:把一次运行的第零轮和第 1 轮导出成一份纯文本,供子 Agent 按附录一问卷阅读。
// 逐行移植对照材料里的 extract.py,输出逐字节一致(含原脚本的写法:结尾不带换行;
// 认领入口名是从调用记录的 JSON 文字里用正则抠的,名字里的引号、换行会带着 JSON 转义)。
// node eval/route/extract.ts <运行目录名> <输出文件>
// 运行目录是 /home/iomgaa/inforsi-eval/<运行目录名>/study/ledger.sqlite,只读打开。
const [run, out] = process.argv.slice(2);
if (run === undefined || out === undefined) throw new Error("用法:node eval/route/extract.ts <运行目录名> <输出文件>");
const rows = readRun(run);
const L: string[] = [];
const subs: J[] = rows.filter(([k]) => k === "submission").map(([, c]) => c);
const decisions = (name: string): J[] => rows.filter(([k, c]) => k === "decision" && c.decision === name).map(([, c]) => c);
L.push(`# 运行 ${run}\n`);
L.push("## 第零轮 一、闭门写的入口(每人按自己写的先后)\n");
for (const s of subs) {
if (s.kind !== "entry") continue;
L.push(`### ${s.author}(第 ${s.version} 版)`);
s.content.entries.forEach((e: J, i: number) => L.push(`#### 第 ${i + 1} 个:${e.name}\n${e.body}\n`));
}
L.push("## 第零轮 二、讨论(按提交先后)\n");
for (const s of subs) {
if (s.kind !== "discussion") continue;
L.push(`### ${s.author}(${s.stageId},第 ${s.version} 版)\n${s.content.body ?? ""}\n`);
if (s.content.entries?.length) {
L.push("换成的新入口清单:");
s.content.entries.forEach((e: J, i: number) => L.push(`#### 新第 ${i + 1} 个:${e.name}\n${e.body}\n`));
}
}
// 认领时的最终入口编号:从第一条认领调用的记录(材料里的入口全文)里抠出来。
const names = new Map<string, [string, string]>();
for (const [k, c] of rows) {
if (k !== "callStarted") continue;
const txt = pyDumps(c, false);
if (!txt.includes("r0-claim")) continue;
for (const m of txt.matchAll(/### 入口 (e-researcher-\d-\d+):(.*?)(作者 (researcher-\d))/g)) names.set(m[1]!, [m[2]!, m[3]!]);
break;
}
L.push("## 第零轮 三、认领时的最终入口编号\n");
for (const [i, [n, a]] of names) L.push(`- ${i}(作者 ${a}):${n}`);
L.push("\n## 第零轮 四、志愿与分配\n");
for (const s of subs) if (s.kind === "claim") L.push(`- ${s.author} 志愿:${s.content.choices.join(" > ")}`);
for (const c of decisions("entries")) L.push("分配:" + pyDumps(c.output, false));
const props = subs.filter((s) => s.kind === "proposal");
if (props.length > 0) {
L.push("\n## 第 1 轮 一、提案全文\n");
for (const s of props) {
L.push(`### ${s.item}(作者 ${s.author})\n${s.content.body}\n`);
L.push("因子:");
for (const f of s.content.factors) L.push(`- ${f.name}:${f.method}`);
L.push("");
}
for (const c of decisions("dedup")) L.push("判重:" + pyDumps(c.output, false));
for (const kind of ["firstPrediction", "secondPrediction"]) {
L.push(`\n## 第 1 轮 ${kind} 三个数(作者→候选)`);
for (const s of subs) if (s.kind === kind) L.push(`- ${s.author} → ${s.target}:${pyRepr(s.content.range)}`);
}
for (const c of decisions("batch")) {
const merged = c.output.candidates.map((x: J) => [x.id, x.merged]);
L.push("\n选批:入选 " + pyRepr(c.output.chosen) + ";合并范围 " + pyDumps(merged, true));
}
L.push("\n## 附:第二次预测正文(看研究者如何判断风险)\n");
for (const s of subs) if (s.kind === "secondPrediction") L.push(`### ${s.author} → ${s.target}\n${s.content.body}\n`);
}
writeFileSync(out, L.join("\n"));
+49
View File
@@ -0,0 +1,49 @@
import { writeFileSync } from "node:fs";
import { pyDumps, readRun, type J } from "./py.ts";
// 任务书 15 第 4.3 节的材料:几次运行的全部最终入口(认领时)和第 1 轮提案,写成一个 items.json,交给 jevroutes.ts 问 Jev。
// 逐行移植对照材料里的 items.py,输出和 Python 的 json.dump(..., ensure_ascii=False, indent=0) 逐字节一致。
// node eval/route/items.ts <输出 items.json> <运行目录名>...
// 运行目录是 /home/iomgaa/inforsi-eval/<运行目录名>/study/ledger.sqlite,只读打开。
const [out, ...runs] = process.argv.slice(2);
if (out === undefined || runs.length === 0) throw new Error("用法:node eval/route/items.ts <输出 items.json> <运行目录名>...");
function* walk(o: J): Generator<string> {
if (typeof o === "string") yield o;
else if (Array.isArray(o)) for (const v of o) yield* walk(v);
else if (o !== null && typeof o === "object") for (const v of Object.values(o)) yield* walk(v);
}
const count = (s: string, sub: string) => s.split(sub).length - 1;
const items: Array<Record<string, string>> = [];
for (const run of runs) {
const rows = readRun(run);
// 最终入口:第一条认领调用的记录里,入口全文最多的那段文字,按"### 入口 e-"切开。
for (const [k, c] of rows) {
if (k !== "callStarted" || !pyDumps(c, false).includes("r0-claim")) continue;
let txt = "";
let best = -1;
for (const s of walk(c)) {
const n = count(s, "### 入口 e-");
if (n > best) [txt, best] = [s, n];
}
for (const p of txt.split(/\n(?=### 入口 e-)/).slice(1)) {
const m = /^### 入口 (e-researcher-\d-\d+):(.*?)(作者 (researcher-\d))\n([\s\S]*)/.exec(p);
if (m === null) throw new Error(`${run} 的认领材料里有一段入口对不上格式:${p.slice(0, 80)}`);
const body = m[4]!.split("\n## ")[0]!.trim();
items.push({ run, id: m[1]!, kind: "entry", name: m[2]!, author: m[3]!, text: body });
}
break;
}
for (const [k, c] of rows) {
if (k !== "submission" || c.kind !== "proposal") continue;
const cc = c.content;
const t = cc.body + "\n\n因子:\n" + cc.factors.map((f: J) => `- ${f.name}:${f.method}`).join("\n");
items.push({ run, id: c.item, kind: "proposal", name: cc.body.split("\n")[0].replace(/^[# ]+/, ""), author: c.author, text: t });
}
}
const indent0 = (o: Record<string, string>) => "{\n" + Object.entries(o).map(([k, v]) => `${pyDumps(k, false)}: ${pyDumps(v, false)}`).join(",\n") + "\n}";
writeFileSync(out, "[\n" + items.map(indent0).join(",\n") + "\n]");
console.log(items.length);
items.forEach((x, i) => console.log(i, x["run"]!.slice(11), x["id"], [...x["text"]!].length, [...x["name"]!].slice(0, 70).join("")));
+77
View File
@@ -0,0 +1,77 @@
import { readFileSync, writeFileSync } from "node:fs";
import { ask, type Noul } from "../lib/jev.ts";
// 任务书 15 第 4.3 节、附录二:对 items.json 里每条最终入口和提案问 Jev 四问(bold、standard、reach、complete),存原始概率。
// 从对照材料的 jevroutes.ts 原样搬来,只改了文件路径(items.json 从命令行读、jev.ts 用相对路径);问题、问法、并发一字未改。
// 判"是""否"的阈值 0.5 在 ../lib/jev.ts 里,这里只存概率。要花钱。
// npm run eval:route:jev -- <items.json> <输出 json>
// items.json 由 items.ts 生成。
const items = JSON.parse(readFileSync(process.argv[2]!, "utf8")) as Array<{ run: string; id: string; kind: string; name: string; text: string }>;
const TASK =
"Research problem: place 26 non-overlapping circles inside the unit square to maximize the sum of their radii. " +
"The current program (baseline) builds circle centers by a fixed concentric-ring construction (many centers get clipped to the border) and assigns radii by a simple pairwise shrinking loop; it scores 0.9598. " +
"The best known value is 2.635983. Each candidate program runs once on 1 CPU core for at most 120 seconds, with numpy and scipy available. " +
"Researchers wrote the text below either as an 'entry' (where to start looking) or as a 'proposal' (a concrete change to run). The text is in Chinese.";
const q = (question: string, focus: string, yes: string, yesEx: string[], no: string, notFor: string, noEx: string[]): Noul => ({
type: "noul",
instructions: { question, focus },
criteria: { true: { what: yes, examples: yesEx }, false: { what: no, not_for: notFor, examples: noEx } },
});
const QUESTIONS: Record<string, Noul> = {
bold: q(
"Relative to the current program, is this a bold direction: it replaces the core way the answer is produced, and its outcome is highly uncertain (could gain a lot or fail)?",
"Compare against the baseline program described in the task, not against the research literature.",
"Replaces how circle positions are produced or improved with a new mechanism whose result is hard to predict.",
["Throw away the ring construction and let a general solver move all circles.", "Simulate circles growing and pushing each other apart from the current layout."],
"Keeps the current way of producing positions, or only measures/diagnoses, or makes a small step with a predictable result.",
"Do not answer yes just because the text is long or technical.",
["Compute the exact optimal radii for the existing centers.", "Study what the evaluator accepts.", "Try a few grid layouts and pick the best."],
),
standard: q(
"Would an expert in numerical optimization regard the method described here as a standard, well-established approach for this kind of packing problem that usually works reliably?",
"Judge the method itself, independent of how far it is from the baseline program.",
"A textbook or widely used method for continuous constrained packing/layout problems that an expert would expect to run reliably.",
["Optimize all centers and radii together with a constrained nonlinear solver from several structured starts.", "Solve a linear program for radii with centers fixed."],
"An unusual or home-made mechanism whose reliability is unclear, or text that is not a method at all (analysis, framing, measurement plan).",
"Do not answer no merely because the method is simple.",
["Move circles along dual prices of a linear program with a hand-tuned line search.", "List which categories of methods exist and how to spend the time budget."],
),
reach: q(
"If this direction were pursued competently within the 120-second, 1-core limit, could it plausibly reach within 1% of the best known value 2.635983?",
"Ask about the ceiling of the route itself, not whether the text is well written.",
"The route can move all circles freely to a locally optimal arrangement and search over arrangements, so near-best values are within reach.",
["Structured or many starting layouts, each refined by moving all centers and radii continuously to a local optimum."],
"The route is capped by a fixed or rigidly parametrized layout, only adjusts one part of the solution, or does not produce an answer.",
"Do not answer yes for a route just because it mentions optimization in passing.",
["Keep the given centers and only optimize radii.", "Scan a few parameters of hexagonal or grid layouts.", "Measure how far the baseline is from optimal."],
),
complete: q(
"Does this text describe a complete route that would itself produce a final answer (where the starting layout comes from, how the circles are improved, how the final answer is made valid), rather than one isolated component, a measurement, or a framing question?",
"Look for a full pipeline that outputs circles and radii.",
"A full pipeline that by itself outputs a packing that could score well.",
["Generate staggered-row starts, refine each with a constrained optimizer, keep the best, shrink radii slightly to make it valid."],
"Only one component (e.g. a radius step), a diagnostic or measurement, a question about categories or budget, or an evaluator analysis.",
"Do not answer no just because some details are left to the implementer.",
["Replace the radius loop with a linear program, keep everything else.", "Ask which family of methods suits a 120-second budget."],
),
};
const PARALLEL = 6;
let i = 0;
const results: unknown[] = new Array(items.length);
async function worker() {
while (i < items.length) {
const k = i++;
const it = items[k]!;
const state = { task: TASK, kind: it.kind, title: it.name, text: it.text };
const answers = await ask(state, QUESTIONS);
results[k] = { index: k, run: it.run, id: it.id, kind: it.kind, name: it.name, answers: Object.fromEntries(answers.map((a) => [a.question, a.probability])) };
process.stderr.write(".");
}
}
await Promise.all(Array.from({ length: PARALLEL }, worker));
writeFileSync(process.argv[3]!, JSON.stringify(results, null, 1));
console.log("done", results.length);
+91
View File
@@ -0,0 +1,91 @@
import { DatabaseSync } from "node:sqlite";
// extract.ts、items.ts 共用:只读打开对照运行的账本,以及照 Python 的写法把值变成文字。
// 对照材料(/home/iomgaa/inforsi-eval/2026-10-03-route-control/)是 Python 脚本导出的,
// 移植要逐字节一致,所以这里复现 Python 的 json.dumps(默认分隔符 ", " 和 ": ")和 repr(单引号字符串、浮点写法)。
// 账本内容是 JSON 解析出来的任意值;照 Python 的动态写法用 any,便于和 extract.py / items.py 逐行对照。
export type J = any;
/** 运行目录名 → 账本全部记录(种类、内容),按序号。只读打开,不经 Ledger.open,不在对照目录里写任何东西。 */
export function readRun(run: string): Array<[string, J]> {
const db = new DatabaseSync(`/home/iomgaa/inforsi-eval/${run}/study/ledger.sqlite`, { readOnly: true });
try {
const rows = db.prepare("SELECT kind, content FROM records ORDER BY seq").all() as Array<{ kind: string; content: string }>;
return rows.map((r) => [r.kind, JSON.parse(r.content)]);
} finally {
db.close();
}
}
/**
* Python 读 JSON 后的数:没有小数点和指数的是 int,照写;其余是 float,用 repr 的写法:
* 最短的能还原的数字,指数在 [-4, 16) 内写成定点,否则写成 1e-05 这种(指数至少两位)。
* 账本由 JSON.stringify 写成,整数(小于 1e21)都写成不带小数点的样子,所以 Number.isInteger 就是 int。
*/
function pyNumber(x: number): string {
if (Number.isInteger(x) && Math.abs(x) < 1e21) return String(x);
const [mantissa, e] = x.toExponential().split("e") as [string, string];
const exp = Number(e);
const negative = mantissa.startsWith("-");
const digits = mantissa.replace("-", "").replace(".", "");
let body: string;
if (exp >= -4 && exp < 16) {
if (exp < 0) body = "0." + "0".repeat(-exp - 1) + digits;
else if (digits.length > exp + 1) body = digits.slice(0, exp + 1) + "." + digits.slice(exp + 1);
else body = digits + "0".repeat(exp + 1 - digits.length) + ".0";
} else {
const m = digits.length > 1 ? digits[0] + "." + digits.slice(1) : digits;
body = m + "e" + (exp < 0 ? "-" : "+") + String(Math.abs(exp)).padStart(2, "0");
}
return (negative ? "-" : "") + body;
}
const hex4 = (c: number) => "\\u" + c.toString(16).padStart(4, "0");
const JSON_ESC: Record<string, string> = { '"': '\\"', "\\": "\\\\", "\n": "\\n", "\r": "\\r", "\t": "\\t", "\b": "\\b", "\f": "\\f" };
/** Python json.dumps(o) / json.dumps(o, ensure_ascii=False)。 */
export function pyDumps(o: J, ensureAscii: boolean): string {
if (o === null) return "null";
if (o === true) return "true";
if (o === false) return "false";
if (typeof o === "number") return pyNumber(o);
if (typeof o === "string") {
let s = '"';
for (const ch of o) {
const c = ch.codePointAt(0)!;
if (JSON_ESC[ch] !== undefined) s += JSON_ESC[ch];
else if (c < 0x20 || (ensureAscii && c > 0x7e)) {
if (c > 0xffff) s += hex4(0xd800 + ((c - 0x10000) >> 10)) + hex4(0xdc00 + ((c - 0x10000) & 0x3ff));
else s += hex4(c);
} else s += ch;
}
return s + '"';
}
if (Array.isArray(o)) return "[" + o.map((v) => pyDumps(v, ensureAscii)).join(", ") + "]";
return "{" + Object.entries(o).map(([k, v]) => pyDumps(k, ensureAscii) + ": " + pyDumps(v, ensureAscii)).join(", ") + "}";
}
/** Python 的 str(o):列表、字典、字符串按 repr。字符串只处理本数据里会出现的转义(引号、反斜杠、控制字符)。 */
export function pyRepr(o: J): string {
if (o === null) return "None";
if (o === true) return "True";
if (o === false) return "False";
if (typeof o === "number") return pyNumber(o);
if (typeof o === "string") {
const q = o.includes("'") && !o.includes('"') ? '"' : "'";
let s = q;
for (const ch of o) {
const c = ch.codePointAt(0)!;
if (ch === q || ch === "\\") s += "\\" + ch;
else if (ch === "\n") s += "\\n";
else if (ch === "\r") s += "\\r";
else if (ch === "\t") s += "\\t";
else if (c < 0x20 || c === 0x7f) s += "\\x" + c.toString(16).padStart(2, "0");
else s += ch;
}
return s + q;
}
if (Array.isArray(o)) return "[" + o.map(pyRepr).join(", ") + "]";
return "{" + Object.entries(o).map(([k, v]) => pyRepr(k) + ": " + pyRepr(v)).join(", ") + "}";
}
+69
View File
@@ -0,0 +1,69 @@
import { readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
import { join } from "node:path";
// 任务书 15 第 4.4 节思考核查的材料:研究者第一次用 read_source 读基线代码之前,思考里想了什么。
// 对每个"闭门写入口"调用(calls/r0-entryDraft.*)和第 1 轮提案调用(calls/r1-proposal.*),
// 读 calls/<调用>/sessions/ 下的 Pi 会话文件,按时间顺序列出每条助手消息里的思考、文字和工具调用(名字和参数摘要),
// 到第一次读取基线源码文件(含)为止;目录列表、资料文件不截断。另标出首次 read_source,保留两种口径。
// 路径类型按保留的调用工作目录核查;每条消息标原始 JSONL 行号,便于回查。
// node eval/route/thinking.ts <研究目录> <输出文件>
// 研究目录是 /home/iomgaa/inforsi-eval/<运行>/study 这一层(下面有 calls/)。
// 会话文件每行一个 JSON;type 为 "message" 的行里 message.role 是 "assistant" 时,message.content 是一串片段:
// {type:"thinking", thinking}、{type:"text", text}、{type:"toolCall", name, arguments}。
const [study, out] = process.argv.slice(2);
if (study === undefined || out === undefined) throw new Error("用法:node eval/route/thinking.ts <研究目录> <输出文件>");
const callsDir = join(study, "calls");
const calls = readdirSync(callsDir)
.filter((c) => c.startsWith("r0-entryDraft.") || c.startsWith("r1-proposal."))
.sort();
if (calls.length === 0) throw new Error(`${callsDir} 下没有闭门写入口或第 1 轮提案调用`);
type Part = { type: "thinking"; thinking: string } | { type: "text"; text: string } | { type: "toolCall"; name: string; arguments: unknown };
const ARG_CHARS = 300;
/** 一个会话文件到第一次读取基线源码(含)为止的文字。 */
function session(path: string): string {
const L: string[] = [];
let step = 0;
let firstRead = true;
const lines = readFileSync(path, "utf8").split("\n");
for (const [lineIndex, line] of lines.entries()) {
if (line.trim() === "") continue;
const entry = JSON.parse(line) as { type: string; timestamp?: string; message?: { role: string; content: Part[] } };
if (entry.type !== "message" || entry.message?.role !== "assistant") continue;
L.push(`---- 助手第 ${++step} 条(${entry.timestamp};原始 JSONL:${lineIndex + 1})`);
for (const p of entry.message.content) {
if (p.type === "thinking") L.push(`【思考】\n${p.thinking}`);
else if (p.type === "text") L.push(`【文字】\n${p.text}`);
else {
const args = JSON.stringify(p.arguments);
L.push(`【工具】${p.name} ${args.length > ARG_CHARS ? args.slice(0, ARG_CHARS) + "……" : args}`);
if (p.name === "read_source") {
if (firstRead) L.push("(首次 read_source;此处不一定读取源码)");
firstRead = false;
const target = (p.arguments as { path: string }).path;
if (statSync(target).isDirectory()) L.push("(仅列目录,继续保留思考)");
else if (/\.(py|ts|js|cpp|c|h|rs)$/.test(target)) {
return [...L, "(到首次读取基线源码文件为止)"].join("\n\n");
}
}
}
}
}
return [...L, "(整个会话没有读取基线源码文件,以上为全部助手消息)"].join("\n\n");
}
const sections: string[] = [`# 思考核查材料:${study}\n`];
for (const call of calls) {
const dir = join(callsDir, call, "sessions");
const files = readdirSync(dir).filter((f) => f.endsWith(".jsonl")).sort();
if (files.length === 0) throw new Error(`${dir} 下没有会话文件`);
// 文件名以开始时间开头,按名字排就是先后。
files.forEach((f, i) => {
const order = files.length > 1 ? `(第 ${i + 1} 个,共 ${files.length} 个,按开始时间)` : "";
sections.push(`## ${call} / ${f}${order}\n\n${session(join(dir, f))}\n`);
});
}
writeFileSync(out, sections.join("\n"));
console.log(`${calls.length} 个调用,写到 ${out}`);
+32
View File
@@ -0,0 +1,32 @@
import assert from "node:assert/strict";
import { readFileSync, writeFileSync } from "node:fs";
import { forwardMixtures, forwardMergedRange, toImprovement } from "../../src/rules/index.ts";
import { readRun, type J } from "./py.ts";
// 只读复核选批记录的合并分位数;不调用模型或启动研究。
// node eval/route/verify-ranges.ts <blind-map.json> <输出json>
const [mapPath, output] = process.argv.slice(2);
if (mapPath === undefined || output === undefined) throw new Error("需要运行映射和输出路径");
const runs = Object.keys(JSON.parse(readFileSync(mapPath, "utf8")));
const results = runs.map((run) => {
const rows = readRun(run);
const batch = rows.find(([kind, c]) => kind === "decision" && c.decision === "batch")?.[1];
if (batch === undefined) return { run, status: "无第1轮" };
const predictions = rows.filter(([kind, c]) => kind === "submission" && c.kind === "secondPrediction").map(([, c]) => c);
const candidates = batch.output.candidates.map((c: J) => ({
id: c.id,
ranges: predictions.filter((p) => p.target === c.id).map((p) => ({ researcher: p.author, range: p.content.range })),
}));
// 读取选批输入里的基线,不由预测反推。
const base = batch.input.baseline;
assert.equal(typeof base, "number", "选批输入应保存基线分");
const mixtures = forwardMixtures(candidates, batch.output.params.lambda);
const checked = candidates.map((c: J, i: number) => {
const computed = forwardMergedRange(toImprovement(mixtures[i]!, base, "maximize"), base, "maximize");
const recorded = batch.output.candidates[i].merged;
assert.deepEqual(computed, recorded);
return { id: c.id, ranges: c.ranges, recorded, computed, equal: true };
});
return { run, baseline: base, lambda: batch.output.params.lambda, candidates: checked };
});
writeFileSync(output, JSON.stringify(results, null, 2));