78 lines
6.0 KiB
TypeScript
78 lines
6.0 KiB
TypeScript
import { readFileSync, writeFileSync } from "node:fs";
|
|
import { ask, type Noul } from "../lib/jev.ts";
|
|
|
|
// 任务书 15 第 4.3 节、附录二：对 items.json 里每条最终入口和提案问 Jev 四问（bold、standard、reach、complete），存原始概率。
|
|
// 从对照材料的 jevroutes.ts 原样搬来，只改了文件路径（items.json 从命令行读、jev.ts 用相对路径）；问题、问法、并发一字未改。
|
|
// 判"是""否"的阈值 0.5 在 ../lib/jev.ts 里，这里只存概率。要花钱。
|
|
// npm run eval:route:jev -- <items.json> <输出 json>
|
|
// items.json 由 items.ts 生成。
|
|
|
|
const items = JSON.parse(readFileSync(process.argv[2]!, "utf8")) as Array<{ run: string; id: string; kind: string; name: string; text: string }>;
|
|
const TASK =
|
|
"Research problem: place 26 non-overlapping circles inside the unit square to maximize the sum of their radii. " +
|
|
"The current program (baseline) builds circle centers by a fixed concentric-ring construction (many centers get clipped to the border) and assigns radii by a simple pairwise shrinking loop; it scores 0.9598. " +
|
|
"The best known value is 2.635983. Each candidate program runs once on 1 CPU core for at most 120 seconds, with numpy and scipy available. " +
|
|
"Researchers wrote the text below either as an 'entry' (where to start looking) or as a 'proposal' (a concrete change to run). The text is in Chinese.";
|
|
|
|
const q = (question: string, focus: string, yes: string, yesEx: string[], no: string, notFor: string, noEx: string[]): Noul => ({
|
|
type: "noul",
|
|
instructions: { question, focus },
|
|
criteria: { true: { what: yes, examples: yesEx }, false: { what: no, not_for: notFor, examples: noEx } },
|
|
});
|
|
|
|
const QUESTIONS: Record<string, Noul> = {
|
|
bold: q(
|
|
"Relative to the current program, is this a bold direction: it replaces the core way the answer is produced, and its outcome is highly uncertain (could gain a lot or fail)?",
|
|
"Compare against the baseline program described in the task, not against the research literature.",
|
|
"Replaces how circle positions are produced or improved with a new mechanism whose result is hard to predict.",
|
|
["Throw away the ring construction and let a general solver move all circles.", "Simulate circles growing and pushing each other apart from the current layout."],
|
|
"Keeps the current way of producing positions, or only measures/diagnoses, or makes a small step with a predictable result.",
|
|
"Do not answer yes just because the text is long or technical.",
|
|
["Compute the exact optimal radii for the existing centers.", "Study what the evaluator accepts.", "Try a few grid layouts and pick the best."],
|
|
),
|
|
standard: q(
|
|
"Would an expert in numerical optimization regard the method described here as a standard, well-established approach for this kind of packing problem that usually works reliably?",
|
|
"Judge the method itself, independent of how far it is from the baseline program.",
|
|
"A textbook or widely used method for continuous constrained packing/layout problems that an expert would expect to run reliably.",
|
|
["Optimize all centers and radii together with a constrained nonlinear solver from several structured starts.", "Solve a linear program for radii with centers fixed."],
|
|
"An unusual or home-made mechanism whose reliability is unclear, or text that is not a method at all (analysis, framing, measurement plan).",
|
|
"Do not answer no merely because the method is simple.",
|
|
["Move circles along dual prices of a linear program with a hand-tuned line search.", "List which categories of methods exist and how to spend the time budget."],
|
|
),
|
|
reach: q(
|
|
"If this direction were pursued competently within the 120-second, 1-core limit, could it plausibly reach within 1% of the best known value 2.635983?",
|
|
"Ask about the ceiling of the route itself, not whether the text is well written.",
|
|
"The route can move all circles freely to a locally optimal arrangement and search over arrangements, so near-best values are within reach.",
|
|
["Structured or many starting layouts, each refined by moving all centers and radii continuously to a local optimum."],
|
|
"The route is capped by a fixed or rigidly parametrized layout, only adjusts one part of the solution, or does not produce an answer.",
|
|
"Do not answer yes for a route just because it mentions optimization in passing.",
|
|
["Keep the given centers and only optimize radii.", "Scan a few parameters of hexagonal or grid layouts.", "Measure how far the baseline is from optimal."],
|
|
),
|
|
complete: q(
|
|
"Does this text describe a complete route that would itself produce a final answer (where the starting layout comes from, how the circles are improved, how the final answer is made valid), rather than one isolated component, a measurement, or a framing question?",
|
|
"Look for a full pipeline that outputs circles and radii.",
|
|
"A full pipeline that by itself outputs a packing that could score well.",
|
|
["Generate staggered-row starts, refine each with a constrained optimizer, keep the best, shrink radii slightly to make it valid."],
|
|
"Only one component (e.g. a radius step), a diagnostic or measurement, a question about categories or budget, or an evaluator analysis.",
|
|
"Do not answer no just because some details are left to the implementer.",
|
|
["Replace the radius loop with a linear program, keep everything else.", "Ask which family of methods suits a 120-second budget."],
|
|
),
|
|
};
|
|
|
|
const PARALLEL = 6;
|
|
let i = 0;
|
|
const results: unknown[] = new Array(items.length);
|
|
async function worker() {
|
|
while (i < items.length) {
|
|
const k = i++;
|
|
const it = items[k]!;
|
|
const state = { task: TASK, kind: it.kind, title: it.name, text: it.text };
|
|
const answers = await ask(state, QUESTIONS);
|
|
results[k] = { index: k, run: it.run, id: it.id, kind: it.kind, name: it.name, answers: Object.fromEntries(answers.map((a) => [a.question, a.probability])) };
|
|
process.stderr.write(".");
|
|
}
|
|
}
|
|
await Promise.all(Array.from({ length: PARALLEL }, worker));
|
|
writeFileSync(process.argv[3]!, JSON.stringify(results, null, 1));
|
|
console.log("done", results.length);
|