feat: flow 第 1 部分（骨架、研究开始、第零轮）与第零轮检验
按任务书 12 实现 src/flow：从账本现算下一步的纯函数、主循环、恢复入口、派模型任务和跑正式运行两个共用动作、冻结配置、基线、第零轮三个研究者阶段和入口分配，走到第 1 轮时停下。materials 导出 finalEntries、加 researcherId 和 StudyRecord。 审阅采纳两条：研究编号本机已用过时冻结前报错；docker run 没启动时写完结果就停下，不在本进程里无限起新编号。 eval/ 新增第零轮检验：起研究命令行、回放假模型、检验用 Jev 客户端、六问质量评估、16-real-1 回放、杀进程与两份同时跑。tsc 与 depcruise 也检查 eval/，新增 eval 只经 index.ts 用正式代码的规则。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,176 @@
|
||||
import type { Noul } from "../lib/jev.ts";
|
||||
|
||||
// 第零轮质量评估问 Jev 的问题（任务书 12 第 5.2 节"问 Jev 的规矩"）：英文、一问只问一件事、
|
||||
// 带材料说明、判"是"判"否"的说明和反例；写法照 src/jev/questions.ts。这些是检验自己的问题，不进 src/jev。
|
||||
// 任务书里两处一句话里有两个条件（第 3 问"D1 或 D2"、第 5 问"写出改成什么，或者预计分数"），按"一问一事"拆成两问，
|
||||
// 在 quality.ts 里用"任一判是"合起来。
|
||||
|
||||
const TASK =
|
||||
"Round zero of a study that improves a program for the circle packing task: place 26 non-overlapping circles inside the unit square " +
|
||||
"so that the sum of their radii is as large as possible (no tolerance: any overlap or circle outside the square is invalid). " +
|
||||
"The baseline program puts the circles on fixed rings around the center and then shrinks radii pair by pair to remove overlaps; " +
|
||||
"it scores about 0.96 and the best known score is about 2.636. " +
|
||||
"In round zero each researcher writes entries: starting points for a line of work that one researcher will follow in the next rounds. " +
|
||||
"An entry has a `name` and a `body` (in Chinese) saying where to start looking and why the route is worth walking. " +
|
||||
"Entries are meant to name a starting point only, not the finished code change or an expected score.";
|
||||
|
||||
export const TASKS = {
|
||||
idea: `${TASK} \`idea\` is a technique from the literature on this task, described in English.`,
|
||||
entry: TASK,
|
||||
pair: `${TASK} \`entry_a\` and \`entry_b\` were written by two different researchers.`,
|
||||
};
|
||||
|
||||
/** 第 1 问：按这个入口走，会直接走到这个想法吗（idea 是 K1–K5 之一）。 */
|
||||
export const LEADS_TO_IDEA: Record<string, Noul> = {
|
||||
leads_to_idea: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Will a researcher who follows `entry` arrive directly at the technique described in `idea.description`?",
|
||||
focus:
|
||||
"Read `entry.body` for what the researcher will look at first and what they intend to try or measure. " +
|
||||
"Answer yes if the entry names this technique, or if this technique is the natural first code change on the route the entry points to. " +
|
||||
"The entry may describe the technique in other words, in Chinese. Sharing the goal of a higher score, or reading the same function, is not enough.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "The entry names this technique, or the route it points to leads straight to this technique as its first code change",
|
||||
examples: [
|
||||
"idea: optimize centers and radii together with a constrained optimizer; entry: 把 26 个圆心和半径一起交给 scipy 的约束优化器",
|
||||
"idea: solve a linear program for the radii with fixed centers; entry: 先问给定圆心下半径和的最大值怎么算，它是不是一个线性规划",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "The entry heads somewhere else: a different technique, only reading rules, or only measuring, so this technique would not be its next step",
|
||||
not_for: "Entries that only share the goal of raising the score, or only read the code this technique would change, without heading for the technique",
|
||||
examples: [
|
||||
"idea: optimize centers and radii together with a constrained optimizer; entry: 只读评测脚本，弄清容差和合法化规则",
|
||||
"idea: solve a linear program for the radii with fixed centers; entry: 把三环布局换成六角网格",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
/** 第 3 问后半：这个入口的路线是不是只停在这条死路上（idea 是 D1 或 D2 之一）。 */
|
||||
export const STOPS_AT_DEAD_END: Record<string, Noul> = {
|
||||
stays_within_idea: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Does the whole route in `entry` stay within the approach described in `idea.description`, with no plan to go beyond it?",
|
||||
focus:
|
||||
"Answer yes only if everything the entry plans to change falls inside this approach. " +
|
||||
"If the entry also plans a continuous optimization that moves the circle centers, or uses the approach only as a helper step " +
|
||||
"(a starting layout, or a way to score layouts inside a larger search), answer no.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "Everything the entry plans to change is inside this approach; it is the whole route",
|
||||
examples: [
|
||||
"idea: hand-designed placement of centers only; entry: 把三环布局换成六角网格，调行数和间距",
|
||||
"idea: change only the radii with fixed centers; entry: 圆心不动，只改半径的分配方法，这就是整条路线",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "The entry goes beyond this approach, or uses it only as one step of a larger route",
|
||||
not_for: "Entries that use this approach as a starting layout or as a scoring helper inside a search that moves the centers",
|
||||
examples: [
|
||||
"idea: hand-designed placement of centers only; entry: 从六角起点出发，再用约束优化同时移动圆心和半径",
|
||||
"idea: change only the radii with fixed centers; entry: 先用精确的半径求解当打分器，再在外层搜索圆心位置",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
/** 第 4、5 问：只看一个入口。第 5 问拆成两问。 */
|
||||
export const ENTRY_ONLY: Record<string, Noul> = {
|
||||
changes_code: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Is `entry` a route that changes the program's code to raise the score, rather than one that only reads rules, measures, or analyzes?",
|
||||
focus:
|
||||
"A route may begin by reading code or measuring something, as long as the reading or measuring is meant to lead to a code change that raises the score. " +
|
||||
"Answer no only if the entry's purpose ends at understanding, measuring, or checking rules, with no plan to change the code.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "The entry intends to change the program, possibly after a first step of reading or measuring",
|
||||
examples: ["先读半径的计算方法，量它离最优差多少，然后把它换掉", "从错排起点出发，跑一次约束优化"],
|
||||
},
|
||||
false: {
|
||||
what: "The entry only reads rules, measures, or analyzes, and does not plan any change to the code",
|
||||
not_for: "Entries that begin with a diagnosis but clearly intend to change the code afterwards",
|
||||
examples: ["只读评测脚本，列出评测接受和拒绝哪些解", "统计基线产物里有多少个圆贴着边，写成一份报告"],
|
||||
},
|
||||
},
|
||||
},
|
||||
names_code_change: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Does `entry` spell out the finished code change to make, such as the exact algorithm with its settings or the replacement code, instead of only a starting point?",
|
||||
focus:
|
||||
"Entries are meant to say where to start looking and why. Mentioning a technique as a question to explore, or as a direction to look in, is a starting point. " +
|
||||
"Stating the change to make with the method and its settings already decided spells out the answer.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "The entry states what the code should be changed into, with the method and its settings decided",
|
||||
examples: [
|
||||
"把布局函数换成 SLSQP，同时优化 78 个变量，起点用 5 行错排，迭代 500 次",
|
||||
"半径计算改成用 scipy linprog 求解，约束是 r_i + r_j ≤ d_ij 和到四边的距离",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "The entry names where to look and what to question, leaving the change itself to be worked out",
|
||||
not_for: "Entries that name a technique only as a question to check or a direction to explore",
|
||||
examples: ["先读半径的计算方法，问它离给定圆心下的最优半径和差多少", "从基线的布局出发，看哪些圆被挤到了边上"],
|
||||
},
|
||||
},
|
||||
},
|
||||
states_expected_score: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Does `entry` state a score, or an improvement of the score, that following this route is expected to reach?",
|
||||
focus:
|
||||
"Look for predictions of the result of this route. Quoting the baseline score, the best known score, or measured facts about the current program is not a prediction.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "The entry predicts the score or the gain that this route will bring",
|
||||
examples: ["这条路预计能到 2.6 以上", "改完能提升约 0.5"],
|
||||
},
|
||||
false: {
|
||||
what: "The entry predicts no score or gain for this route",
|
||||
not_for: "Quoting the current score, the best known score, or the size of the current gap without predicting what the route reaches",
|
||||
examples: ["基线现在是 0.96，已知最好是 2.636", "现在离已知最好还差 1.68"],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
/** 第 2、6 问：两位研究者从这两个入口出发，多半会做同一类代码改动吗。 */
|
||||
export const SAME_KIND: Record<string, Noul> = {
|
||||
same_kind_of_change: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Would two researchers, one starting from `entry_a` and one from `entry_b`, most likely make the same kind of code change?",
|
||||
focus:
|
||||
"Imagine each researcher follows their entry and writes one code change. Compare which part of the program they would change and by what technique. " +
|
||||
"Different wording, a different first file to read, or a different motivation does not make the changes different if both would change the same thing in the same way.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "Both would most likely change the same part of the program with the same technique",
|
||||
examples: [
|
||||
"a: 用约束优化器同时调整圆心和半径; b: 把 78 个变量一起交给 SLSQP",
|
||||
"a: 换成六角网格布局; b: 把圆心改成错排的行，调行数",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "They would most likely change different parts of the program, or the same part with a different technique",
|
||||
not_for: "Entries that merely share the goal of a higher score, or read the same function, but would change different things",
|
||||
examples: ["a: 圆心固定，用线性规划求半径; b: 用约束优化器同时调整圆心和半径", "a: 换成六角网格布局; b: 只读评测规则，弄清容差"],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
Reference in New Issue
Block a user