147 lines
8.9 KiB
TypeScript
147 lines
8.9 KiB
TypeScript
import type { Noul } from "../lib/jev.ts";
|
|
|
|
// 第 1 轮检验问 Jev 的问题（任务书 14 第 3.2 节第 3 步）：英文、一问只问一件事、带材料说明、判"是"判"否"的说明和反例，
|
|
// 写法照 src/jev/questions.ts。这些是检验自己的问题，不进 src/jev；阈值没有校准，一律以概率大于 0.5 判"是"。
|
|
// 提案、挑刺、回应原文是中文，原样放进材料；两个项目的字段名不同（factors 对 points、modification），材料说明里写明。
|
|
|
|
const STUDY =
|
|
"One round of a study that improves a program for the circle packing task: place 26 non-overlapping circles inside the unit square " +
|
|
"so that the sum of their radii is as large as possible (no tolerance: any overlap or circle outside the square is invalid). " +
|
|
"The baseline program puts the circles on fixed rings around the center and then shrinks radii pair by pair to remove overlaps; " +
|
|
"it scores about 0.96 and the best known score is about 2.636. " +
|
|
"In this round each researcher proposed one code change. A proposal's change is given as a list of separate edits " +
|
|
"(each with a `name`, a `method` saying what to change and how, and sometimes `keySettings`), or as one `modification` text. " +
|
|
"The texts are mostly in Chinese.";
|
|
|
|
export const TASKS = {
|
|
pair: `${STUDY} \`proposal_a\` and \`proposal_b\` are the changes of two proposals from the same round.`,
|
|
idea: `${STUDY} \`proposal\` is the change of one proposal. \`idea\` is a technique from the literature on this task, described in English.`,
|
|
critique:
|
|
`${STUDY} After the proposals were written, other researchers wrote critiques of them, and each author replied to every critique ` +
|
|
"and could revise the proposal before it was frozen as the protocol an implementer must follow. " +
|
|
"`critique` is one critique of a proposal; `reply` is the author's reply to that critique; " +
|
|
"`revision` is the rest of the author's response: the changes made to the proposal and the final text, in whatever fields that system used.",
|
|
};
|
|
|
|
/** a. 两条提案是不是同一个改法（照方法.md 4.3 ③ 判重的意思：只差参数的不算同一个改法）。 */
|
|
export const SAME_CHANGE: Record<string, Noul> = {
|
|
same_change: {
|
|
type: "noul",
|
|
instructions: {
|
|
question: "Would carrying out `proposal_a` and carrying out `proposal_b` produce the same code change, so that a single implementation would serve as both?",
|
|
focus:
|
|
"Compare what each change does to the program: which parts it changes and by what method, reading every edit. " +
|
|
"Ignore wording, language, level of detail, names, and how the change is split into edits. " +
|
|
"The same method with only different settings (step sizes, counts, time budgets, layouts) is not the same change, because each setting needs its own run.",
|
|
},
|
|
criteria: {
|
|
true: {
|
|
what: "Both describe the same algorithmic change to the same part of the program with the same settings; they differ only in wording, detail or how the change is split",
|
|
examples: [
|
|
"a: 把圆心和半径一起交给约束优化器，从当前布局出发; b: optimize centers and radii jointly with SLSQP starting from the current layout",
|
|
"a: 半径改用线性规划求解; b: 固定圆心，用 linprog 求半径和最大",
|
|
],
|
|
},
|
|
false: {
|
|
what: "They change different parts of the program, change the same part by a different method, or use the same method with different settings",
|
|
not_for: "Two changes that only share the goal of a higher score, or only edit the same function in different ways",
|
|
examples: [
|
|
"a: 半径改用线性规划求解; b: 把圆心和半径一起交给约束优化器",
|
|
"a: 6 行六角网格; b: 5 行六角网格",
|
|
"a: 多起点重启 20 次; b: 多起点重启 200 次",
|
|
],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
/** b. 作者的回应有没有处理这条挑刺：改了协议，或者给了不改的具体理由。 */
|
|
export const ADDRESSED: Record<string, Noul> = {
|
|
critique_addressed: {
|
|
type: "noul",
|
|
instructions: {
|
|
question:
|
|
"Does `reply` deal with the concern raised in `critique`, either by a concrete change to the proposal (visible in `reply` or `revision`) or by a specific reason why no change is needed?",
|
|
focus:
|
|
"First find the concern in `critique`. Then check whether `reply` engages with that concern: says what was changed and how, or argues specifically why the concern does not apply. " +
|
|
"Agreeing without saying what changed, promising to consider it, or answering a different point does not deal with the concern. " +
|
|
"Whether the reply is right is not the question; only whether it deals with the concern.",
|
|
},
|
|
criteria: {
|
|
true: {
|
|
what: "The reply states a concrete change that answers the concern, or gives a specific reason tied to the concern for not changing",
|
|
examples: [
|
|
"critique: 停止条件没写锚点; reply: 采纳，t0 改为入口处取 perf_counter，并写进协议的停止条件一节",
|
|
"critique: 这个因子会改变评分; reply: 不改：评分只读 solution.json，这个因子只改内部的起点，具体见协议第 2 节",
|
|
],
|
|
},
|
|
false: {
|
|
what: "The reply ignores the concern, only agrees or thanks, promises to look later, or answers something else",
|
|
not_for: "Replies that disagree with the critique but give a specific reason tied to its concern",
|
|
examples: ["critique: 停止条件没写锚点; reply: 感谢意见，同意", "critique: 两个因子分不开; reply: 我们会在实现时注意"],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
/** c. 提案对研究路径上的关键想法（K1–K5）：这条提案的改动是否做了这个想法。 */
|
|
export const CARRIES_IDEA: Record<string, Noul> = {
|
|
carries_idea: {
|
|
type: "noul",
|
|
instructions: {
|
|
question: "Does the code change in `proposal` carry out the technique described in `idea.description`, as the whole change or as one of its parts?",
|
|
focus:
|
|
"Read every edit of the proposal. Answer yes if one of the edits, or the change as a whole, does what the technique describes, possibly in other words or in Chinese. " +
|
|
"Mentioning the technique as a later plan, or sharing only the goal, is not enough.",
|
|
},
|
|
criteria: {
|
|
true: {
|
|
what: "The proposal's change includes this technique",
|
|
examples: [
|
|
"idea: optimize centers and radii together with a constrained optimizer; proposal: 把 78 个变量交给 SLSQP，约束是在正方形内、两两不重叠",
|
|
"idea: solve a linear program for the radii with fixed centers; proposal: 半径分配改成 linprog 求半径和最大",
|
|
],
|
|
},
|
|
false: {
|
|
what: "The proposal's change does not include this technique",
|
|
not_for: "Proposals that only mention the technique as future work, or only share the goal",
|
|
examples: [
|
|
"idea: optimize centers and radii together with a constrained optimizer; proposal: 圆心按六角网格摆，半径用贪心缩放",
|
|
"idea: solve a linear program for the radii with fixed centers; proposal: 从当前最好解加扰动再跑 SLSQP",
|
|
],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
/** c. 提案对死路（D1、D2）：这条提案的改动是不是整个都停在这条死路里。 */
|
|
export const STAYS_IN_DEAD_END: Record<string, Noul> = {
|
|
stays_within_idea: {
|
|
type: "noul",
|
|
instructions: {
|
|
question: "Does the whole code change in `proposal` stay within the approach described in `idea.description`, with no part that goes beyond it?",
|
|
focus:
|
|
"Answer yes only if everything the proposal changes falls inside this approach. " +
|
|
"If any edit uses a continuous optimization that moves the circle centers, or uses the approach only as a helper step " +
|
|
"(a starting layout, or a way to score layouts inside a larger search), answer no.",
|
|
},
|
|
criteria: {
|
|
true: {
|
|
what: "Everything the proposal changes is inside this approach",
|
|
examples: [
|
|
"idea: hand-designed placement of centers only; proposal: 换成 6 行六角网格，枚举行数取最好，半径用贪心",
|
|
"idea: change only the radii with fixed centers; proposal: 圆心不动，半径改用线性规划",
|
|
],
|
|
},
|
|
false: {
|
|
what: "Some part of the proposal goes beyond this approach, or the approach is only one step of a larger change",
|
|
not_for: "Proposals that use this approach as a starting layout or a scoring helper inside a search that moves the centers",
|
|
examples: [
|
|
"idea: hand-designed placement of centers only; proposal: 从六角起点出发，再用 SLSQP 同时移动圆心和半径",
|
|
"idea: change only the radii with fixed centers; proposal: 半径用线性规划打分，外层搜索圆心位置",
|
|
],
|
|
},
|
|
},
|
|
},
|
|
};
|