feat: 任务书 08,jev 模块与账本记 Jev 判断
src/jev 3 个文件 515 行:判重、查冻结条款、查代码改动三个函数,九问问法和阈值照参考项目,只改说法和字段名,阈值注明未在新项目材料上校准,等真实数据。 新增 npm run test:jev(真实 Jev 三处各问一次)。vocab 加 JevJudgment。 ledger 表结构版本 3:callEnded 四种结局对上 agents,墙钟时间用 Measured(恢复时补写出错、原因"进程中断"、测不到);退回、提交记录都带 jev。 config 加必填的 models.jev。 用户原话:"这次就给提交记录加一个和退回记录同样的 Jev 判断字段";"墙钟时间改用 Measured,恢复时写出错加测不到(原因'进程中断')"。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,13 @@
|
||||
// jev:问 Jev 的问法和阈值(docs/结构.md 第二部分)。判断的类型 JevJudgment 在 vocab。
|
||||
export {
|
||||
checkHunk,
|
||||
checkProtocol,
|
||||
codeHunks,
|
||||
judgeSameChange,
|
||||
type CodeHunk,
|
||||
type DedupFactor,
|
||||
type FrozenProtocol,
|
||||
type JevCheck,
|
||||
type ProtocolMetric,
|
||||
type ProtocolRules,
|
||||
} from "./jev.ts";
|
||||
+188
@@ -0,0 +1,188 @@
|
||||
import type { HypothesisMetric, JevJudgment, Json, Submission } from "../vocab/index.ts";
|
||||
import { questionsOf, taskOf, type Question } from "./questions.ts";
|
||||
|
||||
// 问 Jev:从带类型的输入拼材料、发请求、读回答(docs/方法.md 4.3 ③⑥⑨)。
|
||||
// 材料里放什么、字段叫什么只在这里定。分请求照参考项目 src/judge/judge.ts:
|
||||
// 判重一对一次请求;冻结条款三问每问单独一次请求,材料相同;代码改动一个差异块一次请求,五问同在。
|
||||
|
||||
type Factor = Submission<"proposal">["factors"][number];
|
||||
|
||||
/** 判重看的因子:名称和做法。关掉时换成什么、延续哪个因子不放:它们不改变"这条提案改什么"。 */
|
||||
export type DedupFactor = Pick<Factor, "name" | "method">;
|
||||
|
||||
/** 一份冻结协议的条款(docs/方法.md 4.3 ⑥"冻结协议包括"),字段与统一提交相同。 */
|
||||
export type FrozenProtocol = Pick<Submission<"unification">, "factors" | "safeguards" | "metrics" | "body" | "clauses">;
|
||||
|
||||
/** 协议要求候选代码采集的一条指标(新登记的和引用的都在内,由调用方查全)。 */
|
||||
export type ProtocolMetric = Pick<HypothesisMetric, "id" | "howMeasured" | "missing">;
|
||||
|
||||
/** 查代码改动时对照的那部分协议:协议正文和要采集的指标。没有要采集的指标时是空数组。 */
|
||||
export interface ProtocolRules {
|
||||
body: string;
|
||||
metrics: readonly ProtocolMetric[];
|
||||
}
|
||||
|
||||
/** 一个差异块:哪个文件、文件是新增删除还是修改,以及带 3 行上下文的块正文(从 @@ 行起)。 */
|
||||
export interface CodeHunk {
|
||||
path: string;
|
||||
change: "added" | "deleted" | "modified";
|
||||
text: string;
|
||||
}
|
||||
|
||||
/** 查冻结条款、查代码改动的结论:每一问的判断,以及有没有任何一问判"是"。 */
|
||||
export interface JevCheck {
|
||||
flagged: boolean;
|
||||
judgments: JevJudgment[];
|
||||
}
|
||||
|
||||
/** ③ 判重:两条提案的因子清单是不是同一个改法。 */
|
||||
export async function judgeSameChange(model: string, a: readonly DedupFactor[], b: readonly DedupFactor[]): Promise<JevJudgment> {
|
||||
const state = {
|
||||
task: taskOf("dedup"),
|
||||
proposal_a: { factors: a.map(dedupFactor) },
|
||||
proposal_b: { factors: b.map(dedupFactor) },
|
||||
};
|
||||
const [judgment] = await ask(model, state, questionsOf("dedup"));
|
||||
return judgment as JevJudgment;
|
||||
}
|
||||
|
||||
/** ⑥ 查一份冻结协议的条款:改动了评测、自相矛盾、写得太含糊。三问逐个发,材料相同。 */
|
||||
export async function checkProtocol(model: string, protocol: FrozenProtocol): Promise<JevCheck> {
|
||||
const state = {
|
||||
task: taskOf("freeze"),
|
||||
protocol: {
|
||||
factors: protocol.factors,
|
||||
safeguards: protocol.safeguards,
|
||||
body: protocol.body,
|
||||
metrics: protocol.metrics,
|
||||
clauses: protocol.clauses,
|
||||
},
|
||||
};
|
||||
const judgments: JevJudgment[] = [];
|
||||
for (const question of questionsOf("freeze")) judgments.push(...(await ask(model, state, [question])));
|
||||
return check(judgments);
|
||||
}
|
||||
|
||||
/** ⑨ 查一个差异块:五问一次请求,对照协议正文和要采集的指标。 */
|
||||
export async function checkHunk(model: string, hunk: CodeHunk, protocol: ProtocolRules): Promise<JevCheck> {
|
||||
const state = {
|
||||
task: taskOf("diff"),
|
||||
protocol: {
|
||||
body: protocol.body,
|
||||
metrics: protocol.metrics.map((m) => ({ id: m.id, howMeasured: m.howMeasured, missing: m.missing })),
|
||||
},
|
||||
change: { path: hunk.path, change: hunk.change, diff: hunk.text },
|
||||
};
|
||||
return check(await ask(model, state, questionsOf("diff")));
|
||||
}
|
||||
|
||||
function dedupFactor(f: DedupFactor): { [key: string]: Json } {
|
||||
return { name: f.name, method: f.method };
|
||||
}
|
||||
|
||||
function check(judgments: JevJudgment[]): JevCheck {
|
||||
return { flagged: judgments.some((j) => j.yes), judgments };
|
||||
}
|
||||
|
||||
/**
|
||||
* 把 workspace 比较两个版本的结果切成差异块,每块一次请求(一次放多块有位置效应)。
|
||||
* 差异原文里每个文件一段(以 `diff --git ` 开头),与文件清单同序,按位置对上;段里每个 `@@` 行起一块。
|
||||
* 二进制文件、只改可执行位、空文件增删没有 `@@` 行,不出块:这些改动 Jev 看不到内容,由程序检查。
|
||||
*/
|
||||
export function codeHunks(comparison: {
|
||||
files: ReadonlyArray<{ path: string; change: CodeHunk["change"] }>;
|
||||
diff: string;
|
||||
}): CodeHunk[] {
|
||||
// 内容行都以 +、-、空格开头,行首的 `diff --git `、`@@ ` 只会是 git 写的。
|
||||
const sections = comparison.diff.split(/^diff --git /m).slice(1);
|
||||
if (sections.length !== comparison.files.length) {
|
||||
throw new Error(`差异原文有 ${sections.length} 段,文件清单有 ${comparison.files.length} 个,对不上`);
|
||||
}
|
||||
return comparison.files.flatMap((file, i) =>
|
||||
(sections[i] as string)
|
||||
.split(/^(?=@@ )/m)
|
||||
.slice(1)
|
||||
.map((text) => ({ path: file.path, change: file.change, text: text.replace(/\n$/, "") })),
|
||||
);
|
||||
}
|
||||
|
||||
/** 就一份材料问几问,一次请求,交回每一问的判断。 */
|
||||
async function ask(model: string, state: { [key: string]: Json }, questions: readonly Question[]): Promise<JevJudgment[]> {
|
||||
const reply = await post(
|
||||
JSON.stringify({ model, state, questions: Object.fromEntries(questions.map((q) => [q.name, q.noul])) }),
|
||||
);
|
||||
// 回答是外部服务的返回,读不懂就报错停下,带上请求编号(CLAUDE.md"罕见情况怎么办"第 2 条)。
|
||||
if (reply.requestId === null) throw new Error(`Jev 的回答没带请求编号(x-typesafe-request-id),正文:${reply.text.slice(0, 500)}`);
|
||||
const requestId = reply.requestId;
|
||||
const where = `(request-id: ${requestId})`;
|
||||
let response: unknown;
|
||||
try {
|
||||
response = JSON.parse(reply.text);
|
||||
} catch (error) {
|
||||
throw new Error(`Jev 的回答不是 JSON${where}:${reply.text.slice(0, 500)}`, { cause: error });
|
||||
}
|
||||
const answers = (response as { answers?: Record<string, { noul?: unknown } | undefined> } | null)?.answers;
|
||||
return questions.map((q) => {
|
||||
const probability = answers?.[q.name]?.noul;
|
||||
if (typeof probability !== "number" || !(probability >= 0 && probability <= 1)) {
|
||||
throw new Error(`Jev 的回答里问题 ${q.name} 没有 0 到 1 之间的概率${where}:${reply.text.slice(0, 500)}`);
|
||||
}
|
||||
return { question: q.name, questionHash: q.hash, threshold: q.threshold, probability, yes: probability > q.threshold, requestId };
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 发请求
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const ENDPOINT = "https://api.typesafe.ai/v1/systemone";
|
||||
|
||||
/** 一次请求最多等多久。Jev 一般零点几秒就答,等一分钟还没回就当这一次失败。 */
|
||||
const TIMEOUT_MS = 60_000;
|
||||
|
||||
/**
|
||||
* 调不通时第几次重试前等多久,一共最多发 4 次。
|
||||
* 网络抖动、服务短暂过载、限流(HTTP 429)多在几秒到几十秒内恢复,所以先等 2 秒,再拉长到 10 秒、30 秒;
|
||||
* 合计等 42 秒还不通,就不是一时的问题,报错停下由用户决定,不把"没判"当成"没问题"。
|
||||
*/
|
||||
const RETRY_WAITS_MS = [2_000, 10_000, 30_000];
|
||||
|
||||
/**
|
||||
* 发一次请求,拿回正文和请求编号。只有原因在我们控制之外的失败才重试(CLAUDE.md"错误怎么处理"):
|
||||
* 连不上、超时、读正文失败、HTTP 429 和 5xx。其余 4xx 说明请求本身或密钥有问题,重试也一样,直接报错。
|
||||
* 密钥只从环境变量读进请求头,不进正文。
|
||||
*/
|
||||
async function post(body: string): Promise<{ requestId: string | null; text: string }> {
|
||||
const key = process.env["TYPESAFE_API_KEY"];
|
||||
if (key === undefined || key.trim() === "") {
|
||||
throw new Error("环境变量 TYPESAFE_API_KEY 没有设置,Jev 调不通");
|
||||
}
|
||||
const failures: string[] = [];
|
||||
for (let attempt = 0; attempt <= RETRY_WAITS_MS.length; attempt += 1) {
|
||||
if (attempt > 0) await wait(RETRY_WAITS_MS[attempt - 1] as number);
|
||||
let response: Response;
|
||||
let text: string;
|
||||
try {
|
||||
response = await fetch(ENDPOINT, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", authorization: `Bearer ${key}` },
|
||||
body,
|
||||
signal: AbortSignal.timeout(TIMEOUT_MS),
|
||||
});
|
||||
text = await response.text();
|
||||
} catch (error) {
|
||||
failures.push(`第 ${attempt + 1} 次:${error instanceof Error ? error.message : String(error)}`);
|
||||
continue;
|
||||
}
|
||||
const requestId = response.headers.get("x-typesafe-request-id");
|
||||
if (response.ok) return { requestId, text };
|
||||
const what = `HTTP ${response.status}(request-id: ${requestId ?? "无"}):${text.slice(0, 500)}`;
|
||||
if (response.status !== 429 && response.status < 500) throw new Error(`Jev 拒绝了请求,不重试:${what}`);
|
||||
failures.push(`第 ${attempt + 1} 次:${what}`);
|
||||
}
|
||||
throw new Error(`Jev 调不通,发了 ${failures.length} 次都失败:${failures.join(";")}`);
|
||||
}
|
||||
|
||||
function wait(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
import { hash, type Json } from "../vocab/index.ts";
|
||||
|
||||
// Jev 的九问(docs/方法.md 4.3 ③⑥⑨):每一处的材料说明、问题原文、阈值。
|
||||
// 措辞、判"是"判"否"的说明和示例句取自参考项目 src/judge/questions.ts(填圆那一套),
|
||||
// 只改了必须改的:点 → 因子,字段名对上 vocab 的提案和冻结协议,改动对上 workspace 的比较结果。
|
||||
// 写法照参考项目 research-wiki/参考-Jev使用指南.md:问题用英文、一问只问一件事、
|
||||
// 统一问坏的一面(概率高 = 有问题或"是同一个")、判据带 not_for 和例子、问题里用反引号点到材料路径。
|
||||
|
||||
/** 用到 Jev 的三处:判重、冻结条款、一段代码改动。 */
|
||||
export type Site = "dedup" | "freeze" | "diff";
|
||||
|
||||
/** 一条是非题(Jev 的 noul)。 */
|
||||
export interface Noul {
|
||||
type: "noul";
|
||||
instructions: Json;
|
||||
criteria: Json;
|
||||
}
|
||||
|
||||
/**
|
||||
* 每一问的阈值:概率超过阈值判"是"。
|
||||
* 这是参考项目在它自己的材料上校准的值(JEV_THRESHOLDS 填圆那一套),没有在新项目的材料上校准过,等真实数据。
|
||||
*/
|
||||
const THRESHOLDS: Readonly<Record<Site, Readonly<Record<string, number>>>> = {
|
||||
dedup: { same_change: 0.3 },
|
||||
freeze: { self_contradictory: 0.5, changes_measurement: 0.21, too_vague: 0.78 },
|
||||
diff: { unseeded_randomness: 0.55, reads_clock: 0.55, touches_measurement: 0.65, early_exit: 0.63, zero_for_unmeasured: 0.62 },
|
||||
};
|
||||
|
||||
/** 放进材料 `task` 字段的说明:这是什么材料、各部分是什么。 */
|
||||
const TASKS: Readonly<Record<Site, string>> = {
|
||||
dedup:
|
||||
"Two research proposals from the same round of a study that improves a program by changing its code. " +
|
||||
"Each proposal lists its change as `factors`: one or more separate edits, each with a short `name` and a `method` saying what to change and how. " +
|
||||
"Together the factors of one proposal are the whole change that one implementation would carry out. " +
|
||||
"The proposals were written by different researchers, in Chinese or English.",
|
||||
freeze:
|
||||
"The frozen protocol of one candidate in a study that improves a program by changing its code. " +
|
||||
"An implementer will edit the code exactly as `protocol` says, and the unchanged task harness will then run it once and measure the final metric. " +
|
||||
"The implementer may not edit the task's harness or validator files. `protocol.clauses` lists the clauses the implementer must satisfy one by one.",
|
||||
diff:
|
||||
"One hunk of a code change made by an implementer to a candidate program, in unified diff format (`change.diff`; lines starting with + are added, - are removed). " +
|
||||
"`protocol.body` is the frozen protocol the implementer must follow, in Markdown with headings, among them: what must not change, randomness, when to stop, and which runs do not count. " +
|
||||
"`protocol.metrics` lists the metrics the protocol requires the implementer " +
|
||||
"to compute in the candidate code and write to metrics.json (each with its `howMeasured` and `missing` rule; an empty list means none). " +
|
||||
"Only this hunk is shown; judge only what the hunk itself adds or changes.",
|
||||
};
|
||||
|
||||
const QUESTIONS: Readonly<Record<Site, Readonly<Record<string, Noul>>>> = {
|
||||
// ③ 判重:一对提案一次请求。
|
||||
dedup: {
|
||||
same_change: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Do `proposal_a` and `proposal_b` propose the same code change, so that one implementation would carry out both?",
|
||||
focus:
|
||||
"Compare the whole change of each proposal: which parts of the program are changed and by what method, reading every factor's `method`. " +
|
||||
"Ignore wording, language, level of detail, factor names, and how the change is divided into factors: one proposal may describe as two factors what the other describes as one. " +
|
||||
"The same method with different parameter values or sizes is NOT the same change: each setting would need its own run.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "Both describe the same algorithmic change to the same part of the program with the same settings; they differ only in wording, language, detail or how the change is split into factors",
|
||||
examples: [
|
||||
"'sort items by weight before packing' vs 'process items in decreasing order of weight'",
|
||||
"'replace SGD with Adam, lr 1e-3' vs 'switch the optimizer to Adam at learning rate 0.001'",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "They change different parts of the program, change the same part by a different method, or use the same method with different parameter values",
|
||||
not_for: "Two proposals that merely share the same goal, or merely edit the same function in different ways",
|
||||
examples: [
|
||||
"'replace SGD with Adam' vs 'add a learning-rate warmup to SGD'",
|
||||
"'a 6-row hexagonal grid' vs 'a 5-row hexagonal grid'",
|
||||
"'learning rate 1e-3' vs 'learning rate 1e-2' with the same optimizer",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
// ⑥ 冻结条款:每一问单独一次请求,材料相同。
|
||||
freeze: {
|
||||
self_contradictory: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question:
|
||||
"Does the change that `protocol` asks for break a restriction stated elsewhere in `protocol`, or do two of its statements exclude each other, so that no implementation can satisfy all of them?",
|
||||
focus:
|
||||
"Take the change described in `factors` (each factor's `method`), `safeguards` and `clauses`, then check it against every restriction written in `protocol.body` (what must not change, randomness, when to stop). " +
|
||||
"A restriction may name code that must not be edited, or things that must stay as in the baseline, such as positions, sizes or constants; carrying out the change may alter those things even if the change is described in other words. " +
|
||||
"A statement that is merely unusual, ambitious or likely to fail is not a contradiction.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "Carrying out the requested change necessarily violates a stated restriction, or two statements require things that cannot both hold",
|
||||
examples: [
|
||||
"`clauses` says rewrite compute_max_radii while `protocol` says compute_max_radii must not change",
|
||||
"a factor's `method` uses random restarts while `protocol` says the program uses no randomness",
|
||||
"a factor's `method` doubles the step size while `protocol` says every numeric constant keeps its baseline value",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "All statements can hold at the same time in one implementation",
|
||||
not_for: "Plans that may not improve the metric, or statements that are vague but not conflicting",
|
||||
examples: ["rewrite construct_packing and keep compute_max_radii unchanged", "use seed 42 for 10 random restarts and keep the best result"],
|
||||
},
|
||||
},
|
||||
},
|
||||
changes_measurement: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question:
|
||||
"Does `protocol` require changing how the final metric is measured, computed or reported, rather than changing the solution that gets measured?",
|
||||
focus:
|
||||
"The final metric is measured by the task's harness from what the program returns. " +
|
||||
"Changing the algorithm that produces the solution is allowed. Changing how the returned value is summed, scored, validated, rounded or reported is a change of measurement. " +
|
||||
"The program's own handling of its candidate solutions — legality checks, repairs, safety margins such as subtracting 1e-9 from every radius before checking, fallbacks, and rules for picking the best result — changes the answer it returns, not how that answer is measured, even when it copies the validator's formula. " +
|
||||
"Likewise, rules for the protocol's own diagnostic metrics (what they count, how they are aggregated, when they are written) do not change the final metric.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "The protocol asks to alter the scoring or validation: the formula of the metric, what gets counted, tolerances, the harness or validator, or the value reported instead of the real one",
|
||||
examples: [
|
||||
"report the sum of diameters instead of the sum of radii",
|
||||
"count only circles with radius above 0.01 when summing",
|
||||
"raise the overlap tolerance in validate.py to 1e-6",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "The protocol only changes how the solution is built or searched; the metric is computed from its output as before",
|
||||
not_for: "Adding new diagnostic metrics that are recorded next to the final metric without replacing it",
|
||||
examples: ["replace the ring layout with a hexagonal grid", "compute radii with a linear program"],
|
||||
},
|
||||
},
|
||||
},
|
||||
too_vague: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question:
|
||||
"Does `protocol` leave the implementer to guess what to change, for example which function to edit, which method to use, or when an iterative search stops?",
|
||||
focus:
|
||||
"Read `factors` (each factor's `method`), `clauses` and the stopping rule in `protocol.body` as an implementer who must follow them without asking questions. " +
|
||||
"Brief text is fine when it names the function, the method and its settings. " +
|
||||
"Judge only whether each statement is specific. Whether two statements conflict with each other is checked separately; a protocol whose statements are concrete but conflicting is not vague.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "An implementer would have to invent a key part of the change: the target code, the method, its main parameters, or the stopping rule of a loop or search",
|
||||
examples: ["improve the layout of the circles somehow", "optimize the radii better", "run a local search until it is good enough"],
|
||||
},
|
||||
false: {
|
||||
what: "The target code, the method and its settings are stated concretely enough to implement directly",
|
||||
not_for:
|
||||
"Protocols that leave only minor coding details, such as variable names, to the implementer, or protocols whose concrete statements contradict each other",
|
||||
examples: [
|
||||
"in construct_packing, change the outer ring radius constant from 0.7 to 0.45",
|
||||
"replace compute_max_radii with an LP solved by a numpy simplex; stop when no pivot improves",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
// ⑨ 代码改动:一个差异块一次请求,五问同在。
|
||||
diff: {
|
||||
unseeded_randomness: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Do the added lines in `change.diff` use random numbers without the fixed seed that the randomness section of `protocol.body` requires?",
|
||||
focus:
|
||||
"Look for random number generators and random sampling in added lines. " +
|
||||
"Randomness that is seeded exactly as the randomness section of `protocol.body` describes is fine.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "Added code draws random numbers from an unseeded generator, a time-based seed, or a seed that differs from the protocol",
|
||||
examples: ["np.random.rand(26, 2)", "rng = np.random.default_rng()", "random.seed(time.time())"],
|
||||
},
|
||||
false: {
|
||||
what: "Added code uses no random numbers, or seeds them as the protocol says",
|
||||
not_for: "Deterministic code that merely contains the word random in a name or comment",
|
||||
examples: ["rng = np.random.default_rng(42) when the protocol registers seed 42", "centers[i] = [x0 + d * k, y0]"],
|
||||
},
|
||||
},
|
||||
},
|
||||
reads_clock: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question:
|
||||
"Do the added lines in `change.diff` make the program's result depend on the clock in a way that the stopping section of `protocol.body` does not register?",
|
||||
focus:
|
||||
"Look for time.time, time.perf_counter, time.monotonic, datetime.now and deadlines built from them. " +
|
||||
"A wall-clock time limit that the stopping section of `protocol.body` itself describes, implemented with the same limit to stop the same loop, is required by the protocol and is fine. " +
|
||||
"Any other use of the clock that changes what the program computes counts, including a time limit different from the registered one " +
|
||||
"and the registered time used for something other than stopping. Reading the clock only to print or log timing does not make the result depend on it.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "Added code reads the clock and uses it to decide how long to search, when to stop, or which values to use, and the stopping section of `protocol.body` does not register that use or registers a different time limit",
|
||||
examples: [
|
||||
"while time.time() - t0 < 5: # protocol says a fixed 2000 steps",
|
||||
"step = 0.02 if time.time() - t0 < 3 else 0.005",
|
||||
"seed = int(datetime.now().timestamp())",
|
||||
"while time.monotonic() - t0 < 60.0: # protocol registers a 120 s limit",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "Added code does not read the clock, reads it only for printing or logging, or stops a loop on elapsed time exactly as the stopping section of `protocol.body` registers",
|
||||
not_for: "Loops bounded by an iteration count",
|
||||
examples: [
|
||||
"for it in range(2000):",
|
||||
"print(f'took {time.time() - t0:.2f}s')",
|
||||
"while time.monotonic() - t0 < 120.0 and it < 2000: # protocol: stop after 120 s of wall time or 2000 iterations",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
touches_measurement: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question:
|
||||
"Do the added or removed lines in `change.diff` change how the program computes or reports its score, or compute a metric from `protocol.metrics` differently from its `howMeasured`?",
|
||||
focus:
|
||||
"The score is computed from the returned solution, such as summing the radii in run_packing. " +
|
||||
"Changing how circles are placed or how radii are chosen changes the solution, not the score computation. " +
|
||||
"Code that computes a metric listed in `protocol.metrics` the way its `howMeasured` says and writes it to metrics.json is required by the protocol; it is not a change to the score.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "The hunk changes the score formula, scales or rounds the reported value, reports a value other than the one computed from the solution, edits the code that checks validity, or computes a listed metric with a formula or data different from its `howMeasured`",
|
||||
examples: [
|
||||
"sum_radii = np.sum(radii) * 1.01",
|
||||
"sum_radii = 2 * np.sum(radii)",
|
||||
"return centers, radii, 2.7",
|
||||
"val = 1.0 # when the metric's howMeasured gives a formula",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "The hunk only changes how centers or radii are produced, or computes and writes a metric from `protocol.metrics` as its `howMeasured` describes; the score is still computed as before",
|
||||
not_for: "Recording extra diagnostic values next to the score",
|
||||
examples: [
|
||||
"radii = solve_lp(centers)",
|
||||
"centers[i + 9] = [0.5 + 0.45 * np.cos(a), 0.5 + 0.45 * np.sin(a)]",
|
||||
"json.dump({metric_id: float(np.min(gaps))}, f) # when the metric's howMeasured says min gap",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
early_exit: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Do the added lines in `change.diff` end a loop, a search or the program earlier than the stopping section of `protocol.body` says?",
|
||||
focus:
|
||||
"Look for added break, return, sys.exit or raise statements and added limits on iterations. " +
|
||||
"A stop that the stopping section of `protocol.body` itself describes is fine.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "Added code stops work before the protocol's stopping rule is met, or adds a stopping rule the protocol does not mention",
|
||||
examples: ["if it > 100: break # protocol says 2000 iterations", "if total > 2.0: return centers, radii", "sys.exit(0)"],
|
||||
},
|
||||
false: {
|
||||
what: "Added code has no new exits, or exits exactly as the protocol's stopping rule says",
|
||||
not_for: "Returning the final result at the normal end of a function",
|
||||
examples: ["if no_improvement_for >= 100: break # protocol: stop after 100 idle iterations", "return centers, radii"],
|
||||
},
|
||||
},
|
||||
},
|
||||
zero_for_unmeasured: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Do the added lines in `change.diff` record 0 or another fixed number when a value could not be measured or computed?",
|
||||
focus:
|
||||
"Look at exception handlers, missing-value branches and default values of recorded metrics. " +
|
||||
"Recording an explicit 'unavailable' marker or leaving the value out is the correct way; writing 0 hides the failure.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "When a computation fails or a value is missing, added code stores or returns 0 or another made-up number in its place",
|
||||
examples: [
|
||||
"except Exception: min_gap = 0.0",
|
||||
"metrics['clipped-count'] = count if ok else 0",
|
||||
"except ValueError: return centers, np.zeros(n)",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "Added code records failures as unavailable, raises an error, or does not deal with missing values at all",
|
||||
not_for: "Initializing an accumulator to 0 before a loop that always fills it",
|
||||
examples: ["except Exception: metrics['min-gap'] = None", "total = 0.0\nfor r in radii: total += r"],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
/** 一问的全部内容:问题原文、阈值,以及材料说明、问题原文、阈值三者合起来的哈希。 */
|
||||
export interface Question {
|
||||
name: string;
|
||||
noul: Noul;
|
||||
threshold: number;
|
||||
hash: string;
|
||||
}
|
||||
|
||||
/** 一处的全部问题,按上面写的顺序。 */
|
||||
export function questionsOf(site: Site): Question[] {
|
||||
return Object.entries(QUESTIONS[site]).map(([name, noul]) => {
|
||||
const threshold = THRESHOLDS[site][name] as number;
|
||||
return { name, noul, threshold, hash: hash({ task: TASKS[site], name, noul: noul as unknown as Json, threshold }) };
|
||||
});
|
||||
}
|
||||
|
||||
export function taskOf(site: Site): string {
|
||||
return TASKS[site];
|
||||
}
|
||||
Reference in New Issue
Block a user