InfoRSI/src/jev/questions.ts

import { hash, type Json } from "../vocab/index.ts";

// Jev 的九问（docs/方法.md 4.3 ③⑥⑨）：每一处的材料说明、问题原文、阈值。
// 措辞、判"是"判"否"的说明和示例句取自参考项目 src/judge/questions.ts（填圆那一套），
// 只改了必须改的：点 → 因子，字段名对上 vocab 的提案和冻结协议，改动对上 workspace 的比较结果。
// 写法照参考项目 research-wiki/参考-Jev使用指南.md：问题用英文、一问只问一件事、
// 统一问坏的一面（概率高 = 有问题或"是同一个"）、判据带 not_for 和例子、问题里用反引号点到材料路径。

/** 用到 Jev 的三处：判重、冻结条款、一段代码改动。 */
export type Site = "dedup" | "freeze" | "diff";

/** 一条是非题（Jev 的 noul）。 */
export interface Noul {
  type: "noul";
  instructions: Json;
  criteria: Json;
}

/**
 * 每一问的阈值：概率超过阈值判"是"。
 * 这是参考项目在它自己的材料上校准的值（JEV_THRESHOLDS 填圆那一套），没有在新项目的材料上校准过，等真实数据。
 */
const THRESHOLDS: Readonly<Record<Site, Readonly<Record<string, number>>>> = {
  dedup: { same_change: 0.3 },
  freeze: { self_contradictory: 0.5, changes_measurement: 0.21, too_vague: 0.78 },
  diff: { unseeded_randomness: 0.55, reads_clock: 0.55, touches_measurement: 0.65, early_exit: 0.63, zero_for_unmeasured: 0.62 },
};

/** 放进材料 `task` 字段的说明：这是什么材料、各部分是什么。 */
const TASKS: Readonly<Record<Site, string>> = {
  dedup:
    "Two research proposals from the same round of a study that improves a program by changing its code. " +
    "Each proposal lists its change as `factors`: one or more separate edits, each with a short `name` and a `method` saying what to change and how. " +
    "Each factor also has `keySettings`: the setting values that decide its effect, such as a step size or a time window (\"无\" when there are none). " +
    "Together the factors of one proposal are the whole change that one implementation would carry out. " +
    "The proposals were written by different researchers, in Chinese or English.",
  freeze:
    "The frozen protocol of one candidate in a study that improves a program by changing its code. " +
    "An implementer will edit the code exactly as `protocol` says, and the unchanged task harness will then run it once and measure the final metric. " +
    "The implementer may not edit the task's harness or validator files. `protocol.clauses` lists the clauses the implementer must satisfy one by one.",
  diff:
    "One hunk of a code change made by an implementer to a candidate program, in unified diff format (`change.diff`; lines starting with + are added, - are removed). " +
    "`protocol.body` is the frozen protocol the implementer must follow, in Markdown with headings, among them: what must not change, randomness, when to stop, and which runs do not count. " +
    "`protocol.metrics` lists the metrics the protocol requires the implementer " +
    "to compute in the candidate code and write to metrics.json (each with its `howMeasured` and `missing` rule; an empty list means none). " +
    "Only this hunk is shown; judge only what the hunk itself adds or changes.",
};

const QUESTIONS: Readonly<Record<Site, Readonly<Record<string, Noul>>>> = {
  // ③ 判重：一对提案一次请求。
  dedup: {
    same_change: {
      type: "noul",
      instructions: {
        question: "Do `proposal_a` and `proposal_b` propose the same code change, so that one implementation would carry out both?",
        focus:
          "Compare the whole change of each proposal: which parts of the program are changed and by what method, reading every factor's `method`. " +
          "Ignore wording, language, level of detail, factor names, and how the change is divided into factors: one proposal may describe as two factors what the other describes as one. " +
          "The same method with different parameter values or sizes is NOT the same change: each setting would need its own run.",
      },
      criteria: {
        true: {
          what: "Both describe the same algorithmic change to the same part of the program with the same settings; they differ only in wording, language, detail or how the change is split into factors",
          examples: [
            "'sort items by weight before packing' vs 'process items in decreasing order of weight'",
            "'replace SGD with Adam, lr 1e-3' vs 'switch the optimizer to Adam at learning rate 0.001'",
          ],
        },
        false: {
          what: "They change different parts of the program, change the same part by a different method, or use the same method with different parameter values",
          not_for: "Two proposals that merely share the same goal, or merely edit the same function in different ways",
          examples: [
            "'replace SGD with Adam' vs 'add a learning-rate warmup to SGD'",
            "'a 6-row hexagonal grid' vs 'a 5-row hexagonal grid'",
            "'learning rate 1e-3' vs 'learning rate 1e-2' with the same optimizer",
          ],
        },
      },
    },
  },

  // ⑥ 冻结条款：每一问单独一次请求，材料相同。
  freeze: {
    self_contradictory: {
      type: "noul",
      instructions: {
        question:
          "Does the change that `protocol` asks for break a restriction stated elsewhere in `protocol`, or do two of its statements exclude each other, so that no implementation can satisfy all of them?",
        focus:
          "Take the change described in `factors` (each factor's `method`), `safeguards` and `clauses`, then check it against every restriction written in `protocol.body` (what must not change, randomness, when to stop). " +
          "A restriction may name code that must not be edited, or things that must stay as in the baseline, such as positions, sizes or constants; carrying out the change may alter those things even if the change is described in other words. " +
          "A statement that is merely unusual, ambitious or likely to fail is not a contradiction.",
      },
      criteria: {
        true: {
          what: "Carrying out the requested change necessarily violates a stated restriction, or two statements require things that cannot both hold",
          examples: [
            "`clauses` says rewrite compute_max_radii while `protocol` says compute_max_radii must not change",
            "a factor's `method` uses random restarts while `protocol` says the program uses no randomness",
            "a factor's `method` doubles the step size while `protocol` says every numeric constant keeps its baseline value",
          ],
        },
        false: {
          what: "All statements can hold at the same time in one implementation",
          not_for: "Plans that may not improve the metric, or statements that are vague but not conflicting",
          examples: ["rewrite construct_packing and keep compute_max_radii unchanged", "use seed 42 for 10 random restarts and keep the best result"],
        },
      },
    },
    changes_measurement: {
      type: "noul",
      instructions: {
        question:
          "Does `protocol` require changing how the final metric is measured, computed or reported, rather than changing the solution that gets measured?",
        focus:
          "The final metric is measured by the task's harness from what the program returns. " +
          "Changing the algorithm that produces the solution is allowed. Changing how the returned value is summed, scored, validated, rounded or reported is a change of measurement. " +
          "The program's own handling of its candidate solutions — legality checks, repairs, safety margins such as subtracting 1e-9 from every radius before checking, fallbacks, and rules for picking the best result — changes the answer it returns, not how that answer is measured, even when it copies the validator's formula. " +
          "Likewise, rules for the protocol's own diagnostic metrics (what they count, how they are aggregated, when they are written) do not change the final metric.",
      },
      criteria: {
        true: {
          what: "The protocol asks to alter the scoring or validation: the formula of the metric, what gets counted, tolerances, the harness or validator, or the value reported instead of the real one",
          examples: [
            "report the sum of diameters instead of the sum of radii",
            "count only circles with radius above 0.01 when summing",
            "raise the overlap tolerance in validate.py to 1e-6",
          ],
        },
        false: {
          what: "The protocol only changes how the solution is built or searched; the metric is computed from its output as before",
          not_for: "Adding new diagnostic metrics that are recorded next to the final metric without replacing it",
          examples: ["replace the ring layout with a hexagonal grid", "compute radii with a linear program"],
        },
      },
    },
    too_vague: {
      type: "noul",
      instructions: {
        question:
          "Does `protocol` leave the implementer to guess what to change, for example which function to edit, which method to use, or when an iterative search stops?",
        focus:
          "Read `factors` (each factor's `method`), `clauses` and the stopping rule in `protocol.body` as an implementer who must follow them without asking questions. " +
          "Brief text is fine when it names the function, the method and its settings. " +
          "Judge only whether each statement is specific. Whether two statements conflict with each other is checked separately; a protocol whose statements are concrete but conflicting is not vague.",
      },
      criteria: {
        true: {
          what: "An implementer would have to invent a key part of the change: the target code, the method, its main parameters, or the stopping rule of a loop or search",
          examples: ["improve the layout of the circles somehow", "optimize the radii better", "run a local search until it is good enough"],
        },
        false: {
          what: "The target code, the method and its settings are stated concretely enough to implement directly",
          not_for:
            "Protocols that leave only minor coding details, such as variable names, to the implementer, or protocols whose concrete statements contradict each other",
          examples: [
            "in construct_packing, change the outer ring radius constant from 0.7 to 0.45",
            "replace compute_max_radii with an LP solved by a numpy simplex; stop when no pivot improves",
          ],
        },
      },
    },
  },

  // ⑨ 代码改动：一个差异块一次请求，五问同在。
  diff: {
    unseeded_randomness: {
      type: "noul",
      instructions: {
        question: "Do the added lines in `change.diff` use random numbers without the fixed seed that the randomness section of `protocol.body` requires?",
        focus:
          "Look for random number generators and random sampling in added lines. " +
          "Randomness that is seeded exactly as the randomness section of `protocol.body` describes is fine.",
      },
      criteria: {
        true: {
          what: "Added code draws random numbers from an unseeded generator, a time-based seed, or a seed that differs from the protocol",
          examples: ["np.random.rand(26, 2)", "rng = np.random.default_rng()", "random.seed(time.time())"],
        },
        false: {
          what: "Added code uses no random numbers, or seeds them as the protocol says",
          not_for: "Deterministic code that merely contains the word random in a name or comment",
          examples: ["rng = np.random.default_rng(42) when the protocol registers seed 42", "centers[i] = [x0 + d * k, y0]"],
        },
      },
    },
    reads_clock: {
      type: "noul",
      instructions: {
        question:
          "Do the added lines in `change.diff` make the program's result depend on the clock in a way that the stopping section of `protocol.body` does not register?",
        focus:
          "Look for time.time, time.perf_counter, time.monotonic, datetime.now and deadlines built from them. " +
          "A wall-clock time limit that the stopping section of `protocol.body` itself describes, implemented with the same limit to stop the same loop, is required by the protocol and is fine. " +
          "Any other use of the clock that changes what the program computes counts, including a time limit different from the registered one " +
          "and the registered time used for something other than stopping. Reading the clock only to print or log timing does not make the result depend on it.",
      },
      criteria: {
        true: {
          what: "Added code reads the clock and uses it to decide how long to search, when to stop, or which values to use, and the stopping section of `protocol.body` does not register that use or registers a different time limit",
          examples: [
            "while time.time() - t0 < 5:  # protocol says a fixed 2000 steps",
            "step = 0.02 if time.time() - t0 < 3 else 0.005",
            "seed = int(datetime.now().timestamp())",
            "while time.monotonic() - t0 < 60.0:  # protocol registers a 120 s limit",
          ],
        },
        false: {
          what: "Added code does not read the clock, reads it only for printing or logging, or stops a loop on elapsed time exactly as the stopping section of `protocol.body` registers",
          not_for: "Loops bounded by an iteration count",
          examples: [
            "for it in range(2000):",
            "print(f'took {time.time() - t0:.2f}s')",
            "while time.monotonic() - t0 < 120.0 and it < 2000:  # protocol: stop after 120 s of wall time or 2000 iterations",
          ],
        },
      },
    },
    touches_measurement: {
      type: "noul",
      instructions: {
        question:
          "Do the added or removed lines in `change.diff` change how the program computes or reports its score, or compute a metric from `protocol.metrics` differently from its `howMeasured`?",
        focus:
          "The score is computed from the returned solution, such as summing the radii in run_packing. " +
          "Changing how circles are placed or how radii are chosen changes the solution, not the score computation. " +
          "Code that computes a metric listed in `protocol.metrics` the way its `howMeasured` says and writes it to metrics.json is required by the protocol; it is not a change to the score.",
      },
      criteria: {
        true: {
          what: "The hunk changes the score formula, scales or rounds the reported value, reports a value other than the one computed from the solution, edits the code that checks validity, or computes a listed metric with a formula or data different from its `howMeasured`",
          examples: [
            "sum_radii = np.sum(radii) * 1.01",
            "sum_radii = 2 * np.sum(radii)",
            "return centers, radii, 2.7",
            "val = 1.0  # when the metric's howMeasured gives a formula",
          ],
        },
        false: {
          what: "The hunk only changes how centers or radii are produced, or computes and writes a metric from `protocol.metrics` as its `howMeasured` describes; the score is still computed as before",
          not_for: "Recording extra diagnostic values next to the score",
          examples: [
            "radii = solve_lp(centers)",
            "centers[i + 9] = [0.5 + 0.45 * np.cos(a), 0.5 + 0.45 * np.sin(a)]",
            "json.dump({metric_id: float(np.min(gaps))}, f)  # when the metric's howMeasured says min gap",
          ],
        },
      },
    },
    early_exit: {
      type: "noul",
      instructions: {
        question: "Do the added lines in `change.diff` end a loop, a search or the program earlier than the stopping section of `protocol.body` says?",
        focus:
          "Look for added break, return, sys.exit or raise statements and added limits on iterations. " +
          "A stop that the stopping section of `protocol.body` itself describes is fine.",
      },
      criteria: {
        true: {
          what: "Added code stops work before the protocol's stopping rule is met, or adds a stopping rule the protocol does not mention",
          examples: ["if it > 100: break  # protocol says 2000 iterations", "if total > 2.0: return centers, radii", "sys.exit(0)"],
        },
        false: {
          what: "Added code has no new exits, or exits exactly as the protocol's stopping rule says",
          not_for: "Returning the final result at the normal end of a function",
          examples: ["if no_improvement_for >= 100: break  # protocol: stop after 100 idle iterations", "return centers, radii"],
        },
      },
    },
    zero_for_unmeasured: {
      type: "noul",
      instructions: {
        question: "Do the added lines in `change.diff` record 0 or another fixed number when a value could not be measured or computed?",
        focus:
          "Look at exception handlers, missing-value branches and default values of recorded metrics. " +
          "Recording an explicit 'unavailable' marker or leaving the value out is the correct way; writing 0 hides the failure.",
      },
      criteria: {
        true: {
          what: "When a computation fails or a value is missing, added code stores or returns 0 or another made-up number in its place",
          examples: [
            "except Exception: min_gap = 0.0",
            "metrics['clipped-count'] = count if ok else 0",
            "except ValueError: return centers, np.zeros(n)",
          ],
        },
        false: {
          what: "Added code records failures as unavailable, raises an error, or does not deal with missing values at all",
          not_for: "Initializing an accumulator to 0 before a loop that always fills it",
          examples: ["except Exception: metrics['min-gap'] = None", "total = 0.0\nfor r in radii: total += r"],
        },
      },
    },
  },
};

/** 一问的全部内容：问题原文、阈值，以及材料说明、问题原文、阈值三者合起来的哈希。 */
export interface Question {
  name: string;
  noul: Noul;
  threshold: number;
  hash: string;
}

/** 一处的全部问题，按上面写的顺序。 */
export function questionsOf(site: Site): Question[] {
  return Object.entries(QUESTIONS[site]).map(([name, noul]) => {
    const threshold = THRESHOLDS[site][name] as number;
    return { name, noul, threshold, hash: hash({ task: TASKS[site], name, noul: noul as unknown as Json, threshold }) };
  });
}

export function taskOf(site: Site): string {
  return TASKS[site];
}