eval: 第 1 轮检验脚本（跑、杀进程、导出去来历、盲评编号、Jev 三问、数字与过程）

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

eval/round-one/jevcheck.ts

@@ -0,0 +1,97 @@
+import { readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { ask, THRESHOLD, THRESHOLD_NOTE, type Answer, type Noul } from "../lib/jev.ts";
+import { ADDRESSED, CARRIES_IDEA, SAME_CHANGE, STAYS_IN_DEAD_END, TASKS } from "./questions.ts";
+import type { RoundData } from "./round-data.ts";
+
+// 第 1 轮的 Jev 封闭问题（任务书 14 第 3.2 节第 3 步），对 export.ts 导出的每次运行都做（我们的和参考项目的），读 data.json：
+//   a. 判重对不对：本轮提案两两、两个顺序各问一次"是不是同一个改法"，取平均，和程序的判重决定对照；
+//   b. 挑刺有没有被回应：每条挑刺问一次（找不到回应的不问，直接记"没有回应"）；
+//   c. 提案对应研究路径的哪一步：每条提案对 K1–K5（做没做这个想法）和 D1、D2（是不是整个停在这条死路）各问一次，只描述。
+// 阈值未校准，概率大于 0.5 判"是"；原始概率全部存进 <检验目录>/jevcheck.json。要花钱（Jev）。
+//   node eval/round-one/jevcheck.ts <export.ts 写出的检验目录>
+
+type Idea = { id: string; name: string; description: string };
+const ideas = JSON.parse(readFileSync("eval/round-zero/ideas.json", "utf8")) as { key: Idea[]; deadEnds: Idea[] };
+
+const out = process.argv[2];
+if (out === undefined) throw new Error("要写 export.ts 写出的检验目录");
+
+/** 同时最多几个请求在路上：一下发上百个容易被限流。 */
+const PARALLEL = 6;
+let running = 0;
+const waiting: Array<() => void> = [];
+async function limited<T>(fn: () => Promise<T>): Promise<T> {
+  if (running >= PARALLEL) await new Promise<void>((ok) => waiting.push(ok));
+  running++;
+  try {
+    return await fn();
+  } finally {
+    running--;
+    waiting.shift()?.();
+  }
+}
+async function one(state: Record<string, unknown>, questions: Record<string, Noul>): Promise<Answer> {
+  const [answer] = await limited(() => ask(state, questions));
+  return answer!;
+}
+
+async function check(d: RoundData) {
+  const change = new Map(d.proposals.map((p) => [p.id, p.change]));
+  const groupOf = new Map(d.dedup.groups.flatMap((g) => g.proposals.map((p) => [p, g.candidate] as const)));
+
+  // a. 判重
+  const pairQuestion = (a: string, b: string) => one({ task: TASKS.pair, proposal_a: change.get(a), proposal_b: change.get(b) }, SAME_CHANGE);
+  const pairs = await Promise.all(
+    d.proposals.flatMap((a, i) =>
+      d.proposals.slice(i + 1).map(async (b) => {
+        const [ab, ba] = await Promise.all([pairQuestion(a.id, b.id), pairQuestion(b.id, a.id)]);
+        const mean = (ab.probability + ba.probability) / 2;
+        return { pair: [a.id, b.id], ab, ba, mean, jevSame: mean > THRESHOLD, programMerged: groupOf.get(a.id) === groupOf.get(b.id) };
+      }),
+    ),
+  );
+  const dedup = {
+    pairs,
+    jevYesProgramNotMerged: pairs.filter((p) => p.jevSame && !p.programMerged).map((p) => p.pair),
+    jevNoProgramMerged: pairs.filter((p) => !p.jevSame && p.programMerged).map((p) => p.pair),
+  };
+
+  // b. 挑刺有没有被回应
+  const critiques = await Promise.all(
+    d.critiques.map(async (c) => {
+      const response = d.responses.find((r) => r.replies.some((x) => x.critique === c.id));
+      if (response === undefined) return { critique: c.id, author: c.author, proposal: c.proposal, replied: false as const, answer: null };
+      const reply = response.replies.find((x) => x.critique === c.id)!.text;
+      const answer = await one({ task: TASKS.critique, critique: c.content, reply, revision: response.rest }, ADDRESSED);
+      return { critique: c.id, author: c.author, proposal: c.proposal, replied: true as const, answer };
+    }),
+  );
+
+  // c. 研究路径
+  const path = await Promise.all(
+    d.proposals.map(async (p) => {
+      const ask = (idea: Idea, questions: Record<string, Noul>) =>
+        one({ task: TASKS.idea, proposal: p.change, idea: { description: idea.description } }, questions).then((answer) => ({ idea: idea.id, answer }));
+      const answers = await Promise.all([...ideas.key.map((k) => ask(k, CARRIES_IDEA)), ...ideas.deadEnds.map((k) => ask(k, STAYS_IN_DEAD_END))]);
+      return { proposal: p.id, author: p.author, answers };
+    }),
+  );
+  return { dedup, critiques, path };
+}
+
+const report: Record<string, Awaited<ReturnType<typeof check>> & { source: string }> = {};
+for (const run of readdirSync(join(out, "runs")).sort()) {
+  const d = JSON.parse(readFileSync(join(out, "runs", run, "data.json"), "utf8")) as RoundData;
+  report[run] = { source: d.source, ...(await check(d)) };
+  const r = report[run];
+  const p = (x: number) => x.toFixed(2);
+  console.log(`\n== ${run}`);
+  console.log(`a 判重：${r.dedup.pairs.map((x) => `${x.pair.join("~")} ${p(x.ab.probability)}/${p(x.ba.probability)} 平均 ${p(x.mean)}${x.programMerged ? "（程序合了）" : ""}`).join("；")}`);
+  console.log(`  检验判是而程序没合：${JSON.stringify(r.dedup.jevYesProgramNotMerged)}；检验判否而程序合了：${JSON.stringify(r.dedup.jevNoProgramMerged)}`);
+  const answered = r.critiques.filter((c) => c.answer?.yes === true).length;
+  console.log(`b 挑刺被回应：${answered}/${r.critiques.length}；${r.critiques.map((c) => `${c.critique.slice(0, 40)} ${c.replied ? p(c.answer!.probability) : "没有回应"}`).join("；")}`);
+  for (const x of r.path) console.log(`c ${x.proposal}：${x.answers.map((a) => `${a.idea} ${p(a.answer.probability)}${a.answer.yes ? "是" : ""}`).join(" ")}`);
+}
+writeFileSync(join(out, "jevcheck.json"), `${JSON.stringify({ thresholdNote: THRESHOLD_NOTE, runs: report }, null, 2)}\n`);
+console.log(`\n（${THRESHOLD_NOTE}）原始概率：${join(out, "jevcheck.json")}`);