98 lines
5.6 KiB
TypeScript
98 lines
5.6 KiB
TypeScript
import { readdirSync, readFileSync, writeFileSync } from "node:fs";
|
||
import { join } from "node:path";
|
||
import { ask, THRESHOLD, THRESHOLD_NOTE, type Answer, type Noul } from "../lib/jev.ts";
|
||
import { ADDRESSED, CARRIES_IDEA, SAME_CHANGE, STAYS_IN_DEAD_END, TASKS } from "./questions.ts";
|
||
import type { RoundData } from "./round-data.ts";
|
||
|
||
// 第 1 轮的 Jev 封闭问题（任务书 14 第 3.2 节第 3 步），对 export.ts 导出的每次运行都做（我们的和参考项目的），读 data.json：
|
||
// a. 判重对不对：本轮提案两两、两个顺序各问一次"是不是同一个改法"，取平均，和程序的判重决定对照；
|
||
// b. 挑刺有没有被回应：每条挑刺问一次（找不到回应的不问，直接记"没有回应"）；
|
||
// c. 提案对应研究路径的哪一步：每条提案对 K1–K5（做没做这个想法）和 D1、D2（是不是整个停在这条死路）各问一次，只描述。
|
||
// 阈值未校准，概率大于 0.5 判"是"；原始概率全部存进 <检验目录>/jevcheck.json。要花钱（Jev）。
|
||
// node eval/round-one/jevcheck.ts <export.ts 写出的检验目录>
|
||
|
||
type Idea = { id: string; name: string; description: string };
|
||
const ideas = JSON.parse(readFileSync("eval/round-zero/ideas.json", "utf8")) as { key: Idea[]; deadEnds: Idea[] };
|
||
|
||
const out = process.argv[2];
|
||
if (out === undefined) throw new Error("要写 export.ts 写出的检验目录");
|
||
|
||
/** 同时最多几个请求在路上：一下发上百个容易被限流。 */
|
||
const PARALLEL = 6;
|
||
let running = 0;
|
||
const waiting: Array<() => void> = [];
|
||
async function limited<T>(fn: () => Promise<T>): Promise<T> {
|
||
if (running >= PARALLEL) await new Promise<void>((ok) => waiting.push(ok));
|
||
running++;
|
||
try {
|
||
return await fn();
|
||
} finally {
|
||
running--;
|
||
waiting.shift()?.();
|
||
}
|
||
}
|
||
async function one(state: Record<string, unknown>, questions: Record<string, Noul>): Promise<Answer> {
|
||
const [answer] = await limited(() => ask(state, questions));
|
||
return answer!;
|
||
}
|
||
|
||
async function check(d: RoundData) {
|
||
const change = new Map(d.proposals.map((p) => [p.id, p.change]));
|
||
const groupOf = new Map(d.dedup.groups.flatMap((g) => g.proposals.map((p) => [p, g.candidate] as const)));
|
||
|
||
// a. 判重
|
||
const pairQuestion = (a: string, b: string) => one({ task: TASKS.pair, proposal_a: change.get(a), proposal_b: change.get(b) }, SAME_CHANGE);
|
||
const pairs = await Promise.all(
|
||
d.proposals.flatMap((a, i) =>
|
||
d.proposals.slice(i + 1).map(async (b) => {
|
||
const [ab, ba] = await Promise.all([pairQuestion(a.id, b.id), pairQuestion(b.id, a.id)]);
|
||
const mean = (ab.probability + ba.probability) / 2;
|
||
return { pair: [a.id, b.id], ab, ba, mean, jevSame: mean > THRESHOLD, programMerged: groupOf.get(a.id) === groupOf.get(b.id) };
|
||
}),
|
||
),
|
||
);
|
||
const dedup = {
|
||
pairs,
|
||
jevYesProgramNotMerged: pairs.filter((p) => p.jevSame && !p.programMerged).map((p) => p.pair),
|
||
jevNoProgramMerged: pairs.filter((p) => !p.jevSame && p.programMerged).map((p) => p.pair),
|
||
};
|
||
|
||
// b. 挑刺有没有被回应
|
||
const critiques = await Promise.all(
|
||
d.critiques.map(async (c) => {
|
||
const response = d.responses.find((r) => r.replies.some((x) => x.critique === c.id));
|
||
if (response === undefined) return { critique: c.id, author: c.author, proposal: c.proposal, replied: false as const, answer: null };
|
||
const reply = response.replies.find((x) => x.critique === c.id)!.text;
|
||
const answer = await one({ task: TASKS.critique, critique: c.content, reply, revision: response.rest }, ADDRESSED);
|
||
return { critique: c.id, author: c.author, proposal: c.proposal, replied: true as const, answer };
|
||
}),
|
||
);
|
||
|
||
// c. 研究路径
|
||
const path = await Promise.all(
|
||
d.proposals.map(async (p) => {
|
||
const ask = (idea: Idea, questions: Record<string, Noul>) =>
|
||
one({ task: TASKS.idea, proposal: p.change, idea: { description: idea.description } }, questions).then((answer) => ({ idea: idea.id, answer }));
|
||
const answers = await Promise.all([...ideas.key.map((k) => ask(k, CARRIES_IDEA)), ...ideas.deadEnds.map((k) => ask(k, STAYS_IN_DEAD_END))]);
|
||
return { proposal: p.id, author: p.author, answers };
|
||
}),
|
||
);
|
||
return { dedup, critiques, path };
|
||
}
|
||
|
||
const report: Record<string, Awaited<ReturnType<typeof check>> & { source: string }> = {};
|
||
for (const run of readdirSync(join(out, "runs")).sort()) {
|
||
const d = JSON.parse(readFileSync(join(out, "runs", run, "data.json"), "utf8")) as RoundData;
|
||
report[run] = { source: d.source, ...(await check(d)) };
|
||
const r = report[run];
|
||
const p = (x: number) => x.toFixed(2);
|
||
console.log(`\n== ${run}`);
|
||
console.log(`a 判重：${r.dedup.pairs.map((x) => `${x.pair.join("~")} ${p(x.ab.probability)}/${p(x.ba.probability)} 平均 ${p(x.mean)}${x.programMerged ? "（程序合了）" : ""}`).join("；")}`);
|
||
console.log(` 检验判是而程序没合：${JSON.stringify(r.dedup.jevYesProgramNotMerged)}；检验判否而程序合了：${JSON.stringify(r.dedup.jevNoProgramMerged)}`);
|
||
const answered = r.critiques.filter((c) => c.answer?.yes === true).length;
|
||
console.log(`b 挑刺被回应：${answered}/${r.critiques.length}；${r.critiques.map((c) => `${c.critique.slice(0, 40)} ${c.replied ? p(c.answer!.probability) : "没有回应"}`).join("；")}`);
|
||
for (const x of r.path) console.log(`c ${x.proposal}：${x.answers.map((a) => `${a.idea} ${p(a.answer.probability)}${a.answer.yes ? "是" : ""}`).join(" ")}`);
|
||
}
|
||
writeFileSync(join(out, "jevcheck.json"), `${JSON.stringify({ thresholdNote: THRESHOLD_NOTE, runs: report }, null, 2)}\n`);
|
||
console.log(`\n（${THRESHOLD_NOTE}）原始概率：${join(out, "jevcheck.json")}`);
|