InfoRSI/eval/round-zero/quality.ts

import { readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { ask, THRESHOLD, THRESHOLD_NOTE, type Answer, type Noul } from "../lib/jev.ts";
import { entriesIn, entryDecision, readLedger, stages, type Entry } from "../lib/ledger.ts";
import { ENTRY_ONLY, LEADS_TO_IDEA, SAME_KIND, STOPS_AT_DEAD_END, TASKS } from "./questions.ts";

// 第零轮质量评估（任务书 12 第 5.2 节六问）。只读账本，问 Jev，原始概率全部存进 <检验目录>/quality.json。
//   node eval/round-zero/quality.ts <检验目录>（研究在 <检验目录>/study）
// "入口全体"是分配时最终有效的全部入口，"分到的入口"是分配结果里每人分到的那个。

type Idea = { id: string; name: string; description: string };
const ideas = JSON.parse(readFileSync("eval/round-zero/ideas.json", "utf8")) as { key: Idea[]; deadEnds: Idea[] };

const out = process.argv[2];
if (out === undefined) throw new Error("要写检验目录，研究在它下面的 study 里");
const records = readLedger(join(out, "study"));
const decision = entryDecision(records);
if (decision === undefined) throw new Error("账本里还没有第 0 轮的入口分配决定，第零轮没跑完");
const round0 = stages(records).filter((s) => s.round === 0);
const final = entriesIn(records, new Set(round0.map((s) => s.stageId)));
const draft = entriesIn(records, new Set(round0.filter((s) => s.stage === "entryDraft").map((s) => s.stageId)));
// 自己按编号规则算出的入口全体，要和 flow 分配时用的那份一致；对不上说明 flow 或这里有一边取错了。
const decided = decision.input.entries;
if (JSON.stringify(decided.map((e) => e.id).sort()) !== JSON.stringify(final.map((e) => e.id).sort())) {
  throw new Error(`分配决定里的入口 ${decided.map((e) => e.id).join("、")} 和从提交算出的入口全体 ${final.map((e) => e.id).join("、")} 对不上`);
}
const assignment = decision.output;
const assigned = assignment.map((a) => final.find((e) => e.id === a.entry)!);

// ---------------------------------------------------------------------------
// 问 Jev：同样的材料只问一次；每次请求的材料、问题、原始概率都记下
// ---------------------------------------------------------------------------

const raw: Array<{ site: string; ids: string[]; answers: Answer[] }> = [];
const cache = new Map<string, Promise<Answer[]>>();
/** 同时最多几个请求在路上：一下发上百个容易被限流。 */
const PARALLEL = 6;
let running = 0;
const waiting: Array<() => void> = [];
async function limited<T>(fn: () => Promise<T>): Promise<T> {
  if (running >= PARALLEL) await new Promise<void>((ok) => waiting.push(ok));
  running++;
  try {
    return await fn();
  } finally {
    running--;
    waiting.shift()?.();
  }
}
const show = (e: Entry) => ({ name: e.name, body: e.body });
/** 同样的材料、同样的问题只问一次（写入口阶段没换的入口在讨论后还是同一段文字）。 */
function query(site: string, ids: string[], state: Record<string, unknown>, questions: Record<string, Noul>): Promise<Answer[]> {
  const key = JSON.stringify([site, state]);
  if (!cache.has(key)) {
    cache.set(key, limited(() => ask(state, questions)).then((answers) => (raw.push({ site, ids, answers }), answers)));
  }
  return cache.get(key)!;
}
const answer = async (p: Promise<Answer[]>, question: string) => (await p).find((a) => a.question === question)!;
const yes = async (p: Promise<Answer[]>, question: string) => (await answer(p, question)).yes;

const ideaState = (e: Entry, idea: Idea) => ({ task: TASKS.idea, entry: show(e), idea: { description: idea.description } });
const leadsTo = (e: Entry, idea: Idea) => yes(query("leads", [e.id, idea.id], ideaState(e, idea), LEADS_TO_IDEA), "leads_to_idea");
const staysIn = (e: Entry, idea: Idea) => yes(query("stays", [e.id, idea.id], ideaState(e, idea), STOPS_AT_DEAD_END), "stays_within_idea");
const entryOnly = (e: Entry, question: string) => yes(query("entry", [e.id], { task: TASKS.entry, entry: show(e) }, ENTRY_ONLY), question);
const pairProbability = async (a: Entry, b: Entry) =>
  (await answer(query("pair", [a.id, b.id], { task: TASKS.pair, entry_a: show(a), entry_b: show(b) }, SAME_KIND), "same_kind_of_change"))
    .probability;

/** 一对入口是不是同一条路：问题本身对称，所以 A、B 和 B、A 各问一次，取两次概率的平均再比阈值（任务书 13 第一部分第 3 条）。 */
type PairJudgement = { pair: [string, string]; ab: number; ba: number; mean: number; same: boolean };
async function sameKind(a: Entry, b: Entry): Promise<PairJudgement> {
  const [ab, ba] = await Promise.all([pairProbability(a, b), pairProbability(b, a)]);
  const mean = (ab + ba) / 2;
  return { pair: [a.id, b.id], ab, ba, mean, same: mean > THRESHOLD };
}
type PairsResult = { pairs: PairJudgement[]; same: string[][] };
async function judgePairs(pairs: Array<readonly [Entry, Entry]>): Promise<PairsResult> {
  const judged = await Promise.all(pairs.map(([a, b]) => sameKind(a, b)));
  return { pairs: judged, same: judged.filter((j) => j.same).map((j) => j.pair) };
}

async function coverage(list: Entry[]): Promise<Record<string, string[]>> {
  const out: Record<string, string[]> = {};
  for (const idea of ideas.key) {
    const hits = await Promise.all(list.map((e) => leadsTo(e, idea)));
    out[idea.id] = list.filter((_, i) => hits[i]).map((e) => e.id);
  }
  return out;
}
/** 重合度：不同作者的入口两两成对，判"是"的对数占总对数的比例。 */
async function overlap(list: Entry[]): Promise<PairsResult & { ratio: number | null }> {
  const judged = await judgePairs(list.flatMap((a, i) => list.slice(i + 1).filter((b) => b.author !== a.author).map((b) => [a, b] as const)));
  return { ...judged, ratio: judged.pairs.length === 0 ? null : judged.same.length / judged.pairs.length };
}

// ---------------------------------------------------------------------------
// 六问
// ---------------------------------------------------------------------------

const k1 = ideas.key.find((i) => i.id === "K1")!;
const covFinal = await coverage(final);
const q1 = { coverage: covFinal, pass: covFinal["K1"]!.length > 0 };

const assignedPairs = assigned.flatMap((a, i) => assigned.slice(i + 1).map((b) => [a, b] as const));
const assignedJudged = await judgePairs(assignedPairs);
const q2 = { ...assignedJudged, pass: assignedJudged.same.length === 0 };

const assignedK1 = await Promise.all(assigned.map((e) => leadsTo(e, k1)));
const deadEnd = await Promise.all(
  assigned.map(async (e) => {
    const hits = await Promise.all(ideas.deadEnds.map((d) => staysIn(e, d)));
    return ideas.deadEnds.filter((_, i) => hits[i]).map((d) => d.id);
  }),
);
const q3 = {
  k1: assigned.filter((_, i) => assignedK1[i]).map((e) => e.id),
  deadEnds: Object.fromEntries(assigned.map((e, i) => [e.id, deadEnd[i]])),
  pass: assignedK1.some(Boolean) && deadEnd.every((d) => d.length === 0),
};

const perEntry = async (list: Entry[], question: string) => (await Promise.all(list.map((e) => entryOnly(e, question))));
const changesCode = await perEntry(assigned, "changes_code");
const q4 = { notCodeRoute: assigned.filter((_, i) => !changesCode[i]).map((e) => e.id), pass: changesCode.every(Boolean) };

const writesAnswer = async (list: Entry[]) => {
  const [code, score] = [await perEntry(list, "names_code_change"), await perEntry(list, "states_expected_score")];
  return list.filter((_, i) => code[i] || score[i]).map((e) => e.id);
};
const q5 = { assignedWritingAnswer: await writesAnswer(assigned), allWritingAnswer: await writesAnswer(final) };
const q5pass = q5.assignedWritingAnswer.length === 0;

const q6 = {
  beforeDiscussion: { overlap: await overlap(draft), coverage: await coverage(draft) },
  afterDiscussion: { overlap: await overlap(final), coverage: covFinal },
};

const report = {
  study: join(out, "study"),
  thresholdNote: THRESHOLD_NOTE,
  entries: final.map((e) => ({ id: e.id, author: e.author, name: e.name })),
  draftEntries: draft.map((e) => ({ id: e.id, author: e.author, name: e.name })),
  assignment: assignment.map((a, i) => ({ ...a, name: assigned[i]!.name })),
  results: { q1, q2, q3, q4, q5: { ...q5, pass: q5pass }, q6 },
  passed: { q1: q1.pass, q2: q2.pass, q3: q3.pass, q4: q4.pass, q5: q5pass },
  raw,
};
writeFileSync(join(out, "quality.json"), `${JSON.stringify(report, null, 2)}\n`);

const mark = (b: boolean) => (b ? "通过" : "不通过");
const ratio = (o: PairsResult & { ratio: number | null }) => `${o.same.length}/${o.pairs.length}${o.ratio === null ? "" : `（${(o.ratio * 100).toFixed(0)}%）`}`;
const count = (c: Record<string, string[]>) => Object.entries(c).map(([k, v]) => `${k}:${v.length}`).join(" ");
console.log(`（${THRESHOLD_NOTE}）`);
console.log(`入口全体：${final.map((e) => `${e.id} ${e.name}`).join("\n          ")}`);
console.log(`分到的入口：${report.assignment.map((a) => `${a.researcher} → ${a.entry}（第 ${a.rank} 志愿）${a.name}`).join("\n          ")}`);
console.log(`1 方向对不对：${mark(q1.pass)}；覆盖 ${count(covFinal)}`);
const pairLine = (j: PairJudgement) => `${j.pair.join("、")}：${j.ab.toFixed(2)}／${j.ba.toFixed(2)}，平均 ${j.mean.toFixed(2)}`;
console.log(`2 分没分开：${mark(q2.pass)}；判"是"的对 ${JSON.stringify(q2.same)}；每对两个顺序的概率\n          ${q2.pairs.map(pairLine).join("\n          ")}`);
console.log(`3 好方向分出去没有：${mark(q3.pass)}；指向 K1 的 ${JSON.stringify(q3.k1)}，停在死路的 ${JSON.stringify(q3.deadEnds)}`);
console.log(`4 改代码的路线：${mark(q4.pass)}；不是的 ${JSON.stringify(q4.notCodeRoute)}`);
console.log(`5 有没有写答案：${mark(q5pass)}；分到的里写了的 ${JSON.stringify(q5.assignedWritingAnswer)}，入口全体里写了的 ${JSON.stringify(q5.allWritingAnswer)}`);
console.log(`6 讨论有没有用：讨论前重合 ${ratio(q6.beforeDiscussion.overlap)}、覆盖 ${count(q6.beforeDiscussion.coverage)}；讨论后重合 ${ratio(q6.afterDiscussion.overlap)}、覆盖 ${count(covFinal)}`);
console.log(`原始概率：${join(out, "quality.json")}`);