feat: flow 第 1 部分（骨架、研究开始、第零轮）与第零轮检验
按任务书 12 实现 src/flow：从账本现算下一步的纯函数、主循环、恢复入口、派模型任务和跑正式运行两个共用动作、冻结配置、基线、第零轮三个研究者阶段和入口分配，走到第 1 轮时停下。materials 导出 finalEntries、加 researcherId 和 StudyRecord。 审阅采纳两条：研究编号本机已用过时冻结前报错；docker run 没启动时写完结果就停下，不在本进程里无限起新编号。 eval/ 新增第零轮检验：起研究命令行、回放假模型、检验用 Jev 客户端、六问质量评估、16-real-1 回放、杀进程与两份同时跑。tsc 与 depcruise 也检查 eval/，新增 eval 只经 index.ts 用正式代码的规则。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,148 @@
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { ask, THRESHOLD_NOTE, type Answer, type Noul } from "../lib/jev.ts";
|
||||
import { entriesIn, entryDecision, readLedger, stages, type Entry } from "../lib/ledger.ts";
|
||||
import { ENTRY_ONLY, LEADS_TO_IDEA, SAME_KIND, STOPS_AT_DEAD_END, TASKS } from "./questions.ts";
|
||||
|
||||
// 第零轮质量评估（任务书 12 第 5.2 节六问）。只读账本，问 Jev，原始概率全部存进 <检验目录>/quality.json。
|
||||
// node eval/round-zero/quality.ts <检验目录>（研究在 <检验目录>/study）
|
||||
// "入口全体"是分配时最终有效的全部入口，"分到的入口"是分配结果里每人分到的那个。
|
||||
|
||||
type Idea = { id: string; name: string; description: string };
|
||||
const ideas = JSON.parse(readFileSync("eval/round-zero/ideas.json", "utf8")) as { key: Idea[]; deadEnds: Idea[] };
|
||||
|
||||
const out = process.argv[2];
|
||||
if (out === undefined) throw new Error("要写检验目录，研究在它下面的 study 里");
|
||||
const records = readLedger(join(out, "study"));
|
||||
const decision = entryDecision(records);
|
||||
if (decision === undefined) throw new Error("账本里还没有第 0 轮的入口分配决定，第零轮没跑完");
|
||||
const round0 = stages(records).filter((s) => s.round === 0);
|
||||
const final = entriesIn(records, new Set(round0.map((s) => s.stageId)));
|
||||
const draft = entriesIn(records, new Set(round0.filter((s) => s.stage === "entryDraft").map((s) => s.stageId)));
|
||||
// 自己按编号规则算出的入口全体，要和 flow 分配时用的那份一致；对不上说明 flow 或这里有一边取错了。
|
||||
const decided = decision.input.entries;
|
||||
if (JSON.stringify(decided.map((e) => e.id).sort()) !== JSON.stringify(final.map((e) => e.id).sort())) {
|
||||
throw new Error(`分配决定里的入口 ${decided.map((e) => e.id).join("、")} 和从提交算出的入口全体 ${final.map((e) => e.id).join("、")} 对不上`);
|
||||
}
|
||||
const assignment = decision.output;
|
||||
const assigned = assignment.map((a) => final.find((e) => e.id === a.entry)!);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 问 Jev：同样的材料只问一次；每次请求的材料、问题、原始概率都记下
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const raw: Array<{ site: string; ids: string[]; answers: Answer[] }> = [];
|
||||
const cache = new Map<string, Promise<Answer[]>>();
|
||||
/** 同时最多几个请求在路上：一下发上百个容易被限流。 */
|
||||
const PARALLEL = 6;
|
||||
let running = 0;
|
||||
const waiting: Array<() => void> = [];
|
||||
async function limited<T>(fn: () => Promise<T>): Promise<T> {
|
||||
if (running >= PARALLEL) await new Promise<void>((ok) => waiting.push(ok));
|
||||
running++;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
running--;
|
||||
waiting.shift()?.();
|
||||
}
|
||||
}
|
||||
const show = (e: Entry) => ({ name: e.name, body: e.body });
|
||||
/** 同样的材料、同样的问题只问一次（写入口阶段没换的入口在讨论后还是同一段文字）。 */
|
||||
function query(site: string, ids: string[], state: Record<string, unknown>, questions: Record<string, Noul>): Promise<Answer[]> {
|
||||
const key = JSON.stringify([site, state]);
|
||||
if (!cache.has(key)) {
|
||||
cache.set(key, limited(() => ask(state, questions)).then((answers) => (raw.push({ site, ids, answers }), answers)));
|
||||
}
|
||||
return cache.get(key)!;
|
||||
}
|
||||
const yes = async (p: Promise<Answer[]>, question: string) => (await p).find((a) => a.question === question)!.yes;
|
||||
|
||||
const ideaState = (e: Entry, idea: Idea) => ({ task: TASKS.idea, entry: show(e), idea: { description: idea.description } });
|
||||
const leadsTo = (e: Entry, idea: Idea) => yes(query("leads", [e.id, idea.id], ideaState(e, idea), LEADS_TO_IDEA), "leads_to_idea");
|
||||
const staysIn = (e: Entry, idea: Idea) => yes(query("stays", [e.id, idea.id], ideaState(e, idea), STOPS_AT_DEAD_END), "stays_within_idea");
|
||||
const entryOnly = (e: Entry, question: string) => yes(query("entry", [e.id], { task: TASKS.entry, entry: show(e) }, ENTRY_ONLY), question);
|
||||
const sameKind = (a: Entry, b: Entry) =>
|
||||
yes(query("pair", [a.id, b.id], { task: TASKS.pair, entry_a: show(a), entry_b: show(b) }, SAME_KIND), "same_kind_of_change");
|
||||
|
||||
async function coverage(list: Entry[]): Promise<Record<string, string[]>> {
|
||||
const out: Record<string, string[]> = {};
|
||||
for (const idea of ideas.key) {
|
||||
const hits = await Promise.all(list.map((e) => leadsTo(e, idea)));
|
||||
out[idea.id] = list.filter((_, i) => hits[i]).map((e) => e.id);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
async function overlap(list: Entry[]): Promise<{ pairs: number; same: string[][]; ratio: number | null }> {
|
||||
const pairs = list.flatMap((a, i) => list.slice(i + 1).filter((b) => b.author !== a.author).map((b) => [a, b] as const));
|
||||
const hits = await Promise.all(pairs.map(([a, b]) => sameKind(a, b)));
|
||||
const same = pairs.filter((_, i) => hits[i]).map(([a, b]) => [a.id, b.id]);
|
||||
return { pairs: pairs.length, same, ratio: pairs.length === 0 ? null : same.length / pairs.length };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 六问
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const k1 = ideas.key.find((i) => i.id === "K1")!;
|
||||
const covFinal = await coverage(final);
|
||||
const q1 = { coverage: covFinal, pass: covFinal["K1"]!.length > 0 };
|
||||
|
||||
const assignedPairs = assigned.flatMap((a, i) => assigned.slice(i + 1).map((b) => [a, b] as const));
|
||||
const pairHits = await Promise.all(assignedPairs.map(([a, b]) => sameKind(a, b)));
|
||||
const q2 = { same: assignedPairs.filter((_, i) => pairHits[i]).map(([a, b]) => [a.id, b.id]), pass: !pairHits.some(Boolean) };
|
||||
|
||||
const assignedK1 = await Promise.all(assigned.map((e) => leadsTo(e, k1)));
|
||||
const deadEnd = await Promise.all(
|
||||
assigned.map(async (e) => {
|
||||
const hits = await Promise.all(ideas.deadEnds.map((d) => staysIn(e, d)));
|
||||
return ideas.deadEnds.filter((_, i) => hits[i]).map((d) => d.id);
|
||||
}),
|
||||
);
|
||||
const q3 = {
|
||||
k1: assigned.filter((_, i) => assignedK1[i]).map((e) => e.id),
|
||||
deadEnds: Object.fromEntries(assigned.map((e, i) => [e.id, deadEnd[i]])),
|
||||
pass: assignedK1.some(Boolean) && deadEnd.every((d) => d.length === 0),
|
||||
};
|
||||
|
||||
const perEntry = async (list: Entry[], question: string) => (await Promise.all(list.map((e) => entryOnly(e, question))));
|
||||
const changesCode = await perEntry(assigned, "changes_code");
|
||||
const q4 = { notCodeRoute: assigned.filter((_, i) => !changesCode[i]).map((e) => e.id), pass: changesCode.every(Boolean) };
|
||||
|
||||
const writesAnswer = async (list: Entry[]) => {
|
||||
const [code, score] = [await perEntry(list, "names_code_change"), await perEntry(list, "states_expected_score")];
|
||||
return list.filter((_, i) => code[i] || score[i]).map((e) => e.id);
|
||||
};
|
||||
const q5 = { assignedWritingAnswer: await writesAnswer(assigned), allWritingAnswer: await writesAnswer(final) };
|
||||
const q5pass = q5.assignedWritingAnswer.length === 0;
|
||||
|
||||
const q6 = {
|
||||
beforeDiscussion: { overlap: await overlap(draft), coverage: await coverage(draft) },
|
||||
afterDiscussion: { overlap: await overlap(final), coverage: covFinal },
|
||||
};
|
||||
|
||||
const report = {
|
||||
study: join(out, "study"),
|
||||
thresholdNote: THRESHOLD_NOTE,
|
||||
entries: final.map((e) => ({ id: e.id, author: e.author, name: e.name })),
|
||||
draftEntries: draft.map((e) => ({ id: e.id, author: e.author, name: e.name })),
|
||||
assignment: assignment.map((a, i) => ({ ...a, name: assigned[i]!.name })),
|
||||
results: { q1, q2, q3, q4, q5: { ...q5, pass: q5pass }, q6 },
|
||||
passed: { q1: q1.pass, q2: q2.pass, q3: q3.pass, q4: q4.pass, q5: q5pass },
|
||||
raw,
|
||||
};
|
||||
writeFileSync(join(out, "quality.json"), `${JSON.stringify(report, null, 2)}\n`);
|
||||
|
||||
const mark = (b: boolean) => (b ? "通过" : "不通过");
|
||||
const ratio = (o: { pairs: number; same: string[][]; ratio: number | null }) => `${o.same.length}/${o.pairs}${o.ratio === null ? "" : `（${(o.ratio * 100).toFixed(0)}%）`}`;
|
||||
const count = (c: Record<string, string[]>) => Object.entries(c).map(([k, v]) => `${k}:${v.length}`).join(" ");
|
||||
console.log(`（${THRESHOLD_NOTE}）`);
|
||||
console.log(`入口全体：${final.map((e) => `${e.id} ${e.name}`).join("\n ")}`);
|
||||
console.log(`分到的入口：${report.assignment.map((a) => `${a.researcher} → ${a.entry}（第 ${a.rank} 志愿）${a.name}`).join("\n ")}`);
|
||||
console.log(`1 方向对不对：${mark(q1.pass)}；覆盖 ${count(covFinal)}`);
|
||||
console.log(`2 分没分开：${mark(q2.pass)}；判"是"的对 ${JSON.stringify(q2.same)}`);
|
||||
console.log(`3 好方向分出去没有：${mark(q3.pass)}；指向 K1 的 ${JSON.stringify(q3.k1)}，停在死路的 ${JSON.stringify(q3.deadEnds)}`);
|
||||
console.log(`4 改代码的路线：${mark(q4.pass)}；不是的 ${JSON.stringify(q4.notCodeRoute)}`);
|
||||
console.log(`5 有没有写答案：${mark(q5pass)}；分到的里写了的 ${JSON.stringify(q5.assignedWritingAnswer)}，入口全体里写了的 ${JSON.stringify(q5.allWritingAnswer)}`);
|
||||
console.log(`6 讨论有没有用：讨论前重合 ${ratio(q6.beforeDiscussion.overlap)}、覆盖 ${count(q6.beforeDiscussion.coverage)}；讨论后重合 ${ratio(q6.afterDiscussion.overlap)}、覆盖 ${count(covFinal)}`);
|
||||
console.log(`原始概率：${join(out, "quality.json")}`);
|
||||
Reference in New Issue
Block a user