eval: 第 1 轮检验脚本（跑、杀进程、导出去来历、盲评编号、Jev 三问、数字与过程）
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,352 @@
|
||||
import { copyFileSync, mkdirSync, statSync } from "node:fs";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import type { LedgerRecord } from "../../src/ledger/index.ts";
|
||||
import type { DedupOutput, FrozenProtocol } from "../../src/materials/index.ts";
|
||||
import { finalSubmissions, ofKind, stages } from "../lib/ledger.ts";
|
||||
|
||||
// 一次运行第 1 轮（开轮到选批）的记录，读成同一种形状，导出文本和 Jev 检验都用它（任务书 14 第 3.2 节）。
|
||||
// 两种来源：我们的账本（经 src/ledger 读，像用户一样）和参考项目的账本（只读，先复制到输出目录再打开）。
|
||||
// 内容按原样存 Json，字段名不统一（参考项目叫 points，我们叫 factors），只把检验要用的几样取到固定的键上。
|
||||
// 按名字读，读不到就报错停下：flow 第 2 部分还在写，读错了要立刻看出来，不猜。
|
||||
|
||||
export type Json = null | boolean | number | string | Json[] | { [key: string]: Json };
|
||||
export type JsonObject = { [key: string]: Json };
|
||||
export interface Range3 {
|
||||
q1: number;
|
||||
q5: number;
|
||||
q9: number;
|
||||
}
|
||||
export interface Prediction {
|
||||
researcher: string;
|
||||
candidate: string;
|
||||
range: Range3;
|
||||
/** 预测附的理由（我们的 body、参考项目 14 的 rationale）；没有就是 null。 */
|
||||
note: Json;
|
||||
}
|
||||
|
||||
export interface RoundData {
|
||||
/** 来历（运行名、库路径），只进 data.json，不进给审阅者的文本。 */
|
||||
source: string;
|
||||
/** 要从文本里去掉的名字：运行名、研究编号。 */
|
||||
hiddenNames: string[];
|
||||
proposals: Array<{
|
||||
id: string;
|
||||
author: string;
|
||||
content: JsonObject;
|
||||
/** 判重看的"改了什么"：我们是因子的名称、做法、关键设定；参考项目是 points 或 modification。 */
|
||||
change: Json;
|
||||
authorEstimate: Range3 | null;
|
||||
}>;
|
||||
dedup: {
|
||||
/** 本轮新候选各合并了哪几条原提案。 */
|
||||
groups: Array<{ candidate: string; proposals: string[] }>;
|
||||
/** 并进留池候选的新提案。 */
|
||||
carried: Json;
|
||||
/** 程序判重时 Jev 每一对的判断，原样。 */
|
||||
judgments: Json;
|
||||
};
|
||||
critiques: Array<{ id: string; author: string; proposal: string; content: JsonObject }>;
|
||||
responses: Array<{
|
||||
candidate: string;
|
||||
author: string;
|
||||
kind: string;
|
||||
replies: Array<{ critique: string; text: string }>;
|
||||
/** 回应里逐条答复以外的部分（改动、实现条款、正文）。 */
|
||||
rest: JsonObject;
|
||||
}>;
|
||||
protocols: Array<{ candidate: string; content: JsonObject }>;
|
||||
/** null：这次运行没有第一次预测这个阶段。 */
|
||||
firstPrediction: Prediction[] | null;
|
||||
/** null：这次运行用投票，不是范围预测，投票原样在 votes 里。 */
|
||||
secondPrediction: Prediction[] | null;
|
||||
votes: Json;
|
||||
batch: { chosen: string[]; detail: Json };
|
||||
rejections: Array<{ stage: string; author: string; target: string | null; reason: string }>;
|
||||
}
|
||||
|
||||
/** 中性的阶段名，两种来源共用。 */
|
||||
export const STAGE_NAMES = {
|
||||
proposal: "提案",
|
||||
firstPrediction: "第一次预测",
|
||||
critique: "挑刺",
|
||||
response: "回应与统一",
|
||||
secondPrediction: "第二次预测",
|
||||
vote: "投票",
|
||||
} as const;
|
||||
type StageName = keyof typeof STAGE_NAMES;
|
||||
|
||||
function need<T>(value: T | undefined, what: string): T {
|
||||
if (value === undefined) throw new Error(`读不到：${what}`);
|
||||
return value;
|
||||
}
|
||||
function field(obj: JsonObject, key: string, where: string): Json {
|
||||
return need(obj[key], `${where} 的字段 ${key}`);
|
||||
}
|
||||
const omit = (obj: JsonObject, keys: string[]): JsonObject => Object.fromEntries(Object.entries(obj).filter(([k]) => !keys.includes(k)));
|
||||
const asObject = (v: unknown) => v as JsonObject;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 我们的账本
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 我们的研究第 round 轮。records 是 readLedger 读出的全部记录，studyId 是研究编号（要从文本里去掉）。 */
|
||||
export function readOurs(records: LedgerRecord[], round: number, source: string, studyId: string): RoundData {
|
||||
const inRound = stages(records).filter((s) => s.round === round);
|
||||
const stageOf = (name: string) => need(inRound.find((s) => s.stage === name), `第 ${round} 轮 ${name} 阶段的开记录`).stageId;
|
||||
const stageName = new Map(inRound.map((s) => [s.stageId, s.stage]));
|
||||
const finals = finalSubmissions(records);
|
||||
const subs = (stage: string, kinds: string[]) => {
|
||||
const id = stageOf(stage);
|
||||
return finals.filter((s) => s.content.stageId === id && kinds.includes(s.content.kind));
|
||||
};
|
||||
const seals = ofKind(records, "seal");
|
||||
|
||||
const proposals = subs("proposal", ["proposal"]).map((s) => {
|
||||
const content = asObject(s.content.content);
|
||||
const factors = field(content, "factors", `提案 ${s.content.item}`) as JsonObject[];
|
||||
const seal = need(seals.find((x) => x.content.proposal === s.seq), `提案 ${s.content.item}（序号 ${s.seq}）的作者自评封存`);
|
||||
return {
|
||||
id: need(s.content.item ?? undefined, `提案序号 ${s.seq} 的编号 item`),
|
||||
author: s.content.author,
|
||||
content,
|
||||
change: factors.map((f) => ({ name: f["name"]!, method: f["method"]!, keySettings: f["keySettings"]! })),
|
||||
authorEstimate: seal.content.range,
|
||||
};
|
||||
});
|
||||
|
||||
const decision = (kind: string) =>
|
||||
need(ofKind(records, "decision").find((d) => d.content.decision === kind && d.content.round === round), `第 ${round} 轮的 ${kind} 决定`).content;
|
||||
const dedup = decision("dedup");
|
||||
const dedupOut = dedup.output as unknown as DedupOutput;
|
||||
need(dedupOut.fresh, "判重决定输出的 fresh");
|
||||
need(dedupOut.carried, "判重决定输出的 carried");
|
||||
|
||||
const critiques = subs("critique", ["critique"]).map((s) => ({
|
||||
id: need(s.content.item ?? undefined, `挑刺序号 ${s.seq} 的编号 item`),
|
||||
author: s.content.author,
|
||||
proposal: need(s.content.target ?? undefined, `挑刺序号 ${s.seq} 的对象 target`),
|
||||
content: asObject(s.content.content),
|
||||
}));
|
||||
|
||||
const responses = subs("response", ["response", "unification"]).map((s) => {
|
||||
const content = asObject(s.content.content);
|
||||
const replies = field(content, "replies", `回应序号 ${s.seq}`) as JsonObject[];
|
||||
return {
|
||||
candidate: need(s.content.target ?? undefined, `回应序号 ${s.seq} 的对象 target`),
|
||||
author: s.content.author,
|
||||
kind: s.content.kind,
|
||||
replies: replies.map((r) => ({ critique: r["critique"] as string, text: r["reply"] as string })),
|
||||
rest: omit(content, ["replies"]),
|
||||
};
|
||||
});
|
||||
|
||||
const fresh = new Set(dedupOut.fresh.map((g) => g.candidate));
|
||||
const protocols = ofKind(records, "protocol")
|
||||
.filter((p) => fresh.has(p.content.candidate) && (p.content.protocol as unknown as FrozenProtocol).round === round)
|
||||
.map((p) => ({ candidate: p.content.candidate, content: asObject(p.content.protocol) }));
|
||||
|
||||
const predictions = (stage: string): Prediction[] =>
|
||||
subs(stage, [stage]).map((s) => {
|
||||
const content = asObject(s.content.content);
|
||||
return {
|
||||
researcher: s.content.author,
|
||||
candidate: need(s.content.target ?? undefined, `${stage} 序号 ${s.seq} 的对象 target`),
|
||||
range: field(content, "range", `${stage} 序号 ${s.seq}`) as unknown as Range3,
|
||||
note: content["body"] ?? null,
|
||||
};
|
||||
});
|
||||
|
||||
const batch = decision("batch");
|
||||
const chosen = need(asObject(batch.output)["chosen"], "选批决定输出的 chosen") as string[];
|
||||
|
||||
const rejections = ofKind(records, "rejection")
|
||||
.filter((r) => stageName.has(r.content.stageId))
|
||||
.map((r) => ({
|
||||
stage: STAGE_NAMES[need(ourStage(stageName.get(r.content.stageId)!), `阶段 ${r.content.stageId} 的中性名`)],
|
||||
author: r.content.author,
|
||||
target: r.content.target,
|
||||
reason: r.content.reason,
|
||||
}));
|
||||
|
||||
return {
|
||||
source,
|
||||
hiddenNames: [studyId],
|
||||
proposals,
|
||||
dedup: { groups: dedupOut.fresh, carried: dedupOut.carried as unknown as Json, judgments: dedup.input as Json },
|
||||
critiques,
|
||||
responses,
|
||||
protocols,
|
||||
firstPrediction: predictions("firstPrediction"),
|
||||
secondPrediction: predictions("secondPrediction"),
|
||||
votes: null,
|
||||
batch: { chosen, detail: batch.output as Json },
|
||||
rejections,
|
||||
};
|
||||
}
|
||||
|
||||
function ourStage(stage: string): StageName | undefined {
|
||||
return stage in STAGE_NAMES ? (stage as StageName) : undefined;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 参考项目的账本（/home/iomgaa/rb-acceptance/<运行名>/board.sqlite）
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 参考项目的阶段名（stage 编号的最后一段）→ 中性名。挑刺和回应都在 phase review 里，按编号区分。 */
|
||||
const REF_STAGES: Record<string, StageName> = {
|
||||
proposal: "proposal",
|
||||
"predict-blind": "firstPrediction",
|
||||
critique: "critique",
|
||||
response: "response",
|
||||
predict: "secondPrediction",
|
||||
vote: "vote",
|
||||
};
|
||||
|
||||
/** 原库只读：库文件（和非空的 -wal）先复制到 copyDir，再以只读方式打开复制件，原目录里不写任何东西。 */
|
||||
function openCopy(board: string, copyDir: string): DatabaseSync {
|
||||
mkdirSync(copyDir, { recursive: true });
|
||||
const copy = join(copyDir, "board.sqlite");
|
||||
copyFileSync(board, copy);
|
||||
if (statSync(`${board}-wal`, { throwIfNoEntry: false })?.size) copyFileSync(`${board}-wal`, `${copy}-wal`);
|
||||
return new DatabaseSync(copy, { readOnly: true });
|
||||
}
|
||||
|
||||
const refRange = (r: JsonObject): Range3 => ({ q1: r["q10"] as number, q5: r["q50"] as number, q9: r["q90"] as number });
|
||||
|
||||
export function readReference(board: string, copyDir: string): RoundData {
|
||||
const run = basename(dirname(board));
|
||||
const roundId = `${run}.r1`;
|
||||
const db = openCopy(board, copyDir);
|
||||
try {
|
||||
const all = <T>(sql: string, ...args: Array<string | number>) => db.prepare(sql).all(...args) as T[];
|
||||
const stageRows = all<{ id: string }>("SELECT id FROM stages WHERE round_id = ? ORDER BY created_seq", roundId);
|
||||
const stageOf = new Map<StageName, string>();
|
||||
for (const s of stageRows) {
|
||||
const name = REF_STAGES[s.id.split(".").at(-1)!];
|
||||
if (name !== undefined) {
|
||||
if (stageOf.has(name)) throw new Error(`${board}：第 1 轮 ${name} 阶段不止一个（${stageOf.get(name)}、${s.id}）`);
|
||||
stageOf.set(name, s.id);
|
||||
}
|
||||
}
|
||||
// 14、12 那两版的 submissions 表还没有 version 列（一人一个 slot 只有一份），按第 1 版读。
|
||||
const versioned = all<{ name: string }>("PRAGMA table_info(submissions)").some((c) => c.name === "version");
|
||||
/** 每份提交（阶段、人、slot）的最后一版，按第一次出现的先后排。 */
|
||||
const subs = (name: StageName) => {
|
||||
const stage = need(stageOf.get(name), `${board} 第 1 轮 ${name} 阶段`);
|
||||
const rows = all<{ actor: string; slot: string; version: number; payload: string }>(
|
||||
`SELECT actor, slot, ${versioned ? "version" : "1 AS version"}, payload FROM submissions WHERE stage_id = ? ORDER BY created_seq`,
|
||||
stage,
|
||||
);
|
||||
const latest = new Map<string, { stage: string; actor: string; slot: string; version: number; payload: JsonObject }>();
|
||||
for (const r of rows) {
|
||||
const key = `${r.actor}\n${r.slot}`;
|
||||
const seen = latest.get(key);
|
||||
if (seen === undefined || seen.version < r.version) latest.set(key, { stage, actor: r.actor, slot: r.slot, version: r.version, payload: JSON.parse(r.payload) });
|
||||
}
|
||||
return [...latest.values()];
|
||||
};
|
||||
const fact = (kind: string) => {
|
||||
const rows = all<{ payload: string }>(
|
||||
"SELECT payload FROM facts WHERE kind = ? AND id LIKE ? ORDER BY created_seq",
|
||||
kind,
|
||||
`${run}.${kind}.${roundId}.%`,
|
||||
);
|
||||
return JSON.parse(need(rows.at(-1), `${board} 第 1 轮的 ${kind}`).payload) as JsonObject;
|
||||
};
|
||||
|
||||
const proposals = subs("proposal").map((s) => {
|
||||
const p = s.payload;
|
||||
const points = p["points"] as JsonObject[] | undefined;
|
||||
const change: Json =
|
||||
points !== undefined
|
||||
? points.map((x) => ({ name: x["name"]!, method: x["method"]!, ...(x["keySettings"] === undefined ? {} : { keySettings: x["keySettings"] }) }))
|
||||
: { modification: field(p, "modification", `${board} 提案 ${s.slot}`) };
|
||||
const estimate = (p["authorRange"] ?? p["range"]) as JsonObject | undefined;
|
||||
return {
|
||||
id: s.slot,
|
||||
author: s.actor,
|
||||
content: omit(p, ["authorId", "id", "roundId", "authorRange", "range"]),
|
||||
change,
|
||||
authorEstimate: estimate === undefined ? null : refRange(estimate),
|
||||
};
|
||||
});
|
||||
|
||||
const dedup = fact("dedup-record");
|
||||
const groups = (field(dedup, "candidates", `${board} dedup-record`) as JsonObject[]).map((c) => ({
|
||||
candidate: c["candidateId"] as string,
|
||||
proposals: c["proposalIds"] as string[],
|
||||
}));
|
||||
|
||||
// 一份挑刺提交（一人对一条原提案）是一条挑刺，编号是它的 review-submission 事实编号：回应里的 critiqueId 指的就是它。
|
||||
const critiqueStage = need(stageOf.get("critique"), `${board} 第 1 轮挑刺阶段`);
|
||||
const critiqueIds = new Map(
|
||||
all<{ id: string; slot: string }>("SELECT id, slot FROM facts WHERE kind = 'review-submission' AND slot LIKE ?", `${critiqueStage}/%`).map((f) => [f.slot, f.id]),
|
||||
);
|
||||
const critiques = subs("critique").map((s) => ({
|
||||
id: need(critiqueIds.get(`${s.stage}/${s.actor}/${s.slot}`), `${board} 挑刺 ${s.actor}/${s.slot} 的事实编号`),
|
||||
author: s.actor,
|
||||
proposal: s.slot,
|
||||
content: omit(s.payload, ["authorId", "proposalId", "roundId"]),
|
||||
}));
|
||||
|
||||
const responses = subs("response").map((s) => ({
|
||||
candidate: s.slot,
|
||||
author: s.actor,
|
||||
kind: "response",
|
||||
replies: (field(s.payload, "critiqueResponses", `${board} 回应 ${s.slot}`) as JsonObject[]).map((r) => ({
|
||||
critique: r["critiqueId"] as string,
|
||||
text: r["response"] as string,
|
||||
})),
|
||||
rest: omit(s.payload, ["authorId", "candidateId", "roundId", "critiqueResponses"]),
|
||||
}));
|
||||
|
||||
const protocols = all<{ payload: string }>(
|
||||
"SELECT payload FROM facts WHERE kind = 'candidate-protocol' AND id LIKE ? ORDER BY created_seq",
|
||||
`${run}.candidate-protocol.${roundId}.%`,
|
||||
).map((f) => {
|
||||
const p = JSON.parse(f.payload) as JsonObject;
|
||||
return { candidate: field(p, "candidateId", `${board} candidate-protocol`) as string, content: omit(p, ["author", "candidateId", "baselineCodeHash"]) };
|
||||
});
|
||||
|
||||
const predictions = (name: StageName): Prediction[] =>
|
||||
subs(name).map((s) => ({
|
||||
researcher: s.actor,
|
||||
candidate: s.slot,
|
||||
range: refRange(field(s.payload, "range", `${board} ${name} ${s.actor}/${s.slot}`) as JsonObject),
|
||||
note: s.payload["rationale"] ?? null,
|
||||
}));
|
||||
const voting = stageOf.has("vote");
|
||||
|
||||
const batch = voting ? fact("vote-tally") : fact("batch-selection");
|
||||
const chosen = (field(batch, "candidates", `${board} 选批`) as JsonObject[]).filter((c) => c["selected"] === true).map((c) => c["candidateId"] as string);
|
||||
|
||||
// 退回：audit 里的 submission-rejected，taskId 是"<轮编号>.v1.<阶段>.<人>.<对象>"，只取开轮到选批这几个阶段的。
|
||||
const rejections = all<{ payload: string }>("SELECT payload FROM audit WHERE event = 'submission-rejected' ORDER BY seq").flatMap((a) => {
|
||||
const p = JSON.parse(a.payload) as JsonObject;
|
||||
const task = p["taskId"] as string;
|
||||
const m = new RegExp(`^${roundId.replace(/\./g, "\\.")}\\.v\\d+\\.([a-z-]+)\\.([^.]+)\\.(.+)$`).exec(task);
|
||||
const stage = m === null ? undefined : REF_STAGES[m[1]!];
|
||||
if (stage === undefined) return [];
|
||||
const target = m![3]!;
|
||||
return [{ stage: STAGE_NAMES[stage], author: m![2]!, target: target === m![2] ? null : target, reason: p["reason"] as string }];
|
||||
});
|
||||
|
||||
return {
|
||||
source: board,
|
||||
hiddenNames: [run],
|
||||
proposals,
|
||||
dedup: { groups, carried: dedup["pooled"] ?? null, judgments: dedup["pairs"] ?? null },
|
||||
critiques,
|
||||
responses,
|
||||
protocols,
|
||||
firstPrediction: stageOf.has("firstPrediction") ? predictions("firstPrediction") : null,
|
||||
secondPrediction: voting ? null : predictions("secondPrediction"),
|
||||
votes: voting ? subs("vote").map((s) => ({ researcher: s.actor, candidate: s.slot, ...omit(s.payload, ["voterId", "candidateId", "protocolId"]) })) : null,
|
||||
batch: { chosen, detail: batch },
|
||||
rejections,
|
||||
};
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user