224 lines
9.7 KiB
TypeScript
224 lines
9.7 KiB
TypeScript
import { mkdirSync, writeFileSync } from "node:fs";
|
||
import { join, resolve } from "node:path";
|
||
import {
|
||
Board,
|
||
diagnosisId,
|
||
type TaskMaterial,
|
||
} from "../../src/materials/index.ts";
|
||
import { diagnosisOutcome } from "../../src/submissions/index.ts";
|
||
import { checkoutVersion, compareVersions } from "../../src/workspace/index.ts";
|
||
import { jsonFile } from "../lib/forward.ts";
|
||
import { finalSubmissions, ofKind, readLedger } from "../lib/ledger.ts";
|
||
|
||
// 任务书 18 第 3.2 节第 1、3、4 样的审阅材料。第 1 样的目录（blind/）只放诊断结算之前就存在的东西，
|
||
// 不放诊断结果；第 3 样（impl/）放诊断实现与核验的往来；第 4 样（direction/）放本轮结果和本轮提案、候选，
|
||
// 供子 Agent 标注"沿不沿诊断指出的方向"。只导出事实和原文，不替审阅者下结论。
|
||
// node eval/diagnosis/export.ts <研究输出目录> <全新绝对审阅目录>
|
||
const [dir, output, ...extra] = process.argv.slice(2);
|
||
if (!dir || !output || extra.length || output !== resolve(output)) {
|
||
throw new Error(
|
||
"用法：node eval/diagnosis/export.ts <研究输出目录> <全新绝对审阅目录>",
|
||
);
|
||
}
|
||
mkdirSync(output);
|
||
const study = join(dir, "study");
|
||
const records = readLedger(study);
|
||
const board = new Board(records);
|
||
const repo = join(study, "repo.git");
|
||
const finals = finalSubmissions(records);
|
||
const roundOf = (stageId: string) => board.stage(stageId).round;
|
||
const json = (v: unknown) => `${JSON.stringify(v, null, 2)}\n`;
|
||
const checkout = (version: string, path: string) => {
|
||
mkdirSync(path, { recursive: true });
|
||
checkoutVersion(repo, version, path);
|
||
};
|
||
const text = (path: string, body: string) => writeFileSync(path, body);
|
||
|
||
const rounds = [
|
||
...new Set(
|
||
ofKind(records, "decision")
|
||
.filter((d) => d.content.decision === "selectDiagnosis")
|
||
.map((d) => d.content.round),
|
||
),
|
||
].sort((a, b) => a - b);
|
||
|
||
const index = [];
|
||
for (const round of rounds) {
|
||
const roundDir = join(output, `round-${round}`);
|
||
const id = diagnosisId(round);
|
||
const baseline = board.code(board.baselineOf(round));
|
||
const baselineScore = board.baselineScore(board.baselineOf(round));
|
||
const mine = (kind: string) =>
|
||
finals.filter(
|
||
(s) => s.content.kind === kind && roundOf(s.content.stageId) === round,
|
||
);
|
||
|
||
// ---- 第 1 样（看不到诊断结果）----
|
||
const blind = join(roundDir, "blind");
|
||
mkdirSync(blind, { recursive: true });
|
||
checkout(baseline.version, join(blind, "baseline-code"));
|
||
text(
|
||
join(blind, "baseline-score.txt"),
|
||
`本轮当前基线 ${board.baselineOf(round)} 的最终指标：${baselineScore}\n`,
|
||
);
|
||
text(
|
||
join(blind, "diagnosis-proposals.txt"),
|
||
mine("diagnosisProposal")
|
||
.map(
|
||
(s) =>
|
||
`${"=".repeat(20)} ${s.content.author}（记录 ${s.seq}）\n${json(s.content.content)}`,
|
||
)
|
||
.join("\n"),
|
||
);
|
||
text(
|
||
join(blind, "diagnosis-predictions.txt"),
|
||
mine("diagnosisPrediction")
|
||
.map(
|
||
(s) =>
|
||
`${"=".repeat(20)} ${s.content.author}（记录 ${s.seq}）\n${json(s.content.content)}`,
|
||
)
|
||
.join("\n"),
|
||
);
|
||
jsonFile(
|
||
join(blind, "merged-experiments.json"),
|
||
board.diagnosisExperiments(round),
|
||
);
|
||
// 研究者写诊断解释与实验时实际读到的材料（含信念本和上一轮收尾，不含本轮诊断结果）。
|
||
for (const c of ofKind(records, "callStarted").filter((x) => {
|
||
const m = x.content.materials as TaskMaterial;
|
||
return m.kind === "diagnosisProposal" && m.round === round;
|
||
})) {
|
||
text(
|
||
join(blind, `researcher-material-${c.content.actor}.txt`),
|
||
(c.content.materials as TaskMaterial).firstMessage,
|
||
);
|
||
}
|
||
text(
|
||
join(blind, "review.txt"),
|
||
`任务书18第3.2节第1样，只读本目录，中文直接回复，不写文件，看不到诊断结果。\n` +
|
||
`读 diagnosis-proposals.txt（每位研究者的诊断解释与实验）、diagnosis-predictions.txt（三个数）、baseline-code/ 与 baseline-score.txt、researcher-material-*.txt（研究者当时读到的材料，含以前轮次的结论）。回答：\n` +
|
||
`- 每条解释说出了具体的瓶颈吗？写了什么实测会推翻它吗？\n- 同一轮几位研究者的解释有没有真正不同？\n- 每个实验是不是只动一处？是不是真能把几种解释分开？\n- 有没有实验去测以前否掉过的路线的理由？\n`,
|
||
);
|
||
|
||
// ---- 选实验与结算（第 2、5 样的原始数据，供脚本和人读）----
|
||
jsonFile(
|
||
join(roundDir, "select-decision.json"),
|
||
ofKind(records, "decision").find(
|
||
(d) => d.content.decision === "selectDiagnosis" && d.content.round === round,
|
||
) ?? null,
|
||
);
|
||
jsonFile(
|
||
join(roundDir, "settlement.json"),
|
||
ofKind(records, "diagnosisSettlement").find((s) => s.content.round === round) ?? null,
|
||
);
|
||
|
||
// ---- 第 3 样（诊断实现与核验）----
|
||
const impl = join(roundDir, "impl");
|
||
mkdirSync(impl, { recursive: true });
|
||
const chosen = board.decision("selectDiagnosis", round)!.output;
|
||
if (chosen.diagnose) {
|
||
checkout(baseline.version, join(impl, "baseline"));
|
||
const attempts = finals
|
||
.filter(
|
||
(s) =>
|
||
["diagnosisImplementation", "diagnosisRepair"].includes(s.content.kind) &&
|
||
s.content.target === id,
|
||
)
|
||
.map((s) => ({
|
||
submission: s,
|
||
code: ofKind(records, "code").find((c) => c.seq === s.seq + 1) ?? null,
|
||
}));
|
||
jsonFile(join(impl, "implementations.json"), attempts);
|
||
for (const a of attempts) {
|
||
if (a.code === null) continue;
|
||
checkout(a.code.content.version, join(impl, `program-${a.code.seq}`));
|
||
jsonFile(
|
||
join(impl, `diff-${a.code.seq}.json`),
|
||
compareVersions(repo, baseline.version, a.code.content.version),
|
||
);
|
||
}
|
||
const verifications = finals
|
||
.filter(
|
||
(s) => s.content.kind === "diagnosisVerification" && s.content.target === id,
|
||
)
|
||
.map((s) => ({
|
||
submission: s,
|
||
outcome: diagnosisOutcome(
|
||
s.content.content as Parameters<typeof diagnosisOutcome>[0],
|
||
),
|
||
}));
|
||
jsonFile(join(impl, "verifications.json"), verifications);
|
||
jsonFile(
|
||
join(impl, "rejections.json"),
|
||
ofKind(records, "rejection").filter((r) => r.content.target === id),
|
||
);
|
||
jsonFile(
|
||
join(impl, "selected-experiments.json"),
|
||
board.diagnosisExperiments(round).filter((e) => chosen.chosen.includes(e.id)),
|
||
);
|
||
jsonFile(
|
||
join(impl, "calls.json"),
|
||
ofKind(records, "callStarted")
|
||
.filter((c) => {
|
||
const m = c.content.materials as TaskMaterial;
|
||
return (
|
||
m.object === id &&
|
||
["diagnosisImplementation", "diagnosisRepair", "diagnosisVerification"].includes(m.kind)
|
||
);
|
||
})
|
||
.map((c) => ({
|
||
record: c,
|
||
sessions: join(study, "calls", c.content.call, "sessions"),
|
||
})),
|
||
);
|
||
text(
|
||
join(impl, "review.txt"),
|
||
`任务书18第3.2节第3样，只读本目录，中文直接回复，不写文件。\n` +
|
||
`读 selected-experiments.json（实验原文）、baseline/（当前基线代码）、program-<记录号>/ 与 diff-<记录号>.json（每次交上来的诊断程序与基线的差）、implementations.json（位置表、concerns、说明）、verifications.json（核验员逐实验结论）、rejections.json。回答：\n` +
|
||
`- 每个实验是不是只动了写的那一处，有没有偷偷改进别的地方；\n- 核验员该判不通过却通过的、不该判却判了的；有修复的，修复是否只改被指出的地方。\n`,
|
||
);
|
||
}
|
||
|
||
// ---- 第 4 样（沿不沿诊断指出的方向）----
|
||
const direction = join(roundDir, "direction");
|
||
mkdirSync(direction, { recursive: true });
|
||
const settlement = board.diagnosisSettlement(round);
|
||
jsonFile(join(direction, "diagnosis-direction.json"), {
|
||
experiments: board.diagnosisExperiments(round),
|
||
direction: settlement?.output.direction ?? null,
|
||
outcomes: settlement?.output.experiments ?? null,
|
||
judged: settlement?.judged.experiments ?? null,
|
||
});
|
||
const proposals = mine("proposal");
|
||
text(
|
||
join(direction, "proposals.txt"),
|
||
proposals
|
||
.map(
|
||
(s) =>
|
||
`${"=".repeat(20)} ${s.content.item}（${s.content.author}）\n${json(s.content.content)}`,
|
||
)
|
||
.join("\n"),
|
||
);
|
||
const candidates = ofKind(records, "protocol").filter(
|
||
(p) => (p.content.protocol as { round: number }).round === round,
|
||
);
|
||
jsonFile(
|
||
join(direction, "candidates.json"),
|
||
candidates.map((p) => ({
|
||
candidate: p.content.candidate,
|
||
proposals: board.proposalsOf(p.content.candidate),
|
||
protocol: p.content.protocol,
|
||
chosen: board.decision("batch", round)?.output.chosen.includes(p.content.candidate) ?? null,
|
||
})),
|
||
);
|
||
text(
|
||
join(direction, "review.txt"),
|
||
`任务书18第3.2节第4样的标注，只读本目录，中文直接回复，最后给一段 JSON：{"proposals":{"<提案编号>":"along"|"not"},"candidates":{"<候选编号>":"along"|"not"}}。\n` +
|
||
`读 diagnosis-direction.json（本轮诊断推出的方向：direction.experiment 及它的实验原文、实测收益；found 为 false 时没有方向，全部标 not）、proposals.txt（本轮每位研究者交的原提案）、candidates.json（合并后的候选和冻结协议）。\n` +
|
||
`判断每条提案、每个候选是否"沿着诊断指出的方向"：它动的地方或要换的做法，和诊断里收益最大的那个实验所动的地方属于同一处或明确把它当起点；只是在别处小步调参或走别的路线，标 not。同时逐条写一句理由，并写明提案正文有没有引用诊断结果。\n`,
|
||
);
|
||
index.push({ round, dir: roundDir, diagnosed: chosen.diagnose });
|
||
}
|
||
jsonFile(join(output, "index.json"), { study: dir, rounds: index });
|
||
console.log(`审阅材料：${output}`);
|