iomgaa
75d7cffd27
按任务书 12 实现 src/flow：从账本现算下一步的纯函数、主循环、恢复入口、派模型任务和跑正式运行两个共用动作、冻结配置、基线、第零轮三个研究者阶段和入口分配，走到第 1 轮时停下。materials 导出 finalEntries、加 researcherId 和 StudyRecord。 审阅采纳两条：研究编号本机已用过时冻结前报错；docker run 没启动时写完结果就停下，不在本进程里无限起新编号。 eval/ 新增第零轮检验：起研究命令行、回放假模型、检验用 Jev 客户端、六问质量评估、16-real-1 回放、杀进程与两份同时跑。tsc 与 depcruise 也检查 eval/，新增 eval 只经 index.ts 用正式代码的规则。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
52 lines
3.4 KiB
TypeScript
52 lines
3.4 KiB
TypeScript
import { readFileSync, writeFileSync } from "node:fs";
|
||
import { join } from "node:path";
|
||
import { entriesIn, entryDecision, ofKind, readLedger, sameAssignment, stages, type Assignment } from "../lib/ledger.ts";
|
||
import { replayModels, startReplay, type ReplayScript } from "../lib/replay.ts";
|
||
import { launch, newEvalDir, writeConfig } from "../lib/study.ts";
|
||
|
||
// 把参考项目 16-real-1 的第零轮原样交给新 flow 回放一遍（任务书 12 第 5.3 节）：基线照样用真 Docker 跑，
|
||
// 三位研究者由回放假模型按当年的先后交提交。跑完核对分配是否和当年完全相同，列出被退回的提交和没交出去的提交。
|
||
// node eval/round-zero/replay.ts <名字>
|
||
// 研究在 <检验目录>/study；之后用 quality.ts 打分。
|
||
|
||
type Script = ReplayScript & { expected: Assignment[]; expectedEntries: string[] };
|
||
|
||
const name = process.argv[2];
|
||
if (name === undefined) throw new Error("要写检验名字，例如 r0-replay-16");
|
||
const script = JSON.parse(readFileSync("eval/round-zero/replay-16-real-1.json", "utf8")) as Script;
|
||
const out = newEvalDir(name);
|
||
const replay = await startReplay(script, join(out, "pi-agent"), 0);
|
||
const researchers = Object.keys(script.models).sort();
|
||
const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, replayModels(researchers, "jev-latest"));
|
||
const studyDir = join(out, "study");
|
||
const { outcome } = await launch(config, studyDir, replay.env).done;
|
||
await replay.close();
|
||
|
||
const records = readLedger(studyDir);
|
||
const decision = entryDecision(records);
|
||
const got = decision === undefined ? null : decision.output;
|
||
const entries = entriesIn(records, new Set(stages(records).filter((s) => s.round === 0).map((s) => s.stageId))).map((e) => e.id);
|
||
// 入口全体按编号比（账本里按各人最后一版清单的先后排，当年的清单按编号排）；分配按研究者、入口、志愿三样比，不看对象里键的先后。
|
||
const sameEntries = JSON.stringify([...entries].sort()) === JSON.stringify([...script.expectedEntries].sort());
|
||
const same = got !== null && sameAssignment(got, script.expected);
|
||
const report = {
|
||
outcome,
|
||
assignment: got,
|
||
expected: script.expected,
|
||
sameAsReference: same,
|
||
entries,
|
||
expectedEntries: script.expectedEntries,
|
||
sameEntriesAsReference: sameEntries,
|
||
rejections: ofKind(records, "rejection").map((r) => ({ seq: r.seq, stageId: r.content.stageId, author: r.content.author, kind: r.content.kind, reason: r.content.reason })),
|
||
delivered: replay.delivered,
|
||
notDelivered: replay.remaining(),
|
||
};
|
||
writeFileSync(join(out, "replay-report.json"), `${JSON.stringify(report, null, 2)}\n`);
|
||
console.log(`\n结局：${JSON.stringify(outcome)}`);
|
||
console.log(`分配：${got === null ? "没有写出分配决定" : got.map((a) => `${a.researcher} → ${a.entry}（第 ${a.rank} 志愿）`).join("；")}`);
|
||
console.log(`当年：${script.expected.map((a) => `${a.researcher} → ${a.entry}（第 ${a.rank} 志愿）`).join("；")}`);
|
||
console.log(`分配和当年相同：${same ? "是" : "否"}；入口全体和当年相同：${sameEntries ? "是" : "否"}（${entries.join("、")}）`);
|
||
console.log(`退回 ${report.rejections.length} 份${report.rejections.map((r) => `\n ${r.author} ${r.stageId} ${r.kind}：${r.reason}`).join("")}`);
|
||
console.log(`没交出去的：${JSON.stringify(report.notDelivered)}`);
|
||
console.log(`报告：${join(out, "replay-report.json")}；研究目录：${studyDir}`);
|