feat: flow 第 1 部分(骨架、研究开始、第零轮)与第零轮检验
按任务书 12 实现 src/flow:从账本现算下一步的纯函数、主循环、恢复入口、派模型任务和跑正式运行两个共用动作、冻结配置、基线、第零轮三个研究者阶段和入口分配,走到第 1 轮时停下。materials 导出 finalEntries、加 researcherId 和 StudyRecord。 审阅采纳两条:研究编号本机已用过时冻结前报错;docker run 没启动时写完结果就停下,不在本进程里无限起新编号。 eval/ 新增第零轮检验:起研究命令行、回放假模型、检验用 Jev 客户端、六问质量评估、16-real-1 回放、杀进程与两份同时跑。tsc 与 depcruise 也检查 eval/,新增 eval 只经 index.ts 用正式代码的规则。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { entriesIn, entryDecision, ofKind, readLedger, sameAssignment, stages, type Assignment } from "../lib/ledger.ts";
|
||||
import { replayModels, startReplay, type ReplayScript } from "../lib/replay.ts";
|
||||
import { launch, newEvalDir, writeConfig } from "../lib/study.ts";
|
||||
|
||||
// 把参考项目 16-real-1 的第零轮原样交给新 flow 回放一遍(任务书 12 第 5.3 节):基线照样用真 Docker 跑,
|
||||
// 三位研究者由回放假模型按当年的先后交提交。跑完核对分配是否和当年完全相同,列出被退回的提交和没交出去的提交。
|
||||
// node eval/round-zero/replay.ts <名字>
|
||||
// 研究在 <检验目录>/study;之后用 quality.ts 打分。
|
||||
|
||||
type Script = ReplayScript & { expected: Assignment[]; expectedEntries: string[] };
|
||||
|
||||
const name = process.argv[2];
|
||||
if (name === undefined) throw new Error("要写检验名字,例如 r0-replay-16");
|
||||
const script = JSON.parse(readFileSync("eval/round-zero/replay-16-real-1.json", "utf8")) as Script;
|
||||
const out = newEvalDir(name);
|
||||
const replay = await startReplay(script, join(out, "pi-agent"), 0);
|
||||
const researchers = Object.keys(script.models).sort();
|
||||
const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, replayModels(researchers, "jev-latest"));
|
||||
const studyDir = join(out, "study");
|
||||
const { outcome } = await launch(config, studyDir, replay.env).done;
|
||||
await replay.close();
|
||||
|
||||
const records = readLedger(studyDir);
|
||||
const decision = entryDecision(records);
|
||||
const got = decision === undefined ? null : decision.output;
|
||||
const entries = entriesIn(records, new Set(stages(records).filter((s) => s.round === 0).map((s) => s.stageId))).map((e) => e.id);
|
||||
// 入口全体按编号比(账本里按各人最后一版清单的先后排,当年的清单按编号排);分配按研究者、入口、志愿三样比,不看对象里键的先后。
|
||||
const sameEntries = JSON.stringify([...entries].sort()) === JSON.stringify([...script.expectedEntries].sort());
|
||||
const same = got !== null && sameAssignment(got, script.expected);
|
||||
const report = {
|
||||
outcome,
|
||||
assignment: got,
|
||||
expected: script.expected,
|
||||
sameAsReference: same,
|
||||
entries,
|
||||
expectedEntries: script.expectedEntries,
|
||||
sameEntriesAsReference: sameEntries,
|
||||
rejections: ofKind(records, "rejection").map((r) => ({ seq: r.seq, stageId: r.content.stageId, author: r.content.author, kind: r.content.kind, reason: r.content.reason })),
|
||||
delivered: replay.delivered,
|
||||
notDelivered: replay.remaining(),
|
||||
};
|
||||
writeFileSync(join(out, "replay-report.json"), `${JSON.stringify(report, null, 2)}\n`);
|
||||
console.log(`\n结局:${JSON.stringify(outcome)}`);
|
||||
console.log(`分配:${got === null ? "没有写出分配决定" : got.map((a) => `${a.researcher} → ${a.entry}(第 ${a.rank} 志愿)`).join(";")}`);
|
||||
console.log(`当年:${script.expected.map((a) => `${a.researcher} → ${a.entry}(第 ${a.rank} 志愿)`).join(";")}`);
|
||||
console.log(`分配和当年相同:${same ? "是" : "否"};入口全体和当年相同:${sameEntries ? "是" : "否"}(${entries.join("、")})`);
|
||||
console.log(`退回 ${report.rejections.length} 份${report.rejections.map((r) => `\n ${r.author} ${r.stageId} ${r.kind}:${r.reason}`).join("")}`);
|
||||
console.log(`没交出去的:${JSON.stringify(report.notDelivered)}`);
|
||||
console.log(`报告:${join(out, "replay-report.json")};研究目录:${studyDir}`);
|
||||
Reference in New Issue
Block a user