feat: flow 第 1 部分(骨架、研究开始、第零轮)与第零轮检验
按任务书 12 实现 src/flow:从账本现算下一步的纯函数、主循环、恢复入口、派模型任务和跑正式运行两个共用动作、冻结配置、基线、第零轮三个研究者阶段和入口分配,走到第 1 轮时停下。materials 导出 finalEntries、加 researcherId 和 StudyRecord。 审阅采纳两条:研究编号本机已用过时冻结前报错;docker run 没启动时写完结果就停下,不在本进程里无限起新编号。 eval/ 新增第零轮检验:起研究命令行、回放假模型、检验用 Jev 客户端、六问质量评估、16-real-1 回放、杀进程与两份同时跑。tsc 与 depcruise 也检查 eval/,新增 eval 只经 index.ts 用正式代码的规则。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { ofKind, readLedger } from "../lib/ledger.ts";
|
||||
import { replayModels, startReplay } from "../lib/replay.ts";
|
||||
import { containers, launch, newEvalDir, sleep, writeConfig } from "../lib/study.ts";
|
||||
|
||||
// 两份研究同时跑(任务书 12 第 5.4 节):两个研究编号同时启动,都跑到基线结束,查两边各扣各的、容器不撞名。
|
||||
// node eval/round-zero/concurrent.ts
|
||||
// 研究者用一份空清单的回放:基线跑完、派写入口任务时模型一份不交,flow 按"交不出合格提交"停下,不花钱。
|
||||
|
||||
const out = newEvalDir("r0-concurrent");
|
||||
const ids = ["r0-conc-a", "r0-conc-b"];
|
||||
const researchers = ["researcher-1", "researcher-2", "researcher-3"];
|
||||
const replay = await startReplay({ source: "空清单", models: Object.fromEntries([...researchers, "experimenter", "verifier"].map((m) => [m, []])) }, join(out, "pi-agent"), 0);
|
||||
|
||||
const studies = ids.map((id) => {
|
||||
const dir = join(out, id);
|
||||
const config = writeConfig("eval/round-zero/research.json", join(out, `${id}.research.json`), id, replayModels(researchers, "jev-latest"));
|
||||
return { id, dir, run: launch(config, dir, replay.env) };
|
||||
});
|
||||
|
||||
// 跑的时候每 100 毫秒看一次本机 inforsi- 开头的容器,记下见过的名字。
|
||||
const seen = new Set<string>();
|
||||
let running = true;
|
||||
const watcher = (async () => {
|
||||
while (running) {
|
||||
for (const n of containers("inforsi-")) seen.add(n);
|
||||
await sleep(100);
|
||||
}
|
||||
})();
|
||||
const results = await Promise.all(studies.map((s) => s.run.done));
|
||||
running = false;
|
||||
await watcher;
|
||||
await replay.close();
|
||||
|
||||
const report = studies.map((s, i) => {
|
||||
const r = readLedger(s.dir);
|
||||
const base = new Set(ofKind(r, "runPlanned").filter((p) => p.content.purpose === "baseline").map((p) => p.content.run));
|
||||
return {
|
||||
id: s.id,
|
||||
outcome: results[i]!.outcome,
|
||||
baseStarted: ofKind(r, "runStarted").filter((x) => base.has(x.content.run)).length,
|
||||
baseResults: ofKind(r, "runResult").filter((x) => base.has(x.content.run)).map((x) => x.content),
|
||||
containersSeen: [...seen].filter((n) => n.startsWith(`inforsi-${s.id}-`)),
|
||||
};
|
||||
});
|
||||
const shared = report[0]!.containersSeen.filter((n) => report[1]!.containersSeen.includes(n));
|
||||
const pass = report.every((s) => s.baseStarted === 1 && s.baseResults.length === 1 && s.baseResults[0]!.judged !== null) && shared.length === 0;
|
||||
writeFileSync(join(out, "concurrent-report.json"), `${JSON.stringify({ pass, report, allContainersSeen: [...seen], shared }, null, 2)}\n`);
|
||||
for (const s of report) {
|
||||
console.log(`${s.id}:基线确认启动 ${s.baseStarted} 次;结果 ${JSON.stringify(s.baseResults.map((x) => x.judged))};见过的容器 ${s.containersSeen.join("、") || "(没看到)"};结局 ${JSON.stringify(s.outcome)}`);
|
||||
}
|
||||
console.log(`两边共用的容器名:${shared.join("、") || "没有"};${pass ? "通过" : "不通过"}`);
|
||||
console.log(`报告:${join(out, "concurrent-report.json")}`);
|
||||
Reference in New Issue
Block a user