iomgaa
75d7cffd27
按任务书 12 实现 src/flow：从账本现算下一步的纯函数、主循环、恢复入口、派模型任务和跑正式运行两个共用动作、冻结配置、基线、第零轮三个研究者阶段和入口分配，走到第 1 轮时停下。materials 导出 finalEntries、加 researcherId 和 StudyRecord。 审阅采纳两条：研究编号本机已用过时冻结前报错；docker run 没启动时写完结果就停下，不在本进程里无限起新编号。 eval/ 新增第零轮检验：起研究命令行、回放假模型、检验用 Jev 客户端、六问质量评估、16-real-1 回放、杀进程与两份同时跑。tsc 与 depcruise 也检查 eval/，新增 eval 只经 index.ts 用正式代码的规则。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
55 lines
2.9 KiB
TypeScript
55 lines
2.9 KiB
TypeScript
import { writeFileSync } from "node:fs";
|
||
import { join } from "node:path";
|
||
import { ofKind, readLedger } from "../lib/ledger.ts";
|
||
import { replayModels, startReplay } from "../lib/replay.ts";
|
||
import { containers, launch, newEvalDir, sleep, writeConfig } from "../lib/study.ts";
|
||
|
||
// 两份研究同时跑（任务书 12 第 5.4 节）：两个研究编号同时启动，都跑到基线结束，查两边各扣各的、容器不撞名。
|
||
// node eval/round-zero/concurrent.ts
|
||
// 研究者用一份空清单的回放：基线跑完、派写入口任务时模型一份不交，flow 按"交不出合格提交"停下，不花钱。
|
||
|
||
const out = newEvalDir("r0-concurrent");
|
||
const ids = ["r0-conc-a", "r0-conc-b"];
|
||
const researchers = ["researcher-1", "researcher-2", "researcher-3"];
|
||
const replay = await startReplay({ source: "空清单", models: Object.fromEntries([...researchers, "experimenter", "verifier"].map((m) => [m, []])) }, join(out, "pi-agent"), 0);
|
||
|
||
const studies = ids.map((id) => {
|
||
const dir = join(out, id);
|
||
const config = writeConfig("eval/round-zero/research.json", join(out, `${id}.research.json`), id, replayModels(researchers, "jev-latest"));
|
||
return { id, dir, run: launch(config, dir, replay.env) };
|
||
});
|
||
|
||
// 跑的时候每 100 毫秒看一次本机 inforsi- 开头的容器，记下见过的名字。
|
||
const seen = new Set<string>();
|
||
let running = true;
|
||
const watcher = (async () => {
|
||
while (running) {
|
||
for (const n of containers("inforsi-")) seen.add(n);
|
||
await sleep(100);
|
||
}
|
||
})();
|
||
const results = await Promise.all(studies.map((s) => s.run.done));
|
||
running = false;
|
||
await watcher;
|
||
await replay.close();
|
||
|
||
const report = studies.map((s, i) => {
|
||
const r = readLedger(s.dir);
|
||
const base = new Set(ofKind(r, "runPlanned").filter((p) => p.content.purpose === "baseline").map((p) => p.content.run));
|
||
return {
|
||
id: s.id,
|
||
outcome: results[i]!.outcome,
|
||
baseStarted: ofKind(r, "runStarted").filter((x) => base.has(x.content.run)).length,
|
||
baseResults: ofKind(r, "runResult").filter((x) => base.has(x.content.run)).map((x) => x.content),
|
||
containersSeen: [...seen].filter((n) => n.startsWith(`inforsi-${s.id}-`)),
|
||
};
|
||
});
|
||
const shared = report[0]!.containersSeen.filter((n) => report[1]!.containersSeen.includes(n));
|
||
const pass = report.every((s) => s.baseStarted === 1 && s.baseResults.length === 1 && s.baseResults[0]!.judged !== null) && shared.length === 0;
|
||
writeFileSync(join(out, "concurrent-report.json"), `${JSON.stringify({ pass, report, allContainersSeen: [...seen], shared }, null, 2)}\n`);
|
||
for (const s of report) {
|
||
console.log(`${s.id}：基线确认启动 ${s.baseStarted} 次；结果 ${JSON.stringify(s.baseResults.map((x) => x.judged))}；见过的容器 ${s.containersSeen.join("、") || "（没看到）"}；结局 ${JSON.stringify(s.outcome)}`);
|
||
}
|
||
console.log(`两边共用的容器名：${shared.join("、") || "没有"}；${pass ? "通过" : "不通过"}`);
|
||
console.log(`报告：${join(out, "concurrent-report.json")}`);
|