eval: 第 1 轮检验脚本(跑、杀进程、导出去来历、盲评编号、Jev 三问、数字与过程)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { launch, newEvalDir, waitForLedger, writeConfig } from "../lib/study.ts";
|
||||
import { watchTimeline } from "../lib/timeline.ts";
|
||||
|
||||
// 用真模型、真 Docker、真 Jev 从头跑一份研究,直到它自己停下(任务书 14 第 3.1 节;预期停在第 1 轮选批之后,
|
||||
// 原因"实现还没写")。配置照 eval/round-zero/research.json,研究编号换成检验名字。要花钱。
|
||||
// node eval/round-one/run.ts <名字>
|
||||
// 产物:<检验目录>/study(研究目录)、timeline.jsonl(每条记录第一次被看到的时刻,stats 用来算墙钟)、outcome.json(结局)。
|
||||
// 中途按 Ctrl-C 会把停下信号传给研究进程;接着跑用 node eval/lib/cli.ts --resume --dir <检验目录>/study。
|
||||
|
||||
const name = process.argv[2];
|
||||
if (name === undefined) throw new Error("要写检验名字,例如 r1-real-1(只许小写字母、数字和 ._-,它也是研究编号)");
|
||||
const out = newEvalDir(name);
|
||||
const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, null);
|
||||
const studyDir = join(out, "study");
|
||||
const study = launch(config, studyDir, {});
|
||||
process.on("SIGINT", () => study.child.kill("SIGINT"));
|
||||
await waitForLedger(studyDir, study.child);
|
||||
await watchTimeline(studyDir, study.child, join(out, "timeline.jsonl"));
|
||||
const { code, signal, outcome } = await study.done;
|
||||
writeFileSync(join(out, "outcome.json"), `${JSON.stringify({ launches: [{ code, signal, outcome }] }, null, 2)}\n`);
|
||||
console.log(`\n结局:${JSON.stringify(outcome)}\n研究目录:${studyDir}`);
|
||||
Reference in New Issue
Block a user