feat: flow 第 1 部分（骨架、研究开始、第零轮）与第零轮检验
按任务书 12 实现 src/flow：从账本现算下一步的纯函数、主循环、恢复入口、派模型任务和跑正式运行两个共用动作、冻结配置、基线、第零轮三个研究者阶段和入口分配，走到第 1 轮时停下。materials 导出 finalEntries、加 researcherId 和 StudyRecord。 审阅采纳两条：研究编号本机已用过时冻结前报错；docker run 没启动时写完结果就停下，不在本进程里无限起新编号。 eval/ 新增第零轮检验：起研究命令行、回放假模型、检验用 Jev 客户端、六问质量评估、16-real-1 回放、杀进程与两份同时跑。tsc 与 depcruise 也检查 eval/，新增 eval 只经 index.ts 用正式代码的规则。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { ofKind, readLedger } from "../lib/ledger.ts";
|
||||
import { replayModels, startReplay } from "../lib/replay.ts";
|
||||
import { containers, launch, newEvalDir, sleep, writeConfig } from "../lib/study.ts";
|
||||
|
||||
// 两份研究同时跑（任务书 12 第 5.4 节）：两个研究编号同时启动，都跑到基线结束，查两边各扣各的、容器不撞名。
|
||||
// node eval/round-zero/concurrent.ts
|
||||
// 研究者用一份空清单的回放：基线跑完、派写入口任务时模型一份不交，flow 按"交不出合格提交"停下，不花钱。
|
||||
|
||||
const out = newEvalDir("r0-concurrent");
|
||||
const ids = ["r0-conc-a", "r0-conc-b"];
|
||||
const researchers = ["researcher-1", "researcher-2", "researcher-3"];
|
||||
const replay = await startReplay({ source: "空清单", models: Object.fromEntries([...researchers, "experimenter", "verifier"].map((m) => [m, []])) }, join(out, "pi-agent"), 0);
|
||||
|
||||
const studies = ids.map((id) => {
|
||||
const dir = join(out, id);
|
||||
const config = writeConfig("eval/round-zero/research.json", join(out, `${id}.research.json`), id, replayModels(researchers, "jev-latest"));
|
||||
return { id, dir, run: launch(config, dir, replay.env) };
|
||||
});
|
||||
|
||||
// 跑的时候每 100 毫秒看一次本机 inforsi- 开头的容器，记下见过的名字。
|
||||
const seen = new Set<string>();
|
||||
let running = true;
|
||||
const watcher = (async () => {
|
||||
while (running) {
|
||||
for (const n of containers("inforsi-")) seen.add(n);
|
||||
await sleep(100);
|
||||
}
|
||||
})();
|
||||
const results = await Promise.all(studies.map((s) => s.run.done));
|
||||
running = false;
|
||||
await watcher;
|
||||
await replay.close();
|
||||
|
||||
const report = studies.map((s, i) => {
|
||||
const r = readLedger(s.dir);
|
||||
const base = new Set(ofKind(r, "runPlanned").filter((p) => p.content.purpose === "baseline").map((p) => p.content.run));
|
||||
return {
|
||||
id: s.id,
|
||||
outcome: results[i]!.outcome,
|
||||
baseStarted: ofKind(r, "runStarted").filter((x) => base.has(x.content.run)).length,
|
||||
baseResults: ofKind(r, "runResult").filter((x) => base.has(x.content.run)).map((x) => x.content),
|
||||
containersSeen: [...seen].filter((n) => n.startsWith(`inforsi-${s.id}-`)),
|
||||
};
|
||||
});
|
||||
const shared = report[0]!.containersSeen.filter((n) => report[1]!.containersSeen.includes(n));
|
||||
const pass = report.every((s) => s.baseStarted === 1 && s.baseResults.length === 1 && s.baseResults[0]!.judged !== null) && shared.length === 0;
|
||||
writeFileSync(join(out, "concurrent-report.json"), `${JSON.stringify({ pass, report, allContainersSeen: [...seen], shared }, null, 2)}\n`);
|
||||
for (const s of report) {
|
||||
console.log(`${s.id}：基线确认启动 ${s.baseStarted} 次；结果 ${JSON.stringify(s.baseResults.map((x) => x.judged))}；见过的容器 ${s.containersSeen.join("、") || "（没看到）"}；结局 ${JSON.stringify(s.outcome)}`);
|
||||
}
|
||||
console.log(`两边共用的容器名：${shared.join("、") || "没有"}；${pass ? "通过" : "不通过"}`);
|
||||
console.log(`报告：${join(out, "concurrent-report.json")}`);
|
||||
@@ -0,0 +1,71 @@
|
||||
import { copyFileSync, mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { entryId } from "../../src/materials/index.ts";
|
||||
import type { ReplayItem, ReplayScript } from "../lib/replay.ts";
|
||||
|
||||
// 从参考项目 16-real-1 的账本取第零轮的提交，按任务书 12 第 5.3 节转成新项目的格式，写成回放脚本。
|
||||
// 原账本只读：先复制到临时目录，再以只读方式打开复制件，原目录里不写任何东西。
|
||||
// node eval/round-zero/extract-16.ts
|
||||
|
||||
const SOURCE = "/home/iomgaa/rb-acceptance/16-real-1/board.sqlite";
|
||||
export const REPLAY_FILE = "eval/round-zero/replay-16-real-1.json";
|
||||
const PHASE_STAGE: Record<string, string> = { "entry-draft": "entryDraft", "entry-discuss": "discussion", "entry-claim": "claim" };
|
||||
|
||||
const copy = join(mkdtempSync(join(tmpdir(), "inforsi-rb16-")), "board.sqlite");
|
||||
copyFileSync(SOURCE, copy);
|
||||
const db = new DatabaseSync(copy, { readOnly: true });
|
||||
|
||||
type OldEntry = { name: string; startFrom: string; whyWorth: string };
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT st.phase, st.id AS stage, s.actor, s.version, s.created_seq AS seq, s.payload
|
||||
FROM submissions s JOIN stages st ON st.id = s.stage_id
|
||||
WHERE st.phase IN ('entry-draft', 'entry-discuss', 'entry-claim') ORDER BY s.created_seq`,
|
||||
)
|
||||
.all() as Array<{ phase: string; stage: string; actor: string; version: number; seq: number; payload: string }>;
|
||||
const fact = db.prepare("SELECT payload FROM facts WHERE kind = 'entry-assignment'").all() as Array<{ payload: string }>;
|
||||
db.close();
|
||||
if (fact.length !== 1) throw new Error(`entry-assignment 应有 1 条，实有 ${fact.length} 条`);
|
||||
|
||||
/** 当年入口 {name, startFrom, whyWorth} → 新项目 {name, body}。 */
|
||||
const entry = (e: OldEntry) => ({ name: e.name, body: `从哪里开始看：${e.startFrom}\n\n为什么值得走：${e.whyWorth}` });
|
||||
/** 当年入口编号 researcher-1-e4 → 新项目 e-researcher-1-4。 */
|
||||
function newId(old: string): string {
|
||||
const m = /^(researcher-\d+)-e(\d+)$/.exec(old);
|
||||
if (m === null) throw new Error(`入口编号 ${old} 不是 researcher-<i>-e<n> 的写法`);
|
||||
return entryId(m[1]!, Number(m[2]));
|
||||
}
|
||||
|
||||
const models: Record<string, ReplayItem[]> = {};
|
||||
for (const r of rows) {
|
||||
const p = JSON.parse(r.payload) as { entries?: OldEntry[]; body?: string; choices?: string[] };
|
||||
const payload =
|
||||
r.phase === "entry-draft"
|
||||
? { entries: p.entries!.map(entry) }
|
||||
: r.phase === "entry-discuss"
|
||||
? { body: p.body!, ...(p.entries === undefined ? {} : { entries: p.entries.map(entry) }) }
|
||||
: { choices: p.choices!.map(newId) };
|
||||
(models[r.actor] ??= []).push({ from: `${r.stage} ${PHASE_STAGE[r.phase]} seq ${r.seq} v${r.version}`, arguments: { payload } });
|
||||
}
|
||||
|
||||
const old = JSON.parse(fact[0]!.payload) as { assignments: Array<{ researcher: string; entryId: string; rank: number | null }>; entries: Array<{ entryId: string; author: string }> };
|
||||
const expected = old.assignments.map((a) => ({ researcher: a.researcher, entry: newId(a.entryId), rank: a.rank }));
|
||||
const script: ReplayScript & { expected: typeof expected; expectedEntries: string[] } = {
|
||||
source: `${SOURCE}（第零轮 submissions，created_seq ${rows[0]!.seq}–${rows.at(-1)!.seq}）`,
|
||||
models,
|
||||
expected,
|
||||
expectedEntries: old.entries.map((e) => newId(e.entryId)),
|
||||
};
|
||||
writeFileSync(REPLAY_FILE, `${JSON.stringify(script, null, 2)}\n`);
|
||||
|
||||
// 核对：三阶段各几份、作者、版本，以及当年的分配。
|
||||
console.log(`共 ${rows.length} 份提交，序号 ${rows[0]!.seq}–${rows.at(-1)!.seq}`);
|
||||
for (const phase of Object.keys(PHASE_STAGE)) {
|
||||
const these = rows.filter((r) => r.phase === phase);
|
||||
console.log(`${phase}：${these.length} 份；${these.map((r) => `${r.actor}@${r.seq}(v${r.version}${JSON.parse(r.payload).entries ? `，${JSON.parse(r.payload).entries.length} 个入口` : ""})`).join("、")}`);
|
||||
}
|
||||
console.log(`当年分配：${expected.map((a) => `${a.researcher} → ${a.entry}（第 ${a.rank} 志愿）`).join("；")}`);
|
||||
console.log(`当年最终入口：${script.expectedEntries.join("、")}`);
|
||||
console.log(`已写 ${REPLAY_FILE}`);
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"note": "任务书 12 第 5.1 节的关键想法清单和死路清单，出处照抄。英文说明给 Jev 看。",
|
||||
"key": [
|
||||
{
|
||||
"id": "K1",
|
||||
"name": "联合优化",
|
||||
"description": "Optimize the centers and radii of all 26 circles together (78 numbers) with a constrained continuous optimizer such as scipy SLSQP, with constraints that every circle stays inside the square and no two circles overlap.",
|
||||
"source": "最大的一跳：OpenEvolve 2.377→2.634；AdaEvolve 2.541→2.610；参考项目 12-real-1、12-real-2 第 1 轮采纳；16-real-1 关掉它少 0.132"
|
||||
},
|
||||
{
|
||||
"id": "K2",
|
||||
"name": "扰动重优化",
|
||||
"description": "Starting from the current best solution, perturb the circle centers by a moderate amount (about 0.01 to 0.06), run the joint optimization again, and keep the result only if it is better; this includes annealing-style acceptance and larger perturbations after the search stagnates.",
|
||||
"source": "AlphaEvolve、ShinkaEvolve、Discovery Loop；参考项目 12-real-2 第 2 轮靠它从 2.6155 到 2.635983，15-tight-2 也到了 2.635983"
|
||||
},
|
||||
{
|
||||
"id": "K3",
|
||||
"name": "结构化起点与多起点",
|
||||
"description": "Start from structured layouts such as hexagonal or staggered rows, or from many random starting layouts, run the optimization from each start, and keep the best result.",
|
||||
"source": "ThetaEvolve、TTT-Discover 只靠\"错排起点 + 一次 SLSQP\"就到 2.635983；CodeEvolve 10 次重启"
|
||||
},
|
||||
{
|
||||
"id": "K4",
|
||||
"name": "固定圆心求最优半径",
|
||||
"description": "Keep the circle centers fixed and solve a linear program for the radii that maximizes their sum.",
|
||||
"source": "ShinkaEvolve 容差 0 那次的第一步、Discovery Loop 的种子；16-real-1 消融撑 28%"
|
||||
},
|
||||
{
|
||||
"id": "K5",
|
||||
"name": "严格合法收尾",
|
||||
"description": "As a final step, shrink all radii by a tiny amount or refine the solution so that it is strictly valid under zero tolerance.",
|
||||
"source": "ShinkaEvolve、ThetaEvolve 严格版；本任务容差为 0（TASK.md）"
|
||||
}
|
||||
],
|
||||
"deadEnds": [
|
||||
{
|
||||
"id": "D1",
|
||||
"name": "只做几何构造",
|
||||
"description": "Place the circle centers by a hand-designed rule (hexagonal grid, square grid, concentric rings) and only tune the arrangement and its parameters, without using continuous optimization to move the centers.",
|
||||
"source": "各家卡在 2.2–2.54；等半径六角的上限是 2.438966；OpenEvolve 第一阶段提示词要求只写构造，卡在 2.377；参考项目 9a、9b-real-2 卡在 2.517、2.541"
|
||||
},
|
||||
{
|
||||
"id": "D2",
|
||||
"name": "只优化半径",
|
||||
"description": "Keep the circle centers fixed and only change how the radii are allocated, as the whole route rather than as a helper step.",
|
||||
"source": "上限受布局限制，2.5414 是反复出现的平台"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { LedgerRecord } from "../../src/ledger/index.ts";
|
||||
import { entryDecision, finalSubmissions, ofKind, readLedger, sameAssignment, stages, type Assignment } from "../lib/ledger.ts";
|
||||
import { replayModels, startReplay, type ReplayScript } from "../lib/replay.ts";
|
||||
import { containers, launch, newEvalDir, sleep, waitForLedger, writeConfig } from "../lib/study.ts";
|
||||
|
||||
// 杀进程检验（任务书 12 第 5.4 节）：在一处 SIGKILL 控制进程，再启动让它跑完第零轮，逐条查五件事。
|
||||
// node eval/round-zero/kill.ts baseline|drafting|stageFull
|
||||
// 模型一侧用 16-real-1 的回放（不花钱、结果确定，分配应和当年相同）；基线照样用真 Docker。
|
||||
// 回放每次回复前等 DELAY_MS，留出"研究者正在写""最后一份刚收下、阶段还没关"的窗口。
|
||||
|
||||
const DELAY_MS = 3000;
|
||||
type Point = "baseline" | "drafting" | "stageFull";
|
||||
const point = process.argv[2] as Point;
|
||||
|
||||
/** 三处杀点：看到账本里出现这种状态就杀。 */
|
||||
const HITS: Record<Point, (r: LedgerRecord[]) => boolean> = {
|
||||
// 基线容器正在跑：基线运行确认启动了、还没有结果。
|
||||
baseline: (r) => {
|
||||
const base = new Set(ofKind(r, "runPlanned").filter((p) => p.content.purpose === "baseline").map((p) => p.content.run));
|
||||
const done = new Set(ofKind(r, "runResult").map((x) => x.content.run));
|
||||
return ofKind(r, "runStarted").some((s) => base.has(s.content.run) && !done.has(s.content.run));
|
||||
},
|
||||
// 研究者 2 正在写入口：他在写入口阶段的调用开始了、还没结束。
|
||||
drafting: (r) => {
|
||||
const draft = new Set(stages(r).filter((s) => s.stage === "entryDraft").map((s) => s.stageId));
|
||||
return openCalls(r).some((c) => c.content.actor === "researcher-2" && draft.has(c.content.stageId ?? ""));
|
||||
},
|
||||
// 某阶段最后一份提交刚收下、阶段还没关：名单上的人都交了，还没有关闭记录。
|
||||
stageFull: (r) => stages(r).some((s) => !s.closed && s.roster.every((a) => submitted(r, s.stageId, a))),
|
||||
};
|
||||
if (!(point in HITS)) throw new Error(`杀点只能是 ${Object.keys(HITS).join("、")} 之一`);
|
||||
|
||||
function openCalls(r: LedgerRecord[]): Array<LedgerRecord<"callStarted">> {
|
||||
const ended = new Set(ofKind(r, "callEnded").map((e) => e.content.call));
|
||||
return ofKind(r, "callStarted").filter((c) => !ended.has(c.content.call));
|
||||
}
|
||||
const submitted = (r: LedgerRecord[], stageId: string, author: string) =>
|
||||
ofKind(r, "submission").some((s) => s.content.stageId === stageId && s.content.author === author);
|
||||
|
||||
const script = JSON.parse(readFileSync("eval/round-zero/replay-16-real-1.json", "utf8")) as ReplayScript & { expected: Assignment[] };
|
||||
// 研究编号也是镜像标签的一部分，Docker 镜像名只许小写，所以杀点名转成小写。
|
||||
const name = `r0-kill-${point.toLowerCase()}`;
|
||||
const out = newEvalDir(name);
|
||||
const replay = await startReplay(script, join(out, "pi-agent"), DELAY_MS);
|
||||
const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, replayModels(Object.keys(script.models).sort(), "jev-latest"));
|
||||
const studyDir = join(out, "study");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 第一次启动，打中就杀
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const first = launch(config, studyDir, replay.env);
|
||||
await waitForLedger(studyDir, first.child);
|
||||
let triggered: LedgerRecord | undefined;
|
||||
while (first.child.exitCode === null) {
|
||||
const r = readLedger(studyDir);
|
||||
if (HITS[point](r)) {
|
||||
first.child.kill("SIGKILL");
|
||||
triggered = r.at(-1);
|
||||
break;
|
||||
}
|
||||
await sleep(20);
|
||||
}
|
||||
await first.done;
|
||||
if (triggered === undefined) throw new Error(`没打中 ${point}：研究进程自己结束了，账本最后一条是 ${JSON.stringify(readLedger(studyDir).at(-1))}`);
|
||||
// 进程已死，此刻的账本就是被杀时的样子。
|
||||
const atKill = readLedger(studyDir);
|
||||
const killSeq = atKill.at(-1)!.seq;
|
||||
const killedCalls = openCalls(atKill);
|
||||
const openStages = stages(atKill).filter((s) => !s.closed);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 再启动，跑完第零轮
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const second = await launch(null, studyDir, replay.env).done;
|
||||
await replay.close();
|
||||
const r = readLedger(studyDir);
|
||||
const studyId = name;
|
||||
|
||||
const baseRuns = new Set(ofKind(r, "runPlanned").filter((p) => p.content.purpose === "baseline").map((p) => p.content.run));
|
||||
const baseStarted = ofKind(r, "runStarted").filter((s) => baseRuns.has(s.content.run));
|
||||
const leftover = containers(`inforsi-${studyId}-`);
|
||||
const callEnds = killedCalls.map((c) => ({ call: c.content.call, end: ofKind(r, "callEnded").find((e) => e.content.call === c.content.call)?.content ?? null }));
|
||||
const redispatch = openStages.flatMap((s) =>
|
||||
s.roster.map((actor) => {
|
||||
const before = submitted(atKill, s.stageId, actor);
|
||||
const again = ofKind(r, "callStarted").some((c) => c.seq > killSeq && c.content.stageId === s.stageId && c.content.actor === actor);
|
||||
return { stageId: s.stageId, actor, submittedBeforeKill: before, dispatchedAfterRestart: again, ok: before !== again };
|
||||
}),
|
||||
);
|
||||
const finals = finalSubmissions(r);
|
||||
const perStage = stages(r).map((s) => ({
|
||||
stageId: s.stageId,
|
||||
closed: s.closed,
|
||||
finalPerPerson: Object.fromEntries(s.roster.map((a) => [a, finals.filter((f) => f.content.stageId === s.stageId && f.content.author === a).length])),
|
||||
}));
|
||||
const decision = entryDecision(r);
|
||||
const assignment = decision === undefined ? null : decision.output;
|
||||
|
||||
const checks = {
|
||||
"1 基线只扣了 1 次，Docker 里没有留下基线容器": { pass: baseStarted.length === 1 && leftover.length === 0, baseRunsPlanned: [...baseRuns], baseStarted: baseStarted.length, leftoverContainers: leftover },
|
||||
"2 被杀那次调用有结束记录，结局出错、原因进程中断": {
|
||||
pass: callEnds.every((c) => c.end !== null && c.end.status === "failed" && c.end.reason.includes("进程中断")),
|
||||
calls: callEnds,
|
||||
},
|
||||
"3 没交上的人被重新派了，交过的没被重派": { pass: redispatch.every((x) => x.ok), detail: redispatch },
|
||||
"4 每个阶段都关上了，每人每阶段只有一份最终收下的提交": {
|
||||
pass: perStage.every((s) => s.closed && Object.values(s.finalPerPerson).every((n) => n === 1)),
|
||||
detail: perStage,
|
||||
},
|
||||
"5 分配照常写出": { pass: assignment !== null, assignment, sameAsReference: assignment !== null && sameAssignment(assignment, script.expected) },
|
||||
};
|
||||
const report = { point, triggeredAt: triggered, lastRecordAtKill: atKill.at(-1), outcomeAfterRestart: second.outcome, checks };
|
||||
writeFileSync(join(out, "kill-report.json"), `${JSON.stringify(report, null, 2)}\n`);
|
||||
console.log(`\n杀点 ${point}：触发时账本最后一条 #${triggered.seq} ${triggered.kind}；被杀时最后一条 #${killSeq} ${atKill.at(-1)!.kind} ${JSON.stringify(atKill.at(-1)!.content).slice(0, 200)}`);
|
||||
console.log(`再启动后的结局：${JSON.stringify(second.outcome)}`);
|
||||
for (const [k, v] of Object.entries(checks)) console.log(`${v.pass ? "通过" : "不通过"} ${k}`);
|
||||
console.log(`报告：${join(out, "kill-report.json")}`);
|
||||
@@ -0,0 +1,148 @@
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { ask, THRESHOLD_NOTE, type Answer, type Noul } from "../lib/jev.ts";
|
||||
import { entriesIn, entryDecision, readLedger, stages, type Entry } from "../lib/ledger.ts";
|
||||
import { ENTRY_ONLY, LEADS_TO_IDEA, SAME_KIND, STOPS_AT_DEAD_END, TASKS } from "./questions.ts";
|
||||
|
||||
// 第零轮质量评估（任务书 12 第 5.2 节六问）。只读账本，问 Jev，原始概率全部存进 <检验目录>/quality.json。
|
||||
// node eval/round-zero/quality.ts <检验目录>（研究在 <检验目录>/study）
|
||||
// "入口全体"是分配时最终有效的全部入口，"分到的入口"是分配结果里每人分到的那个。
|
||||
|
||||
type Idea = { id: string; name: string; description: string };
|
||||
const ideas = JSON.parse(readFileSync("eval/round-zero/ideas.json", "utf8")) as { key: Idea[]; deadEnds: Idea[] };
|
||||
|
||||
const out = process.argv[2];
|
||||
if (out === undefined) throw new Error("要写检验目录，研究在它下面的 study 里");
|
||||
const records = readLedger(join(out, "study"));
|
||||
const decision = entryDecision(records);
|
||||
if (decision === undefined) throw new Error("账本里还没有第 0 轮的入口分配决定，第零轮没跑完");
|
||||
const round0 = stages(records).filter((s) => s.round === 0);
|
||||
const final = entriesIn(records, new Set(round0.map((s) => s.stageId)));
|
||||
const draft = entriesIn(records, new Set(round0.filter((s) => s.stage === "entryDraft").map((s) => s.stageId)));
|
||||
// 自己按编号规则算出的入口全体，要和 flow 分配时用的那份一致；对不上说明 flow 或这里有一边取错了。
|
||||
const decided = decision.input.entries;
|
||||
if (JSON.stringify(decided.map((e) => e.id).sort()) !== JSON.stringify(final.map((e) => e.id).sort())) {
|
||||
throw new Error(`分配决定里的入口 ${decided.map((e) => e.id).join("、")} 和从提交算出的入口全体 ${final.map((e) => e.id).join("、")} 对不上`);
|
||||
}
|
||||
const assignment = decision.output;
|
||||
const assigned = assignment.map((a) => final.find((e) => e.id === a.entry)!);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 问 Jev：同样的材料只问一次；每次请求的材料、问题、原始概率都记下
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const raw: Array<{ site: string; ids: string[]; answers: Answer[] }> = [];
|
||||
const cache = new Map<string, Promise<Answer[]>>();
|
||||
/** 同时最多几个请求在路上：一下发上百个容易被限流。 */
|
||||
const PARALLEL = 6;
|
||||
let running = 0;
|
||||
const waiting: Array<() => void> = [];
|
||||
async function limited<T>(fn: () => Promise<T>): Promise<T> {
|
||||
if (running >= PARALLEL) await new Promise<void>((ok) => waiting.push(ok));
|
||||
running++;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
running--;
|
||||
waiting.shift()?.();
|
||||
}
|
||||
}
|
||||
const show = (e: Entry) => ({ name: e.name, body: e.body });
|
||||
/** 同样的材料、同样的问题只问一次（写入口阶段没换的入口在讨论后还是同一段文字）。 */
|
||||
function query(site: string, ids: string[], state: Record<string, unknown>, questions: Record<string, Noul>): Promise<Answer[]> {
|
||||
const key = JSON.stringify([site, state]);
|
||||
if (!cache.has(key)) {
|
||||
cache.set(key, limited(() => ask(state, questions)).then((answers) => (raw.push({ site, ids, answers }), answers)));
|
||||
}
|
||||
return cache.get(key)!;
|
||||
}
|
||||
const yes = async (p: Promise<Answer[]>, question: string) => (await p).find((a) => a.question === question)!.yes;
|
||||
|
||||
const ideaState = (e: Entry, idea: Idea) => ({ task: TASKS.idea, entry: show(e), idea: { description: idea.description } });
|
||||
const leadsTo = (e: Entry, idea: Idea) => yes(query("leads", [e.id, idea.id], ideaState(e, idea), LEADS_TO_IDEA), "leads_to_idea");
|
||||
const staysIn = (e: Entry, idea: Idea) => yes(query("stays", [e.id, idea.id], ideaState(e, idea), STOPS_AT_DEAD_END), "stays_within_idea");
|
||||
const entryOnly = (e: Entry, question: string) => yes(query("entry", [e.id], { task: TASKS.entry, entry: show(e) }, ENTRY_ONLY), question);
|
||||
const sameKind = (a: Entry, b: Entry) =>
|
||||
yes(query("pair", [a.id, b.id], { task: TASKS.pair, entry_a: show(a), entry_b: show(b) }, SAME_KIND), "same_kind_of_change");
|
||||
|
||||
async function coverage(list: Entry[]): Promise<Record<string, string[]>> {
|
||||
const out: Record<string, string[]> = {};
|
||||
for (const idea of ideas.key) {
|
||||
const hits = await Promise.all(list.map((e) => leadsTo(e, idea)));
|
||||
out[idea.id] = list.filter((_, i) => hits[i]).map((e) => e.id);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
async function overlap(list: Entry[]): Promise<{ pairs: number; same: string[][]; ratio: number | null }> {
|
||||
const pairs = list.flatMap((a, i) => list.slice(i + 1).filter((b) => b.author !== a.author).map((b) => [a, b] as const));
|
||||
const hits = await Promise.all(pairs.map(([a, b]) => sameKind(a, b)));
|
||||
const same = pairs.filter((_, i) => hits[i]).map(([a, b]) => [a.id, b.id]);
|
||||
return { pairs: pairs.length, same, ratio: pairs.length === 0 ? null : same.length / pairs.length };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 六问
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const k1 = ideas.key.find((i) => i.id === "K1")!;
|
||||
const covFinal = await coverage(final);
|
||||
const q1 = { coverage: covFinal, pass: covFinal["K1"]!.length > 0 };
|
||||
|
||||
const assignedPairs = assigned.flatMap((a, i) => assigned.slice(i + 1).map((b) => [a, b] as const));
|
||||
const pairHits = await Promise.all(assignedPairs.map(([a, b]) => sameKind(a, b)));
|
||||
const q2 = { same: assignedPairs.filter((_, i) => pairHits[i]).map(([a, b]) => [a.id, b.id]), pass: !pairHits.some(Boolean) };
|
||||
|
||||
const assignedK1 = await Promise.all(assigned.map((e) => leadsTo(e, k1)));
|
||||
const deadEnd = await Promise.all(
|
||||
assigned.map(async (e) => {
|
||||
const hits = await Promise.all(ideas.deadEnds.map((d) => staysIn(e, d)));
|
||||
return ideas.deadEnds.filter((_, i) => hits[i]).map((d) => d.id);
|
||||
}),
|
||||
);
|
||||
const q3 = {
|
||||
k1: assigned.filter((_, i) => assignedK1[i]).map((e) => e.id),
|
||||
deadEnds: Object.fromEntries(assigned.map((e, i) => [e.id, deadEnd[i]])),
|
||||
pass: assignedK1.some(Boolean) && deadEnd.every((d) => d.length === 0),
|
||||
};
|
||||
|
||||
const perEntry = async (list: Entry[], question: string) => (await Promise.all(list.map((e) => entryOnly(e, question))));
|
||||
const changesCode = await perEntry(assigned, "changes_code");
|
||||
const q4 = { notCodeRoute: assigned.filter((_, i) => !changesCode[i]).map((e) => e.id), pass: changesCode.every(Boolean) };
|
||||
|
||||
const writesAnswer = async (list: Entry[]) => {
|
||||
const [code, score] = [await perEntry(list, "names_code_change"), await perEntry(list, "states_expected_score")];
|
||||
return list.filter((_, i) => code[i] || score[i]).map((e) => e.id);
|
||||
};
|
||||
const q5 = { assignedWritingAnswer: await writesAnswer(assigned), allWritingAnswer: await writesAnswer(final) };
|
||||
const q5pass = q5.assignedWritingAnswer.length === 0;
|
||||
|
||||
const q6 = {
|
||||
beforeDiscussion: { overlap: await overlap(draft), coverage: await coverage(draft) },
|
||||
afterDiscussion: { overlap: await overlap(final), coverage: covFinal },
|
||||
};
|
||||
|
||||
const report = {
|
||||
study: join(out, "study"),
|
||||
thresholdNote: THRESHOLD_NOTE,
|
||||
entries: final.map((e) => ({ id: e.id, author: e.author, name: e.name })),
|
||||
draftEntries: draft.map((e) => ({ id: e.id, author: e.author, name: e.name })),
|
||||
assignment: assignment.map((a, i) => ({ ...a, name: assigned[i]!.name })),
|
||||
results: { q1, q2, q3, q4, q5: { ...q5, pass: q5pass }, q6 },
|
||||
passed: { q1: q1.pass, q2: q2.pass, q3: q3.pass, q4: q4.pass, q5: q5pass },
|
||||
raw,
|
||||
};
|
||||
writeFileSync(join(out, "quality.json"), `${JSON.stringify(report, null, 2)}\n`);
|
||||
|
||||
const mark = (b: boolean) => (b ? "通过" : "不通过");
|
||||
const ratio = (o: { pairs: number; same: string[][]; ratio: number | null }) => `${o.same.length}/${o.pairs}${o.ratio === null ? "" : `（${(o.ratio * 100).toFixed(0)}%）`}`;
|
||||
const count = (c: Record<string, string[]>) => Object.entries(c).map(([k, v]) => `${k}:${v.length}`).join(" ");
|
||||
console.log(`（${THRESHOLD_NOTE}）`);
|
||||
console.log(`入口全体：${final.map((e) => `${e.id} ${e.name}`).join("\n ")}`);
|
||||
console.log(`分到的入口：${report.assignment.map((a) => `${a.researcher} → ${a.entry}（第 ${a.rank} 志愿）${a.name}`).join("\n ")}`);
|
||||
console.log(`1 方向对不对：${mark(q1.pass)}；覆盖 ${count(covFinal)}`);
|
||||
console.log(`2 分没分开：${mark(q2.pass)}；判"是"的对 ${JSON.stringify(q2.same)}`);
|
||||
console.log(`3 好方向分出去没有：${mark(q3.pass)}；指向 K1 的 ${JSON.stringify(q3.k1)}，停在死路的 ${JSON.stringify(q3.deadEnds)}`);
|
||||
console.log(`4 改代码的路线：${mark(q4.pass)}；不是的 ${JSON.stringify(q4.notCodeRoute)}`);
|
||||
console.log(`5 有没有写答案：${mark(q5pass)}；分到的里写了的 ${JSON.stringify(q5.assignedWritingAnswer)}，入口全体里写了的 ${JSON.stringify(q5.allWritingAnswer)}`);
|
||||
console.log(`6 讨论有没有用：讨论前重合 ${ratio(q6.beforeDiscussion.overlap)}、覆盖 ${count(q6.beforeDiscussion.coverage)}；讨论后重合 ${ratio(q6.afterDiscussion.overlap)}、覆盖 ${count(covFinal)}`);
|
||||
console.log(`原始概率：${join(out, "quality.json")}`);
|
||||
@@ -0,0 +1,176 @@
|
||||
import type { Noul } from "../lib/jev.ts";
|
||||
|
||||
// 第零轮质量评估问 Jev 的问题（任务书 12 第 5.2 节"问 Jev 的规矩"）：英文、一问只问一件事、
|
||||
// 带材料说明、判"是"判"否"的说明和反例；写法照 src/jev/questions.ts。这些是检验自己的问题，不进 src/jev。
|
||||
// 任务书里两处一句话里有两个条件（第 3 问"D1 或 D2"、第 5 问"写出改成什么，或者预计分数"），按"一问一事"拆成两问，
|
||||
// 在 quality.ts 里用"任一判是"合起来。
|
||||
|
||||
const TASK =
|
||||
"Round zero of a study that improves a program for the circle packing task: place 26 non-overlapping circles inside the unit square " +
|
||||
"so that the sum of their radii is as large as possible (no tolerance: any overlap or circle outside the square is invalid). " +
|
||||
"The baseline program puts the circles on fixed rings around the center and then shrinks radii pair by pair to remove overlaps; " +
|
||||
"it scores about 0.96 and the best known score is about 2.636. " +
|
||||
"In round zero each researcher writes entries: starting points for a line of work that one researcher will follow in the next rounds. " +
|
||||
"An entry has a `name` and a `body` (in Chinese) saying where to start looking and why the route is worth walking. " +
|
||||
"Entries are meant to name a starting point only, not the finished code change or an expected score.";
|
||||
|
||||
export const TASKS = {
|
||||
idea: `${TASK} \`idea\` is a technique from the literature on this task, described in English.`,
|
||||
entry: TASK,
|
||||
pair: `${TASK} \`entry_a\` and \`entry_b\` were written by two different researchers.`,
|
||||
};
|
||||
|
||||
/** 第 1 问：按这个入口走，会直接走到这个想法吗（idea 是 K1–K5 之一）。 */
|
||||
export const LEADS_TO_IDEA: Record<string, Noul> = {
|
||||
leads_to_idea: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Will a researcher who follows `entry` arrive directly at the technique described in `idea.description`?",
|
||||
focus:
|
||||
"Read `entry.body` for what the researcher will look at first and what they intend to try or measure. " +
|
||||
"Answer yes if the entry names this technique, or if this technique is the natural first code change on the route the entry points to. " +
|
||||
"The entry may describe the technique in other words, in Chinese. Sharing the goal of a higher score, or reading the same function, is not enough.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "The entry names this technique, or the route it points to leads straight to this technique as its first code change",
|
||||
examples: [
|
||||
"idea: optimize centers and radii together with a constrained optimizer; entry: 把 26 个圆心和半径一起交给 scipy 的约束优化器",
|
||||
"idea: solve a linear program for the radii with fixed centers; entry: 先问给定圆心下半径和的最大值怎么算，它是不是一个线性规划",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "The entry heads somewhere else: a different technique, only reading rules, or only measuring, so this technique would not be its next step",
|
||||
not_for: "Entries that only share the goal of raising the score, or only read the code this technique would change, without heading for the technique",
|
||||
examples: [
|
||||
"idea: optimize centers and radii together with a constrained optimizer; entry: 只读评测脚本，弄清容差和合法化规则",
|
||||
"idea: solve a linear program for the radii with fixed centers; entry: 把三环布局换成六角网格",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
/** 第 3 问后半：这个入口的路线是不是只停在这条死路上（idea 是 D1 或 D2 之一）。 */
|
||||
export const STOPS_AT_DEAD_END: Record<string, Noul> = {
|
||||
stays_within_idea: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Does the whole route in `entry` stay within the approach described in `idea.description`, with no plan to go beyond it?",
|
||||
focus:
|
||||
"Answer yes only if everything the entry plans to change falls inside this approach. " +
|
||||
"If the entry also plans a continuous optimization that moves the circle centers, or uses the approach only as a helper step " +
|
||||
"(a starting layout, or a way to score layouts inside a larger search), answer no.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "Everything the entry plans to change is inside this approach; it is the whole route",
|
||||
examples: [
|
||||
"idea: hand-designed placement of centers only; entry: 把三环布局换成六角网格，调行数和间距",
|
||||
"idea: change only the radii with fixed centers; entry: 圆心不动，只改半径的分配方法，这就是整条路线",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "The entry goes beyond this approach, or uses it only as one step of a larger route",
|
||||
not_for: "Entries that use this approach as a starting layout or as a scoring helper inside a search that moves the centers",
|
||||
examples: [
|
||||
"idea: hand-designed placement of centers only; entry: 从六角起点出发，再用约束优化同时移动圆心和半径",
|
||||
"idea: change only the radii with fixed centers; entry: 先用精确的半径求解当打分器，再在外层搜索圆心位置",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
/** 第 4、5 问：只看一个入口。第 5 问拆成两问。 */
|
||||
export const ENTRY_ONLY: Record<string, Noul> = {
|
||||
changes_code: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Is `entry` a route that changes the program's code to raise the score, rather than one that only reads rules, measures, or analyzes?",
|
||||
focus:
|
||||
"A route may begin by reading code or measuring something, as long as the reading or measuring is meant to lead to a code change that raises the score. " +
|
||||
"Answer no only if the entry's purpose ends at understanding, measuring, or checking rules, with no plan to change the code.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "The entry intends to change the program, possibly after a first step of reading or measuring",
|
||||
examples: ["先读半径的计算方法，量它离最优差多少，然后把它换掉", "从错排起点出发，跑一次约束优化"],
|
||||
},
|
||||
false: {
|
||||
what: "The entry only reads rules, measures, or analyzes, and does not plan any change to the code",
|
||||
not_for: "Entries that begin with a diagnosis but clearly intend to change the code afterwards",
|
||||
examples: ["只读评测脚本，列出评测接受和拒绝哪些解", "统计基线产物里有多少个圆贴着边，写成一份报告"],
|
||||
},
|
||||
},
|
||||
},
|
||||
names_code_change: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Does `entry` spell out the finished code change to make, such as the exact algorithm with its settings or the replacement code, instead of only a starting point?",
|
||||
focus:
|
||||
"Entries are meant to say where to start looking and why. Mentioning a technique as a question to explore, or as a direction to look in, is a starting point. " +
|
||||
"Stating the change to make with the method and its settings already decided spells out the answer.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "The entry states what the code should be changed into, with the method and its settings decided",
|
||||
examples: [
|
||||
"把布局函数换成 SLSQP，同时优化 78 个变量，起点用 5 行错排，迭代 500 次",
|
||||
"半径计算改成用 scipy linprog 求解，约束是 r_i + r_j ≤ d_ij 和到四边的距离",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "The entry names where to look and what to question, leaving the change itself to be worked out",
|
||||
not_for: "Entries that name a technique only as a question to check or a direction to explore",
|
||||
examples: ["先读半径的计算方法，问它离给定圆心下的最优半径和差多少", "从基线的布局出发，看哪些圆被挤到了边上"],
|
||||
},
|
||||
},
|
||||
},
|
||||
states_expected_score: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Does `entry` state a score, or an improvement of the score, that following this route is expected to reach?",
|
||||
focus:
|
||||
"Look for predictions of the result of this route. Quoting the baseline score, the best known score, or measured facts about the current program is not a prediction.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "The entry predicts the score or the gain that this route will bring",
|
||||
examples: ["这条路预计能到 2.6 以上", "改完能提升约 0.5"],
|
||||
},
|
||||
false: {
|
||||
what: "The entry predicts no score or gain for this route",
|
||||
not_for: "Quoting the current score, the best known score, or the size of the current gap without predicting what the route reaches",
|
||||
examples: ["基线现在是 0.96，已知最好是 2.636", "现在离已知最好还差 1.68"],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
/** 第 2、6 问：两位研究者从这两个入口出发，多半会做同一类代码改动吗。 */
|
||||
export const SAME_KIND: Record<string, Noul> = {
|
||||
same_kind_of_change: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question: "Would two researchers, one starting from `entry_a` and one from `entry_b`, most likely make the same kind of code change?",
|
||||
focus:
|
||||
"Imagine each researcher follows their entry and writes one code change. Compare which part of the program they would change and by what technique. " +
|
||||
"Different wording, a different first file to read, or a different motivation does not make the changes different if both would change the same thing in the same way.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "Both would most likely change the same part of the program with the same technique",
|
||||
examples: [
|
||||
"a: 用约束优化器同时调整圆心和半径; b: 把 78 个变量一起交给 SLSQP",
|
||||
"a: 换成六角网格布局; b: 把圆心改成错排的行，调行数",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "They would most likely change different parts of the program, or the same part with a different technique",
|
||||
not_for: "Entries that merely share the goal of a higher score, or read the same function, but would change different things",
|
||||
examples: ["a: 圆心固定，用线性规划求半径; b: 用约束优化器同时调整圆心和半径", "a: 换成六角网格布局; b: 只读评测规则，弄清容差"],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,51 @@
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { entriesIn, entryDecision, ofKind, readLedger, sameAssignment, stages, type Assignment } from "../lib/ledger.ts";
|
||||
import { replayModels, startReplay, type ReplayScript } from "../lib/replay.ts";
|
||||
import { launch, newEvalDir, writeConfig } from "../lib/study.ts";
|
||||
|
||||
// 把参考项目 16-real-1 的第零轮原样交给新 flow 回放一遍（任务书 12 第 5.3 节）：基线照样用真 Docker 跑，
|
||||
// 三位研究者由回放假模型按当年的先后交提交。跑完核对分配是否和当年完全相同，列出被退回的提交和没交出去的提交。
|
||||
// node eval/round-zero/replay.ts <名字>
|
||||
// 研究在 <检验目录>/study；之后用 quality.ts 打分。
|
||||
|
||||
type Script = ReplayScript & { expected: Assignment[]; expectedEntries: string[] };
|
||||
|
||||
const name = process.argv[2];
|
||||
if (name === undefined) throw new Error("要写检验名字，例如 r0-replay-16");
|
||||
const script = JSON.parse(readFileSync("eval/round-zero/replay-16-real-1.json", "utf8")) as Script;
|
||||
const out = newEvalDir(name);
|
||||
const replay = await startReplay(script, join(out, "pi-agent"), 0);
|
||||
const researchers = Object.keys(script.models).sort();
|
||||
const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, replayModels(researchers, "jev-latest"));
|
||||
const studyDir = join(out, "study");
|
||||
const { outcome } = await launch(config, studyDir, replay.env).done;
|
||||
await replay.close();
|
||||
|
||||
const records = readLedger(studyDir);
|
||||
const decision = entryDecision(records);
|
||||
const got = decision === undefined ? null : decision.output;
|
||||
const entries = entriesIn(records, new Set(stages(records).filter((s) => s.round === 0).map((s) => s.stageId))).map((e) => e.id);
|
||||
// 入口全体按编号比（账本里按各人最后一版清单的先后排，当年的清单按编号排）；分配按研究者、入口、志愿三样比，不看对象里键的先后。
|
||||
const sameEntries = JSON.stringify([...entries].sort()) === JSON.stringify([...script.expectedEntries].sort());
|
||||
const same = got !== null && sameAssignment(got, script.expected);
|
||||
const report = {
|
||||
outcome,
|
||||
assignment: got,
|
||||
expected: script.expected,
|
||||
sameAsReference: same,
|
||||
entries,
|
||||
expectedEntries: script.expectedEntries,
|
||||
sameEntriesAsReference: sameEntries,
|
||||
rejections: ofKind(records, "rejection").map((r) => ({ seq: r.seq, stageId: r.content.stageId, author: r.content.author, kind: r.content.kind, reason: r.content.reason })),
|
||||
delivered: replay.delivered,
|
||||
notDelivered: replay.remaining(),
|
||||
};
|
||||
writeFileSync(join(out, "replay-report.json"), `${JSON.stringify(report, null, 2)}\n`);
|
||||
console.log(`\n结局：${JSON.stringify(outcome)}`);
|
||||
console.log(`分配：${got === null ? "没有写出分配决定" : got.map((a) => `${a.researcher} → ${a.entry}（第 ${a.rank} 志愿）`).join("；")}`);
|
||||
console.log(`当年：${script.expected.map((a) => `${a.researcher} → ${a.entry}（第 ${a.rank} 志愿）`).join("；")}`);
|
||||
console.log(`分配和当年相同：${same ? "是" : "否"}；入口全体和当年相同：${sameEntries ? "是" : "否"}（${entries.join("、")}）`);
|
||||
console.log(`退回 ${report.rejections.length} 份${report.rejections.map((r) => `\n ${r.author} ${r.stageId} ${r.kind}：${r.reason}`).join("")}`);
|
||||
console.log(`没交出去的：${JSON.stringify(report.notDelivered)}`);
|
||||
console.log(`报告：${join(out, "replay-report.json")}；研究目录：${studyDir}`);
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"studyId": "r0-eval",
|
||||
"task": "tasks/circle_packing",
|
||||
"models": {
|
||||
"researchers": [
|
||||
{ "provider": "iomgaa", "model": "glm-5.3", "thinking": "medium" },
|
||||
{ "provider": "iomgaa", "model": "glm-5.3", "thinking": "medium" },
|
||||
{ "provider": "iomgaa", "model": "glm-5.3", "thinking": "medium" }
|
||||
],
|
||||
"experimenter": { "provider": "iomgaa", "model": "glm-5.3", "thinking": "medium" },
|
||||
"verifier": { "provider": "iomgaa", "model": "glm-5.3", "thinking": "medium" },
|
||||
"jev": "jev-latest"
|
||||
},
|
||||
"proposalLimit": 1,
|
||||
"batchLimit": 2,
|
||||
"runs": 13,
|
||||
"factorLimit": 3,
|
||||
"maxRepairs": 1,
|
||||
"lightChecks": { "maxCount": 2, "seconds": 5 },
|
||||
"promptDir": "prompts",
|
||||
"devEntry": false,
|
||||
"flow": {
|
||||
"preset": "full",
|
||||
"roundZero": { "enabled": true, "discussions": 2 },
|
||||
"firstPrediction": true,
|
||||
"critique": true,
|
||||
"ablation": { "enabled": true, "cellRule": "valueMinusCost" },
|
||||
"batchRule": "bestExpected",
|
||||
"openReserve": 0.1,
|
||||
"knowledgeUse": 0.5,
|
||||
"decay": 0.5
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { join } from "node:path";
|
||||
import { launch, newEvalDir, writeConfig } from "../lib/study.ts";
|
||||
|
||||
// 用真模型、真 Docker 跑一次第零轮（任务书 12 第 5.2 节"同样的配置真实跑 3 次"）：配置照 eval/round-zero/research.json，
|
||||
// 研究编号换成检验名字。研究在 <检验目录>/study，跑完用 quality.ts 打分。要花钱。
|
||||
// node eval/round-zero/run.ts <名字>
|
||||
// 中途按 Ctrl-C 会把停下信号传给研究进程；接着跑用 node eval/lib/cli.ts --resume --dir <检验目录>/study。
|
||||
|
||||
const name = process.argv[2];
|
||||
if (name === undefined) throw new Error("要写检验名字，例如 r0-real-1");
|
||||
const out = newEvalDir(name);
|
||||
const config = writeConfig("eval/round-zero/research.json", join(out, "research.json"), name, null);
|
||||
const study = launch(config, join(out, "study"), {});
|
||||
process.on("SIGINT", () => study.child.kill("SIGINT"));
|
||||
const { outcome } = await study.done;
|
||||
console.log(`\n结局：${JSON.stringify(outcome)}\n研究目录：${join(out, "study")}`);
|
||||
Reference in New Issue
Block a user