iomgaa
852600df02
消融默认关，关掉时解释阶段一并跳过；选诊断实验用信息价值减成本。 eval/diagnosis/research.json 为第三块配置。vocab 字段说明的出口文字随提示词一起改，见下一次提交的 prompts/ 与同文件改动（本次含在内）。 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
104 lines
3.6 KiB
TypeScript
104 lines
3.6 KiB
TypeScript
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { TestContext } from "node:test";
|
|
import { Ledger } from "../src/ledger/index.ts";
|
|
import type { ModelUsage, ProcessResult, Range, RunDefinition, Stage, Submission } from "../src/vocab/index.ts";
|
|
|
|
// ledger 测试共用：每个测试在自己的临时目录里建一个真实的数据库文件，测试结束时连目录一起删掉。
|
|
|
|
export function tempDir(t: TestContext): string {
|
|
const dir = mkdtempSync(join(tmpdir(), "inforsi-ledger-"));
|
|
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
|
return dir;
|
|
}
|
|
|
|
export function freshLedger(t: TestContext): { ledger: Ledger; path: string } {
|
|
const path = join(tempDir(t), "ledger.db");
|
|
const ledger = Ledger.open(path);
|
|
t.after(() => ledger.close());
|
|
return { ledger, path };
|
|
}
|
|
|
|
export const range = (q1: number, q5: number, q9: number): Range => ({ q1, q5, q9 });
|
|
|
|
export function proposal(estimate: Range, body = "提案正文"): Submission<"proposal"> {
|
|
return {
|
|
factors: [{ name: "换优化器", method: "把 SGD 换成 Adam", keySettings: "学习率 0.01", continues: null }],
|
|
safeguards: [],
|
|
metrics: { defined: [], reused: [] },
|
|
authorEstimate: estimate,
|
|
body,
|
|
};
|
|
}
|
|
|
|
export const prediction = (r: Range): Submission<"firstPrediction"> => ({ range: r, body: "理由" });
|
|
export const critique = (body = "挑刺内容"): Submission<"critique"> => ({ target: "whole", body });
|
|
export const wrapUp = (body = "收尾"): Submission<"wrapUp"> => ({ body });
|
|
|
|
export const implementation = (body = "无"): Submission<"implementation"> => ({
|
|
factorLocations: [{ factor: "p1", locations: ["solve.py:10-20"] }],
|
|
safeguardLocations: [],
|
|
unassigned: [],
|
|
concerns: "无",
|
|
body,
|
|
});
|
|
|
|
export const definition: RunDefinition = {
|
|
command: ["python", "solve.py"],
|
|
workdir: "/work",
|
|
image: { digest: `sha256:${"0".repeat(64)}`, runtime: "runc" },
|
|
wallSeconds: 120,
|
|
cores: 1,
|
|
inheritEnv: false,
|
|
env: {},
|
|
};
|
|
|
|
export const exited: ProcessResult = {
|
|
ending: "exited",
|
|
exitCode: 0,
|
|
cpuSeconds: { status: "measured", value: 100 },
|
|
wallSeconds: { status: "measured", value: 101 },
|
|
peakMemoryBytes: { status: "measured", value: 1e8 },
|
|
};
|
|
|
|
export const notStarted: ProcessResult = { ending: "notStarted", reason: "启动前取消" };
|
|
|
|
/** 开一个阶段，返回开阶段那条记录的序号。 */
|
|
export function openStage(
|
|
ledger: Ledger,
|
|
stageId: string,
|
|
stage: Stage,
|
|
round: number,
|
|
roster: string[] = ["r1", "r2"],
|
|
): number {
|
|
return ledger.append("stageOpen", { stageId, stage, round, roster }).seq;
|
|
}
|
|
|
|
export function closeStage(ledger: Ledger, stageId: string): number {
|
|
return ledger.append("stageClose", { stageId }).seq;
|
|
}
|
|
|
|
/** 登记一次运行。 */
|
|
export function planRun(ledger: Ledger, run: string, purpose: "baseline" | "forward" | "ablation" | "diagnosis", round: number): number {
|
|
return ledger.append("runPlanned", { run, purpose, round, object: purpose === "baseline" ? null : `obj-${run}`, definition }).seq;
|
|
}
|
|
|
|
/** 一次模型调用的开始记录，materials 是这次任务的全部材料。 */
|
|
export const callStart = (call: string, actor = "r1", stageId: string | null = null) => ({
|
|
call,
|
|
actor,
|
|
stageId,
|
|
model: "glm-5.3",
|
|
materials: { firstMessage: "任务说明……", tools: ["read", "submit"] },
|
|
});
|
|
|
|
const unknownCount = { status: "unmeasured" as const, reason: "服务没报" };
|
|
export const usage: ModelUsage = {
|
|
inputTokens: unknownCount,
|
|
cacheReadTokens: unknownCount,
|
|
cacheWriteTokens: unknownCount,
|
|
outputTokens: unknownCount,
|
|
cost: unknownCount,
|
|
};
|