iomgaa
852600df02
消融默认关，关掉时解释阶段一并跳过；选诊断实验用信息价值减成本。 eval/diagnosis/research.json 为第三块配置。vocab 字段说明的出口文字随提示词一起改，见下一次提交的 prompts/ 与同文件改动（本次含在内）。 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
289 lines
12 KiB
TypeScript
289 lines
12 KiB
TypeScript
import assert from "node:assert/strict";
|
||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||
import { tmpdir } from "node:os";
|
||
import { join } from "node:path";
|
||
import { test } from "node:test";
|
||
import { checkConfig, loadConfig, ResearchConfig } from "../src/config/index.ts";
|
||
import { fullConfig, naiveConfig } from "./config-helpers.ts";
|
||
|
||
type Obj = Record<string, unknown>;
|
||
|
||
/** 配置里每个对象字段的路径（JSON 指针），数组里的对象也走进去。 */
|
||
function fieldPaths(value: unknown, at = ""): string[] {
|
||
if (Array.isArray(value)) return value.flatMap((item, i) => fieldPaths(item, `${at}/${i}`));
|
||
if (typeof value !== "object" || value === null) return [];
|
||
return Object.entries(value).flatMap(([k, v]) => [`${at}/${k}`, ...fieldPaths(v, `${at}/${k}`)]);
|
||
}
|
||
|
||
function parentAndKey(root: unknown, path: string): [Obj, string] {
|
||
const keys = path.split("/").slice(1);
|
||
let node = root as Obj;
|
||
for (const k of keys.slice(0, -1)) node = node[k] as Obj;
|
||
return [node, keys[keys.length - 1]!];
|
||
}
|
||
|
||
/** 校验应失败，返回错误消息。 */
|
||
function errorOf(value: unknown): string {
|
||
try {
|
||
checkConfig(value);
|
||
} catch (error) {
|
||
return (error as Error).message;
|
||
}
|
||
assert.fail("应当报错，却通过了校验");
|
||
}
|
||
|
||
test("两份写全的合法配置能通过，返回的就是原内容", () => {
|
||
for (const config of [fullConfig(), naiveConfig()]) {
|
||
assert.deepEqual(checkConfig(structuredClone(config)), config);
|
||
}
|
||
});
|
||
|
||
test("逐个删掉每个字段都报错，报错里有这个字段的路径", () => {
|
||
for (const make of [fullConfig, naiveConfig]) {
|
||
const paths = fieldPaths(make());
|
||
assert.ok(paths.length > 10);
|
||
for (const path of paths) {
|
||
const config = make();
|
||
const [parent, key] = parentAndKey(config, path);
|
||
delete parent[key];
|
||
const message = errorOf(config);
|
||
assert.ok(message.includes(`${path}：`), `删掉 ${path}，报错里应有这个路径：\n${message}`);
|
||
}
|
||
}
|
||
});
|
||
|
||
test("不认识的字段报错：顶层、模型里、两种流程里", () => {
|
||
const top = { ...fullConfig(), seed: "s" };
|
||
assert.match(errorOf(top), /^\/seed：不认识的字段$/m);
|
||
|
||
const model = fullConfig();
|
||
(model.models.researchers[1] as Obj).temperature = 0.2;
|
||
assert.match(errorOf(model), /^\/models\/researchers\/1\/temperature：不认识的字段$/m);
|
||
|
||
const full = fullConfig();
|
||
(full.flow as Obj).researcherNotes = false;
|
||
assert.match(errorOf(full), /^\/flow\/researcherNotes：不认识的字段$/m);
|
||
});
|
||
|
||
test("类型或取值不对报错，带字段路径", () => {
|
||
const cases: Array<[(c: Obj) => void, RegExp]> = [
|
||
[(c) => (c.runs = "13"), /^\/runs：应为整数$/m],
|
||
[(c) => (c.runs = 0), /^\/runs：不能小于 1$/m],
|
||
[(c) => (c.batchLimit = 1.5), /^\/batchLimit：应为整数$/m],
|
||
[(c) => (c.studyId = "../x"), /^\/studyId：格式不对/m],
|
||
[(c) => (c.task = " "), /^\/task：必须是绝对路径（以 \/ 开头）$/m],
|
||
[(c) => (c.devEntry = "false"), /^\/devEntry：应为true 或 false$/m],
|
||
[(c) => ((c.models as Obj).researchers = []), /^\/models\/researchers：至少要有 1 项$/m],
|
||
[(c) => (((c.models as Obj).verifier as Obj).thinking = null), /^\/models\/verifier\/thinking：只能是 "off"、"minimal"、"low"、"medium"、"high"、"xhigh"、"max" 之一$/m],
|
||
[(c) => (((c.models as Obj).verifier as Obj).thinking = "auto"), /^\/models\/verifier\/thinking：只能是/m],
|
||
[(c) => ((c.flow as Obj).openReserve = 0), /^\/flow\/openReserve：必须大于 0$/m],
|
||
[(c) => ((c.flow as Obj).decay = 1.5), /^\/flow\/decay：不能大于 1$/m],
|
||
[(c) => ((c.flow as Obj).roundZero = { enabled: true, discussions: 3 }), /^\/flow\/roundZero\/discussions：只能是 1、2 之一$/m],
|
||
[(c) => ((c.flow as Obj).preset = "fast"), /^\/flow\/preset：缺少或取值不对，只能是 "full"、"naive" 之一$/m],
|
||
[(c) => ((c.lightChecks as Obj).seconds = 0), /^\/lightChecks\/seconds：必须大于 0$/m],
|
||
];
|
||
for (const [change, expected] of cases) {
|
||
const config = fullConfig() as unknown as Obj;
|
||
change(config);
|
||
const message = errorOf(config);
|
||
assert.match(message, expected);
|
||
assert.match(message, /共 1 处/, `只改了一处，应只报一处：\n${message}`);
|
||
}
|
||
});
|
||
|
||
test("研究编号带大写报错：它会进 Docker 镜像名，镜像名只认小写", () => {
|
||
for (const id of ["Circle-1", "circle-A", "R0"]) {
|
||
const config = fullConfig() as unknown as Obj;
|
||
config.studyId = id;
|
||
const message = errorOf(config);
|
||
assert.match(message, /^\/studyId：格式不对/m);
|
||
assert.match(message, /共 1 处/, message);
|
||
}
|
||
});
|
||
|
||
test("任务目录、提示词目录写相对路径报错，报错写明字段路径", () => {
|
||
const cases: Array<[string, string]> = [
|
||
["task", "tasks/circle_packing"],
|
||
["task", "./tasks/circle_packing"],
|
||
["promptDir", "prompts"],
|
||
["promptDir", "../InfoRSI/prompts"],
|
||
];
|
||
for (const [field, path] of cases) {
|
||
const config = fullConfig() as unknown as Obj;
|
||
config[field] = path;
|
||
const message = errorOf(config);
|
||
assert.match(message, new RegExp(`^/${field}：必须是绝对路径（以 / 开头）$`, "m"));
|
||
assert.match(message, /共 1 处/, message);
|
||
}
|
||
});
|
||
|
||
test("Jev 模型必填：少了、写 null、写空白都报错，不补 jev-latest", () => {
|
||
const missing = fullConfig() as unknown as Obj;
|
||
delete (missing.models as Obj).jev;
|
||
assert.match(errorOf(missing), /^\/models\/jev：缺少这一项/m);
|
||
const nulled = fullConfig() as unknown as Obj;
|
||
(nulled.models as Obj).jev = null;
|
||
assert.match(errorOf(nulled), /^\/models\/jev：应为字符串$/m);
|
||
const blank = fullConfig() as unknown as Obj;
|
||
(blank.models as Obj).jev = " ";
|
||
assert.match(errorOf(blank), /^\/models\/jev：不能为空，也不能只有空白$/m);
|
||
const naive = naiveConfig() as unknown as Obj;
|
||
delete (naive.models as Obj).jev;
|
||
assert.match(errorOf(naive), /^\/models\/jev：缺少这一项/m);
|
||
});
|
||
|
||
test("一次列出多处错误，超过 typebox 默认的 8 条、错在 Union 分支里的也都列出", () => {
|
||
const config = fullConfig() as unknown as Obj;
|
||
config.runs = -1;
|
||
delete config.promptDir;
|
||
config.extra = 1;
|
||
((config.models as Obj).experimenter as Obj).model = "";
|
||
((config.models as Obj).researchers as Obj[])[2]!.thinking = null;
|
||
config.devEntry = 0;
|
||
config.factorLimit = 0;
|
||
(config.lightChecks as Obj).maxCount = -1;
|
||
const flow = config.flow as Obj;
|
||
flow.batchRule = "best";
|
||
delete (flow.roundZero as Obj).enabled;
|
||
(flow.ablation as Obj).cellRule = "x";
|
||
flow.decay = -1;
|
||
const message = errorOf(config);
|
||
const paths = [
|
||
"/runs：",
|
||
"/promptDir：",
|
||
"/extra：",
|
||
"/models/experimenter/model：",
|
||
"/models/researchers/2/thinking：",
|
||
"/devEntry：",
|
||
"/factorLimit：",
|
||
"/lightChecks/maxCount：",
|
||
"/flow/batchRule：",
|
||
"/flow/roundZero/enabled：",
|
||
"/flow/ablation/cellRule：",
|
||
"/flow/decay：",
|
||
];
|
||
assert.match(message, new RegExp(`共 ${paths.length} 处`), message);
|
||
for (const path of paths) assert.ok(message.includes(path), `应报出 ${path}：\n${message}`);
|
||
});
|
||
|
||
test("流程写成了不是对象的值：报在流程这一项上", () => {
|
||
const config = fullConfig() as unknown as Obj;
|
||
config.flow = "full";
|
||
assert.match(errorOf(config), /^\/flow：应为对象$/m);
|
||
});
|
||
|
||
test("写不出来的组合：朴素循环里写开关、规则或它用不到的方法参数", () => {
|
||
const extras: Obj = {
|
||
critique: false,
|
||
firstPrediction: false,
|
||
roundZero: { enabled: false },
|
||
diagnosis: { enabled: false },
|
||
ablation: { enabled: false },
|
||
batchRule: "uniformRandom",
|
||
openReserve: 0.1,
|
||
knowledgeUse: 0.5,
|
||
decay: 0.5,
|
||
};
|
||
for (const [key, value] of Object.entries(extras)) {
|
||
const config = naiveConfig() as unknown as Obj;
|
||
(config.flow as Obj)[key] = value;
|
||
assert.match(errorOf(config), new RegExp(`^/flow/${key}：不认识的字段$`, "m"));
|
||
}
|
||
});
|
||
|
||
test("写不出来的组合：关掉的第零轮写讨论次数、关掉的消融写选格规则", () => {
|
||
const zero = fullConfig();
|
||
(zero.flow as Obj).roundZero = { enabled: false, discussions: 2 };
|
||
assert.match(errorOf(zero), /^\/flow\/roundZero\/discussions：不认识的字段$/m);
|
||
|
||
const ablation = fullConfig();
|
||
(ablation.flow as Obj).ablation = { enabled: false, cellRule: "all" };
|
||
assert.match(errorOf(ablation), /^\/flow\/ablation\/cellRule：不认识的字段$/m);
|
||
});
|
||
|
||
/** 诊断开着的完整流程配置。 */
|
||
function diagnosisConfig(): ReturnType<typeof fullConfig> {
|
||
const config = fullConfig();
|
||
(config.flow as Obj).diagnosis = { enabled: true, perResearcher: 2, maxExperiments: 4 };
|
||
return config;
|
||
}
|
||
|
||
test("诊断：开着的配置通过校验；逐个删字段、写 0、写小数、多写字段都报错；关着的不能写两个参数", () => {
|
||
assert.deepEqual(checkConfig(structuredClone(diagnosisConfig())), diagnosisConfig());
|
||
|
||
for (const key of ["enabled", "perResearcher", "maxExperiments"]) {
|
||
const config = diagnosisConfig();
|
||
delete ((config.flow as Obj).diagnosis as Obj)[key];
|
||
assert.match(errorOf(config), new RegExp(`^/flow/diagnosis/${key}：`, "m"));
|
||
}
|
||
const cases: Array<[Obj, RegExp]> = [
|
||
[{ enabled: true, perResearcher: 0, maxExperiments: 4 }, /^\/flow\/diagnosis\/perResearcher：不能小于 1$/m],
|
||
[{ enabled: true, perResearcher: 2, maxExperiments: 2.5 }, /^\/flow\/diagnosis\/maxExperiments：应为整数$/m],
|
||
[{ enabled: true, perResearcher: 2, maxExperiments: 4, rule: "x" }, /^\/flow\/diagnosis\/rule：不认识的字段$/m],
|
||
[{ enabled: false, perResearcher: 2 }, /^\/flow\/diagnosis\/perResearcher：不认识的字段$/m],
|
||
[{ enabled: "yes" }, /^\/flow\/diagnosis\/enabled：缺少或取值不对，只能是 false、true 之一$/m],
|
||
];
|
||
for (const [diagnosis, expected] of cases) {
|
||
const config = fullConfig();
|
||
(config.flow as Obj).diagnosis = diagnosis;
|
||
assert.match(errorOf(config), expected);
|
||
}
|
||
});
|
||
|
||
test("写不出来的组合：规则名不在 rules 登记表里", () => {
|
||
const batch = fullConfig();
|
||
(batch.flow as Obj).batchRule = "expected-best";
|
||
assert.match(errorOf(batch), /^\/flow\/batchRule：只能是 "bestExpected"、"uniformRandom" 之一$/m);
|
||
|
||
const cell = fullConfig();
|
||
(cell.flow as Obj).ablation = { enabled: true, cellRule: "voi" };
|
||
assert.match(
|
||
errorOf(cell),
|
||
/^\/flow\/ablation\/cellRule：只能是 "valueMinusCost"、"all"、"randomSameCount"、"none" 之一$/m,
|
||
);
|
||
});
|
||
|
||
test("字段定义里每个字段都有中文说明，并标明用户配置还是方法参数", () => {
|
||
const seen: string[] = [];
|
||
const visit = (schema: Obj, at: string): void => {
|
||
const props = schema.properties as Record<string, Obj> | undefined;
|
||
for (const [key, field] of Object.entries(props ?? {})) {
|
||
assert.equal(typeof field.description, "string", `${at}/${key} 缺说明`);
|
||
assert.ok(["用户配置", "方法参数"].includes(field.category as string), `${at}/${key} 没标类别`);
|
||
seen.push(`${at}/${key}`);
|
||
visit(field, `${at}/${key}`);
|
||
}
|
||
for (const branch of (schema.anyOf as Obj[] | undefined) ?? []) visit(branch, at);
|
||
if (schema.items) visit(schema.items as Obj, `${at}/0`);
|
||
};
|
||
visit(ResearchConfig as unknown as Obj, "");
|
||
// 抽查几项的类别。
|
||
const category = (path: string[]) => {
|
||
let node = ResearchConfig as unknown as Obj;
|
||
for (const k of path) node = (node.properties as Record<string, Obj>)[k]!;
|
||
return node.category;
|
||
};
|
||
assert.equal(category(["studyId"]), "用户配置");
|
||
assert.equal(category(["runs"]), "用户配置");
|
||
assert.equal(category(["factorLimit"]), "方法参数");
|
||
assert.equal(category(["flow"]), "方法参数");
|
||
assert.equal(category(["models", "jev"]), "用户配置");
|
||
assert.ok(seen.includes("/models/researchers/0/thinking"));
|
||
assert.ok(seen.includes("/flow/ablation/cellRule"));
|
||
});
|
||
|
||
test("loadConfig：读文件并校验；不是 JSON 时报错带文件路径", () => {
|
||
const dir = mkdtempSync(join(tmpdir(), "inforsi-config-"));
|
||
const good = join(dir, "research.json");
|
||
writeFileSync(good, JSON.stringify(fullConfig(), null, 2));
|
||
assert.deepEqual(loadConfig(good), fullConfig());
|
||
|
||
const bad = join(dir, "bad.json");
|
||
writeFileSync(bad, "{ runs: 13 }");
|
||
assert.throws(() => loadConfig(bad), (e: Error) => e.message.startsWith(`${bad} 不是合法的 JSON`));
|
||
|
||
const invalid = join(dir, "invalid.json");
|
||
writeFileSync(invalid, JSON.stringify({ ...fullConfig(), runs: 0 }));
|
||
assert.throws(() => loadConfig(invalid), /\/runs：不能小于 1/);
|
||
});
|