iomgaa
0ac1e01613
src/jev 3 个文件 515 行：判重、查冻结条款、查代码改动三个函数，九问问法和阈值照参考项目，只改说法和字段名，阈值注明未在新项目材料上校准，等真实数据。 新增 npm run test:jev（真实 Jev 三处各问一次）。vocab 加 JevJudgment。 ledger 表结构版本 3：callEnded 四种结局对上 agents，墙钟时间用 Measured（恢复时补写出错、原因"进程中断"、测不到）；退回、提交记录都带 jev。 config 加必填的 models.jev。 用户原话："这次就给提交记录加一个和退回记录同样的 Jev 判断字段"；"墙钟时间改用 Measured，恢复时写出错加测不到（原因'进程中断'）"。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
233 lines
9.9 KiB
TypeScript
233 lines
9.9 KiB
TypeScript
import assert from "node:assert/strict";
|
||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||
import { tmpdir } from "node:os";
|
||
import { join } from "node:path";
|
||
import { test } from "node:test";
|
||
import { checkConfig, loadConfig, ResearchConfig } from "../src/config/index.ts";
|
||
import { fullConfig, naiveConfig } from "./config-helpers.ts";
|
||
|
||
type Obj = Record<string, unknown>;
|
||
|
||
/** 配置里每个对象字段的路径（JSON 指针），数组里的对象也走进去。 */
|
||
function fieldPaths(value: unknown, at = ""): string[] {
|
||
if (Array.isArray(value)) return value.flatMap((item, i) => fieldPaths(item, `${at}/${i}`));
|
||
if (typeof value !== "object" || value === null) return [];
|
||
return Object.entries(value).flatMap(([k, v]) => [`${at}/${k}`, ...fieldPaths(v, `${at}/${k}`)]);
|
||
}
|
||
|
||
function parentAndKey(root: unknown, path: string): [Obj, string] {
|
||
const keys = path.split("/").slice(1);
|
||
let node = root as Obj;
|
||
for (const k of keys.slice(0, -1)) node = node[k] as Obj;
|
||
return [node, keys[keys.length - 1]!];
|
||
}
|
||
|
||
/** 校验应失败，返回错误消息。 */
|
||
function errorOf(value: unknown): string {
|
||
try {
|
||
checkConfig(value);
|
||
} catch (error) {
|
||
return (error as Error).message;
|
||
}
|
||
assert.fail("应当报错，却通过了校验");
|
||
}
|
||
|
||
test("两份写全的合法配置能通过，返回的就是原内容", () => {
|
||
for (const config of [fullConfig(), naiveConfig()]) {
|
||
assert.deepEqual(checkConfig(structuredClone(config)), config);
|
||
}
|
||
});
|
||
|
||
test("逐个删掉每个字段都报错，报错里有这个字段的路径", () => {
|
||
for (const make of [fullConfig, naiveConfig]) {
|
||
const paths = fieldPaths(make());
|
||
assert.ok(paths.length > 10);
|
||
for (const path of paths) {
|
||
const config = make();
|
||
const [parent, key] = parentAndKey(config, path);
|
||
delete parent[key];
|
||
const message = errorOf(config);
|
||
assert.ok(message.includes(`${path}：`), `删掉 ${path}，报错里应有这个路径：\n${message}`);
|
||
}
|
||
}
|
||
});
|
||
|
||
test("不认识的字段报错：顶层、模型里、两种流程里", () => {
|
||
const top = { ...fullConfig(), seed: "s" };
|
||
assert.match(errorOf(top), /^\/seed：不认识的字段$/m);
|
||
|
||
const model = fullConfig();
|
||
(model.models.researchers[1] as Obj).temperature = 0.2;
|
||
assert.match(errorOf(model), /^\/models\/researchers\/1\/temperature：不认识的字段$/m);
|
||
|
||
const full = fullConfig();
|
||
(full.flow as Obj).researcherNotes = false;
|
||
assert.match(errorOf(full), /^\/flow\/researcherNotes：不认识的字段$/m);
|
||
});
|
||
|
||
test("类型或取值不对报错，带字段路径", () => {
|
||
const cases: Array<[(c: Obj) => void, RegExp]> = [
|
||
[(c) => (c.runs = "13"), /^\/runs：应为整数$/m],
|
||
[(c) => (c.runs = 0), /^\/runs：不能小于 1$/m],
|
||
[(c) => (c.batchLimit = 1.5), /^\/batchLimit：应为整数$/m],
|
||
[(c) => (c.studyId = "../x"), /^\/studyId：格式不对/m],
|
||
[(c) => (c.task = " "), /^\/task：不能为空，也不能只有空白$/m],
|
||
[(c) => (c.devEntry = "false"), /^\/devEntry：应为true 或 false$/m],
|
||
[(c) => ((c.models as Obj).researchers = []), /^\/models\/researchers：至少要有 1 项$/m],
|
||
[(c) => (((c.models as Obj).verifier as Obj).thinking = null), /^\/models\/verifier\/thinking：只能是 "off"、"minimal"、"low"、"medium"、"high"、"xhigh"、"max" 之一$/m],
|
||
[(c) => (((c.models as Obj).verifier as Obj).thinking = "auto"), /^\/models\/verifier\/thinking：只能是/m],
|
||
[(c) => ((c.flow as Obj).openReserve = 0), /^\/flow\/openReserve：必须大于 0$/m],
|
||
[(c) => ((c.flow as Obj).decay = 1.5), /^\/flow\/decay：不能大于 1$/m],
|
||
[(c) => ((c.flow as Obj).roundZero = { enabled: true, discussions: 3 }), /^\/flow\/roundZero\/discussions：只能是 1、2 之一$/m],
|
||
[(c) => ((c.flow as Obj).preset = "fast"), /^\/flow\/preset：缺少或取值不对，只能是 "full"、"naive" 之一$/m],
|
||
[(c) => ((c.lightChecks as Obj).seconds = 0), /^\/lightChecks\/seconds：必须大于 0$/m],
|
||
];
|
||
for (const [change, expected] of cases) {
|
||
const config = fullConfig() as unknown as Obj;
|
||
change(config);
|
||
const message = errorOf(config);
|
||
assert.match(message, expected);
|
||
assert.match(message, /共 1 处/, `只改了一处，应只报一处：\n${message}`);
|
||
}
|
||
});
|
||
|
||
test("Jev 模型必填：少了、写 null、写空白都报错，不补 jev-latest", () => {
|
||
const missing = fullConfig() as unknown as Obj;
|
||
delete (missing.models as Obj).jev;
|
||
assert.match(errorOf(missing), /^\/models\/jev：缺少这一项/m);
|
||
const nulled = fullConfig() as unknown as Obj;
|
||
(nulled.models as Obj).jev = null;
|
||
assert.match(errorOf(nulled), /^\/models\/jev：应为字符串$/m);
|
||
const blank = fullConfig() as unknown as Obj;
|
||
(blank.models as Obj).jev = " ";
|
||
assert.match(errorOf(blank), /^\/models\/jev：不能为空，也不能只有空白$/m);
|
||
const naive = naiveConfig() as unknown as Obj;
|
||
delete (naive.models as Obj).jev;
|
||
assert.match(errorOf(naive), /^\/models\/jev：缺少这一项/m);
|
||
});
|
||
|
||
test("一次列出多处错误，超过 typebox 默认的 8 条、错在 Union 分支里的也都列出", () => {
|
||
const config = fullConfig() as unknown as Obj;
|
||
config.runs = -1;
|
||
delete config.promptDir;
|
||
config.extra = 1;
|
||
((config.models as Obj).experimenter as Obj).model = "";
|
||
((config.models as Obj).researchers as Obj[])[2]!.thinking = null;
|
||
config.devEntry = 0;
|
||
config.factorLimit = 0;
|
||
(config.lightChecks as Obj).maxCount = -1;
|
||
const flow = config.flow as Obj;
|
||
flow.batchRule = "best";
|
||
delete (flow.roundZero as Obj).enabled;
|
||
(flow.ablation as Obj).cellRule = "x";
|
||
flow.decay = -1;
|
||
const message = errorOf(config);
|
||
const paths = [
|
||
"/runs：",
|
||
"/promptDir：",
|
||
"/extra：",
|
||
"/models/experimenter/model：",
|
||
"/models/researchers/2/thinking：",
|
||
"/devEntry：",
|
||
"/factorLimit：",
|
||
"/lightChecks/maxCount：",
|
||
"/flow/batchRule：",
|
||
"/flow/roundZero/enabled：",
|
||
"/flow/ablation/cellRule：",
|
||
"/flow/decay：",
|
||
];
|
||
assert.match(message, new RegExp(`共 ${paths.length} 处`), message);
|
||
for (const path of paths) assert.ok(message.includes(path), `应报出 ${path}：\n${message}`);
|
||
});
|
||
|
||
test("流程写成了不是对象的值：报在流程这一项上", () => {
|
||
const config = fullConfig() as unknown as Obj;
|
||
config.flow = "full";
|
||
assert.match(errorOf(config), /^\/flow：应为对象$/m);
|
||
});
|
||
|
||
test("写不出来的组合：朴素循环里写开关、规则或它用不到的方法参数", () => {
|
||
const extras: Obj = {
|
||
critique: false,
|
||
firstPrediction: false,
|
||
roundZero: { enabled: false },
|
||
ablation: { enabled: false },
|
||
batchRule: "uniformRandom",
|
||
openReserve: 0.1,
|
||
knowledgeUse: 0.5,
|
||
decay: 0.5,
|
||
};
|
||
for (const [key, value] of Object.entries(extras)) {
|
||
const config = naiveConfig() as unknown as Obj;
|
||
(config.flow as Obj)[key] = value;
|
||
assert.match(errorOf(config), new RegExp(`^/flow/${key}：不认识的字段$`, "m"));
|
||
}
|
||
});
|
||
|
||
test("写不出来的组合：关掉的第零轮写讨论次数、关掉的消融写选格规则", () => {
|
||
const zero = fullConfig();
|
||
(zero.flow as Obj).roundZero = { enabled: false, discussions: 2 };
|
||
assert.match(errorOf(zero), /^\/flow\/roundZero\/discussions：不认识的字段$/m);
|
||
|
||
const ablation = fullConfig();
|
||
(ablation.flow as Obj).ablation = { enabled: false, cellRule: "all" };
|
||
assert.match(errorOf(ablation), /^\/flow\/ablation\/cellRule：不认识的字段$/m);
|
||
});
|
||
|
||
test("写不出来的组合：规则名不在 rules 登记表里", () => {
|
||
const batch = fullConfig();
|
||
(batch.flow as Obj).batchRule = "expected-best";
|
||
assert.match(errorOf(batch), /^\/flow\/batchRule：只能是 "bestExpected"、"uniformRandom" 之一$/m);
|
||
|
||
const cell = fullConfig();
|
||
(cell.flow as Obj).ablation = { enabled: true, cellRule: "voi" };
|
||
assert.match(
|
||
errorOf(cell),
|
||
/^\/flow\/ablation\/cellRule：只能是 "valueMinusCost"、"all"、"randomSameCount"、"none" 之一$/m,
|
||
);
|
||
});
|
||
|
||
test("字段定义里每个字段都有中文说明，并标明用户配置还是方法参数", () => {
|
||
const seen: string[] = [];
|
||
const visit = (schema: Obj, at: string): void => {
|
||
const props = schema.properties as Record<string, Obj> | undefined;
|
||
for (const [key, field] of Object.entries(props ?? {})) {
|
||
assert.equal(typeof field.description, "string", `${at}/${key} 缺说明`);
|
||
assert.ok(["用户配置", "方法参数"].includes(field.category as string), `${at}/${key} 没标类别`);
|
||
seen.push(`${at}/${key}`);
|
||
visit(field, `${at}/${key}`);
|
||
}
|
||
for (const branch of (schema.anyOf as Obj[] | undefined) ?? []) visit(branch, at);
|
||
if (schema.items) visit(schema.items as Obj, `${at}/0`);
|
||
};
|
||
visit(ResearchConfig as unknown as Obj, "");
|
||
// 抽查几项的类别。
|
||
const category = (path: string[]) => {
|
||
let node = ResearchConfig as unknown as Obj;
|
||
for (const k of path) node = (node.properties as Record<string, Obj>)[k]!;
|
||
return node.category;
|
||
};
|
||
assert.equal(category(["studyId"]), "用户配置");
|
||
assert.equal(category(["runs"]), "用户配置");
|
||
assert.equal(category(["factorLimit"]), "方法参数");
|
||
assert.equal(category(["flow"]), "方法参数");
|
||
assert.equal(category(["models", "jev"]), "用户配置");
|
||
assert.ok(seen.includes("/models/researchers/0/thinking"));
|
||
assert.ok(seen.includes("/flow/ablation/cellRule"));
|
||
});
|
||
|
||
test("loadConfig：读文件并校验；不是 JSON 时报错带文件路径", () => {
|
||
const dir = mkdtempSync(join(tmpdir(), "inforsi-config-"));
|
||
const good = join(dir, "research.json");
|
||
writeFileSync(good, JSON.stringify(fullConfig(), null, 2));
|
||
assert.deepEqual(loadConfig(good), fullConfig());
|
||
|
||
const bad = join(dir, "bad.json");
|
||
writeFileSync(bad, "{ runs: 13 }");
|
||
assert.throws(() => loadConfig(bad), (e: Error) => e.message.startsWith(`${bad} 不是合法的 JSON`));
|
||
|
||
const invalid = join(dir, "invalid.json");
|
||
writeFileSync(invalid, JSON.stringify({ ...fullConfig(), runs: 0 }));
|
||
assert.throws(() => loadConfig(invalid), /\/runs：不能小于 1/);
|
||
});
|