feat: 研究编号只许小写、配置路径必须绝对;检验判重两个顺序取平均
任务书 13 第一部分,用户 2026-10-03 同意。 - config:studyId 限定小写字母、数字和 ._-;task、promptDir 必须以 / 开头。 - eval:research.json 路径改绝对,writeConfig 不再转换;quality.ts 第 2、6 问每对 A、B 与 B、A 各问一次取平均,两次原始概率都存。 - tests:大写编号、相对路径各加测试。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -449,6 +449,8 @@ Jev 是一个判别器,和程序检查地位相同,不单独成一种记录
|
||||
**字段定义**
|
||||
|
||||
- `ResearchConfig`(字段定义):research.json 的全部字段,全部必填,不认识的字段报错。每个字段带中文说明(`description`)和类别(`category`:用户配置或方法参数),配置对话照着它问,不另写一份。`models` 里除了各角色的模型,还有 Jev 用的模型名(`models.jev`,用户配置);Jev 的密钥不进配置,从环境变量读。流程是两选一:完整流程写四个开关、选批规则、它才用得到的 λ、u、衰减;朴素循环只写 `preset`,别的都写不进去。
|
||||
- 研究编号 `studyId` 只许小写字母、数字和 `._-`,首字符是小写字母或数字:它会进 Docker 镜像名,镜像名只认小写。(来源:用户 2026-10-03 同意,任务书 13)
|
||||
- `task`、`promptDir` 必须是绝对路径:从别的目录接着跑一份研究时,相对路径会读到别处,而任务目录没有哈希核对挡着。(来源:用户 2026-10-03 同意,任务书 13)
|
||||
- `FIELD_CATEGORIES`、`FieldCategory`:字段的两种类别,用户配置、方法参数。
|
||||
- `THINKING_LEVELS`、`ThinkingLevel`:Pi 0.85.1 接受的七个思考档位,出处写在代码注释里;config 不能引用 Pi,所以照抄一份。
|
||||
|
||||
|
||||
+2
-7
@@ -1,6 +1,6 @@
|
||||
import { execFileSync, spawn, type ChildProcess } from "node:child_process";
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { join } from "node:path";
|
||||
|
||||
// 检验脚本共用的起研究工具:检验目录放在仓库外,配置照用户的写法写一份,研究在子进程里跑(要能被 SIGKILL)。
|
||||
|
||||
@@ -15,15 +15,10 @@ export function newEvalDir(name: string): string {
|
||||
return dir;
|
||||
}
|
||||
|
||||
/**
|
||||
* 照一份底稿写这次检验用的 research.json:换研究编号(容器名、镜像标签都带它),需要时换各角色模型;
|
||||
* 任务目录、提示词目录写成绝对路径(底稿里相对仓库根目录),这样配置文件放在仓库外也指得对。
|
||||
*/
|
||||
/** 照一份底稿写这次检验用的 research.json:换研究编号(容器名、镜像标签都带它),需要时换各角色模型。 */
|
||||
export function writeConfig(base: string, path: string, studyId: string, models: unknown): string {
|
||||
const config = JSON.parse(readFileSync(base, "utf8")) as Record<string, unknown>;
|
||||
config["studyId"] = studyId;
|
||||
config["task"] = resolve(config["task"] as string);
|
||||
config["promptDir"] = resolve(config["promptDir"] as string);
|
||||
if (models !== null) config["models"] = models;
|
||||
writeFileSync(path, `${JSON.stringify(config, null, 2)}\n`);
|
||||
return path;
|
||||
|
||||
+28
-13
@@ -1,6 +1,6 @@
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { ask, THRESHOLD_NOTE, type Answer, type Noul } from "../lib/jev.ts";
|
||||
import { ask, THRESHOLD, THRESHOLD_NOTE, type Answer, type Noul } from "../lib/jev.ts";
|
||||
import { entriesIn, entryDecision, readLedger, stages, type Entry } from "../lib/ledger.ts";
|
||||
import { ENTRY_ONLY, LEADS_TO_IDEA, SAME_KIND, STOPS_AT_DEAD_END, TASKS } from "./questions.ts";
|
||||
|
||||
@@ -56,14 +56,29 @@ function query(site: string, ids: string[], state: Record<string, unknown>, ques
|
||||
}
|
||||
return cache.get(key)!;
|
||||
}
|
||||
const yes = async (p: Promise<Answer[]>, question: string) => (await p).find((a) => a.question === question)!.yes;
|
||||
const answer = async (p: Promise<Answer[]>, question: string) => (await p).find((a) => a.question === question)!;
|
||||
const yes = async (p: Promise<Answer[]>, question: string) => (await answer(p, question)).yes;
|
||||
|
||||
const ideaState = (e: Entry, idea: Idea) => ({ task: TASKS.idea, entry: show(e), idea: { description: idea.description } });
|
||||
const leadsTo = (e: Entry, idea: Idea) => yes(query("leads", [e.id, idea.id], ideaState(e, idea), LEADS_TO_IDEA), "leads_to_idea");
|
||||
const staysIn = (e: Entry, idea: Idea) => yes(query("stays", [e.id, idea.id], ideaState(e, idea), STOPS_AT_DEAD_END), "stays_within_idea");
|
||||
const entryOnly = (e: Entry, question: string) => yes(query("entry", [e.id], { task: TASKS.entry, entry: show(e) }, ENTRY_ONLY), question);
|
||||
const sameKind = (a: Entry, b: Entry) =>
|
||||
yes(query("pair", [a.id, b.id], { task: TASKS.pair, entry_a: show(a), entry_b: show(b) }, SAME_KIND), "same_kind_of_change");
|
||||
const pairProbability = async (a: Entry, b: Entry) =>
|
||||
(await answer(query("pair", [a.id, b.id], { task: TASKS.pair, entry_a: show(a), entry_b: show(b) }, SAME_KIND), "same_kind_of_change"))
|
||||
.probability;
|
||||
|
||||
/** 一对入口是不是同一条路:问题本身对称,所以 A、B 和 B、A 各问一次,取两次概率的平均再比阈值(任务书 13 第一部分第 3 条)。 */
|
||||
type PairJudgement = { pair: [string, string]; ab: number; ba: number; mean: number; same: boolean };
|
||||
async function sameKind(a: Entry, b: Entry): Promise<PairJudgement> {
|
||||
const [ab, ba] = await Promise.all([pairProbability(a, b), pairProbability(b, a)]);
|
||||
const mean = (ab + ba) / 2;
|
||||
return { pair: [a.id, b.id], ab, ba, mean, same: mean > THRESHOLD };
|
||||
}
|
||||
type PairsResult = { pairs: PairJudgement[]; same: string[][] };
|
||||
async function judgePairs(pairs: Array<readonly [Entry, Entry]>): Promise<PairsResult> {
|
||||
const judged = await Promise.all(pairs.map(([a, b]) => sameKind(a, b)));
|
||||
return { pairs: judged, same: judged.filter((j) => j.same).map((j) => j.pair) };
|
||||
}
|
||||
|
||||
async function coverage(list: Entry[]): Promise<Record<string, string[]>> {
|
||||
const out: Record<string, string[]> = {};
|
||||
@@ -73,11 +88,10 @@ async function coverage(list: Entry[]): Promise<Record<string, string[]>> {
|
||||
}
|
||||
return out;
|
||||
}
|
||||
async function overlap(list: Entry[]): Promise<{ pairs: number; same: string[][]; ratio: number | null }> {
|
||||
const pairs = list.flatMap((a, i) => list.slice(i + 1).filter((b) => b.author !== a.author).map((b) => [a, b] as const));
|
||||
const hits = await Promise.all(pairs.map(([a, b]) => sameKind(a, b)));
|
||||
const same = pairs.filter((_, i) => hits[i]).map(([a, b]) => [a.id, b.id]);
|
||||
return { pairs: pairs.length, same, ratio: pairs.length === 0 ? null : same.length / pairs.length };
|
||||
/** 重合度:不同作者的入口两两成对,判"是"的对数占总对数的比例。 */
|
||||
async function overlap(list: Entry[]): Promise<PairsResult & { ratio: number | null }> {
|
||||
const judged = await judgePairs(list.flatMap((a, i) => list.slice(i + 1).filter((b) => b.author !== a.author).map((b) => [a, b] as const)));
|
||||
return { ...judged, ratio: judged.pairs.length === 0 ? null : judged.same.length / judged.pairs.length };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -89,8 +103,8 @@ const covFinal = await coverage(final);
|
||||
const q1 = { coverage: covFinal, pass: covFinal["K1"]!.length > 0 };
|
||||
|
||||
const assignedPairs = assigned.flatMap((a, i) => assigned.slice(i + 1).map((b) => [a, b] as const));
|
||||
const pairHits = await Promise.all(assignedPairs.map(([a, b]) => sameKind(a, b)));
|
||||
const q2 = { same: assignedPairs.filter((_, i) => pairHits[i]).map(([a, b]) => [a.id, b.id]), pass: !pairHits.some(Boolean) };
|
||||
const assignedJudged = await judgePairs(assignedPairs);
|
||||
const q2 = { ...assignedJudged, pass: assignedJudged.same.length === 0 };
|
||||
|
||||
const assignedK1 = await Promise.all(assigned.map((e) => leadsTo(e, k1)));
|
||||
const deadEnd = await Promise.all(
|
||||
@@ -134,13 +148,14 @@ const report = {
|
||||
writeFileSync(join(out, "quality.json"), `${JSON.stringify(report, null, 2)}\n`);
|
||||
|
||||
const mark = (b: boolean) => (b ? "通过" : "不通过");
|
||||
const ratio = (o: { pairs: number; same: string[][]; ratio: number | null }) => `${o.same.length}/${o.pairs}${o.ratio === null ? "" : `(${(o.ratio * 100).toFixed(0)}%)`}`;
|
||||
const ratio = (o: PairsResult & { ratio: number | null }) => `${o.same.length}/${o.pairs.length}${o.ratio === null ? "" : `(${(o.ratio * 100).toFixed(0)}%)`}`;
|
||||
const count = (c: Record<string, string[]>) => Object.entries(c).map(([k, v]) => `${k}:${v.length}`).join(" ");
|
||||
console.log(`(${THRESHOLD_NOTE})`);
|
||||
console.log(`入口全体:${final.map((e) => `${e.id} ${e.name}`).join("\n ")}`);
|
||||
console.log(`分到的入口:${report.assignment.map((a) => `${a.researcher} → ${a.entry}(第 ${a.rank} 志愿)${a.name}`).join("\n ")}`);
|
||||
console.log(`1 方向对不对:${mark(q1.pass)};覆盖 ${count(covFinal)}`);
|
||||
console.log(`2 分没分开:${mark(q2.pass)};判"是"的对 ${JSON.stringify(q2.same)}`);
|
||||
const pairLine = (j: PairJudgement) => `${j.pair.join("、")}:${j.ab.toFixed(2)}/${j.ba.toFixed(2)},平均 ${j.mean.toFixed(2)}`;
|
||||
console.log(`2 分没分开:${mark(q2.pass)};判"是"的对 ${JSON.stringify(q2.same)};每对两个顺序的概率\n ${q2.pairs.map(pairLine).join("\n ")}`);
|
||||
console.log(`3 好方向分出去没有:${mark(q3.pass)};指向 K1 的 ${JSON.stringify(q3.k1)},停在死路的 ${JSON.stringify(q3.deadEnds)}`);
|
||||
console.log(`4 改代码的路线:${mark(q4.pass)};不是的 ${JSON.stringify(q4.notCodeRoute)}`);
|
||||
console.log(`5 有没有写答案:${mark(q5pass)};分到的里写了的 ${JSON.stringify(q5.assignedWritingAnswer)},入口全体里写了的 ${JSON.stringify(q5.allWritingAnswer)}`);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"studyId": "r0-eval",
|
||||
"task": "tasks/circle_packing",
|
||||
"task": "/home/iomgaa/Projects/InfoRSI/tasks/circle_packing",
|
||||
"models": {
|
||||
"researchers": [
|
||||
{ "provider": "iomgaa", "model": "glm-5.3", "thinking": "medium" },
|
||||
@@ -17,7 +17,7 @@
|
||||
"factorLimit": 3,
|
||||
"maxRepairs": 1,
|
||||
"lightChecks": { "maxCount": 2, "seconds": 5 },
|
||||
"promptDir": "prompts",
|
||||
"promptDir": "/home/iomgaa/Projects/InfoRSI/prompts",
|
||||
"devEntry": false,
|
||||
"flow": {
|
||||
"preset": "full",
|
||||
|
||||
+7
-1
@@ -19,6 +19,12 @@ const TYPE_TEXT: Readonly<Record<string, string>> = {
|
||||
null: "null",
|
||||
};
|
||||
|
||||
/** schema.ts 里文字字段、路径字段的格式,换成人读得懂的话。 */
|
||||
const PATTERN_TEXT: Readonly<Record<string, string>> = {
|
||||
"\\S": "不能为空,也不能只有空白",
|
||||
"^/": "必须是绝对路径(以 / 开头)",
|
||||
};
|
||||
|
||||
function reasonOf(e: SchemaError): string {
|
||||
const p = e.params as Record<string, unknown>;
|
||||
switch (e.keyword) {
|
||||
@@ -39,7 +45,7 @@ function reasonOf(e: SchemaError): string {
|
||||
case "minItems":
|
||||
return `至少要有 ${String(p.limit)} 项`;
|
||||
case "pattern":
|
||||
return p.pattern === "\\S" ? "不能为空,也不能只有空白" : `格式不对,应符合 ${String(p.pattern)}`;
|
||||
return PATTERN_TEXT[String(p.pattern)] ?? `格式不对,应符合 ${String(p.pattern)}`;
|
||||
default:
|
||||
return e.message;
|
||||
}
|
||||
|
||||
+17
-4
@@ -1,6 +1,5 @@
|
||||
import { Type, type Static, type TSchema } from "typebox";
|
||||
import { BATCH_RULES, CELL_RULES, type BatchRuleName, type CellRuleName } from "../rules/index.ts";
|
||||
import { Id } from "../vocab/index.ts";
|
||||
|
||||
// research.json 的字段定义(docs/结构.md 第二部分、第五部分"方法参数""步骤开关")。
|
||||
// 每个字段都必填,不写默认值;不认识的字段报错。可以没有的值写 null(目前没有这样的字段)。
|
||||
@@ -15,6 +14,11 @@ export type FieldCategory = (typeof FIELD_CATEGORIES)[number];
|
||||
const CLOSED = { additionalProperties: false } as const;
|
||||
/** 文字字段:不能为空,也不能只有空白。 */
|
||||
const TEXT = { pattern: "\\S" };
|
||||
/**
|
||||
* 路径字段必须是绝对路径:从别的目录接着跑一份研究时,相对路径会读到别处,而任务目录没有哈希核对挡着
|
||||
* (任务书 13 第一部分第 2 条)。
|
||||
*/
|
||||
const ABSOLUTE_PATH = { pattern: "^/" };
|
||||
|
||||
function user<T extends TSchema>(schema: T, description: string): T {
|
||||
return { ...schema, description, category: "用户配置" };
|
||||
@@ -127,8 +131,14 @@ const PositiveInteger = Type.Integer({ minimum: 1 });
|
||||
|
||||
export const ResearchConfig = Type.Object(
|
||||
{
|
||||
studyId: user(Id, "研究编号:决定的种子由它和轮次、版本算出;只许字母、数字和 ._-,首字符是字母或数字"),
|
||||
task: user(Type.String(TEXT), "用哪个任务:任务目录的路径。指标、方向、ε、时限等任务事实写在任务目录里"),
|
||||
studyId: user(
|
||||
Type.String({ pattern: "^[a-z0-9][a-z0-9._-]{0,127}$" }),
|
||||
"研究编号:决定的种子由它和轮次、版本算出,Docker 镜像名也带它,所以只许小写字母、数字和 ._-,首字符是小写字母或数字",
|
||||
),
|
||||
task: user(
|
||||
Type.String(ABSOLUTE_PATH),
|
||||
"用哪个任务:任务目录的绝对路径。指标、方向、ε、时限等任务事实写在任务目录里",
|
||||
),
|
||||
models: user(
|
||||
Type.Object(
|
||||
{
|
||||
@@ -172,7 +182,10 @@ export const ResearchConfig = Type.Object(
|
||||
),
|
||||
"轻量检查的次数与时长上限:只用来检查语法、接口和极小的样例,不能借它跑正式规模的求解",
|
||||
),
|
||||
promptDir: method(Type.String(TEXT), "提示词目录:各角色的角色提示词和各类任务消息放在这里"),
|
||||
promptDir: method(
|
||||
Type.String(ABSOLUTE_PATH),
|
||||
"提示词目录的绝对路径:各角色的角色提示词和各类任务消息放在这里",
|
||||
),
|
||||
devEntry: method(
|
||||
Type.Boolean(),
|
||||
"开发入口(方法.md 第九部分):正式研究写 false;开发调试时才写 true,打开过的研究不能当验收证据",
|
||||
|
||||
@@ -71,7 +71,7 @@ test("类型或取值不对报错,带字段路径", () => {
|
||||
[(c) => (c.runs = 0), /^\/runs:不能小于 1$/m],
|
||||
[(c) => (c.batchLimit = 1.5), /^\/batchLimit:应为整数$/m],
|
||||
[(c) => (c.studyId = "../x"), /^\/studyId:格式不对/m],
|
||||
[(c) => (c.task = " "), /^\/task:不能为空,也不能只有空白$/m],
|
||||
[(c) => (c.task = " "), /^\/task:必须是绝对路径(以 \/ 开头)$/m],
|
||||
[(c) => (c.devEntry = "false"), /^\/devEntry:应为true 或 false$/m],
|
||||
[(c) => ((c.models as Obj).researchers = []), /^\/models\/researchers:至少要有 1 项$/m],
|
||||
[(c) => (((c.models as Obj).verifier as Obj).thinking = null), /^\/models\/verifier\/thinking:只能是 "off"、"minimal"、"low"、"medium"、"high"、"xhigh"、"max" 之一$/m],
|
||||
@@ -91,6 +91,32 @@ test("类型或取值不对报错,带字段路径", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("研究编号带大写报错:它会进 Docker 镜像名,镜像名只认小写", () => {
|
||||
for (const id of ["Circle-1", "circle-A", "R0"]) {
|
||||
const config = fullConfig() as unknown as Obj;
|
||||
config.studyId = id;
|
||||
const message = errorOf(config);
|
||||
assert.match(message, /^\/studyId:格式不对/m);
|
||||
assert.match(message, /共 1 处/, message);
|
||||
}
|
||||
});
|
||||
|
||||
test("任务目录、提示词目录写相对路径报错,报错写明字段路径", () => {
|
||||
const cases: Array<[string, string]> = [
|
||||
["task", "tasks/circle_packing"],
|
||||
["task", "./tasks/circle_packing"],
|
||||
["promptDir", "prompts"],
|
||||
["promptDir", "../InfoRSI/prompts"],
|
||||
];
|
||||
for (const [field, path] of cases) {
|
||||
const config = fullConfig() as unknown as Obj;
|
||||
config[field] = path;
|
||||
const message = errorOf(config);
|
||||
assert.match(message, new RegExp(`^/${field}:必须是绝对路径(以 / 开头)$`, "m"));
|
||||
assert.match(message, /共 1 处/, message);
|
||||
}
|
||||
});
|
||||
|
||||
test("Jev 模型必填:少了、写 null、写空白都报错,不补 jev-latest", () => {
|
||||
const missing = fullConfig() as unknown as Obj;
|
||||
delete (missing.models as Obj).jev;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { join } from "node:path";
|
||||
import type { ResearchConfig } from "../src/config/index.ts";
|
||||
|
||||
// 两份写全的合法配置:完整流程一份、朴素循环一份。数值取方法.md 6.11 的现行值和参考项目真实用过的配置
|
||||
@@ -8,7 +9,7 @@ const glm = { provider: "iomgaa", model: "glm-5.3", thinking: "medium" } as cons
|
||||
export function fullConfig(): ResearchConfig {
|
||||
return {
|
||||
studyId: "circle-full-1",
|
||||
task: "tasks/circle_packing",
|
||||
task: join(import.meta.dirname, "..", "tasks", "circle_packing"),
|
||||
models: {
|
||||
researchers: [{ ...glm }, { ...glm }, { ...glm }],
|
||||
experimenter: { ...glm },
|
||||
@@ -21,7 +22,7 @@ export function fullConfig(): ResearchConfig {
|
||||
factorLimit: 3,
|
||||
maxRepairs: 1,
|
||||
lightChecks: { maxCount: 2, seconds: 5 },
|
||||
promptDir: "prompts",
|
||||
promptDir: join(import.meta.dirname, "..", "prompts"),
|
||||
devEntry: false,
|
||||
flow: {
|
||||
preset: "full",
|
||||
|
||||
Reference in New Issue
Block a user