diff --git a/docs/结构.md b/docs/结构.md index 42a4571..7b6b1a8 100644 --- a/docs/结构.md +++ b/docs/结构.md @@ -449,6 +449,8 @@ Jev 是一个判别器,和程序检查地位相同,不单独成一种记录 **字段定义** - `ResearchConfig`(字段定义):research.json 的全部字段,全部必填,不认识的字段报错。每个字段带中文说明(`description`)和类别(`category`:用户配置或方法参数),配置对话照着它问,不另写一份。`models` 里除了各角色的模型,还有 Jev 用的模型名(`models.jev`,用户配置);Jev 的密钥不进配置,从环境变量读。流程是两选一:完整流程写四个开关、选批规则、它才用得到的 λ、u、衰减;朴素循环只写 `preset`,别的都写不进去。 + - 研究编号 `studyId` 只许小写字母、数字和 `._-`,首字符是小写字母或数字:它会进 Docker 镜像名,镜像名只认小写。(来源:用户 2026-10-03 同意,任务书 13) + - `task`、`promptDir` 必须是绝对路径:从别的目录接着跑一份研究时,相对路径会读到别处,而任务目录没有哈希核对挡着。(来源:用户 2026-10-03 同意,任务书 13) - `FIELD_CATEGORIES`、`FieldCategory`:字段的两种类别,用户配置、方法参数。 - `THINKING_LEVELS`、`ThinkingLevel`:Pi 0.85.1 接受的七个思考档位,出处写在代码注释里;config 不能引用 Pi,所以照抄一份。 diff --git a/eval/lib/study.ts b/eval/lib/study.ts index e49c86c..d3b3d1f 100644 --- a/eval/lib/study.ts +++ b/eval/lib/study.ts @@ -1,6 +1,6 @@ import { execFileSync, spawn, type ChildProcess } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { join, resolve } from "node:path"; +import { join } from "node:path"; // 检验脚本共用的起研究工具:检验目录放在仓库外,配置照用户的写法写一份,研究在子进程里跑(要能被 SIGKILL)。 @@ -15,15 +15,10 @@ export function newEvalDir(name: string): string { return dir; } -/** - * 照一份底稿写这次检验用的 research.json:换研究编号(容器名、镜像标签都带它),需要时换各角色模型; - * 任务目录、提示词目录写成绝对路径(底稿里相对仓库根目录),这样配置文件放在仓库外也指得对。 - */ +/** 照一份底稿写这次检验用的 research.json:换研究编号(容器名、镜像标签都带它),需要时换各角色模型。 */ export function writeConfig(base: string, path: string, studyId: string, models: unknown): string { const config = JSON.parse(readFileSync(base, "utf8")) as Record; config["studyId"] = studyId; - config["task"] = resolve(config["task"] as string); - config["promptDir"] = resolve(config["promptDir"] as string); if (models !== null) config["models"] = models; writeFileSync(path, `${JSON.stringify(config, null, 2)}\n`); return path; diff --git a/eval/round-zero/quality.ts b/eval/round-zero/quality.ts index 7a481a4..3c6e139 100644 --- a/eval/round-zero/quality.ts +++ b/eval/round-zero/quality.ts @@ -1,6 +1,6 @@ import { readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; -import { ask, THRESHOLD_NOTE, type Answer, type Noul } from "../lib/jev.ts"; +import { ask, THRESHOLD, THRESHOLD_NOTE, type Answer, type Noul } from "../lib/jev.ts"; import { entriesIn, entryDecision, readLedger, stages, type Entry } from "../lib/ledger.ts"; import { ENTRY_ONLY, LEADS_TO_IDEA, SAME_KIND, STOPS_AT_DEAD_END, TASKS } from "./questions.ts"; @@ -56,14 +56,29 @@ function query(site: string, ids: string[], state: Record, ques } return cache.get(key)!; } -const yes = async (p: Promise, question: string) => (await p).find((a) => a.question === question)!.yes; +const answer = async (p: Promise, question: string) => (await p).find((a) => a.question === question)!; +const yes = async (p: Promise, question: string) => (await answer(p, question)).yes; const ideaState = (e: Entry, idea: Idea) => ({ task: TASKS.idea, entry: show(e), idea: { description: idea.description } }); const leadsTo = (e: Entry, idea: Idea) => yes(query("leads", [e.id, idea.id], ideaState(e, idea), LEADS_TO_IDEA), "leads_to_idea"); const staysIn = (e: Entry, idea: Idea) => yes(query("stays", [e.id, idea.id], ideaState(e, idea), STOPS_AT_DEAD_END), "stays_within_idea"); const entryOnly = (e: Entry, question: string) => yes(query("entry", [e.id], { task: TASKS.entry, entry: show(e) }, ENTRY_ONLY), question); -const sameKind = (a: Entry, b: Entry) => - yes(query("pair", [a.id, b.id], { task: TASKS.pair, entry_a: show(a), entry_b: show(b) }, SAME_KIND), "same_kind_of_change"); +const pairProbability = async (a: Entry, b: Entry) => + (await answer(query("pair", [a.id, b.id], { task: TASKS.pair, entry_a: show(a), entry_b: show(b) }, SAME_KIND), "same_kind_of_change")) + .probability; + +/** 一对入口是不是同一条路:问题本身对称,所以 A、B 和 B、A 各问一次,取两次概率的平均再比阈值(任务书 13 第一部分第 3 条)。 */ +type PairJudgement = { pair: [string, string]; ab: number; ba: number; mean: number; same: boolean }; +async function sameKind(a: Entry, b: Entry): Promise { + const [ab, ba] = await Promise.all([pairProbability(a, b), pairProbability(b, a)]); + const mean = (ab + ba) / 2; + return { pair: [a.id, b.id], ab, ba, mean, same: mean > THRESHOLD }; +} +type PairsResult = { pairs: PairJudgement[]; same: string[][] }; +async function judgePairs(pairs: Array): Promise { + const judged = await Promise.all(pairs.map(([a, b]) => sameKind(a, b))); + return { pairs: judged, same: judged.filter((j) => j.same).map((j) => j.pair) }; +} async function coverage(list: Entry[]): Promise> { const out: Record = {}; @@ -73,11 +88,10 @@ async function coverage(list: Entry[]): Promise> { } return out; } -async function overlap(list: Entry[]): Promise<{ pairs: number; same: string[][]; ratio: number | null }> { - const pairs = list.flatMap((a, i) => list.slice(i + 1).filter((b) => b.author !== a.author).map((b) => [a, b] as const)); - const hits = await Promise.all(pairs.map(([a, b]) => sameKind(a, b))); - const same = pairs.filter((_, i) => hits[i]).map(([a, b]) => [a.id, b.id]); - return { pairs: pairs.length, same, ratio: pairs.length === 0 ? null : same.length / pairs.length }; +/** 重合度:不同作者的入口两两成对,判"是"的对数占总对数的比例。 */ +async function overlap(list: Entry[]): Promise { + const judged = await judgePairs(list.flatMap((a, i) => list.slice(i + 1).filter((b) => b.author !== a.author).map((b) => [a, b] as const))); + return { ...judged, ratio: judged.pairs.length === 0 ? null : judged.same.length / judged.pairs.length }; } // --------------------------------------------------------------------------- @@ -89,8 +103,8 @@ const covFinal = await coverage(final); const q1 = { coverage: covFinal, pass: covFinal["K1"]!.length > 0 }; const assignedPairs = assigned.flatMap((a, i) => assigned.slice(i + 1).map((b) => [a, b] as const)); -const pairHits = await Promise.all(assignedPairs.map(([a, b]) => sameKind(a, b))); -const q2 = { same: assignedPairs.filter((_, i) => pairHits[i]).map(([a, b]) => [a.id, b.id]), pass: !pairHits.some(Boolean) }; +const assignedJudged = await judgePairs(assignedPairs); +const q2 = { ...assignedJudged, pass: assignedJudged.same.length === 0 }; const assignedK1 = await Promise.all(assigned.map((e) => leadsTo(e, k1))); const deadEnd = await Promise.all( @@ -134,13 +148,14 @@ const report = { writeFileSync(join(out, "quality.json"), `${JSON.stringify(report, null, 2)}\n`); const mark = (b: boolean) => (b ? "通过" : "不通过"); -const ratio = (o: { pairs: number; same: string[][]; ratio: number | null }) => `${o.same.length}/${o.pairs}${o.ratio === null ? "" : `(${(o.ratio * 100).toFixed(0)}%)`}`; +const ratio = (o: PairsResult & { ratio: number | null }) => `${o.same.length}/${o.pairs.length}${o.ratio === null ? "" : `(${(o.ratio * 100).toFixed(0)}%)`}`; const count = (c: Record) => Object.entries(c).map(([k, v]) => `${k}:${v.length}`).join(" "); console.log(`(${THRESHOLD_NOTE})`); console.log(`入口全体:${final.map((e) => `${e.id} ${e.name}`).join("\n ")}`); console.log(`分到的入口:${report.assignment.map((a) => `${a.researcher} → ${a.entry}(第 ${a.rank} 志愿)${a.name}`).join("\n ")}`); console.log(`1 方向对不对:${mark(q1.pass)};覆盖 ${count(covFinal)}`); -console.log(`2 分没分开:${mark(q2.pass)};判"是"的对 ${JSON.stringify(q2.same)}`); +const pairLine = (j: PairJudgement) => `${j.pair.join("、")}:${j.ab.toFixed(2)}/${j.ba.toFixed(2)},平均 ${j.mean.toFixed(2)}`; +console.log(`2 分没分开:${mark(q2.pass)};判"是"的对 ${JSON.stringify(q2.same)};每对两个顺序的概率\n ${q2.pairs.map(pairLine).join("\n ")}`); console.log(`3 好方向分出去没有:${mark(q3.pass)};指向 K1 的 ${JSON.stringify(q3.k1)},停在死路的 ${JSON.stringify(q3.deadEnds)}`); console.log(`4 改代码的路线:${mark(q4.pass)};不是的 ${JSON.stringify(q4.notCodeRoute)}`); console.log(`5 有没有写答案:${mark(q5pass)};分到的里写了的 ${JSON.stringify(q5.assignedWritingAnswer)},入口全体里写了的 ${JSON.stringify(q5.allWritingAnswer)}`); diff --git a/eval/round-zero/research.json b/eval/round-zero/research.json index 5a1f2aa..7b0881a 100644 --- a/eval/round-zero/research.json +++ b/eval/round-zero/research.json @@ -1,6 +1,6 @@ { "studyId": "r0-eval", - "task": "tasks/circle_packing", + "task": "/home/iomgaa/Projects/InfoRSI/tasks/circle_packing", "models": { "researchers": [ { "provider": "iomgaa", "model": "glm-5.3", "thinking": "medium" }, @@ -17,7 +17,7 @@ "factorLimit": 3, "maxRepairs": 1, "lightChecks": { "maxCount": 2, "seconds": 5 }, - "promptDir": "prompts", + "promptDir": "/home/iomgaa/Projects/InfoRSI/prompts", "devEntry": false, "flow": { "preset": "full", diff --git a/src/config/check.ts b/src/config/check.ts index 224acf7..efed712 100644 --- a/src/config/check.ts +++ b/src/config/check.ts @@ -19,6 +19,12 @@ const TYPE_TEXT: Readonly> = { null: "null", }; +/** schema.ts 里文字字段、路径字段的格式,换成人读得懂的话。 */ +const PATTERN_TEXT: Readonly> = { + "\\S": "不能为空,也不能只有空白", + "^/": "必须是绝对路径(以 / 开头)", +}; + function reasonOf(e: SchemaError): string { const p = e.params as Record; switch (e.keyword) { @@ -39,7 +45,7 @@ function reasonOf(e: SchemaError): string { case "minItems": return `至少要有 ${String(p.limit)} 项`; case "pattern": - return p.pattern === "\\S" ? "不能为空,也不能只有空白" : `格式不对,应符合 ${String(p.pattern)}`; + return PATTERN_TEXT[String(p.pattern)] ?? `格式不对,应符合 ${String(p.pattern)}`; default: return e.message; } diff --git a/src/config/schema.ts b/src/config/schema.ts index ba255d5..e7c65c3 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -1,6 +1,5 @@ import { Type, type Static, type TSchema } from "typebox"; import { BATCH_RULES, CELL_RULES, type BatchRuleName, type CellRuleName } from "../rules/index.ts"; -import { Id } from "../vocab/index.ts"; // research.json 的字段定义(docs/结构.md 第二部分、第五部分"方法参数""步骤开关")。 // 每个字段都必填,不写默认值;不认识的字段报错。可以没有的值写 null(目前没有这样的字段)。 @@ -15,6 +14,11 @@ export type FieldCategory = (typeof FIELD_CATEGORIES)[number]; const CLOSED = { additionalProperties: false } as const; /** 文字字段:不能为空,也不能只有空白。 */ const TEXT = { pattern: "\\S" }; +/** + * 路径字段必须是绝对路径:从别的目录接着跑一份研究时,相对路径会读到别处,而任务目录没有哈希核对挡着 + * (任务书 13 第一部分第 2 条)。 + */ +const ABSOLUTE_PATH = { pattern: "^/" }; function user(schema: T, description: string): T { return { ...schema, description, category: "用户配置" }; @@ -127,8 +131,14 @@ const PositiveInteger = Type.Integer({ minimum: 1 }); export const ResearchConfig = Type.Object( { - studyId: user(Id, "研究编号:决定的种子由它和轮次、版本算出;只许字母、数字和 ._-,首字符是字母或数字"), - task: user(Type.String(TEXT), "用哪个任务:任务目录的路径。指标、方向、ε、时限等任务事实写在任务目录里"), + studyId: user( + Type.String({ pattern: "^[a-z0-9][a-z0-9._-]{0,127}$" }), + "研究编号:决定的种子由它和轮次、版本算出,Docker 镜像名也带它,所以只许小写字母、数字和 ._-,首字符是小写字母或数字", + ), + task: user( + Type.String(ABSOLUTE_PATH), + "用哪个任务:任务目录的绝对路径。指标、方向、ε、时限等任务事实写在任务目录里", + ), models: user( Type.Object( { @@ -172,7 +182,10 @@ export const ResearchConfig = Type.Object( ), "轻量检查的次数与时长上限:只用来检查语法、接口和极小的样例,不能借它跑正式规模的求解", ), - promptDir: method(Type.String(TEXT), "提示词目录:各角色的角色提示词和各类任务消息放在这里"), + promptDir: method( + Type.String(ABSOLUTE_PATH), + "提示词目录的绝对路径:各角色的角色提示词和各类任务消息放在这里", + ), devEntry: method( Type.Boolean(), "开发入口(方法.md 第九部分):正式研究写 false;开发调试时才写 true,打开过的研究不能当验收证据", diff --git a/tests/config-check.test.ts b/tests/config-check.test.ts index e5152f2..1ae7573 100644 --- a/tests/config-check.test.ts +++ b/tests/config-check.test.ts @@ -71,7 +71,7 @@ test("类型或取值不对报错,带字段路径", () => { [(c) => (c.runs = 0), /^\/runs:不能小于 1$/m], [(c) => (c.batchLimit = 1.5), /^\/batchLimit:应为整数$/m], [(c) => (c.studyId = "../x"), /^\/studyId:格式不对/m], - [(c) => (c.task = " "), /^\/task:不能为空,也不能只有空白$/m], + [(c) => (c.task = " "), /^\/task:必须是绝对路径(以 \/ 开头)$/m], [(c) => (c.devEntry = "false"), /^\/devEntry:应为true 或 false$/m], [(c) => ((c.models as Obj).researchers = []), /^\/models\/researchers:至少要有 1 项$/m], [(c) => (((c.models as Obj).verifier as Obj).thinking = null), /^\/models\/verifier\/thinking:只能是 "off"、"minimal"、"low"、"medium"、"high"、"xhigh"、"max" 之一$/m], @@ -91,6 +91,32 @@ test("类型或取值不对报错,带字段路径", () => { } }); +test("研究编号带大写报错:它会进 Docker 镜像名,镜像名只认小写", () => { + for (const id of ["Circle-1", "circle-A", "R0"]) { + const config = fullConfig() as unknown as Obj; + config.studyId = id; + const message = errorOf(config); + assert.match(message, /^\/studyId:格式不对/m); + assert.match(message, /共 1 处/, message); + } +}); + +test("任务目录、提示词目录写相对路径报错,报错写明字段路径", () => { + const cases: Array<[string, string]> = [ + ["task", "tasks/circle_packing"], + ["task", "./tasks/circle_packing"], + ["promptDir", "prompts"], + ["promptDir", "../InfoRSI/prompts"], + ]; + for (const [field, path] of cases) { + const config = fullConfig() as unknown as Obj; + config[field] = path; + const message = errorOf(config); + assert.match(message, new RegExp(`^/${field}:必须是绝对路径(以 / 开头)$`, "m")); + assert.match(message, /共 1 处/, message); + } +}); + test("Jev 模型必填:少了、写 null、写空白都报错,不补 jev-latest", () => { const missing = fullConfig() as unknown as Obj; delete (missing.models as Obj).jev; diff --git a/tests/config-helpers.ts b/tests/config-helpers.ts index 120db57..21eaf13 100644 --- a/tests/config-helpers.ts +++ b/tests/config-helpers.ts @@ -1,3 +1,4 @@ +import { join } from "node:path"; import type { ResearchConfig } from "../src/config/index.ts"; // 两份写全的合法配置:完整流程一份、朴素循环一份。数值取方法.md 6.11 的现行值和参考项目真实用过的配置 @@ -8,7 +9,7 @@ const glm = { provider: "iomgaa", model: "glm-5.3", thinking: "medium" } as cons export function fullConfig(): ResearchConfig { return { studyId: "circle-full-1", - task: "tasks/circle_packing", + task: join(import.meta.dirname, "..", "tasks", "circle_packing"), models: { researchers: [{ ...glm }, { ...glm }, { ...glm }], experimenter: { ...glm }, @@ -21,7 +22,7 @@ export function fullConfig(): ResearchConfig { factorLimit: 3, maxRepairs: 1, lightChecks: { maxCount: 2, seconds: 5 }, - promptDir: "prompts", + promptDir: join(import.meta.dirname, "..", "prompts"), devEntry: false, flow: { preset: "full",