feat: 研究编号只许小写、配置路径必须绝对;检验判重两个顺序取平均

任务书 13 第一部分,用户 2026-10-03 同意。
- config:studyId 限定小写字母、数字和 ._-;task、promptDir 必须以 / 开头。
- eval:research.json 路径改绝对,writeConfig 不再转换;quality.ts 第 2、6 问每对 A、B 与 B、A 各问一次取平均,两次原始概率都存。
- tests:大写编号、相对路径各加测试。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-03 11:42:11 -04:00
parent b516f8a612
commit a42349717c
8 changed files with 88 additions and 30 deletions
+2
View File
@@ -449,6 +449,8 @@ Jev 是一个判别器,和程序检查地位相同,不单独成一种记录
**字段定义**
- `ResearchConfig`(字段定义):research.json 的全部字段,全部必填,不认识的字段报错。每个字段带中文说明(`description`)和类别(`category`:用户配置或方法参数),配置对话照着它问,不另写一份。`models` 里除了各角色的模型,还有 Jev 用的模型名(`models.jev`,用户配置);Jev 的密钥不进配置,从环境变量读。流程是两选一:完整流程写四个开关、选批规则、它才用得到的 λ、u、衰减;朴素循环只写 `preset`,别的都写不进去。
- 研究编号 `studyId` 只许小写字母、数字和 `._-`,首字符是小写字母或数字:它会进 Docker 镜像名,镜像名只认小写。(来源:用户 2026-10-03 同意,任务书 13)
- `task`、`promptDir` 必须是绝对路径:从别的目录接着跑一份研究时,相对路径会读到别处,而任务目录没有哈希核对挡着。(来源:用户 2026-10-03 同意,任务书 13)
- `FIELD_CATEGORIES`、`FieldCategory`:字段的两种类别,用户配置、方法参数。
- `THINKING_LEVELS`、`ThinkingLevel`:Pi 0.85.1 接受的七个思考档位,出处写在代码注释里;config 不能引用 Pi,所以照抄一份。
+2 -7
View File
@@ -1,6 +1,6 @@
import { execFileSync, spawn, type ChildProcess } from "node:child_process";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { join } from "node:path";
// 检验脚本共用的起研究工具:检验目录放在仓库外,配置照用户的写法写一份,研究在子进程里跑(要能被 SIGKILL)。
@@ -15,15 +15,10 @@ export function newEvalDir(name: string): string {
return dir;
}
/**
* 照一份底稿写这次检验用的 research.json:换研究编号(容器名、镜像标签都带它),需要时换各角色模型;
* 任务目录、提示词目录写成绝对路径(底稿里相对仓库根目录),这样配置文件放在仓库外也指得对。
*/
/** 照一份底稿写这次检验用的 research.json:换研究编号(容器名、镜像标签都带它),需要时换各角色模型。 */
export function writeConfig(base: string, path: string, studyId: string, models: unknown): string {
const config = JSON.parse(readFileSync(base, "utf8")) as Record<string, unknown>;
config["studyId"] = studyId;
config["task"] = resolve(config["task"] as string);
config["promptDir"] = resolve(config["promptDir"] as string);
if (models !== null) config["models"] = models;
writeFileSync(path, `${JSON.stringify(config, null, 2)}\n`);
return path;
+28 -13
View File
@@ -1,6 +1,6 @@
import { readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { ask, THRESHOLD_NOTE, type Answer, type Noul } from "../lib/jev.ts";
import { ask, THRESHOLD, THRESHOLD_NOTE, type Answer, type Noul } from "../lib/jev.ts";
import { entriesIn, entryDecision, readLedger, stages, type Entry } from "../lib/ledger.ts";
import { ENTRY_ONLY, LEADS_TO_IDEA, SAME_KIND, STOPS_AT_DEAD_END, TASKS } from "./questions.ts";
@@ -56,14 +56,29 @@ function query(site: string, ids: string[], state: Record<string, unknown>, ques
}
return cache.get(key)!;
}
const yes = async (p: Promise<Answer[]>, question: string) => (await p).find((a) => a.question === question)!.yes;
const answer = async (p: Promise<Answer[]>, question: string) => (await p).find((a) => a.question === question)!;
const yes = async (p: Promise<Answer[]>, question: string) => (await answer(p, question)).yes;
const ideaState = (e: Entry, idea: Idea) => ({ task: TASKS.idea, entry: show(e), idea: { description: idea.description } });
const leadsTo = (e: Entry, idea: Idea) => yes(query("leads", [e.id, idea.id], ideaState(e, idea), LEADS_TO_IDEA), "leads_to_idea");
const staysIn = (e: Entry, idea: Idea) => yes(query("stays", [e.id, idea.id], ideaState(e, idea), STOPS_AT_DEAD_END), "stays_within_idea");
const entryOnly = (e: Entry, question: string) => yes(query("entry", [e.id], { task: TASKS.entry, entry: show(e) }, ENTRY_ONLY), question);
const sameKind = (a: Entry, b: Entry) =>
yes(query("pair", [a.id, b.id], { task: TASKS.pair, entry_a: show(a), entry_b: show(b) }, SAME_KIND), "same_kind_of_change");
const pairProbability = async (a: Entry, b: Entry) =>
(await answer(query("pair", [a.id, b.id], { task: TASKS.pair, entry_a: show(a), entry_b: show(b) }, SAME_KIND), "same_kind_of_change"))
.probability;
/** 一对入口是不是同一条路:问题本身对称,所以 A、B 和 B、A 各问一次,取两次概率的平均再比阈值(任务书 13 第一部分第 3 条)。 */
type PairJudgement = { pair: [string, string]; ab: number; ba: number; mean: number; same: boolean };
async function sameKind(a: Entry, b: Entry): Promise<PairJudgement> {
const [ab, ba] = await Promise.all([pairProbability(a, b), pairProbability(b, a)]);
const mean = (ab + ba) / 2;
return { pair: [a.id, b.id], ab, ba, mean, same: mean > THRESHOLD };
}
type PairsResult = { pairs: PairJudgement[]; same: string[][] };
async function judgePairs(pairs: Array<readonly [Entry, Entry]>): Promise<PairsResult> {
const judged = await Promise.all(pairs.map(([a, b]) => sameKind(a, b)));
return { pairs: judged, same: judged.filter((j) => j.same).map((j) => j.pair) };
}
async function coverage(list: Entry[]): Promise<Record<string, string[]>> {
const out: Record<string, string[]> = {};
@@ -73,11 +88,10 @@ async function coverage(list: Entry[]): Promise<Record<string, string[]>> {
}
return out;
}
async function overlap(list: Entry[]): Promise<{ pairs: number; same: string[][]; ratio: number | null }> {
const pairs = list.flatMap((a, i) => list.slice(i + 1).filter((b) => b.author !== a.author).map((b) => [a, b] as const));
const hits = await Promise.all(pairs.map(([a, b]) => sameKind(a, b)));
const same = pairs.filter((_, i) => hits[i]).map(([a, b]) => [a.id, b.id]);
return { pairs: pairs.length, same, ratio: pairs.length === 0 ? null : same.length / pairs.length };
/** 重合度:不同作者的入口两两成对,判"是"的对数占总对数的比例。 */
async function overlap(list: Entry[]): Promise<PairsResult & { ratio: number | null }> {
const judged = await judgePairs(list.flatMap((a, i) => list.slice(i + 1).filter((b) => b.author !== a.author).map((b) => [a, b] as const)));
return { ...judged, ratio: judged.pairs.length === 0 ? null : judged.same.length / judged.pairs.length };
}
// ---------------------------------------------------------------------------
@@ -89,8 +103,8 @@ const covFinal = await coverage(final);
const q1 = { coverage: covFinal, pass: covFinal["K1"]!.length > 0 };
const assignedPairs = assigned.flatMap((a, i) => assigned.slice(i + 1).map((b) => [a, b] as const));
const pairHits = await Promise.all(assignedPairs.map(([a, b]) => sameKind(a, b)));
const q2 = { same: assignedPairs.filter((_, i) => pairHits[i]).map(([a, b]) => [a.id, b.id]), pass: !pairHits.some(Boolean) };
const assignedJudged = await judgePairs(assignedPairs);
const q2 = { ...assignedJudged, pass: assignedJudged.same.length === 0 };
const assignedK1 = await Promise.all(assigned.map((e) => leadsTo(e, k1)));
const deadEnd = await Promise.all(
@@ -134,13 +148,14 @@ const report = {
writeFileSync(join(out, "quality.json"), `${JSON.stringify(report, null, 2)}\n`);
const mark = (b: boolean) => (b ? "通过" : "不通过");
const ratio = (o: { pairs: number; same: string[][]; ratio: number | null }) => `${o.same.length}/${o.pairs}${o.ratio === null ? "" : `(${(o.ratio * 100).toFixed(0)}%)`}`;
const ratio = (o: PairsResult & { ratio: number | null }) => `${o.same.length}/${o.pairs.length}${o.ratio === null ? "" : `(${(o.ratio * 100).toFixed(0)}%)`}`;
const count = (c: Record<string, string[]>) => Object.entries(c).map(([k, v]) => `${k}:${v.length}`).join(" ");
console.log(`(${THRESHOLD_NOTE})`);
console.log(`入口全体:${final.map((e) => `${e.id} ${e.name}`).join("\n ")}`);
console.log(`分到的入口:${report.assignment.map((a) => `${a.researcher} → ${a.entry}(第 ${a.rank} 志愿)${a.name}`).join("\n ")}`);
console.log(`1 方向对不对:${mark(q1.pass)};覆盖 ${count(covFinal)}`);
console.log(`2 分没分开:${mark(q2.pass)};判"是"的对 ${JSON.stringify(q2.same)}`);
const pairLine = (j: PairJudgement) => `${j.pair.join("、")}:${j.ab.toFixed(2)}/${j.ba.toFixed(2)},平均 ${j.mean.toFixed(2)}`;
console.log(`2 分没分开:${mark(q2.pass)};判"是"的对 ${JSON.stringify(q2.same)};每对两个顺序的概率\n ${q2.pairs.map(pairLine).join("\n ")}`);
console.log(`3 好方向分出去没有:${mark(q3.pass)};指向 K1 的 ${JSON.stringify(q3.k1)},停在死路的 ${JSON.stringify(q3.deadEnds)}`);
console.log(`4 改代码的路线:${mark(q4.pass)};不是的 ${JSON.stringify(q4.notCodeRoute)}`);
console.log(`5 有没有写答案:${mark(q5pass)};分到的里写了的 ${JSON.stringify(q5.assignedWritingAnswer)},入口全体里写了的 ${JSON.stringify(q5.allWritingAnswer)}`);
+2 -2
View File
@@ -1,6 +1,6 @@
{
"studyId": "r0-eval",
"task": "tasks/circle_packing",
"task": "/home/iomgaa/Projects/InfoRSI/tasks/circle_packing",
"models": {
"researchers": [
{ "provider": "iomgaa", "model": "glm-5.3", "thinking": "medium" },
@@ -17,7 +17,7 @@
"factorLimit": 3,
"maxRepairs": 1,
"lightChecks": { "maxCount": 2, "seconds": 5 },
"promptDir": "prompts",
"promptDir": "/home/iomgaa/Projects/InfoRSI/prompts",
"devEntry": false,
"flow": {
"preset": "full",
+7 -1
View File
@@ -19,6 +19,12 @@ const TYPE_TEXT: Readonly<Record<string, string>> = {
null: "null",
};
/** schema.ts 里文字字段、路径字段的格式,换成人读得懂的话。 */
const PATTERN_TEXT: Readonly<Record<string, string>> = {
"\\S": "不能为空,也不能只有空白",
"^/": "必须是绝对路径(以 / 开头)",
};
function reasonOf(e: SchemaError): string {
const p = e.params as Record<string, unknown>;
switch (e.keyword) {
@@ -39,7 +45,7 @@ function reasonOf(e: SchemaError): string {
case "minItems":
return `至少要有 ${String(p.limit)} 项`;
case "pattern":
return p.pattern === "\\S" ? "不能为空,也不能只有空白" : `格式不对,应符合 ${String(p.pattern)}`;
return PATTERN_TEXT[String(p.pattern)] ?? `格式不对,应符合 ${String(p.pattern)}`;
default:
return e.message;
}
+17 -4
View File
@@ -1,6 +1,5 @@
import { Type, type Static, type TSchema } from "typebox";
import { BATCH_RULES, CELL_RULES, type BatchRuleName, type CellRuleName } from "../rules/index.ts";
import { Id } from "../vocab/index.ts";
// research.json 的字段定义(docs/结构.md 第二部分、第五部分"方法参数""步骤开关")。
// 每个字段都必填,不写默认值;不认识的字段报错。可以没有的值写 null(目前没有这样的字段)。
@@ -15,6 +14,11 @@ export type FieldCategory = (typeof FIELD_CATEGORIES)[number];
const CLOSED = { additionalProperties: false } as const;
/** 文字字段:不能为空,也不能只有空白。 */
const TEXT = { pattern: "\\S" };
/**
* 路径字段必须是绝对路径:从别的目录接着跑一份研究时,相对路径会读到别处,而任务目录没有哈希核对挡着
* (任务书 13 第一部分第 2 条)。
*/
const ABSOLUTE_PATH = { pattern: "^/" };
function user<T extends TSchema>(schema: T, description: string): T {
return { ...schema, description, category: "用户配置" };
@@ -127,8 +131,14 @@ const PositiveInteger = Type.Integer({ minimum: 1 });
export const ResearchConfig = Type.Object(
{
studyId: user(Id, "研究编号:决定的种子由它和轮次、版本算出;只许字母、数字和 ._-,首字符是字母或数字"),
task: user(Type.String(TEXT), "用哪个任务:任务目录的路径。指标、方向、ε、时限等任务事实写在任务目录里"),
studyId: user(
Type.String({ pattern: "^[a-z0-9][a-z0-9._-]{0,127}$" }),
"研究编号:决定的种子由它和轮次、版本算出,Docker 镜像名也带它,所以只许小写字母、数字和 ._-,首字符是小写字母或数字",
),
task: user(
Type.String(ABSOLUTE_PATH),
"用哪个任务:任务目录的绝对路径。指标、方向、ε、时限等任务事实写在任务目录里",
),
models: user(
Type.Object(
{
@@ -172,7 +182,10 @@ export const ResearchConfig = Type.Object(
),
"轻量检查的次数与时长上限:只用来检查语法、接口和极小的样例,不能借它跑正式规模的求解",
),
promptDir: method(Type.String(TEXT), "提示词目录:各角色的角色提示词和各类任务消息放在这里"),
promptDir: method(
Type.String(ABSOLUTE_PATH),
"提示词目录的绝对路径:各角色的角色提示词和各类任务消息放在这里",
),
devEntry: method(
Type.Boolean(),
"开发入口(方法.md 第九部分):正式研究写 false;开发调试时才写 true,打开过的研究不能当验收证据",
+27 -1
View File
@@ -71,7 +71,7 @@ test("类型或取值不对报错,带字段路径", () => {
[(c) => (c.runs = 0), /^\/runs:不能小于 1$/m],
[(c) => (c.batchLimit = 1.5), /^\/batchLimit:应为整数$/m],
[(c) => (c.studyId = "../x"), /^\/studyId:格式不对/m],
[(c) => (c.task = " "), /^\/task:不能为空,也不能只有空白$/m],
[(c) => (c.task = " "), /^\/task:必须是绝对路径(以 \/ 开头)$/m],
[(c) => (c.devEntry = "false"), /^\/devEntry:应为true 或 false$/m],
[(c) => ((c.models as Obj).researchers = []), /^\/models\/researchers:至少要有 1 项$/m],
[(c) => (((c.models as Obj).verifier as Obj).thinking = null), /^\/models\/verifier\/thinking:只能是 "off"、"minimal"、"low"、"medium"、"high"、"xhigh"、"max" 之一$/m],
@@ -91,6 +91,32 @@ test("类型或取值不对报错,带字段路径", () => {
}
});
test("研究编号带大写报错:它会进 Docker 镜像名,镜像名只认小写", () => {
for (const id of ["Circle-1", "circle-A", "R0"]) {
const config = fullConfig() as unknown as Obj;
config.studyId = id;
const message = errorOf(config);
assert.match(message, /^\/studyId:格式不对/m);
assert.match(message, /共 1 处/, message);
}
});
test("任务目录、提示词目录写相对路径报错,报错写明字段路径", () => {
const cases: Array<[string, string]> = [
["task", "tasks/circle_packing"],
["task", "./tasks/circle_packing"],
["promptDir", "prompts"],
["promptDir", "../InfoRSI/prompts"],
];
for (const [field, path] of cases) {
const config = fullConfig() as unknown as Obj;
config[field] = path;
const message = errorOf(config);
assert.match(message, new RegExp(`^/${field}:必须是绝对路径(以 / 开头)$`, "m"));
assert.match(message, /共 1 处/, message);
}
});
test("Jev 模型必填:少了、写 null、写空白都报错,不补 jev-latest", () => {
const missing = fullConfig() as unknown as Obj;
delete (missing.models as Obj).jev;
+3 -2
View File
@@ -1,3 +1,4 @@
import { join } from "node:path";
import type { ResearchConfig } from "../src/config/index.ts";
// 两份写全的合法配置:完整流程一份、朴素循环一份。数值取方法.md 6.11 的现行值和参考项目真实用过的配置
@@ -8,7 +9,7 @@ const glm = { provider: "iomgaa", model: "glm-5.3", thinking: "medium" } as cons
export function fullConfig(): ResearchConfig {
return {
studyId: "circle-full-1",
task: "tasks/circle_packing",
task: join(import.meta.dirname, "..", "tasks", "circle_packing"),
models: {
researchers: [{ ...glm }, { ...glm }, { ...glm }],
experimenter: { ...glm },
@@ -21,7 +22,7 @@ export function fullConfig(): ResearchConfig {
factorLimit: 3,
maxRepairs: 1,
lightChecks: { maxCount: 2, seconds: 5 },
promptDir: "prompts",
promptDir: join(import.meta.dirname, "..", "prompts"),
devEntry: false,
flow: {
preset: "full",