feat: flow 第 1 部分(骨架、研究开始、第零轮)与第零轮检验
按任务书 12 实现 src/flow:从账本现算下一步的纯函数、主循环、恢复入口、派模型任务和跑正式运行两个共用动作、冻结配置、基线、第零轮三个研究者阶段和入口分配,走到第 1 轮时停下。materials 导出 finalEntries、加 researcherId 和 StudyRecord。 审阅采纳两条:研究编号本机已用过时冻结前报错;docker run 没启动时写完结果就停下,不在本进程里无限起新编号。 eval/ 新增第零轮检验:起研究命令行、回放假模型、检验用 Jev 客户端、六问质量评估、16-real-1 回放、杀进程与两份同时跑。tsc 与 depcruise 也检查 eval/,新增 eval 只经 index.ts 用正式代码的规则。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,71 @@
|
||||
// 检验自己用的小 Jev 客户端(任务书 12 第四部分第 6 条):不经 src/jev,也不改它。
|
||||
// 请求格式参照 src/jev/jev.ts 和 RivalBits research-wiki/参考-Jev使用指南.md 第二部分:
|
||||
// POST {model, state, questions},回答里 answers.<问题名>.noul 是 0 到 1 的概率。
|
||||
// 检验的问题都没有校准过阈值,一律以概率大于 0.5 判"是",原始概率照存(任务书 12 第 5.2 节)。
|
||||
|
||||
const ENDPOINT = "https://api.typesafe.ai/v1/systemone";
|
||||
/** 检验用的 Jev 模型名,和研究配置的 models.jev 同一个。 */
|
||||
export const JEV_MODEL = "jev-latest";
|
||||
export const THRESHOLD = 0.5;
|
||||
export const THRESHOLD_NOTE = "阈值未校准:一律以概率大于 0.5 判\"是\"";
|
||||
|
||||
/** 一条是非题,写法照 src/jev/questions.ts:问题、关注点、判"是"判"否"的说明和反例。 */
|
||||
export interface Noul {
|
||||
type: "noul";
|
||||
instructions: { question: string; focus: string };
|
||||
criteria: {
|
||||
true: { what: string; examples: string[] };
|
||||
false: { what: string; not_for: string; examples: string[] };
|
||||
};
|
||||
}
|
||||
|
||||
export interface Answer {
|
||||
question: string;
|
||||
probability: number;
|
||||
yes: boolean;
|
||||
requestId: string;
|
||||
}
|
||||
|
||||
/** 调不通时重试前等多久:只重试连不上、超时、429 和 5xx(原因在我们控制之外),一共最多 4 次。 */
|
||||
const RETRY_WAITS_MS = [2_000, 10_000, 30_000];
|
||||
|
||||
/** 就一份材料问几问,一次请求。 */
|
||||
export async function ask(state: Record<string, unknown>, questions: Record<string, Noul>): Promise<Answer[]> {
|
||||
const key = process.env["TYPESAFE_API_KEY"];
|
||||
if (key === undefined || key.trim() === "") throw new Error("环境变量 TYPESAFE_API_KEY 没有设置");
|
||||
const body = JSON.stringify({ model: JEV_MODEL, state, questions });
|
||||
const failures: string[] = [];
|
||||
for (let attempt = 0; attempt <= RETRY_WAITS_MS.length; attempt++) {
|
||||
if (attempt > 0) await new Promise((ok) => setTimeout(ok, RETRY_WAITS_MS[attempt - 1]));
|
||||
let response: Response;
|
||||
let text: string;
|
||||
try {
|
||||
response = await fetch(ENDPOINT, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", authorization: `Bearer ${key}` },
|
||||
body,
|
||||
signal: AbortSignal.timeout(60_000),
|
||||
});
|
||||
text = await response.text();
|
||||
} catch (error) {
|
||||
failures.push((error as Error).message);
|
||||
continue;
|
||||
}
|
||||
const requestId = response.headers.get("x-typesafe-request-id") ?? "无";
|
||||
if (!response.ok) {
|
||||
const what = `HTTP ${response.status}(request-id: ${requestId}):${text.slice(0, 300)}`;
|
||||
if (response.status !== 429 && response.status < 500) throw new Error(`Jev 拒绝了请求:${what}`);
|
||||
failures.push(what);
|
||||
continue;
|
||||
}
|
||||
const answers = (JSON.parse(text) as { answers?: Record<string, { noul?: unknown }> }).answers;
|
||||
return Object.keys(questions).map((question) => {
|
||||
const probability = answers?.[question]?.noul;
|
||||
if (typeof probability !== "number" || !(probability >= 0 && probability <= 1)) {
|
||||
throw new Error(`Jev 的回答里问题 ${question} 没有 0 到 1 之间的概率(request-id: ${requestId}):${text.slice(0, 300)}`);
|
||||
}
|
||||
return { question, probability, yes: probability > THRESHOLD, requestId };
|
||||
});
|
||||
}
|
||||
throw new Error(`Jev 调不通,发了 ${failures.length} 次:${failures.join(";")}`);
|
||||
}
|
||||
Reference in New Issue
Block a user