feat: 诊断(任务书 18 第一块):诊断数学、四种提交、一次运行跑多个实验、账本与材料、flow 诊断各步

消融默认关,关掉时解释阶段一并跳过;选诊断实验用信息价值减成本。
eval/diagnosis/research.json 为第三块配置。vocab 字段说明的出口文字随提示词一起改,见下一次提交的 prompts/ 与同文件改动(本次含在内)。

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-05 10:36:20 -04:00
parent f39f8410e7
commit 852600df02
69 changed files with 6607 additions and 214 deletions
+17 -4
View File
@@ -12,6 +12,7 @@ export const FIELD_CATEGORIES = ["用户配置", "方法参数"] as const;
export type FieldCategory = (typeof FIELD_CATEGORIES)[number];
const CLOSED = { additionalProperties: false } as const;
const PositiveInteger = Type.Integer({ minimum: 1 });
/** 文字字段:不能为空,也不能只有空白。 */
const TEXT = { pattern: "\\S" };
/**
@@ -68,7 +69,7 @@ const RoundZero = Type.Union([
/** 消融(方法.md 第十一部分"步骤开关"):关,或开并写选格规则。 */
const Ablation = Type.Union([
Type.Object(
{ enabled: method(Type.Literal(false), "消融关:不列消融格;解释照样写文字,但不给每格的三个数") },
{ enabled: method(Type.Literal(false), "消融关:不列消融格,也不写解释(解释就是对消融格的预测,方法.md 4.3 ⑫)") },
CLOSED,
),
Type.Object(
@@ -83,6 +84,19 @@ const Ablation = Type.Union([
),
]);
/** 诊断(方法.md 4.3 诊 1–诊 4、第十一部分"步骤开关"):关,或开并写两个方法参数(6.11)。 */
const Diagnosis = Type.Union([
Type.Object({ enabled: method(Type.Literal(false), "诊断关:每轮提案之前不做诊断实验") }, CLOSED),
Type.Object(
{
enabled: method(Type.Literal(true), "诊断开"),
perResearcher: method(PositiveInteger, "每人每轮诊断实验上限 A;现行值 2(方法.md 6.11)"),
maxExperiments: method(PositiveInteger, "一次诊断最多做几个实验 D;现行值 4(方法.md 6.11)"),
},
CLOSED,
),
]);
/**
* 两种流程(方法.md 第十一部分)。完整流程写开关、规则和它才用得到的方法参数;
* 朴素循环整套照搬,不能写任何开关、规则,也不写它用不到的 λ、u、衰减(它不预测、不消融)。
@@ -96,6 +110,7 @@ const FullFlow = Type.Object(
Type.Boolean(),
"挑刺与回应开不开(方法.md 4.3 ⑤ ⑥);关掉时不挑刺、不回应,协议照样冻结;现行值 true",
),
diagnosis: method(Diagnosis, "诊断(方法.md 4.3 诊 1–诊 4)"),
ablation: method(Ablation, "消融(方法.md 4.3 ⑬–⑰)"),
batchRule: method(
Type.Enum(BATCH_RULE_NAMES),
@@ -121,14 +136,12 @@ const NaiveFlow = Type.Object(
{
preset: method(
Type.Literal("naive"),
"朴素循环:方法.md 第十一部分的对照流程整套照搬,不挑刺、不回应、不预测、均匀随机选批、不消融",
"朴素循环:方法.md 第十一部分的对照流程整套照搬,不挑刺、不回应、不预测、均匀随机选批、不诊断、不消融",
),
},
CLOSED,
);
const PositiveInteger = Type.Integer({ minimum: 1 });
export const ResearchConfig = Type.Object(
{
studyId: user(
+41 -8
View File
@@ -1,4 +1,11 @@
import { BATCH_RULES, CELL_RULES, type BatchRuleName, type CellRuleName } from "../rules/index.ts";
import {
BATCH_RULES,
CELL_RULES,
DIAGNOSIS_RULES,
type BatchRuleName,
type CellRuleName,
type DiagnosisRuleName,
} from "../rules/index.ts";
import type { ResearchConfig } from "./schema.ts";
// 步骤清单(docs/结构.md 第三部分,方法.md 第十一部分):把配置里的流程、开关、规则翻译成这份研究要走的步骤。
@@ -23,6 +30,13 @@ export type StudyStepId = (typeof STUDY_STEPS)[number];
export const ROUND_STEPS = [
"openRound", // 1. 开轮:冻结本轮名单和提案上限
"carryOverPool", // 1. 开轮:处理留池候选
"diagnosisProposal", // 诊 1. 诊断解释与诊断实验
"diagnosisDedupe", // 诊 2. 判重合并
"diagnosisPrediction", // 诊 2. 诊断预测
"selectDiagnosis", // 诊 3. 选诊断实验(程序)
"diagnosisImplementation", // 诊 4. 实现、核验、修复
"diagnosisRun", // 诊 4. 诊断运行
"diagnosisSettle", // 诊 4. 诊断结算
"proposal", // 2. 提案
"dedupe", // 3. 判重合并
"firstPrediction", // 4. 第一次预测(看原始提案)
@@ -50,11 +64,11 @@ export const ROUND_STEPS = [
export type RoundStepId = (typeof ROUND_STEPS)[number];
/** 决策点用哪条规则:规则名和版本,从 rules 的登记表取。 */
export type RuleRef = { name: BatchRuleName | CellRuleName; version: number };
export type RuleRef = { name: BatchRuleName | CellRuleName | DiagnosisRuleName; version: number };
type Status = { status: "execute" } | { status: "skip"; reason: string };
/** 一步:执行,或按配置跳过(带原因);只有选批、选格两个决策点带规则,其余为 null。 */
/** 一步:执行,或按配置跳过(带原因);只有选批、选格、选诊断实验三个决策点带规则,其余为 null。 */
export type Step<S extends string> = { id: S; rule: RuleRef | null } & Status;
export interface StepList {
@@ -82,7 +96,7 @@ export function stepList(config: ResearchConfig): StepList {
const flow = config.flow;
if (flow.preset === "naive") {
const naive = skip("朴素循环不做这一步(方法.md 第十一部分)");
return assemble("naive", (id) => (NAIVE_STEPS.has(id) ? EXECUTE : naive), batchRule("uniformRandom"), null);
return assemble("naive", (id) => (NAIVE_STEPS.has(id) ? EXECUTE : naive), batchRule("uniformRandom"), null, null);
}
// 完整流程:只有开关关掉的步骤跳过。
@@ -102,15 +116,27 @@ export function stepList(config: ResearchConfig): StepList {
if (!flow.firstPrediction) off(["firstPrediction"], "配置关掉了第一次预测");
// 挑刺与回应关掉时,协议照样冻结(freezeProtocol 不跟着关)。
if (!flow.critique) off(["critique", "response"], "配置关掉了挑刺与回应");
// 消融关掉时解释照样写文字,只是不给每格的三个数(方法.md 第十一部分"步骤开关"),所以 explanation 不跟着关。
// 消融关掉时,解释也不做:解释就是对消融格的预测,⑬–⑰ 整段都不做(方法.md 4.3 ⑫)。
if (!flow.ablation.enabled) {
off(
["listCells", "selectCells", "ablationImplementation", "ablationVerification", "ablationRun", "settle"],
[
"listCells", "explanation", "selectCells", "ablationImplementation",
"ablationVerification", "ablationRun", "settle",
],
"配置关掉了消融",
);
}
if (!flow.diagnosis.enabled) {
off(
[
"diagnosisProposal", "diagnosisDedupe", "diagnosisPrediction", "selectDiagnosis",
"diagnosisImplementation", "diagnosisRun", "diagnosisSettle",
],
"配置关掉了诊断",
);
}
const cells = flow.ablation.enabled ? cellRule(flow.ablation.cellRule) : null;
return assemble("full", (id) => skipped[id] ?? EXECUTE, batchRule(flow.batchRule), cells);
return assemble("full", (id) => skipped[id] ?? EXECUTE, batchRule(flow.batchRule), cells, flow.diagnosis.enabled ? diagnosisRule() : null);
}
function batchRule(name: BatchRuleName): RuleRef {
@@ -121,13 +147,20 @@ function cellRule(name: CellRuleName): RuleRef {
return { name, version: CELL_RULES[name].version };
}
/** 选诊断实验只有一条规则:信息价值减成本(方法.md 6.12),配置里不选。 */
function diagnosisRule(): RuleRef {
return { name: "valueMinusCost", version: DIAGNOSIS_RULES.valueMinusCost.version };
}
function assemble(
preset: StepList["preset"],
status: (id: StudyStepId | RoundStepId) => Status,
batch: RuleRef,
cells: RuleRef | null,
diagnosis: RuleRef | null,
): StepList {
const ruleOf = (id: RoundStepId) => (id === "selectBatch" ? batch : id === "selectCells" ? cells : null);
const ruleOf = (id: RoundStepId) =>
id === "selectBatch" ? batch : id === "selectCells" ? cells : id === "selectDiagnosis" ? diagnosis : null;
return {
preset,
study: STUDY_STEPS.map((id) => ({ id, rule: null, ...status(id) })),
+56 -6
View File
@@ -1,7 +1,18 @@
import { rmSync } from "node:fs";
import type { Content } from "../ledger/index.ts";
import { formalRun as runDefinition, judge, prepareJudging, prepareRun } from "../tasks/index.ts";
import type { HypothesisMetric, JudgedResult, ProcessResult } from "../vocab/index.ts";
import {
diagnosisRun as diagnosisDefinition,
formalRun as runDefinition,
judge,
judgeDiagnosis,
prepareDiagnosisJudging,
prepareDiagnosisRun,
prepareJudging,
prepareRun,
type DiagnosisExperiment,
type DiagnosisJudged,
} from "../tasks/index.ts";
import type { HypothesisMetric, JudgedResult, ProcessResult, RunDefinition } from "../vocab/index.ts";
import { checkoutVersion } from "../workspace/index.ts";
import { containerName, freshDir, judgeContainerName, runPaths, workspaceRepo } from "./paths.ts";
import type { Study } from "./study.ts";
@@ -9,6 +20,8 @@ import type { Study } from "./study.ts";
// 共用动作二:跑一次正式运行(任务书 12 第 2.7 节)。先登记、再启动、再写结果,前后顺序由这个入口保证
// (方法.md 第七、九部分):登记占名额,确认启动才扣次,确认没启动的写结果"没启动"、退还名额。
// 判分不扣次数,不登记。恢复时接着等一次已经启动的运行,走同一个 finishRun。
// 诊断运行(一次运行跑几个实验,用途 diagnosis)走同一个登记、启动、等待的入口,只是准备目录和判分不同:
// 运行记录不带判分(judged 为 null),逐个实验的判分在诊断结算前由 judgeDiagnosisRun 从运行目录做。
export type RunPlan = Omit<Content<"runPlanned">, "definition">;
@@ -22,11 +35,30 @@ export type RunOutcome = "done" | "stopped" | { notStarted: string };
/** 跑一次正式运行:取出 version 的代码,在任务镜像里跑,判分,写结果。用户停下时不等容器,交回 "stopped"。 */
export async function formalRun(s: Study, plan: RunPlan, version: string, metrics: HypothesisMetric[]): Promise<RunOutcome> {
const definition = runDefinition(s.task, s.frozen.image, String(s.frozen.seed));
return launch(s, plan, definition, metrics, (paths) => {
checkoutVersion(workspaceRepo(s.dir), version, freshDir(paths.code));
prepareRun(s.task, paths.code, freshDir(paths.run), metrics);
});
}
/**
* 跑一次诊断运行:一个容器、一次登记、一次扣次,里面按顺序跑 experiments 里的实验(总时限平分)。
* version 是诊断程序,baselineRunDir 是产出当前基线的那次运行的运行目录(原始产物只读地交给每个实验)。
*/
export async function diagnosisFormalRun(s: Study, plan: RunPlan, version: string, baselineRunDir: string, experiments: DiagnosisExperiment[]): Promise<RunOutcome> {
const definition = diagnosisDefinition(s.task, s.frozen.image, String(s.frozen.seed), experiments.map((e) => e.id));
return launch(s, plan, definition, null, (paths) => {
checkoutVersion(workspaceRepo(s.dir), version, freshDir(paths.code));
prepareDiagnosisRun(s.task, paths.code, baselineRunDir, freshDir(paths.run), experiments);
});
}
/** 先登记、准备运行目录、启动,确认启动才写"确认启动";metrics 为 null 的是诊断运行(运行记录不带判分)。 */
async function launch(s: Study, plan: RunPlan, definition: RunDefinition, metrics: HypothesisMetric[] | null, prepare: (paths: ReturnType<typeof runPaths>) => void): Promise<RunOutcome> {
s.ledger.append("runPlanned", { ...plan, definition });
const paths = runPaths(s.dir, plan.run);
freshDir(paths.base);
checkoutVersion(workspaceRepo(s.dir), version, freshDir(paths.code));
prepareRun(s.task, paths.code, freshDir(paths.run), metrics);
prepare(paths);
const started = await s.runner.start(containerName(s.frozen.config.studyId, plan.run), definition, paths.run);
if (!started.started) {
s.ledger.append("runResult", { run: plan.run, process: { ending: "notStarted", reason: started.reason }, judged: null });
@@ -39,13 +71,14 @@ export async function formalRun(s: Study, plan: RunPlan, version: string, metric
/**
* 等一次已经确认启动的运行结束,判分,写结果,删掉容器。日志写进这次运行自己的日志目录。
* 收到"停下"信号就不等了,直接交回 "stopped";容器不管它,下次启动时恢复入口再来接着等。
* metrics 为 null 的是诊断运行:这里不判分,结果的 judged 写 null。
*/
export async function finishRun(s: Study, run: string, metrics: HypothesisMetric[]): Promise<"done" | "stopped"> {
export async function finishRun(s: Study, run: string, metrics: HypothesisMetric[] | null): Promise<"done" | "stopped"> {
const name = containerName(s.frozen.config.studyId, run);
const paths = runPaths(s.dir, run);
const waited = await untilStopped(s.stop, s.runner.wait(name, emptyDir(paths.logs)));
if (waited === "stopped") return "stopped";
const judged = await judgeRun(s, run, waited, metrics);
const judged = metrics === null ? null : await judgeRun(s, run, waited, metrics);
s.ledger.append("runResult", { run, process: waited, judged });
await s.runner.remove(name);
return "done";
@@ -68,6 +101,23 @@ async function judgeRun(s: Study, run: string, formal: ProcessResult, metrics: H
return judge(s.task, paths.run, paths.judge, formal, validator, metrics);
}
/**
* 诊断运行的判分(诊断结算前做):每个交出答案的实验单独判分。同 judgeRun:判分目录每次重新建空的,
* 判分容器还在就先删掉再用;恢复时从运行目录重新判,不存半截结果。experiments 是这次运行跑的那几个实验,次序同运行。
*/
export async function judgeDiagnosisRun(s: Study, run: string, process: ProcessResult, experiments: DiagnosisExperiment[]): Promise<DiagnosisJudged> {
const paths = runPaths(s.dir, run);
const step = prepareDiagnosisJudging(s.task, paths.run, emptyDir(paths.judge), s.frozen.image, process, experiments);
if (step.status === "judged") return step.result;
const name = judgeContainerName(s.frozen.config.studyId, run);
if ((await s.runner.status(name)) !== "absent") await s.runner.remove(name);
const started = await s.runner.start(name, step.definition, paths.judge);
if (!started.started) throw new Error(`诊断运行 ${run} 的校验器容器没有启动:${started.reason}`);
const validator = await s.runner.wait(name, emptyDir(paths.judgeLogs));
await s.runner.remove(name);
return judgeDiagnosis(s.task, paths.run, paths.judge, process, validator, experiments);
}
/** 清空重建一个目录:恢复时会再等一次、再判一次,上一次没写完的日志和判分目录不留。 */
function emptyDir(dir: string): string {
rmSync(dir, { recursive: true, force: true });
+17 -2
View File
@@ -1,7 +1,7 @@
import { mkdirSync } from "node:fs";
import { resolve } from "node:path";
import type { ResearchConfig } from "../config/index.ts";
import { forwardNotRunEvent, cellNotRunEvent } from "../materials/index.ts";
import { cellNotRunEvent, diagnosisNotRunEvent, forwardNotRunEvent } from "../materials/index.ts";
import { Ledger } from "../ledger/index.ts";
import { Runner } from "../runner/index.ts";
import { fullConfig, readFrozen } from "./frozen.ts";
@@ -10,6 +10,7 @@ import { ledgerFile } from "./paths.ts";
import { recover } from "./recover.ts";
import { baselineRun, saveBaseline } from "./steps/baseline.ts";
import { ablationRun, selectCells, settle, unsealChosen } from "./steps/ablation.ts";
import { diagnosisDedupe, diagnosisRun, diagnosisSettle, selectDiagnosisExperiments } from "./steps/diagnosis.ts";
import { forwardRun, adoptBatch } from "./steps/forward.ts";
import { dispatchStage } from "./steps/dispatch.ts";
import { dedupe, freezeProtocols, selectBatch } from "./steps/round.ts";
@@ -80,10 +81,12 @@ async function doStep(s: Study, next: Exclude<Next, { do: "freezeConfig" }>): Pr
return null;
case "baselineRun":
case "forwardRun":
case "ablationRun": {
case "ablationRun":
case "diagnosisRun": {
const ran =
next.do === "baselineRun" ? await baselineRun(s, next.run)
: next.do === "forwardRun" ? await forwardRun(s, next.round, next.candidate, next.run)
: next.do === "diagnosisRun" ? await diagnosisRun(s, next.round, next.run)
: await ablationRun(s, next.round, next.cell, next.run);
if (ran === "stopped") return USER_STOPPED;
if (ran === "done") return null;
@@ -103,6 +106,18 @@ async function doStep(s: Study, next: Exclude<Next, { do: "freezeConfig" }>): Pr
case "assignEntries":
assignEntries(s);
return null;
case "diagnosisDedupe":
await diagnosisDedupe(s, next.round);
return null;
case "selectDiagnosis":
selectDiagnosisExperiments(s, next.round);
return null;
case "diagnosisNotRun":
s.ledger.append("event", diagnosisNotRunEvent(next.round, next.reason));
return null;
case "diagnosisSettle":
await diagnosisSettle(s, next.round);
return null;
case "dedupe":
await dedupe(s, next.round);
return null;
+24 -9
View File
@@ -4,8 +4,9 @@ import { runSession, type SessionTask, type SessionEnding, type SubmitAttempt, t
import { Board, buildMaterial, callId, queryMaterial, researcherId, type TaskMaterial } from "../materials/index.ts";
import { receive, type Incoming } from "../submissions/index.ts";
import { SUBMISSION_KINDS, type JevJudgment, type Json, type SubmissionKind } from "../vocab/index.ts";
import { placeBaselineArtifact } from "../tasks/index.ts";
import { checkoutVersion } from "../workspace/index.ts";
import { prepareSubmission, taskCode, lightChecker } from "./task-code.ts";
import { prepareSubmission, taskArtifact, taskCode, lightChecker } from "./task-code.ts";
import { callPaths, freshDir, runPaths, workspaceRepo } from "./paths.ts";
import type { Study } from "./study.ts";
@@ -43,7 +44,7 @@ export async function runModelTask<K extends SubmissionKind>(s: Study, t: ModelT
prompts: s.prompts,
codeRoot: paths.code,
changes,
artifacts: t.kind === "explanation" ? runArtifacts(s, t.stageId) : null,
artifacts: t.kind === "explanation" ? roundArtifacts(s, t.stageId) : t.kind === "diagnosisProposal" ? baselineArtifacts(s, t.stageId) : null,
});
const model = role === "researcher" ? researcherModel(s, t.actor) : s.frozen.config.models[role];
// 先登记、后建目录:调用编号只在登记之后才算用掉,被杀在两者之间时下次的编号不会撞上旧目录。
@@ -60,6 +61,8 @@ export async function runModelTask<K extends SubmissionKind>(s: Study, t: ModelT
for (const r of material.readable) checkoutVersion(workspaceRepo(s.dir), r.version, freshDir(r.dir));
const workDir = freshDir(paths.work);
if (startVersion !== null) checkoutVersion(workspaceRepo(s.dir), startVersion, workDir);
const artifactFrom = taskArtifact(s, t);
if (artifactFrom !== null) placeBaselineArtifact(s.task, artifactFrom, workDir);
let reply = 0;
const common = {
model,
@@ -90,22 +93,34 @@ export async function runModelTask<K extends SubmissionKind>(s: Study, t: ModelT
* 解释任务要的产物(materials 的 TaskRequest.artifacts):本轮每次已公开正向运行的 metrics.json、solution.json
* 和标准输出、标准错误日志(运行编号 → 文件名 → 文字)。完整代码那次的答案出自哪里,就靠这些产物判断。
*/
function runArtifacts(s: Study, stageId: string): Record<string, Record<string, string>> {
function roundArtifacts(s: Study, stageId: string): Record<string, Record<string, string>> {
const board = new Board(s.ledger.records());
const round = board.stage(stageId).round;
const artifacts: Record<string, Record<string, string>> = {};
for (const r of board.runs()) {
if (r.plan.purpose !== "forward" || r.plan.round !== round || !board.runPublic(r)) continue;
const paths = runPaths(s.dir, r.plan.run);
const files: Record<string, string> = {};
for (const [dir, name] of [[paths.run, "metrics.json"], [paths.run, "solution.json"], [paths.logs, "stdout.log"], [paths.logs, "stderr.log"]] as const) {
if (existsSync(join(dir, name))) files[name] = readFileSync(join(dir, name), "utf8");
}
artifacts[r.plan.run] = files;
artifacts[r.plan.run] = runFiles(s, r.plan.run);
}
return artifacts;
}
/** 诊 1 要的产物:产出本轮当前基线的那次运行,同样的几个文件(运行编号 → 文件名 → 文字)。 */
function baselineArtifacts(s: Study, stageId: string): Record<string, Record<string, string>> {
const board = new Board(s.ledger.records());
const run = board.runOf(board.baselineOf(board.stage(stageId).round))!.plan.run;
return { [run]: runFiles(s, run) };
}
/** 一次运行留下的产物:metrics.json、solution.json 和两份日志,有哪个读哪个。 */
function runFiles(s: Study, run: string): Record<string, string> {
const paths = runPaths(s.dir, run);
const files: Record<string, string> = {};
for (const [dir, name] of [[paths.run, "metrics.json"], [paths.run, "solution.json"], [paths.logs, "stdout.log"], [paths.logs, "stderr.log"]] as const) {
if (existsSync(join(dir, name))) files[name] = readFileSync(join(dir, name), "utf8");
}
return files;
}
/** 研究者用的模型:第 i 位研究者(researcherId(i))用配置里第 i 个。 */
function researcherModel(s: Study, actor: string) {
const models = s.frozen.config.models.researchers;
+111 -7
View File
@@ -1,8 +1,8 @@
import type { RoundStepId, StudyStepId } from "../config/index.ts";
import type { Content, LedgerRecord } from "../ledger/index.ts";
import { Board, CELL_NOT_RUN, cellId, DECISION_KINDS, experimenterId, factorKey, FORWARD_NOT_RUN, ORIGINAL_BASELINE, researcherId, splitCell, verifierId } from "../materials/index.ts";
import { Board, CELL_NOT_RUN, cellId, DECISION_KINDS, DIAGNOSIS_NOT_RUN, diagnosisId, experimenterId, factorKey, FORWARD_NOT_RUN, ORIGINAL_BASELINE, researcherId, splitCell, verifierId } from "../materials/index.ts";
import { compareIds, listCells, runBalance, sharedSettings } from "../rules/index.ts";
import { ablationOutcome, verificationOutcome } from "../submissions/index.ts";
import { ablationOutcome, diagnosisOutcome, verificationOutcome } from "../submissions/index.ts";
import { SUBMISSION_KINDS, type Stage, type SubmissionKind } from "../vocab/index.ts";
import { readFrozen, type Frozen } from "./frozen.ts";
@@ -23,7 +23,7 @@ const ROUND_ZERO_STAGES = {
type RoundZeroStep = keyof typeof ROUND_ZERO_STAGES;
/** 第 r 轮的研究者阶段:步骤名就是阶段名(vocab STAGES)。 */
type RoundStageStep = Extract<RoundStepId, "proposal" | "firstPrediction" | "critique" | "response" | "secondPrediction">;
type RoundStageStep = Extract<RoundStepId, "diagnosisProposal" | "diagnosisPrediction" | "proposal" | "firstPrediction" | "critique" | "response" | "secondPrediction">;
/** 阶段编号由 flow 起:r<轮>-<步骤>,一个步骤一个阶段。第 r 轮按配置跳过的步骤也用这个编号记"按配置跳过"。 */
export const stageIdOf = (round: number, step: string): string => `r${round}-${step}`;
@@ -52,6 +52,11 @@ export type Next =
| { do: "closeStage"; stageId: string }
| { do: "dispatch"; stageId: string; tasks: TaskRef[] }
| { do: "assignEntries" }
| { do: "diagnosisDedupe"; round: number }
| { do: "selectDiagnosis"; round: number }
| { do: "diagnosisNotRun"; round: number; reason: string }
| { do: "diagnosisRun"; round: number; run: string }
| { do: "diagnosisSettle"; round: number }
| { do: "dedupe"; round: number }
| { do: "freezeProtocols"; round: number }
| { do: "selectBatch"; round: number }
@@ -139,6 +144,28 @@ function roundStepNext(step: RoundStepId, round: number, view: View, frozen: Fro
case "carryOverPool":
// 不写记录:本轮名单和提案上限就是冻结的配置(方法.md 4.3 ①),留池候选由 materials 从记录算出(Board.carried)。
return null;
case "diagnosisProposal":
case "diagnosisPrediction":
// 诊 1、诊 2 → 这个阶段的关闭记录。余额不到 2 次(算到本轮开始前)本轮不诊断:不开阶段、不写记录。
return diagnosing(view, round, frozen) ? stageNext(stageIdOf(round, step), step, round, (roster) => roundTasks(step, round, roster, view.board), view, frozen, researchers(frozen)) : null;
case "diagnosisDedupe":
// 诊 2 判重合并 → 本轮的诊断实验判重合并决定(decision,种类 diagnosisDedup)。
if (!diagnosing(view, round, frozen)) return null;
return view.board.decision("diagnosisDedup", round) === undefined ? { do: "diagnosisDedupe", round } : null;
case "selectDiagnosis":
// 诊 3 → 本轮的选诊断实验决定(decision,种类 selectDiagnosis),定了不诊断也写。
if (!diagnosing(view, round, frozen)) return null;
return view.board.decision("selectDiagnosis", round) === undefined ? { do: "selectDiagnosis", round } : null;
case "diagnosisImplementation":
// 诊 4 实现与核验 → 最近一次核验里至少有一个实验通过,或修复次数用完而一个都不剩、有"不跑"事件。
return diagnosisCodeNext(round, view, frozen);
case "diagnosisRun":
// 诊 4 运行 → 本轮诊断运行最近一次有结果,而且不是"没启动"。
return diagnosisProgramOf(view.board, round, frozen) === null ? null : diagnosisRunNext(round, view);
case "diagnosisSettle":
// 诊 4 结算 → 本轮的诊断结算记录(diagnosisSettlement,按轮次唯一)。
if (diagnosisProgramOf(view.board, round, frozen) === null) return null;
return view.board.diagnosisSettlement(round) === undefined ? { do: "diagnosisSettle", round } : null;
case "proposal":
case "firstPrediction":
case "critique":
@@ -183,9 +210,6 @@ function roundStepNext(step: RoundStepId, round: number, view: View, frozen: Fro
// 解释 → 这一轮解释阶段的关闭记录。
const adopted = view.board.decision("adopt", round)?.output.adopted ?? null;
if (adopted === null || view.board.protocol(adopted).factors.length < 2) return null;
if (frozen.steps.round.some((s) => s.id === "listCells" && s.status === "skip")) {
return { do: "halt", reason: "配置关掉了消融、又采纳了多因子改法:不交三个数的解释还没写(cells 只按完整流程全开实现,做步骤开关时再补)", objects: [] };
}
return stageNext(stageIdOf(round, "explanation"), "explanation", round, (roster) => roster.map((actor) => ({ actor, kind: "explanation" as const, object: null })), view, frozen, researchers(frozen));
}
case "selectCells":
@@ -228,6 +252,7 @@ export function poolOf(board: Board, round: number): string[] {
/**
* 第 r 轮一个研究者阶段的任务(任务书 14 第 2.3 节)。同一个人的几个任务按这里的先后跑。
* - 诊断解释与实验、诊断预测:每人一个任务,没有对象;
* - 提案:每人一个任务,没有对象;
* - 第一次预测:每人 × 本轮每个新候选(含自己的;留池候选不做,方法.md 4.3 ④);
* - 挑刺:每人 × 本轮每条不是自己写的原提案(监督者定,照参考项目);
@@ -238,6 +263,9 @@ function roundTasks(step: RoundStageStep, round: number, roster: readonly string
const each = (kind: TaskRef["kind"], objects: readonly string[]): TaskRef[] =>
roster.flatMap((actor) => objects.map((object) => ({ actor, kind, object })));
switch (step) {
case "diagnosisProposal":
case "diagnosisPrediction":
return roster.map((actor) => ({ actor, kind: step, object: null }));
case "proposal":
return roster.map((actor) => ({ actor, kind: "proposal", object: null }));
case "firstPrediction":
@@ -318,7 +346,7 @@ const researchers = (frozen: Frozen): string[] => frozen.config.models.researche
function assignedStage(
step: string,
round: number,
kind: "implementation" | "repair" | "verification" | "ablationImplementation" | "ablationVerification",
kind: "implementation" | "repair" | "verification" | "ablationImplementation" | "ablationVerification" | "diagnosisImplementation" | "diagnosisRepair" | "diagnosisVerification",
candidates: readonly string[],
view: View,
frozen: Frozen,
@@ -355,6 +383,82 @@ function verificationNext(round: number, view: View, frozen: Frozen): Next | nul
return null;
}
/**
* 第 round 轮诊不诊断的前提(方法.md 4.3 诊断一段):余额至少 2 次,保证本轮至少还能跑 1 次正向。
* 余额只算本轮开始前的运行:本轮诊断运行自己扣了次数以后,诊断这一段的后几步不会反过来算成"不诊断"。
* 诊断关着的步骤清单把这几步都标了跳过,不会走到这里。
*/
function diagnosing(view: View, round: number, frozen: Frozen): boolean {
return runBalance(frozen.config.runs, view.board.runStatesBefore(round)).balance >= 2;
}
/**
* 诊 4 实现、核验与修复(方法.md 4.3 诊 4):一位实验员把选中的全部实验写进一份诊断程序,一位核验员逐个实验核对。
* 阶段编号按"第几次"排:诊断实现-1、核验-1,不通过的退回修复-1、再核验-2……第几次从已开关的阶段数出来,不另存;
* 每次核验都覆盖全部选中的实验。最近一次核验里还有实验不通过、修复次数又用完时,不通过的实验拿掉(diagnosisProgramOf);
* 一个都不剩才写"不跑"事件(方法.md 4.3 诊 4)。
*/
function diagnosisCodeNext(round: number, view: View, frozen: Frozen): Next | null {
if (chosenExperiments(view.board, round) === null) return null;
const object = diagnosisId(round);
for (let n = 1; ; n++) {
const work = n === 1
? assignedStage("diagnosisImplementation-1", round, "diagnosisImplementation", [object], view, frozen)
: assignedStage(`diagnosisRepair-${n - 1}`, round, "diagnosisRepair", [object], view, frozen);
if (work !== null) return work;
const verify = assignedStage(`diagnosisVerification-${n}`, round, "diagnosisVerification", [object], view, frozen);
if (verify !== null) return verify;
const verification = view.board.published("diagnosisVerification").filter((v) => v.round === round).at(-1)!;
if (diagnosisOutcome(verification.content).passed) return null;
if (n > frozen.config.maxRepairs) {
const { experiments, dropped } = diagnosisProgram(view.board, round, frozen.config.maxRepairs);
if (experiments.length > 0 || view.events(DIAGNOSIS_NOT_RUN).some((e) => e.objects.includes(object))) return null;
return { do: "diagnosisNotRun", round, reason: `选中的实验第 ${n} 次核验都不通过,修复次数上限 ${frozen.config.maxRepairs} 已用完:${dropped.map((d) => `${d.experiment}:${d.reason}`).join(";")}` };
}
}
}
/** 本轮诊断选中的实验(诊 3 的决定,按实验编号排);本轮没有诊断(余额不到 2 次、定了不诊断)为 null。 */
function chosenExperiments(board: Board, round: number): string[] | null {
const decision = board.decision("selectDiagnosis", round);
return decision === undefined || !decision.output.diagnose ? null : decision.output.chosen;
}
/**
* 诊 4 核验做完之后本轮诊断实际要跑的实验:选中的实验里最近一次核验通过的;不通过的实验拿掉,各带原因
* (记"证据不足:核验不通过")。拿掉实验后诊断程序的代码不动:核验员核过的就是这一份,只是运行时不调不通过的实验的编号。
* 只在诊断实现与核验这一步做完以后调用。
*/
export function diagnosisProgram(board: Board, round: number, maxRepairs: number): { experiments: string[]; dropped: Array<{ experiment: string; reason: string }> } {
const verifications = board.published("diagnosisVerification").filter((s) => s.round === round);
const last = verifications.at(-1)!.content;
const { failing } = diagnosisOutcome(last);
return {
experiments: board.decision("selectDiagnosis", round)!.output.chosen.filter((e) => !failing.includes(e)),
dropped: failing.map((experiment) => {
const x = last.experiments.find((y) => y.experiment === experiment)!;
const what = `${x.followed ? "" : "没照写法做;"}${x.nothingElse ? "" : "动了写法以外的地方;"}${x.where}`;
return { experiment, reason: `第 ${verifications.length} 次核验仍不通过,修复次数上限 ${maxRepairs} 已用完:${what}` };
}),
};
}
/** 本轮要跑的诊断实验:本轮没有诊断、或一个都没剩下为 null。 */
export function diagnosisProgramOf(board: Board, round: number, frozen: Frozen): string[] | null {
if (chosenExperiments(board, round) === null) return null;
const { experiments } = diagnosisProgram(board, round, frozen.config.maxRepairs);
return experiments.length === 0 ? null : experiments;
}
/** 诊 4 运行:和正向同一套,没启动的下次启动起新编号,已登记没结果的必须先由恢复入口核对。 */
function diagnosisRunNext(round: number, view: View): Next | null {
const runs = view.board.runs().filter((r) => r.plan.purpose === "diagnosis" && r.plan.round === round);
const last = runs.at(-1);
if (last !== undefined && last.result === null) throw new Error(`诊断运行 ${last.plan.run} 登记了还没有结果:恢复入口应当已经核对过`);
if (last !== undefined && last.result!.process.ending !== "notStarted") return null;
return { do: "diagnosisRun", round, run: `r${round}-diagnosis-${runs.length + 1}` };
}
/** ⑩:按选批顺序一次只交回一个运行;没启动的下次启动起新编号,不会把半批提前公开。 */
function forwardNext(round: number, view: View): Next | null {
for (const candidate of view.board.decision("batch", round)!.output.chosen) {
+6 -2
View File
@@ -43,9 +43,13 @@ export async function recover(s: Study): Promise<{ halt: string } | "stopped" |
return null;
}
/** 恢复和第一次运行共用原登记指标定义(引用指标也保留原来的缺失规则);消融格缺失一律只记缺失(方法.md 第五部分"指标")。 */
function metricsOf(s: Study, plan: Content<"runPlanned">): HypothesisMetric[] {
/**
* 恢复和第一次运行共用原登记指标定义(引用指标也保留原来的缺失规则);消融格缺失一律只记缺失(方法.md 第五部分"指标")。
* 诊断运行为 null:运行记录不带判分,逐个实验的判分在诊断结算前做。
*/
function metricsOf(s: Study, plan: Content<"runPlanned">): HypothesisMetric[] | null {
if (plan.purpose === "baseline") return [];
if (plan.purpose === "diagnosis") return null;
const board = new Board(s.ledger.records());
if (plan.purpose === "forward") return protocolMetrics(board, board.protocol(plan.object!));
return ablationMetrics(board, splitCell(plan.object!).candidate);
+165
View File
@@ -0,0 +1,165 @@
import { judgeSameExperiment, type DedupExperiment } from "../../jev/index.ts";
import { Board, diagnosisId, diagnosisKey, type DiagnosisDedupOutput } from "../../materials/index.ts";
import {
compareIds,
runBalance,
runDiagnosisRule,
settleDiagnosis,
type DiagnosisExplanation,
type DiagnosisRuleInput,
type DiagnosisRuleName,
type ExperimentOutcome,
type SettleDiagnosisInput,
} from "../../rules/index.ts";
import type { DiagnosisExperiment, DiagnosisJudged } from "../../tasks/index.ts";
import type { Json } from "../../vocab/index.ts";
import { diagnosisFormalRun, judgeDiagnosisRun, type RunOutcome } from "../formal-run.ts";
import { diagnosisProgram, diagnosisProgramOf } from "../next-step.ts";
import { baselineRunDir } from "../task-code.ts";
import type { Study } from "../study.ts";
import type { SamePair } from "./round.ts";
// 诊断这一段里程序自己做的四步(方法.md 4.3 诊 2–诊 4):判重合并、选诊断实验、诊断运行、诊断结算。
// 判重合并和选实验各写一条决定,结算写一条诊断结算,写下之前被打断就从账本重新算一遍;写下之后恢复时照原样用,不重算。
/** 一个作者交的一个诊断实验:标签 "<作者>.e<第几个>",只在判重合并的输入里认实验用。 */
type Item = { author: string; index: number; label: string; experiment: DedupExperiment };
/**
* 诊 2 判重合并(方法.md 4.3):本轮已公开的诊断实验两两问一次 Jev,按作者编号、再按第几个排好顺序发,
* 全部问完才写决定:中途被打断就整轮重问,判重不扣运行次数。
*/
export async function diagnosisDedupe(s: Study, round: number): Promise<void> {
const board = new Board(s.ledger.records());
const model = s.frozen.config.models.jev;
const items: Item[] = board
.published("diagnosisProposal")
.filter((p) => p.round === round)
.sort((a, b) => compareIds(a.author, b.author))
.flatMap((p) => p.content.experiments.map((experiment, index) => ({ author: p.author, index, label: `${p.author}.e${index + 1}`, experiment })));
const pairs: SamePair[] = [];
for (const [i, a] of items.entries()) {
for (const b of items.slice(i + 1)) pairs.push({ a: a.label, b: b.label, judgment: await judgeSameExperiment(model, a.experiment, b.experiment) });
}
// SAFETY: JevJudgment 和 DiagnosisDedupOutput 都是纯 Json 值;账本在写入时校验 Json。
s.ledger.append("decision", {
decision: "diagnosisDedup",
round,
rule: "jevSameExperiment",
ruleVersion: 1,
params: { jevModel: model },
seed: null,
input: { pairs } as unknown as Json,
output: groupSameExperiment(items, pairs) as unknown as Json,
});
}
/**
* 判为相同的连成一组(同判重合并:A 和 B 相同、B 和 C 相同,A、B、C 就是一个实验)。每组合成一个实验,
* 按组里排最前的那一个排序,编号用 materials 的 diagnosisKey(d1…);组内按作者编号、再按第几个排,第一个的正文就是合并后的正文。
* items 要先按作者编号、第几个排好。
*/
export function groupSameExperiment(items: readonly Pick<Item, "author" | "index" | "label">[], pairs: readonly SamePair[]): DiagnosisDedupOutput {
const parent = new Map(items.map((x) => [x.label, x.label]));
const root = (x: string): string => {
while (parent.get(x) !== x) x = parent.get(x)!;
return x;
};
for (const p of pairs) if (p.judgment.yes) parent.set(root(p.a), root(p.b));
const groups = new Map<string, Array<{ author: string; index: number }>>();
for (const x of items) groups.set(root(x.label), [...(groups.get(root(x.label)) ?? []), { author: x.author, index: x.index }]);
return { experiments: [...groups.values()].map((sources, i) => ({ experiment: diagnosisKey(i), sources })) };
}
/**
* 诊 3 选诊断实验(方法.md 6.12):基线、方向、λ、合并后的全部实验、每条解释对每个实验的三个数,加余额(本轮开始前)、
* 每批上限、一次诊断最多几个实验 D、u、衰减,交给步骤清单里 selectDiagnosis 那一步的规则,写一条决定。
* 规则没有随机,种子写 null;定了不诊断也写。
*/
export function selectDiagnosisExperiments(s: Study, round: number): void {
const board = new Board(s.ledger.records());
const { config, facts, steps } = s.frozen;
const { diagnosis } = config.flow;
if (!diagnosis.enabled) throw new Error("配置关掉了诊断,步骤清单里选诊断实验一步应当标了跳过");
const input: DiagnosisRuleInput = {
baseline: board.baselineScore(board.baselineOf(round)),
direction: facts.finalMetric.direction,
lambda: config.flow.openReserve,
experiments: board.diagnosisExperiments(round).map((e) => e.id),
explanations: predictionsOf(board, round),
balance: runBalance(config.runs, board.runStatesBefore(round)).balance,
batchCap: config.batchLimit,
maxExperiments: diagnosis.maxExperiments,
u: config.flow.knowledgeUse,
decay: config.flow.decay,
};
const rule = steps.round.find((x) => x.id === "selectDiagnosis")!.rule!.name as DiagnosisRuleName;
const { rule: used, ...output } = runDiagnosisRule(rule, input);
// SAFETY: rules 是纯计算,输入输出都是纯 Json 值;账本在写入时校验 Json。
s.ledger.append("decision", {
decision: "selectDiagnosis",
round,
rule: used.name,
ruleVersion: used.version,
params: output.params,
seed: null,
input: input as unknown as Json,
output: output as unknown as Json,
});
}
/** 这一轮已冻结的各条诊断解释:作者的第 i 条(materials 的 diagnosisExplanationId),每条对每个合并后的实验给的三个数。选实验和结算共用。 */
function predictionsOf(board: Board, round: number): DiagnosisExplanation[] {
const predictions = board.published("diagnosisPrediction").filter((p) => p.round === round);
return board.diagnosisExplanations(round).map((x) => ({
id: x.id,
author: x.author,
experiments: predictions.find((p) => p.author === x.author)!.content.predictions.filter((y) => y.explanation === x.id).map((y) => ({ experiment: y.experiment, range: y.range })),
}));
}
/** 要跑的实验的运行输入:编号加它登记的假设专用指标(全名),次序同 ids,也就是运行的次序。 */
function experimentsOf(board: Board, round: number, ids: readonly string[]): DiagnosisExperiment[] {
const merged = board.diagnosisExperiments(round);
return ids.map((id) => ({ id, metrics: merged.find((e) => e.id === id)!.metrics }));
}
/**
* 诊 4 诊断运行:用核验过的诊断程序,复用正式运行的登记、扣次入口,对象是本轮诊断。跑的只是核验里通过的实验:
* 不通过而拿掉的实验,代码留在程序里不动(核验员核过的就是这一份),运行时不调它的编号;每个实验分到总时限 ÷ 跑的实验数。
*/
export function diagnosisRun(s: Study, round: number, run: string): Promise<RunOutcome> {
const board = new Board(s.ledger.records());
const ran = diagnosisProgramOf(board, round, s.frozen)!;
const version = board.code(diagnosisId(round)).version;
return diagnosisFormalRun(s, { run, purpose: "diagnosis", round, object: diagnosisId(round) }, version, baselineRunDir(s, board, round), experimentsOf(board, round, ran));
}
/**
* 诊 4 结算(方法.md 4.3 诊 4、6.12):先从运行目录给每个交出答案的实验单独判分,再照 SettleDiagnosisInput 拼输入,
* 写一条诊断结算。每个合并后的实验的结局:跑了且有效分数是有效;跑了没有效分数、或核验不通过拿掉的是证据不足;没选中的是没做。
* 运行记录不带判分,所以被打断在判分和写结算之间时,恢复后从运行目录重新判。
*/
export async function diagnosisSettle(s: Study, round: number): Promise<void> {
const board = new Board(s.ledger.records());
const { experiments: ran, dropped } = diagnosisProgram(board, round, s.frozen.config.maxRepairs);
const run = board.runs().filter((r) => r.plan.purpose === "diagnosis" && r.plan.round === round).at(-1)!;
const judged = await judgeDiagnosisRun(s, run.plan.run, run.result!.process, experimentsOf(board, round, ran));
const input: SettleDiagnosisInput = {
baseline: board.baselineScore(board.baselineOf(round)),
direction: s.frozen.facts.finalMetric.direction,
epsilon: s.frozen.facts.epsilon,
lambda: s.frozen.config.flow.openReserve,
experiments: board.diagnosisExperiments(round).map((e) => ({ experiment: e.id, outcome: outcomeOf(e.id, ran, dropped.map((d) => d.experiment), judged) })),
explanations: predictionsOf(board, round),
};
// SAFETY: 判分结果、结算的输入输出都是纯 Json 值;账本在写入时校验 Json。
s.ledger.append("diagnosisSettlement", { round, result: { run: run.plan.run, judged, dropped, input, output: settleDiagnosis(input) } as unknown as Json });
}
function outcomeOf(experiment: string, ran: readonly string[], dropped: readonly string[], judged: DiagnosisJudged): ExperimentOutcome {
if (dropped.includes(experiment)) return { kind: "insufficient" };
if (!ran.includes(experiment)) return { kind: "untested" };
const j = judged.experiments.find((x) => x.id === experiment)!;
return j.verdict === "valid" && j.score.status === "measured" ? { kind: "valid", score: j.score.value } : { kind: "insufficient" };
}
+1
View File
@@ -94,6 +94,7 @@ function studyFacts(frozen: Frozen, task: Task): StudyFacts {
lambda: config.flow.openReserve,
totalRuns: config.runs,
proposalLimit: config.proposalLimit,
diagnosisExperimentLimit: config.flow.diagnosis.enabled ? config.flow.diagnosis.perResearcher : null,
factorLimit: config.factorLimit,
metrics: metricDefinitions(task),
taskText: task.guide,
+41 -3
View File
@@ -1,19 +1,20 @@
import { join } from "node:path";
import type { Content } from "../ledger/index.ts";
import { codeHunks } from "../jev/index.ts";
import { Board, protocolMetrics, splitCell, type CodeChanges } from "../materials/index.ts";
import { Board, diagnosisId, protocolMetrics, splitCell, type CodeChanges } from "../materials/index.ts";
import type { KnownOf } from "../submissions/index.ts";
import { lightCheck, protectedChanges, readLightCheck } from "../tasks/index.ts";
import { diagnosisUnversionedNames, lightCheck, protectedChanges, readLightCheck } from "../tasks/index.ts";
import type { Json, Submission, SubmissionKind } from "../vocab/index.ts";
import { compareVersions, saveVersion } from "../workspace/index.ts";
import type { TaskRef } from "./next-step.ts";
import { callPaths, containerName, freshDir, workspaceRepo } from "./paths.ts";
import { callPaths, containerName, freshDir, runPaths, workspaceRepo } from "./paths.ts";
import type { Study } from "./study.ts";
/**
* 实现取本批基线;修复取上次提交。核验、修复的材料差异始终相对本批基线。
* 消融实现第一次取被采纳的完整代码,重做取上一次交的这一格代码(materials 的消融实现材料就是这样写的);
* 消融核验的材料差异是完整代码到这一格代码(方法.md 4.3 ⑯)。
* 诊断实现取当前基线;诊断修复取上次提交的诊断程序;诊断核验、修复的材料差异是当前基线到诊断程序(方法.md 4.3 诊 4)。
*/
export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes: CodeChanges | null; startVersion: string | null } {
const board = new Board(s.ledger.records());
@@ -26,6 +27,13 @@ export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes:
const { candidate } = splitCell(t.object!);
return { changes: compareVersions(workspaceRepo(s.dir), board.code(candidate).version, board.code(t.object!).version), startVersion: null };
}
if (t.kind === "diagnosisImplementation" || t.kind === "diagnosisRepair" || t.kind === "diagnosisVerification") {
const round = board.stage(t.stageId).round;
const baseline = board.code(board.baselineOf(round)).version;
if (t.kind === "diagnosisImplementation") return { changes: null, startVersion: baseline };
const program = board.code(t.object!).version;
return { changes: compareVersions(workspaceRepo(s.dir), baseline, program), startVersion: t.kind === "diagnosisRepair" ? program : null };
}
if (t.kind !== "implementation" && t.kind !== "repair" && t.kind !== "verification") return { changes: null, startVersion: null };
const baseline = board.code(board.baselineOf(board.stage(t.stageId).round)).version;
if (t.kind === "implementation") return { changes: null, startVersion: baseline };
@@ -33,6 +41,18 @@ export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes:
return { changes: compareVersions(workspaceRepo(s.dir), baseline, version), startVersion: t.kind === "repair" ? version : null };
}
/** 产出第 round 轮当前基线的那次运行的运行目录:诊断实验要接着算的原始产物就在里面。 */
export function baselineRunDir(s: Study, board: Board, round: number): string {
return runPaths(s.dir, board.runOf(board.baselineOf(round))!.plan.run).run;
}
/** 诊断实现、修复的工作目录要放进的原始产物所在的运行目录(取出代码之后,tasks 的 placeBaselineArtifact 放);别的任务为 null。 */
export function taskArtifact(s: Study, t: TaskRef & { stageId: string }): string | null {
if (t.kind !== "diagnosisImplementation" && t.kind !== "diagnosisRepair") return null;
const board = new Board(s.ledger.records());
return baselineRunDir(s, board, board.stage(t.stageId).round);
}
/**
* 提交时才知道的东西:提案份数、回应原提案,或实验员当前工作目录的快照和相对本批基线的改动。
* 快照写入 workspace 后才问 Jev;收下才把 code 和 submission 在同一个事务写进账本。
@@ -61,6 +81,24 @@ export function prepareSubmission(
code: { object: t.object!, version: saved.version, locations: null },
};
}
if (t.kind === "diagnosisImplementation" || t.kind === "diagnosisRepair") {
// 诊断程序:存的名单多跳过宿主放进来的原始产物目录;Jev 对照的是选中实验的原文和要采集的指标。
const saved = saveVersion(workspaceRepo(s.dir), workDir, diagnosisUnversionedNames(s.task));
if (!saved.saved) return { known: { code: saved }, code: null };
const round = board.stage(t.stageId).round;
const changes = compareVersions(workspaceRepo(s.dir), board.code(board.baselineOf(round)).version, saved.version);
const chosen = board.diagnosisExperiments(round).filter((e) => board.decision("selectDiagnosis", round)!.output.chosen.includes(e.id));
const impl = content as Submission<"diagnosisImplementation">;
return {
known: { code: {
saved: true,
protectedFiles: protectedChanges(s.task, changes.files.map((f) => f.path)),
hunks: codeHunks(changes),
protocol: { body: chosen.map((e) => `### ${e.id} ${e.name}\n\n动哪一处:${e.spot}\n\n${e.body}`).join("\n\n"), metrics: chosen.flatMap((e) => e.metrics) },
} },
code: { object: t.object!, version: saved.version, locations: impl.locations as Json },
};
}
if (t.kind !== "implementation" && t.kind !== "repair") return { known: null, code: null };
const saved = saveVersion(workspaceRepo(s.dir), workDir, s.frozen.facts.unversionedNames);
if (!saved.saved) return { known: { code: saved }, code: null };
+4
View File
@@ -1,11 +1,15 @@
// jev:问 Jev 的问法和阈值(docs/结构.md 第二部分)。判断的类型 JevJudgment 在 vocab。
export {
checkExperiment,
checkHunk,
checkProtocol,
codeHunks,
judgeSameChange,
judgeSameExperiment,
type CodeHunk,
type DedupExperiment,
type DedupFactor,
type DiagnosisExperiment,
type FrozenProtocol,
type JevCheck,
type ProtocolMetric,
+37 -1
View File
@@ -1,9 +1,10 @@
import type { HypothesisMetric, JevJudgment, Json, Submission } from "../vocab/index.ts";
import { questionsOf, taskOf, type Question } from "./questions.ts";
// 问 Jev:从带类型的输入拼材料、发请求、读回答(docs/方法.md 4.3 ③⑥⑨)。
// 问 Jev:从带类型的输入拼材料、发请求、读回答(docs/方法.md 4.3 ③⑥⑨ 和诊 1、诊 2)。
// 材料里放什么、字段叫什么只在这里定。分请求照参考项目 src/judge/judge.ts:
// 判重一对一次请求;冻结条款三问每问单独一次请求,材料相同;代码改动一个差异块一次请求,五问同在。
// 诊断实验的判重、冻结检查照判重、冻结条款的写法;诊断程序的代码改动直接用 checkHunk。
type Factor = Submission<"proposal">["factors"][number];
@@ -13,6 +14,12 @@ type Factor = Submission<"proposal">["factors"][number];
*/
export type DedupFactor = Pick<Factor, "name" | "method" | "keySettings">;
/** 判重看的诊断实验:名称、动哪一处、怎么动。能分开哪几条解释写在正文里,不影响"是不是同一个实验"。 */
export type DedupExperiment = Pick<Submission<"diagnosisProposal">["experiments"][number], "name" | "spot" | "body">;
/** 冻结检查看的诊断实验:名称、动哪一处、怎么动,加登记的假设专用指标(编号是短名)。 */
export type DiagnosisExperiment = DedupExperiment & Pick<Submission<"diagnosisProposal">["experiments"][number], "metrics">;
/** 一份冻结协议的条款(docs/方法.md 4.3 ⑥"冻结协议包括"),字段与统一提交相同。 */
export type FrozenProtocol = Pick<Submission<"unification">, "factors" | "safeguards" | "metrics" | "body" | "clauses">;
@@ -49,6 +56,31 @@ export async function judgeSameChange(model: string, a: readonly DedupFactor[],
return judgment as JevJudgment;
}
/** 诊 2 判重:两个诊断实验是不是同一个实验。 */
export async function judgeSameExperiment(model: string, a: DedupExperiment, b: DedupExperiment): Promise<JevJudgment> {
const state = {
task: taskOf("diagnosisDedup"),
experiment_a: dedupExperiment(a),
experiment_b: dedupExperiment(b),
};
const [judgment] = await ask(model, state, questionsOf("diagnosisDedup"));
return judgment as JevJudgment;
}
/** 诊 1 查一个诊断实验:改动了评测、自相矛盾、写得太含糊。三问逐个发,材料相同。 */
export async function checkExperiment(model: string, experiment: DiagnosisExperiment): Promise<JevCheck> {
const state = {
task: taskOf("diagnosisFreeze"),
experiment: {
...dedupExperiment(experiment),
metrics: experiment.metrics.map((m) => ({ id: m.id, howMeasured: m.howMeasured, missing: m.missing })),
},
};
const judgments: JevJudgment[] = [];
for (const question of questionsOf("diagnosisFreeze")) judgments.push(...(await ask(model, state, [question])));
return check(judgments);
}
/** ⑥ 查一份冻结协议的条款:改动了评测、自相矛盾、写得太含糊。三问逐个发,材料相同。 */
export async function checkProtocol(model: string, protocol: FrozenProtocol): Promise<JevCheck> {
const state = {
@@ -83,6 +115,10 @@ function dedupFactor(f: DedupFactor): { [key: string]: Json } {
return { name: f.name, method: f.method, keySettings: f.keySettings };
}
function dedupExperiment(e: DedupExperiment): { [key: string]: Json } {
return { name: e.name, spot: e.spot, body: e.body };
}
function check(judgments: JevJudgment[]): JevCheck {
return { flagged: judgments.some((j) => j.yes), judgments };
}
+136 -3
View File
@@ -1,13 +1,17 @@
import { hash, type Json } from "../vocab/index.ts";
// Jev 的九问(docs/方法.md 4.3 ③⑥⑨):每一处的材料说明、问题原文、阈值。
// Jev 的十三问(docs/方法.md 4.3 ③⑥⑨ 和诊 1、诊 2):每一处的材料说明、问题原文、阈值。
// 措辞、判"是"判"否"的说明和示例句取自参考项目 src/judge/questions.ts(填圆那一套),
// 只改了必须改的:点 → 因子,字段名对上 vocab 的提案和冻结协议,改动对上 workspace 的比较结果。
// 写法照参考项目 research-wiki/参考-Jev使用指南.md:问题用英文、一问只问一件事、
// 统一问坏的一面(概率高 = 有问题或"是同一个")、判据带 not_for 和例子、问题里用反引号点到材料路径。
/** 用到 Jev 的三处:判重、冻结条款、一段代码改动。 */
export type Site = "dedup" | "freeze" | "diff";
/**
* 用到 Jev 的五处:判重、冻结条款、一段代码改动,以及诊断实验的判重和冻结检查。
* 诊断的两处材料是"一个诊断实验"(动当前基线的一处),不是候选提案和冻结协议,所以各有自己的材料说明和问法
* (方法.md 4.3 诊 1、诊 2);一段代码改动的五问直接沿用 diff(诊 4)。
*/
export type Site = "dedup" | "freeze" | "diff" | "diagnosisDedup" | "diagnosisFreeze";
/** 一条是非题(Jev 的 noul)。 */
export interface Noul {
@@ -24,6 +28,9 @@ const THRESHOLDS: Readonly<Record<Site, Readonly<Record<string, number>>>> = {
dedup: { same_change: 0.3 },
freeze: { self_contradictory: 0.5, changes_measurement: 0.21, too_vague: 0.78 },
diff: { unseeded_randomness: 0.55, reads_clock: 0.55, touches_measurement: 0.65, early_exit: 0.63, zero_for_unmeasured: 0.62 },
// 诊断实验的两处阈值暂用 dedup、freeze 的值,没有在诊断实验的材料上校准过,等真实数据(任务书 18 第三块)再校准。
diagnosisDedup: { same_experiment: 0.3 },
diagnosisFreeze: { self_contradictory: 0.5, changes_measurement: 0.21, too_vague: 0.78 },
};
/** 放进材料 `task` 字段的说明:这是什么材料、各部分是什么。 */
@@ -44,6 +51,18 @@ const TASKS: Readonly<Record<Site, string>> = {
"`protocol.metrics` lists the metrics the protocol requires the implementer " +
"to compute in the candidate code and write to metrics.json (each with its `howMeasured` and `missing` rule; an empty list means none). " +
"Only this hunk is shown; judge only what the hunk itself adds or changes.",
diagnosisDedup:
"Two diagnosis experiments from the same round of a study that improves a program by changing its code. " +
"A diagnosis experiment takes the current baseline program, changes ONE spot of it (`spot`) as `body` describes " +
"(replace a step with a clearly stronger version, replace it with another method, remove it, or relax a restriction), " +
"runs the changed program once and scores the answer it returns. " +
"The experiments were written by different researchers, in Chinese or English.",
diagnosisFreeze:
"One diagnosis experiment in a study that improves a program by changing its code. " +
"An implementer will change the baseline program at `experiment.spot` exactly as `experiment.body` says and nothing else; " +
"the unchanged task harness will then run the changed program once, and the task's validator will score the answer it returns. " +
"The implementer may not edit the task's harness or validator files. " +
"`experiment.metrics` lists extra diagnostic metrics the changed program must record (an empty list means none).",
};
const QUESTIONS: Readonly<Record<Site, Readonly<Record<string, Noul>>>> = {
@@ -292,6 +311,120 @@ const QUESTIONS: Readonly<Record<Site, Readonly<Record<string, Noul>>>> = {
},
},
},
// 诊 2 诊断实验判重:一对实验一次请求,写法照 dedup。
diagnosisDedup: {
same_experiment: {
type: "noul",
instructions: {
question:
"Do `experiment_a` and `experiment_b` change the same spot of the baseline in the same way, so that one run would answer both?",
focus:
"Compare where each experiment changes the baseline (`spot`) and how (`body`): the kind of change (stronger version, other method, removal, relaxed restriction) and its settings. " +
"Ignore wording, language, level of detail, the experiment's name, and which explanation it says it can separate from which. " +
"The same spot changed in a different way, or the same change with different settings or sizes, is NOT the same experiment: each would need its own run.",
},
criteria: {
true: {
what: "Both change the same spot of the baseline in the same way with the same settings; they differ only in wording, language or detail",
examples: [
"'replace the optimizer step with an exact solver, no time limit' vs 'swap the optimizer for an exact solve without any time cap'",
"'remove the final repair pass' vs 'delete the repair pass at the end of the pipeline'",
],
},
false: {
what: "They change different spots, change the same spot in a different way, or use the same change with different settings",
not_for: "Two experiments that merely test the same explanation, or merely touch the same function at different spots",
examples: [
"'remove the repair pass' vs 'replace the repair pass with an exact solver'",
"'run the search for 10x more steps' vs 'run the search for 100x more steps'",
"'relax the iteration limit of the search' vs 'relax the time limit of the whole program'",
],
},
},
},
},
// 诊 1 诊断实验冻结检查:每问单独一次请求,材料相同;问法照 freeze,材料换成"一个诊断实验"。
diagnosisFreeze: {
self_contradictory: {
type: "noul",
instructions: {
question:
"Does `experiment` ask for something that cannot be done as written: do two of its statements exclude each other, or does `experiment.body` change places other than the one `experiment.spot` names?",
focus:
"Take the change described in `experiment.body` and check it against `experiment.spot`: the experiment must change that one spot and keep everything else of the baseline as it is. " +
"A change that cannot be carried out without also changing something else, or statements that require things that cannot both hold, make it contradictory. " +
"A statement that is merely unusual, ambitious or likely to give a worse score is not a contradiction.",
},
criteria: {
true: {
what: "Carrying out the experiment necessarily changes more than the named spot, or two statements require things that cannot both hold",
examples: [
"`spot` names the radius computation while `body` also asks to replace the starting layout",
"`body` says remove the step and also tune its step size",
"`body` says change only the solver and also keep the solver's code unchanged",
],
},
false: {
what: "All statements can hold at the same time in one implementation that changes only the named spot",
not_for: "Experiments that may not improve the score, or statements that are vague but not conflicting",
examples: ["replace the solver with an exact LP and keep everything else as in the baseline", "remove the final repair pass"],
},
},
},
changes_measurement: {
type: "noul",
instructions: {
question:
"Does `experiment` require changing how the answer is scored or checked, rather than changing the solution that gets scored?",
focus:
"The answer of the changed program is scored by the task's validator and harness. " +
"Changing the algorithm that produces the answer is allowed, including relaxing a limit or a check inside the program itself, as long as the returned answer is still scored as before. " +
"Changing how the returned value is summed, scored, validated, rounded or reported, or the validator or harness themselves, is a change of measurement. " +
"Rules for the experiment's own diagnostic metrics (what they count, how they are aggregated) do not change the score.",
},
criteria: {
true: {
what: "The experiment asks to alter the scoring or validation: the formula of the score, what gets counted, tolerances, the harness or validator, or the value reported instead of the real one",
examples: [
"report the sum of diameters instead of the sum of radii",
"count only items above a size threshold when summing",
"raise the overlap tolerance in the validator",
],
},
false: {
what: "The experiment only changes how the answer is built or searched; the score is computed from its output as before",
not_for: "Adding diagnostic metrics that are recorded next to the score without replacing it",
examples: ["replace the greedy step with an exact solve", "remove the repair pass and see what the raw answer scores"],
},
},
},
too_vague: {
type: "noul",
instructions: {
question:
"Does `experiment` leave the implementer to guess what to change, for example which spot of the baseline to edit, what to replace it with, or the settings of the stronger version?",
focus:
"Read `experiment.spot` and `experiment.body` as an implementer who must follow them without asking questions. " +
"Brief text is fine when it names the spot, the kind of change and its settings. " +
"Judge only whether each statement is specific; whether two statements conflict is checked separately.",
},
criteria: {
true: {
what: "An implementer would have to invent a key part of the experiment: the spot to change, what to put there, or its main settings",
examples: ["make the search better somewhere", "replace the optimizer with a stronger one", "relax some of the limits"],
},
false: {
what: "The spot, the kind of change and its settings are stated concretely enough to implement directly",
not_for: "Experiments that leave only minor coding details, such as variable names, to the implementer",
examples: [
"in the radius step, replace the heuristic with an exact LP solved by scipy linprog; no time limit",
"remove the final repair pass and return the raw answer",
],
},
},
},
},
};
/** 一问的全部内容:问题原文、阈值,以及材料说明、问题原文、阈值三者合起来的哈希。 */
+7 -5
View File
@@ -17,7 +17,7 @@ import {
type SubmissionKind,
} from "../vocab/index.ts";
// 账本记录的种类(docs/结构.md 第四部分的 11 种记录,细分成下面 19 种),每种一份内容定义(TypeBox)。
// 账本记录的种类(docs/结构.md 第四部分的 11 种记录,细分成下面 20 种),每种一份内容定义(TypeBox)。
// 下面写"只有一条""只一次"的,由 schema.ts 的唯一约束保证。
// 配置、冻结协议、决定、结算的内容由别的模块产生,这里只检查最少的标识字段,其余只检查是 Json。
@@ -131,7 +131,7 @@ export const KINDS = {
submission: Type.Unsafe<StoredSubmission>({}),
/**
* 3 退回:被退回的那份内容原样存(它可能正是因为不合格式才被退回),连同退回理由。
* target 是这次任务的对象,与提交的 target 同义(候选、原提案、消融格,没有对象的写 null):
* target 是这次任务的对象,与提交的 target 同义(候选、原提案、消融格、本轮诊断,没有对象的写 null):
* 同一人在同一阶段可能对几个对象各做一次同种任务,重新发起时按它找本人这个对象的上一次退回。
* jev 是这次退回用到的 Jev 判断(见 JevUsed),判"否"的几问也一起存;没用到 Jev 的写 null。
*/
@@ -161,11 +161,11 @@ export const KINDS = {
/** 6 冻结代码:候选或消融格的代码版本号(由 workspace 给出)和因子位置表。位置表先存 Json。 */
code: Type.Object({ object: Id, version: Text, locations: AnyJson }, CLOSED),
/** 7 运行·登记要跑(占名额):运行编号由 flow 起,一个编号只登记一次;用途、轮次、对象(候选或格;基线没有)、运行定义。 */
/** 7 运行·登记要跑(占名额):运行编号由 flow 起,一个编号只登记一次;用途、轮次、对象(候选、格或本轮诊断;基线没有)、运行定义。 */
runPlanned: Type.Object(
{
run: Id,
purpose: Type.Enum(["baseline", "forward", "ablation"], { type: "string" }),
purpose: Type.Enum(["baseline", "forward", "ablation", "diagnosis"], { type: "string" }),
round: Round,
object: OptionalId,
definition: RunDefinition,
@@ -178,7 +178,7 @@ export const KINDS = {
runResult: Type.Object({ run: Id, process: ProcessResult, judged: Type.Union([JudgedResult, Type.Null()]) }, CLOSED),
/**
* 8 决定:选批、采纳、选格、押的因子、入口分配、判重合并。decision 是决定种类。
* 8 决定:选诊断实验、选批、采纳、选格、押的因子、入口分配、判重合并(提案和诊断实验)。decision 是决定种类。
* 输入输出存 Json:ledger 不认识 rules。判重合并的输入里带 Jev 的原始概率。
*/
decision: Type.Object(
@@ -197,6 +197,8 @@ export const KINDS = {
/** 9 结算:每轮最多采纳一个改法,所以最多一次消融,结算按轮次一轮一条。各格结局、配合关系、解释的确认或否定等先存 Json。 */
settlement: Type.Object({ round: Round, result: AnyJson }, CLOSED),
/** 9 诊断结算:每轮最多一次诊断,结算也按轮次一轮一条(方法.md 6.12)。每个实验的结局与实测、解释的确认或否定等先存 Json。 */
diagnosisSettlement: Type.Object({ round: Round, result: AnyJson }, CLOSED),
/** 10 模型调用开始:先登记再调用(方法.md 第九部分)。附这次任务的全部材料,用来重现模型当时看到了什么。一次调用只有一条。 */
callStarted: Type.Object({ call: Id, actor: Id, stageId: OptionalId, model: Text, materials: AnyJson }, CLOSED),
+13 -5
View File
@@ -83,7 +83,7 @@ export class Ledger {
}
/**
* 追加一条记录,返回存下的那一条。先按种类检查内容,再做写入时的四种拒收检查(rejections.ts)。
* 追加一条记录,返回存下的那一条。先按种类检查内容,再做写入时的五种拒收检查(rejections.ts)。
* 写提案时,作者自评从提案里拆出来,在同一个事务里另存成一条封存,返回的是提案那一条。
*/
append<K extends AppendKind>(kind: K, content: Input<K>): LedgerRecord<K> {
@@ -117,16 +117,24 @@ export class Ledger {
this.#first("stageOpen", "json_extract(content, '$.stageId') = ?", stageId) as LedgerRecord<"stageOpen"> | undefined,
latest: (slot) => this.latestSubmission(slot),
runStarted: (run) => this.#first("runStarted", "json_extract(content, '$.run') = ?", run) !== undefined,
ablationStarted: (round) =>
ablationStarted: (round) => this.#purposeStarted("ablation", round),
diagnosisStarted: (round) => this.#purposeStarted("diagnosis", round),
};
/** 这一轮有没有某种用途的运行已经确认启动。 */
#purposeStarted(purpose: "ablation" | "diagnosis", round: number): boolean {
return (
this.#one(
`SELECT s.seq FROM records s JOIN records p
ON p.kind = 'runPlanned' AND json_extract(p.content, '$.run') = json_extract(s.content, '$.run')
WHERE s.kind = 'runStarted'
AND json_extract(p.content, '$.purpose') = 'ablation'
AND json_extract(p.content, '$.purpose') = ?
AND json_extract(p.content, '$.round') = ?`,
purpose,
round,
) !== undefined,
};
) !== undefined
);
}
// -------------------------------------------------------------------------
// 读取
+17 -5
View File
@@ -1,8 +1,8 @@
import { SUBMISSION_KINDS, type Stage } from "../vocab/index.ts";
import type { Content, Kind, LedgerRecord, SubmissionSlot } from "./kinds.ts";
// 写入时由 ledger 拒绝的四种情况(docs/结构.md 第四部分"写入时由 ledger 拒绝的几种情况"),在写入的同一个事务里查。
// 只有这四种。记录之间别的先后(例如运行结果前要先有登记)和提交种类与阶段对不对得上都不查:
// 写入时由 ledger 拒绝的五种情况(docs/结构.md 第四部分"写入时由 ledger 拒绝的几种情况"),在写入的同一个事务里查。
// 只有这五种。记录之间别的先后(例如运行结果前要先有登记)和提交种类与阶段对不对得上都不查:
// flow 调 ledger 是模块之间的调用,不是系统边界。"只发生一次"由 schema.ts 的唯一约束管。
// 提交内容的字段归 submissions,该不该开下一个阶段归 flow。
@@ -16,14 +16,18 @@ export interface WriteState {
runStarted(run: string): boolean;
/** 这一轮有没有消融格的运行已经确认启动。 */
ablationStarted(round: number): boolean;
/** 这一轮的诊断运行有没有已经确认启动。 */
diagnosisStarted(round: number): boolean;
}
/** 实现和核验阶段的提交不能替换(方法.md 第五部分总原则):实现、修复、核验、消融实现、消融核验。 */
/** 实现和核验阶段的提交不能替换(方法.md 第五部分总原则):实现、修复、核验、消融实现、消融核验、诊断实现(含修复)、诊断核验。 */
const NOT_REPLACEABLE: ReadonlySet<Stage> = new Set([
"implementation",
"verification",
"ablationImplementation",
"ablationVerification",
"diagnosisImplementation",
"diagnosisVerification",
]);
export function checkWrite(kind: Kind, content: unknown, state: WriteState): void {
@@ -35,7 +39,7 @@ export function checkWrite(kind: Kind, content: unknown, state: WriteState): voi
/**
* 第 1 条:只收开着的阶段的提交,即有开阶段记录、没有关阶段记录。
* 第 2 条:新的一版必须正好接在本人在这个阶段、对同一个对象的最新一版后面(第一版是 1);
* 实现、修复、核验、消融实现、消融核验只能交第一版。
* 实现、修复、核验、消融实现、消融核验、诊断实现、诊断修复、诊断核验只能交第一版。
*/
function checkSubmission(s: Content<"submission">, state: WriteState): void {
const where = `提交 ${s.kind}(阶段 ${s.stageId},作者 ${s.author},对象 ${s.target ?? "无"},编号 ${s.item ?? "无"},第 ${s.version} 版)`;
@@ -63,7 +67,10 @@ function checkRunResult(r: Content<"runResult">, state: WriteState): void {
}
}
/** 第 4 条:这一轮已经有消融格确认启动了,就不能再关解释阶段(方法.md 4.3 ⑭:解释冻结必须早于任何一格启动)。 */
/**
* 第 4 条:这一轮已经有消融格确认启动了,就不能再关解释阶段(方法.md 4.3 ⑭:解释冻结必须早于任何一格启动)。
* 第 5 条:这一轮的诊断运行已经确认启动了,就不能再关诊断预测阶段(方法.md 4.3 诊 2:冻结必须早于诊断运行启动,范围才算事前预测)。
*/
function checkStageClose(c: Content<"stageClose">, state: WriteState): void {
const open = state.stageOpen(c.stageId);
if (open?.content.stage === "explanation" && state.ablationStarted(open.content.round)) {
@@ -71,4 +78,9 @@ function checkStageClose(c: Content<"stageClose">, state: WriteState): void {
`不能关解释阶段 ${c.stageId}:第 ${open.content.round} 轮已经有消融格确认启动,解释冻结必须早于任何一格启动(方法.md 4.3 ⑭)`,
);
}
if (open?.content.stage === "diagnosisPrediction" && state.diagnosisStarted(open.content.round)) {
throw new Error(
`不能关诊断预测阶段 ${c.stageId}:第 ${open.content.round} 轮的诊断运行已经确认启动,诊断预测冻结必须早于诊断运行启动(方法.md 4.3 诊 2)`,
);
}
}
+3 -1
View File
@@ -3,7 +3,7 @@ import type { DatabaseSync } from "node:sqlite";
// 账本的表结构(docs/结构.md 第四部分"全部只追加"):只有一张表,每一行是序号、种类、内容、内容的哈希。
// 表结构一改就把版本号加 1;旧文件版本号对不上时打开报错,不迁移,也不逐字核对建表语句。
export const SCHEMA_VERSION = 4;
export const SCHEMA_VERSION = 5;
/** 新文件要执行的建表语句,由 Ledger 在一个事务里执行,最后写入版本号。 */
export const CREATE_SCHEMA = [
@@ -33,6 +33,8 @@ export const CREATE_SCHEMA = [
`CREATE UNIQUE INDEX call_ended_once ON records (json_extract(content, '$.call')) WHERE kind = 'callEnded'`,
// 结算以轮次为编号:每轮最多采纳一个改法,所以最多一次消融。
`CREATE UNIQUE INDEX settlement_once ON records (json_extract(content, '$.round')) WHERE kind = 'settlement'`,
// 诊断结算同样以轮次为编号:每轮最多一次诊断。
`CREATE UNIQUE INDEX diagnosis_settlement_once ON records (json_extract(content, '$.round')) WHERE kind = 'diagnosisSettlement'`,
`PRAGMA user_version=${SCHEMA_VERSION}`,
];
+78 -8
View File
@@ -3,7 +3,14 @@ import { type Draft, factorLines, KIND_TEXT, offText, registeredMetrics, taskPoi
import type { Board } from "./read.ts";
import { fenced, json } from "./format.ts";
import { unassignedChanges } from "./locations.ts";
import { clauseKey, factorKey, splitCell, type FrozenProtocol, type LocationTable } from "./shapes.ts";
import {
clauseKey,
diagnosisId,
factorKey,
splitCell,
type FrozenProtocol,
type LocationTable,
} from "./shapes.ts";
// 实验员、核验员的 5 种任务(方法.md 4.3 ⑨ ⑯):只看分给他的那个候选或那一格——冻结协议、起点代码、
// 要实现或核验的东西。不给信念本(用户 2026-10-03 决定 3)、不给任何运行成绩、不给别的候选。
@@ -36,8 +43,7 @@ export function protocolMetrics(board: Board, p: FrozenProtocol): HypothesisMetr
})];
}
function metricSection(d: Draft, p: FrozenProtocol): void {
const all = protocolMetrics(d.board, p);
function metricSection(d: Draft, all: HypothesisMetric[]): void {
d.section("要实现的指标定义", [
all.length === 0 ? "(这份协议没有要实现的假设专用指标)" : json(all),
"",
@@ -58,15 +64,22 @@ function codeLines(d: Draft): string[] {
return [...d.readableLines(), "读这些目录用 read_source(核验员)或 shell(实验员)。"];
}
/** 本轮做过诊断时,诊断程序的代码也能读(只读),冻结协议可以写"照诊断实验 dN 的做法"(方法.md 4.3 ②);只给代码,不给成绩和实验原文。 */
function diagnosisReference(d: Draft): string[] {
if (d.board.diagnosisSettlement(d.round) === undefined) return [];
d.read(diagnosisId(d.round), "本轮诊断程序(只读,冻结协议引用诊断实验时作参考)");
return ["上面还列了本轮诊断程序的代码:冻结协议引用诊断实验时作参考,别把它当成这个候选的起点。"];
}
/** ⑨ 实现:冻结协议、指标定义、要交的位置表、起点代码(本批基线)。 */
export function implementation(d: Draft): ChecklistOf<"implementation"> {
const candidate = d.req.object!;
const { p, items } = frozen(d, candidate);
taskSection(d, "实验员", `候选 ${candidate}`);
d.read(p.baseline, "起点代码(本批基线)");
d.section("基线代码", [...codeLines(d), "你的工作目录是起点代码的副本,改动都落在工作目录里。"]);
d.section("基线代码", [...codeLines(d), "你的工作目录是起点代码的副本,改动都落在工作目录里。", ...diagnosisReference(d)]);
d.section("本任务要处理的东西", [`### 候选 ${candidate} 的冻结协议`, "", json(p)]);
metricSection(d, p);
metricSection(d, protocolMetrics(d.board, p));
locationTableSection(d, p);
return { kind: "implementation", ...items };
}
@@ -78,7 +91,7 @@ export function repair(d: Draft): ChecklistOf<"repair"> {
taskSection(d, "实验员", `候选 ${candidate}`);
d.read(p.baseline, "起点代码(本批基线)");
d.read(candidate, "你上一次提交的代码");
d.section("基线代码", [...codeLines(d), "你的工作目录是上一次提交的代码的副本。"]);
d.section("基线代码", [...codeLines(d), "你的工作目录是上一次提交的代码的副本。", ...diagnosisReference(d)]);
const v = d.board.published("verification").filter((s) => s.target === candidate).at(-1)!.content;
d.section("本任务要处理的东西", [
"### 退回意见",
@@ -102,7 +115,7 @@ export function repair(d: Draft): ChecklistOf<"repair"> {
"",
json(p),
]);
metricSection(d, p);
metricSection(d, protocolMetrics(d.board, p));
locationTableSection(d, p);
return { kind: "repair", ...items };
}
@@ -148,7 +161,7 @@ export function verification(d: Draft): ChecklistOf<"verification"> {
"",
fenced(d.req.changes!.diff, "diff"),
]);
metricSection(d, p);
metricSection(d, protocolMetrics(d.board, p));
return { kind: "verification", unassigned: found.unassigned.map((u) => u.location), ...items };
}
@@ -212,3 +225,60 @@ export function ablationVerification(d: Draft): ChecklistOf<"ablationVerificatio
]);
return { kind: "ablationVerification", cell: a.cell, factors: a.factors };
}
// ---------------------------------------------------------------------------
// 诊断的三种任务(方法.md 4.3 诊 4):对象是本轮诊断,一位实验员把选中的全部实验写进一份诊断程序,一位核验员逐个核对。
// 只看选中的实验原文和当前基线代码,不给信念本、不给任何成绩。
// ---------------------------------------------------------------------------
/** 三种诊断任务共用:任务与本轮、起点代码(当前基线)、选中的实验原文和指标定义;program 是另外要能读的诊断程序的说明。交回选中的实验编号。 */
function diagnosisTask(d: Draft, role: string, program: string | null): string[] {
d.group("分给你的材料", "下面是分给你的本轮诊断的全部材料。");
const chosen = d.board.decision("selectDiagnosis", d.round)!.output.chosen;
const experiments = d.board.diagnosisExperiments(d.round).filter((e) => chosen.includes(e.id));
taskSection(d, role, `本轮诊断(对象 ${diagnosisId(d.round)})`);
d.read(d.board.baselineOf(d.round), "起点代码(当前基线)");
if (program !== null) d.read(diagnosisId(d.round), program);
d.section("基线代码", codeLines(d));
d.section("选中的诊断实验", [
"每个实验都从当前基线的代码出发,只动它写的那一处,其余照基线;每个实验交出自己的答案。",
"",
...experiments.flatMap((e) => [`### ${e.id} ${e.name}`, "", `动哪一处:${e.spot}`, "", e.body, ""]),
]);
metricSection(d, experiments.flatMap((e) => e.metrics));
return chosen;
}
/** 诊断实现,加上原始产物的位置;修复(repair 为 true)另放退回意见(没通过的实验和依据、上一次的位置表和改动),通过的实验不要动。 */
function diagnosisCode(d: Draft, repair: boolean): string[] {
const chosen = diagnosisTask(d, "实验员", repair ? "你上一次提交的诊断程序" : null);
d.section("原始产物", [
`产出当前基线的那次正式运行(${d.board.runOf(d.board.baselineOf(d.round))!.plan.run})的原始产物已经只读地放进你的工作目录,实验要接着算的答案就在里面;位置和怎么读写在任务说明里(research_query 查 task)。`,
"你的工作目录是当前基线代码的副本,加上这份原始产物,改动都落在工作目录里。",
]);
if (!repair) return chosen;
const v = d.board.published("diagnosisVerification").filter((s) => s.round === d.round).at(-1)!.content;
const failed = v.experiments.filter((x) => !x.followed || !x.nothingElse);
d.section("退回意见", [
...failed.map((x) => `- 实验 ${x.experiment}:${x.followed ? "" : "没照写法做;"}${x.nothingElse ? "" : "动了写法以外的地方;"}${x.where}`),
"",
`核验员的说明:${v.body}`,
"只修上面没通过的实验,通过的实验的代码不要动。",
"",
...["### 上一次交的位置表", "", json(d.board.code(diagnosisId(d.round)).locations), "", "### 上一次提交相对起点代码的改动", "", fenced(d.req.changes!.diff, "diff")],
]);
return chosen;
}
export const diagnosisImplementation = (d: Draft): ChecklistOf<"diagnosisImplementation"> => ({ kind: "diagnosisImplementation", experiments: diagnosisCode(d, false) });
export const diagnosisRepair = (d: Draft): ChecklistOf<"diagnosisRepair"> => ({ kind: "diagnosisRepair", experiments: diagnosisCode(d, true) });
/** 诊断核验:选中的实验原文、当前基线到诊断程序的改动、位置表。Jev 的判断不给核验员(同正向核验)。 */
export function diagnosisVerification(d: Draft): ChecklistOf<"diagnosisVerification"> {
const chosen = diagnosisTask(d, "核验员", "要核验的诊断程序");
d.section("本任务要处理的东西", [
...["### 实验员交的位置表", "", json(d.board.code(diagnosisId(d.round)).locations), ""],
...["### 改动全文(诊断程序相对当前基线,每段前后各 3 行上下文)", "", fenced(d.req.changes!.diff, "diff")],
]);
return { kind: "diagnosisVerification", experiments: chosen };
}
+10 -1
View File
@@ -12,6 +12,7 @@ import {
type Phase,
} from "../rules/index.ts";
import type { ProcessResult, Range } from "../vocab/index.ts";
import { diagnosisHistory } from "./diagnosis.ts";
import { measured, num, pct, PLACEMENT_TEXT, rangeText, resourcesText, score, SYNERGY_TEXT, UNTESTED_TEXT } from "./format.ts";
import type { StudyFacts } from "./inputs.ts";
import { answerPathLines } from "./locations.ts";
@@ -21,6 +22,8 @@ import { CELL_NOT_RUN, cellId, factorId, factorKey, ORIGINAL_BASELINE, type Sett
// 信念本(方法.md 6.10):程序从已公开的记录现算,不存。数字全由 rules 的函数算,这里只挑出来写成字。
// 只读这几样:已关阶段的提交、按 Board.runPublic 公开的运行、决定、结算、解封;封存、第一次预测、退回一律不读。
const PHASE_TEXT: Readonly<Record<Phase, string>> = { forward: "正向", diagnosis: "诊断", ablation: "消融" };
/** 范围怎么换算成把握(方法.md 6.2、6.3),写在开头。只说程序怎么用这三个数,不要求研究者照着算。 */
function rangeToBelief(lambda: number): string {
return (
@@ -118,6 +121,9 @@ export function beliefBookOf(board: Board, study: StudyFacts, round: number | nu
}
for (const s of settlements) lines.push(...ablationSummary(board, s));
// 3 诊断历史
lines.push("", "**诊断历史**(每轮的诊断实验与实测、每条解释确认或否定、没做的实验和原因,以及程序推出的方向)", ...diagnosisHistory(board, study, latestRound));
// 4 战绩
lines.push("", "**战绩**(只显示,不改任何权重;偏高偏低都按分数原值判)");
const samples: Array<{ researcher: string; phase: Phase; range: Range; value: number }> = batches.flatMap((d) =>
@@ -127,6 +133,9 @@ export function beliefBookOf(board: Board, study: StudyFacts, round: number | nu
return v === null ? [] : c.ranges.map((r) => ({ researcher: r.researcher, phase: "forward" as const, range: r.range, value: v }));
}),
);
for (const s of board.diagnosisSettlements())
for (const e of s.input.experiments)
if (e.outcome.kind === "valid") for (const x of s.input.explanations) samples.push({ researcher: x.author, phase: "diagnosis", range: x.experiments.find((y) => y.experiment === e.experiment)!.range, value: e.outcome.score });
for (const s of settlements)
for (const c of s.input.cells) {
if (c.outcome.kind !== "valid") continue;
@@ -139,7 +148,7 @@ export function beliefBookOf(board: Board, study: StudyFacts, round: number | nu
if (tallies.length === 0) lines.push("- 还没有实测可以核对");
for (const { researcher, phase, tally: t } of tallies) {
lines.push(
`- ${researcher},${phase === "forward" ? "正向" : "消融"} ${t.total} 次:中位数偏高 ${t.medianHigh}、偏低 ${t.medianLow};` +
`- ${researcher},${PHASE_TEXT[phase]} ${t.total} 次:中位数偏高 ${t.medianHigh}、偏低 ${t.medianLow};` +
`穿过上沿 ${t.aboveTop}、下沿 ${t.belowBottom};包住 ${t.covered};落点 ` +
(Object.keys(PLACEMENT_TEXT) as Array<keyof typeof PLACEMENT_TEXT>).map((k) => `${PLACEMENT_TEXT[k]} ${t.placements[k]}`).join(","),
);
+46 -4
View File
@@ -2,11 +2,12 @@ import type { Ledger } from "../ledger/index.ts";
import { adoptionLine, compareIds, runBalance } from "../rules/index.ts";
import type { Checklist, SubmissionKind } from "../vocab/index.ts";
import { beliefBookOf } from "./belief-book.ts";
import { directionText, json, measured, score } from "./format.ts";
import { diagnosisLines } from "./diagnosis.ts";
import { directionText, json, measured, num, resourcesText, score } from "./format.ts";
import type { TaskRequest } from "./inputs.ts";
import { answerPathLines } from "./locations.ts";
import { Board, type Run } from "./read.ts";
import { entryId, explanationId, factorKey, splitCell, type FrozenProtocol } from "./shapes.ts";
import { entryId, explanationId, factorKey, ORIGINAL_BASELINE, splitCell, type FrozenProtocol } from "./shapes.ts";
// 任务材料的形状,和研究者各任务共用的几块(方法.md 第三部分"给研究者的材料顺序")。
@@ -38,6 +39,11 @@ export const KIND_TEXT: Readonly<Record<SubmissionKind, string>> = {
entry: "第零轮闭门写入口",
discussion: "第零轮讨论",
claim: "第零轮认领入口",
diagnosisProposal: "写诊断解释与诊断实验",
diagnosisPrediction: "诊断预测",
diagnosisImplementation: "实现诊断实验",
diagnosisRepair: "按核验意见修复诊断程序",
diagnosisVerification: "核验诊断程序",
proposal: "提案",
firstPrediction: "第一次预测",
critique: "挑刺",
@@ -142,12 +148,42 @@ export function codeSection(d: Draft): void {
d.read(d.board.baselineOf(d.round), "本批基线");
for (const r of d.board.runs()) {
if (r.plan.object !== null && r.plan.purpose !== "baseline" && d.board.runPublic(r)) {
d.read(r.plan.object, `${r.plan.purpose === "forward" ? "候选" : "消融格"} ${r.plan.object} 的代码`);
d.read(r.plan.object, r.plan.purpose === "diagnosis" ? `第 ${r.plan.round} 轮诊断程序的代码` : `${r.plan.purpose === "forward" ? "候选" : "消融格"} ${r.plan.object} 的代码`);
}
}
d.section("基线", [...baselineLines(d), "", "能读的代码(用 read_source 读):", ...d.readableLines()]);
}
/** 本轮诊断结果(方法.md 第三部分材料顺序第 3 条):从提案起的全部研究者任务都有,没诊断写原因;诊断任务自己读时这一段还没走完,不放。 */
export function diagnosisSection(d: Draft): void {
if (d.board.diagnosisDone(d.round)) d.section("本轮诊断结果", diagnosisLines(d.board, d.req.study, d.round, true));
}
/** 运行的产物只放进查询,这里列编号。 */
export function artifactSection(d: Draft, heading: string): void {
const artifacts = d.req.artifacts!;
const listed = Object.keys(artifacts)
.sort(compareIds)
.flatMap((run) => Object.entries(artifacts[run]!).map(([file, text]) => `- ${d.record(`${run}/${file}`, `运行 ${run} 的产物 ${file}`, text)}`));
d.section(heading, ["用 research_query 按编号查全文:", ...(listed.length === 0 ? ["(没有)"] : listed)]);
}
/** 本批基线那次运行的结果:分数、判分、其余指标、资源、任务的附加记录;原始基线没有位置表,不写答案路径。 */
export function baselineRunSection(d: Draft): void {
const base = d.board.baselineOf(d.round);
const run = d.board.publicRunOf(base)!;
const j = run.result!.judged!;
const others = Object.keys(j.metrics).filter((k) => k !== d.req.study.finalMetric).sort(compareIds);
d.section("本批基线那次运行的结果", [
...(base === ORIGINAL_BASELINE
? [`运行 ${run.plan.run}:任务给的原始代码;${j.verdict === "valid" ? "算数" : "不算数"}(${j.verdictReason});分数 ${measured(j.score, score)}`]
: runLines(d, run)),
resourcesText(run.result!.process, d.board.publicRunOf(ORIGINAL_BASELINE)!.result!.process, d.req.study.wallSeconds),
...(others.length === 0 ? [] : [`其余指标:${others.map((k) => `${k} ${measured(j.metrics[k]!, num)}`).join(";")}`]),
`任务的附加记录(含合法化,原样):${JSON.stringify(j.extra)}`,
]);
}
export function beliefSection(d: Draft): void {
d.section("信念本", [beliefBookOf(d.board, d.req.study, d.round)]);
}
@@ -194,7 +230,13 @@ export function publicRecords(d: Draft): void {
e.content.explanations.forEach((x, i) => d.record(explanationId(e.round, e.author, i + 1), `解释 ${explanationId(e.round, e.author, i + 1)}(${e.author})`, x));
}
for (const w of b.published("wrapUp")) d.record(`${w.stageId}.${w.author}`, `${w.author} 第 ${w.round} 轮的收尾`, w.content.body);
for (const r of b.runs()) if (r.plan.purpose !== "baseline" && b.runPublic(r)) d.record(r.plan.run, `运行 ${r.plan.run}`, [...runLines(d, r), "", json(r.result!.judged)].join("\n"));
for (const r of b.runs()) {
if (r.plan.purpose !== "baseline" && r.plan.purpose !== "diagnosis" && b.runPublic(r)) d.record(r.plan.run, `运行 ${r.plan.run}`, [...runLines(d, r), "", json(r.result!.judged)].join("\n"));
}
// 诊断的完整结果:各人给的三个数(结算输入)不放,只有合并后的范围和逐个实验的判分。
for (const { round, input: _input, ...settled } of b.diagnosisSettlements()) {
d.record(`diagnosis-r${round}`, `第 ${round} 轮诊断的完整结果`, json({ explanations: b.diagnosisExplanations(round), experiments: b.diagnosisExperiments(round), ...settled }));
}
d.section("研究记录", [
"已经公开的记录,一行一条;全文用 research_query 按编号查。",
...d.records.map((r) => `- ${r.id}:${r.title}`),
+5
View File
@@ -12,6 +12,11 @@ const BUILDERS: { [K in SubmissionKind]: (d: Draft) => Extract<Checklist, { kind
entry: researcher.entry,
discussion: researcher.discussion,
claim: researcher.claim,
diagnosisProposal: researcher.diagnosisProposal,
diagnosisPrediction: researcher.diagnosisPrediction,
diagnosisImplementation: assigned.diagnosisImplementation,
diagnosisRepair: assigned.diagnosisRepair,
diagnosisVerification: assigned.diagnosisVerification,
proposal: researcher.proposal,
firstPrediction: researcher.firstPrediction,
critique: researcher.critique,
+78
View File
@@ -0,0 +1,78 @@
import { runBalance } from "../rules/index.ts";
import { measured, num, rangeText, resourcesText, score } from "./format.ts";
import type { StudyFacts } from "./inputs.ts";
import type { Board, MergedExperiment } from "./read.ts";
import { DIAGNOSIS_NOT_RUN, diagnosisId, ORIGINAL_BASELINE, type DiagnosisSettlement } from "./shapes.ts";
// 一轮诊断的结果写成文字(方法.md 4.3 诊 4、6.10 第 3 条),公共材料的"本轮诊断结果"和信念本的"诊断历史"共用。
// 各人对诊断实验给的三个数不放进来,只有合并后的范围。
const ENDING = { finished: "跑完", crashed: "跑崩了", timeout: "超过分到的时间被杀", notRun: "整次运行先结束,没轮到" } as const;
const SKIPPED = { budget: "一次诊断最多做 D 个实验,放不下", value: "信息价值不足以抵过成本" } as const;
/** 第 round 轮没有诊断的原因(方法.md 4.3 诊 3、诊 4)。 */
function noDiagnosisReason(board: Board, study: StudyFacts, round: number): string {
const decision = board.decision("selectDiagnosis", round);
if (decision === undefined) {
const balance = runBalance(study.totalRuns, board.runStatesBefore(round)).balance;
return balance < 2 ? `余额只剩 ${balance} 次,不到 2 次,要留 1 次给本轮正向` : "配置关掉了诊断";
}
if (!decision.output.diagnose) return `信息价值减成本都不大于 0,最好的组合净值 ${num(Math.max(...decision.output.combos.map((c) => c.net)))}`;
return `选中的实验核验都没通过、修复次数用完,一个都没跑:${board.eventReason(DIAGNOSIS_NOT_RUN, diagnosisId(round))}`;
}
const signed = (x: number): string => (x > 0 ? `+${score(x)}` : score(x));
/** 一个合并后的实验:做了什么、实测、结局、测前合并范围、比特、惊讶、用时。 */
function experimentLine(board: Board, study: StudyFacts, round: number, s: DiagnosisSettlement, e: MergedExperiment): string {
const row = s.output.experiments.find((x) => x.experiment === e.id)!;
const judged = s.judged.experiments.find((x) => x.id === e.id);
const dropped = s.dropped.find((x) => x.experiment === e.id);
const skipped = board.decision("selectDiagnosis", round)!.output.experiments.find((y) => y.experiment === e.id)!;
const outcome = {
valid: () => `实测 ${measured(judged!.score, score)},改进量 ${signed(row.gain!)}`,
untested: () => `没做:${SKIPPED[skipped.reason!]}${skipped.bestCombo === null ? "" : `(含它的最好组合净值 ${num(skipped.bestCombo.net)})`}`,
insufficient: () => `证据不足:${dropped === undefined ? `${ENDING[judged!.ending]};${judged!.verdictReason}` : `核验不通过,${dropped.reason}`}`,
}[row.outcome]();
const parts = [outcome, `测前合并范围 ${rangeText(row.merged)}`];
if (row.worldBits !== null) parts.push(`世界比特 ${num(row.worldBits)}`);
if (row.surprise !== null) parts.push(`惊讶 ${measured(row.surprise.bits, (x) => `${num(x)} 比特`)}`);
if (judged !== undefined) {
parts.push(`用时 ${measured(judged.wallSeconds, (x) => `${num(x)} 秒`)}`);
const others = Object.keys(judged.metrics).filter((k) => k !== study.finalMetric).sort();
if (others.length > 0) parts.push(`其余指标:${others.map((k) => `${k} ${measured(judged.metrics[k]!, num)}`).join(";")}`);
}
return `- ${e.id} ${e.name}(作者 ${e.authors.join("、")};动 ${e.spot}):${parts.join(";")}`;
}
/**
* 第 round 轮诊断的结果:每个实验做了什么、实测、用时、结局,每条诊断解释确认或否定,程序推出的方向;没诊断时写原因。
* detail 为 true 时每条解释带正文(公共材料),否则只列确认与否定(信念本的诊断历史)。
*/
export function diagnosisLines(board: Board, study: StudyFacts, round: number, detail: boolean): string[] {
const s = board.diagnosisSettlement(round);
if (s === undefined) return [`本轮没有诊断:${noDiagnosisReason(board, study, round)}`];
const process = board.runs().find((r) => r.plan.run === s.run)!.result!.process;
const lines = [
`诊断运行 ${s.run}:在基线 ${board.baselineOf(round)}(最终指标 ${score(s.input.baseline)})上,一次运行做了 ${s.judged.experiments.length} 个实验,只扣 1 次。` +
"诊断实验的答案不采纳为基线,只用来看这个方向还剩多少空间。",
resourcesText(process, board.publicRunOf(ORIGINAL_BASELINE)!.result!.process, study.wallSeconds),
...["", "诊断实验:", ...board.diagnosisExperiments(round).map((e) => experimentLine(board, study, round, s, e)), "", "诊断解释:"],
];
for (const x of board.diagnosisExplanations(round)) {
const verdicts = s.output.judgments.find((j) => j.id === x.id)!.verdicts;
const of = (v: string) => verdicts.filter((y) => y.verdict === v).map((y) => y.experiment).join("、") || "无";
lines.push(`- ${x.id}(作者 ${x.author}):${verdicts.length === 0 ? "没有有效实验可判" : `确认 ${of("confirmed")};否定 ${of("denied")}`}`);
if (detail) lines.push(...x.body.split("\n").map((l) => ` ${l}`));
}
const d = s.output.direction;
const gain = d.gains.find((g) => g.experiment === d.experiment)!;
const way = `程序按诊断推出的方向:${d.experiment}(${gain.measured ? "实测" : "按更新后的权重估计"} ${signed(d.gain)})。这只是按公式推出来的参考,你可以不跟。`;
return [...lines, "", d.found ? way : "诊断没有指出能涨分的方向:所有实验的收益都不为正。"];
}
/** 信念本的"诊断历史"(方法.md 6.10 第 3 条):每个诊断这一段已走完的轮次一段。 */
export function diagnosisHistory(board: Board, study: StudyFacts, latestRound: number): string[] {
const rounds = Array.from({ length: latestRound }, (_, i) => i + 1).filter((r) => board.diagnosisDone(r));
return rounds.length === 0 ? ["- 还没有"] : rounds.flatMap((r) => [`- 第 ${r} 轮:`, ...diagnosisLines(board, study, r, false).map((l) => (l === "" ? l : ` ${l}`))]);
}
+3 -1
View File
@@ -7,7 +7,7 @@ export { buildMaterial } from "./build.ts";
export type { CodeChanges, PromptFile, Prompts, StudyFacts, TaskRequest } from "./inputs.ts";
export { queryMaterial, readPrompts } from "./prompts.ts";
// flow 推进时读账本也用这一份(本批基线、基线分数、留池候选、各候选合并了哪些原提案),不另写一份。
export { Board, validScore } from "./read.ts";
export { Board, validScore, type MergedExperiment } from "./read.ts";
export {
callId,
candidateId,
@@ -17,6 +17,7 @@ export {
clauseKey,
critiqueId,
DECISION_KINDS,
DIAGNOSIS_NOT_RUN, diagnosisExplanationId, diagnosisId, diagnosisKey, diagnosisNotRunEvent,
entryId,
experimenterId,
verifierId,
@@ -33,6 +34,7 @@ export {
type DecisionKind,
type DecisionShapes,
type DedupOutput,
type DiagnosisDedupOutput, type DiagnosisLocationTable, type DiagnosisSettlement,
type FrozenProtocol,
type LocationTable,
type Settlement,
+6 -4
View File
@@ -17,8 +17,10 @@ export type StudyFacts = {
lambda: number;
/** 正式运行总次数。 */
totalRuns: number;
/** 每人每轮提案上限 M。 */
/** 每人每轮提案上限 M,诊断解释的上限也是它。 */
proposalLimit: number;
/** 每人每轮诊断实验上限 A;诊断关掉时为 null。 */
diagnosisExperimentLimit: number | null;
factorLimit: number;
/** 最终指标和常规辅助指标的定义(tasks 的 metricDefinitions)。 */
metrics: MetricDefinition[];
@@ -50,15 +52,15 @@ export type Prompts = { roles: Record<Role, PromptFile>; tasks: Record<Submissio
/**
* 生成一份任务材料要的全部输入。全部必填,用不到的写 null:
* - changes:核验是起点到候选代码,修复是起点到上一次提交的代码,消融核验是完整代码到这一格的代码;其余为 null;
* - artifacts:解释任务要的本轮各次运行的产物(运行编号 → 文件名 → 文字);其余为 null;
* - changes:核验是起点到候选代码,修复是起点到上一次提交的代码,消融核验是完整代码到这一格的代码;诊断核验是当前基线到诊断程序,诊断修复是当前基线到上一次提交的诊断程序;其余为 null;
* - artifacts:解释任务要的本轮各次运行的产物、诊 1 要的本批基线那次运行的产物(运行编号 → 文件名 → 文字);其余为 null;
* - codeRoot:能读的代码版本取出到 `<codeRoot>/<版本号>`,flow 照材料里的目录取出。
*/
export type TaskRequest = {
kind: SubmissionKind;
actor: string;
stageId: string;
/** 一个任务一个对象:候选、原提案(挑刺)或消融格;入口、讨论、志愿、提案、解释、收尾为 null。 */
/** 一个任务一个对象:候选、原提案(挑刺)、消融格或本轮诊断(diagnosisId);入口、讨论、志愿、提案、诊断解释与预测、解释、收尾为 null。 */
object: string | null;
ledger: Ledger;
study: StudyFacts;
+49 -3
View File
@@ -1,11 +1,13 @@
import type { Content, LedgerRecord, StoredSubmission } from "../ledger/index.ts";
import { compareIds, type RunState } from "../rules/index.ts";
import type { JudgedResult, Range, Stage, SubmissionKind } from "../vocab/index.ts";
import type { HypothesisMetric, JudgedResult, Range, Stage, SubmissionKind } from "../vocab/index.ts";
import {
cellId,
diagnosisExplanationId,
ORIGINAL_BASELINE,
type DecisionKind,
type DecisionShapes,
type DiagnosisSettlement,
type FrozenProtocol,
type LocationTable,
type Settlement,
@@ -22,6 +24,9 @@ export type Run = {
result: Content<"runResult"> | null;
};
/** 本轮合并后的一个诊断实验:编号、全部作者、正文(取作者里编号最小的那一份)、登记的指标(编号已是全名"实验编号.短名")。 */
export type MergedExperiment = { id: string; authors: string[]; name: string; spot: string; body: string; metrics: HypothesisMetric[] };
type StageInfo = { stage: Stage; round: number; roster: string[]; closed: boolean };
export class Board {
@@ -116,12 +121,14 @@ export class Board {
/**
* 运行什么时候公开(任务书 09 设计稿):基线运行有结果即公开;第 r 轮正向运行在第 r 轮的采纳决定写入后
* 整批公开(方法.md 4.3 ⑪);第 r 轮消融运行在第 r 轮结算写入后公开(4.3 ⑰ ⑱)。
* 整批公开(方法.md 4.3 ⑪);第 r 轮消融运行在第 r 轮结算写入后公开(4.3 ⑰ ⑱);
* 第 r 轮诊断运行在第 r 轮诊断结算写入后公开(4.3 诊 4)。
*/
runPublic(run: Run): boolean {
if (run.result === null) return false;
if (run.plan.purpose === "baseline") return true;
if (run.plan.purpose === "forward") return this.decision("adopt", run.plan.round) !== undefined;
if (run.plan.purpose === "diagnosis") return this.diagnosisSettlement(run.plan.round) !== undefined;
return this.settlement(run.plan.round) !== undefined;
}
@@ -140,7 +147,12 @@ export class Board {
/** 每次运行在账上的状态,算余额用(方法.md 第七部分)。 */
runStates(): RunState[] {
return this.runs().map((r) => (r.result?.process.ending === "notStarted" ? "notStarted" : r.started ? "started" : "reserved"));
return this.runs().map(stateOf);
}
/** 第 round 轮开始前各次运行在账上的状态(算诊断开始时的余额)。 */
runStatesBefore(round: number): RunState[] {
return this.runs().filter((r) => r.plan.round < round).map(stateOf);
}
/** 某个候选最新的实现或修复提交(已关阶段)。 */
@@ -169,6 +181,38 @@ export class Board {
return this.#of("settlement").map((r) => ({ round: r.content.round, ...(r.content.result as unknown as Settlement) }));
}
/** 第 round 轮的诊断结算(还没写入为 undefined)。 */
diagnosisSettlement(round: number): DiagnosisSettlement | undefined {
return this.diagnosisSettlements().find((s) => s.round === round);
}
diagnosisSettlements(): Array<{ round: number } & DiagnosisSettlement> {
return this.#of("diagnosisSettlement").map((r) => ({ round: r.content.round, ...(r.content.result as unknown as DiagnosisSettlement) }));
}
/** 第 round 轮诊断这一段走完了没有:提案阶段开了,就是结算写了或本轮不诊断。 */
diagnosisDone(round: number): boolean {
return [...this.stages.values()].some((s) => s.stage === "proposal" && s.round === round);
}
/** 第 round 轮已公开的诊断解释,按作者、条序排。 */
diagnosisExplanations(round: number): Array<{ id: string; author: string; body: string }> {
return this.published("diagnosisProposal")
.filter((s) => s.round === round)
.sort((a, b) => compareIds(a.author, b.author))
.flatMap((s) => s.content.explanations.map((x, i) => ({ id: diagnosisExplanationId(round, s.author, i + 1), author: s.author, body: x.body })));
}
/** 第 round 轮合并后的诊断实验:判重合并的决定指明合成自哪几个作者的哪几个实验,正文取第一个。 */
diagnosisExperiments(round: number): MergedExperiment[] {
const proposals = this.published("diagnosisProposal").filter((s) => s.round === round);
const experimentOf = (src: { author: string; index: number }) => proposals.find((s) => s.author === src.author)!.content.experiments[src.index]!;
return this.decision("diagnosisDedup", round)!.output.experiments.map(({ experiment, sources }) => {
const { name, spot, body, metrics } = experimentOf(sources[0]!);
return { id: experiment, authors: [...new Set(sources.map((x) => x.author))], name, spot, body, metrics: metrics.map((m) => ({ ...m, id: `${experiment}.${m.id}` })) };
});
}
/** 本人最新的私人笔记(方法.md 第三部分"私人笔记"),没写过为 null。 */
note(author: string): string | null {
return this.#of("note").filter((r) => r.content.author === author).at(-1)?.content.body ?? null;
@@ -249,6 +293,8 @@ export class Board {
}
}
const stateOf = (r: Run): RunState => (r.result?.process.ending === "notStarted" ? "notStarted" : r.started ? "started" : "reserved");
/** 判分结果里的有效分数:算数且测到了分数才有,否则 null。 */
export function validScore(judged: JudgedResult | null): number | null {
return judged !== null && judged.verdict === "valid" && judged.score.status === "measured" ? judged.score.value : null;
+38 -12
View File
@@ -1,8 +1,11 @@
import { compareIds } from "../rules/index.ts";
import { ANSWER_PATH_KEY, SAFEGUARD_TRIGGERS_KEY, type ChecklistOf } from "../vocab/index.ts";
import {
artifactSection,
baselineRunSection,
beliefSection,
codeSection,
diagnosisSection,
Draft,
entryVersions,
factorLines,
@@ -33,6 +36,7 @@ function frame(d: Draft, own: Own, middle: () => void): void {
researcherTaskSection(d);
codeSection(d);
if (d.round >= 1) {
diagnosisSection(d);
publicRecords(d);
beliefSection(d);
}
@@ -126,16 +130,42 @@ function concernLines(d: Draft, round: number): string[] {
return lines.length === 0 ? ["(没有)"] : lines;
}
/** 上一轮全部研究者的收尾(提案、诊 1 共用)。 */
function previousWrapSection(d: Draft): void {
const prev = d.round - 1;
const wraps = d.board.published("wrapUp").filter((w) => w.round === prev);
d.section("上一轮的收尾", prev < 1 ? ["(这是第 1 轮,没有上一轮)"] : wraps.flatMap((w) => [`### ${w.author} 的收尾`, "", w.content.body, ""]));
}
/** 诊 1 诊断解释与诊断实验:当前基线的代码,产出它的那次运行的答案、分数、指标,信念本,上一轮收尾;第 1 轮的第零轮全部记录在研究记录里。 */
export function diagnosisProposal(d: Draft): ChecklistOf<"diagnosisProposal"> {
frame(d, [], () => {
baselineRunSection(d);
artifactSection(d, "本批基线那次运行的答案");
previousWrapSection(d);
});
return { kind: "diagnosisProposal", maxExplanations: d.req.study.proposalLimit, maxExperiments: d.req.study.diagnosisExperimentLimit!, takenMetricIds: factorRules(d).takenMetricIds };
}
/** 诊 2 诊断预测:合并后的诊断实验清单(标明作者)、全部诊断解释;自己的解释另列在"你自己的"里。别人在这一步交的数一概不读(只经 Board.mine 读本人的)。 */
export function diagnosisPrediction(d: Draft): ChecklistOf<"diagnosisPrediction"> {
const all = d.board.diagnosisExplanations(d.round);
const mine = all.filter((x) => x.author === d.req.actor);
const experiments = d.board.diagnosisExperiments(d.round);
const text = (x: { id: string; author: string; body: string }) => [`#### ${x.id}(作者 ${x.author})`, "", x.body, ""];
frame(d, [{ heading: "你的诊断解释", lines: ["下面每一条都要对每个合并后的诊断实验各给三个数。", "", ...mine.flatMap(text)] }], () => {
baselineRunSection(d);
const merged = experiments.flatMap((e) => [`#### ${e.id} ${e.name}(作者 ${e.authors.join("、")})`, "", `动哪一处:${e.spot}`, "", e.body, ""]);
d.section("本任务要处理的东西", ["### 合并后的诊断实验", "", ...merged, "### 全部诊断解释(含你自己的)", "", ...all.flatMap(text)]);
});
return { kind: "diagnosisPrediction", explanations: mine.map((x) => x.id), experiments: experiments.map((e) => e.id) };
}
/** ② 提案:公共材料,加上一轮全部收尾、上一轮实验员的提醒、可以写进"延续"的因子。 */
export function proposal(d: Draft): ChecklistOf<"proposal"> {
frame(d, [], () => {
const prev = d.round - 1;
const wraps = d.board.published("wrapUp").filter((w) => w.round === prev);
d.section(
"上一轮的收尾",
prev < 1 ? ["(这是第 1 轮,没有上一轮)"] : wraps.flatMap((w) => [`### ${w.author} 的收尾`, "", w.content.body, ""]),
);
d.section("上一轮实验员实现时的提醒", prev < 1 ? ["(这是第 1 轮,没有上一轮)"] : concernLines(d, prev));
previousWrapSection(d);
d.section("上一轮实验员实现时的提醒", d.round < 2 ? ["(这是第 1 轮,没有上一轮)"] : concernLines(d, d.round - 1));
const lines = d.board.protocols().flatMap((p) =>
p.protocol.factors.map((f, i) => `- ${factorId(p.candidate, factorKey(i))} ${f.name}(关键设定:${f.keySettings};候选 ${p.candidate},基线 ${p.protocol.baseline}${f.continues === null ? "" : `;它自己延续 ${f.continues}`})`),
);
@@ -258,11 +288,7 @@ export function explanation(d: Draft): ChecklistOf<"explanation"> {
"",
...factorLines(p),
]);
const artifacts = d.req.artifacts!;
const listed = Object.keys(artifacts)
.sort(compareIds)
.flatMap((run) => Object.entries(artifacts[run]!).map(([file, text]) => `- ${d.record(`${run}/${file}`, `运行 ${run} 的产物 ${file}`, text)}`));
d.section("本轮运行的产物", ["用 research_query 按编号查全文:", ...(listed.length === 0 ? ["(没有)"] : listed)]);
artifactSection(d, "本轮运行的产物");
});
return { kind: "explanation", cells: p.factors.map((_, i) => factorKey(i)), maxExplanations: d.req.study.proposalLimit };
}
+41 -2
View File
@@ -5,13 +5,17 @@ import type {
BetResult,
CellDecision,
CellRuleInput,
DiagnosisDecision,
DiagnosisRuleInput,
EntryAssignment,
SettleDiagnosisInput,
SettleInput,
adopt,
settleAblation,
settleDiagnosis,
} from "../rules/index.ts";
import type { Content } from "../ledger/index.ts";
import type { Json, Role, Submission, SubmissionKind } from "../vocab/index.ts";
import type { Json, Measured, Role, Submission, SubmissionKind, Verdict } from "../vocab/index.ts";
// 账本里只存成 Json 的几样内容长什么样、编号怎么起(用户 2026-10-03 决定 2):materials 读它们,flow 照着写。
// 决定和结算的输出直接用 rules 的返回类型;ledger 不认识这些类型,只存 Json。
@@ -51,6 +55,12 @@ export function splitCell(cell: string): { candidate: string; key: string } {
}
/** 解释:第 round 轮作者交的第 i 条(从 1 数)。结算里判确认、否定用它。 */
export const explanationId = (round: number, author: string, i: number): string => `r${round}-${author}-x${i}`;
/** 本轮诊断:诊断程序的冻结代码、诊断运行、诊断实现与核验的对象都是它。 */
export const diagnosisId = (round: number): string => `r${round}-diagnosis`;
/** 诊断实验在本轮合并后清单里的编号:第 i 个(从 0 数)是 d(i+1);它登记的指标全名是"实验编号.短名"。 */
export const diagnosisKey = (i: number): string => `d${i + 1}`;
/** 诊断解释:第 round 轮作者交的第 i 条(从 1 数)。 */
export const diagnosisExplanationId = (round: number, author: string, i: number): string => `r${round}-${author}-dx${i}`;
/** 模型调用:同一个任务(阶段、谁、哪种提交、对象)第 attempt 次发起。 */
export const callId = (stageId: string, actor: string, kind: string, object: string | null, attempt: number): string =>
`${stageId}.${actor}.${kind}.${object ?? "none"}.${attempt}`;
@@ -104,11 +114,19 @@ export function freezeProtocol(body: ProtocolBody, candidate: string, round: num
/** 候选代码的位置表,就是实验员交的那三项;基线和消融格的代码记 null(消融格用完整代码的位置表)。 */
export type LocationTable = Pick<Submission<"implementation">, "factorLocations" | "safeguardLocations" | "unassigned">;
/** 诊断程序的位置表:每个选中的实验一项(诊断实现交的 locations)。 */
export type DiagnosisLocationTable = Submission<"diagnosisImplementation">["locations"];
// ---------------------------------------------------------------------------
// 决定(账本记录 decision:decision 是种类,input、output 如下)
// ---------------------------------------------------------------------------
/**
* 诊断实验判重合并的输出(方法.md 4.3 诊 2):合并后的每个实验由哪几个作者的哪几个实验(index 是它在作者那份诊断提案里的位置,从 0 数)合成;
* sources 按作者编号排,第一个就是"作者里编号最小的那一份",合并后的正文取它的。
*/
export type DiagnosisDedupOutput = { experiments: Array<{ experiment: string; sources: Array<{ author: string; index: number }> }> };
/** 判重合并的输出(方法.md 4.3 ③):本轮新候选各合并了哪几条提案;留池候选各并进了哪几条新提案(可以没有)。 */
export type DedupOutput = {
fresh: Array<{ candidate: string; proposals: string[] }>;
@@ -127,9 +145,11 @@ export type DecisionShapes = {
adopt: { input: Parameters<typeof adopt>[0]; output: ReturnType<typeof adopt> };
cells: { input: CellRuleInput; output: CellDecision };
bet: { input: AblationModelInput & { cells: SettleInput["cells"] }; output: BetResult };
diagnosisDedup: { output: DiagnosisDedupOutput };
selectDiagnosis: { input: DiagnosisRuleInput; output: DiagnosisDecision };
};
export type DecisionKind = keyof DecisionShapes;
export const DECISION_KINDS: readonly DecisionKind[] = ["entries", "dedup", "batch", "adopt", "cells", "bet"];
export const DECISION_KINDS: readonly DecisionKind[] = ["entries", "dedup", "batch", "adopt", "cells", "bet", "diagnosisDedup", "selectDiagnosis"];
// ---------------------------------------------------------------------------
// 结算(账本记录 settlement.result,方法.md 4.3 ⑰)
@@ -142,6 +162,25 @@ export type Settlement = {
output: ReturnType<typeof settleAblation>;
};
/**
* 一次诊断的结算(账本记录 diagnosisSettlement.result,方法.md 6.12)。judged 是每个跑过的实验单独判分的结果,与 tasks 的
* DiagnosisJudged 同形(materials 不引用 tasks);诊断运行记录的 judged 为 null,逐个实验的结果只在这里。
* dropped 是核验不通过、修复用完而没跑的实验和原因;input、output 是 settleDiagnosis 的输入输出,没做的实验在里面记 untested。
*/
export type DiagnosisSettlement = {
run: string;
judged: {
experiments: Array<{ id: string; ending: "finished" | "crashed" | "timeout" | "notRun"; wallSeconds: Measured; verdict: Verdict; verdictReason: string; score: Measured; metrics: Record<string, Measured>; extra: Json }>;
};
dropped: Array<{ experiment: string; reason: string }>;
input: SettleDiagnosisInput;
output: ReturnType<typeof settleDiagnosis>;
};
/** 事件(账本记录 event):选中的诊断实验核验都没通过、修复用完,一个都不剩,不跑、不扣次(方法.md 4.3 诊 4)。objects 只有本轮诊断的编号。 */
export const DIAGNOSIS_NOT_RUN = "diagnosisNotRun";
export const diagnosisNotRunEvent = (round: number, reason: string): Content<"event"> => ({ event: DIAGNOSIS_NOT_RUN, reason, objects: [diagnosisId(round)] });
/**
* 事件(账本记录 event):一格消融核验不通过、重做次数用完,这一格不跑、记"证据不足"(方法.md 4.3 ⑯、6.6)。
* 这一格没有运行,信念本从这条事件读原因。objects 只有这一格的编号。
+20 -15
View File
@@ -1,9 +1,17 @@
import type { AblationCell } from "./ablation.ts";
import { compareIds, improvement, NET_TOLERANCE } from "./basics.ts";
import { ablationCellLimit } from "./budget.ts";
import { density } from "./distribution.ts";
import { seededSample } from "./random.ts";
import { ablationModel, evaluateCombos, type AblationModelInput, type ComboParams, type ComboValue } from "./value.ts";
import {
ablationModel,
ablationView,
evaluateCombos,
itemGain,
updatedWeights,
type AblationModelInput,
type ComboParams,
type ComboValue,
} from "./value.ts";
// 方法.md 6.7"选格"、第十一部分的四条选格规则,以及 6.8 押的因子。
@@ -121,10 +129,15 @@ function netOf(combos: readonly ComboValue[], key: string): number {
* "含这一格的净值最大组合"直接按这条挑(只是记录,不要求净值 > 1e-12)。
*/
export function topCombo<C extends { factors: readonly string[]; net: number }>(combos: readonly C[]): C | null {
return topByNet(combos, (c) => c.factors);
}
/** topCombo 的比较规则,组合里的项(因子或实验)怎么取由 items 给;选格和选诊断实验共用(6.7、6.12)。 */
export function topByNet<C extends { net: number }>(combos: readonly C[], items: (c: C) => readonly string[]): C | null {
if (combos.length === 0) return null;
const top = Math.max(...combos.map((c) => c.net));
const near = combos.filter((c) => c.net >= top - NET_TOLERANCE);
return near.reduce((best, c) => (compareCombos(c.factors, best.factors) < 0 ? c : best));
return near.reduce((best, c) => (compareCombos(items(c), items(best)) < 0 ? c : best));
}
/** 格数少的在前;格数相同时按因子编号逐个比。 */
@@ -207,19 +220,11 @@ export function chooseBet(input: AblationModelInput & { cells: readonly Ablation
const o = input.cells.find((c) => c.factor === f)?.outcome;
return o?.kind === "valid" ? o.score : null;
});
const raw = model.weights.map((w, j) =>
scores.reduce<number>((v, x, c) => (x === null ? v : v * density(model.rawKnots[c]![j]!, x)), w),
);
const sum = raw.reduce((t, v) => t + v, 0);
const weightsKept = sum === 0;
const weights = weightsKept ? [...model.weights] : raw.map((v) => v / sum);
const view = ablationView(model);
const { weights, kept: weightsKept } = updatedWeights(view, scores);
const gains = model.factors.map((factor, c) => {
const x = scores[c];
const gain =
x !== null && x !== undefined
? model.space * (improvement(x, input.full, input.direction) / model.gain)
: model.space * model.expected[c]!.reduce((t, e, j) => t + weights[j]! * e, 0);
return { factor, gain, measured: x !== null };
const x = scores[c] ?? null;
return { factor, gain: itemGain(view, c, weights, x), measured: x !== null };
});
let best = 0;
for (let c = 1; c < gains.length; c++) if (gains[c]!.gain > gains[best]!.gain) best = c;
+304
View File
@@ -0,0 +1,304 @@
import type { Direction, Measured, Range } from "../vocab/index.ts";
import { judgeExplanations, type AblationExplanation, type CellOutcome, type ExplanationVerdict } from "./ablation.ts";
import { combinations, compareIds, improvement, NET_TOLERANCE } from "./basics.ts";
import { bitWidth, disagreement, surprise, worldBits } from "./bits.ts";
import { topByNet } from "./cells.ts";
import { ablationCellMixture, knotsMean, mapKnots, mixtureRange } from "./distribution.ts";
import {
itemGain,
OPEN_RESERVE,
runCost,
updatedWeights,
usePotential,
valueOf,
type ComboParams,
type ValueModel,
} from "./value.ts";
// 方法.md 6.12 诊断:成分模型、方向收益、信息价值、价值减成本、选实验、结算。
// 成分、权重更新、精确积分、部分格网格、价值减成本的公式都在 value.ts,这里只给出诊断的"实测 → 收益"换算:
// 收益 g(x) = 改进量 x − b(越小越好的任务按 6.1 换成 b − x),另有收益为 0 的"都不走"。
/** 一条诊断解释:作者,以及对每个合并后的诊断实验给的三个数(实验做完后的分数,原值)。 */
export interface DiagnosisExplanation {
id: string;
author: string;
experiments: ReadonlyArray<{ experiment: string; range: Range }>;
}
export interface DiagnosisModelInput {
/** 本批基线 b。 */
baseline: number;
direction: Direction;
lambda: number;
/** 合并后的全部诊断实验编号(包括没选进组合的)。 */
experiments: readonly string[];
/** 每条解释对每个实验都给了三个数。 */
explanations: readonly DiagnosisExplanation[];
}
export interface DiagnosisModel extends ValueModel {
/** 实验编号,按编号排;ValueModel 里"项的下标"按这个次序。 */
experiments: string[];
/** 成分:各条解释的编号(按编号排),最后是开放预留。 */
components: string[];
}
/**
* 成分模型(6.12"成分与权重"):每条诊断解释一个成分,开放预留一个成分;每个成分对每个实验有一个分布,
* 解释用它自己的三个数补尾巴,开放预留用这个实验自己的均匀分布(6.3"诊断",和消融一格同一个合并函数)。
*/
export function diagnosisModel(input: DiagnosisModelInput): DiagnosisModel {
const experiments = [...input.experiments].sort(compareIds);
const explanations = [...input.explanations].sort((a, b) => compareIds(a.id, b.id));
const mixtures = experiments.map((d) =>
ablationCellMixture(
explanations.map((e) => ({ id: e.id, author: e.author, range: e.experiments.find((x) => x.experiment === d)!.range })),
input.lambda,
),
);
const scale = (x: number): number => improvement(input.baseline, x, input.direction);
const rawKnots = mixtures.map((m) => m.map((c) => c.knots));
const scaledKnots = rawKnots.map((row) => row.map((k) => mapKnots(k, scale, input.direction === "minimize")));
const weights = mixtures[0]!.map((c) => c.weight);
const expected = scaledKnots.map((row) => row.map(knotsMean));
return {
experiments,
components: [...explanations.map((e) => e.id), OPEN_RESERVE],
weights,
rawKnots,
scaledKnots,
expected,
prior: expected.map((row) => row.reduce((s, e, j) => s + weights[j]! * e, 0)),
space: 1,
floor: 0,
scale,
};
}
/** 一个诊断实验组合的价值与成本(6.12"价值减成本")。 */
export interface DiagnosisCombo {
/** 组合里的实验,按编号排。 */
experiments: string[];
voi: number;
/** R̂ = ⌈(余额 − 1) ÷ 每批上限⌉。 */
rHat: number;
mPrime: number;
/** 价值 = u × m′ × VOI。 */
value: number;
/** 成本:诊断只占一次运行,任何组合都一样。 */
cost: number;
net: number;
}
export interface DiagnosisParams extends ComboParams {
/** 一次诊断最多做几个实验 D(6.11)。 */
maxExperiments: number;
}
/**
* 列出可选组合并算价值、成本、净值(6.12"选实验"第 1、2 条):余额不到 2 次一个都没有;实验数不超过 D 时只有"全做",
* 超过 D 时是所有恰好 D 个实验的组合,按实验编号字典序列出。ĝ₀ = max(0, max_d E(g_d))。
*/
export function evaluateDiagnosisCombos(
model: DiagnosisModel,
params: DiagnosisParams,
): { g0: number; combos: DiagnosisCombo[] } {
const g0 = Math.max(0, ...model.prior);
if (params.balance < 2) return { g0, combos: [] };
const rHat = Math.ceil((params.balance - 1) / params.batchCap);
const mPrime = usePotential(rHat, params.decay);
const cost = runCost(g0, params.batchCap, 1);
const size = Math.min(model.experiments.length, params.maxExperiments);
const combos = combinations(model.experiments.length, size).map((S) => {
const voi = valueOf(model, S);
const value = params.u * mPrime * voi;
return { experiments: S.map((d) => model.experiments[d]!), voi, rHat, mPrime, value, cost, net: value - cost };
});
return { g0, combos };
}
export interface DiagnosisRuleInput extends DiagnosisModelInput, DiagnosisParams {}
/**
* 没选中的实验为什么没做(6.12"选实验"第 4 条):budget 放不下(已选的组合装满了 D 个);
* value 信息价值不足以抵过成本。
*/
export type SkippedReason = "budget" | "value";
/** 选诊断实验的结果,含 6.12 要求附上的全部数字;规则名和版本由登记表补上。 */
export interface DiagnosisDecision {
params: { balance: number; batchCap: number; maxExperiments: number; u: number; decay: number; lambda: number };
components: string[];
/** 起始权重,与 components 对齐。 */
weights: number[];
/** 各实验的事前期望收益 E(g_d),按实验编号排。 */
priorGains: Array<{ experiment: string; gain: number }>;
/** ĝ₀ = max(0, max_d E(g_d))。 */
g0: number;
/** 可选的全部组合及其价值、成本、净值(余额不到 2 次时为空)。 */
combos: DiagnosisCombo[];
/** 本轮做不做诊断:选中了净值 > 1e-12 的组合才做。 */
diagnose: boolean;
/** 选中要做的实验,按编号排;不诊断时为空。 */
chosen: string[];
/** 选中组合的净值;不诊断时是 0。 */
chosenNet: number;
/**
* 每个合并后的实验:做不做;没做的写原因,并附"含这个实验的组合里净值最大的那个";
* 没有任何含它的可选组合(余额不到 2 次,或没被任何组合包含)时 bestCombo 为 null。
*/
experiments: Array<{
experiment: string;
done: boolean;
reason: SkippedReason | null;
bestCombo: { experiments: string[]; net: number } | null;
}>;
}
/**
* 选诊断实验,规则"信息价值减成本"(6.12"选实验"):取净值最大、且净值 > 1e-12 的组合,净值相差不到 1e-12 的
* 视为相同、按实验编号从小到大比(和选格同一条挑法,见 topByNet);都不大于 1e-12 时本轮不诊断。
*/
export function selectDiagnosis(input: DiagnosisRuleInput): DiagnosisDecision {
const model = diagnosisModel(input);
const { g0, combos } = evaluateDiagnosisCombos(model, input);
const best = topByNet(combos.filter((c) => c.net > NET_TOLERANCE), (c) => c.experiments);
const chosen = best === null ? [] : [...best.experiments];
return {
params: {
balance: input.balance,
batchCap: input.batchCap,
maxExperiments: input.maxExperiments,
u: input.u,
decay: input.decay,
lambda: input.lambda,
},
components: model.components,
weights: model.weights,
priorGains: model.experiments.map((experiment, d) => ({ experiment, gain: model.prior[d]! })),
g0,
combos,
diagnose: best !== null,
chosen,
chosenNet: best === null ? 0 : best.net,
experiments: model.experiments.map((experiment) => {
const done = chosen.includes(experiment);
const top = topByNet(combos.filter((c) => c.experiments.includes(experiment)), (c) => c.experiments);
return {
experiment,
done,
reason: done ? null : chosen.length > 0 ? "budget" : "value",
bestCombo: done || top === null ? null : { experiments: [...top.experiments], net: top.net },
};
}),
};
}
/**
* 一个实验的结局(6.12"结算"):valid 有效分数;insufficient 证据不足(跑崩、超时、没有产物、修不了、核验不通过);
* untested 没做。和消融一格的三种同形,直接用 ablation.ts 的类型。
*/
export type ExperimentOutcome = Extract<CellOutcome, { kind: "valid" | "insufficient" | "untested" }>;
export interface SettleDiagnosisInput {
/** 本批基线 b。 */
baseline: number;
direction: Direction;
epsilon: number;
lambda: number;
/** 合并后的全部诊断实验及结局,没做的写 untested。 */
experiments: ReadonlyArray<{ experiment: string; outcome: ExperimentOutcome }>;
explanations: readonly DiagnosisExplanation[];
}
/** 诊断推出的方向(6.12"结算")。 */
export interface DiagnosisDirection {
/** 更新后的成分权重(与模型的成分对齐);无从更新时等于起始权重。 */
weights: number[];
/** 实测落在所有成分的范围之外、权重之和为 0 时为 true,此时保持起始权重。 */
weightsKept: boolean;
/** 各实验的收益(按编号排):measured 为 true 的是有效实验实测换算的,其余是按更新后的权重估的。 */
gains: Array<{ experiment: string; gain: number; measured: boolean }>;
/** 收益最大的实验(严格更大才替换,平局取编号小的)及其收益。 */
experiment: string;
gain: number;
/** 最大收益为正才算指出了方向;否则记"诊断没有指出能涨分的方向"。 */
found: boolean;
}
/**
* 一次诊断的结算(6.12"结算"、6.9):每个实验的收益、合并范围,有效实验的世界比特、惊讶、分歧量
* (格宽 ε/100,从 b 起切);每条诊断解释在每个有效实验上的确认或否定(和消融同一个判法);诊断推出的方向。
* 证据不足和没做的实验不判、不记账。
*/
export function settleDiagnosis(input: SettleDiagnosisInput): {
experiments: Array<{
experiment: string;
outcome: ExperimentOutcome["kind"];
gain: number | null;
merged: Range;
worldBits: number | null;
surprise: { cell: number; prior: number; bits: Measured } | null;
disagreement: number | null;
}>;
judgments: Array<{ id: string; verdicts: Array<{ experiment: string; verdict: ExplanationVerdict }> }>;
direction: DiagnosisDirection;
} {
const width = bitWidth(input.epsilon);
const experiments = [...input.experiments].sort((a, b) => compareIds(a.experiment, b.experiment));
const rows = experiments.map(({ experiment, outcome }) => {
const entries = input.explanations.map((e) => ({
id: e.id,
author: e.author,
range: e.experiments.find((x) => x.experiment === experiment)!.range,
}));
const mixture = ablationCellMixture(entries, input.lambda);
const valid = outcome.kind === "valid";
return {
experiment,
outcome: outcome.kind,
gain: valid ? improvement(input.baseline, outcome.score, input.direction) : null,
merged: mixtureRange(mixture),
worldBits: valid ? worldBits(mixture, input.baseline, width) : null,
surprise: valid ? surprise(mixture, input.baseline, width, outcome.score) : null,
disagreement: valid ? disagreement(entries, input.baseline, width) : null,
};
});
// 确认与否定沿用消融的判法:把"实验"当成"格"交给 judgeExplanations,结果再换回实验的叫法。
const asCells: AblationExplanation[] = input.explanations.map((e) => ({
id: e.id,
author: e.author,
cells: e.experiments.map((x) => ({ factor: x.experiment, range: x.range })),
}));
const judgments = judgeExplanations(asCells, experiments.map((e) => ({ factor: e.experiment, outcome: e.outcome }))).map(
(j) => ({ id: j.id, verdicts: j.verdicts.map((v) => ({ experiment: v.factor, verdict: v.verdict })) }),
);
return {
experiments: rows,
judgments,
direction: diagnosisDirection(
diagnosisModel({ ...input, experiments: experiments.map((e) => e.experiment) }),
experiments.map((e) => (e.outcome.kind === "valid" ? e.outcome.score : null)),
),
};
}
/** 诊断推出的方向:按有效实验的实测更新成分权重,有效实验用实测换算收益,其余实验用新权重估,取收益最大的(6.12"结算")。 */
function diagnosisDirection(model: DiagnosisModel, scores: ReadonlyArray<number | null>): DiagnosisDirection {
const { weights, kept } = updatedWeights(model, scores);
const gains = model.experiments.map((experiment, d) => {
const x = scores[d] ?? null;
return { experiment, gain: itemGain(model, d, weights, x), measured: x !== null };
});
let best = 0;
for (let d = 1; d < gains.length; d++) if (gains[d]!.gain > gains[best]!.gain) best = d;
return {
weights,
weightsKept: kept,
gains,
experiment: gains[best]!.experiment,
gain: gains[best]!.gain,
found: gains[best]!.gain > 0,
};
}
+20
View File
@@ -103,6 +103,23 @@ export {
worldBits,
type Placement,
} from "./bits.ts";
export {
diagnosisModel,
evaluateDiagnosisCombos,
selectDiagnosis,
settleDiagnosis,
type DiagnosisCombo,
type DiagnosisDecision,
type DiagnosisDirection,
type DiagnosisExplanation,
type DiagnosisModel,
type DiagnosisModelInput,
type DiagnosisParams,
type DiagnosisRuleInput,
type ExperimentOutcome,
type SettleDiagnosisInput,
type SkippedReason,
} from "./diagnosis.ts";
export { ablationCellLimit, forwardBatchSize, runBalance, type RunState } from "./budget.ts";
export { authorVsMerged, trackRecord, type Phase, type Tally } from "./record.ts";
export { assignEntries, type EntryAssignment } from "./entries.ts";
@@ -110,8 +127,11 @@ export { decisionSeed, seededRandom, seededSample } from "./random.ts";
export {
BATCH_RULES,
CELL_RULES,
DIAGNOSIS_RULES,
runBatchRule,
runCellRule,
runDiagnosisRule,
type BatchRuleName,
type CellRuleName,
type DiagnosisRuleName,
} from "./registry.ts";
+7 -5
View File
@@ -2,10 +2,12 @@ import type { Direction, Range } from "../vocab/index.ts";
import { compareIds, improvement } from "./basics.ts";
import { covers, placement, type Placement } from "./bits.ts";
// 方法.md 6.10 第 4 条"战绩"、6.9"按人分开、按正向与消融分开"。只用来显示,不改任何权重。
// 方法.md 6.10 第 4 条"战绩"、6.9"按人分开、按正向、诊断与消融分开"。只用来显示,不改任何权重。
// 一律按分数原值比,不按方向折算。
export type Phase = "forward" | "ablation";
export type Phase = "forward" | "diagnosis" | "ablation";
const PHASE_ORDER: readonly Phase[] = ["forward", "diagnosis", "ablation"];
/** 一个人的战绩。 */
export interface Tally {
@@ -25,8 +27,8 @@ export interface Tally {
}
/**
* 战绩:按研究者、按正向与消融分开数(6.9、6.10)。samples 是每人每次的范围(第二次预测或解释的每格)和有效实测。
* 结果按研究者编号、再按 forward、ablation 排,只列有样本的组合。
* 战绩:按研究者、按正向、诊断与消融分开数(6.9、6.10)。samples 是每人每次的范围(第二次预测、诊断预测或解释的每格)和有效实测。
* 结果按研究者编号、再按 forward、diagnosis、ablation 排,只列有样本的组合。
*/
export function trackRecord(
samples: ReadonlyArray<{ researcher: string; phase: Phase; range: Range; value: number }>,
@@ -62,7 +64,7 @@ export function trackRecord(
t.placements[placement(r, v)] += 1;
}
return [...groups.values()].sort(
(a, b) => compareIds(a.researcher, b.researcher) || (a.phase === b.phase ? 0 : a.phase === "forward" ? -1 : 1),
(a, b) => compareIds(a.researcher, b.researcher) || PHASE_ORDER.indexOf(a.phase) - PHASE_ORDER.indexOf(b.phase),
);
}
+16
View File
@@ -1,4 +1,5 @@
import { selectAll, selectByValue, selectNone, selectRandomSameCount, type CellDecision, type CellRuleInput } from "./cells.ts";
import { selectDiagnosis, type DiagnosisDecision, type DiagnosisRuleInput } from "./diagnosis.ts";
import { selectBestExpected, selectUniformRandom, type BatchDecision, type BatchInput } from "./forward.ts";
// 规则登记表(结构.md 第五部分"决策规则"、方法.md 第十一部分):名字 → 函数,带版本号。
@@ -22,8 +23,14 @@ export const CELL_RULES = {
none: { version: 1, run: selectNone },
} as const satisfies Record<string, { version: number; run: (input: CellRuleInput) => CellDecision }>;
export const DIAGNOSIS_RULES = {
/** 信息价值减成本(6.12);选诊断实验只有这一条规则,配置里不选。 */
valueMinusCost: { version: 1, run: selectDiagnosis },
} as const satisfies Record<string, { version: number; run: (input: DiagnosisRuleInput) => DiagnosisDecision }>;
export type BatchRuleName = keyof typeof BATCH_RULES;
export type CellRuleName = keyof typeof CELL_RULES;
export type DiagnosisRuleName = keyof typeof DIAGNOSIS_RULES;
/** 按名字跑一条选批规则,结果带上规则名和版本。 */
export function runBatchRule(
@@ -42,3 +49,12 @@ export function runCellRule(
const entry = CELL_RULES[name];
return { rule: { name, version: entry.version }, ...entry.run(input) };
}
/** 按名字跑一条选诊断实验规则,结果带上规则名和版本。 */
export function runDiagnosisRule(
name: DiagnosisRuleName,
input: DiagnosisRuleInput,
): DiagnosisDecision & { rule: { name: DiagnosisRuleName; version: number } } {
const entry = DIAGNOSIS_RULES[name];
return { rule: { name, version: entry.version }, ...entry.run(input) };
}
+139 -31
View File
@@ -9,13 +9,44 @@ import {
GRID_SUBDIVISIONS,
improvement,
} from "./basics.ts";
import { ablationCellMixture, below, integrateUpperTail, knotsMean, mapKnots, type Knots } from "./distribution.ts";
import {
ablationCellMixture,
below,
density,
integrateUpperTail,
knotsMean,
mapKnots,
type Knots,
} from "./distribution.ts";
// 方法.md 6.7:方向收益、成分与权重、信息价值、价值减成本。选格规则和押的因子在 cells.ts。
//
// 一律先在"撑几成"刻度 s(x) = [f(完整) − x] ÷ G(按方向折算)上算,最后乘剩余空间 h:
// 收益_p = h × s_p,h 对所有因子相同且为正,所以信息价值、期望收益、ĝ₀ 都等于 h × (撑几成刻度上的同一个量)。
// 例外是部分格网格的切段:在分数原值上取折点、切小段,再把中点换成撑几成(见 measuredGrid)。
// 成分、权重更新、精确积分、部分格网格、价值减成本这套引擎消融(6.7)和诊断(6.12)共用,只差"实测 → 收益"的换算:
// - 消融:一律先在"撑几成"刻度 s(x) = [f(完整) − x] ÷ G(按方向折算)上算,最后乘剩余空间 h:
// 收益_p = h × s_p,h 对所有因子相同且为正,所以信息价值、期望收益、ĝ₀ 都等于 h × (撑几成刻度上的同一个量);
// 不设"都不走"的选项。
// - 诊断(diagnosis.ts):收益 g(x) = 改进量 x − b,h 取 1,另有收益为 0 的"都不走",即 floor = 0。
// 例外是部分格网格的切段:在分数原值上取折点、切小段,再把中点换成收益刻度(见 gridOf)。
/** 引擎要的成分模型(消融和诊断各自建出来)。下标"项"是因子(消融)或实验(诊断)。 */
export interface ValueModel {
/** 成分起始权重。 */
weights: number[];
/** rawKnots[项][成分]:分数原值上的折线。 */
rawKnots: Knots[][];
/** scaledKnots[项][成分]:收益刻度上的折线。 */
scaledKnots: Knots[][];
/** expected[项][成分]:这个成分下这一项在收益刻度上的期望。 */
expected: number[][];
/** prior[项]:按起始权重平均的期望。 */
prior: number[];
/** 收益 = space × 收益刻度上的值:消融是剩余空间 h,诊断是 1。 */
space: number;
/** "都不走"的收益:诊断是 0,消融没有这个选项,写 null。 */
floor: number | null;
/** 分数原值 → 收益刻度。 */
scale: (x: number) => number;
}
/** 开放预留成分的名字(成分名单里排在最后)。 */
export const OPEN_RESERVE = "openReserve";
@@ -70,6 +101,20 @@ function supportScale(full: number, direction: Direction, gain: number): (x: num
return (x) => improvement(x, full, direction) / gain;
}
/** 消融模型在引擎眼里的样子:收益刻度是撑几成,没有"都不走"。 */
export function ablationView(model: AblationModel): ValueModel {
return {
weights: model.weights,
rawKnots: model.rawKnots,
scaledKnots: model.supportKnots,
expected: model.expected,
prior: model.prior,
space: model.space,
floor: null,
scale: supportScale(model.full, model.direction, model.gain),
};
}
/**
* 成分模型(6.7"成分与权重"):每条解释一个成分,开放预留一个成分;每个成分对每一格有一个分布,
* 解释用它自己的三个数补尾巴,开放预留用这一格的均匀分布(6.3"消融")。同一成分内各格互相独立。
@@ -144,26 +189,33 @@ export function cellGrid(knots: readonly Knots[], k: number): { mids: number[];
}
/**
* 模型里第 c 格的网格(6.7"只测一部分格"),测 k 格时用。在分数原值上取折点、切小段:方法说"每格在自己的开放预留范围内
* 模型里第 c 项的网格(6.7"只测一部分格"),测 k 项时用。在分数原值上取折点、切小段:方法说"每格在自己的开放预留范围内
* 按折点切段",开放预留和折点都是在分数原值上定的(尾巴按 q1 − (q5 − q1)、q9 + (q9 − q5),预留按 L − W/2、U + W/2
* 的浮点原式),去重只合并浮点上完全相等的值。先换到撑几成刻度再去重,相差一个末位的两个原值折点可能换算后变成同一个数,
* 段数和 s 就会跟原值上的不一样。切好后只把小段中点换算成撑几成;把握与刻度无关。
* 的浮点原式),去重只合并浮点上完全相等的值。先换到收益刻度再去重,相差一个末位的两个原值折点可能换算后变成同一个数,
* 段数和 s 就会跟原值上的不一样。切好后只把小段中点换算成收益刻度;把握与刻度无关。
*/
export function measuredGrid(model: AblationModel, c: number, k: number): { mids: number[]; mass: number[][] } {
export function gridOf(model: ValueModel, c: number, k: number): { mids: number[]; mass: number[][] } {
const { mids, mass } = cellGrid(model.rawKnots[c]!, k);
return { mids: mids.map(supportScale(model.full, model.direction, model.gain)), mass };
return { mids: mids.map(model.scale), mass };
}
/** 消融模型里第 c 格的网格,中点换算成撑几成(6.7"只测一部分格")。 */
export function measuredGrid(model: AblationModel, c: number, k: number): { mids: number[]; mass: number[][] } {
return gridOf(ablationView(model), c, k);
}
/**
* 只测 S 里的格时的 E_{S 的结果}[max_p E(撑_p | S 的结果)](6.7"信息价值"),在撑几成刻度上。
* 对测到的格的全部小段组合求和:成分 j 的联合把握 = 起始权重 × 各测到格在所选小段的把握;它们的和 marg 是这个组合的把握,
* 按它归一就是更新后的权重。网格见 measuredGrid。测到的格取小段中点(已换成撑几成),没测的格用更新后的权重平均各成分的期望,取各因子里最大的。
* 只测 S 里的项时的 E_{S 的结果}[max(floor, max_p E(收益_p | S 的结果))](6.7"信息价值"、6.12),在收益刻度上,
* 没有"都不走"(floor 为 null)时不和 floor 取大。
* 对测到的项的全部小段组合求和:成分 j 的联合把握 = 起始权重 × 各测到项在所选小段的把握;它们的和 marg 是这个组合的把握,
* 按它归一就是更新后的权重。网格见 gridOf。测到的项取小段中点(已换成收益刻度),没测的项用更新后的权重平均各成分的期望,取各项里最大的。
*/
function expectedBestGrid(model: AblationModel, S: readonly number[]): number {
function expectedBestGrid(model: ValueModel, S: readonly number[]): number {
const k = S.length;
const J = model.weights.length;
const grids = S.map((c) => measuredGrid(model, c, k));
const rest = model.factors.map((_, c) => c).filter((c) => !S.includes(c));
const floor = model.floor ?? -Infinity;
const grids = S.map((c) => gridOf(model, c, k));
const rest = model.prior.map((_, c) => c).filter((c) => !S.includes(c));
const partial = Array.from({ length: k + 1 }, () => new Float64Array(J));
for (let j = 0; j < J; j++) partial[0]![j] = model.weights[j]!;
let total = 0;
@@ -173,7 +225,7 @@ function expectedBestGrid(model: AblationModel, S: readonly number[]): number {
let marg = 0;
for (let j = 0; j < J; j++) marg += cur[j]!;
if (marg === 0) return;
let best = seen;
let best = Math.max(seen, floor);
for (const c of rest) {
const e = model.expected[c]!;
let num = 0;
@@ -195,36 +247,80 @@ function expectedBestGrid(model: AblationModel, S: readonly number[]): number {
}
/**
* 全部格都测时的 E[max_p 撑_p](6.7"信息价值"):同一成分内"各格撑几成的最大值低于 t"的把握 = 各格把握连乘,
* 期望 = 下端 + ∫(1 − 连乘) dt,逐段多项式精确积分;再按起始权重相加。
* 全部项都测时的 E[max(floor, max_p 收益_p)](6.7"信息价值"、6.12):同一成分内"各项最大值低于 t"的把握 = 各项把握连乘,
* 期望 = 下端 + ∫(1 − 连乘) dt,逐段多项式精确积分;再按起始权重相加。下端:有"都不走"时是 floor(t 低于它时
* max(floor, ·) 一定不低于 t,被积函数是 1),没有时是各项折线的最低点。
*/
function expectedBestAll(model: AblationModel): number {
function expectedBestAll(model: ValueModel): number {
let v = 0;
for (let j = 0; j < model.weights.length; j++) {
const lines = model.supportKnots.map((cell) => cell[j]!);
const lo = Math.min(...lines.map((k) => k[0]![0]));
const lines = model.scaledKnots.map((cell) => cell[j]!);
const lo = model.floor ?? Math.min(...lines.map((k) => k[0]![0]));
v += model.weights[j]! * (lo + integrateUpperTail(lines.map((knots) => [{ weight: 1, knots }]), lo));
}
return v;
}
/** max(floor, max_p E(收益_p)):不测时能拿到的最好期望,在收益刻度上。 */
function bestPrior(model: ValueModel): number {
return Math.max(model.floor ?? -Infinity, ...model.prior);
}
/** 用网格算的信息价值(不论测几项都走网格),S 是项的下标。 */
export function valueByGrid(model: ValueModel, S: readonly number[]): number {
return model.space * (expectedBestGrid(model, [...S].sort((a, b) => a - b)) - bestPrior(model));
}
/** 全部项都测时的信息价值(一维精确积分)。 */
export function valueExact(model: ValueModel): number {
return model.space * (expectedBestAll(model) - bestPrior(model));
}
/**
* 信息价值 VOI(S) = E_{S 的结果}[max(floor, max_p E(收益_p | S 的结果))] − max(floor, max_p E(收益_p))(6.7、6.12),
* S 是项的下标;全部项都测走一维精确积分,只测一部分走网格,空集是 0。不截断。
*/
export function valueOf(model: ValueModel, S: readonly number[]): number {
if (S.length === 0) return 0;
return S.length === model.prior.length ? valueExact(model) : valueByGrid(model, S);
}
/** 网格算法算的信息价值,S 是格的下标(不论测几格都走网格;用来和精确算法互相对照)。 */
export function informationValueGrid(model: AblationModel, S: readonly number[]): number {
return model.space * (expectedBestGrid(model, [...S].sort((a, b) => a - b)) - Math.max(...model.prior));
return valueByGrid(ablationView(model), S);
}
/** 全部格都测时的信息价值(一维精确积分)。 */
export function informationValueExact(model: AblationModel): number {
return model.space * (expectedBestAll(model) - Math.max(...model.prior));
return valueExact(ablationView(model));
}
/** 消融的信息价值 VOI(S)(6.7),S 是格的下标。 */
export function informationValue(model: AblationModel, S: readonly number[]): number {
return valueOf(ablationView(model), S);
}
/**
* 信息价值 VOI(S) = E_{S 的结果}[max_p E(收益_p | S 的结果)] − max_p E(收益_p)(6.7),S 是格的下标;
* 全部格都测走一维精确积分,只测一部分走网格,空集是 0。不截断。
* 成分新权重(6.8、6.12):起始权重 × 各有效项(scores 里不是 null 的)上这个成分在实测处的密度(分数原值上,连乘),
* 再归一;实测落在所有成分之外、权重之和为 0 时保持起始权重。scores 与项的下标对齐,没有有效实测的项为 null。
*/
export function informationValue(model: AblationModel, S: readonly number[]): number {
if (S.length === 0) return 0;
return S.length === model.factors.length ? informationValueExact(model) : informationValueGrid(model, S);
export function updatedWeights(
model: ValueModel,
scores: ReadonlyArray<number | null>,
): { weights: number[]; kept: boolean } {
const raw = model.weights.map((w, j) =>
scores.reduce<number>((v, x, c) => (x === null ? v : v * density(model.rawKnots[c]![j]!, x)), w),
);
const sum = raw.reduce((t, v) => t + v, 0);
const kept = sum === 0;
return { weights: kept ? [...model.weights] : raw.map((v) => v / sum), kept };
}
/** 第 c 项的方向收益(6.8、6.12):有有效实测 x 就用实测换算,否则用 weights 对各成分的期望加权平均。 */
export function itemGain(model: ValueModel, c: number, weights: readonly number[], x: number | null): number {
return x !== null
? model.space * model.scale(x)
: model.space * model.expected[c]!.reduce((t, e, j) => t + weights[j]! * e, 0);
}
/** 一个测格组合的价值与成本(6.7"价值减成本")。 */
@@ -253,6 +349,19 @@ export interface ComboParams {
decay: number;
}
/** m′ = 1 + r + … + r^(R̂−1),R̂ = 0 时是 0(6.7"价值减成本");消融和诊断共用。 */
export function usePotential(rHat: number, decay: number): number {
let mPrime = 0;
for (let j = 0; j < rHat; j++) mPrime += decay ** j;
return mPrime;
}
/** 成本 = 2 max(0, ĝ₀) × [(K + n)/(K + n + 1) − K/(K + 1)],n 是这次占用的运行次数(6.7、6.12);消融和诊断共用。 */
export function runCost(g0: number, batchCap: number, runs: number): number {
const K = batchCap;
return 2 * Math.max(0, g0) * ((K + runs) / (K + runs + 1) - K / (K + 1));
}
/**
* 列出格数 1..min(格数, 余额 − 1) 的全部非空组合并算价值、成本、净值(6.7"选格"第 1 条)。余额为 0 或 1 时一个都没有。
* 组合按格数从少到多、同格数按因子编号字典序列出。ĝ₀ = h × max_p 期望撑几成,原值另给,成本里取 max(0, ĝ₀)。
@@ -265,10 +374,9 @@ export function evaluateCombos(model: AblationModel, params: ComboParams): { g0:
for (const S of combinations(model.factors.length, size)) {
const voi = informationValue(model, S);
const rHat = Math.ceil((params.balance - size) / K);
let mPrime = 0;
for (let j = 0; j < rHat; j++) mPrime += params.decay ** j;
const mPrime = usePotential(rHat, params.decay);
const value = params.u * mPrime * voi;
const cost = 2 * Math.max(0, g0) * ((K + size) / (K + size + 1) - K / (K + 1));
const cost = runCost(g0, K, size);
combos.push({ factors: S.map((c) => model.factors[c]!), voi, rHat, mPrime, value, cost, net: value - cost });
}
}
+48
View File
@@ -23,6 +23,15 @@ export function programProblems(inc: Incoming, protocol: ProtocolBody | null): s
...factorProblems(inc.content, inc.checklist),
...increasing("authorEstimate", inc.content.authorEstimate),
];
case "diagnosisProposal":
return diagnosisProposalProblems(inc.content, inc.checklist);
case "diagnosisPrediction":
return diagnosisPredictionProblems(inc.content, inc.checklist);
case "diagnosisImplementation":
case "diagnosisRepair":
return codeProblems(inc.known.code);
case "diagnosisVerification":
return covers("experiments 的实验", inc.content.experiments.map((x) => x.experiment), inc.checklist.experiments);
case "firstPrediction":
case "secondPrediction":
return increasing("range", inc.content.range);
@@ -63,6 +72,45 @@ function claimProblems(c: Submission<"claim">, k: ChecklistOf<"claim">): string[
];
}
/**
* 诊断解释与诊断实验(方法.md 4.3 诊 1、第五部分"指标"):解释 1 至 M 条、实验 1 至 A 个(下限由字段定义管);
* 诊断实验登记的指标只能选"缺了只记缺失";同一个实验里指标短名不重名、不占用已有编号
* (不同实验的同名短名冻结时各带自己实验的前缀,不会撞)。
*/
function diagnosisProposalProblems(c: Submission<"diagnosisProposal">, k: ChecklistOf<"diagnosisProposal">): string[] {
return [
...(c.explanations.length > k.maxExplanations ? [`交了 ${c.explanations.length} 条诊断解释,最多 ${k.maxExplanations} 条`] : []),
...(c.experiments.length > k.maxExperiments ? [`交了 ${c.experiments.length} 个诊断实验,最多 ${k.maxExperiments} 个`] : []),
...c.experiments.flatMap((e, i) => {
const ids = e.metrics.map((m) => m.id);
return [
...e.metrics
.filter((m) => m.missing !== "recordMissing")
.map((m) => `第 ${i + 1} 个实验的指标 ${m.id} 选了 ${m.missing}:诊断实验里的指标只能选 recordMissing(缺了只记缺失)`),
...repeated(ids).map((x) => `第 ${i + 1} 个实验的 metrics 里编号 ${x} 写了不止一次`),
...unique(ids)
.filter((x) => k.takenMetricIds.includes(x))
.map((x) => `第 ${i + 1} 个实验的指标编号 ${x} 已被占用(已占用的是 ${k.takenMetricIds.join("、")}):换一个编号`),
];
}),
];
}
/**
* 诊断预测(方法.md 4.3 诊 2):覆盖自己每条解释 × 每个合并后的实验,不多不少、每一对一次;每个范围三个数严格递增。
* 一对写成"解释编号:实验编号"(编号里没有冒号)。
*/
function diagnosisPredictionProblems(c: Submission<"diagnosisPrediction">, k: ChecklistOf<"diagnosisPrediction">): string[] {
return [
...covers(
"predictions 的(解释:实验)",
c.predictions.map((p) => `${p.explanation}:${p.experiment}`),
k.explanations.flatMap((e) => k.experiments.map((d) => `${e}:${d}`)),
),
...c.predictions.flatMap((p) => increasing(`${p.explanation}:${p.experiment} 的 range`, p.range)),
];
}
/**
* 提案、回应、统一共用(方法.md 第五部分"延续""指标"、4.3 ②⑥):
* 规则 2 因子数上限;3 两个因子不延续同一个、延续的编号看得到;4 保障短名;10 指标的引用与新登记。
+1 -1
View File
@@ -1,6 +1,6 @@
// submissions:一份提交收下还是退回,先程序检查、再问 Jev(docs/结构.md 第二部分);
// 另有冻结协议的拼法,和核验、消融核验收下之后通过没有。不读也不写账本。
export { ablationOutcome, verificationOutcome } from "./outcomes.ts";
export { ablationOutcome, diagnosisOutcome, verificationOutcome } from "./outcomes.ts";
export { protocolBody } from "./protocol.ts";
export { receive } from "./receive.ts";
export {
+17
View File
@@ -34,3 +34,20 @@ export function ablationOutcome(
const wronglyOff = v.factors.filter((f) => !shouldBeOff.has(f.factor) && f.off).map((f) => f.factor);
return { passed: leftOn.length + wronglyOff.length === 0, leftOn, wronglyOff };
}
/**
* 诊断核验通过没有(方法.md 4.3 诊 4):逐个实验判,任何一项写了 false 的实验不通过。
* notFollowed 是没照写法做的实验,touchedElse 是动了写法以外的地方的实验,failing 是两者的并集,按核验里的次序。
* 一个实验不通过不影响别的实验:通过的实验代码不再改动。
*/
export function diagnosisOutcome(v: Submission<"diagnosisVerification">): {
passed: boolean;
failing: string[];
notFollowed: string[];
touchedElse: string[];
} {
const notFollowed = v.experiments.filter((x) => !x.followed).map((x) => x.experiment);
const touchedElse = v.experiments.filter((x) => !x.nothingElse).map((x) => x.experiment);
const failing = v.experiments.filter((x) => !x.followed || !x.nothingElse).map((x) => x.experiment);
return { passed: failing.length === 0, failing, notFollowed, touchedElse };
}
+16 -3
View File
@@ -1,4 +1,4 @@
import { checkHunk, checkProtocol, type CodeHunk } from "../jev/index.ts";
import { checkExperiment, checkHunk, checkProtocol, type CodeHunk } from "../jev/index.ts";
import type { JevJudgment } from "../vocab/index.ts";
import { programProblems } from "./checks.ts";
import { protocolBody } from "./protocol.ts";
@@ -27,7 +27,8 @@ export async function receive(inc: Incoming): Promise<Verdict> {
}
/**
* 问 Jev(方法.md 4.3 ⑥⑨):回应、统一对拼好的协议问冻结条款三问;实现、修复每个差异块问代码改动五问,顺序发。
* 问 Jev(方法.md 4.3 ⑥⑨、诊 4):回应、统一对拼好的协议问冻结条款三问;实现、修复(含诊断程序)每个差异块问代码改动五问,顺序发。
* 诊断解释与实验(诊 1)对每个实验问冻结条款三问(改动了评测、自相矛盾、太含糊,用诊断的变体问法),顺序发。
* 没有差异块的改动(二进制、只改可执行位、空文件增删)Jev 看不到内容,不问;一块都没有就等于没问,交回 null。
* 消融实现不问:冻结协议里没有登记"关掉以后"的随机性和停止条件,五问没有对照(照参考项目)。
* flagged 是退回原因里每一处的文字:只列判"是"的几问,问题名用 jev 的英文名(用户 2026-10-03 定);判"是"只看 jev 交回的 yes。
@@ -37,7 +38,19 @@ async function askJev(inc: Incoming, protocol: ProtocolBody | null): Promise<{ j
const c = await checkProtocol(inc.jevModel, protocol!);
return { judgments: c.judgments, flagged: c.flagged ? [`- 冻结协议:${flaggedText(c.judgments)}`] : [] };
}
if (inc.kind !== "implementation" && inc.kind !== "repair") return null;
if (inc.kind === "diagnosisProposal") {
const judgments: JevJudgment[] = [];
const flagged: string[] = [];
for (const [i, experiment] of inc.content.experiments.entries()) {
const c = await checkExperiment(inc.jevModel, experiment);
judgments.push(...c.judgments);
if (c.flagged) flagged.push(`- 第 ${i + 1} 个诊断实验(${experiment.name}):${flaggedText(c.judgments)}`);
}
return { judgments, flagged };
}
if (inc.kind !== "implementation" && inc.kind !== "repair" && inc.kind !== "diagnosisImplementation" && inc.kind !== "diagnosisRepair") {
return null;
}
// 走到这里程序检查已过,代码一定存成了版本。
const code = inc.known.code as Extract<typeof inc.known.code, { saved: true }>;
if (code.hunks.length === 0) return null;
+4 -1
View File
@@ -17,7 +17,7 @@ export type OriginalProposal = Pick<Submission<"proposal">, "factors" | "safegua
type NotSaved = { saved: false; path: string; reason: string };
/**
* 实现、修复的代码:存成了版本就带改了的不许改的文件(tasks 的 protectedChanges)、
* 实现、修复(候选和诊断程序同样)的代码:存成了版本就带改了的不许改的文件(tasks 的 protectedChanges)、
* 和本批基线比出的差异块(jev 的 codeHunks 切好的),以及给 Jev 对照的协议正文和要采集的指标。
*/
export type ForwardCode = NotSaved | { saved: true; protectedFiles: string[]; hunks: CodeHunk[]; protocol: ProtocolRules };
@@ -33,6 +33,9 @@ type KnownByKind = {
response: { original: OriginalProposal };
implementation: { code: ForwardCode };
repair: { code: ForwardCode };
/** 诊断程序和候选代码同形:和当前基线比出的差异块,给 Jev 对照的是选中实验的原文和要采集的指标。 */
diagnosisImplementation: { code: ForwardCode };
diagnosisRepair: { code: ForwardCode };
/** 交了"跑不起来"(cannotRun 不是 null)时 flow 不存代码,code 为 null。 */
ablationImplementation: { code: AblationCode | null };
};
+227 -13
View File
@@ -1,4 +1,4 @@
import { copyFileSync, cpSync, existsSync, readFileSync, writeFileSync } from "node:fs";
import { chmodSync, copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { Type, type Static } from "typebox";
import { Value } from "typebox/value";
@@ -13,7 +13,16 @@ import type {
} from "../vocab/index.ts";
import type { TaskFacts } from "./facts.ts";
import { measured, readMetricsFile, unmeasured } from "./metrics-file.ts";
import type { LightCheckOutcome, Task, TaskImplementation } from "./types.ts";
import type {
DiagnosisEnding,
DiagnosisExperiment,
DiagnosisExperimentJudged,
DiagnosisJudged,
DiagnosisJudgingStep,
LightCheckOutcome,
Task,
TaskImplementation,
} from "./types.ts";
// 填圆:在单位正方形里放 26 个圆,最大化半径之和。任务说明见 tasks/circle_packing/TASK.md。
// 凡是要执行任务代码的事(正式运行、校验器、轻量检查)都只交出运行定义,由调用方交给 runner 在任务镜像里跑;
@@ -22,6 +31,7 @@ import type { LightCheckOutcome, Task, TaskImplementation } from "./types.ts";
const CIRCLES = 26;
const ENTRY = "initial.py";
const HARNESS = "harness.py";
const DIAGNOSIS_HARNESS = "harness-diagnosis.py";
const VALIDATOR = "validate.py";
const CHECK_SCRIPT = "check-candidate.py";
const METRICS_SPEC = "metrics-spec.json";
@@ -29,6 +39,12 @@ const METRICS = "metrics.json";
const SOLUTION = "solution.json";
const VERDICT = "verdict.json";
const CHECK_RESULT = "check.json";
/** 诊断运行:原始产物目录、每个实验的答案目录、harness 写的每实验用时报告(路径都相对运行目录)。 */
const BASELINE_ARTIFACT = "baseline-artifact";
const ANSWERS = "answers";
const DIAGNOSIS_REPORT = "diagnosis.json";
/** 判分目录里每个实验写的校验器退出码文件(同一个目录下还有这个实验的 solution.json、verdict.json)。 */
const EXIT_FILE = "exit";
/** 轻量检查目录里放候选代码的子目录;检查脚本放在它的上一层,免得扫到自己。 */
const CHECK_CANDIDATE = "candidate";
/** 运行目录挂进容器的位置。 */
@@ -277,16 +293,16 @@ function readVerdict(judgeDir: string, exitCode: number): ValidatorVerdict {
return verdict;
}
function judge(
_task: Task,
runDir: string,
judgeDir: string,
formal: ProcessResult,
validator: ProcessResult,
hypothesisMetrics: HypothesisMetric[],
): JudgedResult {
const run = started(formal);
const verdict = readVerdict(judgeDir, requireExit("校验器", validator, [0, 1]));
/** 校验器判定读成的结论:算不算数、理由、最终分数、校验器给的辅助指标、合法化明细。 */
interface Scored {
valid: boolean;
reason: string;
score: Measured;
fromValidator: Record<string, Measured>;
extra: Json;
}
function scoreVerdict(verdict: ValidatorVerdict): Scored {
const legal = verdict.legalization;
let deduction: Measured;
let score: Measured;
@@ -328,7 +344,199 @@ function judge(
deduction,
},
};
return assemble(runDir, run, hypothesisMetrics, { valid: verdict.valid, reason }, score, fromValidator, extra);
return { valid: verdict.valid, reason, score, fromValidator, extra };
}
function judge(
_task: Task,
runDir: string,
judgeDir: string,
formal: ProcessResult,
validator: ProcessResult,
hypothesisMetrics: HypothesisMetric[],
): JudgedResult {
const run = started(formal);
const scored = scoreVerdict(readVerdict(judgeDir, requireExit("校验器", validator, [0, 1])));
return assemble(runDir, run, hypothesisMetrics, { valid: scored.valid, reason: scored.reason }, scored.score, scored.fromValidator, scored.extra);
}
// ---------------------------------------------------------------------------
// 诊断运行(方法.md 4.3"诊断"一段):一个容器、一次登记、一次扣次,里面按顺序跑几个实验。
// 全部是新增的一套函数,用自己的 harness(harness-diagnosis.py),不碰上面正式运行的任何一步。
// ---------------------------------------------------------------------------
/** 诊断程序存版本时跳过的名字:正式运行那份加上原始产物目录(它是宿主放进工作目录的,不是实验员的代码)。 */
function diagnosisUnversionedNames(facts: TaskFacts): string[] {
return [...facts.unversionedNames, BASELINE_ARTIFACT];
}
/**
* 把产出当前基线的那次正式运行的原始产物(solution.json)放进 dir 下的 baseline-artifact/,设成只读。
* 实验员的工作目录和诊断运行的运行目录都用它:实验员能看到实验要接着算的那份答案,运行时实验读的是同一份。
* 只读靠文件权限,防的是实验顺手改了它,不是安全边界。
*/
function placeBaselineArtifact(_task: Task, baselineRunDir: string, dir: string): void {
const target = join(dir, BASELINE_ARTIFACT);
mkdirSync(target);
const file = join(target, SOLUTION);
copyFileSync(join(baselineRunDir, SOLUTION), file);
chmodSync(file, 0o444);
}
function prepareDiagnosisRun(task: Task, programDir: string, baselineRunDir: string, runDir: string, experiments: DiagnosisExperiment[]): void {
cpSync(programDir, runDir, { recursive: true });
// 两份 harness 以宿主的为准;指标说明是全部实验登记的指标合在一起,每个实验只写自己的那几条。
copyFileSync(join(task.dir, HARNESS), join(runDir, HARNESS));
copyFileSync(join(task.dir, DIAGNOSIS_HARNESS), join(runDir, DIAGNOSIS_HARNESS));
writeFileSync(join(runDir, METRICS_SPEC), `${JSON.stringify(experiments.flatMap((e) => e.metrics), null, 2)}\n`);
placeBaselineArtifact(task, baselineRunDir, runDir);
mkdirSync(join(runDir, ANSWERS));
}
/** 总时限、核数照任务规定,实验按顺序跑,每个实验分到总时限 ÷ 实验数(由 harness 执行)。 */
function diagnosisRun(task: Task, imageDigest: string, seed: string, experimentIds: string[]): RunDefinition {
const eachSeconds = task.facts.run.wallSeconds / experimentIds.length;
const command = [
"python",
"-B",
`${WORKDIR}/${DIAGNOSIS_HARNESS}`,
"--program",
ENTRY,
"--baseline",
BASELINE_ARTIFACT,
"--answers",
ANSWERS,
"--seconds-each",
String(eachSeconds),
"--report",
DIAGNOSIS_REPORT,
"--experiments",
...experimentIds,
];
return definition(task, imageDigest, command, task.facts.run.wallSeconds, { INFORSI_SEED: seed });
}
/** 一个实验在运行目录里的结局:harness 报告里的一项,或者整次运行先结束、没轮到它。 */
type ExperimentOutcome = { ending: "finished" | "crashed" | "timeout"; wallSeconds: number } | { ending: "notRun"; reason: string };
/** harness 写的报告(我们自己的程序,按约定的格式直接读)。 */
interface ReportEntry {
id: string;
status: "finished" | "crashed" | "timeout";
wall_seconds: number;
}
function experimentOutcomes(
task: Task,
runDir: string,
process: Exclude<ProcessResult, { ending: "notStarted" }>,
experiments: DiagnosisExperiment[],
): ExperimentOutcome[] {
const path = join(runDir, DIAGNOSIS_REPORT);
const report = existsSync(path) ? (readJson(path) as ReportEntry[]) : [];
return experiments.map((e) => {
const entry = report.find((r) => r.id === e.id);
if (entry !== undefined) return { ending: entry.status, wallSeconds: entry.wall_seconds };
const failure = processFailure(task, process);
if (failure === null) throw new Error(`诊断运行正常退出,但 ${DIAGNOSIS_REPORT} 里没有实验 ${e.id}`);
return { ending: "notRun", reason: `没轮到:整次诊断运行${failure}` };
});
}
/** 没有答案可判的实验:不算数,辅助指标和假设专用指标都记测不到并附原因。 */
function unjudgedExperiment(
id: string,
ending: Exclude<DiagnosisEnding, "finished">,
wall: Measured,
reason: string,
hypothesisMetrics: HypothesisMetric[],
): DiagnosisExperimentJudged {
const metrics = {
...Object.fromEntries(FROM_VALIDATOR.map((m) => [m, unmeasured(`没有跑校验器:${reason}`)])),
...Object.fromEntries(hypothesisMetrics.map((m) => [m.id, unmeasured(`没有结果:${reason}`)])),
};
return { id, ending, wallSeconds: wall, verdict: "invalid", verdictReason: reason, score: unmeasured(reason), metrics, extra: { legalization: null } };
}
function judgedExperiments(
task: Task,
runDir: string,
judgeDir: string,
experiments: DiagnosisExperiment[],
outcomes: ExperimentOutcome[],
): DiagnosisJudged {
const eachSeconds = task.facts.run.wallSeconds / experiments.length;
return {
experiments: experiments.map((e, i): DiagnosisExperimentJudged => {
const outcome = outcomes[i]!;
if (outcome.ending === "notRun") return unjudgedExperiment(e.id, "notRun", unmeasured(outcome.reason), outcome.reason, e.metrics);
const wall = measured(outcome.wallSeconds);
if (outcome.ending === "timeout") {
return unjudgedExperiment(e.id, "timeout", wall, `超过分到的时间 ${eachSeconds} 秒,被结束`, e.metrics);
}
if (outcome.ending === "crashed") {
return unjudgedExperiment(e.id, "crashed", wall, `实验的进程以非零退出码结束(报错见这次运行的 stderr.log)`, e.metrics);
}
const dir = join(judgeDir, e.id);
const exitCode = Number(readFileSync(join(dir, EXIT_FILE), "utf8").trim());
const scored = scoreVerdict(readVerdict(dir, exitCode));
const hypothesis = readMetricsFile(join(runDir, ANSWERS, e.id, METRICS), `${ANSWERS}/${e.id}/${METRICS}`, e.metrics).hypothesis;
return {
id: e.id,
ending: "finished",
wallSeconds: wall,
verdict: scored.valid ? "valid" : "invalid",
verdictReason: scored.reason,
score: scored.score,
metrics: { ...scored.fromValidator, ...hypothesis },
extra: scored.extra,
};
}),
};
}
/** 逐个实验判分。校验器一个容器、按顺序对每个有答案的实验各跑一次,退出码各存一个文件。 */
const VALIDATE_EACH = `for id in "$@"; do
python -B ${VALIDATOR} --circles ${CIRCLES} --solution "$id/${SOLUTION}" --atol ${TOLERANCE} --legalize --verdict "$id/${VERDICT}"
echo $? > "$id/${EXIT_FILE}"
done`;
function prepareDiagnosisJudging(
task: Task,
runDir: string,
judgeDir: string,
imageDigest: string,
process: ProcessResult,
experiments: DiagnosisExperiment[],
): DiagnosisJudgingStep {
const run = started(process);
const outcomes = experimentOutcomes(task, runDir, run, experiments);
const finished = experiments.filter((_, i) => outcomes[i]!.ending === "finished");
if (finished.length === 0) return { status: "judged", result: judgedExperiments(task, runDir, judgeDir, experiments, outcomes) };
for (const e of finished) {
mkdirSync(join(judgeDir, e.id));
copyFileSync(join(runDir, ANSWERS, e.id, SOLUTION), join(judgeDir, e.id, SOLUTION));
}
copyFileSync(join(task.dir, VALIDATOR), join(judgeDir, VALIDATOR));
const command = ["sh", "-c", VALIDATE_EACH, "inforsi-validate-each", ...finished.map((e) => e.id)];
return { status: "validate", definition: definition(task, imageDigest, command, VALIDATOR_SECONDS * finished.length, {}) };
}
function judgeDiagnosis(
task: Task,
runDir: string,
judgeDir: string,
process: ProcessResult,
validator: ProcessResult | null,
experiments: DiagnosisExperiment[],
): DiagnosisJudged {
const run = started(process);
const outcomes = experimentOutcomes(task, runDir, run, experiments);
if (outcomes.some((o) => o.ending === "finished")) {
if (validator === null) throw new Error("有实验交出了答案,却没有校验器的运行结果");
requireExit("校验器", validator, [0]);
}
return judgedExperiments(task, runDir, judgeDir, experiments, outcomes);
}
export const circlePacking: TaskImplementation = {
@@ -339,4 +547,10 @@ export const circlePacking: TaskImplementation = {
readLightCheck,
prepareJudging,
judge,
diagnosisUnversionedNames,
placeBaselineArtifact,
prepareDiagnosisRun,
diagnosisRun,
prepareDiagnosisJudging,
judgeDiagnosis,
};
+16 -1
View File
@@ -2,14 +2,29 @@
// 不起子进程:要执行任务代码的事都只交出运行定义,由调用方交给 runner 在任务镜像里跑。
export { checkTaskFacts, TASK_KINDS, TaskFacts, type TaskKind } from "./facts.ts";
export {
diagnosisRun,
diagnosisUnversionedNames,
formalRun,
judge,
judgeDiagnosis,
lightCheck,
loadTask,
metricDefinitions,
placeBaselineArtifact,
prepareDiagnosisJudging,
prepareDiagnosisRun,
prepareJudging,
prepareRun,
protectedChanges,
readLightCheck,
} from "./task.ts";
export type { JudgingStep, LightCheckOutcome, Task } from "./types.ts";
export type {
DiagnosisEnding,
DiagnosisExperiment,
DiagnosisExperimentJudged,
DiagnosisJudged,
DiagnosisJudgingStep,
JudgingStep,
LightCheckOutcome,
Task,
} from "./types.ts";
+80 -1
View File
@@ -3,7 +3,15 @@ import { join, resolve } from "node:path";
import type { HypothesisMetric, JudgedResult, MetricDefinition, ProcessResult, RunDefinition } from "../vocab/index.ts";
import { circlePacking } from "./circle-packing.ts";
import { checkTaskFacts, type TaskKind } from "./facts.ts";
import type { JudgingStep, LightCheckOutcome, Task, TaskImplementation } from "./types.ts";
import type {
DiagnosisExperiment,
DiagnosisJudged,
DiagnosisJudgingStep,
JudgingStep,
LightCheckOutcome,
Task,
TaskImplementation,
} from "./types.ts";
// tasks 对外的函数。任务目录的约定:task.json(任务事实)、TASK.md(给实验员的说明)、
// Dockerfile 与依赖清单(建镜像)、baseline/(基线代码),以及各任务自己的 Python 工具。
@@ -105,3 +113,74 @@ export function judge(
export function protectedChanges(task: Task, changedFiles: string[]): string[] {
return changedFiles.filter((file) => task.facts.protectedFiles.includes(file));
}
// ---------------------------------------------------------------------------
// 诊断运行(方法.md 4.3"诊断"一段):一个容器、一次登记、一次扣次,里面按顺序跑几个实验。
// 是和上面正式运行并列的另一套函数,上面的函数不认识它。各目录同样由调用方建好、传空目录进来。
// ---------------------------------------------------------------------------
/** 存诊断程序版本时跳过的名字:task.json 的清单,加上宿主放进工作目录的原始产物目录。 */
export function diagnosisUnversionedNames(task: Task): string[] {
return implementation(task).diagnosisUnversionedNames(task.facts);
}
/**
* 把产出当前基线的那次正式运行的原始产物,只读地放进 dir(实验员工作目录,调用方已用 checkoutVersion 取出基线代码)。
* baselineRunDir 是那次运行的运行目录。放在哪、叫什么,由任务定,写在 TASK.md 里给实验员看。
*/
export function placeBaselineArtifact(task: Task, baselineRunDir: string, dir: string): void {
implementation(task).placeBaselineArtifact(task, baselineRunDir, dir);
}
/**
* 准备一次诊断运行的运行目录:诊断程序、两份 harness、全部实验登记的指标说明(metrics-spec.json)、
* 只读的原始产物。runDir 由调用方建好,应为空目录;实验的先后顺序就是 experiments 的顺序。
*/
export function prepareDiagnosisRun(
task: Task,
programDir: string,
baselineRunDir: string,
runDir: string,
experiments: DiagnosisExperiment[],
): void {
implementation(task).prepareDiagnosisRun(task, programDir, baselineRunDir, runDir, experiments);
}
/**
* 诊断运行的运行定义:总时限、核数照任务规定,harness 按顺序跑 experimentIds 里的实验,
* 每个实验分到总时限 ÷ 实验数。种子同正式运行(同一份研究共用一个)。
*/
export function diagnosisRun(task: Task, imageDigest: string, seed: string, experimentIds: string[]): RunDefinition {
return implementation(task).diagnosisRun(task, imageDigest, seed, experimentIds);
}
/**
* 诊断判分第一步:每个交出答案的实验的答案复制进 judgeDir 下以实验编号命名的子目录,
* 交回校验器的运行定义(一个容器逐个实验判分);没有一个实验交出答案时直接交回判分结果。
*/
export function prepareDiagnosisJudging(
task: Task,
runDir: string,
judgeDir: string,
imageDigest: string,
process: ProcessResult,
experiments: DiagnosisExperiment[],
): DiagnosisJudgingStep {
return implementation(task).prepareDiagnosisJudging(task, runDir, judgeDir, imageDigest, process, experiments);
}
/**
* 诊断判分第二步:按实验分开交回分数、合法化、登记的指标、用时、结局。validator 是第一步交回过运行定义时跑出的结果,
* 第一步直接判完时传 null。资源用量记在整次运行的 ProcessResult 里,不在这里。
* 校验器自己没正常跑完、判定文件认不出、修后分数高于原分数时报错停下,同正式运行的 judge。
*/
export function judgeDiagnosis(
task: Task,
runDir: string,
judgeDir: string,
process: ProcessResult,
validator: ProcessResult | null,
experiments: DiagnosisExperiment[],
): DiagnosisJudged {
return implementation(task).judgeDiagnosis(task, runDir, judgeDir, process, validator, experiments);
}
+60 -1
View File
@@ -1,4 +1,4 @@
import type { HypothesisMetric, JudgedResult, MetricDefinition, ProcessResult, RunDefinition } from "../vocab/index.ts";
import type { HypothesisMetric, JudgedResult, Json, Measured, MetricDefinition, ProcessResult, RunDefinition, Verdict } from "../vocab/index.ts";
import type { TaskFacts } from "./facts.ts";
/** 读进来的一个任务。 */
@@ -22,6 +22,44 @@ export type LightCheckOutcome = { passed: true } | { passed: false; reasons: str
*/
export type JudgingStep = { status: "judged"; result: JudgedResult } | { status: "validate"; definition: RunDefinition };
/** 一个要跑的诊断实验:编号,以及它自己登记的假设专用指标(冻结后的全名;诊断实验只能登记"缺了只记缺失"的指标)。 */
export interface DiagnosisExperiment {
id: string;
metrics: HypothesisMetric[];
}
/**
* 一个诊断实验怎么结束:finished 跑完交出了答案;crashed 实验自己崩了;timeout 超过分到的时间被杀;
* notRun 整次运行先被总时限或从外面结束,没轮到它。
*/
export type DiagnosisEnding = "finished" | "crashed" | "timeout" | "notRun";
/** 一个诊断实验单独判分的结果。资源用量(CPU、峰值内存)记在整次运行的结果里,这里只有它自己的用时。 */
export interface DiagnosisExperimentJudged {
id: string;
ending: DiagnosisEnding;
/** 这个实验实际用掉的墙钟秒数;notRun 时测不到。 */
wallSeconds: Measured;
verdict: Verdict;
verdictReason: string;
/** 合法化之后的最终指标分数;不算数时测不到并附原因。 */
score: Measured;
/** 校验器给的辅助指标,加上这个实验登记的假设专用指标,键是指标编号。 */
metrics: Record<string, Measured>;
/** 同正式运行的 extra:填圆是合法化的明细。 */
extra: Json;
}
/** 一次诊断运行的判分结果:每个实验一项,顺序同运行定义里的实验顺序。 */
export interface DiagnosisJudged {
experiments: DiagnosisExperimentJudged[];
}
/** 诊断判分第一步:不用跑校验器就已判定(没有一个实验交出答案),或交回校验器的运行定义。 */
export type DiagnosisJudgingStep =
| { status: "judged"; result: DiagnosisJudged }
| { status: "validate"; definition: RunDefinition };
/** 一种任务的实现。对外函数按 task.facts.kind 从对照表里取出它来调。参数的含义见 task.ts 里同名的对外函数。 */
export interface TaskImplementation {
/** 最终指标(编号和方向取自 task.json)与常规辅助指标的定义。 */
@@ -46,4 +84,25 @@ export interface TaskImplementation {
validator: ProcessResult,
hypothesisMetrics: HypothesisMetric[],
): JudgedResult;
/** 诊断运行(方法.md 4.3"诊断"一段):一次运行跑几个实验。上面七个函数的行为与它无关。 */
diagnosisUnversionedNames(facts: TaskFacts): string[];
placeBaselineArtifact(task: Task, baselineRunDir: string, dir: string): void;
prepareDiagnosisRun(task: Task, programDir: string, baselineRunDir: string, runDir: string, experiments: DiagnosisExperiment[]): void;
diagnosisRun(task: Task, imageDigest: string, seed: string, experimentIds: string[]): RunDefinition;
prepareDiagnosisJudging(
task: Task,
runDir: string,
judgeDir: string,
imageDigest: string,
process: ProcessResult,
experiments: DiagnosisExperiment[],
): DiagnosisJudgingStep;
judgeDiagnosis(
task: Task,
runDir: string,
judgeDir: string,
process: ProcessResult,
validator: ProcessResult | null,
experiments: DiagnosisExperiment[],
): DiagnosisJudged;
}
+13
View File
@@ -22,6 +22,9 @@ type FactorRules = {
/** 实现、修复、核验共用:冻结协议的实现条款(c1…)、因子(p1…)、保障短名。 */
type FrozenItems = { candidate: string; clauses: string[]; factors: string[]; safeguards: string[] };
/** 诊断实现、修复、核验共用:本次诊断选中的实验编号(d1…)。 */
type DiagnosisItems = { experiments: string[] };
/** 入口个数的范围(方法.md 4.2:2–3 个)。 */
type EntryCount = { min: 2; max: 3 };
@@ -30,6 +33,16 @@ export type Checklist =
| { kind: "discussion"; entries: EntryCount }
/** minChoices = min(N, 入口总数)(方法.md 4.2 认领)。 */
| { kind: "claim"; entries: string[]; minChoices: number }
/**
* 诊断解释最多 M 条、诊断实验最多 A 个(方法.md 4.3 诊 1);takenMetricIds 是新登记的指标短名不能用的编号
* (最终指标、常规辅助指标和答案路径的两个保留键)。
*/
| { kind: "diagnosisProposal"; maxExplanations: number; maxExperiments: number; takenMetricIds: string[] }
/** 对自己每条解释(explanations)× 每个合并后的实验(experiments)各给三个数(方法.md 4.3 诊 2)。 */
| { kind: "diagnosisPrediction"; explanations: string[]; experiments: string[] }
| ({ kind: "diagnosisImplementation" } & DiagnosisItems)
| ({ kind: "diagnosisRepair" } & DiagnosisItems)
| ({ kind: "diagnosisVerification" } & DiagnosisItems)
| ({ kind: "proposal"; maxProposals: number } & FactorRules)
| { kind: "firstPrediction"; candidate: string }
| { kind: "secondPrediction"; candidate: string }
+9
View File
@@ -10,6 +10,10 @@ export const STAGES = [
"entryDraft", // 0.1 闭门写入口
"discussion", // 0.2 讨论,可配置为 1 或 2 次,每次都是这个阶段
"claim", // 0.3 认领
"diagnosisProposal", // 诊 1 诊断解释与诊断实验
"diagnosisPrediction", // 诊 2 诊断预测
"diagnosisImplementation", // 诊 4 诊断实现,以及核验不通过后的修复
"diagnosisVerification", // 诊 4 诊断核验
"proposal", // ② 提案
"firstPrediction", // ④ 第一次预测
"critique", // ⑤ 挑刺
@@ -33,6 +37,11 @@ export const SUBMISSION_KINDS = {
entry: { stage: "entryDraft", role: "researcher" },
discussion: { stage: "discussion", role: "researcher" },
claim: { stage: "claim", role: "researcher" },
diagnosisProposal: { stage: "diagnosisProposal", role: "researcher" },
diagnosisPrediction: { stage: "diagnosisPrediction", role: "researcher" },
diagnosisImplementation: { stage: "diagnosisImplementation", role: "experimenter" },
diagnosisRepair: { stage: "diagnosisImplementation", role: "experimenter" },
diagnosisVerification: { stage: "diagnosisVerification", role: "verifier" },
proposal: { stage: "proposal", role: "researcher" },
firstPrediction: { stage: "firstPrediction", role: "researcher" },
critique: { stage: "critique", role: "researcher" },
+145
View File
@@ -503,6 +503,146 @@ export const VerificationSubmission = Type.Object(
},
);
// ---------------------------------------------------------------------------
// 诊断:解释与实验、预测、实现、核验(方法.md 4.3 诊 1–诊 4、第五部分"诊断"三行)
// ---------------------------------------------------------------------------
const DiagnosisExplanation = Type.Object(
{
body: Type.String({
...TEXT,
description:
"Markdown:为什么卡在这里,剩下的空间在哪,什么样的实测会推翻它(写不出推翻它的实测,说明这条解释还不能被检验,先改具体再交)。" +
"诊断问的是剩下的空间在哪,不是上一步的功劳怎么分。没有具体判断就直说,写明查了什么、为什么拿不准",
}),
},
CLOSED,
);
const DiagnosisExperiment = Type.Object(
{
name: Type.String({
...TEXT,
description: "名称:离开这份清单单独看也读得懂",
}),
spot: Type.String({
...TEXT,
description: "动哪一处:在当前基线的哪一处代码或哪一步上动手,只写一处(文件、函数或步骤名),其余照基线保留;想测两处就拆成两个实验",
}),
body: Type.String({
...TEXT,
description:
"Markdown:在这一处怎么动(换成明显更强的版本、换成另一种做法、拿掉,或放宽某个限制)、" +
"做完交出什么能判分的答案、几种可能的结果各排除哪几条解释(有分不开的也写明)。不写自己估的分数,分数在下一步交",
}),
metrics: Type.Array(HypothesisMetric, {
description:
"登记假设专用指标:这个实验的代码要额外记录的量,编号只写短名;没有就交空数组。" +
"诊断实验里的指标只能选 recordMissing(缺了只记缺失),选 invalidateRun 会被退回",
}),
},
CLOSED,
);
export const DiagnosisProposalSubmission = Type.Object(
{
explanations: Type.Array(DiagnosisExplanation, {
minItems: 1,
description: "诊断解释清单,编号由程序分配;条数上限见材料。真有第二个看法再写第二条,各条之间要真的不同;没有具体判断,就写一条直说没有的",
}),
experiments: Type.Array(DiagnosisExperiment, {
minItems: 1,
description:
"诊断实验清单,个数上限见材料。好的实验是不同解释对它的预测差得远:大家都预测涨不了多少的实验,测了也不改变下一步。" +
"每个实验只动一处,用时要放得进分到的那份时间;想不出能把解释分开的实验,就交最能暴露你的解释错在哪的那个",
}),
},
{
...CLOSED,
description: "对当前基线的诊断:几条解释,和能把它们分开的几个实验。实验正文里不写自己估的分数",
},
);
export const DiagnosisPredictionSubmission = Type.Object(
{
predictions: Type.Array(
Type.Object(
{
explanation: Type.String({ ...Id, description: "你自己的诊断解释编号,照材料里列的写" }),
experiment: Type.String({ ...Id, description: "合并后的诊断实验编号,照材料里列的写,别人提的也要给" }),
range: Range,
},
CLOSED,
),
{
description:
"你自己的每条诊断解释对每个合并后的实验各给三个数:按这条解释,做了这个实验,分数有一成、一半、九成可能低于多少。" +
"不多不少,每一对只写一次;按这条解释本身写,不往折中的范围靠;解释和这个实验没关系就按基线附近写并放宽;拿不准就把 q1 和 q9 拉开",
},
),
body: Type.String({
...TEXT,
description: "理由,Markdown:这些数是怎么来的,不同解释给的数差在哪里;几条解释给的数差不多,写明它们在这些实验上分不开",
}),
},
CLOSED,
);
/** 诊断实现和修复用同一个定义。代码本身由程序从实验员的工作目录冻结,不作为字段交。 */
export const DiagnosisImplementationSubmission = Type.Object(
{
locations: Type.Array(
Type.Object(
{
experiment: Type.String({ ...Id, description: "诊断实验编号,照材料里列的写" }),
locations: LOCATIONS,
},
CLOSED,
),
{
description: "位置表:选中的每个实验一项,列出本次代码中这个实验实际所在的全部位置,行号以本次提交代码为准",
},
),
concerns: CONCERNS,
body: Type.String({
...TEXT,
description: "实现说明,Markdown:位置表之外需要说明的事;实验原文没规定规模时,写你选了什么、为什么放得进分到的时间。修复时逐条回应核验意见:改在哪里,或以实验原文和代码位置说明为什么不改。没有要补充的就写\"无\"",
}),
},
CLOSED,
);
export const DiagnosisVerificationSubmission = Type.Object(
{
experiments: Type.Array(
Type.Object(
{
experiment: Type.String({ ...Id, description: "诊断实验编号,照材料里列的写" }),
followed: Type.Boolean({
description: "照实验写的做法实现了写 true;没找到、做得不一样或材料不足以核清,写 false",
}),
nothingElse: Type.Boolean({
description: "没有动实验写的那一处以外的地方写 true;顺手改了别处(含顺手改进)写 false",
}),
where: WHERE,
},
CLOSED,
),
{
description: "选中的每个实验一项,逐个判断,一个实验不通过不影响别的实验",
},
),
body: Type.String({
...TEXT,
description: "说明,Markdown:逐项结果之外需要说明的事。没有要补充的就写\"无\"",
}),
},
{
...CLOSED,
description: "只交逐个实验的结果,通过与否由程序推出:任何一项写 false 的实验退回实验员修复,用完修复次数仍不通过的实验从这次诊断里拿掉",
},
);
// ---------------------------------------------------------------------------
// 解释、消融、收尾(方法.md 4.3 ⑭⑯⑱)
// ---------------------------------------------------------------------------
@@ -611,6 +751,11 @@ export const SUBMISSION_SCHEMAS = {
entry: EntrySubmission,
discussion: DiscussionSubmission,
claim: ClaimSubmission,
diagnosisProposal: DiagnosisProposalSubmission,
diagnosisPrediction: DiagnosisPredictionSubmission,
diagnosisImplementation: DiagnosisImplementationSubmission,
diagnosisRepair: DiagnosisImplementationSubmission,
diagnosisVerification: DiagnosisVerificationSubmission,
proposal: ProposalSubmission,
firstPrediction: PredictionSubmission,
critique: CritiqueSubmission,